From 25f3508e846b8ca5c219cc3bcc18a80be6d8d025 Mon Sep 17 00:00:00 2001 From: Adrian Conlon <98398945+AdrianConlon@users.noreply.github.com> Date: Fri, 13 Sep 2024 20:54:17 +0100 Subject: [PATCH] Update simdjson to 3.10.1 --- M6502/HarteTest_6502/simdjson/simdjson.cpp | 64953 +++++++++++-------- M6502/HarteTest_6502/simdjson/simdjson.h | 34431 +++++++++- 2 files changed, 70939 insertions(+), 28445 deletions(-) diff --git a/M6502/HarteTest_6502/simdjson/simdjson.cpp b/M6502/HarteTest_6502/simdjson/simdjson.cpp index 855c3cf..515a3e6 100644 --- a/M6502/HarteTest_6502/simdjson/simdjson.cpp +++ b/M6502/HarteTest_6502/simdjson/simdjson.cpp @@ -1,4 +1,4 @@ -/* auto-generated on 2023-12-07 12:42:28 -0500. Do not edit! */ +/* auto-generated on 2024-08-26 09:37:03 -0400. Do not edit! */ /* including simdjson.cpp: */ /* begin file simdjson.cpp */ #define SIMDJSON_SRC_SIMDJSON_CPP @@ -40,6 +40,16 @@ #endif #endif +// C++ 23 +#if !defined(SIMDJSON_CPLUSPLUS23) && (SIMDJSON_CPLUSPLUS >= 202302L) +#define SIMDJSON_CPLUSPLUS23 1 +#endif + +// C++ 20 +#if !defined(SIMDJSON_CPLUSPLUS20) && (SIMDJSON_CPLUSPLUS >= 202002L) +#define SIMDJSON_CPLUSPLUS20 1 +#endif + // C++ 17 #if !defined(SIMDJSON_CPLUSPLUS17) && (SIMDJSON_CPLUSPLUS >= 201703L) #define SIMDJSON_CPLUSPLUS17 1 @@ -84,6 +94,9 @@ #include #endif +// We are using size_t without namespace std:: throughout the project +using std::size_t; + #ifdef _MSC_VER #define SIMDJSON_VISUAL_STUDIO 1 /** @@ -105,12 +118,14 @@ #endif // __clang__ #endif // _MSC_VER -#if defined(__x86_64__) || defined(_M_AMD64) +#if (defined(__x86_64__) || defined(_M_AMD64)) && !defined(_M_ARM64EC) #define SIMDJSON_IS_X86_64 1 -#elif defined(__aarch64__) || defined(_M_ARM64) +#elif defined(__aarch64__) || defined(_M_ARM64) || defined(_M_ARM64EC) #define SIMDJSON_IS_ARM64 1 #elif defined(__riscv) && __riscv_xlen == 64 #define SIMDJSON_IS_RISCV64 1 +#elif defined(__loongarch_lp64) +#define SIMDJSON_IS_LOONGARCH64 1 #elif defined(__PPC64__) || defined(_M_PPC64) #if defined(__ALTIVEC__) #define SIMDJSON_IS_PPC64_VMX 1 @@ -219,6 +234,11 @@ #define SIMDJSON_NO_SANITIZE_UNDEFINED #endif +#if defined(__clang__) || defined(__GNUC__) +#define simdjson_pure [[gnu::pure]] +#else +#define simdjson_pure +#endif #if defined(__clang__) || defined(__GNUC__) #if defined(__has_feature) @@ -312,6 +332,8 @@ namespace simdjson { #define SIMDJSON_ISALIGNED_N(ptr, n) (((uintptr_t)(ptr) & ((n)-1)) == 0) #if SIMDJSON_REGULAR_VISUAL_STUDIO + // We could use [[deprecated]] but it requires C++14 +#define simdjson_deprecated __declspec(deprecated) #define simdjson_really_inline __forceinline #define simdjson_never_inline __declspec(noinline) @@ -350,6 +372,8 @@ namespace simdjson { #define SIMDJSON_POP_DISABLE_UNUSED_WARNINGS #else // SIMDJSON_REGULAR_VISUAL_STUDIO + // We could use [[deprecated]] but it requires C++14 +#define simdjson_deprecated __attribute__((deprecated)) #define simdjson_really_inline inline __attribute__((always_inline)) #define simdjson_never_inline inline __attribute__((noinline)) @@ -531,7 +555,7 @@ SIMDJSON_PUSH_DISABLE_ALL_WARNINGS #define NONSTD_SV_LITE_H_INCLUDED #define string_view_lite_MAJOR 1 -#define string_view_lite_MINOR 7 +#define string_view_lite_MINOR 8 #define string_view_lite_PATCH 0 #define string_view_lite_VERSION nssv_STRINGIFY(string_view_lite_MAJOR) "." nssv_STRINGIFY(string_view_lite_MINOR) "." nssv_STRINGIFY(string_view_lite_PATCH) @@ -588,6 +612,10 @@ SIMDJSON_PUSH_DISABLE_ALL_WARNINGS # define nssv_CONFIG_NO_STREAM_INSERTION 0 #endif +#ifndef nssv_CONFIG_CONSTEXPR11_STD_SEARCH +# define nssv_CONFIG_CONSTEXPR11_STD_SEARCH 1 +#endif + // Control presence of exception handling (try and auto discover): #ifndef nssv_CONFIG_NO_EXCEPTIONS @@ -649,6 +677,8 @@ SIMDJSON_PUSH_DISABLE_ALL_WARNINGS #if nssv_CONFIG_CONVERSION_STD_STRING_FREE_FUNCTIONS +#include + namespace nonstd { template< class CharT, class Traits, class Allocator = std::allocator > @@ -787,7 +817,7 @@ namespace nonstd { # define nssv_HAS_CPP0X 0 #endif -// Unless defined otherwise below, consider VC14 as C++11 for variant-lite: +// Unless defined otherwise below, consider VC14 as C++11 for string-view-lite: #if nssv_COMPILER_MSVC_VER >= 1900 # undef nssv_CPP11_OR_GREATER @@ -954,9 +984,9 @@ namespace nonstd { # pragma clang diagnostic ignored "-Wreserved-user-defined-literal" # pragma clang diagnostic push # pragma clang diagnostic ignored "-Wuser-defined-literals" -#elif defined(__GNUC__) -# pragma GCC diagnostic push -# pragma GCC diagnostic ignored "-Wliteral-suffix" +#elif nssv_COMPILER_GNUC_VERSION >= 480 +# pragma GCC diagnostic push +# pragma GCC diagnostic ignored "-Wliteral-suffix" #endif // __clang__ #if nssv_COMPILER_MSVC_VERSION >= 140 @@ -971,8 +1001,8 @@ namespace nonstd { #if defined(__clang__) # define nssv_RESTORE_WARNINGS() _Pragma("clang diagnostic pop") -#elif defined(__GNUC__) -# define nssv_RESTORE_WARNINGS() _Pragma("GCC diagnostic pop") +#elif nssv_COMPILER_GNUC_VERSION >= 480 +# define nssv_RESTORE_WARNINGS() _Pragma("GCC diagnostic pop") #elif nssv_COMPILER_MSVC_VERSION >= 140 # define nssv_RESTORE_WARNINGS() __pragma(warning(pop )) #else @@ -1088,12 +1118,31 @@ namespace nonstd { // non-recursive: +#if nssv_CONFIG_CONSTEXPR11_STD_SEARCH + template< class CharT, class Traits = std::char_traits > constexpr const CharT* search(basic_string_view haystack, basic_string_view needle) { return std::search(haystack.begin(), haystack.end(), needle.begin(), needle.end()); } +#else // nssv_CONFIG_CONSTEXPR11_STD_SEARCH + + template< class CharT, class Traits = std::char_traits > + nssv_constexpr14 const CharT* search(basic_string_view haystack, basic_string_view needle) + { + while (needle.size() <= haystack.size()) + { + if (haystack.starts_with(needle)) + { + return haystack.cbegin(); + } + haystack = basic_string_view{ haystack.begin() + 1, haystack.size() - 1U }; + } + return haystack.cend(); + } +#endif // nssv_CONFIG_CONSTEXPR11_STD_SEARCH + #endif // OPTIMIZE #endif // nssv_CPP11_OR_GREATER && ! nssv_CPP17_OR_GREATER @@ -1366,7 +1415,7 @@ namespace nonstd { // find(), 4x: - nssv_constexpr size_type find(basic_string_view v, size_type pos = 0) const nssv_noexcept // (1) + nssv_constexpr14 size_type find(basic_string_view v, size_type pos = 0) const nssv_noexcept // (1) { return assert(v.size() == 0 || v.data() != nssv_nullptr) , pos >= size() @@ -2412,6 +2461,7 @@ namespace simdjson { F_ATOM_ERROR, ///< Problem while parsing an atom starting with the letter 'f' N_ATOM_ERROR, ///< Problem while parsing an atom starting with the letter 'n' NUMBER_ERROR, ///< Problem while parsing a number + BIGINT_ERROR, ///< The integer value exceeds 64 bits UTF8_ERROR, ///< the input is not valid UTF-8 UNINITIALIZED, ///< unknown error, or uninitialized document EMPTY, ///< no structural element found @@ -2423,7 +2473,7 @@ namespace simdjson { INDEX_OUT_OF_BOUNDS, ///< JSON array index too large NO_SUCH_FIELD, ///< JSON field not found in object IO_ERROR, ///< Error reading a file - INVALID_JSON_POINTER, ///< Invalid JSON pointer reference + INVALID_JSON_POINTER, ///< Invalid JSON pointer syntax INVALID_URI_FRAGMENT, ///< Invalid URI fragment UNEXPECTED_ERROR, ///< indicative of a bug in simdjson PARSER_IN_USE, ///< parser is already in use. @@ -4585,6 +4635,7 @@ namespace simdjson { { F_ATOM_ERROR, "F_ATOM_ERROR: Problem while parsing an atom starting with the letter 'f'" }, { N_ATOM_ERROR, "N_ATOM_ERROR: Problem while parsing an atom starting with the letter 'n'" }, { NUMBER_ERROR, "NUMBER_ERROR: Problem while parsing a number" }, + { BIGINT_ERROR, "BIGINT_ERROR: Big integer value that cannot be represented using 64 bits" }, { UTF8_ERROR, "UTF8_ERROR: The input is not valid UTF-8" }, { UNINITIALIZED, "UNINITIALIZED: Uninitialized" }, { EMPTY, "EMPTY: no JSON found" }, @@ -5577,6 +5628,8 @@ SIMDJSON_DLLIMPORTEXPORT const uint64_t simdjson::internal::power_of_five_128[] #define SIMDJSON_IMPLEMENTATION_ID_icelake 4 #define SIMDJSON_IMPLEMENTATION_ID_ppc64 5 #define SIMDJSON_IMPLEMENTATION_ID_westmere 6 +#define SIMDJSON_IMPLEMENTATION_ID_lsx 7 +#define SIMDJSON_IMPLEMENTATION_ID_lasx 8 #define SIMDJSON_IMPLEMENTATION_ID_FOR(IMPL) SIMDJSON_CAT(SIMDJSON_IMPLEMENTATION_ID_, IMPL) #define SIMDJSON_IMPLEMENTATION_ID SIMDJSON_IMPLEMENTATION_ID_FOR(SIMDJSON_IMPLEMENTATION) @@ -5591,7 +5644,11 @@ SIMDJSON_DLLIMPORTEXPORT const uint64_t simdjson::internal::power_of_five_128[] #ifndef SIMDJSON_IMPLEMENTATION_ARM64 #define SIMDJSON_IMPLEMENTATION_ARM64 (SIMDJSON_IS_ARM64) #endif -#define SIMDJSON_CAN_ALWAYS_RUN_ARM64 SIMDJSON_IMPLEMENTATION_ARM64 && SIMDJSON_IS_ARM64 +#if SIMDJSON_IMPLEMENTATION_ARM64 && SIMDJSON_IS_ARM64 +#define SIMDJSON_CAN_ALWAYS_RUN_ARM64 1 +#else +#define SIMDJSON_CAN_ALWAYS_RUN_ARM64 0 +#endif // Default Icelake to on if this is x86-64. Even if we're not compiled for it, it could be selected // at runtime. @@ -5602,9 +5659,20 @@ SIMDJSON_DLLIMPORTEXPORT const uint64_t simdjson::internal::power_of_five_128[] #ifdef _MSC_VER // To see why (__BMI__) && (__PCLMUL__) && (__LZCNT__) are not part of this next line, see // https://github.com/simdjson/simdjson/issues/1247 -#define SIMDJSON_CAN_ALWAYS_RUN_ICELAKE ((SIMDJSON_IMPLEMENTATION_ICELAKE) && (__AVX2__) && (__AVX512F__) && (__AVX512DQ__) && (__AVX512CD__) && (__AVX512BW__) && (__AVX512VL__) && (__AVX512VBMI2__)) +#if ((SIMDJSON_IMPLEMENTATION_ICELAKE) && (__AVX2__) && (__AVX512F__) && (__AVX512DQ__) && (__AVX512CD__) && (__AVX512BW__) && (__AVX512VL__) && (__AVX512VBMI2__)) +#define SIMDJSON_CAN_ALWAYS_RUN_ICELAKE 1 #else -#define SIMDJSON_CAN_ALWAYS_RUN_ICELAKE ((SIMDJSON_IMPLEMENTATION_ICELAKE) && (__AVX2__) && (__BMI__) && (__PCLMUL__) && (__LZCNT__) && (__AVX512F__) && (__AVX512DQ__) && (__AVX512CD__) && (__AVX512BW__) && (__AVX512VL__) && (__AVX512VBMI2__)) +#define SIMDJSON_CAN_ALWAYS_RUN_ICELAKE 0 +#endif + +#else + +#if ((SIMDJSON_IMPLEMENTATION_ICELAKE) && (__AVX2__) && (__BMI__) && (__PCLMUL__) && (__LZCNT__) && (__AVX512F__) && (__AVX512DQ__) && (__AVX512CD__) && (__AVX512BW__) && (__AVX512VL__) && (__AVX512VBMI2__)) +#define SIMDJSON_CAN_ALWAYS_RUN_ICELAKE 1 +#else +#define SIMDJSON_CAN_ALWAYS_RUN_ICELAKE 0 +#endif + #endif // Default Haswell to on if this is x86-64. Even if we're not compiled for it, it could be selected @@ -5620,9 +5688,20 @@ SIMDJSON_DLLIMPORTEXPORT const uint64_t simdjson::internal::power_of_five_128[] #ifdef _MSC_VER // To see why (__BMI__) && (__PCLMUL__) && (__LZCNT__) are not part of this next line, see // https://github.com/simdjson/simdjson/issues/1247 -#define SIMDJSON_CAN_ALWAYS_RUN_HASWELL ((SIMDJSON_IMPLEMENTATION_HASWELL) && (SIMDJSON_IS_X86_64) && (__AVX2__)) +#if ((SIMDJSON_IMPLEMENTATION_HASWELL) && (SIMDJSON_IS_X86_64) && (__AVX2__)) +#define SIMDJSON_CAN_ALWAYS_RUN_HASWELL 1 #else -#define SIMDJSON_CAN_ALWAYS_RUN_HASWELL ((SIMDJSON_IMPLEMENTATION_HASWELL) && (SIMDJSON_IS_X86_64) && (__AVX2__) && (__BMI__) && (__PCLMUL__) && (__LZCNT__)) +#define SIMDJSON_CAN_ALWAYS_RUN_HASWELL 0 +#endif + +#else + +#if ((SIMDJSON_IMPLEMENTATION_HASWELL) && (SIMDJSON_IS_X86_64) && (__AVX2__) && (__BMI__) && (__PCLMUL__) && (__LZCNT__)) +#define SIMDJSON_CAN_ALWAYS_RUN_HASWELL 1 +#else +#define SIMDJSON_CAN_ALWAYS_RUN_HASWELL 0 +#endif + #endif // Default Westmere to on if this is x86-64. @@ -5634,16 +5713,40 @@ SIMDJSON_DLLIMPORTEXPORT const uint64_t simdjson::internal::power_of_five_128[] #define SIMDJSON_IMPLEMENTATION_WESTMERE SIMDJSON_IS_X86_64 #endif #endif -#define SIMDJSON_CAN_ALWAYS_RUN_WESTMERE (SIMDJSON_IMPLEMENTATION_WESTMERE && SIMDJSON_IS_X86_64 && __SSE4_2__ && __PCLMUL__) + +#if (SIMDJSON_IMPLEMENTATION_WESTMERE && SIMDJSON_IS_X86_64 && __SSE4_2__ && __PCLMUL__) +#define SIMDJSON_CAN_ALWAYS_RUN_WESTMERE 1 +#else +#define SIMDJSON_CAN_ALWAYS_RUN_WESTMERE 0 +#endif + #ifndef SIMDJSON_IMPLEMENTATION_PPC64 #define SIMDJSON_IMPLEMENTATION_PPC64 (SIMDJSON_IS_PPC64 && SIMDJSON_IS_PPC64_VMX) #endif -#define SIMDJSON_CAN_ALWAYS_RUN_PPC64 SIMDJSON_IMPLEMENTATION_PPC64 && SIMDJSON_IS_PPC64 && SIMDJSON_IS_PPC64_VMX +#if SIMDJSON_IMPLEMENTATION_PPC64 && SIMDJSON_IS_PPC64 && SIMDJSON_IS_PPC64_VMX +#define SIMDJSON_CAN_ALWAYS_RUN_PPC64 1 +#else +#define SIMDJSON_CAN_ALWAYS_RUN_PPC64 0 +#endif + +#ifndef SIMDJSON_IMPLEMENTATION_LASX +#define SIMDJSON_IMPLEMENTATION_LASX (SIMDJSON_IS_LOONGARCH64 && __loongarch_asx) +#endif +#define SIMDJSON_CAN_ALWAYS_RUN_LASX (SIMDJSON_IMPLEMENTATION_LASX) + +#ifndef SIMDJSON_IMPLEMENTATION_LSX +#if SIMDJSON_CAN_ALWAYS_RUN_LASX +#define SIMDJSON_IMPLEMENTATION_LSX 0 +#else +#define SIMDJSON_IMPLEMENTATION_LSX (SIMDJSON_IS_LOONGARCH64 && __loongarch_sx) +#endif +#endif +#define SIMDJSON_CAN_ALWAYS_RUN_LSX (SIMDJSON_IMPLEMENTATION_LSX) // Default Fallback to on unless a builtin implementation has already been selected. #ifndef SIMDJSON_IMPLEMENTATION_FALLBACK -#if SIMDJSON_CAN_ALWAYS_RUN_ARM64 || SIMDJSON_CAN_ALWAYS_RUN_ICELAKE || SIMDJSON_CAN_ALWAYS_RUN_HASWELL || SIMDJSON_CAN_ALWAYS_RUN_WESTMERE || SIMDJSON_CAN_ALWAYS_RUN_PPC64 +#if SIMDJSON_CAN_ALWAYS_RUN_ARM64 || SIMDJSON_CAN_ALWAYS_RUN_ICELAKE || SIMDJSON_CAN_ALWAYS_RUN_HASWELL || SIMDJSON_CAN_ALWAYS_RUN_WESTMERE || SIMDJSON_CAN_ALWAYS_RUN_PPC64 || SIMDJSON_CAN_ALWAYS_RUN_LSX || SIMDJSON_CAN_ALWAYS_RUN_LASX // if anything at all except fallback can always run, then disable fallback. #define SIMDJSON_IMPLEMENTATION_FALLBACK 0 #else @@ -5665,6 +5768,10 @@ SIMDJSON_DLLIMPORTEXPORT const uint64_t simdjson::internal::power_of_five_128[] #define SIMDJSON_BUILTIN_IMPLEMENTATION arm64 #elif SIMDJSON_CAN_ALWAYS_RUN_PPC64 #define SIMDJSON_BUILTIN_IMPLEMENTATION ppc64 +#elif SIMDJSON_CAN_ALWAYS_RUN_LSX +#define SIMDJSON_BUILTIN_IMPLEMENTATION lsx +#elif SIMDJSON_CAN_ALWAYS_RUN_LASX +#define SIMDJSON_BUILTIN_IMPLEMENTATION lasx #elif SIMDJSON_CAN_ALWAYS_RUN_FALLBACK #define SIMDJSON_BUILTIN_IMPLEMENTATION fallback #else @@ -5679,7 +5786,7 @@ SIMDJSON_DLLIMPORTEXPORT const uint64_t simdjson::internal::power_of_five_128[] #endif // SIMDJSON_IMPLEMENTATION_DETECTION_H /* end file simdjson/implementation_detection.h */ -#if SIMDJSON_IMPLEMENTATION_ARM64 || SIMDJSON_IMPLEMENTATION_ICELAKE || SIMDJSON_IMPLEMENTATION_HASWELL || SIMDJSON_IMPLEMENTATION_WESTMERE || SIMDJSON_IMPLEMENTATION_PPC64 +#if SIMDJSON_IMPLEMENTATION_ARM64 || SIMDJSON_IMPLEMENTATION_ICELAKE || SIMDJSON_IMPLEMENTATION_HASWELL || SIMDJSON_IMPLEMENTATION_WESTMERE || SIMDJSON_IMPLEMENTATION_PPC64 || SIMDJSON_IMPLEMENTATION_LSX || SIMDJSON_IMPLEMENTATION_LASX #include @@ -5809,7 +5916,7 @@ namespace simdjson { // table modified and copied from } // namespace internal } // namespace simdjson -#endif // SIMDJSON_IMPLEMENTATION_ARM64 || SIMDJSON_IMPLEMENTATION_ICELAKE || SIMDJSON_IMPLEMENTATION_HASWELL || SIMDJSON_IMPLEMENTATION_WESTMERE || SIMDJSON_IMPLEMENTATION_PPC64 +#endif // SIMDJSON_IMPLEMENTATION_ARM64 || SIMDJSON_IMPLEMENTATION_ICELAKE || SIMDJSON_IMPLEMENTATION_HASWELL || SIMDJSON_IMPLEMENTATION_WESTMERE || SIMDJSON_IMPLEMENTATION_PPC64 || SIMDJSON_IMPLEMENTATION_LSX || SIMDJSON_IMPLEMENTATION_LASX #endif // SIMDJSON_SRC_SIMDPRUNE_TABLES_CPP /* end file internal/simdprune_tables.cpp */ @@ -5976,7 +6083,7 @@ namespace simdjson { * Unescape a valid UTF-8 string from src to dst, stopping at a final unescaped quote. There * must be an unescaped quote terminating the string. It returns the final output * position as pointer. In case of error (e.g., the string has bad escaped codes), - * then null_nullptrptr is returned. It is assumed that the output buffer is large + * then null_ptr is returned. It is assumed that the output buffer is large * enough. E.g., if src points at 'joe"', then dst needs to have four free bytes + * SIMDJSON_PADDING bytes. * @@ -5993,7 +6100,7 @@ namespace simdjson { * Unescape a NON-valid UTF-8 string from src to dst, stopping at a final unescaped quote. There * must be an unescaped quote terminating the string. It returns the final output * position as pointer. In case of error (e.g., the string has bad escaped codes), - * then null_nullptrptr is returned. It is assumed that the output buffer is large + * then null_ptr is returned. It is assumed that the output buffer is large * enough. E.g., if src points at 'joe"', then dst needs to have four free bytes + * SIMDJSON_PADDING bytes. * @@ -6047,14 +6154,14 @@ namespace simdjson { * * @return Current capacity, in bytes. */ - simdjson_inline size_t capacity() const noexcept; + simdjson_pure simdjson_inline size_t capacity() const noexcept; /** * The maximum level of nested object and arrays supported by this parser. * * @return Maximum depth, in bytes. */ - simdjson_inline size_t max_depth() const noexcept; + simdjson_pure simdjson_inline size_t max_depth() const noexcept; /** * Ensure this parser has enough memory to process JSON documents up to `capacity` bytes in length @@ -6095,11 +6202,11 @@ namespace simdjson { simdjson_inline dom_parser_implementation::dom_parser_implementation(dom_parser_implementation&& other) noexcept = default; simdjson_inline dom_parser_implementation& dom_parser_implementation::operator=(dom_parser_implementation&& other) noexcept = default; - simdjson_inline size_t dom_parser_implementation::capacity() const noexcept { + simdjson_pure simdjson_inline size_t dom_parser_implementation::capacity() const noexcept { return _capacity; } - simdjson_inline size_t dom_parser_implementation::max_depth() const noexcept { + simdjson_pure simdjson_inline size_t dom_parser_implementation::max_depth() const noexcept { return _max_depth; } @@ -6171,7 +6278,7 @@ namespace simdjson { * * @return the name of the implementation, e.g. "haswell", "westmere", "arm64". */ - virtual const std::string& name() const { return _name; } + virtual std::string name() const { return std::string(_name); } /** * The description of this implementation. @@ -6181,7 +6288,7 @@ namespace simdjson { * * @return the description of the implementation, e.g. "Intel/AMD AVX2", "Intel/AMD SSE4.2", "ARM NEON". */ - virtual const std::string& description() const { return _description; } + virtual std::string description() const { return std::string(_description); } /** * The instruction sets this implementation is compiled against @@ -6257,18 +6364,19 @@ namespace simdjson { _required_instruction_sets(required_instruction_sets) { } - virtual ~implementation() = default; + protected: + ~implementation() = default; private: /** * The name of this implementation. */ - const std::string _name; + std::string_view _name; /** * The description of this implementation. */ - const std::string _description; + std::string_view _description; /** * Instruction sets required for this implementation. @@ -6417,7 +6525,9 @@ namespace simdjson { AVX512CD = 0x2000, AVX512BW = 0x4000, AVX512VL = 0x8000, - AVX512VBMI2 = 0x10000 + AVX512VBMI2 = 0x10000, + LSX = 0x20000, + LASX = 0x40000, }; } // namespace internal @@ -6742,7 +6852,7 @@ namespace simdjson { return instruction_set::ALTIVEC; } -#elif defined(__aarch64__) || defined(_M_ARM64) +#elif defined(__aarch64__) || defined(_M_ARM64) || defined(_M_ARM64EC) static inline uint32_t detect_supported_architectures() { return instruction_set::NEON; @@ -6896,6 +7006,19 @@ namespace simdjson { return host_isa; } + +#elif defined(__loongarch_sx) && !defined(__loongarch_asx) + + static inline uint32_t detect_supported_architectures() { + return instruction_set::LSX; + } + +#elif defined(__loongarch_asx) + + static inline uint32_t detect_supported_architectures() { + return instruction_set::LASX; + } + #else // fallback @@ -6913,6 +7036,7 @@ namespace simdjson { /* end file internal/isadetection.h */ #include +#include namespace simdjson { @@ -7221,12 +7345,141 @@ namespace simdjson { } // namespace simdjson #endif // SIMDJSON_IMPLEMENTATION_WESTMERE +#if SIMDJSON_IMPLEMENTATION_LSX +/* including simdjson/lsx/implementation.h: #include */ +/* begin file simdjson/lsx/implementation.h */ +#ifndef SIMDJSON_LSX_IMPLEMENTATION_H +#define SIMDJSON_LSX_IMPLEMENTATION_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/implementation.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/instruction_set.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lsx { + + /** + * @private + */ + class implementation final : public simdjson::implementation { + public: + simdjson_inline implementation() : simdjson::implementation("lsx", "LoongArch SX", internal::instruction_set::LSX) {} + simdjson_warn_unused error_code create_dom_parser_implementation( + size_t capacity, + size_t max_length, + std::unique_ptr& dst + ) const noexcept final; + simdjson_warn_unused error_code minify(const uint8_t* buf, size_t len, uint8_t* dst, size_t& dst_len) const noexcept final; + simdjson_warn_unused bool validate_utf8(const char* buf, size_t len) const noexcept final; + }; + + } // namespace lsx +} // namespace simdjson + +#endif // SIMDJSON_LSX_IMPLEMENTATION_H +/* end file simdjson/lsx/implementation.h */ +namespace simdjson { + namespace internal { + static const simdjson::lsx::implementation* get_lsx_singleton() { + static const simdjson::lsx::implementation lsx_singleton{}; + return &lsx_singleton; + } + } // namespace internal +} // namespace simdjson +#endif // SIMDJSON_IMPLEMENTATION_LSX + +#if SIMDJSON_IMPLEMENTATION_LASX +/* including simdjson/lasx/implementation.h: #include */ +/* begin file simdjson/lasx/implementation.h */ +#ifndef SIMDJSON_LASX_IMPLEMENTATION_H +#define SIMDJSON_LASX_IMPLEMENTATION_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/implementation.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/instruction_set.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lasx { + + /** + * @private + */ + class implementation final : public simdjson::implementation { + public: + simdjson_inline implementation() : simdjson::implementation("lasx", "LoongArch ASX", internal::instruction_set::LASX) {} + simdjson_warn_unused error_code create_dom_parser_implementation( + size_t capacity, + size_t max_length, + std::unique_ptr& dst + ) const noexcept final; + simdjson_warn_unused error_code minify(const uint8_t* buf, size_t len, uint8_t* dst, size_t& dst_len) const noexcept final; + simdjson_warn_unused bool validate_utf8(const char* buf, size_t len) const noexcept final; + }; + + } // namespace lasx +} // namespace simdjson + +#endif // SIMDJSON_LASX_IMPLEMENTATION_H +/* end file simdjson/lasx/implementation.h */ +namespace simdjson { + namespace internal { + static const simdjson::lasx::implementation* get_lasx_singleton() { + static const simdjson::lasx::implementation lasx_singleton{}; + return &lasx_singleton; + } + } // namespace internal +} // namespace simdjson +#endif // SIMDJSON_IMPLEMENTATION_LASX + /* undefining SIMDJSON_CONDITIONAL_INCLUDE */ #undef SIMDJSON_CONDITIONAL_INCLUDE namespace simdjson { namespace internal { + // When there is a single implementation, we should not pay a price + // for dispatching to the best implementation. We should just use the + // one we have. This is a compile-time check. +#define SIMDJSON_SINGLE_IMPLEMENTATION (SIMDJSON_IMPLEMENTATION_ICELAKE \ + + SIMDJSON_IMPLEMENTATION_HASWELL + SIMDJSON_IMPLEMENTATION_WESTMERE \ + + SIMDJSON_IMPLEMENTATION_ARM64 + SIMDJSON_IMPLEMENTATION_PPC64 \ + + SIMDJSON_IMPLEMENTATION_LSX + SIMDJSON_IMPLEMENTATION_LASX \ + + SIMDJSON_IMPLEMENTATION_FALLBACK == 1) + +#if SIMDJSON_SINGLE_IMPLEMENTATION + static const implementation* get_single_implementation() { + return +#if SIMDJSON_IMPLEMENTATION_ICELAKE + get_icelake_singleton(); +#endif +#if SIMDJSON_IMPLEMENTATION_HASWELL + get_haswell_singleton(); +#endif +#if SIMDJSON_IMPLEMENTATION_WESTMERE + get_westmere_singleton(); +#endif +#if SIMDJSON_IMPLEMENTATION_ARM64 + get_arm64_singleton(); +#endif +#if SIMDJSON_IMPLEMENTATION_PPC64 + get_ppc64_singleton(); +#endif +#if SIMDJSON_IMPLEMENTATION_LSX + get_lsx_singleton(); +#endif +#if SIMDJSON_IMPLEMENTATION_LASX + get_lasx_singleton(); +#endif +#if SIMDJSON_IMPLEMENTATION_FALLBACK + get_fallback_singleton(); +#endif + } +#endif + // Static array of known implementations. We're hoping these get baked into the executable // without requiring a static initializer. @@ -7235,8 +7488,8 @@ namespace simdjson { */ class detect_best_supported_implementation_on_first_use final : public implementation { public: - const std::string& name() const noexcept final { return set_best()->name(); } - const std::string& description() const noexcept final { return set_best()->description(); } + std::string name() const noexcept final { return set_best()->name(); } + std::string description() const noexcept final { return set_best()->description(); } uint32_t required_instruction_sets() const noexcept final { return set_best()->required_instruction_sets(); } simdjson_warn_unused error_code create_dom_parser_implementation( size_t capacity, @@ -7256,6 +7509,8 @@ namespace simdjson { const implementation* set_best() const noexcept; }; + static_assert(std::is_trivially_destructible::value, "detect_best_supported_implementation_on_first_use should be trivially destructible"); + static const std::initializer_list& get_available_implementation_pointers() { static const std::initializer_list available_implementation_pointers{ #if SIMDJSON_IMPLEMENTATION_ICELAKE @@ -7273,6 +7528,12 @@ namespace simdjson { #if SIMDJSON_IMPLEMENTATION_PPC64 get_ppc64_singleton(), #endif + #if SIMDJSON_IMPLEMENTATION_LSX + get_lsx_singleton(), + #endif + #if SIMDJSON_IMPLEMENTATION_LASX + get_lasx_singleton(), + #endif #if SIMDJSON_IMPLEMENTATION_FALLBACK get_fallback_singleton(), #endif @@ -7306,6 +7567,8 @@ namespace simdjson { unsupported_implementation() : implementation("unsupported", "Unsupported CPU (no detected SIMD instructions)", 0) {} }; + static_assert(std::is_trivially_destructible::value, "unsupported_singleton should be trivially destructible"); + const unsupported_implementation* get_unsupported_singleton() { static const unsupported_implementation unsupported_singleton{}; return &unsupported_singleton; @@ -7357,9 +7620,16 @@ namespace simdjson { } SIMDJSON_DLLIMPORTEXPORT internal::atomic_ptr& get_active_implementation() { +#if SIMDJSON_SINGLE_IMPLEMENTATION + // We immediately select the only implementation we have, skipping the + // detect_best_supported_implementation_on_first_use_singleton. + static internal::atomic_ptr active_implementation{ internal::get_single_implementation() }; + return active_implementation; +#else static const internal::detect_best_supported_implementation_on_first_use detect_best_supported_implementation_on_first_use_singleton; static internal::atomic_ptr active_implementation{ &detect_best_supported_implementation_on_first_use_singleton }; return active_implementation; +#endif } simdjson_warn_unused error_code minify(const char* buf, size_t len, char* dst, size_t& dst_len) noexcept { @@ -7616,10 +7886,10 @@ namespace simdjson { #include -#if _M_ARM64 +#if SIMDJSON_REGULAR_VISUAL_STUDIO && SIMDJSON_IS_ARM64 // __umulh requires intrin.h #include -#endif // _M_ARM64 +#endif // SIMDJSON_REGULAR_VISUAL_STUDIO && SIMDJSON_IS_ARM64 namespace simdjson { namespace arm64 { @@ -7639,13 +7909,13 @@ namespace simdjson { simdjson_inline internal::value128 full_multiplication(uint64_t value1, uint64_t value2) { internal::value128 answer; #if SIMDJSON_REGULAR_VISUAL_STUDIO || SIMDJSON_IS_32BITS -#ifdef _M_ARM64 +#if SIMDJSON_IS_ARM64 // ARM64 has native support for 64-bit multiplications, no need to emultate answer.high = __umulh(value1, value2); answer.low = value1 * value2; #else answer.low = _umul128(value1, value2, &answer.high); // _umul128 not available on ARM64 -#endif // _M_ARM64 +#endif // SIMDJSON_IS_ARM64 #else // SIMDJSON_REGULAR_VISUAL_STUDIO || SIMDJSON_IS_32BITS __uint128_t r = (static_cast<__uint128_t>(value1)) * value2; answer.low = uint64_t(r); @@ -7800,7 +8070,7 @@ namespace simdjson { tmp = vpaddq_u8(tmp, tmp); return vgetq_lane_u16(vreinterpretq_u16_u8(tmp), 0); } - simdjson_inline bool any() const { return vmaxvq_u8(*this) != 0; } + simdjson_inline bool any() const { return vmaxvq_u32(vreinterpretq_u32_u8(*this)) != 0; } }; // Unsigned bytes @@ -8248,6 +8518,10 @@ namespace simdjson { /* amalgamation skipped (editor-only): #include "simdjson/arm64/begin.h" */ /* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_PPC64 */ /* amalgamation skipped (editor-only): #include "simdjson/ppc64/begin.h" */ +/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_LSX */ +/* amalgamation skipped (editor-only): #include "simdjson/lsx/begin.h" */ +/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_LASX */ +/* amalgamation skipped (editor-only): #include "simdjson/lasx/begin.h" */ /* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_FALLBACK */ /* amalgamation skipped (editor-only): #include "simdjson/fallback/begin.h" */ /* amalgamation skipped (editor-only): #else */ @@ -8268,7 +8542,8 @@ namespace simdjson { enum class number_type { floating_point_number = 1, /// a binary64 number signed_integer, /// a signed integer that fits in a 64-bit word using two's complement - unsigned_integer /// a positive integer larger or equal to 1<<63 + unsigned_integer, /// a positive integer larger or equal to 1<<63 + big_integer /// a big integer that does not fit in a 64-bit word }; } // namespace arm64 @@ -8718,11 +8993,13 @@ namespace simdjson { #define WRITE_INTEGER(VALUE, SRC, WRITER) (found_integer((VALUE), (SRC)), (WRITER).append_s64((VALUE))) #define WRITE_UNSIGNED(VALUE, SRC, WRITER) (found_unsigned_integer((VALUE), (SRC)), (WRITER).append_u64((VALUE))) #define WRITE_DOUBLE(VALUE, SRC, WRITER) (found_float((VALUE), (SRC)), (WRITER).append_double((VALUE))) +#define BIGINT_NUMBER(SRC) (found_invalid_number((SRC)), BIGINT_ERROR) #else #define INVALID_NUMBER(SRC) (NUMBER_ERROR) #define WRITE_INTEGER(VALUE, SRC, WRITER) (WRITER).append_s64((VALUE)) #define WRITE_UNSIGNED(VALUE, SRC, WRITER) (WRITER).append_u64((VALUE)) #define WRITE_DOUBLE(VALUE, SRC, WRITER) (WRITER).append_double((VALUE)) +#define BIGINT_NUMBER(SRC) (BIGINT_ERROR) #endif namespace { @@ -9054,6 +9331,10 @@ namespace simdjson { return true; } + simdjson_inline bool is_digit(const uint8_t c) { + return static_cast(c - '0') <= 9; + } + simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t* const src, const uint8_t*& p, uint64_t& i, int64_t& exponent) { // we continue with the fiction that we have an integer. If the // floating point number is representable as x * 10^z for some integer @@ -9111,7 +9392,7 @@ namespace simdjson { // If there were more than 18 digits, we may have overflowed the integer. We have to do // something!!!! if (simdjson_unlikely(p > start_exp + 18)) { - // Skip leading zeroes: 1e000000000000000000001 is technically valid and doesn't overflow + // Skip leading zeroes: 1e000000000000000000001 is technically valid and does not overflow while (*start_exp == '0') { start_exp++; } // 19 digits could overflow int64_t and is kind of absurd anyway. We don't // support exponents smaller than -999,999,999,999,999,999 and bigger @@ -9133,6 +9414,23 @@ namespace simdjson { return SUCCESS; } + simdjson_inline bool check_if_integer(const uint8_t* const src, size_t max_length) { + const uint8_t* const srcend = src + max_length; + bool negative = (*src == '-'); // we can always read at least one character after the '-' + const uint8_t* p = src + uint8_t(negative); + if (p == srcend) { return false; } + if (*p == '0') { + ++p; + if (p == srcend) { return true; } + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return false; } + return true; + } + while (p != srcend && is_digit(*p)) { ++p; } + if (p == srcend) { return true; } + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return false; } + return true; + } + simdjson_inline size_t significant_digits(const uint8_t* start_digits, size_t digit_count) { // It is possible that the integer had an overflow. // We have to handle the case where we have 0.0000somenumber. @@ -9145,11 +9443,8 @@ namespace simdjson { } // unnamed namespace /** @private */ - template - error_code slow_float_parsing(simdjson_unused const uint8_t* src, W writer) { - double d; - if (parse_float_fallback(src, &d)) { - writer.append_double(d); + static error_code slow_float_parsing(simdjson_unused const uint8_t* src, double* answer) { + if (parse_float_fallback(src, answer)) { return SUCCESS; } return INVALID_NUMBER(src); @@ -9173,13 +9468,13 @@ namespace simdjson { // 10000000000000000000000000000000000000000000e+308 // 3.1415926535897932384626433832795028841971693993751 // - // NOTE: This makes a *copy* of the writer and passes it to slow_float_parsing. This happens - // because slow_float_parsing is a non-inlined function. If we passed our writer reference to - // it, it would force it to be stored in memory, preventing the compiler from picking it apart - // and putting into registers. i.e. if we pass it as reference, it gets slow. - // This is what forces the skip_double, as well. - error_code error = slow_float_parsing(src, writer); - writer.skip_double(); + // NOTE: We do not pass a reference to the to slow_float_parsing. If we passed our writer + // reference to it, it would force it to be stored in memory, preventing the compiler from + // picking it apart and putting into registers. i.e. if we pass it as reference, + // it gets slow. + double d; + error_code error = slow_float_parsing(src, &d); + writer.append_double(d); return error; } // NOTE: it's weird that the simdjson_unlikely() only wraps half the if, but it seems to get slower any other @@ -9211,6 +9506,18 @@ namespace simdjson { return SUCCESS; } + // parse the number at src + // define JSON_TEST_NUMBERS for unit testing + // + // It is assumed that the number is followed by a structural ({,},],[) character + // or a white space character. If that is not the case (e.g., when the JSON + // document is made of a single number), then it is necessary to copy the + // content and append a space before calling this function. + // + // Our objective is accurate parsing (ULP of 0) at high speed. + template + simdjson_inline error_code parse_number(const uint8_t* const src, W& writer); + // for performance analysis, it is sometimes useful to skip parsing #ifdef SIMDJSON_SKIPNUMBERPARSING @@ -9289,11 +9596,11 @@ namespace simdjson { // The longest positive 64-bit number is 20 digits. // We do it this way so we don't trigger this branch unless we must. size_t longest_digit_count = negative ? 19 : 20; - if (digit_count > longest_digit_count) { return INVALID_NUMBER(src); } + if (digit_count > longest_digit_count) { return BIGINT_NUMBER(src); } if (digit_count == longest_digit_count) { if (negative) { // Anything negative above INT64_MAX+1 is invalid - if (i > uint64_t(INT64_MAX) + 1) { return INVALID_NUMBER(src); } + if (i > uint64_t(INT64_MAX) + 1) { return BIGINT_NUMBER(src); } WRITE_INTEGER(~i + 1, src, writer); if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return INVALID_NUMBER(src); } return SUCCESS; @@ -9313,7 +9620,7 @@ namespace simdjson { else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INVALID_NUMBER(src); } } - // Write unsigned if it doesn't fit in a signed integer. + // Write unsigned if it does not fit in a signed integer. if (i > uint64_t(INT64_MAX)) { WRITE_UNSIGNED(i, src, writer); } @@ -9766,19 +10073,32 @@ namespace simdjson { src += uint8_t(negative); const uint8_t* p = src; while (static_cast(*p - '0') <= 9) { p++; } + size_t digit_count = size_t(p - src); if (p == src) { return NUMBER_ERROR; } if (jsoncharutils::is_structural_or_whitespace(*p)) { + static const uint8_t* smaller_big_integer = reinterpret_cast("9223372036854775808"); // We have an integer. + if (simdjson_unlikely(digit_count > 20)) { + return number_type::big_integer; + } // If the number is negative and valid, it must be a signed integer. - if (negative) { return number_type::signed_integer; } + if (negative) { + if (simdjson_unlikely(digit_count > 19)) return number_type::big_integer; + if (simdjson_unlikely(digit_count == 19 && memcmp(src, smaller_big_integer, 19) > 0)) { + return number_type::big_integer; + } + return number_type::signed_integer; + } + // Let us check if we have a big integer (>=2**64). + static const uint8_t* two_to_sixtyfour = reinterpret_cast("18446744073709551616"); + if ((digit_count > 20) || (digit_count == 20 && memcmp(src, two_to_sixtyfour, 20) >= 0)) { + return number_type::big_integer; + } + // The number is positive and smaller than 18446744073709551616 (or 2**64). // We want values larger or equal to 9223372036854775808 to be unsigned // integers, and the other values to be signed integers. - int digit_count = int(p - src); - if (digit_count >= 19) { - const uint8_t* smaller_big_integer = reinterpret_cast("9223372036854775808"); - if ((digit_count >= 20) || (memcmp(src, smaller_big_integer, 19) >= 0)) { - return number_type::unsigned_integer; - } + if ((digit_count == 20) || (digit_count >= 19 && memcmp(src, smaller_big_integer, 19) >= 0)) { + return number_type::unsigned_integer; } return number_type::signed_integer; } @@ -9958,6 +10278,7 @@ namespace simdjson { case number_type::signed_integer: out << "integer in [-9223372036854775808,9223372036854775808)"; break; case number_type::unsigned_integer: out << "unsigned integer in [9223372036854775808,18446744073709551616)"; break; case number_type::floating_point_number: out << "floating-point number (binary64)"; break; + case number_type::big_integer: out << "big integer"; break; default: SIMDJSON_UNREACHABLE(); } return out; @@ -10330,10 +10651,10 @@ namespace simdjson { #include -#if _M_ARM64 +#if SIMDJSON_REGULAR_VISUAL_STUDIO && SIMDJSON_IS_ARM64 // __umulh requires intrin.h #include -#endif // _M_ARM64 +#endif // SIMDJSON_REGULAR_VISUAL_STUDIO && SIMDJSON_IS_ARM64 namespace simdjson { namespace arm64 { @@ -10353,13 +10674,13 @@ namespace simdjson { simdjson_inline internal::value128 full_multiplication(uint64_t value1, uint64_t value2) { internal::value128 answer; #if SIMDJSON_REGULAR_VISUAL_STUDIO || SIMDJSON_IS_32BITS -#ifdef _M_ARM64 +#if SIMDJSON_IS_ARM64 // ARM64 has native support for 64-bit multiplications, no need to emultate answer.high = __umulh(value1, value2); answer.low = value1 * value2; #else answer.low = _umul128(value1, value2, &answer.high); // _umul128 not available on ARM64 -#endif // _M_ARM64 +#endif // SIMDJSON_IS_ARM64 #else // SIMDJSON_REGULAR_VISUAL_STUDIO || SIMDJSON_IS_32BITS __uint128_t r = (static_cast<__uint128_t>(value1)) * value2; answer.low = uint64_t(r); @@ -10514,7 +10835,7 @@ namespace simdjson { tmp = vpaddq_u8(tmp, tmp); return vgetq_lane_u16(vreinterpretq_u16_u8(tmp), 0); } - simdjson_inline bool any() const { return vmaxvq_u8(*this) != 0; } + simdjson_inline bool any() const { return vmaxvq_u32(vreinterpretq_u32_u8(*this)) != 0; } }; // Unsigned bytes @@ -10978,7 +11299,7 @@ namespace simdjson { namespace arm64 { namespace { - simdjson_inline simd8 must_be_2_3_continuation(const simd8 prev2, const simd8 prev3); + simdjson_inline simd8 must_be_2_3_continuation(const simd8 prev2, const simd8 prev3); simdjson_inline bool is_ascii(const simd8x64& input); } // unnamed namespace @@ -11545,7 +11866,7 @@ namespace simdjson { const simd8 prev_input, const simd8 sc) { simd8 prev2 = input.prev<2>(prev_input); simd8 prev3 = input.prev<3>(prev_input); - simd8 must23 = simd8(must_be_2_3_continuation(prev2, prev3)); + simd8 must23 = must_be_2_3_continuation(prev2, prev3); simd8 must23_80 = must23 & uint8_t(0x80); return must23_80 ^ sc; } @@ -12224,7 +12545,7 @@ namespace simdjson { // up enough CPU: the second half of the functions is highly serial, only using 1 execution core // at a time. The second input's scans has some dependency on the first ones finishing it, but // they can make a lot of progress before they need that information. - // 3. Step 1 doesn't use enough capacity, so we run some extra stuff while we're waiting for that + // 3. Step 1 does not use enough capacity, so we run some extra stuff while we're waiting for that // to finish: utf-8 checks and generating the output from the last iteration. // // The reason we run 2 inputs at a time, is steps 2 and 3 are *still* not enough to soak up all @@ -12304,7 +12625,7 @@ namespace simdjson { } parser.n_structural_indexes = uint32_t(indexer.tail - parser.structural_indexes.get()); /*** - * The On Demand API requires special padding. + * The On-Demand API requires special padding. * * This is related to https://github.com/simdjson/simdjson/issues/906 * Basically, we want to make sure that if the parsing continues beyond the last (valid) @@ -13094,8 +13415,8 @@ namespace simdjson { // Use the default Unicode Character 'REPLACEMENT CHARACTER' (U+FFFD) constexpr uint32_t substitution_code_point = 0xfffd; // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the - // conversion isn't valid; we defer the check for this to inside the - // multilingual plane check + // conversion is not valid; we defer the check for this to inside the + // multilingual plane check. uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2); *src_ptr += 6; @@ -13155,8 +13476,8 @@ namespace simdjson { // It is not ideal that this function is nearly identical to handle_unicode_codepoint. // // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the - // conversion isn't valid; we defer the check for this to inside the - // multilingual plane check + // conversion is not valid; we defer the check for this to inside the + // multilingual plane check. uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2); *src_ptr += 6; // If we found a high surrogate, we must @@ -13187,7 +13508,7 @@ namespace simdjson { * Unescape a valid UTF-8 string from src to dst, stopping at a final unescaped quote. There * must be an unescaped quote terminating the string. It returns the final output * position as pointer. In case of error (e.g., the string has bad escaped codes), - * then null_nullptrptr is returned. It is assumed that the output buffer is large + * then null_ptr is returned. It is assumed that the output buffer is large * enough. E.g., if src points at 'joe"', then dst needs to have four free bytes + * SIMDJSON_PADDING bytes. */ @@ -13755,10 +14076,10 @@ namespace simdjson { return is_second_byte ^ is_third_byte ^ is_fourth_byte; } - simdjson_inline simd8 must_be_2_3_continuation(const simd8 prev2, const simd8 prev3) { - simd8 is_third_byte = prev2 >= uint8_t(0xe0u); - simd8 is_fourth_byte = prev3 >= uint8_t(0xf0u); - return is_third_byte ^ is_fourth_byte; + simdjson_inline simd8 must_be_2_3_continuation(const simd8 prev2, const simd8 prev3) { + simd8 is_third_byte = prev2.saturating_sub(0xe0u - 0x80); // Only 111_____ will be >= 0x80 + simd8 is_fourth_byte = prev3.saturating_sub(0xf0u - 0x80); // Only 1111____ will be >= 0x80 + return is_third_byte | is_fourth_byte; } } // unnamed namespace @@ -13828,204 +14149,271 @@ namespace simdjson { #endif // SIMDJSON_SRC_ARM64_CPP /* end file arm64.cpp */ #endif -#if SIMDJSON_IMPLEMENTATION_FALLBACK -/* including fallback.cpp: #include */ -/* begin file fallback.cpp */ -#ifndef SIMDJSON_SRC_FALLBACK_CPP -#define SIMDJSON_SRC_FALLBACK_CPP +#if SIMDJSON_IMPLEMENTATION_HASWELL +/* including haswell.cpp: #include */ +/* begin file haswell.cpp */ +#ifndef SIMDJSON_SRC_HASWELL_CPP +#define SIMDJSON_SRC_HASWELL_CPP /* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ /* amalgamation skipped (editor-only): #include */ /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ -/* including simdjson/fallback.h: #include */ -/* begin file simdjson/fallback.h */ -#ifndef SIMDJSON_FALLBACK_H -#define SIMDJSON_FALLBACK_H +/* including simdjson/haswell.h: #include */ +/* begin file simdjson/haswell.h */ +#ifndef SIMDJSON_HASWELL_H +#define SIMDJSON_HASWELL_H -/* including simdjson/fallback/begin.h: #include "simdjson/fallback/begin.h" */ -/* begin file simdjson/fallback/begin.h */ -/* defining SIMDJSON_IMPLEMENTATION to "fallback" */ -#define SIMDJSON_IMPLEMENTATION fallback -/* including simdjson/fallback/base.h: #include "simdjson/fallback/base.h" */ -/* begin file simdjson/fallback/base.h */ -#ifndef SIMDJSON_FALLBACK_BASE_H -#define SIMDJSON_FALLBACK_BASE_H +/* including simdjson/haswell/begin.h: #include "simdjson/haswell/begin.h" */ +/* begin file simdjson/haswell/begin.h */ +/* defining SIMDJSON_IMPLEMENTATION to "haswell" */ +#define SIMDJSON_IMPLEMENTATION haswell + +/* including simdjson/haswell/base.h: #include "simdjson/haswell/base.h" */ +/* begin file simdjson/haswell/base.h */ +#ifndef SIMDJSON_HASWELL_BASE_H +#define SIMDJSON_HASWELL_BASE_H /* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ /* amalgamation skipped (editor-only): #include "simdjson/base.h" */ /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ +// The constructor may be executed on any host, so we take care not to use SIMDJSON_TARGET_HASWELL namespace simdjson { /** - * Fallback implementation (runs on any machine). + * Implementation for Haswell (Intel AVX2). */ - namespace fallback { + namespace haswell { class implementation; - } // namespace fallback + namespace { + namespace simd { + template struct simd8; + template struct simd8x64; + } // namespace simd + } // unnamed namespace + + } // namespace haswell } // namespace simdjson -#endif // SIMDJSON_FALLBACK_BASE_H -/* end file simdjson/fallback/base.h */ -/* including simdjson/fallback/bitmanipulation.h: #include "simdjson/fallback/bitmanipulation.h" */ -/* begin file simdjson/fallback/bitmanipulation.h */ -#ifndef SIMDJSON_FALLBACK_BITMANIPULATION_H -#define SIMDJSON_FALLBACK_BITMANIPULATION_H +#endif // SIMDJSON_HASWELL_BASE_H +/* end file simdjson/haswell/base.h */ +/* including simdjson/haswell/intrinsics.h: #include "simdjson/haswell/intrinsics.h" */ +/* begin file simdjson/haswell/intrinsics.h */ +#ifndef SIMDJSON_HASWELL_INTRINSICS_H +#define SIMDJSON_HASWELL_INTRINSICS_H /* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/fallback/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/haswell/base.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +#if SIMDJSON_VISUAL_STUDIO +// under clang within visual studio, this will include +#include // visual studio or clang +#else +#include // elsewhere +#endif // SIMDJSON_VISUAL_STUDIO + +#if SIMDJSON_CLANG_VISUAL_STUDIO +/** + * You are not supposed, normally, to include these + * headers directly. Instead you should either include intrin.h + * or x86intrin.h. However, when compiling with clang + * under Windows (i.e., when _MSC_VER is set), these headers + * only get included *if* the corresponding features are detected + * from macros: + * e.g., if __AVX2__ is set... in turn, we normally set these + * macros by compiling against the corresponding architecture + * (e.g., arch:AVX2, -mavx2, etc.) which compiles the whole + * software with these advanced instructions. In simdjson, we + * want to compile the whole program for a generic target, + * and only target our specific kernels. As a workaround, + * we directly include the needed headers. These headers would + * normally guard against such usage, but we carefully included + * (or ) before, so the headers + * are fooled. + */ +#include // for _blsr_u64 +#include // for __lzcnt64 +#include // for most things (AVX2, AVX512, _popcnt64) +#include +#include +#include +#include +#include // for _mm_clmulepi64_si128 + // unfortunately, we may not get _blsr_u64, but, thankfully, clang + // has it as a macro. +#ifndef _blsr_u64 +// we roll our own +#define _blsr_u64(n) ((n - 1) & n) +#endif // _blsr_u64 +#endif // SIMDJSON_CLANG_VISUAL_STUDIO + +static_assert(sizeof(__m256i) <= simdjson::SIMDJSON_PADDING, "insufficient padding for haswell kernel."); + +#endif // SIMDJSON_HASWELL_INTRINSICS_H +/* end file simdjson/haswell/intrinsics.h */ + +#if !SIMDJSON_CAN_ALWAYS_RUN_HASWELL +SIMDJSON_TARGET_REGION("avx2,bmi,pclmul,lzcnt,popcnt") +#endif + +/* including simdjson/haswell/bitmanipulation.h: #include "simdjson/haswell/bitmanipulation.h" */ +/* begin file simdjson/haswell/bitmanipulation.h */ +#ifndef SIMDJSON_HASWELL_BITMANIPULATION_H +#define SIMDJSON_HASWELL_BITMANIPULATION_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/haswell/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/haswell/intrinsics.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/haswell/bitmask.h" */ /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ namespace simdjson { - namespace fallback { + namespace haswell { namespace { -#if defined(_MSC_VER) && !defined(_M_ARM64) && !defined(_M_X64) - static inline unsigned char _BitScanForward64(unsigned long* ret, uint64_t x) { - unsigned long x0 = (unsigned long)x, top, bottom; - _BitScanForward(&top, (unsigned long)(x >> 32)); - _BitScanForward(&bottom, x0); - *ret = x0 ? bottom : 32 + top; - return x != 0; + // We sometimes call trailing_zero on inputs that are zero, + // but the algorithms do not end up using the returned value. + // Sadly, sanitizers are not smart enough to figure it out. + SIMDJSON_NO_SANITIZE_UNDEFINED + // This function can be used safely even if not all bytes have been + // initialized. + // See issue https://github.com/simdjson/simdjson/issues/1965 + SIMDJSON_NO_SANITIZE_MEMORY + simdjson_inline int trailing_zeroes(uint64_t input_num) { +#if SIMDJSON_REGULAR_VISUAL_STUDIO + return (int)_tzcnt_u64(input_num); +#else // SIMDJSON_REGULAR_VISUAL_STUDIO + //////// + // You might expect the next line to be equivalent to + // return (int)_tzcnt_u64(input_num); + // but the generated code differs and might be less efficient? + //////// + return __builtin_ctzll(input_num); +#endif // SIMDJSON_REGULAR_VISUAL_STUDIO } - static unsigned char _BitScanReverse64(unsigned long* ret, uint64_t x) { - unsigned long x1 = (unsigned long)(x >> 32), top, bottom; - _BitScanReverse(&top, x1); - _BitScanReverse(&bottom, (unsigned long)x); - *ret = x1 ? top + 32 : bottom; - return x != 0; + + /* result might be undefined when input_num is zero */ + simdjson_inline uint64_t clear_lowest_bit(uint64_t input_num) { + return _blsr_u64(input_num); } -#endif /* result might be undefined when input_num is zero */ simdjson_inline int leading_zeroes(uint64_t input_num) { -#ifdef _MSC_VER - unsigned long leading_zero = 0; - // Search the mask data from most significant bit (MSB) - // to least significant bit (LSB) for a set bit (1). - if (_BitScanReverse64(&leading_zero, input_num)) - return (int)(63 - leading_zero); - else - return 64; + return int(_lzcnt_u64(input_num)); + } + +#if SIMDJSON_REGULAR_VISUAL_STUDIO + simdjson_inline unsigned __int64 count_ones(uint64_t input_num) { + // note: we do not support legacy 32-bit Windows in this kernel + return __popcnt64(input_num);// Visual Studio wants two underscores + } #else - return __builtin_clzll(input_num); -#endif// _MSC_VER + simdjson_inline long long int count_ones(uint64_t input_num) { + return _popcnt64(input_num); + } +#endif + + simdjson_inline bool add_overflow(uint64_t value1, uint64_t value2, + uint64_t* result) { +#if SIMDJSON_REGULAR_VISUAL_STUDIO + return _addcarry_u64(0, value1, value2, + reinterpret_cast(result)); +#else + return __builtin_uaddll_overflow(value1, value2, + reinterpret_cast(result)); +#endif } } // unnamed namespace - } // namespace fallback + } // namespace haswell } // namespace simdjson -#endif // SIMDJSON_FALLBACK_BITMANIPULATION_H -/* end file simdjson/fallback/bitmanipulation.h */ -/* including simdjson/fallback/stringparsing_defs.h: #include "simdjson/fallback/stringparsing_defs.h" */ -/* begin file simdjson/fallback/stringparsing_defs.h */ -#ifndef SIMDJSON_FALLBACK_STRINGPARSING_DEFS_H -#define SIMDJSON_FALLBACK_STRINGPARSING_DEFS_H +#endif // SIMDJSON_HASWELL_BITMANIPULATION_H +/* end file simdjson/haswell/bitmanipulation.h */ +/* including simdjson/haswell/bitmask.h: #include "simdjson/haswell/bitmask.h" */ +/* begin file simdjson/haswell/bitmask.h */ +#ifndef SIMDJSON_HASWELL_BITMASK_H +#define SIMDJSON_HASWELL_BITMASK_H /* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/fallback/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/haswell/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/haswell/intrinsics.h" */ /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ namespace simdjson { - namespace fallback { + namespace haswell { namespace { - // Holds backslashes and quotes locations. - struct backslash_and_quote { - public: - static constexpr uint32_t BYTES_PROCESSED = 1; - simdjson_inline static backslash_and_quote copy_and_find(const uint8_t* src, uint8_t* dst); - - simdjson_inline bool has_quote_first() { return c == '"'; } - simdjson_inline bool has_backslash() { return c == '\\'; } - simdjson_inline int quote_index() { return c == '"' ? 0 : 1; } - simdjson_inline int backslash_index() { return c == '\\' ? 0 : 1; } - - uint8_t c; - }; // struct backslash_and_quote - - simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8_t* src, uint8_t* dst) { - // store to dest unconditionally - we can overwrite the bits we don't like later - dst[0] = src[0]; - return { src[0] }; + // + // Perform a "cumulative bitwise xor," flipping bits each time a 1 is encountered. + // + // For example, prefix_xor(00100100) == 00011100 + // + simdjson_inline uint64_t prefix_xor(const uint64_t bitmask) { + // There should be no such thing with a processor supporting avx2 + // but not clmul. + __m128i all_ones = _mm_set1_epi8('\xFF'); + __m128i result = _mm_clmulepi64_si128(_mm_set_epi64x(0ULL, bitmask), all_ones, 0); + return _mm_cvtsi128_si64(result); } } // unnamed namespace - } // namespace fallback + } // namespace haswell } // namespace simdjson -#endif // SIMDJSON_FALLBACK_STRINGPARSING_DEFS_H -/* end file simdjson/fallback/stringparsing_defs.h */ -/* including simdjson/fallback/numberparsing_defs.h: #include "simdjson/fallback/numberparsing_defs.h" */ -/* begin file simdjson/fallback/numberparsing_defs.h */ -#ifndef SIMDJSON_FALLBACK_NUMBERPARSING_DEFS_H -#define SIMDJSON_FALLBACK_NUMBERPARSING_DEFS_H +#endif // SIMDJSON_HASWELL_BITMASK_H +/* end file simdjson/haswell/bitmask.h */ +/* including simdjson/haswell/numberparsing_defs.h: #include "simdjson/haswell/numberparsing_defs.h" */ +/* begin file simdjson/haswell/numberparsing_defs.h */ +#ifndef SIMDJSON_HASWELL_NUMBERPARSING_DEFS_H +#define SIMDJSON_HASWELL_NUMBERPARSING_DEFS_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/haswell/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/haswell/intrinsics.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ /* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/fallback/base.h" */ /* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ -#include - -#ifdef JSON_TEST_NUMBERS // for unit testing -void found_invalid_number(const uint8_t* buf); -void found_integer(int64_t result, const uint8_t* buf); -void found_unsigned_integer(uint64_t result, const uint8_t* buf); -void found_float(double result, const uint8_t* buf); -#endif - namespace simdjson { - namespace fallback { + namespace haswell { namespace numberparsing { - // credit: https://johnnylee-sde.github.io/Fast-numeric-string-to-int/ - /** @private */ - static simdjson_inline uint32_t parse_eight_digits_unrolled(const char* chars) { - uint64_t val; - memcpy(&val, chars, sizeof(uint64_t)); - val = (val & 0x0F0F0F0F0F0F0F0F) * 2561 >> 8; - val = (val & 0x00FF00FF00FF00FF) * 6553601 >> 16; - return uint32_t((val & 0x0000FFFF0000FFFF) * 42949672960001 >> 32); - } - /** @private */ static simdjson_inline uint32_t parse_eight_digits_unrolled(const uint8_t* chars) { - return parse_eight_digits_unrolled(reinterpret_cast(chars)); + // this actually computes *16* values so we are being wasteful. + const __m128i ascii0 = _mm_set1_epi8('0'); + const __m128i mul_1_10 = + _mm_setr_epi8(10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1); + const __m128i mul_1_100 = _mm_setr_epi16(100, 1, 100, 1, 100, 1, 100, 1); + const __m128i mul_1_10000 = + _mm_setr_epi16(10000, 1, 10000, 1, 10000, 1, 10000, 1); + const __m128i input = _mm_sub_epi8( + _mm_loadu_si128(reinterpret_cast(chars)), ascii0); + const __m128i t1 = _mm_maddubs_epi16(input, mul_1_10); + const __m128i t2 = _mm_madd_epi16(t1, mul_1_100); + const __m128i t3 = _mm_packus_epi32(t2, t2); + const __m128i t4 = _mm_madd_epi16(t3, mul_1_10000); + return _mm_cvtsi128_si32( + t4); // only captures the sum of the first 8 digits, drop the rest } -#if SIMDJSON_IS_32BITS // _umul128 for x86, arm - // this is a slow emulation routine for 32-bit - // - static simdjson_inline uint64_t __emulu(uint32_t x, uint32_t y) { - return x * (uint64_t)y; - } - static simdjson_inline uint64_t _umul128(uint64_t ab, uint64_t cd, uint64_t* hi) { - uint64_t ad = __emulu((uint32_t)(ab >> 32), (uint32_t)cd); - uint64_t bd = __emulu((uint32_t)ab, (uint32_t)cd); - uint64_t adbc = ad + __emulu((uint32_t)ab, (uint32_t)(cd >> 32)); - uint64_t adbc_carry = !!(adbc < ad); - uint64_t lo = bd + (adbc << 32); - *hi = __emulu((uint32_t)(ab >> 32), (uint32_t)(cd >> 32)) + (adbc >> 32) + - (adbc_carry << 32) + !!(lo < bd); - return lo; - } -#endif - /** @private */ simdjson_inline internal::value128 full_multiplication(uint64_t value1, uint64_t value2) { internal::value128 answer; #if SIMDJSON_REGULAR_VISUAL_STUDIO || SIMDJSON_IS_32BITS -#ifdef _M_ARM64 +#if SIMDJSON_IS_ARM64 // ARM64 has native support for 64-bit multiplications, no need to emultate answer.high = __umulh(value1, value2); answer.low = value1 * value2; #else answer.low = _umul128(value1, value2, &answer.high); // _umul128 not available on ARM64 -#endif // _M_ARM64 +#endif // SIMDJSON_IS_ARM64 #else // SIMDJSON_REGULAR_VISUAL_STUDIO || SIMDJSON_IS_32BITS __uint128_t r = (static_cast<__uint128_t>(value1)) * value2; answer.low = uint64_t(r); @@ -14035,22 +14423,448 @@ namespace simdjson { } } // namespace numberparsing - } // namespace fallback + } // namespace haswell } // namespace simdjson #define SIMDJSON_SWAR_NUMBER_PARSING 1 -#endif // SIMDJSON_FALLBACK_NUMBERPARSING_DEFS_H -/* end file simdjson/fallback/numberparsing_defs.h */ -/* end file simdjson/fallback/begin.h */ -/* including simdjson/generic/amalgamated.h for fallback: #include "simdjson/generic/amalgamated.h" */ -/* begin file simdjson/generic/amalgamated.h for fallback */ +#endif // SIMDJSON_HASWELL_NUMBERPARSING_DEFS_H +/* end file simdjson/haswell/numberparsing_defs.h */ +/* including simdjson/haswell/simd.h: #include "simdjson/haswell/simd.h" */ +/* begin file simdjson/haswell/simd.h */ +#ifndef SIMDJSON_HASWELL_SIMD_H +#define SIMDJSON_HASWELL_SIMD_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/haswell/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/haswell/intrinsics.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/haswell/bitmanipulation.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/simdprune_tables.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace haswell { + namespace { + namespace simd { + + // Forward-declared so they can be used by splat and friends. + template + struct base { + __m256i value; + + // Zero constructor + simdjson_inline base() : value{ __m256i() } {} + + // Conversion from SIMD register + simdjson_inline base(const __m256i _value) : value(_value) {} + + // Conversion to SIMD register + simdjson_inline operator const __m256i& () const { return this->value; } + simdjson_inline operator __m256i& () { return this->value; } + + // Bit operations + simdjson_inline Child operator|(const Child other) const { return _mm256_or_si256(*this, other); } + simdjson_inline Child operator&(const Child other) const { return _mm256_and_si256(*this, other); } + simdjson_inline Child operator^(const Child other) const { return _mm256_xor_si256(*this, other); } + simdjson_inline Child bit_andnot(const Child other) const { return _mm256_andnot_si256(other, *this); } + simdjson_inline Child& operator|=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast | other; return *this_cast; } + simdjson_inline Child& operator&=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast & other; return *this_cast; } + simdjson_inline Child& operator^=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast ^ other; return *this_cast; } + }; + + // Forward-declared so they can be used by splat and friends. + template + struct simd8; + + template> + struct base8 : base> { + typedef uint32_t bitmask_t; + typedef uint64_t bitmask2_t; + + simdjson_inline base8() : base>() {} + simdjson_inline base8(const __m256i _value) : base>(_value) {} + + friend simdjson_really_inline Mask operator==(const simd8 lhs, const simd8 rhs) { return _mm256_cmpeq_epi8(lhs, rhs); } + + static const int SIZE = sizeof(base::value); + + template + simdjson_inline simd8 prev(const simd8 prev_chunk) const { + return _mm256_alignr_epi8(*this, _mm256_permute2x128_si256(prev_chunk, *this, 0x21), 16 - N); + } + }; + + // SIMD byte mask type (returned by things like eq and gt) + template<> + struct simd8 : base8 { + static simdjson_inline simd8 splat(bool _value) { return _mm256_set1_epi8(uint8_t(-(!!_value))); } + + simdjson_inline simd8() : base8() {} + simdjson_inline simd8(const __m256i _value) : base8(_value) {} + // Splat constructor + simdjson_inline simd8(bool _value) : base8(splat(_value)) {} + + simdjson_inline int to_bitmask() const { return _mm256_movemask_epi8(*this); } + simdjson_inline bool any() const { return !_mm256_testz_si256(*this, *this); } + simdjson_inline simd8 operator~() const { return *this ^ true; } + }; + + template + struct base8_numeric : base8 { + static simdjson_inline simd8 splat(T _value) { return _mm256_set1_epi8(_value); } + static simdjson_inline simd8 zero() { return _mm256_setzero_si256(); } + static simdjson_inline simd8 load(const T values[32]) { + return _mm256_loadu_si256(reinterpret_cast(values)); + } + // Repeat 16 values as many times as necessary (usually for lookup tables) + static simdjson_inline simd8 repeat_16( + T v0, T v1, T v2, T v3, T v4, T v5, T v6, T v7, + T v8, T v9, T v10, T v11, T v12, T v13, T v14, T v15 + ) { + return simd8( + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15, + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15 + ); + } + + simdjson_inline base8_numeric() : base8() {} + simdjson_inline base8_numeric(const __m256i _value) : base8(_value) {} + + // Store to array + simdjson_inline void store(T dst[32]) const { return _mm256_storeu_si256(reinterpret_cast<__m256i*>(dst), *this); } + + // Addition/subtraction are the same for signed and unsigned + simdjson_inline simd8 operator+(const simd8 other) const { return _mm256_add_epi8(*this, other); } + simdjson_inline simd8 operator-(const simd8 other) const { return _mm256_sub_epi8(*this, other); } + simdjson_inline simd8& operator+=(const simd8 other) { *this = *this + other; return *static_cast*>(this); } + simdjson_inline simd8& operator-=(const simd8 other) { *this = *this - other; return *static_cast*>(this); } + + // Override to distinguish from bool version + simdjson_inline simd8 operator~() const { return *this ^ 0xFFu; } + + // Perform a lookup assuming the value is between 0 and 16 (undefined behavior for out of range values) + template + simdjson_inline simd8 lookup_16(simd8 lookup_table) const { + return _mm256_shuffle_epi8(lookup_table, *this); + } + + // Copies to 'output" all bytes corresponding to a 0 in the mask (interpreted as a bitset). + // Passing a 0 value for mask would be equivalent to writing out every byte to output. + // Only the first 32 - count_ones(mask) bytes of the result are significant but 32 bytes + // get written. + // Design consideration: it seems like a function with the + // signature simd8 compress(uint32_t mask) would be + // sensible, but the AVX ISA makes this kind of approach difficult. + template + simdjson_inline void compress(uint32_t mask, L* output) const { + using internal::thintable_epi8; + using internal::BitsSetTable256mul2; + using internal::pshufb_combine_table; + // this particular implementation was inspired by work done by @animetosho + // we do it in four steps, first 8 bytes and then second 8 bytes... + uint8_t mask1 = uint8_t(mask); // least significant 8 bits + uint8_t mask2 = uint8_t(mask >> 8); // second least significant 8 bits + uint8_t mask3 = uint8_t(mask >> 16); // ... + uint8_t mask4 = uint8_t(mask >> 24); // ... + // next line just loads the 64-bit values thintable_epi8[mask1] and + // thintable_epi8[mask2] into a 128-bit register, using only + // two instructions on most compilers. + __m256i shufmask = _mm256_set_epi64x(thintable_epi8[mask4], thintable_epi8[mask3], + thintable_epi8[mask2], thintable_epi8[mask1]); + // we increment by 0x08 the second half of the mask and so forth + shufmask = + _mm256_add_epi8(shufmask, _mm256_set_epi32(0x18181818, 0x18181818, + 0x10101010, 0x10101010, 0x08080808, 0x08080808, 0, 0)); + // this is the version "nearly pruned" + __m256i pruned = _mm256_shuffle_epi8(*this, shufmask); + // we still need to put the pieces back together. + // we compute the popcount of the first words: + int pop1 = BitsSetTable256mul2[mask1]; + int pop3 = BitsSetTable256mul2[mask3]; + + // then load the corresponding mask + // could be done with _mm256_loadu2_m128i but many standard libraries omit this intrinsic. + __m256i v256 = _mm256_castsi128_si256( + _mm_loadu_si128(reinterpret_cast(pshufb_combine_table + pop1 * 8))); + __m256i compactmask = _mm256_insertf128_si256(v256, + _mm_loadu_si128(reinterpret_cast(pshufb_combine_table + pop3 * 8)), 1); + __m256i almostthere = _mm256_shuffle_epi8(pruned, compactmask); + // We just need to write out the result. + // This is the tricky bit that is hard to do + // if we want to return a SIMD register, since there + // is no single-instruction approach to recombine + // the two 128-bit lanes with an offset. + __m128i v128; + v128 = _mm256_castsi256_si128(almostthere); + _mm_storeu_si128(reinterpret_cast<__m128i*>(output), v128); + v128 = _mm256_extractf128_si256(almostthere, 1); + _mm_storeu_si128(reinterpret_cast<__m128i*>(output + 16 - count_ones(mask & 0xFFFF)), v128); + } + + template + simdjson_inline simd8 lookup_16( + L replace0, L replace1, L replace2, L replace3, + L replace4, L replace5, L replace6, L replace7, + L replace8, L replace9, L replace10, L replace11, + L replace12, L replace13, L replace14, L replace15) const { + return lookup_16(simd8::repeat_16( + replace0, replace1, replace2, replace3, + replace4, replace5, replace6, replace7, + replace8, replace9, replace10, replace11, + replace12, replace13, replace14, replace15 + )); + } + }; + + // Signed bytes + template<> + struct simd8 : base8_numeric { + simdjson_inline simd8() : base8_numeric() {} + simdjson_inline simd8(const __m256i _value) : base8_numeric(_value) {} + // Splat constructor + simdjson_inline simd8(int8_t _value) : simd8(splat(_value)) {} + // Array constructor + simdjson_inline simd8(const int8_t values[32]) : simd8(load(values)) {} + // Member-by-member initialization + simdjson_inline simd8( + int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, + int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15, + int8_t v16, int8_t v17, int8_t v18, int8_t v19, int8_t v20, int8_t v21, int8_t v22, int8_t v23, + int8_t v24, int8_t v25, int8_t v26, int8_t v27, int8_t v28, int8_t v29, int8_t v30, int8_t v31 + ) : simd8(_mm256_setr_epi8( + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15, + v16, v17, v18, v19, v20, v21, v22, v23, + v24, v25, v26, v27, v28, v29, v30, v31 + )) {} + // Repeat 16 values as many times as necessary (usually for lookup tables) + simdjson_inline static simd8 repeat_16( + int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, + int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 + ) { + return simd8( + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15, + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15 + ); + } + + // Order-sensitive comparisons + simdjson_inline simd8 max_val(const simd8 other) const { return _mm256_max_epi8(*this, other); } + simdjson_inline simd8 min_val(const simd8 other) const { return _mm256_min_epi8(*this, other); } + simdjson_inline simd8 operator>(const simd8 other) const { return _mm256_cmpgt_epi8(*this, other); } + simdjson_inline simd8 operator<(const simd8 other) const { return _mm256_cmpgt_epi8(other, *this); } + }; + + // Unsigned bytes + template<> + struct simd8 : base8_numeric { + simdjson_inline simd8() : base8_numeric() {} + simdjson_inline simd8(const __m256i _value) : base8_numeric(_value) {} + // Splat constructor + simdjson_inline simd8(uint8_t _value) : simd8(splat(_value)) {} + // Array constructor + simdjson_inline simd8(const uint8_t values[32]) : simd8(load(values)) {} + // Member-by-member initialization + simdjson_inline simd8( + uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, + uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15, + uint8_t v16, uint8_t v17, uint8_t v18, uint8_t v19, uint8_t v20, uint8_t v21, uint8_t v22, uint8_t v23, + uint8_t v24, uint8_t v25, uint8_t v26, uint8_t v27, uint8_t v28, uint8_t v29, uint8_t v30, uint8_t v31 + ) : simd8(_mm256_setr_epi8( + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15, + v16, v17, v18, v19, v20, v21, v22, v23, + v24, v25, v26, v27, v28, v29, v30, v31 + )) {} + // Repeat 16 values as many times as necessary (usually for lookup tables) + simdjson_inline static simd8 repeat_16( + uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, + uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 + ) { + return simd8( + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15, + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15 + ); + } + + // Saturated math + simdjson_inline simd8 saturating_add(const simd8 other) const { return _mm256_adds_epu8(*this, other); } + simdjson_inline simd8 saturating_sub(const simd8 other) const { return _mm256_subs_epu8(*this, other); } + + // Order-specific operations + simdjson_inline simd8 max_val(const simd8 other) const { return _mm256_max_epu8(*this, other); } + simdjson_inline simd8 min_val(const simd8 other) const { return _mm256_min_epu8(other, *this); } + // Same as >, but only guarantees true is nonzero (< guarantees true = -1) + simdjson_inline simd8 gt_bits(const simd8 other) const { return this->saturating_sub(other); } + // Same as <, but only guarantees true is nonzero (< guarantees true = -1) + simdjson_inline simd8 lt_bits(const simd8 other) const { return other.saturating_sub(*this); } + simdjson_inline simd8 operator<=(const simd8 other) const { return other.max_val(*this) == other; } + simdjson_inline simd8 operator>=(const simd8 other) const { return other.min_val(*this) == other; } + simdjson_inline simd8 operator>(const simd8 other) const { return this->gt_bits(other).any_bits_set(); } + simdjson_inline simd8 operator<(const simd8 other) const { return this->lt_bits(other).any_bits_set(); } + + // Bit-specific operations + simdjson_inline simd8 bits_not_set() const { return *this == uint8_t(0); } + simdjson_inline simd8 bits_not_set(simd8 bits) const { return (*this & bits).bits_not_set(); } + simdjson_inline simd8 any_bits_set() const { return ~this->bits_not_set(); } + simdjson_inline simd8 any_bits_set(simd8 bits) const { return ~this->bits_not_set(bits); } + simdjson_inline bool is_ascii() const { return _mm256_movemask_epi8(*this) == 0; } + simdjson_inline bool bits_not_set_anywhere() const { return _mm256_testz_si256(*this, *this); } + simdjson_inline bool any_bits_set_anywhere() const { return !bits_not_set_anywhere(); } + simdjson_inline bool bits_not_set_anywhere(simd8 bits) const { return _mm256_testz_si256(*this, bits); } + simdjson_inline bool any_bits_set_anywhere(simd8 bits) const { return !bits_not_set_anywhere(bits); } + template + simdjson_inline simd8 shr() const { return simd8(_mm256_srli_epi16(*this, N)) & uint8_t(0xFFu >> N); } + template + simdjson_inline simd8 shl() const { return simd8(_mm256_slli_epi16(*this, N)) & uint8_t(0xFFu << N); } + // Get one of the bits and make a bitmask out of it. + // e.g. value.get_bit<7>() gets the high bit + template + simdjson_inline int get_bit() const { return _mm256_movemask_epi8(_mm256_slli_epi16(*this, 7 - N)); } + }; + + template + struct simd8x64 { + static constexpr int NUM_CHUNKS = 64 / sizeof(simd8); + static_assert(NUM_CHUNKS == 2, "Haswell kernel should use two registers per 64-byte block."); + const simd8 chunks[NUM_CHUNKS]; + + simd8x64(const simd8x64& o) = delete; // no copy allowed + simd8x64& operator=(const simd8& other) = delete; // no assignment allowed + simd8x64() = delete; // no default constructor allowed + + simdjson_inline simd8x64(const simd8 chunk0, const simd8 chunk1) : chunks{ chunk0, chunk1 } {} + simdjson_inline simd8x64(const T ptr[64]) : chunks{ simd8::load(ptr), simd8::load(ptr + 32) } {} + + simdjson_inline uint64_t compress(uint64_t mask, T* output) const { + uint32_t mask1 = uint32_t(mask); + uint32_t mask2 = uint32_t(mask >> 32); + this->chunks[0].compress(mask1, output); + this->chunks[1].compress(mask2, output + 32 - count_ones(mask1)); + return 64 - count_ones(mask); + } + + simdjson_inline void store(T ptr[64]) const { + this->chunks[0].store(ptr + sizeof(simd8) * 0); + this->chunks[1].store(ptr + sizeof(simd8) * 1); + } + + simdjson_inline uint64_t to_bitmask() const { + uint64_t r_lo = uint32_t(this->chunks[0].to_bitmask()); + uint64_t r_hi = this->chunks[1].to_bitmask(); + return r_lo | (r_hi << 32); + } + + simdjson_inline simd8 reduce_or() const { + return this->chunks[0] | this->chunks[1]; + } + + simdjson_inline simd8x64 bit_or(const T m) const { + const simd8 mask = simd8::splat(m); + return simd8x64( + this->chunks[0] | mask, + this->chunks[1] | mask + ); + } + + simdjson_inline uint64_t eq(const T m) const { + const simd8 mask = simd8::splat(m); + return simd8x64( + this->chunks[0] == mask, + this->chunks[1] == mask + ).to_bitmask(); + } + + simdjson_inline uint64_t eq(const simd8x64& other) const { + return simd8x64( + this->chunks[0] == other.chunks[0], + this->chunks[1] == other.chunks[1] + ).to_bitmask(); + } + + simdjson_inline uint64_t lteq(const T m) const { + const simd8 mask = simd8::splat(m); + return simd8x64( + this->chunks[0] <= mask, + this->chunks[1] <= mask + ).to_bitmask(); + } + }; // struct simd8x64 + + } // namespace simd + + } // unnamed namespace + } // namespace haswell +} // namespace simdjson + +#endif // SIMDJSON_HASWELL_SIMD_H +/* end file simdjson/haswell/simd.h */ +/* including simdjson/haswell/stringparsing_defs.h: #include "simdjson/haswell/stringparsing_defs.h" */ +/* begin file simdjson/haswell/stringparsing_defs.h */ +#ifndef SIMDJSON_HASWELL_STRINGPARSING_DEFS_H +#define SIMDJSON_HASWELL_STRINGPARSING_DEFS_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/haswell/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/haswell/simd.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/haswell/bitmanipulation.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace haswell { + namespace { + + using namespace simd; + + // Holds backslashes and quotes locations. + struct backslash_and_quote { + public: + static constexpr uint32_t BYTES_PROCESSED = 32; + simdjson_inline static backslash_and_quote copy_and_find(const uint8_t* src, uint8_t* dst); + + simdjson_inline bool has_quote_first() { return ((bs_bits - 1) & quote_bits) != 0; } + simdjson_inline bool has_backslash() { return ((quote_bits - 1) & bs_bits) != 0; } + simdjson_inline int quote_index() { return trailing_zeroes(quote_bits); } + simdjson_inline int backslash_index() { return trailing_zeroes(bs_bits); } + + uint32_t bs_bits; + uint32_t quote_bits; + }; // struct backslash_and_quote + + simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8_t* src, uint8_t* dst) { + // this can read up to 15 bytes beyond the buffer size, but we require + // SIMDJSON_PADDING of padding + static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "backslash and quote finder must process fewer than SIMDJSON_PADDING bytes"); + simd8 v(src); + // store to dest unconditionally - we can overwrite the bits we don't like later + v.store(dst); + return { + static_cast((v == '\\').to_bitmask()), // bs_bits + static_cast((v == '"').to_bitmask()), // quote_bits + }; + } + + } // unnamed namespace + } // namespace haswell +} // namespace simdjson + +#endif // SIMDJSON_HASWELL_STRINGPARSING_DEFS_H +/* end file simdjson/haswell/stringparsing_defs.h */ +/* end file simdjson/haswell/begin.h */ +/* including simdjson/generic/amalgamated.h for haswell: #include "simdjson/generic/amalgamated.h" */ +/* begin file simdjson/generic/amalgamated.h for haswell */ #if defined(SIMDJSON_CONDITIONAL_INCLUDE) && !defined(SIMDJSON_GENERIC_DEPENDENCIES_H) #error simdjson/generic/dependencies.h must be included before simdjson/generic/amalgamated.h! #endif -/* including simdjson/generic/base.h for fallback: #include "simdjson/generic/base.h" */ -/* begin file simdjson/generic/base.h for fallback */ +/* including simdjson/generic/base.h for haswell: #include "simdjson/generic/base.h" */ +/* begin file simdjson/generic/base.h for haswell */ #ifndef SIMDJSON_GENERIC_BASE_H /* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ @@ -14070,6 +14884,10 @@ namespace simdjson { /* amalgamation skipped (editor-only): #include "simdjson/arm64/begin.h" */ /* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_PPC64 */ /* amalgamation skipped (editor-only): #include "simdjson/ppc64/begin.h" */ +/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_LSX */ +/* amalgamation skipped (editor-only): #include "simdjson/lsx/begin.h" */ +/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_LASX */ +/* amalgamation skipped (editor-only): #include "simdjson/lasx/begin.h" */ /* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_FALLBACK */ /* amalgamation skipped (editor-only): #include "simdjson/fallback/begin.h" */ /* amalgamation skipped (editor-only): #else */ @@ -14079,7 +14897,7 @@ namespace simdjson { /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ namespace simdjson { - namespace fallback { + namespace haswell { struct open_container; class dom_parser_implementation; @@ -14090,16 +14908,17 @@ namespace simdjson { enum class number_type { floating_point_number = 1, /// a binary64 number signed_integer, /// a signed integer that fits in a 64-bit word using two's complement - unsigned_integer /// a positive integer larger or equal to 1<<63 + unsigned_integer, /// a positive integer larger or equal to 1<<63 + big_integer /// a big integer that does not fit in a 64-bit word }; - } // namespace fallback + } // namespace haswell } // namespace simdjson #endif // SIMDJSON_GENERIC_BASE_H -/* end file simdjson/generic/base.h for fallback */ -/* including simdjson/generic/jsoncharutils.h for fallback: #include "simdjson/generic/jsoncharutils.h" */ -/* begin file simdjson/generic/jsoncharutils.h for fallback */ +/* end file simdjson/generic/base.h for haswell */ +/* including simdjson/generic/jsoncharutils.h for haswell: #include "simdjson/generic/jsoncharutils.h" */ +/* begin file simdjson/generic/jsoncharutils.h for haswell */ #ifndef SIMDJSON_GENERIC_JSONCHARUTILS_H /* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ @@ -14110,7 +14929,7 @@ namespace simdjson { /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ namespace simdjson { - namespace fallback { + namespace haswell { namespace { namespace jsoncharutils { @@ -14202,13 +15021,13 @@ namespace simdjson { } // namespace jsoncharutils } // unnamed namespace - } // namespace fallback + } // namespace haswell } // namespace simdjson #endif // SIMDJSON_GENERIC_JSONCHARUTILS_H -/* end file simdjson/generic/jsoncharutils.h for fallback */ -/* including simdjson/generic/atomparsing.h for fallback: #include "simdjson/generic/atomparsing.h" */ -/* begin file simdjson/generic/atomparsing.h for fallback */ +/* end file simdjson/generic/jsoncharutils.h for haswell */ +/* including simdjson/generic/atomparsing.h for haswell: #include "simdjson/generic/atomparsing.h" */ +/* begin file simdjson/generic/atomparsing.h for haswell */ #ifndef SIMDJSON_GENERIC_ATOMPARSING_H /* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ @@ -14220,7 +15039,7 @@ namespace simdjson { #include namespace simdjson { - namespace fallback { + namespace haswell { namespace { /// @private namespace atomparsing { @@ -14282,13 +15101,13 @@ namespace simdjson { } // namespace atomparsing } // unnamed namespace - } // namespace fallback + } // namespace haswell } // namespace simdjson #endif // SIMDJSON_GENERIC_ATOMPARSING_H -/* end file simdjson/generic/atomparsing.h for fallback */ -/* including simdjson/generic/dom_parser_implementation.h for fallback: #include "simdjson/generic/dom_parser_implementation.h" */ -/* begin file simdjson/generic/dom_parser_implementation.h for fallback */ +/* end file simdjson/generic/atomparsing.h for haswell */ +/* including simdjson/generic/dom_parser_implementation.h for haswell: #include "simdjson/generic/dom_parser_implementation.h" */ +/* begin file simdjson/generic/dom_parser_implementation.h for haswell */ #ifndef SIMDJSON_GENERIC_DOM_PARSER_IMPLEMENTATION_H /* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ @@ -14298,7 +15117,7 @@ namespace simdjson { /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ namespace simdjson { - namespace fallback { + namespace haswell { // expectation: sizeof(open_container) = 64/8. struct open_container { @@ -14340,11 +15159,11 @@ namespace simdjson { }; - } // namespace fallback + } // namespace haswell } // namespace simdjson namespace simdjson { - namespace fallback { + namespace haswell { inline dom_parser_implementation::dom_parser_implementation() noexcept = default; inline dom_parser_implementation::dom_parser_implementation(dom_parser_implementation&& other) noexcept = default; @@ -14374,13 +15193,13 @@ namespace simdjson { return SUCCESS; } - } // namespace fallback + } // namespace haswell } // namespace simdjson #endif // SIMDJSON_GENERIC_DOM_PARSER_IMPLEMENTATION_H -/* end file simdjson/generic/dom_parser_implementation.h for fallback */ -/* including simdjson/generic/implementation_simdjson_result_base.h for fallback: #include "simdjson/generic/implementation_simdjson_result_base.h" */ -/* begin file simdjson/generic/implementation_simdjson_result_base.h for fallback */ +/* end file simdjson/generic/dom_parser_implementation.h for haswell */ +/* including simdjson/generic/implementation_simdjson_result_base.h for haswell: #include "simdjson/generic/implementation_simdjson_result_base.h" */ +/* begin file simdjson/generic/implementation_simdjson_result_base.h for haswell */ #ifndef SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_H /* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ @@ -14389,7 +15208,7 @@ namespace simdjson { /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ namespace simdjson { - namespace fallback { + namespace haswell { // This is a near copy of include/error.h's implementation_simdjson_result_base, except it doesn't use std::pair // so we can avoid inlining errors @@ -14511,13 +15330,13 @@ namespace simdjson { error_code second{ UNINITIALIZED }; /** Users should never directly access 'second'. **/ }; // struct implementation_simdjson_result_base - } // namespace fallback + } // namespace haswell } // namespace simdjson #endif // SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_H -/* end file simdjson/generic/implementation_simdjson_result_base.h for fallback */ -/* including simdjson/generic/numberparsing.h for fallback: #include "simdjson/generic/numberparsing.h" */ -/* begin file simdjson/generic/numberparsing.h for fallback */ +/* end file simdjson/generic/implementation_simdjson_result_base.h for haswell */ +/* including simdjson/generic/numberparsing.h for haswell: #include "simdjson/generic/numberparsing.h" */ +/* begin file simdjson/generic/numberparsing.h for haswell */ #ifndef SIMDJSON_GENERIC_NUMBERPARSING_H /* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ @@ -14532,7 +15351,7 @@ namespace simdjson { #include namespace simdjson { - namespace fallback { + namespace haswell { namespace numberparsing { #ifdef JSON_TEST_NUMBERS @@ -14540,11 +15359,13 @@ namespace simdjson { #define WRITE_INTEGER(VALUE, SRC, WRITER) (found_integer((VALUE), (SRC)), (WRITER).append_s64((VALUE))) #define WRITE_UNSIGNED(VALUE, SRC, WRITER) (found_unsigned_integer((VALUE), (SRC)), (WRITER).append_u64((VALUE))) #define WRITE_DOUBLE(VALUE, SRC, WRITER) (found_float((VALUE), (SRC)), (WRITER).append_double((VALUE))) +#define BIGINT_NUMBER(SRC) (found_invalid_number((SRC)), BIGINT_ERROR) #else #define INVALID_NUMBER(SRC) (NUMBER_ERROR) #define WRITE_INTEGER(VALUE, SRC, WRITER) (WRITER).append_s64((VALUE)) #define WRITE_UNSIGNED(VALUE, SRC, WRITER) (WRITER).append_u64((VALUE)) #define WRITE_DOUBLE(VALUE, SRC, WRITER) (WRITER).append_double((VALUE)) +#define BIGINT_NUMBER(SRC) (BIGINT_ERROR) #endif namespace { @@ -14876,6 +15697,10 @@ namespace simdjson { return true; } + simdjson_inline bool is_digit(const uint8_t c) { + return static_cast(c - '0') <= 9; + } + simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t* const src, const uint8_t*& p, uint64_t& i, int64_t& exponent) { // we continue with the fiction that we have an integer. If the // floating point number is representable as x * 10^z for some integer @@ -14933,7 +15758,7 @@ namespace simdjson { // If there were more than 18 digits, we may have overflowed the integer. We have to do // something!!!! if (simdjson_unlikely(p > start_exp + 18)) { - // Skip leading zeroes: 1e000000000000000000001 is technically valid and doesn't overflow + // Skip leading zeroes: 1e000000000000000000001 is technically valid and does not overflow while (*start_exp == '0') { start_exp++; } // 19 digits could overflow int64_t and is kind of absurd anyway. We don't // support exponents smaller than -999,999,999,999,999,999 and bigger @@ -14955,6 +15780,23 @@ namespace simdjson { return SUCCESS; } + simdjson_inline bool check_if_integer(const uint8_t* const src, size_t max_length) { + const uint8_t* const srcend = src + max_length; + bool negative = (*src == '-'); // we can always read at least one character after the '-' + const uint8_t* p = src + uint8_t(negative); + if (p == srcend) { return false; } + if (*p == '0') { + ++p; + if (p == srcend) { return true; } + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return false; } + return true; + } + while (p != srcend && is_digit(*p)) { ++p; } + if (p == srcend) { return true; } + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return false; } + return true; + } + simdjson_inline size_t significant_digits(const uint8_t* start_digits, size_t digit_count) { // It is possible that the integer had an overflow. // We have to handle the case where we have 0.0000somenumber. @@ -14967,11 +15809,8 @@ namespace simdjson { } // unnamed namespace /** @private */ - template - error_code slow_float_parsing(simdjson_unused const uint8_t* src, W writer) { - double d; - if (parse_float_fallback(src, &d)) { - writer.append_double(d); + static error_code slow_float_parsing(simdjson_unused const uint8_t* src, double* answer) { + if (parse_float_fallback(src, answer)) { return SUCCESS; } return INVALID_NUMBER(src); @@ -14995,13 +15834,13 @@ namespace simdjson { // 10000000000000000000000000000000000000000000e+308 // 3.1415926535897932384626433832795028841971693993751 // - // NOTE: This makes a *copy* of the writer and passes it to slow_float_parsing. This happens - // because slow_float_parsing is a non-inlined function. If we passed our writer reference to - // it, it would force it to be stored in memory, preventing the compiler from picking it apart - // and putting into registers. i.e. if we pass it as reference, it gets slow. - // This is what forces the skip_double, as well. - error_code error = slow_float_parsing(src, writer); - writer.skip_double(); + // NOTE: We do not pass a reference to the to slow_float_parsing. If we passed our writer + // reference to it, it would force it to be stored in memory, preventing the compiler from + // picking it apart and putting into registers. i.e. if we pass it as reference, + // it gets slow. + double d; + error_code error = slow_float_parsing(src, &d); + writer.append_double(d); return error; } // NOTE: it's weird that the simdjson_unlikely() only wraps half the if, but it seems to get slower any other @@ -15033,6 +15872,18 @@ namespace simdjson { return SUCCESS; } + // parse the number at src + // define JSON_TEST_NUMBERS for unit testing + // + // It is assumed that the number is followed by a structural ({,},],[) character + // or a white space character. If that is not the case (e.g., when the JSON + // document is made of a single number), then it is necessary to copy the + // content and append a space before calling this function. + // + // Our objective is accurate parsing (ULP of 0) at high speed. + template + simdjson_inline error_code parse_number(const uint8_t* const src, W& writer); + // for performance analysis, it is sometimes useful to skip parsing #ifdef SIMDJSON_SKIPNUMBERPARSING @@ -15111,11 +15962,11 @@ namespace simdjson { // The longest positive 64-bit number is 20 digits. // We do it this way so we don't trigger this branch unless we must. size_t longest_digit_count = negative ? 19 : 20; - if (digit_count > longest_digit_count) { return INVALID_NUMBER(src); } + if (digit_count > longest_digit_count) { return BIGINT_NUMBER(src); } if (digit_count == longest_digit_count) { if (negative) { // Anything negative above INT64_MAX+1 is invalid - if (i > uint64_t(INT64_MAX) + 1) { return INVALID_NUMBER(src); } + if (i > uint64_t(INT64_MAX) + 1) { return BIGINT_NUMBER(src); } WRITE_INTEGER(~i + 1, src, writer); if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return INVALID_NUMBER(src); } return SUCCESS; @@ -15135,7 +15986,7 @@ namespace simdjson { else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INVALID_NUMBER(src); } } - // Write unsigned if it doesn't fit in a signed integer. + // Write unsigned if it does not fit in a signed integer. if (i > uint64_t(INT64_MAX)) { WRITE_UNSIGNED(i, src, writer); } @@ -15588,19 +16439,32 @@ namespace simdjson { src += uint8_t(negative); const uint8_t* p = src; while (static_cast(*p - '0') <= 9) { p++; } + size_t digit_count = size_t(p - src); if (p == src) { return NUMBER_ERROR; } if (jsoncharutils::is_structural_or_whitespace(*p)) { + static const uint8_t* smaller_big_integer = reinterpret_cast("9223372036854775808"); // We have an integer. + if (simdjson_unlikely(digit_count > 20)) { + return number_type::big_integer; + } // If the number is negative and valid, it must be a signed integer. - if (negative) { return number_type::signed_integer; } + if (negative) { + if (simdjson_unlikely(digit_count > 19)) return number_type::big_integer; + if (simdjson_unlikely(digit_count == 19 && memcmp(src, smaller_big_integer, 19) > 0)) { + return number_type::big_integer; + } + return number_type::signed_integer; + } + // Let us check if we have a big integer (>=2**64). + static const uint8_t* two_to_sixtyfour = reinterpret_cast("18446744073709551616"); + if ((digit_count > 20) || (digit_count == 20 && memcmp(src, two_to_sixtyfour, 20) >= 0)) { + return number_type::big_integer; + } + // The number is positive and smaller than 18446744073709551616 (or 2**64). // We want values larger or equal to 9223372036854775808 to be unsigned // integers, and the other values to be signed integers. - int digit_count = int(p - src); - if (digit_count >= 19) { - const uint8_t* smaller_big_integer = reinterpret_cast("9223372036854775808"); - if ((digit_count >= 20) || (memcmp(src, smaller_big_integer, 19) >= 0)) { - return number_type::unsigned_integer; - } + if ((digit_count == 20) || (digit_count >= 19 && memcmp(src, smaller_big_integer, 19) >= 0)) { + return number_type::unsigned_integer; } return number_type::signed_integer; } @@ -15780,6 +16644,37646 @@ namespace simdjson { case number_type::signed_integer: out << "integer in [-9223372036854775808,9223372036854775808)"; break; case number_type::unsigned_integer: out << "unsigned integer in [9223372036854775808,18446744073709551616)"; break; case number_type::floating_point_number: out << "floating-point number (binary64)"; break; + case number_type::big_integer: out << "big integer"; break; + default: SIMDJSON_UNREACHABLE(); + } + return out; + } + + } // namespace haswell +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_NUMBERPARSING_H +/* end file simdjson/generic/numberparsing.h for haswell */ + +/* including simdjson/generic/implementation_simdjson_result_base-inl.h for haswell: #include "simdjson/generic/implementation_simdjson_result_base-inl.h" */ +/* begin file simdjson/generic/implementation_simdjson_result_base-inl.h for haswell */ +#ifndef SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_INL_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_INL_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/implementation_simdjson_result_base.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace haswell { + + // + // internal::implementation_simdjson_result_base inline implementation + // + + template + simdjson_inline void implementation_simdjson_result_base::tie(T& value, error_code& error) && noexcept { + error = this->second; + if (!error) { + value = std::forward>(*this).first; + } + } + + template + simdjson_warn_unused simdjson_inline error_code implementation_simdjson_result_base::get(T& value) && noexcept { + error_code error; + std::forward>(*this).tie(value, error); + return error; + } + + template + simdjson_inline error_code implementation_simdjson_result_base::error() const noexcept { + return this->second; + } + +#if SIMDJSON_EXCEPTIONS + + template + simdjson_inline T& implementation_simdjson_result_base::value() & noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return this->first; + } + + template + simdjson_inline T&& implementation_simdjson_result_base::value() && noexcept(false) { + return std::forward>(*this).take_value(); + } + + template + simdjson_inline T&& implementation_simdjson_result_base::take_value() && noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return std::forward(this->first); + } + + template + simdjson_inline implementation_simdjson_result_base::operator T && () && noexcept(false) { + return std::forward>(*this).take_value(); + } + +#endif // SIMDJSON_EXCEPTIONS + + template + simdjson_inline const T& implementation_simdjson_result_base::value_unsafe() const& noexcept { + return this->first; + } + + template + simdjson_inline T& implementation_simdjson_result_base::value_unsafe() & noexcept { + return this->first; + } + + template + simdjson_inline T&& implementation_simdjson_result_base::value_unsafe() && noexcept { + return std::forward(this->first); + } + + template + simdjson_inline implementation_simdjson_result_base::implementation_simdjson_result_base(T&& value, error_code error) noexcept + : first{ std::forward(value) }, second{ error } {} + template + simdjson_inline implementation_simdjson_result_base::implementation_simdjson_result_base(error_code error) noexcept + : implementation_simdjson_result_base(T{}, error) {} + template + simdjson_inline implementation_simdjson_result_base::implementation_simdjson_result_base(T&& value) noexcept + : implementation_simdjson_result_base(std::forward(value), SUCCESS) {} + + } // namespace haswell +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_INL_H +/* end file simdjson/generic/implementation_simdjson_result_base-inl.h for haswell */ +/* end file simdjson/generic/amalgamated.h for haswell */ +/* including simdjson/haswell/end.h: #include "simdjson/haswell/end.h" */ +/* begin file simdjson/haswell/end.h */ +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/haswell/base.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +#if !SIMDJSON_CAN_ALWAYS_RUN_HASWELL +SIMDJSON_UNTARGET_REGION +#endif + +/* undefining SIMDJSON_IMPLEMENTATION from "haswell" */ +#undef SIMDJSON_IMPLEMENTATION +/* end file simdjson/haswell/end.h */ + +#endif // SIMDJSON_HASWELL_H +/* end file simdjson/haswell.h */ +/* including simdjson/haswell/implementation.h: #include */ +/* begin file simdjson/haswell/implementation.h */ +#ifndef SIMDJSON_HASWELL_IMPLEMENTATION_H +#define SIMDJSON_HASWELL_IMPLEMENTATION_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/haswell/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/implementation.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/instruction_set.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +// The constructor may be executed on any host, so we take care not to use SIMDJSON_TARGET_HASWELL +namespace simdjson { + namespace haswell { + + /** + * @private + */ + class implementation final : public simdjson::implementation { + public: + simdjson_inline implementation() : simdjson::implementation( + "haswell", + "Intel/AMD AVX2", + internal::instruction_set::AVX2 | internal::instruction_set::PCLMULQDQ | internal::instruction_set::BMI1 | internal::instruction_set::BMI2 + ) {} + simdjson_warn_unused error_code create_dom_parser_implementation( + size_t capacity, + size_t max_length, + std::unique_ptr& dst + ) const noexcept final; + simdjson_warn_unused error_code minify(const uint8_t* buf, size_t len, uint8_t* dst, size_t& dst_len) const noexcept final; + simdjson_warn_unused bool validate_utf8(const char* buf, size_t len) const noexcept final; + }; + + } // namespace haswell +} // namespace simdjson + +#endif // SIMDJSON_HASWELL_IMPLEMENTATION_H +/* end file simdjson/haswell/implementation.h */ + +/* including simdjson/haswell/begin.h: #include */ +/* begin file simdjson/haswell/begin.h */ +/* defining SIMDJSON_IMPLEMENTATION to "haswell" */ +#define SIMDJSON_IMPLEMENTATION haswell + +/* including simdjson/haswell/base.h: #include "simdjson/haswell/base.h" */ +/* begin file simdjson/haswell/base.h */ +#ifndef SIMDJSON_HASWELL_BASE_H +#define SIMDJSON_HASWELL_BASE_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/base.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +// The constructor may be executed on any host, so we take care not to use SIMDJSON_TARGET_HASWELL +namespace simdjson { + /** + * Implementation for Haswell (Intel AVX2). + */ + namespace haswell { + + class implementation; + + namespace { + namespace simd { + template struct simd8; + template struct simd8x64; + } // namespace simd + } // unnamed namespace + + } // namespace haswell +} // namespace simdjson + +#endif // SIMDJSON_HASWELL_BASE_H +/* end file simdjson/haswell/base.h */ +/* including simdjson/haswell/intrinsics.h: #include "simdjson/haswell/intrinsics.h" */ +/* begin file simdjson/haswell/intrinsics.h */ +#ifndef SIMDJSON_HASWELL_INTRINSICS_H +#define SIMDJSON_HASWELL_INTRINSICS_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/haswell/base.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +#if SIMDJSON_VISUAL_STUDIO +// under clang within visual studio, this will include +#include // visual studio or clang +#else +#include // elsewhere +#endif // SIMDJSON_VISUAL_STUDIO + +#if SIMDJSON_CLANG_VISUAL_STUDIO +/** + * You are not supposed, normally, to include these + * headers directly. Instead you should either include intrin.h + * or x86intrin.h. However, when compiling with clang + * under Windows (i.e., when _MSC_VER is set), these headers + * only get included *if* the corresponding features are detected + * from macros: + * e.g., if __AVX2__ is set... in turn, we normally set these + * macros by compiling against the corresponding architecture + * (e.g., arch:AVX2, -mavx2, etc.) which compiles the whole + * software with these advanced instructions. In simdjson, we + * want to compile the whole program for a generic target, + * and only target our specific kernels. As a workaround, + * we directly include the needed headers. These headers would + * normally guard against such usage, but we carefully included + * (or ) before, so the headers + * are fooled. + */ +#include // for _blsr_u64 +#include // for __lzcnt64 +#include // for most things (AVX2, AVX512, _popcnt64) +#include +#include +#include +#include +#include // for _mm_clmulepi64_si128 + // unfortunately, we may not get _blsr_u64, but, thankfully, clang + // has it as a macro. +#ifndef _blsr_u64 +// we roll our own +#define _blsr_u64(n) ((n - 1) & n) +#endif // _blsr_u64 +#endif // SIMDJSON_CLANG_VISUAL_STUDIO + +static_assert(sizeof(__m256i) <= simdjson::SIMDJSON_PADDING, "insufficient padding for haswell kernel."); + +#endif // SIMDJSON_HASWELL_INTRINSICS_H +/* end file simdjson/haswell/intrinsics.h */ + +#if !SIMDJSON_CAN_ALWAYS_RUN_HASWELL +SIMDJSON_TARGET_REGION("avx2,bmi,pclmul,lzcnt,popcnt") +#endif + +/* including simdjson/haswell/bitmanipulation.h: #include "simdjson/haswell/bitmanipulation.h" */ +/* begin file simdjson/haswell/bitmanipulation.h */ +#ifndef SIMDJSON_HASWELL_BITMANIPULATION_H +#define SIMDJSON_HASWELL_BITMANIPULATION_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/haswell/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/haswell/intrinsics.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/haswell/bitmask.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace haswell { + namespace { + + // We sometimes call trailing_zero on inputs that are zero, + // but the algorithms do not end up using the returned value. + // Sadly, sanitizers are not smart enough to figure it out. + SIMDJSON_NO_SANITIZE_UNDEFINED + // This function can be used safely even if not all bytes have been + // initialized. + // See issue https://github.com/simdjson/simdjson/issues/1965 + SIMDJSON_NO_SANITIZE_MEMORY + simdjson_inline int trailing_zeroes(uint64_t input_num) { +#if SIMDJSON_REGULAR_VISUAL_STUDIO + return (int)_tzcnt_u64(input_num); +#else // SIMDJSON_REGULAR_VISUAL_STUDIO + //////// + // You might expect the next line to be equivalent to + // return (int)_tzcnt_u64(input_num); + // but the generated code differs and might be less efficient? + //////// + return __builtin_ctzll(input_num); +#endif // SIMDJSON_REGULAR_VISUAL_STUDIO + } + + /* result might be undefined when input_num is zero */ + simdjson_inline uint64_t clear_lowest_bit(uint64_t input_num) { + return _blsr_u64(input_num); + } + + /* result might be undefined when input_num is zero */ + simdjson_inline int leading_zeroes(uint64_t input_num) { + return int(_lzcnt_u64(input_num)); + } + +#if SIMDJSON_REGULAR_VISUAL_STUDIO + simdjson_inline unsigned __int64 count_ones(uint64_t input_num) { + // note: we do not support legacy 32-bit Windows in this kernel + return __popcnt64(input_num);// Visual Studio wants two underscores + } +#else + simdjson_inline long long int count_ones(uint64_t input_num) { + return _popcnt64(input_num); + } +#endif + + simdjson_inline bool add_overflow(uint64_t value1, uint64_t value2, + uint64_t* result) { +#if SIMDJSON_REGULAR_VISUAL_STUDIO + return _addcarry_u64(0, value1, value2, + reinterpret_cast(result)); +#else + return __builtin_uaddll_overflow(value1, value2, + reinterpret_cast(result)); +#endif + } + + } // unnamed namespace + } // namespace haswell +} // namespace simdjson + +#endif // SIMDJSON_HASWELL_BITMANIPULATION_H +/* end file simdjson/haswell/bitmanipulation.h */ +/* including simdjson/haswell/bitmask.h: #include "simdjson/haswell/bitmask.h" */ +/* begin file simdjson/haswell/bitmask.h */ +#ifndef SIMDJSON_HASWELL_BITMASK_H +#define SIMDJSON_HASWELL_BITMASK_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/haswell/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/haswell/intrinsics.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace haswell { + namespace { + + // + // Perform a "cumulative bitwise xor," flipping bits each time a 1 is encountered. + // + // For example, prefix_xor(00100100) == 00011100 + // + simdjson_inline uint64_t prefix_xor(const uint64_t bitmask) { + // There should be no such thing with a processor supporting avx2 + // but not clmul. + __m128i all_ones = _mm_set1_epi8('\xFF'); + __m128i result = _mm_clmulepi64_si128(_mm_set_epi64x(0ULL, bitmask), all_ones, 0); + return _mm_cvtsi128_si64(result); + } + + } // unnamed namespace + } // namespace haswell +} // namespace simdjson + +#endif // SIMDJSON_HASWELL_BITMASK_H +/* end file simdjson/haswell/bitmask.h */ +/* including simdjson/haswell/numberparsing_defs.h: #include "simdjson/haswell/numberparsing_defs.h" */ +/* begin file simdjson/haswell/numberparsing_defs.h */ +#ifndef SIMDJSON_HASWELL_NUMBERPARSING_DEFS_H +#define SIMDJSON_HASWELL_NUMBERPARSING_DEFS_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/haswell/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/haswell/intrinsics.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace haswell { + namespace numberparsing { + + /** @private */ + static simdjson_inline uint32_t parse_eight_digits_unrolled(const uint8_t* chars) { + // this actually computes *16* values so we are being wasteful. + const __m128i ascii0 = _mm_set1_epi8('0'); + const __m128i mul_1_10 = + _mm_setr_epi8(10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1); + const __m128i mul_1_100 = _mm_setr_epi16(100, 1, 100, 1, 100, 1, 100, 1); + const __m128i mul_1_10000 = + _mm_setr_epi16(10000, 1, 10000, 1, 10000, 1, 10000, 1); + const __m128i input = _mm_sub_epi8( + _mm_loadu_si128(reinterpret_cast(chars)), ascii0); + const __m128i t1 = _mm_maddubs_epi16(input, mul_1_10); + const __m128i t2 = _mm_madd_epi16(t1, mul_1_100); + const __m128i t3 = _mm_packus_epi32(t2, t2); + const __m128i t4 = _mm_madd_epi16(t3, mul_1_10000); + return _mm_cvtsi128_si32( + t4); // only captures the sum of the first 8 digits, drop the rest + } + + /** @private */ + simdjson_inline internal::value128 full_multiplication(uint64_t value1, uint64_t value2) { + internal::value128 answer; +#if SIMDJSON_REGULAR_VISUAL_STUDIO || SIMDJSON_IS_32BITS +#if SIMDJSON_IS_ARM64 + // ARM64 has native support for 64-bit multiplications, no need to emultate + answer.high = __umulh(value1, value2); + answer.low = value1 * value2; +#else + answer.low = _umul128(value1, value2, &answer.high); // _umul128 not available on ARM64 +#endif // SIMDJSON_IS_ARM64 +#else // SIMDJSON_REGULAR_VISUAL_STUDIO || SIMDJSON_IS_32BITS + __uint128_t r = (static_cast<__uint128_t>(value1)) * value2; + answer.low = uint64_t(r); + answer.high = uint64_t(r >> 64); +#endif + return answer; + } + + } // namespace numberparsing + } // namespace haswell +} // namespace simdjson + +#define SIMDJSON_SWAR_NUMBER_PARSING 1 + +#endif // SIMDJSON_HASWELL_NUMBERPARSING_DEFS_H +/* end file simdjson/haswell/numberparsing_defs.h */ +/* including simdjson/haswell/simd.h: #include "simdjson/haswell/simd.h" */ +/* begin file simdjson/haswell/simd.h */ +#ifndef SIMDJSON_HASWELL_SIMD_H +#define SIMDJSON_HASWELL_SIMD_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/haswell/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/haswell/intrinsics.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/haswell/bitmanipulation.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/simdprune_tables.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace haswell { + namespace { + namespace simd { + + // Forward-declared so they can be used by splat and friends. + template + struct base { + __m256i value; + + // Zero constructor + simdjson_inline base() : value{ __m256i() } {} + + // Conversion from SIMD register + simdjson_inline base(const __m256i _value) : value(_value) {} + + // Conversion to SIMD register + simdjson_inline operator const __m256i& () const { return this->value; } + simdjson_inline operator __m256i& () { return this->value; } + + // Bit operations + simdjson_inline Child operator|(const Child other) const { return _mm256_or_si256(*this, other); } + simdjson_inline Child operator&(const Child other) const { return _mm256_and_si256(*this, other); } + simdjson_inline Child operator^(const Child other) const { return _mm256_xor_si256(*this, other); } + simdjson_inline Child bit_andnot(const Child other) const { return _mm256_andnot_si256(other, *this); } + simdjson_inline Child& operator|=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast | other; return *this_cast; } + simdjson_inline Child& operator&=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast & other; return *this_cast; } + simdjson_inline Child& operator^=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast ^ other; return *this_cast; } + }; + + // Forward-declared so they can be used by splat and friends. + template + struct simd8; + + template> + struct base8 : base> { + typedef uint32_t bitmask_t; + typedef uint64_t bitmask2_t; + + simdjson_inline base8() : base>() {} + simdjson_inline base8(const __m256i _value) : base>(_value) {} + + friend simdjson_really_inline Mask operator==(const simd8 lhs, const simd8 rhs) { return _mm256_cmpeq_epi8(lhs, rhs); } + + static const int SIZE = sizeof(base::value); + + template + simdjson_inline simd8 prev(const simd8 prev_chunk) const { + return _mm256_alignr_epi8(*this, _mm256_permute2x128_si256(prev_chunk, *this, 0x21), 16 - N); + } + }; + + // SIMD byte mask type (returned by things like eq and gt) + template<> + struct simd8 : base8 { + static simdjson_inline simd8 splat(bool _value) { return _mm256_set1_epi8(uint8_t(-(!!_value))); } + + simdjson_inline simd8() : base8() {} + simdjson_inline simd8(const __m256i _value) : base8(_value) {} + // Splat constructor + simdjson_inline simd8(bool _value) : base8(splat(_value)) {} + + simdjson_inline int to_bitmask() const { return _mm256_movemask_epi8(*this); } + simdjson_inline bool any() const { return !_mm256_testz_si256(*this, *this); } + simdjson_inline simd8 operator~() const { return *this ^ true; } + }; + + template + struct base8_numeric : base8 { + static simdjson_inline simd8 splat(T _value) { return _mm256_set1_epi8(_value); } + static simdjson_inline simd8 zero() { return _mm256_setzero_si256(); } + static simdjson_inline simd8 load(const T values[32]) { + return _mm256_loadu_si256(reinterpret_cast(values)); + } + // Repeat 16 values as many times as necessary (usually for lookup tables) + static simdjson_inline simd8 repeat_16( + T v0, T v1, T v2, T v3, T v4, T v5, T v6, T v7, + T v8, T v9, T v10, T v11, T v12, T v13, T v14, T v15 + ) { + return simd8( + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15, + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15 + ); + } + + simdjson_inline base8_numeric() : base8() {} + simdjson_inline base8_numeric(const __m256i _value) : base8(_value) {} + + // Store to array + simdjson_inline void store(T dst[32]) const { return _mm256_storeu_si256(reinterpret_cast<__m256i*>(dst), *this); } + + // Addition/subtraction are the same for signed and unsigned + simdjson_inline simd8 operator+(const simd8 other) const { return _mm256_add_epi8(*this, other); } + simdjson_inline simd8 operator-(const simd8 other) const { return _mm256_sub_epi8(*this, other); } + simdjson_inline simd8& operator+=(const simd8 other) { *this = *this + other; return *static_cast*>(this); } + simdjson_inline simd8& operator-=(const simd8 other) { *this = *this - other; return *static_cast*>(this); } + + // Override to distinguish from bool version + simdjson_inline simd8 operator~() const { return *this ^ 0xFFu; } + + // Perform a lookup assuming the value is between 0 and 16 (undefined behavior for out of range values) + template + simdjson_inline simd8 lookup_16(simd8 lookup_table) const { + return _mm256_shuffle_epi8(lookup_table, *this); + } + + // Copies to 'output" all bytes corresponding to a 0 in the mask (interpreted as a bitset). + // Passing a 0 value for mask would be equivalent to writing out every byte to output. + // Only the first 32 - count_ones(mask) bytes of the result are significant but 32 bytes + // get written. + // Design consideration: it seems like a function with the + // signature simd8 compress(uint32_t mask) would be + // sensible, but the AVX ISA makes this kind of approach difficult. + template + simdjson_inline void compress(uint32_t mask, L* output) const { + using internal::thintable_epi8; + using internal::BitsSetTable256mul2; + using internal::pshufb_combine_table; + // this particular implementation was inspired by work done by @animetosho + // we do it in four steps, first 8 bytes and then second 8 bytes... + uint8_t mask1 = uint8_t(mask); // least significant 8 bits + uint8_t mask2 = uint8_t(mask >> 8); // second least significant 8 bits + uint8_t mask3 = uint8_t(mask >> 16); // ... + uint8_t mask4 = uint8_t(mask >> 24); // ... + // next line just loads the 64-bit values thintable_epi8[mask1] and + // thintable_epi8[mask2] into a 128-bit register, using only + // two instructions on most compilers. + __m256i shufmask = _mm256_set_epi64x(thintable_epi8[mask4], thintable_epi8[mask3], + thintable_epi8[mask2], thintable_epi8[mask1]); + // we increment by 0x08 the second half of the mask and so forth + shufmask = + _mm256_add_epi8(shufmask, _mm256_set_epi32(0x18181818, 0x18181818, + 0x10101010, 0x10101010, 0x08080808, 0x08080808, 0, 0)); + // this is the version "nearly pruned" + __m256i pruned = _mm256_shuffle_epi8(*this, shufmask); + // we still need to put the pieces back together. + // we compute the popcount of the first words: + int pop1 = BitsSetTable256mul2[mask1]; + int pop3 = BitsSetTable256mul2[mask3]; + + // then load the corresponding mask + // could be done with _mm256_loadu2_m128i but many standard libraries omit this intrinsic. + __m256i v256 = _mm256_castsi128_si256( + _mm_loadu_si128(reinterpret_cast(pshufb_combine_table + pop1 * 8))); + __m256i compactmask = _mm256_insertf128_si256(v256, + _mm_loadu_si128(reinterpret_cast(pshufb_combine_table + pop3 * 8)), 1); + __m256i almostthere = _mm256_shuffle_epi8(pruned, compactmask); + // We just need to write out the result. + // This is the tricky bit that is hard to do + // if we want to return a SIMD register, since there + // is no single-instruction approach to recombine + // the two 128-bit lanes with an offset. + __m128i v128; + v128 = _mm256_castsi256_si128(almostthere); + _mm_storeu_si128(reinterpret_cast<__m128i*>(output), v128); + v128 = _mm256_extractf128_si256(almostthere, 1); + _mm_storeu_si128(reinterpret_cast<__m128i*>(output + 16 - count_ones(mask & 0xFFFF)), v128); + } + + template + simdjson_inline simd8 lookup_16( + L replace0, L replace1, L replace2, L replace3, + L replace4, L replace5, L replace6, L replace7, + L replace8, L replace9, L replace10, L replace11, + L replace12, L replace13, L replace14, L replace15) const { + return lookup_16(simd8::repeat_16( + replace0, replace1, replace2, replace3, + replace4, replace5, replace6, replace7, + replace8, replace9, replace10, replace11, + replace12, replace13, replace14, replace15 + )); + } + }; + + // Signed bytes + template<> + struct simd8 : base8_numeric { + simdjson_inline simd8() : base8_numeric() {} + simdjson_inline simd8(const __m256i _value) : base8_numeric(_value) {} + // Splat constructor + simdjson_inline simd8(int8_t _value) : simd8(splat(_value)) {} + // Array constructor + simdjson_inline simd8(const int8_t values[32]) : simd8(load(values)) {} + // Member-by-member initialization + simdjson_inline simd8( + int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, + int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15, + int8_t v16, int8_t v17, int8_t v18, int8_t v19, int8_t v20, int8_t v21, int8_t v22, int8_t v23, + int8_t v24, int8_t v25, int8_t v26, int8_t v27, int8_t v28, int8_t v29, int8_t v30, int8_t v31 + ) : simd8(_mm256_setr_epi8( + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15, + v16, v17, v18, v19, v20, v21, v22, v23, + v24, v25, v26, v27, v28, v29, v30, v31 + )) {} + // Repeat 16 values as many times as necessary (usually for lookup tables) + simdjson_inline static simd8 repeat_16( + int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, + int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 + ) { + return simd8( + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15, + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15 + ); + } + + // Order-sensitive comparisons + simdjson_inline simd8 max_val(const simd8 other) const { return _mm256_max_epi8(*this, other); } + simdjson_inline simd8 min_val(const simd8 other) const { return _mm256_min_epi8(*this, other); } + simdjson_inline simd8 operator>(const simd8 other) const { return _mm256_cmpgt_epi8(*this, other); } + simdjson_inline simd8 operator<(const simd8 other) const { return _mm256_cmpgt_epi8(other, *this); } + }; + + // Unsigned bytes + template<> + struct simd8 : base8_numeric { + simdjson_inline simd8() : base8_numeric() {} + simdjson_inline simd8(const __m256i _value) : base8_numeric(_value) {} + // Splat constructor + simdjson_inline simd8(uint8_t _value) : simd8(splat(_value)) {} + // Array constructor + simdjson_inline simd8(const uint8_t values[32]) : simd8(load(values)) {} + // Member-by-member initialization + simdjson_inline simd8( + uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, + uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15, + uint8_t v16, uint8_t v17, uint8_t v18, uint8_t v19, uint8_t v20, uint8_t v21, uint8_t v22, uint8_t v23, + uint8_t v24, uint8_t v25, uint8_t v26, uint8_t v27, uint8_t v28, uint8_t v29, uint8_t v30, uint8_t v31 + ) : simd8(_mm256_setr_epi8( + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15, + v16, v17, v18, v19, v20, v21, v22, v23, + v24, v25, v26, v27, v28, v29, v30, v31 + )) {} + // Repeat 16 values as many times as necessary (usually for lookup tables) + simdjson_inline static simd8 repeat_16( + uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, + uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 + ) { + return simd8( + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15, + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15 + ); + } + + // Saturated math + simdjson_inline simd8 saturating_add(const simd8 other) const { return _mm256_adds_epu8(*this, other); } + simdjson_inline simd8 saturating_sub(const simd8 other) const { return _mm256_subs_epu8(*this, other); } + + // Order-specific operations + simdjson_inline simd8 max_val(const simd8 other) const { return _mm256_max_epu8(*this, other); } + simdjson_inline simd8 min_val(const simd8 other) const { return _mm256_min_epu8(other, *this); } + // Same as >, but only guarantees true is nonzero (< guarantees true = -1) + simdjson_inline simd8 gt_bits(const simd8 other) const { return this->saturating_sub(other); } + // Same as <, but only guarantees true is nonzero (< guarantees true = -1) + simdjson_inline simd8 lt_bits(const simd8 other) const { return other.saturating_sub(*this); } + simdjson_inline simd8 operator<=(const simd8 other) const { return other.max_val(*this) == other; } + simdjson_inline simd8 operator>=(const simd8 other) const { return other.min_val(*this) == other; } + simdjson_inline simd8 operator>(const simd8 other) const { return this->gt_bits(other).any_bits_set(); } + simdjson_inline simd8 operator<(const simd8 other) const { return this->lt_bits(other).any_bits_set(); } + + // Bit-specific operations + simdjson_inline simd8 bits_not_set() const { return *this == uint8_t(0); } + simdjson_inline simd8 bits_not_set(simd8 bits) const { return (*this & bits).bits_not_set(); } + simdjson_inline simd8 any_bits_set() const { return ~this->bits_not_set(); } + simdjson_inline simd8 any_bits_set(simd8 bits) const { return ~this->bits_not_set(bits); } + simdjson_inline bool is_ascii() const { return _mm256_movemask_epi8(*this) == 0; } + simdjson_inline bool bits_not_set_anywhere() const { return _mm256_testz_si256(*this, *this); } + simdjson_inline bool any_bits_set_anywhere() const { return !bits_not_set_anywhere(); } + simdjson_inline bool bits_not_set_anywhere(simd8 bits) const { return _mm256_testz_si256(*this, bits); } + simdjson_inline bool any_bits_set_anywhere(simd8 bits) const { return !bits_not_set_anywhere(bits); } + template + simdjson_inline simd8 shr() const { return simd8(_mm256_srli_epi16(*this, N)) & uint8_t(0xFFu >> N); } + template + simdjson_inline simd8 shl() const { return simd8(_mm256_slli_epi16(*this, N)) & uint8_t(0xFFu << N); } + // Get one of the bits and make a bitmask out of it. + // e.g. value.get_bit<7>() gets the high bit + template + simdjson_inline int get_bit() const { return _mm256_movemask_epi8(_mm256_slli_epi16(*this, 7 - N)); } + }; + + template + struct simd8x64 { + static constexpr int NUM_CHUNKS = 64 / sizeof(simd8); + static_assert(NUM_CHUNKS == 2, "Haswell kernel should use two registers per 64-byte block."); + const simd8 chunks[NUM_CHUNKS]; + + simd8x64(const simd8x64& o) = delete; // no copy allowed + simd8x64& operator=(const simd8& other) = delete; // no assignment allowed + simd8x64() = delete; // no default constructor allowed + + simdjson_inline simd8x64(const simd8 chunk0, const simd8 chunk1) : chunks{ chunk0, chunk1 } {} + simdjson_inline simd8x64(const T ptr[64]) : chunks{ simd8::load(ptr), simd8::load(ptr + 32) } {} + + simdjson_inline uint64_t compress(uint64_t mask, T* output) const { + uint32_t mask1 = uint32_t(mask); + uint32_t mask2 = uint32_t(mask >> 32); + this->chunks[0].compress(mask1, output); + this->chunks[1].compress(mask2, output + 32 - count_ones(mask1)); + return 64 - count_ones(mask); + } + + simdjson_inline void store(T ptr[64]) const { + this->chunks[0].store(ptr + sizeof(simd8) * 0); + this->chunks[1].store(ptr + sizeof(simd8) * 1); + } + + simdjson_inline uint64_t to_bitmask() const { + uint64_t r_lo = uint32_t(this->chunks[0].to_bitmask()); + uint64_t r_hi = this->chunks[1].to_bitmask(); + return r_lo | (r_hi << 32); + } + + simdjson_inline simd8 reduce_or() const { + return this->chunks[0] | this->chunks[1]; + } + + simdjson_inline simd8x64 bit_or(const T m) const { + const simd8 mask = simd8::splat(m); + return simd8x64( + this->chunks[0] | mask, + this->chunks[1] | mask + ); + } + + simdjson_inline uint64_t eq(const T m) const { + const simd8 mask = simd8::splat(m); + return simd8x64( + this->chunks[0] == mask, + this->chunks[1] == mask + ).to_bitmask(); + } + + simdjson_inline uint64_t eq(const simd8x64& other) const { + return simd8x64( + this->chunks[0] == other.chunks[0], + this->chunks[1] == other.chunks[1] + ).to_bitmask(); + } + + simdjson_inline uint64_t lteq(const T m) const { + const simd8 mask = simd8::splat(m); + return simd8x64( + this->chunks[0] <= mask, + this->chunks[1] <= mask + ).to_bitmask(); + } + }; // struct simd8x64 + + } // namespace simd + + } // unnamed namespace + } // namespace haswell +} // namespace simdjson + +#endif // SIMDJSON_HASWELL_SIMD_H +/* end file simdjson/haswell/simd.h */ +/* including simdjson/haswell/stringparsing_defs.h: #include "simdjson/haswell/stringparsing_defs.h" */ +/* begin file simdjson/haswell/stringparsing_defs.h */ +#ifndef SIMDJSON_HASWELL_STRINGPARSING_DEFS_H +#define SIMDJSON_HASWELL_STRINGPARSING_DEFS_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/haswell/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/haswell/simd.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/haswell/bitmanipulation.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace haswell { + namespace { + + using namespace simd; + + // Holds backslashes and quotes locations. + struct backslash_and_quote { + public: + static constexpr uint32_t BYTES_PROCESSED = 32; + simdjson_inline static backslash_and_quote copy_and_find(const uint8_t* src, uint8_t* dst); + + simdjson_inline bool has_quote_first() { return ((bs_bits - 1) & quote_bits) != 0; } + simdjson_inline bool has_backslash() { return ((quote_bits - 1) & bs_bits) != 0; } + simdjson_inline int quote_index() { return trailing_zeroes(quote_bits); } + simdjson_inline int backslash_index() { return trailing_zeroes(bs_bits); } + + uint32_t bs_bits; + uint32_t quote_bits; + }; // struct backslash_and_quote + + simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8_t* src, uint8_t* dst) { + // this can read up to 15 bytes beyond the buffer size, but we require + // SIMDJSON_PADDING of padding + static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "backslash and quote finder must process fewer than SIMDJSON_PADDING bytes"); + simd8 v(src); + // store to dest unconditionally - we can overwrite the bits we don't like later + v.store(dst); + return { + static_cast((v == '\\').to_bitmask()), // bs_bits + static_cast((v == '"').to_bitmask()), // quote_bits + }; + } + + } // unnamed namespace + } // namespace haswell +} // namespace simdjson + +#endif // SIMDJSON_HASWELL_STRINGPARSING_DEFS_H +/* end file simdjson/haswell/stringparsing_defs.h */ +/* end file simdjson/haswell/begin.h */ +/* including generic/amalgamated.h for haswell: #include */ +/* begin file generic/amalgamated.h for haswell */ +#if defined(SIMDJSON_CONDITIONAL_INCLUDE) && !defined(SIMDJSON_SRC_GENERIC_DEPENDENCIES_H) +#error generic/dependencies.h must be included before generic/amalgamated.h! +#endif + +/* including generic/base.h for haswell: #include */ +/* begin file generic/base.h for haswell */ +#ifndef SIMDJSON_SRC_GENERIC_BASE_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_BASE_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace haswell { + namespace { + + struct json_character_block; + + } // unnamed namespace + } // namespace haswell +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_BASE_H +/* end file generic/base.h for haswell */ +/* including generic/dom_parser_implementation.h for haswell: #include */ +/* begin file generic/dom_parser_implementation.h for haswell */ +#ifndef SIMDJSON_SRC_GENERIC_DOM_PARSER_IMPLEMENTATION_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_DOM_PARSER_IMPLEMENTATION_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +// Interface a dom parser implementation must fulfill +namespace simdjson { + namespace haswell { + namespace { + + simdjson_inline simd8 must_be_2_3_continuation(const simd8 prev2, const simd8 prev3); + simdjson_inline bool is_ascii(const simd8x64& input); + + } // unnamed namespace + } // namespace haswell +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_DOM_PARSER_IMPLEMENTATION_H +/* end file generic/dom_parser_implementation.h for haswell */ +/* including generic/json_character_block.h for haswell: #include */ +/* begin file generic/json_character_block.h for haswell */ +#ifndef SIMDJSON_SRC_GENERIC_JSON_CHARACTER_BLOCK_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_JSON_CHARACTER_BLOCK_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace haswell { + namespace { + + struct json_character_block { + static simdjson_inline json_character_block classify(const simd::simd8x64& in); + + simdjson_inline uint64_t whitespace() const noexcept { return _whitespace; } + simdjson_inline uint64_t op() const noexcept { return _op; } + simdjson_inline uint64_t scalar() const noexcept { return ~(op() | whitespace()); } + + uint64_t _whitespace; + uint64_t _op; + }; + + } // unnamed namespace + } // namespace haswell +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_JSON_CHARACTER_BLOCK_H +/* end file generic/json_character_block.h for haswell */ +/* end file generic/amalgamated.h for haswell */ +/* including generic/stage1/amalgamated.h for haswell: #include */ +/* begin file generic/stage1/amalgamated.h for haswell */ +// Stuff other things depend on +/* including generic/stage1/base.h for haswell: #include */ +/* begin file generic/stage1/base.h for haswell */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE1_BASE_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_BASE_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace haswell { + namespace { + namespace stage1 { + + class bit_indexer; + template + struct buf_block_reader; + struct json_block; + class json_minifier; + class json_scanner; + struct json_string_block; + class json_string_scanner; + class json_structural_indexer; + + } // namespace stage1 + + namespace utf8_validation { + struct utf8_checker; + } // namespace utf8_validation + + using utf8_validation::utf8_checker; + + } // unnamed namespace + } // namespace haswell +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE1_BASE_H +/* end file generic/stage1/base.h for haswell */ +/* including generic/stage1/buf_block_reader.h for haswell: #include */ +/* begin file generic/stage1/buf_block_reader.h for haswell */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE1_BUF_BLOCK_READER_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_BUF_BLOCK_READER_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +#include + +namespace simdjson { + namespace haswell { + namespace { + namespace stage1 { + + // Walks through a buffer in block-sized increments, loading the last part with spaces + template + struct buf_block_reader { + public: + simdjson_inline buf_block_reader(const uint8_t* _buf, size_t _len); + simdjson_inline size_t block_index(); + simdjson_inline bool has_full_block() const; + simdjson_inline const uint8_t* full_block() const; + /** + * Get the last block, padded with spaces. + * + * There will always be a last block, with at least 1 byte, unless len == 0 (in which case this + * function fills the buffer with spaces and returns 0. In particular, if len == STEP_SIZE there + * will be 0 full_blocks and 1 remainder block with STEP_SIZE bytes and no spaces for padding. + * + * @return the number of effective characters in the last block. + */ + simdjson_inline size_t get_remainder(uint8_t* dst) const; + simdjson_inline void advance(); + private: + const uint8_t* buf; + const size_t len; + const size_t lenminusstep; + size_t idx; + }; + + // Routines to print masks and text for debugging bitmask operations + simdjson_unused static char* format_input_text_64(const uint8_t* text) { + static char buf[sizeof(simd8x64) + 1]; + for (size_t i = 0; i < sizeof(simd8x64); i++) { + buf[i] = int8_t(text[i]) < ' ' ? '_' : int8_t(text[i]); + } + buf[sizeof(simd8x64)] = '\0'; + return buf; + } + + // Routines to print masks and text for debugging bitmask operations + simdjson_unused static char* format_input_text(const simd8x64& in) { + static char buf[sizeof(simd8x64) + 1]; + in.store(reinterpret_cast(buf)); + for (size_t i = 0; i < sizeof(simd8x64); i++) { + if (buf[i] < ' ') { buf[i] = '_'; } + } + buf[sizeof(simd8x64)] = '\0'; + return buf; + } + + simdjson_unused static char* format_input_text(const simd8x64& in, uint64_t mask) { + static char buf[sizeof(simd8x64) + 1]; + in.store(reinterpret_cast(buf)); + for (size_t i = 0; i < sizeof(simd8x64); i++) { + if (buf[i] <= ' ') { buf[i] = '_'; } + if (!(mask & (size_t(1) << i))) { buf[i] = ' '; } + } + buf[sizeof(simd8x64)] = '\0'; + return buf; + } + + simdjson_unused static char* format_mask(uint64_t mask) { + static char buf[sizeof(simd8x64) + 1]; + for (size_t i = 0; i < 64; i++) { + buf[i] = (mask & (size_t(1) << i)) ? 'X' : ' '; + } + buf[64] = '\0'; + return buf; + } + + template + simdjson_inline buf_block_reader::buf_block_reader(const uint8_t* _buf, size_t _len) : buf{ _buf }, len{ _len }, lenminusstep{ len < STEP_SIZE ? 0 : len - STEP_SIZE }, idx{ 0 } {} + + template + simdjson_inline size_t buf_block_reader::block_index() { return idx; } + + template + simdjson_inline bool buf_block_reader::has_full_block() const { + return idx < lenminusstep; + } + + template + simdjson_inline const uint8_t* buf_block_reader::full_block() const { + return &buf[idx]; + } + + template + simdjson_inline size_t buf_block_reader::get_remainder(uint8_t* dst) const { + if (len == idx) { return 0; } // memcpy(dst, null, 0) will trigger an error with some sanitizers + std::memset(dst, 0x20, STEP_SIZE); // std::memset STEP_SIZE because it's more efficient to write out 8 or 16 bytes at once. + std::memcpy(dst, buf + idx, len - idx); + return len - idx; + } + + template + simdjson_inline void buf_block_reader::advance() { + idx += STEP_SIZE; + } + + } // namespace stage1 + } // unnamed namespace + } // namespace haswell +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE1_BUF_BLOCK_READER_H +/* end file generic/stage1/buf_block_reader.h for haswell */ +/* including generic/stage1/json_escape_scanner.h for haswell: #include */ +/* begin file generic/stage1/json_escape_scanner.h for haswell */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE1_JSON_ESCAPE_SCANNER_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_JSON_ESCAPE_SCANNER_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace haswell { + namespace { + namespace stage1 { + + /** + * Scans for escape characters in JSON, taking care with multiple backslashes (\\n vs. \n). + */ + struct json_escape_scanner { + /** The actual escape characters (the backslashes themselves). */ + uint64_t next_is_escaped = 0ULL; + + struct escaped_and_escape { + /** + * Mask of escaped characters. + * + * ``` + * \n \\n \\\n \\\\n \ + * 0100100010100101000 + * n \ \ n \ \ + * ``` + */ + uint64_t escaped; + /** + * Mask of escape characters. + * + * ``` + * \n \\n \\\n \\\\n \ + * 1001000101001010001 + * \ \ \ \ \ \ \ + * ``` + */ + uint64_t escape; + }; + + /** + * Get a mask of both escape and escaped characters (the characters following a backslash). + * + * @param potential_escape A mask of the character that can escape others (but could be + * escaped itself). e.g. block.eq('\\') + */ + simdjson_really_inline escaped_and_escape next(uint64_t backslash) noexcept { + +#if !SIMDJSON_SKIP_BACKSLASH_SHORT_CIRCUIT + if (!backslash) { return { next_escaped_without_backslashes(), 0 }; } +#endif + + // | | Mask (shows characters instead of 1's) | Depth | Instructions | + // |--------------------------------|----------------------------------------|-------|---------------------| + // | string | `\\n_\\\n___\\\n___\\\\___\\\\__\\\` | | | + // | | ` even odd even odd odd` | | | + // | potential_escape | ` \ \\\ \\\ \\\\ \\\\ \\\` | 1 | 1 (backslash & ~first_is_escaped) + // | escape_and_terminal_code | ` \n \ \n \ \n \ \ \ \ \ \` | 5 | 5 (next_escape_and_terminal_code()) + // | escaped | `\ \ n \ n \ \ \ \ \ ` X | 6 | 7 (escape_and_terminal_code ^ (potential_escape | first_is_escaped)) + // | escape | ` \ \ \ \ \ \ \ \ \ \` | 6 | 8 (escape_and_terminal_code & backslash) + // | first_is_escaped | `\ ` | 7 (*) | 9 (escape >> 63) () + // (*) this is not needed until the next iteration + uint64_t escape_and_terminal_code = next_escape_and_terminal_code(backslash & ~this->next_is_escaped); + uint64_t escaped = escape_and_terminal_code ^ (backslash | this->next_is_escaped); + uint64_t escape = escape_and_terminal_code & backslash; + this->next_is_escaped = escape >> 63; + return { escaped, escape }; + } + + private: + static constexpr const uint64_t ODD_BITS = 0xAAAAAAAAAAAAAAAAULL; + + simdjson_really_inline uint64_t next_escaped_without_backslashes() noexcept { + uint64_t escaped = this->next_is_escaped; + this->next_is_escaped = 0; + return escaped; + } + + /** + * Returns a mask of the next escape characters (masking out escaped backslashes), along with + * any non-backslash escape codes. + * + * \n \\n \\\n \\\\n returns: + * \n \ \ \n \ \ + * 11 100 1011 10100 + * + * You are expected to mask out the first bit yourself if the previous block had a trailing + * escape. + * + * & the result with potential_escape to get just the escape characters. + * ^ the result with (potential_escape | first_is_escaped) to get escaped characters. + */ + static simdjson_really_inline uint64_t next_escape_and_terminal_code(uint64_t potential_escape) noexcept { + // If we were to just shift and mask out any odd bits, we'd actually get a *half* right answer: + // any even-aligned backslash runs would be correct! Odd-aligned backslash runs would be + // inverted (\\\ would be 010 instead of 101). + // + // ``` + // string: | ____\\\\_\\\\_____ | + // maybe_escaped | ODD | \ \ \ \ | + // even-aligned ^^^ ^^^^ odd-aligned + // ``` + // + // Taking that into account, our basic strategy is: + // + // 1. Use subtraction to produce a mask with 1's for even-aligned runs and 0's for + // odd-aligned runs. + // 2. XOR all odd bits, which masks out the odd bits in even-aligned runs, and brings IN the + // odd bits in odd-aligned runs. + // 3. & with backslash to clean up any stray bits. + // runs are set to 0, and then XORing with "odd": + // + // | | Mask (shows characters instead of 1's) | Instructions | + // |--------------------------------|----------------------------------------|---------------------| + // | string | `\\n_\\\n___\\\n___\\\\___\\\\__\\\` | + // | | ` even odd even odd odd` | + // | maybe_escaped | ` n \\n \\n \\\_ \\\_ \\` X | 1 (potential_escape << 1) + // | maybe_escaped_and_odd | ` \n_ \\n _ \\\n_ _ \\\__ _\\\_ \\\` | 1 (maybe_escaped | odd) + // | even_series_codes_and_odd | ` n_\\\ _ n_ _\\\\ _ _ ` | 1 (maybe_escaped_and_odd - potential_escape) + // | escape_and_terminal_code | ` \n \ \n \ \n \ \ \ \ \ \` | 1 (^ odd) + // + + // Escaped characters are characters following an escape. + uint64_t maybe_escaped = potential_escape << 1; + + // To distinguish odd from even escape sequences, therefore, we turn on any *starting* + // escapes that are on an odd byte. (We actually bring in all odd bits, for speed.) + // - Odd runs of backslashes are 0000, and the code at the end ("n" in \n or \\n) is 1. + // - Odd runs of backslashes are 1111, and the code at the end ("n" in \n or \\n) is 0. + // - All other odd bytes are 1, and even bytes are 0. + uint64_t maybe_escaped_and_odd_bits = maybe_escaped | ODD_BITS; + uint64_t even_series_codes_and_odd_bits = maybe_escaped_and_odd_bits - potential_escape; + + // Now we flip all odd bytes back with xor. This: + // - Makes odd runs of backslashes go from 0000 to 1010 + // - Makes even runs of backslashes go from 1111 to 1010 + // - Sets actually-escaped codes to 1 (the n in \n and \\n: \n = 11, \\n = 100) + // - Resets all other bytes to 0 + return even_series_codes_and_odd_bits ^ ODD_BITS; + } + }; + + } // namespace stage1 + } // unnamed namespace + } // namespace haswell +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRING_SCANNER_H +/* end file generic/stage1/json_escape_scanner.h for haswell */ +/* including generic/stage1/json_string_scanner.h for haswell: #include */ +/* begin file generic/stage1/json_string_scanner.h for haswell */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRING_SCANNER_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRING_SCANNER_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace haswell { + namespace { + namespace stage1 { + + struct json_string_block { + // We spell out the constructors in the hope of resolving inlining issues with Visual Studio 2017 + simdjson_really_inline json_string_block(uint64_t escaped, uint64_t quote, uint64_t in_string) : + _escaped(escaped), _quote(quote), _in_string(in_string) {} + + // Escaped characters (characters following an escape() character) + simdjson_really_inline uint64_t escaped() const { return _escaped; } + // Real (non-backslashed) quotes + simdjson_really_inline uint64_t quote() const { return _quote; } + // Only characters inside the string (not including the quotes) + simdjson_really_inline uint64_t string_content() const { return _in_string & ~_quote; } + // Return a mask of whether the given characters are inside a string (only works on non-quotes) + simdjson_really_inline uint64_t non_quote_inside_string(uint64_t mask) const { return mask & _in_string; } + // Return a mask of whether the given characters are inside a string (only works on non-quotes) + simdjson_really_inline uint64_t non_quote_outside_string(uint64_t mask) const { return mask & ~_in_string; } + // Tail of string (everything except the start quote) + simdjson_really_inline uint64_t string_tail() const { return _in_string ^ _quote; } + + // escaped characters (backslashed--does not include the hex characters after \u) + uint64_t _escaped; + // real quotes (non-escaped ones) + uint64_t _quote; + // string characters (includes start quote but not end quote) + uint64_t _in_string; + }; + + // Scans blocks for string characters, storing the state necessary to do so + class json_string_scanner { + public: + simdjson_really_inline json_string_block next(const simd::simd8x64& in); + // Returns either UNCLOSED_STRING or SUCCESS + simdjson_really_inline error_code finish(); + + private: + // Scans for escape characters + json_escape_scanner escape_scanner{}; + // Whether the last iteration was still inside a string (all 1's = true, all 0's = false). + uint64_t prev_in_string = 0ULL; + }; + + // + // Return a mask of all string characters plus end quotes. + // + // prev_escaped is overflow saying whether the next character is escaped. + // prev_in_string is overflow saying whether we're still in a string. + // + // Backslash sequences outside of quotes will be detected in stage 2. + // + simdjson_really_inline json_string_block json_string_scanner::next(const simd::simd8x64& in) { + const uint64_t backslash = in.eq('\\'); + const uint64_t escaped = escape_scanner.next(backslash).escaped; + const uint64_t quote = in.eq('"') & ~escaped; + + // + // prefix_xor flips on bits inside the string (and flips off the end quote). + // + // Then we xor with prev_in_string: if we were in a string already, its effect is flipped + // (characters inside strings are outside, and characters outside strings are inside). + // + const uint64_t in_string = prefix_xor(quote) ^ prev_in_string; + + // + // Check if we're still in a string at the end of the box so the next block will know + // + prev_in_string = uint64_t(static_cast(in_string) >> 63); + + // Use ^ to turn the beginning quote off, and the end quote on. + + // We are returning a function-local object so either we get a move constructor + // or we get copy elision. + return json_string_block(escaped, quote, in_string); + } + + simdjson_really_inline error_code json_string_scanner::finish() { + if (prev_in_string) { + return UNCLOSED_STRING; + } + return SUCCESS; + } + + } // namespace stage1 + } // unnamed namespace + } // namespace haswell +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRING_SCANNER_H +/* end file generic/stage1/json_string_scanner.h for haswell */ +/* including generic/stage1/utf8_lookup4_algorithm.h for haswell: #include */ +/* begin file generic/stage1/utf8_lookup4_algorithm.h for haswell */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE1_UTF8_LOOKUP4_ALGORITHM_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_UTF8_LOOKUP4_ALGORITHM_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace haswell { + namespace { + namespace utf8_validation { + + using namespace simd; + + simdjson_inline simd8 check_special_cases(const simd8 input, const simd8 prev1) { + // Bit 0 = Too Short (lead byte/ASCII followed by lead byte/ASCII) + // Bit 1 = Too Long (ASCII followed by continuation) + // Bit 2 = Overlong 3-byte + // Bit 4 = Surrogate + // Bit 5 = Overlong 2-byte + // Bit 7 = Two Continuations + constexpr const uint8_t TOO_SHORT = 1 << 0; // 11______ 0_______ + // 11______ 11______ + constexpr const uint8_t TOO_LONG = 1 << 1; // 0_______ 10______ + constexpr const uint8_t OVERLONG_3 = 1 << 2; // 11100000 100_____ + constexpr const uint8_t SURROGATE = 1 << 4; // 11101101 101_____ + constexpr const uint8_t OVERLONG_2 = 1 << 5; // 1100000_ 10______ + constexpr const uint8_t TWO_CONTS = 1 << 7; // 10______ 10______ + constexpr const uint8_t TOO_LARGE = 1 << 3; // 11110100 1001____ + // 11110100 101_____ + // 11110101 1001____ + // 11110101 101_____ + // 1111011_ 1001____ + // 1111011_ 101_____ + // 11111___ 1001____ + // 11111___ 101_____ + constexpr const uint8_t TOO_LARGE_1000 = 1 << 6; + // 11110101 1000____ + // 1111011_ 1000____ + // 11111___ 1000____ + constexpr const uint8_t OVERLONG_4 = 1 << 6; // 11110000 1000____ + + const simd8 byte_1_high = prev1.shr<4>().lookup_16( + // 0_______ ________ + TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, + TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, + // 10______ ________ + TWO_CONTS, TWO_CONTS, TWO_CONTS, TWO_CONTS, + // 1100____ ________ + TOO_SHORT | OVERLONG_2, + // 1101____ ________ + TOO_SHORT, + // 1110____ ________ + TOO_SHORT | OVERLONG_3 | SURROGATE, + // 1111____ ________ + TOO_SHORT | TOO_LARGE | TOO_LARGE_1000 | OVERLONG_4 + ); + constexpr const uint8_t CARRY = TOO_SHORT | TOO_LONG | TWO_CONTS; // These all have ____ in byte 1 . + const simd8 byte_1_low = (prev1 & 0x0F).lookup_16( + // ____0000 ________ + CARRY | OVERLONG_3 | OVERLONG_2 | OVERLONG_4, + // ____0001 ________ + CARRY | OVERLONG_2, + // ____001_ ________ + CARRY, + CARRY, + + // ____0100 ________ + CARRY | TOO_LARGE, + // ____0101 ________ + CARRY | TOO_LARGE | TOO_LARGE_1000, + // ____011_ ________ + CARRY | TOO_LARGE | TOO_LARGE_1000, + CARRY | TOO_LARGE | TOO_LARGE_1000, + + // ____1___ ________ + CARRY | TOO_LARGE | TOO_LARGE_1000, + CARRY | TOO_LARGE | TOO_LARGE_1000, + CARRY | TOO_LARGE | TOO_LARGE_1000, + CARRY | TOO_LARGE | TOO_LARGE_1000, + CARRY | TOO_LARGE | TOO_LARGE_1000, + // ____1101 ________ + CARRY | TOO_LARGE | TOO_LARGE_1000 | SURROGATE, + CARRY | TOO_LARGE | TOO_LARGE_1000, + CARRY | TOO_LARGE | TOO_LARGE_1000 + ); + const simd8 byte_2_high = input.shr<4>().lookup_16( + // ________ 0_______ + TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT, + TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT, + + // ________ 1000____ + TOO_LONG | OVERLONG_2 | TWO_CONTS | OVERLONG_3 | TOO_LARGE_1000 | OVERLONG_4, + // ________ 1001____ + TOO_LONG | OVERLONG_2 | TWO_CONTS | OVERLONG_3 | TOO_LARGE, + // ________ 101_____ + TOO_LONG | OVERLONG_2 | TWO_CONTS | SURROGATE | TOO_LARGE, + TOO_LONG | OVERLONG_2 | TWO_CONTS | SURROGATE | TOO_LARGE, + + // ________ 11______ + TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT + ); + return (byte_1_high & byte_1_low & byte_2_high); + } + simdjson_inline simd8 check_multibyte_lengths(const simd8 input, + const simd8 prev_input, const simd8 sc) { + simd8 prev2 = input.prev<2>(prev_input); + simd8 prev3 = input.prev<3>(prev_input); + simd8 must23 = must_be_2_3_continuation(prev2, prev3); + simd8 must23_80 = must23 & uint8_t(0x80); + return must23_80 ^ sc; + } + + // + // Return nonzero if there are incomplete multibyte characters at the end of the block: + // e.g. if there is a 4-byte character, but it's 3 bytes from the end. + // + simdjson_inline simd8 is_incomplete(const simd8 input) { + // If the previous input's last 3 bytes match this, they're too short (they ended at EOF): + // ... 1111____ 111_____ 11______ +#if SIMDJSON_IMPLEMENTATION_ICELAKE + static const uint8_t max_array[64] = { + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 0xf0u - 1, 0xe0u - 1, 0xc0u - 1 + }; +#else + static const uint8_t max_array[32] = { + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 0xf0u - 1, 0xe0u - 1, 0xc0u - 1 + }; +#endif + const simd8 max_value(&max_array[sizeof(max_array) - sizeof(simd8)]); + return input.gt_bits(max_value); + } + + struct utf8_checker { + // If this is nonzero, there has been a UTF-8 error. + simd8 error; + // The last input we received + simd8 prev_input_block; + // Whether the last input we received was incomplete (used for ASCII fast path) + simd8 prev_incomplete; + + // + // Check whether the current bytes are valid UTF-8. + // + simdjson_inline void check_utf8_bytes(const simd8 input, const simd8 prev_input) { + // Flip prev1...prev3 so we can easily determine if they are 2+, 3+ or 4+ lead bytes + // (2, 3, 4-byte leads become large positive numbers instead of small negative numbers) + simd8 prev1 = input.prev<1>(prev_input); + simd8 sc = check_special_cases(input, prev1); + this->error |= check_multibyte_lengths(input, prev_input, sc); + } + + // The only problem that can happen at EOF is that a multibyte character is too short + // or a byte value too large in the last bytes: check_special_cases only checks for bytes + // too large in the first of two bytes. + simdjson_inline void check_eof() { + // If the previous block had incomplete UTF-8 characters at the end, an ASCII block can't + // possibly finish them. + this->error |= this->prev_incomplete; + } + + simdjson_inline void check_next_input(const simd8x64& input) { + if (simdjson_likely(is_ascii(input))) { + this->error |= this->prev_incomplete; + } + else { + // you might think that a for-loop would work, but under Visual Studio, it is not good enough. + static_assert((simd8x64::NUM_CHUNKS == 1) + || (simd8x64::NUM_CHUNKS == 2) + || (simd8x64::NUM_CHUNKS == 4), + "We support one, two or four chunks per 64-byte block."); + SIMDJSON_IF_CONSTEXPR(simd8x64::NUM_CHUNKS == 1) { + this->check_utf8_bytes(input.chunks[0], this->prev_input_block); + } +else SIMDJSON_IF_CONSTEXPR(simd8x64::NUM_CHUNKS == 2) { + this->check_utf8_bytes(input.chunks[0], this->prev_input_block); + this->check_utf8_bytes(input.chunks[1], input.chunks[0]); + } + else SIMDJSON_IF_CONSTEXPR(simd8x64::NUM_CHUNKS == 4) { + this->check_utf8_bytes(input.chunks[0], this->prev_input_block); + this->check_utf8_bytes(input.chunks[1], input.chunks[0]); + this->check_utf8_bytes(input.chunks[2], input.chunks[1]); + this->check_utf8_bytes(input.chunks[3], input.chunks[2]); + } + this->prev_incomplete = is_incomplete(input.chunks[simd8x64::NUM_CHUNKS - 1]); + this->prev_input_block = input.chunks[simd8x64::NUM_CHUNKS - 1]; + } + } + // do not forget to call check_eof! + simdjson_inline error_code errors() { + return this->error.any_bits_set_anywhere() ? error_code::UTF8_ERROR : error_code::SUCCESS; + } + + }; // struct utf8_checker + } // namespace utf8_validation + + } // unnamed namespace + } // namespace haswell +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE1_UTF8_LOOKUP4_ALGORITHM_H +/* end file generic/stage1/utf8_lookup4_algorithm.h for haswell */ +/* including generic/stage1/json_scanner.h for haswell: #include */ +/* begin file generic/stage1/json_scanner.h for haswell */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE1_JSON_SCANNER_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_JSON_SCANNER_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace haswell { + namespace { + namespace stage1 { + + /** + * A block of scanned json, with information on operators and scalars. + * + * We seek to identify pseudo-structural characters. Anything that is inside + * a string must be omitted (hence & ~_string.string_tail()). + * Otherwise, pseudo-structural characters come in two forms. + * 1. We have the structural characters ([,],{,},:, comma). The + * term 'structural character' is from the JSON RFC. + * 2. We have the 'scalar pseudo-structural characters'. + * Scalars are quotes, and any character except structural characters and white space. + * + * To identify the scalar pseudo-structural characters, we must look at what comes + * before them: it must be a space, a quote or a structural characters. + * Starting with simdjson v0.3, we identify them by + * negation: we identify everything that is followed by a non-quote scalar, + * and we negate that. Whatever remains must be a 'scalar pseudo-structural character'. + */ + struct json_block { + public: + // We spell out the constructors in the hope of resolving inlining issues with Visual Studio 2017 + simdjson_inline json_block(json_string_block&& string, json_character_block characters, uint64_t follows_potential_nonquote_scalar) : + _string(std::move(string)), _characters(characters), _follows_potential_nonquote_scalar(follows_potential_nonquote_scalar) {} + simdjson_inline json_block(json_string_block string, json_character_block characters, uint64_t follows_potential_nonquote_scalar) : + _string(string), _characters(characters), _follows_potential_nonquote_scalar(follows_potential_nonquote_scalar) {} + + /** + * The start of structurals. + * In simdjson prior to v0.3, these were called the pseudo-structural characters. + **/ + simdjson_inline uint64_t structural_start() const noexcept { return potential_structural_start() & ~_string.string_tail(); } + /** All JSON whitespace (i.e. not in a string) */ + simdjson_inline uint64_t whitespace() const noexcept { return non_quote_outside_string(_characters.whitespace()); } + + // Helpers + + /** Whether the given characters are inside a string (only works on non-quotes) */ + simdjson_inline uint64_t non_quote_inside_string(uint64_t mask) const noexcept { return _string.non_quote_inside_string(mask); } + /** Whether the given characters are outside a string (only works on non-quotes) */ + simdjson_inline uint64_t non_quote_outside_string(uint64_t mask) const noexcept { return _string.non_quote_outside_string(mask); } + + // string and escape characters + json_string_block _string; + // whitespace, structural characters ('operators'), scalars + json_character_block _characters; + // whether the previous character was a scalar + uint64_t _follows_potential_nonquote_scalar; + private: + // Potential structurals (i.e. disregarding strings) + + /** + * structural elements ([,],{,},:, comma) plus scalar starts like 123, true and "abc". + * They may reside inside a string. + **/ + simdjson_inline uint64_t potential_structural_start() const noexcept { return _characters.op() | potential_scalar_start(); } + /** + * The start of non-operator runs, like 123, true and "abc". + * It main reside inside a string. + **/ + simdjson_inline uint64_t potential_scalar_start() const noexcept { + // The term "scalar" refers to anything except structural characters and white space + // (so letters, numbers, quotes). + // Whenever it is preceded by something that is not a structural element ({,},[,],:, ") nor a white-space + // then we know that it is irrelevant structurally. + return _characters.scalar() & ~follows_potential_scalar(); + } + /** + * Whether the given character is immediately after a non-operator like 123, true. + * The characters following a quote are not included. + */ + simdjson_inline uint64_t follows_potential_scalar() const noexcept { + // _follows_potential_nonquote_scalar: is defined as marking any character that follows a character + // that is not a structural element ({,},[,],:, comma) nor a quote (") and that is not a + // white space. + // It is understood that within quoted region, anything at all could be marked (irrelevant). + return _follows_potential_nonquote_scalar; + } + }; + + /** + * Scans JSON for important bits: structural characters or 'operators', strings, and scalars. + * + * The scanner starts by calculating two distinct things: + * - string characters (taking \" into account) + * - structural characters or 'operators' ([]{},:, comma) + * and scalars (runs of non-operators like 123, true and "abc") + * + * To minimize data dependency (a key component of the scanner's speed), it finds these in parallel: + * in particular, the operator/scalar bit will find plenty of things that are actually part of + * strings. When we're done, json_block will fuse the two together by masking out tokens that are + * part of a string. + */ + class json_scanner { + public: + json_scanner() = default; + simdjson_inline json_block next(const simd::simd8x64& in); + // Returns either UNCLOSED_STRING or SUCCESS + simdjson_inline error_code finish(); + + private: + // Whether the last character of the previous iteration is part of a scalar token + // (anything except whitespace or a structural character/'operator'). + uint64_t prev_scalar = 0ULL; + json_string_scanner string_scanner{}; + }; + + + // + // Check if the current character immediately follows a matching character. + // + // For example, this checks for quotes with backslashes in front of them: + // + // const uint64_t backslashed_quote = in.eq('"') & immediately_follows(in.eq('\'), prev_backslash); + // + simdjson_inline uint64_t follows(const uint64_t match, uint64_t& overflow) { + const uint64_t result = match << 1 | overflow; + overflow = match >> 63; + return result; + } + + simdjson_inline json_block json_scanner::next(const simd::simd8x64& in) { + json_string_block strings = string_scanner.next(in); + // identifies the white-space and the structural characters + json_character_block characters = json_character_block::classify(in); + // The term "scalar" refers to anything except structural characters and white space + // (so letters, numbers, quotes). + // We want follows_scalar to mark anything that follows a non-quote scalar (so letters and numbers). + // + // A terminal quote should either be followed by a structural character (comma, brace, bracket, colon) + // or nothing. However, we still want ' "a string"true ' to mark the 't' of 'true' as a potential + // pseudo-structural character just like we would if we had ' "a string" true '; otherwise we + // may need to add an extra check when parsing strings. + // + // Performance: there are many ways to skin this cat. + const uint64_t nonquote_scalar = characters.scalar() & ~strings.quote(); + uint64_t follows_nonquote_scalar = follows(nonquote_scalar, prev_scalar); + // We are returning a function-local object so either we get a move constructor + // or we get copy elision. + return json_block( + strings,// strings is a function-local object so either it moves or the copy is elided. + characters, + follows_nonquote_scalar + ); + } + + simdjson_inline error_code json_scanner::finish() { + return string_scanner.finish(); + } + + } // namespace stage1 + } // unnamed namespace + } // namespace haswell +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE1_JSON_SCANNER_H +/* end file generic/stage1/json_scanner.h for haswell */ + +// All other declarations +/* including generic/stage1/find_next_document_index.h for haswell: #include */ +/* begin file generic/stage1/find_next_document_index.h for haswell */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE1_FIND_NEXT_DOCUMENT_INDEX_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_FIND_NEXT_DOCUMENT_INDEX_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace haswell { + namespace { + namespace stage1 { + + /** + * This algorithm is used to quickly identify the last structural position that + * makes up a complete document. + * + * It does this by going backwards and finding the last *document boundary* (a + * place where one value follows another without a comma between them). If the + * last document (the characters after the boundary) has an equal number of + * start and end brackets, it is considered complete. + * + * Simply put, we iterate over the structural characters, starting from + * the end. We consider that we found the end of a JSON document when the + * first element of the pair is NOT one of these characters: '{' '[' ':' ',' + * and when the second element is NOT one of these characters: '}' ']' ':' ','. + * + * This simple comparison works most of the time, but it does not cover cases + * where the batch's structural indexes contain a perfect amount of documents. + * In such a case, we do not have access to the structural index which follows + * the last document, therefore, we do not have access to the second element in + * the pair, and that means we cannot identify the last document. To fix this + * issue, we keep a count of the open and closed curly/square braces we found + * while searching for the pair. When we find a pair AND the count of open and + * closed curly/square braces is the same, we know that we just passed a + * complete document, therefore the last json buffer location is the end of the + * batch. + */ + simdjson_inline uint32_t find_next_document_index(dom_parser_implementation& parser) { + // Variant: do not count separately, just figure out depth + if (parser.n_structural_indexes == 0) { return 0; } + auto arr_cnt = 0; + auto obj_cnt = 0; + for (auto i = parser.n_structural_indexes - 1; i > 0; i--) { + auto idxb = parser.structural_indexes[i]; + switch (parser.buf[idxb]) { + case ':': + case ',': + continue; + case '}': + obj_cnt--; + continue; + case ']': + arr_cnt--; + continue; + case '{': + obj_cnt++; + break; + case '[': + arr_cnt++; + break; + } + auto idxa = parser.structural_indexes[i - 1]; + switch (parser.buf[idxa]) { + case '{': + case '[': + case ':': + case ',': + continue; + } + // Last document is complete, so the next document will appear after! + if (!arr_cnt && !obj_cnt) { + return parser.n_structural_indexes; + } + // Last document is incomplete; mark the document at i + 1 as the next one + return i; + } + // If we made it to the end, we want to finish counting to see if we have a full document. + switch (parser.buf[parser.structural_indexes[0]]) { + case '}': + obj_cnt--; + break; + case ']': + arr_cnt--; + break; + case '{': + obj_cnt++; + break; + case '[': + arr_cnt++; + break; + } + if (!arr_cnt && !obj_cnt) { + // We have a complete document. + return parser.n_structural_indexes; + } + return 0; + } + + } // namespace stage1 + } // unnamed namespace + } // namespace haswell +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE1_FIND_NEXT_DOCUMENT_INDEX_H +/* end file generic/stage1/find_next_document_index.h for haswell */ +/* including generic/stage1/json_minifier.h for haswell: #include */ +/* begin file generic/stage1/json_minifier.h for haswell */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE1_JSON_MINIFIER_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_JSON_MINIFIER_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +// This file contains the common code every implementation uses in stage1 +// It is intended to be included multiple times and compiled multiple times +// We assume the file in which it is included already includes +// "simdjson/stage1.h" (this simplifies amalgation) + +namespace simdjson { + namespace haswell { + namespace { + namespace stage1 { + + class json_minifier { + public: + template + static error_code minify(const uint8_t* buf, size_t len, uint8_t* dst, size_t& dst_len) noexcept; + + private: + simdjson_inline json_minifier(uint8_t* _dst) + : dst{ _dst } + {} + template + simdjson_inline void step(const uint8_t* block_buf, buf_block_reader& reader) noexcept; + simdjson_inline void next(const simd::simd8x64& in, const json_block& block); + simdjson_inline error_code finish(uint8_t* dst_start, size_t& dst_len); + json_scanner scanner{}; + uint8_t* dst; + }; + + simdjson_inline void json_minifier::next(const simd::simd8x64& in, const json_block& block) { + uint64_t mask = block.whitespace(); + dst += in.compress(mask, dst); + } + + simdjson_inline error_code json_minifier::finish(uint8_t* dst_start, size_t& dst_len) { + error_code error = scanner.finish(); + if (error) { dst_len = 0; return error; } + dst_len = dst - dst_start; + return SUCCESS; + } + + template<> + simdjson_inline void json_minifier::step<128>(const uint8_t* block_buf, buf_block_reader<128>& reader) noexcept { + simd::simd8x64 in_1(block_buf); + simd::simd8x64 in_2(block_buf + 64); + json_block block_1 = scanner.next(in_1); + json_block block_2 = scanner.next(in_2); + this->next(in_1, block_1); + this->next(in_2, block_2); + reader.advance(); + } + + template<> + simdjson_inline void json_minifier::step<64>(const uint8_t* block_buf, buf_block_reader<64>& reader) noexcept { + simd::simd8x64 in_1(block_buf); + json_block block_1 = scanner.next(in_1); + this->next(block_buf, block_1); + reader.advance(); + } + + template + error_code json_minifier::minify(const uint8_t* buf, size_t len, uint8_t* dst, size_t& dst_len) noexcept { + buf_block_reader reader(buf, len); + json_minifier minifier(dst); + + // Index the first n-1 blocks + while (reader.has_full_block()) { + minifier.step(reader.full_block(), reader); + } + + // Index the last (remainder) block, padded with spaces + uint8_t block[STEP_SIZE]; + size_t remaining_bytes = reader.get_remainder(block); + if (remaining_bytes > 0) { + // We do not want to write directly to the output stream. Rather, we write + // to a local buffer (for safety). + uint8_t out_block[STEP_SIZE]; + uint8_t* const guarded_dst{ minifier.dst }; + minifier.dst = out_block; + minifier.step(block, reader); + size_t to_write = minifier.dst - out_block; + // In some cases, we could be enticed to consider the padded spaces + // as part of the string. This is fine as long as we do not write more + // than we consumed. + if (to_write > remaining_bytes) { to_write = remaining_bytes; } + memcpy(guarded_dst, out_block, to_write); + minifier.dst = guarded_dst + to_write; + } + return minifier.finish(dst, dst_len); + } + + } // namespace stage1 + } // unnamed namespace + } // namespace haswell +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE1_JSON_MINIFIER_H +/* end file generic/stage1/json_minifier.h for haswell */ +/* including generic/stage1/json_structural_indexer.h for haswell: #include */ +/* begin file generic/stage1/json_structural_indexer.h for haswell */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRUCTURAL_INDEXER_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRUCTURAL_INDEXER_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +// This file contains the common code every implementation uses in stage1 +// It is intended to be included multiple times and compiled multiple times +// We assume the file in which it is included already includes +// "simdjson/stage1.h" (this simplifies amalgation) + +namespace simdjson { + namespace haswell { + namespace { + namespace stage1 { + + class bit_indexer { + public: + uint32_t* tail; + + simdjson_inline bit_indexer(uint32_t* index_buf) : tail(index_buf) {} + +#if SIMDJSON_PREFER_REVERSE_BITS + /** + * ARM lacks a fast trailing zero instruction, but it has a fast + * bit reversal instruction and a fast leading zero instruction. + * Thus it may be profitable to reverse the bits (once) and then + * to rely on a sequence of instructions that call the leading + * zero instruction. + * + * Performance notes: + * The chosen routine is not optimal in terms of data dependency + * since zero_leading_bit might require two instructions. However, + * it tends to minimize the total number of instructions which is + * beneficial. + */ + simdjson_inline void write_index(uint32_t idx, uint64_t& rev_bits, int i) { + int lz = leading_zeroes(rev_bits); + this->tail[i] = static_cast(idx) + lz; + rev_bits = zero_leading_bit(rev_bits, lz); + } +#else + /** + * Under recent x64 systems, we often have both a fast trailing zero + * instruction and a fast 'clear-lower-bit' instruction so the following + * algorithm can be competitive. + */ + + simdjson_inline void write_index(uint32_t idx, uint64_t& bits, int i) { + this->tail[i] = idx + trailing_zeroes(bits); + bits = clear_lowest_bit(bits); + } +#endif // SIMDJSON_PREFER_REVERSE_BITS + + template + simdjson_inline int write_indexes(uint32_t idx, uint64_t& bits) { + write_index(idx, bits, START); + SIMDJSON_IF_CONSTEXPR(N > 1) { + write_indexes<(N - 1 > 0 ? START + 1 : START), (N - 1 >= 0 ? N - 1 : 1)>(idx, bits); + } + return START + N; + } + + template + simdjson_inline int write_indexes_stepped(uint32_t idx, uint64_t& bits, int cnt) { + write_indexes(idx, bits); + SIMDJSON_IF_CONSTEXPR((START + STEP) < END) { + if (simdjson_unlikely((START + STEP) < cnt)) { + write_indexes_stepped<(START + STEP < END ? START + STEP : END), END, STEP>(idx, bits, cnt); + } + } + return ((END - START) % STEP) == 0 ? END : (END - START) - ((END - START) % STEP) + STEP; + } + + // flatten out values in 'bits' assuming that they are are to have values of idx + // plus their position in the bitvector, and store these indexes at + // base_ptr[base] incrementing base as we go + // will potentially store extra values beyond end of valid bits, so base_ptr + // needs to be large enough to handle this + // + // If the kernel sets SIMDJSON_GENERIC_JSON_STRUCTURAL_INDEXER_CUSTOM_BIT_INDEXER, then it + // will provide its own version of the code. +#ifdef SIMDJSON_GENERIC_JSON_STRUCTURAL_INDEXER_CUSTOM_BIT_INDEXER + simdjson_inline void write(uint32_t idx, uint64_t bits); +#else + simdjson_inline void write(uint32_t idx, uint64_t bits) { + // In some instances, the next branch is expensive because it is mispredicted. + // Unfortunately, in other cases, + // it helps tremendously. + if (bits == 0) + return; + + int cnt = static_cast(count_ones(bits)); + +#if SIMDJSON_PREFER_REVERSE_BITS + bits = reverse_bits(bits); +#endif +#ifdef SIMDJSON_STRUCTURAL_INDEXER_STEP + static constexpr const int STEP = SIMDJSON_STRUCTURAL_INDEXER_STEP; +#else + static constexpr const int STEP = 4; +#endif + static constexpr const int STEP_UNTIL = 24; + + write_indexes_stepped<0, STEP_UNTIL, STEP>(idx, bits, cnt); + SIMDJSON_IF_CONSTEXPR(STEP_UNTIL < 64) { + if (simdjson_unlikely(STEP_UNTIL < cnt)) { + for (int i = STEP_UNTIL; i < cnt; i++) { + write_index(idx, bits, i); + } + } + } + + this->tail += cnt; + } +#endif // SIMDJSON_GENERIC_JSON_STRUCTURAL_INDEXER_CUSTOM_BIT_INDEXER + + }; + + class json_structural_indexer { + public: + /** + * Find the important bits of JSON in a 128-byte chunk, and add them to structural_indexes. + * + * @param partial Setting the partial parameter to true allows the find_structural_bits to + * tolerate unclosed strings. The caller should still ensure that the input is valid UTF-8. If + * you are processing substrings, you may want to call on a function like trimmed_length_safe_utf8. + */ + template + static error_code index(const uint8_t* buf, size_t len, dom_parser_implementation& parser, stage1_mode partial) noexcept; + + private: + simdjson_inline json_structural_indexer(uint32_t* structural_indexes); + template + simdjson_inline void step(const uint8_t* block, buf_block_reader& reader) noexcept; + simdjson_inline void next(const simd::simd8x64& in, const json_block& block, size_t idx); + simdjson_inline error_code finish(dom_parser_implementation& parser, size_t idx, size_t len, stage1_mode partial); + + json_scanner scanner{}; + utf8_checker checker{}; + bit_indexer indexer; + uint64_t prev_structurals = 0; + uint64_t unescaped_chars_error = 0; + }; + + simdjson_inline json_structural_indexer::json_structural_indexer(uint32_t* structural_indexes) : indexer{ structural_indexes } {} + + // Skip the last character if it is partial + simdjson_inline size_t trim_partial_utf8(const uint8_t* buf, size_t len) { + if (simdjson_unlikely(len < 3)) { + switch (len) { + case 2: + if (buf[len - 1] >= 0xc0) { return len - 1; } // 2-, 3- and 4-byte characters with only 1 byte left + if (buf[len - 2] >= 0xe0) { return len - 2; } // 3- and 4-byte characters with only 2 bytes left + return len; + case 1: + if (buf[len - 1] >= 0xc0) { return len - 1; } // 2-, 3- and 4-byte characters with only 1 byte left + return len; + case 0: + return len; + } + } + if (buf[len - 1] >= 0xc0) { return len - 1; } // 2-, 3- and 4-byte characters with only 1 byte left + if (buf[len - 2] >= 0xe0) { return len - 2; } // 3- and 4-byte characters with only 1 byte left + if (buf[len - 3] >= 0xf0) { return len - 3; } // 4-byte characters with only 3 bytes left + return len; + } + + // + // PERF NOTES: + // We pipe 2 inputs through these stages: + // 1. Load JSON into registers. This takes a long time and is highly parallelizable, so we load + // 2 inputs' worth at once so that by the time step 2 is looking for them input, it's available. + // 2. Scan the JSON for critical data: strings, scalars and operators. This is the critical path. + // The output of step 1 depends entirely on this information. These functions don't quite use + // up enough CPU: the second half of the functions is highly serial, only using 1 execution core + // at a time. The second input's scans has some dependency on the first ones finishing it, but + // they can make a lot of progress before they need that information. + // 3. Step 1 does not use enough capacity, so we run some extra stuff while we're waiting for that + // to finish: utf-8 checks and generating the output from the last iteration. + // + // The reason we run 2 inputs at a time, is steps 2 and 3 are *still* not enough to soak up all + // available capacity with just one input. Running 2 at a time seems to give the CPU a good enough + // workout. + // + template + error_code json_structural_indexer::index(const uint8_t* buf, size_t len, dom_parser_implementation& parser, stage1_mode partial) noexcept { + if (simdjson_unlikely(len > parser.capacity())) { return CAPACITY; } + // We guard the rest of the code so that we can assume that len > 0 throughout. + if (len == 0) { return EMPTY; } + if (is_streaming(partial)) { + len = trim_partial_utf8(buf, len); + // If you end up with an empty window after trimming + // the partial UTF-8 bytes, then chances are good that you + // have an UTF-8 formatting error. + if (len == 0) { return UTF8_ERROR; } + } + buf_block_reader reader(buf, len); + json_structural_indexer indexer(parser.structural_indexes.get()); + + // Read all but the last block + while (reader.has_full_block()) { + indexer.step(reader.full_block(), reader); + } + // Take care of the last block (will always be there unless file is empty which is + // not supposed to happen.) + uint8_t block[STEP_SIZE]; + if (simdjson_unlikely(reader.get_remainder(block) == 0)) { return UNEXPECTED_ERROR; } + indexer.step(block, reader); + return indexer.finish(parser, reader.block_index(), len, partial); + } + + template<> + simdjson_inline void json_structural_indexer::step<128>(const uint8_t* block, buf_block_reader<128>& reader) noexcept { + simd::simd8x64 in_1(block); + simd::simd8x64 in_2(block + 64); + json_block block_1 = scanner.next(in_1); + json_block block_2 = scanner.next(in_2); + this->next(in_1, block_1, reader.block_index()); + this->next(in_2, block_2, reader.block_index() + 64); + reader.advance(); + } + + template<> + simdjson_inline void json_structural_indexer::step<64>(const uint8_t* block, buf_block_reader<64>& reader) noexcept { + simd::simd8x64 in_1(block); + json_block block_1 = scanner.next(in_1); + this->next(in_1, block_1, reader.block_index()); + reader.advance(); + } + + simdjson_inline void json_structural_indexer::next(const simd::simd8x64& in, const json_block& block, size_t idx) { + uint64_t unescaped = in.lteq(0x1F); +#if SIMDJSON_UTF8VALIDATION + checker.check_next_input(in); +#endif + indexer.write(uint32_t(idx - 64), prev_structurals); // Output *last* iteration's structurals to the parser + prev_structurals = block.structural_start(); + unescaped_chars_error |= block.non_quote_inside_string(unescaped); + } + + simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementation& parser, size_t idx, size_t len, stage1_mode partial) { + // Write out the final iteration's structurals + indexer.write(uint32_t(idx - 64), prev_structurals); + error_code error = scanner.finish(); + // We deliberately break down the next expression so that it is + // human readable. + const bool should_we_exit = is_streaming(partial) ? + ((error != SUCCESS) && (error != UNCLOSED_STRING)) // when partial we tolerate UNCLOSED_STRING + : (error != SUCCESS); // if partial is false, we must have SUCCESS + const bool have_unclosed_string = (error == UNCLOSED_STRING); + if (simdjson_unlikely(should_we_exit)) { return error; } + + if (unescaped_chars_error) { + return UNESCAPED_CHARS; + } + parser.n_structural_indexes = uint32_t(indexer.tail - parser.structural_indexes.get()); + /*** + * The On-Demand API requires special padding. + * + * This is related to https://github.com/simdjson/simdjson/issues/906 + * Basically, we want to make sure that if the parsing continues beyond the last (valid) + * structural character, it quickly stops. + * Only three structural characters can be repeated without triggering an error in JSON: [,] and }. + * We repeat the padding character (at 'len'). We don't know what it is, but if the parsing + * continues, then it must be [,] or }. + * Suppose it is ] or }. We backtrack to the first character, what could it be that would + * not trigger an error? It could be ] or } but no, because you can't start a document that way. + * It can't be a comma, a colon or any simple value. So the only way we could continue is + * if the repeated character is [. But if so, the document must start with [. But if the document + * starts with [, it should end with ]. If we enforce that rule, then we would get + * ][[ which is invalid. + * + * This is illustrated with the test array_iterate_unclosed_error() on the following input: + * R"({ "a": [,,)" + **/ + parser.structural_indexes[parser.n_structural_indexes] = uint32_t(len); // used later in partial == stage1_mode::streaming_final + parser.structural_indexes[parser.n_structural_indexes + 1] = uint32_t(len); + parser.structural_indexes[parser.n_structural_indexes + 2] = 0; + parser.next_structural_index = 0; + // a valid JSON file cannot have zero structural indexes - we should have found something + if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { + return EMPTY; + } + if (simdjson_unlikely(parser.structural_indexes[parser.n_structural_indexes - 1] > len)) { + return UNEXPECTED_ERROR; + } + if (partial == stage1_mode::streaming_partial) { + // If we have an unclosed string, then the last structural + // will be the quote and we want to make sure to omit it. + if (have_unclosed_string) { + parser.n_structural_indexes--; + // a valid JSON file cannot have zero structural indexes - we should have found something + if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return CAPACITY; } + } + // We truncate the input to the end of the last complete document (or zero). + auto new_structural_indexes = find_next_document_index(parser); + if (new_structural_indexes == 0 && parser.n_structural_indexes > 0) { + if (parser.structural_indexes[0] == 0) { + // If the buffer is partial and we started at index 0 but the document is + // incomplete, it's too big to parse. + return CAPACITY; + } + else { + // It is possible that the document could be parsed, we just had a lot + // of white space. + parser.n_structural_indexes = 0; + return EMPTY; + } + } + + parser.n_structural_indexes = new_structural_indexes; + } + else if (partial == stage1_mode::streaming_final) { + if (have_unclosed_string) { parser.n_structural_indexes--; } + // We truncate the input to the end of the last complete document (or zero). + // Because partial == stage1_mode::streaming_final, it means that we may + // silently ignore trailing garbage. Though it sounds bad, we do it + // deliberately because many people who have streams of JSON documents + // will truncate them for processing. E.g., imagine that you are uncompressing + // the data from a size file or receiving it in chunks from the network. You + // may not know where exactly the last document will be. Meanwhile the + // document_stream instances allow people to know the JSON documents they are + // parsing (see the iterator.source() method). + parser.n_structural_indexes = find_next_document_index(parser); + // We store the initial n_structural_indexes so that the client can see + // whether we used truncation. If initial_n_structural_indexes == parser.n_structural_indexes, + // then this will query parser.structural_indexes[parser.n_structural_indexes] which is len, + // otherwise, it will copy some prior index. + parser.structural_indexes[parser.n_structural_indexes + 1] = parser.structural_indexes[parser.n_structural_indexes]; + // This next line is critical, do not change it unless you understand what you are + // doing. + parser.structural_indexes[parser.n_structural_indexes] = uint32_t(len); + if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { + // We tolerate an unclosed string at the very end of the stream. Indeed, users + // often load their data in bulk without being careful and they want us to ignore + // the trailing garbage. + return EMPTY; + } + } + checker.check_eof(); + return checker.errors(); + } + + } // namespace stage1 + } // unnamed namespace + } // namespace haswell +} // namespace simdjson + +// Clear CUSTOM_BIT_INDEXER so other implementations can set it if they need to. +#undef SIMDJSON_GENERIC_JSON_STRUCTURAL_INDEXER_CUSTOM_BIT_INDEXER + +#endif // SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRUCTURAL_INDEXER_H +/* end file generic/stage1/json_structural_indexer.h for haswell */ +/* including generic/stage1/utf8_validator.h for haswell: #include */ +/* begin file generic/stage1/utf8_validator.h for haswell */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE1_UTF8_VALIDATOR_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_UTF8_VALIDATOR_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace haswell { + namespace { + namespace stage1 { + + /** + * Validates that the string is actual UTF-8. + */ + template + bool generic_validate_utf8(const uint8_t* input, size_t length) { + checker c{}; + buf_block_reader<64> reader(input, length); + while (reader.has_full_block()) { + simd::simd8x64 in(reader.full_block()); + c.check_next_input(in); + reader.advance(); + } + uint8_t block[64]{}; + reader.get_remainder(block); + simd::simd8x64 in(block); + c.check_next_input(in); + reader.advance(); + c.check_eof(); + return c.errors() == error_code::SUCCESS; + } + + bool generic_validate_utf8(const char* input, size_t length) { + return generic_validate_utf8(reinterpret_cast(input), length); + } + + } // namespace stage1 + } // unnamed namespace + } // namespace haswell +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE1_UTF8_VALIDATOR_H +/* end file generic/stage1/utf8_validator.h for haswell */ +/* end file generic/stage1/amalgamated.h for haswell */ +/* including generic/stage2/amalgamated.h for haswell: #include */ +/* begin file generic/stage2/amalgamated.h for haswell */ +// Stuff other things depend on +/* including generic/stage2/base.h for haswell: #include */ +/* begin file generic/stage2/base.h for haswell */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE2_BASE_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_BASE_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace haswell { + namespace { + namespace stage2 { + + class json_iterator; + class structural_iterator; + struct tape_builder; + struct tape_writer; + + } // namespace stage2 + } // unnamed namespace + } // namespace haswell +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE2_BASE_H +/* end file generic/stage2/base.h for haswell */ +/* including generic/stage2/tape_writer.h for haswell: #include */ +/* begin file generic/stage2/tape_writer.h for haswell */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE2_TAPE_WRITER_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_TAPE_WRITER_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +#include + +namespace simdjson { + namespace haswell { + namespace { + namespace stage2 { + + struct tape_writer { + /** The next place to write to tape */ + uint64_t* next_tape_loc; + + /** Write a signed 64-bit value to tape. */ + simdjson_inline void append_s64(int64_t value) noexcept; + + /** Write an unsigned 64-bit value to tape. */ + simdjson_inline void append_u64(uint64_t value) noexcept; + + /** Write a double value to tape. */ + simdjson_inline void append_double(double value) noexcept; + + /** + * Append a tape entry (an 8-bit type,and 56 bits worth of value). + */ + simdjson_inline void append(uint64_t val, internal::tape_type t) noexcept; + + /** + * Skip the current tape entry without writing. + * + * Used to skip the start of the container, since we'll come back later to fill it in when the + * container ends. + */ + simdjson_inline void skip() noexcept; + + /** + * Skip the number of tape entries necessary to write a large u64 or i64. + */ + simdjson_inline void skip_large_integer() noexcept; + + /** + * Skip the number of tape entries necessary to write a double. + */ + simdjson_inline void skip_double() noexcept; + + /** + * Write a value to a known location on tape. + * + * Used to go back and write out the start of a container after the container ends. + */ + simdjson_inline static void write(uint64_t& tape_loc, uint64_t val, internal::tape_type t) noexcept; + + private: + /** + * Append both the tape entry, and a supplementary value following it. Used for types that need + * all 64 bits, such as double and uint64_t. + */ + template + simdjson_inline void append2(uint64_t val, T val2, internal::tape_type t) noexcept; + }; // struct tape_writer + + simdjson_inline void tape_writer::append_s64(int64_t value) noexcept { + append2(0, value, internal::tape_type::INT64); + } + + simdjson_inline void tape_writer::append_u64(uint64_t value) noexcept { + append(0, internal::tape_type::UINT64); + *next_tape_loc = value; + next_tape_loc++; + } + + /** Write a double value to tape. */ + simdjson_inline void tape_writer::append_double(double value) noexcept { + append2(0, value, internal::tape_type::DOUBLE); + } + + simdjson_inline void tape_writer::skip() noexcept { + next_tape_loc++; + } + + simdjson_inline void tape_writer::skip_large_integer() noexcept { + next_tape_loc += 2; + } + + simdjson_inline void tape_writer::skip_double() noexcept { + next_tape_loc += 2; + } + + simdjson_inline void tape_writer::append(uint64_t val, internal::tape_type t) noexcept { + *next_tape_loc = val | ((uint64_t(char(t))) << 56); + next_tape_loc++; + } + + template + simdjson_inline void tape_writer::append2(uint64_t val, T val2, internal::tape_type t) noexcept { + append(val, t); + static_assert(sizeof(val2) == sizeof(*next_tape_loc), "Type is not 64 bits!"); + memcpy(next_tape_loc, &val2, sizeof(val2)); + next_tape_loc++; + } + + simdjson_inline void tape_writer::write(uint64_t& tape_loc, uint64_t val, internal::tape_type t) noexcept { + tape_loc = val | ((uint64_t(char(t))) << 56); + } + + } // namespace stage2 + } // unnamed namespace + } // namespace haswell +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE2_TAPE_WRITER_H +/* end file generic/stage2/tape_writer.h for haswell */ +/* including generic/stage2/logger.h for haswell: #include */ +/* begin file generic/stage2/logger.h for haswell */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE2_LOGGER_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_LOGGER_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +#include + + +// This is for an internal-only stage 2 specific logger. +// Set LOG_ENABLED = true to log what stage 2 is doing! +namespace simdjson { + namespace haswell { + namespace { + namespace logger { + + static constexpr const char* DASHES = "----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------"; + +#if SIMDJSON_VERBOSE_LOGGING + static constexpr const bool LOG_ENABLED = true; +#else + static constexpr const bool LOG_ENABLED = false; +#endif + static constexpr const int LOG_EVENT_LEN = 20; + static constexpr const int LOG_BUFFER_LEN = 30; + static constexpr const int LOG_SMALL_BUFFER_LEN = 10; + static constexpr const int LOG_INDEX_LEN = 5; + + static int log_depth; // Not threadsafe. Log only. + + // Helper to turn unprintable or newline characters into spaces + static simdjson_inline char printable_char(char c) { + if (c >= 0x20) { + return c; + } + else { + return ' '; + } + } + + // Print the header and set up log_start + static simdjson_inline void log_start() { + if (LOG_ENABLED) { + log_depth = 0; + printf("\n"); + printf("| %-*s | %-*s | %-*s | %-*s | Detail |\n", LOG_EVENT_LEN, "Event", LOG_BUFFER_LEN, "Buffer", LOG_SMALL_BUFFER_LEN, "Next", 5, "Next#"); + printf("|%.*s|%.*s|%.*s|%.*s|--------|\n", LOG_EVENT_LEN + 2, DASHES, LOG_BUFFER_LEN + 2, DASHES, LOG_SMALL_BUFFER_LEN + 2, DASHES, 5 + 2, DASHES); + } + } + + simdjson_unused static simdjson_inline void log_string(const char* message) { + if (LOG_ENABLED) { + printf("%s\n", message); + } + } + + // Logs a single line from the stage 2 DOM parser + template + static simdjson_inline void log_line(S& structurals, const char* title_prefix, const char* title, const char* detail) { + if (LOG_ENABLED) { + printf("| %*s%s%-*s ", log_depth * 2, "", title_prefix, LOG_EVENT_LEN - log_depth * 2 - int(strlen(title_prefix)), title); + auto current_index = structurals.at_beginning() ? nullptr : structurals.next_structural - 1; + auto next_index = structurals.next_structural; + auto current = current_index ? &structurals.buf[*current_index] : reinterpret_cast(" "); + auto next = &structurals.buf[*next_index]; + { + // Print the next N characters in the buffer. + printf("| "); + // Otherwise, print the characters starting from the buffer position. + // Print spaces for unprintable or newline characters. + for (int i = 0; i < LOG_BUFFER_LEN; i++) { + printf("%c", printable_char(current[i])); + } + printf(" "); + // Print the next N characters in the buffer. + printf("| "); + // Otherwise, print the characters starting from the buffer position. + // Print spaces for unprintable or newline characters. + for (int i = 0; i < LOG_SMALL_BUFFER_LEN; i++) { + printf("%c", printable_char(next[i])); + } + printf(" "); + } + if (current_index) { + printf("| %*u ", LOG_INDEX_LEN, *current_index); + } + else { + printf("| %-*s ", LOG_INDEX_LEN, ""); + } + // printf("| %*u ", LOG_INDEX_LEN, structurals.next_tape_index()); + printf("| %-s ", detail); + printf("|\n"); + } + } + + } // namespace logger + } // unnamed namespace + } // namespace haswell +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE2_LOGGER_H +/* end file generic/stage2/logger.h for haswell */ + +// All other declarations +/* including generic/stage2/json_iterator.h for haswell: #include */ +/* begin file generic/stage2/json_iterator.h for haswell */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE2_JSON_ITERATOR_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_JSON_ITERATOR_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace haswell { + namespace { + namespace stage2 { + + class json_iterator { + public: + const uint8_t* const buf; + uint32_t* next_structural; + dom_parser_implementation& dom_parser; + uint32_t depth{ 0 }; + + /** + * Walk the JSON document. + * + * The visitor receives callbacks when values are encountered. All callbacks pass the iterator as + * the first parameter; some callbacks have other parameters as well: + * + * - visit_document_start() - at the beginning. + * - visit_document_end() - at the end (if things were successful). + * + * - visit_array_start() - at the start `[` of a non-empty array. + * - visit_array_end() - at the end `]` of a non-empty array. + * - visit_empty_array() - when an empty array is encountered. + * + * - visit_object_end() - at the start `]` of a non-empty object. + * - visit_object_start() - at the end `]` of a non-empty object. + * - visit_empty_object() - when an empty object is encountered. + * - visit_key(const uint8_t *key) - when a key in an object field is encountered. key is + * guaranteed to point at the first quote of the string (`"key"`). + * - visit_primitive(const uint8_t *value) - when a value is a string, number, boolean or null. + * - visit_root_primitive(iter, uint8_t *value) - when the top-level value is a string, number, boolean or null. + * + * - increment_count(iter) - each time a value is found in an array or object. + */ + template + simdjson_warn_unused simdjson_inline error_code walk_document(V& visitor) noexcept; + + /** + * Create an iterator capable of walking a JSON document. + * + * The document must have already passed through stage 1. + */ + simdjson_inline json_iterator(dom_parser_implementation& _dom_parser, size_t start_structural_index); + + /** + * Look at the next token. + * + * Tokens can be strings, numbers, booleans, null, or operators (`[{]},:`)). + * + * They may include invalid JSON as well (such as `1.2.3` or `ture`). + */ + simdjson_inline const uint8_t* peek() const noexcept; + /** + * Advance to the next token. + * + * Tokens can be strings, numbers, booleans, null, or operators (`[{]},:`)). + * + * They may include invalid JSON as well (such as `1.2.3` or `ture`). + */ + simdjson_inline const uint8_t* advance() noexcept; + /** + * Get the remaining length of the document, from the start of the current token. + */ + simdjson_inline size_t remaining_len() const noexcept; + /** + * Check if we are at the end of the document. + * + * If this is true, there are no more tokens. + */ + simdjson_inline bool at_eof() const noexcept; + /** + * Check if we are at the beginning of the document. + */ + simdjson_inline bool at_beginning() const noexcept; + simdjson_inline uint8_t last_structural() const noexcept; + + /** + * Log that a value has been found. + * + * Set LOG_ENABLED=true in logger.h to see logging. + */ + simdjson_inline void log_value(const char* type) const noexcept; + /** + * Log the start of a multipart value. + * + * Set LOG_ENABLED=true in logger.h to see logging. + */ + simdjson_inline void log_start_value(const char* type) const noexcept; + /** + * Log the end of a multipart value. + * + * Set LOG_ENABLED=true in logger.h to see logging. + */ + simdjson_inline void log_end_value(const char* type) const noexcept; + /** + * Log an error. + * + * Set LOG_ENABLED=true in logger.h to see logging. + */ + simdjson_inline void log_error(const char* error) const noexcept; + + template + simdjson_warn_unused simdjson_inline error_code visit_root_primitive(V& visitor, const uint8_t* value) noexcept; + template + simdjson_warn_unused simdjson_inline error_code visit_primitive(V& visitor, const uint8_t* value) noexcept; + }; + + template + simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V& visitor) noexcept { + logger::log_start(); + + // + // Start the document + // + if (at_eof()) { return EMPTY; } + log_start_value("document"); + SIMDJSON_TRY(visitor.visit_document_start(*this)); + + // + // Read first value + // + { + auto value = advance(); + + // Make sure the outer object or array is closed before continuing; otherwise, there are ways we + // could get into memory corruption. See https://github.com/simdjson/simdjson/issues/906 + if (!STREAMING) { + switch (*value) { + case '{': if (last_structural() != '}') { log_value("starting brace unmatched"); return TAPE_ERROR; }; break; + case '[': if (last_structural() != ']') { log_value("starting bracket unmatched"); return TAPE_ERROR; }; break; + } + } + + switch (*value) { + case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY(visitor.visit_empty_object(*this)); break; } goto object_begin; + case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY(visitor.visit_empty_array(*this)); break; } goto array_begin; + default: SIMDJSON_TRY(visitor.visit_root_primitive(*this, value)); break; + } + } + goto document_end; + + // + // Object parser states + // + object_begin: + log_start_value("object"); + depth++; + if (depth >= dom_parser.max_depth()) { log_error("Exceeded max depth!"); return DEPTH_ERROR; } + dom_parser.is_array[depth] = false; + SIMDJSON_TRY(visitor.visit_object_start(*this)); + + { + auto key = advance(); + if (*key != '"') { log_error("Object does not start with a key"); return TAPE_ERROR; } + SIMDJSON_TRY(visitor.increment_count(*this)); + SIMDJSON_TRY(visitor.visit_key(*this, key)); + } + + object_field: + if (simdjson_unlikely(*advance() != ':')) { log_error("Missing colon after key in object"); return TAPE_ERROR; } + { + auto value = advance(); + switch (*value) { + case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY(visitor.visit_empty_object(*this)); break; } goto object_begin; + case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY(visitor.visit_empty_array(*this)); break; } goto array_begin; + default: SIMDJSON_TRY(visitor.visit_primitive(*this, value)); break; + } + } + + object_continue: + switch (*advance()) { + case ',': + SIMDJSON_TRY(visitor.increment_count(*this)); + { + auto key = advance(); + if (simdjson_unlikely(*key != '"')) { log_error("Key string missing at beginning of field in object"); return TAPE_ERROR; } + SIMDJSON_TRY(visitor.visit_key(*this, key)); + } + goto object_field; + case '}': log_end_value("object"); SIMDJSON_TRY(visitor.visit_object_end(*this)); goto scope_end; + default: log_error("No comma between object fields"); return TAPE_ERROR; + } + + scope_end: + depth--; + if (depth == 0) { goto document_end; } + if (dom_parser.is_array[depth]) { goto array_continue; } + goto object_continue; + + // + // Array parser states + // + array_begin: + log_start_value("array"); + depth++; + if (depth >= dom_parser.max_depth()) { log_error("Exceeded max depth!"); return DEPTH_ERROR; } + dom_parser.is_array[depth] = true; + SIMDJSON_TRY(visitor.visit_array_start(*this)); + SIMDJSON_TRY(visitor.increment_count(*this)); + + array_value: + { + auto value = advance(); + switch (*value) { + case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY(visitor.visit_empty_object(*this)); break; } goto object_begin; + case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY(visitor.visit_empty_array(*this)); break; } goto array_begin; + default: SIMDJSON_TRY(visitor.visit_primitive(*this, value)); break; + } + } + + array_continue: + switch (*advance()) { + case ',': SIMDJSON_TRY(visitor.increment_count(*this)); goto array_value; + case ']': log_end_value("array"); SIMDJSON_TRY(visitor.visit_array_end(*this)); goto scope_end; + default: log_error("Missing comma between array values"); return TAPE_ERROR; + } + + document_end: + log_end_value("document"); + SIMDJSON_TRY(visitor.visit_document_end(*this)); + + dom_parser.next_structural_index = uint32_t(next_structural - &dom_parser.structural_indexes[0]); + + // If we didn't make it to the end, it's an error + if (!STREAMING && dom_parser.next_structural_index != dom_parser.n_structural_indexes) { + log_error("More than one JSON value at the root of the document, or extra characters at the end of the JSON!"); + return TAPE_ERROR; + } + + return SUCCESS; + + } // walk_document() + + simdjson_inline json_iterator::json_iterator(dom_parser_implementation& _dom_parser, size_t start_structural_index) + : buf{ _dom_parser.buf }, + next_structural{ &_dom_parser.structural_indexes[start_structural_index] }, + dom_parser{ _dom_parser } { + } + + simdjson_inline const uint8_t* json_iterator::peek() const noexcept { + return &buf[*(next_structural)]; + } + simdjson_inline const uint8_t* json_iterator::advance() noexcept { + return &buf[*(next_structural++)]; + } + simdjson_inline size_t json_iterator::remaining_len() const noexcept { + return dom_parser.len - *(next_structural - 1); + } + + simdjson_inline bool json_iterator::at_eof() const noexcept { + return next_structural == &dom_parser.structural_indexes[dom_parser.n_structural_indexes]; + } + simdjson_inline bool json_iterator::at_beginning() const noexcept { + return next_structural == dom_parser.structural_indexes.get(); + } + simdjson_inline uint8_t json_iterator::last_structural() const noexcept { + return buf[dom_parser.structural_indexes[dom_parser.n_structural_indexes - 1]]; + } + + simdjson_inline void json_iterator::log_value(const char* type) const noexcept { + logger::log_line(*this, "", type, ""); + } + + simdjson_inline void json_iterator::log_start_value(const char* type) const noexcept { + logger::log_line(*this, "+", type, ""); + if (logger::LOG_ENABLED) { logger::log_depth++; } + } + + simdjson_inline void json_iterator::log_end_value(const char* type) const noexcept { + if (logger::LOG_ENABLED) { logger::log_depth--; } + logger::log_line(*this, "-", type, ""); + } + + simdjson_inline void json_iterator::log_error(const char* error) const noexcept { + logger::log_line(*this, "", "ERROR", error); + } + + template + simdjson_warn_unused simdjson_inline error_code json_iterator::visit_root_primitive(V& visitor, const uint8_t* value) noexcept { + switch (*value) { + case '"': return visitor.visit_root_string(*this, value); + case 't': return visitor.visit_root_true_atom(*this, value); + case 'f': return visitor.visit_root_false_atom(*this, value); + case 'n': return visitor.visit_root_null_atom(*this, value); + case '-': + case '0': case '1': case '2': case '3': case '4': + case '5': case '6': case '7': case '8': case '9': + return visitor.visit_root_number(*this, value); + default: + log_error("Document starts with a non-value character"); + return TAPE_ERROR; + } + } + template + simdjson_warn_unused simdjson_inline error_code json_iterator::visit_primitive(V& visitor, const uint8_t* value) noexcept { + // Use the fact that most scalars are going to be either strings or numbers. + if (*value == '"') { + return visitor.visit_string(*this, value); + } + else if (((*value - '0') < 10) || (*value == '-')) { + return visitor.visit_number(*this, value); + } + // true, false, null are uncommon. + switch (*value) { + case 't': return visitor.visit_true_atom(*this, value); + case 'f': return visitor.visit_false_atom(*this, value); + case 'n': return visitor.visit_null_atom(*this, value); + default: + log_error("Non-value found when value was expected!"); + return TAPE_ERROR; + } + } + + } // namespace stage2 + } // unnamed namespace + } // namespace haswell +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE2_JSON_ITERATOR_H +/* end file generic/stage2/json_iterator.h for haswell */ +/* including generic/stage2/stringparsing.h for haswell: #include */ +/* begin file generic/stage2/stringparsing.h for haswell */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +// This file contains the common code every implementation uses +// It is intended to be included multiple times and compiled multiple times + +namespace simdjson { + namespace haswell { + namespace { + /// @private + namespace stringparsing { + + // begin copypasta + // These chars yield themselves: " \ / + // b -> backspace, f -> formfeed, n -> newline, r -> cr, t -> horizontal tab + // u not handled in this table as it's complex + static const uint8_t escape_map[256] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x0. + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0x22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x2f, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x4. + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x5c, 0, 0, 0, // 0x5. + 0, 0, 0x08, 0, 0, 0, 0x0c, 0, 0, 0, 0, 0, 0, 0, 0x0a, 0, // 0x6. + 0, 0, 0x0d, 0, 0x09, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x7. + + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + }; + + // handle a unicode codepoint + // write appropriate values into dest + // src will advance 6 bytes or 12 bytes + // dest will advance a variable amount (return via pointer) + // return true if the unicode codepoint was valid + // We work in little-endian then swap at write time + simdjson_warn_unused + simdjson_inline bool handle_unicode_codepoint(const uint8_t** src_ptr, + uint8_t** dst_ptr, bool allow_replacement) { + // Use the default Unicode Character 'REPLACEMENT CHARACTER' (U+FFFD) + constexpr uint32_t substitution_code_point = 0xfffd; + // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the + // conversion is not valid; we defer the check for this to inside the + // multilingual plane check. + uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2); + *src_ptr += 6; + + // If we found a high surrogate, we must + // check for low surrogate for characters + // outside the Basic + // Multilingual Plane. + if (code_point >= 0xd800 && code_point < 0xdc00) { + const uint8_t* src_data = *src_ptr; + /* Compiler optimizations convert this to a single 16-bit load and compare on most platforms */ + if (((src_data[0] << 8) | src_data[1]) != ((static_cast ('\\') << 8) | static_cast ('u'))) { + if (!allow_replacement) { return false; } + code_point = substitution_code_point; + } + else { + uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(src_data + 2); + + // We have already checked that the high surrogate is valid and + // (code_point - 0xd800) < 1024. + // + // Check that code_point_2 is in the range 0xdc00..0xdfff + // and that code_point_2 was parsed from valid hex. + uint32_t low_bit = code_point_2 - 0xdc00; + if (low_bit >> 10) { + if (!allow_replacement) { return false; } + code_point = substitution_code_point; + } + else { + code_point = (((code_point - 0xd800) << 10) | low_bit) + 0x10000; + *src_ptr += 6; + } + + } + } + else if (code_point >= 0xdc00 && code_point <= 0xdfff) { + // If we encounter a low surrogate (not preceded by a high surrogate) + // then we have an error. + if (!allow_replacement) { return false; } + code_point = substitution_code_point; + } + size_t offset = jsoncharutils::codepoint_to_utf8(code_point, *dst_ptr); + *dst_ptr += offset; + return offset > 0; + } + + + // handle a unicode codepoint using the wobbly convention + // https://simonsapin.github.io/wtf-8/ + // write appropriate values into dest + // src will advance 6 bytes or 12 bytes + // dest will advance a variable amount (return via pointer) + // return true if the unicode codepoint was valid + // We work in little-endian then swap at write time + simdjson_warn_unused + simdjson_inline bool handle_unicode_codepoint_wobbly(const uint8_t** src_ptr, + uint8_t** dst_ptr) { + // It is not ideal that this function is nearly identical to handle_unicode_codepoint. + // + // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the + // conversion is not valid; we defer the check for this to inside the + // multilingual plane check. + uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2); + *src_ptr += 6; + // If we found a high surrogate, we must + // check for low surrogate for characters + // outside the Basic + // Multilingual Plane. + if (code_point >= 0xd800 && code_point < 0xdc00) { + const uint8_t* src_data = *src_ptr; + /* Compiler optimizations convert this to a single 16-bit load and compare on most platforms */ + if (((src_data[0] << 8) | src_data[1]) == ((static_cast ('\\') << 8) | static_cast ('u'))) { + uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(src_data + 2); + uint32_t low_bit = code_point_2 - 0xdc00; + if ((low_bit >> 10) == 0) { + code_point = + (((code_point - 0xd800) << 10) | low_bit) + 0x10000; + *src_ptr += 6; + } + } + } + + size_t offset = jsoncharutils::codepoint_to_utf8(code_point, *dst_ptr); + *dst_ptr += offset; + return offset > 0; + } + + + /** + * Unescape a valid UTF-8 string from src to dst, stopping at a final unescaped quote. There + * must be an unescaped quote terminating the string. It returns the final output + * position as pointer. In case of error (e.g., the string has bad escaped codes), + * then null_ptr is returned. It is assumed that the output buffer is large + * enough. E.g., if src points at 'joe"', then dst needs to have four free bytes + + * SIMDJSON_PADDING bytes. + */ + simdjson_warn_unused simdjson_inline uint8_t* parse_string(const uint8_t* src, uint8_t* dst, bool allow_replacement) { + while (1) { + // Copy the next n bytes, and find the backslash and quote in them. + auto bs_quote = backslash_and_quote::copy_and_find(src, dst); + // If the next thing is the end quote, copy and return + if (bs_quote.has_quote_first()) { + // we encountered quotes first. Move dst to point to quotes and exit + return dst + bs_quote.quote_index(); + } + if (bs_quote.has_backslash()) { + /* find out where the backspace is */ + auto bs_dist = bs_quote.backslash_index(); + uint8_t escape_char = src[bs_dist + 1]; + /* we encountered backslash first. Handle backslash */ + if (escape_char == 'u') { + /* move src/dst up to the start; they will be further adjusted + within the unicode codepoint handling code. */ + src += bs_dist; + dst += bs_dist; + if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { + return nullptr; + } + } + else { + /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and + * write bs_dist+1 characters to output + * note this may reach beyond the part of the buffer we've actually + * seen. I think this is ok */ + uint8_t escape_result = escape_map[escape_char]; + if (escape_result == 0u) { + return nullptr; /* bogus escape value is an error */ + } + dst[bs_dist] = escape_result; + src += bs_dist + 2; + dst += bs_dist + 1; + } + } + else { + /* they are the same. Since they can't co-occur, it means we + * encountered neither. */ + src += backslash_and_quote::BYTES_PROCESSED; + dst += backslash_and_quote::BYTES_PROCESSED; + } + } + } + + simdjson_warn_unused simdjson_inline uint8_t* parse_wobbly_string(const uint8_t* src, uint8_t* dst) { + // It is not ideal that this function is nearly identical to parse_string. + while (1) { + // Copy the next n bytes, and find the backslash and quote in them. + auto bs_quote = backslash_and_quote::copy_and_find(src, dst); + // If the next thing is the end quote, copy and return + if (bs_quote.has_quote_first()) { + // we encountered quotes first. Move dst to point to quotes and exit + return dst + bs_quote.quote_index(); + } + if (bs_quote.has_backslash()) { + /* find out where the backspace is */ + auto bs_dist = bs_quote.backslash_index(); + uint8_t escape_char = src[bs_dist + 1]; + /* we encountered backslash first. Handle backslash */ + if (escape_char == 'u') { + /* move src/dst up to the start; they will be further adjusted + within the unicode codepoint handling code. */ + src += bs_dist; + dst += bs_dist; + if (!handle_unicode_codepoint_wobbly(&src, &dst)) { + return nullptr; + } + } + else { + /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and + * write bs_dist+1 characters to output + * note this may reach beyond the part of the buffer we've actually + * seen. I think this is ok */ + uint8_t escape_result = escape_map[escape_char]; + if (escape_result == 0u) { + return nullptr; /* bogus escape value is an error */ + } + dst[bs_dist] = escape_result; + src += bs_dist + 2; + dst += bs_dist + 1; + } + } + else { + /* they are the same. Since they can't co-occur, it means we + * encountered neither. */ + src += backslash_and_quote::BYTES_PROCESSED; + dst += backslash_and_quote::BYTES_PROCESSED; + } + } + } + + } // namespace stringparsing + } // unnamed namespace + } // namespace haswell +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H +/* end file generic/stage2/stringparsing.h for haswell */ +/* including generic/stage2/structural_iterator.h for haswell: #include */ +/* begin file generic/stage2/structural_iterator.h for haswell */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE2_STRUCTURAL_ITERATOR_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_STRUCTURAL_ITERATOR_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace haswell { + namespace { + namespace stage2 { + + class structural_iterator { + public: + const uint8_t* const buf; + uint32_t* next_structural; + dom_parser_implementation& dom_parser; + + // Start a structural + simdjson_inline structural_iterator(dom_parser_implementation& _dom_parser, size_t start_structural_index) + : buf{ _dom_parser.buf }, + next_structural{ &_dom_parser.structural_indexes[start_structural_index] }, + dom_parser{ _dom_parser } { + } + // Get the buffer position of the current structural character + simdjson_inline const uint8_t* current() { + return &buf[*(next_structural - 1)]; + } + // Get the current structural character + simdjson_inline char current_char() { + return buf[*(next_structural - 1)]; + } + // Get the next structural character without advancing + simdjson_inline char peek_next_char() { + return buf[*next_structural]; + } + simdjson_inline const uint8_t* peek() { + return &buf[*next_structural]; + } + simdjson_inline const uint8_t* advance() { + return &buf[*(next_structural++)]; + } + simdjson_inline char advance_char() { + return buf[*(next_structural++)]; + } + simdjson_inline size_t remaining_len() { + return dom_parser.len - *(next_structural - 1); + } + + simdjson_inline bool at_end() { + return next_structural == &dom_parser.structural_indexes[dom_parser.n_structural_indexes]; + } + simdjson_inline bool at_beginning() { + return next_structural == dom_parser.structural_indexes.get(); + } + }; + + } // namespace stage2 + } // unnamed namespace + } // namespace haswell +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE2_STRUCTURAL_ITERATOR_H +/* end file generic/stage2/structural_iterator.h for haswell */ +/* including generic/stage2/tape_builder.h for haswell: #include */ +/* begin file generic/stage2/tape_builder.h for haswell */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE2_TAPE_BUILDER_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_TAPE_BUILDER_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + + +namespace simdjson { + namespace haswell { + namespace { + namespace stage2 { + + struct tape_builder { + template + simdjson_warn_unused static simdjson_inline error_code parse_document( + dom_parser_implementation& dom_parser, + dom::document& doc) noexcept; + + /** Called when a non-empty document starts. */ + simdjson_warn_unused simdjson_inline error_code visit_document_start(json_iterator& iter) noexcept; + /** Called when a non-empty document ends without error. */ + simdjson_warn_unused simdjson_inline error_code visit_document_end(json_iterator& iter) noexcept; + + /** Called when a non-empty array starts. */ + simdjson_warn_unused simdjson_inline error_code visit_array_start(json_iterator& iter) noexcept; + /** Called when a non-empty array ends. */ + simdjson_warn_unused simdjson_inline error_code visit_array_end(json_iterator& iter) noexcept; + /** Called when an empty array is found. */ + simdjson_warn_unused simdjson_inline error_code visit_empty_array(json_iterator& iter) noexcept; + + /** Called when a non-empty object starts. */ + simdjson_warn_unused simdjson_inline error_code visit_object_start(json_iterator& iter) noexcept; + /** + * Called when a key in a field is encountered. + * + * primitive, visit_object_start, visit_empty_object, visit_array_start, or visit_empty_array + * will be called after this with the field value. + */ + simdjson_warn_unused simdjson_inline error_code visit_key(json_iterator& iter, const uint8_t* key) noexcept; + /** Called when a non-empty object ends. */ + simdjson_warn_unused simdjson_inline error_code visit_object_end(json_iterator& iter) noexcept; + /** Called when an empty object is found. */ + simdjson_warn_unused simdjson_inline error_code visit_empty_object(json_iterator& iter) noexcept; + + /** + * Called when a string, number, boolean or null is found. + */ + simdjson_warn_unused simdjson_inline error_code visit_primitive(json_iterator& iter, const uint8_t* value) noexcept; + /** + * Called when a string, number, boolean or null is found at the top level of a document (i.e. + * when there is no array or object and the entire document is a single string, number, boolean or + * null. + * + * This is separate from primitive() because simdjson's normal primitive parsing routines assume + * there is at least one more token after the value, which is only true in an array or object. + */ + simdjson_warn_unused simdjson_inline error_code visit_root_primitive(json_iterator& iter, const uint8_t* value) noexcept; + + simdjson_warn_unused simdjson_inline error_code visit_string(json_iterator& iter, const uint8_t* value, bool key = false) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_number(json_iterator& iter, const uint8_t* value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_true_atom(json_iterator& iter, const uint8_t* value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_false_atom(json_iterator& iter, const uint8_t* value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_null_atom(json_iterator& iter, const uint8_t* value) noexcept; + + simdjson_warn_unused simdjson_inline error_code visit_root_string(json_iterator& iter, const uint8_t* value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_root_number(json_iterator& iter, const uint8_t* value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_root_true_atom(json_iterator& iter, const uint8_t* value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_root_false_atom(json_iterator& iter, const uint8_t* value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_root_null_atom(json_iterator& iter, const uint8_t* value) noexcept; + + /** Called each time a new field or element in an array or object is found. */ + simdjson_warn_unused simdjson_inline error_code increment_count(json_iterator& iter) noexcept; + + /** Next location to write to tape */ + tape_writer tape; + private: + /** Next write location in the string buf for stage 2 parsing */ + uint8_t* current_string_buf_loc; + + simdjson_inline tape_builder(dom::document& doc) noexcept; + + simdjson_inline uint32_t next_tape_index(json_iterator& iter) const noexcept; + simdjson_inline void start_container(json_iterator& iter) noexcept; + simdjson_warn_unused simdjson_inline error_code end_container(json_iterator& iter, internal::tape_type start, internal::tape_type end) noexcept; + simdjson_warn_unused simdjson_inline error_code empty_container(json_iterator& iter, internal::tape_type start, internal::tape_type end) noexcept; + simdjson_inline uint8_t* on_start_string(json_iterator& iter) noexcept; + simdjson_inline void on_end_string(uint8_t* dst) noexcept; + }; // struct tape_builder + + template + simdjson_warn_unused simdjson_inline error_code tape_builder::parse_document( + dom_parser_implementation& dom_parser, + dom::document& doc) noexcept { + dom_parser.doc = &doc; + json_iterator iter(dom_parser, STREAMING ? dom_parser.next_structural_index : 0); + tape_builder builder(doc); + return iter.walk_document(builder); + } + + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_primitive(json_iterator& iter, const uint8_t* value) noexcept { + return iter.visit_root_primitive(*this, value); + } + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_primitive(json_iterator& iter, const uint8_t* value) noexcept { + return iter.visit_primitive(*this, value); + } + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_empty_object(json_iterator& iter) noexcept { + return empty_container(iter, internal::tape_type::START_OBJECT, internal::tape_type::END_OBJECT); + } + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_empty_array(json_iterator& iter) noexcept { + return empty_container(iter, internal::tape_type::START_ARRAY, internal::tape_type::END_ARRAY); + } + + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_document_start(json_iterator& iter) noexcept { + start_container(iter); + return SUCCESS; + } + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_object_start(json_iterator& iter) noexcept { + start_container(iter); + return SUCCESS; + } + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_array_start(json_iterator& iter) noexcept { + start_container(iter); + return SUCCESS; + } + + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_object_end(json_iterator& iter) noexcept { + return end_container(iter, internal::tape_type::START_OBJECT, internal::tape_type::END_OBJECT); + } + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_array_end(json_iterator& iter) noexcept { + return end_container(iter, internal::tape_type::START_ARRAY, internal::tape_type::END_ARRAY); + } + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_document_end(json_iterator& iter) noexcept { + constexpr uint32_t start_tape_index = 0; + tape.append(start_tape_index, internal::tape_type::ROOT); + tape_writer::write(iter.dom_parser.doc->tape[start_tape_index], next_tape_index(iter), internal::tape_type::ROOT); + return SUCCESS; + } + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_key(json_iterator& iter, const uint8_t* key) noexcept { + return visit_string(iter, key, true); + } + + simdjson_warn_unused simdjson_inline error_code tape_builder::increment_count(json_iterator& iter) noexcept { + iter.dom_parser.open_containers[iter.depth].count++; // we have a key value pair in the object at parser.dom_parser.depth - 1 + return SUCCESS; + } + + simdjson_inline tape_builder::tape_builder(dom::document& doc) noexcept : tape{ doc.tape.get() }, current_string_buf_loc{ doc.string_buf.get() } {} + + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_string(json_iterator& iter, const uint8_t* value, bool key) noexcept { + iter.log_value(key ? "key" : "string"); + uint8_t* dst = on_start_string(iter); + dst = stringparsing::parse_string(value + 1, dst, false); // We do not allow replacement when the escape characters are invalid. + if (dst == nullptr) { + iter.log_error("Invalid escape in string"); + return STRING_ERROR; + } + on_end_string(dst); + return SUCCESS; + } + + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_string(json_iterator& iter, const uint8_t* value) noexcept { + return visit_string(iter, value); + } + + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_number(json_iterator& iter, const uint8_t* value) noexcept { + iter.log_value("number"); + return numberparsing::parse_number(value, tape); + } + + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_number(json_iterator& iter, const uint8_t* value) noexcept { + // + // We need to make a copy to make sure that the string is space terminated. + // This is not about padding the input, which should already padded up + // to len + SIMDJSON_PADDING. However, we have no control at this stage + // on how the padding was done. What if the input string was padded with nulls? + // It is quite common for an input string to have an extra null character (C string). + // We do not want to allow 9\0 (where \0 is the null character) inside a JSON + // document, but the string "9\0" by itself is fine. So we make a copy and + // pad the input with spaces when we know that there is just one input element. + // This copy is relatively expensive, but it will almost never be called in + // practice unless you are in the strange scenario where you have many JSON + // documents made of single atoms. + // + std::unique_ptrcopy(new (std::nothrow) uint8_t[iter.remaining_len() + SIMDJSON_PADDING]); + if (copy.get() == nullptr) { return MEMALLOC; } + std::memcpy(copy.get(), value, iter.remaining_len()); + std::memset(copy.get() + iter.remaining_len(), ' ', SIMDJSON_PADDING); + error_code error = visit_number(iter, copy.get()); + return error; + } + + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_true_atom(json_iterator& iter, const uint8_t* value) noexcept { + iter.log_value("true"); + if (!atomparsing::is_valid_true_atom(value)) { return T_ATOM_ERROR; } + tape.append(0, internal::tape_type::TRUE_VALUE); + return SUCCESS; + } + + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_true_atom(json_iterator& iter, const uint8_t* value) noexcept { + iter.log_value("true"); + if (!atomparsing::is_valid_true_atom(value, iter.remaining_len())) { return T_ATOM_ERROR; } + tape.append(0, internal::tape_type::TRUE_VALUE); + return SUCCESS; + } + + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_false_atom(json_iterator& iter, const uint8_t* value) noexcept { + iter.log_value("false"); + if (!atomparsing::is_valid_false_atom(value)) { return F_ATOM_ERROR; } + tape.append(0, internal::tape_type::FALSE_VALUE); + return SUCCESS; + } + + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_false_atom(json_iterator& iter, const uint8_t* value) noexcept { + iter.log_value("false"); + if (!atomparsing::is_valid_false_atom(value, iter.remaining_len())) { return F_ATOM_ERROR; } + tape.append(0, internal::tape_type::FALSE_VALUE); + return SUCCESS; + } + + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_null_atom(json_iterator& iter, const uint8_t* value) noexcept { + iter.log_value("null"); + if (!atomparsing::is_valid_null_atom(value)) { return N_ATOM_ERROR; } + tape.append(0, internal::tape_type::NULL_VALUE); + return SUCCESS; + } + + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_null_atom(json_iterator& iter, const uint8_t* value) noexcept { + iter.log_value("null"); + if (!atomparsing::is_valid_null_atom(value, iter.remaining_len())) { return N_ATOM_ERROR; } + tape.append(0, internal::tape_type::NULL_VALUE); + return SUCCESS; + } + + // private: + + simdjson_inline uint32_t tape_builder::next_tape_index(json_iterator& iter) const noexcept { + return uint32_t(tape.next_tape_loc - iter.dom_parser.doc->tape.get()); + } + + simdjson_warn_unused simdjson_inline error_code tape_builder::empty_container(json_iterator& iter, internal::tape_type start, internal::tape_type end) noexcept { + auto start_index = next_tape_index(iter); + tape.append(start_index + 2, start); + tape.append(start_index, end); + return SUCCESS; + } + + simdjson_inline void tape_builder::start_container(json_iterator& iter) noexcept { + iter.dom_parser.open_containers[iter.depth].tape_index = next_tape_index(iter); + iter.dom_parser.open_containers[iter.depth].count = 0; + tape.skip(); // We don't actually *write* the start element until the end. + } + + simdjson_warn_unused simdjson_inline error_code tape_builder::end_container(json_iterator& iter, internal::tape_type start, internal::tape_type end) noexcept { + // Write the ending tape element, pointing at the start location + const uint32_t start_tape_index = iter.dom_parser.open_containers[iter.depth].tape_index; + tape.append(start_tape_index, end); + // Write the start tape element, pointing at the end location (and including count) + // count can overflow if it exceeds 24 bits... so we saturate + // the convention being that a cnt of 0xffffff or more is undetermined in value (>= 0xffffff). + const uint32_t count = iter.dom_parser.open_containers[iter.depth].count; + const uint32_t cntsat = count > 0xFFFFFF ? 0xFFFFFF : count; + tape_writer::write(iter.dom_parser.doc->tape[start_tape_index], next_tape_index(iter) | (uint64_t(cntsat) << 32), start); + return SUCCESS; + } + + simdjson_inline uint8_t* tape_builder::on_start_string(json_iterator& iter) noexcept { + // we advance the point, accounting for the fact that we have a NULL termination + tape.append(current_string_buf_loc - iter.dom_parser.doc->string_buf.get(), internal::tape_type::STRING); + return current_string_buf_loc + sizeof(uint32_t); + } + + simdjson_inline void tape_builder::on_end_string(uint8_t* dst) noexcept { + uint32_t str_length = uint32_t(dst - (current_string_buf_loc + sizeof(uint32_t))); + // TODO check for overflow in case someone has a crazy string (>=4GB?) + // But only add the overflow check when the document itself exceeds 4GB + // Currently unneeded because we refuse to parse docs larger or equal to 4GB. + memcpy(current_string_buf_loc, &str_length, sizeof(uint32_t)); + // NULL termination is still handy if you expect all your strings to + // be NULL terminated? It comes at a small cost + *dst = 0; + current_string_buf_loc = dst + 1; + } + + } // namespace stage2 + } // unnamed namespace + } // namespace haswell +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE2_TAPE_BUILDER_H +/* end file generic/stage2/tape_builder.h for haswell */ +/* end file generic/stage2/amalgamated.h for haswell */ + +// +// Stage 1 +// + +namespace simdjson { + namespace haswell { + + simdjson_warn_unused error_code implementation::create_dom_parser_implementation( + size_t capacity, + size_t max_depth, + std::unique_ptr& dst + ) const noexcept { + dst.reset(new (std::nothrow) dom_parser_implementation()); + if (!dst) { return MEMALLOC; } + if (auto err = dst->set_capacity(capacity)) + return err; + if (auto err = dst->set_max_depth(max_depth)) + return err; + return SUCCESS; + } + + namespace { + + using namespace simd; + + // This identifies structural characters (comma, colon, braces, brackets), + // and ASCII white-space ('\r','\n','\t',' '). + simdjson_inline json_character_block json_character_block::classify(const simd::simd8x64& in) { + // These lookups rely on the fact that anything < 127 will match the lower 4 bits, which is why + // we can't use the generic lookup_16. + const auto whitespace_table = simd8::repeat_16(' ', 100, 100, 100, 17, 100, 113, 2, 100, '\t', '\n', 112, 100, '\r', 100, 100); + + // The 6 operators (:,[]{}) have these values: + // + // , 2C + // : 3A + // [ 5B + // { 7B + // ] 5D + // } 7D + // + // If you use | 0x20 to turn [ and ] into { and }, the lower 4 bits of each character is unique. + // We exploit this, using a simd 4-bit lookup to tell us which character match against, and then + // match it (against | 0x20). + // + // To prevent recognizing other characters, everything else gets compared with 0, which cannot + // match due to the | 0x20. + // + // NOTE: Due to the | 0x20, this ALSO treats and (control characters 0C and 1A) like , + // and :. This gets caught in stage 2, which checks the actual character to ensure the right + // operators are in the right places. + const auto op_table = simd8::repeat_16( + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, ':', '{', // : = 3A, [ = 5B, { = 7B + ',', '}', 0, 0 // , = 2C, ] = 5D, } = 7D + ); + + // We compute whitespace and op separately. If later code only uses one or the + // other, given the fact that all functions are aggressively inlined, we can + // hope that useless computations will be omitted. This is namely case when + // minifying (we only need whitespace). + + const uint64_t whitespace = in.eq({ + _mm256_shuffle_epi8(whitespace_table, in.chunks[0]), + _mm256_shuffle_epi8(whitespace_table, in.chunks[1]) + }); + // Turn [ and ] into { and } + const simd8x64 curlified{ + in.chunks[0] | 0x20, + in.chunks[1] | 0x20 + }; + const uint64_t op = curlified.eq({ + _mm256_shuffle_epi8(op_table, in.chunks[0]), + _mm256_shuffle_epi8(op_table, in.chunks[1]) + }); + + return { whitespace, op }; + } + + simdjson_inline bool is_ascii(const simd8x64& input) { + return input.reduce_or().is_ascii(); + } + + simdjson_unused simdjson_inline simd8 must_be_continuation(const simd8 prev1, const simd8 prev2, const simd8 prev3) { + simd8 is_second_byte = prev1.saturating_sub(0xc0u - 1); // Only 11______ will be > 0 + simd8 is_third_byte = prev2.saturating_sub(0xe0u - 1); // Only 111_____ will be > 0 + simd8 is_fourth_byte = prev3.saturating_sub(0xf0u - 1); // Only 1111____ will be > 0 + // Caller requires a bool (all 1's). All values resulting from the subtraction will be <= 64, so signed comparison is fine. + return simd8(is_second_byte | is_third_byte | is_fourth_byte) > int8_t(0); + } + + simdjson_inline simd8 must_be_2_3_continuation(const simd8 prev2, const simd8 prev3) { + simd8 is_third_byte = prev2.saturating_sub(0xe0u - 0x80); // Only 111_____ will be >= 0x80 + simd8 is_fourth_byte = prev3.saturating_sub(0xf0u - 0x80); // Only 1111____ will be >= 0x80 + return is_third_byte | is_fourth_byte; + } + + } // unnamed namespace + } // namespace haswell +} // namespace simdjson + +// +// Stage 2 +// + +// +// Implementation-specific overrides +// +namespace simdjson { + namespace haswell { + + simdjson_warn_unused error_code implementation::minify(const uint8_t* buf, size_t len, uint8_t* dst, size_t& dst_len) const noexcept { + return haswell::stage1::json_minifier::minify<128>(buf, len, dst, dst_len); + } + + simdjson_warn_unused error_code dom_parser_implementation::stage1(const uint8_t* _buf, size_t _len, stage1_mode streaming) noexcept { + this->buf = _buf; + this->len = _len; + return haswell::stage1::json_structural_indexer::index<128>(_buf, _len, *this, streaming); + } + + simdjson_warn_unused bool implementation::validate_utf8(const char* buf, size_t len) const noexcept { + return haswell::stage1::generic_validate_utf8(buf, len); + } + + simdjson_warn_unused error_code dom_parser_implementation::stage2(dom::document& _doc) noexcept { + return stage2::tape_builder::parse_document(*this, _doc); + } + + simdjson_warn_unused error_code dom_parser_implementation::stage2_next(dom::document& _doc) noexcept { + return stage2::tape_builder::parse_document(*this, _doc); + } + + simdjson_warn_unused uint8_t* dom_parser_implementation::parse_string(const uint8_t* src, uint8_t* dst, bool replacement_char) const noexcept { + return haswell::stringparsing::parse_string(src, dst, replacement_char); + } + + simdjson_warn_unused uint8_t* dom_parser_implementation::parse_wobbly_string(const uint8_t* src, uint8_t* dst) const noexcept { + return haswell::stringparsing::parse_wobbly_string(src, dst); + } + + simdjson_warn_unused error_code dom_parser_implementation::parse(const uint8_t* _buf, size_t _len, dom::document& _doc) noexcept { + auto error = stage1(_buf, _len, stage1_mode::regular); + if (error) { return error; } + return stage2(_doc); + } + + } // namespace haswell +} // namespace simdjson + +/* including simdjson/haswell/end.h: #include */ +/* begin file simdjson/haswell/end.h */ +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/haswell/base.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +#if !SIMDJSON_CAN_ALWAYS_RUN_HASWELL +SIMDJSON_UNTARGET_REGION +#endif + +/* undefining SIMDJSON_IMPLEMENTATION from "haswell" */ +#undef SIMDJSON_IMPLEMENTATION +/* end file simdjson/haswell/end.h */ + +#endif // SIMDJSON_SRC_HASWELL_CPP +/* end file haswell.cpp */ +#endif +#if SIMDJSON_IMPLEMENTATION_ICELAKE +/* including icelake.cpp: #include */ +/* begin file icelake.cpp */ +#ifndef SIMDJSON_SRC_ICELAKE_CPP +#define SIMDJSON_SRC_ICELAKE_CPP + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +/* including simdjson/icelake.h: #include */ +/* begin file simdjson/icelake.h */ +#ifndef SIMDJSON_ICELAKE_H +#define SIMDJSON_ICELAKE_H + +/* including simdjson/icelake/begin.h: #include "simdjson/icelake/begin.h" */ +/* begin file simdjson/icelake/begin.h */ +/* defining SIMDJSON_IMPLEMENTATION to "icelake" */ +#define SIMDJSON_IMPLEMENTATION icelake +/* including simdjson/icelake/base.h: #include "simdjson/icelake/base.h" */ +/* begin file simdjson/icelake/base.h */ +#ifndef SIMDJSON_ICELAKE_BASE_H +#define SIMDJSON_ICELAKE_BASE_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/base.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +// The constructor may be executed on any host, so we take care not to use SIMDJSON_TARGET_ICELAKE +namespace simdjson { + /** + * Implementation for Icelake (Intel AVX512). + */ + namespace icelake { + + class implementation; + + } // namespace icelake +} // namespace simdjson + +#endif // SIMDJSON_ICELAKE_BASE_H +/* end file simdjson/icelake/base.h */ +/* including simdjson/icelake/intrinsics.h: #include "simdjson/icelake/intrinsics.h" */ +/* begin file simdjson/icelake/intrinsics.h */ +#ifndef SIMDJSON_ICELAKE_INTRINSICS_H +#define SIMDJSON_ICELAKE_INTRINSICS_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/icelake/base.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +#if SIMDJSON_VISUAL_STUDIO +// under clang within visual studio, this will include +#include // visual studio or clang +#else +#include // elsewhere +#endif // SIMDJSON_VISUAL_STUDIO + +#if SIMDJSON_CLANG_VISUAL_STUDIO +/** + * You are not supposed, normally, to include these + * headers directly. Instead you should either include intrin.h + * or x86intrin.h. However, when compiling with clang + * under Windows (i.e., when _MSC_VER is set), these headers + * only get included *if* the corresponding features are detected + * from macros: + * e.g., if __AVX2__ is set... in turn, we normally set these + * macros by compiling against the corresponding architecture + * (e.g., arch:AVX2, -mavx2, etc.) which compiles the whole + * software with these advanced instructions. In simdjson, we + * want to compile the whole program for a generic target, + * and only target our specific kernels. As a workaround, + * we directly include the needed headers. These headers would + * normally guard against such usage, but we carefully included + * (or ) before, so the headers + * are fooled. + */ +#include // for _blsr_u64 +#include // for __lzcnt64 +#include // for most things (AVX2, AVX512, _popcnt64) +#include +#include +#include +#include +#include // for _mm_clmulepi64_si128 + // Important: we need the AVX-512 headers: +#include +#include +#include +#include +#include +#include +#include +// unfortunately, we may not get _blsr_u64, but, thankfully, clang +// has it as a macro. +#ifndef _blsr_u64 +// we roll our own +#define _blsr_u64(n) ((n - 1) & n) +#endif // _blsr_u64 +#endif // SIMDJSON_CLANG_VISUAL_STUDIO + +static_assert(sizeof(__m512i) <= simdjson::SIMDJSON_PADDING, "insufficient padding for icelake"); + +#endif // SIMDJSON_ICELAKE_INTRINSICS_H +/* end file simdjson/icelake/intrinsics.h */ + +#if !SIMDJSON_CAN_ALWAYS_RUN_ICELAKE +SIMDJSON_TARGET_REGION("avx512f,avx512dq,avx512cd,avx512bw,avx512vbmi,avx512vbmi2,avx512vl,avx2,bmi,pclmul,lzcnt,popcnt") +#endif + +/* including simdjson/icelake/bitmanipulation.h: #include "simdjson/icelake/bitmanipulation.h" */ +/* begin file simdjson/icelake/bitmanipulation.h */ +#ifndef SIMDJSON_ICELAKE_BITMANIPULATION_H +#define SIMDJSON_ICELAKE_BITMANIPULATION_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/icelake/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/icelake/intrinsics.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace icelake { + namespace { + + // We sometimes call trailing_zero on inputs that are zero, + // but the algorithms do not end up using the returned value. + // Sadly, sanitizers are not smart enough to figure it out. + SIMDJSON_NO_SANITIZE_UNDEFINED + // This function can be used safely even if not all bytes have been + // initialized. + // See issue https://github.com/simdjson/simdjson/issues/1965 + SIMDJSON_NO_SANITIZE_MEMORY + simdjson_inline int trailing_zeroes(uint64_t input_num) { +#if SIMDJSON_REGULAR_VISUAL_STUDIO + return (int)_tzcnt_u64(input_num); +#else // SIMDJSON_REGULAR_VISUAL_STUDIO + //////// + // You might expect the next line to be equivalent to + // return (int)_tzcnt_u64(input_num); + // but the generated code differs and might be less efficient? + //////// + return __builtin_ctzll(input_num); +#endif // SIMDJSON_REGULAR_VISUAL_STUDIO + } + + /* result might be undefined when input_num is zero */ + simdjson_inline uint64_t clear_lowest_bit(uint64_t input_num) { + return _blsr_u64(input_num); + } + + /* result might be undefined when input_num is zero */ + simdjson_inline int leading_zeroes(uint64_t input_num) { + return int(_lzcnt_u64(input_num)); + } + +#if SIMDJSON_REGULAR_VISUAL_STUDIO + simdjson_inline unsigned __int64 count_ones(uint64_t input_num) { + // note: we do not support legacy 32-bit Windows + return __popcnt64(input_num);// Visual Studio wants two underscores + } +#else + simdjson_inline long long int count_ones(uint64_t input_num) { + return _popcnt64(input_num); + } +#endif + + simdjson_inline bool add_overflow(uint64_t value1, uint64_t value2, + uint64_t* result) { +#if SIMDJSON_REGULAR_VISUAL_STUDIO + return _addcarry_u64(0, value1, value2, + reinterpret_cast(result)); +#else + return __builtin_uaddll_overflow(value1, value2, + reinterpret_cast(result)); +#endif + } + + } // unnamed namespace + } // namespace icelake +} // namespace simdjson + +#endif // SIMDJSON_ICELAKE_BITMANIPULATION_H +/* end file simdjson/icelake/bitmanipulation.h */ +/* including simdjson/icelake/bitmask.h: #include "simdjson/icelake/bitmask.h" */ +/* begin file simdjson/icelake/bitmask.h */ +#ifndef SIMDJSON_ICELAKE_BITMASK_H +#define SIMDJSON_ICELAKE_BITMASK_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/icelake/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/icelake/intrinsics.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace icelake { + namespace { + + // + // Perform a "cumulative bitwise xor," flipping bits each time a 1 is encountered. + // + // For example, prefix_xor(00100100) == 00011100 + // + simdjson_inline uint64_t prefix_xor(const uint64_t bitmask) { + // There should be no such thing with a processor supporting avx2 + // but not clmul. + __m128i all_ones = _mm_set1_epi8('\xFF'); + __m128i result = _mm_clmulepi64_si128(_mm_set_epi64x(0ULL, bitmask), all_ones, 0); + return _mm_cvtsi128_si64(result); + } + + } // unnamed namespace + } // namespace icelake +} // namespace simdjson + +#endif // SIMDJSON_ICELAKE_BITMASK_H +/* end file simdjson/icelake/bitmask.h */ +/* including simdjson/icelake/simd.h: #include "simdjson/icelake/simd.h" */ +/* begin file simdjson/icelake/simd.h */ +#ifndef SIMDJSON_ICELAKE_SIMD_H +#define SIMDJSON_ICELAKE_SIMD_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/icelake/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/icelake/intrinsics.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/icelake/bitmanipulation.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/simdprune_tables.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +#if defined(__GNUC__) && !defined(__clang__) +#if __GNUC__ == 8 +#define SIMDJSON_GCC8 1 +#endif // __GNUC__ == 8 +#endif // defined(__GNUC__) && !defined(__clang__) + +#if SIMDJSON_GCC8 +/** + * GCC 8 fails to provide _mm512_set_epi8. We roll our own. + */ +inline __m512i _mm512_set_epi8(uint8_t a0, uint8_t a1, uint8_t a2, uint8_t a3, uint8_t a4, uint8_t a5, uint8_t a6, uint8_t a7, uint8_t a8, uint8_t a9, uint8_t a10, uint8_t a11, uint8_t a12, uint8_t a13, uint8_t a14, uint8_t a15, uint8_t a16, uint8_t a17, uint8_t a18, uint8_t a19, uint8_t a20, uint8_t a21, uint8_t a22, uint8_t a23, uint8_t a24, uint8_t a25, uint8_t a26, uint8_t a27, uint8_t a28, uint8_t a29, uint8_t a30, uint8_t a31, uint8_t a32, uint8_t a33, uint8_t a34, uint8_t a35, uint8_t a36, uint8_t a37, uint8_t a38, uint8_t a39, uint8_t a40, uint8_t a41, uint8_t a42, uint8_t a43, uint8_t a44, uint8_t a45, uint8_t a46, uint8_t a47, uint8_t a48, uint8_t a49, uint8_t a50, uint8_t a51, uint8_t a52, uint8_t a53, uint8_t a54, uint8_t a55, uint8_t a56, uint8_t a57, uint8_t a58, uint8_t a59, uint8_t a60, uint8_t a61, uint8_t a62, uint8_t a63) { + return _mm512_set_epi64(uint64_t(a7) + (uint64_t(a6) << 8) + (uint64_t(a5) << 16) + (uint64_t(a4) << 24) + (uint64_t(a3) << 32) + (uint64_t(a2) << 40) + (uint64_t(a1) << 48) + (uint64_t(a0) << 56), + uint64_t(a15) + (uint64_t(a14) << 8) + (uint64_t(a13) << 16) + (uint64_t(a12) << 24) + (uint64_t(a11) << 32) + (uint64_t(a10) << 40) + (uint64_t(a9) << 48) + (uint64_t(a8) << 56), + uint64_t(a23) + (uint64_t(a22) << 8) + (uint64_t(a21) << 16) + (uint64_t(a20) << 24) + (uint64_t(a19) << 32) + (uint64_t(a18) << 40) + (uint64_t(a17) << 48) + (uint64_t(a16) << 56), + uint64_t(a31) + (uint64_t(a30) << 8) + (uint64_t(a29) << 16) + (uint64_t(a28) << 24) + (uint64_t(a27) << 32) + (uint64_t(a26) << 40) + (uint64_t(a25) << 48) + (uint64_t(a24) << 56), + uint64_t(a39) + (uint64_t(a38) << 8) + (uint64_t(a37) << 16) + (uint64_t(a36) << 24) + (uint64_t(a35) << 32) + (uint64_t(a34) << 40) + (uint64_t(a33) << 48) + (uint64_t(a32) << 56), + uint64_t(a47) + (uint64_t(a46) << 8) + (uint64_t(a45) << 16) + (uint64_t(a44) << 24) + (uint64_t(a43) << 32) + (uint64_t(a42) << 40) + (uint64_t(a41) << 48) + (uint64_t(a40) << 56), + uint64_t(a55) + (uint64_t(a54) << 8) + (uint64_t(a53) << 16) + (uint64_t(a52) << 24) + (uint64_t(a51) << 32) + (uint64_t(a50) << 40) + (uint64_t(a49) << 48) + (uint64_t(a48) << 56), + uint64_t(a63) + (uint64_t(a62) << 8) + (uint64_t(a61) << 16) + (uint64_t(a60) << 24) + (uint64_t(a59) << 32) + (uint64_t(a58) << 40) + (uint64_t(a57) << 48) + (uint64_t(a56) << 56)); +} +#endif // SIMDJSON_GCC8 + + + +namespace simdjson { + namespace icelake { + namespace { + namespace simd { + + // Forward-declared so they can be used by splat and friends. + template + struct base { + __m512i value; + + // Zero constructor + simdjson_inline base() : value{ __m512i() } {} + + // Conversion from SIMD register + simdjson_inline base(const __m512i _value) : value(_value) {} + + // Conversion to SIMD register + simdjson_inline operator const __m512i& () const { return this->value; } + simdjson_inline operator __m512i& () { return this->value; } + + // Bit operations + simdjson_inline Child operator|(const Child other) const { return _mm512_or_si512(*this, other); } + simdjson_inline Child operator&(const Child other) const { return _mm512_and_si512(*this, other); } + simdjson_inline Child operator^(const Child other) const { return _mm512_xor_si512(*this, other); } + simdjson_inline Child bit_andnot(const Child other) const { return _mm512_andnot_si512(other, *this); } + simdjson_inline Child& operator|=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast | other; return *this_cast; } + simdjson_inline Child& operator&=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast & other; return *this_cast; } + simdjson_inline Child& operator^=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast ^ other; return *this_cast; } + }; + + // Forward-declared so they can be used by splat and friends. + template + struct simd8; + + template> + struct base8 : base> { + typedef uint32_t bitmask_t; + typedef uint64_t bitmask2_t; + + simdjson_inline base8() : base>() {} + simdjson_inline base8(const __m512i _value) : base>(_value) {} + + friend simdjson_really_inline uint64_t operator==(const simd8 lhs, const simd8 rhs) { + return _mm512_cmpeq_epi8_mask(lhs, rhs); + } + + static const int SIZE = sizeof(base::value); + + template + simdjson_inline simd8 prev(const simd8 prev_chunk) const { + // workaround for compilers unable to figure out that 16 - N is a constant (GCC 8) + constexpr int shift = 16 - N; + return _mm512_alignr_epi8(*this, _mm512_permutex2var_epi64(prev_chunk, _mm512_set_epi64(13, 12, 11, 10, 9, 8, 7, 6), *this), shift); + } + }; + + // SIMD byte mask type (returned by things like eq and gt) + template<> + struct simd8 : base8 { + static simdjson_inline simd8 splat(bool _value) { return _mm512_set1_epi8(uint8_t(-(!!_value))); } + + simdjson_inline simd8() : base8() {} + simdjson_inline simd8(const __m512i _value) : base8(_value) {} + // Splat constructor + simdjson_inline simd8(bool _value) : base8(splat(_value)) {} + simdjson_inline bool any() const { return !!_mm512_test_epi8_mask(*this, *this); } + simdjson_inline simd8 operator~() const { return *this ^ true; } + }; + + template + struct base8_numeric : base8 { + static simdjson_inline simd8 splat(T _value) { return _mm512_set1_epi8(_value); } + static simdjson_inline simd8 zero() { return _mm512_setzero_si512(); } + static simdjson_inline simd8 load(const T values[64]) { + return _mm512_loadu_si512(reinterpret_cast(values)); + } + // Repeat 16 values as many times as necessary (usually for lookup tables) + static simdjson_inline simd8 repeat_16( + T v0, T v1, T v2, T v3, T v4, T v5, T v6, T v7, + T v8, T v9, T v10, T v11, T v12, T v13, T v14, T v15 + ) { + return simd8( + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15, + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15, + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15, + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15 + ); + } + + simdjson_inline base8_numeric() : base8() {} + simdjson_inline base8_numeric(const __m512i _value) : base8(_value) {} + + // Store to array + simdjson_inline void store(T dst[64]) const { return _mm512_storeu_si512(reinterpret_cast<__m512i*>(dst), *this); } + + // Addition/subtraction are the same for signed and unsigned + simdjson_inline simd8 operator+(const simd8 other) const { return _mm512_add_epi8(*this, other); } + simdjson_inline simd8 operator-(const simd8 other) const { return _mm512_sub_epi8(*this, other); } + simdjson_inline simd8& operator+=(const simd8 other) { *this = *this + other; return *static_cast*>(this); } + simdjson_inline simd8& operator-=(const simd8 other) { *this = *this - other; return *static_cast*>(this); } + + // Override to distinguish from bool version + simdjson_inline simd8 operator~() const { return *this ^ 0xFFu; } + + // Perform a lookup assuming the value is between 0 and 16 (undefined behavior for out of range values) + template + simdjson_inline simd8 lookup_16(simd8 lookup_table) const { + return _mm512_shuffle_epi8(lookup_table, *this); + } + + // Copies to 'output" all bytes corresponding to a 0 in the mask (interpreted as a bitset). + // Passing a 0 value for mask would be equivalent to writing out every byte to output. + // Only the first 32 - count_ones(mask) bytes of the result are significant but 32 bytes + // get written. + // Design consideration: it seems like a function with the + // signature simd8 compress(uint32_t mask) would be + // sensible, but the AVX ISA makes this kind of approach difficult. + template + simdjson_inline void compress(uint64_t mask, L* output) const { + _mm512_mask_compressstoreu_epi8(output, ~mask, *this); + } + + template + simdjson_inline simd8 lookup_16( + L replace0, L replace1, L replace2, L replace3, + L replace4, L replace5, L replace6, L replace7, + L replace8, L replace9, L replace10, L replace11, + L replace12, L replace13, L replace14, L replace15) const { + return lookup_16(simd8::repeat_16( + replace0, replace1, replace2, replace3, + replace4, replace5, replace6, replace7, + replace8, replace9, replace10, replace11, + replace12, replace13, replace14, replace15 + )); + } + }; + + // Signed bytes + template<> + struct simd8 : base8_numeric { + simdjson_inline simd8() : base8_numeric() {} + simdjson_inline simd8(const __m512i _value) : base8_numeric(_value) {} + // Splat constructor + simdjson_inline simd8(int8_t _value) : simd8(splat(_value)) {} + // Array constructor + simdjson_inline simd8(const int8_t values[64]) : simd8(load(values)) {} + // Member-by-member initialization + simdjson_inline simd8( + int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, + int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15, + int8_t v16, int8_t v17, int8_t v18, int8_t v19, int8_t v20, int8_t v21, int8_t v22, int8_t v23, + int8_t v24, int8_t v25, int8_t v26, int8_t v27, int8_t v28, int8_t v29, int8_t v30, int8_t v31, + int8_t v32, int8_t v33, int8_t v34, int8_t v35, int8_t v36, int8_t v37, int8_t v38, int8_t v39, + int8_t v40, int8_t v41, int8_t v42, int8_t v43, int8_t v44, int8_t v45, int8_t v46, int8_t v47, + int8_t v48, int8_t v49, int8_t v50, int8_t v51, int8_t v52, int8_t v53, int8_t v54, int8_t v55, + int8_t v56, int8_t v57, int8_t v58, int8_t v59, int8_t v60, int8_t v61, int8_t v62, int8_t v63 + ) : simd8(_mm512_set_epi8( + v63, v62, v61, v60, v59, v58, v57, v56, + v55, v54, v53, v52, v51, v50, v49, v48, + v47, v46, v45, v44, v43, v42, v41, v40, + v39, v38, v37, v36, v35, v34, v33, v32, + v31, v30, v29, v28, v27, v26, v25, v24, + v23, v22, v21, v20, v19, v18, v17, v16, + v15, v14, v13, v12, v11, v10, v9, v8, + v7, v6, v5, v4, v3, v2, v1, v0 + )) {} + + // Repeat 16 values as many times as necessary (usually for lookup tables) + simdjson_inline static simd8 repeat_16( + int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, + int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 + ) { + return simd8( + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15, + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15, + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15, + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15 + ); + } + + // Order-sensitive comparisons + simdjson_inline simd8 max_val(const simd8 other) const { return _mm512_max_epi8(*this, other); } + simdjson_inline simd8 min_val(const simd8 other) const { return _mm512_min_epi8(*this, other); } + + simdjson_inline simd8 operator>(const simd8 other) const { return _mm512_maskz_abs_epi8(_mm512_cmpgt_epi8_mask(*this, other), _mm512_set1_epi8(uint8_t(0x80))); } + simdjson_inline simd8 operator<(const simd8 other) const { return _mm512_maskz_abs_epi8(_mm512_cmpgt_epi8_mask(other, *this), _mm512_set1_epi8(uint8_t(0x80))); } + }; + + // Unsigned bytes + template<> + struct simd8 : base8_numeric { + simdjson_inline simd8() : base8_numeric() {} + simdjson_inline simd8(const __m512i _value) : base8_numeric(_value) {} + // Splat constructor + simdjson_inline simd8(uint8_t _value) : simd8(splat(_value)) {} + // Array constructor + simdjson_inline simd8(const uint8_t values[64]) : simd8(load(values)) {} + // Member-by-member initialization + simdjson_inline simd8( + uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, + uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15, + uint8_t v16, uint8_t v17, uint8_t v18, uint8_t v19, uint8_t v20, uint8_t v21, uint8_t v22, uint8_t v23, + uint8_t v24, uint8_t v25, uint8_t v26, uint8_t v27, uint8_t v28, uint8_t v29, uint8_t v30, uint8_t v31, + uint8_t v32, uint8_t v33, uint8_t v34, uint8_t v35, uint8_t v36, uint8_t v37, uint8_t v38, uint8_t v39, + uint8_t v40, uint8_t v41, uint8_t v42, uint8_t v43, uint8_t v44, uint8_t v45, uint8_t v46, uint8_t v47, + uint8_t v48, uint8_t v49, uint8_t v50, uint8_t v51, uint8_t v52, uint8_t v53, uint8_t v54, uint8_t v55, + uint8_t v56, uint8_t v57, uint8_t v58, uint8_t v59, uint8_t v60, uint8_t v61, uint8_t v62, uint8_t v63 + ) : simd8(_mm512_set_epi8( + v63, v62, v61, v60, v59, v58, v57, v56, + v55, v54, v53, v52, v51, v50, v49, v48, + v47, v46, v45, v44, v43, v42, v41, v40, + v39, v38, v37, v36, v35, v34, v33, v32, + v31, v30, v29, v28, v27, v26, v25, v24, + v23, v22, v21, v20, v19, v18, v17, v16, + v15, v14, v13, v12, v11, v10, v9, v8, + v7, v6, v5, v4, v3, v2, v1, v0 + )) {} + + // Repeat 16 values as many times as necessary (usually for lookup tables) + simdjson_inline static simd8 repeat_16( + uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, + uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 + ) { + return simd8( + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15, + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15, + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15, + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15 + ); + } + + // Saturated math + simdjson_inline simd8 saturating_add(const simd8 other) const { return _mm512_adds_epu8(*this, other); } + simdjson_inline simd8 saturating_sub(const simd8 other) const { return _mm512_subs_epu8(*this, other); } + + // Order-specific operations + simdjson_inline simd8 max_val(const simd8 other) const { return _mm512_max_epu8(*this, other); } + simdjson_inline simd8 min_val(const simd8 other) const { return _mm512_min_epu8(other, *this); } + // Same as >, but only guarantees true is nonzero (< guarantees true = -1) + simdjson_inline simd8 gt_bits(const simd8 other) const { return this->saturating_sub(other); } + // Same as <, but only guarantees true is nonzero (< guarantees true = -1) + simdjson_inline simd8 lt_bits(const simd8 other) const { return other.saturating_sub(*this); } + simdjson_inline uint64_t operator<=(const simd8 other) const { return other.max_val(*this) == other; } + simdjson_inline uint64_t operator>=(const simd8 other) const { return other.min_val(*this) == other; } + simdjson_inline simd8 operator>(const simd8 other) const { return this->gt_bits(other).any_bits_set(); } + simdjson_inline simd8 operator<(const simd8 other) const { return this->lt_bits(other).any_bits_set(); } + + // Bit-specific operations + simdjson_inline simd8 bits_not_set() const { return _mm512_mask_blend_epi8(*this == uint8_t(0), _mm512_set1_epi8(0), _mm512_set1_epi8(-1)); } + simdjson_inline simd8 bits_not_set(simd8 bits) const { return (*this & bits).bits_not_set(); } + simdjson_inline simd8 any_bits_set() const { return ~this->bits_not_set(); } + simdjson_inline simd8 any_bits_set(simd8 bits) const { return ~this->bits_not_set(bits); } + + simdjson_inline bool is_ascii() const { return _mm512_movepi8_mask(*this) == 0; } + simdjson_inline bool bits_not_set_anywhere() const { + return !_mm512_test_epi8_mask(*this, *this); + } + simdjson_inline bool any_bits_set_anywhere() const { return !bits_not_set_anywhere(); } + simdjson_inline bool bits_not_set_anywhere(simd8 bits) const { return !_mm512_test_epi8_mask(*this, bits); } + simdjson_inline bool any_bits_set_anywhere(simd8 bits) const { return !bits_not_set_anywhere(bits); } + template + simdjson_inline simd8 shr() const { return simd8(_mm512_srli_epi16(*this, N)) & uint8_t(0xFFu >> N); } + template + simdjson_inline simd8 shl() const { return simd8(_mm512_slli_epi16(*this, N)) & uint8_t(0xFFu << N); } + // Get one of the bits and make a bitmask out of it. + // e.g. value.get_bit<7>() gets the high bit + template + simdjson_inline uint64_t get_bit() const { return _mm512_movepi8_mask(_mm512_slli_epi16(*this, 7 - N)); } + }; + + template + struct simd8x64 { + static constexpr int NUM_CHUNKS = 64 / sizeof(simd8); + static_assert(NUM_CHUNKS == 1, "Icelake kernel should use one register per 64-byte block."); + const simd8 chunks[NUM_CHUNKS]; + + simd8x64(const simd8x64& o) = delete; // no copy allowed + simd8x64& operator=(const simd8& other) = delete; // no assignment allowed + simd8x64() = delete; // no default constructor allowed + + simdjson_inline simd8x64(const simd8 chunk0, const simd8 chunk1) : chunks{ chunk0, chunk1 } {} + simdjson_inline simd8x64(const simd8 chunk0) : chunks{ chunk0 } {} + simdjson_inline simd8x64(const T ptr[64]) : chunks{ simd8::load(ptr) } {} + + simdjson_inline uint64_t compress(uint64_t mask, T* output) const { + this->chunks[0].compress(mask, output); + return 64 - count_ones(mask); + } + + simdjson_inline void store(T ptr[64]) const { + this->chunks[0].store(ptr + sizeof(simd8) * 0); + } + + simdjson_inline simd8 reduce_or() const { + return this->chunks[0]; + } + + simdjson_inline simd8x64 bit_or(const T m) const { + const simd8 mask = simd8::splat(m); + return simd8x64( + this->chunks[0] | mask + ); + } + + simdjson_inline uint64_t eq(const T m) const { + const simd8 mask = simd8::splat(m); + return this->chunks[0] == mask; + } + + simdjson_inline uint64_t eq(const simd8x64& other) const { + return this->chunks[0] == other.chunks[0]; + } + + simdjson_inline uint64_t lteq(const T m) const { + const simd8 mask = simd8::splat(m); + return this->chunks[0] <= mask; + } + }; // struct simd8x64 + + } // namespace simd + + } // unnamed namespace + } // namespace icelake +} // namespace simdjson + +#endif // SIMDJSON_ICELAKE_SIMD_H +/* end file simdjson/icelake/simd.h */ +/* including simdjson/icelake/stringparsing_defs.h: #include "simdjson/icelake/stringparsing_defs.h" */ +/* begin file simdjson/icelake/stringparsing_defs.h */ +#ifndef SIMDJSON_ICELAKE_STRINGPARSING_DEFS_H +#define SIMDJSON_ICELAKE_STRINGPARSING_DEFS_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/icelake/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/icelake/simd.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/icelake/bitmanipulation.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace icelake { + namespace { + + using namespace simd; + + // Holds backslashes and quotes locations. + struct backslash_and_quote { + public: + static constexpr uint32_t BYTES_PROCESSED = 64; + simdjson_inline static backslash_and_quote copy_and_find(const uint8_t* src, uint8_t* dst); + + simdjson_inline bool has_quote_first() { return ((bs_bits - 1) & quote_bits) != 0; } + simdjson_inline bool has_backslash() { return ((quote_bits - 1) & bs_bits) != 0; } + simdjson_inline int quote_index() { return trailing_zeroes(quote_bits); } + simdjson_inline int backslash_index() { return trailing_zeroes(bs_bits); } + + uint64_t bs_bits; + uint64_t quote_bits; + }; // struct backslash_and_quote + + simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8_t* src, uint8_t* dst) { + // this can read up to 15 bytes beyond the buffer size, but we require + // SIMDJSON_PADDING of padding + static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "backslash and quote finder must process fewer than SIMDJSON_PADDING bytes"); + simd8 v(src); + // store to dest unconditionally - we can overwrite the bits we don't like later + v.store(dst); + return { + static_cast(v == '\\'), // bs_bits + static_cast(v == '"'), // quote_bits + }; + } + + } // unnamed namespace + } // namespace icelake +} // namespace simdjson + +#endif // SIMDJSON_ICELAKE_STRINGPARSING_DEFS_H +/* end file simdjson/icelake/stringparsing_defs.h */ +/* including simdjson/icelake/numberparsing_defs.h: #include "simdjson/icelake/numberparsing_defs.h" */ +/* begin file simdjson/icelake/numberparsing_defs.h */ +#ifndef SIMDJSON_ICELAKE_NUMBERPARSING_DEFS_H +#define SIMDJSON_ICELAKE_NUMBERPARSING_DEFS_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/icelake/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/icelake/intrinsics.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace icelake { + namespace numberparsing { + + static simdjson_inline uint32_t parse_eight_digits_unrolled(const uint8_t* chars) { + // this actually computes *16* values so we are being wasteful. + const __m128i ascii0 = _mm_set1_epi8('0'); + const __m128i mul_1_10 = + _mm_setr_epi8(10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1); + const __m128i mul_1_100 = _mm_setr_epi16(100, 1, 100, 1, 100, 1, 100, 1); + const __m128i mul_1_10000 = + _mm_setr_epi16(10000, 1, 10000, 1, 10000, 1, 10000, 1); + const __m128i input = _mm_sub_epi8( + _mm_loadu_si128(reinterpret_cast(chars)), ascii0); + const __m128i t1 = _mm_maddubs_epi16(input, mul_1_10); + const __m128i t2 = _mm_madd_epi16(t1, mul_1_100); + const __m128i t3 = _mm_packus_epi32(t2, t2); + const __m128i t4 = _mm_madd_epi16(t3, mul_1_10000); + return _mm_cvtsi128_si32( + t4); // only captures the sum of the first 8 digits, drop the rest + } + + /** @private */ + simdjson_inline internal::value128 full_multiplication(uint64_t value1, uint64_t value2) { + internal::value128 answer; +#if SIMDJSON_REGULAR_VISUAL_STUDIO || SIMDJSON_IS_32BITS +#if SIMDJSON_IS_ARM64 + // ARM64 has native support for 64-bit multiplications, no need to emultate + answer.high = __umulh(value1, value2); + answer.low = value1 * value2; +#else + answer.low = _umul128(value1, value2, &answer.high); // _umul128 not available on ARM64 +#endif // SIMDJSON_IS_ARM64 +#else // SIMDJSON_REGULAR_VISUAL_STUDIO || SIMDJSON_IS_32BITS + __uint128_t r = (static_cast<__uint128_t>(value1)) * value2; + answer.low = uint64_t(r); + answer.high = uint64_t(r >> 64); +#endif + return answer; + } + + } // namespace numberparsing + } // namespace icelake +} // namespace simdjson + +#define SIMDJSON_SWAR_NUMBER_PARSING 1 + +#endif // SIMDJSON_ICELAKE_NUMBERPARSING_DEFS_H +/* end file simdjson/icelake/numberparsing_defs.h */ +/* end file simdjson/icelake/begin.h */ +/* including simdjson/generic/amalgamated.h for icelake: #include "simdjson/generic/amalgamated.h" */ +/* begin file simdjson/generic/amalgamated.h for icelake */ +#if defined(SIMDJSON_CONDITIONAL_INCLUDE) && !defined(SIMDJSON_GENERIC_DEPENDENCIES_H) +#error simdjson/generic/dependencies.h must be included before simdjson/generic/amalgamated.h! +#endif + +/* including simdjson/generic/base.h for icelake: #include "simdjson/generic/base.h" */ +/* begin file simdjson/generic/base.h for icelake */ +#ifndef SIMDJSON_GENERIC_BASE_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_BASE_H */ +/* amalgamation skipped (editor-only): #include "simdjson/base.h" */ +/* amalgamation skipped (editor-only): // If we haven't got an implementation yet, we're in the editor, editing a generic file! Just */ +/* amalgamation skipped (editor-only): // use the most advanced one we can so the most possible stuff can be tested. */ +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_IMPLEMENTATION */ +/* amalgamation skipped (editor-only): #include "simdjson/implementation_detection.h" */ +/* amalgamation skipped (editor-only): #if SIMDJSON_IMPLEMENTATION_ICELAKE */ +/* amalgamation skipped (editor-only): #include "simdjson/icelake/begin.h" */ +/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_HASWELL */ +/* amalgamation skipped (editor-only): #include "simdjson/haswell/begin.h" */ +/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_WESTMERE */ +/* amalgamation skipped (editor-only): #include "simdjson/westmere/begin.h" */ +/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_ARM64 */ +/* amalgamation skipped (editor-only): #include "simdjson/arm64/begin.h" */ +/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_PPC64 */ +/* amalgamation skipped (editor-only): #include "simdjson/ppc64/begin.h" */ +/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_LSX */ +/* amalgamation skipped (editor-only): #include "simdjson/lsx/begin.h" */ +/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_LASX */ +/* amalgamation skipped (editor-only): #include "simdjson/lasx/begin.h" */ +/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_FALLBACK */ +/* amalgamation skipped (editor-only): #include "simdjson/fallback/begin.h" */ +/* amalgamation skipped (editor-only): #else */ +/* amalgamation skipped (editor-only): #error "All possible implementations (including fallback) have been disabled! simdjson will not run." */ +/* amalgamation skipped (editor-only): #endif */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_IMPLEMENTATION */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace icelake { + + struct open_container; + class dom_parser_implementation; + + /** + * The type of a JSON number + */ + enum class number_type { + floating_point_number = 1, /// a binary64 number + signed_integer, /// a signed integer that fits in a 64-bit word using two's complement + unsigned_integer, /// a positive integer larger or equal to 1<<63 + big_integer /// a big integer that does not fit in a 64-bit word + }; + + } // namespace icelake +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_BASE_H +/* end file simdjson/generic/base.h for icelake */ +/* including simdjson/generic/jsoncharutils.h for icelake: #include "simdjson/generic/jsoncharutils.h" */ +/* begin file simdjson/generic/jsoncharutils.h for icelake */ +#ifndef SIMDJSON_GENERIC_JSONCHARUTILS_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_JSONCHARUTILS_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/jsoncharutils_tables.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace icelake { + namespace { + namespace jsoncharutils { + + // return non-zero if not a structural or whitespace char + // zero otherwise + simdjson_inline uint32_t is_not_structural_or_whitespace(uint8_t c) { + return internal::structural_or_whitespace_negated[c]; + } + + simdjson_inline uint32_t is_structural_or_whitespace(uint8_t c) { + return internal::structural_or_whitespace[c]; + } + + // returns a value with the high 16 bits set if not valid + // otherwise returns the conversion of the 4 hex digits at src into the bottom + // 16 bits of the 32-bit return register + // + // see + // https://lemire.me/blog/2019/04/17/parsing-short-hexadecimal-strings-efficiently/ + static inline uint32_t hex_to_u32_nocheck( + const uint8_t* src) { // strictly speaking, static inline is a C-ism + uint32_t v1 = internal::digit_to_val32[630 + src[0]]; + uint32_t v2 = internal::digit_to_val32[420 + src[1]]; + uint32_t v3 = internal::digit_to_val32[210 + src[2]]; + uint32_t v4 = internal::digit_to_val32[0 + src[3]]; + return v1 | v2 | v3 | v4; + } + + // given a code point cp, writes to c + // the utf-8 code, outputting the length in + // bytes, if the length is zero, the code point + // is invalid + // + // This can possibly be made faster using pdep + // and clz and table lookups, but JSON documents + // have few escaped code points, and the following + // function looks cheap. + // + // Note: we assume that surrogates are treated separately + // + simdjson_inline size_t codepoint_to_utf8(uint32_t cp, uint8_t* c) { + if (cp <= 0x7F) { + c[0] = uint8_t(cp); + return 1; // ascii + } + if (cp <= 0x7FF) { + c[0] = uint8_t((cp >> 6) + 192); + c[1] = uint8_t((cp & 63) + 128); + return 2; // universal plane + // Surrogates are treated elsewhere... + //} //else if (0xd800 <= cp && cp <= 0xdfff) { + // return 0; // surrogates // could put assert here + } + else if (cp <= 0xFFFF) { + c[0] = uint8_t((cp >> 12) + 224); + c[1] = uint8_t(((cp >> 6) & 63) + 128); + c[2] = uint8_t((cp & 63) + 128); + return 3; + } + else if (cp <= 0x10FFFF) { // if you know you have a valid code point, this + // is not needed + c[0] = uint8_t((cp >> 18) + 240); + c[1] = uint8_t(((cp >> 12) & 63) + 128); + c[2] = uint8_t(((cp >> 6) & 63) + 128); + c[3] = uint8_t((cp & 63) + 128); + return 4; + } + // will return 0 when the code point was too large. + return 0; // bad r + } + +#if SIMDJSON_IS_32BITS // _umul128 for x86, arm + // this is a slow emulation routine for 32-bit + // + static simdjson_inline uint64_t __emulu(uint32_t x, uint32_t y) { + return x * (uint64_t)y; + } + static simdjson_inline uint64_t _umul128(uint64_t ab, uint64_t cd, uint64_t* hi) { + uint64_t ad = __emulu((uint32_t)(ab >> 32), (uint32_t)cd); + uint64_t bd = __emulu((uint32_t)ab, (uint32_t)cd); + uint64_t adbc = ad + __emulu((uint32_t)ab, (uint32_t)(cd >> 32)); + uint64_t adbc_carry = !!(adbc < ad); + uint64_t lo = bd + (adbc << 32); + *hi = __emulu((uint32_t)(ab >> 32), (uint32_t)(cd >> 32)) + (adbc >> 32) + + (adbc_carry << 32) + !!(lo < bd); + return lo; + } +#endif + + } // namespace jsoncharutils + } // unnamed namespace + } // namespace icelake +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_JSONCHARUTILS_H +/* end file simdjson/generic/jsoncharutils.h for icelake */ +/* including simdjson/generic/atomparsing.h for icelake: #include "simdjson/generic/atomparsing.h" */ +/* begin file simdjson/generic/atomparsing.h for icelake */ +#ifndef SIMDJSON_GENERIC_ATOMPARSING_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_ATOMPARSING_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/jsoncharutils.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +#include + +namespace simdjson { + namespace icelake { + namespace { + /// @private + namespace atomparsing { + + // The string_to_uint32 is exclusively used to map literal strings to 32-bit values. + // We use memcpy instead of a pointer cast to avoid undefined behaviors since we cannot + // be certain that the character pointer will be properly aligned. + // You might think that using memcpy makes this function expensive, but you'd be wrong. + // All decent optimizing compilers (GCC, clang, Visual Studio) will compile string_to_uint32("false"); + // to the compile-time constant 1936482662. + simdjson_inline uint32_t string_to_uint32(const char* str) { uint32_t val; std::memcpy(&val, str, sizeof(uint32_t)); return val; } + + + // Again in str4ncmp we use a memcpy to avoid undefined behavior. The memcpy may appear expensive. + // Yet all decent optimizing compilers will compile memcpy to a single instruction, just about. + simdjson_warn_unused + simdjson_inline uint32_t str4ncmp(const uint8_t* src, const char* atom) { + uint32_t srcval; // we want to avoid unaligned 32-bit loads (undefined in C/C++) + static_assert(sizeof(uint32_t) <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be larger than 4 bytes"); + std::memcpy(&srcval, src, sizeof(uint32_t)); + return srcval ^ string_to_uint32(atom); + } + + simdjson_warn_unused + simdjson_inline bool is_valid_true_atom(const uint8_t* src) { + return (str4ncmp(src, "true") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; + } + + simdjson_warn_unused + simdjson_inline bool is_valid_true_atom(const uint8_t* src, size_t len) { + if (len > 4) { return is_valid_true_atom(src); } + else if (len == 4) { return !str4ncmp(src, "true"); } + else { return false; } + } + + simdjson_warn_unused + simdjson_inline bool is_valid_false_atom(const uint8_t* src) { + return (str4ncmp(src + 1, "alse") | jsoncharutils::is_not_structural_or_whitespace(src[5])) == 0; + } + + simdjson_warn_unused + simdjson_inline bool is_valid_false_atom(const uint8_t* src, size_t len) { + if (len > 5) { return is_valid_false_atom(src); } + else if (len == 5) { return !str4ncmp(src + 1, "alse"); } + else { return false; } + } + + simdjson_warn_unused + simdjson_inline bool is_valid_null_atom(const uint8_t* src) { + return (str4ncmp(src, "null") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; + } + + simdjson_warn_unused + simdjson_inline bool is_valid_null_atom(const uint8_t* src, size_t len) { + if (len > 4) { return is_valid_null_atom(src); } + else if (len == 4) { return !str4ncmp(src, "null"); } + else { return false; } + } + + } // namespace atomparsing + } // unnamed namespace + } // namespace icelake +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_ATOMPARSING_H +/* end file simdjson/generic/atomparsing.h for icelake */ +/* including simdjson/generic/dom_parser_implementation.h for icelake: #include "simdjson/generic/dom_parser_implementation.h" */ +/* begin file simdjson/generic/dom_parser_implementation.h for icelake */ +#ifndef SIMDJSON_GENERIC_DOM_PARSER_IMPLEMENTATION_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_DOM_PARSER_IMPLEMENTATION_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/dom_parser_implementation.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace icelake { + + // expectation: sizeof(open_container) = 64/8. + struct open_container { + uint32_t tape_index; // where, on the tape, does the scope ([,{) begins + uint32_t count; // how many elements in the scope + }; // struct open_container + + static_assert(sizeof(open_container) == 64 / 8, "Open container must be 64 bits"); + + class dom_parser_implementation final : public internal::dom_parser_implementation { + public: + /** Tape location of each open { or [ */ + std::unique_ptr open_containers{}; + /** Whether each open container is a [ or { */ + std::unique_ptr is_array{}; + /** Buffer passed to stage 1 */ + const uint8_t* buf{}; + /** Length passed to stage 1 */ + size_t len{ 0 }; + /** Document passed to stage 2 */ + dom::document* doc{}; + + inline dom_parser_implementation() noexcept; + inline dom_parser_implementation(dom_parser_implementation&& other) noexcept; + inline dom_parser_implementation& operator=(dom_parser_implementation&& other) noexcept; + dom_parser_implementation(const dom_parser_implementation&) = delete; + dom_parser_implementation& operator=(const dom_parser_implementation&) = delete; + + simdjson_warn_unused error_code parse(const uint8_t* buf, size_t len, dom::document& doc) noexcept final; + simdjson_warn_unused error_code stage1(const uint8_t* buf, size_t len, stage1_mode partial) noexcept final; + simdjson_warn_unused error_code stage2(dom::document& doc) noexcept final; + simdjson_warn_unused error_code stage2_next(dom::document& doc) noexcept final; + simdjson_warn_unused uint8_t* parse_string(const uint8_t* src, uint8_t* dst, bool allow_replacement) const noexcept final; + simdjson_warn_unused uint8_t* parse_wobbly_string(const uint8_t* src, uint8_t* dst) const noexcept final; + inline simdjson_warn_unused error_code set_capacity(size_t capacity) noexcept final; + inline simdjson_warn_unused error_code set_max_depth(size_t max_depth) noexcept final; + private: + simdjson_inline simdjson_warn_unused error_code set_capacity_stage1(size_t capacity); + + }; + + } // namespace icelake +} // namespace simdjson + +namespace simdjson { + namespace icelake { + + inline dom_parser_implementation::dom_parser_implementation() noexcept = default; + inline dom_parser_implementation::dom_parser_implementation(dom_parser_implementation&& other) noexcept = default; + inline dom_parser_implementation& dom_parser_implementation::operator=(dom_parser_implementation&& other) noexcept = default; + + // Leaving these here so they can be inlined if so desired + inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(size_t capacity) noexcept { + if (capacity > SIMDJSON_MAXSIZE_BYTES) { return CAPACITY; } + // Stage 1 index output + size_t max_structures = SIMDJSON_ROUNDUP_N(capacity, 64) + 2 + 7; + structural_indexes.reset(new (std::nothrow) uint32_t[max_structures]); + if (!structural_indexes) { _capacity = 0; return MEMALLOC; } + structural_indexes[0] = 0; + n_structural_indexes = 0; + + _capacity = capacity; + return SUCCESS; + } + + inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { + // Stage 2 stacks + open_containers.reset(new (std::nothrow) open_container[max_depth]); + is_array.reset(new (std::nothrow) bool[max_depth]); + if (!is_array || !open_containers) { _max_depth = 0; return MEMALLOC; } + + _max_depth = max_depth; + return SUCCESS; + } + + } // namespace icelake +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_DOM_PARSER_IMPLEMENTATION_H +/* end file simdjson/generic/dom_parser_implementation.h for icelake */ +/* including simdjson/generic/implementation_simdjson_result_base.h for icelake: #include "simdjson/generic/implementation_simdjson_result_base.h" */ +/* begin file simdjson/generic/implementation_simdjson_result_base.h for icelake */ +#ifndef SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace icelake { + + // This is a near copy of include/error.h's implementation_simdjson_result_base, except it doesn't use std::pair + // so we can avoid inlining errors + // TODO reconcile these! + /** + * The result of a simdjson operation that could fail. + * + * Gives the option of reading error codes, or throwing an exception by casting to the desired result. + * + * This is a base class for implementations that want to add functions to the result type for + * chaining. + * + * Override like: + * + * struct simdjson_result : public internal::implementation_simdjson_result_base { + * simdjson_result() noexcept : internal::implementation_simdjson_result_base() {} + * simdjson_result(error_code error) noexcept : internal::implementation_simdjson_result_base(error) {} + * simdjson_result(T &&value) noexcept : internal::implementation_simdjson_result_base(std::forward(value)) {} + * simdjson_result(T &&value, error_code error) noexcept : internal::implementation_simdjson_result_base(value, error) {} + * // Your extra methods here + * } + * + * Then any method returning simdjson_result will be chainable with your methods. + */ + template + struct implementation_simdjson_result_base { + + /** + * Create a new empty result with error = UNINITIALIZED. + */ + simdjson_inline implementation_simdjson_result_base() noexcept = default; + + /** + * Create a new error result. + */ + simdjson_inline implementation_simdjson_result_base(error_code error) noexcept; + + /** + * Create a new successful result. + */ + simdjson_inline implementation_simdjson_result_base(T&& value) noexcept; + + /** + * Create a new result with both things (use if you don't want to branch when creating the result). + */ + simdjson_inline implementation_simdjson_result_base(T&& value, error_code error) noexcept; + + /** + * Move the value and the error to the provided variables. + * + * @param value The variable to assign the value to. May not be set if there is an error. + * @param error The variable to assign the error to. Set to SUCCESS if there is no error. + */ + simdjson_inline void tie(T& value, error_code& error) && noexcept; + + /** + * Move the value to the provided variable. + * + * @param value The variable to assign the value to. May not be set if there is an error. + */ + simdjson_inline error_code get(T& value) && noexcept; + + /** + * The error. + */ + simdjson_inline error_code error() const noexcept; + +#if SIMDJSON_EXCEPTIONS + + /** + * Get the result value. + * + * @throw simdjson_error if there was an error. + */ + simdjson_inline T& value() & noexcept(false); + + /** + * Take the result value (move it). + * + * @throw simdjson_error if there was an error. + */ + simdjson_inline T&& value() && noexcept(false); + + /** + * Take the result value (move it). + * + * @throw simdjson_error if there was an error. + */ + simdjson_inline T&& take_value() && noexcept(false); + + /** + * Cast to the value (will throw on error). + * + * @throw simdjson_error if there was an error. + */ + simdjson_inline operator T && () && noexcept(false); + + +#endif // SIMDJSON_EXCEPTIONS + + /** + * Get the result value. This function is safe if and only + * the error() method returns a value that evaluates to false. + */ + simdjson_inline const T& value_unsafe() const& noexcept; + /** + * Get the result value. This function is safe if and only + * the error() method returns a value that evaluates to false. + */ + simdjson_inline T& value_unsafe() & noexcept; + /** + * Take the result value (move it). This function is safe if and only + * the error() method returns a value that evaluates to false. + */ + simdjson_inline T&& value_unsafe() && noexcept; + protected: + /** users should never directly access first and second. **/ + T first{}; /** Users should never directly access 'first'. **/ + error_code second{ UNINITIALIZED }; /** Users should never directly access 'second'. **/ + }; // struct implementation_simdjson_result_base + + } // namespace icelake +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_H +/* end file simdjson/generic/implementation_simdjson_result_base.h for icelake */ +/* including simdjson/generic/numberparsing.h for icelake: #include "simdjson/generic/numberparsing.h" */ +/* begin file simdjson/generic/numberparsing.h for icelake */ +#ifndef SIMDJSON_GENERIC_NUMBERPARSING_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_NUMBERPARSING_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/jsoncharutils.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +#include +#include +#include + +namespace simdjson { + namespace icelake { + namespace numberparsing { + +#ifdef JSON_TEST_NUMBERS +#define INVALID_NUMBER(SRC) (found_invalid_number((SRC)), NUMBER_ERROR) +#define WRITE_INTEGER(VALUE, SRC, WRITER) (found_integer((VALUE), (SRC)), (WRITER).append_s64((VALUE))) +#define WRITE_UNSIGNED(VALUE, SRC, WRITER) (found_unsigned_integer((VALUE), (SRC)), (WRITER).append_u64((VALUE))) +#define WRITE_DOUBLE(VALUE, SRC, WRITER) (found_float((VALUE), (SRC)), (WRITER).append_double((VALUE))) +#define BIGINT_NUMBER(SRC) (found_invalid_number((SRC)), BIGINT_ERROR) +#else +#define INVALID_NUMBER(SRC) (NUMBER_ERROR) +#define WRITE_INTEGER(VALUE, SRC, WRITER) (WRITER).append_s64((VALUE)) +#define WRITE_UNSIGNED(VALUE, SRC, WRITER) (WRITER).append_u64((VALUE)) +#define WRITE_DOUBLE(VALUE, SRC, WRITER) (WRITER).append_double((VALUE)) +#define BIGINT_NUMBER(SRC) (BIGINT_ERROR) +#endif + + namespace { + + // Convert a mantissa, an exponent and a sign bit into an ieee64 double. + // The real_exponent needs to be in [0, 2046] (technically real_exponent = 2047 would be acceptable). + // The mantissa should be in [0,1<<53). The bit at index (1ULL << 52) while be zeroed. + simdjson_inline double to_double(uint64_t mantissa, uint64_t real_exponent, bool negative) { + double d; + mantissa &= ~(1ULL << 52); + mantissa |= real_exponent << 52; + mantissa |= ((static_cast(negative)) << 63); + std::memcpy(&d, &mantissa, sizeof(d)); + return d; + } + + // Attempts to compute i * 10^(power) exactly; and if "negative" is + // true, negate the result. + // This function will only work in some cases, when it does not work, success is + // set to false. This should work *most of the time* (like 99% of the time). + // We assume that power is in the [smallest_power, + // largest_power] interval: the caller is responsible for this check. + simdjson_inline bool compute_float_64(int64_t power, uint64_t i, bool negative, double& d) { + // we start with a fast path + // It was described in + // Clinger WD. How to read floating point numbers accurately. + // ACM SIGPLAN Notices. 1990 +#ifndef FLT_EVAL_METHOD +#error "FLT_EVAL_METHOD should be defined, please include cfloat." +#endif +#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0) + // We cannot be certain that x/y is rounded to nearest. + if (0 <= power && power <= 22 && i <= 9007199254740991) +#else + if (-22 <= power && power <= 22 && i <= 9007199254740991) +#endif + { + // convert the integer into a double. This is lossless since + // 0 <= i <= 2^53 - 1. + d = double(i); + // + // The general idea is as follows. + // If 0 <= s < 2^53 and if 10^0 <= p <= 10^22 then + // 1) Both s and p can be represented exactly as 64-bit floating-point + // values + // (binary64). + // 2) Because s and p can be represented exactly as floating-point values, + // then s * p + // and s / p will produce correctly rounded values. + // + if (power < 0) { + d = d / simdjson::internal::power_of_ten[-power]; + } + else { + d = d * simdjson::internal::power_of_ten[power]; + } + if (negative) { + d = -d; + } + return true; + } + // When 22 < power && power < 22 + 16, we could + // hope for another, secondary fast path. It was + // described by David M. Gay in "Correctly rounded + // binary-decimal and decimal-binary conversions." (1990) + // If you need to compute i * 10^(22 + x) for x < 16, + // first compute i * 10^x, if you know that result is exact + // (e.g., when i * 10^x < 2^53), + // then you can still proceed and do (i * 10^x) * 10^22. + // Is this worth your time? + // You need 22 < power *and* power < 22 + 16 *and* (i * 10^(x-22) < 2^53) + // for this second fast path to work. + // If you you have 22 < power *and* power < 22 + 16, and then you + // optimistically compute "i * 10^(x-22)", there is still a chance that you + // have wasted your time if i * 10^(x-22) >= 2^53. It makes the use cases of + // this optimization maybe less common than we would like. Source: + // http://www.exploringbinary.com/fast-path-decimal-to-floating-point-conversion/ + // also used in RapidJSON: https://rapidjson.org/strtod_8h_source.html + + // The fast path has now failed, so we are failing back on the slower path. + + // In the slow path, we need to adjust i so that it is > 1<<63 which is always + // possible, except if i == 0, so we handle i == 0 separately. + if (i == 0) { + d = negative ? -0.0 : 0.0; + return true; + } + + + // The exponent is 1024 + 63 + power + // + floor(log(5**power)/log(2)). + // The 1024 comes from the ieee64 standard. + // The 63 comes from the fact that we use a 64-bit word. + // + // Computing floor(log(5**power)/log(2)) could be + // slow. Instead we use a fast function. + // + // For power in (-400,350), we have that + // (((152170 + 65536) * power ) >> 16); + // is equal to + // floor(log(5**power)/log(2)) + power when power >= 0 + // and it is equal to + // ceil(log(5**-power)/log(2)) + power when power < 0 + // + // The 65536 is (1<<16) and corresponds to + // (65536 * power) >> 16 ---> power + // + // ((152170 * power ) >> 16) is equal to + // floor(log(5**power)/log(2)) + // + // Note that this is not magic: 152170/(1<<16) is + // approximatively equal to log(5)/log(2). + // The 1<<16 value is a power of two; we could use a + // larger power of 2 if we wanted to. + // + int64_t exponent = (((152170 + 65536) * power) >> 16) + 1024 + 63; + + + // We want the most significant bit of i to be 1. Shift if needed. + int lz = leading_zeroes(i); + i <<= lz; + + + // We are going to need to do some 64-bit arithmetic to get a precise product. + // We use a table lookup approach. + // It is safe because + // power >= smallest_power + // and power <= largest_power + // We recover the mantissa of the power, it has a leading 1. It is always + // rounded down. + // + // We want the most significant 64 bits of the product. We know + // this will be non-zero because the most significant bit of i is + // 1. + const uint32_t index = 2 * uint32_t(power - simdjson::internal::smallest_power); + // Optimization: It may be that materializing the index as a variable might confuse some compilers and prevent effective complex-addressing loads. (Done for code clarity.) + // + // The full_multiplication function computes the 128-bit product of two 64-bit words + // with a returned value of type value128 with a "low component" corresponding to the + // 64-bit least significant bits of the product and with a "high component" corresponding + // to the 64-bit most significant bits of the product. + simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index]); + // Both i and power_of_five_128[index] have their most significant bit set to 1 which + // implies that the either the most or the second most significant bit of the product + // is 1. We pack values in this manner for efficiency reasons: it maximizes the use + // we make of the product. It also makes it easy to reason about the product: there + // is 0 or 1 leading zero in the product. + + // Unless the least significant 9 bits of the high (64-bit) part of the full + // product are all 1s, then we know that the most significant 55 bits are + // exact and no further work is needed. Having 55 bits is necessary because + // we need 53 bits for the mantissa but we have to have one rounding bit and + // we can waste a bit if the most significant bit of the product is zero. + if ((firstproduct.high & 0x1FF) == 0x1FF) { + // We want to compute i * 5^q, but only care about the top 55 bits at most. + // Consider the scenario where q>=0. Then 5^q may not fit in 64-bits. Doing + // the full computation is wasteful. So we do what is called a "truncated + // multiplication". + // We take the most significant 64-bits, and we put them in + // power_of_five_128[index]. Usually, that's good enough to approximate i * 5^q + // to the desired approximation using one multiplication. Sometimes it does not suffice. + // Then we store the next most significant 64 bits in power_of_five_128[index + 1], and + // then we get a better approximation to i * 5^q. + // + // That's for when q>=0. The logic for q<0 is somewhat similar but it is somewhat + // more complicated. + // + // There is an extra layer of complexity in that we need more than 55 bits of + // accuracy in the round-to-even scenario. + // + // The full_multiplication function computes the 128-bit product of two 64-bit words + // with a returned value of type value128 with a "low component" corresponding to the + // 64-bit least significant bits of the product and with a "high component" corresponding + // to the 64-bit most significant bits of the product. + simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index + 1]); + firstproduct.low += secondproduct.high; + if (secondproduct.high > firstproduct.low) { firstproduct.high++; } + // As it has been proven by Noble Mushtak and Daniel Lemire in "Fast Number Parsing Without + // Fallback" (https://arxiv.org/abs/2212.06644), at this point we are sure that the product + // is sufficiently accurate, and more computation is not needed. + } + uint64_t lower = firstproduct.low; + uint64_t upper = firstproduct.high; + // The final mantissa should be 53 bits with a leading 1. + // We shift it so that it occupies 54 bits with a leading 1. + /////// + uint64_t upperbit = upper >> 63; + uint64_t mantissa = upper >> (upperbit + 9); + lz += int(1 ^ upperbit); + + // Here we have mantissa < (1<<54). + int64_t real_exponent = exponent - lz; + if (simdjson_unlikely(real_exponent <= 0)) { // we have a subnormal? + // Here have that real_exponent <= 0 so -real_exponent >= 0 + if (-real_exponent + 1 >= 64) { // if we have more than 64 bits below the minimum exponent, you have a zero for sure. + d = negative ? -0.0 : 0.0; + return true; + } + // next line is safe because -real_exponent + 1 < 0 + mantissa >>= -real_exponent + 1; + // Thankfully, we can't have both "round-to-even" and subnormals because + // "round-to-even" only occurs for powers close to 0. + mantissa += (mantissa & 1); // round up + mantissa >>= 1; + // There is a weird scenario where we don't have a subnormal but just. + // Suppose we start with 2.2250738585072013e-308, we end up + // with 0x3fffffffffffff x 2^-1023-53 which is technically subnormal + // whereas 0x40000000000000 x 2^-1023-53 is normal. Now, we need to round + // up 0x3fffffffffffff x 2^-1023-53 and once we do, we are no longer + // subnormal, but we can only know this after rounding. + // So we only declare a subnormal if we are smaller than the threshold. + real_exponent = (mantissa < (uint64_t(1) << 52)) ? 0 : 1; + d = to_double(mantissa, real_exponent, negative); + return true; + } + // We have to round to even. The "to even" part + // is only a problem when we are right in between two floats + // which we guard against. + // If we have lots of trailing zeros, we may fall right between two + // floating-point values. + // + // The round-to-even cases take the form of a number 2m+1 which is in (2^53,2^54] + // times a power of two. That is, it is right between a number with binary significand + // m and another number with binary significand m+1; and it must be the case + // that it cannot be represented by a float itself. + // + // We must have that w * 10 ^q == (2m+1) * 2^p for some power of two 2^p. + // Recall that 10^q = 5^q * 2^q. + // When q >= 0, we must have that (2m+1) is divible by 5^q, so 5^q <= 2^54. We have that + // 5^23 <= 2^54 and it is the last power of five to qualify, so q <= 23. + // When q<0, we have w >= (2m+1) x 5^{-q}. We must have that w<2^{64} so + // (2m+1) x 5^{-q} < 2^{64}. We have that 2m+1>2^{53}. Hence, we must have + // 2^{53} x 5^{-q} < 2^{64}. + // Hence we have 5^{-q} < 2^{11}$ or q>= -4. + // + // We require lower <= 1 and not lower == 0 because we could not prove that + // that lower == 0 is implied; but we could prove that lower <= 1 is a necessary and sufficient test. + if (simdjson_unlikely((lower <= 1) && (power >= -4) && (power <= 23) && ((mantissa & 3) == 1))) { + if ((mantissa << (upperbit + 64 - 53 - 2)) == upper) { + mantissa &= ~1; // flip it so that we do not round up + } + } + + mantissa += mantissa & 1; + mantissa >>= 1; + + // Here we have mantissa < (1<<53), unless there was an overflow + if (mantissa >= (1ULL << 53)) { + ////////// + // This will happen when parsing values such as 7.2057594037927933e+16 + //////// + mantissa = (1ULL << 52); + real_exponent++; + } + mantissa &= ~(1ULL << 52); + // we have to check that real_exponent is in range, otherwise we bail out + if (simdjson_unlikely(real_exponent > 2046)) { + // We have an infinite value!!! We could actually throw an error here if we could. + return false; + } + d = to_double(mantissa, real_exponent, negative); + return true; + } + + // We call a fallback floating-point parser that might be slow. Note + // it will accept JSON numbers, but the JSON spec. is more restrictive so + // before you call parse_float_fallback, you need to have validated the input + // string with the JSON grammar. + // It will return an error (false) if the parsed number is infinite. + // The string parsing itself always succeeds. We know that there is at least + // one digit. + static bool parse_float_fallback(const uint8_t* ptr, double* outDouble) { + *outDouble = simdjson::internal::from_chars(reinterpret_cast(ptr)); + // We do not accept infinite values. + + // Detecting finite values in a portable manner is ridiculously hard, ideally + // we would want to do: + // return !std::isfinite(*outDouble); + // but that mysteriously fails under legacy/old libc++ libraries, see + // https://github.com/simdjson/simdjson/issues/1286 + // + // Therefore, fall back to this solution (the extra parens are there + // to handle that max may be a macro on windows). + return !(*outDouble > (std::numeric_limits::max)() || *outDouble < std::numeric_limits::lowest()); + } + + static bool parse_float_fallback(const uint8_t* ptr, const uint8_t* end_ptr, double* outDouble) { + *outDouble = simdjson::internal::from_chars(reinterpret_cast(ptr), reinterpret_cast(end_ptr)); + // We do not accept infinite values. + + // Detecting finite values in a portable manner is ridiculously hard, ideally + // we would want to do: + // return !std::isfinite(*outDouble); + // but that mysteriously fails under legacy/old libc++ libraries, see + // https://github.com/simdjson/simdjson/issues/1286 + // + // Therefore, fall back to this solution (the extra parens are there + // to handle that max may be a macro on windows). + return !(*outDouble > (std::numeric_limits::max)() || *outDouble < std::numeric_limits::lowest()); + } + + // check quickly whether the next 8 chars are made of digits + // at a glance, it looks better than Mula's + // http://0x80.pl/articles/swar-digits-validate.html + simdjson_inline bool is_made_of_eight_digits_fast(const uint8_t* chars) { + uint64_t val; + // this can read up to 7 bytes beyond the buffer size, but we require + // SIMDJSON_PADDING of padding + static_assert(7 <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be bigger than 7"); + std::memcpy(&val, chars, 8); + // a branchy method might be faster: + // return (( val & 0xF0F0F0F0F0F0F0F0 ) == 0x3030303030303030) + // && (( (val + 0x0606060606060606) & 0xF0F0F0F0F0F0F0F0 ) == + // 0x3030303030303030); + return (((val & 0xF0F0F0F0F0F0F0F0) | + (((val + 0x0606060606060606) & 0xF0F0F0F0F0F0F0F0) >> 4)) == + 0x3333333333333333); + } + + template + SIMDJSON_NO_SANITIZE_UNDEFINED // We deliberately allow overflow here and check later + simdjson_inline bool parse_digit(const uint8_t c, I& i) { + const uint8_t digit = static_cast(c - '0'); + if (digit > 9) { + return false; + } + // PERF NOTE: multiplication by 10 is cheaper than arbitrary integer multiplication + i = 10 * i + digit; // might overflow, we will handle the overflow later + return true; + } + + simdjson_inline bool is_digit(const uint8_t c) { + return static_cast(c - '0') <= 9; + } + + simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t* const src, const uint8_t*& p, uint64_t& i, int64_t& exponent) { + // we continue with the fiction that we have an integer. If the + // floating point number is representable as x * 10^z for some integer + // z that fits in 53 bits, then we will be able to convert back the + // the integer into a float in a lossless manner. + const uint8_t* const first_after_period = p; + +#ifdef SIMDJSON_SWAR_NUMBER_PARSING +#if SIMDJSON_SWAR_NUMBER_PARSING + // this helps if we have lots of decimals! + // this turns out to be frequent enough. + if (is_made_of_eight_digits_fast(p)) { + i = i * 100000000 + parse_eight_digits_unrolled(p); + p += 8; + } +#endif // SIMDJSON_SWAR_NUMBER_PARSING +#endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING + // Unrolling the first digit makes a small difference on some implementations (e.g. westmere) + if (parse_digit(*p, i)) { ++p; } + while (parse_digit(*p, i)) { p++; } + exponent = first_after_period - p; + // Decimal without digits (123.) is illegal + if (exponent == 0) { + return INVALID_NUMBER(src); + } + return SUCCESS; + } + + simdjson_inline error_code parse_exponent(simdjson_unused const uint8_t* const src, const uint8_t*& p, int64_t& exponent) { + // Exp Sign: -123.456e[-]78 + bool neg_exp = ('-' == *p); + if (neg_exp || '+' == *p) { p++; } // Skip + as well + + // Exponent: -123.456e-[78] + auto start_exp = p; + int64_t exp_number = 0; + while (parse_digit(*p, exp_number)) { ++p; } + // It is possible for parse_digit to overflow. + // In particular, it could overflow to INT64_MIN, and we cannot do - INT64_MIN. + // Thus we *must* check for possible overflow before we negate exp_number. + + // Performance notes: it may seem like combining the two "simdjson_unlikely checks" below into + // a single simdjson_unlikely path would be faster. The reasoning is sound, but the compiler may + // not oblige and may, in fact, generate two distinct paths in any case. It might be + // possible to do uint64_t(p - start_exp - 1) >= 18 but it could end up trading off + // instructions for a simdjson_likely branch, an unconclusive gain. + + // If there were no digits, it's an error. + if (simdjson_unlikely(p == start_exp)) { + return INVALID_NUMBER(src); + } + // We have a valid positive exponent in exp_number at this point, except that + // it may have overflowed. + + // If there were more than 18 digits, we may have overflowed the integer. We have to do + // something!!!! + if (simdjson_unlikely(p > start_exp + 18)) { + // Skip leading zeroes: 1e000000000000000000001 is technically valid and does not overflow + while (*start_exp == '0') { start_exp++; } + // 19 digits could overflow int64_t and is kind of absurd anyway. We don't + // support exponents smaller than -999,999,999,999,999,999 and bigger + // than 999,999,999,999,999,999. + // We can truncate. + // Note that 999999999999999999 is assuredly too large. The maximal ieee64 value before + // infinity is ~1.8e308. The smallest subnormal is ~5e-324. So, actually, we could + // truncate at 324. + // Note that there is no reason to fail per se at this point in time. + // E.g., 0e999999999999999999999 is a fine number. + if (p > start_exp + 18) { exp_number = 999999999999999999; } + } + // At this point, we know that exp_number is a sane, positive, signed integer. + // It is <= 999,999,999,999,999,999. As long as 'exponent' is in + // [-8223372036854775808, 8223372036854775808], we won't overflow. Because 'exponent' + // is bounded in magnitude by the size of the JSON input, we are fine in this universe. + // To sum it up: the next line should never overflow. + exponent += (neg_exp ? -exp_number : exp_number); + return SUCCESS; + } + + simdjson_inline bool check_if_integer(const uint8_t* const src, size_t max_length) { + const uint8_t* const srcend = src + max_length; + bool negative = (*src == '-'); // we can always read at least one character after the '-' + const uint8_t* p = src + uint8_t(negative); + if (p == srcend) { return false; } + if (*p == '0') { + ++p; + if (p == srcend) { return true; } + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return false; } + return true; + } + while (p != srcend && is_digit(*p)) { ++p; } + if (p == srcend) { return true; } + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return false; } + return true; + } + + simdjson_inline size_t significant_digits(const uint8_t* start_digits, size_t digit_count) { + // It is possible that the integer had an overflow. + // We have to handle the case where we have 0.0000somenumber. + const uint8_t* start = start_digits; + while ((*start == '0') || (*start == '.')) { ++start; } + // we over-decrement by one when there is a '.' + return digit_count - size_t(start - start_digits); + } + + } // unnamed namespace + + /** @private */ + static error_code slow_float_parsing(simdjson_unused const uint8_t* src, double* answer) { + if (parse_float_fallback(src, answer)) { + return SUCCESS; + } + return INVALID_NUMBER(src); + } + + /** @private */ + template + simdjson_inline error_code write_float(const uint8_t* const src, bool negative, uint64_t i, const uint8_t* start_digits, size_t digit_count, int64_t exponent, W& writer) { + // If we frequently had to deal with long strings of digits, + // we could extend our code by using a 128-bit integer instead + // of a 64-bit integer. However, this is uncommon in practice. + // + // 9999999999999999999 < 2**64 so we can accommodate 19 digits. + // If we have a decimal separator, then digit_count - 1 is the number of digits, but we + // may not have a decimal separator! + if (simdjson_unlikely(digit_count > 19 && significant_digits(start_digits, digit_count) > 19)) { + // Ok, chances are good that we had an overflow! + // this is almost never going to get called!!! + // we start anew, going slowly!!! + // This will happen in the following examples: + // 10000000000000000000000000000000000000000000e+308 + // 3.1415926535897932384626433832795028841971693993751 + // + // NOTE: We do not pass a reference to the to slow_float_parsing. If we passed our writer + // reference to it, it would force it to be stored in memory, preventing the compiler from + // picking it apart and putting into registers. i.e. if we pass it as reference, + // it gets slow. + double d; + error_code error = slow_float_parsing(src, &d); + writer.append_double(d); + return error; + } + // NOTE: it's weird that the simdjson_unlikely() only wraps half the if, but it seems to get slower any other + // way we've tried: https://github.com/simdjson/simdjson/pull/990#discussion_r448497331 + // To future reader: we'd love if someone found a better way, or at least could explain this result! + if (simdjson_unlikely(exponent < simdjson::internal::smallest_power) || (exponent > simdjson::internal::largest_power)) { + // + // Important: smallest_power is such that it leads to a zero value. + // Observe that 18446744073709551615e-343 == 0, i.e. (2**64 - 1) e -343 is zero + // so something x 10^-343 goes to zero, but not so with something x 10^-342. + static_assert(simdjson::internal::smallest_power <= -342, "smallest_power is not small enough"); + // + if ((exponent < simdjson::internal::smallest_power) || (i == 0)) { + // E.g. Parse "-0.0e-999" into the same value as "-0.0". See https://en.wikipedia.org/wiki/Signed_zero + WRITE_DOUBLE(negative ? -0.0 : 0.0, src, writer); + return SUCCESS; + } + else { // (exponent > largest_power) and (i != 0) + // We have, for sure, an infinite value and simdjson refuses to parse infinite values. + return INVALID_NUMBER(src); + } + } + double d; + if (!compute_float_64(exponent, i, negative, d)) { + // we are almost never going to get here. + if (!parse_float_fallback(src, &d)) { return INVALID_NUMBER(src); } + } + WRITE_DOUBLE(d, src, writer); + return SUCCESS; + } + + // parse the number at src + // define JSON_TEST_NUMBERS for unit testing + // + // It is assumed that the number is followed by a structural ({,},],[) character + // or a white space character. If that is not the case (e.g., when the JSON + // document is made of a single number), then it is necessary to copy the + // content and append a space before calling this function. + // + // Our objective is accurate parsing (ULP of 0) at high speed. + template + simdjson_inline error_code parse_number(const uint8_t* const src, W& writer); + + // for performance analysis, it is sometimes useful to skip parsing +#ifdef SIMDJSON_SKIPNUMBERPARSING + + template + simdjson_inline error_code parse_number(const uint8_t* const, W& writer) { + writer.append_s64(0); // always write zero + return SUCCESS; // always succeeds + } + + simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t* const src) noexcept { return 0; } + simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t* const src) noexcept { return 0; } + simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t* const src) noexcept { return 0; } + simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t* const src) noexcept { return 0; } + simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t* const src) noexcept { return 0; } + simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t* const src) noexcept { return 0; } + simdjson_unused simdjson_inline bool is_negative(const uint8_t* src) noexcept { return false; } + simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t* src) noexcept { return false; } + simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t* src) noexcept { return number_type::signed_integer; } +#else + +// parse the number at src +// define JSON_TEST_NUMBERS for unit testing +// +// It is assumed that the number is followed by a structural ({,},],[) character +// or a white space character. If that is not the case (e.g., when the JSON +// document is made of a single number), then it is necessary to copy the +// content and append a space before calling this function. +// +// Our objective is accurate parsing (ULP of 0) at high speed. + template + simdjson_inline error_code parse_number(const uint8_t* const src, W& writer) { + + // + // Check for minus sign + // + bool negative = (*src == '-'); + const uint8_t* p = src + uint8_t(negative); + + // + // Parse the integer part. + // + // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare + const uint8_t* const start_digits = p; + uint64_t i = 0; + while (parse_digit(*p, i)) { p++; } + + // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. + // Optimization note: size_t is expected to be unsigned. + size_t digit_count = size_t(p - start_digits); + if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { return INVALID_NUMBER(src); } + + // + // Handle floats if there is a . or e (or both) + // + int64_t exponent = 0; + bool is_float = false; + if ('.' == *p) { + is_float = true; + ++p; + SIMDJSON_TRY(parse_decimal_after_separator(src, p, i, exponent)); + digit_count = int(p - start_digits); // used later to guard against overflows + } + if (('e' == *p) || ('E' == *p)) { + is_float = true; + ++p; + SIMDJSON_TRY(parse_exponent(src, p, exponent)); + } + if (is_float) { + const bool dirty_end = jsoncharutils::is_not_structural_or_whitespace(*p); + SIMDJSON_TRY(write_float(src, negative, i, start_digits, digit_count, exponent, writer)); + if (dirty_end) { return INVALID_NUMBER(src); } + return SUCCESS; + } + + // The longest negative 64-bit number is 19 digits. + // The longest positive 64-bit number is 20 digits. + // We do it this way so we don't trigger this branch unless we must. + size_t longest_digit_count = negative ? 19 : 20; + if (digit_count > longest_digit_count) { return BIGINT_NUMBER(src); } + if (digit_count == longest_digit_count) { + if (negative) { + // Anything negative above INT64_MAX+1 is invalid + if (i > uint64_t(INT64_MAX) + 1) { return BIGINT_NUMBER(src); } + WRITE_INTEGER(~i + 1, src, writer); + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return INVALID_NUMBER(src); } + return SUCCESS; + // Positive overflow check: + // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the + // biggest uint64_t. + // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX. + // If we got here, it's a 20 digit number starting with the digit "1". + // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller + // than 1,553,255,926,290,448,384. + // - That is smaller than the smallest possible 20-digit number the user could write: + // 10,000,000,000,000,000,000. + // - Therefore, if the number is positive and lower than that, it's overflow. + // - The value we are looking at is less than or equal to INT64_MAX. + // + } + else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INVALID_NUMBER(src); } + } + + // Write unsigned if it does not fit in a signed integer. + if (i > uint64_t(INT64_MAX)) { + WRITE_UNSIGNED(i, src, writer); + } + else { + WRITE_INTEGER(negative ? (~i + 1) : i, src, writer); + } + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return INVALID_NUMBER(src); } + return SUCCESS; + } + + // Inlineable functions + namespace { + + // This table can be used to characterize the final character of an integer + // string. For JSON structural character and allowable white space characters, + // we return SUCCESS. For 'e', '.' and 'E', we return INCORRECT_TYPE. Otherwise + // we return NUMBER_ERROR. + // Optimization note: we could easily reduce the size of the table by half (to 128) + // at the cost of an extra branch. + // Optimization note: we want the values to use at most 8 bits (not, e.g., 32 bits): + static_assert(error_code(uint8_t(NUMBER_ERROR)) == NUMBER_ERROR, "bad NUMBER_ERROR cast"); + static_assert(error_code(uint8_t(SUCCESS)) == SUCCESS, "bad NUMBER_ERROR cast"); + static_assert(error_code(uint8_t(INCORRECT_TYPE)) == INCORRECT_TYPE, "bad NUMBER_ERROR cast"); + + const uint8_t integer_string_finisher[256] = { + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, + SUCCESS, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, + NUMBER_ERROR, INCORRECT_TYPE, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, INCORRECT_TYPE, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, SUCCESS, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, INCORRECT_TYPE, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, + SUCCESS, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR }; + + // Parse any number from 0 to 18,446,744,073,709,551,615 + simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t* const src) noexcept { + const uint8_t* p = src; + // + // Parse the integer part. + // + // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare + const uint8_t* const start_digits = p; + uint64_t i = 0; + while (parse_digit(*p, i)) { p++; } + + // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. + // Optimization note: size_t is expected to be unsigned. + size_t digit_count = size_t(p - start_digits); + // The longest positive 64-bit number is 20 digits. + // We do it this way so we don't trigger this branch unless we must. + // Optimization note: the compiler can probably merge + // ((digit_count == 0) || (digit_count > 20)) + // into a single branch since digit_count is unsigned. + if ((digit_count == 0) || (digit_count > 20)) { return INCORRECT_TYPE; } + // Here digit_count > 0. + if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } + // We can do the following... + // if (!jsoncharutils::is_structural_or_whitespace(*p)) { + // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; + // } + // as a single table lookup: + if (integer_string_finisher[*p] != SUCCESS) { return error_code(integer_string_finisher[*p]); } + + if (digit_count == 20) { + // Positive overflow check: + // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the + // biggest uint64_t. + // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX. + // If we got here, it's a 20 digit number starting with the digit "1". + // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller + // than 1,553,255,926,290,448,384. + // - That is smaller than the smallest possible 20-digit number the user could write: + // 10,000,000,000,000,000,000. + // - Therefore, if the number is positive and lower than that, it's overflow. + // - The value we are looking at is less than or equal to INT64_MAX. + // + if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INCORRECT_TYPE; } + } + + return i; + } + + + // Parse any number from 0 to 18,446,744,073,709,551,615 + // Never read at src_end or beyond + simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t* const src, const uint8_t* const src_end) noexcept { + const uint8_t* p = src; + // + // Parse the integer part. + // + // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare + const uint8_t* const start_digits = p; + uint64_t i = 0; + while ((p != src_end) && parse_digit(*p, i)) { p++; } + + // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. + // Optimization note: size_t is expected to be unsigned. + size_t digit_count = size_t(p - start_digits); + // The longest positive 64-bit number is 20 digits. + // We do it this way so we don't trigger this branch unless we must. + // Optimization note: the compiler can probably merge + // ((digit_count == 0) || (digit_count > 20)) + // into a single branch since digit_count is unsigned. + if ((digit_count == 0) || (digit_count > 20)) { return INCORRECT_TYPE; } + // Here digit_count > 0. + if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } + // We can do the following... + // if (!jsoncharutils::is_structural_or_whitespace(*p)) { + // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; + // } + // as a single table lookup: + if ((p != src_end) && integer_string_finisher[*p] != SUCCESS) { return error_code(integer_string_finisher[*p]); } + + if (digit_count == 20) { + // Positive overflow check: + // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the + // biggest uint64_t. + // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX. + // If we got here, it's a 20 digit number starting with the digit "1". + // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller + // than 1,553,255,926,290,448,384. + // - That is smaller than the smallest possible 20-digit number the user could write: + // 10,000,000,000,000,000,000. + // - Therefore, if the number is positive and lower than that, it's overflow. + // - The value we are looking at is less than or equal to INT64_MAX. + // + if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INCORRECT_TYPE; } + } + + return i; + } + + // Parse any number from 0 to 18,446,744,073,709,551,615 + simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t* const src) noexcept { + const uint8_t* p = src + 1; + // + // Parse the integer part. + // + // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare + const uint8_t* const start_digits = p; + uint64_t i = 0; + while (parse_digit(*p, i)) { p++; } + + // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. + // Optimization note: size_t is expected to be unsigned. + size_t digit_count = size_t(p - start_digits); + // The longest positive 64-bit number is 20 digits. + // We do it this way so we don't trigger this branch unless we must. + // Optimization note: the compiler can probably merge + // ((digit_count == 0) || (digit_count > 20)) + // into a single branch since digit_count is unsigned. + if ((digit_count == 0) || (digit_count > 20)) { return INCORRECT_TYPE; } + // Here digit_count > 0. + if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } + // We can do the following... + // if (!jsoncharutils::is_structural_or_whitespace(*p)) { + // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; + // } + // as a single table lookup: + if (*p != '"') { return NUMBER_ERROR; } + + if (digit_count == 20) { + // Positive overflow check: + // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the + // biggest uint64_t. + // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX. + // If we got here, it's a 20 digit number starting with the digit "1". + // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller + // than 1,553,255,926,290,448,384. + // - That is smaller than the smallest possible 20-digit number the user could write: + // 10,000,000,000,000,000,000. + // - Therefore, if the number is positive and lower than that, it's overflow. + // - The value we are looking at is less than or equal to INT64_MAX. + // + // Note: we use src[1] and not src[0] because src[0] is the quote character in this + // instance. + if (src[1] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INCORRECT_TYPE; } + } + + return i; + } + + // Parse any number from -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 + simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t* src) noexcept { + // + // Check for minus sign + // + bool negative = (*src == '-'); + const uint8_t* p = src + uint8_t(negative); + + // + // Parse the integer part. + // + // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare + const uint8_t* const start_digits = p; + uint64_t i = 0; + while (parse_digit(*p, i)) { p++; } + + // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. + // Optimization note: size_t is expected to be unsigned. + size_t digit_count = size_t(p - start_digits); + // We go from + // -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 + // so we can never represent numbers that have more than 19 digits. + size_t longest_digit_count = 19; + // Optimization note: the compiler can probably merge + // ((digit_count == 0) || (digit_count > longest_digit_count)) + // into a single branch since digit_count is unsigned. + if ((digit_count == 0) || (digit_count > longest_digit_count)) { return INCORRECT_TYPE; } + // Here digit_count > 0. + if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } + // We can do the following... + // if (!jsoncharutils::is_structural_or_whitespace(*p)) { + // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; + // } + // as a single table lookup: + if (integer_string_finisher[*p] != SUCCESS) { return error_code(integer_string_finisher[*p]); } + // Negative numbers have can go down to - INT64_MAX - 1 whereas positive numbers are limited to INT64_MAX. + // Performance note: This check is only needed when digit_count == longest_digit_count but it is + // so cheap that we might as well always make it. + if (i > uint64_t(INT64_MAX) + uint64_t(negative)) { return INCORRECT_TYPE; } + return negative ? (~i + 1) : i; + } + + // Parse any number from -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 + // Never read at src_end or beyond + simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t* const src, const uint8_t* const src_end) noexcept { + // + // Check for minus sign + // + if (src == src_end) { return NUMBER_ERROR; } + bool negative = (*src == '-'); + const uint8_t* p = src + uint8_t(negative); + + // + // Parse the integer part. + // + // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare + const uint8_t* const start_digits = p; + uint64_t i = 0; + while ((p != src_end) && parse_digit(*p, i)) { p++; } + + // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. + // Optimization note: size_t is expected to be unsigned. + size_t digit_count = size_t(p - start_digits); + // We go from + // -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 + // so we can never represent numbers that have more than 19 digits. + size_t longest_digit_count = 19; + // Optimization note: the compiler can probably merge + // ((digit_count == 0) || (digit_count > longest_digit_count)) + // into a single branch since digit_count is unsigned. + if ((digit_count == 0) || (digit_count > longest_digit_count)) { return INCORRECT_TYPE; } + // Here digit_count > 0. + if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } + // We can do the following... + // if (!jsoncharutils::is_structural_or_whitespace(*p)) { + // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; + // } + // as a single table lookup: + if ((p != src_end) && integer_string_finisher[*p] != SUCCESS) { return error_code(integer_string_finisher[*p]); } + // Negative numbers have can go down to - INT64_MAX - 1 whereas positive numbers are limited to INT64_MAX. + // Performance note: This check is only needed when digit_count == longest_digit_count but it is + // so cheap that we might as well always make it. + if (i > uint64_t(INT64_MAX) + uint64_t(negative)) { return INCORRECT_TYPE; } + return negative ? (~i + 1) : i; + } + + // Parse any number from -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 + simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t* src) noexcept { + // + // Check for minus sign + // + bool negative = (*(src + 1) == '-'); + src += uint8_t(negative) + 1; + + // + // Parse the integer part. + // + // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare + const uint8_t* const start_digits = src; + uint64_t i = 0; + while (parse_digit(*src, i)) { src++; } + + // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. + // Optimization note: size_t is expected to be unsigned. + size_t digit_count = size_t(src - start_digits); + // We go from + // -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 + // so we can never represent numbers that have more than 19 digits. + size_t longest_digit_count = 19; + // Optimization note: the compiler can probably merge + // ((digit_count == 0) || (digit_count > longest_digit_count)) + // into a single branch since digit_count is unsigned. + if ((digit_count == 0) || (digit_count > longest_digit_count)) { return INCORRECT_TYPE; } + // Here digit_count > 0. + if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } + // We can do the following... + // if (!jsoncharutils::is_structural_or_whitespace(*src)) { + // return (*src == '.' || *src == 'e' || *src == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; + // } + // as a single table lookup: + if (*src != '"') { return NUMBER_ERROR; } + // Negative numbers have can go down to - INT64_MAX - 1 whereas positive numbers are limited to INT64_MAX. + // Performance note: This check is only needed when digit_count == longest_digit_count but it is + // so cheap that we might as well always make it. + if (i > uint64_t(INT64_MAX) + uint64_t(negative)) { return INCORRECT_TYPE; } + return negative ? (~i + 1) : i; + } + + simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t* src) noexcept { + // + // Check for minus sign + // + bool negative = (*src == '-'); + src += uint8_t(negative); + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t* p = src; + p += parse_digit(*p, i); + bool leading_zero = (i == 0); + while (parse_digit(*p, i)) { p++; } + // no integer digits, or 0123 (zero must be solo) + if (p == src) { return INCORRECT_TYPE; } + if ((leading_zero && p != src + 1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely(*p == '.')) { + p++; + const uint8_t* start_decimal_digits = p; + if (!parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits + p++; + while (parse_digit(*p, i)) { p++; } + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p - src - 1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t* start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = p - start_digits > 19; + } + } + else { + overflow = p - src > 19; + } + + // + // Parse the exponent + // + if (*p == 'e' || *p == 'E') { + p++; + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t* start_exp_digits = p; + while (parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p - start_exp_digits == 0 || p - start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0 - exp : exp; + } + + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + double d; + if (simdjson_likely(!overflow)) { + if (compute_float_64(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), &d)) { + return NUMBER_ERROR; + } + return d; + } + + simdjson_unused simdjson_inline bool is_negative(const uint8_t* src) noexcept { + return (*src == '-'); + } + + simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t* src) noexcept { + bool negative = (*src == '-'); + src += uint8_t(negative); + const uint8_t* p = src; + while (static_cast(*p - '0') <= 9) { p++; } + if (p == src) { return NUMBER_ERROR; } + if (jsoncharutils::is_structural_or_whitespace(*p)) { return true; } + return false; + } + + simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t* src) noexcept { + bool negative = (*src == '-'); + src += uint8_t(negative); + const uint8_t* p = src; + while (static_cast(*p - '0') <= 9) { p++; } + size_t digit_count = size_t(p - src); + if (p == src) { return NUMBER_ERROR; } + if (jsoncharutils::is_structural_or_whitespace(*p)) { + static const uint8_t* smaller_big_integer = reinterpret_cast("9223372036854775808"); + // We have an integer. + if (simdjson_unlikely(digit_count > 20)) { + return number_type::big_integer; + } + // If the number is negative and valid, it must be a signed integer. + if (negative) { + if (simdjson_unlikely(digit_count > 19)) return number_type::big_integer; + if (simdjson_unlikely(digit_count == 19 && memcmp(src, smaller_big_integer, 19) > 0)) { + return number_type::big_integer; + } + return number_type::signed_integer; + } + // Let us check if we have a big integer (>=2**64). + static const uint8_t* two_to_sixtyfour = reinterpret_cast("18446744073709551616"); + if ((digit_count > 20) || (digit_count == 20 && memcmp(src, two_to_sixtyfour, 20) >= 0)) { + return number_type::big_integer; + } + // The number is positive and smaller than 18446744073709551616 (or 2**64). + // We want values larger or equal to 9223372036854775808 to be unsigned + // integers, and the other values to be signed integers. + if ((digit_count == 20) || (digit_count >= 19 && memcmp(src, smaller_big_integer, 19) >= 0)) { + return number_type::unsigned_integer; + } + return number_type::signed_integer; + } + // Hopefully, we have 'e' or 'E' or '.'. + return number_type::floating_point_number; + } + + // Never read at src_end or beyond + simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t* src, const uint8_t* const src_end) noexcept { + if (src == src_end) { return NUMBER_ERROR; } + // + // Check for minus sign + // + bool negative = (*src == '-'); + src += uint8_t(negative); + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t* p = src; + if (p == src_end) { return NUMBER_ERROR; } + p += parse_digit(*p, i); + bool leading_zero = (i == 0); + while ((p != src_end) && parse_digit(*p, i)) { p++; } + // no integer digits, or 0123 (zero must be solo) + if (p == src) { return INCORRECT_TYPE; } + if ((leading_zero && p != src + 1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely((p != src_end) && (*p == '.'))) { + p++; + const uint8_t* start_decimal_digits = p; + if ((p == src_end) || !parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits + p++; + while ((p != src_end) && parse_digit(*p, i)) { p++; } + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p - src - 1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t* start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = start_digits - src > 19; + } + } + else { + overflow = p - src > 19; + } + + // + // Parse the exponent + // + if ((p != src_end) && (*p == 'e' || *p == 'E')) { + p++; + if (p == src_end) { return NUMBER_ERROR; } + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t* start_exp_digits = p; + while ((p != src_end) && parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p - start_exp_digits == 0 || p - start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0 - exp : exp; + } + + if ((p != src_end) && jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + double d; + if (simdjson_likely(!overflow)) { + if (compute_float_64(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), src_end, &d)) { + return NUMBER_ERROR; + } + return d; + } + + simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t* src) noexcept { + // + // Check for minus sign + // + bool negative = (*(src + 1) == '-'); + src += uint8_t(negative) + 1; + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t* p = src; + p += parse_digit(*p, i); + bool leading_zero = (i == 0); + while (parse_digit(*p, i)) { p++; } + // no integer digits, or 0123 (zero must be solo) + if (p == src) { return INCORRECT_TYPE; } + if ((leading_zero && p != src + 1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely(*p == '.')) { + p++; + const uint8_t* start_decimal_digits = p; + if (!parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits + p++; + while (parse_digit(*p, i)) { p++; } + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p - src - 1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t* start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = p - start_digits > 19; + } + } + else { + overflow = p - src > 19; + } + + // + // Parse the exponent + // + if (*p == 'e' || *p == 'E') { + p++; + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t* start_exp_digits = p; + while (parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p - start_exp_digits == 0 || p - start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0 - exp : exp; + } + + if (*p != '"') { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + double d; + if (simdjson_likely(!overflow)) { + if (compute_float_64(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), &d)) { + return NUMBER_ERROR; + } + return d; + } + + } // unnamed namespace +#endif // SIMDJSON_SKIPNUMBERPARSING + + } // namespace numberparsing + + inline std::ostream& operator<<(std::ostream& out, number_type type) noexcept { + switch (type) { + case number_type::signed_integer: out << "integer in [-9223372036854775808,9223372036854775808)"; break; + case number_type::unsigned_integer: out << "unsigned integer in [9223372036854775808,18446744073709551616)"; break; + case number_type::floating_point_number: out << "floating-point number (binary64)"; break; + case number_type::big_integer: out << "big integer"; break; + default: SIMDJSON_UNREACHABLE(); + } + return out; + } + + } // namespace icelake +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_NUMBERPARSING_H +/* end file simdjson/generic/numberparsing.h for icelake */ + +/* including simdjson/generic/implementation_simdjson_result_base-inl.h for icelake: #include "simdjson/generic/implementation_simdjson_result_base-inl.h" */ +/* begin file simdjson/generic/implementation_simdjson_result_base-inl.h for icelake */ +#ifndef SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_INL_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_INL_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/implementation_simdjson_result_base.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace icelake { + + // + // internal::implementation_simdjson_result_base inline implementation + // + + template + simdjson_inline void implementation_simdjson_result_base::tie(T& value, error_code& error) && noexcept { + error = this->second; + if (!error) { + value = std::forward>(*this).first; + } + } + + template + simdjson_warn_unused simdjson_inline error_code implementation_simdjson_result_base::get(T& value) && noexcept { + error_code error; + std::forward>(*this).tie(value, error); + return error; + } + + template + simdjson_inline error_code implementation_simdjson_result_base::error() const noexcept { + return this->second; + } + +#if SIMDJSON_EXCEPTIONS + + template + simdjson_inline T& implementation_simdjson_result_base::value() & noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return this->first; + } + + template + simdjson_inline T&& implementation_simdjson_result_base::value() && noexcept(false) { + return std::forward>(*this).take_value(); + } + + template + simdjson_inline T&& implementation_simdjson_result_base::take_value() && noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return std::forward(this->first); + } + + template + simdjson_inline implementation_simdjson_result_base::operator T && () && noexcept(false) { + return std::forward>(*this).take_value(); + } + +#endif // SIMDJSON_EXCEPTIONS + + template + simdjson_inline const T& implementation_simdjson_result_base::value_unsafe() const& noexcept { + return this->first; + } + + template + simdjson_inline T& implementation_simdjson_result_base::value_unsafe() & noexcept { + return this->first; + } + + template + simdjson_inline T&& implementation_simdjson_result_base::value_unsafe() && noexcept { + return std::forward(this->first); + } + + template + simdjson_inline implementation_simdjson_result_base::implementation_simdjson_result_base(T&& value, error_code error) noexcept + : first{ std::forward(value) }, second{ error } {} + template + simdjson_inline implementation_simdjson_result_base::implementation_simdjson_result_base(error_code error) noexcept + : implementation_simdjson_result_base(T{}, error) {} + template + simdjson_inline implementation_simdjson_result_base::implementation_simdjson_result_base(T&& value) noexcept + : implementation_simdjson_result_base(std::forward(value), SUCCESS) {} + + } // namespace icelake +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_INL_H +/* end file simdjson/generic/implementation_simdjson_result_base-inl.h for icelake */ +/* end file simdjson/generic/amalgamated.h for icelake */ +/* including simdjson/icelake/end.h: #include "simdjson/icelake/end.h" */ +/* begin file simdjson/icelake/end.h */ +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/icelake/base.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +#if !SIMDJSON_CAN_ALWAYS_RUN_ICELAKE +SIMDJSON_UNTARGET_REGION +#endif + +/* undefining SIMDJSON_IMPLEMENTATION from "icelake" */ +#undef SIMDJSON_IMPLEMENTATION +/* end file simdjson/icelake/end.h */ + +#endif // SIMDJSON_ICELAKE_H +/* end file simdjson/icelake.h */ +/* including simdjson/icelake/implementation.h: #include */ +/* begin file simdjson/icelake/implementation.h */ +#ifndef SIMDJSON_ICELAKE_IMPLEMENTATION_H +#define SIMDJSON_ICELAKE_IMPLEMENTATION_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/icelake/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/implementation.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/instruction_set.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +// The constructor may be executed on any host, so we take care not to use SIMDJSON_TARGET_ICELAKE +namespace simdjson { + namespace icelake { + + /** + * @private + */ + class implementation final : public simdjson::implementation { + public: + simdjson_inline implementation() : simdjson::implementation( + "icelake", + "Intel/AMD AVX512", + internal::instruction_set::AVX2 | internal::instruction_set::PCLMULQDQ | internal::instruction_set::BMI1 | internal::instruction_set::BMI2 | internal::instruction_set::AVX512F | internal::instruction_set::AVX512DQ | internal::instruction_set::AVX512CD | internal::instruction_set::AVX512BW | internal::instruction_set::AVX512VL | internal::instruction_set::AVX512VBMI2 + ) {} + simdjson_warn_unused error_code create_dom_parser_implementation( + size_t capacity, + size_t max_length, + std::unique_ptr& dst + ) const noexcept final; + simdjson_warn_unused error_code minify(const uint8_t* buf, size_t len, uint8_t* dst, size_t& dst_len) const noexcept final; + simdjson_warn_unused bool validate_utf8(const char* buf, size_t len) const noexcept final; + }; + + } // namespace icelake +} // namespace simdjson + +#endif // SIMDJSON_ICELAKE_IMPLEMENTATION_H +/* end file simdjson/icelake/implementation.h */ + +// defining SIMDJSON_GENERIC_JSON_STRUCTURAL_INDEXER_CUSTOM_BIT_INDEXER allows us to provide our own bit_indexer::write +#define SIMDJSON_GENERIC_JSON_STRUCTURAL_INDEXER_CUSTOM_BIT_INDEXER + +/* including simdjson/icelake/begin.h: #include */ +/* begin file simdjson/icelake/begin.h */ +/* defining SIMDJSON_IMPLEMENTATION to "icelake" */ +#define SIMDJSON_IMPLEMENTATION icelake +/* including simdjson/icelake/base.h: #include "simdjson/icelake/base.h" */ +/* begin file simdjson/icelake/base.h */ +#ifndef SIMDJSON_ICELAKE_BASE_H +#define SIMDJSON_ICELAKE_BASE_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/base.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +// The constructor may be executed on any host, so we take care not to use SIMDJSON_TARGET_ICELAKE +namespace simdjson { + /** + * Implementation for Icelake (Intel AVX512). + */ + namespace icelake { + + class implementation; + + } // namespace icelake +} // namespace simdjson + +#endif // SIMDJSON_ICELAKE_BASE_H +/* end file simdjson/icelake/base.h */ +/* including simdjson/icelake/intrinsics.h: #include "simdjson/icelake/intrinsics.h" */ +/* begin file simdjson/icelake/intrinsics.h */ +#ifndef SIMDJSON_ICELAKE_INTRINSICS_H +#define SIMDJSON_ICELAKE_INTRINSICS_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/icelake/base.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +#if SIMDJSON_VISUAL_STUDIO +// under clang within visual studio, this will include +#include // visual studio or clang +#else +#include // elsewhere +#endif // SIMDJSON_VISUAL_STUDIO + +#if SIMDJSON_CLANG_VISUAL_STUDIO +/** + * You are not supposed, normally, to include these + * headers directly. Instead you should either include intrin.h + * or x86intrin.h. However, when compiling with clang + * under Windows (i.e., when _MSC_VER is set), these headers + * only get included *if* the corresponding features are detected + * from macros: + * e.g., if __AVX2__ is set... in turn, we normally set these + * macros by compiling against the corresponding architecture + * (e.g., arch:AVX2, -mavx2, etc.) which compiles the whole + * software with these advanced instructions. In simdjson, we + * want to compile the whole program for a generic target, + * and only target our specific kernels. As a workaround, + * we directly include the needed headers. These headers would + * normally guard against such usage, but we carefully included + * (or ) before, so the headers + * are fooled. + */ +#include // for _blsr_u64 +#include // for __lzcnt64 +#include // for most things (AVX2, AVX512, _popcnt64) +#include +#include +#include +#include +#include // for _mm_clmulepi64_si128 + // Important: we need the AVX-512 headers: +#include +#include +#include +#include +#include +#include +#include +// unfortunately, we may not get _blsr_u64, but, thankfully, clang +// has it as a macro. +#ifndef _blsr_u64 +// we roll our own +#define _blsr_u64(n) ((n - 1) & n) +#endif // _blsr_u64 +#endif // SIMDJSON_CLANG_VISUAL_STUDIO + +static_assert(sizeof(__m512i) <= simdjson::SIMDJSON_PADDING, "insufficient padding for icelake"); + +#endif // SIMDJSON_ICELAKE_INTRINSICS_H +/* end file simdjson/icelake/intrinsics.h */ + +#if !SIMDJSON_CAN_ALWAYS_RUN_ICELAKE +SIMDJSON_TARGET_REGION("avx512f,avx512dq,avx512cd,avx512bw,avx512vbmi,avx512vbmi2,avx512vl,avx2,bmi,pclmul,lzcnt,popcnt") +#endif + +/* including simdjson/icelake/bitmanipulation.h: #include "simdjson/icelake/bitmanipulation.h" */ +/* begin file simdjson/icelake/bitmanipulation.h */ +#ifndef SIMDJSON_ICELAKE_BITMANIPULATION_H +#define SIMDJSON_ICELAKE_BITMANIPULATION_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/icelake/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/icelake/intrinsics.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace icelake { + namespace { + + // We sometimes call trailing_zero on inputs that are zero, + // but the algorithms do not end up using the returned value. + // Sadly, sanitizers are not smart enough to figure it out. + SIMDJSON_NO_SANITIZE_UNDEFINED + // This function can be used safely even if not all bytes have been + // initialized. + // See issue https://github.com/simdjson/simdjson/issues/1965 + SIMDJSON_NO_SANITIZE_MEMORY + simdjson_inline int trailing_zeroes(uint64_t input_num) { +#if SIMDJSON_REGULAR_VISUAL_STUDIO + return (int)_tzcnt_u64(input_num); +#else // SIMDJSON_REGULAR_VISUAL_STUDIO + //////// + // You might expect the next line to be equivalent to + // return (int)_tzcnt_u64(input_num); + // but the generated code differs and might be less efficient? + //////// + return __builtin_ctzll(input_num); +#endif // SIMDJSON_REGULAR_VISUAL_STUDIO + } + + /* result might be undefined when input_num is zero */ + simdjson_inline uint64_t clear_lowest_bit(uint64_t input_num) { + return _blsr_u64(input_num); + } + + /* result might be undefined when input_num is zero */ + simdjson_inline int leading_zeroes(uint64_t input_num) { + return int(_lzcnt_u64(input_num)); + } + +#if SIMDJSON_REGULAR_VISUAL_STUDIO + simdjson_inline unsigned __int64 count_ones(uint64_t input_num) { + // note: we do not support legacy 32-bit Windows + return __popcnt64(input_num);// Visual Studio wants two underscores + } +#else + simdjson_inline long long int count_ones(uint64_t input_num) { + return _popcnt64(input_num); + } +#endif + + simdjson_inline bool add_overflow(uint64_t value1, uint64_t value2, + uint64_t* result) { +#if SIMDJSON_REGULAR_VISUAL_STUDIO + return _addcarry_u64(0, value1, value2, + reinterpret_cast(result)); +#else + return __builtin_uaddll_overflow(value1, value2, + reinterpret_cast(result)); +#endif + } + + } // unnamed namespace + } // namespace icelake +} // namespace simdjson + +#endif // SIMDJSON_ICELAKE_BITMANIPULATION_H +/* end file simdjson/icelake/bitmanipulation.h */ +/* including simdjson/icelake/bitmask.h: #include "simdjson/icelake/bitmask.h" */ +/* begin file simdjson/icelake/bitmask.h */ +#ifndef SIMDJSON_ICELAKE_BITMASK_H +#define SIMDJSON_ICELAKE_BITMASK_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/icelake/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/icelake/intrinsics.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace icelake { + namespace { + + // + // Perform a "cumulative bitwise xor," flipping bits each time a 1 is encountered. + // + // For example, prefix_xor(00100100) == 00011100 + // + simdjson_inline uint64_t prefix_xor(const uint64_t bitmask) { + // There should be no such thing with a processor supporting avx2 + // but not clmul. + __m128i all_ones = _mm_set1_epi8('\xFF'); + __m128i result = _mm_clmulepi64_si128(_mm_set_epi64x(0ULL, bitmask), all_ones, 0); + return _mm_cvtsi128_si64(result); + } + + } // unnamed namespace + } // namespace icelake +} // namespace simdjson + +#endif // SIMDJSON_ICELAKE_BITMASK_H +/* end file simdjson/icelake/bitmask.h */ +/* including simdjson/icelake/simd.h: #include "simdjson/icelake/simd.h" */ +/* begin file simdjson/icelake/simd.h */ +#ifndef SIMDJSON_ICELAKE_SIMD_H +#define SIMDJSON_ICELAKE_SIMD_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/icelake/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/icelake/intrinsics.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/icelake/bitmanipulation.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/simdprune_tables.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +#if defined(__GNUC__) && !defined(__clang__) +#if __GNUC__ == 8 +#define SIMDJSON_GCC8 1 +#endif // __GNUC__ == 8 +#endif // defined(__GNUC__) && !defined(__clang__) + +#if SIMDJSON_GCC8 +/** + * GCC 8 fails to provide _mm512_set_epi8. We roll our own. + */ +inline __m512i _mm512_set_epi8(uint8_t a0, uint8_t a1, uint8_t a2, uint8_t a3, uint8_t a4, uint8_t a5, uint8_t a6, uint8_t a7, uint8_t a8, uint8_t a9, uint8_t a10, uint8_t a11, uint8_t a12, uint8_t a13, uint8_t a14, uint8_t a15, uint8_t a16, uint8_t a17, uint8_t a18, uint8_t a19, uint8_t a20, uint8_t a21, uint8_t a22, uint8_t a23, uint8_t a24, uint8_t a25, uint8_t a26, uint8_t a27, uint8_t a28, uint8_t a29, uint8_t a30, uint8_t a31, uint8_t a32, uint8_t a33, uint8_t a34, uint8_t a35, uint8_t a36, uint8_t a37, uint8_t a38, uint8_t a39, uint8_t a40, uint8_t a41, uint8_t a42, uint8_t a43, uint8_t a44, uint8_t a45, uint8_t a46, uint8_t a47, uint8_t a48, uint8_t a49, uint8_t a50, uint8_t a51, uint8_t a52, uint8_t a53, uint8_t a54, uint8_t a55, uint8_t a56, uint8_t a57, uint8_t a58, uint8_t a59, uint8_t a60, uint8_t a61, uint8_t a62, uint8_t a63) { + return _mm512_set_epi64(uint64_t(a7) + (uint64_t(a6) << 8) + (uint64_t(a5) << 16) + (uint64_t(a4) << 24) + (uint64_t(a3) << 32) + (uint64_t(a2) << 40) + (uint64_t(a1) << 48) + (uint64_t(a0) << 56), + uint64_t(a15) + (uint64_t(a14) << 8) + (uint64_t(a13) << 16) + (uint64_t(a12) << 24) + (uint64_t(a11) << 32) + (uint64_t(a10) << 40) + (uint64_t(a9) << 48) + (uint64_t(a8) << 56), + uint64_t(a23) + (uint64_t(a22) << 8) + (uint64_t(a21) << 16) + (uint64_t(a20) << 24) + (uint64_t(a19) << 32) + (uint64_t(a18) << 40) + (uint64_t(a17) << 48) + (uint64_t(a16) << 56), + uint64_t(a31) + (uint64_t(a30) << 8) + (uint64_t(a29) << 16) + (uint64_t(a28) << 24) + (uint64_t(a27) << 32) + (uint64_t(a26) << 40) + (uint64_t(a25) << 48) + (uint64_t(a24) << 56), + uint64_t(a39) + (uint64_t(a38) << 8) + (uint64_t(a37) << 16) + (uint64_t(a36) << 24) + (uint64_t(a35) << 32) + (uint64_t(a34) << 40) + (uint64_t(a33) << 48) + (uint64_t(a32) << 56), + uint64_t(a47) + (uint64_t(a46) << 8) + (uint64_t(a45) << 16) + (uint64_t(a44) << 24) + (uint64_t(a43) << 32) + (uint64_t(a42) << 40) + (uint64_t(a41) << 48) + (uint64_t(a40) << 56), + uint64_t(a55) + (uint64_t(a54) << 8) + (uint64_t(a53) << 16) + (uint64_t(a52) << 24) + (uint64_t(a51) << 32) + (uint64_t(a50) << 40) + (uint64_t(a49) << 48) + (uint64_t(a48) << 56), + uint64_t(a63) + (uint64_t(a62) << 8) + (uint64_t(a61) << 16) + (uint64_t(a60) << 24) + (uint64_t(a59) << 32) + (uint64_t(a58) << 40) + (uint64_t(a57) << 48) + (uint64_t(a56) << 56)); +} +#endif // SIMDJSON_GCC8 + + + +namespace simdjson { + namespace icelake { + namespace { + namespace simd { + + // Forward-declared so they can be used by splat and friends. + template + struct base { + __m512i value; + + // Zero constructor + simdjson_inline base() : value{ __m512i() } {} + + // Conversion from SIMD register + simdjson_inline base(const __m512i _value) : value(_value) {} + + // Conversion to SIMD register + simdjson_inline operator const __m512i& () const { return this->value; } + simdjson_inline operator __m512i& () { return this->value; } + + // Bit operations + simdjson_inline Child operator|(const Child other) const { return _mm512_or_si512(*this, other); } + simdjson_inline Child operator&(const Child other) const { return _mm512_and_si512(*this, other); } + simdjson_inline Child operator^(const Child other) const { return _mm512_xor_si512(*this, other); } + simdjson_inline Child bit_andnot(const Child other) const { return _mm512_andnot_si512(other, *this); } + simdjson_inline Child& operator|=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast | other; return *this_cast; } + simdjson_inline Child& operator&=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast & other; return *this_cast; } + simdjson_inline Child& operator^=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast ^ other; return *this_cast; } + }; + + // Forward-declared so they can be used by splat and friends. + template + struct simd8; + + template> + struct base8 : base> { + typedef uint32_t bitmask_t; + typedef uint64_t bitmask2_t; + + simdjson_inline base8() : base>() {} + simdjson_inline base8(const __m512i _value) : base>(_value) {} + + friend simdjson_really_inline uint64_t operator==(const simd8 lhs, const simd8 rhs) { + return _mm512_cmpeq_epi8_mask(lhs, rhs); + } + + static const int SIZE = sizeof(base::value); + + template + simdjson_inline simd8 prev(const simd8 prev_chunk) const { + // workaround for compilers unable to figure out that 16 - N is a constant (GCC 8) + constexpr int shift = 16 - N; + return _mm512_alignr_epi8(*this, _mm512_permutex2var_epi64(prev_chunk, _mm512_set_epi64(13, 12, 11, 10, 9, 8, 7, 6), *this), shift); + } + }; + + // SIMD byte mask type (returned by things like eq and gt) + template<> + struct simd8 : base8 { + static simdjson_inline simd8 splat(bool _value) { return _mm512_set1_epi8(uint8_t(-(!!_value))); } + + simdjson_inline simd8() : base8() {} + simdjson_inline simd8(const __m512i _value) : base8(_value) {} + // Splat constructor + simdjson_inline simd8(bool _value) : base8(splat(_value)) {} + simdjson_inline bool any() const { return !!_mm512_test_epi8_mask(*this, *this); } + simdjson_inline simd8 operator~() const { return *this ^ true; } + }; + + template + struct base8_numeric : base8 { + static simdjson_inline simd8 splat(T _value) { return _mm512_set1_epi8(_value); } + static simdjson_inline simd8 zero() { return _mm512_setzero_si512(); } + static simdjson_inline simd8 load(const T values[64]) { + return _mm512_loadu_si512(reinterpret_cast(values)); + } + // Repeat 16 values as many times as necessary (usually for lookup tables) + static simdjson_inline simd8 repeat_16( + T v0, T v1, T v2, T v3, T v4, T v5, T v6, T v7, + T v8, T v9, T v10, T v11, T v12, T v13, T v14, T v15 + ) { + return simd8( + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15, + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15, + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15, + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15 + ); + } + + simdjson_inline base8_numeric() : base8() {} + simdjson_inline base8_numeric(const __m512i _value) : base8(_value) {} + + // Store to array + simdjson_inline void store(T dst[64]) const { return _mm512_storeu_si512(reinterpret_cast<__m512i*>(dst), *this); } + + // Addition/subtraction are the same for signed and unsigned + simdjson_inline simd8 operator+(const simd8 other) const { return _mm512_add_epi8(*this, other); } + simdjson_inline simd8 operator-(const simd8 other) const { return _mm512_sub_epi8(*this, other); } + simdjson_inline simd8& operator+=(const simd8 other) { *this = *this + other; return *static_cast*>(this); } + simdjson_inline simd8& operator-=(const simd8 other) { *this = *this - other; return *static_cast*>(this); } + + // Override to distinguish from bool version + simdjson_inline simd8 operator~() const { return *this ^ 0xFFu; } + + // Perform a lookup assuming the value is between 0 and 16 (undefined behavior for out of range values) + template + simdjson_inline simd8 lookup_16(simd8 lookup_table) const { + return _mm512_shuffle_epi8(lookup_table, *this); + } + + // Copies to 'output" all bytes corresponding to a 0 in the mask (interpreted as a bitset). + // Passing a 0 value for mask would be equivalent to writing out every byte to output. + // Only the first 32 - count_ones(mask) bytes of the result are significant but 32 bytes + // get written. + // Design consideration: it seems like a function with the + // signature simd8 compress(uint32_t mask) would be + // sensible, but the AVX ISA makes this kind of approach difficult. + template + simdjson_inline void compress(uint64_t mask, L* output) const { + _mm512_mask_compressstoreu_epi8(output, ~mask, *this); + } + + template + simdjson_inline simd8 lookup_16( + L replace0, L replace1, L replace2, L replace3, + L replace4, L replace5, L replace6, L replace7, + L replace8, L replace9, L replace10, L replace11, + L replace12, L replace13, L replace14, L replace15) const { + return lookup_16(simd8::repeat_16( + replace0, replace1, replace2, replace3, + replace4, replace5, replace6, replace7, + replace8, replace9, replace10, replace11, + replace12, replace13, replace14, replace15 + )); + } + }; + + // Signed bytes + template<> + struct simd8 : base8_numeric { + simdjson_inline simd8() : base8_numeric() {} + simdjson_inline simd8(const __m512i _value) : base8_numeric(_value) {} + // Splat constructor + simdjson_inline simd8(int8_t _value) : simd8(splat(_value)) {} + // Array constructor + simdjson_inline simd8(const int8_t values[64]) : simd8(load(values)) {} + // Member-by-member initialization + simdjson_inline simd8( + int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, + int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15, + int8_t v16, int8_t v17, int8_t v18, int8_t v19, int8_t v20, int8_t v21, int8_t v22, int8_t v23, + int8_t v24, int8_t v25, int8_t v26, int8_t v27, int8_t v28, int8_t v29, int8_t v30, int8_t v31, + int8_t v32, int8_t v33, int8_t v34, int8_t v35, int8_t v36, int8_t v37, int8_t v38, int8_t v39, + int8_t v40, int8_t v41, int8_t v42, int8_t v43, int8_t v44, int8_t v45, int8_t v46, int8_t v47, + int8_t v48, int8_t v49, int8_t v50, int8_t v51, int8_t v52, int8_t v53, int8_t v54, int8_t v55, + int8_t v56, int8_t v57, int8_t v58, int8_t v59, int8_t v60, int8_t v61, int8_t v62, int8_t v63 + ) : simd8(_mm512_set_epi8( + v63, v62, v61, v60, v59, v58, v57, v56, + v55, v54, v53, v52, v51, v50, v49, v48, + v47, v46, v45, v44, v43, v42, v41, v40, + v39, v38, v37, v36, v35, v34, v33, v32, + v31, v30, v29, v28, v27, v26, v25, v24, + v23, v22, v21, v20, v19, v18, v17, v16, + v15, v14, v13, v12, v11, v10, v9, v8, + v7, v6, v5, v4, v3, v2, v1, v0 + )) {} + + // Repeat 16 values as many times as necessary (usually for lookup tables) + simdjson_inline static simd8 repeat_16( + int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, + int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 + ) { + return simd8( + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15, + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15, + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15, + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15 + ); + } + + // Order-sensitive comparisons + simdjson_inline simd8 max_val(const simd8 other) const { return _mm512_max_epi8(*this, other); } + simdjson_inline simd8 min_val(const simd8 other) const { return _mm512_min_epi8(*this, other); } + + simdjson_inline simd8 operator>(const simd8 other) const { return _mm512_maskz_abs_epi8(_mm512_cmpgt_epi8_mask(*this, other), _mm512_set1_epi8(uint8_t(0x80))); } + simdjson_inline simd8 operator<(const simd8 other) const { return _mm512_maskz_abs_epi8(_mm512_cmpgt_epi8_mask(other, *this), _mm512_set1_epi8(uint8_t(0x80))); } + }; + + // Unsigned bytes + template<> + struct simd8 : base8_numeric { + simdjson_inline simd8() : base8_numeric() {} + simdjson_inline simd8(const __m512i _value) : base8_numeric(_value) {} + // Splat constructor + simdjson_inline simd8(uint8_t _value) : simd8(splat(_value)) {} + // Array constructor + simdjson_inline simd8(const uint8_t values[64]) : simd8(load(values)) {} + // Member-by-member initialization + simdjson_inline simd8( + uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, + uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15, + uint8_t v16, uint8_t v17, uint8_t v18, uint8_t v19, uint8_t v20, uint8_t v21, uint8_t v22, uint8_t v23, + uint8_t v24, uint8_t v25, uint8_t v26, uint8_t v27, uint8_t v28, uint8_t v29, uint8_t v30, uint8_t v31, + uint8_t v32, uint8_t v33, uint8_t v34, uint8_t v35, uint8_t v36, uint8_t v37, uint8_t v38, uint8_t v39, + uint8_t v40, uint8_t v41, uint8_t v42, uint8_t v43, uint8_t v44, uint8_t v45, uint8_t v46, uint8_t v47, + uint8_t v48, uint8_t v49, uint8_t v50, uint8_t v51, uint8_t v52, uint8_t v53, uint8_t v54, uint8_t v55, + uint8_t v56, uint8_t v57, uint8_t v58, uint8_t v59, uint8_t v60, uint8_t v61, uint8_t v62, uint8_t v63 + ) : simd8(_mm512_set_epi8( + v63, v62, v61, v60, v59, v58, v57, v56, + v55, v54, v53, v52, v51, v50, v49, v48, + v47, v46, v45, v44, v43, v42, v41, v40, + v39, v38, v37, v36, v35, v34, v33, v32, + v31, v30, v29, v28, v27, v26, v25, v24, + v23, v22, v21, v20, v19, v18, v17, v16, + v15, v14, v13, v12, v11, v10, v9, v8, + v7, v6, v5, v4, v3, v2, v1, v0 + )) {} + + // Repeat 16 values as many times as necessary (usually for lookup tables) + simdjson_inline static simd8 repeat_16( + uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, + uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 + ) { + return simd8( + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15, + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15, + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15, + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15 + ); + } + + // Saturated math + simdjson_inline simd8 saturating_add(const simd8 other) const { return _mm512_adds_epu8(*this, other); } + simdjson_inline simd8 saturating_sub(const simd8 other) const { return _mm512_subs_epu8(*this, other); } + + // Order-specific operations + simdjson_inline simd8 max_val(const simd8 other) const { return _mm512_max_epu8(*this, other); } + simdjson_inline simd8 min_val(const simd8 other) const { return _mm512_min_epu8(other, *this); } + // Same as >, but only guarantees true is nonzero (< guarantees true = -1) + simdjson_inline simd8 gt_bits(const simd8 other) const { return this->saturating_sub(other); } + // Same as <, but only guarantees true is nonzero (< guarantees true = -1) + simdjson_inline simd8 lt_bits(const simd8 other) const { return other.saturating_sub(*this); } + simdjson_inline uint64_t operator<=(const simd8 other) const { return other.max_val(*this) == other; } + simdjson_inline uint64_t operator>=(const simd8 other) const { return other.min_val(*this) == other; } + simdjson_inline simd8 operator>(const simd8 other) const { return this->gt_bits(other).any_bits_set(); } + simdjson_inline simd8 operator<(const simd8 other) const { return this->lt_bits(other).any_bits_set(); } + + // Bit-specific operations + simdjson_inline simd8 bits_not_set() const { return _mm512_mask_blend_epi8(*this == uint8_t(0), _mm512_set1_epi8(0), _mm512_set1_epi8(-1)); } + simdjson_inline simd8 bits_not_set(simd8 bits) const { return (*this & bits).bits_not_set(); } + simdjson_inline simd8 any_bits_set() const { return ~this->bits_not_set(); } + simdjson_inline simd8 any_bits_set(simd8 bits) const { return ~this->bits_not_set(bits); } + + simdjson_inline bool is_ascii() const { return _mm512_movepi8_mask(*this) == 0; } + simdjson_inline bool bits_not_set_anywhere() const { + return !_mm512_test_epi8_mask(*this, *this); + } + simdjson_inline bool any_bits_set_anywhere() const { return !bits_not_set_anywhere(); } + simdjson_inline bool bits_not_set_anywhere(simd8 bits) const { return !_mm512_test_epi8_mask(*this, bits); } + simdjson_inline bool any_bits_set_anywhere(simd8 bits) const { return !bits_not_set_anywhere(bits); } + template + simdjson_inline simd8 shr() const { return simd8(_mm512_srli_epi16(*this, N)) & uint8_t(0xFFu >> N); } + template + simdjson_inline simd8 shl() const { return simd8(_mm512_slli_epi16(*this, N)) & uint8_t(0xFFu << N); } + // Get one of the bits and make a bitmask out of it. + // e.g. value.get_bit<7>() gets the high bit + template + simdjson_inline uint64_t get_bit() const { return _mm512_movepi8_mask(_mm512_slli_epi16(*this, 7 - N)); } + }; + + template + struct simd8x64 { + static constexpr int NUM_CHUNKS = 64 / sizeof(simd8); + static_assert(NUM_CHUNKS == 1, "Icelake kernel should use one register per 64-byte block."); + const simd8 chunks[NUM_CHUNKS]; + + simd8x64(const simd8x64& o) = delete; // no copy allowed + simd8x64& operator=(const simd8& other) = delete; // no assignment allowed + simd8x64() = delete; // no default constructor allowed + + simdjson_inline simd8x64(const simd8 chunk0, const simd8 chunk1) : chunks{ chunk0, chunk1 } {} + simdjson_inline simd8x64(const simd8 chunk0) : chunks{ chunk0 } {} + simdjson_inline simd8x64(const T ptr[64]) : chunks{ simd8::load(ptr) } {} + + simdjson_inline uint64_t compress(uint64_t mask, T* output) const { + this->chunks[0].compress(mask, output); + return 64 - count_ones(mask); + } + + simdjson_inline void store(T ptr[64]) const { + this->chunks[0].store(ptr + sizeof(simd8) * 0); + } + + simdjson_inline simd8 reduce_or() const { + return this->chunks[0]; + } + + simdjson_inline simd8x64 bit_or(const T m) const { + const simd8 mask = simd8::splat(m); + return simd8x64( + this->chunks[0] | mask + ); + } + + simdjson_inline uint64_t eq(const T m) const { + const simd8 mask = simd8::splat(m); + return this->chunks[0] == mask; + } + + simdjson_inline uint64_t eq(const simd8x64& other) const { + return this->chunks[0] == other.chunks[0]; + } + + simdjson_inline uint64_t lteq(const T m) const { + const simd8 mask = simd8::splat(m); + return this->chunks[0] <= mask; + } + }; // struct simd8x64 + + } // namespace simd + + } // unnamed namespace + } // namespace icelake +} // namespace simdjson + +#endif // SIMDJSON_ICELAKE_SIMD_H +/* end file simdjson/icelake/simd.h */ +/* including simdjson/icelake/stringparsing_defs.h: #include "simdjson/icelake/stringparsing_defs.h" */ +/* begin file simdjson/icelake/stringparsing_defs.h */ +#ifndef SIMDJSON_ICELAKE_STRINGPARSING_DEFS_H +#define SIMDJSON_ICELAKE_STRINGPARSING_DEFS_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/icelake/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/icelake/simd.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/icelake/bitmanipulation.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace icelake { + namespace { + + using namespace simd; + + // Holds backslashes and quotes locations. + struct backslash_and_quote { + public: + static constexpr uint32_t BYTES_PROCESSED = 64; + simdjson_inline static backslash_and_quote copy_and_find(const uint8_t* src, uint8_t* dst); + + simdjson_inline bool has_quote_first() { return ((bs_bits - 1) & quote_bits) != 0; } + simdjson_inline bool has_backslash() { return ((quote_bits - 1) & bs_bits) != 0; } + simdjson_inline int quote_index() { return trailing_zeroes(quote_bits); } + simdjson_inline int backslash_index() { return trailing_zeroes(bs_bits); } + + uint64_t bs_bits; + uint64_t quote_bits; + }; // struct backslash_and_quote + + simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8_t* src, uint8_t* dst) { + // this can read up to 15 bytes beyond the buffer size, but we require + // SIMDJSON_PADDING of padding + static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "backslash and quote finder must process fewer than SIMDJSON_PADDING bytes"); + simd8 v(src); + // store to dest unconditionally - we can overwrite the bits we don't like later + v.store(dst); + return { + static_cast(v == '\\'), // bs_bits + static_cast(v == '"'), // quote_bits + }; + } + + } // unnamed namespace + } // namespace icelake +} // namespace simdjson + +#endif // SIMDJSON_ICELAKE_STRINGPARSING_DEFS_H +/* end file simdjson/icelake/stringparsing_defs.h */ +/* including simdjson/icelake/numberparsing_defs.h: #include "simdjson/icelake/numberparsing_defs.h" */ +/* begin file simdjson/icelake/numberparsing_defs.h */ +#ifndef SIMDJSON_ICELAKE_NUMBERPARSING_DEFS_H +#define SIMDJSON_ICELAKE_NUMBERPARSING_DEFS_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/icelake/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/icelake/intrinsics.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace icelake { + namespace numberparsing { + + static simdjson_inline uint32_t parse_eight_digits_unrolled(const uint8_t* chars) { + // this actually computes *16* values so we are being wasteful. + const __m128i ascii0 = _mm_set1_epi8('0'); + const __m128i mul_1_10 = + _mm_setr_epi8(10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1); + const __m128i mul_1_100 = _mm_setr_epi16(100, 1, 100, 1, 100, 1, 100, 1); + const __m128i mul_1_10000 = + _mm_setr_epi16(10000, 1, 10000, 1, 10000, 1, 10000, 1); + const __m128i input = _mm_sub_epi8( + _mm_loadu_si128(reinterpret_cast(chars)), ascii0); + const __m128i t1 = _mm_maddubs_epi16(input, mul_1_10); + const __m128i t2 = _mm_madd_epi16(t1, mul_1_100); + const __m128i t3 = _mm_packus_epi32(t2, t2); + const __m128i t4 = _mm_madd_epi16(t3, mul_1_10000); + return _mm_cvtsi128_si32( + t4); // only captures the sum of the first 8 digits, drop the rest + } + + /** @private */ + simdjson_inline internal::value128 full_multiplication(uint64_t value1, uint64_t value2) { + internal::value128 answer; +#if SIMDJSON_REGULAR_VISUAL_STUDIO || SIMDJSON_IS_32BITS +#if SIMDJSON_IS_ARM64 + // ARM64 has native support for 64-bit multiplications, no need to emultate + answer.high = __umulh(value1, value2); + answer.low = value1 * value2; +#else + answer.low = _umul128(value1, value2, &answer.high); // _umul128 not available on ARM64 +#endif // SIMDJSON_IS_ARM64 +#else // SIMDJSON_REGULAR_VISUAL_STUDIO || SIMDJSON_IS_32BITS + __uint128_t r = (static_cast<__uint128_t>(value1)) * value2; + answer.low = uint64_t(r); + answer.high = uint64_t(r >> 64); +#endif + return answer; + } + + } // namespace numberparsing + } // namespace icelake +} // namespace simdjson + +#define SIMDJSON_SWAR_NUMBER_PARSING 1 + +#endif // SIMDJSON_ICELAKE_NUMBERPARSING_DEFS_H +/* end file simdjson/icelake/numberparsing_defs.h */ +/* end file simdjson/icelake/begin.h */ +/* including generic/amalgamated.h for icelake: #include */ +/* begin file generic/amalgamated.h for icelake */ +#if defined(SIMDJSON_CONDITIONAL_INCLUDE) && !defined(SIMDJSON_SRC_GENERIC_DEPENDENCIES_H) +#error generic/dependencies.h must be included before generic/amalgamated.h! +#endif + +/* including generic/base.h for icelake: #include */ +/* begin file generic/base.h for icelake */ +#ifndef SIMDJSON_SRC_GENERIC_BASE_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_BASE_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace icelake { + namespace { + + struct json_character_block; + + } // unnamed namespace + } // namespace icelake +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_BASE_H +/* end file generic/base.h for icelake */ +/* including generic/dom_parser_implementation.h for icelake: #include */ +/* begin file generic/dom_parser_implementation.h for icelake */ +#ifndef SIMDJSON_SRC_GENERIC_DOM_PARSER_IMPLEMENTATION_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_DOM_PARSER_IMPLEMENTATION_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +// Interface a dom parser implementation must fulfill +namespace simdjson { + namespace icelake { + namespace { + + simdjson_inline simd8 must_be_2_3_continuation(const simd8 prev2, const simd8 prev3); + simdjson_inline bool is_ascii(const simd8x64& input); + + } // unnamed namespace + } // namespace icelake +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_DOM_PARSER_IMPLEMENTATION_H +/* end file generic/dom_parser_implementation.h for icelake */ +/* including generic/json_character_block.h for icelake: #include */ +/* begin file generic/json_character_block.h for icelake */ +#ifndef SIMDJSON_SRC_GENERIC_JSON_CHARACTER_BLOCK_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_JSON_CHARACTER_BLOCK_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace icelake { + namespace { + + struct json_character_block { + static simdjson_inline json_character_block classify(const simd::simd8x64& in); + + simdjson_inline uint64_t whitespace() const noexcept { return _whitespace; } + simdjson_inline uint64_t op() const noexcept { return _op; } + simdjson_inline uint64_t scalar() const noexcept { return ~(op() | whitespace()); } + + uint64_t _whitespace; + uint64_t _op; + }; + + } // unnamed namespace + } // namespace icelake +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_JSON_CHARACTER_BLOCK_H +/* end file generic/json_character_block.h for icelake */ +/* end file generic/amalgamated.h for icelake */ +/* including generic/stage1/amalgamated.h for icelake: #include */ +/* begin file generic/stage1/amalgamated.h for icelake */ +// Stuff other things depend on +/* including generic/stage1/base.h for icelake: #include */ +/* begin file generic/stage1/base.h for icelake */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE1_BASE_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_BASE_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace icelake { + namespace { + namespace stage1 { + + class bit_indexer; + template + struct buf_block_reader; + struct json_block; + class json_minifier; + class json_scanner; + struct json_string_block; + class json_string_scanner; + class json_structural_indexer; + + } // namespace stage1 + + namespace utf8_validation { + struct utf8_checker; + } // namespace utf8_validation + + using utf8_validation::utf8_checker; + + } // unnamed namespace + } // namespace icelake +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE1_BASE_H +/* end file generic/stage1/base.h for icelake */ +/* including generic/stage1/buf_block_reader.h for icelake: #include */ +/* begin file generic/stage1/buf_block_reader.h for icelake */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE1_BUF_BLOCK_READER_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_BUF_BLOCK_READER_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +#include + +namespace simdjson { + namespace icelake { + namespace { + namespace stage1 { + + // Walks through a buffer in block-sized increments, loading the last part with spaces + template + struct buf_block_reader { + public: + simdjson_inline buf_block_reader(const uint8_t* _buf, size_t _len); + simdjson_inline size_t block_index(); + simdjson_inline bool has_full_block() const; + simdjson_inline const uint8_t* full_block() const; + /** + * Get the last block, padded with spaces. + * + * There will always be a last block, with at least 1 byte, unless len == 0 (in which case this + * function fills the buffer with spaces and returns 0. In particular, if len == STEP_SIZE there + * will be 0 full_blocks and 1 remainder block with STEP_SIZE bytes and no spaces for padding. + * + * @return the number of effective characters in the last block. + */ + simdjson_inline size_t get_remainder(uint8_t* dst) const; + simdjson_inline void advance(); + private: + const uint8_t* buf; + const size_t len; + const size_t lenminusstep; + size_t idx; + }; + + // Routines to print masks and text for debugging bitmask operations + simdjson_unused static char* format_input_text_64(const uint8_t* text) { + static char buf[sizeof(simd8x64) + 1]; + for (size_t i = 0; i < sizeof(simd8x64); i++) { + buf[i] = int8_t(text[i]) < ' ' ? '_' : int8_t(text[i]); + } + buf[sizeof(simd8x64)] = '\0'; + return buf; + } + + // Routines to print masks and text for debugging bitmask operations + simdjson_unused static char* format_input_text(const simd8x64& in) { + static char buf[sizeof(simd8x64) + 1]; + in.store(reinterpret_cast(buf)); + for (size_t i = 0; i < sizeof(simd8x64); i++) { + if (buf[i] < ' ') { buf[i] = '_'; } + } + buf[sizeof(simd8x64)] = '\0'; + return buf; + } + + simdjson_unused static char* format_input_text(const simd8x64& in, uint64_t mask) { + static char buf[sizeof(simd8x64) + 1]; + in.store(reinterpret_cast(buf)); + for (size_t i = 0; i < sizeof(simd8x64); i++) { + if (buf[i] <= ' ') { buf[i] = '_'; } + if (!(mask & (size_t(1) << i))) { buf[i] = ' '; } + } + buf[sizeof(simd8x64)] = '\0'; + return buf; + } + + simdjson_unused static char* format_mask(uint64_t mask) { + static char buf[sizeof(simd8x64) + 1]; + for (size_t i = 0; i < 64; i++) { + buf[i] = (mask & (size_t(1) << i)) ? 'X' : ' '; + } + buf[64] = '\0'; + return buf; + } + + template + simdjson_inline buf_block_reader::buf_block_reader(const uint8_t* _buf, size_t _len) : buf{ _buf }, len{ _len }, lenminusstep{ len < STEP_SIZE ? 0 : len - STEP_SIZE }, idx{ 0 } {} + + template + simdjson_inline size_t buf_block_reader::block_index() { return idx; } + + template + simdjson_inline bool buf_block_reader::has_full_block() const { + return idx < lenminusstep; + } + + template + simdjson_inline const uint8_t* buf_block_reader::full_block() const { + return &buf[idx]; + } + + template + simdjson_inline size_t buf_block_reader::get_remainder(uint8_t* dst) const { + if (len == idx) { return 0; } // memcpy(dst, null, 0) will trigger an error with some sanitizers + std::memset(dst, 0x20, STEP_SIZE); // std::memset STEP_SIZE because it's more efficient to write out 8 or 16 bytes at once. + std::memcpy(dst, buf + idx, len - idx); + return len - idx; + } + + template + simdjson_inline void buf_block_reader::advance() { + idx += STEP_SIZE; + } + + } // namespace stage1 + } // unnamed namespace + } // namespace icelake +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE1_BUF_BLOCK_READER_H +/* end file generic/stage1/buf_block_reader.h for icelake */ +/* including generic/stage1/json_escape_scanner.h for icelake: #include */ +/* begin file generic/stage1/json_escape_scanner.h for icelake */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE1_JSON_ESCAPE_SCANNER_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_JSON_ESCAPE_SCANNER_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace icelake { + namespace { + namespace stage1 { + + /** + * Scans for escape characters in JSON, taking care with multiple backslashes (\\n vs. \n). + */ + struct json_escape_scanner { + /** The actual escape characters (the backslashes themselves). */ + uint64_t next_is_escaped = 0ULL; + + struct escaped_and_escape { + /** + * Mask of escaped characters. + * + * ``` + * \n \\n \\\n \\\\n \ + * 0100100010100101000 + * n \ \ n \ \ + * ``` + */ + uint64_t escaped; + /** + * Mask of escape characters. + * + * ``` + * \n \\n \\\n \\\\n \ + * 1001000101001010001 + * \ \ \ \ \ \ \ + * ``` + */ + uint64_t escape; + }; + + /** + * Get a mask of both escape and escaped characters (the characters following a backslash). + * + * @param potential_escape A mask of the character that can escape others (but could be + * escaped itself). e.g. block.eq('\\') + */ + simdjson_really_inline escaped_and_escape next(uint64_t backslash) noexcept { + +#if !SIMDJSON_SKIP_BACKSLASH_SHORT_CIRCUIT + if (!backslash) { return { next_escaped_without_backslashes(), 0 }; } +#endif + + // | | Mask (shows characters instead of 1's) | Depth | Instructions | + // |--------------------------------|----------------------------------------|-------|---------------------| + // | string | `\\n_\\\n___\\\n___\\\\___\\\\__\\\` | | | + // | | ` even odd even odd odd` | | | + // | potential_escape | ` \ \\\ \\\ \\\\ \\\\ \\\` | 1 | 1 (backslash & ~first_is_escaped) + // | escape_and_terminal_code | ` \n \ \n \ \n \ \ \ \ \ \` | 5 | 5 (next_escape_and_terminal_code()) + // | escaped | `\ \ n \ n \ \ \ \ \ ` X | 6 | 7 (escape_and_terminal_code ^ (potential_escape | first_is_escaped)) + // | escape | ` \ \ \ \ \ \ \ \ \ \` | 6 | 8 (escape_and_terminal_code & backslash) + // | first_is_escaped | `\ ` | 7 (*) | 9 (escape >> 63) () + // (*) this is not needed until the next iteration + uint64_t escape_and_terminal_code = next_escape_and_terminal_code(backslash & ~this->next_is_escaped); + uint64_t escaped = escape_and_terminal_code ^ (backslash | this->next_is_escaped); + uint64_t escape = escape_and_terminal_code & backslash; + this->next_is_escaped = escape >> 63; + return { escaped, escape }; + } + + private: + static constexpr const uint64_t ODD_BITS = 0xAAAAAAAAAAAAAAAAULL; + + simdjson_really_inline uint64_t next_escaped_without_backslashes() noexcept { + uint64_t escaped = this->next_is_escaped; + this->next_is_escaped = 0; + return escaped; + } + + /** + * Returns a mask of the next escape characters (masking out escaped backslashes), along with + * any non-backslash escape codes. + * + * \n \\n \\\n \\\\n returns: + * \n \ \ \n \ \ + * 11 100 1011 10100 + * + * You are expected to mask out the first bit yourself if the previous block had a trailing + * escape. + * + * & the result with potential_escape to get just the escape characters. + * ^ the result with (potential_escape | first_is_escaped) to get escaped characters. + */ + static simdjson_really_inline uint64_t next_escape_and_terminal_code(uint64_t potential_escape) noexcept { + // If we were to just shift and mask out any odd bits, we'd actually get a *half* right answer: + // any even-aligned backslash runs would be correct! Odd-aligned backslash runs would be + // inverted (\\\ would be 010 instead of 101). + // + // ``` + // string: | ____\\\\_\\\\_____ | + // maybe_escaped | ODD | \ \ \ \ | + // even-aligned ^^^ ^^^^ odd-aligned + // ``` + // + // Taking that into account, our basic strategy is: + // + // 1. Use subtraction to produce a mask with 1's for even-aligned runs and 0's for + // odd-aligned runs. + // 2. XOR all odd bits, which masks out the odd bits in even-aligned runs, and brings IN the + // odd bits in odd-aligned runs. + // 3. & with backslash to clean up any stray bits. + // runs are set to 0, and then XORing with "odd": + // + // | | Mask (shows characters instead of 1's) | Instructions | + // |--------------------------------|----------------------------------------|---------------------| + // | string | `\\n_\\\n___\\\n___\\\\___\\\\__\\\` | + // | | ` even odd even odd odd` | + // | maybe_escaped | ` n \\n \\n \\\_ \\\_ \\` X | 1 (potential_escape << 1) + // | maybe_escaped_and_odd | ` \n_ \\n _ \\\n_ _ \\\__ _\\\_ \\\` | 1 (maybe_escaped | odd) + // | even_series_codes_and_odd | ` n_\\\ _ n_ _\\\\ _ _ ` | 1 (maybe_escaped_and_odd - potential_escape) + // | escape_and_terminal_code | ` \n \ \n \ \n \ \ \ \ \ \` | 1 (^ odd) + // + + // Escaped characters are characters following an escape. + uint64_t maybe_escaped = potential_escape << 1; + + // To distinguish odd from even escape sequences, therefore, we turn on any *starting* + // escapes that are on an odd byte. (We actually bring in all odd bits, for speed.) + // - Odd runs of backslashes are 0000, and the code at the end ("n" in \n or \\n) is 1. + // - Odd runs of backslashes are 1111, and the code at the end ("n" in \n or \\n) is 0. + // - All other odd bytes are 1, and even bytes are 0. + uint64_t maybe_escaped_and_odd_bits = maybe_escaped | ODD_BITS; + uint64_t even_series_codes_and_odd_bits = maybe_escaped_and_odd_bits - potential_escape; + + // Now we flip all odd bytes back with xor. This: + // - Makes odd runs of backslashes go from 0000 to 1010 + // - Makes even runs of backslashes go from 1111 to 1010 + // - Sets actually-escaped codes to 1 (the n in \n and \\n: \n = 11, \\n = 100) + // - Resets all other bytes to 0 + return even_series_codes_and_odd_bits ^ ODD_BITS; + } + }; + + } // namespace stage1 + } // unnamed namespace + } // namespace icelake +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRING_SCANNER_H +/* end file generic/stage1/json_escape_scanner.h for icelake */ +/* including generic/stage1/json_string_scanner.h for icelake: #include */ +/* begin file generic/stage1/json_string_scanner.h for icelake */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRING_SCANNER_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRING_SCANNER_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace icelake { + namespace { + namespace stage1 { + + struct json_string_block { + // We spell out the constructors in the hope of resolving inlining issues with Visual Studio 2017 + simdjson_really_inline json_string_block(uint64_t escaped, uint64_t quote, uint64_t in_string) : + _escaped(escaped), _quote(quote), _in_string(in_string) {} + + // Escaped characters (characters following an escape() character) + simdjson_really_inline uint64_t escaped() const { return _escaped; } + // Real (non-backslashed) quotes + simdjson_really_inline uint64_t quote() const { return _quote; } + // Only characters inside the string (not including the quotes) + simdjson_really_inline uint64_t string_content() const { return _in_string & ~_quote; } + // Return a mask of whether the given characters are inside a string (only works on non-quotes) + simdjson_really_inline uint64_t non_quote_inside_string(uint64_t mask) const { return mask & _in_string; } + // Return a mask of whether the given characters are inside a string (only works on non-quotes) + simdjson_really_inline uint64_t non_quote_outside_string(uint64_t mask) const { return mask & ~_in_string; } + // Tail of string (everything except the start quote) + simdjson_really_inline uint64_t string_tail() const { return _in_string ^ _quote; } + + // escaped characters (backslashed--does not include the hex characters after \u) + uint64_t _escaped; + // real quotes (non-escaped ones) + uint64_t _quote; + // string characters (includes start quote but not end quote) + uint64_t _in_string; + }; + + // Scans blocks for string characters, storing the state necessary to do so + class json_string_scanner { + public: + simdjson_really_inline json_string_block next(const simd::simd8x64& in); + // Returns either UNCLOSED_STRING or SUCCESS + simdjson_really_inline error_code finish(); + + private: + // Scans for escape characters + json_escape_scanner escape_scanner{}; + // Whether the last iteration was still inside a string (all 1's = true, all 0's = false). + uint64_t prev_in_string = 0ULL; + }; + + // + // Return a mask of all string characters plus end quotes. + // + // prev_escaped is overflow saying whether the next character is escaped. + // prev_in_string is overflow saying whether we're still in a string. + // + // Backslash sequences outside of quotes will be detected in stage 2. + // + simdjson_really_inline json_string_block json_string_scanner::next(const simd::simd8x64& in) { + const uint64_t backslash = in.eq('\\'); + const uint64_t escaped = escape_scanner.next(backslash).escaped; + const uint64_t quote = in.eq('"') & ~escaped; + + // + // prefix_xor flips on bits inside the string (and flips off the end quote). + // + // Then we xor with prev_in_string: if we were in a string already, its effect is flipped + // (characters inside strings are outside, and characters outside strings are inside). + // + const uint64_t in_string = prefix_xor(quote) ^ prev_in_string; + + // + // Check if we're still in a string at the end of the box so the next block will know + // + prev_in_string = uint64_t(static_cast(in_string) >> 63); + + // Use ^ to turn the beginning quote off, and the end quote on. + + // We are returning a function-local object so either we get a move constructor + // or we get copy elision. + return json_string_block(escaped, quote, in_string); + } + + simdjson_really_inline error_code json_string_scanner::finish() { + if (prev_in_string) { + return UNCLOSED_STRING; + } + return SUCCESS; + } + + } // namespace stage1 + } // unnamed namespace + } // namespace icelake +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRING_SCANNER_H +/* end file generic/stage1/json_string_scanner.h for icelake */ +/* including generic/stage1/utf8_lookup4_algorithm.h for icelake: #include */ +/* begin file generic/stage1/utf8_lookup4_algorithm.h for icelake */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE1_UTF8_LOOKUP4_ALGORITHM_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_UTF8_LOOKUP4_ALGORITHM_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace icelake { + namespace { + namespace utf8_validation { + + using namespace simd; + + simdjson_inline simd8 check_special_cases(const simd8 input, const simd8 prev1) { + // Bit 0 = Too Short (lead byte/ASCII followed by lead byte/ASCII) + // Bit 1 = Too Long (ASCII followed by continuation) + // Bit 2 = Overlong 3-byte + // Bit 4 = Surrogate + // Bit 5 = Overlong 2-byte + // Bit 7 = Two Continuations + constexpr const uint8_t TOO_SHORT = 1 << 0; // 11______ 0_______ + // 11______ 11______ + constexpr const uint8_t TOO_LONG = 1 << 1; // 0_______ 10______ + constexpr const uint8_t OVERLONG_3 = 1 << 2; // 11100000 100_____ + constexpr const uint8_t SURROGATE = 1 << 4; // 11101101 101_____ + constexpr const uint8_t OVERLONG_2 = 1 << 5; // 1100000_ 10______ + constexpr const uint8_t TWO_CONTS = 1 << 7; // 10______ 10______ + constexpr const uint8_t TOO_LARGE = 1 << 3; // 11110100 1001____ + // 11110100 101_____ + // 11110101 1001____ + // 11110101 101_____ + // 1111011_ 1001____ + // 1111011_ 101_____ + // 11111___ 1001____ + // 11111___ 101_____ + constexpr const uint8_t TOO_LARGE_1000 = 1 << 6; + // 11110101 1000____ + // 1111011_ 1000____ + // 11111___ 1000____ + constexpr const uint8_t OVERLONG_4 = 1 << 6; // 11110000 1000____ + + const simd8 byte_1_high = prev1.shr<4>().lookup_16( + // 0_______ ________ + TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, + TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, + // 10______ ________ + TWO_CONTS, TWO_CONTS, TWO_CONTS, TWO_CONTS, + // 1100____ ________ + TOO_SHORT | OVERLONG_2, + // 1101____ ________ + TOO_SHORT, + // 1110____ ________ + TOO_SHORT | OVERLONG_3 | SURROGATE, + // 1111____ ________ + TOO_SHORT | TOO_LARGE | TOO_LARGE_1000 | OVERLONG_4 + ); + constexpr const uint8_t CARRY = TOO_SHORT | TOO_LONG | TWO_CONTS; // These all have ____ in byte 1 . + const simd8 byte_1_low = (prev1 & 0x0F).lookup_16( + // ____0000 ________ + CARRY | OVERLONG_3 | OVERLONG_2 | OVERLONG_4, + // ____0001 ________ + CARRY | OVERLONG_2, + // ____001_ ________ + CARRY, + CARRY, + + // ____0100 ________ + CARRY | TOO_LARGE, + // ____0101 ________ + CARRY | TOO_LARGE | TOO_LARGE_1000, + // ____011_ ________ + CARRY | TOO_LARGE | TOO_LARGE_1000, + CARRY | TOO_LARGE | TOO_LARGE_1000, + + // ____1___ ________ + CARRY | TOO_LARGE | TOO_LARGE_1000, + CARRY | TOO_LARGE | TOO_LARGE_1000, + CARRY | TOO_LARGE | TOO_LARGE_1000, + CARRY | TOO_LARGE | TOO_LARGE_1000, + CARRY | TOO_LARGE | TOO_LARGE_1000, + // ____1101 ________ + CARRY | TOO_LARGE | TOO_LARGE_1000 | SURROGATE, + CARRY | TOO_LARGE | TOO_LARGE_1000, + CARRY | TOO_LARGE | TOO_LARGE_1000 + ); + const simd8 byte_2_high = input.shr<4>().lookup_16( + // ________ 0_______ + TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT, + TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT, + + // ________ 1000____ + TOO_LONG | OVERLONG_2 | TWO_CONTS | OVERLONG_3 | TOO_LARGE_1000 | OVERLONG_4, + // ________ 1001____ + TOO_LONG | OVERLONG_2 | TWO_CONTS | OVERLONG_3 | TOO_LARGE, + // ________ 101_____ + TOO_LONG | OVERLONG_2 | TWO_CONTS | SURROGATE | TOO_LARGE, + TOO_LONG | OVERLONG_2 | TWO_CONTS | SURROGATE | TOO_LARGE, + + // ________ 11______ + TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT + ); + return (byte_1_high & byte_1_low & byte_2_high); + } + simdjson_inline simd8 check_multibyte_lengths(const simd8 input, + const simd8 prev_input, const simd8 sc) { + simd8 prev2 = input.prev<2>(prev_input); + simd8 prev3 = input.prev<3>(prev_input); + simd8 must23 = must_be_2_3_continuation(prev2, prev3); + simd8 must23_80 = must23 & uint8_t(0x80); + return must23_80 ^ sc; + } + + // + // Return nonzero if there are incomplete multibyte characters at the end of the block: + // e.g. if there is a 4-byte character, but it's 3 bytes from the end. + // + simdjson_inline simd8 is_incomplete(const simd8 input) { + // If the previous input's last 3 bytes match this, they're too short (they ended at EOF): + // ... 1111____ 111_____ 11______ +#if SIMDJSON_IMPLEMENTATION_ICELAKE + static const uint8_t max_array[64] = { + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 0xf0u - 1, 0xe0u - 1, 0xc0u - 1 + }; +#else + static const uint8_t max_array[32] = { + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 0xf0u - 1, 0xe0u - 1, 0xc0u - 1 + }; +#endif + const simd8 max_value(&max_array[sizeof(max_array) - sizeof(simd8)]); + return input.gt_bits(max_value); + } + + struct utf8_checker { + // If this is nonzero, there has been a UTF-8 error. + simd8 error; + // The last input we received + simd8 prev_input_block; + // Whether the last input we received was incomplete (used for ASCII fast path) + simd8 prev_incomplete; + + // + // Check whether the current bytes are valid UTF-8. + // + simdjson_inline void check_utf8_bytes(const simd8 input, const simd8 prev_input) { + // Flip prev1...prev3 so we can easily determine if they are 2+, 3+ or 4+ lead bytes + // (2, 3, 4-byte leads become large positive numbers instead of small negative numbers) + simd8 prev1 = input.prev<1>(prev_input); + simd8 sc = check_special_cases(input, prev1); + this->error |= check_multibyte_lengths(input, prev_input, sc); + } + + // The only problem that can happen at EOF is that a multibyte character is too short + // or a byte value too large in the last bytes: check_special_cases only checks for bytes + // too large in the first of two bytes. + simdjson_inline void check_eof() { + // If the previous block had incomplete UTF-8 characters at the end, an ASCII block can't + // possibly finish them. + this->error |= this->prev_incomplete; + } + + simdjson_inline void check_next_input(const simd8x64& input) { + if (simdjson_likely(is_ascii(input))) { + this->error |= this->prev_incomplete; + } + else { + // you might think that a for-loop would work, but under Visual Studio, it is not good enough. + static_assert((simd8x64::NUM_CHUNKS == 1) + || (simd8x64::NUM_CHUNKS == 2) + || (simd8x64::NUM_CHUNKS == 4), + "We support one, two or four chunks per 64-byte block."); + SIMDJSON_IF_CONSTEXPR(simd8x64::NUM_CHUNKS == 1) { + this->check_utf8_bytes(input.chunks[0], this->prev_input_block); + } +else SIMDJSON_IF_CONSTEXPR(simd8x64::NUM_CHUNKS == 2) { + this->check_utf8_bytes(input.chunks[0], this->prev_input_block); + this->check_utf8_bytes(input.chunks[1], input.chunks[0]); + } + else SIMDJSON_IF_CONSTEXPR(simd8x64::NUM_CHUNKS == 4) { + this->check_utf8_bytes(input.chunks[0], this->prev_input_block); + this->check_utf8_bytes(input.chunks[1], input.chunks[0]); + this->check_utf8_bytes(input.chunks[2], input.chunks[1]); + this->check_utf8_bytes(input.chunks[3], input.chunks[2]); + } + this->prev_incomplete = is_incomplete(input.chunks[simd8x64::NUM_CHUNKS - 1]); + this->prev_input_block = input.chunks[simd8x64::NUM_CHUNKS - 1]; + } + } + // do not forget to call check_eof! + simdjson_inline error_code errors() { + return this->error.any_bits_set_anywhere() ? error_code::UTF8_ERROR : error_code::SUCCESS; + } + + }; // struct utf8_checker + } // namespace utf8_validation + + } // unnamed namespace + } // namespace icelake +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE1_UTF8_LOOKUP4_ALGORITHM_H +/* end file generic/stage1/utf8_lookup4_algorithm.h for icelake */ +/* including generic/stage1/json_scanner.h for icelake: #include */ +/* begin file generic/stage1/json_scanner.h for icelake */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE1_JSON_SCANNER_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_JSON_SCANNER_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace icelake { + namespace { + namespace stage1 { + + /** + * A block of scanned json, with information on operators and scalars. + * + * We seek to identify pseudo-structural characters. Anything that is inside + * a string must be omitted (hence & ~_string.string_tail()). + * Otherwise, pseudo-structural characters come in two forms. + * 1. We have the structural characters ([,],{,},:, comma). The + * term 'structural character' is from the JSON RFC. + * 2. We have the 'scalar pseudo-structural characters'. + * Scalars are quotes, and any character except structural characters and white space. + * + * To identify the scalar pseudo-structural characters, we must look at what comes + * before them: it must be a space, a quote or a structural characters. + * Starting with simdjson v0.3, we identify them by + * negation: we identify everything that is followed by a non-quote scalar, + * and we negate that. Whatever remains must be a 'scalar pseudo-structural character'. + */ + struct json_block { + public: + // We spell out the constructors in the hope of resolving inlining issues with Visual Studio 2017 + simdjson_inline json_block(json_string_block&& string, json_character_block characters, uint64_t follows_potential_nonquote_scalar) : + _string(std::move(string)), _characters(characters), _follows_potential_nonquote_scalar(follows_potential_nonquote_scalar) {} + simdjson_inline json_block(json_string_block string, json_character_block characters, uint64_t follows_potential_nonquote_scalar) : + _string(string), _characters(characters), _follows_potential_nonquote_scalar(follows_potential_nonquote_scalar) {} + + /** + * The start of structurals. + * In simdjson prior to v0.3, these were called the pseudo-structural characters. + **/ + simdjson_inline uint64_t structural_start() const noexcept { return potential_structural_start() & ~_string.string_tail(); } + /** All JSON whitespace (i.e. not in a string) */ + simdjson_inline uint64_t whitespace() const noexcept { return non_quote_outside_string(_characters.whitespace()); } + + // Helpers + + /** Whether the given characters are inside a string (only works on non-quotes) */ + simdjson_inline uint64_t non_quote_inside_string(uint64_t mask) const noexcept { return _string.non_quote_inside_string(mask); } + /** Whether the given characters are outside a string (only works on non-quotes) */ + simdjson_inline uint64_t non_quote_outside_string(uint64_t mask) const noexcept { return _string.non_quote_outside_string(mask); } + + // string and escape characters + json_string_block _string; + // whitespace, structural characters ('operators'), scalars + json_character_block _characters; + // whether the previous character was a scalar + uint64_t _follows_potential_nonquote_scalar; + private: + // Potential structurals (i.e. disregarding strings) + + /** + * structural elements ([,],{,},:, comma) plus scalar starts like 123, true and "abc". + * They may reside inside a string. + **/ + simdjson_inline uint64_t potential_structural_start() const noexcept { return _characters.op() | potential_scalar_start(); } + /** + * The start of non-operator runs, like 123, true and "abc". + * It main reside inside a string. + **/ + simdjson_inline uint64_t potential_scalar_start() const noexcept { + // The term "scalar" refers to anything except structural characters and white space + // (so letters, numbers, quotes). + // Whenever it is preceded by something that is not a structural element ({,},[,],:, ") nor a white-space + // then we know that it is irrelevant structurally. + return _characters.scalar() & ~follows_potential_scalar(); + } + /** + * Whether the given character is immediately after a non-operator like 123, true. + * The characters following a quote are not included. + */ + simdjson_inline uint64_t follows_potential_scalar() const noexcept { + // _follows_potential_nonquote_scalar: is defined as marking any character that follows a character + // that is not a structural element ({,},[,],:, comma) nor a quote (") and that is not a + // white space. + // It is understood that within quoted region, anything at all could be marked (irrelevant). + return _follows_potential_nonquote_scalar; + } + }; + + /** + * Scans JSON for important bits: structural characters or 'operators', strings, and scalars. + * + * The scanner starts by calculating two distinct things: + * - string characters (taking \" into account) + * - structural characters or 'operators' ([]{},:, comma) + * and scalars (runs of non-operators like 123, true and "abc") + * + * To minimize data dependency (a key component of the scanner's speed), it finds these in parallel: + * in particular, the operator/scalar bit will find plenty of things that are actually part of + * strings. When we're done, json_block will fuse the two together by masking out tokens that are + * part of a string. + */ + class json_scanner { + public: + json_scanner() = default; + simdjson_inline json_block next(const simd::simd8x64& in); + // Returns either UNCLOSED_STRING or SUCCESS + simdjson_inline error_code finish(); + + private: + // Whether the last character of the previous iteration is part of a scalar token + // (anything except whitespace or a structural character/'operator'). + uint64_t prev_scalar = 0ULL; + json_string_scanner string_scanner{}; + }; + + + // + // Check if the current character immediately follows a matching character. + // + // For example, this checks for quotes with backslashes in front of them: + // + // const uint64_t backslashed_quote = in.eq('"') & immediately_follows(in.eq('\'), prev_backslash); + // + simdjson_inline uint64_t follows(const uint64_t match, uint64_t& overflow) { + const uint64_t result = match << 1 | overflow; + overflow = match >> 63; + return result; + } + + simdjson_inline json_block json_scanner::next(const simd::simd8x64& in) { + json_string_block strings = string_scanner.next(in); + // identifies the white-space and the structural characters + json_character_block characters = json_character_block::classify(in); + // The term "scalar" refers to anything except structural characters and white space + // (so letters, numbers, quotes). + // We want follows_scalar to mark anything that follows a non-quote scalar (so letters and numbers). + // + // A terminal quote should either be followed by a structural character (comma, brace, bracket, colon) + // or nothing. However, we still want ' "a string"true ' to mark the 't' of 'true' as a potential + // pseudo-structural character just like we would if we had ' "a string" true '; otherwise we + // may need to add an extra check when parsing strings. + // + // Performance: there are many ways to skin this cat. + const uint64_t nonquote_scalar = characters.scalar() & ~strings.quote(); + uint64_t follows_nonquote_scalar = follows(nonquote_scalar, prev_scalar); + // We are returning a function-local object so either we get a move constructor + // or we get copy elision. + return json_block( + strings,// strings is a function-local object so either it moves or the copy is elided. + characters, + follows_nonquote_scalar + ); + } + + simdjson_inline error_code json_scanner::finish() { + return string_scanner.finish(); + } + + } // namespace stage1 + } // unnamed namespace + } // namespace icelake +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE1_JSON_SCANNER_H +/* end file generic/stage1/json_scanner.h for icelake */ + +// All other declarations +/* including generic/stage1/find_next_document_index.h for icelake: #include */ +/* begin file generic/stage1/find_next_document_index.h for icelake */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE1_FIND_NEXT_DOCUMENT_INDEX_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_FIND_NEXT_DOCUMENT_INDEX_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace icelake { + namespace { + namespace stage1 { + + /** + * This algorithm is used to quickly identify the last structural position that + * makes up a complete document. + * + * It does this by going backwards and finding the last *document boundary* (a + * place where one value follows another without a comma between them). If the + * last document (the characters after the boundary) has an equal number of + * start and end brackets, it is considered complete. + * + * Simply put, we iterate over the structural characters, starting from + * the end. We consider that we found the end of a JSON document when the + * first element of the pair is NOT one of these characters: '{' '[' ':' ',' + * and when the second element is NOT one of these characters: '}' ']' ':' ','. + * + * This simple comparison works most of the time, but it does not cover cases + * where the batch's structural indexes contain a perfect amount of documents. + * In such a case, we do not have access to the structural index which follows + * the last document, therefore, we do not have access to the second element in + * the pair, and that means we cannot identify the last document. To fix this + * issue, we keep a count of the open and closed curly/square braces we found + * while searching for the pair. When we find a pair AND the count of open and + * closed curly/square braces is the same, we know that we just passed a + * complete document, therefore the last json buffer location is the end of the + * batch. + */ + simdjson_inline uint32_t find_next_document_index(dom_parser_implementation& parser) { + // Variant: do not count separately, just figure out depth + if (parser.n_structural_indexes == 0) { return 0; } + auto arr_cnt = 0; + auto obj_cnt = 0; + for (auto i = parser.n_structural_indexes - 1; i > 0; i--) { + auto idxb = parser.structural_indexes[i]; + switch (parser.buf[idxb]) { + case ':': + case ',': + continue; + case '}': + obj_cnt--; + continue; + case ']': + arr_cnt--; + continue; + case '{': + obj_cnt++; + break; + case '[': + arr_cnt++; + break; + } + auto idxa = parser.structural_indexes[i - 1]; + switch (parser.buf[idxa]) { + case '{': + case '[': + case ':': + case ',': + continue; + } + // Last document is complete, so the next document will appear after! + if (!arr_cnt && !obj_cnt) { + return parser.n_structural_indexes; + } + // Last document is incomplete; mark the document at i + 1 as the next one + return i; + } + // If we made it to the end, we want to finish counting to see if we have a full document. + switch (parser.buf[parser.structural_indexes[0]]) { + case '}': + obj_cnt--; + break; + case ']': + arr_cnt--; + break; + case '{': + obj_cnt++; + break; + case '[': + arr_cnt++; + break; + } + if (!arr_cnt && !obj_cnt) { + // We have a complete document. + return parser.n_structural_indexes; + } + return 0; + } + + } // namespace stage1 + } // unnamed namespace + } // namespace icelake +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE1_FIND_NEXT_DOCUMENT_INDEX_H +/* end file generic/stage1/find_next_document_index.h for icelake */ +/* including generic/stage1/json_minifier.h for icelake: #include */ +/* begin file generic/stage1/json_minifier.h for icelake */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE1_JSON_MINIFIER_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_JSON_MINIFIER_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +// This file contains the common code every implementation uses in stage1 +// It is intended to be included multiple times and compiled multiple times +// We assume the file in which it is included already includes +// "simdjson/stage1.h" (this simplifies amalgation) + +namespace simdjson { + namespace icelake { + namespace { + namespace stage1 { + + class json_minifier { + public: + template + static error_code minify(const uint8_t* buf, size_t len, uint8_t* dst, size_t& dst_len) noexcept; + + private: + simdjson_inline json_minifier(uint8_t* _dst) + : dst{ _dst } + {} + template + simdjson_inline void step(const uint8_t* block_buf, buf_block_reader& reader) noexcept; + simdjson_inline void next(const simd::simd8x64& in, const json_block& block); + simdjson_inline error_code finish(uint8_t* dst_start, size_t& dst_len); + json_scanner scanner{}; + uint8_t* dst; + }; + + simdjson_inline void json_minifier::next(const simd::simd8x64& in, const json_block& block) { + uint64_t mask = block.whitespace(); + dst += in.compress(mask, dst); + } + + simdjson_inline error_code json_minifier::finish(uint8_t* dst_start, size_t& dst_len) { + error_code error = scanner.finish(); + if (error) { dst_len = 0; return error; } + dst_len = dst - dst_start; + return SUCCESS; + } + + template<> + simdjson_inline void json_minifier::step<128>(const uint8_t* block_buf, buf_block_reader<128>& reader) noexcept { + simd::simd8x64 in_1(block_buf); + simd::simd8x64 in_2(block_buf + 64); + json_block block_1 = scanner.next(in_1); + json_block block_2 = scanner.next(in_2); + this->next(in_1, block_1); + this->next(in_2, block_2); + reader.advance(); + } + + template<> + simdjson_inline void json_minifier::step<64>(const uint8_t* block_buf, buf_block_reader<64>& reader) noexcept { + simd::simd8x64 in_1(block_buf); + json_block block_1 = scanner.next(in_1); + this->next(block_buf, block_1); + reader.advance(); + } + + template + error_code json_minifier::minify(const uint8_t* buf, size_t len, uint8_t* dst, size_t& dst_len) noexcept { + buf_block_reader reader(buf, len); + json_minifier minifier(dst); + + // Index the first n-1 blocks + while (reader.has_full_block()) { + minifier.step(reader.full_block(), reader); + } + + // Index the last (remainder) block, padded with spaces + uint8_t block[STEP_SIZE]; + size_t remaining_bytes = reader.get_remainder(block); + if (remaining_bytes > 0) { + // We do not want to write directly to the output stream. Rather, we write + // to a local buffer (for safety). + uint8_t out_block[STEP_SIZE]; + uint8_t* const guarded_dst{ minifier.dst }; + minifier.dst = out_block; + minifier.step(block, reader); + size_t to_write = minifier.dst - out_block; + // In some cases, we could be enticed to consider the padded spaces + // as part of the string. This is fine as long as we do not write more + // than we consumed. + if (to_write > remaining_bytes) { to_write = remaining_bytes; } + memcpy(guarded_dst, out_block, to_write); + minifier.dst = guarded_dst + to_write; + } + return minifier.finish(dst, dst_len); + } + + } // namespace stage1 + } // unnamed namespace + } // namespace icelake +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE1_JSON_MINIFIER_H +/* end file generic/stage1/json_minifier.h for icelake */ +/* including generic/stage1/json_structural_indexer.h for icelake: #include */ +/* begin file generic/stage1/json_structural_indexer.h for icelake */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRUCTURAL_INDEXER_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRUCTURAL_INDEXER_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +// This file contains the common code every implementation uses in stage1 +// It is intended to be included multiple times and compiled multiple times +// We assume the file in which it is included already includes +// "simdjson/stage1.h" (this simplifies amalgation) + +namespace simdjson { + namespace icelake { + namespace { + namespace stage1 { + + class bit_indexer { + public: + uint32_t* tail; + + simdjson_inline bit_indexer(uint32_t* index_buf) : tail(index_buf) {} + +#if SIMDJSON_PREFER_REVERSE_BITS + /** + * ARM lacks a fast trailing zero instruction, but it has a fast + * bit reversal instruction and a fast leading zero instruction. + * Thus it may be profitable to reverse the bits (once) and then + * to rely on a sequence of instructions that call the leading + * zero instruction. + * + * Performance notes: + * The chosen routine is not optimal in terms of data dependency + * since zero_leading_bit might require two instructions. However, + * it tends to minimize the total number of instructions which is + * beneficial. + */ + simdjson_inline void write_index(uint32_t idx, uint64_t& rev_bits, int i) { + int lz = leading_zeroes(rev_bits); + this->tail[i] = static_cast(idx) + lz; + rev_bits = zero_leading_bit(rev_bits, lz); + } +#else + /** + * Under recent x64 systems, we often have both a fast trailing zero + * instruction and a fast 'clear-lower-bit' instruction so the following + * algorithm can be competitive. + */ + + simdjson_inline void write_index(uint32_t idx, uint64_t& bits, int i) { + this->tail[i] = idx + trailing_zeroes(bits); + bits = clear_lowest_bit(bits); + } +#endif // SIMDJSON_PREFER_REVERSE_BITS + + template + simdjson_inline int write_indexes(uint32_t idx, uint64_t& bits) { + write_index(idx, bits, START); + SIMDJSON_IF_CONSTEXPR(N > 1) { + write_indexes<(N - 1 > 0 ? START + 1 : START), (N - 1 >= 0 ? N - 1 : 1)>(idx, bits); + } + return START + N; + } + + template + simdjson_inline int write_indexes_stepped(uint32_t idx, uint64_t& bits, int cnt) { + write_indexes(idx, bits); + SIMDJSON_IF_CONSTEXPR((START + STEP) < END) { + if (simdjson_unlikely((START + STEP) < cnt)) { + write_indexes_stepped<(START + STEP < END ? START + STEP : END), END, STEP>(idx, bits, cnt); + } + } + return ((END - START) % STEP) == 0 ? END : (END - START) - ((END - START) % STEP) + STEP; + } + + // flatten out values in 'bits' assuming that they are are to have values of idx + // plus their position in the bitvector, and store these indexes at + // base_ptr[base] incrementing base as we go + // will potentially store extra values beyond end of valid bits, so base_ptr + // needs to be large enough to handle this + // + // If the kernel sets SIMDJSON_GENERIC_JSON_STRUCTURAL_INDEXER_CUSTOM_BIT_INDEXER, then it + // will provide its own version of the code. +#ifdef SIMDJSON_GENERIC_JSON_STRUCTURAL_INDEXER_CUSTOM_BIT_INDEXER + simdjson_inline void write(uint32_t idx, uint64_t bits); +#else + simdjson_inline void write(uint32_t idx, uint64_t bits) { + // In some instances, the next branch is expensive because it is mispredicted. + // Unfortunately, in other cases, + // it helps tremendously. + if (bits == 0) + return; + + int cnt = static_cast(count_ones(bits)); + +#if SIMDJSON_PREFER_REVERSE_BITS + bits = reverse_bits(bits); +#endif +#ifdef SIMDJSON_STRUCTURAL_INDEXER_STEP + static constexpr const int STEP = SIMDJSON_STRUCTURAL_INDEXER_STEP; +#else + static constexpr const int STEP = 4; +#endif + static constexpr const int STEP_UNTIL = 24; + + write_indexes_stepped<0, STEP_UNTIL, STEP>(idx, bits, cnt); + SIMDJSON_IF_CONSTEXPR(STEP_UNTIL < 64) { + if (simdjson_unlikely(STEP_UNTIL < cnt)) { + for (int i = STEP_UNTIL; i < cnt; i++) { + write_index(idx, bits, i); + } + } + } + + this->tail += cnt; + } +#endif // SIMDJSON_GENERIC_JSON_STRUCTURAL_INDEXER_CUSTOM_BIT_INDEXER + + }; + + class json_structural_indexer { + public: + /** + * Find the important bits of JSON in a 128-byte chunk, and add them to structural_indexes. + * + * @param partial Setting the partial parameter to true allows the find_structural_bits to + * tolerate unclosed strings. The caller should still ensure that the input is valid UTF-8. If + * you are processing substrings, you may want to call on a function like trimmed_length_safe_utf8. + */ + template + static error_code index(const uint8_t* buf, size_t len, dom_parser_implementation& parser, stage1_mode partial) noexcept; + + private: + simdjson_inline json_structural_indexer(uint32_t* structural_indexes); + template + simdjson_inline void step(const uint8_t* block, buf_block_reader& reader) noexcept; + simdjson_inline void next(const simd::simd8x64& in, const json_block& block, size_t idx); + simdjson_inline error_code finish(dom_parser_implementation& parser, size_t idx, size_t len, stage1_mode partial); + + json_scanner scanner{}; + utf8_checker checker{}; + bit_indexer indexer; + uint64_t prev_structurals = 0; + uint64_t unescaped_chars_error = 0; + }; + + simdjson_inline json_structural_indexer::json_structural_indexer(uint32_t* structural_indexes) : indexer{ structural_indexes } {} + + // Skip the last character if it is partial + simdjson_inline size_t trim_partial_utf8(const uint8_t* buf, size_t len) { + if (simdjson_unlikely(len < 3)) { + switch (len) { + case 2: + if (buf[len - 1] >= 0xc0) { return len - 1; } // 2-, 3- and 4-byte characters with only 1 byte left + if (buf[len - 2] >= 0xe0) { return len - 2; } // 3- and 4-byte characters with only 2 bytes left + return len; + case 1: + if (buf[len - 1] >= 0xc0) { return len - 1; } // 2-, 3- and 4-byte characters with only 1 byte left + return len; + case 0: + return len; + } + } + if (buf[len - 1] >= 0xc0) { return len - 1; } // 2-, 3- and 4-byte characters with only 1 byte left + if (buf[len - 2] >= 0xe0) { return len - 2; } // 3- and 4-byte characters with only 1 byte left + if (buf[len - 3] >= 0xf0) { return len - 3; } // 4-byte characters with only 3 bytes left + return len; + } + + // + // PERF NOTES: + // We pipe 2 inputs through these stages: + // 1. Load JSON into registers. This takes a long time and is highly parallelizable, so we load + // 2 inputs' worth at once so that by the time step 2 is looking for them input, it's available. + // 2. Scan the JSON for critical data: strings, scalars and operators. This is the critical path. + // The output of step 1 depends entirely on this information. These functions don't quite use + // up enough CPU: the second half of the functions is highly serial, only using 1 execution core + // at a time. The second input's scans has some dependency on the first ones finishing it, but + // they can make a lot of progress before they need that information. + // 3. Step 1 does not use enough capacity, so we run some extra stuff while we're waiting for that + // to finish: utf-8 checks and generating the output from the last iteration. + // + // The reason we run 2 inputs at a time, is steps 2 and 3 are *still* not enough to soak up all + // available capacity with just one input. Running 2 at a time seems to give the CPU a good enough + // workout. + // + template + error_code json_structural_indexer::index(const uint8_t* buf, size_t len, dom_parser_implementation& parser, stage1_mode partial) noexcept { + if (simdjson_unlikely(len > parser.capacity())) { return CAPACITY; } + // We guard the rest of the code so that we can assume that len > 0 throughout. + if (len == 0) { return EMPTY; } + if (is_streaming(partial)) { + len = trim_partial_utf8(buf, len); + // If you end up with an empty window after trimming + // the partial UTF-8 bytes, then chances are good that you + // have an UTF-8 formatting error. + if (len == 0) { return UTF8_ERROR; } + } + buf_block_reader reader(buf, len); + json_structural_indexer indexer(parser.structural_indexes.get()); + + // Read all but the last block + while (reader.has_full_block()) { + indexer.step(reader.full_block(), reader); + } + // Take care of the last block (will always be there unless file is empty which is + // not supposed to happen.) + uint8_t block[STEP_SIZE]; + if (simdjson_unlikely(reader.get_remainder(block) == 0)) { return UNEXPECTED_ERROR; } + indexer.step(block, reader); + return indexer.finish(parser, reader.block_index(), len, partial); + } + + template<> + simdjson_inline void json_structural_indexer::step<128>(const uint8_t* block, buf_block_reader<128>& reader) noexcept { + simd::simd8x64 in_1(block); + simd::simd8x64 in_2(block + 64); + json_block block_1 = scanner.next(in_1); + json_block block_2 = scanner.next(in_2); + this->next(in_1, block_1, reader.block_index()); + this->next(in_2, block_2, reader.block_index() + 64); + reader.advance(); + } + + template<> + simdjson_inline void json_structural_indexer::step<64>(const uint8_t* block, buf_block_reader<64>& reader) noexcept { + simd::simd8x64 in_1(block); + json_block block_1 = scanner.next(in_1); + this->next(in_1, block_1, reader.block_index()); + reader.advance(); + } + + simdjson_inline void json_structural_indexer::next(const simd::simd8x64& in, const json_block& block, size_t idx) { + uint64_t unescaped = in.lteq(0x1F); +#if SIMDJSON_UTF8VALIDATION + checker.check_next_input(in); +#endif + indexer.write(uint32_t(idx - 64), prev_structurals); // Output *last* iteration's structurals to the parser + prev_structurals = block.structural_start(); + unescaped_chars_error |= block.non_quote_inside_string(unescaped); + } + + simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementation& parser, size_t idx, size_t len, stage1_mode partial) { + // Write out the final iteration's structurals + indexer.write(uint32_t(idx - 64), prev_structurals); + error_code error = scanner.finish(); + // We deliberately break down the next expression so that it is + // human readable. + const bool should_we_exit = is_streaming(partial) ? + ((error != SUCCESS) && (error != UNCLOSED_STRING)) // when partial we tolerate UNCLOSED_STRING + : (error != SUCCESS); // if partial is false, we must have SUCCESS + const bool have_unclosed_string = (error == UNCLOSED_STRING); + if (simdjson_unlikely(should_we_exit)) { return error; } + + if (unescaped_chars_error) { + return UNESCAPED_CHARS; + } + parser.n_structural_indexes = uint32_t(indexer.tail - parser.structural_indexes.get()); + /*** + * The On-Demand API requires special padding. + * + * This is related to https://github.com/simdjson/simdjson/issues/906 + * Basically, we want to make sure that if the parsing continues beyond the last (valid) + * structural character, it quickly stops. + * Only three structural characters can be repeated without triggering an error in JSON: [,] and }. + * We repeat the padding character (at 'len'). We don't know what it is, but if the parsing + * continues, then it must be [,] or }. + * Suppose it is ] or }. We backtrack to the first character, what could it be that would + * not trigger an error? It could be ] or } but no, because you can't start a document that way. + * It can't be a comma, a colon or any simple value. So the only way we could continue is + * if the repeated character is [. But if so, the document must start with [. But if the document + * starts with [, it should end with ]. If we enforce that rule, then we would get + * ][[ which is invalid. + * + * This is illustrated with the test array_iterate_unclosed_error() on the following input: + * R"({ "a": [,,)" + **/ + parser.structural_indexes[parser.n_structural_indexes] = uint32_t(len); // used later in partial == stage1_mode::streaming_final + parser.structural_indexes[parser.n_structural_indexes + 1] = uint32_t(len); + parser.structural_indexes[parser.n_structural_indexes + 2] = 0; + parser.next_structural_index = 0; + // a valid JSON file cannot have zero structural indexes - we should have found something + if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { + return EMPTY; + } + if (simdjson_unlikely(parser.structural_indexes[parser.n_structural_indexes - 1] > len)) { + return UNEXPECTED_ERROR; + } + if (partial == stage1_mode::streaming_partial) { + // If we have an unclosed string, then the last structural + // will be the quote and we want to make sure to omit it. + if (have_unclosed_string) { + parser.n_structural_indexes--; + // a valid JSON file cannot have zero structural indexes - we should have found something + if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return CAPACITY; } + } + // We truncate the input to the end of the last complete document (or zero). + auto new_structural_indexes = find_next_document_index(parser); + if (new_structural_indexes == 0 && parser.n_structural_indexes > 0) { + if (parser.structural_indexes[0] == 0) { + // If the buffer is partial and we started at index 0 but the document is + // incomplete, it's too big to parse. + return CAPACITY; + } + else { + // It is possible that the document could be parsed, we just had a lot + // of white space. + parser.n_structural_indexes = 0; + return EMPTY; + } + } + + parser.n_structural_indexes = new_structural_indexes; + } + else if (partial == stage1_mode::streaming_final) { + if (have_unclosed_string) { parser.n_structural_indexes--; } + // We truncate the input to the end of the last complete document (or zero). + // Because partial == stage1_mode::streaming_final, it means that we may + // silently ignore trailing garbage. Though it sounds bad, we do it + // deliberately because many people who have streams of JSON documents + // will truncate them for processing. E.g., imagine that you are uncompressing + // the data from a size file or receiving it in chunks from the network. You + // may not know where exactly the last document will be. Meanwhile the + // document_stream instances allow people to know the JSON documents they are + // parsing (see the iterator.source() method). + parser.n_structural_indexes = find_next_document_index(parser); + // We store the initial n_structural_indexes so that the client can see + // whether we used truncation. If initial_n_structural_indexes == parser.n_structural_indexes, + // then this will query parser.structural_indexes[parser.n_structural_indexes] which is len, + // otherwise, it will copy some prior index. + parser.structural_indexes[parser.n_structural_indexes + 1] = parser.structural_indexes[parser.n_structural_indexes]; + // This next line is critical, do not change it unless you understand what you are + // doing. + parser.structural_indexes[parser.n_structural_indexes] = uint32_t(len); + if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { + // We tolerate an unclosed string at the very end of the stream. Indeed, users + // often load their data in bulk without being careful and they want us to ignore + // the trailing garbage. + return EMPTY; + } + } + checker.check_eof(); + return checker.errors(); + } + + } // namespace stage1 + } // unnamed namespace + } // namespace icelake +} // namespace simdjson + +// Clear CUSTOM_BIT_INDEXER so other implementations can set it if they need to. +#undef SIMDJSON_GENERIC_JSON_STRUCTURAL_INDEXER_CUSTOM_BIT_INDEXER + +#endif // SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRUCTURAL_INDEXER_H +/* end file generic/stage1/json_structural_indexer.h for icelake */ +/* including generic/stage1/utf8_validator.h for icelake: #include */ +/* begin file generic/stage1/utf8_validator.h for icelake */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE1_UTF8_VALIDATOR_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_UTF8_VALIDATOR_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace icelake { + namespace { + namespace stage1 { + + /** + * Validates that the string is actual UTF-8. + */ + template + bool generic_validate_utf8(const uint8_t* input, size_t length) { + checker c{}; + buf_block_reader<64> reader(input, length); + while (reader.has_full_block()) { + simd::simd8x64 in(reader.full_block()); + c.check_next_input(in); + reader.advance(); + } + uint8_t block[64]{}; + reader.get_remainder(block); + simd::simd8x64 in(block); + c.check_next_input(in); + reader.advance(); + c.check_eof(); + return c.errors() == error_code::SUCCESS; + } + + bool generic_validate_utf8(const char* input, size_t length) { + return generic_validate_utf8(reinterpret_cast(input), length); + } + + } // namespace stage1 + } // unnamed namespace + } // namespace icelake +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE1_UTF8_VALIDATOR_H +/* end file generic/stage1/utf8_validator.h for icelake */ +/* end file generic/stage1/amalgamated.h for icelake */ +/* including generic/stage2/amalgamated.h for icelake: #include */ +/* begin file generic/stage2/amalgamated.h for icelake */ +// Stuff other things depend on +/* including generic/stage2/base.h for icelake: #include */ +/* begin file generic/stage2/base.h for icelake */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE2_BASE_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_BASE_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace icelake { + namespace { + namespace stage2 { + + class json_iterator; + class structural_iterator; + struct tape_builder; + struct tape_writer; + + } // namespace stage2 + } // unnamed namespace + } // namespace icelake +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE2_BASE_H +/* end file generic/stage2/base.h for icelake */ +/* including generic/stage2/tape_writer.h for icelake: #include */ +/* begin file generic/stage2/tape_writer.h for icelake */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE2_TAPE_WRITER_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_TAPE_WRITER_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +#include + +namespace simdjson { + namespace icelake { + namespace { + namespace stage2 { + + struct tape_writer { + /** The next place to write to tape */ + uint64_t* next_tape_loc; + + /** Write a signed 64-bit value to tape. */ + simdjson_inline void append_s64(int64_t value) noexcept; + + /** Write an unsigned 64-bit value to tape. */ + simdjson_inline void append_u64(uint64_t value) noexcept; + + /** Write a double value to tape. */ + simdjson_inline void append_double(double value) noexcept; + + /** + * Append a tape entry (an 8-bit type,and 56 bits worth of value). + */ + simdjson_inline void append(uint64_t val, internal::tape_type t) noexcept; + + /** + * Skip the current tape entry without writing. + * + * Used to skip the start of the container, since we'll come back later to fill it in when the + * container ends. + */ + simdjson_inline void skip() noexcept; + + /** + * Skip the number of tape entries necessary to write a large u64 or i64. + */ + simdjson_inline void skip_large_integer() noexcept; + + /** + * Skip the number of tape entries necessary to write a double. + */ + simdjson_inline void skip_double() noexcept; + + /** + * Write a value to a known location on tape. + * + * Used to go back and write out the start of a container after the container ends. + */ + simdjson_inline static void write(uint64_t& tape_loc, uint64_t val, internal::tape_type t) noexcept; + + private: + /** + * Append both the tape entry, and a supplementary value following it. Used for types that need + * all 64 bits, such as double and uint64_t. + */ + template + simdjson_inline void append2(uint64_t val, T val2, internal::tape_type t) noexcept; + }; // struct tape_writer + + simdjson_inline void tape_writer::append_s64(int64_t value) noexcept { + append2(0, value, internal::tape_type::INT64); + } + + simdjson_inline void tape_writer::append_u64(uint64_t value) noexcept { + append(0, internal::tape_type::UINT64); + *next_tape_loc = value; + next_tape_loc++; + } + + /** Write a double value to tape. */ + simdjson_inline void tape_writer::append_double(double value) noexcept { + append2(0, value, internal::tape_type::DOUBLE); + } + + simdjson_inline void tape_writer::skip() noexcept { + next_tape_loc++; + } + + simdjson_inline void tape_writer::skip_large_integer() noexcept { + next_tape_loc += 2; + } + + simdjson_inline void tape_writer::skip_double() noexcept { + next_tape_loc += 2; + } + + simdjson_inline void tape_writer::append(uint64_t val, internal::tape_type t) noexcept { + *next_tape_loc = val | ((uint64_t(char(t))) << 56); + next_tape_loc++; + } + + template + simdjson_inline void tape_writer::append2(uint64_t val, T val2, internal::tape_type t) noexcept { + append(val, t); + static_assert(sizeof(val2) == sizeof(*next_tape_loc), "Type is not 64 bits!"); + memcpy(next_tape_loc, &val2, sizeof(val2)); + next_tape_loc++; + } + + simdjson_inline void tape_writer::write(uint64_t& tape_loc, uint64_t val, internal::tape_type t) noexcept { + tape_loc = val | ((uint64_t(char(t))) << 56); + } + + } // namespace stage2 + } // unnamed namespace + } // namespace icelake +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE2_TAPE_WRITER_H +/* end file generic/stage2/tape_writer.h for icelake */ +/* including generic/stage2/logger.h for icelake: #include */ +/* begin file generic/stage2/logger.h for icelake */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE2_LOGGER_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_LOGGER_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +#include + + +// This is for an internal-only stage 2 specific logger. +// Set LOG_ENABLED = true to log what stage 2 is doing! +namespace simdjson { + namespace icelake { + namespace { + namespace logger { + + static constexpr const char* DASHES = "----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------"; + +#if SIMDJSON_VERBOSE_LOGGING + static constexpr const bool LOG_ENABLED = true; +#else + static constexpr const bool LOG_ENABLED = false; +#endif + static constexpr const int LOG_EVENT_LEN = 20; + static constexpr const int LOG_BUFFER_LEN = 30; + static constexpr const int LOG_SMALL_BUFFER_LEN = 10; + static constexpr const int LOG_INDEX_LEN = 5; + + static int log_depth; // Not threadsafe. Log only. + + // Helper to turn unprintable or newline characters into spaces + static simdjson_inline char printable_char(char c) { + if (c >= 0x20) { + return c; + } + else { + return ' '; + } + } + + // Print the header and set up log_start + static simdjson_inline void log_start() { + if (LOG_ENABLED) { + log_depth = 0; + printf("\n"); + printf("| %-*s | %-*s | %-*s | %-*s | Detail |\n", LOG_EVENT_LEN, "Event", LOG_BUFFER_LEN, "Buffer", LOG_SMALL_BUFFER_LEN, "Next", 5, "Next#"); + printf("|%.*s|%.*s|%.*s|%.*s|--------|\n", LOG_EVENT_LEN + 2, DASHES, LOG_BUFFER_LEN + 2, DASHES, LOG_SMALL_BUFFER_LEN + 2, DASHES, 5 + 2, DASHES); + } + } + + simdjson_unused static simdjson_inline void log_string(const char* message) { + if (LOG_ENABLED) { + printf("%s\n", message); + } + } + + // Logs a single line from the stage 2 DOM parser + template + static simdjson_inline void log_line(S& structurals, const char* title_prefix, const char* title, const char* detail) { + if (LOG_ENABLED) { + printf("| %*s%s%-*s ", log_depth * 2, "", title_prefix, LOG_EVENT_LEN - log_depth * 2 - int(strlen(title_prefix)), title); + auto current_index = structurals.at_beginning() ? nullptr : structurals.next_structural - 1; + auto next_index = structurals.next_structural; + auto current = current_index ? &structurals.buf[*current_index] : reinterpret_cast(" "); + auto next = &structurals.buf[*next_index]; + { + // Print the next N characters in the buffer. + printf("| "); + // Otherwise, print the characters starting from the buffer position. + // Print spaces for unprintable or newline characters. + for (int i = 0; i < LOG_BUFFER_LEN; i++) { + printf("%c", printable_char(current[i])); + } + printf(" "); + // Print the next N characters in the buffer. + printf("| "); + // Otherwise, print the characters starting from the buffer position. + // Print spaces for unprintable or newline characters. + for (int i = 0; i < LOG_SMALL_BUFFER_LEN; i++) { + printf("%c", printable_char(next[i])); + } + printf(" "); + } + if (current_index) { + printf("| %*u ", LOG_INDEX_LEN, *current_index); + } + else { + printf("| %-*s ", LOG_INDEX_LEN, ""); + } + // printf("| %*u ", LOG_INDEX_LEN, structurals.next_tape_index()); + printf("| %-s ", detail); + printf("|\n"); + } + } + + } // namespace logger + } // unnamed namespace + } // namespace icelake +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE2_LOGGER_H +/* end file generic/stage2/logger.h for icelake */ + +// All other declarations +/* including generic/stage2/json_iterator.h for icelake: #include */ +/* begin file generic/stage2/json_iterator.h for icelake */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE2_JSON_ITERATOR_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_JSON_ITERATOR_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace icelake { + namespace { + namespace stage2 { + + class json_iterator { + public: + const uint8_t* const buf; + uint32_t* next_structural; + dom_parser_implementation& dom_parser; + uint32_t depth{ 0 }; + + /** + * Walk the JSON document. + * + * The visitor receives callbacks when values are encountered. All callbacks pass the iterator as + * the first parameter; some callbacks have other parameters as well: + * + * - visit_document_start() - at the beginning. + * - visit_document_end() - at the end (if things were successful). + * + * - visit_array_start() - at the start `[` of a non-empty array. + * - visit_array_end() - at the end `]` of a non-empty array. + * - visit_empty_array() - when an empty array is encountered. + * + * - visit_object_end() - at the start `]` of a non-empty object. + * - visit_object_start() - at the end `]` of a non-empty object. + * - visit_empty_object() - when an empty object is encountered. + * - visit_key(const uint8_t *key) - when a key in an object field is encountered. key is + * guaranteed to point at the first quote of the string (`"key"`). + * - visit_primitive(const uint8_t *value) - when a value is a string, number, boolean or null. + * - visit_root_primitive(iter, uint8_t *value) - when the top-level value is a string, number, boolean or null. + * + * - increment_count(iter) - each time a value is found in an array or object. + */ + template + simdjson_warn_unused simdjson_inline error_code walk_document(V& visitor) noexcept; + + /** + * Create an iterator capable of walking a JSON document. + * + * The document must have already passed through stage 1. + */ + simdjson_inline json_iterator(dom_parser_implementation& _dom_parser, size_t start_structural_index); + + /** + * Look at the next token. + * + * Tokens can be strings, numbers, booleans, null, or operators (`[{]},:`)). + * + * They may include invalid JSON as well (such as `1.2.3` or `ture`). + */ + simdjson_inline const uint8_t* peek() const noexcept; + /** + * Advance to the next token. + * + * Tokens can be strings, numbers, booleans, null, or operators (`[{]},:`)). + * + * They may include invalid JSON as well (such as `1.2.3` or `ture`). + */ + simdjson_inline const uint8_t* advance() noexcept; + /** + * Get the remaining length of the document, from the start of the current token. + */ + simdjson_inline size_t remaining_len() const noexcept; + /** + * Check if we are at the end of the document. + * + * If this is true, there are no more tokens. + */ + simdjson_inline bool at_eof() const noexcept; + /** + * Check if we are at the beginning of the document. + */ + simdjson_inline bool at_beginning() const noexcept; + simdjson_inline uint8_t last_structural() const noexcept; + + /** + * Log that a value has been found. + * + * Set LOG_ENABLED=true in logger.h to see logging. + */ + simdjson_inline void log_value(const char* type) const noexcept; + /** + * Log the start of a multipart value. + * + * Set LOG_ENABLED=true in logger.h to see logging. + */ + simdjson_inline void log_start_value(const char* type) const noexcept; + /** + * Log the end of a multipart value. + * + * Set LOG_ENABLED=true in logger.h to see logging. + */ + simdjson_inline void log_end_value(const char* type) const noexcept; + /** + * Log an error. + * + * Set LOG_ENABLED=true in logger.h to see logging. + */ + simdjson_inline void log_error(const char* error) const noexcept; + + template + simdjson_warn_unused simdjson_inline error_code visit_root_primitive(V& visitor, const uint8_t* value) noexcept; + template + simdjson_warn_unused simdjson_inline error_code visit_primitive(V& visitor, const uint8_t* value) noexcept; + }; + + template + simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V& visitor) noexcept { + logger::log_start(); + + // + // Start the document + // + if (at_eof()) { return EMPTY; } + log_start_value("document"); + SIMDJSON_TRY(visitor.visit_document_start(*this)); + + // + // Read first value + // + { + auto value = advance(); + + // Make sure the outer object or array is closed before continuing; otherwise, there are ways we + // could get into memory corruption. See https://github.com/simdjson/simdjson/issues/906 + if (!STREAMING) { + switch (*value) { + case '{': if (last_structural() != '}') { log_value("starting brace unmatched"); return TAPE_ERROR; }; break; + case '[': if (last_structural() != ']') { log_value("starting bracket unmatched"); return TAPE_ERROR; }; break; + } + } + + switch (*value) { + case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY(visitor.visit_empty_object(*this)); break; } goto object_begin; + case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY(visitor.visit_empty_array(*this)); break; } goto array_begin; + default: SIMDJSON_TRY(visitor.visit_root_primitive(*this, value)); break; + } + } + goto document_end; + + // + // Object parser states + // + object_begin: + log_start_value("object"); + depth++; + if (depth >= dom_parser.max_depth()) { log_error("Exceeded max depth!"); return DEPTH_ERROR; } + dom_parser.is_array[depth] = false; + SIMDJSON_TRY(visitor.visit_object_start(*this)); + + { + auto key = advance(); + if (*key != '"') { log_error("Object does not start with a key"); return TAPE_ERROR; } + SIMDJSON_TRY(visitor.increment_count(*this)); + SIMDJSON_TRY(visitor.visit_key(*this, key)); + } + + object_field: + if (simdjson_unlikely(*advance() != ':')) { log_error("Missing colon after key in object"); return TAPE_ERROR; } + { + auto value = advance(); + switch (*value) { + case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY(visitor.visit_empty_object(*this)); break; } goto object_begin; + case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY(visitor.visit_empty_array(*this)); break; } goto array_begin; + default: SIMDJSON_TRY(visitor.visit_primitive(*this, value)); break; + } + } + + object_continue: + switch (*advance()) { + case ',': + SIMDJSON_TRY(visitor.increment_count(*this)); + { + auto key = advance(); + if (simdjson_unlikely(*key != '"')) { log_error("Key string missing at beginning of field in object"); return TAPE_ERROR; } + SIMDJSON_TRY(visitor.visit_key(*this, key)); + } + goto object_field; + case '}': log_end_value("object"); SIMDJSON_TRY(visitor.visit_object_end(*this)); goto scope_end; + default: log_error("No comma between object fields"); return TAPE_ERROR; + } + + scope_end: + depth--; + if (depth == 0) { goto document_end; } + if (dom_parser.is_array[depth]) { goto array_continue; } + goto object_continue; + + // + // Array parser states + // + array_begin: + log_start_value("array"); + depth++; + if (depth >= dom_parser.max_depth()) { log_error("Exceeded max depth!"); return DEPTH_ERROR; } + dom_parser.is_array[depth] = true; + SIMDJSON_TRY(visitor.visit_array_start(*this)); + SIMDJSON_TRY(visitor.increment_count(*this)); + + array_value: + { + auto value = advance(); + switch (*value) { + case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY(visitor.visit_empty_object(*this)); break; } goto object_begin; + case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY(visitor.visit_empty_array(*this)); break; } goto array_begin; + default: SIMDJSON_TRY(visitor.visit_primitive(*this, value)); break; + } + } + + array_continue: + switch (*advance()) { + case ',': SIMDJSON_TRY(visitor.increment_count(*this)); goto array_value; + case ']': log_end_value("array"); SIMDJSON_TRY(visitor.visit_array_end(*this)); goto scope_end; + default: log_error("Missing comma between array values"); return TAPE_ERROR; + } + + document_end: + log_end_value("document"); + SIMDJSON_TRY(visitor.visit_document_end(*this)); + + dom_parser.next_structural_index = uint32_t(next_structural - &dom_parser.structural_indexes[0]); + + // If we didn't make it to the end, it's an error + if (!STREAMING && dom_parser.next_structural_index != dom_parser.n_structural_indexes) { + log_error("More than one JSON value at the root of the document, or extra characters at the end of the JSON!"); + return TAPE_ERROR; + } + + return SUCCESS; + + } // walk_document() + + simdjson_inline json_iterator::json_iterator(dom_parser_implementation& _dom_parser, size_t start_structural_index) + : buf{ _dom_parser.buf }, + next_structural{ &_dom_parser.structural_indexes[start_structural_index] }, + dom_parser{ _dom_parser } { + } + + simdjson_inline const uint8_t* json_iterator::peek() const noexcept { + return &buf[*(next_structural)]; + } + simdjson_inline const uint8_t* json_iterator::advance() noexcept { + return &buf[*(next_structural++)]; + } + simdjson_inline size_t json_iterator::remaining_len() const noexcept { + return dom_parser.len - *(next_structural - 1); + } + + simdjson_inline bool json_iterator::at_eof() const noexcept { + return next_structural == &dom_parser.structural_indexes[dom_parser.n_structural_indexes]; + } + simdjson_inline bool json_iterator::at_beginning() const noexcept { + return next_structural == dom_parser.structural_indexes.get(); + } + simdjson_inline uint8_t json_iterator::last_structural() const noexcept { + return buf[dom_parser.structural_indexes[dom_parser.n_structural_indexes - 1]]; + } + + simdjson_inline void json_iterator::log_value(const char* type) const noexcept { + logger::log_line(*this, "", type, ""); + } + + simdjson_inline void json_iterator::log_start_value(const char* type) const noexcept { + logger::log_line(*this, "+", type, ""); + if (logger::LOG_ENABLED) { logger::log_depth++; } + } + + simdjson_inline void json_iterator::log_end_value(const char* type) const noexcept { + if (logger::LOG_ENABLED) { logger::log_depth--; } + logger::log_line(*this, "-", type, ""); + } + + simdjson_inline void json_iterator::log_error(const char* error) const noexcept { + logger::log_line(*this, "", "ERROR", error); + } + + template + simdjson_warn_unused simdjson_inline error_code json_iterator::visit_root_primitive(V& visitor, const uint8_t* value) noexcept { + switch (*value) { + case '"': return visitor.visit_root_string(*this, value); + case 't': return visitor.visit_root_true_atom(*this, value); + case 'f': return visitor.visit_root_false_atom(*this, value); + case 'n': return visitor.visit_root_null_atom(*this, value); + case '-': + case '0': case '1': case '2': case '3': case '4': + case '5': case '6': case '7': case '8': case '9': + return visitor.visit_root_number(*this, value); + default: + log_error("Document starts with a non-value character"); + return TAPE_ERROR; + } + } + template + simdjson_warn_unused simdjson_inline error_code json_iterator::visit_primitive(V& visitor, const uint8_t* value) noexcept { + // Use the fact that most scalars are going to be either strings or numbers. + if (*value == '"') { + return visitor.visit_string(*this, value); + } + else if (((*value - '0') < 10) || (*value == '-')) { + return visitor.visit_number(*this, value); + } + // true, false, null are uncommon. + switch (*value) { + case 't': return visitor.visit_true_atom(*this, value); + case 'f': return visitor.visit_false_atom(*this, value); + case 'n': return visitor.visit_null_atom(*this, value); + default: + log_error("Non-value found when value was expected!"); + return TAPE_ERROR; + } + } + + } // namespace stage2 + } // unnamed namespace + } // namespace icelake +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE2_JSON_ITERATOR_H +/* end file generic/stage2/json_iterator.h for icelake */ +/* including generic/stage2/stringparsing.h for icelake: #include */ +/* begin file generic/stage2/stringparsing.h for icelake */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +// This file contains the common code every implementation uses +// It is intended to be included multiple times and compiled multiple times + +namespace simdjson { + namespace icelake { + namespace { + /// @private + namespace stringparsing { + + // begin copypasta + // These chars yield themselves: " \ / + // b -> backspace, f -> formfeed, n -> newline, r -> cr, t -> horizontal tab + // u not handled in this table as it's complex + static const uint8_t escape_map[256] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x0. + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0x22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x2f, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x4. + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x5c, 0, 0, 0, // 0x5. + 0, 0, 0x08, 0, 0, 0, 0x0c, 0, 0, 0, 0, 0, 0, 0, 0x0a, 0, // 0x6. + 0, 0, 0x0d, 0, 0x09, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x7. + + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + }; + + // handle a unicode codepoint + // write appropriate values into dest + // src will advance 6 bytes or 12 bytes + // dest will advance a variable amount (return via pointer) + // return true if the unicode codepoint was valid + // We work in little-endian then swap at write time + simdjson_warn_unused + simdjson_inline bool handle_unicode_codepoint(const uint8_t** src_ptr, + uint8_t** dst_ptr, bool allow_replacement) { + // Use the default Unicode Character 'REPLACEMENT CHARACTER' (U+FFFD) + constexpr uint32_t substitution_code_point = 0xfffd; + // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the + // conversion is not valid; we defer the check for this to inside the + // multilingual plane check. + uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2); + *src_ptr += 6; + + // If we found a high surrogate, we must + // check for low surrogate for characters + // outside the Basic + // Multilingual Plane. + if (code_point >= 0xd800 && code_point < 0xdc00) { + const uint8_t* src_data = *src_ptr; + /* Compiler optimizations convert this to a single 16-bit load and compare on most platforms */ + if (((src_data[0] << 8) | src_data[1]) != ((static_cast ('\\') << 8) | static_cast ('u'))) { + if (!allow_replacement) { return false; } + code_point = substitution_code_point; + } + else { + uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(src_data + 2); + + // We have already checked that the high surrogate is valid and + // (code_point - 0xd800) < 1024. + // + // Check that code_point_2 is in the range 0xdc00..0xdfff + // and that code_point_2 was parsed from valid hex. + uint32_t low_bit = code_point_2 - 0xdc00; + if (low_bit >> 10) { + if (!allow_replacement) { return false; } + code_point = substitution_code_point; + } + else { + code_point = (((code_point - 0xd800) << 10) | low_bit) + 0x10000; + *src_ptr += 6; + } + + } + } + else if (code_point >= 0xdc00 && code_point <= 0xdfff) { + // If we encounter a low surrogate (not preceded by a high surrogate) + // then we have an error. + if (!allow_replacement) { return false; } + code_point = substitution_code_point; + } + size_t offset = jsoncharutils::codepoint_to_utf8(code_point, *dst_ptr); + *dst_ptr += offset; + return offset > 0; + } + + + // handle a unicode codepoint using the wobbly convention + // https://simonsapin.github.io/wtf-8/ + // write appropriate values into dest + // src will advance 6 bytes or 12 bytes + // dest will advance a variable amount (return via pointer) + // return true if the unicode codepoint was valid + // We work in little-endian then swap at write time + simdjson_warn_unused + simdjson_inline bool handle_unicode_codepoint_wobbly(const uint8_t** src_ptr, + uint8_t** dst_ptr) { + // It is not ideal that this function is nearly identical to handle_unicode_codepoint. + // + // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the + // conversion is not valid; we defer the check for this to inside the + // multilingual plane check. + uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2); + *src_ptr += 6; + // If we found a high surrogate, we must + // check for low surrogate for characters + // outside the Basic + // Multilingual Plane. + if (code_point >= 0xd800 && code_point < 0xdc00) { + const uint8_t* src_data = *src_ptr; + /* Compiler optimizations convert this to a single 16-bit load and compare on most platforms */ + if (((src_data[0] << 8) | src_data[1]) == ((static_cast ('\\') << 8) | static_cast ('u'))) { + uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(src_data + 2); + uint32_t low_bit = code_point_2 - 0xdc00; + if ((low_bit >> 10) == 0) { + code_point = + (((code_point - 0xd800) << 10) | low_bit) + 0x10000; + *src_ptr += 6; + } + } + } + + size_t offset = jsoncharutils::codepoint_to_utf8(code_point, *dst_ptr); + *dst_ptr += offset; + return offset > 0; + } + + + /** + * Unescape a valid UTF-8 string from src to dst, stopping at a final unescaped quote. There + * must be an unescaped quote terminating the string. It returns the final output + * position as pointer. In case of error (e.g., the string has bad escaped codes), + * then null_ptr is returned. It is assumed that the output buffer is large + * enough. E.g., if src points at 'joe"', then dst needs to have four free bytes + + * SIMDJSON_PADDING bytes. + */ + simdjson_warn_unused simdjson_inline uint8_t* parse_string(const uint8_t* src, uint8_t* dst, bool allow_replacement) { + while (1) { + // Copy the next n bytes, and find the backslash and quote in them. + auto bs_quote = backslash_and_quote::copy_and_find(src, dst); + // If the next thing is the end quote, copy and return + if (bs_quote.has_quote_first()) { + // we encountered quotes first. Move dst to point to quotes and exit + return dst + bs_quote.quote_index(); + } + if (bs_quote.has_backslash()) { + /* find out where the backspace is */ + auto bs_dist = bs_quote.backslash_index(); + uint8_t escape_char = src[bs_dist + 1]; + /* we encountered backslash first. Handle backslash */ + if (escape_char == 'u') { + /* move src/dst up to the start; they will be further adjusted + within the unicode codepoint handling code. */ + src += bs_dist; + dst += bs_dist; + if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { + return nullptr; + } + } + else { + /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and + * write bs_dist+1 characters to output + * note this may reach beyond the part of the buffer we've actually + * seen. I think this is ok */ + uint8_t escape_result = escape_map[escape_char]; + if (escape_result == 0u) { + return nullptr; /* bogus escape value is an error */ + } + dst[bs_dist] = escape_result; + src += bs_dist + 2; + dst += bs_dist + 1; + } + } + else { + /* they are the same. Since they can't co-occur, it means we + * encountered neither. */ + src += backslash_and_quote::BYTES_PROCESSED; + dst += backslash_and_quote::BYTES_PROCESSED; + } + } + } + + simdjson_warn_unused simdjson_inline uint8_t* parse_wobbly_string(const uint8_t* src, uint8_t* dst) { + // It is not ideal that this function is nearly identical to parse_string. + while (1) { + // Copy the next n bytes, and find the backslash and quote in them. + auto bs_quote = backslash_and_quote::copy_and_find(src, dst); + // If the next thing is the end quote, copy and return + if (bs_quote.has_quote_first()) { + // we encountered quotes first. Move dst to point to quotes and exit + return dst + bs_quote.quote_index(); + } + if (bs_quote.has_backslash()) { + /* find out where the backspace is */ + auto bs_dist = bs_quote.backslash_index(); + uint8_t escape_char = src[bs_dist + 1]; + /* we encountered backslash first. Handle backslash */ + if (escape_char == 'u') { + /* move src/dst up to the start; they will be further adjusted + within the unicode codepoint handling code. */ + src += bs_dist; + dst += bs_dist; + if (!handle_unicode_codepoint_wobbly(&src, &dst)) { + return nullptr; + } + } + else { + /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and + * write bs_dist+1 characters to output + * note this may reach beyond the part of the buffer we've actually + * seen. I think this is ok */ + uint8_t escape_result = escape_map[escape_char]; + if (escape_result == 0u) { + return nullptr; /* bogus escape value is an error */ + } + dst[bs_dist] = escape_result; + src += bs_dist + 2; + dst += bs_dist + 1; + } + } + else { + /* they are the same. Since they can't co-occur, it means we + * encountered neither. */ + src += backslash_and_quote::BYTES_PROCESSED; + dst += backslash_and_quote::BYTES_PROCESSED; + } + } + } + + } // namespace stringparsing + } // unnamed namespace + } // namespace icelake +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H +/* end file generic/stage2/stringparsing.h for icelake */ +/* including generic/stage2/structural_iterator.h for icelake: #include */ +/* begin file generic/stage2/structural_iterator.h for icelake */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE2_STRUCTURAL_ITERATOR_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_STRUCTURAL_ITERATOR_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace icelake { + namespace { + namespace stage2 { + + class structural_iterator { + public: + const uint8_t* const buf; + uint32_t* next_structural; + dom_parser_implementation& dom_parser; + + // Start a structural + simdjson_inline structural_iterator(dom_parser_implementation& _dom_parser, size_t start_structural_index) + : buf{ _dom_parser.buf }, + next_structural{ &_dom_parser.structural_indexes[start_structural_index] }, + dom_parser{ _dom_parser } { + } + // Get the buffer position of the current structural character + simdjson_inline const uint8_t* current() { + return &buf[*(next_structural - 1)]; + } + // Get the current structural character + simdjson_inline char current_char() { + return buf[*(next_structural - 1)]; + } + // Get the next structural character without advancing + simdjson_inline char peek_next_char() { + return buf[*next_structural]; + } + simdjson_inline const uint8_t* peek() { + return &buf[*next_structural]; + } + simdjson_inline const uint8_t* advance() { + return &buf[*(next_structural++)]; + } + simdjson_inline char advance_char() { + return buf[*(next_structural++)]; + } + simdjson_inline size_t remaining_len() { + return dom_parser.len - *(next_structural - 1); + } + + simdjson_inline bool at_end() { + return next_structural == &dom_parser.structural_indexes[dom_parser.n_structural_indexes]; + } + simdjson_inline bool at_beginning() { + return next_structural == dom_parser.structural_indexes.get(); + } + }; + + } // namespace stage2 + } // unnamed namespace + } // namespace icelake +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE2_STRUCTURAL_ITERATOR_H +/* end file generic/stage2/structural_iterator.h for icelake */ +/* including generic/stage2/tape_builder.h for icelake: #include */ +/* begin file generic/stage2/tape_builder.h for icelake */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE2_TAPE_BUILDER_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_TAPE_BUILDER_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + + +namespace simdjson { + namespace icelake { + namespace { + namespace stage2 { + + struct tape_builder { + template + simdjson_warn_unused static simdjson_inline error_code parse_document( + dom_parser_implementation& dom_parser, + dom::document& doc) noexcept; + + /** Called when a non-empty document starts. */ + simdjson_warn_unused simdjson_inline error_code visit_document_start(json_iterator& iter) noexcept; + /** Called when a non-empty document ends without error. */ + simdjson_warn_unused simdjson_inline error_code visit_document_end(json_iterator& iter) noexcept; + + /** Called when a non-empty array starts. */ + simdjson_warn_unused simdjson_inline error_code visit_array_start(json_iterator& iter) noexcept; + /** Called when a non-empty array ends. */ + simdjson_warn_unused simdjson_inline error_code visit_array_end(json_iterator& iter) noexcept; + /** Called when an empty array is found. */ + simdjson_warn_unused simdjson_inline error_code visit_empty_array(json_iterator& iter) noexcept; + + /** Called when a non-empty object starts. */ + simdjson_warn_unused simdjson_inline error_code visit_object_start(json_iterator& iter) noexcept; + /** + * Called when a key in a field is encountered. + * + * primitive, visit_object_start, visit_empty_object, visit_array_start, or visit_empty_array + * will be called after this with the field value. + */ + simdjson_warn_unused simdjson_inline error_code visit_key(json_iterator& iter, const uint8_t* key) noexcept; + /** Called when a non-empty object ends. */ + simdjson_warn_unused simdjson_inline error_code visit_object_end(json_iterator& iter) noexcept; + /** Called when an empty object is found. */ + simdjson_warn_unused simdjson_inline error_code visit_empty_object(json_iterator& iter) noexcept; + + /** + * Called when a string, number, boolean or null is found. + */ + simdjson_warn_unused simdjson_inline error_code visit_primitive(json_iterator& iter, const uint8_t* value) noexcept; + /** + * Called when a string, number, boolean or null is found at the top level of a document (i.e. + * when there is no array or object and the entire document is a single string, number, boolean or + * null. + * + * This is separate from primitive() because simdjson's normal primitive parsing routines assume + * there is at least one more token after the value, which is only true in an array or object. + */ + simdjson_warn_unused simdjson_inline error_code visit_root_primitive(json_iterator& iter, const uint8_t* value) noexcept; + + simdjson_warn_unused simdjson_inline error_code visit_string(json_iterator& iter, const uint8_t* value, bool key = false) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_number(json_iterator& iter, const uint8_t* value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_true_atom(json_iterator& iter, const uint8_t* value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_false_atom(json_iterator& iter, const uint8_t* value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_null_atom(json_iterator& iter, const uint8_t* value) noexcept; + + simdjson_warn_unused simdjson_inline error_code visit_root_string(json_iterator& iter, const uint8_t* value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_root_number(json_iterator& iter, const uint8_t* value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_root_true_atom(json_iterator& iter, const uint8_t* value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_root_false_atom(json_iterator& iter, const uint8_t* value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_root_null_atom(json_iterator& iter, const uint8_t* value) noexcept; + + /** Called each time a new field or element in an array or object is found. */ + simdjson_warn_unused simdjson_inline error_code increment_count(json_iterator& iter) noexcept; + + /** Next location to write to tape */ + tape_writer tape; + private: + /** Next write location in the string buf for stage 2 parsing */ + uint8_t* current_string_buf_loc; + + simdjson_inline tape_builder(dom::document& doc) noexcept; + + simdjson_inline uint32_t next_tape_index(json_iterator& iter) const noexcept; + simdjson_inline void start_container(json_iterator& iter) noexcept; + simdjson_warn_unused simdjson_inline error_code end_container(json_iterator& iter, internal::tape_type start, internal::tape_type end) noexcept; + simdjson_warn_unused simdjson_inline error_code empty_container(json_iterator& iter, internal::tape_type start, internal::tape_type end) noexcept; + simdjson_inline uint8_t* on_start_string(json_iterator& iter) noexcept; + simdjson_inline void on_end_string(uint8_t* dst) noexcept; + }; // struct tape_builder + + template + simdjson_warn_unused simdjson_inline error_code tape_builder::parse_document( + dom_parser_implementation& dom_parser, + dom::document& doc) noexcept { + dom_parser.doc = &doc; + json_iterator iter(dom_parser, STREAMING ? dom_parser.next_structural_index : 0); + tape_builder builder(doc); + return iter.walk_document(builder); + } + + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_primitive(json_iterator& iter, const uint8_t* value) noexcept { + return iter.visit_root_primitive(*this, value); + } + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_primitive(json_iterator& iter, const uint8_t* value) noexcept { + return iter.visit_primitive(*this, value); + } + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_empty_object(json_iterator& iter) noexcept { + return empty_container(iter, internal::tape_type::START_OBJECT, internal::tape_type::END_OBJECT); + } + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_empty_array(json_iterator& iter) noexcept { + return empty_container(iter, internal::tape_type::START_ARRAY, internal::tape_type::END_ARRAY); + } + + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_document_start(json_iterator& iter) noexcept { + start_container(iter); + return SUCCESS; + } + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_object_start(json_iterator& iter) noexcept { + start_container(iter); + return SUCCESS; + } + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_array_start(json_iterator& iter) noexcept { + start_container(iter); + return SUCCESS; + } + + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_object_end(json_iterator& iter) noexcept { + return end_container(iter, internal::tape_type::START_OBJECT, internal::tape_type::END_OBJECT); + } + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_array_end(json_iterator& iter) noexcept { + return end_container(iter, internal::tape_type::START_ARRAY, internal::tape_type::END_ARRAY); + } + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_document_end(json_iterator& iter) noexcept { + constexpr uint32_t start_tape_index = 0; + tape.append(start_tape_index, internal::tape_type::ROOT); + tape_writer::write(iter.dom_parser.doc->tape[start_tape_index], next_tape_index(iter), internal::tape_type::ROOT); + return SUCCESS; + } + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_key(json_iterator& iter, const uint8_t* key) noexcept { + return visit_string(iter, key, true); + } + + simdjson_warn_unused simdjson_inline error_code tape_builder::increment_count(json_iterator& iter) noexcept { + iter.dom_parser.open_containers[iter.depth].count++; // we have a key value pair in the object at parser.dom_parser.depth - 1 + return SUCCESS; + } + + simdjson_inline tape_builder::tape_builder(dom::document& doc) noexcept : tape{ doc.tape.get() }, current_string_buf_loc{ doc.string_buf.get() } {} + + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_string(json_iterator& iter, const uint8_t* value, bool key) noexcept { + iter.log_value(key ? "key" : "string"); + uint8_t* dst = on_start_string(iter); + dst = stringparsing::parse_string(value + 1, dst, false); // We do not allow replacement when the escape characters are invalid. + if (dst == nullptr) { + iter.log_error("Invalid escape in string"); + return STRING_ERROR; + } + on_end_string(dst); + return SUCCESS; + } + + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_string(json_iterator& iter, const uint8_t* value) noexcept { + return visit_string(iter, value); + } + + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_number(json_iterator& iter, const uint8_t* value) noexcept { + iter.log_value("number"); + return numberparsing::parse_number(value, tape); + } + + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_number(json_iterator& iter, const uint8_t* value) noexcept { + // + // We need to make a copy to make sure that the string is space terminated. + // This is not about padding the input, which should already padded up + // to len + SIMDJSON_PADDING. However, we have no control at this stage + // on how the padding was done. What if the input string was padded with nulls? + // It is quite common for an input string to have an extra null character (C string). + // We do not want to allow 9\0 (where \0 is the null character) inside a JSON + // document, but the string "9\0" by itself is fine. So we make a copy and + // pad the input with spaces when we know that there is just one input element. + // This copy is relatively expensive, but it will almost never be called in + // practice unless you are in the strange scenario where you have many JSON + // documents made of single atoms. + // + std::unique_ptrcopy(new (std::nothrow) uint8_t[iter.remaining_len() + SIMDJSON_PADDING]); + if (copy.get() == nullptr) { return MEMALLOC; } + std::memcpy(copy.get(), value, iter.remaining_len()); + std::memset(copy.get() + iter.remaining_len(), ' ', SIMDJSON_PADDING); + error_code error = visit_number(iter, copy.get()); + return error; + } + + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_true_atom(json_iterator& iter, const uint8_t* value) noexcept { + iter.log_value("true"); + if (!atomparsing::is_valid_true_atom(value)) { return T_ATOM_ERROR; } + tape.append(0, internal::tape_type::TRUE_VALUE); + return SUCCESS; + } + + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_true_atom(json_iterator& iter, const uint8_t* value) noexcept { + iter.log_value("true"); + if (!atomparsing::is_valid_true_atom(value, iter.remaining_len())) { return T_ATOM_ERROR; } + tape.append(0, internal::tape_type::TRUE_VALUE); + return SUCCESS; + } + + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_false_atom(json_iterator& iter, const uint8_t* value) noexcept { + iter.log_value("false"); + if (!atomparsing::is_valid_false_atom(value)) { return F_ATOM_ERROR; } + tape.append(0, internal::tape_type::FALSE_VALUE); + return SUCCESS; + } + + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_false_atom(json_iterator& iter, const uint8_t* value) noexcept { + iter.log_value("false"); + if (!atomparsing::is_valid_false_atom(value, iter.remaining_len())) { return F_ATOM_ERROR; } + tape.append(0, internal::tape_type::FALSE_VALUE); + return SUCCESS; + } + + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_null_atom(json_iterator& iter, const uint8_t* value) noexcept { + iter.log_value("null"); + if (!atomparsing::is_valid_null_atom(value)) { return N_ATOM_ERROR; } + tape.append(0, internal::tape_type::NULL_VALUE); + return SUCCESS; + } + + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_null_atom(json_iterator& iter, const uint8_t* value) noexcept { + iter.log_value("null"); + if (!atomparsing::is_valid_null_atom(value, iter.remaining_len())) { return N_ATOM_ERROR; } + tape.append(0, internal::tape_type::NULL_VALUE); + return SUCCESS; + } + + // private: + + simdjson_inline uint32_t tape_builder::next_tape_index(json_iterator& iter) const noexcept { + return uint32_t(tape.next_tape_loc - iter.dom_parser.doc->tape.get()); + } + + simdjson_warn_unused simdjson_inline error_code tape_builder::empty_container(json_iterator& iter, internal::tape_type start, internal::tape_type end) noexcept { + auto start_index = next_tape_index(iter); + tape.append(start_index + 2, start); + tape.append(start_index, end); + return SUCCESS; + } + + simdjson_inline void tape_builder::start_container(json_iterator& iter) noexcept { + iter.dom_parser.open_containers[iter.depth].tape_index = next_tape_index(iter); + iter.dom_parser.open_containers[iter.depth].count = 0; + tape.skip(); // We don't actually *write* the start element until the end. + } + + simdjson_warn_unused simdjson_inline error_code tape_builder::end_container(json_iterator& iter, internal::tape_type start, internal::tape_type end) noexcept { + // Write the ending tape element, pointing at the start location + const uint32_t start_tape_index = iter.dom_parser.open_containers[iter.depth].tape_index; + tape.append(start_tape_index, end); + // Write the start tape element, pointing at the end location (and including count) + // count can overflow if it exceeds 24 bits... so we saturate + // the convention being that a cnt of 0xffffff or more is undetermined in value (>= 0xffffff). + const uint32_t count = iter.dom_parser.open_containers[iter.depth].count; + const uint32_t cntsat = count > 0xFFFFFF ? 0xFFFFFF : count; + tape_writer::write(iter.dom_parser.doc->tape[start_tape_index], next_tape_index(iter) | (uint64_t(cntsat) << 32), start); + return SUCCESS; + } + + simdjson_inline uint8_t* tape_builder::on_start_string(json_iterator& iter) noexcept { + // we advance the point, accounting for the fact that we have a NULL termination + tape.append(current_string_buf_loc - iter.dom_parser.doc->string_buf.get(), internal::tape_type::STRING); + return current_string_buf_loc + sizeof(uint32_t); + } + + simdjson_inline void tape_builder::on_end_string(uint8_t* dst) noexcept { + uint32_t str_length = uint32_t(dst - (current_string_buf_loc + sizeof(uint32_t))); + // TODO check for overflow in case someone has a crazy string (>=4GB?) + // But only add the overflow check when the document itself exceeds 4GB + // Currently unneeded because we refuse to parse docs larger or equal to 4GB. + memcpy(current_string_buf_loc, &str_length, sizeof(uint32_t)); + // NULL termination is still handy if you expect all your strings to + // be NULL terminated? It comes at a small cost + *dst = 0; + current_string_buf_loc = dst + 1; + } + + } // namespace stage2 + } // unnamed namespace + } // namespace icelake +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE2_TAPE_BUILDER_H +/* end file generic/stage2/tape_builder.h for icelake */ +/* end file generic/stage2/amalgamated.h for icelake */ + +#undef SIMDJSON_GENERIC_JSON_STRUCTURAL_INDEXER_CUSTOM_BIT_INDEXER + +// +// Stage 1 +// + +namespace simdjson { + namespace icelake { + + simdjson_warn_unused error_code implementation::create_dom_parser_implementation( + size_t capacity, + size_t max_depth, + std::unique_ptr& dst + ) const noexcept { + dst.reset(new (std::nothrow) dom_parser_implementation()); + if (!dst) { return MEMALLOC; } + if (auto err = dst->set_capacity(capacity)) + return err; + if (auto err = dst->set_max_depth(max_depth)) + return err; + return SUCCESS; + } + + namespace { + + using namespace simd; + + // This identifies structural characters (comma, colon, braces, brackets), + // and ASCII white-space ('\r','\n','\t',' '). + simdjson_inline json_character_block json_character_block::classify(const simd::simd8x64& in) { + // These lookups rely on the fact that anything < 127 will match the lower 4 bits, which is why + // we can't use the generic lookup_16. + const auto whitespace_table = simd8::repeat_16(' ', 100, 100, 100, 17, 100, 113, 2, 100, '\t', '\n', 112, 100, '\r', 100, 100); + + // The 6 operators (:,[]{}) have these values: + // + // , 2C + // : 3A + // [ 5B + // { 7B + // ] 5D + // } 7D + // + // If you use | 0x20 to turn [ and ] into { and }, the lower 4 bits of each character is unique. + // We exploit this, using a simd 4-bit lookup to tell us which character match against, and then + // match it (against | 0x20). + // + // To prevent recognizing other characters, everything else gets compared with 0, which cannot + // match due to the | 0x20. + // + // NOTE: Due to the | 0x20, this ALSO treats and (control characters 0C and 1A) like , + // and :. This gets caught in stage 2, which checks the actual character to ensure the right + // operators are in the right places. + const auto op_table = simd8::repeat_16( + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, ':', '{', // : = 3A, [ = 5B, { = 7B + ',', '}', 0, 0 // , = 2C, ] = 5D, } = 7D + ); + + // We compute whitespace and op separately. If later code only uses one or the + // other, given the fact that all functions are aggressively inlined, we can + // hope that useless computations will be omitted. This is namely case when + // minifying (we only need whitespace). + + const uint64_t whitespace = in.eq({ + _mm512_shuffle_epi8(whitespace_table, in.chunks[0]) + }); + // Turn [ and ] into { and } + const simd8x64 curlified{ + in.chunks[0] | 0x20 + }; + const uint64_t op = curlified.eq({ + _mm512_shuffle_epi8(op_table, in.chunks[0]) + }); + + return { whitespace, op }; + } + + simdjson_inline bool is_ascii(const simd8x64& input) { + return input.reduce_or().is_ascii(); + } + + simdjson_unused simdjson_inline simd8 must_be_continuation(const simd8 prev1, const simd8 prev2, const simd8 prev3) { + simd8 is_second_byte = prev1.saturating_sub(0xc0u - 1); // Only 11______ will be > 0 + simd8 is_third_byte = prev2.saturating_sub(0xe0u - 1); // Only 111_____ will be > 0 + simd8 is_fourth_byte = prev3.saturating_sub(0xf0u - 1); // Only 1111____ will be > 0 + // Caller requires a bool (all 1's). All values resulting from the subtraction will be <= 64, so signed comparison is fine. + return simd8(is_second_byte | is_third_byte | is_fourth_byte) > int8_t(0); + } + + simdjson_inline simd8 must_be_2_3_continuation(const simd8 prev2, const simd8 prev3) { + simd8 is_third_byte = prev2.saturating_sub(0xe0u - 0x80); // Only 111_____ will be >= 0x80 + simd8 is_fourth_byte = prev3.saturating_sub(0xf0u - 0x80); // Only 1111____ will be >= 0x80 + return is_third_byte | is_fourth_byte; + } + + } // unnamed namespace + } // namespace icelake +} // namespace simdjson + +/** + * We provide a custom version of bit_indexer::write using + * naked intrinsics. + * TODO: make this code more elegant. + */ + // Under GCC 12, the intrinsic _mm512_extracti32x4_epi32 may generate 'maybe uninitialized'. + // as a workaround, we disable warnings within the following function. +SIMDJSON_PUSH_DISABLE_ALL_WARNINGS +namespace simdjson { + namespace icelake { + namespace { + namespace stage1 { + simdjson_inline void bit_indexer::write(uint32_t idx, uint64_t bits) { + // In some instances, the next branch is expensive because it is mispredicted. + // Unfortunately, in other cases, + // it helps tremendously. + if (bits == 0) { return; } + + const __m512i indexes = _mm512_maskz_compress_epi8(bits, _mm512_set_epi32( + 0x3f3e3d3c, 0x3b3a3938, 0x37363534, 0x33323130, + 0x2f2e2d2c, 0x2b2a2928, 0x27262524, 0x23222120, + 0x1f1e1d1c, 0x1b1a1918, 0x17161514, 0x13121110, + 0x0f0e0d0c, 0x0b0a0908, 0x07060504, 0x03020100 + )); + const __m512i start_index = _mm512_set1_epi32(idx); + + const auto count = count_ones(bits); + __m512i t0 = _mm512_cvtepu8_epi32(_mm512_castsi512_si128(indexes)); + _mm512_storeu_si512(this->tail, _mm512_add_epi32(t0, start_index)); + + if (count > 16) { + const __m512i t1 = _mm512_cvtepu8_epi32(_mm512_extracti32x4_epi32(indexes, 1)); + _mm512_storeu_si512(this->tail + 16, _mm512_add_epi32(t1, start_index)); + if (count > 32) { + const __m512i t2 = _mm512_cvtepu8_epi32(_mm512_extracti32x4_epi32(indexes, 2)); + _mm512_storeu_si512(this->tail + 32, _mm512_add_epi32(t2, start_index)); + if (count > 48) { + const __m512i t3 = _mm512_cvtepu8_epi32(_mm512_extracti32x4_epi32(indexes, 3)); + _mm512_storeu_si512(this->tail + 48, _mm512_add_epi32(t3, start_index)); + } + } + } + this->tail += count; + } + } + } + } +} +SIMDJSON_POP_DISABLE_WARNINGS + +// +// Stage 2 +// + +// +// Implementation-specific overrides +// +namespace simdjson { + namespace icelake { + + simdjson_warn_unused error_code implementation::minify(const uint8_t* buf, size_t len, uint8_t* dst, size_t& dst_len) const noexcept { + return icelake::stage1::json_minifier::minify<128>(buf, len, dst, dst_len); + } + + simdjson_warn_unused error_code dom_parser_implementation::stage1(const uint8_t* _buf, size_t _len, stage1_mode streaming) noexcept { + this->buf = _buf; + this->len = _len; + return icelake::stage1::json_structural_indexer::index<128>(_buf, _len, *this, streaming); + } + + simdjson_warn_unused bool implementation::validate_utf8(const char* buf, size_t len) const noexcept { + return icelake::stage1::generic_validate_utf8(buf, len); + } + + simdjson_warn_unused error_code dom_parser_implementation::stage2(dom::document& _doc) noexcept { + return stage2::tape_builder::parse_document(*this, _doc); + } + + simdjson_warn_unused error_code dom_parser_implementation::stage2_next(dom::document& _doc) noexcept { + return stage2::tape_builder::parse_document(*this, _doc); + } + + simdjson_warn_unused uint8_t* dom_parser_implementation::parse_string(const uint8_t* src, uint8_t* dst, bool replacement_char) const noexcept { + return icelake::stringparsing::parse_string(src, dst, replacement_char); + } + + simdjson_warn_unused uint8_t* dom_parser_implementation::parse_wobbly_string(const uint8_t* src, uint8_t* dst) const noexcept { + return icelake::stringparsing::parse_wobbly_string(src, dst); + } + + simdjson_warn_unused error_code dom_parser_implementation::parse(const uint8_t* _buf, size_t _len, dom::document& _doc) noexcept { + auto error = stage1(_buf, _len, stage1_mode::regular); + if (error) { return error; } + return stage2(_doc); + } + + } // namespace icelake +} // namespace simdjson + +/* including simdjson/icelake/end.h: #include */ +/* begin file simdjson/icelake/end.h */ +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/icelake/base.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +#if !SIMDJSON_CAN_ALWAYS_RUN_ICELAKE +SIMDJSON_UNTARGET_REGION +#endif + +/* undefining SIMDJSON_IMPLEMENTATION from "icelake" */ +#undef SIMDJSON_IMPLEMENTATION +/* end file simdjson/icelake/end.h */ + +#endif // SIMDJSON_SRC_ICELAKE_CPP +/* end file icelake.cpp */ +#endif +#if SIMDJSON_IMPLEMENTATION_PPC64 +/* including ppc64.cpp: #include */ +/* begin file ppc64.cpp */ +#ifndef SIMDJSON_SRC_PPC64_CPP +#define SIMDJSON_SRC_PPC64_CPP + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +/* including simdjson/ppc64.h: #include */ +/* begin file simdjson/ppc64.h */ +#ifndef SIMDJSON_PPC64_H +#define SIMDJSON_PPC64_H + +/* including simdjson/ppc64/begin.h: #include "simdjson/ppc64/begin.h" */ +/* begin file simdjson/ppc64/begin.h */ +/* defining SIMDJSON_IMPLEMENTATION to "ppc64" */ +#define SIMDJSON_IMPLEMENTATION ppc64 +/* including simdjson/ppc64/base.h: #include "simdjson/ppc64/base.h" */ +/* begin file simdjson/ppc64/base.h */ +#ifndef SIMDJSON_PPC64_BASE_H +#define SIMDJSON_PPC64_BASE_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/base.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + /** + * Implementation for ALTIVEC (PPC64). + */ + namespace ppc64 { + + class implementation; + + namespace { + namespace simd { + template struct simd8; + template struct simd8x64; + } // namespace simd + } // unnamed namespace + + } // namespace ppc64 +} // namespace simdjson + +#endif // SIMDJSON_PPC64_BASE_H +/* end file simdjson/ppc64/base.h */ +/* including simdjson/ppc64/intrinsics.h: #include "simdjson/ppc64/intrinsics.h" */ +/* begin file simdjson/ppc64/intrinsics.h */ +#ifndef SIMDJSON_PPC64_INTRINSICS_H +#define SIMDJSON_PPC64_INTRINSICS_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/ppc64/base.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +// This should be the correct header whether +// you use visual studio or other compilers. +#include + +// These are defined by altivec.h in GCC toolchain, it is safe to undef them. +#ifdef bool +#undef bool +#endif + +#ifdef vector +#undef vector +#endif + +static_assert(sizeof(__vector unsigned char) <= simdjson::SIMDJSON_PADDING, "insufficient padding for ppc64"); + +#endif // SIMDJSON_PPC64_INTRINSICS_H +/* end file simdjson/ppc64/intrinsics.h */ +/* including simdjson/ppc64/bitmanipulation.h: #include "simdjson/ppc64/bitmanipulation.h" */ +/* begin file simdjson/ppc64/bitmanipulation.h */ +#ifndef SIMDJSON_PPC64_BITMANIPULATION_H +#define SIMDJSON_PPC64_BITMANIPULATION_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/ppc64/base.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace ppc64 { + namespace { + + // We sometimes call trailing_zero on inputs that are zero, + // but the algorithms do not end up using the returned value. + // Sadly, sanitizers are not smart enough to figure it out. + SIMDJSON_NO_SANITIZE_UNDEFINED + // This function can be used safely even if not all bytes have been + // initialized. + // See issue https://github.com/simdjson/simdjson/issues/1965 + SIMDJSON_NO_SANITIZE_MEMORY + simdjson_inline int trailing_zeroes(uint64_t input_num) { +#if SIMDJSON_REGULAR_VISUAL_STUDIO + unsigned long ret; + // Search the mask data from least significant bit (LSB) + // to the most significant bit (MSB) for a set bit (1). + _BitScanForward64(&ret, input_num); + return (int)ret; +#else // SIMDJSON_REGULAR_VISUAL_STUDIO + return __builtin_ctzll(input_num); +#endif // SIMDJSON_REGULAR_VISUAL_STUDIO + } + + /* result might be undefined when input_num is zero */ + simdjson_inline uint64_t clear_lowest_bit(uint64_t input_num) { + return input_num & (input_num - 1); + } + + /* result might be undefined when input_num is zero */ + simdjson_inline int leading_zeroes(uint64_t input_num) { +#if SIMDJSON_REGULAR_VISUAL_STUDIO + unsigned long leading_zero = 0; + // Search the mask data from most significant bit (MSB) + // to least significant bit (LSB) for a set bit (1). + if (_BitScanReverse64(&leading_zero, input_num)) + return (int)(63 - leading_zero); + else + return 64; +#else + return __builtin_clzll(input_num); +#endif // SIMDJSON_REGULAR_VISUAL_STUDIO + } + +#if SIMDJSON_REGULAR_VISUAL_STUDIO + simdjson_inline int count_ones(uint64_t input_num) { + // note: we do not support legacy 32-bit Windows in this kernel + return __popcnt64(input_num); // Visual Studio wants two underscores + } +#else + simdjson_inline int count_ones(uint64_t input_num) { + return __builtin_popcountll(input_num); + } +#endif + + simdjson_inline bool add_overflow(uint64_t value1, uint64_t value2, + uint64_t* result) { +#if SIMDJSON_REGULAR_VISUAL_STUDIO + * result = value1 + value2; + return *result < value1; +#else + return __builtin_uaddll_overflow(value1, value2, + reinterpret_cast(result)); +#endif + } + + } // unnamed namespace + } // namespace ppc64 +} // namespace simdjson + +#endif // SIMDJSON_PPC64_BITMANIPULATION_H +/* end file simdjson/ppc64/bitmanipulation.h */ +/* including simdjson/ppc64/bitmask.h: #include "simdjson/ppc64/bitmask.h" */ +/* begin file simdjson/ppc64/bitmask.h */ +#ifndef SIMDJSON_PPC64_BITMASK_H +#define SIMDJSON_PPC64_BITMASK_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/ppc64/base.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace ppc64 { + namespace { + + // + // Perform a "cumulative bitwise xor," flipping bits each time a 1 is + // encountered. + // + // For example, prefix_xor(00100100) == 00011100 + // + simdjson_inline uint64_t prefix_xor(uint64_t bitmask) { + // You can use the version below, however gcc sometimes miscompiles + // vec_pmsum_be, it happens somewhere around between 8 and 9th version. + // The performance boost was not noticeable, falling back to a usual + // implementation. + // __vector unsigned long long all_ones = {~0ull, ~0ull}; + // __vector unsigned long long mask = {bitmask, 0}; + // // Clang and GCC return different values for pmsum for ull so cast it to one. + // // Generally it is not specified by ALTIVEC ISA what is returned by + // // vec_pmsum_be. + // #if defined(__LITTLE_ENDIAN__) + // return (uint64_t)(((__vector unsigned long long)vec_pmsum_be(all_ones, mask))[0]); + // #else + // return (uint64_t)(((__vector unsigned long long)vec_pmsum_be(all_ones, mask))[1]); + // #endif + bitmask ^= bitmask << 1; + bitmask ^= bitmask << 2; + bitmask ^= bitmask << 4; + bitmask ^= bitmask << 8; + bitmask ^= bitmask << 16; + bitmask ^= bitmask << 32; + return bitmask; + } + + } // unnamed namespace + } // namespace ppc64 +} // namespace simdjson + +#endif +/* end file simdjson/ppc64/bitmask.h */ +/* including simdjson/ppc64/numberparsing_defs.h: #include "simdjson/ppc64/numberparsing_defs.h" */ +/* begin file simdjson/ppc64/numberparsing_defs.h */ +#ifndef SIMDJSON_PPC64_NUMBERPARSING_DEFS_H +#define SIMDJSON_PPC64_NUMBERPARSING_DEFS_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/ppc64/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/ppc64/intrinsics.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +#include + +#if defined(__linux__) +#include +#elif defined(__FreeBSD__) +#include +#endif + +namespace simdjson { + namespace ppc64 { + namespace numberparsing { + + // we don't have appropriate instructions, so let us use a scalar function + // credit: https://johnnylee-sde.github.io/Fast-numeric-string-to-int/ + /** @private */ + static simdjson_inline uint32_t parse_eight_digits_unrolled(const uint8_t* chars) { + uint64_t val; + std::memcpy(&val, chars, sizeof(uint64_t)); +#ifdef __BIG_ENDIAN__ +#if defined(__linux__) + val = bswap_64(val); +#elif defined(__FreeBSD__) + val = bswap64(val); +#endif +#endif + val = (val & 0x0F0F0F0F0F0F0F0F) * 2561 >> 8; + val = (val & 0x00FF00FF00FF00FF) * 6553601 >> 16; + return uint32_t((val & 0x0000FFFF0000FFFF) * 42949672960001 >> 32); + } + + /** @private */ + simdjson_inline internal::value128 full_multiplication(uint64_t value1, uint64_t value2) { + internal::value128 answer; +#if SIMDJSON_REGULAR_VISUAL_STUDIO || SIMDJSON_IS_32BITS +#if SIMDJSON_IS_ARM64 + // ARM64 has native support for 64-bit multiplications, no need to emultate + answer.high = __umulh(value1, value2); + answer.low = value1 * value2; +#else + answer.low = _umul128(value1, value2, &answer.high); // _umul128 not available on ARM64 +#endif // SIMDJSON_IS_ARM64 +#else // SIMDJSON_REGULAR_VISUAL_STUDIO || SIMDJSON_IS_32BITS + __uint128_t r = (static_cast<__uint128_t>(value1)) * value2; + answer.low = uint64_t(r); + answer.high = uint64_t(r >> 64); +#endif + return answer; + } + + } // namespace numberparsing + } // namespace ppc64 +} // namespace simdjson + +#define SIMDJSON_SWAR_NUMBER_PARSING 1 + +#endif // SIMDJSON_PPC64_NUMBERPARSING_DEFS_H +/* end file simdjson/ppc64/numberparsing_defs.h */ +/* including simdjson/ppc64/simd.h: #include "simdjson/ppc64/simd.h" */ +/* begin file simdjson/ppc64/simd.h */ +#ifndef SIMDJSON_PPC64_SIMD_H +#define SIMDJSON_PPC64_SIMD_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/ppc64/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/ppc64/bitmanipulation.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/simdprune_tables.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +#include + +namespace simdjson { + namespace ppc64 { + namespace { + namespace simd { + + using __m128i = __vector unsigned char; + + template struct base { + __m128i value; + + // Zero constructor + simdjson_inline base() : value{ __m128i() } {} + + // Conversion from SIMD register + simdjson_inline base(const __m128i _value) : value(_value) {} + + // Conversion to SIMD register + simdjson_inline operator const __m128i& () const { + return this->value; + } + simdjson_inline operator __m128i& () { return this->value; } + + // Bit operations + simdjson_inline Child operator|(const Child other) const { + return vec_or(this->value, (__m128i)other); + } + simdjson_inline Child operator&(const Child other) const { + return vec_and(this->value, (__m128i)other); + } + simdjson_inline Child operator^(const Child other) const { + return vec_xor(this->value, (__m128i)other); + } + simdjson_inline Child bit_andnot(const Child other) const { + return vec_andc(this->value, (__m128i)other); + } + simdjson_inline Child& operator|=(const Child other) { + auto this_cast = static_cast(this); + *this_cast = *this_cast | other; + return *this_cast; + } + simdjson_inline Child& operator&=(const Child other) { + auto this_cast = static_cast(this); + *this_cast = *this_cast & other; + return *this_cast; + } + simdjson_inline Child& operator^=(const Child other) { + auto this_cast = static_cast(this); + *this_cast = *this_cast ^ other; + return *this_cast; + } + }; + + template > + struct base8 : base> { + typedef uint16_t bitmask_t; + typedef uint32_t bitmask2_t; + + simdjson_inline base8() : base>() {} + simdjson_inline base8(const __m128i _value) : base>(_value) {} + + friend simdjson_inline Mask operator==(const simd8 lhs, const simd8 rhs) { + return (__m128i)vec_cmpeq(lhs.value, (__m128i)rhs); + } + + static const int SIZE = sizeof(base>::value); + + template + simdjson_inline simd8 prev(simd8 prev_chunk) const { + __m128i chunk = this->value; +#ifdef __LITTLE_ENDIAN__ + chunk = (__m128i)vec_reve(this->value); + prev_chunk = (__m128i)vec_reve((__m128i)prev_chunk); +#endif + chunk = (__m128i)vec_sld((__m128i)prev_chunk, (__m128i)chunk, 16 - N); +#ifdef __LITTLE_ENDIAN__ + chunk = (__m128i)vec_reve((__m128i)chunk); +#endif + return chunk; + } + }; + + // SIMD byte mask type (returned by things like eq and gt) + template <> struct simd8 : base8 { + static simdjson_inline simd8 splat(bool _value) { + return (__m128i)vec_splats((unsigned char)(-(!!_value))); + } + + simdjson_inline simd8() : base8() {} + simdjson_inline simd8(const __m128i _value) + : base8(_value) {} + // Splat constructor + simdjson_inline simd8(bool _value) + : base8(splat(_value)) {} + + simdjson_inline int to_bitmask() const { + __vector unsigned long long result; + const __m128i perm_mask = { 0x78, 0x70, 0x68, 0x60, 0x58, 0x50, 0x48, 0x40, + 0x38, 0x30, 0x28, 0x20, 0x18, 0x10, 0x08, 0x00 }; + + result = ((__vector unsigned long long)vec_vbpermq((__m128i)this->value, + (__m128i)perm_mask)); +#ifdef __LITTLE_ENDIAN__ + return static_cast(result[1]); +#else + return static_cast(result[0]); +#endif + } + simdjson_inline bool any() const { + return !vec_all_eq(this->value, (__m128i)vec_splats(0)); + } + simdjson_inline simd8 operator~() const { + return this->value ^ (__m128i)splat(true); + } + }; + + template struct base8_numeric : base8 { + static simdjson_inline simd8 splat(T value) { + (void)value; + return (__m128i)vec_splats(value); + } + static simdjson_inline simd8 zero() { return splat(0); } + static simdjson_inline simd8 load(const T values[16]) { + return (__m128i)(vec_vsx_ld(0, reinterpret_cast(values))); + } + // Repeat 16 values as many times as necessary (usually for lookup tables) + static simdjson_inline simd8 repeat_16(T v0, T v1, T v2, T v3, T v4, + T v5, T v6, T v7, T v8, T v9, + T v10, T v11, T v12, T v13, + T v14, T v15) { + return simd8(v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, + v14, v15); + } + + simdjson_inline base8_numeric() : base8() {} + simdjson_inline base8_numeric(const __m128i _value) + : base8(_value) {} + + // Store to array + simdjson_inline void store(T dst[16]) const { + vec_vsx_st(this->value, 0, reinterpret_cast<__m128i*>(dst)); + } + + // Override to distinguish from bool version + simdjson_inline simd8 operator~() const { return *this ^ 0xFFu; } + + // Addition/subtraction are the same for signed and unsigned + simdjson_inline simd8 operator+(const simd8 other) const { + return (__m128i)((__m128i)this->value + (__m128i)other); + } + simdjson_inline simd8 operator-(const simd8 other) const { + return (__m128i)((__m128i)this->value - (__m128i)other); + } + simdjson_inline simd8& operator+=(const simd8 other) { + *this = *this + other; + return *static_cast *>(this); + } + simdjson_inline simd8& operator-=(const simd8 other) { + *this = *this - other; + return *static_cast *>(this); + } + + // Perform a lookup assuming the value is between 0 and 16 (undefined behavior + // for out of range values) + template + simdjson_inline simd8 lookup_16(simd8 lookup_table) const { + return (__m128i)vec_perm((__m128i)lookup_table, (__m128i)lookup_table, this->value); + } + + // Copies to 'output" all bytes corresponding to a 0 in the mask (interpreted + // as a bitset). Passing a 0 value for mask would be equivalent to writing out + // every byte to output. Only the first 16 - count_ones(mask) bytes of the + // result are significant but 16 bytes get written. Design consideration: it + // seems like a function with the signature simd8 compress(uint32_t mask) + // would be sensible, but the AVX ISA makes this kind of approach difficult. + template + simdjson_inline void compress(uint16_t mask, L* output) const { + using internal::BitsSetTable256mul2; + using internal::pshufb_combine_table; + using internal::thintable_epi8; + // this particular implementation was inspired by work done by @animetosho + // we do it in two steps, first 8 bytes and then second 8 bytes + uint8_t mask1 = uint8_t(mask); // least significant 8 bits + uint8_t mask2 = uint8_t(mask >> 8); // most significant 8 bits + // next line just loads the 64-bit values thintable_epi8[mask1] and + // thintable_epi8[mask2] into a 128-bit register, using only + // two instructions on most compilers. +#ifdef __LITTLE_ENDIAN__ + __m128i shufmask = (__m128i)(__vector unsigned long long) { + thintable_epi8[mask1], thintable_epi8[mask2] + }; +#else + __m128i shufmask = (__m128i)(__vector unsigned long long) { + thintable_epi8[mask2], thintable_epi8[mask1] + }; + shufmask = (__m128i)vec_reve((__m128i)shufmask); +#endif + // we increment by 0x08 the second half of the mask + shufmask = ((__m128i)shufmask) + + ((__m128i)(__vector int) { 0, 0, 0x08080808, 0x08080808 }); + + // this is the version "nearly pruned" + __m128i pruned = vec_perm(this->value, this->value, shufmask); + // we still need to put the two halves together. + // we compute the popcount of the first half: + int pop1 = BitsSetTable256mul2[mask1]; + // then load the corresponding mask, what it does is to write + // only the first pop1 bytes from the first 8 bytes, and then + // it fills in with the bytes from the second 8 bytes + some filling + // at the end. + __m128i compactmask = + vec_vsx_ld(0, reinterpret_cast(pshufb_combine_table + pop1 * 8)); + __m128i answer = vec_perm(pruned, (__m128i)vec_splats(0), compactmask); + vec_vsx_st(answer, 0, reinterpret_cast<__m128i*>(output)); + } + + template + simdjson_inline simd8 + lookup_16(L replace0, L replace1, L replace2, L replace3, L replace4, + L replace5, L replace6, L replace7, L replace8, L replace9, + L replace10, L replace11, L replace12, L replace13, L replace14, + L replace15) const { + return lookup_16(simd8::repeat_16( + replace0, replace1, replace2, replace3, replace4, replace5, replace6, + replace7, replace8, replace9, replace10, replace11, replace12, + replace13, replace14, replace15)); + } + }; + + // Signed bytes + template <> struct simd8 : base8_numeric { + simdjson_inline simd8() : base8_numeric() {} + simdjson_inline simd8(const __m128i _value) + : base8_numeric(_value) {} + // Splat constructor + simdjson_inline simd8(int8_t _value) : simd8(splat(_value)) {} + // Array constructor + simdjson_inline simd8(const int8_t* values) : simd8(load(values)) {} + // Member-by-member initialization + simdjson_inline simd8(int8_t v0, int8_t v1, int8_t v2, int8_t v3, + int8_t v4, int8_t v5, int8_t v6, int8_t v7, + int8_t v8, int8_t v9, int8_t v10, int8_t v11, + int8_t v12, int8_t v13, int8_t v14, int8_t v15) + : simd8((__m128i)(__vector signed char) { + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, + v15 + }) {} + // Repeat 16 values as many times as necessary (usually for lookup tables) + simdjson_inline static simd8 + repeat_16(int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, + int8_t v6, int8_t v7, int8_t v8, int8_t v9, int8_t v10, int8_t v11, + int8_t v12, int8_t v13, int8_t v14, int8_t v15) { + return simd8(v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, + v13, v14, v15); + } + + // Order-sensitive comparisons + simdjson_inline simd8 + max_val(const simd8 other) const { + return (__m128i)vec_max((__vector signed char)this->value, + (__vector signed char)(__m128i)other); + } + simdjson_inline simd8 + min_val(const simd8 other) const { + return (__m128i)vec_min((__vector signed char)this->value, + (__vector signed char)(__m128i)other); + } + simdjson_inline simd8 + operator>(const simd8 other) const { + return (__m128i)vec_cmpgt((__vector signed char)this->value, + (__vector signed char)(__m128i)other); + } + simdjson_inline simd8 + operator<(const simd8 other) const { + return (__m128i)vec_cmplt((__vector signed char)this->value, + (__vector signed char)(__m128i)other); + } + }; + + // Unsigned bytes + template <> struct simd8 : base8_numeric { + simdjson_inline simd8() : base8_numeric() {} + simdjson_inline simd8(const __m128i _value) + : base8_numeric(_value) {} + // Splat constructor + simdjson_inline simd8(uint8_t _value) : simd8(splat(_value)) {} + // Array constructor + simdjson_inline simd8(const uint8_t* values) : simd8(load(values)) {} + // Member-by-member initialization + simdjson_inline + simd8(uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, + uint8_t v6, uint8_t v7, uint8_t v8, uint8_t v9, uint8_t v10, + uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15) + : simd8((__m128i) { + v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, + v13, v14, v15 + }) {} + // Repeat 16 values as many times as necessary (usually for lookup tables) + simdjson_inline static simd8 + repeat_16(uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, + uint8_t v5, uint8_t v6, uint8_t v7, uint8_t v8, uint8_t v9, + uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, + uint8_t v15) { + return simd8(v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, + v13, v14, v15); + } + + // Saturated math + simdjson_inline simd8 + saturating_add(const simd8 other) const { + return (__m128i)vec_adds(this->value, (__m128i)other); + } + simdjson_inline simd8 + saturating_sub(const simd8 other) const { + return (__m128i)vec_subs(this->value, (__m128i)other); + } + + // Order-specific operations + simdjson_inline simd8 + max_val(const simd8 other) const { + return (__m128i)vec_max(this->value, (__m128i)other); + } + simdjson_inline simd8 + min_val(const simd8 other) const { + return (__m128i)vec_min(this->value, (__m128i)other); + } + // Same as >, but only guarantees true is nonzero (< guarantees true = -1) + simdjson_inline simd8 + gt_bits(const simd8 other) const { + return this->saturating_sub(other); + } + // Same as <, but only guarantees true is nonzero (< guarantees true = -1) + simdjson_inline simd8 + lt_bits(const simd8 other) const { + return other.saturating_sub(*this); + } + simdjson_inline simd8 + operator<=(const simd8 other) const { + return other.max_val(*this) == other; + } + simdjson_inline simd8 + operator>=(const simd8 other) const { + return other.min_val(*this) == other; + } + simdjson_inline simd8 + operator>(const simd8 other) const { + return this->gt_bits(other).any_bits_set(); + } + simdjson_inline simd8 + operator<(const simd8 other) const { + return this->gt_bits(other).any_bits_set(); + } + + // Bit-specific operations + simdjson_inline simd8 bits_not_set() const { + return (__m128i)vec_cmpeq(this->value, (__m128i)vec_splats(uint8_t(0))); + } + simdjson_inline simd8 bits_not_set(simd8 bits) const { + return (*this & bits).bits_not_set(); + } + simdjson_inline simd8 any_bits_set() const { + return ~this->bits_not_set(); + } + simdjson_inline simd8 any_bits_set(simd8 bits) const { + return ~this->bits_not_set(bits); + } + simdjson_inline bool bits_not_set_anywhere() const { + return vec_all_eq(this->value, (__m128i)vec_splats(0)); + } + simdjson_inline bool any_bits_set_anywhere() const { + return !bits_not_set_anywhere(); + } + simdjson_inline bool bits_not_set_anywhere(simd8 bits) const { + return vec_all_eq(vec_and(this->value, (__m128i)bits), + (__m128i)vec_splats(0)); + } + simdjson_inline bool any_bits_set_anywhere(simd8 bits) const { + return !bits_not_set_anywhere(bits); + } + template simdjson_inline simd8 shr() const { + return simd8( + (__m128i)vec_sr(this->value, (__m128i)vec_splat_u8(N))); + } + template simdjson_inline simd8 shl() const { + return simd8( + (__m128i)vec_sl(this->value, (__m128i)vec_splat_u8(N))); + } + }; + + template struct simd8x64 { + static constexpr int NUM_CHUNKS = 64 / sizeof(simd8); + static_assert(NUM_CHUNKS == 4, + "PPC64 kernel should use four registers per 64-byte block."); + const simd8 chunks[NUM_CHUNKS]; + + simd8x64(const simd8x64& o) = delete; // no copy allowed + simd8x64& + operator=(const simd8& other) = delete; // no assignment allowed + simd8x64() = delete; // no default constructor allowed + + simdjson_inline simd8x64(const simd8 chunk0, const simd8 chunk1, + const simd8 chunk2, const simd8 chunk3) + : chunks{ chunk0, chunk1, chunk2, chunk3 } {} + simdjson_inline simd8x64(const T ptr[64]) + : chunks{ simd8::load(ptr), simd8::load(ptr + 16), + simd8::load(ptr + 32), simd8::load(ptr + 48) } {} + + simdjson_inline void store(T ptr[64]) const { + this->chunks[0].store(ptr + sizeof(simd8) * 0); + this->chunks[1].store(ptr + sizeof(simd8) * 1); + this->chunks[2].store(ptr + sizeof(simd8) * 2); + this->chunks[3].store(ptr + sizeof(simd8) * 3); + } + + simdjson_inline simd8 reduce_or() const { + return (this->chunks[0] | this->chunks[1]) | + (this->chunks[2] | this->chunks[3]); + } + + simdjson_inline uint64_t compress(uint64_t mask, T* output) const { + this->chunks[0].compress(uint16_t(mask), output); + this->chunks[1].compress(uint16_t(mask >> 16), + output + 16 - count_ones(mask & 0xFFFF)); + this->chunks[2].compress(uint16_t(mask >> 32), + output + 32 - count_ones(mask & 0xFFFFFFFF)); + this->chunks[3].compress(uint16_t(mask >> 48), + output + 48 - count_ones(mask & 0xFFFFFFFFFFFF)); + return 64 - count_ones(mask); + } + + simdjson_inline uint64_t to_bitmask() const { + uint64_t r0 = uint32_t(this->chunks[0].to_bitmask()); + uint64_t r1 = this->chunks[1].to_bitmask(); + uint64_t r2 = this->chunks[2].to_bitmask(); + uint64_t r3 = this->chunks[3].to_bitmask(); + return r0 | (r1 << 16) | (r2 << 32) | (r3 << 48); + } + + simdjson_inline uint64_t eq(const T m) const { + const simd8 mask = simd8::splat(m); + return simd8x64(this->chunks[0] == mask, this->chunks[1] == mask, + this->chunks[2] == mask, this->chunks[3] == mask) + .to_bitmask(); + } + + simdjson_inline uint64_t eq(const simd8x64& other) const { + return simd8x64(this->chunks[0] == other.chunks[0], + this->chunks[1] == other.chunks[1], + this->chunks[2] == other.chunks[2], + this->chunks[3] == other.chunks[3]) + .to_bitmask(); + } + + simdjson_inline uint64_t lteq(const T m) const { + const simd8 mask = simd8::splat(m); + return simd8x64(this->chunks[0] <= mask, this->chunks[1] <= mask, + this->chunks[2] <= mask, this->chunks[3] <= mask) + .to_bitmask(); + } + }; // struct simd8x64 + + } // namespace simd + } // unnamed namespace + } // namespace ppc64 +} // namespace simdjson + +#endif // SIMDJSON_PPC64_SIMD_INPUT_H +/* end file simdjson/ppc64/simd.h */ +/* including simdjson/ppc64/stringparsing_defs.h: #include "simdjson/ppc64/stringparsing_defs.h" */ +/* begin file simdjson/ppc64/stringparsing_defs.h */ +#ifndef SIMDJSON_PPC64_STRINGPARSING_DEFS_H +#define SIMDJSON_PPC64_STRINGPARSING_DEFS_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/ppc64/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/ppc64/bitmanipulation.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/ppc64/simd.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace ppc64 { + namespace { + + using namespace simd; + + // Holds backslashes and quotes locations. + struct backslash_and_quote { + public: + static constexpr uint32_t BYTES_PROCESSED = 32; + simdjson_inline static backslash_and_quote + copy_and_find(const uint8_t* src, uint8_t* dst); + + simdjson_inline bool has_quote_first() { + return ((bs_bits - 1) & quote_bits) != 0; + } + simdjson_inline bool has_backslash() { return bs_bits != 0; } + simdjson_inline int quote_index() { + return trailing_zeroes(quote_bits); + } + simdjson_inline int backslash_index() { + return trailing_zeroes(bs_bits); + } + + uint32_t bs_bits; + uint32_t quote_bits; + }; // struct backslash_and_quote + + simdjson_inline backslash_and_quote + backslash_and_quote::copy_and_find(const uint8_t* src, uint8_t* dst) { + // this can read up to 31 bytes beyond the buffer size, but we require + // SIMDJSON_PADDING of padding + static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), + "backslash and quote finder must process fewer than " + "SIMDJSON_PADDING bytes"); + simd8 v0(src); + simd8 v1(src + sizeof(v0)); + v0.store(dst); + v1.store(dst + sizeof(v0)); + + // Getting a 64-bit bitmask is much cheaper than multiple 16-bit bitmasks on + // PPC; therefore, we smash them together into a 64-byte mask and get the + // bitmask from there. + uint64_t bs_and_quote = + simd8x64(v0 == '\\', v1 == '\\', v0 == '"', v1 == '"').to_bitmask(); + return { + uint32_t(bs_and_quote), // bs_bits + uint32_t(bs_and_quote >> 32) // quote_bits + }; + } + + } // unnamed namespace + } // namespace ppc64 +} // namespace simdjson + +#endif // SIMDJSON_PPC64_STRINGPARSING_DEFS_H +/* end file simdjson/ppc64/stringparsing_defs.h */ + +#define SIMDJSON_SKIP_BACKSLASH_SHORT_CIRCUIT 1 +/* end file simdjson/ppc64/begin.h */ +/* including simdjson/generic/amalgamated.h for ppc64: #include "simdjson/generic/amalgamated.h" */ +/* begin file simdjson/generic/amalgamated.h for ppc64 */ +#if defined(SIMDJSON_CONDITIONAL_INCLUDE) && !defined(SIMDJSON_GENERIC_DEPENDENCIES_H) +#error simdjson/generic/dependencies.h must be included before simdjson/generic/amalgamated.h! +#endif + +/* including simdjson/generic/base.h for ppc64: #include "simdjson/generic/base.h" */ +/* begin file simdjson/generic/base.h for ppc64 */ +#ifndef SIMDJSON_GENERIC_BASE_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_BASE_H */ +/* amalgamation skipped (editor-only): #include "simdjson/base.h" */ +/* amalgamation skipped (editor-only): // If we haven't got an implementation yet, we're in the editor, editing a generic file! Just */ +/* amalgamation skipped (editor-only): // use the most advanced one we can so the most possible stuff can be tested. */ +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_IMPLEMENTATION */ +/* amalgamation skipped (editor-only): #include "simdjson/implementation_detection.h" */ +/* amalgamation skipped (editor-only): #if SIMDJSON_IMPLEMENTATION_ICELAKE */ +/* amalgamation skipped (editor-only): #include "simdjson/icelake/begin.h" */ +/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_HASWELL */ +/* amalgamation skipped (editor-only): #include "simdjson/haswell/begin.h" */ +/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_WESTMERE */ +/* amalgamation skipped (editor-only): #include "simdjson/westmere/begin.h" */ +/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_ARM64 */ +/* amalgamation skipped (editor-only): #include "simdjson/arm64/begin.h" */ +/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_PPC64 */ +/* amalgamation skipped (editor-only): #include "simdjson/ppc64/begin.h" */ +/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_LSX */ +/* amalgamation skipped (editor-only): #include "simdjson/lsx/begin.h" */ +/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_LASX */ +/* amalgamation skipped (editor-only): #include "simdjson/lasx/begin.h" */ +/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_FALLBACK */ +/* amalgamation skipped (editor-only): #include "simdjson/fallback/begin.h" */ +/* amalgamation skipped (editor-only): #else */ +/* amalgamation skipped (editor-only): #error "All possible implementations (including fallback) have been disabled! simdjson will not run." */ +/* amalgamation skipped (editor-only): #endif */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_IMPLEMENTATION */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace ppc64 { + + struct open_container; + class dom_parser_implementation; + + /** + * The type of a JSON number + */ + enum class number_type { + floating_point_number = 1, /// a binary64 number + signed_integer, /// a signed integer that fits in a 64-bit word using two's complement + unsigned_integer, /// a positive integer larger or equal to 1<<63 + big_integer /// a big integer that does not fit in a 64-bit word + }; + + } // namespace ppc64 +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_BASE_H +/* end file simdjson/generic/base.h for ppc64 */ +/* including simdjson/generic/jsoncharutils.h for ppc64: #include "simdjson/generic/jsoncharutils.h" */ +/* begin file simdjson/generic/jsoncharutils.h for ppc64 */ +#ifndef SIMDJSON_GENERIC_JSONCHARUTILS_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_JSONCHARUTILS_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/jsoncharutils_tables.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace ppc64 { + namespace { + namespace jsoncharutils { + + // return non-zero if not a structural or whitespace char + // zero otherwise + simdjson_inline uint32_t is_not_structural_or_whitespace(uint8_t c) { + return internal::structural_or_whitespace_negated[c]; + } + + simdjson_inline uint32_t is_structural_or_whitespace(uint8_t c) { + return internal::structural_or_whitespace[c]; + } + + // returns a value with the high 16 bits set if not valid + // otherwise returns the conversion of the 4 hex digits at src into the bottom + // 16 bits of the 32-bit return register + // + // see + // https://lemire.me/blog/2019/04/17/parsing-short-hexadecimal-strings-efficiently/ + static inline uint32_t hex_to_u32_nocheck( + const uint8_t* src) { // strictly speaking, static inline is a C-ism + uint32_t v1 = internal::digit_to_val32[630 + src[0]]; + uint32_t v2 = internal::digit_to_val32[420 + src[1]]; + uint32_t v3 = internal::digit_to_val32[210 + src[2]]; + uint32_t v4 = internal::digit_to_val32[0 + src[3]]; + return v1 | v2 | v3 | v4; + } + + // given a code point cp, writes to c + // the utf-8 code, outputting the length in + // bytes, if the length is zero, the code point + // is invalid + // + // This can possibly be made faster using pdep + // and clz and table lookups, but JSON documents + // have few escaped code points, and the following + // function looks cheap. + // + // Note: we assume that surrogates are treated separately + // + simdjson_inline size_t codepoint_to_utf8(uint32_t cp, uint8_t* c) { + if (cp <= 0x7F) { + c[0] = uint8_t(cp); + return 1; // ascii + } + if (cp <= 0x7FF) { + c[0] = uint8_t((cp >> 6) + 192); + c[1] = uint8_t((cp & 63) + 128); + return 2; // universal plane + // Surrogates are treated elsewhere... + //} //else if (0xd800 <= cp && cp <= 0xdfff) { + // return 0; // surrogates // could put assert here + } + else if (cp <= 0xFFFF) { + c[0] = uint8_t((cp >> 12) + 224); + c[1] = uint8_t(((cp >> 6) & 63) + 128); + c[2] = uint8_t((cp & 63) + 128); + return 3; + } + else if (cp <= 0x10FFFF) { // if you know you have a valid code point, this + // is not needed + c[0] = uint8_t((cp >> 18) + 240); + c[1] = uint8_t(((cp >> 12) & 63) + 128); + c[2] = uint8_t(((cp >> 6) & 63) + 128); + c[3] = uint8_t((cp & 63) + 128); + return 4; + } + // will return 0 when the code point was too large. + return 0; // bad r + } + +#if SIMDJSON_IS_32BITS // _umul128 for x86, arm + // this is a slow emulation routine for 32-bit + // + static simdjson_inline uint64_t __emulu(uint32_t x, uint32_t y) { + return x * (uint64_t)y; + } + static simdjson_inline uint64_t _umul128(uint64_t ab, uint64_t cd, uint64_t* hi) { + uint64_t ad = __emulu((uint32_t)(ab >> 32), (uint32_t)cd); + uint64_t bd = __emulu((uint32_t)ab, (uint32_t)cd); + uint64_t adbc = ad + __emulu((uint32_t)ab, (uint32_t)(cd >> 32)); + uint64_t adbc_carry = !!(adbc < ad); + uint64_t lo = bd + (adbc << 32); + *hi = __emulu((uint32_t)(ab >> 32), (uint32_t)(cd >> 32)) + (adbc >> 32) + + (adbc_carry << 32) + !!(lo < bd); + return lo; + } +#endif + + } // namespace jsoncharutils + } // unnamed namespace + } // namespace ppc64 +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_JSONCHARUTILS_H +/* end file simdjson/generic/jsoncharutils.h for ppc64 */ +/* including simdjson/generic/atomparsing.h for ppc64: #include "simdjson/generic/atomparsing.h" */ +/* begin file simdjson/generic/atomparsing.h for ppc64 */ +#ifndef SIMDJSON_GENERIC_ATOMPARSING_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_ATOMPARSING_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/jsoncharutils.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +#include + +namespace simdjson { + namespace ppc64 { + namespace { + /// @private + namespace atomparsing { + + // The string_to_uint32 is exclusively used to map literal strings to 32-bit values. + // We use memcpy instead of a pointer cast to avoid undefined behaviors since we cannot + // be certain that the character pointer will be properly aligned. + // You might think that using memcpy makes this function expensive, but you'd be wrong. + // All decent optimizing compilers (GCC, clang, Visual Studio) will compile string_to_uint32("false"); + // to the compile-time constant 1936482662. + simdjson_inline uint32_t string_to_uint32(const char* str) { uint32_t val; std::memcpy(&val, str, sizeof(uint32_t)); return val; } + + + // Again in str4ncmp we use a memcpy to avoid undefined behavior. The memcpy may appear expensive. + // Yet all decent optimizing compilers will compile memcpy to a single instruction, just about. + simdjson_warn_unused + simdjson_inline uint32_t str4ncmp(const uint8_t* src, const char* atom) { + uint32_t srcval; // we want to avoid unaligned 32-bit loads (undefined in C/C++) + static_assert(sizeof(uint32_t) <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be larger than 4 bytes"); + std::memcpy(&srcval, src, sizeof(uint32_t)); + return srcval ^ string_to_uint32(atom); + } + + simdjson_warn_unused + simdjson_inline bool is_valid_true_atom(const uint8_t* src) { + return (str4ncmp(src, "true") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; + } + + simdjson_warn_unused + simdjson_inline bool is_valid_true_atom(const uint8_t* src, size_t len) { + if (len > 4) { return is_valid_true_atom(src); } + else if (len == 4) { return !str4ncmp(src, "true"); } + else { return false; } + } + + simdjson_warn_unused + simdjson_inline bool is_valid_false_atom(const uint8_t* src) { + return (str4ncmp(src + 1, "alse") | jsoncharutils::is_not_structural_or_whitespace(src[5])) == 0; + } + + simdjson_warn_unused + simdjson_inline bool is_valid_false_atom(const uint8_t* src, size_t len) { + if (len > 5) { return is_valid_false_atom(src); } + else if (len == 5) { return !str4ncmp(src + 1, "alse"); } + else { return false; } + } + + simdjson_warn_unused + simdjson_inline bool is_valid_null_atom(const uint8_t* src) { + return (str4ncmp(src, "null") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; + } + + simdjson_warn_unused + simdjson_inline bool is_valid_null_atom(const uint8_t* src, size_t len) { + if (len > 4) { return is_valid_null_atom(src); } + else if (len == 4) { return !str4ncmp(src, "null"); } + else { return false; } + } + + } // namespace atomparsing + } // unnamed namespace + } // namespace ppc64 +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_ATOMPARSING_H +/* end file simdjson/generic/atomparsing.h for ppc64 */ +/* including simdjson/generic/dom_parser_implementation.h for ppc64: #include "simdjson/generic/dom_parser_implementation.h" */ +/* begin file simdjson/generic/dom_parser_implementation.h for ppc64 */ +#ifndef SIMDJSON_GENERIC_DOM_PARSER_IMPLEMENTATION_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_DOM_PARSER_IMPLEMENTATION_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/dom_parser_implementation.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace ppc64 { + + // expectation: sizeof(open_container) = 64/8. + struct open_container { + uint32_t tape_index; // where, on the tape, does the scope ([,{) begins + uint32_t count; // how many elements in the scope + }; // struct open_container + + static_assert(sizeof(open_container) == 64 / 8, "Open container must be 64 bits"); + + class dom_parser_implementation final : public internal::dom_parser_implementation { + public: + /** Tape location of each open { or [ */ + std::unique_ptr open_containers{}; + /** Whether each open container is a [ or { */ + std::unique_ptr is_array{}; + /** Buffer passed to stage 1 */ + const uint8_t* buf{}; + /** Length passed to stage 1 */ + size_t len{ 0 }; + /** Document passed to stage 2 */ + dom::document* doc{}; + + inline dom_parser_implementation() noexcept; + inline dom_parser_implementation(dom_parser_implementation&& other) noexcept; + inline dom_parser_implementation& operator=(dom_parser_implementation&& other) noexcept; + dom_parser_implementation(const dom_parser_implementation&) = delete; + dom_parser_implementation& operator=(const dom_parser_implementation&) = delete; + + simdjson_warn_unused error_code parse(const uint8_t* buf, size_t len, dom::document& doc) noexcept final; + simdjson_warn_unused error_code stage1(const uint8_t* buf, size_t len, stage1_mode partial) noexcept final; + simdjson_warn_unused error_code stage2(dom::document& doc) noexcept final; + simdjson_warn_unused error_code stage2_next(dom::document& doc) noexcept final; + simdjson_warn_unused uint8_t* parse_string(const uint8_t* src, uint8_t* dst, bool allow_replacement) const noexcept final; + simdjson_warn_unused uint8_t* parse_wobbly_string(const uint8_t* src, uint8_t* dst) const noexcept final; + inline simdjson_warn_unused error_code set_capacity(size_t capacity) noexcept final; + inline simdjson_warn_unused error_code set_max_depth(size_t max_depth) noexcept final; + private: + simdjson_inline simdjson_warn_unused error_code set_capacity_stage1(size_t capacity); + + }; + + } // namespace ppc64 +} // namespace simdjson + +namespace simdjson { + namespace ppc64 { + + inline dom_parser_implementation::dom_parser_implementation() noexcept = default; + inline dom_parser_implementation::dom_parser_implementation(dom_parser_implementation&& other) noexcept = default; + inline dom_parser_implementation& dom_parser_implementation::operator=(dom_parser_implementation&& other) noexcept = default; + + // Leaving these here so they can be inlined if so desired + inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(size_t capacity) noexcept { + if (capacity > SIMDJSON_MAXSIZE_BYTES) { return CAPACITY; } + // Stage 1 index output + size_t max_structures = SIMDJSON_ROUNDUP_N(capacity, 64) + 2 + 7; + structural_indexes.reset(new (std::nothrow) uint32_t[max_structures]); + if (!structural_indexes) { _capacity = 0; return MEMALLOC; } + structural_indexes[0] = 0; + n_structural_indexes = 0; + + _capacity = capacity; + return SUCCESS; + } + + inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { + // Stage 2 stacks + open_containers.reset(new (std::nothrow) open_container[max_depth]); + is_array.reset(new (std::nothrow) bool[max_depth]); + if (!is_array || !open_containers) { _max_depth = 0; return MEMALLOC; } + + _max_depth = max_depth; + return SUCCESS; + } + + } // namespace ppc64 +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_DOM_PARSER_IMPLEMENTATION_H +/* end file simdjson/generic/dom_parser_implementation.h for ppc64 */ +/* including simdjson/generic/implementation_simdjson_result_base.h for ppc64: #include "simdjson/generic/implementation_simdjson_result_base.h" */ +/* begin file simdjson/generic/implementation_simdjson_result_base.h for ppc64 */ +#ifndef SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace ppc64 { + + // This is a near copy of include/error.h's implementation_simdjson_result_base, except it doesn't use std::pair + // so we can avoid inlining errors + // TODO reconcile these! + /** + * The result of a simdjson operation that could fail. + * + * Gives the option of reading error codes, or throwing an exception by casting to the desired result. + * + * This is a base class for implementations that want to add functions to the result type for + * chaining. + * + * Override like: + * + * struct simdjson_result : public internal::implementation_simdjson_result_base { + * simdjson_result() noexcept : internal::implementation_simdjson_result_base() {} + * simdjson_result(error_code error) noexcept : internal::implementation_simdjson_result_base(error) {} + * simdjson_result(T &&value) noexcept : internal::implementation_simdjson_result_base(std::forward(value)) {} + * simdjson_result(T &&value, error_code error) noexcept : internal::implementation_simdjson_result_base(value, error) {} + * // Your extra methods here + * } + * + * Then any method returning simdjson_result will be chainable with your methods. + */ + template + struct implementation_simdjson_result_base { + + /** + * Create a new empty result with error = UNINITIALIZED. + */ + simdjson_inline implementation_simdjson_result_base() noexcept = default; + + /** + * Create a new error result. + */ + simdjson_inline implementation_simdjson_result_base(error_code error) noexcept; + + /** + * Create a new successful result. + */ + simdjson_inline implementation_simdjson_result_base(T&& value) noexcept; + + /** + * Create a new result with both things (use if you don't want to branch when creating the result). + */ + simdjson_inline implementation_simdjson_result_base(T&& value, error_code error) noexcept; + + /** + * Move the value and the error to the provided variables. + * + * @param value The variable to assign the value to. May not be set if there is an error. + * @param error The variable to assign the error to. Set to SUCCESS if there is no error. + */ + simdjson_inline void tie(T& value, error_code& error) && noexcept; + + /** + * Move the value to the provided variable. + * + * @param value The variable to assign the value to. May not be set if there is an error. + */ + simdjson_inline error_code get(T& value) && noexcept; + + /** + * The error. + */ + simdjson_inline error_code error() const noexcept; + +#if SIMDJSON_EXCEPTIONS + + /** + * Get the result value. + * + * @throw simdjson_error if there was an error. + */ + simdjson_inline T& value() & noexcept(false); + + /** + * Take the result value (move it). + * + * @throw simdjson_error if there was an error. + */ + simdjson_inline T&& value() && noexcept(false); + + /** + * Take the result value (move it). + * + * @throw simdjson_error if there was an error. + */ + simdjson_inline T&& take_value() && noexcept(false); + + /** + * Cast to the value (will throw on error). + * + * @throw simdjson_error if there was an error. + */ + simdjson_inline operator T && () && noexcept(false); + + +#endif // SIMDJSON_EXCEPTIONS + + /** + * Get the result value. This function is safe if and only + * the error() method returns a value that evaluates to false. + */ + simdjson_inline const T& value_unsafe() const& noexcept; + /** + * Get the result value. This function is safe if and only + * the error() method returns a value that evaluates to false. + */ + simdjson_inline T& value_unsafe() & noexcept; + /** + * Take the result value (move it). This function is safe if and only + * the error() method returns a value that evaluates to false. + */ + simdjson_inline T&& value_unsafe() && noexcept; + protected: + /** users should never directly access first and second. **/ + T first{}; /** Users should never directly access 'first'. **/ + error_code second{ UNINITIALIZED }; /** Users should never directly access 'second'. **/ + }; // struct implementation_simdjson_result_base + + } // namespace ppc64 +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_H +/* end file simdjson/generic/implementation_simdjson_result_base.h for ppc64 */ +/* including simdjson/generic/numberparsing.h for ppc64: #include "simdjson/generic/numberparsing.h" */ +/* begin file simdjson/generic/numberparsing.h for ppc64 */ +#ifndef SIMDJSON_GENERIC_NUMBERPARSING_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_NUMBERPARSING_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/jsoncharutils.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +#include +#include +#include + +namespace simdjson { + namespace ppc64 { + namespace numberparsing { + +#ifdef JSON_TEST_NUMBERS +#define INVALID_NUMBER(SRC) (found_invalid_number((SRC)), NUMBER_ERROR) +#define WRITE_INTEGER(VALUE, SRC, WRITER) (found_integer((VALUE), (SRC)), (WRITER).append_s64((VALUE))) +#define WRITE_UNSIGNED(VALUE, SRC, WRITER) (found_unsigned_integer((VALUE), (SRC)), (WRITER).append_u64((VALUE))) +#define WRITE_DOUBLE(VALUE, SRC, WRITER) (found_float((VALUE), (SRC)), (WRITER).append_double((VALUE))) +#define BIGINT_NUMBER(SRC) (found_invalid_number((SRC)), BIGINT_ERROR) +#else +#define INVALID_NUMBER(SRC) (NUMBER_ERROR) +#define WRITE_INTEGER(VALUE, SRC, WRITER) (WRITER).append_s64((VALUE)) +#define WRITE_UNSIGNED(VALUE, SRC, WRITER) (WRITER).append_u64((VALUE)) +#define WRITE_DOUBLE(VALUE, SRC, WRITER) (WRITER).append_double((VALUE)) +#define BIGINT_NUMBER(SRC) (BIGINT_ERROR) +#endif + + namespace { + + // Convert a mantissa, an exponent and a sign bit into an ieee64 double. + // The real_exponent needs to be in [0, 2046] (technically real_exponent = 2047 would be acceptable). + // The mantissa should be in [0,1<<53). The bit at index (1ULL << 52) while be zeroed. + simdjson_inline double to_double(uint64_t mantissa, uint64_t real_exponent, bool negative) { + double d; + mantissa &= ~(1ULL << 52); + mantissa |= real_exponent << 52; + mantissa |= ((static_cast(negative)) << 63); + std::memcpy(&d, &mantissa, sizeof(d)); + return d; + } + + // Attempts to compute i * 10^(power) exactly; and if "negative" is + // true, negate the result. + // This function will only work in some cases, when it does not work, success is + // set to false. This should work *most of the time* (like 99% of the time). + // We assume that power is in the [smallest_power, + // largest_power] interval: the caller is responsible for this check. + simdjson_inline bool compute_float_64(int64_t power, uint64_t i, bool negative, double& d) { + // we start with a fast path + // It was described in + // Clinger WD. How to read floating point numbers accurately. + // ACM SIGPLAN Notices. 1990 +#ifndef FLT_EVAL_METHOD +#error "FLT_EVAL_METHOD should be defined, please include cfloat." +#endif +#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0) + // We cannot be certain that x/y is rounded to nearest. + if (0 <= power && power <= 22 && i <= 9007199254740991) +#else + if (-22 <= power && power <= 22 && i <= 9007199254740991) +#endif + { + // convert the integer into a double. This is lossless since + // 0 <= i <= 2^53 - 1. + d = double(i); + // + // The general idea is as follows. + // If 0 <= s < 2^53 and if 10^0 <= p <= 10^22 then + // 1) Both s and p can be represented exactly as 64-bit floating-point + // values + // (binary64). + // 2) Because s and p can be represented exactly as floating-point values, + // then s * p + // and s / p will produce correctly rounded values. + // + if (power < 0) { + d = d / simdjson::internal::power_of_ten[-power]; + } + else { + d = d * simdjson::internal::power_of_ten[power]; + } + if (negative) { + d = -d; + } + return true; + } + // When 22 < power && power < 22 + 16, we could + // hope for another, secondary fast path. It was + // described by David M. Gay in "Correctly rounded + // binary-decimal and decimal-binary conversions." (1990) + // If you need to compute i * 10^(22 + x) for x < 16, + // first compute i * 10^x, if you know that result is exact + // (e.g., when i * 10^x < 2^53), + // then you can still proceed and do (i * 10^x) * 10^22. + // Is this worth your time? + // You need 22 < power *and* power < 22 + 16 *and* (i * 10^(x-22) < 2^53) + // for this second fast path to work. + // If you you have 22 < power *and* power < 22 + 16, and then you + // optimistically compute "i * 10^(x-22)", there is still a chance that you + // have wasted your time if i * 10^(x-22) >= 2^53. It makes the use cases of + // this optimization maybe less common than we would like. Source: + // http://www.exploringbinary.com/fast-path-decimal-to-floating-point-conversion/ + // also used in RapidJSON: https://rapidjson.org/strtod_8h_source.html + + // The fast path has now failed, so we are failing back on the slower path. + + // In the slow path, we need to adjust i so that it is > 1<<63 which is always + // possible, except if i == 0, so we handle i == 0 separately. + if (i == 0) { + d = negative ? -0.0 : 0.0; + return true; + } + + + // The exponent is 1024 + 63 + power + // + floor(log(5**power)/log(2)). + // The 1024 comes from the ieee64 standard. + // The 63 comes from the fact that we use a 64-bit word. + // + // Computing floor(log(5**power)/log(2)) could be + // slow. Instead we use a fast function. + // + // For power in (-400,350), we have that + // (((152170 + 65536) * power ) >> 16); + // is equal to + // floor(log(5**power)/log(2)) + power when power >= 0 + // and it is equal to + // ceil(log(5**-power)/log(2)) + power when power < 0 + // + // The 65536 is (1<<16) and corresponds to + // (65536 * power) >> 16 ---> power + // + // ((152170 * power ) >> 16) is equal to + // floor(log(5**power)/log(2)) + // + // Note that this is not magic: 152170/(1<<16) is + // approximatively equal to log(5)/log(2). + // The 1<<16 value is a power of two; we could use a + // larger power of 2 if we wanted to. + // + int64_t exponent = (((152170 + 65536) * power) >> 16) + 1024 + 63; + + + // We want the most significant bit of i to be 1. Shift if needed. + int lz = leading_zeroes(i); + i <<= lz; + + + // We are going to need to do some 64-bit arithmetic to get a precise product. + // We use a table lookup approach. + // It is safe because + // power >= smallest_power + // and power <= largest_power + // We recover the mantissa of the power, it has a leading 1. It is always + // rounded down. + // + // We want the most significant 64 bits of the product. We know + // this will be non-zero because the most significant bit of i is + // 1. + const uint32_t index = 2 * uint32_t(power - simdjson::internal::smallest_power); + // Optimization: It may be that materializing the index as a variable might confuse some compilers and prevent effective complex-addressing loads. (Done for code clarity.) + // + // The full_multiplication function computes the 128-bit product of two 64-bit words + // with a returned value of type value128 with a "low component" corresponding to the + // 64-bit least significant bits of the product and with a "high component" corresponding + // to the 64-bit most significant bits of the product. + simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index]); + // Both i and power_of_five_128[index] have their most significant bit set to 1 which + // implies that the either the most or the second most significant bit of the product + // is 1. We pack values in this manner for efficiency reasons: it maximizes the use + // we make of the product. It also makes it easy to reason about the product: there + // is 0 or 1 leading zero in the product. + + // Unless the least significant 9 bits of the high (64-bit) part of the full + // product are all 1s, then we know that the most significant 55 bits are + // exact and no further work is needed. Having 55 bits is necessary because + // we need 53 bits for the mantissa but we have to have one rounding bit and + // we can waste a bit if the most significant bit of the product is zero. + if ((firstproduct.high & 0x1FF) == 0x1FF) { + // We want to compute i * 5^q, but only care about the top 55 bits at most. + // Consider the scenario where q>=0. Then 5^q may not fit in 64-bits. Doing + // the full computation is wasteful. So we do what is called a "truncated + // multiplication". + // We take the most significant 64-bits, and we put them in + // power_of_five_128[index]. Usually, that's good enough to approximate i * 5^q + // to the desired approximation using one multiplication. Sometimes it does not suffice. + // Then we store the next most significant 64 bits in power_of_five_128[index + 1], and + // then we get a better approximation to i * 5^q. + // + // That's for when q>=0. The logic for q<0 is somewhat similar but it is somewhat + // more complicated. + // + // There is an extra layer of complexity in that we need more than 55 bits of + // accuracy in the round-to-even scenario. + // + // The full_multiplication function computes the 128-bit product of two 64-bit words + // with a returned value of type value128 with a "low component" corresponding to the + // 64-bit least significant bits of the product and with a "high component" corresponding + // to the 64-bit most significant bits of the product. + simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index + 1]); + firstproduct.low += secondproduct.high; + if (secondproduct.high > firstproduct.low) { firstproduct.high++; } + // As it has been proven by Noble Mushtak and Daniel Lemire in "Fast Number Parsing Without + // Fallback" (https://arxiv.org/abs/2212.06644), at this point we are sure that the product + // is sufficiently accurate, and more computation is not needed. + } + uint64_t lower = firstproduct.low; + uint64_t upper = firstproduct.high; + // The final mantissa should be 53 bits with a leading 1. + // We shift it so that it occupies 54 bits with a leading 1. + /////// + uint64_t upperbit = upper >> 63; + uint64_t mantissa = upper >> (upperbit + 9); + lz += int(1 ^ upperbit); + + // Here we have mantissa < (1<<54). + int64_t real_exponent = exponent - lz; + if (simdjson_unlikely(real_exponent <= 0)) { // we have a subnormal? + // Here have that real_exponent <= 0 so -real_exponent >= 0 + if (-real_exponent + 1 >= 64) { // if we have more than 64 bits below the minimum exponent, you have a zero for sure. + d = negative ? -0.0 : 0.0; + return true; + } + // next line is safe because -real_exponent + 1 < 0 + mantissa >>= -real_exponent + 1; + // Thankfully, we can't have both "round-to-even" and subnormals because + // "round-to-even" only occurs for powers close to 0. + mantissa += (mantissa & 1); // round up + mantissa >>= 1; + // There is a weird scenario where we don't have a subnormal but just. + // Suppose we start with 2.2250738585072013e-308, we end up + // with 0x3fffffffffffff x 2^-1023-53 which is technically subnormal + // whereas 0x40000000000000 x 2^-1023-53 is normal. Now, we need to round + // up 0x3fffffffffffff x 2^-1023-53 and once we do, we are no longer + // subnormal, but we can only know this after rounding. + // So we only declare a subnormal if we are smaller than the threshold. + real_exponent = (mantissa < (uint64_t(1) << 52)) ? 0 : 1; + d = to_double(mantissa, real_exponent, negative); + return true; + } + // We have to round to even. The "to even" part + // is only a problem when we are right in between two floats + // which we guard against. + // If we have lots of trailing zeros, we may fall right between two + // floating-point values. + // + // The round-to-even cases take the form of a number 2m+1 which is in (2^53,2^54] + // times a power of two. That is, it is right between a number with binary significand + // m and another number with binary significand m+1; and it must be the case + // that it cannot be represented by a float itself. + // + // We must have that w * 10 ^q == (2m+1) * 2^p for some power of two 2^p. + // Recall that 10^q = 5^q * 2^q. + // When q >= 0, we must have that (2m+1) is divible by 5^q, so 5^q <= 2^54. We have that + // 5^23 <= 2^54 and it is the last power of five to qualify, so q <= 23. + // When q<0, we have w >= (2m+1) x 5^{-q}. We must have that w<2^{64} so + // (2m+1) x 5^{-q} < 2^{64}. We have that 2m+1>2^{53}. Hence, we must have + // 2^{53} x 5^{-q} < 2^{64}. + // Hence we have 5^{-q} < 2^{11}$ or q>= -4. + // + // We require lower <= 1 and not lower == 0 because we could not prove that + // that lower == 0 is implied; but we could prove that lower <= 1 is a necessary and sufficient test. + if (simdjson_unlikely((lower <= 1) && (power >= -4) && (power <= 23) && ((mantissa & 3) == 1))) { + if ((mantissa << (upperbit + 64 - 53 - 2)) == upper) { + mantissa &= ~1; // flip it so that we do not round up + } + } + + mantissa += mantissa & 1; + mantissa >>= 1; + + // Here we have mantissa < (1<<53), unless there was an overflow + if (mantissa >= (1ULL << 53)) { + ////////// + // This will happen when parsing values such as 7.2057594037927933e+16 + //////// + mantissa = (1ULL << 52); + real_exponent++; + } + mantissa &= ~(1ULL << 52); + // we have to check that real_exponent is in range, otherwise we bail out + if (simdjson_unlikely(real_exponent > 2046)) { + // We have an infinite value!!! We could actually throw an error here if we could. + return false; + } + d = to_double(mantissa, real_exponent, negative); + return true; + } + + // We call a fallback floating-point parser that might be slow. Note + // it will accept JSON numbers, but the JSON spec. is more restrictive so + // before you call parse_float_fallback, you need to have validated the input + // string with the JSON grammar. + // It will return an error (false) if the parsed number is infinite. + // The string parsing itself always succeeds. We know that there is at least + // one digit. + static bool parse_float_fallback(const uint8_t* ptr, double* outDouble) { + *outDouble = simdjson::internal::from_chars(reinterpret_cast(ptr)); + // We do not accept infinite values. + + // Detecting finite values in a portable manner is ridiculously hard, ideally + // we would want to do: + // return !std::isfinite(*outDouble); + // but that mysteriously fails under legacy/old libc++ libraries, see + // https://github.com/simdjson/simdjson/issues/1286 + // + // Therefore, fall back to this solution (the extra parens are there + // to handle that max may be a macro on windows). + return !(*outDouble > (std::numeric_limits::max)() || *outDouble < std::numeric_limits::lowest()); + } + + static bool parse_float_fallback(const uint8_t* ptr, const uint8_t* end_ptr, double* outDouble) { + *outDouble = simdjson::internal::from_chars(reinterpret_cast(ptr), reinterpret_cast(end_ptr)); + // We do not accept infinite values. + + // Detecting finite values in a portable manner is ridiculously hard, ideally + // we would want to do: + // return !std::isfinite(*outDouble); + // but that mysteriously fails under legacy/old libc++ libraries, see + // https://github.com/simdjson/simdjson/issues/1286 + // + // Therefore, fall back to this solution (the extra parens are there + // to handle that max may be a macro on windows). + return !(*outDouble > (std::numeric_limits::max)() || *outDouble < std::numeric_limits::lowest()); + } + + // check quickly whether the next 8 chars are made of digits + // at a glance, it looks better than Mula's + // http://0x80.pl/articles/swar-digits-validate.html + simdjson_inline bool is_made_of_eight_digits_fast(const uint8_t* chars) { + uint64_t val; + // this can read up to 7 bytes beyond the buffer size, but we require + // SIMDJSON_PADDING of padding + static_assert(7 <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be bigger than 7"); + std::memcpy(&val, chars, 8); + // a branchy method might be faster: + // return (( val & 0xF0F0F0F0F0F0F0F0 ) == 0x3030303030303030) + // && (( (val + 0x0606060606060606) & 0xF0F0F0F0F0F0F0F0 ) == + // 0x3030303030303030); + return (((val & 0xF0F0F0F0F0F0F0F0) | + (((val + 0x0606060606060606) & 0xF0F0F0F0F0F0F0F0) >> 4)) == + 0x3333333333333333); + } + + template + SIMDJSON_NO_SANITIZE_UNDEFINED // We deliberately allow overflow here and check later + simdjson_inline bool parse_digit(const uint8_t c, I& i) { + const uint8_t digit = static_cast(c - '0'); + if (digit > 9) { + return false; + } + // PERF NOTE: multiplication by 10 is cheaper than arbitrary integer multiplication + i = 10 * i + digit; // might overflow, we will handle the overflow later + return true; + } + + simdjson_inline bool is_digit(const uint8_t c) { + return static_cast(c - '0') <= 9; + } + + simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t* const src, const uint8_t*& p, uint64_t& i, int64_t& exponent) { + // we continue with the fiction that we have an integer. If the + // floating point number is representable as x * 10^z for some integer + // z that fits in 53 bits, then we will be able to convert back the + // the integer into a float in a lossless manner. + const uint8_t* const first_after_period = p; + +#ifdef SIMDJSON_SWAR_NUMBER_PARSING +#if SIMDJSON_SWAR_NUMBER_PARSING + // this helps if we have lots of decimals! + // this turns out to be frequent enough. + if (is_made_of_eight_digits_fast(p)) { + i = i * 100000000 + parse_eight_digits_unrolled(p); + p += 8; + } +#endif // SIMDJSON_SWAR_NUMBER_PARSING +#endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING + // Unrolling the first digit makes a small difference on some implementations (e.g. westmere) + if (parse_digit(*p, i)) { ++p; } + while (parse_digit(*p, i)) { p++; } + exponent = first_after_period - p; + // Decimal without digits (123.) is illegal + if (exponent == 0) { + return INVALID_NUMBER(src); + } + return SUCCESS; + } + + simdjson_inline error_code parse_exponent(simdjson_unused const uint8_t* const src, const uint8_t*& p, int64_t& exponent) { + // Exp Sign: -123.456e[-]78 + bool neg_exp = ('-' == *p); + if (neg_exp || '+' == *p) { p++; } // Skip + as well + + // Exponent: -123.456e-[78] + auto start_exp = p; + int64_t exp_number = 0; + while (parse_digit(*p, exp_number)) { ++p; } + // It is possible for parse_digit to overflow. + // In particular, it could overflow to INT64_MIN, and we cannot do - INT64_MIN. + // Thus we *must* check for possible overflow before we negate exp_number. + + // Performance notes: it may seem like combining the two "simdjson_unlikely checks" below into + // a single simdjson_unlikely path would be faster. The reasoning is sound, but the compiler may + // not oblige and may, in fact, generate two distinct paths in any case. It might be + // possible to do uint64_t(p - start_exp - 1) >= 18 but it could end up trading off + // instructions for a simdjson_likely branch, an unconclusive gain. + + // If there were no digits, it's an error. + if (simdjson_unlikely(p == start_exp)) { + return INVALID_NUMBER(src); + } + // We have a valid positive exponent in exp_number at this point, except that + // it may have overflowed. + + // If there were more than 18 digits, we may have overflowed the integer. We have to do + // something!!!! + if (simdjson_unlikely(p > start_exp + 18)) { + // Skip leading zeroes: 1e000000000000000000001 is technically valid and does not overflow + while (*start_exp == '0') { start_exp++; } + // 19 digits could overflow int64_t and is kind of absurd anyway. We don't + // support exponents smaller than -999,999,999,999,999,999 and bigger + // than 999,999,999,999,999,999. + // We can truncate. + // Note that 999999999999999999 is assuredly too large. The maximal ieee64 value before + // infinity is ~1.8e308. The smallest subnormal is ~5e-324. So, actually, we could + // truncate at 324. + // Note that there is no reason to fail per se at this point in time. + // E.g., 0e999999999999999999999 is a fine number. + if (p > start_exp + 18) { exp_number = 999999999999999999; } + } + // At this point, we know that exp_number is a sane, positive, signed integer. + // It is <= 999,999,999,999,999,999. As long as 'exponent' is in + // [-8223372036854775808, 8223372036854775808], we won't overflow. Because 'exponent' + // is bounded in magnitude by the size of the JSON input, we are fine in this universe. + // To sum it up: the next line should never overflow. + exponent += (neg_exp ? -exp_number : exp_number); + return SUCCESS; + } + + simdjson_inline bool check_if_integer(const uint8_t* const src, size_t max_length) { + const uint8_t* const srcend = src + max_length; + bool negative = (*src == '-'); // we can always read at least one character after the '-' + const uint8_t* p = src + uint8_t(negative); + if (p == srcend) { return false; } + if (*p == '0') { + ++p; + if (p == srcend) { return true; } + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return false; } + return true; + } + while (p != srcend && is_digit(*p)) { ++p; } + if (p == srcend) { return true; } + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return false; } + return true; + } + + simdjson_inline size_t significant_digits(const uint8_t* start_digits, size_t digit_count) { + // It is possible that the integer had an overflow. + // We have to handle the case where we have 0.0000somenumber. + const uint8_t* start = start_digits; + while ((*start == '0') || (*start == '.')) { ++start; } + // we over-decrement by one when there is a '.' + return digit_count - size_t(start - start_digits); + } + + } // unnamed namespace + + /** @private */ + static error_code slow_float_parsing(simdjson_unused const uint8_t* src, double* answer) { + if (parse_float_fallback(src, answer)) { + return SUCCESS; + } + return INVALID_NUMBER(src); + } + + /** @private */ + template + simdjson_inline error_code write_float(const uint8_t* const src, bool negative, uint64_t i, const uint8_t* start_digits, size_t digit_count, int64_t exponent, W& writer) { + // If we frequently had to deal with long strings of digits, + // we could extend our code by using a 128-bit integer instead + // of a 64-bit integer. However, this is uncommon in practice. + // + // 9999999999999999999 < 2**64 so we can accommodate 19 digits. + // If we have a decimal separator, then digit_count - 1 is the number of digits, but we + // may not have a decimal separator! + if (simdjson_unlikely(digit_count > 19 && significant_digits(start_digits, digit_count) > 19)) { + // Ok, chances are good that we had an overflow! + // this is almost never going to get called!!! + // we start anew, going slowly!!! + // This will happen in the following examples: + // 10000000000000000000000000000000000000000000e+308 + // 3.1415926535897932384626433832795028841971693993751 + // + // NOTE: We do not pass a reference to the to slow_float_parsing. If we passed our writer + // reference to it, it would force it to be stored in memory, preventing the compiler from + // picking it apart and putting into registers. i.e. if we pass it as reference, + // it gets slow. + double d; + error_code error = slow_float_parsing(src, &d); + writer.append_double(d); + return error; + } + // NOTE: it's weird that the simdjson_unlikely() only wraps half the if, but it seems to get slower any other + // way we've tried: https://github.com/simdjson/simdjson/pull/990#discussion_r448497331 + // To future reader: we'd love if someone found a better way, or at least could explain this result! + if (simdjson_unlikely(exponent < simdjson::internal::smallest_power) || (exponent > simdjson::internal::largest_power)) { + // + // Important: smallest_power is such that it leads to a zero value. + // Observe that 18446744073709551615e-343 == 0, i.e. (2**64 - 1) e -343 is zero + // so something x 10^-343 goes to zero, but not so with something x 10^-342. + static_assert(simdjson::internal::smallest_power <= -342, "smallest_power is not small enough"); + // + if ((exponent < simdjson::internal::smallest_power) || (i == 0)) { + // E.g. Parse "-0.0e-999" into the same value as "-0.0". See https://en.wikipedia.org/wiki/Signed_zero + WRITE_DOUBLE(negative ? -0.0 : 0.0, src, writer); + return SUCCESS; + } + else { // (exponent > largest_power) and (i != 0) + // We have, for sure, an infinite value and simdjson refuses to parse infinite values. + return INVALID_NUMBER(src); + } + } + double d; + if (!compute_float_64(exponent, i, negative, d)) { + // we are almost never going to get here. + if (!parse_float_fallback(src, &d)) { return INVALID_NUMBER(src); } + } + WRITE_DOUBLE(d, src, writer); + return SUCCESS; + } + + // parse the number at src + // define JSON_TEST_NUMBERS for unit testing + // + // It is assumed that the number is followed by a structural ({,},],[) character + // or a white space character. If that is not the case (e.g., when the JSON + // document is made of a single number), then it is necessary to copy the + // content and append a space before calling this function. + // + // Our objective is accurate parsing (ULP of 0) at high speed. + template + simdjson_inline error_code parse_number(const uint8_t* const src, W& writer); + + // for performance analysis, it is sometimes useful to skip parsing +#ifdef SIMDJSON_SKIPNUMBERPARSING + + template + simdjson_inline error_code parse_number(const uint8_t* const, W& writer) { + writer.append_s64(0); // always write zero + return SUCCESS; // always succeeds + } + + simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t* const src) noexcept { return 0; } + simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t* const src) noexcept { return 0; } + simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t* const src) noexcept { return 0; } + simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t* const src) noexcept { return 0; } + simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t* const src) noexcept { return 0; } + simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t* const src) noexcept { return 0; } + simdjson_unused simdjson_inline bool is_negative(const uint8_t* src) noexcept { return false; } + simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t* src) noexcept { return false; } + simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t* src) noexcept { return number_type::signed_integer; } +#else + +// parse the number at src +// define JSON_TEST_NUMBERS for unit testing +// +// It is assumed that the number is followed by a structural ({,},],[) character +// or a white space character. If that is not the case (e.g., when the JSON +// document is made of a single number), then it is necessary to copy the +// content and append a space before calling this function. +// +// Our objective is accurate parsing (ULP of 0) at high speed. + template + simdjson_inline error_code parse_number(const uint8_t* const src, W& writer) { + + // + // Check for minus sign + // + bool negative = (*src == '-'); + const uint8_t* p = src + uint8_t(negative); + + // + // Parse the integer part. + // + // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare + const uint8_t* const start_digits = p; + uint64_t i = 0; + while (parse_digit(*p, i)) { p++; } + + // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. + // Optimization note: size_t is expected to be unsigned. + size_t digit_count = size_t(p - start_digits); + if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { return INVALID_NUMBER(src); } + + // + // Handle floats if there is a . or e (or both) + // + int64_t exponent = 0; + bool is_float = false; + if ('.' == *p) { + is_float = true; + ++p; + SIMDJSON_TRY(parse_decimal_after_separator(src, p, i, exponent)); + digit_count = int(p - start_digits); // used later to guard against overflows + } + if (('e' == *p) || ('E' == *p)) { + is_float = true; + ++p; + SIMDJSON_TRY(parse_exponent(src, p, exponent)); + } + if (is_float) { + const bool dirty_end = jsoncharutils::is_not_structural_or_whitespace(*p); + SIMDJSON_TRY(write_float(src, negative, i, start_digits, digit_count, exponent, writer)); + if (dirty_end) { return INVALID_NUMBER(src); } + return SUCCESS; + } + + // The longest negative 64-bit number is 19 digits. + // The longest positive 64-bit number is 20 digits. + // We do it this way so we don't trigger this branch unless we must. + size_t longest_digit_count = negative ? 19 : 20; + if (digit_count > longest_digit_count) { return BIGINT_NUMBER(src); } + if (digit_count == longest_digit_count) { + if (negative) { + // Anything negative above INT64_MAX+1 is invalid + if (i > uint64_t(INT64_MAX) + 1) { return BIGINT_NUMBER(src); } + WRITE_INTEGER(~i + 1, src, writer); + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return INVALID_NUMBER(src); } + return SUCCESS; + // Positive overflow check: + // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the + // biggest uint64_t. + // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX. + // If we got here, it's a 20 digit number starting with the digit "1". + // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller + // than 1,553,255,926,290,448,384. + // - That is smaller than the smallest possible 20-digit number the user could write: + // 10,000,000,000,000,000,000. + // - Therefore, if the number is positive and lower than that, it's overflow. + // - The value we are looking at is less than or equal to INT64_MAX. + // + } + else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INVALID_NUMBER(src); } + } + + // Write unsigned if it does not fit in a signed integer. + if (i > uint64_t(INT64_MAX)) { + WRITE_UNSIGNED(i, src, writer); + } + else { + WRITE_INTEGER(negative ? (~i + 1) : i, src, writer); + } + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return INVALID_NUMBER(src); } + return SUCCESS; + } + + // Inlineable functions + namespace { + + // This table can be used to characterize the final character of an integer + // string. For JSON structural character and allowable white space characters, + // we return SUCCESS. For 'e', '.' and 'E', we return INCORRECT_TYPE. Otherwise + // we return NUMBER_ERROR. + // Optimization note: we could easily reduce the size of the table by half (to 128) + // at the cost of an extra branch. + // Optimization note: we want the values to use at most 8 bits (not, e.g., 32 bits): + static_assert(error_code(uint8_t(NUMBER_ERROR)) == NUMBER_ERROR, "bad NUMBER_ERROR cast"); + static_assert(error_code(uint8_t(SUCCESS)) == SUCCESS, "bad NUMBER_ERROR cast"); + static_assert(error_code(uint8_t(INCORRECT_TYPE)) == INCORRECT_TYPE, "bad NUMBER_ERROR cast"); + + const uint8_t integer_string_finisher[256] = { + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, + SUCCESS, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, + NUMBER_ERROR, INCORRECT_TYPE, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, INCORRECT_TYPE, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, SUCCESS, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, INCORRECT_TYPE, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, + SUCCESS, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR }; + + // Parse any number from 0 to 18,446,744,073,709,551,615 + simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t* const src) noexcept { + const uint8_t* p = src; + // + // Parse the integer part. + // + // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare + const uint8_t* const start_digits = p; + uint64_t i = 0; + while (parse_digit(*p, i)) { p++; } + + // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. + // Optimization note: size_t is expected to be unsigned. + size_t digit_count = size_t(p - start_digits); + // The longest positive 64-bit number is 20 digits. + // We do it this way so we don't trigger this branch unless we must. + // Optimization note: the compiler can probably merge + // ((digit_count == 0) || (digit_count > 20)) + // into a single branch since digit_count is unsigned. + if ((digit_count == 0) || (digit_count > 20)) { return INCORRECT_TYPE; } + // Here digit_count > 0. + if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } + // We can do the following... + // if (!jsoncharutils::is_structural_or_whitespace(*p)) { + // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; + // } + // as a single table lookup: + if (integer_string_finisher[*p] != SUCCESS) { return error_code(integer_string_finisher[*p]); } + + if (digit_count == 20) { + // Positive overflow check: + // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the + // biggest uint64_t. + // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX. + // If we got here, it's a 20 digit number starting with the digit "1". + // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller + // than 1,553,255,926,290,448,384. + // - That is smaller than the smallest possible 20-digit number the user could write: + // 10,000,000,000,000,000,000. + // - Therefore, if the number is positive and lower than that, it's overflow. + // - The value we are looking at is less than or equal to INT64_MAX. + // + if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INCORRECT_TYPE; } + } + + return i; + } + + + // Parse any number from 0 to 18,446,744,073,709,551,615 + // Never read at src_end or beyond + simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t* const src, const uint8_t* const src_end) noexcept { + const uint8_t* p = src; + // + // Parse the integer part. + // + // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare + const uint8_t* const start_digits = p; + uint64_t i = 0; + while ((p != src_end) && parse_digit(*p, i)) { p++; } + + // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. + // Optimization note: size_t is expected to be unsigned. + size_t digit_count = size_t(p - start_digits); + // The longest positive 64-bit number is 20 digits. + // We do it this way so we don't trigger this branch unless we must. + // Optimization note: the compiler can probably merge + // ((digit_count == 0) || (digit_count > 20)) + // into a single branch since digit_count is unsigned. + if ((digit_count == 0) || (digit_count > 20)) { return INCORRECT_TYPE; } + // Here digit_count > 0. + if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } + // We can do the following... + // if (!jsoncharutils::is_structural_or_whitespace(*p)) { + // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; + // } + // as a single table lookup: + if ((p != src_end) && integer_string_finisher[*p] != SUCCESS) { return error_code(integer_string_finisher[*p]); } + + if (digit_count == 20) { + // Positive overflow check: + // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the + // biggest uint64_t. + // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX. + // If we got here, it's a 20 digit number starting with the digit "1". + // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller + // than 1,553,255,926,290,448,384. + // - That is smaller than the smallest possible 20-digit number the user could write: + // 10,000,000,000,000,000,000. + // - Therefore, if the number is positive and lower than that, it's overflow. + // - The value we are looking at is less than or equal to INT64_MAX. + // + if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INCORRECT_TYPE; } + } + + return i; + } + + // Parse any number from 0 to 18,446,744,073,709,551,615 + simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t* const src) noexcept { + const uint8_t* p = src + 1; + // + // Parse the integer part. + // + // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare + const uint8_t* const start_digits = p; + uint64_t i = 0; + while (parse_digit(*p, i)) { p++; } + + // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. + // Optimization note: size_t is expected to be unsigned. + size_t digit_count = size_t(p - start_digits); + // The longest positive 64-bit number is 20 digits. + // We do it this way so we don't trigger this branch unless we must. + // Optimization note: the compiler can probably merge + // ((digit_count == 0) || (digit_count > 20)) + // into a single branch since digit_count is unsigned. + if ((digit_count == 0) || (digit_count > 20)) { return INCORRECT_TYPE; } + // Here digit_count > 0. + if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } + // We can do the following... + // if (!jsoncharutils::is_structural_or_whitespace(*p)) { + // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; + // } + // as a single table lookup: + if (*p != '"') { return NUMBER_ERROR; } + + if (digit_count == 20) { + // Positive overflow check: + // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the + // biggest uint64_t. + // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX. + // If we got here, it's a 20 digit number starting with the digit "1". + // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller + // than 1,553,255,926,290,448,384. + // - That is smaller than the smallest possible 20-digit number the user could write: + // 10,000,000,000,000,000,000. + // - Therefore, if the number is positive and lower than that, it's overflow. + // - The value we are looking at is less than or equal to INT64_MAX. + // + // Note: we use src[1] and not src[0] because src[0] is the quote character in this + // instance. + if (src[1] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INCORRECT_TYPE; } + } + + return i; + } + + // Parse any number from -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 + simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t* src) noexcept { + // + // Check for minus sign + // + bool negative = (*src == '-'); + const uint8_t* p = src + uint8_t(negative); + + // + // Parse the integer part. + // + // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare + const uint8_t* const start_digits = p; + uint64_t i = 0; + while (parse_digit(*p, i)) { p++; } + + // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. + // Optimization note: size_t is expected to be unsigned. + size_t digit_count = size_t(p - start_digits); + // We go from + // -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 + // so we can never represent numbers that have more than 19 digits. + size_t longest_digit_count = 19; + // Optimization note: the compiler can probably merge + // ((digit_count == 0) || (digit_count > longest_digit_count)) + // into a single branch since digit_count is unsigned. + if ((digit_count == 0) || (digit_count > longest_digit_count)) { return INCORRECT_TYPE; } + // Here digit_count > 0. + if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } + // We can do the following... + // if (!jsoncharutils::is_structural_or_whitespace(*p)) { + // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; + // } + // as a single table lookup: + if (integer_string_finisher[*p] != SUCCESS) { return error_code(integer_string_finisher[*p]); } + // Negative numbers have can go down to - INT64_MAX - 1 whereas positive numbers are limited to INT64_MAX. + // Performance note: This check is only needed when digit_count == longest_digit_count but it is + // so cheap that we might as well always make it. + if (i > uint64_t(INT64_MAX) + uint64_t(negative)) { return INCORRECT_TYPE; } + return negative ? (~i + 1) : i; + } + + // Parse any number from -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 + // Never read at src_end or beyond + simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t* const src, const uint8_t* const src_end) noexcept { + // + // Check for minus sign + // + if (src == src_end) { return NUMBER_ERROR; } + bool negative = (*src == '-'); + const uint8_t* p = src + uint8_t(negative); + + // + // Parse the integer part. + // + // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare + const uint8_t* const start_digits = p; + uint64_t i = 0; + while ((p != src_end) && parse_digit(*p, i)) { p++; } + + // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. + // Optimization note: size_t is expected to be unsigned. + size_t digit_count = size_t(p - start_digits); + // We go from + // -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 + // so we can never represent numbers that have more than 19 digits. + size_t longest_digit_count = 19; + // Optimization note: the compiler can probably merge + // ((digit_count == 0) || (digit_count > longest_digit_count)) + // into a single branch since digit_count is unsigned. + if ((digit_count == 0) || (digit_count > longest_digit_count)) { return INCORRECT_TYPE; } + // Here digit_count > 0. + if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } + // We can do the following... + // if (!jsoncharutils::is_structural_or_whitespace(*p)) { + // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; + // } + // as a single table lookup: + if ((p != src_end) && integer_string_finisher[*p] != SUCCESS) { return error_code(integer_string_finisher[*p]); } + // Negative numbers have can go down to - INT64_MAX - 1 whereas positive numbers are limited to INT64_MAX. + // Performance note: This check is only needed when digit_count == longest_digit_count but it is + // so cheap that we might as well always make it. + if (i > uint64_t(INT64_MAX) + uint64_t(negative)) { return INCORRECT_TYPE; } + return negative ? (~i + 1) : i; + } + + // Parse any number from -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 + simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t* src) noexcept { + // + // Check for minus sign + // + bool negative = (*(src + 1) == '-'); + src += uint8_t(negative) + 1; + + // + // Parse the integer part. + // + // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare + const uint8_t* const start_digits = src; + uint64_t i = 0; + while (parse_digit(*src, i)) { src++; } + + // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. + // Optimization note: size_t is expected to be unsigned. + size_t digit_count = size_t(src - start_digits); + // We go from + // -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 + // so we can never represent numbers that have more than 19 digits. + size_t longest_digit_count = 19; + // Optimization note: the compiler can probably merge + // ((digit_count == 0) || (digit_count > longest_digit_count)) + // into a single branch since digit_count is unsigned. + if ((digit_count == 0) || (digit_count > longest_digit_count)) { return INCORRECT_TYPE; } + // Here digit_count > 0. + if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } + // We can do the following... + // if (!jsoncharutils::is_structural_or_whitespace(*src)) { + // return (*src == '.' || *src == 'e' || *src == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; + // } + // as a single table lookup: + if (*src != '"') { return NUMBER_ERROR; } + // Negative numbers have can go down to - INT64_MAX - 1 whereas positive numbers are limited to INT64_MAX. + // Performance note: This check is only needed when digit_count == longest_digit_count but it is + // so cheap that we might as well always make it. + if (i > uint64_t(INT64_MAX) + uint64_t(negative)) { return INCORRECT_TYPE; } + return negative ? (~i + 1) : i; + } + + simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t* src) noexcept { + // + // Check for minus sign + // + bool negative = (*src == '-'); + src += uint8_t(negative); + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t* p = src; + p += parse_digit(*p, i); + bool leading_zero = (i == 0); + while (parse_digit(*p, i)) { p++; } + // no integer digits, or 0123 (zero must be solo) + if (p == src) { return INCORRECT_TYPE; } + if ((leading_zero && p != src + 1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely(*p == '.')) { + p++; + const uint8_t* start_decimal_digits = p; + if (!parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits + p++; + while (parse_digit(*p, i)) { p++; } + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p - src - 1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t* start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = p - start_digits > 19; + } + } + else { + overflow = p - src > 19; + } + + // + // Parse the exponent + // + if (*p == 'e' || *p == 'E') { + p++; + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t* start_exp_digits = p; + while (parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p - start_exp_digits == 0 || p - start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0 - exp : exp; + } + + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + double d; + if (simdjson_likely(!overflow)) { + if (compute_float_64(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), &d)) { + return NUMBER_ERROR; + } + return d; + } + + simdjson_unused simdjson_inline bool is_negative(const uint8_t* src) noexcept { + return (*src == '-'); + } + + simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t* src) noexcept { + bool negative = (*src == '-'); + src += uint8_t(negative); + const uint8_t* p = src; + while (static_cast(*p - '0') <= 9) { p++; } + if (p == src) { return NUMBER_ERROR; } + if (jsoncharutils::is_structural_or_whitespace(*p)) { return true; } + return false; + } + + simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t* src) noexcept { + bool negative = (*src == '-'); + src += uint8_t(negative); + const uint8_t* p = src; + while (static_cast(*p - '0') <= 9) { p++; } + size_t digit_count = size_t(p - src); + if (p == src) { return NUMBER_ERROR; } + if (jsoncharutils::is_structural_or_whitespace(*p)) { + static const uint8_t* smaller_big_integer = reinterpret_cast("9223372036854775808"); + // We have an integer. + if (simdjson_unlikely(digit_count > 20)) { + return number_type::big_integer; + } + // If the number is negative and valid, it must be a signed integer. + if (negative) { + if (simdjson_unlikely(digit_count > 19)) return number_type::big_integer; + if (simdjson_unlikely(digit_count == 19 && memcmp(src, smaller_big_integer, 19) > 0)) { + return number_type::big_integer; + } + return number_type::signed_integer; + } + // Let us check if we have a big integer (>=2**64). + static const uint8_t* two_to_sixtyfour = reinterpret_cast("18446744073709551616"); + if ((digit_count > 20) || (digit_count == 20 && memcmp(src, two_to_sixtyfour, 20) >= 0)) { + return number_type::big_integer; + } + // The number is positive and smaller than 18446744073709551616 (or 2**64). + // We want values larger or equal to 9223372036854775808 to be unsigned + // integers, and the other values to be signed integers. + if ((digit_count == 20) || (digit_count >= 19 && memcmp(src, smaller_big_integer, 19) >= 0)) { + return number_type::unsigned_integer; + } + return number_type::signed_integer; + } + // Hopefully, we have 'e' or 'E' or '.'. + return number_type::floating_point_number; + } + + // Never read at src_end or beyond + simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t* src, const uint8_t* const src_end) noexcept { + if (src == src_end) { return NUMBER_ERROR; } + // + // Check for minus sign + // + bool negative = (*src == '-'); + src += uint8_t(negative); + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t* p = src; + if (p == src_end) { return NUMBER_ERROR; } + p += parse_digit(*p, i); + bool leading_zero = (i == 0); + while ((p != src_end) && parse_digit(*p, i)) { p++; } + // no integer digits, or 0123 (zero must be solo) + if (p == src) { return INCORRECT_TYPE; } + if ((leading_zero && p != src + 1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely((p != src_end) && (*p == '.'))) { + p++; + const uint8_t* start_decimal_digits = p; + if ((p == src_end) || !parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits + p++; + while ((p != src_end) && parse_digit(*p, i)) { p++; } + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p - src - 1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t* start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = start_digits - src > 19; + } + } + else { + overflow = p - src > 19; + } + + // + // Parse the exponent + // + if ((p != src_end) && (*p == 'e' || *p == 'E')) { + p++; + if (p == src_end) { return NUMBER_ERROR; } + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t* start_exp_digits = p; + while ((p != src_end) && parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p - start_exp_digits == 0 || p - start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0 - exp : exp; + } + + if ((p != src_end) && jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + double d; + if (simdjson_likely(!overflow)) { + if (compute_float_64(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), src_end, &d)) { + return NUMBER_ERROR; + } + return d; + } + + simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t* src) noexcept { + // + // Check for minus sign + // + bool negative = (*(src + 1) == '-'); + src += uint8_t(negative) + 1; + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t* p = src; + p += parse_digit(*p, i); + bool leading_zero = (i == 0); + while (parse_digit(*p, i)) { p++; } + // no integer digits, or 0123 (zero must be solo) + if (p == src) { return INCORRECT_TYPE; } + if ((leading_zero && p != src + 1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely(*p == '.')) { + p++; + const uint8_t* start_decimal_digits = p; + if (!parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits + p++; + while (parse_digit(*p, i)) { p++; } + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p - src - 1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t* start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = p - start_digits > 19; + } + } + else { + overflow = p - src > 19; + } + + // + // Parse the exponent + // + if (*p == 'e' || *p == 'E') { + p++; + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t* start_exp_digits = p; + while (parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p - start_exp_digits == 0 || p - start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0 - exp : exp; + } + + if (*p != '"') { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + double d; + if (simdjson_likely(!overflow)) { + if (compute_float_64(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), &d)) { + return NUMBER_ERROR; + } + return d; + } + + } // unnamed namespace +#endif // SIMDJSON_SKIPNUMBERPARSING + + } // namespace numberparsing + + inline std::ostream& operator<<(std::ostream& out, number_type type) noexcept { + switch (type) { + case number_type::signed_integer: out << "integer in [-9223372036854775808,9223372036854775808)"; break; + case number_type::unsigned_integer: out << "unsigned integer in [9223372036854775808,18446744073709551616)"; break; + case number_type::floating_point_number: out << "floating-point number (binary64)"; break; + case number_type::big_integer: out << "big integer"; break; + default: SIMDJSON_UNREACHABLE(); + } + return out; + } + + } // namespace ppc64 +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_NUMBERPARSING_H +/* end file simdjson/generic/numberparsing.h for ppc64 */ + +/* including simdjson/generic/implementation_simdjson_result_base-inl.h for ppc64: #include "simdjson/generic/implementation_simdjson_result_base-inl.h" */ +/* begin file simdjson/generic/implementation_simdjson_result_base-inl.h for ppc64 */ +#ifndef SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_INL_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_INL_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/implementation_simdjson_result_base.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace ppc64 { + + // + // internal::implementation_simdjson_result_base inline implementation + // + + template + simdjson_inline void implementation_simdjson_result_base::tie(T& value, error_code& error) && noexcept { + error = this->second; + if (!error) { + value = std::forward>(*this).first; + } + } + + template + simdjson_warn_unused simdjson_inline error_code implementation_simdjson_result_base::get(T& value) && noexcept { + error_code error; + std::forward>(*this).tie(value, error); + return error; + } + + template + simdjson_inline error_code implementation_simdjson_result_base::error() const noexcept { + return this->second; + } + +#if SIMDJSON_EXCEPTIONS + + template + simdjson_inline T& implementation_simdjson_result_base::value() & noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return this->first; + } + + template + simdjson_inline T&& implementation_simdjson_result_base::value() && noexcept(false) { + return std::forward>(*this).take_value(); + } + + template + simdjson_inline T&& implementation_simdjson_result_base::take_value() && noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return std::forward(this->first); + } + + template + simdjson_inline implementation_simdjson_result_base::operator T && () && noexcept(false) { + return std::forward>(*this).take_value(); + } + +#endif // SIMDJSON_EXCEPTIONS + + template + simdjson_inline const T& implementation_simdjson_result_base::value_unsafe() const& noexcept { + return this->first; + } + + template + simdjson_inline T& implementation_simdjson_result_base::value_unsafe() & noexcept { + return this->first; + } + + template + simdjson_inline T&& implementation_simdjson_result_base::value_unsafe() && noexcept { + return std::forward(this->first); + } + + template + simdjson_inline implementation_simdjson_result_base::implementation_simdjson_result_base(T&& value, error_code error) noexcept + : first{ std::forward(value) }, second{ error } {} + template + simdjson_inline implementation_simdjson_result_base::implementation_simdjson_result_base(error_code error) noexcept + : implementation_simdjson_result_base(T{}, error) {} + template + simdjson_inline implementation_simdjson_result_base::implementation_simdjson_result_base(T&& value) noexcept + : implementation_simdjson_result_base(std::forward(value), SUCCESS) {} + + } // namespace ppc64 +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_INL_H +/* end file simdjson/generic/implementation_simdjson_result_base-inl.h for ppc64 */ +/* end file simdjson/generic/amalgamated.h for ppc64 */ +/* including simdjson/ppc64/end.h: #include "simdjson/ppc64/end.h" */ +/* begin file simdjson/ppc64/end.h */ +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/ppc64/base.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +#undef SIMDJSON_SKIP_BACKSLASH_SHORT_CIRCUIT +/* undefining SIMDJSON_IMPLEMENTATION from "ppc64" */ +#undef SIMDJSON_IMPLEMENTATION +/* end file simdjson/ppc64/end.h */ + +#endif // SIMDJSON_PPC64_H +/* end file simdjson/ppc64.h */ +/* including simdjson/ppc64/implementation.h: #include */ +/* begin file simdjson/ppc64/implementation.h */ +#ifndef SIMDJSON_PPC64_IMPLEMENTATION_H +#define SIMDJSON_PPC64_IMPLEMENTATION_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/ppc64/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/implementation.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/instruction_set.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + + /** + * Implementation for ALTIVEC (PPC64). + */ + namespace ppc64 { + + /** + * @private + */ + class implementation final : public simdjson::implementation { + public: + simdjson_inline implementation() + : simdjson::implementation("ppc64", "PPC64 ALTIVEC", + internal::instruction_set::ALTIVEC) {} + + simdjson_warn_unused error_code create_dom_parser_implementation( + size_t capacity, size_t max_length, + std::unique_ptr& dst) + const noexcept final; + simdjson_warn_unused error_code minify(const uint8_t* buf, size_t len, + uint8_t* dst, + size_t& dst_len) const noexcept final; + simdjson_warn_unused bool validate_utf8(const char* buf, + size_t len) const noexcept final; + }; + + } // namespace ppc64 +} // namespace simdjson + +#endif // SIMDJSON_PPC64_IMPLEMENTATION_H +/* end file simdjson/ppc64/implementation.h */ + +/* including simdjson/ppc64/begin.h: #include */ +/* begin file simdjson/ppc64/begin.h */ +/* defining SIMDJSON_IMPLEMENTATION to "ppc64" */ +#define SIMDJSON_IMPLEMENTATION ppc64 +/* including simdjson/ppc64/base.h: #include "simdjson/ppc64/base.h" */ +/* begin file simdjson/ppc64/base.h */ +#ifndef SIMDJSON_PPC64_BASE_H +#define SIMDJSON_PPC64_BASE_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/base.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + /** + * Implementation for ALTIVEC (PPC64). + */ + namespace ppc64 { + + class implementation; + + namespace { + namespace simd { + template struct simd8; + template struct simd8x64; + } // namespace simd + } // unnamed namespace + + } // namespace ppc64 +} // namespace simdjson + +#endif // SIMDJSON_PPC64_BASE_H +/* end file simdjson/ppc64/base.h */ +/* including simdjson/ppc64/intrinsics.h: #include "simdjson/ppc64/intrinsics.h" */ +/* begin file simdjson/ppc64/intrinsics.h */ +#ifndef SIMDJSON_PPC64_INTRINSICS_H +#define SIMDJSON_PPC64_INTRINSICS_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/ppc64/base.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +// This should be the correct header whether +// you use visual studio or other compilers. +#include + +// These are defined by altivec.h in GCC toolchain, it is safe to undef them. +#ifdef bool +#undef bool +#endif + +#ifdef vector +#undef vector +#endif + +static_assert(sizeof(__vector unsigned char) <= simdjson::SIMDJSON_PADDING, "insufficient padding for ppc64"); + +#endif // SIMDJSON_PPC64_INTRINSICS_H +/* end file simdjson/ppc64/intrinsics.h */ +/* including simdjson/ppc64/bitmanipulation.h: #include "simdjson/ppc64/bitmanipulation.h" */ +/* begin file simdjson/ppc64/bitmanipulation.h */ +#ifndef SIMDJSON_PPC64_BITMANIPULATION_H +#define SIMDJSON_PPC64_BITMANIPULATION_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/ppc64/base.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace ppc64 { + namespace { + + // We sometimes call trailing_zero on inputs that are zero, + // but the algorithms do not end up using the returned value. + // Sadly, sanitizers are not smart enough to figure it out. + SIMDJSON_NO_SANITIZE_UNDEFINED + // This function can be used safely even if not all bytes have been + // initialized. + // See issue https://github.com/simdjson/simdjson/issues/1965 + SIMDJSON_NO_SANITIZE_MEMORY + simdjson_inline int trailing_zeroes(uint64_t input_num) { +#if SIMDJSON_REGULAR_VISUAL_STUDIO + unsigned long ret; + // Search the mask data from least significant bit (LSB) + // to the most significant bit (MSB) for a set bit (1). + _BitScanForward64(&ret, input_num); + return (int)ret; +#else // SIMDJSON_REGULAR_VISUAL_STUDIO + return __builtin_ctzll(input_num); +#endif // SIMDJSON_REGULAR_VISUAL_STUDIO + } + + /* result might be undefined when input_num is zero */ + simdjson_inline uint64_t clear_lowest_bit(uint64_t input_num) { + return input_num & (input_num - 1); + } + + /* result might be undefined when input_num is zero */ + simdjson_inline int leading_zeroes(uint64_t input_num) { +#if SIMDJSON_REGULAR_VISUAL_STUDIO + unsigned long leading_zero = 0; + // Search the mask data from most significant bit (MSB) + // to least significant bit (LSB) for a set bit (1). + if (_BitScanReverse64(&leading_zero, input_num)) + return (int)(63 - leading_zero); + else + return 64; +#else + return __builtin_clzll(input_num); +#endif // SIMDJSON_REGULAR_VISUAL_STUDIO + } + +#if SIMDJSON_REGULAR_VISUAL_STUDIO + simdjson_inline int count_ones(uint64_t input_num) { + // note: we do not support legacy 32-bit Windows in this kernel + return __popcnt64(input_num); // Visual Studio wants two underscores + } +#else + simdjson_inline int count_ones(uint64_t input_num) { + return __builtin_popcountll(input_num); + } +#endif + + simdjson_inline bool add_overflow(uint64_t value1, uint64_t value2, + uint64_t* result) { +#if SIMDJSON_REGULAR_VISUAL_STUDIO + * result = value1 + value2; + return *result < value1; +#else + return __builtin_uaddll_overflow(value1, value2, + reinterpret_cast(result)); +#endif + } + + } // unnamed namespace + } // namespace ppc64 +} // namespace simdjson + +#endif // SIMDJSON_PPC64_BITMANIPULATION_H +/* end file simdjson/ppc64/bitmanipulation.h */ +/* including simdjson/ppc64/bitmask.h: #include "simdjson/ppc64/bitmask.h" */ +/* begin file simdjson/ppc64/bitmask.h */ +#ifndef SIMDJSON_PPC64_BITMASK_H +#define SIMDJSON_PPC64_BITMASK_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/ppc64/base.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace ppc64 { + namespace { + + // + // Perform a "cumulative bitwise xor," flipping bits each time a 1 is + // encountered. + // + // For example, prefix_xor(00100100) == 00011100 + // + simdjson_inline uint64_t prefix_xor(uint64_t bitmask) { + // You can use the version below, however gcc sometimes miscompiles + // vec_pmsum_be, it happens somewhere around between 8 and 9th version. + // The performance boost was not noticeable, falling back to a usual + // implementation. + // __vector unsigned long long all_ones = {~0ull, ~0ull}; + // __vector unsigned long long mask = {bitmask, 0}; + // // Clang and GCC return different values for pmsum for ull so cast it to one. + // // Generally it is not specified by ALTIVEC ISA what is returned by + // // vec_pmsum_be. + // #if defined(__LITTLE_ENDIAN__) + // return (uint64_t)(((__vector unsigned long long)vec_pmsum_be(all_ones, mask))[0]); + // #else + // return (uint64_t)(((__vector unsigned long long)vec_pmsum_be(all_ones, mask))[1]); + // #endif + bitmask ^= bitmask << 1; + bitmask ^= bitmask << 2; + bitmask ^= bitmask << 4; + bitmask ^= bitmask << 8; + bitmask ^= bitmask << 16; + bitmask ^= bitmask << 32; + return bitmask; + } + + } // unnamed namespace + } // namespace ppc64 +} // namespace simdjson + +#endif +/* end file simdjson/ppc64/bitmask.h */ +/* including simdjson/ppc64/numberparsing_defs.h: #include "simdjson/ppc64/numberparsing_defs.h" */ +/* begin file simdjson/ppc64/numberparsing_defs.h */ +#ifndef SIMDJSON_PPC64_NUMBERPARSING_DEFS_H +#define SIMDJSON_PPC64_NUMBERPARSING_DEFS_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/ppc64/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/ppc64/intrinsics.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +#include + +#if defined(__linux__) +#include +#elif defined(__FreeBSD__) +#include +#endif + +namespace simdjson { + namespace ppc64 { + namespace numberparsing { + + // we don't have appropriate instructions, so let us use a scalar function + // credit: https://johnnylee-sde.github.io/Fast-numeric-string-to-int/ + /** @private */ + static simdjson_inline uint32_t parse_eight_digits_unrolled(const uint8_t* chars) { + uint64_t val; + std::memcpy(&val, chars, sizeof(uint64_t)); +#ifdef __BIG_ENDIAN__ +#if defined(__linux__) + val = bswap_64(val); +#elif defined(__FreeBSD__) + val = bswap64(val); +#endif +#endif + val = (val & 0x0F0F0F0F0F0F0F0F) * 2561 >> 8; + val = (val & 0x00FF00FF00FF00FF) * 6553601 >> 16; + return uint32_t((val & 0x0000FFFF0000FFFF) * 42949672960001 >> 32); + } + + /** @private */ + simdjson_inline internal::value128 full_multiplication(uint64_t value1, uint64_t value2) { + internal::value128 answer; +#if SIMDJSON_REGULAR_VISUAL_STUDIO || SIMDJSON_IS_32BITS +#if SIMDJSON_IS_ARM64 + // ARM64 has native support for 64-bit multiplications, no need to emultate + answer.high = __umulh(value1, value2); + answer.low = value1 * value2; +#else + answer.low = _umul128(value1, value2, &answer.high); // _umul128 not available on ARM64 +#endif // SIMDJSON_IS_ARM64 +#else // SIMDJSON_REGULAR_VISUAL_STUDIO || SIMDJSON_IS_32BITS + __uint128_t r = (static_cast<__uint128_t>(value1)) * value2; + answer.low = uint64_t(r); + answer.high = uint64_t(r >> 64); +#endif + return answer; + } + + } // namespace numberparsing + } // namespace ppc64 +} // namespace simdjson + +#define SIMDJSON_SWAR_NUMBER_PARSING 1 + +#endif // SIMDJSON_PPC64_NUMBERPARSING_DEFS_H +/* end file simdjson/ppc64/numberparsing_defs.h */ +/* including simdjson/ppc64/simd.h: #include "simdjson/ppc64/simd.h" */ +/* begin file simdjson/ppc64/simd.h */ +#ifndef SIMDJSON_PPC64_SIMD_H +#define SIMDJSON_PPC64_SIMD_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/ppc64/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/ppc64/bitmanipulation.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/simdprune_tables.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +#include + +namespace simdjson { + namespace ppc64 { + namespace { + namespace simd { + + using __m128i = __vector unsigned char; + + template struct base { + __m128i value; + + // Zero constructor + simdjson_inline base() : value{ __m128i() } {} + + // Conversion from SIMD register + simdjson_inline base(const __m128i _value) : value(_value) {} + + // Conversion to SIMD register + simdjson_inline operator const __m128i& () const { + return this->value; + } + simdjson_inline operator __m128i& () { return this->value; } + + // Bit operations + simdjson_inline Child operator|(const Child other) const { + return vec_or(this->value, (__m128i)other); + } + simdjson_inline Child operator&(const Child other) const { + return vec_and(this->value, (__m128i)other); + } + simdjson_inline Child operator^(const Child other) const { + return vec_xor(this->value, (__m128i)other); + } + simdjson_inline Child bit_andnot(const Child other) const { + return vec_andc(this->value, (__m128i)other); + } + simdjson_inline Child& operator|=(const Child other) { + auto this_cast = static_cast(this); + *this_cast = *this_cast | other; + return *this_cast; + } + simdjson_inline Child& operator&=(const Child other) { + auto this_cast = static_cast(this); + *this_cast = *this_cast & other; + return *this_cast; + } + simdjson_inline Child& operator^=(const Child other) { + auto this_cast = static_cast(this); + *this_cast = *this_cast ^ other; + return *this_cast; + } + }; + + template > + struct base8 : base> { + typedef uint16_t bitmask_t; + typedef uint32_t bitmask2_t; + + simdjson_inline base8() : base>() {} + simdjson_inline base8(const __m128i _value) : base>(_value) {} + + friend simdjson_inline Mask operator==(const simd8 lhs, const simd8 rhs) { + return (__m128i)vec_cmpeq(lhs.value, (__m128i)rhs); + } + + static const int SIZE = sizeof(base>::value); + + template + simdjson_inline simd8 prev(simd8 prev_chunk) const { + __m128i chunk = this->value; +#ifdef __LITTLE_ENDIAN__ + chunk = (__m128i)vec_reve(this->value); + prev_chunk = (__m128i)vec_reve((__m128i)prev_chunk); +#endif + chunk = (__m128i)vec_sld((__m128i)prev_chunk, (__m128i)chunk, 16 - N); +#ifdef __LITTLE_ENDIAN__ + chunk = (__m128i)vec_reve((__m128i)chunk); +#endif + return chunk; + } + }; + + // SIMD byte mask type (returned by things like eq and gt) + template <> struct simd8 : base8 { + static simdjson_inline simd8 splat(bool _value) { + return (__m128i)vec_splats((unsigned char)(-(!!_value))); + } + + simdjson_inline simd8() : base8() {} + simdjson_inline simd8(const __m128i _value) + : base8(_value) {} + // Splat constructor + simdjson_inline simd8(bool _value) + : base8(splat(_value)) {} + + simdjson_inline int to_bitmask() const { + __vector unsigned long long result; + const __m128i perm_mask = { 0x78, 0x70, 0x68, 0x60, 0x58, 0x50, 0x48, 0x40, + 0x38, 0x30, 0x28, 0x20, 0x18, 0x10, 0x08, 0x00 }; + + result = ((__vector unsigned long long)vec_vbpermq((__m128i)this->value, + (__m128i)perm_mask)); +#ifdef __LITTLE_ENDIAN__ + return static_cast(result[1]); +#else + return static_cast(result[0]); +#endif + } + simdjson_inline bool any() const { + return !vec_all_eq(this->value, (__m128i)vec_splats(0)); + } + simdjson_inline simd8 operator~() const { + return this->value ^ (__m128i)splat(true); + } + }; + + template struct base8_numeric : base8 { + static simdjson_inline simd8 splat(T value) { + (void)value; + return (__m128i)vec_splats(value); + } + static simdjson_inline simd8 zero() { return splat(0); } + static simdjson_inline simd8 load(const T values[16]) { + return (__m128i)(vec_vsx_ld(0, reinterpret_cast(values))); + } + // Repeat 16 values as many times as necessary (usually for lookup tables) + static simdjson_inline simd8 repeat_16(T v0, T v1, T v2, T v3, T v4, + T v5, T v6, T v7, T v8, T v9, + T v10, T v11, T v12, T v13, + T v14, T v15) { + return simd8(v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, + v14, v15); + } + + simdjson_inline base8_numeric() : base8() {} + simdjson_inline base8_numeric(const __m128i _value) + : base8(_value) {} + + // Store to array + simdjson_inline void store(T dst[16]) const { + vec_vsx_st(this->value, 0, reinterpret_cast<__m128i*>(dst)); + } + + // Override to distinguish from bool version + simdjson_inline simd8 operator~() const { return *this ^ 0xFFu; } + + // Addition/subtraction are the same for signed and unsigned + simdjson_inline simd8 operator+(const simd8 other) const { + return (__m128i)((__m128i)this->value + (__m128i)other); + } + simdjson_inline simd8 operator-(const simd8 other) const { + return (__m128i)((__m128i)this->value - (__m128i)other); + } + simdjson_inline simd8& operator+=(const simd8 other) { + *this = *this + other; + return *static_cast *>(this); + } + simdjson_inline simd8& operator-=(const simd8 other) { + *this = *this - other; + return *static_cast *>(this); + } + + // Perform a lookup assuming the value is between 0 and 16 (undefined behavior + // for out of range values) + template + simdjson_inline simd8 lookup_16(simd8 lookup_table) const { + return (__m128i)vec_perm((__m128i)lookup_table, (__m128i)lookup_table, this->value); + } + + // Copies to 'output" all bytes corresponding to a 0 in the mask (interpreted + // as a bitset). Passing a 0 value for mask would be equivalent to writing out + // every byte to output. Only the first 16 - count_ones(mask) bytes of the + // result are significant but 16 bytes get written. Design consideration: it + // seems like a function with the signature simd8 compress(uint32_t mask) + // would be sensible, but the AVX ISA makes this kind of approach difficult. + template + simdjson_inline void compress(uint16_t mask, L* output) const { + using internal::BitsSetTable256mul2; + using internal::pshufb_combine_table; + using internal::thintable_epi8; + // this particular implementation was inspired by work done by @animetosho + // we do it in two steps, first 8 bytes and then second 8 bytes + uint8_t mask1 = uint8_t(mask); // least significant 8 bits + uint8_t mask2 = uint8_t(mask >> 8); // most significant 8 bits + // next line just loads the 64-bit values thintable_epi8[mask1] and + // thintable_epi8[mask2] into a 128-bit register, using only + // two instructions on most compilers. +#ifdef __LITTLE_ENDIAN__ + __m128i shufmask = (__m128i)(__vector unsigned long long) { + thintable_epi8[mask1], thintable_epi8[mask2] + }; +#else + __m128i shufmask = (__m128i)(__vector unsigned long long) { + thintable_epi8[mask2], thintable_epi8[mask1] + }; + shufmask = (__m128i)vec_reve((__m128i)shufmask); +#endif + // we increment by 0x08 the second half of the mask + shufmask = ((__m128i)shufmask) + + ((__m128i)(__vector int) { 0, 0, 0x08080808, 0x08080808 }); + + // this is the version "nearly pruned" + __m128i pruned = vec_perm(this->value, this->value, shufmask); + // we still need to put the two halves together. + // we compute the popcount of the first half: + int pop1 = BitsSetTable256mul2[mask1]; + // then load the corresponding mask, what it does is to write + // only the first pop1 bytes from the first 8 bytes, and then + // it fills in with the bytes from the second 8 bytes + some filling + // at the end. + __m128i compactmask = + vec_vsx_ld(0, reinterpret_cast(pshufb_combine_table + pop1 * 8)); + __m128i answer = vec_perm(pruned, (__m128i)vec_splats(0), compactmask); + vec_vsx_st(answer, 0, reinterpret_cast<__m128i*>(output)); + } + + template + simdjson_inline simd8 + lookup_16(L replace0, L replace1, L replace2, L replace3, L replace4, + L replace5, L replace6, L replace7, L replace8, L replace9, + L replace10, L replace11, L replace12, L replace13, L replace14, + L replace15) const { + return lookup_16(simd8::repeat_16( + replace0, replace1, replace2, replace3, replace4, replace5, replace6, + replace7, replace8, replace9, replace10, replace11, replace12, + replace13, replace14, replace15)); + } + }; + + // Signed bytes + template <> struct simd8 : base8_numeric { + simdjson_inline simd8() : base8_numeric() {} + simdjson_inline simd8(const __m128i _value) + : base8_numeric(_value) {} + // Splat constructor + simdjson_inline simd8(int8_t _value) : simd8(splat(_value)) {} + // Array constructor + simdjson_inline simd8(const int8_t* values) : simd8(load(values)) {} + // Member-by-member initialization + simdjson_inline simd8(int8_t v0, int8_t v1, int8_t v2, int8_t v3, + int8_t v4, int8_t v5, int8_t v6, int8_t v7, + int8_t v8, int8_t v9, int8_t v10, int8_t v11, + int8_t v12, int8_t v13, int8_t v14, int8_t v15) + : simd8((__m128i)(__vector signed char) { + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, + v15 + }) {} + // Repeat 16 values as many times as necessary (usually for lookup tables) + simdjson_inline static simd8 + repeat_16(int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, + int8_t v6, int8_t v7, int8_t v8, int8_t v9, int8_t v10, int8_t v11, + int8_t v12, int8_t v13, int8_t v14, int8_t v15) { + return simd8(v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, + v13, v14, v15); + } + + // Order-sensitive comparisons + simdjson_inline simd8 + max_val(const simd8 other) const { + return (__m128i)vec_max((__vector signed char)this->value, + (__vector signed char)(__m128i)other); + } + simdjson_inline simd8 + min_val(const simd8 other) const { + return (__m128i)vec_min((__vector signed char)this->value, + (__vector signed char)(__m128i)other); + } + simdjson_inline simd8 + operator>(const simd8 other) const { + return (__m128i)vec_cmpgt((__vector signed char)this->value, + (__vector signed char)(__m128i)other); + } + simdjson_inline simd8 + operator<(const simd8 other) const { + return (__m128i)vec_cmplt((__vector signed char)this->value, + (__vector signed char)(__m128i)other); + } + }; + + // Unsigned bytes + template <> struct simd8 : base8_numeric { + simdjson_inline simd8() : base8_numeric() {} + simdjson_inline simd8(const __m128i _value) + : base8_numeric(_value) {} + // Splat constructor + simdjson_inline simd8(uint8_t _value) : simd8(splat(_value)) {} + // Array constructor + simdjson_inline simd8(const uint8_t* values) : simd8(load(values)) {} + // Member-by-member initialization + simdjson_inline + simd8(uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, + uint8_t v6, uint8_t v7, uint8_t v8, uint8_t v9, uint8_t v10, + uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15) + : simd8((__m128i) { + v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, + v13, v14, v15 + }) {} + // Repeat 16 values as many times as necessary (usually for lookup tables) + simdjson_inline static simd8 + repeat_16(uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, + uint8_t v5, uint8_t v6, uint8_t v7, uint8_t v8, uint8_t v9, + uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, + uint8_t v15) { + return simd8(v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, + v13, v14, v15); + } + + // Saturated math + simdjson_inline simd8 + saturating_add(const simd8 other) const { + return (__m128i)vec_adds(this->value, (__m128i)other); + } + simdjson_inline simd8 + saturating_sub(const simd8 other) const { + return (__m128i)vec_subs(this->value, (__m128i)other); + } + + // Order-specific operations + simdjson_inline simd8 + max_val(const simd8 other) const { + return (__m128i)vec_max(this->value, (__m128i)other); + } + simdjson_inline simd8 + min_val(const simd8 other) const { + return (__m128i)vec_min(this->value, (__m128i)other); + } + // Same as >, but only guarantees true is nonzero (< guarantees true = -1) + simdjson_inline simd8 + gt_bits(const simd8 other) const { + return this->saturating_sub(other); + } + // Same as <, but only guarantees true is nonzero (< guarantees true = -1) + simdjson_inline simd8 + lt_bits(const simd8 other) const { + return other.saturating_sub(*this); + } + simdjson_inline simd8 + operator<=(const simd8 other) const { + return other.max_val(*this) == other; + } + simdjson_inline simd8 + operator>=(const simd8 other) const { + return other.min_val(*this) == other; + } + simdjson_inline simd8 + operator>(const simd8 other) const { + return this->gt_bits(other).any_bits_set(); + } + simdjson_inline simd8 + operator<(const simd8 other) const { + return this->gt_bits(other).any_bits_set(); + } + + // Bit-specific operations + simdjson_inline simd8 bits_not_set() const { + return (__m128i)vec_cmpeq(this->value, (__m128i)vec_splats(uint8_t(0))); + } + simdjson_inline simd8 bits_not_set(simd8 bits) const { + return (*this & bits).bits_not_set(); + } + simdjson_inline simd8 any_bits_set() const { + return ~this->bits_not_set(); + } + simdjson_inline simd8 any_bits_set(simd8 bits) const { + return ~this->bits_not_set(bits); + } + simdjson_inline bool bits_not_set_anywhere() const { + return vec_all_eq(this->value, (__m128i)vec_splats(0)); + } + simdjson_inline bool any_bits_set_anywhere() const { + return !bits_not_set_anywhere(); + } + simdjson_inline bool bits_not_set_anywhere(simd8 bits) const { + return vec_all_eq(vec_and(this->value, (__m128i)bits), + (__m128i)vec_splats(0)); + } + simdjson_inline bool any_bits_set_anywhere(simd8 bits) const { + return !bits_not_set_anywhere(bits); + } + template simdjson_inline simd8 shr() const { + return simd8( + (__m128i)vec_sr(this->value, (__m128i)vec_splat_u8(N))); + } + template simdjson_inline simd8 shl() const { + return simd8( + (__m128i)vec_sl(this->value, (__m128i)vec_splat_u8(N))); + } + }; + + template struct simd8x64 { + static constexpr int NUM_CHUNKS = 64 / sizeof(simd8); + static_assert(NUM_CHUNKS == 4, + "PPC64 kernel should use four registers per 64-byte block."); + const simd8 chunks[NUM_CHUNKS]; + + simd8x64(const simd8x64& o) = delete; // no copy allowed + simd8x64& + operator=(const simd8& other) = delete; // no assignment allowed + simd8x64() = delete; // no default constructor allowed + + simdjson_inline simd8x64(const simd8 chunk0, const simd8 chunk1, + const simd8 chunk2, const simd8 chunk3) + : chunks{ chunk0, chunk1, chunk2, chunk3 } {} + simdjson_inline simd8x64(const T ptr[64]) + : chunks{ simd8::load(ptr), simd8::load(ptr + 16), + simd8::load(ptr + 32), simd8::load(ptr + 48) } {} + + simdjson_inline void store(T ptr[64]) const { + this->chunks[0].store(ptr + sizeof(simd8) * 0); + this->chunks[1].store(ptr + sizeof(simd8) * 1); + this->chunks[2].store(ptr + sizeof(simd8) * 2); + this->chunks[3].store(ptr + sizeof(simd8) * 3); + } + + simdjson_inline simd8 reduce_or() const { + return (this->chunks[0] | this->chunks[1]) | + (this->chunks[2] | this->chunks[3]); + } + + simdjson_inline uint64_t compress(uint64_t mask, T* output) const { + this->chunks[0].compress(uint16_t(mask), output); + this->chunks[1].compress(uint16_t(mask >> 16), + output + 16 - count_ones(mask & 0xFFFF)); + this->chunks[2].compress(uint16_t(mask >> 32), + output + 32 - count_ones(mask & 0xFFFFFFFF)); + this->chunks[3].compress(uint16_t(mask >> 48), + output + 48 - count_ones(mask & 0xFFFFFFFFFFFF)); + return 64 - count_ones(mask); + } + + simdjson_inline uint64_t to_bitmask() const { + uint64_t r0 = uint32_t(this->chunks[0].to_bitmask()); + uint64_t r1 = this->chunks[1].to_bitmask(); + uint64_t r2 = this->chunks[2].to_bitmask(); + uint64_t r3 = this->chunks[3].to_bitmask(); + return r0 | (r1 << 16) | (r2 << 32) | (r3 << 48); + } + + simdjson_inline uint64_t eq(const T m) const { + const simd8 mask = simd8::splat(m); + return simd8x64(this->chunks[0] == mask, this->chunks[1] == mask, + this->chunks[2] == mask, this->chunks[3] == mask) + .to_bitmask(); + } + + simdjson_inline uint64_t eq(const simd8x64& other) const { + return simd8x64(this->chunks[0] == other.chunks[0], + this->chunks[1] == other.chunks[1], + this->chunks[2] == other.chunks[2], + this->chunks[3] == other.chunks[3]) + .to_bitmask(); + } + + simdjson_inline uint64_t lteq(const T m) const { + const simd8 mask = simd8::splat(m); + return simd8x64(this->chunks[0] <= mask, this->chunks[1] <= mask, + this->chunks[2] <= mask, this->chunks[3] <= mask) + .to_bitmask(); + } + }; // struct simd8x64 + + } // namespace simd + } // unnamed namespace + } // namespace ppc64 +} // namespace simdjson + +#endif // SIMDJSON_PPC64_SIMD_INPUT_H +/* end file simdjson/ppc64/simd.h */ +/* including simdjson/ppc64/stringparsing_defs.h: #include "simdjson/ppc64/stringparsing_defs.h" */ +/* begin file simdjson/ppc64/stringparsing_defs.h */ +#ifndef SIMDJSON_PPC64_STRINGPARSING_DEFS_H +#define SIMDJSON_PPC64_STRINGPARSING_DEFS_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/ppc64/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/ppc64/bitmanipulation.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/ppc64/simd.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace ppc64 { + namespace { + + using namespace simd; + + // Holds backslashes and quotes locations. + struct backslash_and_quote { + public: + static constexpr uint32_t BYTES_PROCESSED = 32; + simdjson_inline static backslash_and_quote + copy_and_find(const uint8_t* src, uint8_t* dst); + + simdjson_inline bool has_quote_first() { + return ((bs_bits - 1) & quote_bits) != 0; + } + simdjson_inline bool has_backslash() { return bs_bits != 0; } + simdjson_inline int quote_index() { + return trailing_zeroes(quote_bits); + } + simdjson_inline int backslash_index() { + return trailing_zeroes(bs_bits); + } + + uint32_t bs_bits; + uint32_t quote_bits; + }; // struct backslash_and_quote + + simdjson_inline backslash_and_quote + backslash_and_quote::copy_and_find(const uint8_t* src, uint8_t* dst) { + // this can read up to 31 bytes beyond the buffer size, but we require + // SIMDJSON_PADDING of padding + static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), + "backslash and quote finder must process fewer than " + "SIMDJSON_PADDING bytes"); + simd8 v0(src); + simd8 v1(src + sizeof(v0)); + v0.store(dst); + v1.store(dst + sizeof(v0)); + + // Getting a 64-bit bitmask is much cheaper than multiple 16-bit bitmasks on + // PPC; therefore, we smash them together into a 64-byte mask and get the + // bitmask from there. + uint64_t bs_and_quote = + simd8x64(v0 == '\\', v1 == '\\', v0 == '"', v1 == '"').to_bitmask(); + return { + uint32_t(bs_and_quote), // bs_bits + uint32_t(bs_and_quote >> 32) // quote_bits + }; + } + + } // unnamed namespace + } // namespace ppc64 +} // namespace simdjson + +#endif // SIMDJSON_PPC64_STRINGPARSING_DEFS_H +/* end file simdjson/ppc64/stringparsing_defs.h */ + +#define SIMDJSON_SKIP_BACKSLASH_SHORT_CIRCUIT 1 +/* end file simdjson/ppc64/begin.h */ +/* including generic/amalgamated.h for ppc64: #include */ +/* begin file generic/amalgamated.h for ppc64 */ +#if defined(SIMDJSON_CONDITIONAL_INCLUDE) && !defined(SIMDJSON_SRC_GENERIC_DEPENDENCIES_H) +#error generic/dependencies.h must be included before generic/amalgamated.h! +#endif + +/* including generic/base.h for ppc64: #include */ +/* begin file generic/base.h for ppc64 */ +#ifndef SIMDJSON_SRC_GENERIC_BASE_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_BASE_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace ppc64 { + namespace { + + struct json_character_block; + + } // unnamed namespace + } // namespace ppc64 +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_BASE_H +/* end file generic/base.h for ppc64 */ +/* including generic/dom_parser_implementation.h for ppc64: #include */ +/* begin file generic/dom_parser_implementation.h for ppc64 */ +#ifndef SIMDJSON_SRC_GENERIC_DOM_PARSER_IMPLEMENTATION_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_DOM_PARSER_IMPLEMENTATION_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +// Interface a dom parser implementation must fulfill +namespace simdjson { + namespace ppc64 { + namespace { + + simdjson_inline simd8 must_be_2_3_continuation(const simd8 prev2, const simd8 prev3); + simdjson_inline bool is_ascii(const simd8x64& input); + + } // unnamed namespace + } // namespace ppc64 +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_DOM_PARSER_IMPLEMENTATION_H +/* end file generic/dom_parser_implementation.h for ppc64 */ +/* including generic/json_character_block.h for ppc64: #include */ +/* begin file generic/json_character_block.h for ppc64 */ +#ifndef SIMDJSON_SRC_GENERIC_JSON_CHARACTER_BLOCK_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_JSON_CHARACTER_BLOCK_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace ppc64 { + namespace { + + struct json_character_block { + static simdjson_inline json_character_block classify(const simd::simd8x64& in); + + simdjson_inline uint64_t whitespace() const noexcept { return _whitespace; } + simdjson_inline uint64_t op() const noexcept { return _op; } + simdjson_inline uint64_t scalar() const noexcept { return ~(op() | whitespace()); } + + uint64_t _whitespace; + uint64_t _op; + }; + + } // unnamed namespace + } // namespace ppc64 +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_JSON_CHARACTER_BLOCK_H +/* end file generic/json_character_block.h for ppc64 */ +/* end file generic/amalgamated.h for ppc64 */ +/* including generic/stage1/amalgamated.h for ppc64: #include */ +/* begin file generic/stage1/amalgamated.h for ppc64 */ +// Stuff other things depend on +/* including generic/stage1/base.h for ppc64: #include */ +/* begin file generic/stage1/base.h for ppc64 */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE1_BASE_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_BASE_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace ppc64 { + namespace { + namespace stage1 { + + class bit_indexer; + template + struct buf_block_reader; + struct json_block; + class json_minifier; + class json_scanner; + struct json_string_block; + class json_string_scanner; + class json_structural_indexer; + + } // namespace stage1 + + namespace utf8_validation { + struct utf8_checker; + } // namespace utf8_validation + + using utf8_validation::utf8_checker; + + } // unnamed namespace + } // namespace ppc64 +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE1_BASE_H +/* end file generic/stage1/base.h for ppc64 */ +/* including generic/stage1/buf_block_reader.h for ppc64: #include */ +/* begin file generic/stage1/buf_block_reader.h for ppc64 */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE1_BUF_BLOCK_READER_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_BUF_BLOCK_READER_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +#include + +namespace simdjson { + namespace ppc64 { + namespace { + namespace stage1 { + + // Walks through a buffer in block-sized increments, loading the last part with spaces + template + struct buf_block_reader { + public: + simdjson_inline buf_block_reader(const uint8_t* _buf, size_t _len); + simdjson_inline size_t block_index(); + simdjson_inline bool has_full_block() const; + simdjson_inline const uint8_t* full_block() const; + /** + * Get the last block, padded with spaces. + * + * There will always be a last block, with at least 1 byte, unless len == 0 (in which case this + * function fills the buffer with spaces and returns 0. In particular, if len == STEP_SIZE there + * will be 0 full_blocks and 1 remainder block with STEP_SIZE bytes and no spaces for padding. + * + * @return the number of effective characters in the last block. + */ + simdjson_inline size_t get_remainder(uint8_t* dst) const; + simdjson_inline void advance(); + private: + const uint8_t* buf; + const size_t len; + const size_t lenminusstep; + size_t idx; + }; + + // Routines to print masks and text for debugging bitmask operations + simdjson_unused static char* format_input_text_64(const uint8_t* text) { + static char buf[sizeof(simd8x64) + 1]; + for (size_t i = 0; i < sizeof(simd8x64); i++) { + buf[i] = int8_t(text[i]) < ' ' ? '_' : int8_t(text[i]); + } + buf[sizeof(simd8x64)] = '\0'; + return buf; + } + + // Routines to print masks and text for debugging bitmask operations + simdjson_unused static char* format_input_text(const simd8x64& in) { + static char buf[sizeof(simd8x64) + 1]; + in.store(reinterpret_cast(buf)); + for (size_t i = 0; i < sizeof(simd8x64); i++) { + if (buf[i] < ' ') { buf[i] = '_'; } + } + buf[sizeof(simd8x64)] = '\0'; + return buf; + } + + simdjson_unused static char* format_input_text(const simd8x64& in, uint64_t mask) { + static char buf[sizeof(simd8x64) + 1]; + in.store(reinterpret_cast(buf)); + for (size_t i = 0; i < sizeof(simd8x64); i++) { + if (buf[i] <= ' ') { buf[i] = '_'; } + if (!(mask & (size_t(1) << i))) { buf[i] = ' '; } + } + buf[sizeof(simd8x64)] = '\0'; + return buf; + } + + simdjson_unused static char* format_mask(uint64_t mask) { + static char buf[sizeof(simd8x64) + 1]; + for (size_t i = 0; i < 64; i++) { + buf[i] = (mask & (size_t(1) << i)) ? 'X' : ' '; + } + buf[64] = '\0'; + return buf; + } + + template + simdjson_inline buf_block_reader::buf_block_reader(const uint8_t* _buf, size_t _len) : buf{ _buf }, len{ _len }, lenminusstep{ len < STEP_SIZE ? 0 : len - STEP_SIZE }, idx{ 0 } {} + + template + simdjson_inline size_t buf_block_reader::block_index() { return idx; } + + template + simdjson_inline bool buf_block_reader::has_full_block() const { + return idx < lenminusstep; + } + + template + simdjson_inline const uint8_t* buf_block_reader::full_block() const { + return &buf[idx]; + } + + template + simdjson_inline size_t buf_block_reader::get_remainder(uint8_t* dst) const { + if (len == idx) { return 0; } // memcpy(dst, null, 0) will trigger an error with some sanitizers + std::memset(dst, 0x20, STEP_SIZE); // std::memset STEP_SIZE because it's more efficient to write out 8 or 16 bytes at once. + std::memcpy(dst, buf + idx, len - idx); + return len - idx; + } + + template + simdjson_inline void buf_block_reader::advance() { + idx += STEP_SIZE; + } + + } // namespace stage1 + } // unnamed namespace + } // namespace ppc64 +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE1_BUF_BLOCK_READER_H +/* end file generic/stage1/buf_block_reader.h for ppc64 */ +/* including generic/stage1/json_escape_scanner.h for ppc64: #include */ +/* begin file generic/stage1/json_escape_scanner.h for ppc64 */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE1_JSON_ESCAPE_SCANNER_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_JSON_ESCAPE_SCANNER_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace ppc64 { + namespace { + namespace stage1 { + + /** + * Scans for escape characters in JSON, taking care with multiple backslashes (\\n vs. \n). + */ + struct json_escape_scanner { + /** The actual escape characters (the backslashes themselves). */ + uint64_t next_is_escaped = 0ULL; + + struct escaped_and_escape { + /** + * Mask of escaped characters. + * + * ``` + * \n \\n \\\n \\\\n \ + * 0100100010100101000 + * n \ \ n \ \ + * ``` + */ + uint64_t escaped; + /** + * Mask of escape characters. + * + * ``` + * \n \\n \\\n \\\\n \ + * 1001000101001010001 + * \ \ \ \ \ \ \ + * ``` + */ + uint64_t escape; + }; + + /** + * Get a mask of both escape and escaped characters (the characters following a backslash). + * + * @param potential_escape A mask of the character that can escape others (but could be + * escaped itself). e.g. block.eq('\\') + */ + simdjson_really_inline escaped_and_escape next(uint64_t backslash) noexcept { + +#if !SIMDJSON_SKIP_BACKSLASH_SHORT_CIRCUIT + if (!backslash) { return { next_escaped_without_backslashes(), 0 }; } +#endif + + // | | Mask (shows characters instead of 1's) | Depth | Instructions | + // |--------------------------------|----------------------------------------|-------|---------------------| + // | string | `\\n_\\\n___\\\n___\\\\___\\\\__\\\` | | | + // | | ` even odd even odd odd` | | | + // | potential_escape | ` \ \\\ \\\ \\\\ \\\\ \\\` | 1 | 1 (backslash & ~first_is_escaped) + // | escape_and_terminal_code | ` \n \ \n \ \n \ \ \ \ \ \` | 5 | 5 (next_escape_and_terminal_code()) + // | escaped | `\ \ n \ n \ \ \ \ \ ` X | 6 | 7 (escape_and_terminal_code ^ (potential_escape | first_is_escaped)) + // | escape | ` \ \ \ \ \ \ \ \ \ \` | 6 | 8 (escape_and_terminal_code & backslash) + // | first_is_escaped | `\ ` | 7 (*) | 9 (escape >> 63) () + // (*) this is not needed until the next iteration + uint64_t escape_and_terminal_code = next_escape_and_terminal_code(backslash & ~this->next_is_escaped); + uint64_t escaped = escape_and_terminal_code ^ (backslash | this->next_is_escaped); + uint64_t escape = escape_and_terminal_code & backslash; + this->next_is_escaped = escape >> 63; + return { escaped, escape }; + } + + private: + static constexpr const uint64_t ODD_BITS = 0xAAAAAAAAAAAAAAAAULL; + + simdjson_really_inline uint64_t next_escaped_without_backslashes() noexcept { + uint64_t escaped = this->next_is_escaped; + this->next_is_escaped = 0; + return escaped; + } + + /** + * Returns a mask of the next escape characters (masking out escaped backslashes), along with + * any non-backslash escape codes. + * + * \n \\n \\\n \\\\n returns: + * \n \ \ \n \ \ + * 11 100 1011 10100 + * + * You are expected to mask out the first bit yourself if the previous block had a trailing + * escape. + * + * & the result with potential_escape to get just the escape characters. + * ^ the result with (potential_escape | first_is_escaped) to get escaped characters. + */ + static simdjson_really_inline uint64_t next_escape_and_terminal_code(uint64_t potential_escape) noexcept { + // If we were to just shift and mask out any odd bits, we'd actually get a *half* right answer: + // any even-aligned backslash runs would be correct! Odd-aligned backslash runs would be + // inverted (\\\ would be 010 instead of 101). + // + // ``` + // string: | ____\\\\_\\\\_____ | + // maybe_escaped | ODD | \ \ \ \ | + // even-aligned ^^^ ^^^^ odd-aligned + // ``` + // + // Taking that into account, our basic strategy is: + // + // 1. Use subtraction to produce a mask with 1's for even-aligned runs and 0's for + // odd-aligned runs. + // 2. XOR all odd bits, which masks out the odd bits in even-aligned runs, and brings IN the + // odd bits in odd-aligned runs. + // 3. & with backslash to clean up any stray bits. + // runs are set to 0, and then XORing with "odd": + // + // | | Mask (shows characters instead of 1's) | Instructions | + // |--------------------------------|----------------------------------------|---------------------| + // | string | `\\n_\\\n___\\\n___\\\\___\\\\__\\\` | + // | | ` even odd even odd odd` | + // | maybe_escaped | ` n \\n \\n \\\_ \\\_ \\` X | 1 (potential_escape << 1) + // | maybe_escaped_and_odd | ` \n_ \\n _ \\\n_ _ \\\__ _\\\_ \\\` | 1 (maybe_escaped | odd) + // | even_series_codes_and_odd | ` n_\\\ _ n_ _\\\\ _ _ ` | 1 (maybe_escaped_and_odd - potential_escape) + // | escape_and_terminal_code | ` \n \ \n \ \n \ \ \ \ \ \` | 1 (^ odd) + // + + // Escaped characters are characters following an escape. + uint64_t maybe_escaped = potential_escape << 1; + + // To distinguish odd from even escape sequences, therefore, we turn on any *starting* + // escapes that are on an odd byte. (We actually bring in all odd bits, for speed.) + // - Odd runs of backslashes are 0000, and the code at the end ("n" in \n or \\n) is 1. + // - Odd runs of backslashes are 1111, and the code at the end ("n" in \n or \\n) is 0. + // - All other odd bytes are 1, and even bytes are 0. + uint64_t maybe_escaped_and_odd_bits = maybe_escaped | ODD_BITS; + uint64_t even_series_codes_and_odd_bits = maybe_escaped_and_odd_bits - potential_escape; + + // Now we flip all odd bytes back with xor. This: + // - Makes odd runs of backslashes go from 0000 to 1010 + // - Makes even runs of backslashes go from 1111 to 1010 + // - Sets actually-escaped codes to 1 (the n in \n and \\n: \n = 11, \\n = 100) + // - Resets all other bytes to 0 + return even_series_codes_and_odd_bits ^ ODD_BITS; + } + }; + + } // namespace stage1 + } // unnamed namespace + } // namespace ppc64 +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRING_SCANNER_H +/* end file generic/stage1/json_escape_scanner.h for ppc64 */ +/* including generic/stage1/json_string_scanner.h for ppc64: #include */ +/* begin file generic/stage1/json_string_scanner.h for ppc64 */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRING_SCANNER_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRING_SCANNER_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace ppc64 { + namespace { + namespace stage1 { + + struct json_string_block { + // We spell out the constructors in the hope of resolving inlining issues with Visual Studio 2017 + simdjson_really_inline json_string_block(uint64_t escaped, uint64_t quote, uint64_t in_string) : + _escaped(escaped), _quote(quote), _in_string(in_string) {} + + // Escaped characters (characters following an escape() character) + simdjson_really_inline uint64_t escaped() const { return _escaped; } + // Real (non-backslashed) quotes + simdjson_really_inline uint64_t quote() const { return _quote; } + // Only characters inside the string (not including the quotes) + simdjson_really_inline uint64_t string_content() const { return _in_string & ~_quote; } + // Return a mask of whether the given characters are inside a string (only works on non-quotes) + simdjson_really_inline uint64_t non_quote_inside_string(uint64_t mask) const { return mask & _in_string; } + // Return a mask of whether the given characters are inside a string (only works on non-quotes) + simdjson_really_inline uint64_t non_quote_outside_string(uint64_t mask) const { return mask & ~_in_string; } + // Tail of string (everything except the start quote) + simdjson_really_inline uint64_t string_tail() const { return _in_string ^ _quote; } + + // escaped characters (backslashed--does not include the hex characters after \u) + uint64_t _escaped; + // real quotes (non-escaped ones) + uint64_t _quote; + // string characters (includes start quote but not end quote) + uint64_t _in_string; + }; + + // Scans blocks for string characters, storing the state necessary to do so + class json_string_scanner { + public: + simdjson_really_inline json_string_block next(const simd::simd8x64& in); + // Returns either UNCLOSED_STRING or SUCCESS + simdjson_really_inline error_code finish(); + + private: + // Scans for escape characters + json_escape_scanner escape_scanner{}; + // Whether the last iteration was still inside a string (all 1's = true, all 0's = false). + uint64_t prev_in_string = 0ULL; + }; + + // + // Return a mask of all string characters plus end quotes. + // + // prev_escaped is overflow saying whether the next character is escaped. + // prev_in_string is overflow saying whether we're still in a string. + // + // Backslash sequences outside of quotes will be detected in stage 2. + // + simdjson_really_inline json_string_block json_string_scanner::next(const simd::simd8x64& in) { + const uint64_t backslash = in.eq('\\'); + const uint64_t escaped = escape_scanner.next(backslash).escaped; + const uint64_t quote = in.eq('"') & ~escaped; + + // + // prefix_xor flips on bits inside the string (and flips off the end quote). + // + // Then we xor with prev_in_string: if we were in a string already, its effect is flipped + // (characters inside strings are outside, and characters outside strings are inside). + // + const uint64_t in_string = prefix_xor(quote) ^ prev_in_string; + + // + // Check if we're still in a string at the end of the box so the next block will know + // + prev_in_string = uint64_t(static_cast(in_string) >> 63); + + // Use ^ to turn the beginning quote off, and the end quote on. + + // We are returning a function-local object so either we get a move constructor + // or we get copy elision. + return json_string_block(escaped, quote, in_string); + } + + simdjson_really_inline error_code json_string_scanner::finish() { + if (prev_in_string) { + return UNCLOSED_STRING; + } + return SUCCESS; + } + + } // namespace stage1 + } // unnamed namespace + } // namespace ppc64 +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRING_SCANNER_H +/* end file generic/stage1/json_string_scanner.h for ppc64 */ +/* including generic/stage1/utf8_lookup4_algorithm.h for ppc64: #include */ +/* begin file generic/stage1/utf8_lookup4_algorithm.h for ppc64 */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE1_UTF8_LOOKUP4_ALGORITHM_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_UTF8_LOOKUP4_ALGORITHM_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace ppc64 { + namespace { + namespace utf8_validation { + + using namespace simd; + + simdjson_inline simd8 check_special_cases(const simd8 input, const simd8 prev1) { + // Bit 0 = Too Short (lead byte/ASCII followed by lead byte/ASCII) + // Bit 1 = Too Long (ASCII followed by continuation) + // Bit 2 = Overlong 3-byte + // Bit 4 = Surrogate + // Bit 5 = Overlong 2-byte + // Bit 7 = Two Continuations + constexpr const uint8_t TOO_SHORT = 1 << 0; // 11______ 0_______ + // 11______ 11______ + constexpr const uint8_t TOO_LONG = 1 << 1; // 0_______ 10______ + constexpr const uint8_t OVERLONG_3 = 1 << 2; // 11100000 100_____ + constexpr const uint8_t SURROGATE = 1 << 4; // 11101101 101_____ + constexpr const uint8_t OVERLONG_2 = 1 << 5; // 1100000_ 10______ + constexpr const uint8_t TWO_CONTS = 1 << 7; // 10______ 10______ + constexpr const uint8_t TOO_LARGE = 1 << 3; // 11110100 1001____ + // 11110100 101_____ + // 11110101 1001____ + // 11110101 101_____ + // 1111011_ 1001____ + // 1111011_ 101_____ + // 11111___ 1001____ + // 11111___ 101_____ + constexpr const uint8_t TOO_LARGE_1000 = 1 << 6; + // 11110101 1000____ + // 1111011_ 1000____ + // 11111___ 1000____ + constexpr const uint8_t OVERLONG_4 = 1 << 6; // 11110000 1000____ + + const simd8 byte_1_high = prev1.shr<4>().lookup_16( + // 0_______ ________ + TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, + TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, + // 10______ ________ + TWO_CONTS, TWO_CONTS, TWO_CONTS, TWO_CONTS, + // 1100____ ________ + TOO_SHORT | OVERLONG_2, + // 1101____ ________ + TOO_SHORT, + // 1110____ ________ + TOO_SHORT | OVERLONG_3 | SURROGATE, + // 1111____ ________ + TOO_SHORT | TOO_LARGE | TOO_LARGE_1000 | OVERLONG_4 + ); + constexpr const uint8_t CARRY = TOO_SHORT | TOO_LONG | TWO_CONTS; // These all have ____ in byte 1 . + const simd8 byte_1_low = (prev1 & 0x0F).lookup_16( + // ____0000 ________ + CARRY | OVERLONG_3 | OVERLONG_2 | OVERLONG_4, + // ____0001 ________ + CARRY | OVERLONG_2, + // ____001_ ________ + CARRY, + CARRY, + + // ____0100 ________ + CARRY | TOO_LARGE, + // ____0101 ________ + CARRY | TOO_LARGE | TOO_LARGE_1000, + // ____011_ ________ + CARRY | TOO_LARGE | TOO_LARGE_1000, + CARRY | TOO_LARGE | TOO_LARGE_1000, + + // ____1___ ________ + CARRY | TOO_LARGE | TOO_LARGE_1000, + CARRY | TOO_LARGE | TOO_LARGE_1000, + CARRY | TOO_LARGE | TOO_LARGE_1000, + CARRY | TOO_LARGE | TOO_LARGE_1000, + CARRY | TOO_LARGE | TOO_LARGE_1000, + // ____1101 ________ + CARRY | TOO_LARGE | TOO_LARGE_1000 | SURROGATE, + CARRY | TOO_LARGE | TOO_LARGE_1000, + CARRY | TOO_LARGE | TOO_LARGE_1000 + ); + const simd8 byte_2_high = input.shr<4>().lookup_16( + // ________ 0_______ + TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT, + TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT, + + // ________ 1000____ + TOO_LONG | OVERLONG_2 | TWO_CONTS | OVERLONG_3 | TOO_LARGE_1000 | OVERLONG_4, + // ________ 1001____ + TOO_LONG | OVERLONG_2 | TWO_CONTS | OVERLONG_3 | TOO_LARGE, + // ________ 101_____ + TOO_LONG | OVERLONG_2 | TWO_CONTS | SURROGATE | TOO_LARGE, + TOO_LONG | OVERLONG_2 | TWO_CONTS | SURROGATE | TOO_LARGE, + + // ________ 11______ + TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT + ); + return (byte_1_high & byte_1_low & byte_2_high); + } + simdjson_inline simd8 check_multibyte_lengths(const simd8 input, + const simd8 prev_input, const simd8 sc) { + simd8 prev2 = input.prev<2>(prev_input); + simd8 prev3 = input.prev<3>(prev_input); + simd8 must23 = must_be_2_3_continuation(prev2, prev3); + simd8 must23_80 = must23 & uint8_t(0x80); + return must23_80 ^ sc; + } + + // + // Return nonzero if there are incomplete multibyte characters at the end of the block: + // e.g. if there is a 4-byte character, but it's 3 bytes from the end. + // + simdjson_inline simd8 is_incomplete(const simd8 input) { + // If the previous input's last 3 bytes match this, they're too short (they ended at EOF): + // ... 1111____ 111_____ 11______ +#if SIMDJSON_IMPLEMENTATION_ICELAKE + static const uint8_t max_array[64] = { + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 0xf0u - 1, 0xe0u - 1, 0xc0u - 1 + }; +#else + static const uint8_t max_array[32] = { + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 0xf0u - 1, 0xe0u - 1, 0xc0u - 1 + }; +#endif + const simd8 max_value(&max_array[sizeof(max_array) - sizeof(simd8)]); + return input.gt_bits(max_value); + } + + struct utf8_checker { + // If this is nonzero, there has been a UTF-8 error. + simd8 error; + // The last input we received + simd8 prev_input_block; + // Whether the last input we received was incomplete (used for ASCII fast path) + simd8 prev_incomplete; + + // + // Check whether the current bytes are valid UTF-8. + // + simdjson_inline void check_utf8_bytes(const simd8 input, const simd8 prev_input) { + // Flip prev1...prev3 so we can easily determine if they are 2+, 3+ or 4+ lead bytes + // (2, 3, 4-byte leads become large positive numbers instead of small negative numbers) + simd8 prev1 = input.prev<1>(prev_input); + simd8 sc = check_special_cases(input, prev1); + this->error |= check_multibyte_lengths(input, prev_input, sc); + } + + // The only problem that can happen at EOF is that a multibyte character is too short + // or a byte value too large in the last bytes: check_special_cases only checks for bytes + // too large in the first of two bytes. + simdjson_inline void check_eof() { + // If the previous block had incomplete UTF-8 characters at the end, an ASCII block can't + // possibly finish them. + this->error |= this->prev_incomplete; + } + + simdjson_inline void check_next_input(const simd8x64& input) { + if (simdjson_likely(is_ascii(input))) { + this->error |= this->prev_incomplete; + } + else { + // you might think that a for-loop would work, but under Visual Studio, it is not good enough. + static_assert((simd8x64::NUM_CHUNKS == 1) + || (simd8x64::NUM_CHUNKS == 2) + || (simd8x64::NUM_CHUNKS == 4), + "We support one, two or four chunks per 64-byte block."); + SIMDJSON_IF_CONSTEXPR(simd8x64::NUM_CHUNKS == 1) { + this->check_utf8_bytes(input.chunks[0], this->prev_input_block); + } +else SIMDJSON_IF_CONSTEXPR(simd8x64::NUM_CHUNKS == 2) { + this->check_utf8_bytes(input.chunks[0], this->prev_input_block); + this->check_utf8_bytes(input.chunks[1], input.chunks[0]); + } + else SIMDJSON_IF_CONSTEXPR(simd8x64::NUM_CHUNKS == 4) { + this->check_utf8_bytes(input.chunks[0], this->prev_input_block); + this->check_utf8_bytes(input.chunks[1], input.chunks[0]); + this->check_utf8_bytes(input.chunks[2], input.chunks[1]); + this->check_utf8_bytes(input.chunks[3], input.chunks[2]); + } + this->prev_incomplete = is_incomplete(input.chunks[simd8x64::NUM_CHUNKS - 1]); + this->prev_input_block = input.chunks[simd8x64::NUM_CHUNKS - 1]; + } + } + // do not forget to call check_eof! + simdjson_inline error_code errors() { + return this->error.any_bits_set_anywhere() ? error_code::UTF8_ERROR : error_code::SUCCESS; + } + + }; // struct utf8_checker + } // namespace utf8_validation + + } // unnamed namespace + } // namespace ppc64 +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE1_UTF8_LOOKUP4_ALGORITHM_H +/* end file generic/stage1/utf8_lookup4_algorithm.h for ppc64 */ +/* including generic/stage1/json_scanner.h for ppc64: #include */ +/* begin file generic/stage1/json_scanner.h for ppc64 */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE1_JSON_SCANNER_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_JSON_SCANNER_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace ppc64 { + namespace { + namespace stage1 { + + /** + * A block of scanned json, with information on operators and scalars. + * + * We seek to identify pseudo-structural characters. Anything that is inside + * a string must be omitted (hence & ~_string.string_tail()). + * Otherwise, pseudo-structural characters come in two forms. + * 1. We have the structural characters ([,],{,},:, comma). The + * term 'structural character' is from the JSON RFC. + * 2. We have the 'scalar pseudo-structural characters'. + * Scalars are quotes, and any character except structural characters and white space. + * + * To identify the scalar pseudo-structural characters, we must look at what comes + * before them: it must be a space, a quote or a structural characters. + * Starting with simdjson v0.3, we identify them by + * negation: we identify everything that is followed by a non-quote scalar, + * and we negate that. Whatever remains must be a 'scalar pseudo-structural character'. + */ + struct json_block { + public: + // We spell out the constructors in the hope of resolving inlining issues with Visual Studio 2017 + simdjson_inline json_block(json_string_block&& string, json_character_block characters, uint64_t follows_potential_nonquote_scalar) : + _string(std::move(string)), _characters(characters), _follows_potential_nonquote_scalar(follows_potential_nonquote_scalar) {} + simdjson_inline json_block(json_string_block string, json_character_block characters, uint64_t follows_potential_nonquote_scalar) : + _string(string), _characters(characters), _follows_potential_nonquote_scalar(follows_potential_nonquote_scalar) {} + + /** + * The start of structurals. + * In simdjson prior to v0.3, these were called the pseudo-structural characters. + **/ + simdjson_inline uint64_t structural_start() const noexcept { return potential_structural_start() & ~_string.string_tail(); } + /** All JSON whitespace (i.e. not in a string) */ + simdjson_inline uint64_t whitespace() const noexcept { return non_quote_outside_string(_characters.whitespace()); } + + // Helpers + + /** Whether the given characters are inside a string (only works on non-quotes) */ + simdjson_inline uint64_t non_quote_inside_string(uint64_t mask) const noexcept { return _string.non_quote_inside_string(mask); } + /** Whether the given characters are outside a string (only works on non-quotes) */ + simdjson_inline uint64_t non_quote_outside_string(uint64_t mask) const noexcept { return _string.non_quote_outside_string(mask); } + + // string and escape characters + json_string_block _string; + // whitespace, structural characters ('operators'), scalars + json_character_block _characters; + // whether the previous character was a scalar + uint64_t _follows_potential_nonquote_scalar; + private: + // Potential structurals (i.e. disregarding strings) + + /** + * structural elements ([,],{,},:, comma) plus scalar starts like 123, true and "abc". + * They may reside inside a string. + **/ + simdjson_inline uint64_t potential_structural_start() const noexcept { return _characters.op() | potential_scalar_start(); } + /** + * The start of non-operator runs, like 123, true and "abc". + * It main reside inside a string. + **/ + simdjson_inline uint64_t potential_scalar_start() const noexcept { + // The term "scalar" refers to anything except structural characters and white space + // (so letters, numbers, quotes). + // Whenever it is preceded by something that is not a structural element ({,},[,],:, ") nor a white-space + // then we know that it is irrelevant structurally. + return _characters.scalar() & ~follows_potential_scalar(); + } + /** + * Whether the given character is immediately after a non-operator like 123, true. + * The characters following a quote are not included. + */ + simdjson_inline uint64_t follows_potential_scalar() const noexcept { + // _follows_potential_nonquote_scalar: is defined as marking any character that follows a character + // that is not a structural element ({,},[,],:, comma) nor a quote (") and that is not a + // white space. + // It is understood that within quoted region, anything at all could be marked (irrelevant). + return _follows_potential_nonquote_scalar; + } + }; + + /** + * Scans JSON for important bits: structural characters or 'operators', strings, and scalars. + * + * The scanner starts by calculating two distinct things: + * - string characters (taking \" into account) + * - structural characters or 'operators' ([]{},:, comma) + * and scalars (runs of non-operators like 123, true and "abc") + * + * To minimize data dependency (a key component of the scanner's speed), it finds these in parallel: + * in particular, the operator/scalar bit will find plenty of things that are actually part of + * strings. When we're done, json_block will fuse the two together by masking out tokens that are + * part of a string. + */ + class json_scanner { + public: + json_scanner() = default; + simdjson_inline json_block next(const simd::simd8x64& in); + // Returns either UNCLOSED_STRING or SUCCESS + simdjson_inline error_code finish(); + + private: + // Whether the last character of the previous iteration is part of a scalar token + // (anything except whitespace or a structural character/'operator'). + uint64_t prev_scalar = 0ULL; + json_string_scanner string_scanner{}; + }; + + + // + // Check if the current character immediately follows a matching character. + // + // For example, this checks for quotes with backslashes in front of them: + // + // const uint64_t backslashed_quote = in.eq('"') & immediately_follows(in.eq('\'), prev_backslash); + // + simdjson_inline uint64_t follows(const uint64_t match, uint64_t& overflow) { + const uint64_t result = match << 1 | overflow; + overflow = match >> 63; + return result; + } + + simdjson_inline json_block json_scanner::next(const simd::simd8x64& in) { + json_string_block strings = string_scanner.next(in); + // identifies the white-space and the structural characters + json_character_block characters = json_character_block::classify(in); + // The term "scalar" refers to anything except structural characters and white space + // (so letters, numbers, quotes). + // We want follows_scalar to mark anything that follows a non-quote scalar (so letters and numbers). + // + // A terminal quote should either be followed by a structural character (comma, brace, bracket, colon) + // or nothing. However, we still want ' "a string"true ' to mark the 't' of 'true' as a potential + // pseudo-structural character just like we would if we had ' "a string" true '; otherwise we + // may need to add an extra check when parsing strings. + // + // Performance: there are many ways to skin this cat. + const uint64_t nonquote_scalar = characters.scalar() & ~strings.quote(); + uint64_t follows_nonquote_scalar = follows(nonquote_scalar, prev_scalar); + // We are returning a function-local object so either we get a move constructor + // or we get copy elision. + return json_block( + strings,// strings is a function-local object so either it moves or the copy is elided. + characters, + follows_nonquote_scalar + ); + } + + simdjson_inline error_code json_scanner::finish() { + return string_scanner.finish(); + } + + } // namespace stage1 + } // unnamed namespace + } // namespace ppc64 +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE1_JSON_SCANNER_H +/* end file generic/stage1/json_scanner.h for ppc64 */ + +// All other declarations +/* including generic/stage1/find_next_document_index.h for ppc64: #include */ +/* begin file generic/stage1/find_next_document_index.h for ppc64 */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE1_FIND_NEXT_DOCUMENT_INDEX_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_FIND_NEXT_DOCUMENT_INDEX_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace ppc64 { + namespace { + namespace stage1 { + + /** + * This algorithm is used to quickly identify the last structural position that + * makes up a complete document. + * + * It does this by going backwards and finding the last *document boundary* (a + * place where one value follows another without a comma between them). If the + * last document (the characters after the boundary) has an equal number of + * start and end brackets, it is considered complete. + * + * Simply put, we iterate over the structural characters, starting from + * the end. We consider that we found the end of a JSON document when the + * first element of the pair is NOT one of these characters: '{' '[' ':' ',' + * and when the second element is NOT one of these characters: '}' ']' ':' ','. + * + * This simple comparison works most of the time, but it does not cover cases + * where the batch's structural indexes contain a perfect amount of documents. + * In such a case, we do not have access to the structural index which follows + * the last document, therefore, we do not have access to the second element in + * the pair, and that means we cannot identify the last document. To fix this + * issue, we keep a count of the open and closed curly/square braces we found + * while searching for the pair. When we find a pair AND the count of open and + * closed curly/square braces is the same, we know that we just passed a + * complete document, therefore the last json buffer location is the end of the + * batch. + */ + simdjson_inline uint32_t find_next_document_index(dom_parser_implementation& parser) { + // Variant: do not count separately, just figure out depth + if (parser.n_structural_indexes == 0) { return 0; } + auto arr_cnt = 0; + auto obj_cnt = 0; + for (auto i = parser.n_structural_indexes - 1; i > 0; i--) { + auto idxb = parser.structural_indexes[i]; + switch (parser.buf[idxb]) { + case ':': + case ',': + continue; + case '}': + obj_cnt--; + continue; + case ']': + arr_cnt--; + continue; + case '{': + obj_cnt++; + break; + case '[': + arr_cnt++; + break; + } + auto idxa = parser.structural_indexes[i - 1]; + switch (parser.buf[idxa]) { + case '{': + case '[': + case ':': + case ',': + continue; + } + // Last document is complete, so the next document will appear after! + if (!arr_cnt && !obj_cnt) { + return parser.n_structural_indexes; + } + // Last document is incomplete; mark the document at i + 1 as the next one + return i; + } + // If we made it to the end, we want to finish counting to see if we have a full document. + switch (parser.buf[parser.structural_indexes[0]]) { + case '}': + obj_cnt--; + break; + case ']': + arr_cnt--; + break; + case '{': + obj_cnt++; + break; + case '[': + arr_cnt++; + break; + } + if (!arr_cnt && !obj_cnt) { + // We have a complete document. + return parser.n_structural_indexes; + } + return 0; + } + + } // namespace stage1 + } // unnamed namespace + } // namespace ppc64 +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE1_FIND_NEXT_DOCUMENT_INDEX_H +/* end file generic/stage1/find_next_document_index.h for ppc64 */ +/* including generic/stage1/json_minifier.h for ppc64: #include */ +/* begin file generic/stage1/json_minifier.h for ppc64 */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE1_JSON_MINIFIER_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_JSON_MINIFIER_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +// This file contains the common code every implementation uses in stage1 +// It is intended to be included multiple times and compiled multiple times +// We assume the file in which it is included already includes +// "simdjson/stage1.h" (this simplifies amalgation) + +namespace simdjson { + namespace ppc64 { + namespace { + namespace stage1 { + + class json_minifier { + public: + template + static error_code minify(const uint8_t* buf, size_t len, uint8_t* dst, size_t& dst_len) noexcept; + + private: + simdjson_inline json_minifier(uint8_t* _dst) + : dst{ _dst } + {} + template + simdjson_inline void step(const uint8_t* block_buf, buf_block_reader& reader) noexcept; + simdjson_inline void next(const simd::simd8x64& in, const json_block& block); + simdjson_inline error_code finish(uint8_t* dst_start, size_t& dst_len); + json_scanner scanner{}; + uint8_t* dst; + }; + + simdjson_inline void json_minifier::next(const simd::simd8x64& in, const json_block& block) { + uint64_t mask = block.whitespace(); + dst += in.compress(mask, dst); + } + + simdjson_inline error_code json_minifier::finish(uint8_t* dst_start, size_t& dst_len) { + error_code error = scanner.finish(); + if (error) { dst_len = 0; return error; } + dst_len = dst - dst_start; + return SUCCESS; + } + + template<> + simdjson_inline void json_minifier::step<128>(const uint8_t* block_buf, buf_block_reader<128>& reader) noexcept { + simd::simd8x64 in_1(block_buf); + simd::simd8x64 in_2(block_buf + 64); + json_block block_1 = scanner.next(in_1); + json_block block_2 = scanner.next(in_2); + this->next(in_1, block_1); + this->next(in_2, block_2); + reader.advance(); + } + + template<> + simdjson_inline void json_minifier::step<64>(const uint8_t* block_buf, buf_block_reader<64>& reader) noexcept { + simd::simd8x64 in_1(block_buf); + json_block block_1 = scanner.next(in_1); + this->next(block_buf, block_1); + reader.advance(); + } + + template + error_code json_minifier::minify(const uint8_t* buf, size_t len, uint8_t* dst, size_t& dst_len) noexcept { + buf_block_reader reader(buf, len); + json_minifier minifier(dst); + + // Index the first n-1 blocks + while (reader.has_full_block()) { + minifier.step(reader.full_block(), reader); + } + + // Index the last (remainder) block, padded with spaces + uint8_t block[STEP_SIZE]; + size_t remaining_bytes = reader.get_remainder(block); + if (remaining_bytes > 0) { + // We do not want to write directly to the output stream. Rather, we write + // to a local buffer (for safety). + uint8_t out_block[STEP_SIZE]; + uint8_t* const guarded_dst{ minifier.dst }; + minifier.dst = out_block; + minifier.step(block, reader); + size_t to_write = minifier.dst - out_block; + // In some cases, we could be enticed to consider the padded spaces + // as part of the string. This is fine as long as we do not write more + // than we consumed. + if (to_write > remaining_bytes) { to_write = remaining_bytes; } + memcpy(guarded_dst, out_block, to_write); + minifier.dst = guarded_dst + to_write; + } + return minifier.finish(dst, dst_len); + } + + } // namespace stage1 + } // unnamed namespace + } // namespace ppc64 +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE1_JSON_MINIFIER_H +/* end file generic/stage1/json_minifier.h for ppc64 */ +/* including generic/stage1/json_structural_indexer.h for ppc64: #include */ +/* begin file generic/stage1/json_structural_indexer.h for ppc64 */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRUCTURAL_INDEXER_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRUCTURAL_INDEXER_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +// This file contains the common code every implementation uses in stage1 +// It is intended to be included multiple times and compiled multiple times +// We assume the file in which it is included already includes +// "simdjson/stage1.h" (this simplifies amalgation) + +namespace simdjson { + namespace ppc64 { + namespace { + namespace stage1 { + + class bit_indexer { + public: + uint32_t* tail; + + simdjson_inline bit_indexer(uint32_t* index_buf) : tail(index_buf) {} + +#if SIMDJSON_PREFER_REVERSE_BITS + /** + * ARM lacks a fast trailing zero instruction, but it has a fast + * bit reversal instruction and a fast leading zero instruction. + * Thus it may be profitable to reverse the bits (once) and then + * to rely on a sequence of instructions that call the leading + * zero instruction. + * + * Performance notes: + * The chosen routine is not optimal in terms of data dependency + * since zero_leading_bit might require two instructions. However, + * it tends to minimize the total number of instructions which is + * beneficial. + */ + simdjson_inline void write_index(uint32_t idx, uint64_t& rev_bits, int i) { + int lz = leading_zeroes(rev_bits); + this->tail[i] = static_cast(idx) + lz; + rev_bits = zero_leading_bit(rev_bits, lz); + } +#else + /** + * Under recent x64 systems, we often have both a fast trailing zero + * instruction and a fast 'clear-lower-bit' instruction so the following + * algorithm can be competitive. + */ + + simdjson_inline void write_index(uint32_t idx, uint64_t& bits, int i) { + this->tail[i] = idx + trailing_zeroes(bits); + bits = clear_lowest_bit(bits); + } +#endif // SIMDJSON_PREFER_REVERSE_BITS + + template + simdjson_inline int write_indexes(uint32_t idx, uint64_t& bits) { + write_index(idx, bits, START); + SIMDJSON_IF_CONSTEXPR(N > 1) { + write_indexes<(N - 1 > 0 ? START + 1 : START), (N - 1 >= 0 ? N - 1 : 1)>(idx, bits); + } + return START + N; + } + + template + simdjson_inline int write_indexes_stepped(uint32_t idx, uint64_t& bits, int cnt) { + write_indexes(idx, bits); + SIMDJSON_IF_CONSTEXPR((START + STEP) < END) { + if (simdjson_unlikely((START + STEP) < cnt)) { + write_indexes_stepped<(START + STEP < END ? START + STEP : END), END, STEP>(idx, bits, cnt); + } + } + return ((END - START) % STEP) == 0 ? END : (END - START) - ((END - START) % STEP) + STEP; + } + + // flatten out values in 'bits' assuming that they are are to have values of idx + // plus their position in the bitvector, and store these indexes at + // base_ptr[base] incrementing base as we go + // will potentially store extra values beyond end of valid bits, so base_ptr + // needs to be large enough to handle this + // + // If the kernel sets SIMDJSON_GENERIC_JSON_STRUCTURAL_INDEXER_CUSTOM_BIT_INDEXER, then it + // will provide its own version of the code. +#ifdef SIMDJSON_GENERIC_JSON_STRUCTURAL_INDEXER_CUSTOM_BIT_INDEXER + simdjson_inline void write(uint32_t idx, uint64_t bits); +#else + simdjson_inline void write(uint32_t idx, uint64_t bits) { + // In some instances, the next branch is expensive because it is mispredicted. + // Unfortunately, in other cases, + // it helps tremendously. + if (bits == 0) + return; + + int cnt = static_cast(count_ones(bits)); + +#if SIMDJSON_PREFER_REVERSE_BITS + bits = reverse_bits(bits); +#endif +#ifdef SIMDJSON_STRUCTURAL_INDEXER_STEP + static constexpr const int STEP = SIMDJSON_STRUCTURAL_INDEXER_STEP; +#else + static constexpr const int STEP = 4; +#endif + static constexpr const int STEP_UNTIL = 24; + + write_indexes_stepped<0, STEP_UNTIL, STEP>(idx, bits, cnt); + SIMDJSON_IF_CONSTEXPR(STEP_UNTIL < 64) { + if (simdjson_unlikely(STEP_UNTIL < cnt)) { + for (int i = STEP_UNTIL; i < cnt; i++) { + write_index(idx, bits, i); + } + } + } + + this->tail += cnt; + } +#endif // SIMDJSON_GENERIC_JSON_STRUCTURAL_INDEXER_CUSTOM_BIT_INDEXER + + }; + + class json_structural_indexer { + public: + /** + * Find the important bits of JSON in a 128-byte chunk, and add them to structural_indexes. + * + * @param partial Setting the partial parameter to true allows the find_structural_bits to + * tolerate unclosed strings. The caller should still ensure that the input is valid UTF-8. If + * you are processing substrings, you may want to call on a function like trimmed_length_safe_utf8. + */ + template + static error_code index(const uint8_t* buf, size_t len, dom_parser_implementation& parser, stage1_mode partial) noexcept; + + private: + simdjson_inline json_structural_indexer(uint32_t* structural_indexes); + template + simdjson_inline void step(const uint8_t* block, buf_block_reader& reader) noexcept; + simdjson_inline void next(const simd::simd8x64& in, const json_block& block, size_t idx); + simdjson_inline error_code finish(dom_parser_implementation& parser, size_t idx, size_t len, stage1_mode partial); + + json_scanner scanner{}; + utf8_checker checker{}; + bit_indexer indexer; + uint64_t prev_structurals = 0; + uint64_t unescaped_chars_error = 0; + }; + + simdjson_inline json_structural_indexer::json_structural_indexer(uint32_t* structural_indexes) : indexer{ structural_indexes } {} + + // Skip the last character if it is partial + simdjson_inline size_t trim_partial_utf8(const uint8_t* buf, size_t len) { + if (simdjson_unlikely(len < 3)) { + switch (len) { + case 2: + if (buf[len - 1] >= 0xc0) { return len - 1; } // 2-, 3- and 4-byte characters with only 1 byte left + if (buf[len - 2] >= 0xe0) { return len - 2; } // 3- and 4-byte characters with only 2 bytes left + return len; + case 1: + if (buf[len - 1] >= 0xc0) { return len - 1; } // 2-, 3- and 4-byte characters with only 1 byte left + return len; + case 0: + return len; + } + } + if (buf[len - 1] >= 0xc0) { return len - 1; } // 2-, 3- and 4-byte characters with only 1 byte left + if (buf[len - 2] >= 0xe0) { return len - 2; } // 3- and 4-byte characters with only 1 byte left + if (buf[len - 3] >= 0xf0) { return len - 3; } // 4-byte characters with only 3 bytes left + return len; + } + + // + // PERF NOTES: + // We pipe 2 inputs through these stages: + // 1. Load JSON into registers. This takes a long time and is highly parallelizable, so we load + // 2 inputs' worth at once so that by the time step 2 is looking for them input, it's available. + // 2. Scan the JSON for critical data: strings, scalars and operators. This is the critical path. + // The output of step 1 depends entirely on this information. These functions don't quite use + // up enough CPU: the second half of the functions is highly serial, only using 1 execution core + // at a time. The second input's scans has some dependency on the first ones finishing it, but + // they can make a lot of progress before they need that information. + // 3. Step 1 does not use enough capacity, so we run some extra stuff while we're waiting for that + // to finish: utf-8 checks and generating the output from the last iteration. + // + // The reason we run 2 inputs at a time, is steps 2 and 3 are *still* not enough to soak up all + // available capacity with just one input. Running 2 at a time seems to give the CPU a good enough + // workout. + // + template + error_code json_structural_indexer::index(const uint8_t* buf, size_t len, dom_parser_implementation& parser, stage1_mode partial) noexcept { + if (simdjson_unlikely(len > parser.capacity())) { return CAPACITY; } + // We guard the rest of the code so that we can assume that len > 0 throughout. + if (len == 0) { return EMPTY; } + if (is_streaming(partial)) { + len = trim_partial_utf8(buf, len); + // If you end up with an empty window after trimming + // the partial UTF-8 bytes, then chances are good that you + // have an UTF-8 formatting error. + if (len == 0) { return UTF8_ERROR; } + } + buf_block_reader reader(buf, len); + json_structural_indexer indexer(parser.structural_indexes.get()); + + // Read all but the last block + while (reader.has_full_block()) { + indexer.step(reader.full_block(), reader); + } + // Take care of the last block (will always be there unless file is empty which is + // not supposed to happen.) + uint8_t block[STEP_SIZE]; + if (simdjson_unlikely(reader.get_remainder(block) == 0)) { return UNEXPECTED_ERROR; } + indexer.step(block, reader); + return indexer.finish(parser, reader.block_index(), len, partial); + } + + template<> + simdjson_inline void json_structural_indexer::step<128>(const uint8_t* block, buf_block_reader<128>& reader) noexcept { + simd::simd8x64 in_1(block); + simd::simd8x64 in_2(block + 64); + json_block block_1 = scanner.next(in_1); + json_block block_2 = scanner.next(in_2); + this->next(in_1, block_1, reader.block_index()); + this->next(in_2, block_2, reader.block_index() + 64); + reader.advance(); + } + + template<> + simdjson_inline void json_structural_indexer::step<64>(const uint8_t* block, buf_block_reader<64>& reader) noexcept { + simd::simd8x64 in_1(block); + json_block block_1 = scanner.next(in_1); + this->next(in_1, block_1, reader.block_index()); + reader.advance(); + } + + simdjson_inline void json_structural_indexer::next(const simd::simd8x64& in, const json_block& block, size_t idx) { + uint64_t unescaped = in.lteq(0x1F); +#if SIMDJSON_UTF8VALIDATION + checker.check_next_input(in); +#endif + indexer.write(uint32_t(idx - 64), prev_structurals); // Output *last* iteration's structurals to the parser + prev_structurals = block.structural_start(); + unescaped_chars_error |= block.non_quote_inside_string(unescaped); + } + + simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementation& parser, size_t idx, size_t len, stage1_mode partial) { + // Write out the final iteration's structurals + indexer.write(uint32_t(idx - 64), prev_structurals); + error_code error = scanner.finish(); + // We deliberately break down the next expression so that it is + // human readable. + const bool should_we_exit = is_streaming(partial) ? + ((error != SUCCESS) && (error != UNCLOSED_STRING)) // when partial we tolerate UNCLOSED_STRING + : (error != SUCCESS); // if partial is false, we must have SUCCESS + const bool have_unclosed_string = (error == UNCLOSED_STRING); + if (simdjson_unlikely(should_we_exit)) { return error; } + + if (unescaped_chars_error) { + return UNESCAPED_CHARS; + } + parser.n_structural_indexes = uint32_t(indexer.tail - parser.structural_indexes.get()); + /*** + * The On-Demand API requires special padding. + * + * This is related to https://github.com/simdjson/simdjson/issues/906 + * Basically, we want to make sure that if the parsing continues beyond the last (valid) + * structural character, it quickly stops. + * Only three structural characters can be repeated without triggering an error in JSON: [,] and }. + * We repeat the padding character (at 'len'). We don't know what it is, but if the parsing + * continues, then it must be [,] or }. + * Suppose it is ] or }. We backtrack to the first character, what could it be that would + * not trigger an error? It could be ] or } but no, because you can't start a document that way. + * It can't be a comma, a colon or any simple value. So the only way we could continue is + * if the repeated character is [. But if so, the document must start with [. But if the document + * starts with [, it should end with ]. If we enforce that rule, then we would get + * ][[ which is invalid. + * + * This is illustrated with the test array_iterate_unclosed_error() on the following input: + * R"({ "a": [,,)" + **/ + parser.structural_indexes[parser.n_structural_indexes] = uint32_t(len); // used later in partial == stage1_mode::streaming_final + parser.structural_indexes[parser.n_structural_indexes + 1] = uint32_t(len); + parser.structural_indexes[parser.n_structural_indexes + 2] = 0; + parser.next_structural_index = 0; + // a valid JSON file cannot have zero structural indexes - we should have found something + if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { + return EMPTY; + } + if (simdjson_unlikely(parser.structural_indexes[parser.n_structural_indexes - 1] > len)) { + return UNEXPECTED_ERROR; + } + if (partial == stage1_mode::streaming_partial) { + // If we have an unclosed string, then the last structural + // will be the quote and we want to make sure to omit it. + if (have_unclosed_string) { + parser.n_structural_indexes--; + // a valid JSON file cannot have zero structural indexes - we should have found something + if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return CAPACITY; } + } + // We truncate the input to the end of the last complete document (or zero). + auto new_structural_indexes = find_next_document_index(parser); + if (new_structural_indexes == 0 && parser.n_structural_indexes > 0) { + if (parser.structural_indexes[0] == 0) { + // If the buffer is partial and we started at index 0 but the document is + // incomplete, it's too big to parse. + return CAPACITY; + } + else { + // It is possible that the document could be parsed, we just had a lot + // of white space. + parser.n_structural_indexes = 0; + return EMPTY; + } + } + + parser.n_structural_indexes = new_structural_indexes; + } + else if (partial == stage1_mode::streaming_final) { + if (have_unclosed_string) { parser.n_structural_indexes--; } + // We truncate the input to the end of the last complete document (or zero). + // Because partial == stage1_mode::streaming_final, it means that we may + // silently ignore trailing garbage. Though it sounds bad, we do it + // deliberately because many people who have streams of JSON documents + // will truncate them for processing. E.g., imagine that you are uncompressing + // the data from a size file or receiving it in chunks from the network. You + // may not know where exactly the last document will be. Meanwhile the + // document_stream instances allow people to know the JSON documents they are + // parsing (see the iterator.source() method). + parser.n_structural_indexes = find_next_document_index(parser); + // We store the initial n_structural_indexes so that the client can see + // whether we used truncation. If initial_n_structural_indexes == parser.n_structural_indexes, + // then this will query parser.structural_indexes[parser.n_structural_indexes] which is len, + // otherwise, it will copy some prior index. + parser.structural_indexes[parser.n_structural_indexes + 1] = parser.structural_indexes[parser.n_structural_indexes]; + // This next line is critical, do not change it unless you understand what you are + // doing. + parser.structural_indexes[parser.n_structural_indexes] = uint32_t(len); + if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { + // We tolerate an unclosed string at the very end of the stream. Indeed, users + // often load their data in bulk without being careful and they want us to ignore + // the trailing garbage. + return EMPTY; + } + } + checker.check_eof(); + return checker.errors(); + } + + } // namespace stage1 + } // unnamed namespace + } // namespace ppc64 +} // namespace simdjson + +// Clear CUSTOM_BIT_INDEXER so other implementations can set it if they need to. +#undef SIMDJSON_GENERIC_JSON_STRUCTURAL_INDEXER_CUSTOM_BIT_INDEXER + +#endif // SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRUCTURAL_INDEXER_H +/* end file generic/stage1/json_structural_indexer.h for ppc64 */ +/* including generic/stage1/utf8_validator.h for ppc64: #include */ +/* begin file generic/stage1/utf8_validator.h for ppc64 */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE1_UTF8_VALIDATOR_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_UTF8_VALIDATOR_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace ppc64 { + namespace { + namespace stage1 { + + /** + * Validates that the string is actual UTF-8. + */ + template + bool generic_validate_utf8(const uint8_t* input, size_t length) { + checker c{}; + buf_block_reader<64> reader(input, length); + while (reader.has_full_block()) { + simd::simd8x64 in(reader.full_block()); + c.check_next_input(in); + reader.advance(); + } + uint8_t block[64]{}; + reader.get_remainder(block); + simd::simd8x64 in(block); + c.check_next_input(in); + reader.advance(); + c.check_eof(); + return c.errors() == error_code::SUCCESS; + } + + bool generic_validate_utf8(const char* input, size_t length) { + return generic_validate_utf8(reinterpret_cast(input), length); + } + + } // namespace stage1 + } // unnamed namespace + } // namespace ppc64 +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE1_UTF8_VALIDATOR_H +/* end file generic/stage1/utf8_validator.h for ppc64 */ +/* end file generic/stage1/amalgamated.h for ppc64 */ +/* including generic/stage2/amalgamated.h for ppc64: #include */ +/* begin file generic/stage2/amalgamated.h for ppc64 */ +// Stuff other things depend on +/* including generic/stage2/base.h for ppc64: #include */ +/* begin file generic/stage2/base.h for ppc64 */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE2_BASE_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_BASE_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace ppc64 { + namespace { + namespace stage2 { + + class json_iterator; + class structural_iterator; + struct tape_builder; + struct tape_writer; + + } // namespace stage2 + } // unnamed namespace + } // namespace ppc64 +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE2_BASE_H +/* end file generic/stage2/base.h for ppc64 */ +/* including generic/stage2/tape_writer.h for ppc64: #include */ +/* begin file generic/stage2/tape_writer.h for ppc64 */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE2_TAPE_WRITER_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_TAPE_WRITER_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +#include + +namespace simdjson { + namespace ppc64 { + namespace { + namespace stage2 { + + struct tape_writer { + /** The next place to write to tape */ + uint64_t* next_tape_loc; + + /** Write a signed 64-bit value to tape. */ + simdjson_inline void append_s64(int64_t value) noexcept; + + /** Write an unsigned 64-bit value to tape. */ + simdjson_inline void append_u64(uint64_t value) noexcept; + + /** Write a double value to tape. */ + simdjson_inline void append_double(double value) noexcept; + + /** + * Append a tape entry (an 8-bit type,and 56 bits worth of value). + */ + simdjson_inline void append(uint64_t val, internal::tape_type t) noexcept; + + /** + * Skip the current tape entry without writing. + * + * Used to skip the start of the container, since we'll come back later to fill it in when the + * container ends. + */ + simdjson_inline void skip() noexcept; + + /** + * Skip the number of tape entries necessary to write a large u64 or i64. + */ + simdjson_inline void skip_large_integer() noexcept; + + /** + * Skip the number of tape entries necessary to write a double. + */ + simdjson_inline void skip_double() noexcept; + + /** + * Write a value to a known location on tape. + * + * Used to go back and write out the start of a container after the container ends. + */ + simdjson_inline static void write(uint64_t& tape_loc, uint64_t val, internal::tape_type t) noexcept; + + private: + /** + * Append both the tape entry, and a supplementary value following it. Used for types that need + * all 64 bits, such as double and uint64_t. + */ + template + simdjson_inline void append2(uint64_t val, T val2, internal::tape_type t) noexcept; + }; // struct tape_writer + + simdjson_inline void tape_writer::append_s64(int64_t value) noexcept { + append2(0, value, internal::tape_type::INT64); + } + + simdjson_inline void tape_writer::append_u64(uint64_t value) noexcept { + append(0, internal::tape_type::UINT64); + *next_tape_loc = value; + next_tape_loc++; + } + + /** Write a double value to tape. */ + simdjson_inline void tape_writer::append_double(double value) noexcept { + append2(0, value, internal::tape_type::DOUBLE); + } + + simdjson_inline void tape_writer::skip() noexcept { + next_tape_loc++; + } + + simdjson_inline void tape_writer::skip_large_integer() noexcept { + next_tape_loc += 2; + } + + simdjson_inline void tape_writer::skip_double() noexcept { + next_tape_loc += 2; + } + + simdjson_inline void tape_writer::append(uint64_t val, internal::tape_type t) noexcept { + *next_tape_loc = val | ((uint64_t(char(t))) << 56); + next_tape_loc++; + } + + template + simdjson_inline void tape_writer::append2(uint64_t val, T val2, internal::tape_type t) noexcept { + append(val, t); + static_assert(sizeof(val2) == sizeof(*next_tape_loc), "Type is not 64 bits!"); + memcpy(next_tape_loc, &val2, sizeof(val2)); + next_tape_loc++; + } + + simdjson_inline void tape_writer::write(uint64_t& tape_loc, uint64_t val, internal::tape_type t) noexcept { + tape_loc = val | ((uint64_t(char(t))) << 56); + } + + } // namespace stage2 + } // unnamed namespace + } // namespace ppc64 +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE2_TAPE_WRITER_H +/* end file generic/stage2/tape_writer.h for ppc64 */ +/* including generic/stage2/logger.h for ppc64: #include */ +/* begin file generic/stage2/logger.h for ppc64 */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE2_LOGGER_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_LOGGER_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +#include + + +// This is for an internal-only stage 2 specific logger. +// Set LOG_ENABLED = true to log what stage 2 is doing! +namespace simdjson { + namespace ppc64 { + namespace { + namespace logger { + + static constexpr const char* DASHES = "----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------"; + +#if SIMDJSON_VERBOSE_LOGGING + static constexpr const bool LOG_ENABLED = true; +#else + static constexpr const bool LOG_ENABLED = false; +#endif + static constexpr const int LOG_EVENT_LEN = 20; + static constexpr const int LOG_BUFFER_LEN = 30; + static constexpr const int LOG_SMALL_BUFFER_LEN = 10; + static constexpr const int LOG_INDEX_LEN = 5; + + static int log_depth; // Not threadsafe. Log only. + + // Helper to turn unprintable or newline characters into spaces + static simdjson_inline char printable_char(char c) { + if (c >= 0x20) { + return c; + } + else { + return ' '; + } + } + + // Print the header and set up log_start + static simdjson_inline void log_start() { + if (LOG_ENABLED) { + log_depth = 0; + printf("\n"); + printf("| %-*s | %-*s | %-*s | %-*s | Detail |\n", LOG_EVENT_LEN, "Event", LOG_BUFFER_LEN, "Buffer", LOG_SMALL_BUFFER_LEN, "Next", 5, "Next#"); + printf("|%.*s|%.*s|%.*s|%.*s|--------|\n", LOG_EVENT_LEN + 2, DASHES, LOG_BUFFER_LEN + 2, DASHES, LOG_SMALL_BUFFER_LEN + 2, DASHES, 5 + 2, DASHES); + } + } + + simdjson_unused static simdjson_inline void log_string(const char* message) { + if (LOG_ENABLED) { + printf("%s\n", message); + } + } + + // Logs a single line from the stage 2 DOM parser + template + static simdjson_inline void log_line(S& structurals, const char* title_prefix, const char* title, const char* detail) { + if (LOG_ENABLED) { + printf("| %*s%s%-*s ", log_depth * 2, "", title_prefix, LOG_EVENT_LEN - log_depth * 2 - int(strlen(title_prefix)), title); + auto current_index = structurals.at_beginning() ? nullptr : structurals.next_structural - 1; + auto next_index = structurals.next_structural; + auto current = current_index ? &structurals.buf[*current_index] : reinterpret_cast(" "); + auto next = &structurals.buf[*next_index]; + { + // Print the next N characters in the buffer. + printf("| "); + // Otherwise, print the characters starting from the buffer position. + // Print spaces for unprintable or newline characters. + for (int i = 0; i < LOG_BUFFER_LEN; i++) { + printf("%c", printable_char(current[i])); + } + printf(" "); + // Print the next N characters in the buffer. + printf("| "); + // Otherwise, print the characters starting from the buffer position. + // Print spaces for unprintable or newline characters. + for (int i = 0; i < LOG_SMALL_BUFFER_LEN; i++) { + printf("%c", printable_char(next[i])); + } + printf(" "); + } + if (current_index) { + printf("| %*u ", LOG_INDEX_LEN, *current_index); + } + else { + printf("| %-*s ", LOG_INDEX_LEN, ""); + } + // printf("| %*u ", LOG_INDEX_LEN, structurals.next_tape_index()); + printf("| %-s ", detail); + printf("|\n"); + } + } + + } // namespace logger + } // unnamed namespace + } // namespace ppc64 +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE2_LOGGER_H +/* end file generic/stage2/logger.h for ppc64 */ + +// All other declarations +/* including generic/stage2/json_iterator.h for ppc64: #include */ +/* begin file generic/stage2/json_iterator.h for ppc64 */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE2_JSON_ITERATOR_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_JSON_ITERATOR_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace ppc64 { + namespace { + namespace stage2 { + + class json_iterator { + public: + const uint8_t* const buf; + uint32_t* next_structural; + dom_parser_implementation& dom_parser; + uint32_t depth{ 0 }; + + /** + * Walk the JSON document. + * + * The visitor receives callbacks when values are encountered. All callbacks pass the iterator as + * the first parameter; some callbacks have other parameters as well: + * + * - visit_document_start() - at the beginning. + * - visit_document_end() - at the end (if things were successful). + * + * - visit_array_start() - at the start `[` of a non-empty array. + * - visit_array_end() - at the end `]` of a non-empty array. + * - visit_empty_array() - when an empty array is encountered. + * + * - visit_object_end() - at the start `]` of a non-empty object. + * - visit_object_start() - at the end `]` of a non-empty object. + * - visit_empty_object() - when an empty object is encountered. + * - visit_key(const uint8_t *key) - when a key in an object field is encountered. key is + * guaranteed to point at the first quote of the string (`"key"`). + * - visit_primitive(const uint8_t *value) - when a value is a string, number, boolean or null. + * - visit_root_primitive(iter, uint8_t *value) - when the top-level value is a string, number, boolean or null. + * + * - increment_count(iter) - each time a value is found in an array or object. + */ + template + simdjson_warn_unused simdjson_inline error_code walk_document(V& visitor) noexcept; + + /** + * Create an iterator capable of walking a JSON document. + * + * The document must have already passed through stage 1. + */ + simdjson_inline json_iterator(dom_parser_implementation& _dom_parser, size_t start_structural_index); + + /** + * Look at the next token. + * + * Tokens can be strings, numbers, booleans, null, or operators (`[{]},:`)). + * + * They may include invalid JSON as well (such as `1.2.3` or `ture`). + */ + simdjson_inline const uint8_t* peek() const noexcept; + /** + * Advance to the next token. + * + * Tokens can be strings, numbers, booleans, null, or operators (`[{]},:`)). + * + * They may include invalid JSON as well (such as `1.2.3` or `ture`). + */ + simdjson_inline const uint8_t* advance() noexcept; + /** + * Get the remaining length of the document, from the start of the current token. + */ + simdjson_inline size_t remaining_len() const noexcept; + /** + * Check if we are at the end of the document. + * + * If this is true, there are no more tokens. + */ + simdjson_inline bool at_eof() const noexcept; + /** + * Check if we are at the beginning of the document. + */ + simdjson_inline bool at_beginning() const noexcept; + simdjson_inline uint8_t last_structural() const noexcept; + + /** + * Log that a value has been found. + * + * Set LOG_ENABLED=true in logger.h to see logging. + */ + simdjson_inline void log_value(const char* type) const noexcept; + /** + * Log the start of a multipart value. + * + * Set LOG_ENABLED=true in logger.h to see logging. + */ + simdjson_inline void log_start_value(const char* type) const noexcept; + /** + * Log the end of a multipart value. + * + * Set LOG_ENABLED=true in logger.h to see logging. + */ + simdjson_inline void log_end_value(const char* type) const noexcept; + /** + * Log an error. + * + * Set LOG_ENABLED=true in logger.h to see logging. + */ + simdjson_inline void log_error(const char* error) const noexcept; + + template + simdjson_warn_unused simdjson_inline error_code visit_root_primitive(V& visitor, const uint8_t* value) noexcept; + template + simdjson_warn_unused simdjson_inline error_code visit_primitive(V& visitor, const uint8_t* value) noexcept; + }; + + template + simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V& visitor) noexcept { + logger::log_start(); + + // + // Start the document + // + if (at_eof()) { return EMPTY; } + log_start_value("document"); + SIMDJSON_TRY(visitor.visit_document_start(*this)); + + // + // Read first value + // + { + auto value = advance(); + + // Make sure the outer object or array is closed before continuing; otherwise, there are ways we + // could get into memory corruption. See https://github.com/simdjson/simdjson/issues/906 + if (!STREAMING) { + switch (*value) { + case '{': if (last_structural() != '}') { log_value("starting brace unmatched"); return TAPE_ERROR; }; break; + case '[': if (last_structural() != ']') { log_value("starting bracket unmatched"); return TAPE_ERROR; }; break; + } + } + + switch (*value) { + case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY(visitor.visit_empty_object(*this)); break; } goto object_begin; + case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY(visitor.visit_empty_array(*this)); break; } goto array_begin; + default: SIMDJSON_TRY(visitor.visit_root_primitive(*this, value)); break; + } + } + goto document_end; + + // + // Object parser states + // + object_begin: + log_start_value("object"); + depth++; + if (depth >= dom_parser.max_depth()) { log_error("Exceeded max depth!"); return DEPTH_ERROR; } + dom_parser.is_array[depth] = false; + SIMDJSON_TRY(visitor.visit_object_start(*this)); + + { + auto key = advance(); + if (*key != '"') { log_error("Object does not start with a key"); return TAPE_ERROR; } + SIMDJSON_TRY(visitor.increment_count(*this)); + SIMDJSON_TRY(visitor.visit_key(*this, key)); + } + + object_field: + if (simdjson_unlikely(*advance() != ':')) { log_error("Missing colon after key in object"); return TAPE_ERROR; } + { + auto value = advance(); + switch (*value) { + case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY(visitor.visit_empty_object(*this)); break; } goto object_begin; + case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY(visitor.visit_empty_array(*this)); break; } goto array_begin; + default: SIMDJSON_TRY(visitor.visit_primitive(*this, value)); break; + } + } + + object_continue: + switch (*advance()) { + case ',': + SIMDJSON_TRY(visitor.increment_count(*this)); + { + auto key = advance(); + if (simdjson_unlikely(*key != '"')) { log_error("Key string missing at beginning of field in object"); return TAPE_ERROR; } + SIMDJSON_TRY(visitor.visit_key(*this, key)); + } + goto object_field; + case '}': log_end_value("object"); SIMDJSON_TRY(visitor.visit_object_end(*this)); goto scope_end; + default: log_error("No comma between object fields"); return TAPE_ERROR; + } + + scope_end: + depth--; + if (depth == 0) { goto document_end; } + if (dom_parser.is_array[depth]) { goto array_continue; } + goto object_continue; + + // + // Array parser states + // + array_begin: + log_start_value("array"); + depth++; + if (depth >= dom_parser.max_depth()) { log_error("Exceeded max depth!"); return DEPTH_ERROR; } + dom_parser.is_array[depth] = true; + SIMDJSON_TRY(visitor.visit_array_start(*this)); + SIMDJSON_TRY(visitor.increment_count(*this)); + + array_value: + { + auto value = advance(); + switch (*value) { + case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY(visitor.visit_empty_object(*this)); break; } goto object_begin; + case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY(visitor.visit_empty_array(*this)); break; } goto array_begin; + default: SIMDJSON_TRY(visitor.visit_primitive(*this, value)); break; + } + } + + array_continue: + switch (*advance()) { + case ',': SIMDJSON_TRY(visitor.increment_count(*this)); goto array_value; + case ']': log_end_value("array"); SIMDJSON_TRY(visitor.visit_array_end(*this)); goto scope_end; + default: log_error("Missing comma between array values"); return TAPE_ERROR; + } + + document_end: + log_end_value("document"); + SIMDJSON_TRY(visitor.visit_document_end(*this)); + + dom_parser.next_structural_index = uint32_t(next_structural - &dom_parser.structural_indexes[0]); + + // If we didn't make it to the end, it's an error + if (!STREAMING && dom_parser.next_structural_index != dom_parser.n_structural_indexes) { + log_error("More than one JSON value at the root of the document, or extra characters at the end of the JSON!"); + return TAPE_ERROR; + } + + return SUCCESS; + + } // walk_document() + + simdjson_inline json_iterator::json_iterator(dom_parser_implementation& _dom_parser, size_t start_structural_index) + : buf{ _dom_parser.buf }, + next_structural{ &_dom_parser.structural_indexes[start_structural_index] }, + dom_parser{ _dom_parser } { + } + + simdjson_inline const uint8_t* json_iterator::peek() const noexcept { + return &buf[*(next_structural)]; + } + simdjson_inline const uint8_t* json_iterator::advance() noexcept { + return &buf[*(next_structural++)]; + } + simdjson_inline size_t json_iterator::remaining_len() const noexcept { + return dom_parser.len - *(next_structural - 1); + } + + simdjson_inline bool json_iterator::at_eof() const noexcept { + return next_structural == &dom_parser.structural_indexes[dom_parser.n_structural_indexes]; + } + simdjson_inline bool json_iterator::at_beginning() const noexcept { + return next_structural == dom_parser.structural_indexes.get(); + } + simdjson_inline uint8_t json_iterator::last_structural() const noexcept { + return buf[dom_parser.structural_indexes[dom_parser.n_structural_indexes - 1]]; + } + + simdjson_inline void json_iterator::log_value(const char* type) const noexcept { + logger::log_line(*this, "", type, ""); + } + + simdjson_inline void json_iterator::log_start_value(const char* type) const noexcept { + logger::log_line(*this, "+", type, ""); + if (logger::LOG_ENABLED) { logger::log_depth++; } + } + + simdjson_inline void json_iterator::log_end_value(const char* type) const noexcept { + if (logger::LOG_ENABLED) { logger::log_depth--; } + logger::log_line(*this, "-", type, ""); + } + + simdjson_inline void json_iterator::log_error(const char* error) const noexcept { + logger::log_line(*this, "", "ERROR", error); + } + + template + simdjson_warn_unused simdjson_inline error_code json_iterator::visit_root_primitive(V& visitor, const uint8_t* value) noexcept { + switch (*value) { + case '"': return visitor.visit_root_string(*this, value); + case 't': return visitor.visit_root_true_atom(*this, value); + case 'f': return visitor.visit_root_false_atom(*this, value); + case 'n': return visitor.visit_root_null_atom(*this, value); + case '-': + case '0': case '1': case '2': case '3': case '4': + case '5': case '6': case '7': case '8': case '9': + return visitor.visit_root_number(*this, value); + default: + log_error("Document starts with a non-value character"); + return TAPE_ERROR; + } + } + template + simdjson_warn_unused simdjson_inline error_code json_iterator::visit_primitive(V& visitor, const uint8_t* value) noexcept { + // Use the fact that most scalars are going to be either strings or numbers. + if (*value == '"') { + return visitor.visit_string(*this, value); + } + else if (((*value - '0') < 10) || (*value == '-')) { + return visitor.visit_number(*this, value); + } + // true, false, null are uncommon. + switch (*value) { + case 't': return visitor.visit_true_atom(*this, value); + case 'f': return visitor.visit_false_atom(*this, value); + case 'n': return visitor.visit_null_atom(*this, value); + default: + log_error("Non-value found when value was expected!"); + return TAPE_ERROR; + } + } + + } // namespace stage2 + } // unnamed namespace + } // namespace ppc64 +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE2_JSON_ITERATOR_H +/* end file generic/stage2/json_iterator.h for ppc64 */ +/* including generic/stage2/stringparsing.h for ppc64: #include */ +/* begin file generic/stage2/stringparsing.h for ppc64 */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +// This file contains the common code every implementation uses +// It is intended to be included multiple times and compiled multiple times + +namespace simdjson { + namespace ppc64 { + namespace { + /// @private + namespace stringparsing { + + // begin copypasta + // These chars yield themselves: " \ / + // b -> backspace, f -> formfeed, n -> newline, r -> cr, t -> horizontal tab + // u not handled in this table as it's complex + static const uint8_t escape_map[256] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x0. + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0x22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x2f, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x4. + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x5c, 0, 0, 0, // 0x5. + 0, 0, 0x08, 0, 0, 0, 0x0c, 0, 0, 0, 0, 0, 0, 0, 0x0a, 0, // 0x6. + 0, 0, 0x0d, 0, 0x09, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x7. + + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + }; + + // handle a unicode codepoint + // write appropriate values into dest + // src will advance 6 bytes or 12 bytes + // dest will advance a variable amount (return via pointer) + // return true if the unicode codepoint was valid + // We work in little-endian then swap at write time + simdjson_warn_unused + simdjson_inline bool handle_unicode_codepoint(const uint8_t** src_ptr, + uint8_t** dst_ptr, bool allow_replacement) { + // Use the default Unicode Character 'REPLACEMENT CHARACTER' (U+FFFD) + constexpr uint32_t substitution_code_point = 0xfffd; + // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the + // conversion is not valid; we defer the check for this to inside the + // multilingual plane check. + uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2); + *src_ptr += 6; + + // If we found a high surrogate, we must + // check for low surrogate for characters + // outside the Basic + // Multilingual Plane. + if (code_point >= 0xd800 && code_point < 0xdc00) { + const uint8_t* src_data = *src_ptr; + /* Compiler optimizations convert this to a single 16-bit load and compare on most platforms */ + if (((src_data[0] << 8) | src_data[1]) != ((static_cast ('\\') << 8) | static_cast ('u'))) { + if (!allow_replacement) { return false; } + code_point = substitution_code_point; + } + else { + uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(src_data + 2); + + // We have already checked that the high surrogate is valid and + // (code_point - 0xd800) < 1024. + // + // Check that code_point_2 is in the range 0xdc00..0xdfff + // and that code_point_2 was parsed from valid hex. + uint32_t low_bit = code_point_2 - 0xdc00; + if (low_bit >> 10) { + if (!allow_replacement) { return false; } + code_point = substitution_code_point; + } + else { + code_point = (((code_point - 0xd800) << 10) | low_bit) + 0x10000; + *src_ptr += 6; + } + + } + } + else if (code_point >= 0xdc00 && code_point <= 0xdfff) { + // If we encounter a low surrogate (not preceded by a high surrogate) + // then we have an error. + if (!allow_replacement) { return false; } + code_point = substitution_code_point; + } + size_t offset = jsoncharutils::codepoint_to_utf8(code_point, *dst_ptr); + *dst_ptr += offset; + return offset > 0; + } + + + // handle a unicode codepoint using the wobbly convention + // https://simonsapin.github.io/wtf-8/ + // write appropriate values into dest + // src will advance 6 bytes or 12 bytes + // dest will advance a variable amount (return via pointer) + // return true if the unicode codepoint was valid + // We work in little-endian then swap at write time + simdjson_warn_unused + simdjson_inline bool handle_unicode_codepoint_wobbly(const uint8_t** src_ptr, + uint8_t** dst_ptr) { + // It is not ideal that this function is nearly identical to handle_unicode_codepoint. + // + // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the + // conversion is not valid; we defer the check for this to inside the + // multilingual plane check. + uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2); + *src_ptr += 6; + // If we found a high surrogate, we must + // check for low surrogate for characters + // outside the Basic + // Multilingual Plane. + if (code_point >= 0xd800 && code_point < 0xdc00) { + const uint8_t* src_data = *src_ptr; + /* Compiler optimizations convert this to a single 16-bit load and compare on most platforms */ + if (((src_data[0] << 8) | src_data[1]) == ((static_cast ('\\') << 8) | static_cast ('u'))) { + uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(src_data + 2); + uint32_t low_bit = code_point_2 - 0xdc00; + if ((low_bit >> 10) == 0) { + code_point = + (((code_point - 0xd800) << 10) | low_bit) + 0x10000; + *src_ptr += 6; + } + } + } + + size_t offset = jsoncharutils::codepoint_to_utf8(code_point, *dst_ptr); + *dst_ptr += offset; + return offset > 0; + } + + + /** + * Unescape a valid UTF-8 string from src to dst, stopping at a final unescaped quote. There + * must be an unescaped quote terminating the string. It returns the final output + * position as pointer. In case of error (e.g., the string has bad escaped codes), + * then null_ptr is returned. It is assumed that the output buffer is large + * enough. E.g., if src points at 'joe"', then dst needs to have four free bytes + + * SIMDJSON_PADDING bytes. + */ + simdjson_warn_unused simdjson_inline uint8_t* parse_string(const uint8_t* src, uint8_t* dst, bool allow_replacement) { + while (1) { + // Copy the next n bytes, and find the backslash and quote in them. + auto bs_quote = backslash_and_quote::copy_and_find(src, dst); + // If the next thing is the end quote, copy and return + if (bs_quote.has_quote_first()) { + // we encountered quotes first. Move dst to point to quotes and exit + return dst + bs_quote.quote_index(); + } + if (bs_quote.has_backslash()) { + /* find out where the backspace is */ + auto bs_dist = bs_quote.backslash_index(); + uint8_t escape_char = src[bs_dist + 1]; + /* we encountered backslash first. Handle backslash */ + if (escape_char == 'u') { + /* move src/dst up to the start; they will be further adjusted + within the unicode codepoint handling code. */ + src += bs_dist; + dst += bs_dist; + if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { + return nullptr; + } + } + else { + /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and + * write bs_dist+1 characters to output + * note this may reach beyond the part of the buffer we've actually + * seen. I think this is ok */ + uint8_t escape_result = escape_map[escape_char]; + if (escape_result == 0u) { + return nullptr; /* bogus escape value is an error */ + } + dst[bs_dist] = escape_result; + src += bs_dist + 2; + dst += bs_dist + 1; + } + } + else { + /* they are the same. Since they can't co-occur, it means we + * encountered neither. */ + src += backslash_and_quote::BYTES_PROCESSED; + dst += backslash_and_quote::BYTES_PROCESSED; + } + } + } + + simdjson_warn_unused simdjson_inline uint8_t* parse_wobbly_string(const uint8_t* src, uint8_t* dst) { + // It is not ideal that this function is nearly identical to parse_string. + while (1) { + // Copy the next n bytes, and find the backslash and quote in them. + auto bs_quote = backslash_and_quote::copy_and_find(src, dst); + // If the next thing is the end quote, copy and return + if (bs_quote.has_quote_first()) { + // we encountered quotes first. Move dst to point to quotes and exit + return dst + bs_quote.quote_index(); + } + if (bs_quote.has_backslash()) { + /* find out where the backspace is */ + auto bs_dist = bs_quote.backslash_index(); + uint8_t escape_char = src[bs_dist + 1]; + /* we encountered backslash first. Handle backslash */ + if (escape_char == 'u') { + /* move src/dst up to the start; they will be further adjusted + within the unicode codepoint handling code. */ + src += bs_dist; + dst += bs_dist; + if (!handle_unicode_codepoint_wobbly(&src, &dst)) { + return nullptr; + } + } + else { + /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and + * write bs_dist+1 characters to output + * note this may reach beyond the part of the buffer we've actually + * seen. I think this is ok */ + uint8_t escape_result = escape_map[escape_char]; + if (escape_result == 0u) { + return nullptr; /* bogus escape value is an error */ + } + dst[bs_dist] = escape_result; + src += bs_dist + 2; + dst += bs_dist + 1; + } + } + else { + /* they are the same. Since they can't co-occur, it means we + * encountered neither. */ + src += backslash_and_quote::BYTES_PROCESSED; + dst += backslash_and_quote::BYTES_PROCESSED; + } + } + } + + } // namespace stringparsing + } // unnamed namespace + } // namespace ppc64 +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H +/* end file generic/stage2/stringparsing.h for ppc64 */ +/* including generic/stage2/structural_iterator.h for ppc64: #include */ +/* begin file generic/stage2/structural_iterator.h for ppc64 */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE2_STRUCTURAL_ITERATOR_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_STRUCTURAL_ITERATOR_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace ppc64 { + namespace { + namespace stage2 { + + class structural_iterator { + public: + const uint8_t* const buf; + uint32_t* next_structural; + dom_parser_implementation& dom_parser; + + // Start a structural + simdjson_inline structural_iterator(dom_parser_implementation& _dom_parser, size_t start_structural_index) + : buf{ _dom_parser.buf }, + next_structural{ &_dom_parser.structural_indexes[start_structural_index] }, + dom_parser{ _dom_parser } { + } + // Get the buffer position of the current structural character + simdjson_inline const uint8_t* current() { + return &buf[*(next_structural - 1)]; + } + // Get the current structural character + simdjson_inline char current_char() { + return buf[*(next_structural - 1)]; + } + // Get the next structural character without advancing + simdjson_inline char peek_next_char() { + return buf[*next_structural]; + } + simdjson_inline const uint8_t* peek() { + return &buf[*next_structural]; + } + simdjson_inline const uint8_t* advance() { + return &buf[*(next_structural++)]; + } + simdjson_inline char advance_char() { + return buf[*(next_structural++)]; + } + simdjson_inline size_t remaining_len() { + return dom_parser.len - *(next_structural - 1); + } + + simdjson_inline bool at_end() { + return next_structural == &dom_parser.structural_indexes[dom_parser.n_structural_indexes]; + } + simdjson_inline bool at_beginning() { + return next_structural == dom_parser.structural_indexes.get(); + } + }; + + } // namespace stage2 + } // unnamed namespace + } // namespace ppc64 +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE2_STRUCTURAL_ITERATOR_H +/* end file generic/stage2/structural_iterator.h for ppc64 */ +/* including generic/stage2/tape_builder.h for ppc64: #include */ +/* begin file generic/stage2/tape_builder.h for ppc64 */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE2_TAPE_BUILDER_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_TAPE_BUILDER_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + + +namespace simdjson { + namespace ppc64 { + namespace { + namespace stage2 { + + struct tape_builder { + template + simdjson_warn_unused static simdjson_inline error_code parse_document( + dom_parser_implementation& dom_parser, + dom::document& doc) noexcept; + + /** Called when a non-empty document starts. */ + simdjson_warn_unused simdjson_inline error_code visit_document_start(json_iterator& iter) noexcept; + /** Called when a non-empty document ends without error. */ + simdjson_warn_unused simdjson_inline error_code visit_document_end(json_iterator& iter) noexcept; + + /** Called when a non-empty array starts. */ + simdjson_warn_unused simdjson_inline error_code visit_array_start(json_iterator& iter) noexcept; + /** Called when a non-empty array ends. */ + simdjson_warn_unused simdjson_inline error_code visit_array_end(json_iterator& iter) noexcept; + /** Called when an empty array is found. */ + simdjson_warn_unused simdjson_inline error_code visit_empty_array(json_iterator& iter) noexcept; + + /** Called when a non-empty object starts. */ + simdjson_warn_unused simdjson_inline error_code visit_object_start(json_iterator& iter) noexcept; + /** + * Called when a key in a field is encountered. + * + * primitive, visit_object_start, visit_empty_object, visit_array_start, or visit_empty_array + * will be called after this with the field value. + */ + simdjson_warn_unused simdjson_inline error_code visit_key(json_iterator& iter, const uint8_t* key) noexcept; + /** Called when a non-empty object ends. */ + simdjson_warn_unused simdjson_inline error_code visit_object_end(json_iterator& iter) noexcept; + /** Called when an empty object is found. */ + simdjson_warn_unused simdjson_inline error_code visit_empty_object(json_iterator& iter) noexcept; + + /** + * Called when a string, number, boolean or null is found. + */ + simdjson_warn_unused simdjson_inline error_code visit_primitive(json_iterator& iter, const uint8_t* value) noexcept; + /** + * Called when a string, number, boolean or null is found at the top level of a document (i.e. + * when there is no array or object and the entire document is a single string, number, boolean or + * null. + * + * This is separate from primitive() because simdjson's normal primitive parsing routines assume + * there is at least one more token after the value, which is only true in an array or object. + */ + simdjson_warn_unused simdjson_inline error_code visit_root_primitive(json_iterator& iter, const uint8_t* value) noexcept; + + simdjson_warn_unused simdjson_inline error_code visit_string(json_iterator& iter, const uint8_t* value, bool key = false) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_number(json_iterator& iter, const uint8_t* value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_true_atom(json_iterator& iter, const uint8_t* value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_false_atom(json_iterator& iter, const uint8_t* value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_null_atom(json_iterator& iter, const uint8_t* value) noexcept; + + simdjson_warn_unused simdjson_inline error_code visit_root_string(json_iterator& iter, const uint8_t* value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_root_number(json_iterator& iter, const uint8_t* value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_root_true_atom(json_iterator& iter, const uint8_t* value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_root_false_atom(json_iterator& iter, const uint8_t* value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_root_null_atom(json_iterator& iter, const uint8_t* value) noexcept; + + /** Called each time a new field or element in an array or object is found. */ + simdjson_warn_unused simdjson_inline error_code increment_count(json_iterator& iter) noexcept; + + /** Next location to write to tape */ + tape_writer tape; + private: + /** Next write location in the string buf for stage 2 parsing */ + uint8_t* current_string_buf_loc; + + simdjson_inline tape_builder(dom::document& doc) noexcept; + + simdjson_inline uint32_t next_tape_index(json_iterator& iter) const noexcept; + simdjson_inline void start_container(json_iterator& iter) noexcept; + simdjson_warn_unused simdjson_inline error_code end_container(json_iterator& iter, internal::tape_type start, internal::tape_type end) noexcept; + simdjson_warn_unused simdjson_inline error_code empty_container(json_iterator& iter, internal::tape_type start, internal::tape_type end) noexcept; + simdjson_inline uint8_t* on_start_string(json_iterator& iter) noexcept; + simdjson_inline void on_end_string(uint8_t* dst) noexcept; + }; // struct tape_builder + + template + simdjson_warn_unused simdjson_inline error_code tape_builder::parse_document( + dom_parser_implementation& dom_parser, + dom::document& doc) noexcept { + dom_parser.doc = &doc; + json_iterator iter(dom_parser, STREAMING ? dom_parser.next_structural_index : 0); + tape_builder builder(doc); + return iter.walk_document(builder); + } + + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_primitive(json_iterator& iter, const uint8_t* value) noexcept { + return iter.visit_root_primitive(*this, value); + } + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_primitive(json_iterator& iter, const uint8_t* value) noexcept { + return iter.visit_primitive(*this, value); + } + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_empty_object(json_iterator& iter) noexcept { + return empty_container(iter, internal::tape_type::START_OBJECT, internal::tape_type::END_OBJECT); + } + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_empty_array(json_iterator& iter) noexcept { + return empty_container(iter, internal::tape_type::START_ARRAY, internal::tape_type::END_ARRAY); + } + + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_document_start(json_iterator& iter) noexcept { + start_container(iter); + return SUCCESS; + } + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_object_start(json_iterator& iter) noexcept { + start_container(iter); + return SUCCESS; + } + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_array_start(json_iterator& iter) noexcept { + start_container(iter); + return SUCCESS; + } + + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_object_end(json_iterator& iter) noexcept { + return end_container(iter, internal::tape_type::START_OBJECT, internal::tape_type::END_OBJECT); + } + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_array_end(json_iterator& iter) noexcept { + return end_container(iter, internal::tape_type::START_ARRAY, internal::tape_type::END_ARRAY); + } + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_document_end(json_iterator& iter) noexcept { + constexpr uint32_t start_tape_index = 0; + tape.append(start_tape_index, internal::tape_type::ROOT); + tape_writer::write(iter.dom_parser.doc->tape[start_tape_index], next_tape_index(iter), internal::tape_type::ROOT); + return SUCCESS; + } + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_key(json_iterator& iter, const uint8_t* key) noexcept { + return visit_string(iter, key, true); + } + + simdjson_warn_unused simdjson_inline error_code tape_builder::increment_count(json_iterator& iter) noexcept { + iter.dom_parser.open_containers[iter.depth].count++; // we have a key value pair in the object at parser.dom_parser.depth - 1 + return SUCCESS; + } + + simdjson_inline tape_builder::tape_builder(dom::document& doc) noexcept : tape{ doc.tape.get() }, current_string_buf_loc{ doc.string_buf.get() } {} + + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_string(json_iterator& iter, const uint8_t* value, bool key) noexcept { + iter.log_value(key ? "key" : "string"); + uint8_t* dst = on_start_string(iter); + dst = stringparsing::parse_string(value + 1, dst, false); // We do not allow replacement when the escape characters are invalid. + if (dst == nullptr) { + iter.log_error("Invalid escape in string"); + return STRING_ERROR; + } + on_end_string(dst); + return SUCCESS; + } + + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_string(json_iterator& iter, const uint8_t* value) noexcept { + return visit_string(iter, value); + } + + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_number(json_iterator& iter, const uint8_t* value) noexcept { + iter.log_value("number"); + return numberparsing::parse_number(value, tape); + } + + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_number(json_iterator& iter, const uint8_t* value) noexcept { + // + // We need to make a copy to make sure that the string is space terminated. + // This is not about padding the input, which should already padded up + // to len + SIMDJSON_PADDING. However, we have no control at this stage + // on how the padding was done. What if the input string was padded with nulls? + // It is quite common for an input string to have an extra null character (C string). + // We do not want to allow 9\0 (where \0 is the null character) inside a JSON + // document, but the string "9\0" by itself is fine. So we make a copy and + // pad the input with spaces when we know that there is just one input element. + // This copy is relatively expensive, but it will almost never be called in + // practice unless you are in the strange scenario where you have many JSON + // documents made of single atoms. + // + std::unique_ptrcopy(new (std::nothrow) uint8_t[iter.remaining_len() + SIMDJSON_PADDING]); + if (copy.get() == nullptr) { return MEMALLOC; } + std::memcpy(copy.get(), value, iter.remaining_len()); + std::memset(copy.get() + iter.remaining_len(), ' ', SIMDJSON_PADDING); + error_code error = visit_number(iter, copy.get()); + return error; + } + + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_true_atom(json_iterator& iter, const uint8_t* value) noexcept { + iter.log_value("true"); + if (!atomparsing::is_valid_true_atom(value)) { return T_ATOM_ERROR; } + tape.append(0, internal::tape_type::TRUE_VALUE); + return SUCCESS; + } + + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_true_atom(json_iterator& iter, const uint8_t* value) noexcept { + iter.log_value("true"); + if (!atomparsing::is_valid_true_atom(value, iter.remaining_len())) { return T_ATOM_ERROR; } + tape.append(0, internal::tape_type::TRUE_VALUE); + return SUCCESS; + } + + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_false_atom(json_iterator& iter, const uint8_t* value) noexcept { + iter.log_value("false"); + if (!atomparsing::is_valid_false_atom(value)) { return F_ATOM_ERROR; } + tape.append(0, internal::tape_type::FALSE_VALUE); + return SUCCESS; + } + + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_false_atom(json_iterator& iter, const uint8_t* value) noexcept { + iter.log_value("false"); + if (!atomparsing::is_valid_false_atom(value, iter.remaining_len())) { return F_ATOM_ERROR; } + tape.append(0, internal::tape_type::FALSE_VALUE); + return SUCCESS; + } + + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_null_atom(json_iterator& iter, const uint8_t* value) noexcept { + iter.log_value("null"); + if (!atomparsing::is_valid_null_atom(value)) { return N_ATOM_ERROR; } + tape.append(0, internal::tape_type::NULL_VALUE); + return SUCCESS; + } + + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_null_atom(json_iterator& iter, const uint8_t* value) noexcept { + iter.log_value("null"); + if (!atomparsing::is_valid_null_atom(value, iter.remaining_len())) { return N_ATOM_ERROR; } + tape.append(0, internal::tape_type::NULL_VALUE); + return SUCCESS; + } + + // private: + + simdjson_inline uint32_t tape_builder::next_tape_index(json_iterator& iter) const noexcept { + return uint32_t(tape.next_tape_loc - iter.dom_parser.doc->tape.get()); + } + + simdjson_warn_unused simdjson_inline error_code tape_builder::empty_container(json_iterator& iter, internal::tape_type start, internal::tape_type end) noexcept { + auto start_index = next_tape_index(iter); + tape.append(start_index + 2, start); + tape.append(start_index, end); + return SUCCESS; + } + + simdjson_inline void tape_builder::start_container(json_iterator& iter) noexcept { + iter.dom_parser.open_containers[iter.depth].tape_index = next_tape_index(iter); + iter.dom_parser.open_containers[iter.depth].count = 0; + tape.skip(); // We don't actually *write* the start element until the end. + } + + simdjson_warn_unused simdjson_inline error_code tape_builder::end_container(json_iterator& iter, internal::tape_type start, internal::tape_type end) noexcept { + // Write the ending tape element, pointing at the start location + const uint32_t start_tape_index = iter.dom_parser.open_containers[iter.depth].tape_index; + tape.append(start_tape_index, end); + // Write the start tape element, pointing at the end location (and including count) + // count can overflow if it exceeds 24 bits... so we saturate + // the convention being that a cnt of 0xffffff or more is undetermined in value (>= 0xffffff). + const uint32_t count = iter.dom_parser.open_containers[iter.depth].count; + const uint32_t cntsat = count > 0xFFFFFF ? 0xFFFFFF : count; + tape_writer::write(iter.dom_parser.doc->tape[start_tape_index], next_tape_index(iter) | (uint64_t(cntsat) << 32), start); + return SUCCESS; + } + + simdjson_inline uint8_t* tape_builder::on_start_string(json_iterator& iter) noexcept { + // we advance the point, accounting for the fact that we have a NULL termination + tape.append(current_string_buf_loc - iter.dom_parser.doc->string_buf.get(), internal::tape_type::STRING); + return current_string_buf_loc + sizeof(uint32_t); + } + + simdjson_inline void tape_builder::on_end_string(uint8_t* dst) noexcept { + uint32_t str_length = uint32_t(dst - (current_string_buf_loc + sizeof(uint32_t))); + // TODO check for overflow in case someone has a crazy string (>=4GB?) + // But only add the overflow check when the document itself exceeds 4GB + // Currently unneeded because we refuse to parse docs larger or equal to 4GB. + memcpy(current_string_buf_loc, &str_length, sizeof(uint32_t)); + // NULL termination is still handy if you expect all your strings to + // be NULL terminated? It comes at a small cost + *dst = 0; + current_string_buf_loc = dst + 1; + } + + } // namespace stage2 + } // unnamed namespace + } // namespace ppc64 +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE2_TAPE_BUILDER_H +/* end file generic/stage2/tape_builder.h for ppc64 */ +/* end file generic/stage2/amalgamated.h for ppc64 */ + +// +// Stage 1 +// +namespace simdjson { + namespace ppc64 { + + simdjson_warn_unused error_code implementation::create_dom_parser_implementation( + size_t capacity, + size_t max_depth, + std::unique_ptr& dst + ) const noexcept { + dst.reset(new (std::nothrow) dom_parser_implementation()); + if (!dst) { return MEMALLOC; } + if (auto err = dst->set_capacity(capacity)) + return err; + if (auto err = dst->set_max_depth(max_depth)) + return err; + return SUCCESS; + } + + namespace { + + using namespace simd; + + simdjson_inline json_character_block json_character_block::classify(const simd::simd8x64& in) { + const simd8 table1(16, 0, 0, 0, 0, 0, 0, 0, 0, 8, 12, 1, 2, 9, 0, 0); + const simd8 table2(8, 0, 18, 4, 0, 1, 0, 1, 0, 0, 0, 3, 2, 1, 0, 0); + + simd8x64 v( + (in.chunks[0] & 0xf).lookup_16(table1) & (in.chunks[0].shr<4>()).lookup_16(table2), + (in.chunks[1] & 0xf).lookup_16(table1) & (in.chunks[1].shr<4>()).lookup_16(table2), + (in.chunks[2] & 0xf).lookup_16(table1) & (in.chunks[2].shr<4>()).lookup_16(table2), + (in.chunks[3] & 0xf).lookup_16(table1) & (in.chunks[3].shr<4>()).lookup_16(table2) + ); + + uint64_t op = simd8x64( + v.chunks[0].any_bits_set(0x7), + v.chunks[1].any_bits_set(0x7), + v.chunks[2].any_bits_set(0x7), + v.chunks[3].any_bits_set(0x7) + ).to_bitmask(); + + uint64_t whitespace = simd8x64( + v.chunks[0].any_bits_set(0x18), + v.chunks[1].any_bits_set(0x18), + v.chunks[2].any_bits_set(0x18), + v.chunks[3].any_bits_set(0x18) + ).to_bitmask(); + + return { whitespace, op }; + } + + simdjson_inline bool is_ascii(const simd8x64& input) { + // careful: 0x80 is not ascii. + return input.reduce_or().saturating_sub(0x7fu).bits_not_set_anywhere(); + } + + simdjson_unused simdjson_inline simd8 must_be_continuation(const simd8 prev1, const simd8 prev2, const simd8 prev3) { + simd8 is_second_byte = prev1.saturating_sub(0xc0u - 1); // Only 11______ will be > 0 + simd8 is_third_byte = prev2.saturating_sub(0xe0u - 1); // Only 111_____ will be > 0 + simd8 is_fourth_byte = prev3.saturating_sub(0xf0u - 1); // Only 1111____ will be > 0 + // Caller requires a bool (all 1's). All values resulting from the subtraction will be <= 64, so signed comparison is fine. + return simd8(is_second_byte | is_third_byte | is_fourth_byte) > int8_t(0); + } + + simdjson_inline simd8 must_be_2_3_continuation(const simd8 prev2, const simd8 prev3) { + simd8 is_third_byte = prev2.saturating_sub(0xe0u - 0x80); // Only 111_____ will be >= 0x80 + simd8 is_fourth_byte = prev3.saturating_sub(0xf0u - 0x80); // Only 1111____ will be >= 0x80 + return is_third_byte | is_fourth_byte; + } + + } // unnamed namespace + } // namespace ppc64 +} // namespace simdjson + +// +// Stage 2 +// + +// +// Implementation-specific overrides +// +namespace simdjson { + namespace ppc64 { + + simdjson_warn_unused error_code implementation::minify(const uint8_t* buf, size_t len, uint8_t* dst, size_t& dst_len) const noexcept { + return ppc64::stage1::json_minifier::minify<64>(buf, len, dst, dst_len); + } + + simdjson_warn_unused error_code dom_parser_implementation::stage1(const uint8_t* _buf, size_t _len, stage1_mode streaming) noexcept { + this->buf = _buf; + this->len = _len; + return ppc64::stage1::json_structural_indexer::index<64>(buf, len, *this, streaming); + } + + simdjson_warn_unused bool implementation::validate_utf8(const char* buf, size_t len) const noexcept { + return ppc64::stage1::generic_validate_utf8(buf, len); + } + + simdjson_warn_unused error_code dom_parser_implementation::stage2(dom::document& _doc) noexcept { + return stage2::tape_builder::parse_document(*this, _doc); + } + + simdjson_warn_unused error_code dom_parser_implementation::stage2_next(dom::document& _doc) noexcept { + return stage2::tape_builder::parse_document(*this, _doc); + } + + simdjson_warn_unused uint8_t* dom_parser_implementation::parse_string(const uint8_t* src, uint8_t* dst, bool replacement_char) const noexcept { + return ppc64::stringparsing::parse_string(src, dst, replacement_char); + } + + simdjson_warn_unused uint8_t* dom_parser_implementation::parse_wobbly_string(const uint8_t* src, uint8_t* dst) const noexcept { + return ppc64::stringparsing::parse_wobbly_string(src, dst); + } + + simdjson_warn_unused error_code dom_parser_implementation::parse(const uint8_t* _buf, size_t _len, dom::document& _doc) noexcept { + auto error = stage1(_buf, _len, stage1_mode::regular); + if (error) { return error; } + return stage2(_doc); + } + + } // namespace ppc64 +} // namespace simdjson + +/* including simdjson/ppc64/end.h: #include */ +/* begin file simdjson/ppc64/end.h */ +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/ppc64/base.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +#undef SIMDJSON_SKIP_BACKSLASH_SHORT_CIRCUIT +/* undefining SIMDJSON_IMPLEMENTATION from "ppc64" */ +#undef SIMDJSON_IMPLEMENTATION +/* end file simdjson/ppc64/end.h */ + +#endif // SIMDJSON_SRC_PPC64_CPP +/* end file ppc64.cpp */ +#endif +#if SIMDJSON_IMPLEMENTATION_WESTMERE +/* including westmere.cpp: #include */ +/* begin file westmere.cpp */ +#ifndef SIMDJSON_SRC_WESTMERE_CPP +#define SIMDJSON_SRC_WESTMERE_CPP + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +/* including simdjson/westmere.h: #include */ +/* begin file simdjson/westmere.h */ +#ifndef SIMDJSON_WESTMERE_H +#define SIMDJSON_WESTMERE_H + +/* including simdjson/westmere/begin.h: #include "simdjson/westmere/begin.h" */ +/* begin file simdjson/westmere/begin.h */ +/* defining SIMDJSON_IMPLEMENTATION to "westmere" */ +#define SIMDJSON_IMPLEMENTATION westmere +/* including simdjson/westmere/base.h: #include "simdjson/westmere/base.h" */ +/* begin file simdjson/westmere/base.h */ +#ifndef SIMDJSON_WESTMERE_BASE_H +#define SIMDJSON_WESTMERE_BASE_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/base.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +// The constructor may be executed on any host, so we take care not to use SIMDJSON_TARGET_WESTMERE +namespace simdjson { + /** + * Implementation for Westmere (Intel SSE4.2). + */ + namespace westmere { + + class implementation; + + namespace { + namespace simd { + + template struct simd8; + template struct simd8x64; + + } // namespace simd + } // unnamed namespace + + } // namespace westmere +} // namespace simdjson + +#endif // SIMDJSON_WESTMERE_BASE_H +/* end file simdjson/westmere/base.h */ +/* including simdjson/westmere/intrinsics.h: #include "simdjson/westmere/intrinsics.h" */ +/* begin file simdjson/westmere/intrinsics.h */ +#ifndef SIMDJSON_WESTMERE_INTRINSICS_H +#define SIMDJSON_WESTMERE_INTRINSICS_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/westmere/base.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +#if SIMDJSON_VISUAL_STUDIO +// under clang within visual studio, this will include +#include // visual studio or clang +#else +#include // elsewhere +#endif // SIMDJSON_VISUAL_STUDIO + + +#if SIMDJSON_CLANG_VISUAL_STUDIO +/** + * You are not supposed, normally, to include these + * headers directly. Instead you should either include intrin.h + * or x86intrin.h. However, when compiling with clang + * under Windows (i.e., when _MSC_VER is set), these headers + * only get included *if* the corresponding features are detected + * from macros: + */ +#include // for _mm_alignr_epi8 +#include // for _mm_clmulepi64_si128 +#endif + +static_assert(sizeof(__m128i) <= simdjson::SIMDJSON_PADDING, "insufficient padding for westmere"); + +#endif // SIMDJSON_WESTMERE_INTRINSICS_H +/* end file simdjson/westmere/intrinsics.h */ + +#if !SIMDJSON_CAN_ALWAYS_RUN_WESTMERE +SIMDJSON_TARGET_REGION("sse4.2,pclmul,popcnt") +#endif + +/* including simdjson/westmere/bitmanipulation.h: #include "simdjson/westmere/bitmanipulation.h" */ +/* begin file simdjson/westmere/bitmanipulation.h */ +#ifndef SIMDJSON_WESTMERE_BITMANIPULATION_H +#define SIMDJSON_WESTMERE_BITMANIPULATION_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/westmere/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/westmere/intrinsics.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace westmere { + namespace { + + // We sometimes call trailing_zero on inputs that are zero, + // but the algorithms do not end up using the returned value. + // Sadly, sanitizers are not smart enough to figure it out. + SIMDJSON_NO_SANITIZE_UNDEFINED + // This function can be used safely even if not all bytes have been + // initialized. + // See issue https://github.com/simdjson/simdjson/issues/1965 + SIMDJSON_NO_SANITIZE_MEMORY + simdjson_inline int trailing_zeroes(uint64_t input_num) { +#if SIMDJSON_REGULAR_VISUAL_STUDIO + unsigned long ret; + // Search the mask data from least significant bit (LSB) + // to the most significant bit (MSB) for a set bit (1). + _BitScanForward64(&ret, input_num); + return (int)ret; +#else // SIMDJSON_REGULAR_VISUAL_STUDIO + return __builtin_ctzll(input_num); +#endif // SIMDJSON_REGULAR_VISUAL_STUDIO + } + + /* result might be undefined when input_num is zero */ + simdjson_inline uint64_t clear_lowest_bit(uint64_t input_num) { + return input_num & (input_num - 1); + } + + /* result might be undefined when input_num is zero */ + simdjson_inline int leading_zeroes(uint64_t input_num) { +#if SIMDJSON_REGULAR_VISUAL_STUDIO + unsigned long leading_zero = 0; + // Search the mask data from most significant bit (MSB) + // to least significant bit (LSB) for a set bit (1). + if (_BitScanReverse64(&leading_zero, input_num)) + return (int)(63 - leading_zero); + else + return 64; +#else + return __builtin_clzll(input_num); +#endif// SIMDJSON_REGULAR_VISUAL_STUDIO + } + +#if SIMDJSON_REGULAR_VISUAL_STUDIO + simdjson_inline unsigned __int64 count_ones(uint64_t input_num) { + // note: we do not support legacy 32-bit Windows in this kernel + return __popcnt64(input_num);// Visual Studio wants two underscores + } +#else + simdjson_inline long long int count_ones(uint64_t input_num) { + return _popcnt64(input_num); + } +#endif + + simdjson_inline bool add_overflow(uint64_t value1, uint64_t value2, + uint64_t* result) { +#if SIMDJSON_REGULAR_VISUAL_STUDIO + return _addcarry_u64(0, value1, value2, + reinterpret_cast(result)); +#else + return __builtin_uaddll_overflow(value1, value2, + reinterpret_cast(result)); +#endif + } + + } // unnamed namespace + } // namespace westmere +} // namespace simdjson + +#endif // SIMDJSON_WESTMERE_BITMANIPULATION_H +/* end file simdjson/westmere/bitmanipulation.h */ +/* including simdjson/westmere/bitmask.h: #include "simdjson/westmere/bitmask.h" */ +/* begin file simdjson/westmere/bitmask.h */ +#ifndef SIMDJSON_WESTMERE_BITMASK_H +#define SIMDJSON_WESTMERE_BITMASK_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/westmere/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/westmere/intrinsics.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace westmere { + namespace { + + // + // Perform a "cumulative bitwise xor," flipping bits each time a 1 is encountered. + // + // For example, prefix_xor(00100100) == 00011100 + // + simdjson_inline uint64_t prefix_xor(const uint64_t bitmask) { + // There should be no such thing with a processing supporting avx2 + // but not clmul. + __m128i all_ones = _mm_set1_epi8('\xFF'); + __m128i result = _mm_clmulepi64_si128(_mm_set_epi64x(0ULL, bitmask), all_ones, 0); + return _mm_cvtsi128_si64(result); + } + + } // unnamed namespace + } // namespace westmere +} // namespace simdjson + +#endif // SIMDJSON_WESTMERE_BITMASK_H +/* end file simdjson/westmere/bitmask.h */ +/* including simdjson/westmere/numberparsing_defs.h: #include "simdjson/westmere/numberparsing_defs.h" */ +/* begin file simdjson/westmere/numberparsing_defs.h */ +#ifndef SIMDJSON_WESTMERE_NUMBERPARSING_DEFS_H +#define SIMDJSON_WESTMERE_NUMBERPARSING_DEFS_H + +/* including simdjson/westmere/base.h: #include "simdjson/westmere/base.h" */ +/* begin file simdjson/westmere/base.h */ +#ifndef SIMDJSON_WESTMERE_BASE_H +#define SIMDJSON_WESTMERE_BASE_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/base.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +// The constructor may be executed on any host, so we take care not to use SIMDJSON_TARGET_WESTMERE +namespace simdjson { + /** + * Implementation for Westmere (Intel SSE4.2). + */ + namespace westmere { + + class implementation; + + namespace { + namespace simd { + + template struct simd8; + template struct simd8x64; + + } // namespace simd + } // unnamed namespace + + } // namespace westmere +} // namespace simdjson + +#endif // SIMDJSON_WESTMERE_BASE_H +/* end file simdjson/westmere/base.h */ +/* including simdjson/westmere/intrinsics.h: #include "simdjson/westmere/intrinsics.h" */ +/* begin file simdjson/westmere/intrinsics.h */ +#ifndef SIMDJSON_WESTMERE_INTRINSICS_H +#define SIMDJSON_WESTMERE_INTRINSICS_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/westmere/base.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +#if SIMDJSON_VISUAL_STUDIO +// under clang within visual studio, this will include +#include // visual studio or clang +#else +#include // elsewhere +#endif // SIMDJSON_VISUAL_STUDIO + + +#if SIMDJSON_CLANG_VISUAL_STUDIO +/** + * You are not supposed, normally, to include these + * headers directly. Instead you should either include intrin.h + * or x86intrin.h. However, when compiling with clang + * under Windows (i.e., when _MSC_VER is set), these headers + * only get included *if* the corresponding features are detected + * from macros: + */ +#include // for _mm_alignr_epi8 +#include // for _mm_clmulepi64_si128 +#endif + +static_assert(sizeof(__m128i) <= simdjson::SIMDJSON_PADDING, "insufficient padding for westmere"); + +#endif // SIMDJSON_WESTMERE_INTRINSICS_H +/* end file simdjson/westmere/intrinsics.h */ + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace westmere { + namespace numberparsing { + + /** @private */ + static simdjson_inline uint32_t parse_eight_digits_unrolled(const uint8_t* chars) { + // this actually computes *16* values so we are being wasteful. + const __m128i ascii0 = _mm_set1_epi8('0'); + const __m128i mul_1_10 = + _mm_setr_epi8(10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1); + const __m128i mul_1_100 = _mm_setr_epi16(100, 1, 100, 1, 100, 1, 100, 1); + const __m128i mul_1_10000 = + _mm_setr_epi16(10000, 1, 10000, 1, 10000, 1, 10000, 1); + const __m128i input = _mm_sub_epi8( + _mm_loadu_si128(reinterpret_cast(chars)), ascii0); + const __m128i t1 = _mm_maddubs_epi16(input, mul_1_10); + const __m128i t2 = _mm_madd_epi16(t1, mul_1_100); + const __m128i t3 = _mm_packus_epi32(t2, t2); + const __m128i t4 = _mm_madd_epi16(t3, mul_1_10000); + return _mm_cvtsi128_si32( + t4); // only captures the sum of the first 8 digits, drop the rest + } + + /** @private */ + simdjson_inline internal::value128 full_multiplication(uint64_t value1, uint64_t value2) { + internal::value128 answer; +#if SIMDJSON_REGULAR_VISUAL_STUDIO || SIMDJSON_IS_32BITS +#if SIMDJSON_IS_ARM64 + // ARM64 has native support for 64-bit multiplications, no need to emultate + answer.high = __umulh(value1, value2); + answer.low = value1 * value2; +#else + answer.low = _umul128(value1, value2, &answer.high); // _umul128 not available on ARM64 +#endif // SIMDJSON_IS_ARM64 +#else // SIMDJSON_REGULAR_VISUAL_STUDIO || SIMDJSON_IS_32BITS + __uint128_t r = (static_cast<__uint128_t>(value1)) * value2; + answer.low = uint64_t(r); + answer.high = uint64_t(r >> 64); +#endif + return answer; + } + + } // namespace numberparsing + } // namespace westmere +} // namespace simdjson + +#define SIMDJSON_SWAR_NUMBER_PARSING 1 + +#endif // SIMDJSON_WESTMERE_NUMBERPARSING_DEFS_H +/* end file simdjson/westmere/numberparsing_defs.h */ +/* including simdjson/westmere/simd.h: #include "simdjson/westmere/simd.h" */ +/* begin file simdjson/westmere/simd.h */ +#ifndef SIMDJSON_WESTMERE_SIMD_H +#define SIMDJSON_WESTMERE_SIMD_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/westmere/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/westmere/bitmanipulation.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/simdprune_tables.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace westmere { + namespace { + namespace simd { + + template + struct base { + __m128i value; + + // Zero constructor + simdjson_inline base() : value{ __m128i() } {} + + // Conversion from SIMD register + simdjson_inline base(const __m128i _value) : value(_value) {} + + // Conversion to SIMD register + simdjson_inline operator const __m128i& () const { return this->value; } + simdjson_inline operator __m128i& () { return this->value; } + + // Bit operations + simdjson_inline Child operator|(const Child other) const { return _mm_or_si128(*this, other); } + simdjson_inline Child operator&(const Child other) const { return _mm_and_si128(*this, other); } + simdjson_inline Child operator^(const Child other) const { return _mm_xor_si128(*this, other); } + simdjson_inline Child bit_andnot(const Child other) const { return _mm_andnot_si128(other, *this); } + simdjson_inline Child& operator|=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast | other; return *this_cast; } + simdjson_inline Child& operator&=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast & other; return *this_cast; } + simdjson_inline Child& operator^=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast ^ other; return *this_cast; } + }; + + template> + struct base8 : base> { + typedef uint16_t bitmask_t; + typedef uint32_t bitmask2_t; + + simdjson_inline base8() : base>() {} + simdjson_inline base8(const __m128i _value) : base>(_value) {} + + friend simdjson_inline Mask operator==(const simd8 lhs, const simd8 rhs) { return _mm_cmpeq_epi8(lhs, rhs); } + + static const int SIZE = sizeof(base>::value); + + template + simdjson_inline simd8 prev(const simd8 prev_chunk) const { + return _mm_alignr_epi8(*this, prev_chunk, 16 - N); + } + }; + + // SIMD byte mask type (returned by things like eq and gt) + template<> + struct simd8 : base8 { + static simdjson_inline simd8 splat(bool _value) { return _mm_set1_epi8(uint8_t(-(!!_value))); } + + simdjson_inline simd8() : base8() {} + simdjson_inline simd8(const __m128i _value) : base8(_value) {} + // Splat constructor + simdjson_inline simd8(bool _value) : base8(splat(_value)) {} + + simdjson_inline int to_bitmask() const { return _mm_movemask_epi8(*this); } + simdjson_inline bool any() const { return !_mm_testz_si128(*this, *this); } + simdjson_inline simd8 operator~() const { return *this ^ true; } + }; + + template + struct base8_numeric : base8 { + static simdjson_inline simd8 splat(T _value) { return _mm_set1_epi8(_value); } + static simdjson_inline simd8 zero() { return _mm_setzero_si128(); } + static simdjson_inline simd8 load(const T values[16]) { + return _mm_loadu_si128(reinterpret_cast(values)); + } + // Repeat 16 values as many times as necessary (usually for lookup tables) + static simdjson_inline simd8 repeat_16( + T v0, T v1, T v2, T v3, T v4, T v5, T v6, T v7, + T v8, T v9, T v10, T v11, T v12, T v13, T v14, T v15 + ) { + return simd8( + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15 + ); + } + + simdjson_inline base8_numeric() : base8() {} + simdjson_inline base8_numeric(const __m128i _value) : base8(_value) {} + + // Store to array + simdjson_inline void store(T dst[16]) const { return _mm_storeu_si128(reinterpret_cast<__m128i*>(dst), *this); } + + // Override to distinguish from bool version + simdjson_inline simd8 operator~() const { return *this ^ 0xFFu; } + + // Addition/subtraction are the same for signed and unsigned + simdjson_inline simd8 operator+(const simd8 other) const { return _mm_add_epi8(*this, other); } + simdjson_inline simd8 operator-(const simd8 other) const { return _mm_sub_epi8(*this, other); } + simdjson_inline simd8& operator+=(const simd8 other) { *this = *this + other; return *static_cast*>(this); } + simdjson_inline simd8& operator-=(const simd8 other) { *this = *this - other; return *static_cast*>(this); } + + // Perform a lookup assuming the value is between 0 and 16 (undefined behavior for out of range values) + template + simdjson_inline simd8 lookup_16(simd8 lookup_table) const { + return _mm_shuffle_epi8(lookup_table, *this); + } + + // Copies to 'output" all bytes corresponding to a 0 in the mask (interpreted as a bitset). + // Passing a 0 value for mask would be equivalent to writing out every byte to output. + // Only the first 16 - count_ones(mask) bytes of the result are significant but 16 bytes + // get written. + // Design consideration: it seems like a function with the + // signature simd8 compress(uint32_t mask) would be + // sensible, but the AVX ISA makes this kind of approach difficult. + template + simdjson_inline void compress(uint16_t mask, L* output) const { + using internal::thintable_epi8; + using internal::BitsSetTable256mul2; + using internal::pshufb_combine_table; + // this particular implementation was inspired by work done by @animetosho + // we do it in two steps, first 8 bytes and then second 8 bytes + uint8_t mask1 = uint8_t(mask); // least significant 8 bits + uint8_t mask2 = uint8_t(mask >> 8); // most significant 8 bits + // next line just loads the 64-bit values thintable_epi8[mask1] and + // thintable_epi8[mask2] into a 128-bit register, using only + // two instructions on most compilers. + __m128i shufmask = _mm_set_epi64x(thintable_epi8[mask2], thintable_epi8[mask1]); + // we increment by 0x08 the second half of the mask + shufmask = + _mm_add_epi8(shufmask, _mm_set_epi32(0x08080808, 0x08080808, 0, 0)); + // this is the version "nearly pruned" + __m128i pruned = _mm_shuffle_epi8(*this, shufmask); + // we still need to put the two halves together. + // we compute the popcount of the first half: + int pop1 = BitsSetTable256mul2[mask1]; + // then load the corresponding mask, what it does is to write + // only the first pop1 bytes from the first 8 bytes, and then + // it fills in with the bytes from the second 8 bytes + some filling + // at the end. + __m128i compactmask = + _mm_loadu_si128(reinterpret_cast(pshufb_combine_table + pop1 * 8)); + __m128i answer = _mm_shuffle_epi8(pruned, compactmask); + _mm_storeu_si128(reinterpret_cast<__m128i*>(output), answer); + } + + template + simdjson_inline simd8 lookup_16( + L replace0, L replace1, L replace2, L replace3, + L replace4, L replace5, L replace6, L replace7, + L replace8, L replace9, L replace10, L replace11, + L replace12, L replace13, L replace14, L replace15) const { + return lookup_16(simd8::repeat_16( + replace0, replace1, replace2, replace3, + replace4, replace5, replace6, replace7, + replace8, replace9, replace10, replace11, + replace12, replace13, replace14, replace15 + )); + } + }; + + // Signed bytes + template<> + struct simd8 : base8_numeric { + simdjson_inline simd8() : base8_numeric() {} + simdjson_inline simd8(const __m128i _value) : base8_numeric(_value) {} + // Splat constructor + simdjson_inline simd8(int8_t _value) : simd8(splat(_value)) {} + // Array constructor + simdjson_inline simd8(const int8_t* values) : simd8(load(values)) {} + // Member-by-member initialization + simdjson_inline simd8( + int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, + int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 + ) : simd8(_mm_setr_epi8( + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15 + )) {} + // Repeat 16 values as many times as necessary (usually for lookup tables) + simdjson_inline static simd8 repeat_16( + int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, + int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 + ) { + return simd8( + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15 + ); + } + + // Order-sensitive comparisons + simdjson_inline simd8 max_val(const simd8 other) const { return _mm_max_epi8(*this, other); } + simdjson_inline simd8 min_val(const simd8 other) const { return _mm_min_epi8(*this, other); } + simdjson_inline simd8 operator>(const simd8 other) const { return _mm_cmpgt_epi8(*this, other); } + simdjson_inline simd8 operator<(const simd8 other) const { return _mm_cmpgt_epi8(other, *this); } + }; + + // Unsigned bytes + template<> + struct simd8 : base8_numeric { + simdjson_inline simd8() : base8_numeric() {} + simdjson_inline simd8(const __m128i _value) : base8_numeric(_value) {} + // Splat constructor + simdjson_inline simd8(uint8_t _value) : simd8(splat(_value)) {} + // Array constructor + simdjson_inline simd8(const uint8_t* values) : simd8(load(values)) {} + // Member-by-member initialization + simdjson_inline simd8( + uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, + uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 + ) : simd8(_mm_setr_epi8( + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15 + )) {} + // Repeat 16 values as many times as necessary (usually for lookup tables) + simdjson_inline static simd8 repeat_16( + uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, + uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 + ) { + return simd8( + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15 + ); + } + + // Saturated math + simdjson_inline simd8 saturating_add(const simd8 other) const { return _mm_adds_epu8(*this, other); } + simdjson_inline simd8 saturating_sub(const simd8 other) const { return _mm_subs_epu8(*this, other); } + + // Order-specific operations + simdjson_inline simd8 max_val(const simd8 other) const { return _mm_max_epu8(*this, other); } + simdjson_inline simd8 min_val(const simd8 other) const { return _mm_min_epu8(*this, other); } + // Same as >, but only guarantees true is nonzero (< guarantees true = -1) + simdjson_inline simd8 gt_bits(const simd8 other) const { return this->saturating_sub(other); } + // Same as <, but only guarantees true is nonzero (< guarantees true = -1) + simdjson_inline simd8 lt_bits(const simd8 other) const { return other.saturating_sub(*this); } + simdjson_inline simd8 operator<=(const simd8 other) const { return other.max_val(*this) == other; } + simdjson_inline simd8 operator>=(const simd8 other) const { return other.min_val(*this) == other; } + simdjson_inline simd8 operator>(const simd8 other) const { return this->gt_bits(other).any_bits_set(); } + simdjson_inline simd8 operator<(const simd8 other) const { return this->gt_bits(other).any_bits_set(); } + + // Bit-specific operations + simdjson_inline simd8 bits_not_set() const { return *this == uint8_t(0); } + simdjson_inline simd8 bits_not_set(simd8 bits) const { return (*this & bits).bits_not_set(); } + simdjson_inline simd8 any_bits_set() const { return ~this->bits_not_set(); } + simdjson_inline simd8 any_bits_set(simd8 bits) const { return ~this->bits_not_set(bits); } + simdjson_inline bool is_ascii() const { return _mm_movemask_epi8(*this) == 0; } + simdjson_inline bool bits_not_set_anywhere() const { return _mm_testz_si128(*this, *this); } + simdjson_inline bool any_bits_set_anywhere() const { return !bits_not_set_anywhere(); } + simdjson_inline bool bits_not_set_anywhere(simd8 bits) const { return _mm_testz_si128(*this, bits); } + simdjson_inline bool any_bits_set_anywhere(simd8 bits) const { return !bits_not_set_anywhere(bits); } + template + simdjson_inline simd8 shr() const { return simd8(_mm_srli_epi16(*this, N)) & uint8_t(0xFFu >> N); } + template + simdjson_inline simd8 shl() const { return simd8(_mm_slli_epi16(*this, N)) & uint8_t(0xFFu << N); } + // Get one of the bits and make a bitmask out of it. + // e.g. value.get_bit<7>() gets the high bit + template + simdjson_inline int get_bit() const { return _mm_movemask_epi8(_mm_slli_epi16(*this, 7 - N)); } + }; + + template + struct simd8x64 { + static constexpr int NUM_CHUNKS = 64 / sizeof(simd8); + static_assert(NUM_CHUNKS == 4, "Westmere kernel should use four registers per 64-byte block."); + const simd8 chunks[NUM_CHUNKS]; + + simd8x64(const simd8x64& o) = delete; // no copy allowed + simd8x64& operator=(const simd8& other) = delete; // no assignment allowed + simd8x64() = delete; // no default constructor allowed + + simdjson_inline simd8x64(const simd8 chunk0, const simd8 chunk1, const simd8 chunk2, const simd8 chunk3) : chunks{ chunk0, chunk1, chunk2, chunk3 } {} + simdjson_inline simd8x64(const T ptr[64]) : chunks{ simd8::load(ptr), simd8::load(ptr + 16), simd8::load(ptr + 32), simd8::load(ptr + 48) } {} + + simdjson_inline void store(T ptr[64]) const { + this->chunks[0].store(ptr + sizeof(simd8) * 0); + this->chunks[1].store(ptr + sizeof(simd8) * 1); + this->chunks[2].store(ptr + sizeof(simd8) * 2); + this->chunks[3].store(ptr + sizeof(simd8) * 3); + } + + simdjson_inline simd8 reduce_or() const { + return (this->chunks[0] | this->chunks[1]) | (this->chunks[2] | this->chunks[3]); + } + + simdjson_inline uint64_t compress(uint64_t mask, T* output) const { + this->chunks[0].compress(uint16_t(mask), output); + this->chunks[1].compress(uint16_t(mask >> 16), output + 16 - count_ones(mask & 0xFFFF)); + this->chunks[2].compress(uint16_t(mask >> 32), output + 32 - count_ones(mask & 0xFFFFFFFF)); + this->chunks[3].compress(uint16_t(mask >> 48), output + 48 - count_ones(mask & 0xFFFFFFFFFFFF)); + return 64 - count_ones(mask); + } + + simdjson_inline uint64_t to_bitmask() const { + uint64_t r0 = uint32_t(this->chunks[0].to_bitmask()); + uint64_t r1 = this->chunks[1].to_bitmask(); + uint64_t r2 = this->chunks[2].to_bitmask(); + uint64_t r3 = this->chunks[3].to_bitmask(); + return r0 | (r1 << 16) | (r2 << 32) | (r3 << 48); + } + + simdjson_inline uint64_t eq(const T m) const { + const simd8 mask = simd8::splat(m); + return simd8x64( + this->chunks[0] == mask, + this->chunks[1] == mask, + this->chunks[2] == mask, + this->chunks[3] == mask + ).to_bitmask(); + } + + simdjson_inline uint64_t eq(const simd8x64& other) const { + return simd8x64( + this->chunks[0] == other.chunks[0], + this->chunks[1] == other.chunks[1], + this->chunks[2] == other.chunks[2], + this->chunks[3] == other.chunks[3] + ).to_bitmask(); + } + + simdjson_inline uint64_t lteq(const T m) const { + const simd8 mask = simd8::splat(m); + return simd8x64( + this->chunks[0] <= mask, + this->chunks[1] <= mask, + this->chunks[2] <= mask, + this->chunks[3] <= mask + ).to_bitmask(); + } + }; // struct simd8x64 + + } // namespace simd + } // unnamed namespace + } // namespace westmere +} // namespace simdjson + +#endif // SIMDJSON_WESTMERE_SIMD_INPUT_H +/* end file simdjson/westmere/simd.h */ +/* including simdjson/westmere/stringparsing_defs.h: #include "simdjson/westmere/stringparsing_defs.h" */ +/* begin file simdjson/westmere/stringparsing_defs.h */ +#ifndef SIMDJSON_WESTMERE_STRINGPARSING_DEFS_H +#define SIMDJSON_WESTMERE_STRINGPARSING_DEFS_H + +/* including simdjson/westmere/bitmanipulation.h: #include "simdjson/westmere/bitmanipulation.h" */ +/* begin file simdjson/westmere/bitmanipulation.h */ +#ifndef SIMDJSON_WESTMERE_BITMANIPULATION_H +#define SIMDJSON_WESTMERE_BITMANIPULATION_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/westmere/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/westmere/intrinsics.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace westmere { + namespace { + + // We sometimes call trailing_zero on inputs that are zero, + // but the algorithms do not end up using the returned value. + // Sadly, sanitizers are not smart enough to figure it out. + SIMDJSON_NO_SANITIZE_UNDEFINED + // This function can be used safely even if not all bytes have been + // initialized. + // See issue https://github.com/simdjson/simdjson/issues/1965 + SIMDJSON_NO_SANITIZE_MEMORY + simdjson_inline int trailing_zeroes(uint64_t input_num) { +#if SIMDJSON_REGULAR_VISUAL_STUDIO + unsigned long ret; + // Search the mask data from least significant bit (LSB) + // to the most significant bit (MSB) for a set bit (1). + _BitScanForward64(&ret, input_num); + return (int)ret; +#else // SIMDJSON_REGULAR_VISUAL_STUDIO + return __builtin_ctzll(input_num); +#endif // SIMDJSON_REGULAR_VISUAL_STUDIO + } + + /* result might be undefined when input_num is zero */ + simdjson_inline uint64_t clear_lowest_bit(uint64_t input_num) { + return input_num & (input_num - 1); + } + + /* result might be undefined when input_num is zero */ + simdjson_inline int leading_zeroes(uint64_t input_num) { +#if SIMDJSON_REGULAR_VISUAL_STUDIO + unsigned long leading_zero = 0; + // Search the mask data from most significant bit (MSB) + // to least significant bit (LSB) for a set bit (1). + if (_BitScanReverse64(&leading_zero, input_num)) + return (int)(63 - leading_zero); + else + return 64; +#else + return __builtin_clzll(input_num); +#endif// SIMDJSON_REGULAR_VISUAL_STUDIO + } + +#if SIMDJSON_REGULAR_VISUAL_STUDIO + simdjson_inline unsigned __int64 count_ones(uint64_t input_num) { + // note: we do not support legacy 32-bit Windows in this kernel + return __popcnt64(input_num);// Visual Studio wants two underscores + } +#else + simdjson_inline long long int count_ones(uint64_t input_num) { + return _popcnt64(input_num); + } +#endif + + simdjson_inline bool add_overflow(uint64_t value1, uint64_t value2, + uint64_t* result) { +#if SIMDJSON_REGULAR_VISUAL_STUDIO + return _addcarry_u64(0, value1, value2, + reinterpret_cast(result)); +#else + return __builtin_uaddll_overflow(value1, value2, + reinterpret_cast(result)); +#endif + } + + } // unnamed namespace + } // namespace westmere +} // namespace simdjson + +#endif // SIMDJSON_WESTMERE_BITMANIPULATION_H +/* end file simdjson/westmere/bitmanipulation.h */ +/* including simdjson/westmere/simd.h: #include "simdjson/westmere/simd.h" */ +/* begin file simdjson/westmere/simd.h */ +#ifndef SIMDJSON_WESTMERE_SIMD_H +#define SIMDJSON_WESTMERE_SIMD_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/westmere/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/westmere/bitmanipulation.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/simdprune_tables.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace westmere { + namespace { + namespace simd { + + template + struct base { + __m128i value; + + // Zero constructor + simdjson_inline base() : value{ __m128i() } {} + + // Conversion from SIMD register + simdjson_inline base(const __m128i _value) : value(_value) {} + + // Conversion to SIMD register + simdjson_inline operator const __m128i& () const { return this->value; } + simdjson_inline operator __m128i& () { return this->value; } + + // Bit operations + simdjson_inline Child operator|(const Child other) const { return _mm_or_si128(*this, other); } + simdjson_inline Child operator&(const Child other) const { return _mm_and_si128(*this, other); } + simdjson_inline Child operator^(const Child other) const { return _mm_xor_si128(*this, other); } + simdjson_inline Child bit_andnot(const Child other) const { return _mm_andnot_si128(other, *this); } + simdjson_inline Child& operator|=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast | other; return *this_cast; } + simdjson_inline Child& operator&=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast & other; return *this_cast; } + simdjson_inline Child& operator^=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast ^ other; return *this_cast; } + }; + + template> + struct base8 : base> { + typedef uint16_t bitmask_t; + typedef uint32_t bitmask2_t; + + simdjson_inline base8() : base>() {} + simdjson_inline base8(const __m128i _value) : base>(_value) {} + + friend simdjson_inline Mask operator==(const simd8 lhs, const simd8 rhs) { return _mm_cmpeq_epi8(lhs, rhs); } + + static const int SIZE = sizeof(base>::value); + + template + simdjson_inline simd8 prev(const simd8 prev_chunk) const { + return _mm_alignr_epi8(*this, prev_chunk, 16 - N); + } + }; + + // SIMD byte mask type (returned by things like eq and gt) + template<> + struct simd8 : base8 { + static simdjson_inline simd8 splat(bool _value) { return _mm_set1_epi8(uint8_t(-(!!_value))); } + + simdjson_inline simd8() : base8() {} + simdjson_inline simd8(const __m128i _value) : base8(_value) {} + // Splat constructor + simdjson_inline simd8(bool _value) : base8(splat(_value)) {} + + simdjson_inline int to_bitmask() const { return _mm_movemask_epi8(*this); } + simdjson_inline bool any() const { return !_mm_testz_si128(*this, *this); } + simdjson_inline simd8 operator~() const { return *this ^ true; } + }; + + template + struct base8_numeric : base8 { + static simdjson_inline simd8 splat(T _value) { return _mm_set1_epi8(_value); } + static simdjson_inline simd8 zero() { return _mm_setzero_si128(); } + static simdjson_inline simd8 load(const T values[16]) { + return _mm_loadu_si128(reinterpret_cast(values)); + } + // Repeat 16 values as many times as necessary (usually for lookup tables) + static simdjson_inline simd8 repeat_16( + T v0, T v1, T v2, T v3, T v4, T v5, T v6, T v7, + T v8, T v9, T v10, T v11, T v12, T v13, T v14, T v15 + ) { + return simd8( + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15 + ); + } + + simdjson_inline base8_numeric() : base8() {} + simdjson_inline base8_numeric(const __m128i _value) : base8(_value) {} + + // Store to array + simdjson_inline void store(T dst[16]) const { return _mm_storeu_si128(reinterpret_cast<__m128i*>(dst), *this); } + + // Override to distinguish from bool version + simdjson_inline simd8 operator~() const { return *this ^ 0xFFu; } + + // Addition/subtraction are the same for signed and unsigned + simdjson_inline simd8 operator+(const simd8 other) const { return _mm_add_epi8(*this, other); } + simdjson_inline simd8 operator-(const simd8 other) const { return _mm_sub_epi8(*this, other); } + simdjson_inline simd8& operator+=(const simd8 other) { *this = *this + other; return *static_cast*>(this); } + simdjson_inline simd8& operator-=(const simd8 other) { *this = *this - other; return *static_cast*>(this); } + + // Perform a lookup assuming the value is between 0 and 16 (undefined behavior for out of range values) + template + simdjson_inline simd8 lookup_16(simd8 lookup_table) const { + return _mm_shuffle_epi8(lookup_table, *this); + } + + // Copies to 'output" all bytes corresponding to a 0 in the mask (interpreted as a bitset). + // Passing a 0 value for mask would be equivalent to writing out every byte to output. + // Only the first 16 - count_ones(mask) bytes of the result are significant but 16 bytes + // get written. + // Design consideration: it seems like a function with the + // signature simd8 compress(uint32_t mask) would be + // sensible, but the AVX ISA makes this kind of approach difficult. + template + simdjson_inline void compress(uint16_t mask, L* output) const { + using internal::thintable_epi8; + using internal::BitsSetTable256mul2; + using internal::pshufb_combine_table; + // this particular implementation was inspired by work done by @animetosho + // we do it in two steps, first 8 bytes and then second 8 bytes + uint8_t mask1 = uint8_t(mask); // least significant 8 bits + uint8_t mask2 = uint8_t(mask >> 8); // most significant 8 bits + // next line just loads the 64-bit values thintable_epi8[mask1] and + // thintable_epi8[mask2] into a 128-bit register, using only + // two instructions on most compilers. + __m128i shufmask = _mm_set_epi64x(thintable_epi8[mask2], thintable_epi8[mask1]); + // we increment by 0x08 the second half of the mask + shufmask = + _mm_add_epi8(shufmask, _mm_set_epi32(0x08080808, 0x08080808, 0, 0)); + // this is the version "nearly pruned" + __m128i pruned = _mm_shuffle_epi8(*this, shufmask); + // we still need to put the two halves together. + // we compute the popcount of the first half: + int pop1 = BitsSetTable256mul2[mask1]; + // then load the corresponding mask, what it does is to write + // only the first pop1 bytes from the first 8 bytes, and then + // it fills in with the bytes from the second 8 bytes + some filling + // at the end. + __m128i compactmask = + _mm_loadu_si128(reinterpret_cast(pshufb_combine_table + pop1 * 8)); + __m128i answer = _mm_shuffle_epi8(pruned, compactmask); + _mm_storeu_si128(reinterpret_cast<__m128i*>(output), answer); + } + + template + simdjson_inline simd8 lookup_16( + L replace0, L replace1, L replace2, L replace3, + L replace4, L replace5, L replace6, L replace7, + L replace8, L replace9, L replace10, L replace11, + L replace12, L replace13, L replace14, L replace15) const { + return lookup_16(simd8::repeat_16( + replace0, replace1, replace2, replace3, + replace4, replace5, replace6, replace7, + replace8, replace9, replace10, replace11, + replace12, replace13, replace14, replace15 + )); + } + }; + + // Signed bytes + template<> + struct simd8 : base8_numeric { + simdjson_inline simd8() : base8_numeric() {} + simdjson_inline simd8(const __m128i _value) : base8_numeric(_value) {} + // Splat constructor + simdjson_inline simd8(int8_t _value) : simd8(splat(_value)) {} + // Array constructor + simdjson_inline simd8(const int8_t* values) : simd8(load(values)) {} + // Member-by-member initialization + simdjson_inline simd8( + int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, + int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 + ) : simd8(_mm_setr_epi8( + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15 + )) {} + // Repeat 16 values as many times as necessary (usually for lookup tables) + simdjson_inline static simd8 repeat_16( + int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, + int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 + ) { + return simd8( + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15 + ); + } + + // Order-sensitive comparisons + simdjson_inline simd8 max_val(const simd8 other) const { return _mm_max_epi8(*this, other); } + simdjson_inline simd8 min_val(const simd8 other) const { return _mm_min_epi8(*this, other); } + simdjson_inline simd8 operator>(const simd8 other) const { return _mm_cmpgt_epi8(*this, other); } + simdjson_inline simd8 operator<(const simd8 other) const { return _mm_cmpgt_epi8(other, *this); } + }; + + // Unsigned bytes + template<> + struct simd8 : base8_numeric { + simdjson_inline simd8() : base8_numeric() {} + simdjson_inline simd8(const __m128i _value) : base8_numeric(_value) {} + // Splat constructor + simdjson_inline simd8(uint8_t _value) : simd8(splat(_value)) {} + // Array constructor + simdjson_inline simd8(const uint8_t* values) : simd8(load(values)) {} + // Member-by-member initialization + simdjson_inline simd8( + uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, + uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 + ) : simd8(_mm_setr_epi8( + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15 + )) {} + // Repeat 16 values as many times as necessary (usually for lookup tables) + simdjson_inline static simd8 repeat_16( + uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, + uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 + ) { + return simd8( + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15 + ); + } + + // Saturated math + simdjson_inline simd8 saturating_add(const simd8 other) const { return _mm_adds_epu8(*this, other); } + simdjson_inline simd8 saturating_sub(const simd8 other) const { return _mm_subs_epu8(*this, other); } + + // Order-specific operations + simdjson_inline simd8 max_val(const simd8 other) const { return _mm_max_epu8(*this, other); } + simdjson_inline simd8 min_val(const simd8 other) const { return _mm_min_epu8(*this, other); } + // Same as >, but only guarantees true is nonzero (< guarantees true = -1) + simdjson_inline simd8 gt_bits(const simd8 other) const { return this->saturating_sub(other); } + // Same as <, but only guarantees true is nonzero (< guarantees true = -1) + simdjson_inline simd8 lt_bits(const simd8 other) const { return other.saturating_sub(*this); } + simdjson_inline simd8 operator<=(const simd8 other) const { return other.max_val(*this) == other; } + simdjson_inline simd8 operator>=(const simd8 other) const { return other.min_val(*this) == other; } + simdjson_inline simd8 operator>(const simd8 other) const { return this->gt_bits(other).any_bits_set(); } + simdjson_inline simd8 operator<(const simd8 other) const { return this->gt_bits(other).any_bits_set(); } + + // Bit-specific operations + simdjson_inline simd8 bits_not_set() const { return *this == uint8_t(0); } + simdjson_inline simd8 bits_not_set(simd8 bits) const { return (*this & bits).bits_not_set(); } + simdjson_inline simd8 any_bits_set() const { return ~this->bits_not_set(); } + simdjson_inline simd8 any_bits_set(simd8 bits) const { return ~this->bits_not_set(bits); } + simdjson_inline bool is_ascii() const { return _mm_movemask_epi8(*this) == 0; } + simdjson_inline bool bits_not_set_anywhere() const { return _mm_testz_si128(*this, *this); } + simdjson_inline bool any_bits_set_anywhere() const { return !bits_not_set_anywhere(); } + simdjson_inline bool bits_not_set_anywhere(simd8 bits) const { return _mm_testz_si128(*this, bits); } + simdjson_inline bool any_bits_set_anywhere(simd8 bits) const { return !bits_not_set_anywhere(bits); } + template + simdjson_inline simd8 shr() const { return simd8(_mm_srli_epi16(*this, N)) & uint8_t(0xFFu >> N); } + template + simdjson_inline simd8 shl() const { return simd8(_mm_slli_epi16(*this, N)) & uint8_t(0xFFu << N); } + // Get one of the bits and make a bitmask out of it. + // e.g. value.get_bit<7>() gets the high bit + template + simdjson_inline int get_bit() const { return _mm_movemask_epi8(_mm_slli_epi16(*this, 7 - N)); } + }; + + template + struct simd8x64 { + static constexpr int NUM_CHUNKS = 64 / sizeof(simd8); + static_assert(NUM_CHUNKS == 4, "Westmere kernel should use four registers per 64-byte block."); + const simd8 chunks[NUM_CHUNKS]; + + simd8x64(const simd8x64& o) = delete; // no copy allowed + simd8x64& operator=(const simd8& other) = delete; // no assignment allowed + simd8x64() = delete; // no default constructor allowed + + simdjson_inline simd8x64(const simd8 chunk0, const simd8 chunk1, const simd8 chunk2, const simd8 chunk3) : chunks{ chunk0, chunk1, chunk2, chunk3 } {} + simdjson_inline simd8x64(const T ptr[64]) : chunks{ simd8::load(ptr), simd8::load(ptr + 16), simd8::load(ptr + 32), simd8::load(ptr + 48) } {} + + simdjson_inline void store(T ptr[64]) const { + this->chunks[0].store(ptr + sizeof(simd8) * 0); + this->chunks[1].store(ptr + sizeof(simd8) * 1); + this->chunks[2].store(ptr + sizeof(simd8) * 2); + this->chunks[3].store(ptr + sizeof(simd8) * 3); + } + + simdjson_inline simd8 reduce_or() const { + return (this->chunks[0] | this->chunks[1]) | (this->chunks[2] | this->chunks[3]); + } + + simdjson_inline uint64_t compress(uint64_t mask, T* output) const { + this->chunks[0].compress(uint16_t(mask), output); + this->chunks[1].compress(uint16_t(mask >> 16), output + 16 - count_ones(mask & 0xFFFF)); + this->chunks[2].compress(uint16_t(mask >> 32), output + 32 - count_ones(mask & 0xFFFFFFFF)); + this->chunks[3].compress(uint16_t(mask >> 48), output + 48 - count_ones(mask & 0xFFFFFFFFFFFF)); + return 64 - count_ones(mask); + } + + simdjson_inline uint64_t to_bitmask() const { + uint64_t r0 = uint32_t(this->chunks[0].to_bitmask()); + uint64_t r1 = this->chunks[1].to_bitmask(); + uint64_t r2 = this->chunks[2].to_bitmask(); + uint64_t r3 = this->chunks[3].to_bitmask(); + return r0 | (r1 << 16) | (r2 << 32) | (r3 << 48); + } + + simdjson_inline uint64_t eq(const T m) const { + const simd8 mask = simd8::splat(m); + return simd8x64( + this->chunks[0] == mask, + this->chunks[1] == mask, + this->chunks[2] == mask, + this->chunks[3] == mask + ).to_bitmask(); + } + + simdjson_inline uint64_t eq(const simd8x64& other) const { + return simd8x64( + this->chunks[0] == other.chunks[0], + this->chunks[1] == other.chunks[1], + this->chunks[2] == other.chunks[2], + this->chunks[3] == other.chunks[3] + ).to_bitmask(); + } + + simdjson_inline uint64_t lteq(const T m) const { + const simd8 mask = simd8::splat(m); + return simd8x64( + this->chunks[0] <= mask, + this->chunks[1] <= mask, + this->chunks[2] <= mask, + this->chunks[3] <= mask + ).to_bitmask(); + } + }; // struct simd8x64 + + } // namespace simd + } // unnamed namespace + } // namespace westmere +} // namespace simdjson + +#endif // SIMDJSON_WESTMERE_SIMD_INPUT_H +/* end file simdjson/westmere/simd.h */ + +namespace simdjson { + namespace westmere { + namespace { + + using namespace simd; + + // Holds backslashes and quotes locations. + struct backslash_and_quote { + public: + static constexpr uint32_t BYTES_PROCESSED = 32; + simdjson_inline static backslash_and_quote copy_and_find(const uint8_t* src, uint8_t* dst); + + simdjson_inline bool has_quote_first() { return ((bs_bits - 1) & quote_bits) != 0; } + simdjson_inline bool has_backslash() { return bs_bits != 0; } + simdjson_inline int quote_index() { return trailing_zeroes(quote_bits); } + simdjson_inline int backslash_index() { return trailing_zeroes(bs_bits); } + + uint32_t bs_bits; + uint32_t quote_bits; + }; // struct backslash_and_quote + + simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8_t* src, uint8_t* dst) { + // this can read up to 31 bytes beyond the buffer size, but we require + // SIMDJSON_PADDING of padding + static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "backslash and quote finder must process fewer than SIMDJSON_PADDING bytes"); + simd8 v0(src); + simd8 v1(src + 16); + v0.store(dst); + v1.store(dst + 16); + uint64_t bs_and_quote = simd8x64(v0 == '\\', v1 == '\\', v0 == '"', v1 == '"').to_bitmask(); + return { + uint32_t(bs_and_quote), // bs_bits + uint32_t(bs_and_quote >> 32) // quote_bits + }; + } + + } // unnamed namespace + } // namespace westmere +} // namespace simdjson + +#endif // SIMDJSON_WESTMERE_STRINGPARSING_DEFS_H +/* end file simdjson/westmere/stringparsing_defs.h */ +/* end file simdjson/westmere/begin.h */ +/* including simdjson/generic/amalgamated.h for westmere: #include "simdjson/generic/amalgamated.h" */ +/* begin file simdjson/generic/amalgamated.h for westmere */ +#if defined(SIMDJSON_CONDITIONAL_INCLUDE) && !defined(SIMDJSON_GENERIC_DEPENDENCIES_H) +#error simdjson/generic/dependencies.h must be included before simdjson/generic/amalgamated.h! +#endif + +/* including simdjson/generic/base.h for westmere: #include "simdjson/generic/base.h" */ +/* begin file simdjson/generic/base.h for westmere */ +#ifndef SIMDJSON_GENERIC_BASE_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_BASE_H */ +/* amalgamation skipped (editor-only): #include "simdjson/base.h" */ +/* amalgamation skipped (editor-only): // If we haven't got an implementation yet, we're in the editor, editing a generic file! Just */ +/* amalgamation skipped (editor-only): // use the most advanced one we can so the most possible stuff can be tested. */ +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_IMPLEMENTATION */ +/* amalgamation skipped (editor-only): #include "simdjson/implementation_detection.h" */ +/* amalgamation skipped (editor-only): #if SIMDJSON_IMPLEMENTATION_ICELAKE */ +/* amalgamation skipped (editor-only): #include "simdjson/icelake/begin.h" */ +/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_HASWELL */ +/* amalgamation skipped (editor-only): #include "simdjson/haswell/begin.h" */ +/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_WESTMERE */ +/* amalgamation skipped (editor-only): #include "simdjson/westmere/begin.h" */ +/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_ARM64 */ +/* amalgamation skipped (editor-only): #include "simdjson/arm64/begin.h" */ +/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_PPC64 */ +/* amalgamation skipped (editor-only): #include "simdjson/ppc64/begin.h" */ +/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_LSX */ +/* amalgamation skipped (editor-only): #include "simdjson/lsx/begin.h" */ +/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_LASX */ +/* amalgamation skipped (editor-only): #include "simdjson/lasx/begin.h" */ +/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_FALLBACK */ +/* amalgamation skipped (editor-only): #include "simdjson/fallback/begin.h" */ +/* amalgamation skipped (editor-only): #else */ +/* amalgamation skipped (editor-only): #error "All possible implementations (including fallback) have been disabled! simdjson will not run." */ +/* amalgamation skipped (editor-only): #endif */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_IMPLEMENTATION */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace westmere { + + struct open_container; + class dom_parser_implementation; + + /** + * The type of a JSON number + */ + enum class number_type { + floating_point_number = 1, /// a binary64 number + signed_integer, /// a signed integer that fits in a 64-bit word using two's complement + unsigned_integer, /// a positive integer larger or equal to 1<<63 + big_integer /// a big integer that does not fit in a 64-bit word + }; + + } // namespace westmere +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_BASE_H +/* end file simdjson/generic/base.h for westmere */ +/* including simdjson/generic/jsoncharutils.h for westmere: #include "simdjson/generic/jsoncharutils.h" */ +/* begin file simdjson/generic/jsoncharutils.h for westmere */ +#ifndef SIMDJSON_GENERIC_JSONCHARUTILS_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_JSONCHARUTILS_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/jsoncharutils_tables.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace westmere { + namespace { + namespace jsoncharutils { + + // return non-zero if not a structural or whitespace char + // zero otherwise + simdjson_inline uint32_t is_not_structural_or_whitespace(uint8_t c) { + return internal::structural_or_whitespace_negated[c]; + } + + simdjson_inline uint32_t is_structural_or_whitespace(uint8_t c) { + return internal::structural_or_whitespace[c]; + } + + // returns a value with the high 16 bits set if not valid + // otherwise returns the conversion of the 4 hex digits at src into the bottom + // 16 bits of the 32-bit return register + // + // see + // https://lemire.me/blog/2019/04/17/parsing-short-hexadecimal-strings-efficiently/ + static inline uint32_t hex_to_u32_nocheck( + const uint8_t* src) { // strictly speaking, static inline is a C-ism + uint32_t v1 = internal::digit_to_val32[630 + src[0]]; + uint32_t v2 = internal::digit_to_val32[420 + src[1]]; + uint32_t v3 = internal::digit_to_val32[210 + src[2]]; + uint32_t v4 = internal::digit_to_val32[0 + src[3]]; + return v1 | v2 | v3 | v4; + } + + // given a code point cp, writes to c + // the utf-8 code, outputting the length in + // bytes, if the length is zero, the code point + // is invalid + // + // This can possibly be made faster using pdep + // and clz and table lookups, but JSON documents + // have few escaped code points, and the following + // function looks cheap. + // + // Note: we assume that surrogates are treated separately + // + simdjson_inline size_t codepoint_to_utf8(uint32_t cp, uint8_t* c) { + if (cp <= 0x7F) { + c[0] = uint8_t(cp); + return 1; // ascii + } + if (cp <= 0x7FF) { + c[0] = uint8_t((cp >> 6) + 192); + c[1] = uint8_t((cp & 63) + 128); + return 2; // universal plane + // Surrogates are treated elsewhere... + //} //else if (0xd800 <= cp && cp <= 0xdfff) { + // return 0; // surrogates // could put assert here + } + else if (cp <= 0xFFFF) { + c[0] = uint8_t((cp >> 12) + 224); + c[1] = uint8_t(((cp >> 6) & 63) + 128); + c[2] = uint8_t((cp & 63) + 128); + return 3; + } + else if (cp <= 0x10FFFF) { // if you know you have a valid code point, this + // is not needed + c[0] = uint8_t((cp >> 18) + 240); + c[1] = uint8_t(((cp >> 12) & 63) + 128); + c[2] = uint8_t(((cp >> 6) & 63) + 128); + c[3] = uint8_t((cp & 63) + 128); + return 4; + } + // will return 0 when the code point was too large. + return 0; // bad r + } + +#if SIMDJSON_IS_32BITS // _umul128 for x86, arm + // this is a slow emulation routine for 32-bit + // + static simdjson_inline uint64_t __emulu(uint32_t x, uint32_t y) { + return x * (uint64_t)y; + } + static simdjson_inline uint64_t _umul128(uint64_t ab, uint64_t cd, uint64_t* hi) { + uint64_t ad = __emulu((uint32_t)(ab >> 32), (uint32_t)cd); + uint64_t bd = __emulu((uint32_t)ab, (uint32_t)cd); + uint64_t adbc = ad + __emulu((uint32_t)ab, (uint32_t)(cd >> 32)); + uint64_t adbc_carry = !!(adbc < ad); + uint64_t lo = bd + (adbc << 32); + *hi = __emulu((uint32_t)(ab >> 32), (uint32_t)(cd >> 32)) + (adbc >> 32) + + (adbc_carry << 32) + !!(lo < bd); + return lo; + } +#endif + + } // namespace jsoncharutils + } // unnamed namespace + } // namespace westmere +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_JSONCHARUTILS_H +/* end file simdjson/generic/jsoncharutils.h for westmere */ +/* including simdjson/generic/atomparsing.h for westmere: #include "simdjson/generic/atomparsing.h" */ +/* begin file simdjson/generic/atomparsing.h for westmere */ +#ifndef SIMDJSON_GENERIC_ATOMPARSING_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_ATOMPARSING_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/jsoncharutils.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +#include + +namespace simdjson { + namespace westmere { + namespace { + /// @private + namespace atomparsing { + + // The string_to_uint32 is exclusively used to map literal strings to 32-bit values. + // We use memcpy instead of a pointer cast to avoid undefined behaviors since we cannot + // be certain that the character pointer will be properly aligned. + // You might think that using memcpy makes this function expensive, but you'd be wrong. + // All decent optimizing compilers (GCC, clang, Visual Studio) will compile string_to_uint32("false"); + // to the compile-time constant 1936482662. + simdjson_inline uint32_t string_to_uint32(const char* str) { uint32_t val; std::memcpy(&val, str, sizeof(uint32_t)); return val; } + + + // Again in str4ncmp we use a memcpy to avoid undefined behavior. The memcpy may appear expensive. + // Yet all decent optimizing compilers will compile memcpy to a single instruction, just about. + simdjson_warn_unused + simdjson_inline uint32_t str4ncmp(const uint8_t* src, const char* atom) { + uint32_t srcval; // we want to avoid unaligned 32-bit loads (undefined in C/C++) + static_assert(sizeof(uint32_t) <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be larger than 4 bytes"); + std::memcpy(&srcval, src, sizeof(uint32_t)); + return srcval ^ string_to_uint32(atom); + } + + simdjson_warn_unused + simdjson_inline bool is_valid_true_atom(const uint8_t* src) { + return (str4ncmp(src, "true") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; + } + + simdjson_warn_unused + simdjson_inline bool is_valid_true_atom(const uint8_t* src, size_t len) { + if (len > 4) { return is_valid_true_atom(src); } + else if (len == 4) { return !str4ncmp(src, "true"); } + else { return false; } + } + + simdjson_warn_unused + simdjson_inline bool is_valid_false_atom(const uint8_t* src) { + return (str4ncmp(src + 1, "alse") | jsoncharutils::is_not_structural_or_whitespace(src[5])) == 0; + } + + simdjson_warn_unused + simdjson_inline bool is_valid_false_atom(const uint8_t* src, size_t len) { + if (len > 5) { return is_valid_false_atom(src); } + else if (len == 5) { return !str4ncmp(src + 1, "alse"); } + else { return false; } + } + + simdjson_warn_unused + simdjson_inline bool is_valid_null_atom(const uint8_t* src) { + return (str4ncmp(src, "null") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; + } + + simdjson_warn_unused + simdjson_inline bool is_valid_null_atom(const uint8_t* src, size_t len) { + if (len > 4) { return is_valid_null_atom(src); } + else if (len == 4) { return !str4ncmp(src, "null"); } + else { return false; } + } + + } // namespace atomparsing + } // unnamed namespace + } // namespace westmere +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_ATOMPARSING_H +/* end file simdjson/generic/atomparsing.h for westmere */ +/* including simdjson/generic/dom_parser_implementation.h for westmere: #include "simdjson/generic/dom_parser_implementation.h" */ +/* begin file simdjson/generic/dom_parser_implementation.h for westmere */ +#ifndef SIMDJSON_GENERIC_DOM_PARSER_IMPLEMENTATION_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_DOM_PARSER_IMPLEMENTATION_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/dom_parser_implementation.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace westmere { + + // expectation: sizeof(open_container) = 64/8. + struct open_container { + uint32_t tape_index; // where, on the tape, does the scope ([,{) begins + uint32_t count; // how many elements in the scope + }; // struct open_container + + static_assert(sizeof(open_container) == 64 / 8, "Open container must be 64 bits"); + + class dom_parser_implementation final : public internal::dom_parser_implementation { + public: + /** Tape location of each open { or [ */ + std::unique_ptr open_containers{}; + /** Whether each open container is a [ or { */ + std::unique_ptr is_array{}; + /** Buffer passed to stage 1 */ + const uint8_t* buf{}; + /** Length passed to stage 1 */ + size_t len{ 0 }; + /** Document passed to stage 2 */ + dom::document* doc{}; + + inline dom_parser_implementation() noexcept; + inline dom_parser_implementation(dom_parser_implementation&& other) noexcept; + inline dom_parser_implementation& operator=(dom_parser_implementation&& other) noexcept; + dom_parser_implementation(const dom_parser_implementation&) = delete; + dom_parser_implementation& operator=(const dom_parser_implementation&) = delete; + + simdjson_warn_unused error_code parse(const uint8_t* buf, size_t len, dom::document& doc) noexcept final; + simdjson_warn_unused error_code stage1(const uint8_t* buf, size_t len, stage1_mode partial) noexcept final; + simdjson_warn_unused error_code stage2(dom::document& doc) noexcept final; + simdjson_warn_unused error_code stage2_next(dom::document& doc) noexcept final; + simdjson_warn_unused uint8_t* parse_string(const uint8_t* src, uint8_t* dst, bool allow_replacement) const noexcept final; + simdjson_warn_unused uint8_t* parse_wobbly_string(const uint8_t* src, uint8_t* dst) const noexcept final; + inline simdjson_warn_unused error_code set_capacity(size_t capacity) noexcept final; + inline simdjson_warn_unused error_code set_max_depth(size_t max_depth) noexcept final; + private: + simdjson_inline simdjson_warn_unused error_code set_capacity_stage1(size_t capacity); + + }; + + } // namespace westmere +} // namespace simdjson + +namespace simdjson { + namespace westmere { + + inline dom_parser_implementation::dom_parser_implementation() noexcept = default; + inline dom_parser_implementation::dom_parser_implementation(dom_parser_implementation&& other) noexcept = default; + inline dom_parser_implementation& dom_parser_implementation::operator=(dom_parser_implementation&& other) noexcept = default; + + // Leaving these here so they can be inlined if so desired + inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(size_t capacity) noexcept { + if (capacity > SIMDJSON_MAXSIZE_BYTES) { return CAPACITY; } + // Stage 1 index output + size_t max_structures = SIMDJSON_ROUNDUP_N(capacity, 64) + 2 + 7; + structural_indexes.reset(new (std::nothrow) uint32_t[max_structures]); + if (!structural_indexes) { _capacity = 0; return MEMALLOC; } + structural_indexes[0] = 0; + n_structural_indexes = 0; + + _capacity = capacity; + return SUCCESS; + } + + inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { + // Stage 2 stacks + open_containers.reset(new (std::nothrow) open_container[max_depth]); + is_array.reset(new (std::nothrow) bool[max_depth]); + if (!is_array || !open_containers) { _max_depth = 0; return MEMALLOC; } + + _max_depth = max_depth; + return SUCCESS; + } + + } // namespace westmere +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_DOM_PARSER_IMPLEMENTATION_H +/* end file simdjson/generic/dom_parser_implementation.h for westmere */ +/* including simdjson/generic/implementation_simdjson_result_base.h for westmere: #include "simdjson/generic/implementation_simdjson_result_base.h" */ +/* begin file simdjson/generic/implementation_simdjson_result_base.h for westmere */ +#ifndef SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace westmere { + + // This is a near copy of include/error.h's implementation_simdjson_result_base, except it doesn't use std::pair + // so we can avoid inlining errors + // TODO reconcile these! + /** + * The result of a simdjson operation that could fail. + * + * Gives the option of reading error codes, or throwing an exception by casting to the desired result. + * + * This is a base class for implementations that want to add functions to the result type for + * chaining. + * + * Override like: + * + * struct simdjson_result : public internal::implementation_simdjson_result_base { + * simdjson_result() noexcept : internal::implementation_simdjson_result_base() {} + * simdjson_result(error_code error) noexcept : internal::implementation_simdjson_result_base(error) {} + * simdjson_result(T &&value) noexcept : internal::implementation_simdjson_result_base(std::forward(value)) {} + * simdjson_result(T &&value, error_code error) noexcept : internal::implementation_simdjson_result_base(value, error) {} + * // Your extra methods here + * } + * + * Then any method returning simdjson_result will be chainable with your methods. + */ + template + struct implementation_simdjson_result_base { + + /** + * Create a new empty result with error = UNINITIALIZED. + */ + simdjson_inline implementation_simdjson_result_base() noexcept = default; + + /** + * Create a new error result. + */ + simdjson_inline implementation_simdjson_result_base(error_code error) noexcept; + + /** + * Create a new successful result. + */ + simdjson_inline implementation_simdjson_result_base(T&& value) noexcept; + + /** + * Create a new result with both things (use if you don't want to branch when creating the result). + */ + simdjson_inline implementation_simdjson_result_base(T&& value, error_code error) noexcept; + + /** + * Move the value and the error to the provided variables. + * + * @param value The variable to assign the value to. May not be set if there is an error. + * @param error The variable to assign the error to. Set to SUCCESS if there is no error. + */ + simdjson_inline void tie(T& value, error_code& error) && noexcept; + + /** + * Move the value to the provided variable. + * + * @param value The variable to assign the value to. May not be set if there is an error. + */ + simdjson_inline error_code get(T& value) && noexcept; + + /** + * The error. + */ + simdjson_inline error_code error() const noexcept; + +#if SIMDJSON_EXCEPTIONS + + /** + * Get the result value. + * + * @throw simdjson_error if there was an error. + */ + simdjson_inline T& value() & noexcept(false); + + /** + * Take the result value (move it). + * + * @throw simdjson_error if there was an error. + */ + simdjson_inline T&& value() && noexcept(false); + + /** + * Take the result value (move it). + * + * @throw simdjson_error if there was an error. + */ + simdjson_inline T&& take_value() && noexcept(false); + + /** + * Cast to the value (will throw on error). + * + * @throw simdjson_error if there was an error. + */ + simdjson_inline operator T && () && noexcept(false); + + +#endif // SIMDJSON_EXCEPTIONS + + /** + * Get the result value. This function is safe if and only + * the error() method returns a value that evaluates to false. + */ + simdjson_inline const T& value_unsafe() const& noexcept; + /** + * Get the result value. This function is safe if and only + * the error() method returns a value that evaluates to false. + */ + simdjson_inline T& value_unsafe() & noexcept; + /** + * Take the result value (move it). This function is safe if and only + * the error() method returns a value that evaluates to false. + */ + simdjson_inline T&& value_unsafe() && noexcept; + protected: + /** users should never directly access first and second. **/ + T first{}; /** Users should never directly access 'first'. **/ + error_code second{ UNINITIALIZED }; /** Users should never directly access 'second'. **/ + }; // struct implementation_simdjson_result_base + + } // namespace westmere +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_H +/* end file simdjson/generic/implementation_simdjson_result_base.h for westmere */ +/* including simdjson/generic/numberparsing.h for westmere: #include "simdjson/generic/numberparsing.h" */ +/* begin file simdjson/generic/numberparsing.h for westmere */ +#ifndef SIMDJSON_GENERIC_NUMBERPARSING_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_NUMBERPARSING_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/jsoncharutils.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +#include +#include +#include + +namespace simdjson { + namespace westmere { + namespace numberparsing { + +#ifdef JSON_TEST_NUMBERS +#define INVALID_NUMBER(SRC) (found_invalid_number((SRC)), NUMBER_ERROR) +#define WRITE_INTEGER(VALUE, SRC, WRITER) (found_integer((VALUE), (SRC)), (WRITER).append_s64((VALUE))) +#define WRITE_UNSIGNED(VALUE, SRC, WRITER) (found_unsigned_integer((VALUE), (SRC)), (WRITER).append_u64((VALUE))) +#define WRITE_DOUBLE(VALUE, SRC, WRITER) (found_float((VALUE), (SRC)), (WRITER).append_double((VALUE))) +#define BIGINT_NUMBER(SRC) (found_invalid_number((SRC)), BIGINT_ERROR) +#else +#define INVALID_NUMBER(SRC) (NUMBER_ERROR) +#define WRITE_INTEGER(VALUE, SRC, WRITER) (WRITER).append_s64((VALUE)) +#define WRITE_UNSIGNED(VALUE, SRC, WRITER) (WRITER).append_u64((VALUE)) +#define WRITE_DOUBLE(VALUE, SRC, WRITER) (WRITER).append_double((VALUE)) +#define BIGINT_NUMBER(SRC) (BIGINT_ERROR) +#endif + + namespace { + + // Convert a mantissa, an exponent and a sign bit into an ieee64 double. + // The real_exponent needs to be in [0, 2046] (technically real_exponent = 2047 would be acceptable). + // The mantissa should be in [0,1<<53). The bit at index (1ULL << 52) while be zeroed. + simdjson_inline double to_double(uint64_t mantissa, uint64_t real_exponent, bool negative) { + double d; + mantissa &= ~(1ULL << 52); + mantissa |= real_exponent << 52; + mantissa |= ((static_cast(negative)) << 63); + std::memcpy(&d, &mantissa, sizeof(d)); + return d; + } + + // Attempts to compute i * 10^(power) exactly; and if "negative" is + // true, negate the result. + // This function will only work in some cases, when it does not work, success is + // set to false. This should work *most of the time* (like 99% of the time). + // We assume that power is in the [smallest_power, + // largest_power] interval: the caller is responsible for this check. + simdjson_inline bool compute_float_64(int64_t power, uint64_t i, bool negative, double& d) { + // we start with a fast path + // It was described in + // Clinger WD. How to read floating point numbers accurately. + // ACM SIGPLAN Notices. 1990 +#ifndef FLT_EVAL_METHOD +#error "FLT_EVAL_METHOD should be defined, please include cfloat." +#endif +#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0) + // We cannot be certain that x/y is rounded to nearest. + if (0 <= power && power <= 22 && i <= 9007199254740991) +#else + if (-22 <= power && power <= 22 && i <= 9007199254740991) +#endif + { + // convert the integer into a double. This is lossless since + // 0 <= i <= 2^53 - 1. + d = double(i); + // + // The general idea is as follows. + // If 0 <= s < 2^53 and if 10^0 <= p <= 10^22 then + // 1) Both s and p can be represented exactly as 64-bit floating-point + // values + // (binary64). + // 2) Because s and p can be represented exactly as floating-point values, + // then s * p + // and s / p will produce correctly rounded values. + // + if (power < 0) { + d = d / simdjson::internal::power_of_ten[-power]; + } + else { + d = d * simdjson::internal::power_of_ten[power]; + } + if (negative) { + d = -d; + } + return true; + } + // When 22 < power && power < 22 + 16, we could + // hope for another, secondary fast path. It was + // described by David M. Gay in "Correctly rounded + // binary-decimal and decimal-binary conversions." (1990) + // If you need to compute i * 10^(22 + x) for x < 16, + // first compute i * 10^x, if you know that result is exact + // (e.g., when i * 10^x < 2^53), + // then you can still proceed and do (i * 10^x) * 10^22. + // Is this worth your time? + // You need 22 < power *and* power < 22 + 16 *and* (i * 10^(x-22) < 2^53) + // for this second fast path to work. + // If you you have 22 < power *and* power < 22 + 16, and then you + // optimistically compute "i * 10^(x-22)", there is still a chance that you + // have wasted your time if i * 10^(x-22) >= 2^53. It makes the use cases of + // this optimization maybe less common than we would like. Source: + // http://www.exploringbinary.com/fast-path-decimal-to-floating-point-conversion/ + // also used in RapidJSON: https://rapidjson.org/strtod_8h_source.html + + // The fast path has now failed, so we are failing back on the slower path. + + // In the slow path, we need to adjust i so that it is > 1<<63 which is always + // possible, except if i == 0, so we handle i == 0 separately. + if (i == 0) { + d = negative ? -0.0 : 0.0; + return true; + } + + + // The exponent is 1024 + 63 + power + // + floor(log(5**power)/log(2)). + // The 1024 comes from the ieee64 standard. + // The 63 comes from the fact that we use a 64-bit word. + // + // Computing floor(log(5**power)/log(2)) could be + // slow. Instead we use a fast function. + // + // For power in (-400,350), we have that + // (((152170 + 65536) * power ) >> 16); + // is equal to + // floor(log(5**power)/log(2)) + power when power >= 0 + // and it is equal to + // ceil(log(5**-power)/log(2)) + power when power < 0 + // + // The 65536 is (1<<16) and corresponds to + // (65536 * power) >> 16 ---> power + // + // ((152170 * power ) >> 16) is equal to + // floor(log(5**power)/log(2)) + // + // Note that this is not magic: 152170/(1<<16) is + // approximatively equal to log(5)/log(2). + // The 1<<16 value is a power of two; we could use a + // larger power of 2 if we wanted to. + // + int64_t exponent = (((152170 + 65536) * power) >> 16) + 1024 + 63; + + + // We want the most significant bit of i to be 1. Shift if needed. + int lz = leading_zeroes(i); + i <<= lz; + + + // We are going to need to do some 64-bit arithmetic to get a precise product. + // We use a table lookup approach. + // It is safe because + // power >= smallest_power + // and power <= largest_power + // We recover the mantissa of the power, it has a leading 1. It is always + // rounded down. + // + // We want the most significant 64 bits of the product. We know + // this will be non-zero because the most significant bit of i is + // 1. + const uint32_t index = 2 * uint32_t(power - simdjson::internal::smallest_power); + // Optimization: It may be that materializing the index as a variable might confuse some compilers and prevent effective complex-addressing loads. (Done for code clarity.) + // + // The full_multiplication function computes the 128-bit product of two 64-bit words + // with a returned value of type value128 with a "low component" corresponding to the + // 64-bit least significant bits of the product and with a "high component" corresponding + // to the 64-bit most significant bits of the product. + simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index]); + // Both i and power_of_five_128[index] have their most significant bit set to 1 which + // implies that the either the most or the second most significant bit of the product + // is 1. We pack values in this manner for efficiency reasons: it maximizes the use + // we make of the product. It also makes it easy to reason about the product: there + // is 0 or 1 leading zero in the product. + + // Unless the least significant 9 bits of the high (64-bit) part of the full + // product are all 1s, then we know that the most significant 55 bits are + // exact and no further work is needed. Having 55 bits is necessary because + // we need 53 bits for the mantissa but we have to have one rounding bit and + // we can waste a bit if the most significant bit of the product is zero. + if ((firstproduct.high & 0x1FF) == 0x1FF) { + // We want to compute i * 5^q, but only care about the top 55 bits at most. + // Consider the scenario where q>=0. Then 5^q may not fit in 64-bits. Doing + // the full computation is wasteful. So we do what is called a "truncated + // multiplication". + // We take the most significant 64-bits, and we put them in + // power_of_five_128[index]. Usually, that's good enough to approximate i * 5^q + // to the desired approximation using one multiplication. Sometimes it does not suffice. + // Then we store the next most significant 64 bits in power_of_five_128[index + 1], and + // then we get a better approximation to i * 5^q. + // + // That's for when q>=0. The logic for q<0 is somewhat similar but it is somewhat + // more complicated. + // + // There is an extra layer of complexity in that we need more than 55 bits of + // accuracy in the round-to-even scenario. + // + // The full_multiplication function computes the 128-bit product of two 64-bit words + // with a returned value of type value128 with a "low component" corresponding to the + // 64-bit least significant bits of the product and with a "high component" corresponding + // to the 64-bit most significant bits of the product. + simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index + 1]); + firstproduct.low += secondproduct.high; + if (secondproduct.high > firstproduct.low) { firstproduct.high++; } + // As it has been proven by Noble Mushtak and Daniel Lemire in "Fast Number Parsing Without + // Fallback" (https://arxiv.org/abs/2212.06644), at this point we are sure that the product + // is sufficiently accurate, and more computation is not needed. + } + uint64_t lower = firstproduct.low; + uint64_t upper = firstproduct.high; + // The final mantissa should be 53 bits with a leading 1. + // We shift it so that it occupies 54 bits with a leading 1. + /////// + uint64_t upperbit = upper >> 63; + uint64_t mantissa = upper >> (upperbit + 9); + lz += int(1 ^ upperbit); + + // Here we have mantissa < (1<<54). + int64_t real_exponent = exponent - lz; + if (simdjson_unlikely(real_exponent <= 0)) { // we have a subnormal? + // Here have that real_exponent <= 0 so -real_exponent >= 0 + if (-real_exponent + 1 >= 64) { // if we have more than 64 bits below the minimum exponent, you have a zero for sure. + d = negative ? -0.0 : 0.0; + return true; + } + // next line is safe because -real_exponent + 1 < 0 + mantissa >>= -real_exponent + 1; + // Thankfully, we can't have both "round-to-even" and subnormals because + // "round-to-even" only occurs for powers close to 0. + mantissa += (mantissa & 1); // round up + mantissa >>= 1; + // There is a weird scenario where we don't have a subnormal but just. + // Suppose we start with 2.2250738585072013e-308, we end up + // with 0x3fffffffffffff x 2^-1023-53 which is technically subnormal + // whereas 0x40000000000000 x 2^-1023-53 is normal. Now, we need to round + // up 0x3fffffffffffff x 2^-1023-53 and once we do, we are no longer + // subnormal, but we can only know this after rounding. + // So we only declare a subnormal if we are smaller than the threshold. + real_exponent = (mantissa < (uint64_t(1) << 52)) ? 0 : 1; + d = to_double(mantissa, real_exponent, negative); + return true; + } + // We have to round to even. The "to even" part + // is only a problem when we are right in between two floats + // which we guard against. + // If we have lots of trailing zeros, we may fall right between two + // floating-point values. + // + // The round-to-even cases take the form of a number 2m+1 which is in (2^53,2^54] + // times a power of two. That is, it is right between a number with binary significand + // m and another number with binary significand m+1; and it must be the case + // that it cannot be represented by a float itself. + // + // We must have that w * 10 ^q == (2m+1) * 2^p for some power of two 2^p. + // Recall that 10^q = 5^q * 2^q. + // When q >= 0, we must have that (2m+1) is divible by 5^q, so 5^q <= 2^54. We have that + // 5^23 <= 2^54 and it is the last power of five to qualify, so q <= 23. + // When q<0, we have w >= (2m+1) x 5^{-q}. We must have that w<2^{64} so + // (2m+1) x 5^{-q} < 2^{64}. We have that 2m+1>2^{53}. Hence, we must have + // 2^{53} x 5^{-q} < 2^{64}. + // Hence we have 5^{-q} < 2^{11}$ or q>= -4. + // + // We require lower <= 1 and not lower == 0 because we could not prove that + // that lower == 0 is implied; but we could prove that lower <= 1 is a necessary and sufficient test. + if (simdjson_unlikely((lower <= 1) && (power >= -4) && (power <= 23) && ((mantissa & 3) == 1))) { + if ((mantissa << (upperbit + 64 - 53 - 2)) == upper) { + mantissa &= ~1; // flip it so that we do not round up + } + } + + mantissa += mantissa & 1; + mantissa >>= 1; + + // Here we have mantissa < (1<<53), unless there was an overflow + if (mantissa >= (1ULL << 53)) { + ////////// + // This will happen when parsing values such as 7.2057594037927933e+16 + //////// + mantissa = (1ULL << 52); + real_exponent++; + } + mantissa &= ~(1ULL << 52); + // we have to check that real_exponent is in range, otherwise we bail out + if (simdjson_unlikely(real_exponent > 2046)) { + // We have an infinite value!!! We could actually throw an error here if we could. + return false; + } + d = to_double(mantissa, real_exponent, negative); + return true; + } + + // We call a fallback floating-point parser that might be slow. Note + // it will accept JSON numbers, but the JSON spec. is more restrictive so + // before you call parse_float_fallback, you need to have validated the input + // string with the JSON grammar. + // It will return an error (false) if the parsed number is infinite. + // The string parsing itself always succeeds. We know that there is at least + // one digit. + static bool parse_float_fallback(const uint8_t* ptr, double* outDouble) { + *outDouble = simdjson::internal::from_chars(reinterpret_cast(ptr)); + // We do not accept infinite values. + + // Detecting finite values in a portable manner is ridiculously hard, ideally + // we would want to do: + // return !std::isfinite(*outDouble); + // but that mysteriously fails under legacy/old libc++ libraries, see + // https://github.com/simdjson/simdjson/issues/1286 + // + // Therefore, fall back to this solution (the extra parens are there + // to handle that max may be a macro on windows). + return !(*outDouble > (std::numeric_limits::max)() || *outDouble < std::numeric_limits::lowest()); + } + + static bool parse_float_fallback(const uint8_t* ptr, const uint8_t* end_ptr, double* outDouble) { + *outDouble = simdjson::internal::from_chars(reinterpret_cast(ptr), reinterpret_cast(end_ptr)); + // We do not accept infinite values. + + // Detecting finite values in a portable manner is ridiculously hard, ideally + // we would want to do: + // return !std::isfinite(*outDouble); + // but that mysteriously fails under legacy/old libc++ libraries, see + // https://github.com/simdjson/simdjson/issues/1286 + // + // Therefore, fall back to this solution (the extra parens are there + // to handle that max may be a macro on windows). + return !(*outDouble > (std::numeric_limits::max)() || *outDouble < std::numeric_limits::lowest()); + } + + // check quickly whether the next 8 chars are made of digits + // at a glance, it looks better than Mula's + // http://0x80.pl/articles/swar-digits-validate.html + simdjson_inline bool is_made_of_eight_digits_fast(const uint8_t* chars) { + uint64_t val; + // this can read up to 7 bytes beyond the buffer size, but we require + // SIMDJSON_PADDING of padding + static_assert(7 <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be bigger than 7"); + std::memcpy(&val, chars, 8); + // a branchy method might be faster: + // return (( val & 0xF0F0F0F0F0F0F0F0 ) == 0x3030303030303030) + // && (( (val + 0x0606060606060606) & 0xF0F0F0F0F0F0F0F0 ) == + // 0x3030303030303030); + return (((val & 0xF0F0F0F0F0F0F0F0) | + (((val + 0x0606060606060606) & 0xF0F0F0F0F0F0F0F0) >> 4)) == + 0x3333333333333333); + } + + template + SIMDJSON_NO_SANITIZE_UNDEFINED // We deliberately allow overflow here and check later + simdjson_inline bool parse_digit(const uint8_t c, I& i) { + const uint8_t digit = static_cast(c - '0'); + if (digit > 9) { + return false; + } + // PERF NOTE: multiplication by 10 is cheaper than arbitrary integer multiplication + i = 10 * i + digit; // might overflow, we will handle the overflow later + return true; + } + + simdjson_inline bool is_digit(const uint8_t c) { + return static_cast(c - '0') <= 9; + } + + simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t* const src, const uint8_t*& p, uint64_t& i, int64_t& exponent) { + // we continue with the fiction that we have an integer. If the + // floating point number is representable as x * 10^z for some integer + // z that fits in 53 bits, then we will be able to convert back the + // the integer into a float in a lossless manner. + const uint8_t* const first_after_period = p; + +#ifdef SIMDJSON_SWAR_NUMBER_PARSING +#if SIMDJSON_SWAR_NUMBER_PARSING + // this helps if we have lots of decimals! + // this turns out to be frequent enough. + if (is_made_of_eight_digits_fast(p)) { + i = i * 100000000 + parse_eight_digits_unrolled(p); + p += 8; + } +#endif // SIMDJSON_SWAR_NUMBER_PARSING +#endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING + // Unrolling the first digit makes a small difference on some implementations (e.g. westmere) + if (parse_digit(*p, i)) { ++p; } + while (parse_digit(*p, i)) { p++; } + exponent = first_after_period - p; + // Decimal without digits (123.) is illegal + if (exponent == 0) { + return INVALID_NUMBER(src); + } + return SUCCESS; + } + + simdjson_inline error_code parse_exponent(simdjson_unused const uint8_t* const src, const uint8_t*& p, int64_t& exponent) { + // Exp Sign: -123.456e[-]78 + bool neg_exp = ('-' == *p); + if (neg_exp || '+' == *p) { p++; } // Skip + as well + + // Exponent: -123.456e-[78] + auto start_exp = p; + int64_t exp_number = 0; + while (parse_digit(*p, exp_number)) { ++p; } + // It is possible for parse_digit to overflow. + // In particular, it could overflow to INT64_MIN, and we cannot do - INT64_MIN. + // Thus we *must* check for possible overflow before we negate exp_number. + + // Performance notes: it may seem like combining the two "simdjson_unlikely checks" below into + // a single simdjson_unlikely path would be faster. The reasoning is sound, but the compiler may + // not oblige and may, in fact, generate two distinct paths in any case. It might be + // possible to do uint64_t(p - start_exp - 1) >= 18 but it could end up trading off + // instructions for a simdjson_likely branch, an unconclusive gain. + + // If there were no digits, it's an error. + if (simdjson_unlikely(p == start_exp)) { + return INVALID_NUMBER(src); + } + // We have a valid positive exponent in exp_number at this point, except that + // it may have overflowed. + + // If there were more than 18 digits, we may have overflowed the integer. We have to do + // something!!!! + if (simdjson_unlikely(p > start_exp + 18)) { + // Skip leading zeroes: 1e000000000000000000001 is technically valid and does not overflow + while (*start_exp == '0') { start_exp++; } + // 19 digits could overflow int64_t and is kind of absurd anyway. We don't + // support exponents smaller than -999,999,999,999,999,999 and bigger + // than 999,999,999,999,999,999. + // We can truncate. + // Note that 999999999999999999 is assuredly too large. The maximal ieee64 value before + // infinity is ~1.8e308. The smallest subnormal is ~5e-324. So, actually, we could + // truncate at 324. + // Note that there is no reason to fail per se at this point in time. + // E.g., 0e999999999999999999999 is a fine number. + if (p > start_exp + 18) { exp_number = 999999999999999999; } + } + // At this point, we know that exp_number is a sane, positive, signed integer. + // It is <= 999,999,999,999,999,999. As long as 'exponent' is in + // [-8223372036854775808, 8223372036854775808], we won't overflow. Because 'exponent' + // is bounded in magnitude by the size of the JSON input, we are fine in this universe. + // To sum it up: the next line should never overflow. + exponent += (neg_exp ? -exp_number : exp_number); + return SUCCESS; + } + + simdjson_inline bool check_if_integer(const uint8_t* const src, size_t max_length) { + const uint8_t* const srcend = src + max_length; + bool negative = (*src == '-'); // we can always read at least one character after the '-' + const uint8_t* p = src + uint8_t(negative); + if (p == srcend) { return false; } + if (*p == '0') { + ++p; + if (p == srcend) { return true; } + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return false; } + return true; + } + while (p != srcend && is_digit(*p)) { ++p; } + if (p == srcend) { return true; } + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return false; } + return true; + } + + simdjson_inline size_t significant_digits(const uint8_t* start_digits, size_t digit_count) { + // It is possible that the integer had an overflow. + // We have to handle the case where we have 0.0000somenumber. + const uint8_t* start = start_digits; + while ((*start == '0') || (*start == '.')) { ++start; } + // we over-decrement by one when there is a '.' + return digit_count - size_t(start - start_digits); + } + + } // unnamed namespace + + /** @private */ + static error_code slow_float_parsing(simdjson_unused const uint8_t* src, double* answer) { + if (parse_float_fallback(src, answer)) { + return SUCCESS; + } + return INVALID_NUMBER(src); + } + + /** @private */ + template + simdjson_inline error_code write_float(const uint8_t* const src, bool negative, uint64_t i, const uint8_t* start_digits, size_t digit_count, int64_t exponent, W& writer) { + // If we frequently had to deal with long strings of digits, + // we could extend our code by using a 128-bit integer instead + // of a 64-bit integer. However, this is uncommon in practice. + // + // 9999999999999999999 < 2**64 so we can accommodate 19 digits. + // If we have a decimal separator, then digit_count - 1 is the number of digits, but we + // may not have a decimal separator! + if (simdjson_unlikely(digit_count > 19 && significant_digits(start_digits, digit_count) > 19)) { + // Ok, chances are good that we had an overflow! + // this is almost never going to get called!!! + // we start anew, going slowly!!! + // This will happen in the following examples: + // 10000000000000000000000000000000000000000000e+308 + // 3.1415926535897932384626433832795028841971693993751 + // + // NOTE: We do not pass a reference to the to slow_float_parsing. If we passed our writer + // reference to it, it would force it to be stored in memory, preventing the compiler from + // picking it apart and putting into registers. i.e. if we pass it as reference, + // it gets slow. + double d; + error_code error = slow_float_parsing(src, &d); + writer.append_double(d); + return error; + } + // NOTE: it's weird that the simdjson_unlikely() only wraps half the if, but it seems to get slower any other + // way we've tried: https://github.com/simdjson/simdjson/pull/990#discussion_r448497331 + // To future reader: we'd love if someone found a better way, or at least could explain this result! + if (simdjson_unlikely(exponent < simdjson::internal::smallest_power) || (exponent > simdjson::internal::largest_power)) { + // + // Important: smallest_power is such that it leads to a zero value. + // Observe that 18446744073709551615e-343 == 0, i.e. (2**64 - 1) e -343 is zero + // so something x 10^-343 goes to zero, but not so with something x 10^-342. + static_assert(simdjson::internal::smallest_power <= -342, "smallest_power is not small enough"); + // + if ((exponent < simdjson::internal::smallest_power) || (i == 0)) { + // E.g. Parse "-0.0e-999" into the same value as "-0.0". See https://en.wikipedia.org/wiki/Signed_zero + WRITE_DOUBLE(negative ? -0.0 : 0.0, src, writer); + return SUCCESS; + } + else { // (exponent > largest_power) and (i != 0) + // We have, for sure, an infinite value and simdjson refuses to parse infinite values. + return INVALID_NUMBER(src); + } + } + double d; + if (!compute_float_64(exponent, i, negative, d)) { + // we are almost never going to get here. + if (!parse_float_fallback(src, &d)) { return INVALID_NUMBER(src); } + } + WRITE_DOUBLE(d, src, writer); + return SUCCESS; + } + + // parse the number at src + // define JSON_TEST_NUMBERS for unit testing + // + // It is assumed that the number is followed by a structural ({,},],[) character + // or a white space character. If that is not the case (e.g., when the JSON + // document is made of a single number), then it is necessary to copy the + // content and append a space before calling this function. + // + // Our objective is accurate parsing (ULP of 0) at high speed. + template + simdjson_inline error_code parse_number(const uint8_t* const src, W& writer); + + // for performance analysis, it is sometimes useful to skip parsing +#ifdef SIMDJSON_SKIPNUMBERPARSING + + template + simdjson_inline error_code parse_number(const uint8_t* const, W& writer) { + writer.append_s64(0); // always write zero + return SUCCESS; // always succeeds + } + + simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t* const src) noexcept { return 0; } + simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t* const src) noexcept { return 0; } + simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t* const src) noexcept { return 0; } + simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t* const src) noexcept { return 0; } + simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t* const src) noexcept { return 0; } + simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t* const src) noexcept { return 0; } + simdjson_unused simdjson_inline bool is_negative(const uint8_t* src) noexcept { return false; } + simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t* src) noexcept { return false; } + simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t* src) noexcept { return number_type::signed_integer; } +#else + +// parse the number at src +// define JSON_TEST_NUMBERS for unit testing +// +// It is assumed that the number is followed by a structural ({,},],[) character +// or a white space character. If that is not the case (e.g., when the JSON +// document is made of a single number), then it is necessary to copy the +// content and append a space before calling this function. +// +// Our objective is accurate parsing (ULP of 0) at high speed. + template + simdjson_inline error_code parse_number(const uint8_t* const src, W& writer) { + + // + // Check for minus sign + // + bool negative = (*src == '-'); + const uint8_t* p = src + uint8_t(negative); + + // + // Parse the integer part. + // + // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare + const uint8_t* const start_digits = p; + uint64_t i = 0; + while (parse_digit(*p, i)) { p++; } + + // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. + // Optimization note: size_t is expected to be unsigned. + size_t digit_count = size_t(p - start_digits); + if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { return INVALID_NUMBER(src); } + + // + // Handle floats if there is a . or e (or both) + // + int64_t exponent = 0; + bool is_float = false; + if ('.' == *p) { + is_float = true; + ++p; + SIMDJSON_TRY(parse_decimal_after_separator(src, p, i, exponent)); + digit_count = int(p - start_digits); // used later to guard against overflows + } + if (('e' == *p) || ('E' == *p)) { + is_float = true; + ++p; + SIMDJSON_TRY(parse_exponent(src, p, exponent)); + } + if (is_float) { + const bool dirty_end = jsoncharutils::is_not_structural_or_whitespace(*p); + SIMDJSON_TRY(write_float(src, negative, i, start_digits, digit_count, exponent, writer)); + if (dirty_end) { return INVALID_NUMBER(src); } + return SUCCESS; + } + + // The longest negative 64-bit number is 19 digits. + // The longest positive 64-bit number is 20 digits. + // We do it this way so we don't trigger this branch unless we must. + size_t longest_digit_count = negative ? 19 : 20; + if (digit_count > longest_digit_count) { return BIGINT_NUMBER(src); } + if (digit_count == longest_digit_count) { + if (negative) { + // Anything negative above INT64_MAX+1 is invalid + if (i > uint64_t(INT64_MAX) + 1) { return BIGINT_NUMBER(src); } + WRITE_INTEGER(~i + 1, src, writer); + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return INVALID_NUMBER(src); } + return SUCCESS; + // Positive overflow check: + // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the + // biggest uint64_t. + // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX. + // If we got here, it's a 20 digit number starting with the digit "1". + // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller + // than 1,553,255,926,290,448,384. + // - That is smaller than the smallest possible 20-digit number the user could write: + // 10,000,000,000,000,000,000. + // - Therefore, if the number is positive and lower than that, it's overflow. + // - The value we are looking at is less than or equal to INT64_MAX. + // + } + else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INVALID_NUMBER(src); } + } + + // Write unsigned if it does not fit in a signed integer. + if (i > uint64_t(INT64_MAX)) { + WRITE_UNSIGNED(i, src, writer); + } + else { + WRITE_INTEGER(negative ? (~i + 1) : i, src, writer); + } + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return INVALID_NUMBER(src); } + return SUCCESS; + } + + // Inlineable functions + namespace { + + // This table can be used to characterize the final character of an integer + // string. For JSON structural character and allowable white space characters, + // we return SUCCESS. For 'e', '.' and 'E', we return INCORRECT_TYPE. Otherwise + // we return NUMBER_ERROR. + // Optimization note: we could easily reduce the size of the table by half (to 128) + // at the cost of an extra branch. + // Optimization note: we want the values to use at most 8 bits (not, e.g., 32 bits): + static_assert(error_code(uint8_t(NUMBER_ERROR)) == NUMBER_ERROR, "bad NUMBER_ERROR cast"); + static_assert(error_code(uint8_t(SUCCESS)) == SUCCESS, "bad NUMBER_ERROR cast"); + static_assert(error_code(uint8_t(INCORRECT_TYPE)) == INCORRECT_TYPE, "bad NUMBER_ERROR cast"); + + const uint8_t integer_string_finisher[256] = { + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, + SUCCESS, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, + NUMBER_ERROR, INCORRECT_TYPE, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, INCORRECT_TYPE, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, SUCCESS, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, INCORRECT_TYPE, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, + SUCCESS, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR }; + + // Parse any number from 0 to 18,446,744,073,709,551,615 + simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t* const src) noexcept { + const uint8_t* p = src; + // + // Parse the integer part. + // + // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare + const uint8_t* const start_digits = p; + uint64_t i = 0; + while (parse_digit(*p, i)) { p++; } + + // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. + // Optimization note: size_t is expected to be unsigned. + size_t digit_count = size_t(p - start_digits); + // The longest positive 64-bit number is 20 digits. + // We do it this way so we don't trigger this branch unless we must. + // Optimization note: the compiler can probably merge + // ((digit_count == 0) || (digit_count > 20)) + // into a single branch since digit_count is unsigned. + if ((digit_count == 0) || (digit_count > 20)) { return INCORRECT_TYPE; } + // Here digit_count > 0. + if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } + // We can do the following... + // if (!jsoncharutils::is_structural_or_whitespace(*p)) { + // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; + // } + // as a single table lookup: + if (integer_string_finisher[*p] != SUCCESS) { return error_code(integer_string_finisher[*p]); } + + if (digit_count == 20) { + // Positive overflow check: + // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the + // biggest uint64_t. + // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX. + // If we got here, it's a 20 digit number starting with the digit "1". + // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller + // than 1,553,255,926,290,448,384. + // - That is smaller than the smallest possible 20-digit number the user could write: + // 10,000,000,000,000,000,000. + // - Therefore, if the number is positive and lower than that, it's overflow. + // - The value we are looking at is less than or equal to INT64_MAX. + // + if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INCORRECT_TYPE; } + } + + return i; + } + + + // Parse any number from 0 to 18,446,744,073,709,551,615 + // Never read at src_end or beyond + simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t* const src, const uint8_t* const src_end) noexcept { + const uint8_t* p = src; + // + // Parse the integer part. + // + // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare + const uint8_t* const start_digits = p; + uint64_t i = 0; + while ((p != src_end) && parse_digit(*p, i)) { p++; } + + // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. + // Optimization note: size_t is expected to be unsigned. + size_t digit_count = size_t(p - start_digits); + // The longest positive 64-bit number is 20 digits. + // We do it this way so we don't trigger this branch unless we must. + // Optimization note: the compiler can probably merge + // ((digit_count == 0) || (digit_count > 20)) + // into a single branch since digit_count is unsigned. + if ((digit_count == 0) || (digit_count > 20)) { return INCORRECT_TYPE; } + // Here digit_count > 0. + if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } + // We can do the following... + // if (!jsoncharutils::is_structural_or_whitespace(*p)) { + // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; + // } + // as a single table lookup: + if ((p != src_end) && integer_string_finisher[*p] != SUCCESS) { return error_code(integer_string_finisher[*p]); } + + if (digit_count == 20) { + // Positive overflow check: + // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the + // biggest uint64_t. + // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX. + // If we got here, it's a 20 digit number starting with the digit "1". + // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller + // than 1,553,255,926,290,448,384. + // - That is smaller than the smallest possible 20-digit number the user could write: + // 10,000,000,000,000,000,000. + // - Therefore, if the number is positive and lower than that, it's overflow. + // - The value we are looking at is less than or equal to INT64_MAX. + // + if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INCORRECT_TYPE; } + } + + return i; + } + + // Parse any number from 0 to 18,446,744,073,709,551,615 + simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t* const src) noexcept { + const uint8_t* p = src + 1; + // + // Parse the integer part. + // + // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare + const uint8_t* const start_digits = p; + uint64_t i = 0; + while (parse_digit(*p, i)) { p++; } + + // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. + // Optimization note: size_t is expected to be unsigned. + size_t digit_count = size_t(p - start_digits); + // The longest positive 64-bit number is 20 digits. + // We do it this way so we don't trigger this branch unless we must. + // Optimization note: the compiler can probably merge + // ((digit_count == 0) || (digit_count > 20)) + // into a single branch since digit_count is unsigned. + if ((digit_count == 0) || (digit_count > 20)) { return INCORRECT_TYPE; } + // Here digit_count > 0. + if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } + // We can do the following... + // if (!jsoncharutils::is_structural_or_whitespace(*p)) { + // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; + // } + // as a single table lookup: + if (*p != '"') { return NUMBER_ERROR; } + + if (digit_count == 20) { + // Positive overflow check: + // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the + // biggest uint64_t. + // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX. + // If we got here, it's a 20 digit number starting with the digit "1". + // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller + // than 1,553,255,926,290,448,384. + // - That is smaller than the smallest possible 20-digit number the user could write: + // 10,000,000,000,000,000,000. + // - Therefore, if the number is positive and lower than that, it's overflow. + // - The value we are looking at is less than or equal to INT64_MAX. + // + // Note: we use src[1] and not src[0] because src[0] is the quote character in this + // instance. + if (src[1] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INCORRECT_TYPE; } + } + + return i; + } + + // Parse any number from -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 + simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t* src) noexcept { + // + // Check for minus sign + // + bool negative = (*src == '-'); + const uint8_t* p = src + uint8_t(negative); + + // + // Parse the integer part. + // + // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare + const uint8_t* const start_digits = p; + uint64_t i = 0; + while (parse_digit(*p, i)) { p++; } + + // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. + // Optimization note: size_t is expected to be unsigned. + size_t digit_count = size_t(p - start_digits); + // We go from + // -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 + // so we can never represent numbers that have more than 19 digits. + size_t longest_digit_count = 19; + // Optimization note: the compiler can probably merge + // ((digit_count == 0) || (digit_count > longest_digit_count)) + // into a single branch since digit_count is unsigned. + if ((digit_count == 0) || (digit_count > longest_digit_count)) { return INCORRECT_TYPE; } + // Here digit_count > 0. + if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } + // We can do the following... + // if (!jsoncharutils::is_structural_or_whitespace(*p)) { + // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; + // } + // as a single table lookup: + if (integer_string_finisher[*p] != SUCCESS) { return error_code(integer_string_finisher[*p]); } + // Negative numbers have can go down to - INT64_MAX - 1 whereas positive numbers are limited to INT64_MAX. + // Performance note: This check is only needed when digit_count == longest_digit_count but it is + // so cheap that we might as well always make it. + if (i > uint64_t(INT64_MAX) + uint64_t(negative)) { return INCORRECT_TYPE; } + return negative ? (~i + 1) : i; + } + + // Parse any number from -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 + // Never read at src_end or beyond + simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t* const src, const uint8_t* const src_end) noexcept { + // + // Check for minus sign + // + if (src == src_end) { return NUMBER_ERROR; } + bool negative = (*src == '-'); + const uint8_t* p = src + uint8_t(negative); + + // + // Parse the integer part. + // + // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare + const uint8_t* const start_digits = p; + uint64_t i = 0; + while ((p != src_end) && parse_digit(*p, i)) { p++; } + + // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. + // Optimization note: size_t is expected to be unsigned. + size_t digit_count = size_t(p - start_digits); + // We go from + // -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 + // so we can never represent numbers that have more than 19 digits. + size_t longest_digit_count = 19; + // Optimization note: the compiler can probably merge + // ((digit_count == 0) || (digit_count > longest_digit_count)) + // into a single branch since digit_count is unsigned. + if ((digit_count == 0) || (digit_count > longest_digit_count)) { return INCORRECT_TYPE; } + // Here digit_count > 0. + if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } + // We can do the following... + // if (!jsoncharutils::is_structural_or_whitespace(*p)) { + // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; + // } + // as a single table lookup: + if ((p != src_end) && integer_string_finisher[*p] != SUCCESS) { return error_code(integer_string_finisher[*p]); } + // Negative numbers have can go down to - INT64_MAX - 1 whereas positive numbers are limited to INT64_MAX. + // Performance note: This check is only needed when digit_count == longest_digit_count but it is + // so cheap that we might as well always make it. + if (i > uint64_t(INT64_MAX) + uint64_t(negative)) { return INCORRECT_TYPE; } + return negative ? (~i + 1) : i; + } + + // Parse any number from -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 + simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t* src) noexcept { + // + // Check for minus sign + // + bool negative = (*(src + 1) == '-'); + src += uint8_t(negative) + 1; + + // + // Parse the integer part. + // + // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare + const uint8_t* const start_digits = src; + uint64_t i = 0; + while (parse_digit(*src, i)) { src++; } + + // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. + // Optimization note: size_t is expected to be unsigned. + size_t digit_count = size_t(src - start_digits); + // We go from + // -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 + // so we can never represent numbers that have more than 19 digits. + size_t longest_digit_count = 19; + // Optimization note: the compiler can probably merge + // ((digit_count == 0) || (digit_count > longest_digit_count)) + // into a single branch since digit_count is unsigned. + if ((digit_count == 0) || (digit_count > longest_digit_count)) { return INCORRECT_TYPE; } + // Here digit_count > 0. + if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } + // We can do the following... + // if (!jsoncharutils::is_structural_or_whitespace(*src)) { + // return (*src == '.' || *src == 'e' || *src == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; + // } + // as a single table lookup: + if (*src != '"') { return NUMBER_ERROR; } + // Negative numbers have can go down to - INT64_MAX - 1 whereas positive numbers are limited to INT64_MAX. + // Performance note: This check is only needed when digit_count == longest_digit_count but it is + // so cheap that we might as well always make it. + if (i > uint64_t(INT64_MAX) + uint64_t(negative)) { return INCORRECT_TYPE; } + return negative ? (~i + 1) : i; + } + + simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t* src) noexcept { + // + // Check for minus sign + // + bool negative = (*src == '-'); + src += uint8_t(negative); + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t* p = src; + p += parse_digit(*p, i); + bool leading_zero = (i == 0); + while (parse_digit(*p, i)) { p++; } + // no integer digits, or 0123 (zero must be solo) + if (p == src) { return INCORRECT_TYPE; } + if ((leading_zero && p != src + 1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely(*p == '.')) { + p++; + const uint8_t* start_decimal_digits = p; + if (!parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits + p++; + while (parse_digit(*p, i)) { p++; } + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p - src - 1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t* start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = p - start_digits > 19; + } + } + else { + overflow = p - src > 19; + } + + // + // Parse the exponent + // + if (*p == 'e' || *p == 'E') { + p++; + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t* start_exp_digits = p; + while (parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p - start_exp_digits == 0 || p - start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0 - exp : exp; + } + + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + double d; + if (simdjson_likely(!overflow)) { + if (compute_float_64(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), &d)) { + return NUMBER_ERROR; + } + return d; + } + + simdjson_unused simdjson_inline bool is_negative(const uint8_t* src) noexcept { + return (*src == '-'); + } + + simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t* src) noexcept { + bool negative = (*src == '-'); + src += uint8_t(negative); + const uint8_t* p = src; + while (static_cast(*p - '0') <= 9) { p++; } + if (p == src) { return NUMBER_ERROR; } + if (jsoncharutils::is_structural_or_whitespace(*p)) { return true; } + return false; + } + + simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t* src) noexcept { + bool negative = (*src == '-'); + src += uint8_t(negative); + const uint8_t* p = src; + while (static_cast(*p - '0') <= 9) { p++; } + size_t digit_count = size_t(p - src); + if (p == src) { return NUMBER_ERROR; } + if (jsoncharutils::is_structural_or_whitespace(*p)) { + static const uint8_t* smaller_big_integer = reinterpret_cast("9223372036854775808"); + // We have an integer. + if (simdjson_unlikely(digit_count > 20)) { + return number_type::big_integer; + } + // If the number is negative and valid, it must be a signed integer. + if (negative) { + if (simdjson_unlikely(digit_count > 19)) return number_type::big_integer; + if (simdjson_unlikely(digit_count == 19 && memcmp(src, smaller_big_integer, 19) > 0)) { + return number_type::big_integer; + } + return number_type::signed_integer; + } + // Let us check if we have a big integer (>=2**64). + static const uint8_t* two_to_sixtyfour = reinterpret_cast("18446744073709551616"); + if ((digit_count > 20) || (digit_count == 20 && memcmp(src, two_to_sixtyfour, 20) >= 0)) { + return number_type::big_integer; + } + // The number is positive and smaller than 18446744073709551616 (or 2**64). + // We want values larger or equal to 9223372036854775808 to be unsigned + // integers, and the other values to be signed integers. + if ((digit_count == 20) || (digit_count >= 19 && memcmp(src, smaller_big_integer, 19) >= 0)) { + return number_type::unsigned_integer; + } + return number_type::signed_integer; + } + // Hopefully, we have 'e' or 'E' or '.'. + return number_type::floating_point_number; + } + + // Never read at src_end or beyond + simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t* src, const uint8_t* const src_end) noexcept { + if (src == src_end) { return NUMBER_ERROR; } + // + // Check for minus sign + // + bool negative = (*src == '-'); + src += uint8_t(negative); + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t* p = src; + if (p == src_end) { return NUMBER_ERROR; } + p += parse_digit(*p, i); + bool leading_zero = (i == 0); + while ((p != src_end) && parse_digit(*p, i)) { p++; } + // no integer digits, or 0123 (zero must be solo) + if (p == src) { return INCORRECT_TYPE; } + if ((leading_zero && p != src + 1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely((p != src_end) && (*p == '.'))) { + p++; + const uint8_t* start_decimal_digits = p; + if ((p == src_end) || !parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits + p++; + while ((p != src_end) && parse_digit(*p, i)) { p++; } + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p - src - 1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t* start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = start_digits - src > 19; + } + } + else { + overflow = p - src > 19; + } + + // + // Parse the exponent + // + if ((p != src_end) && (*p == 'e' || *p == 'E')) { + p++; + if (p == src_end) { return NUMBER_ERROR; } + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t* start_exp_digits = p; + while ((p != src_end) && parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p - start_exp_digits == 0 || p - start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0 - exp : exp; + } + + if ((p != src_end) && jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + double d; + if (simdjson_likely(!overflow)) { + if (compute_float_64(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), src_end, &d)) { + return NUMBER_ERROR; + } + return d; + } + + simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t* src) noexcept { + // + // Check for minus sign + // + bool negative = (*(src + 1) == '-'); + src += uint8_t(negative) + 1; + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t* p = src; + p += parse_digit(*p, i); + bool leading_zero = (i == 0); + while (parse_digit(*p, i)) { p++; } + // no integer digits, or 0123 (zero must be solo) + if (p == src) { return INCORRECT_TYPE; } + if ((leading_zero && p != src + 1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely(*p == '.')) { + p++; + const uint8_t* start_decimal_digits = p; + if (!parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits + p++; + while (parse_digit(*p, i)) { p++; } + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p - src - 1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t* start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = p - start_digits > 19; + } + } + else { + overflow = p - src > 19; + } + + // + // Parse the exponent + // + if (*p == 'e' || *p == 'E') { + p++; + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t* start_exp_digits = p; + while (parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p - start_exp_digits == 0 || p - start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0 - exp : exp; + } + + if (*p != '"') { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + double d; + if (simdjson_likely(!overflow)) { + if (compute_float_64(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), &d)) { + return NUMBER_ERROR; + } + return d; + } + + } // unnamed namespace +#endif // SIMDJSON_SKIPNUMBERPARSING + + } // namespace numberparsing + + inline std::ostream& operator<<(std::ostream& out, number_type type) noexcept { + switch (type) { + case number_type::signed_integer: out << "integer in [-9223372036854775808,9223372036854775808)"; break; + case number_type::unsigned_integer: out << "unsigned integer in [9223372036854775808,18446744073709551616)"; break; + case number_type::floating_point_number: out << "floating-point number (binary64)"; break; + case number_type::big_integer: out << "big integer"; break; + default: SIMDJSON_UNREACHABLE(); + } + return out; + } + + } // namespace westmere +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_NUMBERPARSING_H +/* end file simdjson/generic/numberparsing.h for westmere */ + +/* including simdjson/generic/implementation_simdjson_result_base-inl.h for westmere: #include "simdjson/generic/implementation_simdjson_result_base-inl.h" */ +/* begin file simdjson/generic/implementation_simdjson_result_base-inl.h for westmere */ +#ifndef SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_INL_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_INL_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/implementation_simdjson_result_base.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace westmere { + + // + // internal::implementation_simdjson_result_base inline implementation + // + + template + simdjson_inline void implementation_simdjson_result_base::tie(T& value, error_code& error) && noexcept { + error = this->second; + if (!error) { + value = std::forward>(*this).first; + } + } + + template + simdjson_warn_unused simdjson_inline error_code implementation_simdjson_result_base::get(T& value) && noexcept { + error_code error; + std::forward>(*this).tie(value, error); + return error; + } + + template + simdjson_inline error_code implementation_simdjson_result_base::error() const noexcept { + return this->second; + } + +#if SIMDJSON_EXCEPTIONS + + template + simdjson_inline T& implementation_simdjson_result_base::value() & noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return this->first; + } + + template + simdjson_inline T&& implementation_simdjson_result_base::value() && noexcept(false) { + return std::forward>(*this).take_value(); + } + + template + simdjson_inline T&& implementation_simdjson_result_base::take_value() && noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return std::forward(this->first); + } + + template + simdjson_inline implementation_simdjson_result_base::operator T && () && noexcept(false) { + return std::forward>(*this).take_value(); + } + +#endif // SIMDJSON_EXCEPTIONS + + template + simdjson_inline const T& implementation_simdjson_result_base::value_unsafe() const& noexcept { + return this->first; + } + + template + simdjson_inline T& implementation_simdjson_result_base::value_unsafe() & noexcept { + return this->first; + } + + template + simdjson_inline T&& implementation_simdjson_result_base::value_unsafe() && noexcept { + return std::forward(this->first); + } + + template + simdjson_inline implementation_simdjson_result_base::implementation_simdjson_result_base(T&& value, error_code error) noexcept + : first{ std::forward(value) }, second{ error } {} + template + simdjson_inline implementation_simdjson_result_base::implementation_simdjson_result_base(error_code error) noexcept + : implementation_simdjson_result_base(T{}, error) {} + template + simdjson_inline implementation_simdjson_result_base::implementation_simdjson_result_base(T&& value) noexcept + : implementation_simdjson_result_base(std::forward(value), SUCCESS) {} + + } // namespace westmere +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_INL_H +/* end file simdjson/generic/implementation_simdjson_result_base-inl.h for westmere */ +/* end file simdjson/generic/amalgamated.h for westmere */ +/* including simdjson/westmere/end.h: #include "simdjson/westmere/end.h" */ +/* begin file simdjson/westmere/end.h */ +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/westmere/base.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +#if !SIMDJSON_CAN_ALWAYS_RUN_WESTMERE +SIMDJSON_UNTARGET_REGION +#endif + +/* undefining SIMDJSON_IMPLEMENTATION from "westmere" */ +#undef SIMDJSON_IMPLEMENTATION +/* end file simdjson/westmere/end.h */ + +#endif // SIMDJSON_WESTMERE_H +/* end file simdjson/westmere.h */ +/* including simdjson/westmere/implementation.h: #include */ +/* begin file simdjson/westmere/implementation.h */ +#ifndef SIMDJSON_WESTMERE_IMPLEMENTATION_H +#define SIMDJSON_WESTMERE_IMPLEMENTATION_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/westmere/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/implementation.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/instruction_set.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +// The constructor may be executed on any host, so we take care not to use SIMDJSON_TARGET_WESTMERE +namespace simdjson { + namespace westmere { + + /** + * @private + */ + class implementation final : public simdjson::implementation { + public: + simdjson_inline implementation() : simdjson::implementation("westmere", "Intel/AMD SSE4.2", internal::instruction_set::SSE42 | internal::instruction_set::PCLMULQDQ) {} + simdjson_warn_unused error_code create_dom_parser_implementation( + size_t capacity, + size_t max_length, + std::unique_ptr& dst + ) const noexcept final; + simdjson_warn_unused error_code minify(const uint8_t* buf, size_t len, uint8_t* dst, size_t& dst_len) const noexcept final; + simdjson_warn_unused bool validate_utf8(const char* buf, size_t len) const noexcept final; + }; + + } // namespace westmere +} // namespace simdjson + +#endif // SIMDJSON_WESTMERE_IMPLEMENTATION_H +/* end file simdjson/westmere/implementation.h */ + +/* including simdjson/westmere/begin.h: #include */ +/* begin file simdjson/westmere/begin.h */ +/* defining SIMDJSON_IMPLEMENTATION to "westmere" */ +#define SIMDJSON_IMPLEMENTATION westmere +/* including simdjson/westmere/base.h: #include "simdjson/westmere/base.h" */ +/* begin file simdjson/westmere/base.h */ +#ifndef SIMDJSON_WESTMERE_BASE_H +#define SIMDJSON_WESTMERE_BASE_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/base.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +// The constructor may be executed on any host, so we take care not to use SIMDJSON_TARGET_WESTMERE +namespace simdjson { + /** + * Implementation for Westmere (Intel SSE4.2). + */ + namespace westmere { + + class implementation; + + namespace { + namespace simd { + + template struct simd8; + template struct simd8x64; + + } // namespace simd + } // unnamed namespace + + } // namespace westmere +} // namespace simdjson + +#endif // SIMDJSON_WESTMERE_BASE_H +/* end file simdjson/westmere/base.h */ +/* including simdjson/westmere/intrinsics.h: #include "simdjson/westmere/intrinsics.h" */ +/* begin file simdjson/westmere/intrinsics.h */ +#ifndef SIMDJSON_WESTMERE_INTRINSICS_H +#define SIMDJSON_WESTMERE_INTRINSICS_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/westmere/base.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +#if SIMDJSON_VISUAL_STUDIO +// under clang within visual studio, this will include +#include // visual studio or clang +#else +#include // elsewhere +#endif // SIMDJSON_VISUAL_STUDIO + + +#if SIMDJSON_CLANG_VISUAL_STUDIO +/** + * You are not supposed, normally, to include these + * headers directly. Instead you should either include intrin.h + * or x86intrin.h. However, when compiling with clang + * under Windows (i.e., when _MSC_VER is set), these headers + * only get included *if* the corresponding features are detected + * from macros: + */ +#include // for _mm_alignr_epi8 +#include // for _mm_clmulepi64_si128 +#endif + +static_assert(sizeof(__m128i) <= simdjson::SIMDJSON_PADDING, "insufficient padding for westmere"); + +#endif // SIMDJSON_WESTMERE_INTRINSICS_H +/* end file simdjson/westmere/intrinsics.h */ + +#if !SIMDJSON_CAN_ALWAYS_RUN_WESTMERE +SIMDJSON_TARGET_REGION("sse4.2,pclmul,popcnt") +#endif + +/* including simdjson/westmere/bitmanipulation.h: #include "simdjson/westmere/bitmanipulation.h" */ +/* begin file simdjson/westmere/bitmanipulation.h */ +#ifndef SIMDJSON_WESTMERE_BITMANIPULATION_H +#define SIMDJSON_WESTMERE_BITMANIPULATION_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/westmere/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/westmere/intrinsics.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace westmere { + namespace { + + // We sometimes call trailing_zero on inputs that are zero, + // but the algorithms do not end up using the returned value. + // Sadly, sanitizers are not smart enough to figure it out. + SIMDJSON_NO_SANITIZE_UNDEFINED + // This function can be used safely even if not all bytes have been + // initialized. + // See issue https://github.com/simdjson/simdjson/issues/1965 + SIMDJSON_NO_SANITIZE_MEMORY + simdjson_inline int trailing_zeroes(uint64_t input_num) { +#if SIMDJSON_REGULAR_VISUAL_STUDIO + unsigned long ret; + // Search the mask data from least significant bit (LSB) + // to the most significant bit (MSB) for a set bit (1). + _BitScanForward64(&ret, input_num); + return (int)ret; +#else // SIMDJSON_REGULAR_VISUAL_STUDIO + return __builtin_ctzll(input_num); +#endif // SIMDJSON_REGULAR_VISUAL_STUDIO + } + + /* result might be undefined when input_num is zero */ + simdjson_inline uint64_t clear_lowest_bit(uint64_t input_num) { + return input_num & (input_num - 1); + } + + /* result might be undefined when input_num is zero */ + simdjson_inline int leading_zeroes(uint64_t input_num) { +#if SIMDJSON_REGULAR_VISUAL_STUDIO + unsigned long leading_zero = 0; + // Search the mask data from most significant bit (MSB) + // to least significant bit (LSB) for a set bit (1). + if (_BitScanReverse64(&leading_zero, input_num)) + return (int)(63 - leading_zero); + else + return 64; +#else + return __builtin_clzll(input_num); +#endif// SIMDJSON_REGULAR_VISUAL_STUDIO + } + +#if SIMDJSON_REGULAR_VISUAL_STUDIO + simdjson_inline unsigned __int64 count_ones(uint64_t input_num) { + // note: we do not support legacy 32-bit Windows in this kernel + return __popcnt64(input_num);// Visual Studio wants two underscores + } +#else + simdjson_inline long long int count_ones(uint64_t input_num) { + return _popcnt64(input_num); + } +#endif + + simdjson_inline bool add_overflow(uint64_t value1, uint64_t value2, + uint64_t* result) { +#if SIMDJSON_REGULAR_VISUAL_STUDIO + return _addcarry_u64(0, value1, value2, + reinterpret_cast(result)); +#else + return __builtin_uaddll_overflow(value1, value2, + reinterpret_cast(result)); +#endif + } + + } // unnamed namespace + } // namespace westmere +} // namespace simdjson + +#endif // SIMDJSON_WESTMERE_BITMANIPULATION_H +/* end file simdjson/westmere/bitmanipulation.h */ +/* including simdjson/westmere/bitmask.h: #include "simdjson/westmere/bitmask.h" */ +/* begin file simdjson/westmere/bitmask.h */ +#ifndef SIMDJSON_WESTMERE_BITMASK_H +#define SIMDJSON_WESTMERE_BITMASK_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/westmere/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/westmere/intrinsics.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace westmere { + namespace { + + // + // Perform a "cumulative bitwise xor," flipping bits each time a 1 is encountered. + // + // For example, prefix_xor(00100100) == 00011100 + // + simdjson_inline uint64_t prefix_xor(const uint64_t bitmask) { + // There should be no such thing with a processing supporting avx2 + // but not clmul. + __m128i all_ones = _mm_set1_epi8('\xFF'); + __m128i result = _mm_clmulepi64_si128(_mm_set_epi64x(0ULL, bitmask), all_ones, 0); + return _mm_cvtsi128_si64(result); + } + + } // unnamed namespace + } // namespace westmere +} // namespace simdjson + +#endif // SIMDJSON_WESTMERE_BITMASK_H +/* end file simdjson/westmere/bitmask.h */ +/* including simdjson/westmere/numberparsing_defs.h: #include "simdjson/westmere/numberparsing_defs.h" */ +/* begin file simdjson/westmere/numberparsing_defs.h */ +#ifndef SIMDJSON_WESTMERE_NUMBERPARSING_DEFS_H +#define SIMDJSON_WESTMERE_NUMBERPARSING_DEFS_H + +/* including simdjson/westmere/base.h: #include "simdjson/westmere/base.h" */ +/* begin file simdjson/westmere/base.h */ +#ifndef SIMDJSON_WESTMERE_BASE_H +#define SIMDJSON_WESTMERE_BASE_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/base.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +// The constructor may be executed on any host, so we take care not to use SIMDJSON_TARGET_WESTMERE +namespace simdjson { + /** + * Implementation for Westmere (Intel SSE4.2). + */ + namespace westmere { + + class implementation; + + namespace { + namespace simd { + + template struct simd8; + template struct simd8x64; + + } // namespace simd + } // unnamed namespace + + } // namespace westmere +} // namespace simdjson + +#endif // SIMDJSON_WESTMERE_BASE_H +/* end file simdjson/westmere/base.h */ +/* including simdjson/westmere/intrinsics.h: #include "simdjson/westmere/intrinsics.h" */ +/* begin file simdjson/westmere/intrinsics.h */ +#ifndef SIMDJSON_WESTMERE_INTRINSICS_H +#define SIMDJSON_WESTMERE_INTRINSICS_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/westmere/base.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +#if SIMDJSON_VISUAL_STUDIO +// under clang within visual studio, this will include +#include // visual studio or clang +#else +#include // elsewhere +#endif // SIMDJSON_VISUAL_STUDIO + + +#if SIMDJSON_CLANG_VISUAL_STUDIO +/** + * You are not supposed, normally, to include these + * headers directly. Instead you should either include intrin.h + * or x86intrin.h. However, when compiling with clang + * under Windows (i.e., when _MSC_VER is set), these headers + * only get included *if* the corresponding features are detected + * from macros: + */ +#include // for _mm_alignr_epi8 +#include // for _mm_clmulepi64_si128 +#endif + +static_assert(sizeof(__m128i) <= simdjson::SIMDJSON_PADDING, "insufficient padding for westmere"); + +#endif // SIMDJSON_WESTMERE_INTRINSICS_H +/* end file simdjson/westmere/intrinsics.h */ + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace westmere { + namespace numberparsing { + + /** @private */ + static simdjson_inline uint32_t parse_eight_digits_unrolled(const uint8_t* chars) { + // this actually computes *16* values so we are being wasteful. + const __m128i ascii0 = _mm_set1_epi8('0'); + const __m128i mul_1_10 = + _mm_setr_epi8(10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1); + const __m128i mul_1_100 = _mm_setr_epi16(100, 1, 100, 1, 100, 1, 100, 1); + const __m128i mul_1_10000 = + _mm_setr_epi16(10000, 1, 10000, 1, 10000, 1, 10000, 1); + const __m128i input = _mm_sub_epi8( + _mm_loadu_si128(reinterpret_cast(chars)), ascii0); + const __m128i t1 = _mm_maddubs_epi16(input, mul_1_10); + const __m128i t2 = _mm_madd_epi16(t1, mul_1_100); + const __m128i t3 = _mm_packus_epi32(t2, t2); + const __m128i t4 = _mm_madd_epi16(t3, mul_1_10000); + return _mm_cvtsi128_si32( + t4); // only captures the sum of the first 8 digits, drop the rest + } + + /** @private */ + simdjson_inline internal::value128 full_multiplication(uint64_t value1, uint64_t value2) { + internal::value128 answer; +#if SIMDJSON_REGULAR_VISUAL_STUDIO || SIMDJSON_IS_32BITS +#if SIMDJSON_IS_ARM64 + // ARM64 has native support for 64-bit multiplications, no need to emultate + answer.high = __umulh(value1, value2); + answer.low = value1 * value2; +#else + answer.low = _umul128(value1, value2, &answer.high); // _umul128 not available on ARM64 +#endif // SIMDJSON_IS_ARM64 +#else // SIMDJSON_REGULAR_VISUAL_STUDIO || SIMDJSON_IS_32BITS + __uint128_t r = (static_cast<__uint128_t>(value1)) * value2; + answer.low = uint64_t(r); + answer.high = uint64_t(r >> 64); +#endif + return answer; + } + + } // namespace numberparsing + } // namespace westmere +} // namespace simdjson + +#define SIMDJSON_SWAR_NUMBER_PARSING 1 + +#endif // SIMDJSON_WESTMERE_NUMBERPARSING_DEFS_H +/* end file simdjson/westmere/numberparsing_defs.h */ +/* including simdjson/westmere/simd.h: #include "simdjson/westmere/simd.h" */ +/* begin file simdjson/westmere/simd.h */ +#ifndef SIMDJSON_WESTMERE_SIMD_H +#define SIMDJSON_WESTMERE_SIMD_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/westmere/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/westmere/bitmanipulation.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/simdprune_tables.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace westmere { + namespace { + namespace simd { + + template + struct base { + __m128i value; + + // Zero constructor + simdjson_inline base() : value{ __m128i() } {} + + // Conversion from SIMD register + simdjson_inline base(const __m128i _value) : value(_value) {} + + // Conversion to SIMD register + simdjson_inline operator const __m128i& () const { return this->value; } + simdjson_inline operator __m128i& () { return this->value; } + + // Bit operations + simdjson_inline Child operator|(const Child other) const { return _mm_or_si128(*this, other); } + simdjson_inline Child operator&(const Child other) const { return _mm_and_si128(*this, other); } + simdjson_inline Child operator^(const Child other) const { return _mm_xor_si128(*this, other); } + simdjson_inline Child bit_andnot(const Child other) const { return _mm_andnot_si128(other, *this); } + simdjson_inline Child& operator|=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast | other; return *this_cast; } + simdjson_inline Child& operator&=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast & other; return *this_cast; } + simdjson_inline Child& operator^=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast ^ other; return *this_cast; } + }; + + template> + struct base8 : base> { + typedef uint16_t bitmask_t; + typedef uint32_t bitmask2_t; + + simdjson_inline base8() : base>() {} + simdjson_inline base8(const __m128i _value) : base>(_value) {} + + friend simdjson_inline Mask operator==(const simd8 lhs, const simd8 rhs) { return _mm_cmpeq_epi8(lhs, rhs); } + + static const int SIZE = sizeof(base>::value); + + template + simdjson_inline simd8 prev(const simd8 prev_chunk) const { + return _mm_alignr_epi8(*this, prev_chunk, 16 - N); + } + }; + + // SIMD byte mask type (returned by things like eq and gt) + template<> + struct simd8 : base8 { + static simdjson_inline simd8 splat(bool _value) { return _mm_set1_epi8(uint8_t(-(!!_value))); } + + simdjson_inline simd8() : base8() {} + simdjson_inline simd8(const __m128i _value) : base8(_value) {} + // Splat constructor + simdjson_inline simd8(bool _value) : base8(splat(_value)) {} + + simdjson_inline int to_bitmask() const { return _mm_movemask_epi8(*this); } + simdjson_inline bool any() const { return !_mm_testz_si128(*this, *this); } + simdjson_inline simd8 operator~() const { return *this ^ true; } + }; + + template + struct base8_numeric : base8 { + static simdjson_inline simd8 splat(T _value) { return _mm_set1_epi8(_value); } + static simdjson_inline simd8 zero() { return _mm_setzero_si128(); } + static simdjson_inline simd8 load(const T values[16]) { + return _mm_loadu_si128(reinterpret_cast(values)); + } + // Repeat 16 values as many times as necessary (usually for lookup tables) + static simdjson_inline simd8 repeat_16( + T v0, T v1, T v2, T v3, T v4, T v5, T v6, T v7, + T v8, T v9, T v10, T v11, T v12, T v13, T v14, T v15 + ) { + return simd8( + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15 + ); + } + + simdjson_inline base8_numeric() : base8() {} + simdjson_inline base8_numeric(const __m128i _value) : base8(_value) {} + + // Store to array + simdjson_inline void store(T dst[16]) const { return _mm_storeu_si128(reinterpret_cast<__m128i*>(dst), *this); } + + // Override to distinguish from bool version + simdjson_inline simd8 operator~() const { return *this ^ 0xFFu; } + + // Addition/subtraction are the same for signed and unsigned + simdjson_inline simd8 operator+(const simd8 other) const { return _mm_add_epi8(*this, other); } + simdjson_inline simd8 operator-(const simd8 other) const { return _mm_sub_epi8(*this, other); } + simdjson_inline simd8& operator+=(const simd8 other) { *this = *this + other; return *static_cast*>(this); } + simdjson_inline simd8& operator-=(const simd8 other) { *this = *this - other; return *static_cast*>(this); } + + // Perform a lookup assuming the value is between 0 and 16 (undefined behavior for out of range values) + template + simdjson_inline simd8 lookup_16(simd8 lookup_table) const { + return _mm_shuffle_epi8(lookup_table, *this); + } + + // Copies to 'output" all bytes corresponding to a 0 in the mask (interpreted as a bitset). + // Passing a 0 value for mask would be equivalent to writing out every byte to output. + // Only the first 16 - count_ones(mask) bytes of the result are significant but 16 bytes + // get written. + // Design consideration: it seems like a function with the + // signature simd8 compress(uint32_t mask) would be + // sensible, but the AVX ISA makes this kind of approach difficult. + template + simdjson_inline void compress(uint16_t mask, L* output) const { + using internal::thintable_epi8; + using internal::BitsSetTable256mul2; + using internal::pshufb_combine_table; + // this particular implementation was inspired by work done by @animetosho + // we do it in two steps, first 8 bytes and then second 8 bytes + uint8_t mask1 = uint8_t(mask); // least significant 8 bits + uint8_t mask2 = uint8_t(mask >> 8); // most significant 8 bits + // next line just loads the 64-bit values thintable_epi8[mask1] and + // thintable_epi8[mask2] into a 128-bit register, using only + // two instructions on most compilers. + __m128i shufmask = _mm_set_epi64x(thintable_epi8[mask2], thintable_epi8[mask1]); + // we increment by 0x08 the second half of the mask + shufmask = + _mm_add_epi8(shufmask, _mm_set_epi32(0x08080808, 0x08080808, 0, 0)); + // this is the version "nearly pruned" + __m128i pruned = _mm_shuffle_epi8(*this, shufmask); + // we still need to put the two halves together. + // we compute the popcount of the first half: + int pop1 = BitsSetTable256mul2[mask1]; + // then load the corresponding mask, what it does is to write + // only the first pop1 bytes from the first 8 bytes, and then + // it fills in with the bytes from the second 8 bytes + some filling + // at the end. + __m128i compactmask = + _mm_loadu_si128(reinterpret_cast(pshufb_combine_table + pop1 * 8)); + __m128i answer = _mm_shuffle_epi8(pruned, compactmask); + _mm_storeu_si128(reinterpret_cast<__m128i*>(output), answer); + } + + template + simdjson_inline simd8 lookup_16( + L replace0, L replace1, L replace2, L replace3, + L replace4, L replace5, L replace6, L replace7, + L replace8, L replace9, L replace10, L replace11, + L replace12, L replace13, L replace14, L replace15) const { + return lookup_16(simd8::repeat_16( + replace0, replace1, replace2, replace3, + replace4, replace5, replace6, replace7, + replace8, replace9, replace10, replace11, + replace12, replace13, replace14, replace15 + )); + } + }; + + // Signed bytes + template<> + struct simd8 : base8_numeric { + simdjson_inline simd8() : base8_numeric() {} + simdjson_inline simd8(const __m128i _value) : base8_numeric(_value) {} + // Splat constructor + simdjson_inline simd8(int8_t _value) : simd8(splat(_value)) {} + // Array constructor + simdjson_inline simd8(const int8_t* values) : simd8(load(values)) {} + // Member-by-member initialization + simdjson_inline simd8( + int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, + int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 + ) : simd8(_mm_setr_epi8( + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15 + )) {} + // Repeat 16 values as many times as necessary (usually for lookup tables) + simdjson_inline static simd8 repeat_16( + int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, + int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 + ) { + return simd8( + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15 + ); + } + + // Order-sensitive comparisons + simdjson_inline simd8 max_val(const simd8 other) const { return _mm_max_epi8(*this, other); } + simdjson_inline simd8 min_val(const simd8 other) const { return _mm_min_epi8(*this, other); } + simdjson_inline simd8 operator>(const simd8 other) const { return _mm_cmpgt_epi8(*this, other); } + simdjson_inline simd8 operator<(const simd8 other) const { return _mm_cmpgt_epi8(other, *this); } + }; + + // Unsigned bytes + template<> + struct simd8 : base8_numeric { + simdjson_inline simd8() : base8_numeric() {} + simdjson_inline simd8(const __m128i _value) : base8_numeric(_value) {} + // Splat constructor + simdjson_inline simd8(uint8_t _value) : simd8(splat(_value)) {} + // Array constructor + simdjson_inline simd8(const uint8_t* values) : simd8(load(values)) {} + // Member-by-member initialization + simdjson_inline simd8( + uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, + uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 + ) : simd8(_mm_setr_epi8( + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15 + )) {} + // Repeat 16 values as many times as necessary (usually for lookup tables) + simdjson_inline static simd8 repeat_16( + uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, + uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 + ) { + return simd8( + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15 + ); + } + + // Saturated math + simdjson_inline simd8 saturating_add(const simd8 other) const { return _mm_adds_epu8(*this, other); } + simdjson_inline simd8 saturating_sub(const simd8 other) const { return _mm_subs_epu8(*this, other); } + + // Order-specific operations + simdjson_inline simd8 max_val(const simd8 other) const { return _mm_max_epu8(*this, other); } + simdjson_inline simd8 min_val(const simd8 other) const { return _mm_min_epu8(*this, other); } + // Same as >, but only guarantees true is nonzero (< guarantees true = -1) + simdjson_inline simd8 gt_bits(const simd8 other) const { return this->saturating_sub(other); } + // Same as <, but only guarantees true is nonzero (< guarantees true = -1) + simdjson_inline simd8 lt_bits(const simd8 other) const { return other.saturating_sub(*this); } + simdjson_inline simd8 operator<=(const simd8 other) const { return other.max_val(*this) == other; } + simdjson_inline simd8 operator>=(const simd8 other) const { return other.min_val(*this) == other; } + simdjson_inline simd8 operator>(const simd8 other) const { return this->gt_bits(other).any_bits_set(); } + simdjson_inline simd8 operator<(const simd8 other) const { return this->gt_bits(other).any_bits_set(); } + + // Bit-specific operations + simdjson_inline simd8 bits_not_set() const { return *this == uint8_t(0); } + simdjson_inline simd8 bits_not_set(simd8 bits) const { return (*this & bits).bits_not_set(); } + simdjson_inline simd8 any_bits_set() const { return ~this->bits_not_set(); } + simdjson_inline simd8 any_bits_set(simd8 bits) const { return ~this->bits_not_set(bits); } + simdjson_inline bool is_ascii() const { return _mm_movemask_epi8(*this) == 0; } + simdjson_inline bool bits_not_set_anywhere() const { return _mm_testz_si128(*this, *this); } + simdjson_inline bool any_bits_set_anywhere() const { return !bits_not_set_anywhere(); } + simdjson_inline bool bits_not_set_anywhere(simd8 bits) const { return _mm_testz_si128(*this, bits); } + simdjson_inline bool any_bits_set_anywhere(simd8 bits) const { return !bits_not_set_anywhere(bits); } + template + simdjson_inline simd8 shr() const { return simd8(_mm_srli_epi16(*this, N)) & uint8_t(0xFFu >> N); } + template + simdjson_inline simd8 shl() const { return simd8(_mm_slli_epi16(*this, N)) & uint8_t(0xFFu << N); } + // Get one of the bits and make a bitmask out of it. + // e.g. value.get_bit<7>() gets the high bit + template + simdjson_inline int get_bit() const { return _mm_movemask_epi8(_mm_slli_epi16(*this, 7 - N)); } + }; + + template + struct simd8x64 { + static constexpr int NUM_CHUNKS = 64 / sizeof(simd8); + static_assert(NUM_CHUNKS == 4, "Westmere kernel should use four registers per 64-byte block."); + const simd8 chunks[NUM_CHUNKS]; + + simd8x64(const simd8x64& o) = delete; // no copy allowed + simd8x64& operator=(const simd8& other) = delete; // no assignment allowed + simd8x64() = delete; // no default constructor allowed + + simdjson_inline simd8x64(const simd8 chunk0, const simd8 chunk1, const simd8 chunk2, const simd8 chunk3) : chunks{ chunk0, chunk1, chunk2, chunk3 } {} + simdjson_inline simd8x64(const T ptr[64]) : chunks{ simd8::load(ptr), simd8::load(ptr + 16), simd8::load(ptr + 32), simd8::load(ptr + 48) } {} + + simdjson_inline void store(T ptr[64]) const { + this->chunks[0].store(ptr + sizeof(simd8) * 0); + this->chunks[1].store(ptr + sizeof(simd8) * 1); + this->chunks[2].store(ptr + sizeof(simd8) * 2); + this->chunks[3].store(ptr + sizeof(simd8) * 3); + } + + simdjson_inline simd8 reduce_or() const { + return (this->chunks[0] | this->chunks[1]) | (this->chunks[2] | this->chunks[3]); + } + + simdjson_inline uint64_t compress(uint64_t mask, T* output) const { + this->chunks[0].compress(uint16_t(mask), output); + this->chunks[1].compress(uint16_t(mask >> 16), output + 16 - count_ones(mask & 0xFFFF)); + this->chunks[2].compress(uint16_t(mask >> 32), output + 32 - count_ones(mask & 0xFFFFFFFF)); + this->chunks[3].compress(uint16_t(mask >> 48), output + 48 - count_ones(mask & 0xFFFFFFFFFFFF)); + return 64 - count_ones(mask); + } + + simdjson_inline uint64_t to_bitmask() const { + uint64_t r0 = uint32_t(this->chunks[0].to_bitmask()); + uint64_t r1 = this->chunks[1].to_bitmask(); + uint64_t r2 = this->chunks[2].to_bitmask(); + uint64_t r3 = this->chunks[3].to_bitmask(); + return r0 | (r1 << 16) | (r2 << 32) | (r3 << 48); + } + + simdjson_inline uint64_t eq(const T m) const { + const simd8 mask = simd8::splat(m); + return simd8x64( + this->chunks[0] == mask, + this->chunks[1] == mask, + this->chunks[2] == mask, + this->chunks[3] == mask + ).to_bitmask(); + } + + simdjson_inline uint64_t eq(const simd8x64& other) const { + return simd8x64( + this->chunks[0] == other.chunks[0], + this->chunks[1] == other.chunks[1], + this->chunks[2] == other.chunks[2], + this->chunks[3] == other.chunks[3] + ).to_bitmask(); + } + + simdjson_inline uint64_t lteq(const T m) const { + const simd8 mask = simd8::splat(m); + return simd8x64( + this->chunks[0] <= mask, + this->chunks[1] <= mask, + this->chunks[2] <= mask, + this->chunks[3] <= mask + ).to_bitmask(); + } + }; // struct simd8x64 + + } // namespace simd + } // unnamed namespace + } // namespace westmere +} // namespace simdjson + +#endif // SIMDJSON_WESTMERE_SIMD_INPUT_H +/* end file simdjson/westmere/simd.h */ +/* including simdjson/westmere/stringparsing_defs.h: #include "simdjson/westmere/stringparsing_defs.h" */ +/* begin file simdjson/westmere/stringparsing_defs.h */ +#ifndef SIMDJSON_WESTMERE_STRINGPARSING_DEFS_H +#define SIMDJSON_WESTMERE_STRINGPARSING_DEFS_H + +/* including simdjson/westmere/bitmanipulation.h: #include "simdjson/westmere/bitmanipulation.h" */ +/* begin file simdjson/westmere/bitmanipulation.h */ +#ifndef SIMDJSON_WESTMERE_BITMANIPULATION_H +#define SIMDJSON_WESTMERE_BITMANIPULATION_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/westmere/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/westmere/intrinsics.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace westmere { + namespace { + + // We sometimes call trailing_zero on inputs that are zero, + // but the algorithms do not end up using the returned value. + // Sadly, sanitizers are not smart enough to figure it out. + SIMDJSON_NO_SANITIZE_UNDEFINED + // This function can be used safely even if not all bytes have been + // initialized. + // See issue https://github.com/simdjson/simdjson/issues/1965 + SIMDJSON_NO_SANITIZE_MEMORY + simdjson_inline int trailing_zeroes(uint64_t input_num) { +#if SIMDJSON_REGULAR_VISUAL_STUDIO + unsigned long ret; + // Search the mask data from least significant bit (LSB) + // to the most significant bit (MSB) for a set bit (1). + _BitScanForward64(&ret, input_num); + return (int)ret; +#else // SIMDJSON_REGULAR_VISUAL_STUDIO + return __builtin_ctzll(input_num); +#endif // SIMDJSON_REGULAR_VISUAL_STUDIO + } + + /* result might be undefined when input_num is zero */ + simdjson_inline uint64_t clear_lowest_bit(uint64_t input_num) { + return input_num & (input_num - 1); + } + + /* result might be undefined when input_num is zero */ + simdjson_inline int leading_zeroes(uint64_t input_num) { +#if SIMDJSON_REGULAR_VISUAL_STUDIO + unsigned long leading_zero = 0; + // Search the mask data from most significant bit (MSB) + // to least significant bit (LSB) for a set bit (1). + if (_BitScanReverse64(&leading_zero, input_num)) + return (int)(63 - leading_zero); + else + return 64; +#else + return __builtin_clzll(input_num); +#endif// SIMDJSON_REGULAR_VISUAL_STUDIO + } + +#if SIMDJSON_REGULAR_VISUAL_STUDIO + simdjson_inline unsigned __int64 count_ones(uint64_t input_num) { + // note: we do not support legacy 32-bit Windows in this kernel + return __popcnt64(input_num);// Visual Studio wants two underscores + } +#else + simdjson_inline long long int count_ones(uint64_t input_num) { + return _popcnt64(input_num); + } +#endif + + simdjson_inline bool add_overflow(uint64_t value1, uint64_t value2, + uint64_t* result) { +#if SIMDJSON_REGULAR_VISUAL_STUDIO + return _addcarry_u64(0, value1, value2, + reinterpret_cast(result)); +#else + return __builtin_uaddll_overflow(value1, value2, + reinterpret_cast(result)); +#endif + } + + } // unnamed namespace + } // namespace westmere +} // namespace simdjson + +#endif // SIMDJSON_WESTMERE_BITMANIPULATION_H +/* end file simdjson/westmere/bitmanipulation.h */ +/* including simdjson/westmere/simd.h: #include "simdjson/westmere/simd.h" */ +/* begin file simdjson/westmere/simd.h */ +#ifndef SIMDJSON_WESTMERE_SIMD_H +#define SIMDJSON_WESTMERE_SIMD_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/westmere/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/westmere/bitmanipulation.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/simdprune_tables.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace westmere { + namespace { + namespace simd { + + template + struct base { + __m128i value; + + // Zero constructor + simdjson_inline base() : value{ __m128i() } {} + + // Conversion from SIMD register + simdjson_inline base(const __m128i _value) : value(_value) {} + + // Conversion to SIMD register + simdjson_inline operator const __m128i& () const { return this->value; } + simdjson_inline operator __m128i& () { return this->value; } + + // Bit operations + simdjson_inline Child operator|(const Child other) const { return _mm_or_si128(*this, other); } + simdjson_inline Child operator&(const Child other) const { return _mm_and_si128(*this, other); } + simdjson_inline Child operator^(const Child other) const { return _mm_xor_si128(*this, other); } + simdjson_inline Child bit_andnot(const Child other) const { return _mm_andnot_si128(other, *this); } + simdjson_inline Child& operator|=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast | other; return *this_cast; } + simdjson_inline Child& operator&=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast & other; return *this_cast; } + simdjson_inline Child& operator^=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast ^ other; return *this_cast; } + }; + + template> + struct base8 : base> { + typedef uint16_t bitmask_t; + typedef uint32_t bitmask2_t; + + simdjson_inline base8() : base>() {} + simdjson_inline base8(const __m128i _value) : base>(_value) {} + + friend simdjson_inline Mask operator==(const simd8 lhs, const simd8 rhs) { return _mm_cmpeq_epi8(lhs, rhs); } + + static const int SIZE = sizeof(base>::value); + + template + simdjson_inline simd8 prev(const simd8 prev_chunk) const { + return _mm_alignr_epi8(*this, prev_chunk, 16 - N); + } + }; + + // SIMD byte mask type (returned by things like eq and gt) + template<> + struct simd8 : base8 { + static simdjson_inline simd8 splat(bool _value) { return _mm_set1_epi8(uint8_t(-(!!_value))); } + + simdjson_inline simd8() : base8() {} + simdjson_inline simd8(const __m128i _value) : base8(_value) {} + // Splat constructor + simdjson_inline simd8(bool _value) : base8(splat(_value)) {} + + simdjson_inline int to_bitmask() const { return _mm_movemask_epi8(*this); } + simdjson_inline bool any() const { return !_mm_testz_si128(*this, *this); } + simdjson_inline simd8 operator~() const { return *this ^ true; } + }; + + template + struct base8_numeric : base8 { + static simdjson_inline simd8 splat(T _value) { return _mm_set1_epi8(_value); } + static simdjson_inline simd8 zero() { return _mm_setzero_si128(); } + static simdjson_inline simd8 load(const T values[16]) { + return _mm_loadu_si128(reinterpret_cast(values)); + } + // Repeat 16 values as many times as necessary (usually for lookup tables) + static simdjson_inline simd8 repeat_16( + T v0, T v1, T v2, T v3, T v4, T v5, T v6, T v7, + T v8, T v9, T v10, T v11, T v12, T v13, T v14, T v15 + ) { + return simd8( + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15 + ); + } + + simdjson_inline base8_numeric() : base8() {} + simdjson_inline base8_numeric(const __m128i _value) : base8(_value) {} + + // Store to array + simdjson_inline void store(T dst[16]) const { return _mm_storeu_si128(reinterpret_cast<__m128i*>(dst), *this); } + + // Override to distinguish from bool version + simdjson_inline simd8 operator~() const { return *this ^ 0xFFu; } + + // Addition/subtraction are the same for signed and unsigned + simdjson_inline simd8 operator+(const simd8 other) const { return _mm_add_epi8(*this, other); } + simdjson_inline simd8 operator-(const simd8 other) const { return _mm_sub_epi8(*this, other); } + simdjson_inline simd8& operator+=(const simd8 other) { *this = *this + other; return *static_cast*>(this); } + simdjson_inline simd8& operator-=(const simd8 other) { *this = *this - other; return *static_cast*>(this); } + + // Perform a lookup assuming the value is between 0 and 16 (undefined behavior for out of range values) + template + simdjson_inline simd8 lookup_16(simd8 lookup_table) const { + return _mm_shuffle_epi8(lookup_table, *this); + } + + // Copies to 'output" all bytes corresponding to a 0 in the mask (interpreted as a bitset). + // Passing a 0 value for mask would be equivalent to writing out every byte to output. + // Only the first 16 - count_ones(mask) bytes of the result are significant but 16 bytes + // get written. + // Design consideration: it seems like a function with the + // signature simd8 compress(uint32_t mask) would be + // sensible, but the AVX ISA makes this kind of approach difficult. + template + simdjson_inline void compress(uint16_t mask, L* output) const { + using internal::thintable_epi8; + using internal::BitsSetTable256mul2; + using internal::pshufb_combine_table; + // this particular implementation was inspired by work done by @animetosho + // we do it in two steps, first 8 bytes and then second 8 bytes + uint8_t mask1 = uint8_t(mask); // least significant 8 bits + uint8_t mask2 = uint8_t(mask >> 8); // most significant 8 bits + // next line just loads the 64-bit values thintable_epi8[mask1] and + // thintable_epi8[mask2] into a 128-bit register, using only + // two instructions on most compilers. + __m128i shufmask = _mm_set_epi64x(thintable_epi8[mask2], thintable_epi8[mask1]); + // we increment by 0x08 the second half of the mask + shufmask = + _mm_add_epi8(shufmask, _mm_set_epi32(0x08080808, 0x08080808, 0, 0)); + // this is the version "nearly pruned" + __m128i pruned = _mm_shuffle_epi8(*this, shufmask); + // we still need to put the two halves together. + // we compute the popcount of the first half: + int pop1 = BitsSetTable256mul2[mask1]; + // then load the corresponding mask, what it does is to write + // only the first pop1 bytes from the first 8 bytes, and then + // it fills in with the bytes from the second 8 bytes + some filling + // at the end. + __m128i compactmask = + _mm_loadu_si128(reinterpret_cast(pshufb_combine_table + pop1 * 8)); + __m128i answer = _mm_shuffle_epi8(pruned, compactmask); + _mm_storeu_si128(reinterpret_cast<__m128i*>(output), answer); + } + + template + simdjson_inline simd8 lookup_16( + L replace0, L replace1, L replace2, L replace3, + L replace4, L replace5, L replace6, L replace7, + L replace8, L replace9, L replace10, L replace11, + L replace12, L replace13, L replace14, L replace15) const { + return lookup_16(simd8::repeat_16( + replace0, replace1, replace2, replace3, + replace4, replace5, replace6, replace7, + replace8, replace9, replace10, replace11, + replace12, replace13, replace14, replace15 + )); + } + }; + + // Signed bytes + template<> + struct simd8 : base8_numeric { + simdjson_inline simd8() : base8_numeric() {} + simdjson_inline simd8(const __m128i _value) : base8_numeric(_value) {} + // Splat constructor + simdjson_inline simd8(int8_t _value) : simd8(splat(_value)) {} + // Array constructor + simdjson_inline simd8(const int8_t* values) : simd8(load(values)) {} + // Member-by-member initialization + simdjson_inline simd8( + int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, + int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 + ) : simd8(_mm_setr_epi8( + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15 + )) {} + // Repeat 16 values as many times as necessary (usually for lookup tables) + simdjson_inline static simd8 repeat_16( + int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, + int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 + ) { + return simd8( + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15 + ); + } + + // Order-sensitive comparisons + simdjson_inline simd8 max_val(const simd8 other) const { return _mm_max_epi8(*this, other); } + simdjson_inline simd8 min_val(const simd8 other) const { return _mm_min_epi8(*this, other); } + simdjson_inline simd8 operator>(const simd8 other) const { return _mm_cmpgt_epi8(*this, other); } + simdjson_inline simd8 operator<(const simd8 other) const { return _mm_cmpgt_epi8(other, *this); } + }; + + // Unsigned bytes + template<> + struct simd8 : base8_numeric { + simdjson_inline simd8() : base8_numeric() {} + simdjson_inline simd8(const __m128i _value) : base8_numeric(_value) {} + // Splat constructor + simdjson_inline simd8(uint8_t _value) : simd8(splat(_value)) {} + // Array constructor + simdjson_inline simd8(const uint8_t* values) : simd8(load(values)) {} + // Member-by-member initialization + simdjson_inline simd8( + uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, + uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 + ) : simd8(_mm_setr_epi8( + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15 + )) {} + // Repeat 16 values as many times as necessary (usually for lookup tables) + simdjson_inline static simd8 repeat_16( + uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, + uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 + ) { + return simd8( + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15 + ); + } + + // Saturated math + simdjson_inline simd8 saturating_add(const simd8 other) const { return _mm_adds_epu8(*this, other); } + simdjson_inline simd8 saturating_sub(const simd8 other) const { return _mm_subs_epu8(*this, other); } + + // Order-specific operations + simdjson_inline simd8 max_val(const simd8 other) const { return _mm_max_epu8(*this, other); } + simdjson_inline simd8 min_val(const simd8 other) const { return _mm_min_epu8(*this, other); } + // Same as >, but only guarantees true is nonzero (< guarantees true = -1) + simdjson_inline simd8 gt_bits(const simd8 other) const { return this->saturating_sub(other); } + // Same as <, but only guarantees true is nonzero (< guarantees true = -1) + simdjson_inline simd8 lt_bits(const simd8 other) const { return other.saturating_sub(*this); } + simdjson_inline simd8 operator<=(const simd8 other) const { return other.max_val(*this) == other; } + simdjson_inline simd8 operator>=(const simd8 other) const { return other.min_val(*this) == other; } + simdjson_inline simd8 operator>(const simd8 other) const { return this->gt_bits(other).any_bits_set(); } + simdjson_inline simd8 operator<(const simd8 other) const { return this->gt_bits(other).any_bits_set(); } + + // Bit-specific operations + simdjson_inline simd8 bits_not_set() const { return *this == uint8_t(0); } + simdjson_inline simd8 bits_not_set(simd8 bits) const { return (*this & bits).bits_not_set(); } + simdjson_inline simd8 any_bits_set() const { return ~this->bits_not_set(); } + simdjson_inline simd8 any_bits_set(simd8 bits) const { return ~this->bits_not_set(bits); } + simdjson_inline bool is_ascii() const { return _mm_movemask_epi8(*this) == 0; } + simdjson_inline bool bits_not_set_anywhere() const { return _mm_testz_si128(*this, *this); } + simdjson_inline bool any_bits_set_anywhere() const { return !bits_not_set_anywhere(); } + simdjson_inline bool bits_not_set_anywhere(simd8 bits) const { return _mm_testz_si128(*this, bits); } + simdjson_inline bool any_bits_set_anywhere(simd8 bits) const { return !bits_not_set_anywhere(bits); } + template + simdjson_inline simd8 shr() const { return simd8(_mm_srli_epi16(*this, N)) & uint8_t(0xFFu >> N); } + template + simdjson_inline simd8 shl() const { return simd8(_mm_slli_epi16(*this, N)) & uint8_t(0xFFu << N); } + // Get one of the bits and make a bitmask out of it. + // e.g. value.get_bit<7>() gets the high bit + template + simdjson_inline int get_bit() const { return _mm_movemask_epi8(_mm_slli_epi16(*this, 7 - N)); } + }; + + template + struct simd8x64 { + static constexpr int NUM_CHUNKS = 64 / sizeof(simd8); + static_assert(NUM_CHUNKS == 4, "Westmere kernel should use four registers per 64-byte block."); + const simd8 chunks[NUM_CHUNKS]; + + simd8x64(const simd8x64& o) = delete; // no copy allowed + simd8x64& operator=(const simd8& other) = delete; // no assignment allowed + simd8x64() = delete; // no default constructor allowed + + simdjson_inline simd8x64(const simd8 chunk0, const simd8 chunk1, const simd8 chunk2, const simd8 chunk3) : chunks{ chunk0, chunk1, chunk2, chunk3 } {} + simdjson_inline simd8x64(const T ptr[64]) : chunks{ simd8::load(ptr), simd8::load(ptr + 16), simd8::load(ptr + 32), simd8::load(ptr + 48) } {} + + simdjson_inline void store(T ptr[64]) const { + this->chunks[0].store(ptr + sizeof(simd8) * 0); + this->chunks[1].store(ptr + sizeof(simd8) * 1); + this->chunks[2].store(ptr + sizeof(simd8) * 2); + this->chunks[3].store(ptr + sizeof(simd8) * 3); + } + + simdjson_inline simd8 reduce_or() const { + return (this->chunks[0] | this->chunks[1]) | (this->chunks[2] | this->chunks[3]); + } + + simdjson_inline uint64_t compress(uint64_t mask, T* output) const { + this->chunks[0].compress(uint16_t(mask), output); + this->chunks[1].compress(uint16_t(mask >> 16), output + 16 - count_ones(mask & 0xFFFF)); + this->chunks[2].compress(uint16_t(mask >> 32), output + 32 - count_ones(mask & 0xFFFFFFFF)); + this->chunks[3].compress(uint16_t(mask >> 48), output + 48 - count_ones(mask & 0xFFFFFFFFFFFF)); + return 64 - count_ones(mask); + } + + simdjson_inline uint64_t to_bitmask() const { + uint64_t r0 = uint32_t(this->chunks[0].to_bitmask()); + uint64_t r1 = this->chunks[1].to_bitmask(); + uint64_t r2 = this->chunks[2].to_bitmask(); + uint64_t r3 = this->chunks[3].to_bitmask(); + return r0 | (r1 << 16) | (r2 << 32) | (r3 << 48); + } + + simdjson_inline uint64_t eq(const T m) const { + const simd8 mask = simd8::splat(m); + return simd8x64( + this->chunks[0] == mask, + this->chunks[1] == mask, + this->chunks[2] == mask, + this->chunks[3] == mask + ).to_bitmask(); + } + + simdjson_inline uint64_t eq(const simd8x64& other) const { + return simd8x64( + this->chunks[0] == other.chunks[0], + this->chunks[1] == other.chunks[1], + this->chunks[2] == other.chunks[2], + this->chunks[3] == other.chunks[3] + ).to_bitmask(); + } + + simdjson_inline uint64_t lteq(const T m) const { + const simd8 mask = simd8::splat(m); + return simd8x64( + this->chunks[0] <= mask, + this->chunks[1] <= mask, + this->chunks[2] <= mask, + this->chunks[3] <= mask + ).to_bitmask(); + } + }; // struct simd8x64 + + } // namespace simd + } // unnamed namespace + } // namespace westmere +} // namespace simdjson + +#endif // SIMDJSON_WESTMERE_SIMD_INPUT_H +/* end file simdjson/westmere/simd.h */ + +namespace simdjson { + namespace westmere { + namespace { + + using namespace simd; + + // Holds backslashes and quotes locations. + struct backslash_and_quote { + public: + static constexpr uint32_t BYTES_PROCESSED = 32; + simdjson_inline static backslash_and_quote copy_and_find(const uint8_t* src, uint8_t* dst); + + simdjson_inline bool has_quote_first() { return ((bs_bits - 1) & quote_bits) != 0; } + simdjson_inline bool has_backslash() { return bs_bits != 0; } + simdjson_inline int quote_index() { return trailing_zeroes(quote_bits); } + simdjson_inline int backslash_index() { return trailing_zeroes(bs_bits); } + + uint32_t bs_bits; + uint32_t quote_bits; + }; // struct backslash_and_quote + + simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8_t* src, uint8_t* dst) { + // this can read up to 31 bytes beyond the buffer size, but we require + // SIMDJSON_PADDING of padding + static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "backslash and quote finder must process fewer than SIMDJSON_PADDING bytes"); + simd8 v0(src); + simd8 v1(src + 16); + v0.store(dst); + v1.store(dst + 16); + uint64_t bs_and_quote = simd8x64(v0 == '\\', v1 == '\\', v0 == '"', v1 == '"').to_bitmask(); + return { + uint32_t(bs_and_quote), // bs_bits + uint32_t(bs_and_quote >> 32) // quote_bits + }; + } + + } // unnamed namespace + } // namespace westmere +} // namespace simdjson + +#endif // SIMDJSON_WESTMERE_STRINGPARSING_DEFS_H +/* end file simdjson/westmere/stringparsing_defs.h */ +/* end file simdjson/westmere/begin.h */ +/* including generic/amalgamated.h for westmere: #include */ +/* begin file generic/amalgamated.h for westmere */ +#if defined(SIMDJSON_CONDITIONAL_INCLUDE) && !defined(SIMDJSON_SRC_GENERIC_DEPENDENCIES_H) +#error generic/dependencies.h must be included before generic/amalgamated.h! +#endif + +/* including generic/base.h for westmere: #include */ +/* begin file generic/base.h for westmere */ +#ifndef SIMDJSON_SRC_GENERIC_BASE_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_BASE_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace westmere { + namespace { + + struct json_character_block; + + } // unnamed namespace + } // namespace westmere +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_BASE_H +/* end file generic/base.h for westmere */ +/* including generic/dom_parser_implementation.h for westmere: #include */ +/* begin file generic/dom_parser_implementation.h for westmere */ +#ifndef SIMDJSON_SRC_GENERIC_DOM_PARSER_IMPLEMENTATION_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_DOM_PARSER_IMPLEMENTATION_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +// Interface a dom parser implementation must fulfill +namespace simdjson { + namespace westmere { + namespace { + + simdjson_inline simd8 must_be_2_3_continuation(const simd8 prev2, const simd8 prev3); + simdjson_inline bool is_ascii(const simd8x64& input); + + } // unnamed namespace + } // namespace westmere +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_DOM_PARSER_IMPLEMENTATION_H +/* end file generic/dom_parser_implementation.h for westmere */ +/* including generic/json_character_block.h for westmere: #include */ +/* begin file generic/json_character_block.h for westmere */ +#ifndef SIMDJSON_SRC_GENERIC_JSON_CHARACTER_BLOCK_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_JSON_CHARACTER_BLOCK_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace westmere { + namespace { + + struct json_character_block { + static simdjson_inline json_character_block classify(const simd::simd8x64& in); + + simdjson_inline uint64_t whitespace() const noexcept { return _whitespace; } + simdjson_inline uint64_t op() const noexcept { return _op; } + simdjson_inline uint64_t scalar() const noexcept { return ~(op() | whitespace()); } + + uint64_t _whitespace; + uint64_t _op; + }; + + } // unnamed namespace + } // namespace westmere +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_JSON_CHARACTER_BLOCK_H +/* end file generic/json_character_block.h for westmere */ +/* end file generic/amalgamated.h for westmere */ +/* including generic/stage1/amalgamated.h for westmere: #include */ +/* begin file generic/stage1/amalgamated.h for westmere */ +// Stuff other things depend on +/* including generic/stage1/base.h for westmere: #include */ +/* begin file generic/stage1/base.h for westmere */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE1_BASE_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_BASE_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace westmere { + namespace { + namespace stage1 { + + class bit_indexer; + template + struct buf_block_reader; + struct json_block; + class json_minifier; + class json_scanner; + struct json_string_block; + class json_string_scanner; + class json_structural_indexer; + + } // namespace stage1 + + namespace utf8_validation { + struct utf8_checker; + } // namespace utf8_validation + + using utf8_validation::utf8_checker; + + } // unnamed namespace + } // namespace westmere +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE1_BASE_H +/* end file generic/stage1/base.h for westmere */ +/* including generic/stage1/buf_block_reader.h for westmere: #include */ +/* begin file generic/stage1/buf_block_reader.h for westmere */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE1_BUF_BLOCK_READER_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_BUF_BLOCK_READER_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +#include + +namespace simdjson { + namespace westmere { + namespace { + namespace stage1 { + + // Walks through a buffer in block-sized increments, loading the last part with spaces + template + struct buf_block_reader { + public: + simdjson_inline buf_block_reader(const uint8_t* _buf, size_t _len); + simdjson_inline size_t block_index(); + simdjson_inline bool has_full_block() const; + simdjson_inline const uint8_t* full_block() const; + /** + * Get the last block, padded with spaces. + * + * There will always be a last block, with at least 1 byte, unless len == 0 (in which case this + * function fills the buffer with spaces and returns 0. In particular, if len == STEP_SIZE there + * will be 0 full_blocks and 1 remainder block with STEP_SIZE bytes and no spaces for padding. + * + * @return the number of effective characters in the last block. + */ + simdjson_inline size_t get_remainder(uint8_t* dst) const; + simdjson_inline void advance(); + private: + const uint8_t* buf; + const size_t len; + const size_t lenminusstep; + size_t idx; + }; + + // Routines to print masks and text for debugging bitmask operations + simdjson_unused static char* format_input_text_64(const uint8_t* text) { + static char buf[sizeof(simd8x64) + 1]; + for (size_t i = 0; i < sizeof(simd8x64); i++) { + buf[i] = int8_t(text[i]) < ' ' ? '_' : int8_t(text[i]); + } + buf[sizeof(simd8x64)] = '\0'; + return buf; + } + + // Routines to print masks and text for debugging bitmask operations + simdjson_unused static char* format_input_text(const simd8x64& in) { + static char buf[sizeof(simd8x64) + 1]; + in.store(reinterpret_cast(buf)); + for (size_t i = 0; i < sizeof(simd8x64); i++) { + if (buf[i] < ' ') { buf[i] = '_'; } + } + buf[sizeof(simd8x64)] = '\0'; + return buf; + } + + simdjson_unused static char* format_input_text(const simd8x64& in, uint64_t mask) { + static char buf[sizeof(simd8x64) + 1]; + in.store(reinterpret_cast(buf)); + for (size_t i = 0; i < sizeof(simd8x64); i++) { + if (buf[i] <= ' ') { buf[i] = '_'; } + if (!(mask & (size_t(1) << i))) { buf[i] = ' '; } + } + buf[sizeof(simd8x64)] = '\0'; + return buf; + } + + simdjson_unused static char* format_mask(uint64_t mask) { + static char buf[sizeof(simd8x64) + 1]; + for (size_t i = 0; i < 64; i++) { + buf[i] = (mask & (size_t(1) << i)) ? 'X' : ' '; + } + buf[64] = '\0'; + return buf; + } + + template + simdjson_inline buf_block_reader::buf_block_reader(const uint8_t* _buf, size_t _len) : buf{ _buf }, len{ _len }, lenminusstep{ len < STEP_SIZE ? 0 : len - STEP_SIZE }, idx{ 0 } {} + + template + simdjson_inline size_t buf_block_reader::block_index() { return idx; } + + template + simdjson_inline bool buf_block_reader::has_full_block() const { + return idx < lenminusstep; + } + + template + simdjson_inline const uint8_t* buf_block_reader::full_block() const { + return &buf[idx]; + } + + template + simdjson_inline size_t buf_block_reader::get_remainder(uint8_t* dst) const { + if (len == idx) { return 0; } // memcpy(dst, null, 0) will trigger an error with some sanitizers + std::memset(dst, 0x20, STEP_SIZE); // std::memset STEP_SIZE because it's more efficient to write out 8 or 16 bytes at once. + std::memcpy(dst, buf + idx, len - idx); + return len - idx; + } + + template + simdjson_inline void buf_block_reader::advance() { + idx += STEP_SIZE; + } + + } // namespace stage1 + } // unnamed namespace + } // namespace westmere +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE1_BUF_BLOCK_READER_H +/* end file generic/stage1/buf_block_reader.h for westmere */ +/* including generic/stage1/json_escape_scanner.h for westmere: #include */ +/* begin file generic/stage1/json_escape_scanner.h for westmere */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE1_JSON_ESCAPE_SCANNER_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_JSON_ESCAPE_SCANNER_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace westmere { + namespace { + namespace stage1 { + + /** + * Scans for escape characters in JSON, taking care with multiple backslashes (\\n vs. \n). + */ + struct json_escape_scanner { + /** The actual escape characters (the backslashes themselves). */ + uint64_t next_is_escaped = 0ULL; + + struct escaped_and_escape { + /** + * Mask of escaped characters. + * + * ``` + * \n \\n \\\n \\\\n \ + * 0100100010100101000 + * n \ \ n \ \ + * ``` + */ + uint64_t escaped; + /** + * Mask of escape characters. + * + * ``` + * \n \\n \\\n \\\\n \ + * 1001000101001010001 + * \ \ \ \ \ \ \ + * ``` + */ + uint64_t escape; + }; + + /** + * Get a mask of both escape and escaped characters (the characters following a backslash). + * + * @param potential_escape A mask of the character that can escape others (but could be + * escaped itself). e.g. block.eq('\\') + */ + simdjson_really_inline escaped_and_escape next(uint64_t backslash) noexcept { + +#if !SIMDJSON_SKIP_BACKSLASH_SHORT_CIRCUIT + if (!backslash) { return { next_escaped_without_backslashes(), 0 }; } +#endif + + // | | Mask (shows characters instead of 1's) | Depth | Instructions | + // |--------------------------------|----------------------------------------|-------|---------------------| + // | string | `\\n_\\\n___\\\n___\\\\___\\\\__\\\` | | | + // | | ` even odd even odd odd` | | | + // | potential_escape | ` \ \\\ \\\ \\\\ \\\\ \\\` | 1 | 1 (backslash & ~first_is_escaped) + // | escape_and_terminal_code | ` \n \ \n \ \n \ \ \ \ \ \` | 5 | 5 (next_escape_and_terminal_code()) + // | escaped | `\ \ n \ n \ \ \ \ \ ` X | 6 | 7 (escape_and_terminal_code ^ (potential_escape | first_is_escaped)) + // | escape | ` \ \ \ \ \ \ \ \ \ \` | 6 | 8 (escape_and_terminal_code & backslash) + // | first_is_escaped | `\ ` | 7 (*) | 9 (escape >> 63) () + // (*) this is not needed until the next iteration + uint64_t escape_and_terminal_code = next_escape_and_terminal_code(backslash & ~this->next_is_escaped); + uint64_t escaped = escape_and_terminal_code ^ (backslash | this->next_is_escaped); + uint64_t escape = escape_and_terminal_code & backslash; + this->next_is_escaped = escape >> 63; + return { escaped, escape }; + } + + private: + static constexpr const uint64_t ODD_BITS = 0xAAAAAAAAAAAAAAAAULL; + + simdjson_really_inline uint64_t next_escaped_without_backslashes() noexcept { + uint64_t escaped = this->next_is_escaped; + this->next_is_escaped = 0; + return escaped; + } + + /** + * Returns a mask of the next escape characters (masking out escaped backslashes), along with + * any non-backslash escape codes. + * + * \n \\n \\\n \\\\n returns: + * \n \ \ \n \ \ + * 11 100 1011 10100 + * + * You are expected to mask out the first bit yourself if the previous block had a trailing + * escape. + * + * & the result with potential_escape to get just the escape characters. + * ^ the result with (potential_escape | first_is_escaped) to get escaped characters. + */ + static simdjson_really_inline uint64_t next_escape_and_terminal_code(uint64_t potential_escape) noexcept { + // If we were to just shift and mask out any odd bits, we'd actually get a *half* right answer: + // any even-aligned backslash runs would be correct! Odd-aligned backslash runs would be + // inverted (\\\ would be 010 instead of 101). + // + // ``` + // string: | ____\\\\_\\\\_____ | + // maybe_escaped | ODD | \ \ \ \ | + // even-aligned ^^^ ^^^^ odd-aligned + // ``` + // + // Taking that into account, our basic strategy is: + // + // 1. Use subtraction to produce a mask with 1's for even-aligned runs and 0's for + // odd-aligned runs. + // 2. XOR all odd bits, which masks out the odd bits in even-aligned runs, and brings IN the + // odd bits in odd-aligned runs. + // 3. & with backslash to clean up any stray bits. + // runs are set to 0, and then XORing with "odd": + // + // | | Mask (shows characters instead of 1's) | Instructions | + // |--------------------------------|----------------------------------------|---------------------| + // | string | `\\n_\\\n___\\\n___\\\\___\\\\__\\\` | + // | | ` even odd even odd odd` | + // | maybe_escaped | ` n \\n \\n \\\_ \\\_ \\` X | 1 (potential_escape << 1) + // | maybe_escaped_and_odd | ` \n_ \\n _ \\\n_ _ \\\__ _\\\_ \\\` | 1 (maybe_escaped | odd) + // | even_series_codes_and_odd | ` n_\\\ _ n_ _\\\\ _ _ ` | 1 (maybe_escaped_and_odd - potential_escape) + // | escape_and_terminal_code | ` \n \ \n \ \n \ \ \ \ \ \` | 1 (^ odd) + // + + // Escaped characters are characters following an escape. + uint64_t maybe_escaped = potential_escape << 1; + + // To distinguish odd from even escape sequences, therefore, we turn on any *starting* + // escapes that are on an odd byte. (We actually bring in all odd bits, for speed.) + // - Odd runs of backslashes are 0000, and the code at the end ("n" in \n or \\n) is 1. + // - Odd runs of backslashes are 1111, and the code at the end ("n" in \n or \\n) is 0. + // - All other odd bytes are 1, and even bytes are 0. + uint64_t maybe_escaped_and_odd_bits = maybe_escaped | ODD_BITS; + uint64_t even_series_codes_and_odd_bits = maybe_escaped_and_odd_bits - potential_escape; + + // Now we flip all odd bytes back with xor. This: + // - Makes odd runs of backslashes go from 0000 to 1010 + // - Makes even runs of backslashes go from 1111 to 1010 + // - Sets actually-escaped codes to 1 (the n in \n and \\n: \n = 11, \\n = 100) + // - Resets all other bytes to 0 + return even_series_codes_and_odd_bits ^ ODD_BITS; + } + }; + + } // namespace stage1 + } // unnamed namespace + } // namespace westmere +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRING_SCANNER_H +/* end file generic/stage1/json_escape_scanner.h for westmere */ +/* including generic/stage1/json_string_scanner.h for westmere: #include */ +/* begin file generic/stage1/json_string_scanner.h for westmere */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRING_SCANNER_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRING_SCANNER_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace westmere { + namespace { + namespace stage1 { + + struct json_string_block { + // We spell out the constructors in the hope of resolving inlining issues with Visual Studio 2017 + simdjson_really_inline json_string_block(uint64_t escaped, uint64_t quote, uint64_t in_string) : + _escaped(escaped), _quote(quote), _in_string(in_string) {} + + // Escaped characters (characters following an escape() character) + simdjson_really_inline uint64_t escaped() const { return _escaped; } + // Real (non-backslashed) quotes + simdjson_really_inline uint64_t quote() const { return _quote; } + // Only characters inside the string (not including the quotes) + simdjson_really_inline uint64_t string_content() const { return _in_string & ~_quote; } + // Return a mask of whether the given characters are inside a string (only works on non-quotes) + simdjson_really_inline uint64_t non_quote_inside_string(uint64_t mask) const { return mask & _in_string; } + // Return a mask of whether the given characters are inside a string (only works on non-quotes) + simdjson_really_inline uint64_t non_quote_outside_string(uint64_t mask) const { return mask & ~_in_string; } + // Tail of string (everything except the start quote) + simdjson_really_inline uint64_t string_tail() const { return _in_string ^ _quote; } + + // escaped characters (backslashed--does not include the hex characters after \u) + uint64_t _escaped; + // real quotes (non-escaped ones) + uint64_t _quote; + // string characters (includes start quote but not end quote) + uint64_t _in_string; + }; + + // Scans blocks for string characters, storing the state necessary to do so + class json_string_scanner { + public: + simdjson_really_inline json_string_block next(const simd::simd8x64& in); + // Returns either UNCLOSED_STRING or SUCCESS + simdjson_really_inline error_code finish(); + + private: + // Scans for escape characters + json_escape_scanner escape_scanner{}; + // Whether the last iteration was still inside a string (all 1's = true, all 0's = false). + uint64_t prev_in_string = 0ULL; + }; + + // + // Return a mask of all string characters plus end quotes. + // + // prev_escaped is overflow saying whether the next character is escaped. + // prev_in_string is overflow saying whether we're still in a string. + // + // Backslash sequences outside of quotes will be detected in stage 2. + // + simdjson_really_inline json_string_block json_string_scanner::next(const simd::simd8x64& in) { + const uint64_t backslash = in.eq('\\'); + const uint64_t escaped = escape_scanner.next(backslash).escaped; + const uint64_t quote = in.eq('"') & ~escaped; + + // + // prefix_xor flips on bits inside the string (and flips off the end quote). + // + // Then we xor with prev_in_string: if we were in a string already, its effect is flipped + // (characters inside strings are outside, and characters outside strings are inside). + // + const uint64_t in_string = prefix_xor(quote) ^ prev_in_string; + + // + // Check if we're still in a string at the end of the box so the next block will know + // + prev_in_string = uint64_t(static_cast(in_string) >> 63); + + // Use ^ to turn the beginning quote off, and the end quote on. + + // We are returning a function-local object so either we get a move constructor + // or we get copy elision. + return json_string_block(escaped, quote, in_string); + } + + simdjson_really_inline error_code json_string_scanner::finish() { + if (prev_in_string) { + return UNCLOSED_STRING; + } + return SUCCESS; + } + + } // namespace stage1 + } // unnamed namespace + } // namespace westmere +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRING_SCANNER_H +/* end file generic/stage1/json_string_scanner.h for westmere */ +/* including generic/stage1/utf8_lookup4_algorithm.h for westmere: #include */ +/* begin file generic/stage1/utf8_lookup4_algorithm.h for westmere */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE1_UTF8_LOOKUP4_ALGORITHM_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_UTF8_LOOKUP4_ALGORITHM_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace westmere { + namespace { + namespace utf8_validation { + + using namespace simd; + + simdjson_inline simd8 check_special_cases(const simd8 input, const simd8 prev1) { + // Bit 0 = Too Short (lead byte/ASCII followed by lead byte/ASCII) + // Bit 1 = Too Long (ASCII followed by continuation) + // Bit 2 = Overlong 3-byte + // Bit 4 = Surrogate + // Bit 5 = Overlong 2-byte + // Bit 7 = Two Continuations + constexpr const uint8_t TOO_SHORT = 1 << 0; // 11______ 0_______ + // 11______ 11______ + constexpr const uint8_t TOO_LONG = 1 << 1; // 0_______ 10______ + constexpr const uint8_t OVERLONG_3 = 1 << 2; // 11100000 100_____ + constexpr const uint8_t SURROGATE = 1 << 4; // 11101101 101_____ + constexpr const uint8_t OVERLONG_2 = 1 << 5; // 1100000_ 10______ + constexpr const uint8_t TWO_CONTS = 1 << 7; // 10______ 10______ + constexpr const uint8_t TOO_LARGE = 1 << 3; // 11110100 1001____ + // 11110100 101_____ + // 11110101 1001____ + // 11110101 101_____ + // 1111011_ 1001____ + // 1111011_ 101_____ + // 11111___ 1001____ + // 11111___ 101_____ + constexpr const uint8_t TOO_LARGE_1000 = 1 << 6; + // 11110101 1000____ + // 1111011_ 1000____ + // 11111___ 1000____ + constexpr const uint8_t OVERLONG_4 = 1 << 6; // 11110000 1000____ + + const simd8 byte_1_high = prev1.shr<4>().lookup_16( + // 0_______ ________ + TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, + TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, + // 10______ ________ + TWO_CONTS, TWO_CONTS, TWO_CONTS, TWO_CONTS, + // 1100____ ________ + TOO_SHORT | OVERLONG_2, + // 1101____ ________ + TOO_SHORT, + // 1110____ ________ + TOO_SHORT | OVERLONG_3 | SURROGATE, + // 1111____ ________ + TOO_SHORT | TOO_LARGE | TOO_LARGE_1000 | OVERLONG_4 + ); + constexpr const uint8_t CARRY = TOO_SHORT | TOO_LONG | TWO_CONTS; // These all have ____ in byte 1 . + const simd8 byte_1_low = (prev1 & 0x0F).lookup_16( + // ____0000 ________ + CARRY | OVERLONG_3 | OVERLONG_2 | OVERLONG_4, + // ____0001 ________ + CARRY | OVERLONG_2, + // ____001_ ________ + CARRY, + CARRY, + + // ____0100 ________ + CARRY | TOO_LARGE, + // ____0101 ________ + CARRY | TOO_LARGE | TOO_LARGE_1000, + // ____011_ ________ + CARRY | TOO_LARGE | TOO_LARGE_1000, + CARRY | TOO_LARGE | TOO_LARGE_1000, + + // ____1___ ________ + CARRY | TOO_LARGE | TOO_LARGE_1000, + CARRY | TOO_LARGE | TOO_LARGE_1000, + CARRY | TOO_LARGE | TOO_LARGE_1000, + CARRY | TOO_LARGE | TOO_LARGE_1000, + CARRY | TOO_LARGE | TOO_LARGE_1000, + // ____1101 ________ + CARRY | TOO_LARGE | TOO_LARGE_1000 | SURROGATE, + CARRY | TOO_LARGE | TOO_LARGE_1000, + CARRY | TOO_LARGE | TOO_LARGE_1000 + ); + const simd8 byte_2_high = input.shr<4>().lookup_16( + // ________ 0_______ + TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT, + TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT, + + // ________ 1000____ + TOO_LONG | OVERLONG_2 | TWO_CONTS | OVERLONG_3 | TOO_LARGE_1000 | OVERLONG_4, + // ________ 1001____ + TOO_LONG | OVERLONG_2 | TWO_CONTS | OVERLONG_3 | TOO_LARGE, + // ________ 101_____ + TOO_LONG | OVERLONG_2 | TWO_CONTS | SURROGATE | TOO_LARGE, + TOO_LONG | OVERLONG_2 | TWO_CONTS | SURROGATE | TOO_LARGE, + + // ________ 11______ + TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT + ); + return (byte_1_high & byte_1_low & byte_2_high); + } + simdjson_inline simd8 check_multibyte_lengths(const simd8 input, + const simd8 prev_input, const simd8 sc) { + simd8 prev2 = input.prev<2>(prev_input); + simd8 prev3 = input.prev<3>(prev_input); + simd8 must23 = must_be_2_3_continuation(prev2, prev3); + simd8 must23_80 = must23 & uint8_t(0x80); + return must23_80 ^ sc; + } + + // + // Return nonzero if there are incomplete multibyte characters at the end of the block: + // e.g. if there is a 4-byte character, but it's 3 bytes from the end. + // + simdjson_inline simd8 is_incomplete(const simd8 input) { + // If the previous input's last 3 bytes match this, they're too short (they ended at EOF): + // ... 1111____ 111_____ 11______ +#if SIMDJSON_IMPLEMENTATION_ICELAKE + static const uint8_t max_array[64] = { + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 0xf0u - 1, 0xe0u - 1, 0xc0u - 1 + }; +#else + static const uint8_t max_array[32] = { + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 0xf0u - 1, 0xe0u - 1, 0xc0u - 1 + }; +#endif + const simd8 max_value(&max_array[sizeof(max_array) - sizeof(simd8)]); + return input.gt_bits(max_value); + } + + struct utf8_checker { + // If this is nonzero, there has been a UTF-8 error. + simd8 error; + // The last input we received + simd8 prev_input_block; + // Whether the last input we received was incomplete (used for ASCII fast path) + simd8 prev_incomplete; + + // + // Check whether the current bytes are valid UTF-8. + // + simdjson_inline void check_utf8_bytes(const simd8 input, const simd8 prev_input) { + // Flip prev1...prev3 so we can easily determine if they are 2+, 3+ or 4+ lead bytes + // (2, 3, 4-byte leads become large positive numbers instead of small negative numbers) + simd8 prev1 = input.prev<1>(prev_input); + simd8 sc = check_special_cases(input, prev1); + this->error |= check_multibyte_lengths(input, prev_input, sc); + } + + // The only problem that can happen at EOF is that a multibyte character is too short + // or a byte value too large in the last bytes: check_special_cases only checks for bytes + // too large in the first of two bytes. + simdjson_inline void check_eof() { + // If the previous block had incomplete UTF-8 characters at the end, an ASCII block can't + // possibly finish them. + this->error |= this->prev_incomplete; + } + + simdjson_inline void check_next_input(const simd8x64& input) { + if (simdjson_likely(is_ascii(input))) { + this->error |= this->prev_incomplete; + } + else { + // you might think that a for-loop would work, but under Visual Studio, it is not good enough. + static_assert((simd8x64::NUM_CHUNKS == 1) + || (simd8x64::NUM_CHUNKS == 2) + || (simd8x64::NUM_CHUNKS == 4), + "We support one, two or four chunks per 64-byte block."); + SIMDJSON_IF_CONSTEXPR(simd8x64::NUM_CHUNKS == 1) { + this->check_utf8_bytes(input.chunks[0], this->prev_input_block); + } +else SIMDJSON_IF_CONSTEXPR(simd8x64::NUM_CHUNKS == 2) { + this->check_utf8_bytes(input.chunks[0], this->prev_input_block); + this->check_utf8_bytes(input.chunks[1], input.chunks[0]); + } + else SIMDJSON_IF_CONSTEXPR(simd8x64::NUM_CHUNKS == 4) { + this->check_utf8_bytes(input.chunks[0], this->prev_input_block); + this->check_utf8_bytes(input.chunks[1], input.chunks[0]); + this->check_utf8_bytes(input.chunks[2], input.chunks[1]); + this->check_utf8_bytes(input.chunks[3], input.chunks[2]); + } + this->prev_incomplete = is_incomplete(input.chunks[simd8x64::NUM_CHUNKS - 1]); + this->prev_input_block = input.chunks[simd8x64::NUM_CHUNKS - 1]; + } + } + // do not forget to call check_eof! + simdjson_inline error_code errors() { + return this->error.any_bits_set_anywhere() ? error_code::UTF8_ERROR : error_code::SUCCESS; + } + + }; // struct utf8_checker + } // namespace utf8_validation + + } // unnamed namespace + } // namespace westmere +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE1_UTF8_LOOKUP4_ALGORITHM_H +/* end file generic/stage1/utf8_lookup4_algorithm.h for westmere */ +/* including generic/stage1/json_scanner.h for westmere: #include */ +/* begin file generic/stage1/json_scanner.h for westmere */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE1_JSON_SCANNER_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_JSON_SCANNER_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace westmere { + namespace { + namespace stage1 { + + /** + * A block of scanned json, with information on operators and scalars. + * + * We seek to identify pseudo-structural characters. Anything that is inside + * a string must be omitted (hence & ~_string.string_tail()). + * Otherwise, pseudo-structural characters come in two forms. + * 1. We have the structural characters ([,],{,},:, comma). The + * term 'structural character' is from the JSON RFC. + * 2. We have the 'scalar pseudo-structural characters'. + * Scalars are quotes, and any character except structural characters and white space. + * + * To identify the scalar pseudo-structural characters, we must look at what comes + * before them: it must be a space, a quote or a structural characters. + * Starting with simdjson v0.3, we identify them by + * negation: we identify everything that is followed by a non-quote scalar, + * and we negate that. Whatever remains must be a 'scalar pseudo-structural character'. + */ + struct json_block { + public: + // We spell out the constructors in the hope of resolving inlining issues with Visual Studio 2017 + simdjson_inline json_block(json_string_block&& string, json_character_block characters, uint64_t follows_potential_nonquote_scalar) : + _string(std::move(string)), _characters(characters), _follows_potential_nonquote_scalar(follows_potential_nonquote_scalar) {} + simdjson_inline json_block(json_string_block string, json_character_block characters, uint64_t follows_potential_nonquote_scalar) : + _string(string), _characters(characters), _follows_potential_nonquote_scalar(follows_potential_nonquote_scalar) {} + + /** + * The start of structurals. + * In simdjson prior to v0.3, these were called the pseudo-structural characters. + **/ + simdjson_inline uint64_t structural_start() const noexcept { return potential_structural_start() & ~_string.string_tail(); } + /** All JSON whitespace (i.e. not in a string) */ + simdjson_inline uint64_t whitespace() const noexcept { return non_quote_outside_string(_characters.whitespace()); } + + // Helpers + + /** Whether the given characters are inside a string (only works on non-quotes) */ + simdjson_inline uint64_t non_quote_inside_string(uint64_t mask) const noexcept { return _string.non_quote_inside_string(mask); } + /** Whether the given characters are outside a string (only works on non-quotes) */ + simdjson_inline uint64_t non_quote_outside_string(uint64_t mask) const noexcept { return _string.non_quote_outside_string(mask); } + + // string and escape characters + json_string_block _string; + // whitespace, structural characters ('operators'), scalars + json_character_block _characters; + // whether the previous character was a scalar + uint64_t _follows_potential_nonquote_scalar; + private: + // Potential structurals (i.e. disregarding strings) + + /** + * structural elements ([,],{,},:, comma) plus scalar starts like 123, true and "abc". + * They may reside inside a string. + **/ + simdjson_inline uint64_t potential_structural_start() const noexcept { return _characters.op() | potential_scalar_start(); } + /** + * The start of non-operator runs, like 123, true and "abc". + * It main reside inside a string. + **/ + simdjson_inline uint64_t potential_scalar_start() const noexcept { + // The term "scalar" refers to anything except structural characters and white space + // (so letters, numbers, quotes). + // Whenever it is preceded by something that is not a structural element ({,},[,],:, ") nor a white-space + // then we know that it is irrelevant structurally. + return _characters.scalar() & ~follows_potential_scalar(); + } + /** + * Whether the given character is immediately after a non-operator like 123, true. + * The characters following a quote are not included. + */ + simdjson_inline uint64_t follows_potential_scalar() const noexcept { + // _follows_potential_nonquote_scalar: is defined as marking any character that follows a character + // that is not a structural element ({,},[,],:, comma) nor a quote (") and that is not a + // white space. + // It is understood that within quoted region, anything at all could be marked (irrelevant). + return _follows_potential_nonquote_scalar; + } + }; + + /** + * Scans JSON for important bits: structural characters or 'operators', strings, and scalars. + * + * The scanner starts by calculating two distinct things: + * - string characters (taking \" into account) + * - structural characters or 'operators' ([]{},:, comma) + * and scalars (runs of non-operators like 123, true and "abc") + * + * To minimize data dependency (a key component of the scanner's speed), it finds these in parallel: + * in particular, the operator/scalar bit will find plenty of things that are actually part of + * strings. When we're done, json_block will fuse the two together by masking out tokens that are + * part of a string. + */ + class json_scanner { + public: + json_scanner() = default; + simdjson_inline json_block next(const simd::simd8x64& in); + // Returns either UNCLOSED_STRING or SUCCESS + simdjson_inline error_code finish(); + + private: + // Whether the last character of the previous iteration is part of a scalar token + // (anything except whitespace or a structural character/'operator'). + uint64_t prev_scalar = 0ULL; + json_string_scanner string_scanner{}; + }; + + + // + // Check if the current character immediately follows a matching character. + // + // For example, this checks for quotes with backslashes in front of them: + // + // const uint64_t backslashed_quote = in.eq('"') & immediately_follows(in.eq('\'), prev_backslash); + // + simdjson_inline uint64_t follows(const uint64_t match, uint64_t& overflow) { + const uint64_t result = match << 1 | overflow; + overflow = match >> 63; + return result; + } + + simdjson_inline json_block json_scanner::next(const simd::simd8x64& in) { + json_string_block strings = string_scanner.next(in); + // identifies the white-space and the structural characters + json_character_block characters = json_character_block::classify(in); + // The term "scalar" refers to anything except structural characters and white space + // (so letters, numbers, quotes). + // We want follows_scalar to mark anything that follows a non-quote scalar (so letters and numbers). + // + // A terminal quote should either be followed by a structural character (comma, brace, bracket, colon) + // or nothing. However, we still want ' "a string"true ' to mark the 't' of 'true' as a potential + // pseudo-structural character just like we would if we had ' "a string" true '; otherwise we + // may need to add an extra check when parsing strings. + // + // Performance: there are many ways to skin this cat. + const uint64_t nonquote_scalar = characters.scalar() & ~strings.quote(); + uint64_t follows_nonquote_scalar = follows(nonquote_scalar, prev_scalar); + // We are returning a function-local object so either we get a move constructor + // or we get copy elision. + return json_block( + strings,// strings is a function-local object so either it moves or the copy is elided. + characters, + follows_nonquote_scalar + ); + } + + simdjson_inline error_code json_scanner::finish() { + return string_scanner.finish(); + } + + } // namespace stage1 + } // unnamed namespace + } // namespace westmere +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE1_JSON_SCANNER_H +/* end file generic/stage1/json_scanner.h for westmere */ + +// All other declarations +/* including generic/stage1/find_next_document_index.h for westmere: #include */ +/* begin file generic/stage1/find_next_document_index.h for westmere */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE1_FIND_NEXT_DOCUMENT_INDEX_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_FIND_NEXT_DOCUMENT_INDEX_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace westmere { + namespace { + namespace stage1 { + + /** + * This algorithm is used to quickly identify the last structural position that + * makes up a complete document. + * + * It does this by going backwards and finding the last *document boundary* (a + * place where one value follows another without a comma between them). If the + * last document (the characters after the boundary) has an equal number of + * start and end brackets, it is considered complete. + * + * Simply put, we iterate over the structural characters, starting from + * the end. We consider that we found the end of a JSON document when the + * first element of the pair is NOT one of these characters: '{' '[' ':' ',' + * and when the second element is NOT one of these characters: '}' ']' ':' ','. + * + * This simple comparison works most of the time, but it does not cover cases + * where the batch's structural indexes contain a perfect amount of documents. + * In such a case, we do not have access to the structural index which follows + * the last document, therefore, we do not have access to the second element in + * the pair, and that means we cannot identify the last document. To fix this + * issue, we keep a count of the open and closed curly/square braces we found + * while searching for the pair. When we find a pair AND the count of open and + * closed curly/square braces is the same, we know that we just passed a + * complete document, therefore the last json buffer location is the end of the + * batch. + */ + simdjson_inline uint32_t find_next_document_index(dom_parser_implementation& parser) { + // Variant: do not count separately, just figure out depth + if (parser.n_structural_indexes == 0) { return 0; } + auto arr_cnt = 0; + auto obj_cnt = 0; + for (auto i = parser.n_structural_indexes - 1; i > 0; i--) { + auto idxb = parser.structural_indexes[i]; + switch (parser.buf[idxb]) { + case ':': + case ',': + continue; + case '}': + obj_cnt--; + continue; + case ']': + arr_cnt--; + continue; + case '{': + obj_cnt++; + break; + case '[': + arr_cnt++; + break; + } + auto idxa = parser.structural_indexes[i - 1]; + switch (parser.buf[idxa]) { + case '{': + case '[': + case ':': + case ',': + continue; + } + // Last document is complete, so the next document will appear after! + if (!arr_cnt && !obj_cnt) { + return parser.n_structural_indexes; + } + // Last document is incomplete; mark the document at i + 1 as the next one + return i; + } + // If we made it to the end, we want to finish counting to see if we have a full document. + switch (parser.buf[parser.structural_indexes[0]]) { + case '}': + obj_cnt--; + break; + case ']': + arr_cnt--; + break; + case '{': + obj_cnt++; + break; + case '[': + arr_cnt++; + break; + } + if (!arr_cnt && !obj_cnt) { + // We have a complete document. + return parser.n_structural_indexes; + } + return 0; + } + + } // namespace stage1 + } // unnamed namespace + } // namespace westmere +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE1_FIND_NEXT_DOCUMENT_INDEX_H +/* end file generic/stage1/find_next_document_index.h for westmere */ +/* including generic/stage1/json_minifier.h for westmere: #include */ +/* begin file generic/stage1/json_minifier.h for westmere */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE1_JSON_MINIFIER_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_JSON_MINIFIER_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +// This file contains the common code every implementation uses in stage1 +// It is intended to be included multiple times and compiled multiple times +// We assume the file in which it is included already includes +// "simdjson/stage1.h" (this simplifies amalgation) + +namespace simdjson { + namespace westmere { + namespace { + namespace stage1 { + + class json_minifier { + public: + template + static error_code minify(const uint8_t* buf, size_t len, uint8_t* dst, size_t& dst_len) noexcept; + + private: + simdjson_inline json_minifier(uint8_t* _dst) + : dst{ _dst } + {} + template + simdjson_inline void step(const uint8_t* block_buf, buf_block_reader& reader) noexcept; + simdjson_inline void next(const simd::simd8x64& in, const json_block& block); + simdjson_inline error_code finish(uint8_t* dst_start, size_t& dst_len); + json_scanner scanner{}; + uint8_t* dst; + }; + + simdjson_inline void json_minifier::next(const simd::simd8x64& in, const json_block& block) { + uint64_t mask = block.whitespace(); + dst += in.compress(mask, dst); + } + + simdjson_inline error_code json_minifier::finish(uint8_t* dst_start, size_t& dst_len) { + error_code error = scanner.finish(); + if (error) { dst_len = 0; return error; } + dst_len = dst - dst_start; + return SUCCESS; + } + + template<> + simdjson_inline void json_minifier::step<128>(const uint8_t* block_buf, buf_block_reader<128>& reader) noexcept { + simd::simd8x64 in_1(block_buf); + simd::simd8x64 in_2(block_buf + 64); + json_block block_1 = scanner.next(in_1); + json_block block_2 = scanner.next(in_2); + this->next(in_1, block_1); + this->next(in_2, block_2); + reader.advance(); + } + + template<> + simdjson_inline void json_minifier::step<64>(const uint8_t* block_buf, buf_block_reader<64>& reader) noexcept { + simd::simd8x64 in_1(block_buf); + json_block block_1 = scanner.next(in_1); + this->next(block_buf, block_1); + reader.advance(); + } + + template + error_code json_minifier::minify(const uint8_t* buf, size_t len, uint8_t* dst, size_t& dst_len) noexcept { + buf_block_reader reader(buf, len); + json_minifier minifier(dst); + + // Index the first n-1 blocks + while (reader.has_full_block()) { + minifier.step(reader.full_block(), reader); + } + + // Index the last (remainder) block, padded with spaces + uint8_t block[STEP_SIZE]; + size_t remaining_bytes = reader.get_remainder(block); + if (remaining_bytes > 0) { + // We do not want to write directly to the output stream. Rather, we write + // to a local buffer (for safety). + uint8_t out_block[STEP_SIZE]; + uint8_t* const guarded_dst{ minifier.dst }; + minifier.dst = out_block; + minifier.step(block, reader); + size_t to_write = minifier.dst - out_block; + // In some cases, we could be enticed to consider the padded spaces + // as part of the string. This is fine as long as we do not write more + // than we consumed. + if (to_write > remaining_bytes) { to_write = remaining_bytes; } + memcpy(guarded_dst, out_block, to_write); + minifier.dst = guarded_dst + to_write; + } + return minifier.finish(dst, dst_len); + } + + } // namespace stage1 + } // unnamed namespace + } // namespace westmere +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE1_JSON_MINIFIER_H +/* end file generic/stage1/json_minifier.h for westmere */ +/* including generic/stage1/json_structural_indexer.h for westmere: #include */ +/* begin file generic/stage1/json_structural_indexer.h for westmere */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRUCTURAL_INDEXER_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRUCTURAL_INDEXER_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +// This file contains the common code every implementation uses in stage1 +// It is intended to be included multiple times and compiled multiple times +// We assume the file in which it is included already includes +// "simdjson/stage1.h" (this simplifies amalgation) + +namespace simdjson { + namespace westmere { + namespace { + namespace stage1 { + + class bit_indexer { + public: + uint32_t* tail; + + simdjson_inline bit_indexer(uint32_t* index_buf) : tail(index_buf) {} + +#if SIMDJSON_PREFER_REVERSE_BITS + /** + * ARM lacks a fast trailing zero instruction, but it has a fast + * bit reversal instruction and a fast leading zero instruction. + * Thus it may be profitable to reverse the bits (once) and then + * to rely on a sequence of instructions that call the leading + * zero instruction. + * + * Performance notes: + * The chosen routine is not optimal in terms of data dependency + * since zero_leading_bit might require two instructions. However, + * it tends to minimize the total number of instructions which is + * beneficial. + */ + simdjson_inline void write_index(uint32_t idx, uint64_t& rev_bits, int i) { + int lz = leading_zeroes(rev_bits); + this->tail[i] = static_cast(idx) + lz; + rev_bits = zero_leading_bit(rev_bits, lz); + } +#else + /** + * Under recent x64 systems, we often have both a fast trailing zero + * instruction and a fast 'clear-lower-bit' instruction so the following + * algorithm can be competitive. + */ + + simdjson_inline void write_index(uint32_t idx, uint64_t& bits, int i) { + this->tail[i] = idx + trailing_zeroes(bits); + bits = clear_lowest_bit(bits); + } +#endif // SIMDJSON_PREFER_REVERSE_BITS + + template + simdjson_inline int write_indexes(uint32_t idx, uint64_t& bits) { + write_index(idx, bits, START); + SIMDJSON_IF_CONSTEXPR(N > 1) { + write_indexes<(N - 1 > 0 ? START + 1 : START), (N - 1 >= 0 ? N - 1 : 1)>(idx, bits); + } + return START + N; + } + + template + simdjson_inline int write_indexes_stepped(uint32_t idx, uint64_t& bits, int cnt) { + write_indexes(idx, bits); + SIMDJSON_IF_CONSTEXPR((START + STEP) < END) { + if (simdjson_unlikely((START + STEP) < cnt)) { + write_indexes_stepped<(START + STEP < END ? START + STEP : END), END, STEP>(idx, bits, cnt); + } + } + return ((END - START) % STEP) == 0 ? END : (END - START) - ((END - START) % STEP) + STEP; + } + + // flatten out values in 'bits' assuming that they are are to have values of idx + // plus their position in the bitvector, and store these indexes at + // base_ptr[base] incrementing base as we go + // will potentially store extra values beyond end of valid bits, so base_ptr + // needs to be large enough to handle this + // + // If the kernel sets SIMDJSON_GENERIC_JSON_STRUCTURAL_INDEXER_CUSTOM_BIT_INDEXER, then it + // will provide its own version of the code. +#ifdef SIMDJSON_GENERIC_JSON_STRUCTURAL_INDEXER_CUSTOM_BIT_INDEXER + simdjson_inline void write(uint32_t idx, uint64_t bits); +#else + simdjson_inline void write(uint32_t idx, uint64_t bits) { + // In some instances, the next branch is expensive because it is mispredicted. + // Unfortunately, in other cases, + // it helps tremendously. + if (bits == 0) + return; + + int cnt = static_cast(count_ones(bits)); + +#if SIMDJSON_PREFER_REVERSE_BITS + bits = reverse_bits(bits); +#endif +#ifdef SIMDJSON_STRUCTURAL_INDEXER_STEP + static constexpr const int STEP = SIMDJSON_STRUCTURAL_INDEXER_STEP; +#else + static constexpr const int STEP = 4; +#endif + static constexpr const int STEP_UNTIL = 24; + + write_indexes_stepped<0, STEP_UNTIL, STEP>(idx, bits, cnt); + SIMDJSON_IF_CONSTEXPR(STEP_UNTIL < 64) { + if (simdjson_unlikely(STEP_UNTIL < cnt)) { + for (int i = STEP_UNTIL; i < cnt; i++) { + write_index(idx, bits, i); + } + } + } + + this->tail += cnt; + } +#endif // SIMDJSON_GENERIC_JSON_STRUCTURAL_INDEXER_CUSTOM_BIT_INDEXER + + }; + + class json_structural_indexer { + public: + /** + * Find the important bits of JSON in a 128-byte chunk, and add them to structural_indexes. + * + * @param partial Setting the partial parameter to true allows the find_structural_bits to + * tolerate unclosed strings. The caller should still ensure that the input is valid UTF-8. If + * you are processing substrings, you may want to call on a function like trimmed_length_safe_utf8. + */ + template + static error_code index(const uint8_t* buf, size_t len, dom_parser_implementation& parser, stage1_mode partial) noexcept; + + private: + simdjson_inline json_structural_indexer(uint32_t* structural_indexes); + template + simdjson_inline void step(const uint8_t* block, buf_block_reader& reader) noexcept; + simdjson_inline void next(const simd::simd8x64& in, const json_block& block, size_t idx); + simdjson_inline error_code finish(dom_parser_implementation& parser, size_t idx, size_t len, stage1_mode partial); + + json_scanner scanner{}; + utf8_checker checker{}; + bit_indexer indexer; + uint64_t prev_structurals = 0; + uint64_t unescaped_chars_error = 0; + }; + + simdjson_inline json_structural_indexer::json_structural_indexer(uint32_t* structural_indexes) : indexer{ structural_indexes } {} + + // Skip the last character if it is partial + simdjson_inline size_t trim_partial_utf8(const uint8_t* buf, size_t len) { + if (simdjson_unlikely(len < 3)) { + switch (len) { + case 2: + if (buf[len - 1] >= 0xc0) { return len - 1; } // 2-, 3- and 4-byte characters with only 1 byte left + if (buf[len - 2] >= 0xe0) { return len - 2; } // 3- and 4-byte characters with only 2 bytes left + return len; + case 1: + if (buf[len - 1] >= 0xc0) { return len - 1; } // 2-, 3- and 4-byte characters with only 1 byte left + return len; + case 0: + return len; + } + } + if (buf[len - 1] >= 0xc0) { return len - 1; } // 2-, 3- and 4-byte characters with only 1 byte left + if (buf[len - 2] >= 0xe0) { return len - 2; } // 3- and 4-byte characters with only 1 byte left + if (buf[len - 3] >= 0xf0) { return len - 3; } // 4-byte characters with only 3 bytes left + return len; + } + + // + // PERF NOTES: + // We pipe 2 inputs through these stages: + // 1. Load JSON into registers. This takes a long time and is highly parallelizable, so we load + // 2 inputs' worth at once so that by the time step 2 is looking for them input, it's available. + // 2. Scan the JSON for critical data: strings, scalars and operators. This is the critical path. + // The output of step 1 depends entirely on this information. These functions don't quite use + // up enough CPU: the second half of the functions is highly serial, only using 1 execution core + // at a time. The second input's scans has some dependency on the first ones finishing it, but + // they can make a lot of progress before they need that information. + // 3. Step 1 does not use enough capacity, so we run some extra stuff while we're waiting for that + // to finish: utf-8 checks and generating the output from the last iteration. + // + // The reason we run 2 inputs at a time, is steps 2 and 3 are *still* not enough to soak up all + // available capacity with just one input. Running 2 at a time seems to give the CPU a good enough + // workout. + // + template + error_code json_structural_indexer::index(const uint8_t* buf, size_t len, dom_parser_implementation& parser, stage1_mode partial) noexcept { + if (simdjson_unlikely(len > parser.capacity())) { return CAPACITY; } + // We guard the rest of the code so that we can assume that len > 0 throughout. + if (len == 0) { return EMPTY; } + if (is_streaming(partial)) { + len = trim_partial_utf8(buf, len); + // If you end up with an empty window after trimming + // the partial UTF-8 bytes, then chances are good that you + // have an UTF-8 formatting error. + if (len == 0) { return UTF8_ERROR; } + } + buf_block_reader reader(buf, len); + json_structural_indexer indexer(parser.structural_indexes.get()); + + // Read all but the last block + while (reader.has_full_block()) { + indexer.step(reader.full_block(), reader); + } + // Take care of the last block (will always be there unless file is empty which is + // not supposed to happen.) + uint8_t block[STEP_SIZE]; + if (simdjson_unlikely(reader.get_remainder(block) == 0)) { return UNEXPECTED_ERROR; } + indexer.step(block, reader); + return indexer.finish(parser, reader.block_index(), len, partial); + } + + template<> + simdjson_inline void json_structural_indexer::step<128>(const uint8_t* block, buf_block_reader<128>& reader) noexcept { + simd::simd8x64 in_1(block); + simd::simd8x64 in_2(block + 64); + json_block block_1 = scanner.next(in_1); + json_block block_2 = scanner.next(in_2); + this->next(in_1, block_1, reader.block_index()); + this->next(in_2, block_2, reader.block_index() + 64); + reader.advance(); + } + + template<> + simdjson_inline void json_structural_indexer::step<64>(const uint8_t* block, buf_block_reader<64>& reader) noexcept { + simd::simd8x64 in_1(block); + json_block block_1 = scanner.next(in_1); + this->next(in_1, block_1, reader.block_index()); + reader.advance(); + } + + simdjson_inline void json_structural_indexer::next(const simd::simd8x64& in, const json_block& block, size_t idx) { + uint64_t unescaped = in.lteq(0x1F); +#if SIMDJSON_UTF8VALIDATION + checker.check_next_input(in); +#endif + indexer.write(uint32_t(idx - 64), prev_structurals); // Output *last* iteration's structurals to the parser + prev_structurals = block.structural_start(); + unescaped_chars_error |= block.non_quote_inside_string(unescaped); + } + + simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementation& parser, size_t idx, size_t len, stage1_mode partial) { + // Write out the final iteration's structurals + indexer.write(uint32_t(idx - 64), prev_structurals); + error_code error = scanner.finish(); + // We deliberately break down the next expression so that it is + // human readable. + const bool should_we_exit = is_streaming(partial) ? + ((error != SUCCESS) && (error != UNCLOSED_STRING)) // when partial we tolerate UNCLOSED_STRING + : (error != SUCCESS); // if partial is false, we must have SUCCESS + const bool have_unclosed_string = (error == UNCLOSED_STRING); + if (simdjson_unlikely(should_we_exit)) { return error; } + + if (unescaped_chars_error) { + return UNESCAPED_CHARS; + } + parser.n_structural_indexes = uint32_t(indexer.tail - parser.structural_indexes.get()); + /*** + * The On-Demand API requires special padding. + * + * This is related to https://github.com/simdjson/simdjson/issues/906 + * Basically, we want to make sure that if the parsing continues beyond the last (valid) + * structural character, it quickly stops. + * Only three structural characters can be repeated without triggering an error in JSON: [,] and }. + * We repeat the padding character (at 'len'). We don't know what it is, but if the parsing + * continues, then it must be [,] or }. + * Suppose it is ] or }. We backtrack to the first character, what could it be that would + * not trigger an error? It could be ] or } but no, because you can't start a document that way. + * It can't be a comma, a colon or any simple value. So the only way we could continue is + * if the repeated character is [. But if so, the document must start with [. But if the document + * starts with [, it should end with ]. If we enforce that rule, then we would get + * ][[ which is invalid. + * + * This is illustrated with the test array_iterate_unclosed_error() on the following input: + * R"({ "a": [,,)" + **/ + parser.structural_indexes[parser.n_structural_indexes] = uint32_t(len); // used later in partial == stage1_mode::streaming_final + parser.structural_indexes[parser.n_structural_indexes + 1] = uint32_t(len); + parser.structural_indexes[parser.n_structural_indexes + 2] = 0; + parser.next_structural_index = 0; + // a valid JSON file cannot have zero structural indexes - we should have found something + if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { + return EMPTY; + } + if (simdjson_unlikely(parser.structural_indexes[parser.n_structural_indexes - 1] > len)) { + return UNEXPECTED_ERROR; + } + if (partial == stage1_mode::streaming_partial) { + // If we have an unclosed string, then the last structural + // will be the quote and we want to make sure to omit it. + if (have_unclosed_string) { + parser.n_structural_indexes--; + // a valid JSON file cannot have zero structural indexes - we should have found something + if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return CAPACITY; } + } + // We truncate the input to the end of the last complete document (or zero). + auto new_structural_indexes = find_next_document_index(parser); + if (new_structural_indexes == 0 && parser.n_structural_indexes > 0) { + if (parser.structural_indexes[0] == 0) { + // If the buffer is partial and we started at index 0 but the document is + // incomplete, it's too big to parse. + return CAPACITY; + } + else { + // It is possible that the document could be parsed, we just had a lot + // of white space. + parser.n_structural_indexes = 0; + return EMPTY; + } + } + + parser.n_structural_indexes = new_structural_indexes; + } + else if (partial == stage1_mode::streaming_final) { + if (have_unclosed_string) { parser.n_structural_indexes--; } + // We truncate the input to the end of the last complete document (or zero). + // Because partial == stage1_mode::streaming_final, it means that we may + // silently ignore trailing garbage. Though it sounds bad, we do it + // deliberately because many people who have streams of JSON documents + // will truncate them for processing. E.g., imagine that you are uncompressing + // the data from a size file or receiving it in chunks from the network. You + // may not know where exactly the last document will be. Meanwhile the + // document_stream instances allow people to know the JSON documents they are + // parsing (see the iterator.source() method). + parser.n_structural_indexes = find_next_document_index(parser); + // We store the initial n_structural_indexes so that the client can see + // whether we used truncation. If initial_n_structural_indexes == parser.n_structural_indexes, + // then this will query parser.structural_indexes[parser.n_structural_indexes] which is len, + // otherwise, it will copy some prior index. + parser.structural_indexes[parser.n_structural_indexes + 1] = parser.structural_indexes[parser.n_structural_indexes]; + // This next line is critical, do not change it unless you understand what you are + // doing. + parser.structural_indexes[parser.n_structural_indexes] = uint32_t(len); + if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { + // We tolerate an unclosed string at the very end of the stream. Indeed, users + // often load their data in bulk without being careful and they want us to ignore + // the trailing garbage. + return EMPTY; + } + } + checker.check_eof(); + return checker.errors(); + } + + } // namespace stage1 + } // unnamed namespace + } // namespace westmere +} // namespace simdjson + +// Clear CUSTOM_BIT_INDEXER so other implementations can set it if they need to. +#undef SIMDJSON_GENERIC_JSON_STRUCTURAL_INDEXER_CUSTOM_BIT_INDEXER + +#endif // SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRUCTURAL_INDEXER_H +/* end file generic/stage1/json_structural_indexer.h for westmere */ +/* including generic/stage1/utf8_validator.h for westmere: #include */ +/* begin file generic/stage1/utf8_validator.h for westmere */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE1_UTF8_VALIDATOR_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_UTF8_VALIDATOR_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace westmere { + namespace { + namespace stage1 { + + /** + * Validates that the string is actual UTF-8. + */ + template + bool generic_validate_utf8(const uint8_t* input, size_t length) { + checker c{}; + buf_block_reader<64> reader(input, length); + while (reader.has_full_block()) { + simd::simd8x64 in(reader.full_block()); + c.check_next_input(in); + reader.advance(); + } + uint8_t block[64]{}; + reader.get_remainder(block); + simd::simd8x64 in(block); + c.check_next_input(in); + reader.advance(); + c.check_eof(); + return c.errors() == error_code::SUCCESS; + } + + bool generic_validate_utf8(const char* input, size_t length) { + return generic_validate_utf8(reinterpret_cast(input), length); + } + + } // namespace stage1 + } // unnamed namespace + } // namespace westmere +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE1_UTF8_VALIDATOR_H +/* end file generic/stage1/utf8_validator.h for westmere */ +/* end file generic/stage1/amalgamated.h for westmere */ +/* including generic/stage2/amalgamated.h for westmere: #include */ +/* begin file generic/stage2/amalgamated.h for westmere */ +// Stuff other things depend on +/* including generic/stage2/base.h for westmere: #include */ +/* begin file generic/stage2/base.h for westmere */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE2_BASE_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_BASE_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace westmere { + namespace { + namespace stage2 { + + class json_iterator; + class structural_iterator; + struct tape_builder; + struct tape_writer; + + } // namespace stage2 + } // unnamed namespace + } // namespace westmere +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE2_BASE_H +/* end file generic/stage2/base.h for westmere */ +/* including generic/stage2/tape_writer.h for westmere: #include */ +/* begin file generic/stage2/tape_writer.h for westmere */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE2_TAPE_WRITER_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_TAPE_WRITER_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +#include + +namespace simdjson { + namespace westmere { + namespace { + namespace stage2 { + + struct tape_writer { + /** The next place to write to tape */ + uint64_t* next_tape_loc; + + /** Write a signed 64-bit value to tape. */ + simdjson_inline void append_s64(int64_t value) noexcept; + + /** Write an unsigned 64-bit value to tape. */ + simdjson_inline void append_u64(uint64_t value) noexcept; + + /** Write a double value to tape. */ + simdjson_inline void append_double(double value) noexcept; + + /** + * Append a tape entry (an 8-bit type,and 56 bits worth of value). + */ + simdjson_inline void append(uint64_t val, internal::tape_type t) noexcept; + + /** + * Skip the current tape entry without writing. + * + * Used to skip the start of the container, since we'll come back later to fill it in when the + * container ends. + */ + simdjson_inline void skip() noexcept; + + /** + * Skip the number of tape entries necessary to write a large u64 or i64. + */ + simdjson_inline void skip_large_integer() noexcept; + + /** + * Skip the number of tape entries necessary to write a double. + */ + simdjson_inline void skip_double() noexcept; + + /** + * Write a value to a known location on tape. + * + * Used to go back and write out the start of a container after the container ends. + */ + simdjson_inline static void write(uint64_t& tape_loc, uint64_t val, internal::tape_type t) noexcept; + + private: + /** + * Append both the tape entry, and a supplementary value following it. Used for types that need + * all 64 bits, such as double and uint64_t. + */ + template + simdjson_inline void append2(uint64_t val, T val2, internal::tape_type t) noexcept; + }; // struct tape_writer + + simdjson_inline void tape_writer::append_s64(int64_t value) noexcept { + append2(0, value, internal::tape_type::INT64); + } + + simdjson_inline void tape_writer::append_u64(uint64_t value) noexcept { + append(0, internal::tape_type::UINT64); + *next_tape_loc = value; + next_tape_loc++; + } + + /** Write a double value to tape. */ + simdjson_inline void tape_writer::append_double(double value) noexcept { + append2(0, value, internal::tape_type::DOUBLE); + } + + simdjson_inline void tape_writer::skip() noexcept { + next_tape_loc++; + } + + simdjson_inline void tape_writer::skip_large_integer() noexcept { + next_tape_loc += 2; + } + + simdjson_inline void tape_writer::skip_double() noexcept { + next_tape_loc += 2; + } + + simdjson_inline void tape_writer::append(uint64_t val, internal::tape_type t) noexcept { + *next_tape_loc = val | ((uint64_t(char(t))) << 56); + next_tape_loc++; + } + + template + simdjson_inline void tape_writer::append2(uint64_t val, T val2, internal::tape_type t) noexcept { + append(val, t); + static_assert(sizeof(val2) == sizeof(*next_tape_loc), "Type is not 64 bits!"); + memcpy(next_tape_loc, &val2, sizeof(val2)); + next_tape_loc++; + } + + simdjson_inline void tape_writer::write(uint64_t& tape_loc, uint64_t val, internal::tape_type t) noexcept { + tape_loc = val | ((uint64_t(char(t))) << 56); + } + + } // namespace stage2 + } // unnamed namespace + } // namespace westmere +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE2_TAPE_WRITER_H +/* end file generic/stage2/tape_writer.h for westmere */ +/* including generic/stage2/logger.h for westmere: #include */ +/* begin file generic/stage2/logger.h for westmere */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE2_LOGGER_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_LOGGER_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +#include + + +// This is for an internal-only stage 2 specific logger. +// Set LOG_ENABLED = true to log what stage 2 is doing! +namespace simdjson { + namespace westmere { + namespace { + namespace logger { + + static constexpr const char* DASHES = "----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------"; + +#if SIMDJSON_VERBOSE_LOGGING + static constexpr const bool LOG_ENABLED = true; +#else + static constexpr const bool LOG_ENABLED = false; +#endif + static constexpr const int LOG_EVENT_LEN = 20; + static constexpr const int LOG_BUFFER_LEN = 30; + static constexpr const int LOG_SMALL_BUFFER_LEN = 10; + static constexpr const int LOG_INDEX_LEN = 5; + + static int log_depth; // Not threadsafe. Log only. + + // Helper to turn unprintable or newline characters into spaces + static simdjson_inline char printable_char(char c) { + if (c >= 0x20) { + return c; + } + else { + return ' '; + } + } + + // Print the header and set up log_start + static simdjson_inline void log_start() { + if (LOG_ENABLED) { + log_depth = 0; + printf("\n"); + printf("| %-*s | %-*s | %-*s | %-*s | Detail |\n", LOG_EVENT_LEN, "Event", LOG_BUFFER_LEN, "Buffer", LOG_SMALL_BUFFER_LEN, "Next", 5, "Next#"); + printf("|%.*s|%.*s|%.*s|%.*s|--------|\n", LOG_EVENT_LEN + 2, DASHES, LOG_BUFFER_LEN + 2, DASHES, LOG_SMALL_BUFFER_LEN + 2, DASHES, 5 + 2, DASHES); + } + } + + simdjson_unused static simdjson_inline void log_string(const char* message) { + if (LOG_ENABLED) { + printf("%s\n", message); + } + } + + // Logs a single line from the stage 2 DOM parser + template + static simdjson_inline void log_line(S& structurals, const char* title_prefix, const char* title, const char* detail) { + if (LOG_ENABLED) { + printf("| %*s%s%-*s ", log_depth * 2, "", title_prefix, LOG_EVENT_LEN - log_depth * 2 - int(strlen(title_prefix)), title); + auto current_index = structurals.at_beginning() ? nullptr : structurals.next_structural - 1; + auto next_index = structurals.next_structural; + auto current = current_index ? &structurals.buf[*current_index] : reinterpret_cast(" "); + auto next = &structurals.buf[*next_index]; + { + // Print the next N characters in the buffer. + printf("| "); + // Otherwise, print the characters starting from the buffer position. + // Print spaces for unprintable or newline characters. + for (int i = 0; i < LOG_BUFFER_LEN; i++) { + printf("%c", printable_char(current[i])); + } + printf(" "); + // Print the next N characters in the buffer. + printf("| "); + // Otherwise, print the characters starting from the buffer position. + // Print spaces for unprintable or newline characters. + for (int i = 0; i < LOG_SMALL_BUFFER_LEN; i++) { + printf("%c", printable_char(next[i])); + } + printf(" "); + } + if (current_index) { + printf("| %*u ", LOG_INDEX_LEN, *current_index); + } + else { + printf("| %-*s ", LOG_INDEX_LEN, ""); + } + // printf("| %*u ", LOG_INDEX_LEN, structurals.next_tape_index()); + printf("| %-s ", detail); + printf("|\n"); + } + } + + } // namespace logger + } // unnamed namespace + } // namespace westmere +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE2_LOGGER_H +/* end file generic/stage2/logger.h for westmere */ + +// All other declarations +/* including generic/stage2/json_iterator.h for westmere: #include */ +/* begin file generic/stage2/json_iterator.h for westmere */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE2_JSON_ITERATOR_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_JSON_ITERATOR_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace westmere { + namespace { + namespace stage2 { + + class json_iterator { + public: + const uint8_t* const buf; + uint32_t* next_structural; + dom_parser_implementation& dom_parser; + uint32_t depth{ 0 }; + + /** + * Walk the JSON document. + * + * The visitor receives callbacks when values are encountered. All callbacks pass the iterator as + * the first parameter; some callbacks have other parameters as well: + * + * - visit_document_start() - at the beginning. + * - visit_document_end() - at the end (if things were successful). + * + * - visit_array_start() - at the start `[` of a non-empty array. + * - visit_array_end() - at the end `]` of a non-empty array. + * - visit_empty_array() - when an empty array is encountered. + * + * - visit_object_end() - at the start `]` of a non-empty object. + * - visit_object_start() - at the end `]` of a non-empty object. + * - visit_empty_object() - when an empty object is encountered. + * - visit_key(const uint8_t *key) - when a key in an object field is encountered. key is + * guaranteed to point at the first quote of the string (`"key"`). + * - visit_primitive(const uint8_t *value) - when a value is a string, number, boolean or null. + * - visit_root_primitive(iter, uint8_t *value) - when the top-level value is a string, number, boolean or null. + * + * - increment_count(iter) - each time a value is found in an array or object. + */ + template + simdjson_warn_unused simdjson_inline error_code walk_document(V& visitor) noexcept; + + /** + * Create an iterator capable of walking a JSON document. + * + * The document must have already passed through stage 1. + */ + simdjson_inline json_iterator(dom_parser_implementation& _dom_parser, size_t start_structural_index); + + /** + * Look at the next token. + * + * Tokens can be strings, numbers, booleans, null, or operators (`[{]},:`)). + * + * They may include invalid JSON as well (such as `1.2.3` or `ture`). + */ + simdjson_inline const uint8_t* peek() const noexcept; + /** + * Advance to the next token. + * + * Tokens can be strings, numbers, booleans, null, or operators (`[{]},:`)). + * + * They may include invalid JSON as well (such as `1.2.3` or `ture`). + */ + simdjson_inline const uint8_t* advance() noexcept; + /** + * Get the remaining length of the document, from the start of the current token. + */ + simdjson_inline size_t remaining_len() const noexcept; + /** + * Check if we are at the end of the document. + * + * If this is true, there are no more tokens. + */ + simdjson_inline bool at_eof() const noexcept; + /** + * Check if we are at the beginning of the document. + */ + simdjson_inline bool at_beginning() const noexcept; + simdjson_inline uint8_t last_structural() const noexcept; + + /** + * Log that a value has been found. + * + * Set LOG_ENABLED=true in logger.h to see logging. + */ + simdjson_inline void log_value(const char* type) const noexcept; + /** + * Log the start of a multipart value. + * + * Set LOG_ENABLED=true in logger.h to see logging. + */ + simdjson_inline void log_start_value(const char* type) const noexcept; + /** + * Log the end of a multipart value. + * + * Set LOG_ENABLED=true in logger.h to see logging. + */ + simdjson_inline void log_end_value(const char* type) const noexcept; + /** + * Log an error. + * + * Set LOG_ENABLED=true in logger.h to see logging. + */ + simdjson_inline void log_error(const char* error) const noexcept; + + template + simdjson_warn_unused simdjson_inline error_code visit_root_primitive(V& visitor, const uint8_t* value) noexcept; + template + simdjson_warn_unused simdjson_inline error_code visit_primitive(V& visitor, const uint8_t* value) noexcept; + }; + + template + simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V& visitor) noexcept { + logger::log_start(); + + // + // Start the document + // + if (at_eof()) { return EMPTY; } + log_start_value("document"); + SIMDJSON_TRY(visitor.visit_document_start(*this)); + + // + // Read first value + // + { + auto value = advance(); + + // Make sure the outer object or array is closed before continuing; otherwise, there are ways we + // could get into memory corruption. See https://github.com/simdjson/simdjson/issues/906 + if (!STREAMING) { + switch (*value) { + case '{': if (last_structural() != '}') { log_value("starting brace unmatched"); return TAPE_ERROR; }; break; + case '[': if (last_structural() != ']') { log_value("starting bracket unmatched"); return TAPE_ERROR; }; break; + } + } + + switch (*value) { + case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY(visitor.visit_empty_object(*this)); break; } goto object_begin; + case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY(visitor.visit_empty_array(*this)); break; } goto array_begin; + default: SIMDJSON_TRY(visitor.visit_root_primitive(*this, value)); break; + } + } + goto document_end; + + // + // Object parser states + // + object_begin: + log_start_value("object"); + depth++; + if (depth >= dom_parser.max_depth()) { log_error("Exceeded max depth!"); return DEPTH_ERROR; } + dom_parser.is_array[depth] = false; + SIMDJSON_TRY(visitor.visit_object_start(*this)); + + { + auto key = advance(); + if (*key != '"') { log_error("Object does not start with a key"); return TAPE_ERROR; } + SIMDJSON_TRY(visitor.increment_count(*this)); + SIMDJSON_TRY(visitor.visit_key(*this, key)); + } + + object_field: + if (simdjson_unlikely(*advance() != ':')) { log_error("Missing colon after key in object"); return TAPE_ERROR; } + { + auto value = advance(); + switch (*value) { + case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY(visitor.visit_empty_object(*this)); break; } goto object_begin; + case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY(visitor.visit_empty_array(*this)); break; } goto array_begin; + default: SIMDJSON_TRY(visitor.visit_primitive(*this, value)); break; + } + } + + object_continue: + switch (*advance()) { + case ',': + SIMDJSON_TRY(visitor.increment_count(*this)); + { + auto key = advance(); + if (simdjson_unlikely(*key != '"')) { log_error("Key string missing at beginning of field in object"); return TAPE_ERROR; } + SIMDJSON_TRY(visitor.visit_key(*this, key)); + } + goto object_field; + case '}': log_end_value("object"); SIMDJSON_TRY(visitor.visit_object_end(*this)); goto scope_end; + default: log_error("No comma between object fields"); return TAPE_ERROR; + } + + scope_end: + depth--; + if (depth == 0) { goto document_end; } + if (dom_parser.is_array[depth]) { goto array_continue; } + goto object_continue; + + // + // Array parser states + // + array_begin: + log_start_value("array"); + depth++; + if (depth >= dom_parser.max_depth()) { log_error("Exceeded max depth!"); return DEPTH_ERROR; } + dom_parser.is_array[depth] = true; + SIMDJSON_TRY(visitor.visit_array_start(*this)); + SIMDJSON_TRY(visitor.increment_count(*this)); + + array_value: + { + auto value = advance(); + switch (*value) { + case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY(visitor.visit_empty_object(*this)); break; } goto object_begin; + case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY(visitor.visit_empty_array(*this)); break; } goto array_begin; + default: SIMDJSON_TRY(visitor.visit_primitive(*this, value)); break; + } + } + + array_continue: + switch (*advance()) { + case ',': SIMDJSON_TRY(visitor.increment_count(*this)); goto array_value; + case ']': log_end_value("array"); SIMDJSON_TRY(visitor.visit_array_end(*this)); goto scope_end; + default: log_error("Missing comma between array values"); return TAPE_ERROR; + } + + document_end: + log_end_value("document"); + SIMDJSON_TRY(visitor.visit_document_end(*this)); + + dom_parser.next_structural_index = uint32_t(next_structural - &dom_parser.structural_indexes[0]); + + // If we didn't make it to the end, it's an error + if (!STREAMING && dom_parser.next_structural_index != dom_parser.n_structural_indexes) { + log_error("More than one JSON value at the root of the document, or extra characters at the end of the JSON!"); + return TAPE_ERROR; + } + + return SUCCESS; + + } // walk_document() + + simdjson_inline json_iterator::json_iterator(dom_parser_implementation& _dom_parser, size_t start_structural_index) + : buf{ _dom_parser.buf }, + next_structural{ &_dom_parser.structural_indexes[start_structural_index] }, + dom_parser{ _dom_parser } { + } + + simdjson_inline const uint8_t* json_iterator::peek() const noexcept { + return &buf[*(next_structural)]; + } + simdjson_inline const uint8_t* json_iterator::advance() noexcept { + return &buf[*(next_structural++)]; + } + simdjson_inline size_t json_iterator::remaining_len() const noexcept { + return dom_parser.len - *(next_structural - 1); + } + + simdjson_inline bool json_iterator::at_eof() const noexcept { + return next_structural == &dom_parser.structural_indexes[dom_parser.n_structural_indexes]; + } + simdjson_inline bool json_iterator::at_beginning() const noexcept { + return next_structural == dom_parser.structural_indexes.get(); + } + simdjson_inline uint8_t json_iterator::last_structural() const noexcept { + return buf[dom_parser.structural_indexes[dom_parser.n_structural_indexes - 1]]; + } + + simdjson_inline void json_iterator::log_value(const char* type) const noexcept { + logger::log_line(*this, "", type, ""); + } + + simdjson_inline void json_iterator::log_start_value(const char* type) const noexcept { + logger::log_line(*this, "+", type, ""); + if (logger::LOG_ENABLED) { logger::log_depth++; } + } + + simdjson_inline void json_iterator::log_end_value(const char* type) const noexcept { + if (logger::LOG_ENABLED) { logger::log_depth--; } + logger::log_line(*this, "-", type, ""); + } + + simdjson_inline void json_iterator::log_error(const char* error) const noexcept { + logger::log_line(*this, "", "ERROR", error); + } + + template + simdjson_warn_unused simdjson_inline error_code json_iterator::visit_root_primitive(V& visitor, const uint8_t* value) noexcept { + switch (*value) { + case '"': return visitor.visit_root_string(*this, value); + case 't': return visitor.visit_root_true_atom(*this, value); + case 'f': return visitor.visit_root_false_atom(*this, value); + case 'n': return visitor.visit_root_null_atom(*this, value); + case '-': + case '0': case '1': case '2': case '3': case '4': + case '5': case '6': case '7': case '8': case '9': + return visitor.visit_root_number(*this, value); + default: + log_error("Document starts with a non-value character"); + return TAPE_ERROR; + } + } + template + simdjson_warn_unused simdjson_inline error_code json_iterator::visit_primitive(V& visitor, const uint8_t* value) noexcept { + // Use the fact that most scalars are going to be either strings or numbers. + if (*value == '"') { + return visitor.visit_string(*this, value); + } + else if (((*value - '0') < 10) || (*value == '-')) { + return visitor.visit_number(*this, value); + } + // true, false, null are uncommon. + switch (*value) { + case 't': return visitor.visit_true_atom(*this, value); + case 'f': return visitor.visit_false_atom(*this, value); + case 'n': return visitor.visit_null_atom(*this, value); + default: + log_error("Non-value found when value was expected!"); + return TAPE_ERROR; + } + } + + } // namespace stage2 + } // unnamed namespace + } // namespace westmere +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE2_JSON_ITERATOR_H +/* end file generic/stage2/json_iterator.h for westmere */ +/* including generic/stage2/stringparsing.h for westmere: #include */ +/* begin file generic/stage2/stringparsing.h for westmere */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +// This file contains the common code every implementation uses +// It is intended to be included multiple times and compiled multiple times + +namespace simdjson { + namespace westmere { + namespace { + /// @private + namespace stringparsing { + + // begin copypasta + // These chars yield themselves: " \ / + // b -> backspace, f -> formfeed, n -> newline, r -> cr, t -> horizontal tab + // u not handled in this table as it's complex + static const uint8_t escape_map[256] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x0. + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0x22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x2f, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x4. + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x5c, 0, 0, 0, // 0x5. + 0, 0, 0x08, 0, 0, 0, 0x0c, 0, 0, 0, 0, 0, 0, 0, 0x0a, 0, // 0x6. + 0, 0, 0x0d, 0, 0x09, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x7. + + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + }; + + // handle a unicode codepoint + // write appropriate values into dest + // src will advance 6 bytes or 12 bytes + // dest will advance a variable amount (return via pointer) + // return true if the unicode codepoint was valid + // We work in little-endian then swap at write time + simdjson_warn_unused + simdjson_inline bool handle_unicode_codepoint(const uint8_t** src_ptr, + uint8_t** dst_ptr, bool allow_replacement) { + // Use the default Unicode Character 'REPLACEMENT CHARACTER' (U+FFFD) + constexpr uint32_t substitution_code_point = 0xfffd; + // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the + // conversion is not valid; we defer the check for this to inside the + // multilingual plane check. + uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2); + *src_ptr += 6; + + // If we found a high surrogate, we must + // check for low surrogate for characters + // outside the Basic + // Multilingual Plane. + if (code_point >= 0xd800 && code_point < 0xdc00) { + const uint8_t* src_data = *src_ptr; + /* Compiler optimizations convert this to a single 16-bit load and compare on most platforms */ + if (((src_data[0] << 8) | src_data[1]) != ((static_cast ('\\') << 8) | static_cast ('u'))) { + if (!allow_replacement) { return false; } + code_point = substitution_code_point; + } + else { + uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(src_data + 2); + + // We have already checked that the high surrogate is valid and + // (code_point - 0xd800) < 1024. + // + // Check that code_point_2 is in the range 0xdc00..0xdfff + // and that code_point_2 was parsed from valid hex. + uint32_t low_bit = code_point_2 - 0xdc00; + if (low_bit >> 10) { + if (!allow_replacement) { return false; } + code_point = substitution_code_point; + } + else { + code_point = (((code_point - 0xd800) << 10) | low_bit) + 0x10000; + *src_ptr += 6; + } + + } + } + else if (code_point >= 0xdc00 && code_point <= 0xdfff) { + // If we encounter a low surrogate (not preceded by a high surrogate) + // then we have an error. + if (!allow_replacement) { return false; } + code_point = substitution_code_point; + } + size_t offset = jsoncharutils::codepoint_to_utf8(code_point, *dst_ptr); + *dst_ptr += offset; + return offset > 0; + } + + + // handle a unicode codepoint using the wobbly convention + // https://simonsapin.github.io/wtf-8/ + // write appropriate values into dest + // src will advance 6 bytes or 12 bytes + // dest will advance a variable amount (return via pointer) + // return true if the unicode codepoint was valid + // We work in little-endian then swap at write time + simdjson_warn_unused + simdjson_inline bool handle_unicode_codepoint_wobbly(const uint8_t** src_ptr, + uint8_t** dst_ptr) { + // It is not ideal that this function is nearly identical to handle_unicode_codepoint. + // + // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the + // conversion is not valid; we defer the check for this to inside the + // multilingual plane check. + uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2); + *src_ptr += 6; + // If we found a high surrogate, we must + // check for low surrogate for characters + // outside the Basic + // Multilingual Plane. + if (code_point >= 0xd800 && code_point < 0xdc00) { + const uint8_t* src_data = *src_ptr; + /* Compiler optimizations convert this to a single 16-bit load and compare on most platforms */ + if (((src_data[0] << 8) | src_data[1]) == ((static_cast ('\\') << 8) | static_cast ('u'))) { + uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(src_data + 2); + uint32_t low_bit = code_point_2 - 0xdc00; + if ((low_bit >> 10) == 0) { + code_point = + (((code_point - 0xd800) << 10) | low_bit) + 0x10000; + *src_ptr += 6; + } + } + } + + size_t offset = jsoncharutils::codepoint_to_utf8(code_point, *dst_ptr); + *dst_ptr += offset; + return offset > 0; + } + + + /** + * Unescape a valid UTF-8 string from src to dst, stopping at a final unescaped quote. There + * must be an unescaped quote terminating the string. It returns the final output + * position as pointer. In case of error (e.g., the string has bad escaped codes), + * then null_ptr is returned. It is assumed that the output buffer is large + * enough. E.g., if src points at 'joe"', then dst needs to have four free bytes + + * SIMDJSON_PADDING bytes. + */ + simdjson_warn_unused simdjson_inline uint8_t* parse_string(const uint8_t* src, uint8_t* dst, bool allow_replacement) { + while (1) { + // Copy the next n bytes, and find the backslash and quote in them. + auto bs_quote = backslash_and_quote::copy_and_find(src, dst); + // If the next thing is the end quote, copy and return + if (bs_quote.has_quote_first()) { + // we encountered quotes first. Move dst to point to quotes and exit + return dst + bs_quote.quote_index(); + } + if (bs_quote.has_backslash()) { + /* find out where the backspace is */ + auto bs_dist = bs_quote.backslash_index(); + uint8_t escape_char = src[bs_dist + 1]; + /* we encountered backslash first. Handle backslash */ + if (escape_char == 'u') { + /* move src/dst up to the start; they will be further adjusted + within the unicode codepoint handling code. */ + src += bs_dist; + dst += bs_dist; + if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { + return nullptr; + } + } + else { + /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and + * write bs_dist+1 characters to output + * note this may reach beyond the part of the buffer we've actually + * seen. I think this is ok */ + uint8_t escape_result = escape_map[escape_char]; + if (escape_result == 0u) { + return nullptr; /* bogus escape value is an error */ + } + dst[bs_dist] = escape_result; + src += bs_dist + 2; + dst += bs_dist + 1; + } + } + else { + /* they are the same. Since they can't co-occur, it means we + * encountered neither. */ + src += backslash_and_quote::BYTES_PROCESSED; + dst += backslash_and_quote::BYTES_PROCESSED; + } + } + } + + simdjson_warn_unused simdjson_inline uint8_t* parse_wobbly_string(const uint8_t* src, uint8_t* dst) { + // It is not ideal that this function is nearly identical to parse_string. + while (1) { + // Copy the next n bytes, and find the backslash and quote in them. + auto bs_quote = backslash_and_quote::copy_and_find(src, dst); + // If the next thing is the end quote, copy and return + if (bs_quote.has_quote_first()) { + // we encountered quotes first. Move dst to point to quotes and exit + return dst + bs_quote.quote_index(); + } + if (bs_quote.has_backslash()) { + /* find out where the backspace is */ + auto bs_dist = bs_quote.backslash_index(); + uint8_t escape_char = src[bs_dist + 1]; + /* we encountered backslash first. Handle backslash */ + if (escape_char == 'u') { + /* move src/dst up to the start; they will be further adjusted + within the unicode codepoint handling code. */ + src += bs_dist; + dst += bs_dist; + if (!handle_unicode_codepoint_wobbly(&src, &dst)) { + return nullptr; + } + } + else { + /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and + * write bs_dist+1 characters to output + * note this may reach beyond the part of the buffer we've actually + * seen. I think this is ok */ + uint8_t escape_result = escape_map[escape_char]; + if (escape_result == 0u) { + return nullptr; /* bogus escape value is an error */ + } + dst[bs_dist] = escape_result; + src += bs_dist + 2; + dst += bs_dist + 1; + } + } + else { + /* they are the same. Since they can't co-occur, it means we + * encountered neither. */ + src += backslash_and_quote::BYTES_PROCESSED; + dst += backslash_and_quote::BYTES_PROCESSED; + } + } + } + + } // namespace stringparsing + } // unnamed namespace + } // namespace westmere +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H +/* end file generic/stage2/stringparsing.h for westmere */ +/* including generic/stage2/structural_iterator.h for westmere: #include */ +/* begin file generic/stage2/structural_iterator.h for westmere */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE2_STRUCTURAL_ITERATOR_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_STRUCTURAL_ITERATOR_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace westmere { + namespace { + namespace stage2 { + + class structural_iterator { + public: + const uint8_t* const buf; + uint32_t* next_structural; + dom_parser_implementation& dom_parser; + + // Start a structural + simdjson_inline structural_iterator(dom_parser_implementation& _dom_parser, size_t start_structural_index) + : buf{ _dom_parser.buf }, + next_structural{ &_dom_parser.structural_indexes[start_structural_index] }, + dom_parser{ _dom_parser } { + } + // Get the buffer position of the current structural character + simdjson_inline const uint8_t* current() { + return &buf[*(next_structural - 1)]; + } + // Get the current structural character + simdjson_inline char current_char() { + return buf[*(next_structural - 1)]; + } + // Get the next structural character without advancing + simdjson_inline char peek_next_char() { + return buf[*next_structural]; + } + simdjson_inline const uint8_t* peek() { + return &buf[*next_structural]; + } + simdjson_inline const uint8_t* advance() { + return &buf[*(next_structural++)]; + } + simdjson_inline char advance_char() { + return buf[*(next_structural++)]; + } + simdjson_inline size_t remaining_len() { + return dom_parser.len - *(next_structural - 1); + } + + simdjson_inline bool at_end() { + return next_structural == &dom_parser.structural_indexes[dom_parser.n_structural_indexes]; + } + simdjson_inline bool at_beginning() { + return next_structural == dom_parser.structural_indexes.get(); + } + }; + + } // namespace stage2 + } // unnamed namespace + } // namespace westmere +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE2_STRUCTURAL_ITERATOR_H +/* end file generic/stage2/structural_iterator.h for westmere */ +/* including generic/stage2/tape_builder.h for westmere: #include */ +/* begin file generic/stage2/tape_builder.h for westmere */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE2_TAPE_BUILDER_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_TAPE_BUILDER_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + + +namespace simdjson { + namespace westmere { + namespace { + namespace stage2 { + + struct tape_builder { + template + simdjson_warn_unused static simdjson_inline error_code parse_document( + dom_parser_implementation& dom_parser, + dom::document& doc) noexcept; + + /** Called when a non-empty document starts. */ + simdjson_warn_unused simdjson_inline error_code visit_document_start(json_iterator& iter) noexcept; + /** Called when a non-empty document ends without error. */ + simdjson_warn_unused simdjson_inline error_code visit_document_end(json_iterator& iter) noexcept; + + /** Called when a non-empty array starts. */ + simdjson_warn_unused simdjson_inline error_code visit_array_start(json_iterator& iter) noexcept; + /** Called when a non-empty array ends. */ + simdjson_warn_unused simdjson_inline error_code visit_array_end(json_iterator& iter) noexcept; + /** Called when an empty array is found. */ + simdjson_warn_unused simdjson_inline error_code visit_empty_array(json_iterator& iter) noexcept; + + /** Called when a non-empty object starts. */ + simdjson_warn_unused simdjson_inline error_code visit_object_start(json_iterator& iter) noexcept; + /** + * Called when a key in a field is encountered. + * + * primitive, visit_object_start, visit_empty_object, visit_array_start, or visit_empty_array + * will be called after this with the field value. + */ + simdjson_warn_unused simdjson_inline error_code visit_key(json_iterator& iter, const uint8_t* key) noexcept; + /** Called when a non-empty object ends. */ + simdjson_warn_unused simdjson_inline error_code visit_object_end(json_iterator& iter) noexcept; + /** Called when an empty object is found. */ + simdjson_warn_unused simdjson_inline error_code visit_empty_object(json_iterator& iter) noexcept; + + /** + * Called when a string, number, boolean or null is found. + */ + simdjson_warn_unused simdjson_inline error_code visit_primitive(json_iterator& iter, const uint8_t* value) noexcept; + /** + * Called when a string, number, boolean or null is found at the top level of a document (i.e. + * when there is no array or object and the entire document is a single string, number, boolean or + * null. + * + * This is separate from primitive() because simdjson's normal primitive parsing routines assume + * there is at least one more token after the value, which is only true in an array or object. + */ + simdjson_warn_unused simdjson_inline error_code visit_root_primitive(json_iterator& iter, const uint8_t* value) noexcept; + + simdjson_warn_unused simdjson_inline error_code visit_string(json_iterator& iter, const uint8_t* value, bool key = false) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_number(json_iterator& iter, const uint8_t* value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_true_atom(json_iterator& iter, const uint8_t* value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_false_atom(json_iterator& iter, const uint8_t* value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_null_atom(json_iterator& iter, const uint8_t* value) noexcept; + + simdjson_warn_unused simdjson_inline error_code visit_root_string(json_iterator& iter, const uint8_t* value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_root_number(json_iterator& iter, const uint8_t* value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_root_true_atom(json_iterator& iter, const uint8_t* value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_root_false_atom(json_iterator& iter, const uint8_t* value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_root_null_atom(json_iterator& iter, const uint8_t* value) noexcept; + + /** Called each time a new field or element in an array or object is found. */ + simdjson_warn_unused simdjson_inline error_code increment_count(json_iterator& iter) noexcept; + + /** Next location to write to tape */ + tape_writer tape; + private: + /** Next write location in the string buf for stage 2 parsing */ + uint8_t* current_string_buf_loc; + + simdjson_inline tape_builder(dom::document& doc) noexcept; + + simdjson_inline uint32_t next_tape_index(json_iterator& iter) const noexcept; + simdjson_inline void start_container(json_iterator& iter) noexcept; + simdjson_warn_unused simdjson_inline error_code end_container(json_iterator& iter, internal::tape_type start, internal::tape_type end) noexcept; + simdjson_warn_unused simdjson_inline error_code empty_container(json_iterator& iter, internal::tape_type start, internal::tape_type end) noexcept; + simdjson_inline uint8_t* on_start_string(json_iterator& iter) noexcept; + simdjson_inline void on_end_string(uint8_t* dst) noexcept; + }; // struct tape_builder + + template + simdjson_warn_unused simdjson_inline error_code tape_builder::parse_document( + dom_parser_implementation& dom_parser, + dom::document& doc) noexcept { + dom_parser.doc = &doc; + json_iterator iter(dom_parser, STREAMING ? dom_parser.next_structural_index : 0); + tape_builder builder(doc); + return iter.walk_document(builder); + } + + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_primitive(json_iterator& iter, const uint8_t* value) noexcept { + return iter.visit_root_primitive(*this, value); + } + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_primitive(json_iterator& iter, const uint8_t* value) noexcept { + return iter.visit_primitive(*this, value); + } + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_empty_object(json_iterator& iter) noexcept { + return empty_container(iter, internal::tape_type::START_OBJECT, internal::tape_type::END_OBJECT); + } + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_empty_array(json_iterator& iter) noexcept { + return empty_container(iter, internal::tape_type::START_ARRAY, internal::tape_type::END_ARRAY); + } + + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_document_start(json_iterator& iter) noexcept { + start_container(iter); + return SUCCESS; + } + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_object_start(json_iterator& iter) noexcept { + start_container(iter); + return SUCCESS; + } + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_array_start(json_iterator& iter) noexcept { + start_container(iter); + return SUCCESS; + } + + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_object_end(json_iterator& iter) noexcept { + return end_container(iter, internal::tape_type::START_OBJECT, internal::tape_type::END_OBJECT); + } + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_array_end(json_iterator& iter) noexcept { + return end_container(iter, internal::tape_type::START_ARRAY, internal::tape_type::END_ARRAY); + } + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_document_end(json_iterator& iter) noexcept { + constexpr uint32_t start_tape_index = 0; + tape.append(start_tape_index, internal::tape_type::ROOT); + tape_writer::write(iter.dom_parser.doc->tape[start_tape_index], next_tape_index(iter), internal::tape_type::ROOT); + return SUCCESS; + } + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_key(json_iterator& iter, const uint8_t* key) noexcept { + return visit_string(iter, key, true); + } + + simdjson_warn_unused simdjson_inline error_code tape_builder::increment_count(json_iterator& iter) noexcept { + iter.dom_parser.open_containers[iter.depth].count++; // we have a key value pair in the object at parser.dom_parser.depth - 1 + return SUCCESS; + } + + simdjson_inline tape_builder::tape_builder(dom::document& doc) noexcept : tape{ doc.tape.get() }, current_string_buf_loc{ doc.string_buf.get() } {} + + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_string(json_iterator& iter, const uint8_t* value, bool key) noexcept { + iter.log_value(key ? "key" : "string"); + uint8_t* dst = on_start_string(iter); + dst = stringparsing::parse_string(value + 1, dst, false); // We do not allow replacement when the escape characters are invalid. + if (dst == nullptr) { + iter.log_error("Invalid escape in string"); + return STRING_ERROR; + } + on_end_string(dst); + return SUCCESS; + } + + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_string(json_iterator& iter, const uint8_t* value) noexcept { + return visit_string(iter, value); + } + + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_number(json_iterator& iter, const uint8_t* value) noexcept { + iter.log_value("number"); + return numberparsing::parse_number(value, tape); + } + + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_number(json_iterator& iter, const uint8_t* value) noexcept { + // + // We need to make a copy to make sure that the string is space terminated. + // This is not about padding the input, which should already padded up + // to len + SIMDJSON_PADDING. However, we have no control at this stage + // on how the padding was done. What if the input string was padded with nulls? + // It is quite common for an input string to have an extra null character (C string). + // We do not want to allow 9\0 (where \0 is the null character) inside a JSON + // document, but the string "9\0" by itself is fine. So we make a copy and + // pad the input with spaces when we know that there is just one input element. + // This copy is relatively expensive, but it will almost never be called in + // practice unless you are in the strange scenario where you have many JSON + // documents made of single atoms. + // + std::unique_ptrcopy(new (std::nothrow) uint8_t[iter.remaining_len() + SIMDJSON_PADDING]); + if (copy.get() == nullptr) { return MEMALLOC; } + std::memcpy(copy.get(), value, iter.remaining_len()); + std::memset(copy.get() + iter.remaining_len(), ' ', SIMDJSON_PADDING); + error_code error = visit_number(iter, copy.get()); + return error; + } + + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_true_atom(json_iterator& iter, const uint8_t* value) noexcept { + iter.log_value("true"); + if (!atomparsing::is_valid_true_atom(value)) { return T_ATOM_ERROR; } + tape.append(0, internal::tape_type::TRUE_VALUE); + return SUCCESS; + } + + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_true_atom(json_iterator& iter, const uint8_t* value) noexcept { + iter.log_value("true"); + if (!atomparsing::is_valid_true_atom(value, iter.remaining_len())) { return T_ATOM_ERROR; } + tape.append(0, internal::tape_type::TRUE_VALUE); + return SUCCESS; + } + + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_false_atom(json_iterator& iter, const uint8_t* value) noexcept { + iter.log_value("false"); + if (!atomparsing::is_valid_false_atom(value)) { return F_ATOM_ERROR; } + tape.append(0, internal::tape_type::FALSE_VALUE); + return SUCCESS; + } + + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_false_atom(json_iterator& iter, const uint8_t* value) noexcept { + iter.log_value("false"); + if (!atomparsing::is_valid_false_atom(value, iter.remaining_len())) { return F_ATOM_ERROR; } + tape.append(0, internal::tape_type::FALSE_VALUE); + return SUCCESS; + } + + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_null_atom(json_iterator& iter, const uint8_t* value) noexcept { + iter.log_value("null"); + if (!atomparsing::is_valid_null_atom(value)) { return N_ATOM_ERROR; } + tape.append(0, internal::tape_type::NULL_VALUE); + return SUCCESS; + } + + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_null_atom(json_iterator& iter, const uint8_t* value) noexcept { + iter.log_value("null"); + if (!atomparsing::is_valid_null_atom(value, iter.remaining_len())) { return N_ATOM_ERROR; } + tape.append(0, internal::tape_type::NULL_VALUE); + return SUCCESS; + } + + // private: + + simdjson_inline uint32_t tape_builder::next_tape_index(json_iterator& iter) const noexcept { + return uint32_t(tape.next_tape_loc - iter.dom_parser.doc->tape.get()); + } + + simdjson_warn_unused simdjson_inline error_code tape_builder::empty_container(json_iterator& iter, internal::tape_type start, internal::tape_type end) noexcept { + auto start_index = next_tape_index(iter); + tape.append(start_index + 2, start); + tape.append(start_index, end); + return SUCCESS; + } + + simdjson_inline void tape_builder::start_container(json_iterator& iter) noexcept { + iter.dom_parser.open_containers[iter.depth].tape_index = next_tape_index(iter); + iter.dom_parser.open_containers[iter.depth].count = 0; + tape.skip(); // We don't actually *write* the start element until the end. + } + + simdjson_warn_unused simdjson_inline error_code tape_builder::end_container(json_iterator& iter, internal::tape_type start, internal::tape_type end) noexcept { + // Write the ending tape element, pointing at the start location + const uint32_t start_tape_index = iter.dom_parser.open_containers[iter.depth].tape_index; + tape.append(start_tape_index, end); + // Write the start tape element, pointing at the end location (and including count) + // count can overflow if it exceeds 24 bits... so we saturate + // the convention being that a cnt of 0xffffff or more is undetermined in value (>= 0xffffff). + const uint32_t count = iter.dom_parser.open_containers[iter.depth].count; + const uint32_t cntsat = count > 0xFFFFFF ? 0xFFFFFF : count; + tape_writer::write(iter.dom_parser.doc->tape[start_tape_index], next_tape_index(iter) | (uint64_t(cntsat) << 32), start); + return SUCCESS; + } + + simdjson_inline uint8_t* tape_builder::on_start_string(json_iterator& iter) noexcept { + // we advance the point, accounting for the fact that we have a NULL termination + tape.append(current_string_buf_loc - iter.dom_parser.doc->string_buf.get(), internal::tape_type::STRING); + return current_string_buf_loc + sizeof(uint32_t); + } + + simdjson_inline void tape_builder::on_end_string(uint8_t* dst) noexcept { + uint32_t str_length = uint32_t(dst - (current_string_buf_loc + sizeof(uint32_t))); + // TODO check for overflow in case someone has a crazy string (>=4GB?) + // But only add the overflow check when the document itself exceeds 4GB + // Currently unneeded because we refuse to parse docs larger or equal to 4GB. + memcpy(current_string_buf_loc, &str_length, sizeof(uint32_t)); + // NULL termination is still handy if you expect all your strings to + // be NULL terminated? It comes at a small cost + *dst = 0; + current_string_buf_loc = dst + 1; + } + + } // namespace stage2 + } // unnamed namespace + } // namespace westmere +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE2_TAPE_BUILDER_H +/* end file generic/stage2/tape_builder.h for westmere */ +/* end file generic/stage2/amalgamated.h for westmere */ + +// +// Stage 1 +// + +namespace simdjson { + namespace westmere { + + simdjson_warn_unused error_code implementation::create_dom_parser_implementation( + size_t capacity, + size_t max_depth, + std::unique_ptr& dst + ) const noexcept { + dst.reset(new (std::nothrow) dom_parser_implementation()); + if (!dst) { return MEMALLOC; } + if (auto err = dst->set_capacity(capacity)) + return err; + if (auto err = dst->set_max_depth(max_depth)) + return err; + return SUCCESS; + } + + namespace { + + using namespace simd; + + simdjson_inline json_character_block json_character_block::classify(const simd::simd8x64& in) { + // These lookups rely on the fact that anything < 127 will match the lower 4 bits, which is why + // we can't use the generic lookup_16. + auto whitespace_table = simd8::repeat_16(' ', 100, 100, 100, 17, 100, 113, 2, 100, '\t', '\n', 112, 100, '\r', 100, 100); + + // The 6 operators (:,[]{}) have these values: + // + // , 2C + // : 3A + // [ 5B + // { 7B + // ] 5D + // } 7D + // + // If you use | 0x20 to turn [ and ] into { and }, the lower 4 bits of each character is unique. + // We exploit this, using a simd 4-bit lookup to tell us which character match against, and then + // match it (against | 0x20). + // + // To prevent recognizing other characters, everything else gets compared with 0, which cannot + // match due to the | 0x20. + // + // NOTE: Due to the | 0x20, this ALSO treats and (control characters 0C and 1A) like , + // and :. This gets caught in stage 2, which checks the actual character to ensure the right + // operators are in the right places. + const auto op_table = simd8::repeat_16( + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 0, ':', '{', // : = 3A, [ = 5B, { = 7B + ',', '}', 0, 0 // , = 2C, ] = 5D, } = 7D + ); + + // We compute whitespace and op separately. If the code later only use one or the + // other, given the fact that all functions are aggressively inlined, we can + // hope that useless computations will be omitted. This is namely case when + // minifying (we only need whitespace). + + + const uint64_t whitespace = in.eq({ + _mm_shuffle_epi8(whitespace_table, in.chunks[0]), + _mm_shuffle_epi8(whitespace_table, in.chunks[1]), + _mm_shuffle_epi8(whitespace_table, in.chunks[2]), + _mm_shuffle_epi8(whitespace_table, in.chunks[3]) + }); + // Turn [ and ] into { and } + const simd8x64 curlified{ + in.chunks[0] | 0x20, + in.chunks[1] | 0x20, + in.chunks[2] | 0x20, + in.chunks[3] | 0x20 + }; + const uint64_t op = curlified.eq({ + _mm_shuffle_epi8(op_table, in.chunks[0]), + _mm_shuffle_epi8(op_table, in.chunks[1]), + _mm_shuffle_epi8(op_table, in.chunks[2]), + _mm_shuffle_epi8(op_table, in.chunks[3]) + }); + return { whitespace, op }; + } + + simdjson_inline bool is_ascii(const simd8x64& input) { + return input.reduce_or().is_ascii(); + } + + simdjson_unused simdjson_inline simd8 must_be_continuation(const simd8 prev1, const simd8 prev2, const simd8 prev3) { + simd8 is_second_byte = prev1.saturating_sub(0xc0u - 1); // Only 11______ will be > 0 + simd8 is_third_byte = prev2.saturating_sub(0xe0u - 1); // Only 111_____ will be > 0 + simd8 is_fourth_byte = prev3.saturating_sub(0xf0u - 1); // Only 1111____ will be > 0 + // Caller requires a bool (all 1's). All values resulting from the subtraction will be <= 64, so signed comparison is fine. + return simd8(is_second_byte | is_third_byte | is_fourth_byte) > int8_t(0); + } + + simdjson_inline simd8 must_be_2_3_continuation(const simd8 prev2, const simd8 prev3) { + simd8 is_third_byte = prev2.saturating_sub(0xe0u - 0x80); // Only 111_____ will be >= 0x80 + simd8 is_fourth_byte = prev3.saturating_sub(0xf0u - 0x80); // Only 1111____ will be >= 0x80 + return is_third_byte | is_fourth_byte; + } + + } // unnamed namespace + } // namespace westmere +} // namespace simdjson + +// +// Stage 2 +// + +// +// Implementation-specific overrides +// + +namespace simdjson { + namespace westmere { + + simdjson_warn_unused error_code implementation::minify(const uint8_t* buf, size_t len, uint8_t* dst, size_t& dst_len) const noexcept { + return westmere::stage1::json_minifier::minify<64>(buf, len, dst, dst_len); + } + + simdjson_warn_unused error_code dom_parser_implementation::stage1(const uint8_t* _buf, size_t _len, stage1_mode streaming) noexcept { + this->buf = _buf; + this->len = _len; + return westmere::stage1::json_structural_indexer::index<64>(_buf, _len, *this, streaming); + } + + simdjson_warn_unused bool implementation::validate_utf8(const char* buf, size_t len) const noexcept { + return westmere::stage1::generic_validate_utf8(buf, len); + } + + simdjson_warn_unused error_code dom_parser_implementation::stage2(dom::document& _doc) noexcept { + return stage2::tape_builder::parse_document(*this, _doc); + } + + simdjson_warn_unused error_code dom_parser_implementation::stage2_next(dom::document& _doc) noexcept { + return stage2::tape_builder::parse_document(*this, _doc); + } + + simdjson_warn_unused uint8_t* dom_parser_implementation::parse_string(const uint8_t* src, uint8_t* dst, bool replacement_char) const noexcept { + return westmere::stringparsing::parse_string(src, dst, replacement_char); + } + + simdjson_warn_unused uint8_t* dom_parser_implementation::parse_wobbly_string(const uint8_t* src, uint8_t* dst) const noexcept { + return westmere::stringparsing::parse_wobbly_string(src, dst); + } + + simdjson_warn_unused error_code dom_parser_implementation::parse(const uint8_t* _buf, size_t _len, dom::document& _doc) noexcept { + auto error = stage1(_buf, _len, stage1_mode::regular); + if (error) { return error; } + return stage2(_doc); + } + + } // namespace westmere +} // namespace simdjson + +/* including simdjson/westmere/end.h: #include */ +/* begin file simdjson/westmere/end.h */ +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/westmere/base.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +#if !SIMDJSON_CAN_ALWAYS_RUN_WESTMERE +SIMDJSON_UNTARGET_REGION +#endif + +/* undefining SIMDJSON_IMPLEMENTATION from "westmere" */ +#undef SIMDJSON_IMPLEMENTATION +/* end file simdjson/westmere/end.h */ + +#endif // SIMDJSON_SRC_WESTMERE_CPP +/* end file westmere.cpp */ +#endif +#if SIMDJSON_IMPLEMENTATION_LSX +/* including lsx.cpp: #include */ +/* begin file lsx.cpp */ +#ifndef SIMDJSON_SRC_LSX_CPP +#define SIMDJSON_SRC_LSX_CPP + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +/* including simdjson/lsx.h: #include */ +/* begin file simdjson/lsx.h */ +#ifndef SIMDJSON_LSX_H +#define SIMDJSON_LSX_H + +/* including simdjson/lsx/begin.h: #include "simdjson/lsx/begin.h" */ +/* begin file simdjson/lsx/begin.h */ +/* defining SIMDJSON_IMPLEMENTATION to "lsx" */ +#define SIMDJSON_IMPLEMENTATION lsx +/* including simdjson/lsx/base.h: #include "simdjson/lsx/base.h" */ +/* begin file simdjson/lsx/base.h */ +#ifndef SIMDJSON_LSX_BASE_H +#define SIMDJSON_LSX_BASE_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/base.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + /** + * Implementation for LSX. + */ + namespace lsx { + + class implementation; + + namespace { + namespace simd { + template struct simd8; + template struct simd8x64; + } // namespace simd + } // unnamed namespace + + } // namespace lsx +} // namespace simdjson + +#endif // SIMDJSON_LSX_BASE_H +/* end file simdjson/lsx/base.h */ +/* including simdjson/lsx/intrinsics.h: #include "simdjson/lsx/intrinsics.h" */ +/* begin file simdjson/lsx/intrinsics.h */ +#ifndef SIMDJSON_LSX_INTRINSICS_H +#define SIMDJSON_LSX_INTRINSICS_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/lsx/base.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +// This should be the correct header whether +// you use visual studio or other compilers. +#include + +static_assert(sizeof(__m128i) <= simdjson::SIMDJSON_PADDING, "insufficient padding for LoongArch SX"); + +#endif // SIMDJSON_LSX_INTRINSICS_H +/* end file simdjson/lsx/intrinsics.h */ +/* including simdjson/lsx/bitmanipulation.h: #include "simdjson/lsx/bitmanipulation.h" */ +/* begin file simdjson/lsx/bitmanipulation.h */ +#ifndef SIMDJSON_LSX_BITMANIPULATION_H +#define SIMDJSON_LSX_BITMANIPULATION_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/lsx/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/lsx/intrinsics.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/lsx/bitmask.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lsx { + namespace { + + // We sometimes call trailing_zero on inputs that are zero, + // but the algorithms do not end up using the returned value. + // Sadly, sanitizers are not smart enough to figure it out. + SIMDJSON_NO_SANITIZE_UNDEFINED + // This function can be used safely even if not all bytes have been + // initialized. + // See issue https://github.com/simdjson/simdjson/issues/1965 + SIMDJSON_NO_SANITIZE_MEMORY + simdjson_inline int trailing_zeroes(uint64_t input_num) { + return __builtin_ctzll(input_num); + } + + /* result might be undefined when input_num is zero */ + simdjson_inline uint64_t clear_lowest_bit(uint64_t input_num) { + return input_num & (input_num - 1); + } + + /* result might be undefined when input_num is zero */ + simdjson_inline int leading_zeroes(uint64_t input_num) { + return __builtin_clzll(input_num); + } + + /* result might be undefined when input_num is zero */ + simdjson_inline int count_ones(uint64_t input_num) { + return __lsx_vpickve2gr_w(__lsx_vpcnt_d(__m128i(v2u64{ input_num, 0 })), 0); + } + + simdjson_inline bool add_overflow(uint64_t value1, uint64_t value2, uint64_t* result) { + return __builtin_uaddll_overflow(value1, value2, + reinterpret_cast(result)); + } + + } // unnamed namespace + } // namespace lsx +} // namespace simdjson + +#endif // SIMDJSON_LSX_BITMANIPULATION_H +/* end file simdjson/lsx/bitmanipulation.h */ +/* including simdjson/lsx/bitmask.h: #include "simdjson/lsx/bitmask.h" */ +/* begin file simdjson/lsx/bitmask.h */ +#ifndef SIMDJSON_LSX_BITMASK_H +#define SIMDJSON_LSX_BITMASK_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/lsx/base.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lsx { + namespace { + + // + // Perform a "cumulative bitwise xor," flipping bits each time a 1 is encountered. + // + // For example, prefix_xor(00100100) == 00011100 + // + simdjson_inline uint64_t prefix_xor(uint64_t bitmask) { + bitmask ^= bitmask << 1; + bitmask ^= bitmask << 2; + bitmask ^= bitmask << 4; + bitmask ^= bitmask << 8; + bitmask ^= bitmask << 16; + bitmask ^= bitmask << 32; + return bitmask; + } + + } // unnamed namespace + } // namespace lsx +} // namespace simdjson + +#endif +/* end file simdjson/lsx/bitmask.h */ +/* including simdjson/lsx/numberparsing_defs.h: #include "simdjson/lsx/numberparsing_defs.h" */ +/* begin file simdjson/lsx/numberparsing_defs.h */ +#ifndef SIMDJSON_LSX_NUMBERPARSING_DEFS_H +#define SIMDJSON_LSX_NUMBERPARSING_DEFS_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/lsx/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/lsx/intrinsics.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +#include + +namespace simdjson { + namespace lsx { + namespace numberparsing { + + // we don't have appropriate instructions, so let us use a scalar function + // credit: https://johnnylee-sde.github.io/Fast-numeric-string-to-int/ + /** @private */ + static simdjson_inline uint32_t parse_eight_digits_unrolled(const uint8_t* chars) { + uint64_t val; + std::memcpy(&val, chars, sizeof(uint64_t)); + val = (val & 0x0F0F0F0F0F0F0F0F) * 2561 >> 8; + val = (val & 0x00FF00FF00FF00FF) * 6553601 >> 16; + return uint32_t((val & 0x0000FFFF0000FFFF) * 42949672960001 >> 32); + } + + simdjson_inline internal::value128 full_multiplication(uint64_t value1, uint64_t value2) { + internal::value128 answer; + __uint128_t r = (static_cast<__uint128_t>(value1)) * value2; + answer.low = uint64_t(r); + answer.high = uint64_t(r >> 64); + return answer; + } + + } // namespace numberparsing + } // namespace lsx +} // namespace simdjson + +#define SIMDJSON_SWAR_NUMBER_PARSING 1 + +#endif // SIMDJSON_LSX_NUMBERPARSING_DEFS_H +/* end file simdjson/lsx/numberparsing_defs.h */ +/* including simdjson/lsx/simd.h: #include "simdjson/lsx/simd.h" */ +/* begin file simdjson/lsx/simd.h */ +#ifndef SIMDJSON_LSX_SIMD_H +#define SIMDJSON_LSX_SIMD_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/lsx/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/lsx/bitmanipulation.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/simdprune_tables.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lsx { + namespace { + namespace simd { + + // Forward-declared so they can be used by splat and friends. + template + struct base { + __m128i value; + + // Zero constructor + simdjson_inline base() : value{ __m128i() } {} + + // Conversion from SIMD register + simdjson_inline base(const __m128i _value) : value(_value) {} + + // Conversion to SIMD register + simdjson_inline operator const __m128i& () const { return this->value; } + simdjson_inline operator __m128i& () { return this->value; } + simdjson_inline operator const v16i8& () const { return (v16i8&)this->value; } + simdjson_inline operator v16i8& () { return (v16i8&)this->value; } + + // Bit operations + simdjson_inline Child operator|(const Child other) const { return __lsx_vor_v(*this, other); } + simdjson_inline Child operator&(const Child other) const { return __lsx_vand_v(*this, other); } + simdjson_inline Child operator^(const Child other) const { return __lsx_vxor_v(*this, other); } + simdjson_inline Child bit_andnot(const Child other) const { return __lsx_vandn_v(other, *this); } + simdjson_inline Child& operator|=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast | other; return *this_cast; } + simdjson_inline Child& operator&=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast & other; return *this_cast; } + simdjson_inline Child& operator^=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast ^ other; return *this_cast; } + }; + + // Forward-declared so they can be used by splat and friends. + template + struct simd8; + + template> + struct base8 : base> { + simdjson_inline base8() : base>() {} + simdjson_inline base8(const __m128i _value) : base>(_value) {} + + friend simdjson_really_inline Mask operator==(const simd8 lhs, const simd8 rhs) { return __lsx_vseq_b(lhs, rhs); } + + static const int SIZE = sizeof(base>::value); + + template + simdjson_inline simd8 prev(const simd8 prev_chunk) const { + return __lsx_vor_v(__lsx_vbsll_v(*this, N), __lsx_vbsrl_v(prev_chunk, 16 - N)); + } + }; + + // SIMD byte mask type (returned by things like eq and gt) + template<> + struct simd8 : base8 { + static simdjson_inline simd8 splat(bool _value) { + return __lsx_vreplgr2vr_b(uint8_t(-(!!_value))); + } + + simdjson_inline simd8() : base8() {} + simdjson_inline simd8(const __m128i _value) : base8(_value) {} + // Splat constructor + simdjson_inline simd8(bool _value) : base8(splat(_value)) {} + + simdjson_inline int to_bitmask() const { return __lsx_vpickve2gr_w(__lsx_vmskltz_b(*this), 0); } + simdjson_inline bool any() const { return 0 == __lsx_vpickve2gr_hu(__lsx_vmsknz_b(*this), 0); } + simdjson_inline simd8 operator~() const { return *this ^ true; } + }; + + template + struct base8_numeric : base8 { + static simdjson_inline simd8 splat(T _value) { return __lsx_vreplgr2vr_b(_value); } + static simdjson_inline simd8 zero() { return __lsx_vldi(0); } + static simdjson_inline simd8 load(const T values[16]) { + return __lsx_vld(reinterpret_cast(values), 0); + } + // Repeat 16 values as many times as necessary (usually for lookup tables) + static simdjson_inline simd8 repeat_16( + T v0, T v1, T v2, T v3, T v4, T v5, T v6, T v7, + T v8, T v9, T v10, T v11, T v12, T v13, T v14, T v15 + ) { + return simd8( + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15 + ); + } + + simdjson_inline base8_numeric() : base8() {} + simdjson_inline base8_numeric(const __m128i _value) : base8(_value) {} + + // Store to array + simdjson_inline void store(T dst[16]) const { + return __lsx_vst(*this, reinterpret_cast<__m128i*>(dst), 0); + } + + // Addition/subtraction are the same for signed and unsigned + simdjson_inline simd8 operator+(const simd8 other) const { return __lsx_vadd_b(*this, other); } + simdjson_inline simd8 operator-(const simd8 other) const { return __lsx_vsub_b(*this, other); } + simdjson_inline simd8& operator+=(const simd8 other) { *this = *this + other; return *static_cast*>(this); } + simdjson_inline simd8& operator-=(const simd8 other) { *this = *this - other; return *static_cast*>(this); } + + // Override to distinguish from bool version + simdjson_inline simd8 operator~() const { return *this ^ 0xFFu; } + + // Perform a lookup assuming the value is between 0 and 16 (undefined behavior for out of range values) + template + simdjson_inline simd8 lookup_16(simd8 lookup_table) const { + return __lsx_vshuf_b(lookup_table, lookup_table, *this); + } + + // Copies to 'output" all bytes corresponding to a 0 in the mask (interpreted as a bitset). + // Passing a 0 value for mask would be equivalent to writing out every byte to output. + // Only the first 16 - count_ones(mask) bytes of the result are significant but 16 bytes + // get written. + template + simdjson_inline void compress(uint16_t mask, L* output) const { + using internal::thintable_epi8; + using internal::BitsSetTable256mul2; + using internal::pshufb_combine_table; + // this particular implementation was inspired by haswell + // lsx do it in 2 steps, first 8 bytes and then second 8 bytes... + uint8_t mask1 = uint8_t(mask); // least significant 8 bits + uint8_t mask2 = uint8_t(mask >> 8); // second least significant 8 bits + // next line just loads the 64-bit values thintable_epi8[mask1] and + // thintable_epi8[mask2] into a 128-bit register. + __m128i shufmask = { int64_t(thintable_epi8[mask1]), int64_t(thintable_epi8[mask2]) + 0x0808080808080808 }; + // this is the version "nearly pruned" + __m128i pruned = __lsx_vshuf_b(*this, *this, shufmask); + // we still need to put the pieces back together. + // we compute the popcount of the first words: + int pop1 = BitsSetTable256mul2[mask1]; + // then load the corresponding mask + __m128i compactmask = __lsx_vldx(reinterpret_cast(reinterpret_cast(pshufb_combine_table)), pop1 * 8); + __m128i answer = __lsx_vshuf_b(pruned, pruned, compactmask); + __lsx_vst(answer, reinterpret_cast(output), 0); + } + + template + simdjson_inline simd8 lookup_16( + L replace0, L replace1, L replace2, L replace3, + L replace4, L replace5, L replace6, L replace7, + L replace8, L replace9, L replace10, L replace11, + L replace12, L replace13, L replace14, L replace15) const { + return lookup_16(simd8::repeat_16( + replace0, replace1, replace2, replace3, + replace4, replace5, replace6, replace7, + replace8, replace9, replace10, replace11, + replace12, replace13, replace14, replace15 + )); + } + }; + + // Signed bytes + template<> + struct simd8 : base8_numeric { + simdjson_inline simd8() : base8_numeric() {} + simdjson_inline simd8(const __m128i _value) : base8_numeric(_value) {} + // Splat constructor + simdjson_inline simd8(int8_t _value) : simd8(splat(_value)) {} + // Array constructor + simdjson_inline simd8(const int8_t values[16]) : simd8(load(values)) {} + // Member-by-member initialization + simdjson_inline simd8( + int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, + int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 + ) : simd8({ + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10,v11,v12,v13,v14,v15 + }) {} + // Repeat 16 values as many times as necessary (usually for lookup tables) + simdjson_inline static simd8 repeat_16( + int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, + int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 + ) { + return simd8( + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15 + ); + } + + // Order-sensitive comparisons + simdjson_inline simd8 max_val(const simd8 other) const { return __lsx_vmax_b(*this, other); } + simdjson_inline simd8 min_val(const simd8 other) const { return __lsx_vmin_b(*this, other); } + simdjson_inline simd8 operator>(const simd8 other) const { return __lsx_vslt_b(other, *this); } + simdjson_inline simd8 operator<(const simd8 other) const { return __lsx_vslt_b(*this, other); } + }; + + // Unsigned bytes + template<> + struct simd8 : base8_numeric { + simdjson_inline simd8() : base8_numeric() {} + simdjson_inline simd8(const __m128i _value) : base8_numeric(_value) {} + // Splat constructor + simdjson_inline simd8(uint8_t _value) : simd8(splat(_value)) {} + // Array constructor + simdjson_inline simd8(const uint8_t values[16]) : simd8(load(values)) {} + // Member-by-member initialization + simdjson_inline simd8( + uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, + uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 + ) : simd8(__m128i(v16u8{ + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10,v11,v12,v13,v14,v15 + })) {} + // Repeat 16 values as many times as necessary (usually for lookup tables) + simdjson_inline static simd8 repeat_16( + uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, + uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 + ) { + return simd8( + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15 + ); + } + + // Saturated math + simdjson_inline simd8 saturating_add(const simd8 other) const { return __lsx_vsadd_bu(*this, other); } + simdjson_inline simd8 saturating_sub(const simd8 other) const { return __lsx_vssub_bu(*this, other); } + + // Order-specific operations + simdjson_inline simd8 max_val(const simd8 other) const { return __lsx_vmax_bu(*this, other); } + simdjson_inline simd8 min_val(const simd8 other) const { return __lsx_vmin_bu(other, *this); } + // Same as >, but only guarantees true is nonzero (< guarantees true = -1) + simdjson_inline simd8 gt_bits(const simd8 other) const { return this->saturating_sub(other); } + // Same as <, but only guarantees true is nonzero (< guarantees true = -1) + simdjson_inline simd8 lt_bits(const simd8 other) const { return other.saturating_sub(*this); } + simdjson_inline simd8 operator<=(const simd8 other) const { return other.max_val(*this) == other; } + simdjson_inline simd8 operator>=(const simd8 other) const { return other.min_val(*this) == other; } + simdjson_inline simd8 operator>(const simd8 other) const { return this->gt_bits(other).any_bits_set(); } + simdjson_inline simd8 operator<(const simd8 other) const { return this->lt_bits(other).any_bits_set(); } + + // Bit-specific operations + simdjson_inline simd8 bits_not_set() const { return *this == uint8_t(0); } + simdjson_inline simd8 bits_not_set(simd8 bits) const { return (*this & bits).bits_not_set(); } + simdjson_inline simd8 any_bits_set() const { return ~this->bits_not_set(); } + simdjson_inline simd8 any_bits_set(simd8 bits) const { return ~this->bits_not_set(bits); } + simdjson_inline bool is_ascii() const { return 0 == __lsx_vpickve2gr_w(__lsx_vmskltz_b(*this), 0); } + simdjson_inline bool bits_not_set_anywhere() const { return 0 == __lsx_vpickve2gr_hu(__lsx_vmsknz_b(*this), 0); } + simdjson_inline bool any_bits_set_anywhere() const { return !bits_not_set_anywhere(); } + simdjson_inline bool bits_not_set_anywhere(simd8 bits) const { + return 0 == __lsx_vpickve2gr_hu(__lsx_vmsknz_b(__lsx_vand_v(*this, bits)), 0); + } + simdjson_inline bool any_bits_set_anywhere(simd8 bits) const { return !bits_not_set_anywhere(bits); } + template + simdjson_inline simd8 shr() const { return simd8(__lsx_vsrli_b(*this, N)); } + template + simdjson_inline simd8 shl() const { return simd8(__lsx_vslli_b(*this, N)); } + }; + + template + struct simd8x64 { + static constexpr int NUM_CHUNKS = 64 / sizeof(simd8); + static_assert(NUM_CHUNKS == 4, "LSX kernel should use four registers per 64-byte block."); + const simd8 chunks[NUM_CHUNKS]; + + simd8x64(const simd8x64& o) = delete; // no copy allowed + simd8x64& operator=(const simd8& other) = delete; // no assignment allowed + simd8x64() = delete; // no default constructor allowed + + simdjson_inline simd8x64(const simd8 chunk0, const simd8 chunk1, const simd8 chunk2, const simd8 chunk3) : chunks{ chunk0, chunk1, chunk2, chunk3 } {} + simdjson_inline simd8x64(const T ptr[64]) : chunks{ simd8::load(ptr), simd8::load(ptr + 16), simd8::load(ptr + 32), simd8::load(ptr + 48) } {} + + simdjson_inline uint64_t compress(uint64_t mask, T* output) const { + uint16_t mask1 = uint16_t(mask); + uint16_t mask2 = uint16_t(mask >> 16); + uint16_t mask3 = uint16_t(mask >> 32); + uint16_t mask4 = uint16_t(mask >> 48); + __m128i zcnt = __lsx_vpcnt_h(__m128i(v2u64{ ~mask, 0 })); + uint64_t zcnt1 = __lsx_vpickve2gr_hu(zcnt, 0); + uint64_t zcnt2 = __lsx_vpickve2gr_hu(zcnt, 1); + uint64_t zcnt3 = __lsx_vpickve2gr_hu(zcnt, 2); + uint64_t zcnt4 = __lsx_vpickve2gr_hu(zcnt, 3); + uint8_t* voutput = reinterpret_cast(output); + // There should be a critical value which processes in scaler is faster. + if (zcnt1) + this->chunks[0].compress(mask1, reinterpret_cast(voutput)); + voutput += zcnt1; + if (zcnt2) + this->chunks[1].compress(mask2, reinterpret_cast(voutput)); + voutput += zcnt2; + if (zcnt3) + this->chunks[2].compress(mask3, reinterpret_cast(voutput)); + voutput += zcnt3; + if (zcnt4) + this->chunks[3].compress(mask4, reinterpret_cast(voutput)); + voutput += zcnt4; + return reinterpret_cast(voutput) - reinterpret_cast(output); + } + + simdjson_inline void store(T ptr[64]) const { + this->chunks[0].store(ptr + sizeof(simd8) * 0); + this->chunks[1].store(ptr + sizeof(simd8) * 1); + this->chunks[2].store(ptr + sizeof(simd8) * 2); + this->chunks[3].store(ptr + sizeof(simd8) * 3); + } + + simdjson_inline uint64_t to_bitmask() const { + __m128i mask1 = __lsx_vmskltz_b(this->chunks[0]); + __m128i mask2 = __lsx_vmskltz_b(this->chunks[1]); + __m128i mask3 = __lsx_vmskltz_b(this->chunks[2]); + __m128i mask4 = __lsx_vmskltz_b(this->chunks[3]); + mask1 = __lsx_vilvl_h(mask2, mask1); + mask2 = __lsx_vilvl_h(mask4, mask3); + return __lsx_vpickve2gr_du(__lsx_vilvl_w(mask2, mask1), 0); + } + + simdjson_inline simd8 reduce_or() const { + return (this->chunks[0] | this->chunks[1]) | (this->chunks[2] | this->chunks[3]); + } + + simdjson_inline uint64_t eq(const T m) const { + const simd8 mask = simd8::splat(m); + return simd8x64( + this->chunks[0] == mask, + this->chunks[1] == mask, + this->chunks[2] == mask, + this->chunks[3] == mask + ).to_bitmask(); + } + + simdjson_inline uint64_t eq(const simd8x64& other) const { + return simd8x64( + this->chunks[0] == other.chunks[0], + this->chunks[1] == other.chunks[1], + this->chunks[2] == other.chunks[2], + this->chunks[3] == other.chunks[3] + ).to_bitmask(); + } + + simdjson_inline uint64_t lteq(const T m) const { + const simd8 mask = simd8::splat(m); + return simd8x64( + this->chunks[0] <= mask, + this->chunks[1] <= mask, + this->chunks[2] <= mask, + this->chunks[3] <= mask + ).to_bitmask(); + } + }; // struct simd8x64 + + } // namespace simd + } // unnamed namespace + } // namespace lsx +} // namespace simdjson + +#endif // SIMDJSON_LSX_SIMD_H +/* end file simdjson/lsx/simd.h */ +/* including simdjson/lsx/stringparsing_defs.h: #include "simdjson/lsx/stringparsing_defs.h" */ +/* begin file simdjson/lsx/stringparsing_defs.h */ +#ifndef SIMDJSON_LSX_STRINGPARSING_DEFS_H +#define SIMDJSON_LSX_STRINGPARSING_DEFS_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/lsx/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/lsx/simd.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/lsx/bitmanipulation.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lsx { + namespace { + + using namespace simd; + + // Holds backslashes and quotes locations. + struct backslash_and_quote { + public: + static constexpr uint32_t BYTES_PROCESSED = 32; + simdjson_inline static backslash_and_quote copy_and_find(const uint8_t* src, uint8_t* dst); + + simdjson_inline bool has_quote_first() { return ((bs_bits - 1) & quote_bits) != 0; } + simdjson_inline bool has_backslash() { return bs_bits != 0; } + simdjson_inline int quote_index() { return trailing_zeroes(quote_bits); } + simdjson_inline int backslash_index() { return trailing_zeroes(bs_bits); } + + uint32_t bs_bits; + uint32_t quote_bits; + }; // struct backslash_and_quote + + simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8_t* src, uint8_t* dst) { + // this can read up to 31 bytes beyond the buffer size, but we require + // SIMDJSON_PADDING of padding + static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "backslash and quote finder must process fewer than SIMDJSON_PADDING bytes"); + simd8 v0(src); + simd8 v1(src + sizeof(v0)); + v0.store(dst); + v1.store(dst + sizeof(v0)); + + // Getting a 64-bit bitmask is much cheaper than multiple 16-bit bitmasks on LSX; therefore, we + // smash them together into a 64-byte mask and get the bitmask from there. + uint64_t bs_and_quote = simd8x64(v0 == '\\', v1 == '\\', v0 == '"', v1 == '"').to_bitmask(); + return { + uint32_t(bs_and_quote), // bs_bits + uint32_t(bs_and_quote >> 32) // quote_bits + }; + } + + } // unnamed namespace + } // namespace lsx +} // namespace simdjson + +#endif // SIMDJSON_LSX_STRINGPARSING_DEFS_H +/* end file simdjson/lsx/stringparsing_defs.h */ + +#define SIMDJSON_SKIP_BACKSLASH_SHORT_CIRCUIT 1 +/* end file simdjson/lsx/begin.h */ +/* including simdjson/generic/amalgamated.h for lsx: #include "simdjson/generic/amalgamated.h" */ +/* begin file simdjson/generic/amalgamated.h for lsx */ +#if defined(SIMDJSON_CONDITIONAL_INCLUDE) && !defined(SIMDJSON_GENERIC_DEPENDENCIES_H) +#error simdjson/generic/dependencies.h must be included before simdjson/generic/amalgamated.h! +#endif + +/* including simdjson/generic/base.h for lsx: #include "simdjson/generic/base.h" */ +/* begin file simdjson/generic/base.h for lsx */ +#ifndef SIMDJSON_GENERIC_BASE_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_BASE_H */ +/* amalgamation skipped (editor-only): #include "simdjson/base.h" */ +/* amalgamation skipped (editor-only): // If we haven't got an implementation yet, we're in the editor, editing a generic file! Just */ +/* amalgamation skipped (editor-only): // use the most advanced one we can so the most possible stuff can be tested. */ +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_IMPLEMENTATION */ +/* amalgamation skipped (editor-only): #include "simdjson/implementation_detection.h" */ +/* amalgamation skipped (editor-only): #if SIMDJSON_IMPLEMENTATION_ICELAKE */ +/* amalgamation skipped (editor-only): #include "simdjson/icelake/begin.h" */ +/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_HASWELL */ +/* amalgamation skipped (editor-only): #include "simdjson/haswell/begin.h" */ +/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_WESTMERE */ +/* amalgamation skipped (editor-only): #include "simdjson/westmere/begin.h" */ +/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_ARM64 */ +/* amalgamation skipped (editor-only): #include "simdjson/arm64/begin.h" */ +/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_PPC64 */ +/* amalgamation skipped (editor-only): #include "simdjson/ppc64/begin.h" */ +/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_LSX */ +/* amalgamation skipped (editor-only): #include "simdjson/lsx/begin.h" */ +/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_LASX */ +/* amalgamation skipped (editor-only): #include "simdjson/lasx/begin.h" */ +/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_FALLBACK */ +/* amalgamation skipped (editor-only): #include "simdjson/fallback/begin.h" */ +/* amalgamation skipped (editor-only): #else */ +/* amalgamation skipped (editor-only): #error "All possible implementations (including fallback) have been disabled! simdjson will not run." */ +/* amalgamation skipped (editor-only): #endif */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_IMPLEMENTATION */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lsx { + + struct open_container; + class dom_parser_implementation; + + /** + * The type of a JSON number + */ + enum class number_type { + floating_point_number = 1, /// a binary64 number + signed_integer, /// a signed integer that fits in a 64-bit word using two's complement + unsigned_integer, /// a positive integer larger or equal to 1<<63 + big_integer /// a big integer that does not fit in a 64-bit word + }; + + } // namespace lsx +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_BASE_H +/* end file simdjson/generic/base.h for lsx */ +/* including simdjson/generic/jsoncharutils.h for lsx: #include "simdjson/generic/jsoncharutils.h" */ +/* begin file simdjson/generic/jsoncharutils.h for lsx */ +#ifndef SIMDJSON_GENERIC_JSONCHARUTILS_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_JSONCHARUTILS_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/jsoncharutils_tables.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lsx { + namespace { + namespace jsoncharutils { + + // return non-zero if not a structural or whitespace char + // zero otherwise + simdjson_inline uint32_t is_not_structural_or_whitespace(uint8_t c) { + return internal::structural_or_whitespace_negated[c]; + } + + simdjson_inline uint32_t is_structural_or_whitespace(uint8_t c) { + return internal::structural_or_whitespace[c]; + } + + // returns a value with the high 16 bits set if not valid + // otherwise returns the conversion of the 4 hex digits at src into the bottom + // 16 bits of the 32-bit return register + // + // see + // https://lemire.me/blog/2019/04/17/parsing-short-hexadecimal-strings-efficiently/ + static inline uint32_t hex_to_u32_nocheck( + const uint8_t* src) { // strictly speaking, static inline is a C-ism + uint32_t v1 = internal::digit_to_val32[630 + src[0]]; + uint32_t v2 = internal::digit_to_val32[420 + src[1]]; + uint32_t v3 = internal::digit_to_val32[210 + src[2]]; + uint32_t v4 = internal::digit_to_val32[0 + src[3]]; + return v1 | v2 | v3 | v4; + } + + // given a code point cp, writes to c + // the utf-8 code, outputting the length in + // bytes, if the length is zero, the code point + // is invalid + // + // This can possibly be made faster using pdep + // and clz and table lookups, but JSON documents + // have few escaped code points, and the following + // function looks cheap. + // + // Note: we assume that surrogates are treated separately + // + simdjson_inline size_t codepoint_to_utf8(uint32_t cp, uint8_t* c) { + if (cp <= 0x7F) { + c[0] = uint8_t(cp); + return 1; // ascii + } + if (cp <= 0x7FF) { + c[0] = uint8_t((cp >> 6) + 192); + c[1] = uint8_t((cp & 63) + 128); + return 2; // universal plane + // Surrogates are treated elsewhere... + //} //else if (0xd800 <= cp && cp <= 0xdfff) { + // return 0; // surrogates // could put assert here + } + else if (cp <= 0xFFFF) { + c[0] = uint8_t((cp >> 12) + 224); + c[1] = uint8_t(((cp >> 6) & 63) + 128); + c[2] = uint8_t((cp & 63) + 128); + return 3; + } + else if (cp <= 0x10FFFF) { // if you know you have a valid code point, this + // is not needed + c[0] = uint8_t((cp >> 18) + 240); + c[1] = uint8_t(((cp >> 12) & 63) + 128); + c[2] = uint8_t(((cp >> 6) & 63) + 128); + c[3] = uint8_t((cp & 63) + 128); + return 4; + } + // will return 0 when the code point was too large. + return 0; // bad r + } + +#if SIMDJSON_IS_32BITS // _umul128 for x86, arm + // this is a slow emulation routine for 32-bit + // + static simdjson_inline uint64_t __emulu(uint32_t x, uint32_t y) { + return x * (uint64_t)y; + } + static simdjson_inline uint64_t _umul128(uint64_t ab, uint64_t cd, uint64_t* hi) { + uint64_t ad = __emulu((uint32_t)(ab >> 32), (uint32_t)cd); + uint64_t bd = __emulu((uint32_t)ab, (uint32_t)cd); + uint64_t adbc = ad + __emulu((uint32_t)ab, (uint32_t)(cd >> 32)); + uint64_t adbc_carry = !!(adbc < ad); + uint64_t lo = bd + (adbc << 32); + *hi = __emulu((uint32_t)(ab >> 32), (uint32_t)(cd >> 32)) + (adbc >> 32) + + (adbc_carry << 32) + !!(lo < bd); + return lo; + } +#endif + + } // namespace jsoncharutils + } // unnamed namespace + } // namespace lsx +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_JSONCHARUTILS_H +/* end file simdjson/generic/jsoncharutils.h for lsx */ +/* including simdjson/generic/atomparsing.h for lsx: #include "simdjson/generic/atomparsing.h" */ +/* begin file simdjson/generic/atomparsing.h for lsx */ +#ifndef SIMDJSON_GENERIC_ATOMPARSING_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_ATOMPARSING_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/jsoncharutils.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +#include + +namespace simdjson { + namespace lsx { + namespace { + /// @private + namespace atomparsing { + + // The string_to_uint32 is exclusively used to map literal strings to 32-bit values. + // We use memcpy instead of a pointer cast to avoid undefined behaviors since we cannot + // be certain that the character pointer will be properly aligned. + // You might think that using memcpy makes this function expensive, but you'd be wrong. + // All decent optimizing compilers (GCC, clang, Visual Studio) will compile string_to_uint32("false"); + // to the compile-time constant 1936482662. + simdjson_inline uint32_t string_to_uint32(const char* str) { uint32_t val; std::memcpy(&val, str, sizeof(uint32_t)); return val; } + + + // Again in str4ncmp we use a memcpy to avoid undefined behavior. The memcpy may appear expensive. + // Yet all decent optimizing compilers will compile memcpy to a single instruction, just about. + simdjson_warn_unused + simdjson_inline uint32_t str4ncmp(const uint8_t* src, const char* atom) { + uint32_t srcval; // we want to avoid unaligned 32-bit loads (undefined in C/C++) + static_assert(sizeof(uint32_t) <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be larger than 4 bytes"); + std::memcpy(&srcval, src, sizeof(uint32_t)); + return srcval ^ string_to_uint32(atom); + } + + simdjson_warn_unused + simdjson_inline bool is_valid_true_atom(const uint8_t* src) { + return (str4ncmp(src, "true") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; + } + + simdjson_warn_unused + simdjson_inline bool is_valid_true_atom(const uint8_t* src, size_t len) { + if (len > 4) { return is_valid_true_atom(src); } + else if (len == 4) { return !str4ncmp(src, "true"); } + else { return false; } + } + + simdjson_warn_unused + simdjson_inline bool is_valid_false_atom(const uint8_t* src) { + return (str4ncmp(src + 1, "alse") | jsoncharutils::is_not_structural_or_whitespace(src[5])) == 0; + } + + simdjson_warn_unused + simdjson_inline bool is_valid_false_atom(const uint8_t* src, size_t len) { + if (len > 5) { return is_valid_false_atom(src); } + else if (len == 5) { return !str4ncmp(src + 1, "alse"); } + else { return false; } + } + + simdjson_warn_unused + simdjson_inline bool is_valid_null_atom(const uint8_t* src) { + return (str4ncmp(src, "null") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; + } + + simdjson_warn_unused + simdjson_inline bool is_valid_null_atom(const uint8_t* src, size_t len) { + if (len > 4) { return is_valid_null_atom(src); } + else if (len == 4) { return !str4ncmp(src, "null"); } + else { return false; } + } + + } // namespace atomparsing + } // unnamed namespace + } // namespace lsx +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_ATOMPARSING_H +/* end file simdjson/generic/atomparsing.h for lsx */ +/* including simdjson/generic/dom_parser_implementation.h for lsx: #include "simdjson/generic/dom_parser_implementation.h" */ +/* begin file simdjson/generic/dom_parser_implementation.h for lsx */ +#ifndef SIMDJSON_GENERIC_DOM_PARSER_IMPLEMENTATION_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_DOM_PARSER_IMPLEMENTATION_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/dom_parser_implementation.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lsx { + + // expectation: sizeof(open_container) = 64/8. + struct open_container { + uint32_t tape_index; // where, on the tape, does the scope ([,{) begins + uint32_t count; // how many elements in the scope + }; // struct open_container + + static_assert(sizeof(open_container) == 64 / 8, "Open container must be 64 bits"); + + class dom_parser_implementation final : public internal::dom_parser_implementation { + public: + /** Tape location of each open { or [ */ + std::unique_ptr open_containers{}; + /** Whether each open container is a [ or { */ + std::unique_ptr is_array{}; + /** Buffer passed to stage 1 */ + const uint8_t* buf{}; + /** Length passed to stage 1 */ + size_t len{ 0 }; + /** Document passed to stage 2 */ + dom::document* doc{}; + + inline dom_parser_implementation() noexcept; + inline dom_parser_implementation(dom_parser_implementation&& other) noexcept; + inline dom_parser_implementation& operator=(dom_parser_implementation&& other) noexcept; + dom_parser_implementation(const dom_parser_implementation&) = delete; + dom_parser_implementation& operator=(const dom_parser_implementation&) = delete; + + simdjson_warn_unused error_code parse(const uint8_t* buf, size_t len, dom::document& doc) noexcept final; + simdjson_warn_unused error_code stage1(const uint8_t* buf, size_t len, stage1_mode partial) noexcept final; + simdjson_warn_unused error_code stage2(dom::document& doc) noexcept final; + simdjson_warn_unused error_code stage2_next(dom::document& doc) noexcept final; + simdjson_warn_unused uint8_t* parse_string(const uint8_t* src, uint8_t* dst, bool allow_replacement) const noexcept final; + simdjson_warn_unused uint8_t* parse_wobbly_string(const uint8_t* src, uint8_t* dst) const noexcept final; + inline simdjson_warn_unused error_code set_capacity(size_t capacity) noexcept final; + inline simdjson_warn_unused error_code set_max_depth(size_t max_depth) noexcept final; + private: + simdjson_inline simdjson_warn_unused error_code set_capacity_stage1(size_t capacity); + + }; + + } // namespace lsx +} // namespace simdjson + +namespace simdjson { + namespace lsx { + + inline dom_parser_implementation::dom_parser_implementation() noexcept = default; + inline dom_parser_implementation::dom_parser_implementation(dom_parser_implementation&& other) noexcept = default; + inline dom_parser_implementation& dom_parser_implementation::operator=(dom_parser_implementation&& other) noexcept = default; + + // Leaving these here so they can be inlined if so desired + inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(size_t capacity) noexcept { + if (capacity > SIMDJSON_MAXSIZE_BYTES) { return CAPACITY; } + // Stage 1 index output + size_t max_structures = SIMDJSON_ROUNDUP_N(capacity, 64) + 2 + 7; + structural_indexes.reset(new (std::nothrow) uint32_t[max_structures]); + if (!structural_indexes) { _capacity = 0; return MEMALLOC; } + structural_indexes[0] = 0; + n_structural_indexes = 0; + + _capacity = capacity; + return SUCCESS; + } + + inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { + // Stage 2 stacks + open_containers.reset(new (std::nothrow) open_container[max_depth]); + is_array.reset(new (std::nothrow) bool[max_depth]); + if (!is_array || !open_containers) { _max_depth = 0; return MEMALLOC; } + + _max_depth = max_depth; + return SUCCESS; + } + + } // namespace lsx +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_DOM_PARSER_IMPLEMENTATION_H +/* end file simdjson/generic/dom_parser_implementation.h for lsx */ +/* including simdjson/generic/implementation_simdjson_result_base.h for lsx: #include "simdjson/generic/implementation_simdjson_result_base.h" */ +/* begin file simdjson/generic/implementation_simdjson_result_base.h for lsx */ +#ifndef SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lsx { + + // This is a near copy of include/error.h's implementation_simdjson_result_base, except it doesn't use std::pair + // so we can avoid inlining errors + // TODO reconcile these! + /** + * The result of a simdjson operation that could fail. + * + * Gives the option of reading error codes, or throwing an exception by casting to the desired result. + * + * This is a base class for implementations that want to add functions to the result type for + * chaining. + * + * Override like: + * + * struct simdjson_result : public internal::implementation_simdjson_result_base { + * simdjson_result() noexcept : internal::implementation_simdjson_result_base() {} + * simdjson_result(error_code error) noexcept : internal::implementation_simdjson_result_base(error) {} + * simdjson_result(T &&value) noexcept : internal::implementation_simdjson_result_base(std::forward(value)) {} + * simdjson_result(T &&value, error_code error) noexcept : internal::implementation_simdjson_result_base(value, error) {} + * // Your extra methods here + * } + * + * Then any method returning simdjson_result will be chainable with your methods. + */ + template + struct implementation_simdjson_result_base { + + /** + * Create a new empty result with error = UNINITIALIZED. + */ + simdjson_inline implementation_simdjson_result_base() noexcept = default; + + /** + * Create a new error result. + */ + simdjson_inline implementation_simdjson_result_base(error_code error) noexcept; + + /** + * Create a new successful result. + */ + simdjson_inline implementation_simdjson_result_base(T&& value) noexcept; + + /** + * Create a new result with both things (use if you don't want to branch when creating the result). + */ + simdjson_inline implementation_simdjson_result_base(T&& value, error_code error) noexcept; + + /** + * Move the value and the error to the provided variables. + * + * @param value The variable to assign the value to. May not be set if there is an error. + * @param error The variable to assign the error to. Set to SUCCESS if there is no error. + */ + simdjson_inline void tie(T& value, error_code& error) && noexcept; + + /** + * Move the value to the provided variable. + * + * @param value The variable to assign the value to. May not be set if there is an error. + */ + simdjson_inline error_code get(T& value) && noexcept; + + /** + * The error. + */ + simdjson_inline error_code error() const noexcept; + +#if SIMDJSON_EXCEPTIONS + + /** + * Get the result value. + * + * @throw simdjson_error if there was an error. + */ + simdjson_inline T& value() & noexcept(false); + + /** + * Take the result value (move it). + * + * @throw simdjson_error if there was an error. + */ + simdjson_inline T&& value() && noexcept(false); + + /** + * Take the result value (move it). + * + * @throw simdjson_error if there was an error. + */ + simdjson_inline T&& take_value() && noexcept(false); + + /** + * Cast to the value (will throw on error). + * + * @throw simdjson_error if there was an error. + */ + simdjson_inline operator T && () && noexcept(false); + + +#endif // SIMDJSON_EXCEPTIONS + + /** + * Get the result value. This function is safe if and only + * the error() method returns a value that evaluates to false. + */ + simdjson_inline const T& value_unsafe() const& noexcept; + /** + * Get the result value. This function is safe if and only + * the error() method returns a value that evaluates to false. + */ + simdjson_inline T& value_unsafe() & noexcept; + /** + * Take the result value (move it). This function is safe if and only + * the error() method returns a value that evaluates to false. + */ + simdjson_inline T&& value_unsafe() && noexcept; + protected: + /** users should never directly access first and second. **/ + T first{}; /** Users should never directly access 'first'. **/ + error_code second{ UNINITIALIZED }; /** Users should never directly access 'second'. **/ + }; // struct implementation_simdjson_result_base + + } // namespace lsx +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_H +/* end file simdjson/generic/implementation_simdjson_result_base.h for lsx */ +/* including simdjson/generic/numberparsing.h for lsx: #include "simdjson/generic/numberparsing.h" */ +/* begin file simdjson/generic/numberparsing.h for lsx */ +#ifndef SIMDJSON_GENERIC_NUMBERPARSING_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_NUMBERPARSING_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/jsoncharutils.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +#include +#include +#include + +namespace simdjson { + namespace lsx { + namespace numberparsing { + +#ifdef JSON_TEST_NUMBERS +#define INVALID_NUMBER(SRC) (found_invalid_number((SRC)), NUMBER_ERROR) +#define WRITE_INTEGER(VALUE, SRC, WRITER) (found_integer((VALUE), (SRC)), (WRITER).append_s64((VALUE))) +#define WRITE_UNSIGNED(VALUE, SRC, WRITER) (found_unsigned_integer((VALUE), (SRC)), (WRITER).append_u64((VALUE))) +#define WRITE_DOUBLE(VALUE, SRC, WRITER) (found_float((VALUE), (SRC)), (WRITER).append_double((VALUE))) +#define BIGINT_NUMBER(SRC) (found_invalid_number((SRC)), BIGINT_ERROR) +#else +#define INVALID_NUMBER(SRC) (NUMBER_ERROR) +#define WRITE_INTEGER(VALUE, SRC, WRITER) (WRITER).append_s64((VALUE)) +#define WRITE_UNSIGNED(VALUE, SRC, WRITER) (WRITER).append_u64((VALUE)) +#define WRITE_DOUBLE(VALUE, SRC, WRITER) (WRITER).append_double((VALUE)) +#define BIGINT_NUMBER(SRC) (BIGINT_ERROR) +#endif + + namespace { + + // Convert a mantissa, an exponent and a sign bit into an ieee64 double. + // The real_exponent needs to be in [0, 2046] (technically real_exponent = 2047 would be acceptable). + // The mantissa should be in [0,1<<53). The bit at index (1ULL << 52) while be zeroed. + simdjson_inline double to_double(uint64_t mantissa, uint64_t real_exponent, bool negative) { + double d; + mantissa &= ~(1ULL << 52); + mantissa |= real_exponent << 52; + mantissa |= ((static_cast(negative)) << 63); + std::memcpy(&d, &mantissa, sizeof(d)); + return d; + } + + // Attempts to compute i * 10^(power) exactly; and if "negative" is + // true, negate the result. + // This function will only work in some cases, when it does not work, success is + // set to false. This should work *most of the time* (like 99% of the time). + // We assume that power is in the [smallest_power, + // largest_power] interval: the caller is responsible for this check. + simdjson_inline bool compute_float_64(int64_t power, uint64_t i, bool negative, double& d) { + // we start with a fast path + // It was described in + // Clinger WD. How to read floating point numbers accurately. + // ACM SIGPLAN Notices. 1990 +#ifndef FLT_EVAL_METHOD +#error "FLT_EVAL_METHOD should be defined, please include cfloat." +#endif +#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0) + // We cannot be certain that x/y is rounded to nearest. + if (0 <= power && power <= 22 && i <= 9007199254740991) +#else + if (-22 <= power && power <= 22 && i <= 9007199254740991) +#endif + { + // convert the integer into a double. This is lossless since + // 0 <= i <= 2^53 - 1. + d = double(i); + // + // The general idea is as follows. + // If 0 <= s < 2^53 and if 10^0 <= p <= 10^22 then + // 1) Both s and p can be represented exactly as 64-bit floating-point + // values + // (binary64). + // 2) Because s and p can be represented exactly as floating-point values, + // then s * p + // and s / p will produce correctly rounded values. + // + if (power < 0) { + d = d / simdjson::internal::power_of_ten[-power]; + } + else { + d = d * simdjson::internal::power_of_ten[power]; + } + if (negative) { + d = -d; + } + return true; + } + // When 22 < power && power < 22 + 16, we could + // hope for another, secondary fast path. It was + // described by David M. Gay in "Correctly rounded + // binary-decimal and decimal-binary conversions." (1990) + // If you need to compute i * 10^(22 + x) for x < 16, + // first compute i * 10^x, if you know that result is exact + // (e.g., when i * 10^x < 2^53), + // then you can still proceed and do (i * 10^x) * 10^22. + // Is this worth your time? + // You need 22 < power *and* power < 22 + 16 *and* (i * 10^(x-22) < 2^53) + // for this second fast path to work. + // If you you have 22 < power *and* power < 22 + 16, and then you + // optimistically compute "i * 10^(x-22)", there is still a chance that you + // have wasted your time if i * 10^(x-22) >= 2^53. It makes the use cases of + // this optimization maybe less common than we would like. Source: + // http://www.exploringbinary.com/fast-path-decimal-to-floating-point-conversion/ + // also used in RapidJSON: https://rapidjson.org/strtod_8h_source.html + + // The fast path has now failed, so we are failing back on the slower path. + + // In the slow path, we need to adjust i so that it is > 1<<63 which is always + // possible, except if i == 0, so we handle i == 0 separately. + if (i == 0) { + d = negative ? -0.0 : 0.0; + return true; + } + + + // The exponent is 1024 + 63 + power + // + floor(log(5**power)/log(2)). + // The 1024 comes from the ieee64 standard. + // The 63 comes from the fact that we use a 64-bit word. + // + // Computing floor(log(5**power)/log(2)) could be + // slow. Instead we use a fast function. + // + // For power in (-400,350), we have that + // (((152170 + 65536) * power ) >> 16); + // is equal to + // floor(log(5**power)/log(2)) + power when power >= 0 + // and it is equal to + // ceil(log(5**-power)/log(2)) + power when power < 0 + // + // The 65536 is (1<<16) and corresponds to + // (65536 * power) >> 16 ---> power + // + // ((152170 * power ) >> 16) is equal to + // floor(log(5**power)/log(2)) + // + // Note that this is not magic: 152170/(1<<16) is + // approximatively equal to log(5)/log(2). + // The 1<<16 value is a power of two; we could use a + // larger power of 2 if we wanted to. + // + int64_t exponent = (((152170 + 65536) * power) >> 16) + 1024 + 63; + + + // We want the most significant bit of i to be 1. Shift if needed. + int lz = leading_zeroes(i); + i <<= lz; + + + // We are going to need to do some 64-bit arithmetic to get a precise product. + // We use a table lookup approach. + // It is safe because + // power >= smallest_power + // and power <= largest_power + // We recover the mantissa of the power, it has a leading 1. It is always + // rounded down. + // + // We want the most significant 64 bits of the product. We know + // this will be non-zero because the most significant bit of i is + // 1. + const uint32_t index = 2 * uint32_t(power - simdjson::internal::smallest_power); + // Optimization: It may be that materializing the index as a variable might confuse some compilers and prevent effective complex-addressing loads. (Done for code clarity.) + // + // The full_multiplication function computes the 128-bit product of two 64-bit words + // with a returned value of type value128 with a "low component" corresponding to the + // 64-bit least significant bits of the product and with a "high component" corresponding + // to the 64-bit most significant bits of the product. + simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index]); + // Both i and power_of_five_128[index] have their most significant bit set to 1 which + // implies that the either the most or the second most significant bit of the product + // is 1. We pack values in this manner for efficiency reasons: it maximizes the use + // we make of the product. It also makes it easy to reason about the product: there + // is 0 or 1 leading zero in the product. + + // Unless the least significant 9 bits of the high (64-bit) part of the full + // product are all 1s, then we know that the most significant 55 bits are + // exact and no further work is needed. Having 55 bits is necessary because + // we need 53 bits for the mantissa but we have to have one rounding bit and + // we can waste a bit if the most significant bit of the product is zero. + if ((firstproduct.high & 0x1FF) == 0x1FF) { + // We want to compute i * 5^q, but only care about the top 55 bits at most. + // Consider the scenario where q>=0. Then 5^q may not fit in 64-bits. Doing + // the full computation is wasteful. So we do what is called a "truncated + // multiplication". + // We take the most significant 64-bits, and we put them in + // power_of_five_128[index]. Usually, that's good enough to approximate i * 5^q + // to the desired approximation using one multiplication. Sometimes it does not suffice. + // Then we store the next most significant 64 bits in power_of_five_128[index + 1], and + // then we get a better approximation to i * 5^q. + // + // That's for when q>=0. The logic for q<0 is somewhat similar but it is somewhat + // more complicated. + // + // There is an extra layer of complexity in that we need more than 55 bits of + // accuracy in the round-to-even scenario. + // + // The full_multiplication function computes the 128-bit product of two 64-bit words + // with a returned value of type value128 with a "low component" corresponding to the + // 64-bit least significant bits of the product and with a "high component" corresponding + // to the 64-bit most significant bits of the product. + simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index + 1]); + firstproduct.low += secondproduct.high; + if (secondproduct.high > firstproduct.low) { firstproduct.high++; } + // As it has been proven by Noble Mushtak and Daniel Lemire in "Fast Number Parsing Without + // Fallback" (https://arxiv.org/abs/2212.06644), at this point we are sure that the product + // is sufficiently accurate, and more computation is not needed. + } + uint64_t lower = firstproduct.low; + uint64_t upper = firstproduct.high; + // The final mantissa should be 53 bits with a leading 1. + // We shift it so that it occupies 54 bits with a leading 1. + /////// + uint64_t upperbit = upper >> 63; + uint64_t mantissa = upper >> (upperbit + 9); + lz += int(1 ^ upperbit); + + // Here we have mantissa < (1<<54). + int64_t real_exponent = exponent - lz; + if (simdjson_unlikely(real_exponent <= 0)) { // we have a subnormal? + // Here have that real_exponent <= 0 so -real_exponent >= 0 + if (-real_exponent + 1 >= 64) { // if we have more than 64 bits below the minimum exponent, you have a zero for sure. + d = negative ? -0.0 : 0.0; + return true; + } + // next line is safe because -real_exponent + 1 < 0 + mantissa >>= -real_exponent + 1; + // Thankfully, we can't have both "round-to-even" and subnormals because + // "round-to-even" only occurs for powers close to 0. + mantissa += (mantissa & 1); // round up + mantissa >>= 1; + // There is a weird scenario where we don't have a subnormal but just. + // Suppose we start with 2.2250738585072013e-308, we end up + // with 0x3fffffffffffff x 2^-1023-53 which is technically subnormal + // whereas 0x40000000000000 x 2^-1023-53 is normal. Now, we need to round + // up 0x3fffffffffffff x 2^-1023-53 and once we do, we are no longer + // subnormal, but we can only know this after rounding. + // So we only declare a subnormal if we are smaller than the threshold. + real_exponent = (mantissa < (uint64_t(1) << 52)) ? 0 : 1; + d = to_double(mantissa, real_exponent, negative); + return true; + } + // We have to round to even. The "to even" part + // is only a problem when we are right in between two floats + // which we guard against. + // If we have lots of trailing zeros, we may fall right between two + // floating-point values. + // + // The round-to-even cases take the form of a number 2m+1 which is in (2^53,2^54] + // times a power of two. That is, it is right between a number with binary significand + // m and another number with binary significand m+1; and it must be the case + // that it cannot be represented by a float itself. + // + // We must have that w * 10 ^q == (2m+1) * 2^p for some power of two 2^p. + // Recall that 10^q = 5^q * 2^q. + // When q >= 0, we must have that (2m+1) is divible by 5^q, so 5^q <= 2^54. We have that + // 5^23 <= 2^54 and it is the last power of five to qualify, so q <= 23. + // When q<0, we have w >= (2m+1) x 5^{-q}. We must have that w<2^{64} so + // (2m+1) x 5^{-q} < 2^{64}. We have that 2m+1>2^{53}. Hence, we must have + // 2^{53} x 5^{-q} < 2^{64}. + // Hence we have 5^{-q} < 2^{11}$ or q>= -4. + // + // We require lower <= 1 and not lower == 0 because we could not prove that + // that lower == 0 is implied; but we could prove that lower <= 1 is a necessary and sufficient test. + if (simdjson_unlikely((lower <= 1) && (power >= -4) && (power <= 23) && ((mantissa & 3) == 1))) { + if ((mantissa << (upperbit + 64 - 53 - 2)) == upper) { + mantissa &= ~1; // flip it so that we do not round up + } + } + + mantissa += mantissa & 1; + mantissa >>= 1; + + // Here we have mantissa < (1<<53), unless there was an overflow + if (mantissa >= (1ULL << 53)) { + ////////// + // This will happen when parsing values such as 7.2057594037927933e+16 + //////// + mantissa = (1ULL << 52); + real_exponent++; + } + mantissa &= ~(1ULL << 52); + // we have to check that real_exponent is in range, otherwise we bail out + if (simdjson_unlikely(real_exponent > 2046)) { + // We have an infinite value!!! We could actually throw an error here if we could. + return false; + } + d = to_double(mantissa, real_exponent, negative); + return true; + } + + // We call a fallback floating-point parser that might be slow. Note + // it will accept JSON numbers, but the JSON spec. is more restrictive so + // before you call parse_float_fallback, you need to have validated the input + // string with the JSON grammar. + // It will return an error (false) if the parsed number is infinite. + // The string parsing itself always succeeds. We know that there is at least + // one digit. + static bool parse_float_fallback(const uint8_t* ptr, double* outDouble) { + *outDouble = simdjson::internal::from_chars(reinterpret_cast(ptr)); + // We do not accept infinite values. + + // Detecting finite values in a portable manner is ridiculously hard, ideally + // we would want to do: + // return !std::isfinite(*outDouble); + // but that mysteriously fails under legacy/old libc++ libraries, see + // https://github.com/simdjson/simdjson/issues/1286 + // + // Therefore, fall back to this solution (the extra parens are there + // to handle that max may be a macro on windows). + return !(*outDouble > (std::numeric_limits::max)() || *outDouble < std::numeric_limits::lowest()); + } + + static bool parse_float_fallback(const uint8_t* ptr, const uint8_t* end_ptr, double* outDouble) { + *outDouble = simdjson::internal::from_chars(reinterpret_cast(ptr), reinterpret_cast(end_ptr)); + // We do not accept infinite values. + + // Detecting finite values in a portable manner is ridiculously hard, ideally + // we would want to do: + // return !std::isfinite(*outDouble); + // but that mysteriously fails under legacy/old libc++ libraries, see + // https://github.com/simdjson/simdjson/issues/1286 + // + // Therefore, fall back to this solution (the extra parens are there + // to handle that max may be a macro on windows). + return !(*outDouble > (std::numeric_limits::max)() || *outDouble < std::numeric_limits::lowest()); + } + + // check quickly whether the next 8 chars are made of digits + // at a glance, it looks better than Mula's + // http://0x80.pl/articles/swar-digits-validate.html + simdjson_inline bool is_made_of_eight_digits_fast(const uint8_t* chars) { + uint64_t val; + // this can read up to 7 bytes beyond the buffer size, but we require + // SIMDJSON_PADDING of padding + static_assert(7 <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be bigger than 7"); + std::memcpy(&val, chars, 8); + // a branchy method might be faster: + // return (( val & 0xF0F0F0F0F0F0F0F0 ) == 0x3030303030303030) + // && (( (val + 0x0606060606060606) & 0xF0F0F0F0F0F0F0F0 ) == + // 0x3030303030303030); + return (((val & 0xF0F0F0F0F0F0F0F0) | + (((val + 0x0606060606060606) & 0xF0F0F0F0F0F0F0F0) >> 4)) == + 0x3333333333333333); + } + + template + SIMDJSON_NO_SANITIZE_UNDEFINED // We deliberately allow overflow here and check later + simdjson_inline bool parse_digit(const uint8_t c, I& i) { + const uint8_t digit = static_cast(c - '0'); + if (digit > 9) { + return false; + } + // PERF NOTE: multiplication by 10 is cheaper than arbitrary integer multiplication + i = 10 * i + digit; // might overflow, we will handle the overflow later + return true; + } + + simdjson_inline bool is_digit(const uint8_t c) { + return static_cast(c - '0') <= 9; + } + + simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t* const src, const uint8_t*& p, uint64_t& i, int64_t& exponent) { + // we continue with the fiction that we have an integer. If the + // floating point number is representable as x * 10^z for some integer + // z that fits in 53 bits, then we will be able to convert back the + // the integer into a float in a lossless manner. + const uint8_t* const first_after_period = p; + +#ifdef SIMDJSON_SWAR_NUMBER_PARSING +#if SIMDJSON_SWAR_NUMBER_PARSING + // this helps if we have lots of decimals! + // this turns out to be frequent enough. + if (is_made_of_eight_digits_fast(p)) { + i = i * 100000000 + parse_eight_digits_unrolled(p); + p += 8; + } +#endif // SIMDJSON_SWAR_NUMBER_PARSING +#endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING + // Unrolling the first digit makes a small difference on some implementations (e.g. westmere) + if (parse_digit(*p, i)) { ++p; } + while (parse_digit(*p, i)) { p++; } + exponent = first_after_period - p; + // Decimal without digits (123.) is illegal + if (exponent == 0) { + return INVALID_NUMBER(src); + } + return SUCCESS; + } + + simdjson_inline error_code parse_exponent(simdjson_unused const uint8_t* const src, const uint8_t*& p, int64_t& exponent) { + // Exp Sign: -123.456e[-]78 + bool neg_exp = ('-' == *p); + if (neg_exp || '+' == *p) { p++; } // Skip + as well + + // Exponent: -123.456e-[78] + auto start_exp = p; + int64_t exp_number = 0; + while (parse_digit(*p, exp_number)) { ++p; } + // It is possible for parse_digit to overflow. + // In particular, it could overflow to INT64_MIN, and we cannot do - INT64_MIN. + // Thus we *must* check for possible overflow before we negate exp_number. + + // Performance notes: it may seem like combining the two "simdjson_unlikely checks" below into + // a single simdjson_unlikely path would be faster. The reasoning is sound, but the compiler may + // not oblige and may, in fact, generate two distinct paths in any case. It might be + // possible to do uint64_t(p - start_exp - 1) >= 18 but it could end up trading off + // instructions for a simdjson_likely branch, an unconclusive gain. + + // If there were no digits, it's an error. + if (simdjson_unlikely(p == start_exp)) { + return INVALID_NUMBER(src); + } + // We have a valid positive exponent in exp_number at this point, except that + // it may have overflowed. + + // If there were more than 18 digits, we may have overflowed the integer. We have to do + // something!!!! + if (simdjson_unlikely(p > start_exp + 18)) { + // Skip leading zeroes: 1e000000000000000000001 is technically valid and does not overflow + while (*start_exp == '0') { start_exp++; } + // 19 digits could overflow int64_t and is kind of absurd anyway. We don't + // support exponents smaller than -999,999,999,999,999,999 and bigger + // than 999,999,999,999,999,999. + // We can truncate. + // Note that 999999999999999999 is assuredly too large. The maximal ieee64 value before + // infinity is ~1.8e308. The smallest subnormal is ~5e-324. So, actually, we could + // truncate at 324. + // Note that there is no reason to fail per se at this point in time. + // E.g., 0e999999999999999999999 is a fine number. + if (p > start_exp + 18) { exp_number = 999999999999999999; } + } + // At this point, we know that exp_number is a sane, positive, signed integer. + // It is <= 999,999,999,999,999,999. As long as 'exponent' is in + // [-8223372036854775808, 8223372036854775808], we won't overflow. Because 'exponent' + // is bounded in magnitude by the size of the JSON input, we are fine in this universe. + // To sum it up: the next line should never overflow. + exponent += (neg_exp ? -exp_number : exp_number); + return SUCCESS; + } + + simdjson_inline bool check_if_integer(const uint8_t* const src, size_t max_length) { + const uint8_t* const srcend = src + max_length; + bool negative = (*src == '-'); // we can always read at least one character after the '-' + const uint8_t* p = src + uint8_t(negative); + if (p == srcend) { return false; } + if (*p == '0') { + ++p; + if (p == srcend) { return true; } + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return false; } + return true; + } + while (p != srcend && is_digit(*p)) { ++p; } + if (p == srcend) { return true; } + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return false; } + return true; + } + + simdjson_inline size_t significant_digits(const uint8_t* start_digits, size_t digit_count) { + // It is possible that the integer had an overflow. + // We have to handle the case where we have 0.0000somenumber. + const uint8_t* start = start_digits; + while ((*start == '0') || (*start == '.')) { ++start; } + // we over-decrement by one when there is a '.' + return digit_count - size_t(start - start_digits); + } + + } // unnamed namespace + + /** @private */ + static error_code slow_float_parsing(simdjson_unused const uint8_t* src, double* answer) { + if (parse_float_fallback(src, answer)) { + return SUCCESS; + } + return INVALID_NUMBER(src); + } + + /** @private */ + template + simdjson_inline error_code write_float(const uint8_t* const src, bool negative, uint64_t i, const uint8_t* start_digits, size_t digit_count, int64_t exponent, W& writer) { + // If we frequently had to deal with long strings of digits, + // we could extend our code by using a 128-bit integer instead + // of a 64-bit integer. However, this is uncommon in practice. + // + // 9999999999999999999 < 2**64 so we can accommodate 19 digits. + // If we have a decimal separator, then digit_count - 1 is the number of digits, but we + // may not have a decimal separator! + if (simdjson_unlikely(digit_count > 19 && significant_digits(start_digits, digit_count) > 19)) { + // Ok, chances are good that we had an overflow! + // this is almost never going to get called!!! + // we start anew, going slowly!!! + // This will happen in the following examples: + // 10000000000000000000000000000000000000000000e+308 + // 3.1415926535897932384626433832795028841971693993751 + // + // NOTE: We do not pass a reference to the to slow_float_parsing. If we passed our writer + // reference to it, it would force it to be stored in memory, preventing the compiler from + // picking it apart and putting into registers. i.e. if we pass it as reference, + // it gets slow. + double d; + error_code error = slow_float_parsing(src, &d); + writer.append_double(d); + return error; + } + // NOTE: it's weird that the simdjson_unlikely() only wraps half the if, but it seems to get slower any other + // way we've tried: https://github.com/simdjson/simdjson/pull/990#discussion_r448497331 + // To future reader: we'd love if someone found a better way, or at least could explain this result! + if (simdjson_unlikely(exponent < simdjson::internal::smallest_power) || (exponent > simdjson::internal::largest_power)) { + // + // Important: smallest_power is such that it leads to a zero value. + // Observe that 18446744073709551615e-343 == 0, i.e. (2**64 - 1) e -343 is zero + // so something x 10^-343 goes to zero, but not so with something x 10^-342. + static_assert(simdjson::internal::smallest_power <= -342, "smallest_power is not small enough"); + // + if ((exponent < simdjson::internal::smallest_power) || (i == 0)) { + // E.g. Parse "-0.0e-999" into the same value as "-0.0". See https://en.wikipedia.org/wiki/Signed_zero + WRITE_DOUBLE(negative ? -0.0 : 0.0, src, writer); + return SUCCESS; + } + else { // (exponent > largest_power) and (i != 0) + // We have, for sure, an infinite value and simdjson refuses to parse infinite values. + return INVALID_NUMBER(src); + } + } + double d; + if (!compute_float_64(exponent, i, negative, d)) { + // we are almost never going to get here. + if (!parse_float_fallback(src, &d)) { return INVALID_NUMBER(src); } + } + WRITE_DOUBLE(d, src, writer); + return SUCCESS; + } + + // parse the number at src + // define JSON_TEST_NUMBERS for unit testing + // + // It is assumed that the number is followed by a structural ({,},],[) character + // or a white space character. If that is not the case (e.g., when the JSON + // document is made of a single number), then it is necessary to copy the + // content and append a space before calling this function. + // + // Our objective is accurate parsing (ULP of 0) at high speed. + template + simdjson_inline error_code parse_number(const uint8_t* const src, W& writer); + + // for performance analysis, it is sometimes useful to skip parsing +#ifdef SIMDJSON_SKIPNUMBERPARSING + + template + simdjson_inline error_code parse_number(const uint8_t* const, W& writer) { + writer.append_s64(0); // always write zero + return SUCCESS; // always succeeds + } + + simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t* const src) noexcept { return 0; } + simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t* const src) noexcept { return 0; } + simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t* const src) noexcept { return 0; } + simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t* const src) noexcept { return 0; } + simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t* const src) noexcept { return 0; } + simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t* const src) noexcept { return 0; } + simdjson_unused simdjson_inline bool is_negative(const uint8_t* src) noexcept { return false; } + simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t* src) noexcept { return false; } + simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t* src) noexcept { return number_type::signed_integer; } +#else + +// parse the number at src +// define JSON_TEST_NUMBERS for unit testing +// +// It is assumed that the number is followed by a structural ({,},],[) character +// or a white space character. If that is not the case (e.g., when the JSON +// document is made of a single number), then it is necessary to copy the +// content and append a space before calling this function. +// +// Our objective is accurate parsing (ULP of 0) at high speed. + template + simdjson_inline error_code parse_number(const uint8_t* const src, W& writer) { + + // + // Check for minus sign + // + bool negative = (*src == '-'); + const uint8_t* p = src + uint8_t(negative); + + // + // Parse the integer part. + // + // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare + const uint8_t* const start_digits = p; + uint64_t i = 0; + while (parse_digit(*p, i)) { p++; } + + // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. + // Optimization note: size_t is expected to be unsigned. + size_t digit_count = size_t(p - start_digits); + if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { return INVALID_NUMBER(src); } + + // + // Handle floats if there is a . or e (or both) + // + int64_t exponent = 0; + bool is_float = false; + if ('.' == *p) { + is_float = true; + ++p; + SIMDJSON_TRY(parse_decimal_after_separator(src, p, i, exponent)); + digit_count = int(p - start_digits); // used later to guard against overflows + } + if (('e' == *p) || ('E' == *p)) { + is_float = true; + ++p; + SIMDJSON_TRY(parse_exponent(src, p, exponent)); + } + if (is_float) { + const bool dirty_end = jsoncharutils::is_not_structural_or_whitespace(*p); + SIMDJSON_TRY(write_float(src, negative, i, start_digits, digit_count, exponent, writer)); + if (dirty_end) { return INVALID_NUMBER(src); } + return SUCCESS; + } + + // The longest negative 64-bit number is 19 digits. + // The longest positive 64-bit number is 20 digits. + // We do it this way so we don't trigger this branch unless we must. + size_t longest_digit_count = negative ? 19 : 20; + if (digit_count > longest_digit_count) { return BIGINT_NUMBER(src); } + if (digit_count == longest_digit_count) { + if (negative) { + // Anything negative above INT64_MAX+1 is invalid + if (i > uint64_t(INT64_MAX) + 1) { return BIGINT_NUMBER(src); } + WRITE_INTEGER(~i + 1, src, writer); + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return INVALID_NUMBER(src); } + return SUCCESS; + // Positive overflow check: + // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the + // biggest uint64_t. + // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX. + // If we got here, it's a 20 digit number starting with the digit "1". + // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller + // than 1,553,255,926,290,448,384. + // - That is smaller than the smallest possible 20-digit number the user could write: + // 10,000,000,000,000,000,000. + // - Therefore, if the number is positive and lower than that, it's overflow. + // - The value we are looking at is less than or equal to INT64_MAX. + // + } + else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INVALID_NUMBER(src); } + } + + // Write unsigned if it does not fit in a signed integer. + if (i > uint64_t(INT64_MAX)) { + WRITE_UNSIGNED(i, src, writer); + } + else { + WRITE_INTEGER(negative ? (~i + 1) : i, src, writer); + } + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return INVALID_NUMBER(src); } + return SUCCESS; + } + + // Inlineable functions + namespace { + + // This table can be used to characterize the final character of an integer + // string. For JSON structural character and allowable white space characters, + // we return SUCCESS. For 'e', '.' and 'E', we return INCORRECT_TYPE. Otherwise + // we return NUMBER_ERROR. + // Optimization note: we could easily reduce the size of the table by half (to 128) + // at the cost of an extra branch. + // Optimization note: we want the values to use at most 8 bits (not, e.g., 32 bits): + static_assert(error_code(uint8_t(NUMBER_ERROR)) == NUMBER_ERROR, "bad NUMBER_ERROR cast"); + static_assert(error_code(uint8_t(SUCCESS)) == SUCCESS, "bad NUMBER_ERROR cast"); + static_assert(error_code(uint8_t(INCORRECT_TYPE)) == INCORRECT_TYPE, "bad NUMBER_ERROR cast"); + + const uint8_t integer_string_finisher[256] = { + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, + SUCCESS, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, + NUMBER_ERROR, INCORRECT_TYPE, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, INCORRECT_TYPE, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, SUCCESS, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, INCORRECT_TYPE, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, + SUCCESS, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR }; + + // Parse any number from 0 to 18,446,744,073,709,551,615 + simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t* const src) noexcept { + const uint8_t* p = src; + // + // Parse the integer part. + // + // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare + const uint8_t* const start_digits = p; + uint64_t i = 0; + while (parse_digit(*p, i)) { p++; } + + // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. + // Optimization note: size_t is expected to be unsigned. + size_t digit_count = size_t(p - start_digits); + // The longest positive 64-bit number is 20 digits. + // We do it this way so we don't trigger this branch unless we must. + // Optimization note: the compiler can probably merge + // ((digit_count == 0) || (digit_count > 20)) + // into a single branch since digit_count is unsigned. + if ((digit_count == 0) || (digit_count > 20)) { return INCORRECT_TYPE; } + // Here digit_count > 0. + if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } + // We can do the following... + // if (!jsoncharutils::is_structural_or_whitespace(*p)) { + // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; + // } + // as a single table lookup: + if (integer_string_finisher[*p] != SUCCESS) { return error_code(integer_string_finisher[*p]); } + + if (digit_count == 20) { + // Positive overflow check: + // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the + // biggest uint64_t. + // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX. + // If we got here, it's a 20 digit number starting with the digit "1". + // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller + // than 1,553,255,926,290,448,384. + // - That is smaller than the smallest possible 20-digit number the user could write: + // 10,000,000,000,000,000,000. + // - Therefore, if the number is positive and lower than that, it's overflow. + // - The value we are looking at is less than or equal to INT64_MAX. + // + if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INCORRECT_TYPE; } + } + + return i; + } + + + // Parse any number from 0 to 18,446,744,073,709,551,615 + // Never read at src_end or beyond + simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t* const src, const uint8_t* const src_end) noexcept { + const uint8_t* p = src; + // + // Parse the integer part. + // + // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare + const uint8_t* const start_digits = p; + uint64_t i = 0; + while ((p != src_end) && parse_digit(*p, i)) { p++; } + + // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. + // Optimization note: size_t is expected to be unsigned. + size_t digit_count = size_t(p - start_digits); + // The longest positive 64-bit number is 20 digits. + // We do it this way so we don't trigger this branch unless we must. + // Optimization note: the compiler can probably merge + // ((digit_count == 0) || (digit_count > 20)) + // into a single branch since digit_count is unsigned. + if ((digit_count == 0) || (digit_count > 20)) { return INCORRECT_TYPE; } + // Here digit_count > 0. + if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } + // We can do the following... + // if (!jsoncharutils::is_structural_or_whitespace(*p)) { + // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; + // } + // as a single table lookup: + if ((p != src_end) && integer_string_finisher[*p] != SUCCESS) { return error_code(integer_string_finisher[*p]); } + + if (digit_count == 20) { + // Positive overflow check: + // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the + // biggest uint64_t. + // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX. + // If we got here, it's a 20 digit number starting with the digit "1". + // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller + // than 1,553,255,926,290,448,384. + // - That is smaller than the smallest possible 20-digit number the user could write: + // 10,000,000,000,000,000,000. + // - Therefore, if the number is positive and lower than that, it's overflow. + // - The value we are looking at is less than or equal to INT64_MAX. + // + if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INCORRECT_TYPE; } + } + + return i; + } + + // Parse any number from 0 to 18,446,744,073,709,551,615 + simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t* const src) noexcept { + const uint8_t* p = src + 1; + // + // Parse the integer part. + // + // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare + const uint8_t* const start_digits = p; + uint64_t i = 0; + while (parse_digit(*p, i)) { p++; } + + // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. + // Optimization note: size_t is expected to be unsigned. + size_t digit_count = size_t(p - start_digits); + // The longest positive 64-bit number is 20 digits. + // We do it this way so we don't trigger this branch unless we must. + // Optimization note: the compiler can probably merge + // ((digit_count == 0) || (digit_count > 20)) + // into a single branch since digit_count is unsigned. + if ((digit_count == 0) || (digit_count > 20)) { return INCORRECT_TYPE; } + // Here digit_count > 0. + if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } + // We can do the following... + // if (!jsoncharutils::is_structural_or_whitespace(*p)) { + // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; + // } + // as a single table lookup: + if (*p != '"') { return NUMBER_ERROR; } + + if (digit_count == 20) { + // Positive overflow check: + // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the + // biggest uint64_t. + // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX. + // If we got here, it's a 20 digit number starting with the digit "1". + // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller + // than 1,553,255,926,290,448,384. + // - That is smaller than the smallest possible 20-digit number the user could write: + // 10,000,000,000,000,000,000. + // - Therefore, if the number is positive and lower than that, it's overflow. + // - The value we are looking at is less than or equal to INT64_MAX. + // + // Note: we use src[1] and not src[0] because src[0] is the quote character in this + // instance. + if (src[1] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INCORRECT_TYPE; } + } + + return i; + } + + // Parse any number from -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 + simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t* src) noexcept { + // + // Check for minus sign + // + bool negative = (*src == '-'); + const uint8_t* p = src + uint8_t(negative); + + // + // Parse the integer part. + // + // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare + const uint8_t* const start_digits = p; + uint64_t i = 0; + while (parse_digit(*p, i)) { p++; } + + // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. + // Optimization note: size_t is expected to be unsigned. + size_t digit_count = size_t(p - start_digits); + // We go from + // -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 + // so we can never represent numbers that have more than 19 digits. + size_t longest_digit_count = 19; + // Optimization note: the compiler can probably merge + // ((digit_count == 0) || (digit_count > longest_digit_count)) + // into a single branch since digit_count is unsigned. + if ((digit_count == 0) || (digit_count > longest_digit_count)) { return INCORRECT_TYPE; } + // Here digit_count > 0. + if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } + // We can do the following... + // if (!jsoncharutils::is_structural_or_whitespace(*p)) { + // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; + // } + // as a single table lookup: + if (integer_string_finisher[*p] != SUCCESS) { return error_code(integer_string_finisher[*p]); } + // Negative numbers have can go down to - INT64_MAX - 1 whereas positive numbers are limited to INT64_MAX. + // Performance note: This check is only needed when digit_count == longest_digit_count but it is + // so cheap that we might as well always make it. + if (i > uint64_t(INT64_MAX) + uint64_t(negative)) { return INCORRECT_TYPE; } + return negative ? (~i + 1) : i; + } + + // Parse any number from -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 + // Never read at src_end or beyond + simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t* const src, const uint8_t* const src_end) noexcept { + // + // Check for minus sign + // + if (src == src_end) { return NUMBER_ERROR; } + bool negative = (*src == '-'); + const uint8_t* p = src + uint8_t(negative); + + // + // Parse the integer part. + // + // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare + const uint8_t* const start_digits = p; + uint64_t i = 0; + while ((p != src_end) && parse_digit(*p, i)) { p++; } + + // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. + // Optimization note: size_t is expected to be unsigned. + size_t digit_count = size_t(p - start_digits); + // We go from + // -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 + // so we can never represent numbers that have more than 19 digits. + size_t longest_digit_count = 19; + // Optimization note: the compiler can probably merge + // ((digit_count == 0) || (digit_count > longest_digit_count)) + // into a single branch since digit_count is unsigned. + if ((digit_count == 0) || (digit_count > longest_digit_count)) { return INCORRECT_TYPE; } + // Here digit_count > 0. + if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } + // We can do the following... + // if (!jsoncharutils::is_structural_or_whitespace(*p)) { + // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; + // } + // as a single table lookup: + if ((p != src_end) && integer_string_finisher[*p] != SUCCESS) { return error_code(integer_string_finisher[*p]); } + // Negative numbers have can go down to - INT64_MAX - 1 whereas positive numbers are limited to INT64_MAX. + // Performance note: This check is only needed when digit_count == longest_digit_count but it is + // so cheap that we might as well always make it. + if (i > uint64_t(INT64_MAX) + uint64_t(negative)) { return INCORRECT_TYPE; } + return negative ? (~i + 1) : i; + } + + // Parse any number from -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 + simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t* src) noexcept { + // + // Check for minus sign + // + bool negative = (*(src + 1) == '-'); + src += uint8_t(negative) + 1; + + // + // Parse the integer part. + // + // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare + const uint8_t* const start_digits = src; + uint64_t i = 0; + while (parse_digit(*src, i)) { src++; } + + // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. + // Optimization note: size_t is expected to be unsigned. + size_t digit_count = size_t(src - start_digits); + // We go from + // -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 + // so we can never represent numbers that have more than 19 digits. + size_t longest_digit_count = 19; + // Optimization note: the compiler can probably merge + // ((digit_count == 0) || (digit_count > longest_digit_count)) + // into a single branch since digit_count is unsigned. + if ((digit_count == 0) || (digit_count > longest_digit_count)) { return INCORRECT_TYPE; } + // Here digit_count > 0. + if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } + // We can do the following... + // if (!jsoncharutils::is_structural_or_whitespace(*src)) { + // return (*src == '.' || *src == 'e' || *src == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; + // } + // as a single table lookup: + if (*src != '"') { return NUMBER_ERROR; } + // Negative numbers have can go down to - INT64_MAX - 1 whereas positive numbers are limited to INT64_MAX. + // Performance note: This check is only needed when digit_count == longest_digit_count but it is + // so cheap that we might as well always make it. + if (i > uint64_t(INT64_MAX) + uint64_t(negative)) { return INCORRECT_TYPE; } + return negative ? (~i + 1) : i; + } + + simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t* src) noexcept { + // + // Check for minus sign + // + bool negative = (*src == '-'); + src += uint8_t(negative); + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t* p = src; + p += parse_digit(*p, i); + bool leading_zero = (i == 0); + while (parse_digit(*p, i)) { p++; } + // no integer digits, or 0123 (zero must be solo) + if (p == src) { return INCORRECT_TYPE; } + if ((leading_zero && p != src + 1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely(*p == '.')) { + p++; + const uint8_t* start_decimal_digits = p; + if (!parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits + p++; + while (parse_digit(*p, i)) { p++; } + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p - src - 1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t* start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = p - start_digits > 19; + } + } + else { + overflow = p - src > 19; + } + + // + // Parse the exponent + // + if (*p == 'e' || *p == 'E') { + p++; + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t* start_exp_digits = p; + while (parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p - start_exp_digits == 0 || p - start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0 - exp : exp; + } + + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + double d; + if (simdjson_likely(!overflow)) { + if (compute_float_64(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), &d)) { + return NUMBER_ERROR; + } + return d; + } + + simdjson_unused simdjson_inline bool is_negative(const uint8_t* src) noexcept { + return (*src == '-'); + } + + simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t* src) noexcept { + bool negative = (*src == '-'); + src += uint8_t(negative); + const uint8_t* p = src; + while (static_cast(*p - '0') <= 9) { p++; } + if (p == src) { return NUMBER_ERROR; } + if (jsoncharutils::is_structural_or_whitespace(*p)) { return true; } + return false; + } + + simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t* src) noexcept { + bool negative = (*src == '-'); + src += uint8_t(negative); + const uint8_t* p = src; + while (static_cast(*p - '0') <= 9) { p++; } + size_t digit_count = size_t(p - src); + if (p == src) { return NUMBER_ERROR; } + if (jsoncharutils::is_structural_or_whitespace(*p)) { + static const uint8_t* smaller_big_integer = reinterpret_cast("9223372036854775808"); + // We have an integer. + if (simdjson_unlikely(digit_count > 20)) { + return number_type::big_integer; + } + // If the number is negative and valid, it must be a signed integer. + if (negative) { + if (simdjson_unlikely(digit_count > 19)) return number_type::big_integer; + if (simdjson_unlikely(digit_count == 19 && memcmp(src, smaller_big_integer, 19) > 0)) { + return number_type::big_integer; + } + return number_type::signed_integer; + } + // Let us check if we have a big integer (>=2**64). + static const uint8_t* two_to_sixtyfour = reinterpret_cast("18446744073709551616"); + if ((digit_count > 20) || (digit_count == 20 && memcmp(src, two_to_sixtyfour, 20) >= 0)) { + return number_type::big_integer; + } + // The number is positive and smaller than 18446744073709551616 (or 2**64). + // We want values larger or equal to 9223372036854775808 to be unsigned + // integers, and the other values to be signed integers. + if ((digit_count == 20) || (digit_count >= 19 && memcmp(src, smaller_big_integer, 19) >= 0)) { + return number_type::unsigned_integer; + } + return number_type::signed_integer; + } + // Hopefully, we have 'e' or 'E' or '.'. + return number_type::floating_point_number; + } + + // Never read at src_end or beyond + simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t* src, const uint8_t* const src_end) noexcept { + if (src == src_end) { return NUMBER_ERROR; } + // + // Check for minus sign + // + bool negative = (*src == '-'); + src += uint8_t(negative); + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t* p = src; + if (p == src_end) { return NUMBER_ERROR; } + p += parse_digit(*p, i); + bool leading_zero = (i == 0); + while ((p != src_end) && parse_digit(*p, i)) { p++; } + // no integer digits, or 0123 (zero must be solo) + if (p == src) { return INCORRECT_TYPE; } + if ((leading_zero && p != src + 1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely((p != src_end) && (*p == '.'))) { + p++; + const uint8_t* start_decimal_digits = p; + if ((p == src_end) || !parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits + p++; + while ((p != src_end) && parse_digit(*p, i)) { p++; } + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p - src - 1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t* start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = start_digits - src > 19; + } + } + else { + overflow = p - src > 19; + } + + // + // Parse the exponent + // + if ((p != src_end) && (*p == 'e' || *p == 'E')) { + p++; + if (p == src_end) { return NUMBER_ERROR; } + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t* start_exp_digits = p; + while ((p != src_end) && parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p - start_exp_digits == 0 || p - start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0 - exp : exp; + } + + if ((p != src_end) && jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + double d; + if (simdjson_likely(!overflow)) { + if (compute_float_64(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), src_end, &d)) { + return NUMBER_ERROR; + } + return d; + } + + simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t* src) noexcept { + // + // Check for minus sign + // + bool negative = (*(src + 1) == '-'); + src += uint8_t(negative) + 1; + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t* p = src; + p += parse_digit(*p, i); + bool leading_zero = (i == 0); + while (parse_digit(*p, i)) { p++; } + // no integer digits, or 0123 (zero must be solo) + if (p == src) { return INCORRECT_TYPE; } + if ((leading_zero && p != src + 1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely(*p == '.')) { + p++; + const uint8_t* start_decimal_digits = p; + if (!parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits + p++; + while (parse_digit(*p, i)) { p++; } + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p - src - 1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t* start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = p - start_digits > 19; + } + } + else { + overflow = p - src > 19; + } + + // + // Parse the exponent + // + if (*p == 'e' || *p == 'E') { + p++; + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t* start_exp_digits = p; + while (parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p - start_exp_digits == 0 || p - start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0 - exp : exp; + } + + if (*p != '"') { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + double d; + if (simdjson_likely(!overflow)) { + if (compute_float_64(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), &d)) { + return NUMBER_ERROR; + } + return d; + } + + } // unnamed namespace +#endif // SIMDJSON_SKIPNUMBERPARSING + + } // namespace numberparsing + + inline std::ostream& operator<<(std::ostream& out, number_type type) noexcept { + switch (type) { + case number_type::signed_integer: out << "integer in [-9223372036854775808,9223372036854775808)"; break; + case number_type::unsigned_integer: out << "unsigned integer in [9223372036854775808,18446744073709551616)"; break; + case number_type::floating_point_number: out << "floating-point number (binary64)"; break; + case number_type::big_integer: out << "big integer"; break; + default: SIMDJSON_UNREACHABLE(); + } + return out; + } + + } // namespace lsx +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_NUMBERPARSING_H +/* end file simdjson/generic/numberparsing.h for lsx */ + +/* including simdjson/generic/implementation_simdjson_result_base-inl.h for lsx: #include "simdjson/generic/implementation_simdjson_result_base-inl.h" */ +/* begin file simdjson/generic/implementation_simdjson_result_base-inl.h for lsx */ +#ifndef SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_INL_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_INL_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/implementation_simdjson_result_base.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lsx { + + // + // internal::implementation_simdjson_result_base inline implementation + // + + template + simdjson_inline void implementation_simdjson_result_base::tie(T& value, error_code& error) && noexcept { + error = this->second; + if (!error) { + value = std::forward>(*this).first; + } + } + + template + simdjson_warn_unused simdjson_inline error_code implementation_simdjson_result_base::get(T& value) && noexcept { + error_code error; + std::forward>(*this).tie(value, error); + return error; + } + + template + simdjson_inline error_code implementation_simdjson_result_base::error() const noexcept { + return this->second; + } + +#if SIMDJSON_EXCEPTIONS + + template + simdjson_inline T& implementation_simdjson_result_base::value() & noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return this->first; + } + + template + simdjson_inline T&& implementation_simdjson_result_base::value() && noexcept(false) { + return std::forward>(*this).take_value(); + } + + template + simdjson_inline T&& implementation_simdjson_result_base::take_value() && noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return std::forward(this->first); + } + + template + simdjson_inline implementation_simdjson_result_base::operator T && () && noexcept(false) { + return std::forward>(*this).take_value(); + } + +#endif // SIMDJSON_EXCEPTIONS + + template + simdjson_inline const T& implementation_simdjson_result_base::value_unsafe() const& noexcept { + return this->first; + } + + template + simdjson_inline T& implementation_simdjson_result_base::value_unsafe() & noexcept { + return this->first; + } + + template + simdjson_inline T&& implementation_simdjson_result_base::value_unsafe() && noexcept { + return std::forward(this->first); + } + + template + simdjson_inline implementation_simdjson_result_base::implementation_simdjson_result_base(T&& value, error_code error) noexcept + : first{ std::forward(value) }, second{ error } {} + template + simdjson_inline implementation_simdjson_result_base::implementation_simdjson_result_base(error_code error) noexcept + : implementation_simdjson_result_base(T{}, error) {} + template + simdjson_inline implementation_simdjson_result_base::implementation_simdjson_result_base(T&& value) noexcept + : implementation_simdjson_result_base(std::forward(value), SUCCESS) {} + + } // namespace lsx +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_INL_H +/* end file simdjson/generic/implementation_simdjson_result_base-inl.h for lsx */ +/* end file simdjson/generic/amalgamated.h for lsx */ +/* including simdjson/lsx/end.h: #include "simdjson/lsx/end.h" */ +/* begin file simdjson/lsx/end.h */ +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/lsx/base.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +#undef SIMDJSON_SKIP_BACKSLASH_SHORT_CIRCUIT +/* undefining SIMDJSON_IMPLEMENTATION from "lsx" */ +#undef SIMDJSON_IMPLEMENTATION +/* end file simdjson/lsx/end.h */ + +#endif // SIMDJSON_LSX_H +/* end file simdjson/lsx.h */ +/* including simdjson/lsx/implementation.h: #include */ +/* begin file simdjson/lsx/implementation.h */ +#ifndef SIMDJSON_LSX_IMPLEMENTATION_H +#define SIMDJSON_LSX_IMPLEMENTATION_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/implementation.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/instruction_set.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lsx { + + /** + * @private + */ + class implementation final : public simdjson::implementation { + public: + simdjson_inline implementation() : simdjson::implementation("lsx", "LoongArch SX", internal::instruction_set::LSX) {} + simdjson_warn_unused error_code create_dom_parser_implementation( + size_t capacity, + size_t max_length, + std::unique_ptr& dst + ) const noexcept final; + simdjson_warn_unused error_code minify(const uint8_t* buf, size_t len, uint8_t* dst, size_t& dst_len) const noexcept final; + simdjson_warn_unused bool validate_utf8(const char* buf, size_t len) const noexcept final; + }; + + } // namespace lsx +} // namespace simdjson + +#endif // SIMDJSON_LSX_IMPLEMENTATION_H +/* end file simdjson/lsx/implementation.h */ + +/* including simdjson/lsx/begin.h: #include */ +/* begin file simdjson/lsx/begin.h */ +/* defining SIMDJSON_IMPLEMENTATION to "lsx" */ +#define SIMDJSON_IMPLEMENTATION lsx +/* including simdjson/lsx/base.h: #include "simdjson/lsx/base.h" */ +/* begin file simdjson/lsx/base.h */ +#ifndef SIMDJSON_LSX_BASE_H +#define SIMDJSON_LSX_BASE_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/base.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + /** + * Implementation for LSX. + */ + namespace lsx { + + class implementation; + + namespace { + namespace simd { + template struct simd8; + template struct simd8x64; + } // namespace simd + } // unnamed namespace + + } // namespace lsx +} // namespace simdjson + +#endif // SIMDJSON_LSX_BASE_H +/* end file simdjson/lsx/base.h */ +/* including simdjson/lsx/intrinsics.h: #include "simdjson/lsx/intrinsics.h" */ +/* begin file simdjson/lsx/intrinsics.h */ +#ifndef SIMDJSON_LSX_INTRINSICS_H +#define SIMDJSON_LSX_INTRINSICS_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/lsx/base.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +// This should be the correct header whether +// you use visual studio or other compilers. +#include + +static_assert(sizeof(__m128i) <= simdjson::SIMDJSON_PADDING, "insufficient padding for LoongArch SX"); + +#endif // SIMDJSON_LSX_INTRINSICS_H +/* end file simdjson/lsx/intrinsics.h */ +/* including simdjson/lsx/bitmanipulation.h: #include "simdjson/lsx/bitmanipulation.h" */ +/* begin file simdjson/lsx/bitmanipulation.h */ +#ifndef SIMDJSON_LSX_BITMANIPULATION_H +#define SIMDJSON_LSX_BITMANIPULATION_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/lsx/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/lsx/intrinsics.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/lsx/bitmask.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lsx { + namespace { + + // We sometimes call trailing_zero on inputs that are zero, + // but the algorithms do not end up using the returned value. + // Sadly, sanitizers are not smart enough to figure it out. + SIMDJSON_NO_SANITIZE_UNDEFINED + // This function can be used safely even if not all bytes have been + // initialized. + // See issue https://github.com/simdjson/simdjson/issues/1965 + SIMDJSON_NO_SANITIZE_MEMORY + simdjson_inline int trailing_zeroes(uint64_t input_num) { + return __builtin_ctzll(input_num); + } + + /* result might be undefined when input_num is zero */ + simdjson_inline uint64_t clear_lowest_bit(uint64_t input_num) { + return input_num & (input_num - 1); + } + + /* result might be undefined when input_num is zero */ + simdjson_inline int leading_zeroes(uint64_t input_num) { + return __builtin_clzll(input_num); + } + + /* result might be undefined when input_num is zero */ + simdjson_inline int count_ones(uint64_t input_num) { + return __lsx_vpickve2gr_w(__lsx_vpcnt_d(__m128i(v2u64{ input_num, 0 })), 0); + } + + simdjson_inline bool add_overflow(uint64_t value1, uint64_t value2, uint64_t* result) { + return __builtin_uaddll_overflow(value1, value2, + reinterpret_cast(result)); + } + + } // unnamed namespace + } // namespace lsx +} // namespace simdjson + +#endif // SIMDJSON_LSX_BITMANIPULATION_H +/* end file simdjson/lsx/bitmanipulation.h */ +/* including simdjson/lsx/bitmask.h: #include "simdjson/lsx/bitmask.h" */ +/* begin file simdjson/lsx/bitmask.h */ +#ifndef SIMDJSON_LSX_BITMASK_H +#define SIMDJSON_LSX_BITMASK_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/lsx/base.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lsx { + namespace { + + // + // Perform a "cumulative bitwise xor," flipping bits each time a 1 is encountered. + // + // For example, prefix_xor(00100100) == 00011100 + // + simdjson_inline uint64_t prefix_xor(uint64_t bitmask) { + bitmask ^= bitmask << 1; + bitmask ^= bitmask << 2; + bitmask ^= bitmask << 4; + bitmask ^= bitmask << 8; + bitmask ^= bitmask << 16; + bitmask ^= bitmask << 32; + return bitmask; + } + + } // unnamed namespace + } // namespace lsx +} // namespace simdjson + +#endif +/* end file simdjson/lsx/bitmask.h */ +/* including simdjson/lsx/numberparsing_defs.h: #include "simdjson/lsx/numberparsing_defs.h" */ +/* begin file simdjson/lsx/numberparsing_defs.h */ +#ifndef SIMDJSON_LSX_NUMBERPARSING_DEFS_H +#define SIMDJSON_LSX_NUMBERPARSING_DEFS_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/lsx/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/lsx/intrinsics.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +#include + +namespace simdjson { + namespace lsx { + namespace numberparsing { + + // we don't have appropriate instructions, so let us use a scalar function + // credit: https://johnnylee-sde.github.io/Fast-numeric-string-to-int/ + /** @private */ + static simdjson_inline uint32_t parse_eight_digits_unrolled(const uint8_t* chars) { + uint64_t val; + std::memcpy(&val, chars, sizeof(uint64_t)); + val = (val & 0x0F0F0F0F0F0F0F0F) * 2561 >> 8; + val = (val & 0x00FF00FF00FF00FF) * 6553601 >> 16; + return uint32_t((val & 0x0000FFFF0000FFFF) * 42949672960001 >> 32); + } + + simdjson_inline internal::value128 full_multiplication(uint64_t value1, uint64_t value2) { + internal::value128 answer; + __uint128_t r = (static_cast<__uint128_t>(value1)) * value2; + answer.low = uint64_t(r); + answer.high = uint64_t(r >> 64); + return answer; + } + + } // namespace numberparsing + } // namespace lsx +} // namespace simdjson + +#define SIMDJSON_SWAR_NUMBER_PARSING 1 + +#endif // SIMDJSON_LSX_NUMBERPARSING_DEFS_H +/* end file simdjson/lsx/numberparsing_defs.h */ +/* including simdjson/lsx/simd.h: #include "simdjson/lsx/simd.h" */ +/* begin file simdjson/lsx/simd.h */ +#ifndef SIMDJSON_LSX_SIMD_H +#define SIMDJSON_LSX_SIMD_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/lsx/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/lsx/bitmanipulation.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/simdprune_tables.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lsx { + namespace { + namespace simd { + + // Forward-declared so they can be used by splat and friends. + template + struct base { + __m128i value; + + // Zero constructor + simdjson_inline base() : value{ __m128i() } {} + + // Conversion from SIMD register + simdjson_inline base(const __m128i _value) : value(_value) {} + + // Conversion to SIMD register + simdjson_inline operator const __m128i& () const { return this->value; } + simdjson_inline operator __m128i& () { return this->value; } + simdjson_inline operator const v16i8& () const { return (v16i8&)this->value; } + simdjson_inline operator v16i8& () { return (v16i8&)this->value; } + + // Bit operations + simdjson_inline Child operator|(const Child other) const { return __lsx_vor_v(*this, other); } + simdjson_inline Child operator&(const Child other) const { return __lsx_vand_v(*this, other); } + simdjson_inline Child operator^(const Child other) const { return __lsx_vxor_v(*this, other); } + simdjson_inline Child bit_andnot(const Child other) const { return __lsx_vandn_v(other, *this); } + simdjson_inline Child& operator|=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast | other; return *this_cast; } + simdjson_inline Child& operator&=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast & other; return *this_cast; } + simdjson_inline Child& operator^=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast ^ other; return *this_cast; } + }; + + // Forward-declared so they can be used by splat and friends. + template + struct simd8; + + template> + struct base8 : base> { + simdjson_inline base8() : base>() {} + simdjson_inline base8(const __m128i _value) : base>(_value) {} + + friend simdjson_really_inline Mask operator==(const simd8 lhs, const simd8 rhs) { return __lsx_vseq_b(lhs, rhs); } + + static const int SIZE = sizeof(base>::value); + + template + simdjson_inline simd8 prev(const simd8 prev_chunk) const { + return __lsx_vor_v(__lsx_vbsll_v(*this, N), __lsx_vbsrl_v(prev_chunk, 16 - N)); + } + }; + + // SIMD byte mask type (returned by things like eq and gt) + template<> + struct simd8 : base8 { + static simdjson_inline simd8 splat(bool _value) { + return __lsx_vreplgr2vr_b(uint8_t(-(!!_value))); + } + + simdjson_inline simd8() : base8() {} + simdjson_inline simd8(const __m128i _value) : base8(_value) {} + // Splat constructor + simdjson_inline simd8(bool _value) : base8(splat(_value)) {} + + simdjson_inline int to_bitmask() const { return __lsx_vpickve2gr_w(__lsx_vmskltz_b(*this), 0); } + simdjson_inline bool any() const { return 0 == __lsx_vpickve2gr_hu(__lsx_vmsknz_b(*this), 0); } + simdjson_inline simd8 operator~() const { return *this ^ true; } + }; + + template + struct base8_numeric : base8 { + static simdjson_inline simd8 splat(T _value) { return __lsx_vreplgr2vr_b(_value); } + static simdjson_inline simd8 zero() { return __lsx_vldi(0); } + static simdjson_inline simd8 load(const T values[16]) { + return __lsx_vld(reinterpret_cast(values), 0); + } + // Repeat 16 values as many times as necessary (usually for lookup tables) + static simdjson_inline simd8 repeat_16( + T v0, T v1, T v2, T v3, T v4, T v5, T v6, T v7, + T v8, T v9, T v10, T v11, T v12, T v13, T v14, T v15 + ) { + return simd8( + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15 + ); + } + + simdjson_inline base8_numeric() : base8() {} + simdjson_inline base8_numeric(const __m128i _value) : base8(_value) {} + + // Store to array + simdjson_inline void store(T dst[16]) const { + return __lsx_vst(*this, reinterpret_cast<__m128i*>(dst), 0); + } + + // Addition/subtraction are the same for signed and unsigned + simdjson_inline simd8 operator+(const simd8 other) const { return __lsx_vadd_b(*this, other); } + simdjson_inline simd8 operator-(const simd8 other) const { return __lsx_vsub_b(*this, other); } + simdjson_inline simd8& operator+=(const simd8 other) { *this = *this + other; return *static_cast*>(this); } + simdjson_inline simd8& operator-=(const simd8 other) { *this = *this - other; return *static_cast*>(this); } + + // Override to distinguish from bool version + simdjson_inline simd8 operator~() const { return *this ^ 0xFFu; } + + // Perform a lookup assuming the value is between 0 and 16 (undefined behavior for out of range values) + template + simdjson_inline simd8 lookup_16(simd8 lookup_table) const { + return __lsx_vshuf_b(lookup_table, lookup_table, *this); + } + + // Copies to 'output" all bytes corresponding to a 0 in the mask (interpreted as a bitset). + // Passing a 0 value for mask would be equivalent to writing out every byte to output. + // Only the first 16 - count_ones(mask) bytes of the result are significant but 16 bytes + // get written. + template + simdjson_inline void compress(uint16_t mask, L* output) const { + using internal::thintable_epi8; + using internal::BitsSetTable256mul2; + using internal::pshufb_combine_table; + // this particular implementation was inspired by haswell + // lsx do it in 2 steps, first 8 bytes and then second 8 bytes... + uint8_t mask1 = uint8_t(mask); // least significant 8 bits + uint8_t mask2 = uint8_t(mask >> 8); // second least significant 8 bits + // next line just loads the 64-bit values thintable_epi8[mask1] and + // thintable_epi8[mask2] into a 128-bit register. + __m128i shufmask = { int64_t(thintable_epi8[mask1]), int64_t(thintable_epi8[mask2]) + 0x0808080808080808 }; + // this is the version "nearly pruned" + __m128i pruned = __lsx_vshuf_b(*this, *this, shufmask); + // we still need to put the pieces back together. + // we compute the popcount of the first words: + int pop1 = BitsSetTable256mul2[mask1]; + // then load the corresponding mask + __m128i compactmask = __lsx_vldx(reinterpret_cast(reinterpret_cast(pshufb_combine_table)), pop1 * 8); + __m128i answer = __lsx_vshuf_b(pruned, pruned, compactmask); + __lsx_vst(answer, reinterpret_cast(output), 0); + } + + template + simdjson_inline simd8 lookup_16( + L replace0, L replace1, L replace2, L replace3, + L replace4, L replace5, L replace6, L replace7, + L replace8, L replace9, L replace10, L replace11, + L replace12, L replace13, L replace14, L replace15) const { + return lookup_16(simd8::repeat_16( + replace0, replace1, replace2, replace3, + replace4, replace5, replace6, replace7, + replace8, replace9, replace10, replace11, + replace12, replace13, replace14, replace15 + )); + } + }; + + // Signed bytes + template<> + struct simd8 : base8_numeric { + simdjson_inline simd8() : base8_numeric() {} + simdjson_inline simd8(const __m128i _value) : base8_numeric(_value) {} + // Splat constructor + simdjson_inline simd8(int8_t _value) : simd8(splat(_value)) {} + // Array constructor + simdjson_inline simd8(const int8_t values[16]) : simd8(load(values)) {} + // Member-by-member initialization + simdjson_inline simd8( + int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, + int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 + ) : simd8({ + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10,v11,v12,v13,v14,v15 + }) {} + // Repeat 16 values as many times as necessary (usually for lookup tables) + simdjson_inline static simd8 repeat_16( + int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, + int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 + ) { + return simd8( + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15 + ); + } + + // Order-sensitive comparisons + simdjson_inline simd8 max_val(const simd8 other) const { return __lsx_vmax_b(*this, other); } + simdjson_inline simd8 min_val(const simd8 other) const { return __lsx_vmin_b(*this, other); } + simdjson_inline simd8 operator>(const simd8 other) const { return __lsx_vslt_b(other, *this); } + simdjson_inline simd8 operator<(const simd8 other) const { return __lsx_vslt_b(*this, other); } + }; + + // Unsigned bytes + template<> + struct simd8 : base8_numeric { + simdjson_inline simd8() : base8_numeric() {} + simdjson_inline simd8(const __m128i _value) : base8_numeric(_value) {} + // Splat constructor + simdjson_inline simd8(uint8_t _value) : simd8(splat(_value)) {} + // Array constructor + simdjson_inline simd8(const uint8_t values[16]) : simd8(load(values)) {} + // Member-by-member initialization + simdjson_inline simd8( + uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, + uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 + ) : simd8(__m128i(v16u8{ + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10,v11,v12,v13,v14,v15 + })) {} + // Repeat 16 values as many times as necessary (usually for lookup tables) + simdjson_inline static simd8 repeat_16( + uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, + uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 + ) { + return simd8( + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15 + ); + } + + // Saturated math + simdjson_inline simd8 saturating_add(const simd8 other) const { return __lsx_vsadd_bu(*this, other); } + simdjson_inline simd8 saturating_sub(const simd8 other) const { return __lsx_vssub_bu(*this, other); } + + // Order-specific operations + simdjson_inline simd8 max_val(const simd8 other) const { return __lsx_vmax_bu(*this, other); } + simdjson_inline simd8 min_val(const simd8 other) const { return __lsx_vmin_bu(other, *this); } + // Same as >, but only guarantees true is nonzero (< guarantees true = -1) + simdjson_inline simd8 gt_bits(const simd8 other) const { return this->saturating_sub(other); } + // Same as <, but only guarantees true is nonzero (< guarantees true = -1) + simdjson_inline simd8 lt_bits(const simd8 other) const { return other.saturating_sub(*this); } + simdjson_inline simd8 operator<=(const simd8 other) const { return other.max_val(*this) == other; } + simdjson_inline simd8 operator>=(const simd8 other) const { return other.min_val(*this) == other; } + simdjson_inline simd8 operator>(const simd8 other) const { return this->gt_bits(other).any_bits_set(); } + simdjson_inline simd8 operator<(const simd8 other) const { return this->lt_bits(other).any_bits_set(); } + + // Bit-specific operations + simdjson_inline simd8 bits_not_set() const { return *this == uint8_t(0); } + simdjson_inline simd8 bits_not_set(simd8 bits) const { return (*this & bits).bits_not_set(); } + simdjson_inline simd8 any_bits_set() const { return ~this->bits_not_set(); } + simdjson_inline simd8 any_bits_set(simd8 bits) const { return ~this->bits_not_set(bits); } + simdjson_inline bool is_ascii() const { return 0 == __lsx_vpickve2gr_w(__lsx_vmskltz_b(*this), 0); } + simdjson_inline bool bits_not_set_anywhere() const { return 0 == __lsx_vpickve2gr_hu(__lsx_vmsknz_b(*this), 0); } + simdjson_inline bool any_bits_set_anywhere() const { return !bits_not_set_anywhere(); } + simdjson_inline bool bits_not_set_anywhere(simd8 bits) const { + return 0 == __lsx_vpickve2gr_hu(__lsx_vmsknz_b(__lsx_vand_v(*this, bits)), 0); + } + simdjson_inline bool any_bits_set_anywhere(simd8 bits) const { return !bits_not_set_anywhere(bits); } + template + simdjson_inline simd8 shr() const { return simd8(__lsx_vsrli_b(*this, N)); } + template + simdjson_inline simd8 shl() const { return simd8(__lsx_vslli_b(*this, N)); } + }; + + template + struct simd8x64 { + static constexpr int NUM_CHUNKS = 64 / sizeof(simd8); + static_assert(NUM_CHUNKS == 4, "LSX kernel should use four registers per 64-byte block."); + const simd8 chunks[NUM_CHUNKS]; + + simd8x64(const simd8x64& o) = delete; // no copy allowed + simd8x64& operator=(const simd8& other) = delete; // no assignment allowed + simd8x64() = delete; // no default constructor allowed + + simdjson_inline simd8x64(const simd8 chunk0, const simd8 chunk1, const simd8 chunk2, const simd8 chunk3) : chunks{ chunk0, chunk1, chunk2, chunk3 } {} + simdjson_inline simd8x64(const T ptr[64]) : chunks{ simd8::load(ptr), simd8::load(ptr + 16), simd8::load(ptr + 32), simd8::load(ptr + 48) } {} + + simdjson_inline uint64_t compress(uint64_t mask, T* output) const { + uint16_t mask1 = uint16_t(mask); + uint16_t mask2 = uint16_t(mask >> 16); + uint16_t mask3 = uint16_t(mask >> 32); + uint16_t mask4 = uint16_t(mask >> 48); + __m128i zcnt = __lsx_vpcnt_h(__m128i(v2u64{ ~mask, 0 })); + uint64_t zcnt1 = __lsx_vpickve2gr_hu(zcnt, 0); + uint64_t zcnt2 = __lsx_vpickve2gr_hu(zcnt, 1); + uint64_t zcnt3 = __lsx_vpickve2gr_hu(zcnt, 2); + uint64_t zcnt4 = __lsx_vpickve2gr_hu(zcnt, 3); + uint8_t* voutput = reinterpret_cast(output); + // There should be a critical value which processes in scaler is faster. + if (zcnt1) + this->chunks[0].compress(mask1, reinterpret_cast(voutput)); + voutput += zcnt1; + if (zcnt2) + this->chunks[1].compress(mask2, reinterpret_cast(voutput)); + voutput += zcnt2; + if (zcnt3) + this->chunks[2].compress(mask3, reinterpret_cast(voutput)); + voutput += zcnt3; + if (zcnt4) + this->chunks[3].compress(mask4, reinterpret_cast(voutput)); + voutput += zcnt4; + return reinterpret_cast(voutput) - reinterpret_cast(output); + } + + simdjson_inline void store(T ptr[64]) const { + this->chunks[0].store(ptr + sizeof(simd8) * 0); + this->chunks[1].store(ptr + sizeof(simd8) * 1); + this->chunks[2].store(ptr + sizeof(simd8) * 2); + this->chunks[3].store(ptr + sizeof(simd8) * 3); + } + + simdjson_inline uint64_t to_bitmask() const { + __m128i mask1 = __lsx_vmskltz_b(this->chunks[0]); + __m128i mask2 = __lsx_vmskltz_b(this->chunks[1]); + __m128i mask3 = __lsx_vmskltz_b(this->chunks[2]); + __m128i mask4 = __lsx_vmskltz_b(this->chunks[3]); + mask1 = __lsx_vilvl_h(mask2, mask1); + mask2 = __lsx_vilvl_h(mask4, mask3); + return __lsx_vpickve2gr_du(__lsx_vilvl_w(mask2, mask1), 0); + } + + simdjson_inline simd8 reduce_or() const { + return (this->chunks[0] | this->chunks[1]) | (this->chunks[2] | this->chunks[3]); + } + + simdjson_inline uint64_t eq(const T m) const { + const simd8 mask = simd8::splat(m); + return simd8x64( + this->chunks[0] == mask, + this->chunks[1] == mask, + this->chunks[2] == mask, + this->chunks[3] == mask + ).to_bitmask(); + } + + simdjson_inline uint64_t eq(const simd8x64& other) const { + return simd8x64( + this->chunks[0] == other.chunks[0], + this->chunks[1] == other.chunks[1], + this->chunks[2] == other.chunks[2], + this->chunks[3] == other.chunks[3] + ).to_bitmask(); + } + + simdjson_inline uint64_t lteq(const T m) const { + const simd8 mask = simd8::splat(m); + return simd8x64( + this->chunks[0] <= mask, + this->chunks[1] <= mask, + this->chunks[2] <= mask, + this->chunks[3] <= mask + ).to_bitmask(); + } + }; // struct simd8x64 + + } // namespace simd + } // unnamed namespace + } // namespace lsx +} // namespace simdjson + +#endif // SIMDJSON_LSX_SIMD_H +/* end file simdjson/lsx/simd.h */ +/* including simdjson/lsx/stringparsing_defs.h: #include "simdjson/lsx/stringparsing_defs.h" */ +/* begin file simdjson/lsx/stringparsing_defs.h */ +#ifndef SIMDJSON_LSX_STRINGPARSING_DEFS_H +#define SIMDJSON_LSX_STRINGPARSING_DEFS_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/lsx/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/lsx/simd.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/lsx/bitmanipulation.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lsx { + namespace { + + using namespace simd; + + // Holds backslashes and quotes locations. + struct backslash_and_quote { + public: + static constexpr uint32_t BYTES_PROCESSED = 32; + simdjson_inline static backslash_and_quote copy_and_find(const uint8_t* src, uint8_t* dst); + + simdjson_inline bool has_quote_first() { return ((bs_bits - 1) & quote_bits) != 0; } + simdjson_inline bool has_backslash() { return bs_bits != 0; } + simdjson_inline int quote_index() { return trailing_zeroes(quote_bits); } + simdjson_inline int backslash_index() { return trailing_zeroes(bs_bits); } + + uint32_t bs_bits; + uint32_t quote_bits; + }; // struct backslash_and_quote + + simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8_t* src, uint8_t* dst) { + // this can read up to 31 bytes beyond the buffer size, but we require + // SIMDJSON_PADDING of padding + static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "backslash and quote finder must process fewer than SIMDJSON_PADDING bytes"); + simd8 v0(src); + simd8 v1(src + sizeof(v0)); + v0.store(dst); + v1.store(dst + sizeof(v0)); + + // Getting a 64-bit bitmask is much cheaper than multiple 16-bit bitmasks on LSX; therefore, we + // smash them together into a 64-byte mask and get the bitmask from there. + uint64_t bs_and_quote = simd8x64(v0 == '\\', v1 == '\\', v0 == '"', v1 == '"').to_bitmask(); + return { + uint32_t(bs_and_quote), // bs_bits + uint32_t(bs_and_quote >> 32) // quote_bits + }; + } + + } // unnamed namespace + } // namespace lsx +} // namespace simdjson + +#endif // SIMDJSON_LSX_STRINGPARSING_DEFS_H +/* end file simdjson/lsx/stringparsing_defs.h */ + +#define SIMDJSON_SKIP_BACKSLASH_SHORT_CIRCUIT 1 +/* end file simdjson/lsx/begin.h */ +/* including generic/amalgamated.h for lsx: #include */ +/* begin file generic/amalgamated.h for lsx */ +#if defined(SIMDJSON_CONDITIONAL_INCLUDE) && !defined(SIMDJSON_SRC_GENERIC_DEPENDENCIES_H) +#error generic/dependencies.h must be included before generic/amalgamated.h! +#endif + +/* including generic/base.h for lsx: #include */ +/* begin file generic/base.h for lsx */ +#ifndef SIMDJSON_SRC_GENERIC_BASE_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_BASE_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lsx { + namespace { + + struct json_character_block; + + } // unnamed namespace + } // namespace lsx +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_BASE_H +/* end file generic/base.h for lsx */ +/* including generic/dom_parser_implementation.h for lsx: #include */ +/* begin file generic/dom_parser_implementation.h for lsx */ +#ifndef SIMDJSON_SRC_GENERIC_DOM_PARSER_IMPLEMENTATION_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_DOM_PARSER_IMPLEMENTATION_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +// Interface a dom parser implementation must fulfill +namespace simdjson { + namespace lsx { + namespace { + + simdjson_inline simd8 must_be_2_3_continuation(const simd8 prev2, const simd8 prev3); + simdjson_inline bool is_ascii(const simd8x64& input); + + } // unnamed namespace + } // namespace lsx +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_DOM_PARSER_IMPLEMENTATION_H +/* end file generic/dom_parser_implementation.h for lsx */ +/* including generic/json_character_block.h for lsx: #include */ +/* begin file generic/json_character_block.h for lsx */ +#ifndef SIMDJSON_SRC_GENERIC_JSON_CHARACTER_BLOCK_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_JSON_CHARACTER_BLOCK_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lsx { + namespace { + + struct json_character_block { + static simdjson_inline json_character_block classify(const simd::simd8x64& in); + + simdjson_inline uint64_t whitespace() const noexcept { return _whitespace; } + simdjson_inline uint64_t op() const noexcept { return _op; } + simdjson_inline uint64_t scalar() const noexcept { return ~(op() | whitespace()); } + + uint64_t _whitespace; + uint64_t _op; + }; + + } // unnamed namespace + } // namespace lsx +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_JSON_CHARACTER_BLOCK_H +/* end file generic/json_character_block.h for lsx */ +/* end file generic/amalgamated.h for lsx */ +/* including generic/stage1/amalgamated.h for lsx: #include */ +/* begin file generic/stage1/amalgamated.h for lsx */ +// Stuff other things depend on +/* including generic/stage1/base.h for lsx: #include */ +/* begin file generic/stage1/base.h for lsx */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE1_BASE_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_BASE_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lsx { + namespace { + namespace stage1 { + + class bit_indexer; + template + struct buf_block_reader; + struct json_block; + class json_minifier; + class json_scanner; + struct json_string_block; + class json_string_scanner; + class json_structural_indexer; + + } // namespace stage1 + + namespace utf8_validation { + struct utf8_checker; + } // namespace utf8_validation + + using utf8_validation::utf8_checker; + + } // unnamed namespace + } // namespace lsx +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE1_BASE_H +/* end file generic/stage1/base.h for lsx */ +/* including generic/stage1/buf_block_reader.h for lsx: #include */ +/* begin file generic/stage1/buf_block_reader.h for lsx */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE1_BUF_BLOCK_READER_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_BUF_BLOCK_READER_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +#include + +namespace simdjson { + namespace lsx { + namespace { + namespace stage1 { + + // Walks through a buffer in block-sized increments, loading the last part with spaces + template + struct buf_block_reader { + public: + simdjson_inline buf_block_reader(const uint8_t* _buf, size_t _len); + simdjson_inline size_t block_index(); + simdjson_inline bool has_full_block() const; + simdjson_inline const uint8_t* full_block() const; + /** + * Get the last block, padded with spaces. + * + * There will always be a last block, with at least 1 byte, unless len == 0 (in which case this + * function fills the buffer with spaces and returns 0. In particular, if len == STEP_SIZE there + * will be 0 full_blocks and 1 remainder block with STEP_SIZE bytes and no spaces for padding. + * + * @return the number of effective characters in the last block. + */ + simdjson_inline size_t get_remainder(uint8_t* dst) const; + simdjson_inline void advance(); + private: + const uint8_t* buf; + const size_t len; + const size_t lenminusstep; + size_t idx; + }; + + // Routines to print masks and text for debugging bitmask operations + simdjson_unused static char* format_input_text_64(const uint8_t* text) { + static char buf[sizeof(simd8x64) + 1]; + for (size_t i = 0; i < sizeof(simd8x64); i++) { + buf[i] = int8_t(text[i]) < ' ' ? '_' : int8_t(text[i]); + } + buf[sizeof(simd8x64)] = '\0'; + return buf; + } + + // Routines to print masks and text for debugging bitmask operations + simdjson_unused static char* format_input_text(const simd8x64& in) { + static char buf[sizeof(simd8x64) + 1]; + in.store(reinterpret_cast(buf)); + for (size_t i = 0; i < sizeof(simd8x64); i++) { + if (buf[i] < ' ') { buf[i] = '_'; } + } + buf[sizeof(simd8x64)] = '\0'; + return buf; + } + + simdjson_unused static char* format_input_text(const simd8x64& in, uint64_t mask) { + static char buf[sizeof(simd8x64) + 1]; + in.store(reinterpret_cast(buf)); + for (size_t i = 0; i < sizeof(simd8x64); i++) { + if (buf[i] <= ' ') { buf[i] = '_'; } + if (!(mask & (size_t(1) << i))) { buf[i] = ' '; } + } + buf[sizeof(simd8x64)] = '\0'; + return buf; + } + + simdjson_unused static char* format_mask(uint64_t mask) { + static char buf[sizeof(simd8x64) + 1]; + for (size_t i = 0; i < 64; i++) { + buf[i] = (mask & (size_t(1) << i)) ? 'X' : ' '; + } + buf[64] = '\0'; + return buf; + } + + template + simdjson_inline buf_block_reader::buf_block_reader(const uint8_t* _buf, size_t _len) : buf{ _buf }, len{ _len }, lenminusstep{ len < STEP_SIZE ? 0 : len - STEP_SIZE }, idx{ 0 } {} + + template + simdjson_inline size_t buf_block_reader::block_index() { return idx; } + + template + simdjson_inline bool buf_block_reader::has_full_block() const { + return idx < lenminusstep; + } + + template + simdjson_inline const uint8_t* buf_block_reader::full_block() const { + return &buf[idx]; + } + + template + simdjson_inline size_t buf_block_reader::get_remainder(uint8_t* dst) const { + if (len == idx) { return 0; } // memcpy(dst, null, 0) will trigger an error with some sanitizers + std::memset(dst, 0x20, STEP_SIZE); // std::memset STEP_SIZE because it's more efficient to write out 8 or 16 bytes at once. + std::memcpy(dst, buf + idx, len - idx); + return len - idx; + } + + template + simdjson_inline void buf_block_reader::advance() { + idx += STEP_SIZE; + } + + } // namespace stage1 + } // unnamed namespace + } // namespace lsx +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE1_BUF_BLOCK_READER_H +/* end file generic/stage1/buf_block_reader.h for lsx */ +/* including generic/stage1/json_escape_scanner.h for lsx: #include */ +/* begin file generic/stage1/json_escape_scanner.h for lsx */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE1_JSON_ESCAPE_SCANNER_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_JSON_ESCAPE_SCANNER_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lsx { + namespace { + namespace stage1 { + + /** + * Scans for escape characters in JSON, taking care with multiple backslashes (\\n vs. \n). + */ + struct json_escape_scanner { + /** The actual escape characters (the backslashes themselves). */ + uint64_t next_is_escaped = 0ULL; + + struct escaped_and_escape { + /** + * Mask of escaped characters. + * + * ``` + * \n \\n \\\n \\\\n \ + * 0100100010100101000 + * n \ \ n \ \ + * ``` + */ + uint64_t escaped; + /** + * Mask of escape characters. + * + * ``` + * \n \\n \\\n \\\\n \ + * 1001000101001010001 + * \ \ \ \ \ \ \ + * ``` + */ + uint64_t escape; + }; + + /** + * Get a mask of both escape and escaped characters (the characters following a backslash). + * + * @param potential_escape A mask of the character that can escape others (but could be + * escaped itself). e.g. block.eq('\\') + */ + simdjson_really_inline escaped_and_escape next(uint64_t backslash) noexcept { + +#if !SIMDJSON_SKIP_BACKSLASH_SHORT_CIRCUIT + if (!backslash) { return { next_escaped_without_backslashes(), 0 }; } +#endif + + // | | Mask (shows characters instead of 1's) | Depth | Instructions | + // |--------------------------------|----------------------------------------|-------|---------------------| + // | string | `\\n_\\\n___\\\n___\\\\___\\\\__\\\` | | | + // | | ` even odd even odd odd` | | | + // | potential_escape | ` \ \\\ \\\ \\\\ \\\\ \\\` | 1 | 1 (backslash & ~first_is_escaped) + // | escape_and_terminal_code | ` \n \ \n \ \n \ \ \ \ \ \` | 5 | 5 (next_escape_and_terminal_code()) + // | escaped | `\ \ n \ n \ \ \ \ \ ` X | 6 | 7 (escape_and_terminal_code ^ (potential_escape | first_is_escaped)) + // | escape | ` \ \ \ \ \ \ \ \ \ \` | 6 | 8 (escape_and_terminal_code & backslash) + // | first_is_escaped | `\ ` | 7 (*) | 9 (escape >> 63) () + // (*) this is not needed until the next iteration + uint64_t escape_and_terminal_code = next_escape_and_terminal_code(backslash & ~this->next_is_escaped); + uint64_t escaped = escape_and_terminal_code ^ (backslash | this->next_is_escaped); + uint64_t escape = escape_and_terminal_code & backslash; + this->next_is_escaped = escape >> 63; + return { escaped, escape }; + } + + private: + static constexpr const uint64_t ODD_BITS = 0xAAAAAAAAAAAAAAAAULL; + + simdjson_really_inline uint64_t next_escaped_without_backslashes() noexcept { + uint64_t escaped = this->next_is_escaped; + this->next_is_escaped = 0; + return escaped; + } + + /** + * Returns a mask of the next escape characters (masking out escaped backslashes), along with + * any non-backslash escape codes. + * + * \n \\n \\\n \\\\n returns: + * \n \ \ \n \ \ + * 11 100 1011 10100 + * + * You are expected to mask out the first bit yourself if the previous block had a trailing + * escape. + * + * & the result with potential_escape to get just the escape characters. + * ^ the result with (potential_escape | first_is_escaped) to get escaped characters. + */ + static simdjson_really_inline uint64_t next_escape_and_terminal_code(uint64_t potential_escape) noexcept { + // If we were to just shift and mask out any odd bits, we'd actually get a *half* right answer: + // any even-aligned backslash runs would be correct! Odd-aligned backslash runs would be + // inverted (\\\ would be 010 instead of 101). + // + // ``` + // string: | ____\\\\_\\\\_____ | + // maybe_escaped | ODD | \ \ \ \ | + // even-aligned ^^^ ^^^^ odd-aligned + // ``` + // + // Taking that into account, our basic strategy is: + // + // 1. Use subtraction to produce a mask with 1's for even-aligned runs and 0's for + // odd-aligned runs. + // 2. XOR all odd bits, which masks out the odd bits in even-aligned runs, and brings IN the + // odd bits in odd-aligned runs. + // 3. & with backslash to clean up any stray bits. + // runs are set to 0, and then XORing with "odd": + // + // | | Mask (shows characters instead of 1's) | Instructions | + // |--------------------------------|----------------------------------------|---------------------| + // | string | `\\n_\\\n___\\\n___\\\\___\\\\__\\\` | + // | | ` even odd even odd odd` | + // | maybe_escaped | ` n \\n \\n \\\_ \\\_ \\` X | 1 (potential_escape << 1) + // | maybe_escaped_and_odd | ` \n_ \\n _ \\\n_ _ \\\__ _\\\_ \\\` | 1 (maybe_escaped | odd) + // | even_series_codes_and_odd | ` n_\\\ _ n_ _\\\\ _ _ ` | 1 (maybe_escaped_and_odd - potential_escape) + // | escape_and_terminal_code | ` \n \ \n \ \n \ \ \ \ \ \` | 1 (^ odd) + // + + // Escaped characters are characters following an escape. + uint64_t maybe_escaped = potential_escape << 1; + + // To distinguish odd from even escape sequences, therefore, we turn on any *starting* + // escapes that are on an odd byte. (We actually bring in all odd bits, for speed.) + // - Odd runs of backslashes are 0000, and the code at the end ("n" in \n or \\n) is 1. + // - Odd runs of backslashes are 1111, and the code at the end ("n" in \n or \\n) is 0. + // - All other odd bytes are 1, and even bytes are 0. + uint64_t maybe_escaped_and_odd_bits = maybe_escaped | ODD_BITS; + uint64_t even_series_codes_and_odd_bits = maybe_escaped_and_odd_bits - potential_escape; + + // Now we flip all odd bytes back with xor. This: + // - Makes odd runs of backslashes go from 0000 to 1010 + // - Makes even runs of backslashes go from 1111 to 1010 + // - Sets actually-escaped codes to 1 (the n in \n and \\n: \n = 11, \\n = 100) + // - Resets all other bytes to 0 + return even_series_codes_and_odd_bits ^ ODD_BITS; + } + }; + + } // namespace stage1 + } // unnamed namespace + } // namespace lsx +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRING_SCANNER_H +/* end file generic/stage1/json_escape_scanner.h for lsx */ +/* including generic/stage1/json_string_scanner.h for lsx: #include */ +/* begin file generic/stage1/json_string_scanner.h for lsx */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRING_SCANNER_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRING_SCANNER_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lsx { + namespace { + namespace stage1 { + + struct json_string_block { + // We spell out the constructors in the hope of resolving inlining issues with Visual Studio 2017 + simdjson_really_inline json_string_block(uint64_t escaped, uint64_t quote, uint64_t in_string) : + _escaped(escaped), _quote(quote), _in_string(in_string) {} + + // Escaped characters (characters following an escape() character) + simdjson_really_inline uint64_t escaped() const { return _escaped; } + // Real (non-backslashed) quotes + simdjson_really_inline uint64_t quote() const { return _quote; } + // Only characters inside the string (not including the quotes) + simdjson_really_inline uint64_t string_content() const { return _in_string & ~_quote; } + // Return a mask of whether the given characters are inside a string (only works on non-quotes) + simdjson_really_inline uint64_t non_quote_inside_string(uint64_t mask) const { return mask & _in_string; } + // Return a mask of whether the given characters are inside a string (only works on non-quotes) + simdjson_really_inline uint64_t non_quote_outside_string(uint64_t mask) const { return mask & ~_in_string; } + // Tail of string (everything except the start quote) + simdjson_really_inline uint64_t string_tail() const { return _in_string ^ _quote; } + + // escaped characters (backslashed--does not include the hex characters after \u) + uint64_t _escaped; + // real quotes (non-escaped ones) + uint64_t _quote; + // string characters (includes start quote but not end quote) + uint64_t _in_string; + }; + + // Scans blocks for string characters, storing the state necessary to do so + class json_string_scanner { + public: + simdjson_really_inline json_string_block next(const simd::simd8x64& in); + // Returns either UNCLOSED_STRING or SUCCESS + simdjson_really_inline error_code finish(); + + private: + // Scans for escape characters + json_escape_scanner escape_scanner{}; + // Whether the last iteration was still inside a string (all 1's = true, all 0's = false). + uint64_t prev_in_string = 0ULL; + }; + + // + // Return a mask of all string characters plus end quotes. + // + // prev_escaped is overflow saying whether the next character is escaped. + // prev_in_string is overflow saying whether we're still in a string. + // + // Backslash sequences outside of quotes will be detected in stage 2. + // + simdjson_really_inline json_string_block json_string_scanner::next(const simd::simd8x64& in) { + const uint64_t backslash = in.eq('\\'); + const uint64_t escaped = escape_scanner.next(backslash).escaped; + const uint64_t quote = in.eq('"') & ~escaped; + + // + // prefix_xor flips on bits inside the string (and flips off the end quote). + // + // Then we xor with prev_in_string: if we were in a string already, its effect is flipped + // (characters inside strings are outside, and characters outside strings are inside). + // + const uint64_t in_string = prefix_xor(quote) ^ prev_in_string; + + // + // Check if we're still in a string at the end of the box so the next block will know + // + prev_in_string = uint64_t(static_cast(in_string) >> 63); + + // Use ^ to turn the beginning quote off, and the end quote on. + + // We are returning a function-local object so either we get a move constructor + // or we get copy elision. + return json_string_block(escaped, quote, in_string); + } + + simdjson_really_inline error_code json_string_scanner::finish() { + if (prev_in_string) { + return UNCLOSED_STRING; + } + return SUCCESS; + } + + } // namespace stage1 + } // unnamed namespace + } // namespace lsx +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRING_SCANNER_H +/* end file generic/stage1/json_string_scanner.h for lsx */ +/* including generic/stage1/utf8_lookup4_algorithm.h for lsx: #include */ +/* begin file generic/stage1/utf8_lookup4_algorithm.h for lsx */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE1_UTF8_LOOKUP4_ALGORITHM_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_UTF8_LOOKUP4_ALGORITHM_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lsx { + namespace { + namespace utf8_validation { + + using namespace simd; + + simdjson_inline simd8 check_special_cases(const simd8 input, const simd8 prev1) { + // Bit 0 = Too Short (lead byte/ASCII followed by lead byte/ASCII) + // Bit 1 = Too Long (ASCII followed by continuation) + // Bit 2 = Overlong 3-byte + // Bit 4 = Surrogate + // Bit 5 = Overlong 2-byte + // Bit 7 = Two Continuations + constexpr const uint8_t TOO_SHORT = 1 << 0; // 11______ 0_______ + // 11______ 11______ + constexpr const uint8_t TOO_LONG = 1 << 1; // 0_______ 10______ + constexpr const uint8_t OVERLONG_3 = 1 << 2; // 11100000 100_____ + constexpr const uint8_t SURROGATE = 1 << 4; // 11101101 101_____ + constexpr const uint8_t OVERLONG_2 = 1 << 5; // 1100000_ 10______ + constexpr const uint8_t TWO_CONTS = 1 << 7; // 10______ 10______ + constexpr const uint8_t TOO_LARGE = 1 << 3; // 11110100 1001____ + // 11110100 101_____ + // 11110101 1001____ + // 11110101 101_____ + // 1111011_ 1001____ + // 1111011_ 101_____ + // 11111___ 1001____ + // 11111___ 101_____ + constexpr const uint8_t TOO_LARGE_1000 = 1 << 6; + // 11110101 1000____ + // 1111011_ 1000____ + // 11111___ 1000____ + constexpr const uint8_t OVERLONG_4 = 1 << 6; // 11110000 1000____ + + const simd8 byte_1_high = prev1.shr<4>().lookup_16( + // 0_______ ________ + TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, + TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, + // 10______ ________ + TWO_CONTS, TWO_CONTS, TWO_CONTS, TWO_CONTS, + // 1100____ ________ + TOO_SHORT | OVERLONG_2, + // 1101____ ________ + TOO_SHORT, + // 1110____ ________ + TOO_SHORT | OVERLONG_3 | SURROGATE, + // 1111____ ________ + TOO_SHORT | TOO_LARGE | TOO_LARGE_1000 | OVERLONG_4 + ); + constexpr const uint8_t CARRY = TOO_SHORT | TOO_LONG | TWO_CONTS; // These all have ____ in byte 1 . + const simd8 byte_1_low = (prev1 & 0x0F).lookup_16( + // ____0000 ________ + CARRY | OVERLONG_3 | OVERLONG_2 | OVERLONG_4, + // ____0001 ________ + CARRY | OVERLONG_2, + // ____001_ ________ + CARRY, + CARRY, + + // ____0100 ________ + CARRY | TOO_LARGE, + // ____0101 ________ + CARRY | TOO_LARGE | TOO_LARGE_1000, + // ____011_ ________ + CARRY | TOO_LARGE | TOO_LARGE_1000, + CARRY | TOO_LARGE | TOO_LARGE_1000, + + // ____1___ ________ + CARRY | TOO_LARGE | TOO_LARGE_1000, + CARRY | TOO_LARGE | TOO_LARGE_1000, + CARRY | TOO_LARGE | TOO_LARGE_1000, + CARRY | TOO_LARGE | TOO_LARGE_1000, + CARRY | TOO_LARGE | TOO_LARGE_1000, + // ____1101 ________ + CARRY | TOO_LARGE | TOO_LARGE_1000 | SURROGATE, + CARRY | TOO_LARGE | TOO_LARGE_1000, + CARRY | TOO_LARGE | TOO_LARGE_1000 + ); + const simd8 byte_2_high = input.shr<4>().lookup_16( + // ________ 0_______ + TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT, + TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT, + + // ________ 1000____ + TOO_LONG | OVERLONG_2 | TWO_CONTS | OVERLONG_3 | TOO_LARGE_1000 | OVERLONG_4, + // ________ 1001____ + TOO_LONG | OVERLONG_2 | TWO_CONTS | OVERLONG_3 | TOO_LARGE, + // ________ 101_____ + TOO_LONG | OVERLONG_2 | TWO_CONTS | SURROGATE | TOO_LARGE, + TOO_LONG | OVERLONG_2 | TWO_CONTS | SURROGATE | TOO_LARGE, + + // ________ 11______ + TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT + ); + return (byte_1_high & byte_1_low & byte_2_high); + } + simdjson_inline simd8 check_multibyte_lengths(const simd8 input, + const simd8 prev_input, const simd8 sc) { + simd8 prev2 = input.prev<2>(prev_input); + simd8 prev3 = input.prev<3>(prev_input); + simd8 must23 = must_be_2_3_continuation(prev2, prev3); + simd8 must23_80 = must23 & uint8_t(0x80); + return must23_80 ^ sc; + } + + // + // Return nonzero if there are incomplete multibyte characters at the end of the block: + // e.g. if there is a 4-byte character, but it's 3 bytes from the end. + // + simdjson_inline simd8 is_incomplete(const simd8 input) { + // If the previous input's last 3 bytes match this, they're too short (they ended at EOF): + // ... 1111____ 111_____ 11______ +#if SIMDJSON_IMPLEMENTATION_ICELAKE + static const uint8_t max_array[64] = { + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 0xf0u - 1, 0xe0u - 1, 0xc0u - 1 + }; +#else + static const uint8_t max_array[32] = { + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 0xf0u - 1, 0xe0u - 1, 0xc0u - 1 + }; +#endif + const simd8 max_value(&max_array[sizeof(max_array) - sizeof(simd8)]); + return input.gt_bits(max_value); + } + + struct utf8_checker { + // If this is nonzero, there has been a UTF-8 error. + simd8 error; + // The last input we received + simd8 prev_input_block; + // Whether the last input we received was incomplete (used for ASCII fast path) + simd8 prev_incomplete; + + // + // Check whether the current bytes are valid UTF-8. + // + simdjson_inline void check_utf8_bytes(const simd8 input, const simd8 prev_input) { + // Flip prev1...prev3 so we can easily determine if they are 2+, 3+ or 4+ lead bytes + // (2, 3, 4-byte leads become large positive numbers instead of small negative numbers) + simd8 prev1 = input.prev<1>(prev_input); + simd8 sc = check_special_cases(input, prev1); + this->error |= check_multibyte_lengths(input, prev_input, sc); + } + + // The only problem that can happen at EOF is that a multibyte character is too short + // or a byte value too large in the last bytes: check_special_cases only checks for bytes + // too large in the first of two bytes. + simdjson_inline void check_eof() { + // If the previous block had incomplete UTF-8 characters at the end, an ASCII block can't + // possibly finish them. + this->error |= this->prev_incomplete; + } + + simdjson_inline void check_next_input(const simd8x64& input) { + if (simdjson_likely(is_ascii(input))) { + this->error |= this->prev_incomplete; + } + else { + // you might think that a for-loop would work, but under Visual Studio, it is not good enough. + static_assert((simd8x64::NUM_CHUNKS == 1) + || (simd8x64::NUM_CHUNKS == 2) + || (simd8x64::NUM_CHUNKS == 4), + "We support one, two or four chunks per 64-byte block."); + SIMDJSON_IF_CONSTEXPR(simd8x64::NUM_CHUNKS == 1) { + this->check_utf8_bytes(input.chunks[0], this->prev_input_block); + } +else SIMDJSON_IF_CONSTEXPR(simd8x64::NUM_CHUNKS == 2) { + this->check_utf8_bytes(input.chunks[0], this->prev_input_block); + this->check_utf8_bytes(input.chunks[1], input.chunks[0]); + } + else SIMDJSON_IF_CONSTEXPR(simd8x64::NUM_CHUNKS == 4) { + this->check_utf8_bytes(input.chunks[0], this->prev_input_block); + this->check_utf8_bytes(input.chunks[1], input.chunks[0]); + this->check_utf8_bytes(input.chunks[2], input.chunks[1]); + this->check_utf8_bytes(input.chunks[3], input.chunks[2]); + } + this->prev_incomplete = is_incomplete(input.chunks[simd8x64::NUM_CHUNKS - 1]); + this->prev_input_block = input.chunks[simd8x64::NUM_CHUNKS - 1]; + } + } + // do not forget to call check_eof! + simdjson_inline error_code errors() { + return this->error.any_bits_set_anywhere() ? error_code::UTF8_ERROR : error_code::SUCCESS; + } + + }; // struct utf8_checker + } // namespace utf8_validation + + } // unnamed namespace + } // namespace lsx +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE1_UTF8_LOOKUP4_ALGORITHM_H +/* end file generic/stage1/utf8_lookup4_algorithm.h for lsx */ +/* including generic/stage1/json_scanner.h for lsx: #include */ +/* begin file generic/stage1/json_scanner.h for lsx */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE1_JSON_SCANNER_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_JSON_SCANNER_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lsx { + namespace { + namespace stage1 { + + /** + * A block of scanned json, with information on operators and scalars. + * + * We seek to identify pseudo-structural characters. Anything that is inside + * a string must be omitted (hence & ~_string.string_tail()). + * Otherwise, pseudo-structural characters come in two forms. + * 1. We have the structural characters ([,],{,},:, comma). The + * term 'structural character' is from the JSON RFC. + * 2. We have the 'scalar pseudo-structural characters'. + * Scalars are quotes, and any character except structural characters and white space. + * + * To identify the scalar pseudo-structural characters, we must look at what comes + * before them: it must be a space, a quote or a structural characters. + * Starting with simdjson v0.3, we identify them by + * negation: we identify everything that is followed by a non-quote scalar, + * and we negate that. Whatever remains must be a 'scalar pseudo-structural character'. + */ + struct json_block { + public: + // We spell out the constructors in the hope of resolving inlining issues with Visual Studio 2017 + simdjson_inline json_block(json_string_block&& string, json_character_block characters, uint64_t follows_potential_nonquote_scalar) : + _string(std::move(string)), _characters(characters), _follows_potential_nonquote_scalar(follows_potential_nonquote_scalar) {} + simdjson_inline json_block(json_string_block string, json_character_block characters, uint64_t follows_potential_nonquote_scalar) : + _string(string), _characters(characters), _follows_potential_nonquote_scalar(follows_potential_nonquote_scalar) {} + + /** + * The start of structurals. + * In simdjson prior to v0.3, these were called the pseudo-structural characters. + **/ + simdjson_inline uint64_t structural_start() const noexcept { return potential_structural_start() & ~_string.string_tail(); } + /** All JSON whitespace (i.e. not in a string) */ + simdjson_inline uint64_t whitespace() const noexcept { return non_quote_outside_string(_characters.whitespace()); } + + // Helpers + + /** Whether the given characters are inside a string (only works on non-quotes) */ + simdjson_inline uint64_t non_quote_inside_string(uint64_t mask) const noexcept { return _string.non_quote_inside_string(mask); } + /** Whether the given characters are outside a string (only works on non-quotes) */ + simdjson_inline uint64_t non_quote_outside_string(uint64_t mask) const noexcept { return _string.non_quote_outside_string(mask); } + + // string and escape characters + json_string_block _string; + // whitespace, structural characters ('operators'), scalars + json_character_block _characters; + // whether the previous character was a scalar + uint64_t _follows_potential_nonquote_scalar; + private: + // Potential structurals (i.e. disregarding strings) + + /** + * structural elements ([,],{,},:, comma) plus scalar starts like 123, true and "abc". + * They may reside inside a string. + **/ + simdjson_inline uint64_t potential_structural_start() const noexcept { return _characters.op() | potential_scalar_start(); } + /** + * The start of non-operator runs, like 123, true and "abc". + * It main reside inside a string. + **/ + simdjson_inline uint64_t potential_scalar_start() const noexcept { + // The term "scalar" refers to anything except structural characters and white space + // (so letters, numbers, quotes). + // Whenever it is preceded by something that is not a structural element ({,},[,],:, ") nor a white-space + // then we know that it is irrelevant structurally. + return _characters.scalar() & ~follows_potential_scalar(); + } + /** + * Whether the given character is immediately after a non-operator like 123, true. + * The characters following a quote are not included. + */ + simdjson_inline uint64_t follows_potential_scalar() const noexcept { + // _follows_potential_nonquote_scalar: is defined as marking any character that follows a character + // that is not a structural element ({,},[,],:, comma) nor a quote (") and that is not a + // white space. + // It is understood that within quoted region, anything at all could be marked (irrelevant). + return _follows_potential_nonquote_scalar; + } + }; + + /** + * Scans JSON for important bits: structural characters or 'operators', strings, and scalars. + * + * The scanner starts by calculating two distinct things: + * - string characters (taking \" into account) + * - structural characters or 'operators' ([]{},:, comma) + * and scalars (runs of non-operators like 123, true and "abc") + * + * To minimize data dependency (a key component of the scanner's speed), it finds these in parallel: + * in particular, the operator/scalar bit will find plenty of things that are actually part of + * strings. When we're done, json_block will fuse the two together by masking out tokens that are + * part of a string. + */ + class json_scanner { + public: + json_scanner() = default; + simdjson_inline json_block next(const simd::simd8x64& in); + // Returns either UNCLOSED_STRING or SUCCESS + simdjson_inline error_code finish(); + + private: + // Whether the last character of the previous iteration is part of a scalar token + // (anything except whitespace or a structural character/'operator'). + uint64_t prev_scalar = 0ULL; + json_string_scanner string_scanner{}; + }; + + + // + // Check if the current character immediately follows a matching character. + // + // For example, this checks for quotes with backslashes in front of them: + // + // const uint64_t backslashed_quote = in.eq('"') & immediately_follows(in.eq('\'), prev_backslash); + // + simdjson_inline uint64_t follows(const uint64_t match, uint64_t& overflow) { + const uint64_t result = match << 1 | overflow; + overflow = match >> 63; + return result; + } + + simdjson_inline json_block json_scanner::next(const simd::simd8x64& in) { + json_string_block strings = string_scanner.next(in); + // identifies the white-space and the structural characters + json_character_block characters = json_character_block::classify(in); + // The term "scalar" refers to anything except structural characters and white space + // (so letters, numbers, quotes). + // We want follows_scalar to mark anything that follows a non-quote scalar (so letters and numbers). + // + // A terminal quote should either be followed by a structural character (comma, brace, bracket, colon) + // or nothing. However, we still want ' "a string"true ' to mark the 't' of 'true' as a potential + // pseudo-structural character just like we would if we had ' "a string" true '; otherwise we + // may need to add an extra check when parsing strings. + // + // Performance: there are many ways to skin this cat. + const uint64_t nonquote_scalar = characters.scalar() & ~strings.quote(); + uint64_t follows_nonquote_scalar = follows(nonquote_scalar, prev_scalar); + // We are returning a function-local object so either we get a move constructor + // or we get copy elision. + return json_block( + strings,// strings is a function-local object so either it moves or the copy is elided. + characters, + follows_nonquote_scalar + ); + } + + simdjson_inline error_code json_scanner::finish() { + return string_scanner.finish(); + } + + } // namespace stage1 + } // unnamed namespace + } // namespace lsx +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE1_JSON_SCANNER_H +/* end file generic/stage1/json_scanner.h for lsx */ + +// All other declarations +/* including generic/stage1/find_next_document_index.h for lsx: #include */ +/* begin file generic/stage1/find_next_document_index.h for lsx */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE1_FIND_NEXT_DOCUMENT_INDEX_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_FIND_NEXT_DOCUMENT_INDEX_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lsx { + namespace { + namespace stage1 { + + /** + * This algorithm is used to quickly identify the last structural position that + * makes up a complete document. + * + * It does this by going backwards and finding the last *document boundary* (a + * place where one value follows another without a comma between them). If the + * last document (the characters after the boundary) has an equal number of + * start and end brackets, it is considered complete. + * + * Simply put, we iterate over the structural characters, starting from + * the end. We consider that we found the end of a JSON document when the + * first element of the pair is NOT one of these characters: '{' '[' ':' ',' + * and when the second element is NOT one of these characters: '}' ']' ':' ','. + * + * This simple comparison works most of the time, but it does not cover cases + * where the batch's structural indexes contain a perfect amount of documents. + * In such a case, we do not have access to the structural index which follows + * the last document, therefore, we do not have access to the second element in + * the pair, and that means we cannot identify the last document. To fix this + * issue, we keep a count of the open and closed curly/square braces we found + * while searching for the pair. When we find a pair AND the count of open and + * closed curly/square braces is the same, we know that we just passed a + * complete document, therefore the last json buffer location is the end of the + * batch. + */ + simdjson_inline uint32_t find_next_document_index(dom_parser_implementation& parser) { + // Variant: do not count separately, just figure out depth + if (parser.n_structural_indexes == 0) { return 0; } + auto arr_cnt = 0; + auto obj_cnt = 0; + for (auto i = parser.n_structural_indexes - 1; i > 0; i--) { + auto idxb = parser.structural_indexes[i]; + switch (parser.buf[idxb]) { + case ':': + case ',': + continue; + case '}': + obj_cnt--; + continue; + case ']': + arr_cnt--; + continue; + case '{': + obj_cnt++; + break; + case '[': + arr_cnt++; + break; + } + auto idxa = parser.structural_indexes[i - 1]; + switch (parser.buf[idxa]) { + case '{': + case '[': + case ':': + case ',': + continue; + } + // Last document is complete, so the next document will appear after! + if (!arr_cnt && !obj_cnt) { + return parser.n_structural_indexes; + } + // Last document is incomplete; mark the document at i + 1 as the next one + return i; + } + // If we made it to the end, we want to finish counting to see if we have a full document. + switch (parser.buf[parser.structural_indexes[0]]) { + case '}': + obj_cnt--; + break; + case ']': + arr_cnt--; + break; + case '{': + obj_cnt++; + break; + case '[': + arr_cnt++; + break; + } + if (!arr_cnt && !obj_cnt) { + // We have a complete document. + return parser.n_structural_indexes; + } + return 0; + } + + } // namespace stage1 + } // unnamed namespace + } // namespace lsx +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE1_FIND_NEXT_DOCUMENT_INDEX_H +/* end file generic/stage1/find_next_document_index.h for lsx */ +/* including generic/stage1/json_minifier.h for lsx: #include */ +/* begin file generic/stage1/json_minifier.h for lsx */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE1_JSON_MINIFIER_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_JSON_MINIFIER_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +// This file contains the common code every implementation uses in stage1 +// It is intended to be included multiple times and compiled multiple times +// We assume the file in which it is included already includes +// "simdjson/stage1.h" (this simplifies amalgation) + +namespace simdjson { + namespace lsx { + namespace { + namespace stage1 { + + class json_minifier { + public: + template + static error_code minify(const uint8_t* buf, size_t len, uint8_t* dst, size_t& dst_len) noexcept; + + private: + simdjson_inline json_minifier(uint8_t* _dst) + : dst{ _dst } + {} + template + simdjson_inline void step(const uint8_t* block_buf, buf_block_reader& reader) noexcept; + simdjson_inline void next(const simd::simd8x64& in, const json_block& block); + simdjson_inline error_code finish(uint8_t* dst_start, size_t& dst_len); + json_scanner scanner{}; + uint8_t* dst; + }; + + simdjson_inline void json_minifier::next(const simd::simd8x64& in, const json_block& block) { + uint64_t mask = block.whitespace(); + dst += in.compress(mask, dst); + } + + simdjson_inline error_code json_minifier::finish(uint8_t* dst_start, size_t& dst_len) { + error_code error = scanner.finish(); + if (error) { dst_len = 0; return error; } + dst_len = dst - dst_start; + return SUCCESS; + } + + template<> + simdjson_inline void json_minifier::step<128>(const uint8_t* block_buf, buf_block_reader<128>& reader) noexcept { + simd::simd8x64 in_1(block_buf); + simd::simd8x64 in_2(block_buf + 64); + json_block block_1 = scanner.next(in_1); + json_block block_2 = scanner.next(in_2); + this->next(in_1, block_1); + this->next(in_2, block_2); + reader.advance(); + } + + template<> + simdjson_inline void json_minifier::step<64>(const uint8_t* block_buf, buf_block_reader<64>& reader) noexcept { + simd::simd8x64 in_1(block_buf); + json_block block_1 = scanner.next(in_1); + this->next(block_buf, block_1); + reader.advance(); + } + + template + error_code json_minifier::minify(const uint8_t* buf, size_t len, uint8_t* dst, size_t& dst_len) noexcept { + buf_block_reader reader(buf, len); + json_minifier minifier(dst); + + // Index the first n-1 blocks + while (reader.has_full_block()) { + minifier.step(reader.full_block(), reader); + } + + // Index the last (remainder) block, padded with spaces + uint8_t block[STEP_SIZE]; + size_t remaining_bytes = reader.get_remainder(block); + if (remaining_bytes > 0) { + // We do not want to write directly to the output stream. Rather, we write + // to a local buffer (for safety). + uint8_t out_block[STEP_SIZE]; + uint8_t* const guarded_dst{ minifier.dst }; + minifier.dst = out_block; + minifier.step(block, reader); + size_t to_write = minifier.dst - out_block; + // In some cases, we could be enticed to consider the padded spaces + // as part of the string. This is fine as long as we do not write more + // than we consumed. + if (to_write > remaining_bytes) { to_write = remaining_bytes; } + memcpy(guarded_dst, out_block, to_write); + minifier.dst = guarded_dst + to_write; + } + return minifier.finish(dst, dst_len); + } + + } // namespace stage1 + } // unnamed namespace + } // namespace lsx +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE1_JSON_MINIFIER_H +/* end file generic/stage1/json_minifier.h for lsx */ +/* including generic/stage1/json_structural_indexer.h for lsx: #include */ +/* begin file generic/stage1/json_structural_indexer.h for lsx */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRUCTURAL_INDEXER_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRUCTURAL_INDEXER_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +// This file contains the common code every implementation uses in stage1 +// It is intended to be included multiple times and compiled multiple times +// We assume the file in which it is included already includes +// "simdjson/stage1.h" (this simplifies amalgation) + +namespace simdjson { + namespace lsx { + namespace { + namespace stage1 { + + class bit_indexer { + public: + uint32_t* tail; + + simdjson_inline bit_indexer(uint32_t* index_buf) : tail(index_buf) {} + +#if SIMDJSON_PREFER_REVERSE_BITS + /** + * ARM lacks a fast trailing zero instruction, but it has a fast + * bit reversal instruction and a fast leading zero instruction. + * Thus it may be profitable to reverse the bits (once) and then + * to rely on a sequence of instructions that call the leading + * zero instruction. + * + * Performance notes: + * The chosen routine is not optimal in terms of data dependency + * since zero_leading_bit might require two instructions. However, + * it tends to minimize the total number of instructions which is + * beneficial. + */ + simdjson_inline void write_index(uint32_t idx, uint64_t& rev_bits, int i) { + int lz = leading_zeroes(rev_bits); + this->tail[i] = static_cast(idx) + lz; + rev_bits = zero_leading_bit(rev_bits, lz); + } +#else + /** + * Under recent x64 systems, we often have both a fast trailing zero + * instruction and a fast 'clear-lower-bit' instruction so the following + * algorithm can be competitive. + */ + + simdjson_inline void write_index(uint32_t idx, uint64_t& bits, int i) { + this->tail[i] = idx + trailing_zeroes(bits); + bits = clear_lowest_bit(bits); + } +#endif // SIMDJSON_PREFER_REVERSE_BITS + + template + simdjson_inline int write_indexes(uint32_t idx, uint64_t& bits) { + write_index(idx, bits, START); + SIMDJSON_IF_CONSTEXPR(N > 1) { + write_indexes<(N - 1 > 0 ? START + 1 : START), (N - 1 >= 0 ? N - 1 : 1)>(idx, bits); + } + return START + N; + } + + template + simdjson_inline int write_indexes_stepped(uint32_t idx, uint64_t& bits, int cnt) { + write_indexes(idx, bits); + SIMDJSON_IF_CONSTEXPR((START + STEP) < END) { + if (simdjson_unlikely((START + STEP) < cnt)) { + write_indexes_stepped<(START + STEP < END ? START + STEP : END), END, STEP>(idx, bits, cnt); + } + } + return ((END - START) % STEP) == 0 ? END : (END - START) - ((END - START) % STEP) + STEP; + } + + // flatten out values in 'bits' assuming that they are are to have values of idx + // plus their position in the bitvector, and store these indexes at + // base_ptr[base] incrementing base as we go + // will potentially store extra values beyond end of valid bits, so base_ptr + // needs to be large enough to handle this + // + // If the kernel sets SIMDJSON_GENERIC_JSON_STRUCTURAL_INDEXER_CUSTOM_BIT_INDEXER, then it + // will provide its own version of the code. +#ifdef SIMDJSON_GENERIC_JSON_STRUCTURAL_INDEXER_CUSTOM_BIT_INDEXER + simdjson_inline void write(uint32_t idx, uint64_t bits); +#else + simdjson_inline void write(uint32_t idx, uint64_t bits) { + // In some instances, the next branch is expensive because it is mispredicted. + // Unfortunately, in other cases, + // it helps tremendously. + if (bits == 0) + return; + + int cnt = static_cast(count_ones(bits)); + +#if SIMDJSON_PREFER_REVERSE_BITS + bits = reverse_bits(bits); +#endif +#ifdef SIMDJSON_STRUCTURAL_INDEXER_STEP + static constexpr const int STEP = SIMDJSON_STRUCTURAL_INDEXER_STEP; +#else + static constexpr const int STEP = 4; +#endif + static constexpr const int STEP_UNTIL = 24; + + write_indexes_stepped<0, STEP_UNTIL, STEP>(idx, bits, cnt); + SIMDJSON_IF_CONSTEXPR(STEP_UNTIL < 64) { + if (simdjson_unlikely(STEP_UNTIL < cnt)) { + for (int i = STEP_UNTIL; i < cnt; i++) { + write_index(idx, bits, i); + } + } + } + + this->tail += cnt; + } +#endif // SIMDJSON_GENERIC_JSON_STRUCTURAL_INDEXER_CUSTOM_BIT_INDEXER + + }; + + class json_structural_indexer { + public: + /** + * Find the important bits of JSON in a 128-byte chunk, and add them to structural_indexes. + * + * @param partial Setting the partial parameter to true allows the find_structural_bits to + * tolerate unclosed strings. The caller should still ensure that the input is valid UTF-8. If + * you are processing substrings, you may want to call on a function like trimmed_length_safe_utf8. + */ + template + static error_code index(const uint8_t* buf, size_t len, dom_parser_implementation& parser, stage1_mode partial) noexcept; + + private: + simdjson_inline json_structural_indexer(uint32_t* structural_indexes); + template + simdjson_inline void step(const uint8_t* block, buf_block_reader& reader) noexcept; + simdjson_inline void next(const simd::simd8x64& in, const json_block& block, size_t idx); + simdjson_inline error_code finish(dom_parser_implementation& parser, size_t idx, size_t len, stage1_mode partial); + + json_scanner scanner{}; + utf8_checker checker{}; + bit_indexer indexer; + uint64_t prev_structurals = 0; + uint64_t unescaped_chars_error = 0; + }; + + simdjson_inline json_structural_indexer::json_structural_indexer(uint32_t* structural_indexes) : indexer{ structural_indexes } {} + + // Skip the last character if it is partial + simdjson_inline size_t trim_partial_utf8(const uint8_t* buf, size_t len) { + if (simdjson_unlikely(len < 3)) { + switch (len) { + case 2: + if (buf[len - 1] >= 0xc0) { return len - 1; } // 2-, 3- and 4-byte characters with only 1 byte left + if (buf[len - 2] >= 0xe0) { return len - 2; } // 3- and 4-byte characters with only 2 bytes left + return len; + case 1: + if (buf[len - 1] >= 0xc0) { return len - 1; } // 2-, 3- and 4-byte characters with only 1 byte left + return len; + case 0: + return len; + } + } + if (buf[len - 1] >= 0xc0) { return len - 1; } // 2-, 3- and 4-byte characters with only 1 byte left + if (buf[len - 2] >= 0xe0) { return len - 2; } // 3- and 4-byte characters with only 1 byte left + if (buf[len - 3] >= 0xf0) { return len - 3; } // 4-byte characters with only 3 bytes left + return len; + } + + // + // PERF NOTES: + // We pipe 2 inputs through these stages: + // 1. Load JSON into registers. This takes a long time and is highly parallelizable, so we load + // 2 inputs' worth at once so that by the time step 2 is looking for them input, it's available. + // 2. Scan the JSON for critical data: strings, scalars and operators. This is the critical path. + // The output of step 1 depends entirely on this information. These functions don't quite use + // up enough CPU: the second half of the functions is highly serial, only using 1 execution core + // at a time. The second input's scans has some dependency on the first ones finishing it, but + // they can make a lot of progress before they need that information. + // 3. Step 1 does not use enough capacity, so we run some extra stuff while we're waiting for that + // to finish: utf-8 checks and generating the output from the last iteration. + // + // The reason we run 2 inputs at a time, is steps 2 and 3 are *still* not enough to soak up all + // available capacity with just one input. Running 2 at a time seems to give the CPU a good enough + // workout. + // + template + error_code json_structural_indexer::index(const uint8_t* buf, size_t len, dom_parser_implementation& parser, stage1_mode partial) noexcept { + if (simdjson_unlikely(len > parser.capacity())) { return CAPACITY; } + // We guard the rest of the code so that we can assume that len > 0 throughout. + if (len == 0) { return EMPTY; } + if (is_streaming(partial)) { + len = trim_partial_utf8(buf, len); + // If you end up with an empty window after trimming + // the partial UTF-8 bytes, then chances are good that you + // have an UTF-8 formatting error. + if (len == 0) { return UTF8_ERROR; } + } + buf_block_reader reader(buf, len); + json_structural_indexer indexer(parser.structural_indexes.get()); + + // Read all but the last block + while (reader.has_full_block()) { + indexer.step(reader.full_block(), reader); + } + // Take care of the last block (will always be there unless file is empty which is + // not supposed to happen.) + uint8_t block[STEP_SIZE]; + if (simdjson_unlikely(reader.get_remainder(block) == 0)) { return UNEXPECTED_ERROR; } + indexer.step(block, reader); + return indexer.finish(parser, reader.block_index(), len, partial); + } + + template<> + simdjson_inline void json_structural_indexer::step<128>(const uint8_t* block, buf_block_reader<128>& reader) noexcept { + simd::simd8x64 in_1(block); + simd::simd8x64 in_2(block + 64); + json_block block_1 = scanner.next(in_1); + json_block block_2 = scanner.next(in_2); + this->next(in_1, block_1, reader.block_index()); + this->next(in_2, block_2, reader.block_index() + 64); + reader.advance(); + } + + template<> + simdjson_inline void json_structural_indexer::step<64>(const uint8_t* block, buf_block_reader<64>& reader) noexcept { + simd::simd8x64 in_1(block); + json_block block_1 = scanner.next(in_1); + this->next(in_1, block_1, reader.block_index()); + reader.advance(); + } + + simdjson_inline void json_structural_indexer::next(const simd::simd8x64& in, const json_block& block, size_t idx) { + uint64_t unescaped = in.lteq(0x1F); +#if SIMDJSON_UTF8VALIDATION + checker.check_next_input(in); +#endif + indexer.write(uint32_t(idx - 64), prev_structurals); // Output *last* iteration's structurals to the parser + prev_structurals = block.structural_start(); + unescaped_chars_error |= block.non_quote_inside_string(unescaped); + } + + simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementation& parser, size_t idx, size_t len, stage1_mode partial) { + // Write out the final iteration's structurals + indexer.write(uint32_t(idx - 64), prev_structurals); + error_code error = scanner.finish(); + // We deliberately break down the next expression so that it is + // human readable. + const bool should_we_exit = is_streaming(partial) ? + ((error != SUCCESS) && (error != UNCLOSED_STRING)) // when partial we tolerate UNCLOSED_STRING + : (error != SUCCESS); // if partial is false, we must have SUCCESS + const bool have_unclosed_string = (error == UNCLOSED_STRING); + if (simdjson_unlikely(should_we_exit)) { return error; } + + if (unescaped_chars_error) { + return UNESCAPED_CHARS; + } + parser.n_structural_indexes = uint32_t(indexer.tail - parser.structural_indexes.get()); + /*** + * The On-Demand API requires special padding. + * + * This is related to https://github.com/simdjson/simdjson/issues/906 + * Basically, we want to make sure that if the parsing continues beyond the last (valid) + * structural character, it quickly stops. + * Only three structural characters can be repeated without triggering an error in JSON: [,] and }. + * We repeat the padding character (at 'len'). We don't know what it is, but if the parsing + * continues, then it must be [,] or }. + * Suppose it is ] or }. We backtrack to the first character, what could it be that would + * not trigger an error? It could be ] or } but no, because you can't start a document that way. + * It can't be a comma, a colon or any simple value. So the only way we could continue is + * if the repeated character is [. But if so, the document must start with [. But if the document + * starts with [, it should end with ]. If we enforce that rule, then we would get + * ][[ which is invalid. + * + * This is illustrated with the test array_iterate_unclosed_error() on the following input: + * R"({ "a": [,,)" + **/ + parser.structural_indexes[parser.n_structural_indexes] = uint32_t(len); // used later in partial == stage1_mode::streaming_final + parser.structural_indexes[parser.n_structural_indexes + 1] = uint32_t(len); + parser.structural_indexes[parser.n_structural_indexes + 2] = 0; + parser.next_structural_index = 0; + // a valid JSON file cannot have zero structural indexes - we should have found something + if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { + return EMPTY; + } + if (simdjson_unlikely(parser.structural_indexes[parser.n_structural_indexes - 1] > len)) { + return UNEXPECTED_ERROR; + } + if (partial == stage1_mode::streaming_partial) { + // If we have an unclosed string, then the last structural + // will be the quote and we want to make sure to omit it. + if (have_unclosed_string) { + parser.n_structural_indexes--; + // a valid JSON file cannot have zero structural indexes - we should have found something + if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return CAPACITY; } + } + // We truncate the input to the end of the last complete document (or zero). + auto new_structural_indexes = find_next_document_index(parser); + if (new_structural_indexes == 0 && parser.n_structural_indexes > 0) { + if (parser.structural_indexes[0] == 0) { + // If the buffer is partial and we started at index 0 but the document is + // incomplete, it's too big to parse. + return CAPACITY; + } + else { + // It is possible that the document could be parsed, we just had a lot + // of white space. + parser.n_structural_indexes = 0; + return EMPTY; + } + } + + parser.n_structural_indexes = new_structural_indexes; + } + else if (partial == stage1_mode::streaming_final) { + if (have_unclosed_string) { parser.n_structural_indexes--; } + // We truncate the input to the end of the last complete document (or zero). + // Because partial == stage1_mode::streaming_final, it means that we may + // silently ignore trailing garbage. Though it sounds bad, we do it + // deliberately because many people who have streams of JSON documents + // will truncate them for processing. E.g., imagine that you are uncompressing + // the data from a size file or receiving it in chunks from the network. You + // may not know where exactly the last document will be. Meanwhile the + // document_stream instances allow people to know the JSON documents they are + // parsing (see the iterator.source() method). + parser.n_structural_indexes = find_next_document_index(parser); + // We store the initial n_structural_indexes so that the client can see + // whether we used truncation. If initial_n_structural_indexes == parser.n_structural_indexes, + // then this will query parser.structural_indexes[parser.n_structural_indexes] which is len, + // otherwise, it will copy some prior index. + parser.structural_indexes[parser.n_structural_indexes + 1] = parser.structural_indexes[parser.n_structural_indexes]; + // This next line is critical, do not change it unless you understand what you are + // doing. + parser.structural_indexes[parser.n_structural_indexes] = uint32_t(len); + if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { + // We tolerate an unclosed string at the very end of the stream. Indeed, users + // often load their data in bulk without being careful and they want us to ignore + // the trailing garbage. + return EMPTY; + } + } + checker.check_eof(); + return checker.errors(); + } + + } // namespace stage1 + } // unnamed namespace + } // namespace lsx +} // namespace simdjson + +// Clear CUSTOM_BIT_INDEXER so other implementations can set it if they need to. +#undef SIMDJSON_GENERIC_JSON_STRUCTURAL_INDEXER_CUSTOM_BIT_INDEXER + +#endif // SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRUCTURAL_INDEXER_H +/* end file generic/stage1/json_structural_indexer.h for lsx */ +/* including generic/stage1/utf8_validator.h for lsx: #include */ +/* begin file generic/stage1/utf8_validator.h for lsx */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE1_UTF8_VALIDATOR_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_UTF8_VALIDATOR_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lsx { + namespace { + namespace stage1 { + + /** + * Validates that the string is actual UTF-8. + */ + template + bool generic_validate_utf8(const uint8_t* input, size_t length) { + checker c{}; + buf_block_reader<64> reader(input, length); + while (reader.has_full_block()) { + simd::simd8x64 in(reader.full_block()); + c.check_next_input(in); + reader.advance(); + } + uint8_t block[64]{}; + reader.get_remainder(block); + simd::simd8x64 in(block); + c.check_next_input(in); + reader.advance(); + c.check_eof(); + return c.errors() == error_code::SUCCESS; + } + + bool generic_validate_utf8(const char* input, size_t length) { + return generic_validate_utf8(reinterpret_cast(input), length); + } + + } // namespace stage1 + } // unnamed namespace + } // namespace lsx +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE1_UTF8_VALIDATOR_H +/* end file generic/stage1/utf8_validator.h for lsx */ +/* end file generic/stage1/amalgamated.h for lsx */ +/* including generic/stage2/amalgamated.h for lsx: #include */ +/* begin file generic/stage2/amalgamated.h for lsx */ +// Stuff other things depend on +/* including generic/stage2/base.h for lsx: #include */ +/* begin file generic/stage2/base.h for lsx */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE2_BASE_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_BASE_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lsx { + namespace { + namespace stage2 { + + class json_iterator; + class structural_iterator; + struct tape_builder; + struct tape_writer; + + } // namespace stage2 + } // unnamed namespace + } // namespace lsx +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE2_BASE_H +/* end file generic/stage2/base.h for lsx */ +/* including generic/stage2/tape_writer.h for lsx: #include */ +/* begin file generic/stage2/tape_writer.h for lsx */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE2_TAPE_WRITER_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_TAPE_WRITER_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +#include + +namespace simdjson { + namespace lsx { + namespace { + namespace stage2 { + + struct tape_writer { + /** The next place to write to tape */ + uint64_t* next_tape_loc; + + /** Write a signed 64-bit value to tape. */ + simdjson_inline void append_s64(int64_t value) noexcept; + + /** Write an unsigned 64-bit value to tape. */ + simdjson_inline void append_u64(uint64_t value) noexcept; + + /** Write a double value to tape. */ + simdjson_inline void append_double(double value) noexcept; + + /** + * Append a tape entry (an 8-bit type,and 56 bits worth of value). + */ + simdjson_inline void append(uint64_t val, internal::tape_type t) noexcept; + + /** + * Skip the current tape entry without writing. + * + * Used to skip the start of the container, since we'll come back later to fill it in when the + * container ends. + */ + simdjson_inline void skip() noexcept; + + /** + * Skip the number of tape entries necessary to write a large u64 or i64. + */ + simdjson_inline void skip_large_integer() noexcept; + + /** + * Skip the number of tape entries necessary to write a double. + */ + simdjson_inline void skip_double() noexcept; + + /** + * Write a value to a known location on tape. + * + * Used to go back and write out the start of a container after the container ends. + */ + simdjson_inline static void write(uint64_t& tape_loc, uint64_t val, internal::tape_type t) noexcept; + + private: + /** + * Append both the tape entry, and a supplementary value following it. Used for types that need + * all 64 bits, such as double and uint64_t. + */ + template + simdjson_inline void append2(uint64_t val, T val2, internal::tape_type t) noexcept; + }; // struct tape_writer + + simdjson_inline void tape_writer::append_s64(int64_t value) noexcept { + append2(0, value, internal::tape_type::INT64); + } + + simdjson_inline void tape_writer::append_u64(uint64_t value) noexcept { + append(0, internal::tape_type::UINT64); + *next_tape_loc = value; + next_tape_loc++; + } + + /** Write a double value to tape. */ + simdjson_inline void tape_writer::append_double(double value) noexcept { + append2(0, value, internal::tape_type::DOUBLE); + } + + simdjson_inline void tape_writer::skip() noexcept { + next_tape_loc++; + } + + simdjson_inline void tape_writer::skip_large_integer() noexcept { + next_tape_loc += 2; + } + + simdjson_inline void tape_writer::skip_double() noexcept { + next_tape_loc += 2; + } + + simdjson_inline void tape_writer::append(uint64_t val, internal::tape_type t) noexcept { + *next_tape_loc = val | ((uint64_t(char(t))) << 56); + next_tape_loc++; + } + + template + simdjson_inline void tape_writer::append2(uint64_t val, T val2, internal::tape_type t) noexcept { + append(val, t); + static_assert(sizeof(val2) == sizeof(*next_tape_loc), "Type is not 64 bits!"); + memcpy(next_tape_loc, &val2, sizeof(val2)); + next_tape_loc++; + } + + simdjson_inline void tape_writer::write(uint64_t& tape_loc, uint64_t val, internal::tape_type t) noexcept { + tape_loc = val | ((uint64_t(char(t))) << 56); + } + + } // namespace stage2 + } // unnamed namespace + } // namespace lsx +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE2_TAPE_WRITER_H +/* end file generic/stage2/tape_writer.h for lsx */ +/* including generic/stage2/logger.h for lsx: #include */ +/* begin file generic/stage2/logger.h for lsx */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE2_LOGGER_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_LOGGER_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +#include + + +// This is for an internal-only stage 2 specific logger. +// Set LOG_ENABLED = true to log what stage 2 is doing! +namespace simdjson { + namespace lsx { + namespace { + namespace logger { + + static constexpr const char* DASHES = "----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------"; + +#if SIMDJSON_VERBOSE_LOGGING + static constexpr const bool LOG_ENABLED = true; +#else + static constexpr const bool LOG_ENABLED = false; +#endif + static constexpr const int LOG_EVENT_LEN = 20; + static constexpr const int LOG_BUFFER_LEN = 30; + static constexpr const int LOG_SMALL_BUFFER_LEN = 10; + static constexpr const int LOG_INDEX_LEN = 5; + + static int log_depth; // Not threadsafe. Log only. + + // Helper to turn unprintable or newline characters into spaces + static simdjson_inline char printable_char(char c) { + if (c >= 0x20) { + return c; + } + else { + return ' '; + } + } + + // Print the header and set up log_start + static simdjson_inline void log_start() { + if (LOG_ENABLED) { + log_depth = 0; + printf("\n"); + printf("| %-*s | %-*s | %-*s | %-*s | Detail |\n", LOG_EVENT_LEN, "Event", LOG_BUFFER_LEN, "Buffer", LOG_SMALL_BUFFER_LEN, "Next", 5, "Next#"); + printf("|%.*s|%.*s|%.*s|%.*s|--------|\n", LOG_EVENT_LEN + 2, DASHES, LOG_BUFFER_LEN + 2, DASHES, LOG_SMALL_BUFFER_LEN + 2, DASHES, 5 + 2, DASHES); + } + } + + simdjson_unused static simdjson_inline void log_string(const char* message) { + if (LOG_ENABLED) { + printf("%s\n", message); + } + } + + // Logs a single line from the stage 2 DOM parser + template + static simdjson_inline void log_line(S& structurals, const char* title_prefix, const char* title, const char* detail) { + if (LOG_ENABLED) { + printf("| %*s%s%-*s ", log_depth * 2, "", title_prefix, LOG_EVENT_LEN - log_depth * 2 - int(strlen(title_prefix)), title); + auto current_index = structurals.at_beginning() ? nullptr : structurals.next_structural - 1; + auto next_index = structurals.next_structural; + auto current = current_index ? &structurals.buf[*current_index] : reinterpret_cast(" "); + auto next = &structurals.buf[*next_index]; + { + // Print the next N characters in the buffer. + printf("| "); + // Otherwise, print the characters starting from the buffer position. + // Print spaces for unprintable or newline characters. + for (int i = 0; i < LOG_BUFFER_LEN; i++) { + printf("%c", printable_char(current[i])); + } + printf(" "); + // Print the next N characters in the buffer. + printf("| "); + // Otherwise, print the characters starting from the buffer position. + // Print spaces for unprintable or newline characters. + for (int i = 0; i < LOG_SMALL_BUFFER_LEN; i++) { + printf("%c", printable_char(next[i])); + } + printf(" "); + } + if (current_index) { + printf("| %*u ", LOG_INDEX_LEN, *current_index); + } + else { + printf("| %-*s ", LOG_INDEX_LEN, ""); + } + // printf("| %*u ", LOG_INDEX_LEN, structurals.next_tape_index()); + printf("| %-s ", detail); + printf("|\n"); + } + } + + } // namespace logger + } // unnamed namespace + } // namespace lsx +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE2_LOGGER_H +/* end file generic/stage2/logger.h for lsx */ + +// All other declarations +/* including generic/stage2/json_iterator.h for lsx: #include */ +/* begin file generic/stage2/json_iterator.h for lsx */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE2_JSON_ITERATOR_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_JSON_ITERATOR_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lsx { + namespace { + namespace stage2 { + + class json_iterator { + public: + const uint8_t* const buf; + uint32_t* next_structural; + dom_parser_implementation& dom_parser; + uint32_t depth{ 0 }; + + /** + * Walk the JSON document. + * + * The visitor receives callbacks when values are encountered. All callbacks pass the iterator as + * the first parameter; some callbacks have other parameters as well: + * + * - visit_document_start() - at the beginning. + * - visit_document_end() - at the end (if things were successful). + * + * - visit_array_start() - at the start `[` of a non-empty array. + * - visit_array_end() - at the end `]` of a non-empty array. + * - visit_empty_array() - when an empty array is encountered. + * + * - visit_object_end() - at the start `]` of a non-empty object. + * - visit_object_start() - at the end `]` of a non-empty object. + * - visit_empty_object() - when an empty object is encountered. + * - visit_key(const uint8_t *key) - when a key in an object field is encountered. key is + * guaranteed to point at the first quote of the string (`"key"`). + * - visit_primitive(const uint8_t *value) - when a value is a string, number, boolean or null. + * - visit_root_primitive(iter, uint8_t *value) - when the top-level value is a string, number, boolean or null. + * + * - increment_count(iter) - each time a value is found in an array or object. + */ + template + simdjson_warn_unused simdjson_inline error_code walk_document(V& visitor) noexcept; + + /** + * Create an iterator capable of walking a JSON document. + * + * The document must have already passed through stage 1. + */ + simdjson_inline json_iterator(dom_parser_implementation& _dom_parser, size_t start_structural_index); + + /** + * Look at the next token. + * + * Tokens can be strings, numbers, booleans, null, or operators (`[{]},:`)). + * + * They may include invalid JSON as well (such as `1.2.3` or `ture`). + */ + simdjson_inline const uint8_t* peek() const noexcept; + /** + * Advance to the next token. + * + * Tokens can be strings, numbers, booleans, null, or operators (`[{]},:`)). + * + * They may include invalid JSON as well (such as `1.2.3` or `ture`). + */ + simdjson_inline const uint8_t* advance() noexcept; + /** + * Get the remaining length of the document, from the start of the current token. + */ + simdjson_inline size_t remaining_len() const noexcept; + /** + * Check if we are at the end of the document. + * + * If this is true, there are no more tokens. + */ + simdjson_inline bool at_eof() const noexcept; + /** + * Check if we are at the beginning of the document. + */ + simdjson_inline bool at_beginning() const noexcept; + simdjson_inline uint8_t last_structural() const noexcept; + + /** + * Log that a value has been found. + * + * Set LOG_ENABLED=true in logger.h to see logging. + */ + simdjson_inline void log_value(const char* type) const noexcept; + /** + * Log the start of a multipart value. + * + * Set LOG_ENABLED=true in logger.h to see logging. + */ + simdjson_inline void log_start_value(const char* type) const noexcept; + /** + * Log the end of a multipart value. + * + * Set LOG_ENABLED=true in logger.h to see logging. + */ + simdjson_inline void log_end_value(const char* type) const noexcept; + /** + * Log an error. + * + * Set LOG_ENABLED=true in logger.h to see logging. + */ + simdjson_inline void log_error(const char* error) const noexcept; + + template + simdjson_warn_unused simdjson_inline error_code visit_root_primitive(V& visitor, const uint8_t* value) noexcept; + template + simdjson_warn_unused simdjson_inline error_code visit_primitive(V& visitor, const uint8_t* value) noexcept; + }; + + template + simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V& visitor) noexcept { + logger::log_start(); + + // + // Start the document + // + if (at_eof()) { return EMPTY; } + log_start_value("document"); + SIMDJSON_TRY(visitor.visit_document_start(*this)); + + // + // Read first value + // + { + auto value = advance(); + + // Make sure the outer object or array is closed before continuing; otherwise, there are ways we + // could get into memory corruption. See https://github.com/simdjson/simdjson/issues/906 + if (!STREAMING) { + switch (*value) { + case '{': if (last_structural() != '}') { log_value("starting brace unmatched"); return TAPE_ERROR; }; break; + case '[': if (last_structural() != ']') { log_value("starting bracket unmatched"); return TAPE_ERROR; }; break; + } + } + + switch (*value) { + case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY(visitor.visit_empty_object(*this)); break; } goto object_begin; + case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY(visitor.visit_empty_array(*this)); break; } goto array_begin; + default: SIMDJSON_TRY(visitor.visit_root_primitive(*this, value)); break; + } + } + goto document_end; + + // + // Object parser states + // + object_begin: + log_start_value("object"); + depth++; + if (depth >= dom_parser.max_depth()) { log_error("Exceeded max depth!"); return DEPTH_ERROR; } + dom_parser.is_array[depth] = false; + SIMDJSON_TRY(visitor.visit_object_start(*this)); + + { + auto key = advance(); + if (*key != '"') { log_error("Object does not start with a key"); return TAPE_ERROR; } + SIMDJSON_TRY(visitor.increment_count(*this)); + SIMDJSON_TRY(visitor.visit_key(*this, key)); + } + + object_field: + if (simdjson_unlikely(*advance() != ':')) { log_error("Missing colon after key in object"); return TAPE_ERROR; } + { + auto value = advance(); + switch (*value) { + case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY(visitor.visit_empty_object(*this)); break; } goto object_begin; + case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY(visitor.visit_empty_array(*this)); break; } goto array_begin; + default: SIMDJSON_TRY(visitor.visit_primitive(*this, value)); break; + } + } + + object_continue: + switch (*advance()) { + case ',': + SIMDJSON_TRY(visitor.increment_count(*this)); + { + auto key = advance(); + if (simdjson_unlikely(*key != '"')) { log_error("Key string missing at beginning of field in object"); return TAPE_ERROR; } + SIMDJSON_TRY(visitor.visit_key(*this, key)); + } + goto object_field; + case '}': log_end_value("object"); SIMDJSON_TRY(visitor.visit_object_end(*this)); goto scope_end; + default: log_error("No comma between object fields"); return TAPE_ERROR; + } + + scope_end: + depth--; + if (depth == 0) { goto document_end; } + if (dom_parser.is_array[depth]) { goto array_continue; } + goto object_continue; + + // + // Array parser states + // + array_begin: + log_start_value("array"); + depth++; + if (depth >= dom_parser.max_depth()) { log_error("Exceeded max depth!"); return DEPTH_ERROR; } + dom_parser.is_array[depth] = true; + SIMDJSON_TRY(visitor.visit_array_start(*this)); + SIMDJSON_TRY(visitor.increment_count(*this)); + + array_value: + { + auto value = advance(); + switch (*value) { + case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY(visitor.visit_empty_object(*this)); break; } goto object_begin; + case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY(visitor.visit_empty_array(*this)); break; } goto array_begin; + default: SIMDJSON_TRY(visitor.visit_primitive(*this, value)); break; + } + } + + array_continue: + switch (*advance()) { + case ',': SIMDJSON_TRY(visitor.increment_count(*this)); goto array_value; + case ']': log_end_value("array"); SIMDJSON_TRY(visitor.visit_array_end(*this)); goto scope_end; + default: log_error("Missing comma between array values"); return TAPE_ERROR; + } + + document_end: + log_end_value("document"); + SIMDJSON_TRY(visitor.visit_document_end(*this)); + + dom_parser.next_structural_index = uint32_t(next_structural - &dom_parser.structural_indexes[0]); + + // If we didn't make it to the end, it's an error + if (!STREAMING && dom_parser.next_structural_index != dom_parser.n_structural_indexes) { + log_error("More than one JSON value at the root of the document, or extra characters at the end of the JSON!"); + return TAPE_ERROR; + } + + return SUCCESS; + + } // walk_document() + + simdjson_inline json_iterator::json_iterator(dom_parser_implementation& _dom_parser, size_t start_structural_index) + : buf{ _dom_parser.buf }, + next_structural{ &_dom_parser.structural_indexes[start_structural_index] }, + dom_parser{ _dom_parser } { + } + + simdjson_inline const uint8_t* json_iterator::peek() const noexcept { + return &buf[*(next_structural)]; + } + simdjson_inline const uint8_t* json_iterator::advance() noexcept { + return &buf[*(next_structural++)]; + } + simdjson_inline size_t json_iterator::remaining_len() const noexcept { + return dom_parser.len - *(next_structural - 1); + } + + simdjson_inline bool json_iterator::at_eof() const noexcept { + return next_structural == &dom_parser.structural_indexes[dom_parser.n_structural_indexes]; + } + simdjson_inline bool json_iterator::at_beginning() const noexcept { + return next_structural == dom_parser.structural_indexes.get(); + } + simdjson_inline uint8_t json_iterator::last_structural() const noexcept { + return buf[dom_parser.structural_indexes[dom_parser.n_structural_indexes - 1]]; + } + + simdjson_inline void json_iterator::log_value(const char* type) const noexcept { + logger::log_line(*this, "", type, ""); + } + + simdjson_inline void json_iterator::log_start_value(const char* type) const noexcept { + logger::log_line(*this, "+", type, ""); + if (logger::LOG_ENABLED) { logger::log_depth++; } + } + + simdjson_inline void json_iterator::log_end_value(const char* type) const noexcept { + if (logger::LOG_ENABLED) { logger::log_depth--; } + logger::log_line(*this, "-", type, ""); + } + + simdjson_inline void json_iterator::log_error(const char* error) const noexcept { + logger::log_line(*this, "", "ERROR", error); + } + + template + simdjson_warn_unused simdjson_inline error_code json_iterator::visit_root_primitive(V& visitor, const uint8_t* value) noexcept { + switch (*value) { + case '"': return visitor.visit_root_string(*this, value); + case 't': return visitor.visit_root_true_atom(*this, value); + case 'f': return visitor.visit_root_false_atom(*this, value); + case 'n': return visitor.visit_root_null_atom(*this, value); + case '-': + case '0': case '1': case '2': case '3': case '4': + case '5': case '6': case '7': case '8': case '9': + return visitor.visit_root_number(*this, value); + default: + log_error("Document starts with a non-value character"); + return TAPE_ERROR; + } + } + template + simdjson_warn_unused simdjson_inline error_code json_iterator::visit_primitive(V& visitor, const uint8_t* value) noexcept { + // Use the fact that most scalars are going to be either strings or numbers. + if (*value == '"') { + return visitor.visit_string(*this, value); + } + else if (((*value - '0') < 10) || (*value == '-')) { + return visitor.visit_number(*this, value); + } + // true, false, null are uncommon. + switch (*value) { + case 't': return visitor.visit_true_atom(*this, value); + case 'f': return visitor.visit_false_atom(*this, value); + case 'n': return visitor.visit_null_atom(*this, value); + default: + log_error("Non-value found when value was expected!"); + return TAPE_ERROR; + } + } + + } // namespace stage2 + } // unnamed namespace + } // namespace lsx +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE2_JSON_ITERATOR_H +/* end file generic/stage2/json_iterator.h for lsx */ +/* including generic/stage2/stringparsing.h for lsx: #include */ +/* begin file generic/stage2/stringparsing.h for lsx */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +// This file contains the common code every implementation uses +// It is intended to be included multiple times and compiled multiple times + +namespace simdjson { + namespace lsx { + namespace { + /// @private + namespace stringparsing { + + // begin copypasta + // These chars yield themselves: " \ / + // b -> backspace, f -> formfeed, n -> newline, r -> cr, t -> horizontal tab + // u not handled in this table as it's complex + static const uint8_t escape_map[256] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x0. + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0x22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x2f, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x4. + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x5c, 0, 0, 0, // 0x5. + 0, 0, 0x08, 0, 0, 0, 0x0c, 0, 0, 0, 0, 0, 0, 0, 0x0a, 0, // 0x6. + 0, 0, 0x0d, 0, 0x09, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x7. + + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + }; + + // handle a unicode codepoint + // write appropriate values into dest + // src will advance 6 bytes or 12 bytes + // dest will advance a variable amount (return via pointer) + // return true if the unicode codepoint was valid + // We work in little-endian then swap at write time + simdjson_warn_unused + simdjson_inline bool handle_unicode_codepoint(const uint8_t** src_ptr, + uint8_t** dst_ptr, bool allow_replacement) { + // Use the default Unicode Character 'REPLACEMENT CHARACTER' (U+FFFD) + constexpr uint32_t substitution_code_point = 0xfffd; + // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the + // conversion is not valid; we defer the check for this to inside the + // multilingual plane check. + uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2); + *src_ptr += 6; + + // If we found a high surrogate, we must + // check for low surrogate for characters + // outside the Basic + // Multilingual Plane. + if (code_point >= 0xd800 && code_point < 0xdc00) { + const uint8_t* src_data = *src_ptr; + /* Compiler optimizations convert this to a single 16-bit load and compare on most platforms */ + if (((src_data[0] << 8) | src_data[1]) != ((static_cast ('\\') << 8) | static_cast ('u'))) { + if (!allow_replacement) { return false; } + code_point = substitution_code_point; + } + else { + uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(src_data + 2); + + // We have already checked that the high surrogate is valid and + // (code_point - 0xd800) < 1024. + // + // Check that code_point_2 is in the range 0xdc00..0xdfff + // and that code_point_2 was parsed from valid hex. + uint32_t low_bit = code_point_2 - 0xdc00; + if (low_bit >> 10) { + if (!allow_replacement) { return false; } + code_point = substitution_code_point; + } + else { + code_point = (((code_point - 0xd800) << 10) | low_bit) + 0x10000; + *src_ptr += 6; + } + + } + } + else if (code_point >= 0xdc00 && code_point <= 0xdfff) { + // If we encounter a low surrogate (not preceded by a high surrogate) + // then we have an error. + if (!allow_replacement) { return false; } + code_point = substitution_code_point; + } + size_t offset = jsoncharutils::codepoint_to_utf8(code_point, *dst_ptr); + *dst_ptr += offset; + return offset > 0; + } + + + // handle a unicode codepoint using the wobbly convention + // https://simonsapin.github.io/wtf-8/ + // write appropriate values into dest + // src will advance 6 bytes or 12 bytes + // dest will advance a variable amount (return via pointer) + // return true if the unicode codepoint was valid + // We work in little-endian then swap at write time + simdjson_warn_unused + simdjson_inline bool handle_unicode_codepoint_wobbly(const uint8_t** src_ptr, + uint8_t** dst_ptr) { + // It is not ideal that this function is nearly identical to handle_unicode_codepoint. + // + // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the + // conversion is not valid; we defer the check for this to inside the + // multilingual plane check. + uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2); + *src_ptr += 6; + // If we found a high surrogate, we must + // check for low surrogate for characters + // outside the Basic + // Multilingual Plane. + if (code_point >= 0xd800 && code_point < 0xdc00) { + const uint8_t* src_data = *src_ptr; + /* Compiler optimizations convert this to a single 16-bit load and compare on most platforms */ + if (((src_data[0] << 8) | src_data[1]) == ((static_cast ('\\') << 8) | static_cast ('u'))) { + uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(src_data + 2); + uint32_t low_bit = code_point_2 - 0xdc00; + if ((low_bit >> 10) == 0) { + code_point = + (((code_point - 0xd800) << 10) | low_bit) + 0x10000; + *src_ptr += 6; + } + } + } + + size_t offset = jsoncharutils::codepoint_to_utf8(code_point, *dst_ptr); + *dst_ptr += offset; + return offset > 0; + } + + + /** + * Unescape a valid UTF-8 string from src to dst, stopping at a final unescaped quote. There + * must be an unescaped quote terminating the string. It returns the final output + * position as pointer. In case of error (e.g., the string has bad escaped codes), + * then null_ptr is returned. It is assumed that the output buffer is large + * enough. E.g., if src points at 'joe"', then dst needs to have four free bytes + + * SIMDJSON_PADDING bytes. + */ + simdjson_warn_unused simdjson_inline uint8_t* parse_string(const uint8_t* src, uint8_t* dst, bool allow_replacement) { + while (1) { + // Copy the next n bytes, and find the backslash and quote in them. + auto bs_quote = backslash_and_quote::copy_and_find(src, dst); + // If the next thing is the end quote, copy and return + if (bs_quote.has_quote_first()) { + // we encountered quotes first. Move dst to point to quotes and exit + return dst + bs_quote.quote_index(); + } + if (bs_quote.has_backslash()) { + /* find out where the backspace is */ + auto bs_dist = bs_quote.backslash_index(); + uint8_t escape_char = src[bs_dist + 1]; + /* we encountered backslash first. Handle backslash */ + if (escape_char == 'u') { + /* move src/dst up to the start; they will be further adjusted + within the unicode codepoint handling code. */ + src += bs_dist; + dst += bs_dist; + if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { + return nullptr; + } + } + else { + /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and + * write bs_dist+1 characters to output + * note this may reach beyond the part of the buffer we've actually + * seen. I think this is ok */ + uint8_t escape_result = escape_map[escape_char]; + if (escape_result == 0u) { + return nullptr; /* bogus escape value is an error */ + } + dst[bs_dist] = escape_result; + src += bs_dist + 2; + dst += bs_dist + 1; + } + } + else { + /* they are the same. Since they can't co-occur, it means we + * encountered neither. */ + src += backslash_and_quote::BYTES_PROCESSED; + dst += backslash_and_quote::BYTES_PROCESSED; + } + } + } + + simdjson_warn_unused simdjson_inline uint8_t* parse_wobbly_string(const uint8_t* src, uint8_t* dst) { + // It is not ideal that this function is nearly identical to parse_string. + while (1) { + // Copy the next n bytes, and find the backslash and quote in them. + auto bs_quote = backslash_and_quote::copy_and_find(src, dst); + // If the next thing is the end quote, copy and return + if (bs_quote.has_quote_first()) { + // we encountered quotes first. Move dst to point to quotes and exit + return dst + bs_quote.quote_index(); + } + if (bs_quote.has_backslash()) { + /* find out where the backspace is */ + auto bs_dist = bs_quote.backslash_index(); + uint8_t escape_char = src[bs_dist + 1]; + /* we encountered backslash first. Handle backslash */ + if (escape_char == 'u') { + /* move src/dst up to the start; they will be further adjusted + within the unicode codepoint handling code. */ + src += bs_dist; + dst += bs_dist; + if (!handle_unicode_codepoint_wobbly(&src, &dst)) { + return nullptr; + } + } + else { + /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and + * write bs_dist+1 characters to output + * note this may reach beyond the part of the buffer we've actually + * seen. I think this is ok */ + uint8_t escape_result = escape_map[escape_char]; + if (escape_result == 0u) { + return nullptr; /* bogus escape value is an error */ + } + dst[bs_dist] = escape_result; + src += bs_dist + 2; + dst += bs_dist + 1; + } + } + else { + /* they are the same. Since they can't co-occur, it means we + * encountered neither. */ + src += backslash_and_quote::BYTES_PROCESSED; + dst += backslash_and_quote::BYTES_PROCESSED; + } + } + } + + } // namespace stringparsing + } // unnamed namespace + } // namespace lsx +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H +/* end file generic/stage2/stringparsing.h for lsx */ +/* including generic/stage2/structural_iterator.h for lsx: #include */ +/* begin file generic/stage2/structural_iterator.h for lsx */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE2_STRUCTURAL_ITERATOR_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_STRUCTURAL_ITERATOR_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lsx { + namespace { + namespace stage2 { + + class structural_iterator { + public: + const uint8_t* const buf; + uint32_t* next_structural; + dom_parser_implementation& dom_parser; + + // Start a structural + simdjson_inline structural_iterator(dom_parser_implementation& _dom_parser, size_t start_structural_index) + : buf{ _dom_parser.buf }, + next_structural{ &_dom_parser.structural_indexes[start_structural_index] }, + dom_parser{ _dom_parser } { + } + // Get the buffer position of the current structural character + simdjson_inline const uint8_t* current() { + return &buf[*(next_structural - 1)]; + } + // Get the current structural character + simdjson_inline char current_char() { + return buf[*(next_structural - 1)]; + } + // Get the next structural character without advancing + simdjson_inline char peek_next_char() { + return buf[*next_structural]; + } + simdjson_inline const uint8_t* peek() { + return &buf[*next_structural]; + } + simdjson_inline const uint8_t* advance() { + return &buf[*(next_structural++)]; + } + simdjson_inline char advance_char() { + return buf[*(next_structural++)]; + } + simdjson_inline size_t remaining_len() { + return dom_parser.len - *(next_structural - 1); + } + + simdjson_inline bool at_end() { + return next_structural == &dom_parser.structural_indexes[dom_parser.n_structural_indexes]; + } + simdjson_inline bool at_beginning() { + return next_structural == dom_parser.structural_indexes.get(); + } + }; + + } // namespace stage2 + } // unnamed namespace + } // namespace lsx +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE2_STRUCTURAL_ITERATOR_H +/* end file generic/stage2/structural_iterator.h for lsx */ +/* including generic/stage2/tape_builder.h for lsx: #include */ +/* begin file generic/stage2/tape_builder.h for lsx */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE2_TAPE_BUILDER_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_TAPE_BUILDER_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + + +namespace simdjson { + namespace lsx { + namespace { + namespace stage2 { + + struct tape_builder { + template + simdjson_warn_unused static simdjson_inline error_code parse_document( + dom_parser_implementation& dom_parser, + dom::document& doc) noexcept; + + /** Called when a non-empty document starts. */ + simdjson_warn_unused simdjson_inline error_code visit_document_start(json_iterator& iter) noexcept; + /** Called when a non-empty document ends without error. */ + simdjson_warn_unused simdjson_inline error_code visit_document_end(json_iterator& iter) noexcept; + + /** Called when a non-empty array starts. */ + simdjson_warn_unused simdjson_inline error_code visit_array_start(json_iterator& iter) noexcept; + /** Called when a non-empty array ends. */ + simdjson_warn_unused simdjson_inline error_code visit_array_end(json_iterator& iter) noexcept; + /** Called when an empty array is found. */ + simdjson_warn_unused simdjson_inline error_code visit_empty_array(json_iterator& iter) noexcept; + + /** Called when a non-empty object starts. */ + simdjson_warn_unused simdjson_inline error_code visit_object_start(json_iterator& iter) noexcept; + /** + * Called when a key in a field is encountered. + * + * primitive, visit_object_start, visit_empty_object, visit_array_start, or visit_empty_array + * will be called after this with the field value. + */ + simdjson_warn_unused simdjson_inline error_code visit_key(json_iterator& iter, const uint8_t* key) noexcept; + /** Called when a non-empty object ends. */ + simdjson_warn_unused simdjson_inline error_code visit_object_end(json_iterator& iter) noexcept; + /** Called when an empty object is found. */ + simdjson_warn_unused simdjson_inline error_code visit_empty_object(json_iterator& iter) noexcept; + + /** + * Called when a string, number, boolean or null is found. + */ + simdjson_warn_unused simdjson_inline error_code visit_primitive(json_iterator& iter, const uint8_t* value) noexcept; + /** + * Called when a string, number, boolean or null is found at the top level of a document (i.e. + * when there is no array or object and the entire document is a single string, number, boolean or + * null. + * + * This is separate from primitive() because simdjson's normal primitive parsing routines assume + * there is at least one more token after the value, which is only true in an array or object. + */ + simdjson_warn_unused simdjson_inline error_code visit_root_primitive(json_iterator& iter, const uint8_t* value) noexcept; + + simdjson_warn_unused simdjson_inline error_code visit_string(json_iterator& iter, const uint8_t* value, bool key = false) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_number(json_iterator& iter, const uint8_t* value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_true_atom(json_iterator& iter, const uint8_t* value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_false_atom(json_iterator& iter, const uint8_t* value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_null_atom(json_iterator& iter, const uint8_t* value) noexcept; + + simdjson_warn_unused simdjson_inline error_code visit_root_string(json_iterator& iter, const uint8_t* value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_root_number(json_iterator& iter, const uint8_t* value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_root_true_atom(json_iterator& iter, const uint8_t* value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_root_false_atom(json_iterator& iter, const uint8_t* value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_root_null_atom(json_iterator& iter, const uint8_t* value) noexcept; + + /** Called each time a new field or element in an array or object is found. */ + simdjson_warn_unused simdjson_inline error_code increment_count(json_iterator& iter) noexcept; + + /** Next location to write to tape */ + tape_writer tape; + private: + /** Next write location in the string buf for stage 2 parsing */ + uint8_t* current_string_buf_loc; + + simdjson_inline tape_builder(dom::document& doc) noexcept; + + simdjson_inline uint32_t next_tape_index(json_iterator& iter) const noexcept; + simdjson_inline void start_container(json_iterator& iter) noexcept; + simdjson_warn_unused simdjson_inline error_code end_container(json_iterator& iter, internal::tape_type start, internal::tape_type end) noexcept; + simdjson_warn_unused simdjson_inline error_code empty_container(json_iterator& iter, internal::tape_type start, internal::tape_type end) noexcept; + simdjson_inline uint8_t* on_start_string(json_iterator& iter) noexcept; + simdjson_inline void on_end_string(uint8_t* dst) noexcept; + }; // struct tape_builder + + template + simdjson_warn_unused simdjson_inline error_code tape_builder::parse_document( + dom_parser_implementation& dom_parser, + dom::document& doc) noexcept { + dom_parser.doc = &doc; + json_iterator iter(dom_parser, STREAMING ? dom_parser.next_structural_index : 0); + tape_builder builder(doc); + return iter.walk_document(builder); + } + + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_primitive(json_iterator& iter, const uint8_t* value) noexcept { + return iter.visit_root_primitive(*this, value); + } + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_primitive(json_iterator& iter, const uint8_t* value) noexcept { + return iter.visit_primitive(*this, value); + } + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_empty_object(json_iterator& iter) noexcept { + return empty_container(iter, internal::tape_type::START_OBJECT, internal::tape_type::END_OBJECT); + } + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_empty_array(json_iterator& iter) noexcept { + return empty_container(iter, internal::tape_type::START_ARRAY, internal::tape_type::END_ARRAY); + } + + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_document_start(json_iterator& iter) noexcept { + start_container(iter); + return SUCCESS; + } + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_object_start(json_iterator& iter) noexcept { + start_container(iter); + return SUCCESS; + } + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_array_start(json_iterator& iter) noexcept { + start_container(iter); + return SUCCESS; + } + + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_object_end(json_iterator& iter) noexcept { + return end_container(iter, internal::tape_type::START_OBJECT, internal::tape_type::END_OBJECT); + } + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_array_end(json_iterator& iter) noexcept { + return end_container(iter, internal::tape_type::START_ARRAY, internal::tape_type::END_ARRAY); + } + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_document_end(json_iterator& iter) noexcept { + constexpr uint32_t start_tape_index = 0; + tape.append(start_tape_index, internal::tape_type::ROOT); + tape_writer::write(iter.dom_parser.doc->tape[start_tape_index], next_tape_index(iter), internal::tape_type::ROOT); + return SUCCESS; + } + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_key(json_iterator& iter, const uint8_t* key) noexcept { + return visit_string(iter, key, true); + } + + simdjson_warn_unused simdjson_inline error_code tape_builder::increment_count(json_iterator& iter) noexcept { + iter.dom_parser.open_containers[iter.depth].count++; // we have a key value pair in the object at parser.dom_parser.depth - 1 + return SUCCESS; + } + + simdjson_inline tape_builder::tape_builder(dom::document& doc) noexcept : tape{ doc.tape.get() }, current_string_buf_loc{ doc.string_buf.get() } {} + + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_string(json_iterator& iter, const uint8_t* value, bool key) noexcept { + iter.log_value(key ? "key" : "string"); + uint8_t* dst = on_start_string(iter); + dst = stringparsing::parse_string(value + 1, dst, false); // We do not allow replacement when the escape characters are invalid. + if (dst == nullptr) { + iter.log_error("Invalid escape in string"); + return STRING_ERROR; + } + on_end_string(dst); + return SUCCESS; + } + + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_string(json_iterator& iter, const uint8_t* value) noexcept { + return visit_string(iter, value); + } + + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_number(json_iterator& iter, const uint8_t* value) noexcept { + iter.log_value("number"); + return numberparsing::parse_number(value, tape); + } + + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_number(json_iterator& iter, const uint8_t* value) noexcept { + // + // We need to make a copy to make sure that the string is space terminated. + // This is not about padding the input, which should already padded up + // to len + SIMDJSON_PADDING. However, we have no control at this stage + // on how the padding was done. What if the input string was padded with nulls? + // It is quite common for an input string to have an extra null character (C string). + // We do not want to allow 9\0 (where \0 is the null character) inside a JSON + // document, but the string "9\0" by itself is fine. So we make a copy and + // pad the input with spaces when we know that there is just one input element. + // This copy is relatively expensive, but it will almost never be called in + // practice unless you are in the strange scenario where you have many JSON + // documents made of single atoms. + // + std::unique_ptrcopy(new (std::nothrow) uint8_t[iter.remaining_len() + SIMDJSON_PADDING]); + if (copy.get() == nullptr) { return MEMALLOC; } + std::memcpy(copy.get(), value, iter.remaining_len()); + std::memset(copy.get() + iter.remaining_len(), ' ', SIMDJSON_PADDING); + error_code error = visit_number(iter, copy.get()); + return error; + } + + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_true_atom(json_iterator& iter, const uint8_t* value) noexcept { + iter.log_value("true"); + if (!atomparsing::is_valid_true_atom(value)) { return T_ATOM_ERROR; } + tape.append(0, internal::tape_type::TRUE_VALUE); + return SUCCESS; + } + + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_true_atom(json_iterator& iter, const uint8_t* value) noexcept { + iter.log_value("true"); + if (!atomparsing::is_valid_true_atom(value, iter.remaining_len())) { return T_ATOM_ERROR; } + tape.append(0, internal::tape_type::TRUE_VALUE); + return SUCCESS; + } + + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_false_atom(json_iterator& iter, const uint8_t* value) noexcept { + iter.log_value("false"); + if (!atomparsing::is_valid_false_atom(value)) { return F_ATOM_ERROR; } + tape.append(0, internal::tape_type::FALSE_VALUE); + return SUCCESS; + } + + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_false_atom(json_iterator& iter, const uint8_t* value) noexcept { + iter.log_value("false"); + if (!atomparsing::is_valid_false_atom(value, iter.remaining_len())) { return F_ATOM_ERROR; } + tape.append(0, internal::tape_type::FALSE_VALUE); + return SUCCESS; + } + + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_null_atom(json_iterator& iter, const uint8_t* value) noexcept { + iter.log_value("null"); + if (!atomparsing::is_valid_null_atom(value)) { return N_ATOM_ERROR; } + tape.append(0, internal::tape_type::NULL_VALUE); + return SUCCESS; + } + + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_null_atom(json_iterator& iter, const uint8_t* value) noexcept { + iter.log_value("null"); + if (!atomparsing::is_valid_null_atom(value, iter.remaining_len())) { return N_ATOM_ERROR; } + tape.append(0, internal::tape_type::NULL_VALUE); + return SUCCESS; + } + + // private: + + simdjson_inline uint32_t tape_builder::next_tape_index(json_iterator& iter) const noexcept { + return uint32_t(tape.next_tape_loc - iter.dom_parser.doc->tape.get()); + } + + simdjson_warn_unused simdjson_inline error_code tape_builder::empty_container(json_iterator& iter, internal::tape_type start, internal::tape_type end) noexcept { + auto start_index = next_tape_index(iter); + tape.append(start_index + 2, start); + tape.append(start_index, end); + return SUCCESS; + } + + simdjson_inline void tape_builder::start_container(json_iterator& iter) noexcept { + iter.dom_parser.open_containers[iter.depth].tape_index = next_tape_index(iter); + iter.dom_parser.open_containers[iter.depth].count = 0; + tape.skip(); // We don't actually *write* the start element until the end. + } + + simdjson_warn_unused simdjson_inline error_code tape_builder::end_container(json_iterator& iter, internal::tape_type start, internal::tape_type end) noexcept { + // Write the ending tape element, pointing at the start location + const uint32_t start_tape_index = iter.dom_parser.open_containers[iter.depth].tape_index; + tape.append(start_tape_index, end); + // Write the start tape element, pointing at the end location (and including count) + // count can overflow if it exceeds 24 bits... so we saturate + // the convention being that a cnt of 0xffffff or more is undetermined in value (>= 0xffffff). + const uint32_t count = iter.dom_parser.open_containers[iter.depth].count; + const uint32_t cntsat = count > 0xFFFFFF ? 0xFFFFFF : count; + tape_writer::write(iter.dom_parser.doc->tape[start_tape_index], next_tape_index(iter) | (uint64_t(cntsat) << 32), start); + return SUCCESS; + } + + simdjson_inline uint8_t* tape_builder::on_start_string(json_iterator& iter) noexcept { + // we advance the point, accounting for the fact that we have a NULL termination + tape.append(current_string_buf_loc - iter.dom_parser.doc->string_buf.get(), internal::tape_type::STRING); + return current_string_buf_loc + sizeof(uint32_t); + } + + simdjson_inline void tape_builder::on_end_string(uint8_t* dst) noexcept { + uint32_t str_length = uint32_t(dst - (current_string_buf_loc + sizeof(uint32_t))); + // TODO check for overflow in case someone has a crazy string (>=4GB?) + // But only add the overflow check when the document itself exceeds 4GB + // Currently unneeded because we refuse to parse docs larger or equal to 4GB. + memcpy(current_string_buf_loc, &str_length, sizeof(uint32_t)); + // NULL termination is still handy if you expect all your strings to + // be NULL terminated? It comes at a small cost + *dst = 0; + current_string_buf_loc = dst + 1; + } + + } // namespace stage2 + } // unnamed namespace + } // namespace lsx +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE2_TAPE_BUILDER_H +/* end file generic/stage2/tape_builder.h for lsx */ +/* end file generic/stage2/amalgamated.h for lsx */ + +// +// Stage 1 +// +namespace simdjson { + namespace lsx { + + simdjson_warn_unused error_code implementation::create_dom_parser_implementation( + size_t capacity, + size_t max_depth, + std::unique_ptr& dst + ) const noexcept { + dst.reset(new (std::nothrow) dom_parser_implementation()); + if (!dst) { return MEMALLOC; } + if (auto err = dst->set_capacity(capacity)) + return err; + if (auto err = dst->set_max_depth(max_depth)) + return err; + return SUCCESS; + } + + namespace { + + using namespace simd; + + simdjson_inline json_character_block json_character_block::classify(const simd::simd8x64& in) { + // Inspired by haswell. + // LSX use low 5 bits as index. For the 6 operators (:,[]{}), the unique-5bits is [6:2]. + // The ASCII white-space and operators have these values: (char, hex, unique-5bits) + // (' ', 20, 00000) ('\t', 09, 01001) ('\n', 0A, 01010) ('\r', 0D, 01101) + // (',', 2C, 01011) (':', 3A, 01110) ('[', 5B, 10110) ('{', 7B, 11110) (']', 5D, 10111) ('}', 7D, 11111) + const simd8 ws_table = simd8::repeat_16( + ' ', 0, 0, 0, 0, 0, 0, 0, 0, '\t', '\n', 0, 0, '\r', 0, 0 + ); + const simd8 op_table_lo = simd8::repeat_16( + 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ',', 0, 0, ':', 0 + ); + const simd8 op_table_hi = simd8::repeat_16( + 0, 0, 0, 0, 0, 0, '[', ']', 0, 0, 0, 0, 0, 0, '{', '}' + ); + uint64_t ws = in.eq({ + in.chunks[0].lookup_16(ws_table), + in.chunks[1].lookup_16(ws_table), + in.chunks[2].lookup_16(ws_table), + in.chunks[3].lookup_16(ws_table) + }); + uint64_t op = in.eq({ + __lsx_vshuf_b(op_table_hi, op_table_lo, in.chunks[0].shr<2>()), + __lsx_vshuf_b(op_table_hi, op_table_lo, in.chunks[1].shr<2>()), + __lsx_vshuf_b(op_table_hi, op_table_lo, in.chunks[2].shr<2>()), + __lsx_vshuf_b(op_table_hi, op_table_lo, in.chunks[3].shr<2>()) + }); + + return { ws, op }; + } + + simdjson_inline bool is_ascii(const simd8x64& input) { + return input.reduce_or().is_ascii(); + } + + simdjson_inline simd8 must_be_2_3_continuation(const simd8 prev2, const simd8 prev3) { + simd8 is_third_byte = prev2.saturating_sub(0xe0u - 0x80); // Only 111_____ will be >= 0x80 + simd8 is_fourth_byte = prev3.saturating_sub(0xf0u - 0x80); // Only 1111____ will be >= 0x80 + return is_third_byte | is_fourth_byte; + } + + } // unnamed namespace + } // namespace lsx +} // namespace simdjson + +// +// Stage 2 +// + +// +// Implementation-specific overrides +// +namespace simdjson { + namespace lsx { + + simdjson_warn_unused error_code implementation::minify(const uint8_t* buf, size_t len, uint8_t* dst, size_t& dst_len) const noexcept { + return lsx::stage1::json_minifier::minify<64>(buf, len, dst, dst_len); + } + + simdjson_warn_unused error_code dom_parser_implementation::stage1(const uint8_t* _buf, size_t _len, stage1_mode streaming) noexcept { + this->buf = _buf; + this->len = _len; + return lsx::stage1::json_structural_indexer::index<64>(buf, len, *this, streaming); + } + + simdjson_warn_unused bool implementation::validate_utf8(const char* buf, size_t len) const noexcept { + return lsx::stage1::generic_validate_utf8(buf, len); + } + + simdjson_warn_unused error_code dom_parser_implementation::stage2(dom::document& _doc) noexcept { + return stage2::tape_builder::parse_document(*this, _doc); + } + + simdjson_warn_unused error_code dom_parser_implementation::stage2_next(dom::document& _doc) noexcept { + return stage2::tape_builder::parse_document(*this, _doc); + } + + simdjson_warn_unused uint8_t* dom_parser_implementation::parse_string(const uint8_t* src, uint8_t* dst, bool allow_replacement) const noexcept { + return lsx::stringparsing::parse_string(src, dst, allow_replacement); + } + + simdjson_warn_unused uint8_t* dom_parser_implementation::parse_wobbly_string(const uint8_t* src, uint8_t* dst) const noexcept { + return lsx::stringparsing::parse_wobbly_string(src, dst); + } + + simdjson_warn_unused error_code dom_parser_implementation::parse(const uint8_t* _buf, size_t _len, dom::document& _doc) noexcept { + auto error = stage1(_buf, _len, stage1_mode::regular); + if (error) { return error; } + return stage2(_doc); + } + + } // namespace lsx +} // namespace simdjson + +/* including simdjson/lsx/end.h: #include */ +/* begin file simdjson/lsx/end.h */ +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/lsx/base.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +#undef SIMDJSON_SKIP_BACKSLASH_SHORT_CIRCUIT +/* undefining SIMDJSON_IMPLEMENTATION from "lsx" */ +#undef SIMDJSON_IMPLEMENTATION +/* end file simdjson/lsx/end.h */ + +#endif // SIMDJSON_SRC_LSX_CPP +/* end file lsx.cpp */ +#endif +#if SIMDJSON_IMPLEMENTATION_LASX +/* including lasx.cpp: #include */ +/* begin file lasx.cpp */ +#ifndef SIMDJSON_SRC_LASX_CPP +#define SIMDJSON_SRC_LASX_CPP + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +/* including simdjson/lasx.h: #include */ +/* begin file simdjson/lasx.h */ +#ifndef SIMDJSON_LASX_H +#define SIMDJSON_LASX_H + +/* including simdjson/lasx/begin.h: #include "simdjson/lasx/begin.h" */ +/* begin file simdjson/lasx/begin.h */ +/* defining SIMDJSON_IMPLEMENTATION to "lasx" */ +#define SIMDJSON_IMPLEMENTATION lasx +/* including simdjson/lasx/base.h: #include "simdjson/lasx/base.h" */ +/* begin file simdjson/lasx/base.h */ +#ifndef SIMDJSON_LASX_BASE_H +#define SIMDJSON_LASX_BASE_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/base.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + /** + * Implementation for LASX. + */ + namespace lasx { + + class implementation; + + namespace { + namespace simd { + template struct simd8; + template struct simd8x64; + } // namespace simd + } // unnamed namespace + + } // namespace lasx +} // namespace simdjson + +#endif // SIMDJSON_LASX_BASE_H +/* end file simdjson/lasx/base.h */ +/* including simdjson/lasx/intrinsics.h: #include "simdjson/lasx/intrinsics.h" */ +/* begin file simdjson/lasx/intrinsics.h */ +#ifndef SIMDJSON_LASX_INTRINSICS_H +#define SIMDJSON_LASX_INTRINSICS_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/lasx/base.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +// This should be the correct header whether +// you use visual studio or other compilers. +#include + +static_assert(sizeof(__m256i) <= simdjson::SIMDJSON_PADDING, "insufficient padding for LoongArch ASX"); + +#endif // SIMDJSON_LASX_INTRINSICS_H +/* end file simdjson/lasx/intrinsics.h */ +/* including simdjson/lasx/bitmanipulation.h: #include "simdjson/lasx/bitmanipulation.h" */ +/* begin file simdjson/lasx/bitmanipulation.h */ +#ifndef SIMDJSON_LASX_BITMANIPULATION_H +#define SIMDJSON_LASX_BITMANIPULATION_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/lasx/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/lasx/intrinsics.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/lasx/bitmask.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lasx { + namespace { + + // We sometimes call trailing_zero on inputs that are zero, + // but the algorithms do not end up using the returned value. + // Sadly, sanitizers are not smart enough to figure it out. + SIMDJSON_NO_SANITIZE_UNDEFINED + // This function can be used safely even if not all bytes have been + // initialized. + // See issue https://github.com/simdjson/simdjson/issues/1965 + SIMDJSON_NO_SANITIZE_MEMORY + simdjson_inline int trailing_zeroes(uint64_t input_num) { + return __builtin_ctzll(input_num); + } + + /* result might be undefined when input_num is zero */ + simdjson_inline uint64_t clear_lowest_bit(uint64_t input_num) { + return input_num & (input_num - 1); + } + + /* result might be undefined when input_num is zero */ + simdjson_inline int leading_zeroes(uint64_t input_num) { + return __builtin_clzll(input_num); + } + + /* result might be undefined when input_num is zero */ + simdjson_inline int count_ones(uint64_t input_num) { + return __lasx_xvpickve2gr_w(__lasx_xvpcnt_d(__m256i(v4u64{ input_num, 0, 0, 0 })), 0); + } + + simdjson_inline bool add_overflow(uint64_t value1, uint64_t value2, uint64_t* result) { + return __builtin_uaddll_overflow(value1, value2, + reinterpret_cast(result)); + } + + } // unnamed namespace + } // namespace lasx +} // namespace simdjson + +#endif // SIMDJSON_LASX_BITMANIPULATION_H +/* end file simdjson/lasx/bitmanipulation.h */ +/* including simdjson/lasx/bitmask.h: #include "simdjson/lasx/bitmask.h" */ +/* begin file simdjson/lasx/bitmask.h */ +#ifndef SIMDJSON_LASX_BITMASK_H +#define SIMDJSON_LASX_BITMASK_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/lasx/base.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lasx { + namespace { + + // + // Perform a "cumulative bitwise xor," flipping bits each time a 1 is encountered. + // + // For example, prefix_xor(00100100) == 00011100 + // + simdjson_inline uint64_t prefix_xor(uint64_t bitmask) { + bitmask ^= bitmask << 1; + bitmask ^= bitmask << 2; + bitmask ^= bitmask << 4; + bitmask ^= bitmask << 8; + bitmask ^= bitmask << 16; + bitmask ^= bitmask << 32; + return bitmask; + } + + } // unnamed namespace + } // namespace lasx +} // namespace simdjson + +#endif +/* end file simdjson/lasx/bitmask.h */ +/* including simdjson/lasx/numberparsing_defs.h: #include "simdjson/lasx/numberparsing_defs.h" */ +/* begin file simdjson/lasx/numberparsing_defs.h */ +#ifndef SIMDJSON_LASX_NUMBERPARSING_DEFS_H +#define SIMDJSON_LASX_NUMBERPARSING_DEFS_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/lasx/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/lasx/intrinsics.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +#include + +namespace simdjson { + namespace lasx { + namespace numberparsing { + + // we don't have appropriate instructions, so let us use a scalar function + // credit: https://johnnylee-sde.github.io/Fast-numeric-string-to-int/ + /** @private */ + static simdjson_inline uint32_t parse_eight_digits_unrolled(const uint8_t* chars) { + uint64_t val; + std::memcpy(&val, chars, sizeof(uint64_t)); + val = (val & 0x0F0F0F0F0F0F0F0F) * 2561 >> 8; + val = (val & 0x00FF00FF00FF00FF) * 6553601 >> 16; + return uint32_t((val & 0x0000FFFF0000FFFF) * 42949672960001 >> 32); + } + + simdjson_inline internal::value128 full_multiplication(uint64_t value1, uint64_t value2) { + internal::value128 answer; + __uint128_t r = (static_cast<__uint128_t>(value1)) * value2; + answer.low = uint64_t(r); + answer.high = uint64_t(r >> 64); + return answer; + } + + } // namespace numberparsing + } // namespace lasx +} // namespace simdjson + +#define SIMDJSON_SWAR_NUMBER_PARSING 1 + +#endif // SIMDJSON_LASX_NUMBERPARSING_DEFS_H +/* end file simdjson/lasx/numberparsing_defs.h */ +/* including simdjson/lasx/simd.h: #include "simdjson/lasx/simd.h" */ +/* begin file simdjson/lasx/simd.h */ +#ifndef SIMDJSON_LASX_SIMD_H +#define SIMDJSON_LASX_SIMD_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/lasx/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/lasx/bitmanipulation.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/simdprune_tables.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lasx { + namespace { + namespace simd { + + // Forward-declared so they can be used by splat and friends. + template + struct base { + __m256i value; + + // Zero constructor + simdjson_inline base() : value{ __m256i() } {} + + // Conversion from SIMD register + simdjson_inline base(const __m256i _value) : value(_value) {} + + // Conversion to SIMD register + simdjson_inline operator const __m256i& () const { return this->value; } + simdjson_inline operator __m256i& () { return this->value; } + simdjson_inline operator const v32i8& () const { return (v32i8&)this->value; } + simdjson_inline operator v32i8& () { return (v32i8&)this->value; } + + // Bit operations + simdjson_inline Child operator|(const Child other) const { return __lasx_xvor_v(*this, other); } + simdjson_inline Child operator&(const Child other) const { return __lasx_xvand_v(*this, other); } + simdjson_inline Child operator^(const Child other) const { return __lasx_xvxor_v(*this, other); } + simdjson_inline Child bit_andnot(const Child other) const { return __lasx_xvandn_v(other, *this); } + simdjson_inline Child& operator|=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast | other; return *this_cast; } + simdjson_inline Child& operator&=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast & other; return *this_cast; } + simdjson_inline Child& operator^=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast ^ other; return *this_cast; } + }; + + // Forward-declared so they can be used by splat and friends. + template + struct simd8; + + template> + struct base8 : base> { + simdjson_inline base8() : base>() {} + simdjson_inline base8(const __m256i _value) : base>(_value) {} + + friend simdjson_really_inline Mask operator==(const simd8 lhs, const simd8 rhs) { return __lasx_xvseq_b(lhs, rhs); } + + static const int SIZE = sizeof(base>::value); + + template + simdjson_inline simd8 prev(const simd8 prev_chunk) const { + __m256i hi = __lasx_xvbsll_v(*this, N); + __m256i lo = __lasx_xvbsrl_v(*this, 16 - N); + __m256i tmp = __lasx_xvbsrl_v(prev_chunk, 16 - N); + lo = __lasx_xvpermi_q(lo, tmp, 0x21); + return __lasx_xvor_v(hi, lo); + } + }; + + // SIMD byte mask type (returned by things like eq and gt) + template<> + struct simd8 : base8 { + static simdjson_inline simd8 splat(bool _value) { return __lasx_xvreplgr2vr_b(uint8_t(-(!!_value))); } + + simdjson_inline simd8() : base8() {} + simdjson_inline simd8(const __m256i _value) : base8(_value) {} + // Splat constructor + simdjson_inline simd8(bool _value) : base8(splat(_value)) {} + + simdjson_inline int to_bitmask() const { + __m256i mask = __lasx_xvmskltz_b(*this); + return (__lasx_xvpickve2gr_w(mask, 4) << 16) | (__lasx_xvpickve2gr_w(mask, 0)); + } + simdjson_inline bool any() const { + __m256i v = __lasx_xvmsknz_b(*this); + return (0 == __lasx_xvpickve2gr_w(v, 0)) && (0 == __lasx_xvpickve2gr_w(v, 4)); + } + simdjson_inline simd8 operator~() const { return *this ^ true; } + }; + + template + struct base8_numeric : base8 { + static simdjson_inline simd8 splat(T _value) { + return __lasx_xvreplgr2vr_b(_value); + } + static simdjson_inline simd8 zero() { return __lasx_xvldi(0); } + static simdjson_inline simd8 load(const T values[32]) { + return __lasx_xvld(reinterpret_cast(values), 0); + } + // Repeat 16 values as many times as necessary (usually for lookup tables) + static simdjson_inline simd8 repeat_16( + T v0, T v1, T v2, T v3, T v4, T v5, T v6, T v7, + T v8, T v9, T v10, T v11, T v12, T v13, T v14, T v15 + ) { + return simd8( + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15, + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15 + ); + } + + simdjson_inline base8_numeric() : base8() {} + simdjson_inline base8_numeric(const __m256i _value) : base8(_value) {} + + // Store to array + simdjson_inline void store(T dst[32]) const { + return __lasx_xvst(*this, reinterpret_cast<__m256i*>(dst), 0); + } + + // Addition/subtraction are the same for signed and unsigned + simdjson_inline simd8 operator+(const simd8 other) const { return __lasx_xvadd_b(*this, other); } + simdjson_inline simd8 operator-(const simd8 other) const { return __lasx_xvsub_b(*this, other); } + simdjson_inline simd8& operator+=(const simd8 other) { *this = *this + other; return *static_cast*>(this); } + simdjson_inline simd8& operator-=(const simd8 other) { *this = *this - other; return *static_cast*>(this); } + + // Override to distinguish from bool version + simdjson_inline simd8 operator~() const { return *this ^ 0xFFu; } + + // Perform a lookup assuming the value is between 0 and 16 (undefined behavior for out of range values) + template + simdjson_inline simd8 lookup_16(simd8 lookup_table) const { + return __lasx_xvshuf_b(lookup_table, lookup_table, *this); + } + + // Copies to 'output" all bytes corresponding to a 0 in the mask (interpreted as a bitset). + // Passing a 0 value for mask would be equivalent to writing out every byte to output. + // Only the first 16 - count_ones(mask) bytes of the result are significant but 16 bytes + // get written. + template + simdjson_inline void compress(uint32_t mask, L* output) const { + using internal::thintable_epi8; + using internal::BitsSetTable256mul2; + using internal::pshufb_combine_table; + // this particular implementation was inspired by haswell + // lasx do it in 4 steps, first 8 bytes and then second 8 bytes... + uint8_t mask1 = uint8_t(mask); // least significant 8 bits + uint8_t mask2 = uint8_t(mask >> 8); // second significant 8 bits + uint8_t mask3 = uint8_t(mask >> 16); // ... + uint8_t mask4 = uint8_t(mask >> 24); // ... + // next line just loads the 64-bit values thintable_epi8[mask{1,2,3,4}] + // into a 256-bit register. + __m256i shufmask = { int64_t(thintable_epi8[mask1]), int64_t(thintable_epi8[mask2]) + 0x0808080808080808, int64_t(thintable_epi8[mask3]), int64_t(thintable_epi8[mask4]) + 0x0808080808080808 }; + // this is the version "nearly pruned" + __m256i pruned = __lasx_xvshuf_b(*this, *this, shufmask); + // we still need to put the pieces back together. + // we compute the popcount of the first words: + int pop1 = BitsSetTable256mul2[mask1]; + int pop2 = BitsSetTable256mul2[mask2]; + int pop3 = BitsSetTable256mul2[mask3]; + + // then load the corresponding mask + __m256i masklo = __lasx_xvldx(reinterpret_cast(reinterpret_cast(pshufb_combine_table)), pop1 * 8); + __m256i maskhi = __lasx_xvldx(reinterpret_cast(reinterpret_cast(pshufb_combine_table)), pop3 * 8); + __m256i compactmask = __lasx_xvpermi_q(maskhi, masklo, 0x20); + __m256i answer = __lasx_xvshuf_b(pruned, pruned, compactmask); + __lasx_xvst(answer, reinterpret_cast(output), 0); + uint64_t value3 = __lasx_xvpickve2gr_du(answer, 2); + uint64_t value4 = __lasx_xvpickve2gr_du(answer, 3); + uint64_t* pos = reinterpret_cast(reinterpret_cast(output) + 16 - (pop1 + pop2) / 2); + pos[0] = value3; + pos[1] = value4; + } + + template + simdjson_inline simd8 lookup_16( + L replace0, L replace1, L replace2, L replace3, + L replace4, L replace5, L replace6, L replace7, + L replace8, L replace9, L replace10, L replace11, + L replace12, L replace13, L replace14, L replace15) const { + return lookup_16(simd8::repeat_16( + replace0, replace1, replace2, replace3, + replace4, replace5, replace6, replace7, + replace8, replace9, replace10, replace11, + replace12, replace13, replace14, replace15 + )); + } + }; + + // Signed bytes + template<> + struct simd8 : base8_numeric { + simdjson_inline simd8() : base8_numeric() {} + simdjson_inline simd8(const __m256i _value) : base8_numeric(_value) {} + // Splat constructor + simdjson_inline simd8(int8_t _value) : simd8(splat(_value)) {} + // Array constructor + simdjson_inline simd8(const int8_t values[32]) : simd8(load(values)) {} + // Member-by-member initialization + simdjson_inline simd8( + int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, + int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15, + int8_t v16, int8_t v17, int8_t v18, int8_t v19, int8_t v20, int8_t v21, int8_t v22, int8_t v23, + int8_t v24, int8_t v25, int8_t v26, int8_t v27, int8_t v28, int8_t v29, int8_t v30, int8_t v31 + ) : simd8({ + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10,v11,v12,v13,v14,v15, + v16,v17,v18,v19,v20,v21,v22,v23, + v24,v25,v26,v27,v28,v29,v30,v31 + }) {} + // Repeat 16 values as many times as necessary (usually for lookup tables) + simdjson_inline static simd8 repeat_16( + int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, + int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 + ) { + return simd8( + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15, + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15 + ); + } + + // Order-sensitive comparisons + simdjson_inline simd8 max_val(const simd8 other) const { return __lasx_xvmax_b(*this, other); } + simdjson_inline simd8 min_val(const simd8 other) const { return __lasx_xvmin_b(*this, other); } + simdjson_inline simd8 operator>(const simd8 other) const { return __lasx_xvslt_b(other, *this); } + simdjson_inline simd8 operator<(const simd8 other) const { return __lasx_xvslt_b(*this, other); } + }; + + // Unsigned bytes + template<> + struct simd8 : base8_numeric { + simdjson_inline simd8() : base8_numeric() {} + simdjson_inline simd8(const __m256i _value) : base8_numeric(_value) {} + // Splat constructor + simdjson_inline simd8(uint8_t _value) : simd8(splat(_value)) {} + // Array constructor + simdjson_inline simd8(const uint8_t values[32]) : simd8(load(values)) {} + // Member-by-member initialization + simdjson_inline simd8( + uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, + uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15, + uint8_t v16, uint8_t v17, uint8_t v18, uint8_t v19, uint8_t v20, uint8_t v21, uint8_t v22, uint8_t v23, + uint8_t v24, uint8_t v25, uint8_t v26, uint8_t v27, uint8_t v28, uint8_t v29, uint8_t v30, uint8_t v31 + ) : simd8(__m256i(v32u8{ + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10,v11,v12,v13,v14,v15, + v16,v17,v18,v19,v20,v21,v22,v23, + v24,v25,v26,v27,v28,v29,v30,v31 + })) {} + // Repeat 16 values as many times as necessary (usually for lookup tables) + simdjson_inline static simd8 repeat_16( + uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, + uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 + ) { + return simd8( + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15, + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15 + ); + } + + // Saturated math + simdjson_inline simd8 saturating_add(const simd8 other) const { return __lasx_xvsadd_bu(*this, other); } + simdjson_inline simd8 saturating_sub(const simd8 other) const { return __lasx_xvssub_bu(*this, other); } + + // Order-specific operations + simdjson_inline simd8 max_val(const simd8 other) const { return __lasx_xvmax_bu(*this, other); } + simdjson_inline simd8 min_val(const simd8 other) const { return __lasx_xvmin_bu(other, *this); } + // Same as >, but only guarantees true is nonzero (< guarantees true = -1) + simdjson_inline simd8 gt_bits(const simd8 other) const { return this->saturating_sub(other); } + // Same as <, but only guarantees true is nonzero (< guarantees true = -1) + simdjson_inline simd8 lt_bits(const simd8 other) const { return other.saturating_sub(*this); } + simdjson_inline simd8 operator<=(const simd8 other) const { return other.max_val(*this) == other; } + simdjson_inline simd8 operator>=(const simd8 other) const { return other.min_val(*this) == other; } + simdjson_inline simd8 operator>(const simd8 other) const { return this->gt_bits(other).any_bits_set(); } + simdjson_inline simd8 operator<(const simd8 other) const { return this->lt_bits(other).any_bits_set(); } + + // Bit-specific operations + simdjson_inline simd8 bits_not_set() const { return *this == uint8_t(0); } + simdjson_inline simd8 bits_not_set(simd8 bits) const { return (*this & bits).bits_not_set(); } + simdjson_inline simd8 any_bits_set() const { return ~this->bits_not_set(); } + simdjson_inline simd8 any_bits_set(simd8 bits) const { return ~this->bits_not_set(bits); } + simdjson_inline bool is_ascii() const { + __m256i mask = __lasx_xvmskltz_b(*this); + return (0 == __lasx_xvpickve2gr_w(mask, 0)) && (0 == __lasx_xvpickve2gr_w(mask, 4)); + } + simdjson_inline bool bits_not_set_anywhere() const { + __m256i v = __lasx_xvmsknz_b(*this); + return (0 == __lasx_xvpickve2gr_w(v, 0)) && (0 == __lasx_xvpickve2gr_w(v, 4)); + } + simdjson_inline bool any_bits_set_anywhere() const { return !bits_not_set_anywhere(); } + simdjson_inline bool bits_not_set_anywhere(simd8 bits) const { + __m256i v = __lasx_xvmsknz_b(__lasx_xvand_v(*this, bits)); + return (0 == __lasx_xvpickve2gr_w(v, 0)) && (0 == __lasx_xvpickve2gr_w(v, 4)); + } + simdjson_inline bool any_bits_set_anywhere(simd8 bits) const { return !bits_not_set_anywhere(bits); } + template + simdjson_inline simd8 shr() const { return simd8(__lasx_xvsrli_b(*this, N)); } + template + simdjson_inline simd8 shl() const { return simd8(__lasx_xvslli_b(*this, N)); } + }; + + template + struct simd8x64 { + static constexpr int NUM_CHUNKS = 64 / sizeof(simd8); + static_assert(NUM_CHUNKS == 2, "LASX kernel should use two registers per 64-byte block."); + const simd8 chunks[NUM_CHUNKS]; + + simd8x64(const simd8x64& o) = delete; // no copy allowed + simd8x64& operator=(const simd8& other) = delete; // no assignment allowed + simd8x64() = delete; // no default constructor allowed + + simdjson_inline simd8x64(const simd8 chunk0, const simd8 chunk1) : chunks{ chunk0, chunk1 } {} + simdjson_inline simd8x64(const T ptr[64]) : chunks{ simd8::load(ptr), simd8::load(ptr + 32) } {} + + simdjson_inline uint64_t compress(uint64_t mask, T* output) const { + uint32_t mask1 = uint32_t(mask); + uint32_t mask2 = uint32_t(mask >> 32); + __m256i zcnt = __lasx_xvpcnt_w(__m256i(v4u64{ ~mask, 0, 0, 0 })); + uint64_t zcnt1 = __lasx_xvpickve2gr_wu(zcnt, 0); + uint64_t zcnt2 = __lasx_xvpickve2gr_wu(zcnt, 1); + // There should be a critical value which processes in scaler is faster. + if (zcnt1) + this->chunks[0].compress(mask1, output); + if (zcnt2) + this->chunks[1].compress(mask2, output + zcnt1); + return zcnt1 + zcnt2; + } + + simdjson_inline void store(T ptr[64]) const { + this->chunks[0].store(ptr + sizeof(simd8) * 0); + this->chunks[1].store(ptr + sizeof(simd8) * 1); + } + + simdjson_inline uint64_t to_bitmask() const { + __m256i mask0 = __lasx_xvmskltz_b(this->chunks[0]); + __m256i mask1 = __lasx_xvmskltz_b(this->chunks[1]); + __m256i mask_tmp = __lasx_xvpickve_w(mask0, 4); + __m256i tmp = __lasx_xvpickve_w(mask1, 4); + mask0 = __lasx_xvinsve0_w(mask0, mask1, 1); + mask_tmp = __lasx_xvinsve0_w(mask_tmp, tmp, 1); + return __lasx_xvpickve2gr_du(__lasx_xvpackev_h(mask_tmp, mask0), 0); + } + + simdjson_inline simd8 reduce_or() const { + return this->chunks[0] | this->chunks[1]; + } + + simdjson_inline uint64_t eq(const T m) const { + const simd8 mask = simd8::splat(m); + return simd8x64( + this->chunks[0] == mask, + this->chunks[1] == mask + ).to_bitmask(); + } + + simdjson_inline uint64_t eq(const simd8x64& other) const { + return simd8x64( + this->chunks[0] == other.chunks[0], + this->chunks[1] == other.chunks[1] + ).to_bitmask(); + } + + simdjson_inline uint64_t lteq(const T m) const { + const simd8 mask = simd8::splat(m); + return simd8x64( + this->chunks[0] <= mask, + this->chunks[1] <= mask + ).to_bitmask(); + } + }; // struct simd8x64 + + } // namespace simd + } // unnamed namespace + } // namespace lasx +} // namespace simdjson + +#endif // SIMDJSON_LASX_SIMD_H +/* end file simdjson/lasx/simd.h */ +/* including simdjson/lasx/stringparsing_defs.h: #include "simdjson/lasx/stringparsing_defs.h" */ +/* begin file simdjson/lasx/stringparsing_defs.h */ +#ifndef SIMDJSON_LASX_STRINGPARSING_DEFS_H +#define SIMDJSON_LASX_STRINGPARSING_DEFS_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/lasx/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/lasx/simd.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/lasx/bitmanipulation.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lasx { + namespace { + + using namespace simd; + + // Holds backslashes and quotes locations. + struct backslash_and_quote { + public: + static constexpr uint32_t BYTES_PROCESSED = 32; + simdjson_inline static backslash_and_quote copy_and_find(const uint8_t* src, uint8_t* dst); + + simdjson_inline bool has_quote_first() { return ((bs_bits - 1) & quote_bits) != 0; } + simdjson_inline bool has_backslash() { return bs_bits != 0; } + simdjson_inline int quote_index() { return trailing_zeroes(quote_bits); } + simdjson_inline int backslash_index() { return trailing_zeroes(bs_bits); } + + uint32_t bs_bits; + uint32_t quote_bits; + }; // struct backslash_and_quote + + simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8_t* src, uint8_t* dst) { + // this can read up to 31 bytes beyond the buffer size, but we require + // SIMDJSON_PADDING of padding + static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "backslash and quote finder must process fewer than SIMDJSON_PADDING bytes"); + simd8 v(src); + v.store(dst); + return { + static_cast((v == '\\').to_bitmask()), // bs_bits + static_cast((v == '"').to_bitmask()), // quote_bits + }; + } + + } // unnamed namespace + } // namespace lasx +} // namespace simdjson + +#endif // SIMDJSON_LASX_STRINGPARSING_DEFS_H +/* end file simdjson/lasx/stringparsing_defs.h */ + +#define SIMDJSON_SKIP_BACKSLASH_SHORT_CIRCUIT 1 +/* end file simdjson/lasx/begin.h */ +/* including simdjson/generic/amalgamated.h for lasx: #include "simdjson/generic/amalgamated.h" */ +/* begin file simdjson/generic/amalgamated.h for lasx */ +#if defined(SIMDJSON_CONDITIONAL_INCLUDE) && !defined(SIMDJSON_GENERIC_DEPENDENCIES_H) +#error simdjson/generic/dependencies.h must be included before simdjson/generic/amalgamated.h! +#endif + +/* including simdjson/generic/base.h for lasx: #include "simdjson/generic/base.h" */ +/* begin file simdjson/generic/base.h for lasx */ +#ifndef SIMDJSON_GENERIC_BASE_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_BASE_H */ +/* amalgamation skipped (editor-only): #include "simdjson/base.h" */ +/* amalgamation skipped (editor-only): // If we haven't got an implementation yet, we're in the editor, editing a generic file! Just */ +/* amalgamation skipped (editor-only): // use the most advanced one we can so the most possible stuff can be tested. */ +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_IMPLEMENTATION */ +/* amalgamation skipped (editor-only): #include "simdjson/implementation_detection.h" */ +/* amalgamation skipped (editor-only): #if SIMDJSON_IMPLEMENTATION_ICELAKE */ +/* amalgamation skipped (editor-only): #include "simdjson/icelake/begin.h" */ +/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_HASWELL */ +/* amalgamation skipped (editor-only): #include "simdjson/haswell/begin.h" */ +/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_WESTMERE */ +/* amalgamation skipped (editor-only): #include "simdjson/westmere/begin.h" */ +/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_ARM64 */ +/* amalgamation skipped (editor-only): #include "simdjson/arm64/begin.h" */ +/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_PPC64 */ +/* amalgamation skipped (editor-only): #include "simdjson/ppc64/begin.h" */ +/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_LSX */ +/* amalgamation skipped (editor-only): #include "simdjson/lsx/begin.h" */ +/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_LASX */ +/* amalgamation skipped (editor-only): #include "simdjson/lasx/begin.h" */ +/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_FALLBACK */ +/* amalgamation skipped (editor-only): #include "simdjson/fallback/begin.h" */ +/* amalgamation skipped (editor-only): #else */ +/* amalgamation skipped (editor-only): #error "All possible implementations (including fallback) have been disabled! simdjson will not run." */ +/* amalgamation skipped (editor-only): #endif */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_IMPLEMENTATION */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lasx { + + struct open_container; + class dom_parser_implementation; + + /** + * The type of a JSON number + */ + enum class number_type { + floating_point_number = 1, /// a binary64 number + signed_integer, /// a signed integer that fits in a 64-bit word using two's complement + unsigned_integer, /// a positive integer larger or equal to 1<<63 + big_integer /// a big integer that does not fit in a 64-bit word + }; + + } // namespace lasx +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_BASE_H +/* end file simdjson/generic/base.h for lasx */ +/* including simdjson/generic/jsoncharutils.h for lasx: #include "simdjson/generic/jsoncharutils.h" */ +/* begin file simdjson/generic/jsoncharutils.h for lasx */ +#ifndef SIMDJSON_GENERIC_JSONCHARUTILS_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_JSONCHARUTILS_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/jsoncharutils_tables.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lasx { + namespace { + namespace jsoncharutils { + + // return non-zero if not a structural or whitespace char + // zero otherwise + simdjson_inline uint32_t is_not_structural_or_whitespace(uint8_t c) { + return internal::structural_or_whitespace_negated[c]; + } + + simdjson_inline uint32_t is_structural_or_whitespace(uint8_t c) { + return internal::structural_or_whitespace[c]; + } + + // returns a value with the high 16 bits set if not valid + // otherwise returns the conversion of the 4 hex digits at src into the bottom + // 16 bits of the 32-bit return register + // + // see + // https://lemire.me/blog/2019/04/17/parsing-short-hexadecimal-strings-efficiently/ + static inline uint32_t hex_to_u32_nocheck( + const uint8_t* src) { // strictly speaking, static inline is a C-ism + uint32_t v1 = internal::digit_to_val32[630 + src[0]]; + uint32_t v2 = internal::digit_to_val32[420 + src[1]]; + uint32_t v3 = internal::digit_to_val32[210 + src[2]]; + uint32_t v4 = internal::digit_to_val32[0 + src[3]]; + return v1 | v2 | v3 | v4; + } + + // given a code point cp, writes to c + // the utf-8 code, outputting the length in + // bytes, if the length is zero, the code point + // is invalid + // + // This can possibly be made faster using pdep + // and clz and table lookups, but JSON documents + // have few escaped code points, and the following + // function looks cheap. + // + // Note: we assume that surrogates are treated separately + // + simdjson_inline size_t codepoint_to_utf8(uint32_t cp, uint8_t* c) { + if (cp <= 0x7F) { + c[0] = uint8_t(cp); + return 1; // ascii + } + if (cp <= 0x7FF) { + c[0] = uint8_t((cp >> 6) + 192); + c[1] = uint8_t((cp & 63) + 128); + return 2; // universal plane + // Surrogates are treated elsewhere... + //} //else if (0xd800 <= cp && cp <= 0xdfff) { + // return 0; // surrogates // could put assert here + } + else if (cp <= 0xFFFF) { + c[0] = uint8_t((cp >> 12) + 224); + c[1] = uint8_t(((cp >> 6) & 63) + 128); + c[2] = uint8_t((cp & 63) + 128); + return 3; + } + else if (cp <= 0x10FFFF) { // if you know you have a valid code point, this + // is not needed + c[0] = uint8_t((cp >> 18) + 240); + c[1] = uint8_t(((cp >> 12) & 63) + 128); + c[2] = uint8_t(((cp >> 6) & 63) + 128); + c[3] = uint8_t((cp & 63) + 128); + return 4; + } + // will return 0 when the code point was too large. + return 0; // bad r + } + +#if SIMDJSON_IS_32BITS // _umul128 for x86, arm + // this is a slow emulation routine for 32-bit + // + static simdjson_inline uint64_t __emulu(uint32_t x, uint32_t y) { + return x * (uint64_t)y; + } + static simdjson_inline uint64_t _umul128(uint64_t ab, uint64_t cd, uint64_t* hi) { + uint64_t ad = __emulu((uint32_t)(ab >> 32), (uint32_t)cd); + uint64_t bd = __emulu((uint32_t)ab, (uint32_t)cd); + uint64_t adbc = ad + __emulu((uint32_t)ab, (uint32_t)(cd >> 32)); + uint64_t adbc_carry = !!(adbc < ad); + uint64_t lo = bd + (adbc << 32); + *hi = __emulu((uint32_t)(ab >> 32), (uint32_t)(cd >> 32)) + (adbc >> 32) + + (adbc_carry << 32) + !!(lo < bd); + return lo; + } +#endif + + } // namespace jsoncharutils + } // unnamed namespace + } // namespace lasx +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_JSONCHARUTILS_H +/* end file simdjson/generic/jsoncharutils.h for lasx */ +/* including simdjson/generic/atomparsing.h for lasx: #include "simdjson/generic/atomparsing.h" */ +/* begin file simdjson/generic/atomparsing.h for lasx */ +#ifndef SIMDJSON_GENERIC_ATOMPARSING_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_ATOMPARSING_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/jsoncharutils.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +#include + +namespace simdjson { + namespace lasx { + namespace { + /// @private + namespace atomparsing { + + // The string_to_uint32 is exclusively used to map literal strings to 32-bit values. + // We use memcpy instead of a pointer cast to avoid undefined behaviors since we cannot + // be certain that the character pointer will be properly aligned. + // You might think that using memcpy makes this function expensive, but you'd be wrong. + // All decent optimizing compilers (GCC, clang, Visual Studio) will compile string_to_uint32("false"); + // to the compile-time constant 1936482662. + simdjson_inline uint32_t string_to_uint32(const char* str) { uint32_t val; std::memcpy(&val, str, sizeof(uint32_t)); return val; } + + + // Again in str4ncmp we use a memcpy to avoid undefined behavior. The memcpy may appear expensive. + // Yet all decent optimizing compilers will compile memcpy to a single instruction, just about. + simdjson_warn_unused + simdjson_inline uint32_t str4ncmp(const uint8_t* src, const char* atom) { + uint32_t srcval; // we want to avoid unaligned 32-bit loads (undefined in C/C++) + static_assert(sizeof(uint32_t) <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be larger than 4 bytes"); + std::memcpy(&srcval, src, sizeof(uint32_t)); + return srcval ^ string_to_uint32(atom); + } + + simdjson_warn_unused + simdjson_inline bool is_valid_true_atom(const uint8_t* src) { + return (str4ncmp(src, "true") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; + } + + simdjson_warn_unused + simdjson_inline bool is_valid_true_atom(const uint8_t* src, size_t len) { + if (len > 4) { return is_valid_true_atom(src); } + else if (len == 4) { return !str4ncmp(src, "true"); } + else { return false; } + } + + simdjson_warn_unused + simdjson_inline bool is_valid_false_atom(const uint8_t* src) { + return (str4ncmp(src + 1, "alse") | jsoncharutils::is_not_structural_or_whitespace(src[5])) == 0; + } + + simdjson_warn_unused + simdjson_inline bool is_valid_false_atom(const uint8_t* src, size_t len) { + if (len > 5) { return is_valid_false_atom(src); } + else if (len == 5) { return !str4ncmp(src + 1, "alse"); } + else { return false; } + } + + simdjson_warn_unused + simdjson_inline bool is_valid_null_atom(const uint8_t* src) { + return (str4ncmp(src, "null") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; + } + + simdjson_warn_unused + simdjson_inline bool is_valid_null_atom(const uint8_t* src, size_t len) { + if (len > 4) { return is_valid_null_atom(src); } + else if (len == 4) { return !str4ncmp(src, "null"); } + else { return false; } + } + + } // namespace atomparsing + } // unnamed namespace + } // namespace lasx +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_ATOMPARSING_H +/* end file simdjson/generic/atomparsing.h for lasx */ +/* including simdjson/generic/dom_parser_implementation.h for lasx: #include "simdjson/generic/dom_parser_implementation.h" */ +/* begin file simdjson/generic/dom_parser_implementation.h for lasx */ +#ifndef SIMDJSON_GENERIC_DOM_PARSER_IMPLEMENTATION_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_DOM_PARSER_IMPLEMENTATION_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/dom_parser_implementation.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lasx { + + // expectation: sizeof(open_container) = 64/8. + struct open_container { + uint32_t tape_index; // where, on the tape, does the scope ([,{) begins + uint32_t count; // how many elements in the scope + }; // struct open_container + + static_assert(sizeof(open_container) == 64 / 8, "Open container must be 64 bits"); + + class dom_parser_implementation final : public internal::dom_parser_implementation { + public: + /** Tape location of each open { or [ */ + std::unique_ptr open_containers{}; + /** Whether each open container is a [ or { */ + std::unique_ptr is_array{}; + /** Buffer passed to stage 1 */ + const uint8_t* buf{}; + /** Length passed to stage 1 */ + size_t len{ 0 }; + /** Document passed to stage 2 */ + dom::document* doc{}; + + inline dom_parser_implementation() noexcept; + inline dom_parser_implementation(dom_parser_implementation&& other) noexcept; + inline dom_parser_implementation& operator=(dom_parser_implementation&& other) noexcept; + dom_parser_implementation(const dom_parser_implementation&) = delete; + dom_parser_implementation& operator=(const dom_parser_implementation&) = delete; + + simdjson_warn_unused error_code parse(const uint8_t* buf, size_t len, dom::document& doc) noexcept final; + simdjson_warn_unused error_code stage1(const uint8_t* buf, size_t len, stage1_mode partial) noexcept final; + simdjson_warn_unused error_code stage2(dom::document& doc) noexcept final; + simdjson_warn_unused error_code stage2_next(dom::document& doc) noexcept final; + simdjson_warn_unused uint8_t* parse_string(const uint8_t* src, uint8_t* dst, bool allow_replacement) const noexcept final; + simdjson_warn_unused uint8_t* parse_wobbly_string(const uint8_t* src, uint8_t* dst) const noexcept final; + inline simdjson_warn_unused error_code set_capacity(size_t capacity) noexcept final; + inline simdjson_warn_unused error_code set_max_depth(size_t max_depth) noexcept final; + private: + simdjson_inline simdjson_warn_unused error_code set_capacity_stage1(size_t capacity); + + }; + + } // namespace lasx +} // namespace simdjson + +namespace simdjson { + namespace lasx { + + inline dom_parser_implementation::dom_parser_implementation() noexcept = default; + inline dom_parser_implementation::dom_parser_implementation(dom_parser_implementation&& other) noexcept = default; + inline dom_parser_implementation& dom_parser_implementation::operator=(dom_parser_implementation&& other) noexcept = default; + + // Leaving these here so they can be inlined if so desired + inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(size_t capacity) noexcept { + if (capacity > SIMDJSON_MAXSIZE_BYTES) { return CAPACITY; } + // Stage 1 index output + size_t max_structures = SIMDJSON_ROUNDUP_N(capacity, 64) + 2 + 7; + structural_indexes.reset(new (std::nothrow) uint32_t[max_structures]); + if (!structural_indexes) { _capacity = 0; return MEMALLOC; } + structural_indexes[0] = 0; + n_structural_indexes = 0; + + _capacity = capacity; + return SUCCESS; + } + + inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { + // Stage 2 stacks + open_containers.reset(new (std::nothrow) open_container[max_depth]); + is_array.reset(new (std::nothrow) bool[max_depth]); + if (!is_array || !open_containers) { _max_depth = 0; return MEMALLOC; } + + _max_depth = max_depth; + return SUCCESS; + } + + } // namespace lasx +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_DOM_PARSER_IMPLEMENTATION_H +/* end file simdjson/generic/dom_parser_implementation.h for lasx */ +/* including simdjson/generic/implementation_simdjson_result_base.h for lasx: #include "simdjson/generic/implementation_simdjson_result_base.h" */ +/* begin file simdjson/generic/implementation_simdjson_result_base.h for lasx */ +#ifndef SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lasx { + + // This is a near copy of include/error.h's implementation_simdjson_result_base, except it doesn't use std::pair + // so we can avoid inlining errors + // TODO reconcile these! + /** + * The result of a simdjson operation that could fail. + * + * Gives the option of reading error codes, or throwing an exception by casting to the desired result. + * + * This is a base class for implementations that want to add functions to the result type for + * chaining. + * + * Override like: + * + * struct simdjson_result : public internal::implementation_simdjson_result_base { + * simdjson_result() noexcept : internal::implementation_simdjson_result_base() {} + * simdjson_result(error_code error) noexcept : internal::implementation_simdjson_result_base(error) {} + * simdjson_result(T &&value) noexcept : internal::implementation_simdjson_result_base(std::forward(value)) {} + * simdjson_result(T &&value, error_code error) noexcept : internal::implementation_simdjson_result_base(value, error) {} + * // Your extra methods here + * } + * + * Then any method returning simdjson_result will be chainable with your methods. + */ + template + struct implementation_simdjson_result_base { + + /** + * Create a new empty result with error = UNINITIALIZED. + */ + simdjson_inline implementation_simdjson_result_base() noexcept = default; + + /** + * Create a new error result. + */ + simdjson_inline implementation_simdjson_result_base(error_code error) noexcept; + + /** + * Create a new successful result. + */ + simdjson_inline implementation_simdjson_result_base(T&& value) noexcept; + + /** + * Create a new result with both things (use if you don't want to branch when creating the result). + */ + simdjson_inline implementation_simdjson_result_base(T&& value, error_code error) noexcept; + + /** + * Move the value and the error to the provided variables. + * + * @param value The variable to assign the value to. May not be set if there is an error. + * @param error The variable to assign the error to. Set to SUCCESS if there is no error. + */ + simdjson_inline void tie(T& value, error_code& error) && noexcept; + + /** + * Move the value to the provided variable. + * + * @param value The variable to assign the value to. May not be set if there is an error. + */ + simdjson_inline error_code get(T& value) && noexcept; + + /** + * The error. + */ + simdjson_inline error_code error() const noexcept; + +#if SIMDJSON_EXCEPTIONS + + /** + * Get the result value. + * + * @throw simdjson_error if there was an error. + */ + simdjson_inline T& value() & noexcept(false); + + /** + * Take the result value (move it). + * + * @throw simdjson_error if there was an error. + */ + simdjson_inline T&& value() && noexcept(false); + + /** + * Take the result value (move it). + * + * @throw simdjson_error if there was an error. + */ + simdjson_inline T&& take_value() && noexcept(false); + + /** + * Cast to the value (will throw on error). + * + * @throw simdjson_error if there was an error. + */ + simdjson_inline operator T && () && noexcept(false); + + +#endif // SIMDJSON_EXCEPTIONS + + /** + * Get the result value. This function is safe if and only + * the error() method returns a value that evaluates to false. + */ + simdjson_inline const T& value_unsafe() const& noexcept; + /** + * Get the result value. This function is safe if and only + * the error() method returns a value that evaluates to false. + */ + simdjson_inline T& value_unsafe() & noexcept; + /** + * Take the result value (move it). This function is safe if and only + * the error() method returns a value that evaluates to false. + */ + simdjson_inline T&& value_unsafe() && noexcept; + protected: + /** users should never directly access first and second. **/ + T first{}; /** Users should never directly access 'first'. **/ + error_code second{ UNINITIALIZED }; /** Users should never directly access 'second'. **/ + }; // struct implementation_simdjson_result_base + + } // namespace lasx +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_H +/* end file simdjson/generic/implementation_simdjson_result_base.h for lasx */ +/* including simdjson/generic/numberparsing.h for lasx: #include "simdjson/generic/numberparsing.h" */ +/* begin file simdjson/generic/numberparsing.h for lasx */ +#ifndef SIMDJSON_GENERIC_NUMBERPARSING_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_NUMBERPARSING_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/jsoncharutils.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +#include +#include +#include + +namespace simdjson { + namespace lasx { + namespace numberparsing { + +#ifdef JSON_TEST_NUMBERS +#define INVALID_NUMBER(SRC) (found_invalid_number((SRC)), NUMBER_ERROR) +#define WRITE_INTEGER(VALUE, SRC, WRITER) (found_integer((VALUE), (SRC)), (WRITER).append_s64((VALUE))) +#define WRITE_UNSIGNED(VALUE, SRC, WRITER) (found_unsigned_integer((VALUE), (SRC)), (WRITER).append_u64((VALUE))) +#define WRITE_DOUBLE(VALUE, SRC, WRITER) (found_float((VALUE), (SRC)), (WRITER).append_double((VALUE))) +#define BIGINT_NUMBER(SRC) (found_invalid_number((SRC)), BIGINT_ERROR) +#else +#define INVALID_NUMBER(SRC) (NUMBER_ERROR) +#define WRITE_INTEGER(VALUE, SRC, WRITER) (WRITER).append_s64((VALUE)) +#define WRITE_UNSIGNED(VALUE, SRC, WRITER) (WRITER).append_u64((VALUE)) +#define WRITE_DOUBLE(VALUE, SRC, WRITER) (WRITER).append_double((VALUE)) +#define BIGINT_NUMBER(SRC) (BIGINT_ERROR) +#endif + + namespace { + + // Convert a mantissa, an exponent and a sign bit into an ieee64 double. + // The real_exponent needs to be in [0, 2046] (technically real_exponent = 2047 would be acceptable). + // The mantissa should be in [0,1<<53). The bit at index (1ULL << 52) while be zeroed. + simdjson_inline double to_double(uint64_t mantissa, uint64_t real_exponent, bool negative) { + double d; + mantissa &= ~(1ULL << 52); + mantissa |= real_exponent << 52; + mantissa |= ((static_cast(negative)) << 63); + std::memcpy(&d, &mantissa, sizeof(d)); + return d; + } + + // Attempts to compute i * 10^(power) exactly; and if "negative" is + // true, negate the result. + // This function will only work in some cases, when it does not work, success is + // set to false. This should work *most of the time* (like 99% of the time). + // We assume that power is in the [smallest_power, + // largest_power] interval: the caller is responsible for this check. + simdjson_inline bool compute_float_64(int64_t power, uint64_t i, bool negative, double& d) { + // we start with a fast path + // It was described in + // Clinger WD. How to read floating point numbers accurately. + // ACM SIGPLAN Notices. 1990 +#ifndef FLT_EVAL_METHOD +#error "FLT_EVAL_METHOD should be defined, please include cfloat." +#endif +#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0) + // We cannot be certain that x/y is rounded to nearest. + if (0 <= power && power <= 22 && i <= 9007199254740991) +#else + if (-22 <= power && power <= 22 && i <= 9007199254740991) +#endif + { + // convert the integer into a double. This is lossless since + // 0 <= i <= 2^53 - 1. + d = double(i); + // + // The general idea is as follows. + // If 0 <= s < 2^53 and if 10^0 <= p <= 10^22 then + // 1) Both s and p can be represented exactly as 64-bit floating-point + // values + // (binary64). + // 2) Because s and p can be represented exactly as floating-point values, + // then s * p + // and s / p will produce correctly rounded values. + // + if (power < 0) { + d = d / simdjson::internal::power_of_ten[-power]; + } + else { + d = d * simdjson::internal::power_of_ten[power]; + } + if (negative) { + d = -d; + } + return true; + } + // When 22 < power && power < 22 + 16, we could + // hope for another, secondary fast path. It was + // described by David M. Gay in "Correctly rounded + // binary-decimal and decimal-binary conversions." (1990) + // If you need to compute i * 10^(22 + x) for x < 16, + // first compute i * 10^x, if you know that result is exact + // (e.g., when i * 10^x < 2^53), + // then you can still proceed and do (i * 10^x) * 10^22. + // Is this worth your time? + // You need 22 < power *and* power < 22 + 16 *and* (i * 10^(x-22) < 2^53) + // for this second fast path to work. + // If you you have 22 < power *and* power < 22 + 16, and then you + // optimistically compute "i * 10^(x-22)", there is still a chance that you + // have wasted your time if i * 10^(x-22) >= 2^53. It makes the use cases of + // this optimization maybe less common than we would like. Source: + // http://www.exploringbinary.com/fast-path-decimal-to-floating-point-conversion/ + // also used in RapidJSON: https://rapidjson.org/strtod_8h_source.html + + // The fast path has now failed, so we are failing back on the slower path. + + // In the slow path, we need to adjust i so that it is > 1<<63 which is always + // possible, except if i == 0, so we handle i == 0 separately. + if (i == 0) { + d = negative ? -0.0 : 0.0; + return true; + } + + + // The exponent is 1024 + 63 + power + // + floor(log(5**power)/log(2)). + // The 1024 comes from the ieee64 standard. + // The 63 comes from the fact that we use a 64-bit word. + // + // Computing floor(log(5**power)/log(2)) could be + // slow. Instead we use a fast function. + // + // For power in (-400,350), we have that + // (((152170 + 65536) * power ) >> 16); + // is equal to + // floor(log(5**power)/log(2)) + power when power >= 0 + // and it is equal to + // ceil(log(5**-power)/log(2)) + power when power < 0 + // + // The 65536 is (1<<16) and corresponds to + // (65536 * power) >> 16 ---> power + // + // ((152170 * power ) >> 16) is equal to + // floor(log(5**power)/log(2)) + // + // Note that this is not magic: 152170/(1<<16) is + // approximatively equal to log(5)/log(2). + // The 1<<16 value is a power of two; we could use a + // larger power of 2 if we wanted to. + // + int64_t exponent = (((152170 + 65536) * power) >> 16) + 1024 + 63; + + + // We want the most significant bit of i to be 1. Shift if needed. + int lz = leading_zeroes(i); + i <<= lz; + + + // We are going to need to do some 64-bit arithmetic to get a precise product. + // We use a table lookup approach. + // It is safe because + // power >= smallest_power + // and power <= largest_power + // We recover the mantissa of the power, it has a leading 1. It is always + // rounded down. + // + // We want the most significant 64 bits of the product. We know + // this will be non-zero because the most significant bit of i is + // 1. + const uint32_t index = 2 * uint32_t(power - simdjson::internal::smallest_power); + // Optimization: It may be that materializing the index as a variable might confuse some compilers and prevent effective complex-addressing loads. (Done for code clarity.) + // + // The full_multiplication function computes the 128-bit product of two 64-bit words + // with a returned value of type value128 with a "low component" corresponding to the + // 64-bit least significant bits of the product and with a "high component" corresponding + // to the 64-bit most significant bits of the product. + simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index]); + // Both i and power_of_five_128[index] have their most significant bit set to 1 which + // implies that the either the most or the second most significant bit of the product + // is 1. We pack values in this manner for efficiency reasons: it maximizes the use + // we make of the product. It also makes it easy to reason about the product: there + // is 0 or 1 leading zero in the product. + + // Unless the least significant 9 bits of the high (64-bit) part of the full + // product are all 1s, then we know that the most significant 55 bits are + // exact and no further work is needed. Having 55 bits is necessary because + // we need 53 bits for the mantissa but we have to have one rounding bit and + // we can waste a bit if the most significant bit of the product is zero. + if ((firstproduct.high & 0x1FF) == 0x1FF) { + // We want to compute i * 5^q, but only care about the top 55 bits at most. + // Consider the scenario where q>=0. Then 5^q may not fit in 64-bits. Doing + // the full computation is wasteful. So we do what is called a "truncated + // multiplication". + // We take the most significant 64-bits, and we put them in + // power_of_five_128[index]. Usually, that's good enough to approximate i * 5^q + // to the desired approximation using one multiplication. Sometimes it does not suffice. + // Then we store the next most significant 64 bits in power_of_five_128[index + 1], and + // then we get a better approximation to i * 5^q. + // + // That's for when q>=0. The logic for q<0 is somewhat similar but it is somewhat + // more complicated. + // + // There is an extra layer of complexity in that we need more than 55 bits of + // accuracy in the round-to-even scenario. + // + // The full_multiplication function computes the 128-bit product of two 64-bit words + // with a returned value of type value128 with a "low component" corresponding to the + // 64-bit least significant bits of the product and with a "high component" corresponding + // to the 64-bit most significant bits of the product. + simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index + 1]); + firstproduct.low += secondproduct.high; + if (secondproduct.high > firstproduct.low) { firstproduct.high++; } + // As it has been proven by Noble Mushtak and Daniel Lemire in "Fast Number Parsing Without + // Fallback" (https://arxiv.org/abs/2212.06644), at this point we are sure that the product + // is sufficiently accurate, and more computation is not needed. + } + uint64_t lower = firstproduct.low; + uint64_t upper = firstproduct.high; + // The final mantissa should be 53 bits with a leading 1. + // We shift it so that it occupies 54 bits with a leading 1. + /////// + uint64_t upperbit = upper >> 63; + uint64_t mantissa = upper >> (upperbit + 9); + lz += int(1 ^ upperbit); + + // Here we have mantissa < (1<<54). + int64_t real_exponent = exponent - lz; + if (simdjson_unlikely(real_exponent <= 0)) { // we have a subnormal? + // Here have that real_exponent <= 0 so -real_exponent >= 0 + if (-real_exponent + 1 >= 64) { // if we have more than 64 bits below the minimum exponent, you have a zero for sure. + d = negative ? -0.0 : 0.0; + return true; + } + // next line is safe because -real_exponent + 1 < 0 + mantissa >>= -real_exponent + 1; + // Thankfully, we can't have both "round-to-even" and subnormals because + // "round-to-even" only occurs for powers close to 0. + mantissa += (mantissa & 1); // round up + mantissa >>= 1; + // There is a weird scenario where we don't have a subnormal but just. + // Suppose we start with 2.2250738585072013e-308, we end up + // with 0x3fffffffffffff x 2^-1023-53 which is technically subnormal + // whereas 0x40000000000000 x 2^-1023-53 is normal. Now, we need to round + // up 0x3fffffffffffff x 2^-1023-53 and once we do, we are no longer + // subnormal, but we can only know this after rounding. + // So we only declare a subnormal if we are smaller than the threshold. + real_exponent = (mantissa < (uint64_t(1) << 52)) ? 0 : 1; + d = to_double(mantissa, real_exponent, negative); + return true; + } + // We have to round to even. The "to even" part + // is only a problem when we are right in between two floats + // which we guard against. + // If we have lots of trailing zeros, we may fall right between two + // floating-point values. + // + // The round-to-even cases take the form of a number 2m+1 which is in (2^53,2^54] + // times a power of two. That is, it is right between a number with binary significand + // m and another number with binary significand m+1; and it must be the case + // that it cannot be represented by a float itself. + // + // We must have that w * 10 ^q == (2m+1) * 2^p for some power of two 2^p. + // Recall that 10^q = 5^q * 2^q. + // When q >= 0, we must have that (2m+1) is divible by 5^q, so 5^q <= 2^54. We have that + // 5^23 <= 2^54 and it is the last power of five to qualify, so q <= 23. + // When q<0, we have w >= (2m+1) x 5^{-q}. We must have that w<2^{64} so + // (2m+1) x 5^{-q} < 2^{64}. We have that 2m+1>2^{53}. Hence, we must have + // 2^{53} x 5^{-q} < 2^{64}. + // Hence we have 5^{-q} < 2^{11}$ or q>= -4. + // + // We require lower <= 1 and not lower == 0 because we could not prove that + // that lower == 0 is implied; but we could prove that lower <= 1 is a necessary and sufficient test. + if (simdjson_unlikely((lower <= 1) && (power >= -4) && (power <= 23) && ((mantissa & 3) == 1))) { + if ((mantissa << (upperbit + 64 - 53 - 2)) == upper) { + mantissa &= ~1; // flip it so that we do not round up + } + } + + mantissa += mantissa & 1; + mantissa >>= 1; + + // Here we have mantissa < (1<<53), unless there was an overflow + if (mantissa >= (1ULL << 53)) { + ////////// + // This will happen when parsing values such as 7.2057594037927933e+16 + //////// + mantissa = (1ULL << 52); + real_exponent++; + } + mantissa &= ~(1ULL << 52); + // we have to check that real_exponent is in range, otherwise we bail out + if (simdjson_unlikely(real_exponent > 2046)) { + // We have an infinite value!!! We could actually throw an error here if we could. + return false; + } + d = to_double(mantissa, real_exponent, negative); + return true; + } + + // We call a fallback floating-point parser that might be slow. Note + // it will accept JSON numbers, but the JSON spec. is more restrictive so + // before you call parse_float_fallback, you need to have validated the input + // string with the JSON grammar. + // It will return an error (false) if the parsed number is infinite. + // The string parsing itself always succeeds. We know that there is at least + // one digit. + static bool parse_float_fallback(const uint8_t* ptr, double* outDouble) { + *outDouble = simdjson::internal::from_chars(reinterpret_cast(ptr)); + // We do not accept infinite values. + + // Detecting finite values in a portable manner is ridiculously hard, ideally + // we would want to do: + // return !std::isfinite(*outDouble); + // but that mysteriously fails under legacy/old libc++ libraries, see + // https://github.com/simdjson/simdjson/issues/1286 + // + // Therefore, fall back to this solution (the extra parens are there + // to handle that max may be a macro on windows). + return !(*outDouble > (std::numeric_limits::max)() || *outDouble < std::numeric_limits::lowest()); + } + + static bool parse_float_fallback(const uint8_t* ptr, const uint8_t* end_ptr, double* outDouble) { + *outDouble = simdjson::internal::from_chars(reinterpret_cast(ptr), reinterpret_cast(end_ptr)); + // We do not accept infinite values. + + // Detecting finite values in a portable manner is ridiculously hard, ideally + // we would want to do: + // return !std::isfinite(*outDouble); + // but that mysteriously fails under legacy/old libc++ libraries, see + // https://github.com/simdjson/simdjson/issues/1286 + // + // Therefore, fall back to this solution (the extra parens are there + // to handle that max may be a macro on windows). + return !(*outDouble > (std::numeric_limits::max)() || *outDouble < std::numeric_limits::lowest()); + } + + // check quickly whether the next 8 chars are made of digits + // at a glance, it looks better than Mula's + // http://0x80.pl/articles/swar-digits-validate.html + simdjson_inline bool is_made_of_eight_digits_fast(const uint8_t* chars) { + uint64_t val; + // this can read up to 7 bytes beyond the buffer size, but we require + // SIMDJSON_PADDING of padding + static_assert(7 <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be bigger than 7"); + std::memcpy(&val, chars, 8); + // a branchy method might be faster: + // return (( val & 0xF0F0F0F0F0F0F0F0 ) == 0x3030303030303030) + // && (( (val + 0x0606060606060606) & 0xF0F0F0F0F0F0F0F0 ) == + // 0x3030303030303030); + return (((val & 0xF0F0F0F0F0F0F0F0) | + (((val + 0x0606060606060606) & 0xF0F0F0F0F0F0F0F0) >> 4)) == + 0x3333333333333333); + } + + template + SIMDJSON_NO_SANITIZE_UNDEFINED // We deliberately allow overflow here and check later + simdjson_inline bool parse_digit(const uint8_t c, I& i) { + const uint8_t digit = static_cast(c - '0'); + if (digit > 9) { + return false; + } + // PERF NOTE: multiplication by 10 is cheaper than arbitrary integer multiplication + i = 10 * i + digit; // might overflow, we will handle the overflow later + return true; + } + + simdjson_inline bool is_digit(const uint8_t c) { + return static_cast(c - '0') <= 9; + } + + simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t* const src, const uint8_t*& p, uint64_t& i, int64_t& exponent) { + // we continue with the fiction that we have an integer. If the + // floating point number is representable as x * 10^z for some integer + // z that fits in 53 bits, then we will be able to convert back the + // the integer into a float in a lossless manner. + const uint8_t* const first_after_period = p; + +#ifdef SIMDJSON_SWAR_NUMBER_PARSING +#if SIMDJSON_SWAR_NUMBER_PARSING + // this helps if we have lots of decimals! + // this turns out to be frequent enough. + if (is_made_of_eight_digits_fast(p)) { + i = i * 100000000 + parse_eight_digits_unrolled(p); + p += 8; + } +#endif // SIMDJSON_SWAR_NUMBER_PARSING +#endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING + // Unrolling the first digit makes a small difference on some implementations (e.g. westmere) + if (parse_digit(*p, i)) { ++p; } + while (parse_digit(*p, i)) { p++; } + exponent = first_after_period - p; + // Decimal without digits (123.) is illegal + if (exponent == 0) { + return INVALID_NUMBER(src); + } + return SUCCESS; + } + + simdjson_inline error_code parse_exponent(simdjson_unused const uint8_t* const src, const uint8_t*& p, int64_t& exponent) { + // Exp Sign: -123.456e[-]78 + bool neg_exp = ('-' == *p); + if (neg_exp || '+' == *p) { p++; } // Skip + as well + + // Exponent: -123.456e-[78] + auto start_exp = p; + int64_t exp_number = 0; + while (parse_digit(*p, exp_number)) { ++p; } + // It is possible for parse_digit to overflow. + // In particular, it could overflow to INT64_MIN, and we cannot do - INT64_MIN. + // Thus we *must* check for possible overflow before we negate exp_number. + + // Performance notes: it may seem like combining the two "simdjson_unlikely checks" below into + // a single simdjson_unlikely path would be faster. The reasoning is sound, but the compiler may + // not oblige and may, in fact, generate two distinct paths in any case. It might be + // possible to do uint64_t(p - start_exp - 1) >= 18 but it could end up trading off + // instructions for a simdjson_likely branch, an unconclusive gain. + + // If there were no digits, it's an error. + if (simdjson_unlikely(p == start_exp)) { + return INVALID_NUMBER(src); + } + // We have a valid positive exponent in exp_number at this point, except that + // it may have overflowed. + + // If there were more than 18 digits, we may have overflowed the integer. We have to do + // something!!!! + if (simdjson_unlikely(p > start_exp + 18)) { + // Skip leading zeroes: 1e000000000000000000001 is technically valid and does not overflow + while (*start_exp == '0') { start_exp++; } + // 19 digits could overflow int64_t and is kind of absurd anyway. We don't + // support exponents smaller than -999,999,999,999,999,999 and bigger + // than 999,999,999,999,999,999. + // We can truncate. + // Note that 999999999999999999 is assuredly too large. The maximal ieee64 value before + // infinity is ~1.8e308. The smallest subnormal is ~5e-324. So, actually, we could + // truncate at 324. + // Note that there is no reason to fail per se at this point in time. + // E.g., 0e999999999999999999999 is a fine number. + if (p > start_exp + 18) { exp_number = 999999999999999999; } + } + // At this point, we know that exp_number is a sane, positive, signed integer. + // It is <= 999,999,999,999,999,999. As long as 'exponent' is in + // [-8223372036854775808, 8223372036854775808], we won't overflow. Because 'exponent' + // is bounded in magnitude by the size of the JSON input, we are fine in this universe. + // To sum it up: the next line should never overflow. + exponent += (neg_exp ? -exp_number : exp_number); + return SUCCESS; + } + + simdjson_inline bool check_if_integer(const uint8_t* const src, size_t max_length) { + const uint8_t* const srcend = src + max_length; + bool negative = (*src == '-'); // we can always read at least one character after the '-' + const uint8_t* p = src + uint8_t(negative); + if (p == srcend) { return false; } + if (*p == '0') { + ++p; + if (p == srcend) { return true; } + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return false; } + return true; + } + while (p != srcend && is_digit(*p)) { ++p; } + if (p == srcend) { return true; } + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return false; } + return true; + } + + simdjson_inline size_t significant_digits(const uint8_t* start_digits, size_t digit_count) { + // It is possible that the integer had an overflow. + // We have to handle the case where we have 0.0000somenumber. + const uint8_t* start = start_digits; + while ((*start == '0') || (*start == '.')) { ++start; } + // we over-decrement by one when there is a '.' + return digit_count - size_t(start - start_digits); + } + + } // unnamed namespace + + /** @private */ + static error_code slow_float_parsing(simdjson_unused const uint8_t* src, double* answer) { + if (parse_float_fallback(src, answer)) { + return SUCCESS; + } + return INVALID_NUMBER(src); + } + + /** @private */ + template + simdjson_inline error_code write_float(const uint8_t* const src, bool negative, uint64_t i, const uint8_t* start_digits, size_t digit_count, int64_t exponent, W& writer) { + // If we frequently had to deal with long strings of digits, + // we could extend our code by using a 128-bit integer instead + // of a 64-bit integer. However, this is uncommon in practice. + // + // 9999999999999999999 < 2**64 so we can accommodate 19 digits. + // If we have a decimal separator, then digit_count - 1 is the number of digits, but we + // may not have a decimal separator! + if (simdjson_unlikely(digit_count > 19 && significant_digits(start_digits, digit_count) > 19)) { + // Ok, chances are good that we had an overflow! + // this is almost never going to get called!!! + // we start anew, going slowly!!! + // This will happen in the following examples: + // 10000000000000000000000000000000000000000000e+308 + // 3.1415926535897932384626433832795028841971693993751 + // + // NOTE: We do not pass a reference to the to slow_float_parsing. If we passed our writer + // reference to it, it would force it to be stored in memory, preventing the compiler from + // picking it apart and putting into registers. i.e. if we pass it as reference, + // it gets slow. + double d; + error_code error = slow_float_parsing(src, &d); + writer.append_double(d); + return error; + } + // NOTE: it's weird that the simdjson_unlikely() only wraps half the if, but it seems to get slower any other + // way we've tried: https://github.com/simdjson/simdjson/pull/990#discussion_r448497331 + // To future reader: we'd love if someone found a better way, or at least could explain this result! + if (simdjson_unlikely(exponent < simdjson::internal::smallest_power) || (exponent > simdjson::internal::largest_power)) { + // + // Important: smallest_power is such that it leads to a zero value. + // Observe that 18446744073709551615e-343 == 0, i.e. (2**64 - 1) e -343 is zero + // so something x 10^-343 goes to zero, but not so with something x 10^-342. + static_assert(simdjson::internal::smallest_power <= -342, "smallest_power is not small enough"); + // + if ((exponent < simdjson::internal::smallest_power) || (i == 0)) { + // E.g. Parse "-0.0e-999" into the same value as "-0.0". See https://en.wikipedia.org/wiki/Signed_zero + WRITE_DOUBLE(negative ? -0.0 : 0.0, src, writer); + return SUCCESS; + } + else { // (exponent > largest_power) and (i != 0) + // We have, for sure, an infinite value and simdjson refuses to parse infinite values. + return INVALID_NUMBER(src); + } + } + double d; + if (!compute_float_64(exponent, i, negative, d)) { + // we are almost never going to get here. + if (!parse_float_fallback(src, &d)) { return INVALID_NUMBER(src); } + } + WRITE_DOUBLE(d, src, writer); + return SUCCESS; + } + + // parse the number at src + // define JSON_TEST_NUMBERS for unit testing + // + // It is assumed that the number is followed by a structural ({,},],[) character + // or a white space character. If that is not the case (e.g., when the JSON + // document is made of a single number), then it is necessary to copy the + // content and append a space before calling this function. + // + // Our objective is accurate parsing (ULP of 0) at high speed. + template + simdjson_inline error_code parse_number(const uint8_t* const src, W& writer); + + // for performance analysis, it is sometimes useful to skip parsing +#ifdef SIMDJSON_SKIPNUMBERPARSING + + template + simdjson_inline error_code parse_number(const uint8_t* const, W& writer) { + writer.append_s64(0); // always write zero + return SUCCESS; // always succeeds + } + + simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t* const src) noexcept { return 0; } + simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t* const src) noexcept { return 0; } + simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t* const src) noexcept { return 0; } + simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t* const src) noexcept { return 0; } + simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t* const src) noexcept { return 0; } + simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t* const src) noexcept { return 0; } + simdjson_unused simdjson_inline bool is_negative(const uint8_t* src) noexcept { return false; } + simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t* src) noexcept { return false; } + simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t* src) noexcept { return number_type::signed_integer; } +#else + +// parse the number at src +// define JSON_TEST_NUMBERS for unit testing +// +// It is assumed that the number is followed by a structural ({,},],[) character +// or a white space character. If that is not the case (e.g., when the JSON +// document is made of a single number), then it is necessary to copy the +// content and append a space before calling this function. +// +// Our objective is accurate parsing (ULP of 0) at high speed. + template + simdjson_inline error_code parse_number(const uint8_t* const src, W& writer) { + + // + // Check for minus sign + // + bool negative = (*src == '-'); + const uint8_t* p = src + uint8_t(negative); + + // + // Parse the integer part. + // + // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare + const uint8_t* const start_digits = p; + uint64_t i = 0; + while (parse_digit(*p, i)) { p++; } + + // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. + // Optimization note: size_t is expected to be unsigned. + size_t digit_count = size_t(p - start_digits); + if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { return INVALID_NUMBER(src); } + + // + // Handle floats if there is a . or e (or both) + // + int64_t exponent = 0; + bool is_float = false; + if ('.' == *p) { + is_float = true; + ++p; + SIMDJSON_TRY(parse_decimal_after_separator(src, p, i, exponent)); + digit_count = int(p - start_digits); // used later to guard against overflows + } + if (('e' == *p) || ('E' == *p)) { + is_float = true; + ++p; + SIMDJSON_TRY(parse_exponent(src, p, exponent)); + } + if (is_float) { + const bool dirty_end = jsoncharutils::is_not_structural_or_whitespace(*p); + SIMDJSON_TRY(write_float(src, negative, i, start_digits, digit_count, exponent, writer)); + if (dirty_end) { return INVALID_NUMBER(src); } + return SUCCESS; + } + + // The longest negative 64-bit number is 19 digits. + // The longest positive 64-bit number is 20 digits. + // We do it this way so we don't trigger this branch unless we must. + size_t longest_digit_count = negative ? 19 : 20; + if (digit_count > longest_digit_count) { return BIGINT_NUMBER(src); } + if (digit_count == longest_digit_count) { + if (negative) { + // Anything negative above INT64_MAX+1 is invalid + if (i > uint64_t(INT64_MAX) + 1) { return BIGINT_NUMBER(src); } + WRITE_INTEGER(~i + 1, src, writer); + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return INVALID_NUMBER(src); } + return SUCCESS; + // Positive overflow check: + // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the + // biggest uint64_t. + // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX. + // If we got here, it's a 20 digit number starting with the digit "1". + // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller + // than 1,553,255,926,290,448,384. + // - That is smaller than the smallest possible 20-digit number the user could write: + // 10,000,000,000,000,000,000. + // - Therefore, if the number is positive and lower than that, it's overflow. + // - The value we are looking at is less than or equal to INT64_MAX. + // + } + else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INVALID_NUMBER(src); } + } + + // Write unsigned if it does not fit in a signed integer. + if (i > uint64_t(INT64_MAX)) { + WRITE_UNSIGNED(i, src, writer); + } + else { + WRITE_INTEGER(negative ? (~i + 1) : i, src, writer); + } + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return INVALID_NUMBER(src); } + return SUCCESS; + } + + // Inlineable functions + namespace { + + // This table can be used to characterize the final character of an integer + // string. For JSON structural character and allowable white space characters, + // we return SUCCESS. For 'e', '.' and 'E', we return INCORRECT_TYPE. Otherwise + // we return NUMBER_ERROR. + // Optimization note: we could easily reduce the size of the table by half (to 128) + // at the cost of an extra branch. + // Optimization note: we want the values to use at most 8 bits (not, e.g., 32 bits): + static_assert(error_code(uint8_t(NUMBER_ERROR)) == NUMBER_ERROR, "bad NUMBER_ERROR cast"); + static_assert(error_code(uint8_t(SUCCESS)) == SUCCESS, "bad NUMBER_ERROR cast"); + static_assert(error_code(uint8_t(INCORRECT_TYPE)) == INCORRECT_TYPE, "bad NUMBER_ERROR cast"); + + const uint8_t integer_string_finisher[256] = { + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, + SUCCESS, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, + NUMBER_ERROR, INCORRECT_TYPE, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, INCORRECT_TYPE, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, SUCCESS, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, INCORRECT_TYPE, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, + SUCCESS, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR }; + + // Parse any number from 0 to 18,446,744,073,709,551,615 + simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t* const src) noexcept { + const uint8_t* p = src; + // + // Parse the integer part. + // + // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare + const uint8_t* const start_digits = p; + uint64_t i = 0; + while (parse_digit(*p, i)) { p++; } + + // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. + // Optimization note: size_t is expected to be unsigned. + size_t digit_count = size_t(p - start_digits); + // The longest positive 64-bit number is 20 digits. + // We do it this way so we don't trigger this branch unless we must. + // Optimization note: the compiler can probably merge + // ((digit_count == 0) || (digit_count > 20)) + // into a single branch since digit_count is unsigned. + if ((digit_count == 0) || (digit_count > 20)) { return INCORRECT_TYPE; } + // Here digit_count > 0. + if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } + // We can do the following... + // if (!jsoncharutils::is_structural_or_whitespace(*p)) { + // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; + // } + // as a single table lookup: + if (integer_string_finisher[*p] != SUCCESS) { return error_code(integer_string_finisher[*p]); } + + if (digit_count == 20) { + // Positive overflow check: + // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the + // biggest uint64_t. + // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX. + // If we got here, it's a 20 digit number starting with the digit "1". + // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller + // than 1,553,255,926,290,448,384. + // - That is smaller than the smallest possible 20-digit number the user could write: + // 10,000,000,000,000,000,000. + // - Therefore, if the number is positive and lower than that, it's overflow. + // - The value we are looking at is less than or equal to INT64_MAX. + // + if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INCORRECT_TYPE; } + } + + return i; + } + + + // Parse any number from 0 to 18,446,744,073,709,551,615 + // Never read at src_end or beyond + simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t* const src, const uint8_t* const src_end) noexcept { + const uint8_t* p = src; + // + // Parse the integer part. + // + // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare + const uint8_t* const start_digits = p; + uint64_t i = 0; + while ((p != src_end) && parse_digit(*p, i)) { p++; } + + // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. + // Optimization note: size_t is expected to be unsigned. + size_t digit_count = size_t(p - start_digits); + // The longest positive 64-bit number is 20 digits. + // We do it this way so we don't trigger this branch unless we must. + // Optimization note: the compiler can probably merge + // ((digit_count == 0) || (digit_count > 20)) + // into a single branch since digit_count is unsigned. + if ((digit_count == 0) || (digit_count > 20)) { return INCORRECT_TYPE; } + // Here digit_count > 0. + if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } + // We can do the following... + // if (!jsoncharutils::is_structural_or_whitespace(*p)) { + // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; + // } + // as a single table lookup: + if ((p != src_end) && integer_string_finisher[*p] != SUCCESS) { return error_code(integer_string_finisher[*p]); } + + if (digit_count == 20) { + // Positive overflow check: + // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the + // biggest uint64_t. + // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX. + // If we got here, it's a 20 digit number starting with the digit "1". + // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller + // than 1,553,255,926,290,448,384. + // - That is smaller than the smallest possible 20-digit number the user could write: + // 10,000,000,000,000,000,000. + // - Therefore, if the number is positive and lower than that, it's overflow. + // - The value we are looking at is less than or equal to INT64_MAX. + // + if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INCORRECT_TYPE; } + } + + return i; + } + + // Parse any number from 0 to 18,446,744,073,709,551,615 + simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t* const src) noexcept { + const uint8_t* p = src + 1; + // + // Parse the integer part. + // + // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare + const uint8_t* const start_digits = p; + uint64_t i = 0; + while (parse_digit(*p, i)) { p++; } + + // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. + // Optimization note: size_t is expected to be unsigned. + size_t digit_count = size_t(p - start_digits); + // The longest positive 64-bit number is 20 digits. + // We do it this way so we don't trigger this branch unless we must. + // Optimization note: the compiler can probably merge + // ((digit_count == 0) || (digit_count > 20)) + // into a single branch since digit_count is unsigned. + if ((digit_count == 0) || (digit_count > 20)) { return INCORRECT_TYPE; } + // Here digit_count > 0. + if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } + // We can do the following... + // if (!jsoncharutils::is_structural_or_whitespace(*p)) { + // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; + // } + // as a single table lookup: + if (*p != '"') { return NUMBER_ERROR; } + + if (digit_count == 20) { + // Positive overflow check: + // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the + // biggest uint64_t. + // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX. + // If we got here, it's a 20 digit number starting with the digit "1". + // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller + // than 1,553,255,926,290,448,384. + // - That is smaller than the smallest possible 20-digit number the user could write: + // 10,000,000,000,000,000,000. + // - Therefore, if the number is positive and lower than that, it's overflow. + // - The value we are looking at is less than or equal to INT64_MAX. + // + // Note: we use src[1] and not src[0] because src[0] is the quote character in this + // instance. + if (src[1] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INCORRECT_TYPE; } + } + + return i; + } + + // Parse any number from -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 + simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t* src) noexcept { + // + // Check for minus sign + // + bool negative = (*src == '-'); + const uint8_t* p = src + uint8_t(negative); + + // + // Parse the integer part. + // + // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare + const uint8_t* const start_digits = p; + uint64_t i = 0; + while (parse_digit(*p, i)) { p++; } + + // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. + // Optimization note: size_t is expected to be unsigned. + size_t digit_count = size_t(p - start_digits); + // We go from + // -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 + // so we can never represent numbers that have more than 19 digits. + size_t longest_digit_count = 19; + // Optimization note: the compiler can probably merge + // ((digit_count == 0) || (digit_count > longest_digit_count)) + // into a single branch since digit_count is unsigned. + if ((digit_count == 0) || (digit_count > longest_digit_count)) { return INCORRECT_TYPE; } + // Here digit_count > 0. + if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } + // We can do the following... + // if (!jsoncharutils::is_structural_or_whitespace(*p)) { + // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; + // } + // as a single table lookup: + if (integer_string_finisher[*p] != SUCCESS) { return error_code(integer_string_finisher[*p]); } + // Negative numbers have can go down to - INT64_MAX - 1 whereas positive numbers are limited to INT64_MAX. + // Performance note: This check is only needed when digit_count == longest_digit_count but it is + // so cheap that we might as well always make it. + if (i > uint64_t(INT64_MAX) + uint64_t(negative)) { return INCORRECT_TYPE; } + return negative ? (~i + 1) : i; + } + + // Parse any number from -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 + // Never read at src_end or beyond + simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t* const src, const uint8_t* const src_end) noexcept { + // + // Check for minus sign + // + if (src == src_end) { return NUMBER_ERROR; } + bool negative = (*src == '-'); + const uint8_t* p = src + uint8_t(negative); + + // + // Parse the integer part. + // + // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare + const uint8_t* const start_digits = p; + uint64_t i = 0; + while ((p != src_end) && parse_digit(*p, i)) { p++; } + + // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. + // Optimization note: size_t is expected to be unsigned. + size_t digit_count = size_t(p - start_digits); + // We go from + // -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 + // so we can never represent numbers that have more than 19 digits. + size_t longest_digit_count = 19; + // Optimization note: the compiler can probably merge + // ((digit_count == 0) || (digit_count > longest_digit_count)) + // into a single branch since digit_count is unsigned. + if ((digit_count == 0) || (digit_count > longest_digit_count)) { return INCORRECT_TYPE; } + // Here digit_count > 0. + if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } + // We can do the following... + // if (!jsoncharutils::is_structural_or_whitespace(*p)) { + // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; + // } + // as a single table lookup: + if ((p != src_end) && integer_string_finisher[*p] != SUCCESS) { return error_code(integer_string_finisher[*p]); } + // Negative numbers have can go down to - INT64_MAX - 1 whereas positive numbers are limited to INT64_MAX. + // Performance note: This check is only needed when digit_count == longest_digit_count but it is + // so cheap that we might as well always make it. + if (i > uint64_t(INT64_MAX) + uint64_t(negative)) { return INCORRECT_TYPE; } + return negative ? (~i + 1) : i; + } + + // Parse any number from -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 + simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t* src) noexcept { + // + // Check for minus sign + // + bool negative = (*(src + 1) == '-'); + src += uint8_t(negative) + 1; + + // + // Parse the integer part. + // + // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare + const uint8_t* const start_digits = src; + uint64_t i = 0; + while (parse_digit(*src, i)) { src++; } + + // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. + // Optimization note: size_t is expected to be unsigned. + size_t digit_count = size_t(src - start_digits); + // We go from + // -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 + // so we can never represent numbers that have more than 19 digits. + size_t longest_digit_count = 19; + // Optimization note: the compiler can probably merge + // ((digit_count == 0) || (digit_count > longest_digit_count)) + // into a single branch since digit_count is unsigned. + if ((digit_count == 0) || (digit_count > longest_digit_count)) { return INCORRECT_TYPE; } + // Here digit_count > 0. + if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } + // We can do the following... + // if (!jsoncharutils::is_structural_or_whitespace(*src)) { + // return (*src == '.' || *src == 'e' || *src == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; + // } + // as a single table lookup: + if (*src != '"') { return NUMBER_ERROR; } + // Negative numbers have can go down to - INT64_MAX - 1 whereas positive numbers are limited to INT64_MAX. + // Performance note: This check is only needed when digit_count == longest_digit_count but it is + // so cheap that we might as well always make it. + if (i > uint64_t(INT64_MAX) + uint64_t(negative)) { return INCORRECT_TYPE; } + return negative ? (~i + 1) : i; + } + + simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t* src) noexcept { + // + // Check for minus sign + // + bool negative = (*src == '-'); + src += uint8_t(negative); + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t* p = src; + p += parse_digit(*p, i); + bool leading_zero = (i == 0); + while (parse_digit(*p, i)) { p++; } + // no integer digits, or 0123 (zero must be solo) + if (p == src) { return INCORRECT_TYPE; } + if ((leading_zero && p != src + 1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely(*p == '.')) { + p++; + const uint8_t* start_decimal_digits = p; + if (!parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits + p++; + while (parse_digit(*p, i)) { p++; } + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p - src - 1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t* start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = p - start_digits > 19; + } + } + else { + overflow = p - src > 19; + } + + // + // Parse the exponent + // + if (*p == 'e' || *p == 'E') { + p++; + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t* start_exp_digits = p; + while (parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p - start_exp_digits == 0 || p - start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0 - exp : exp; + } + + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + double d; + if (simdjson_likely(!overflow)) { + if (compute_float_64(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), &d)) { + return NUMBER_ERROR; + } + return d; + } + + simdjson_unused simdjson_inline bool is_negative(const uint8_t* src) noexcept { + return (*src == '-'); + } + + simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t* src) noexcept { + bool negative = (*src == '-'); + src += uint8_t(negative); + const uint8_t* p = src; + while (static_cast(*p - '0') <= 9) { p++; } + if (p == src) { return NUMBER_ERROR; } + if (jsoncharutils::is_structural_or_whitespace(*p)) { return true; } + return false; + } + + simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t* src) noexcept { + bool negative = (*src == '-'); + src += uint8_t(negative); + const uint8_t* p = src; + while (static_cast(*p - '0') <= 9) { p++; } + size_t digit_count = size_t(p - src); + if (p == src) { return NUMBER_ERROR; } + if (jsoncharutils::is_structural_or_whitespace(*p)) { + static const uint8_t* smaller_big_integer = reinterpret_cast("9223372036854775808"); + // We have an integer. + if (simdjson_unlikely(digit_count > 20)) { + return number_type::big_integer; + } + // If the number is negative and valid, it must be a signed integer. + if (negative) { + if (simdjson_unlikely(digit_count > 19)) return number_type::big_integer; + if (simdjson_unlikely(digit_count == 19 && memcmp(src, smaller_big_integer, 19) > 0)) { + return number_type::big_integer; + } + return number_type::signed_integer; + } + // Let us check if we have a big integer (>=2**64). + static const uint8_t* two_to_sixtyfour = reinterpret_cast("18446744073709551616"); + if ((digit_count > 20) || (digit_count == 20 && memcmp(src, two_to_sixtyfour, 20) >= 0)) { + return number_type::big_integer; + } + // The number is positive and smaller than 18446744073709551616 (or 2**64). + // We want values larger or equal to 9223372036854775808 to be unsigned + // integers, and the other values to be signed integers. + if ((digit_count == 20) || (digit_count >= 19 && memcmp(src, smaller_big_integer, 19) >= 0)) { + return number_type::unsigned_integer; + } + return number_type::signed_integer; + } + // Hopefully, we have 'e' or 'E' or '.'. + return number_type::floating_point_number; + } + + // Never read at src_end or beyond + simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t* src, const uint8_t* const src_end) noexcept { + if (src == src_end) { return NUMBER_ERROR; } + // + // Check for minus sign + // + bool negative = (*src == '-'); + src += uint8_t(negative); + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t* p = src; + if (p == src_end) { return NUMBER_ERROR; } + p += parse_digit(*p, i); + bool leading_zero = (i == 0); + while ((p != src_end) && parse_digit(*p, i)) { p++; } + // no integer digits, or 0123 (zero must be solo) + if (p == src) { return INCORRECT_TYPE; } + if ((leading_zero && p != src + 1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely((p != src_end) && (*p == '.'))) { + p++; + const uint8_t* start_decimal_digits = p; + if ((p == src_end) || !parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits + p++; + while ((p != src_end) && parse_digit(*p, i)) { p++; } + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p - src - 1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t* start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = start_digits - src > 19; + } + } + else { + overflow = p - src > 19; + } + + // + // Parse the exponent + // + if ((p != src_end) && (*p == 'e' || *p == 'E')) { + p++; + if (p == src_end) { return NUMBER_ERROR; } + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t* start_exp_digits = p; + while ((p != src_end) && parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p - start_exp_digits == 0 || p - start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0 - exp : exp; + } + + if ((p != src_end) && jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + double d; + if (simdjson_likely(!overflow)) { + if (compute_float_64(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), src_end, &d)) { + return NUMBER_ERROR; + } + return d; + } + + simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t* src) noexcept { + // + // Check for minus sign + // + bool negative = (*(src + 1) == '-'); + src += uint8_t(negative) + 1; + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t* p = src; + p += parse_digit(*p, i); + bool leading_zero = (i == 0); + while (parse_digit(*p, i)) { p++; } + // no integer digits, or 0123 (zero must be solo) + if (p == src) { return INCORRECT_TYPE; } + if ((leading_zero && p != src + 1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely(*p == '.')) { + p++; + const uint8_t* start_decimal_digits = p; + if (!parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits + p++; + while (parse_digit(*p, i)) { p++; } + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p - src - 1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t* start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = p - start_digits > 19; + } + } + else { + overflow = p - src > 19; + } + + // + // Parse the exponent + // + if (*p == 'e' || *p == 'E') { + p++; + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t* start_exp_digits = p; + while (parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p - start_exp_digits == 0 || p - start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0 - exp : exp; + } + + if (*p != '"') { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + double d; + if (simdjson_likely(!overflow)) { + if (compute_float_64(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), &d)) { + return NUMBER_ERROR; + } + return d; + } + + } // unnamed namespace +#endif // SIMDJSON_SKIPNUMBERPARSING + + } // namespace numberparsing + + inline std::ostream& operator<<(std::ostream& out, number_type type) noexcept { + switch (type) { + case number_type::signed_integer: out << "integer in [-9223372036854775808,9223372036854775808)"; break; + case number_type::unsigned_integer: out << "unsigned integer in [9223372036854775808,18446744073709551616)"; break; + case number_type::floating_point_number: out << "floating-point number (binary64)"; break; + case number_type::big_integer: out << "big integer"; break; + default: SIMDJSON_UNREACHABLE(); + } + return out; + } + + } // namespace lasx +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_NUMBERPARSING_H +/* end file simdjson/generic/numberparsing.h for lasx */ + +/* including simdjson/generic/implementation_simdjson_result_base-inl.h for lasx: #include "simdjson/generic/implementation_simdjson_result_base-inl.h" */ +/* begin file simdjson/generic/implementation_simdjson_result_base-inl.h for lasx */ +#ifndef SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_INL_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_INL_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/implementation_simdjson_result_base.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lasx { + + // + // internal::implementation_simdjson_result_base inline implementation + // + + template + simdjson_inline void implementation_simdjson_result_base::tie(T& value, error_code& error) && noexcept { + error = this->second; + if (!error) { + value = std::forward>(*this).first; + } + } + + template + simdjson_warn_unused simdjson_inline error_code implementation_simdjson_result_base::get(T& value) && noexcept { + error_code error; + std::forward>(*this).tie(value, error); + return error; + } + + template + simdjson_inline error_code implementation_simdjson_result_base::error() const noexcept { + return this->second; + } + +#if SIMDJSON_EXCEPTIONS + + template + simdjson_inline T& implementation_simdjson_result_base::value() & noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return this->first; + } + + template + simdjson_inline T&& implementation_simdjson_result_base::value() && noexcept(false) { + return std::forward>(*this).take_value(); + } + + template + simdjson_inline T&& implementation_simdjson_result_base::take_value() && noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return std::forward(this->first); + } + + template + simdjson_inline implementation_simdjson_result_base::operator T && () && noexcept(false) { + return std::forward>(*this).take_value(); + } + +#endif // SIMDJSON_EXCEPTIONS + + template + simdjson_inline const T& implementation_simdjson_result_base::value_unsafe() const& noexcept { + return this->first; + } + + template + simdjson_inline T& implementation_simdjson_result_base::value_unsafe() & noexcept { + return this->first; + } + + template + simdjson_inline T&& implementation_simdjson_result_base::value_unsafe() && noexcept { + return std::forward(this->first); + } + + template + simdjson_inline implementation_simdjson_result_base::implementation_simdjson_result_base(T&& value, error_code error) noexcept + : first{ std::forward(value) }, second{ error } {} + template + simdjson_inline implementation_simdjson_result_base::implementation_simdjson_result_base(error_code error) noexcept + : implementation_simdjson_result_base(T{}, error) {} + template + simdjson_inline implementation_simdjson_result_base::implementation_simdjson_result_base(T&& value) noexcept + : implementation_simdjson_result_base(std::forward(value), SUCCESS) {} + + } // namespace lasx +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_INL_H +/* end file simdjson/generic/implementation_simdjson_result_base-inl.h for lasx */ +/* end file simdjson/generic/amalgamated.h for lasx */ +/* including simdjson/lasx/end.h: #include "simdjson/lasx/end.h" */ +/* begin file simdjson/lasx/end.h */ +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/lasx/base.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +#undef SIMDJSON_SKIP_BACKSLASH_SHORT_CIRCUIT +/* undefining SIMDJSON_IMPLEMENTATION from "lasx" */ +#undef SIMDJSON_IMPLEMENTATION +/* end file simdjson/lasx/end.h */ + +#endif // SIMDJSON_LASX_H +/* end file simdjson/lasx.h */ +/* including simdjson/lasx/implementation.h: #include */ +/* begin file simdjson/lasx/implementation.h */ +#ifndef SIMDJSON_LASX_IMPLEMENTATION_H +#define SIMDJSON_LASX_IMPLEMENTATION_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/implementation.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/instruction_set.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lasx { + + /** + * @private + */ + class implementation final : public simdjson::implementation { + public: + simdjson_inline implementation() : simdjson::implementation("lasx", "LoongArch ASX", internal::instruction_set::LASX) {} + simdjson_warn_unused error_code create_dom_parser_implementation( + size_t capacity, + size_t max_length, + std::unique_ptr& dst + ) const noexcept final; + simdjson_warn_unused error_code minify(const uint8_t* buf, size_t len, uint8_t* dst, size_t& dst_len) const noexcept final; + simdjson_warn_unused bool validate_utf8(const char* buf, size_t len) const noexcept final; + }; + + } // namespace lasx +} // namespace simdjson + +#endif // SIMDJSON_LASX_IMPLEMENTATION_H +/* end file simdjson/lasx/implementation.h */ + +/* including simdjson/lasx/begin.h: #include */ +/* begin file simdjson/lasx/begin.h */ +/* defining SIMDJSON_IMPLEMENTATION to "lasx" */ +#define SIMDJSON_IMPLEMENTATION lasx +/* including simdjson/lasx/base.h: #include "simdjson/lasx/base.h" */ +/* begin file simdjson/lasx/base.h */ +#ifndef SIMDJSON_LASX_BASE_H +#define SIMDJSON_LASX_BASE_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/base.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + /** + * Implementation for LASX. + */ + namespace lasx { + + class implementation; + + namespace { + namespace simd { + template struct simd8; + template struct simd8x64; + } // namespace simd + } // unnamed namespace + + } // namespace lasx +} // namespace simdjson + +#endif // SIMDJSON_LASX_BASE_H +/* end file simdjson/lasx/base.h */ +/* including simdjson/lasx/intrinsics.h: #include "simdjson/lasx/intrinsics.h" */ +/* begin file simdjson/lasx/intrinsics.h */ +#ifndef SIMDJSON_LASX_INTRINSICS_H +#define SIMDJSON_LASX_INTRINSICS_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/lasx/base.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +// This should be the correct header whether +// you use visual studio or other compilers. +#include + +static_assert(sizeof(__m256i) <= simdjson::SIMDJSON_PADDING, "insufficient padding for LoongArch ASX"); + +#endif // SIMDJSON_LASX_INTRINSICS_H +/* end file simdjson/lasx/intrinsics.h */ +/* including simdjson/lasx/bitmanipulation.h: #include "simdjson/lasx/bitmanipulation.h" */ +/* begin file simdjson/lasx/bitmanipulation.h */ +#ifndef SIMDJSON_LASX_BITMANIPULATION_H +#define SIMDJSON_LASX_BITMANIPULATION_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/lasx/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/lasx/intrinsics.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/lasx/bitmask.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lasx { + namespace { + + // We sometimes call trailing_zero on inputs that are zero, + // but the algorithms do not end up using the returned value. + // Sadly, sanitizers are not smart enough to figure it out. + SIMDJSON_NO_SANITIZE_UNDEFINED + // This function can be used safely even if not all bytes have been + // initialized. + // See issue https://github.com/simdjson/simdjson/issues/1965 + SIMDJSON_NO_SANITIZE_MEMORY + simdjson_inline int trailing_zeroes(uint64_t input_num) { + return __builtin_ctzll(input_num); + } + + /* result might be undefined when input_num is zero */ + simdjson_inline uint64_t clear_lowest_bit(uint64_t input_num) { + return input_num & (input_num - 1); + } + + /* result might be undefined when input_num is zero */ + simdjson_inline int leading_zeroes(uint64_t input_num) { + return __builtin_clzll(input_num); + } + + /* result might be undefined when input_num is zero */ + simdjson_inline int count_ones(uint64_t input_num) { + return __lasx_xvpickve2gr_w(__lasx_xvpcnt_d(__m256i(v4u64{ input_num, 0, 0, 0 })), 0); + } + + simdjson_inline bool add_overflow(uint64_t value1, uint64_t value2, uint64_t* result) { + return __builtin_uaddll_overflow(value1, value2, + reinterpret_cast(result)); + } + + } // unnamed namespace + } // namespace lasx +} // namespace simdjson + +#endif // SIMDJSON_LASX_BITMANIPULATION_H +/* end file simdjson/lasx/bitmanipulation.h */ +/* including simdjson/lasx/bitmask.h: #include "simdjson/lasx/bitmask.h" */ +/* begin file simdjson/lasx/bitmask.h */ +#ifndef SIMDJSON_LASX_BITMASK_H +#define SIMDJSON_LASX_BITMASK_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/lasx/base.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lasx { + namespace { + + // + // Perform a "cumulative bitwise xor," flipping bits each time a 1 is encountered. + // + // For example, prefix_xor(00100100) == 00011100 + // + simdjson_inline uint64_t prefix_xor(uint64_t bitmask) { + bitmask ^= bitmask << 1; + bitmask ^= bitmask << 2; + bitmask ^= bitmask << 4; + bitmask ^= bitmask << 8; + bitmask ^= bitmask << 16; + bitmask ^= bitmask << 32; + return bitmask; + } + + } // unnamed namespace + } // namespace lasx +} // namespace simdjson + +#endif +/* end file simdjson/lasx/bitmask.h */ +/* including simdjson/lasx/numberparsing_defs.h: #include "simdjson/lasx/numberparsing_defs.h" */ +/* begin file simdjson/lasx/numberparsing_defs.h */ +#ifndef SIMDJSON_LASX_NUMBERPARSING_DEFS_H +#define SIMDJSON_LASX_NUMBERPARSING_DEFS_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/lasx/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/lasx/intrinsics.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +#include + +namespace simdjson { + namespace lasx { + namespace numberparsing { + + // we don't have appropriate instructions, so let us use a scalar function + // credit: https://johnnylee-sde.github.io/Fast-numeric-string-to-int/ + /** @private */ + static simdjson_inline uint32_t parse_eight_digits_unrolled(const uint8_t* chars) { + uint64_t val; + std::memcpy(&val, chars, sizeof(uint64_t)); + val = (val & 0x0F0F0F0F0F0F0F0F) * 2561 >> 8; + val = (val & 0x00FF00FF00FF00FF) * 6553601 >> 16; + return uint32_t((val & 0x0000FFFF0000FFFF) * 42949672960001 >> 32); + } + + simdjson_inline internal::value128 full_multiplication(uint64_t value1, uint64_t value2) { + internal::value128 answer; + __uint128_t r = (static_cast<__uint128_t>(value1)) * value2; + answer.low = uint64_t(r); + answer.high = uint64_t(r >> 64); + return answer; + } + + } // namespace numberparsing + } // namespace lasx +} // namespace simdjson + +#define SIMDJSON_SWAR_NUMBER_PARSING 1 + +#endif // SIMDJSON_LASX_NUMBERPARSING_DEFS_H +/* end file simdjson/lasx/numberparsing_defs.h */ +/* including simdjson/lasx/simd.h: #include "simdjson/lasx/simd.h" */ +/* begin file simdjson/lasx/simd.h */ +#ifndef SIMDJSON_LASX_SIMD_H +#define SIMDJSON_LASX_SIMD_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/lasx/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/lasx/bitmanipulation.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/simdprune_tables.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lasx { + namespace { + namespace simd { + + // Forward-declared so they can be used by splat and friends. + template + struct base { + __m256i value; + + // Zero constructor + simdjson_inline base() : value{ __m256i() } {} + + // Conversion from SIMD register + simdjson_inline base(const __m256i _value) : value(_value) {} + + // Conversion to SIMD register + simdjson_inline operator const __m256i& () const { return this->value; } + simdjson_inline operator __m256i& () { return this->value; } + simdjson_inline operator const v32i8& () const { return (v32i8&)this->value; } + simdjson_inline operator v32i8& () { return (v32i8&)this->value; } + + // Bit operations + simdjson_inline Child operator|(const Child other) const { return __lasx_xvor_v(*this, other); } + simdjson_inline Child operator&(const Child other) const { return __lasx_xvand_v(*this, other); } + simdjson_inline Child operator^(const Child other) const { return __lasx_xvxor_v(*this, other); } + simdjson_inline Child bit_andnot(const Child other) const { return __lasx_xvandn_v(other, *this); } + simdjson_inline Child& operator|=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast | other; return *this_cast; } + simdjson_inline Child& operator&=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast & other; return *this_cast; } + simdjson_inline Child& operator^=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast ^ other; return *this_cast; } + }; + + // Forward-declared so they can be used by splat and friends. + template + struct simd8; + + template> + struct base8 : base> { + simdjson_inline base8() : base>() {} + simdjson_inline base8(const __m256i _value) : base>(_value) {} + + friend simdjson_really_inline Mask operator==(const simd8 lhs, const simd8 rhs) { return __lasx_xvseq_b(lhs, rhs); } + + static const int SIZE = sizeof(base>::value); + + template + simdjson_inline simd8 prev(const simd8 prev_chunk) const { + __m256i hi = __lasx_xvbsll_v(*this, N); + __m256i lo = __lasx_xvbsrl_v(*this, 16 - N); + __m256i tmp = __lasx_xvbsrl_v(prev_chunk, 16 - N); + lo = __lasx_xvpermi_q(lo, tmp, 0x21); + return __lasx_xvor_v(hi, lo); + } + }; + + // SIMD byte mask type (returned by things like eq and gt) + template<> + struct simd8 : base8 { + static simdjson_inline simd8 splat(bool _value) { return __lasx_xvreplgr2vr_b(uint8_t(-(!!_value))); } + + simdjson_inline simd8() : base8() {} + simdjson_inline simd8(const __m256i _value) : base8(_value) {} + // Splat constructor + simdjson_inline simd8(bool _value) : base8(splat(_value)) {} + + simdjson_inline int to_bitmask() const { + __m256i mask = __lasx_xvmskltz_b(*this); + return (__lasx_xvpickve2gr_w(mask, 4) << 16) | (__lasx_xvpickve2gr_w(mask, 0)); + } + simdjson_inline bool any() const { + __m256i v = __lasx_xvmsknz_b(*this); + return (0 == __lasx_xvpickve2gr_w(v, 0)) && (0 == __lasx_xvpickve2gr_w(v, 4)); + } + simdjson_inline simd8 operator~() const { return *this ^ true; } + }; + + template + struct base8_numeric : base8 { + static simdjson_inline simd8 splat(T _value) { + return __lasx_xvreplgr2vr_b(_value); + } + static simdjson_inline simd8 zero() { return __lasx_xvldi(0); } + static simdjson_inline simd8 load(const T values[32]) { + return __lasx_xvld(reinterpret_cast(values), 0); + } + // Repeat 16 values as many times as necessary (usually for lookup tables) + static simdjson_inline simd8 repeat_16( + T v0, T v1, T v2, T v3, T v4, T v5, T v6, T v7, + T v8, T v9, T v10, T v11, T v12, T v13, T v14, T v15 + ) { + return simd8( + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15, + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15 + ); + } + + simdjson_inline base8_numeric() : base8() {} + simdjson_inline base8_numeric(const __m256i _value) : base8(_value) {} + + // Store to array + simdjson_inline void store(T dst[32]) const { + return __lasx_xvst(*this, reinterpret_cast<__m256i*>(dst), 0); + } + + // Addition/subtraction are the same for signed and unsigned + simdjson_inline simd8 operator+(const simd8 other) const { return __lasx_xvadd_b(*this, other); } + simdjson_inline simd8 operator-(const simd8 other) const { return __lasx_xvsub_b(*this, other); } + simdjson_inline simd8& operator+=(const simd8 other) { *this = *this + other; return *static_cast*>(this); } + simdjson_inline simd8& operator-=(const simd8 other) { *this = *this - other; return *static_cast*>(this); } + + // Override to distinguish from bool version + simdjson_inline simd8 operator~() const { return *this ^ 0xFFu; } + + // Perform a lookup assuming the value is between 0 and 16 (undefined behavior for out of range values) + template + simdjson_inline simd8 lookup_16(simd8 lookup_table) const { + return __lasx_xvshuf_b(lookup_table, lookup_table, *this); + } + + // Copies to 'output" all bytes corresponding to a 0 in the mask (interpreted as a bitset). + // Passing a 0 value for mask would be equivalent to writing out every byte to output. + // Only the first 16 - count_ones(mask) bytes of the result are significant but 16 bytes + // get written. + template + simdjson_inline void compress(uint32_t mask, L* output) const { + using internal::thintable_epi8; + using internal::BitsSetTable256mul2; + using internal::pshufb_combine_table; + // this particular implementation was inspired by haswell + // lasx do it in 4 steps, first 8 bytes and then second 8 bytes... + uint8_t mask1 = uint8_t(mask); // least significant 8 bits + uint8_t mask2 = uint8_t(mask >> 8); // second significant 8 bits + uint8_t mask3 = uint8_t(mask >> 16); // ... + uint8_t mask4 = uint8_t(mask >> 24); // ... + // next line just loads the 64-bit values thintable_epi8[mask{1,2,3,4}] + // into a 256-bit register. + __m256i shufmask = { int64_t(thintable_epi8[mask1]), int64_t(thintable_epi8[mask2]) + 0x0808080808080808, int64_t(thintable_epi8[mask3]), int64_t(thintable_epi8[mask4]) + 0x0808080808080808 }; + // this is the version "nearly pruned" + __m256i pruned = __lasx_xvshuf_b(*this, *this, shufmask); + // we still need to put the pieces back together. + // we compute the popcount of the first words: + int pop1 = BitsSetTable256mul2[mask1]; + int pop2 = BitsSetTable256mul2[mask2]; + int pop3 = BitsSetTable256mul2[mask3]; + + // then load the corresponding mask + __m256i masklo = __lasx_xvldx(reinterpret_cast(reinterpret_cast(pshufb_combine_table)), pop1 * 8); + __m256i maskhi = __lasx_xvldx(reinterpret_cast(reinterpret_cast(pshufb_combine_table)), pop3 * 8); + __m256i compactmask = __lasx_xvpermi_q(maskhi, masklo, 0x20); + __m256i answer = __lasx_xvshuf_b(pruned, pruned, compactmask); + __lasx_xvst(answer, reinterpret_cast(output), 0); + uint64_t value3 = __lasx_xvpickve2gr_du(answer, 2); + uint64_t value4 = __lasx_xvpickve2gr_du(answer, 3); + uint64_t* pos = reinterpret_cast(reinterpret_cast(output) + 16 - (pop1 + pop2) / 2); + pos[0] = value3; + pos[1] = value4; + } + + template + simdjson_inline simd8 lookup_16( + L replace0, L replace1, L replace2, L replace3, + L replace4, L replace5, L replace6, L replace7, + L replace8, L replace9, L replace10, L replace11, + L replace12, L replace13, L replace14, L replace15) const { + return lookup_16(simd8::repeat_16( + replace0, replace1, replace2, replace3, + replace4, replace5, replace6, replace7, + replace8, replace9, replace10, replace11, + replace12, replace13, replace14, replace15 + )); + } + }; + + // Signed bytes + template<> + struct simd8 : base8_numeric { + simdjson_inline simd8() : base8_numeric() {} + simdjson_inline simd8(const __m256i _value) : base8_numeric(_value) {} + // Splat constructor + simdjson_inline simd8(int8_t _value) : simd8(splat(_value)) {} + // Array constructor + simdjson_inline simd8(const int8_t values[32]) : simd8(load(values)) {} + // Member-by-member initialization + simdjson_inline simd8( + int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, + int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15, + int8_t v16, int8_t v17, int8_t v18, int8_t v19, int8_t v20, int8_t v21, int8_t v22, int8_t v23, + int8_t v24, int8_t v25, int8_t v26, int8_t v27, int8_t v28, int8_t v29, int8_t v30, int8_t v31 + ) : simd8({ + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10,v11,v12,v13,v14,v15, + v16,v17,v18,v19,v20,v21,v22,v23, + v24,v25,v26,v27,v28,v29,v30,v31 + }) {} + // Repeat 16 values as many times as necessary (usually for lookup tables) + simdjson_inline static simd8 repeat_16( + int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, + int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 + ) { + return simd8( + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15, + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15 + ); + } + + // Order-sensitive comparisons + simdjson_inline simd8 max_val(const simd8 other) const { return __lasx_xvmax_b(*this, other); } + simdjson_inline simd8 min_val(const simd8 other) const { return __lasx_xvmin_b(*this, other); } + simdjson_inline simd8 operator>(const simd8 other) const { return __lasx_xvslt_b(other, *this); } + simdjson_inline simd8 operator<(const simd8 other) const { return __lasx_xvslt_b(*this, other); } + }; + + // Unsigned bytes + template<> + struct simd8 : base8_numeric { + simdjson_inline simd8() : base8_numeric() {} + simdjson_inline simd8(const __m256i _value) : base8_numeric(_value) {} + // Splat constructor + simdjson_inline simd8(uint8_t _value) : simd8(splat(_value)) {} + // Array constructor + simdjson_inline simd8(const uint8_t values[32]) : simd8(load(values)) {} + // Member-by-member initialization + simdjson_inline simd8( + uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, + uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15, + uint8_t v16, uint8_t v17, uint8_t v18, uint8_t v19, uint8_t v20, uint8_t v21, uint8_t v22, uint8_t v23, + uint8_t v24, uint8_t v25, uint8_t v26, uint8_t v27, uint8_t v28, uint8_t v29, uint8_t v30, uint8_t v31 + ) : simd8(__m256i(v32u8{ + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10,v11,v12,v13,v14,v15, + v16,v17,v18,v19,v20,v21,v22,v23, + v24,v25,v26,v27,v28,v29,v30,v31 + })) {} + // Repeat 16 values as many times as necessary (usually for lookup tables) + simdjson_inline static simd8 repeat_16( + uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, + uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 + ) { + return simd8( + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15, + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15 + ); + } + + // Saturated math + simdjson_inline simd8 saturating_add(const simd8 other) const { return __lasx_xvsadd_bu(*this, other); } + simdjson_inline simd8 saturating_sub(const simd8 other) const { return __lasx_xvssub_bu(*this, other); } + + // Order-specific operations + simdjson_inline simd8 max_val(const simd8 other) const { return __lasx_xvmax_bu(*this, other); } + simdjson_inline simd8 min_val(const simd8 other) const { return __lasx_xvmin_bu(other, *this); } + // Same as >, but only guarantees true is nonzero (< guarantees true = -1) + simdjson_inline simd8 gt_bits(const simd8 other) const { return this->saturating_sub(other); } + // Same as <, but only guarantees true is nonzero (< guarantees true = -1) + simdjson_inline simd8 lt_bits(const simd8 other) const { return other.saturating_sub(*this); } + simdjson_inline simd8 operator<=(const simd8 other) const { return other.max_val(*this) == other; } + simdjson_inline simd8 operator>=(const simd8 other) const { return other.min_val(*this) == other; } + simdjson_inline simd8 operator>(const simd8 other) const { return this->gt_bits(other).any_bits_set(); } + simdjson_inline simd8 operator<(const simd8 other) const { return this->lt_bits(other).any_bits_set(); } + + // Bit-specific operations + simdjson_inline simd8 bits_not_set() const { return *this == uint8_t(0); } + simdjson_inline simd8 bits_not_set(simd8 bits) const { return (*this & bits).bits_not_set(); } + simdjson_inline simd8 any_bits_set() const { return ~this->bits_not_set(); } + simdjson_inline simd8 any_bits_set(simd8 bits) const { return ~this->bits_not_set(bits); } + simdjson_inline bool is_ascii() const { + __m256i mask = __lasx_xvmskltz_b(*this); + return (0 == __lasx_xvpickve2gr_w(mask, 0)) && (0 == __lasx_xvpickve2gr_w(mask, 4)); + } + simdjson_inline bool bits_not_set_anywhere() const { + __m256i v = __lasx_xvmsknz_b(*this); + return (0 == __lasx_xvpickve2gr_w(v, 0)) && (0 == __lasx_xvpickve2gr_w(v, 4)); + } + simdjson_inline bool any_bits_set_anywhere() const { return !bits_not_set_anywhere(); } + simdjson_inline bool bits_not_set_anywhere(simd8 bits) const { + __m256i v = __lasx_xvmsknz_b(__lasx_xvand_v(*this, bits)); + return (0 == __lasx_xvpickve2gr_w(v, 0)) && (0 == __lasx_xvpickve2gr_w(v, 4)); + } + simdjson_inline bool any_bits_set_anywhere(simd8 bits) const { return !bits_not_set_anywhere(bits); } + template + simdjson_inline simd8 shr() const { return simd8(__lasx_xvsrli_b(*this, N)); } + template + simdjson_inline simd8 shl() const { return simd8(__lasx_xvslli_b(*this, N)); } + }; + + template + struct simd8x64 { + static constexpr int NUM_CHUNKS = 64 / sizeof(simd8); + static_assert(NUM_CHUNKS == 2, "LASX kernel should use two registers per 64-byte block."); + const simd8 chunks[NUM_CHUNKS]; + + simd8x64(const simd8x64& o) = delete; // no copy allowed + simd8x64& operator=(const simd8& other) = delete; // no assignment allowed + simd8x64() = delete; // no default constructor allowed + + simdjson_inline simd8x64(const simd8 chunk0, const simd8 chunk1) : chunks{ chunk0, chunk1 } {} + simdjson_inline simd8x64(const T ptr[64]) : chunks{ simd8::load(ptr), simd8::load(ptr + 32) } {} + + simdjson_inline uint64_t compress(uint64_t mask, T* output) const { + uint32_t mask1 = uint32_t(mask); + uint32_t mask2 = uint32_t(mask >> 32); + __m256i zcnt = __lasx_xvpcnt_w(__m256i(v4u64{ ~mask, 0, 0, 0 })); + uint64_t zcnt1 = __lasx_xvpickve2gr_wu(zcnt, 0); + uint64_t zcnt2 = __lasx_xvpickve2gr_wu(zcnt, 1); + // There should be a critical value which processes in scaler is faster. + if (zcnt1) + this->chunks[0].compress(mask1, output); + if (zcnt2) + this->chunks[1].compress(mask2, output + zcnt1); + return zcnt1 + zcnt2; + } + + simdjson_inline void store(T ptr[64]) const { + this->chunks[0].store(ptr + sizeof(simd8) * 0); + this->chunks[1].store(ptr + sizeof(simd8) * 1); + } + + simdjson_inline uint64_t to_bitmask() const { + __m256i mask0 = __lasx_xvmskltz_b(this->chunks[0]); + __m256i mask1 = __lasx_xvmskltz_b(this->chunks[1]); + __m256i mask_tmp = __lasx_xvpickve_w(mask0, 4); + __m256i tmp = __lasx_xvpickve_w(mask1, 4); + mask0 = __lasx_xvinsve0_w(mask0, mask1, 1); + mask_tmp = __lasx_xvinsve0_w(mask_tmp, tmp, 1); + return __lasx_xvpickve2gr_du(__lasx_xvpackev_h(mask_tmp, mask0), 0); + } + + simdjson_inline simd8 reduce_or() const { + return this->chunks[0] | this->chunks[1]; + } + + simdjson_inline uint64_t eq(const T m) const { + const simd8 mask = simd8::splat(m); + return simd8x64( + this->chunks[0] == mask, + this->chunks[1] == mask + ).to_bitmask(); + } + + simdjson_inline uint64_t eq(const simd8x64& other) const { + return simd8x64( + this->chunks[0] == other.chunks[0], + this->chunks[1] == other.chunks[1] + ).to_bitmask(); + } + + simdjson_inline uint64_t lteq(const T m) const { + const simd8 mask = simd8::splat(m); + return simd8x64( + this->chunks[0] <= mask, + this->chunks[1] <= mask + ).to_bitmask(); + } + }; // struct simd8x64 + + } // namespace simd + } // unnamed namespace + } // namespace lasx +} // namespace simdjson + +#endif // SIMDJSON_LASX_SIMD_H +/* end file simdjson/lasx/simd.h */ +/* including simdjson/lasx/stringparsing_defs.h: #include "simdjson/lasx/stringparsing_defs.h" */ +/* begin file simdjson/lasx/stringparsing_defs.h */ +#ifndef SIMDJSON_LASX_STRINGPARSING_DEFS_H +#define SIMDJSON_LASX_STRINGPARSING_DEFS_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/lasx/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/lasx/simd.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/lasx/bitmanipulation.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lasx { + namespace { + + using namespace simd; + + // Holds backslashes and quotes locations. + struct backslash_and_quote { + public: + static constexpr uint32_t BYTES_PROCESSED = 32; + simdjson_inline static backslash_and_quote copy_and_find(const uint8_t* src, uint8_t* dst); + + simdjson_inline bool has_quote_first() { return ((bs_bits - 1) & quote_bits) != 0; } + simdjson_inline bool has_backslash() { return bs_bits != 0; } + simdjson_inline int quote_index() { return trailing_zeroes(quote_bits); } + simdjson_inline int backslash_index() { return trailing_zeroes(bs_bits); } + + uint32_t bs_bits; + uint32_t quote_bits; + }; // struct backslash_and_quote + + simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8_t* src, uint8_t* dst) { + // this can read up to 31 bytes beyond the buffer size, but we require + // SIMDJSON_PADDING of padding + static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "backslash and quote finder must process fewer than SIMDJSON_PADDING bytes"); + simd8 v(src); + v.store(dst); + return { + static_cast((v == '\\').to_bitmask()), // bs_bits + static_cast((v == '"').to_bitmask()), // quote_bits + }; + } + + } // unnamed namespace + } // namespace lasx +} // namespace simdjson + +#endif // SIMDJSON_LASX_STRINGPARSING_DEFS_H +/* end file simdjson/lasx/stringparsing_defs.h */ + +#define SIMDJSON_SKIP_BACKSLASH_SHORT_CIRCUIT 1 +/* end file simdjson/lasx/begin.h */ +/* including generic/amalgamated.h for lasx: #include */ +/* begin file generic/amalgamated.h for lasx */ +#if defined(SIMDJSON_CONDITIONAL_INCLUDE) && !defined(SIMDJSON_SRC_GENERIC_DEPENDENCIES_H) +#error generic/dependencies.h must be included before generic/amalgamated.h! +#endif + +/* including generic/base.h for lasx: #include */ +/* begin file generic/base.h for lasx */ +#ifndef SIMDJSON_SRC_GENERIC_BASE_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_BASE_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lasx { + namespace { + + struct json_character_block; + + } // unnamed namespace + } // namespace lasx +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_BASE_H +/* end file generic/base.h for lasx */ +/* including generic/dom_parser_implementation.h for lasx: #include */ +/* begin file generic/dom_parser_implementation.h for lasx */ +#ifndef SIMDJSON_SRC_GENERIC_DOM_PARSER_IMPLEMENTATION_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_DOM_PARSER_IMPLEMENTATION_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +// Interface a dom parser implementation must fulfill +namespace simdjson { + namespace lasx { + namespace { + + simdjson_inline simd8 must_be_2_3_continuation(const simd8 prev2, const simd8 prev3); + simdjson_inline bool is_ascii(const simd8x64& input); + + } // unnamed namespace + } // namespace lasx +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_DOM_PARSER_IMPLEMENTATION_H +/* end file generic/dom_parser_implementation.h for lasx */ +/* including generic/json_character_block.h for lasx: #include */ +/* begin file generic/json_character_block.h for lasx */ +#ifndef SIMDJSON_SRC_GENERIC_JSON_CHARACTER_BLOCK_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_JSON_CHARACTER_BLOCK_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lasx { + namespace { + + struct json_character_block { + static simdjson_inline json_character_block classify(const simd::simd8x64& in); + + simdjson_inline uint64_t whitespace() const noexcept { return _whitespace; } + simdjson_inline uint64_t op() const noexcept { return _op; } + simdjson_inline uint64_t scalar() const noexcept { return ~(op() | whitespace()); } + + uint64_t _whitespace; + uint64_t _op; + }; + + } // unnamed namespace + } // namespace lasx +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_JSON_CHARACTER_BLOCK_H +/* end file generic/json_character_block.h for lasx */ +/* end file generic/amalgamated.h for lasx */ +/* including generic/stage1/amalgamated.h for lasx: #include */ +/* begin file generic/stage1/amalgamated.h for lasx */ +// Stuff other things depend on +/* including generic/stage1/base.h for lasx: #include */ +/* begin file generic/stage1/base.h for lasx */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE1_BASE_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_BASE_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lasx { + namespace { + namespace stage1 { + + class bit_indexer; + template + struct buf_block_reader; + struct json_block; + class json_minifier; + class json_scanner; + struct json_string_block; + class json_string_scanner; + class json_structural_indexer; + + } // namespace stage1 + + namespace utf8_validation { + struct utf8_checker; + } // namespace utf8_validation + + using utf8_validation::utf8_checker; + + } // unnamed namespace + } // namespace lasx +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE1_BASE_H +/* end file generic/stage1/base.h for lasx */ +/* including generic/stage1/buf_block_reader.h for lasx: #include */ +/* begin file generic/stage1/buf_block_reader.h for lasx */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE1_BUF_BLOCK_READER_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_BUF_BLOCK_READER_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +#include + +namespace simdjson { + namespace lasx { + namespace { + namespace stage1 { + + // Walks through a buffer in block-sized increments, loading the last part with spaces + template + struct buf_block_reader { + public: + simdjson_inline buf_block_reader(const uint8_t* _buf, size_t _len); + simdjson_inline size_t block_index(); + simdjson_inline bool has_full_block() const; + simdjson_inline const uint8_t* full_block() const; + /** + * Get the last block, padded with spaces. + * + * There will always be a last block, with at least 1 byte, unless len == 0 (in which case this + * function fills the buffer with spaces and returns 0. In particular, if len == STEP_SIZE there + * will be 0 full_blocks and 1 remainder block with STEP_SIZE bytes and no spaces for padding. + * + * @return the number of effective characters in the last block. + */ + simdjson_inline size_t get_remainder(uint8_t* dst) const; + simdjson_inline void advance(); + private: + const uint8_t* buf; + const size_t len; + const size_t lenminusstep; + size_t idx; + }; + + // Routines to print masks and text for debugging bitmask operations + simdjson_unused static char* format_input_text_64(const uint8_t* text) { + static char buf[sizeof(simd8x64) + 1]; + for (size_t i = 0; i < sizeof(simd8x64); i++) { + buf[i] = int8_t(text[i]) < ' ' ? '_' : int8_t(text[i]); + } + buf[sizeof(simd8x64)] = '\0'; + return buf; + } + + // Routines to print masks and text for debugging bitmask operations + simdjson_unused static char* format_input_text(const simd8x64& in) { + static char buf[sizeof(simd8x64) + 1]; + in.store(reinterpret_cast(buf)); + for (size_t i = 0; i < sizeof(simd8x64); i++) { + if (buf[i] < ' ') { buf[i] = '_'; } + } + buf[sizeof(simd8x64)] = '\0'; + return buf; + } + + simdjson_unused static char* format_input_text(const simd8x64& in, uint64_t mask) { + static char buf[sizeof(simd8x64) + 1]; + in.store(reinterpret_cast(buf)); + for (size_t i = 0; i < sizeof(simd8x64); i++) { + if (buf[i] <= ' ') { buf[i] = '_'; } + if (!(mask & (size_t(1) << i))) { buf[i] = ' '; } + } + buf[sizeof(simd8x64)] = '\0'; + return buf; + } + + simdjson_unused static char* format_mask(uint64_t mask) { + static char buf[sizeof(simd8x64) + 1]; + for (size_t i = 0; i < 64; i++) { + buf[i] = (mask & (size_t(1) << i)) ? 'X' : ' '; + } + buf[64] = '\0'; + return buf; + } + + template + simdjson_inline buf_block_reader::buf_block_reader(const uint8_t* _buf, size_t _len) : buf{ _buf }, len{ _len }, lenminusstep{ len < STEP_SIZE ? 0 : len - STEP_SIZE }, idx{ 0 } {} + + template + simdjson_inline size_t buf_block_reader::block_index() { return idx; } + + template + simdjson_inline bool buf_block_reader::has_full_block() const { + return idx < lenminusstep; + } + + template + simdjson_inline const uint8_t* buf_block_reader::full_block() const { + return &buf[idx]; + } + + template + simdjson_inline size_t buf_block_reader::get_remainder(uint8_t* dst) const { + if (len == idx) { return 0; } // memcpy(dst, null, 0) will trigger an error with some sanitizers + std::memset(dst, 0x20, STEP_SIZE); // std::memset STEP_SIZE because it's more efficient to write out 8 or 16 bytes at once. + std::memcpy(dst, buf + idx, len - idx); + return len - idx; + } + + template + simdjson_inline void buf_block_reader::advance() { + idx += STEP_SIZE; + } + + } // namespace stage1 + } // unnamed namespace + } // namespace lasx +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE1_BUF_BLOCK_READER_H +/* end file generic/stage1/buf_block_reader.h for lasx */ +/* including generic/stage1/json_escape_scanner.h for lasx: #include */ +/* begin file generic/stage1/json_escape_scanner.h for lasx */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE1_JSON_ESCAPE_SCANNER_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_JSON_ESCAPE_SCANNER_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lasx { + namespace { + namespace stage1 { + + /** + * Scans for escape characters in JSON, taking care with multiple backslashes (\\n vs. \n). + */ + struct json_escape_scanner { + /** The actual escape characters (the backslashes themselves). */ + uint64_t next_is_escaped = 0ULL; + + struct escaped_and_escape { + /** + * Mask of escaped characters. + * + * ``` + * \n \\n \\\n \\\\n \ + * 0100100010100101000 + * n \ \ n \ \ + * ``` + */ + uint64_t escaped; + /** + * Mask of escape characters. + * + * ``` + * \n \\n \\\n \\\\n \ + * 1001000101001010001 + * \ \ \ \ \ \ \ + * ``` + */ + uint64_t escape; + }; + + /** + * Get a mask of both escape and escaped characters (the characters following a backslash). + * + * @param potential_escape A mask of the character that can escape others (but could be + * escaped itself). e.g. block.eq('\\') + */ + simdjson_really_inline escaped_and_escape next(uint64_t backslash) noexcept { + +#if !SIMDJSON_SKIP_BACKSLASH_SHORT_CIRCUIT + if (!backslash) { return { next_escaped_without_backslashes(), 0 }; } +#endif + + // | | Mask (shows characters instead of 1's) | Depth | Instructions | + // |--------------------------------|----------------------------------------|-------|---------------------| + // | string | `\\n_\\\n___\\\n___\\\\___\\\\__\\\` | | | + // | | ` even odd even odd odd` | | | + // | potential_escape | ` \ \\\ \\\ \\\\ \\\\ \\\` | 1 | 1 (backslash & ~first_is_escaped) + // | escape_and_terminal_code | ` \n \ \n \ \n \ \ \ \ \ \` | 5 | 5 (next_escape_and_terminal_code()) + // | escaped | `\ \ n \ n \ \ \ \ \ ` X | 6 | 7 (escape_and_terminal_code ^ (potential_escape | first_is_escaped)) + // | escape | ` \ \ \ \ \ \ \ \ \ \` | 6 | 8 (escape_and_terminal_code & backslash) + // | first_is_escaped | `\ ` | 7 (*) | 9 (escape >> 63) () + // (*) this is not needed until the next iteration + uint64_t escape_and_terminal_code = next_escape_and_terminal_code(backslash & ~this->next_is_escaped); + uint64_t escaped = escape_and_terminal_code ^ (backslash | this->next_is_escaped); + uint64_t escape = escape_and_terminal_code & backslash; + this->next_is_escaped = escape >> 63; + return { escaped, escape }; + } + + private: + static constexpr const uint64_t ODD_BITS = 0xAAAAAAAAAAAAAAAAULL; + + simdjson_really_inline uint64_t next_escaped_without_backslashes() noexcept { + uint64_t escaped = this->next_is_escaped; + this->next_is_escaped = 0; + return escaped; + } + + /** + * Returns a mask of the next escape characters (masking out escaped backslashes), along with + * any non-backslash escape codes. + * + * \n \\n \\\n \\\\n returns: + * \n \ \ \n \ \ + * 11 100 1011 10100 + * + * You are expected to mask out the first bit yourself if the previous block had a trailing + * escape. + * + * & the result with potential_escape to get just the escape characters. + * ^ the result with (potential_escape | first_is_escaped) to get escaped characters. + */ + static simdjson_really_inline uint64_t next_escape_and_terminal_code(uint64_t potential_escape) noexcept { + // If we were to just shift and mask out any odd bits, we'd actually get a *half* right answer: + // any even-aligned backslash runs would be correct! Odd-aligned backslash runs would be + // inverted (\\\ would be 010 instead of 101). + // + // ``` + // string: | ____\\\\_\\\\_____ | + // maybe_escaped | ODD | \ \ \ \ | + // even-aligned ^^^ ^^^^ odd-aligned + // ``` + // + // Taking that into account, our basic strategy is: + // + // 1. Use subtraction to produce a mask with 1's for even-aligned runs and 0's for + // odd-aligned runs. + // 2. XOR all odd bits, which masks out the odd bits in even-aligned runs, and brings IN the + // odd bits in odd-aligned runs. + // 3. & with backslash to clean up any stray bits. + // runs are set to 0, and then XORing with "odd": + // + // | | Mask (shows characters instead of 1's) | Instructions | + // |--------------------------------|----------------------------------------|---------------------| + // | string | `\\n_\\\n___\\\n___\\\\___\\\\__\\\` | + // | | ` even odd even odd odd` | + // | maybe_escaped | ` n \\n \\n \\\_ \\\_ \\` X | 1 (potential_escape << 1) + // | maybe_escaped_and_odd | ` \n_ \\n _ \\\n_ _ \\\__ _\\\_ \\\` | 1 (maybe_escaped | odd) + // | even_series_codes_and_odd | ` n_\\\ _ n_ _\\\\ _ _ ` | 1 (maybe_escaped_and_odd - potential_escape) + // | escape_and_terminal_code | ` \n \ \n \ \n \ \ \ \ \ \` | 1 (^ odd) + // + + // Escaped characters are characters following an escape. + uint64_t maybe_escaped = potential_escape << 1; + + // To distinguish odd from even escape sequences, therefore, we turn on any *starting* + // escapes that are on an odd byte. (We actually bring in all odd bits, for speed.) + // - Odd runs of backslashes are 0000, and the code at the end ("n" in \n or \\n) is 1. + // - Odd runs of backslashes are 1111, and the code at the end ("n" in \n or \\n) is 0. + // - All other odd bytes are 1, and even bytes are 0. + uint64_t maybe_escaped_and_odd_bits = maybe_escaped | ODD_BITS; + uint64_t even_series_codes_and_odd_bits = maybe_escaped_and_odd_bits - potential_escape; + + // Now we flip all odd bytes back with xor. This: + // - Makes odd runs of backslashes go from 0000 to 1010 + // - Makes even runs of backslashes go from 1111 to 1010 + // - Sets actually-escaped codes to 1 (the n in \n and \\n: \n = 11, \\n = 100) + // - Resets all other bytes to 0 + return even_series_codes_and_odd_bits ^ ODD_BITS; + } + }; + + } // namespace stage1 + } // unnamed namespace + } // namespace lasx +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRING_SCANNER_H +/* end file generic/stage1/json_escape_scanner.h for lasx */ +/* including generic/stage1/json_string_scanner.h for lasx: #include */ +/* begin file generic/stage1/json_string_scanner.h for lasx */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRING_SCANNER_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRING_SCANNER_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lasx { + namespace { + namespace stage1 { + + struct json_string_block { + // We spell out the constructors in the hope of resolving inlining issues with Visual Studio 2017 + simdjson_really_inline json_string_block(uint64_t escaped, uint64_t quote, uint64_t in_string) : + _escaped(escaped), _quote(quote), _in_string(in_string) {} + + // Escaped characters (characters following an escape() character) + simdjson_really_inline uint64_t escaped() const { return _escaped; } + // Real (non-backslashed) quotes + simdjson_really_inline uint64_t quote() const { return _quote; } + // Only characters inside the string (not including the quotes) + simdjson_really_inline uint64_t string_content() const { return _in_string & ~_quote; } + // Return a mask of whether the given characters are inside a string (only works on non-quotes) + simdjson_really_inline uint64_t non_quote_inside_string(uint64_t mask) const { return mask & _in_string; } + // Return a mask of whether the given characters are inside a string (only works on non-quotes) + simdjson_really_inline uint64_t non_quote_outside_string(uint64_t mask) const { return mask & ~_in_string; } + // Tail of string (everything except the start quote) + simdjson_really_inline uint64_t string_tail() const { return _in_string ^ _quote; } + + // escaped characters (backslashed--does not include the hex characters after \u) + uint64_t _escaped; + // real quotes (non-escaped ones) + uint64_t _quote; + // string characters (includes start quote but not end quote) + uint64_t _in_string; + }; + + // Scans blocks for string characters, storing the state necessary to do so + class json_string_scanner { + public: + simdjson_really_inline json_string_block next(const simd::simd8x64& in); + // Returns either UNCLOSED_STRING or SUCCESS + simdjson_really_inline error_code finish(); + + private: + // Scans for escape characters + json_escape_scanner escape_scanner{}; + // Whether the last iteration was still inside a string (all 1's = true, all 0's = false). + uint64_t prev_in_string = 0ULL; + }; + + // + // Return a mask of all string characters plus end quotes. + // + // prev_escaped is overflow saying whether the next character is escaped. + // prev_in_string is overflow saying whether we're still in a string. + // + // Backslash sequences outside of quotes will be detected in stage 2. + // + simdjson_really_inline json_string_block json_string_scanner::next(const simd::simd8x64& in) { + const uint64_t backslash = in.eq('\\'); + const uint64_t escaped = escape_scanner.next(backslash).escaped; + const uint64_t quote = in.eq('"') & ~escaped; + + // + // prefix_xor flips on bits inside the string (and flips off the end quote). + // + // Then we xor with prev_in_string: if we were in a string already, its effect is flipped + // (characters inside strings are outside, and characters outside strings are inside). + // + const uint64_t in_string = prefix_xor(quote) ^ prev_in_string; + + // + // Check if we're still in a string at the end of the box so the next block will know + // + prev_in_string = uint64_t(static_cast(in_string) >> 63); + + // Use ^ to turn the beginning quote off, and the end quote on. + + // We are returning a function-local object so either we get a move constructor + // or we get copy elision. + return json_string_block(escaped, quote, in_string); + } + + simdjson_really_inline error_code json_string_scanner::finish() { + if (prev_in_string) { + return UNCLOSED_STRING; + } + return SUCCESS; + } + + } // namespace stage1 + } // unnamed namespace + } // namespace lasx +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRING_SCANNER_H +/* end file generic/stage1/json_string_scanner.h for lasx */ +/* including generic/stage1/utf8_lookup4_algorithm.h for lasx: #include */ +/* begin file generic/stage1/utf8_lookup4_algorithm.h for lasx */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE1_UTF8_LOOKUP4_ALGORITHM_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_UTF8_LOOKUP4_ALGORITHM_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lasx { + namespace { + namespace utf8_validation { + + using namespace simd; + + simdjson_inline simd8 check_special_cases(const simd8 input, const simd8 prev1) { + // Bit 0 = Too Short (lead byte/ASCII followed by lead byte/ASCII) + // Bit 1 = Too Long (ASCII followed by continuation) + // Bit 2 = Overlong 3-byte + // Bit 4 = Surrogate + // Bit 5 = Overlong 2-byte + // Bit 7 = Two Continuations + constexpr const uint8_t TOO_SHORT = 1 << 0; // 11______ 0_______ + // 11______ 11______ + constexpr const uint8_t TOO_LONG = 1 << 1; // 0_______ 10______ + constexpr const uint8_t OVERLONG_3 = 1 << 2; // 11100000 100_____ + constexpr const uint8_t SURROGATE = 1 << 4; // 11101101 101_____ + constexpr const uint8_t OVERLONG_2 = 1 << 5; // 1100000_ 10______ + constexpr const uint8_t TWO_CONTS = 1 << 7; // 10______ 10______ + constexpr const uint8_t TOO_LARGE = 1 << 3; // 11110100 1001____ + // 11110100 101_____ + // 11110101 1001____ + // 11110101 101_____ + // 1111011_ 1001____ + // 1111011_ 101_____ + // 11111___ 1001____ + // 11111___ 101_____ + constexpr const uint8_t TOO_LARGE_1000 = 1 << 6; + // 11110101 1000____ + // 1111011_ 1000____ + // 11111___ 1000____ + constexpr const uint8_t OVERLONG_4 = 1 << 6; // 11110000 1000____ + + const simd8 byte_1_high = prev1.shr<4>().lookup_16( + // 0_______ ________ + TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, + TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, + // 10______ ________ + TWO_CONTS, TWO_CONTS, TWO_CONTS, TWO_CONTS, + // 1100____ ________ + TOO_SHORT | OVERLONG_2, + // 1101____ ________ + TOO_SHORT, + // 1110____ ________ + TOO_SHORT | OVERLONG_3 | SURROGATE, + // 1111____ ________ + TOO_SHORT | TOO_LARGE | TOO_LARGE_1000 | OVERLONG_4 + ); + constexpr const uint8_t CARRY = TOO_SHORT | TOO_LONG | TWO_CONTS; // These all have ____ in byte 1 . + const simd8 byte_1_low = (prev1 & 0x0F).lookup_16( + // ____0000 ________ + CARRY | OVERLONG_3 | OVERLONG_2 | OVERLONG_4, + // ____0001 ________ + CARRY | OVERLONG_2, + // ____001_ ________ + CARRY, + CARRY, + + // ____0100 ________ + CARRY | TOO_LARGE, + // ____0101 ________ + CARRY | TOO_LARGE | TOO_LARGE_1000, + // ____011_ ________ + CARRY | TOO_LARGE | TOO_LARGE_1000, + CARRY | TOO_LARGE | TOO_LARGE_1000, + + // ____1___ ________ + CARRY | TOO_LARGE | TOO_LARGE_1000, + CARRY | TOO_LARGE | TOO_LARGE_1000, + CARRY | TOO_LARGE | TOO_LARGE_1000, + CARRY | TOO_LARGE | TOO_LARGE_1000, + CARRY | TOO_LARGE | TOO_LARGE_1000, + // ____1101 ________ + CARRY | TOO_LARGE | TOO_LARGE_1000 | SURROGATE, + CARRY | TOO_LARGE | TOO_LARGE_1000, + CARRY | TOO_LARGE | TOO_LARGE_1000 + ); + const simd8 byte_2_high = input.shr<4>().lookup_16( + // ________ 0_______ + TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT, + TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT, + + // ________ 1000____ + TOO_LONG | OVERLONG_2 | TWO_CONTS | OVERLONG_3 | TOO_LARGE_1000 | OVERLONG_4, + // ________ 1001____ + TOO_LONG | OVERLONG_2 | TWO_CONTS | OVERLONG_3 | TOO_LARGE, + // ________ 101_____ + TOO_LONG | OVERLONG_2 | TWO_CONTS | SURROGATE | TOO_LARGE, + TOO_LONG | OVERLONG_2 | TWO_CONTS | SURROGATE | TOO_LARGE, + + // ________ 11______ + TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT + ); + return (byte_1_high & byte_1_low & byte_2_high); + } + simdjson_inline simd8 check_multibyte_lengths(const simd8 input, + const simd8 prev_input, const simd8 sc) { + simd8 prev2 = input.prev<2>(prev_input); + simd8 prev3 = input.prev<3>(prev_input); + simd8 must23 = must_be_2_3_continuation(prev2, prev3); + simd8 must23_80 = must23 & uint8_t(0x80); + return must23_80 ^ sc; + } + + // + // Return nonzero if there are incomplete multibyte characters at the end of the block: + // e.g. if there is a 4-byte character, but it's 3 bytes from the end. + // + simdjson_inline simd8 is_incomplete(const simd8 input) { + // If the previous input's last 3 bytes match this, they're too short (they ended at EOF): + // ... 1111____ 111_____ 11______ +#if SIMDJSON_IMPLEMENTATION_ICELAKE + static const uint8_t max_array[64] = { + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 0xf0u - 1, 0xe0u - 1, 0xc0u - 1 + }; +#else + static const uint8_t max_array[32] = { + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 0xf0u - 1, 0xe0u - 1, 0xc0u - 1 + }; +#endif + const simd8 max_value(&max_array[sizeof(max_array) - sizeof(simd8)]); + return input.gt_bits(max_value); + } + + struct utf8_checker { + // If this is nonzero, there has been a UTF-8 error. + simd8 error; + // The last input we received + simd8 prev_input_block; + // Whether the last input we received was incomplete (used for ASCII fast path) + simd8 prev_incomplete; + + // + // Check whether the current bytes are valid UTF-8. + // + simdjson_inline void check_utf8_bytes(const simd8 input, const simd8 prev_input) { + // Flip prev1...prev3 so we can easily determine if they are 2+, 3+ or 4+ lead bytes + // (2, 3, 4-byte leads become large positive numbers instead of small negative numbers) + simd8 prev1 = input.prev<1>(prev_input); + simd8 sc = check_special_cases(input, prev1); + this->error |= check_multibyte_lengths(input, prev_input, sc); + } + + // The only problem that can happen at EOF is that a multibyte character is too short + // or a byte value too large in the last bytes: check_special_cases only checks for bytes + // too large in the first of two bytes. + simdjson_inline void check_eof() { + // If the previous block had incomplete UTF-8 characters at the end, an ASCII block can't + // possibly finish them. + this->error |= this->prev_incomplete; + } + + simdjson_inline void check_next_input(const simd8x64& input) { + if (simdjson_likely(is_ascii(input))) { + this->error |= this->prev_incomplete; + } + else { + // you might think that a for-loop would work, but under Visual Studio, it is not good enough. + static_assert((simd8x64::NUM_CHUNKS == 1) + || (simd8x64::NUM_CHUNKS == 2) + || (simd8x64::NUM_CHUNKS == 4), + "We support one, two or four chunks per 64-byte block."); + SIMDJSON_IF_CONSTEXPR(simd8x64::NUM_CHUNKS == 1) { + this->check_utf8_bytes(input.chunks[0], this->prev_input_block); + } +else SIMDJSON_IF_CONSTEXPR(simd8x64::NUM_CHUNKS == 2) { + this->check_utf8_bytes(input.chunks[0], this->prev_input_block); + this->check_utf8_bytes(input.chunks[1], input.chunks[0]); + } + else SIMDJSON_IF_CONSTEXPR(simd8x64::NUM_CHUNKS == 4) { + this->check_utf8_bytes(input.chunks[0], this->prev_input_block); + this->check_utf8_bytes(input.chunks[1], input.chunks[0]); + this->check_utf8_bytes(input.chunks[2], input.chunks[1]); + this->check_utf8_bytes(input.chunks[3], input.chunks[2]); + } + this->prev_incomplete = is_incomplete(input.chunks[simd8x64::NUM_CHUNKS - 1]); + this->prev_input_block = input.chunks[simd8x64::NUM_CHUNKS - 1]; + } + } + // do not forget to call check_eof! + simdjson_inline error_code errors() { + return this->error.any_bits_set_anywhere() ? error_code::UTF8_ERROR : error_code::SUCCESS; + } + + }; // struct utf8_checker + } // namespace utf8_validation + + } // unnamed namespace + } // namespace lasx +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE1_UTF8_LOOKUP4_ALGORITHM_H +/* end file generic/stage1/utf8_lookup4_algorithm.h for lasx */ +/* including generic/stage1/json_scanner.h for lasx: #include */ +/* begin file generic/stage1/json_scanner.h for lasx */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE1_JSON_SCANNER_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_JSON_SCANNER_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lasx { + namespace { + namespace stage1 { + + /** + * A block of scanned json, with information on operators and scalars. + * + * We seek to identify pseudo-structural characters. Anything that is inside + * a string must be omitted (hence & ~_string.string_tail()). + * Otherwise, pseudo-structural characters come in two forms. + * 1. We have the structural characters ([,],{,},:, comma). The + * term 'structural character' is from the JSON RFC. + * 2. We have the 'scalar pseudo-structural characters'. + * Scalars are quotes, and any character except structural characters and white space. + * + * To identify the scalar pseudo-structural characters, we must look at what comes + * before them: it must be a space, a quote or a structural characters. + * Starting with simdjson v0.3, we identify them by + * negation: we identify everything that is followed by a non-quote scalar, + * and we negate that. Whatever remains must be a 'scalar pseudo-structural character'. + */ + struct json_block { + public: + // We spell out the constructors in the hope of resolving inlining issues with Visual Studio 2017 + simdjson_inline json_block(json_string_block&& string, json_character_block characters, uint64_t follows_potential_nonquote_scalar) : + _string(std::move(string)), _characters(characters), _follows_potential_nonquote_scalar(follows_potential_nonquote_scalar) {} + simdjson_inline json_block(json_string_block string, json_character_block characters, uint64_t follows_potential_nonquote_scalar) : + _string(string), _characters(characters), _follows_potential_nonquote_scalar(follows_potential_nonquote_scalar) {} + + /** + * The start of structurals. + * In simdjson prior to v0.3, these were called the pseudo-structural characters. + **/ + simdjson_inline uint64_t structural_start() const noexcept { return potential_structural_start() & ~_string.string_tail(); } + /** All JSON whitespace (i.e. not in a string) */ + simdjson_inline uint64_t whitespace() const noexcept { return non_quote_outside_string(_characters.whitespace()); } + + // Helpers + + /** Whether the given characters are inside a string (only works on non-quotes) */ + simdjson_inline uint64_t non_quote_inside_string(uint64_t mask) const noexcept { return _string.non_quote_inside_string(mask); } + /** Whether the given characters are outside a string (only works on non-quotes) */ + simdjson_inline uint64_t non_quote_outside_string(uint64_t mask) const noexcept { return _string.non_quote_outside_string(mask); } + + // string and escape characters + json_string_block _string; + // whitespace, structural characters ('operators'), scalars + json_character_block _characters; + // whether the previous character was a scalar + uint64_t _follows_potential_nonquote_scalar; + private: + // Potential structurals (i.e. disregarding strings) + + /** + * structural elements ([,],{,},:, comma) plus scalar starts like 123, true and "abc". + * They may reside inside a string. + **/ + simdjson_inline uint64_t potential_structural_start() const noexcept { return _characters.op() | potential_scalar_start(); } + /** + * The start of non-operator runs, like 123, true and "abc". + * It main reside inside a string. + **/ + simdjson_inline uint64_t potential_scalar_start() const noexcept { + // The term "scalar" refers to anything except structural characters and white space + // (so letters, numbers, quotes). + // Whenever it is preceded by something that is not a structural element ({,},[,],:, ") nor a white-space + // then we know that it is irrelevant structurally. + return _characters.scalar() & ~follows_potential_scalar(); + } + /** + * Whether the given character is immediately after a non-operator like 123, true. + * The characters following a quote are not included. + */ + simdjson_inline uint64_t follows_potential_scalar() const noexcept { + // _follows_potential_nonquote_scalar: is defined as marking any character that follows a character + // that is not a structural element ({,},[,],:, comma) nor a quote (") and that is not a + // white space. + // It is understood that within quoted region, anything at all could be marked (irrelevant). + return _follows_potential_nonquote_scalar; + } + }; + + /** + * Scans JSON for important bits: structural characters or 'operators', strings, and scalars. + * + * The scanner starts by calculating two distinct things: + * - string characters (taking \" into account) + * - structural characters or 'operators' ([]{},:, comma) + * and scalars (runs of non-operators like 123, true and "abc") + * + * To minimize data dependency (a key component of the scanner's speed), it finds these in parallel: + * in particular, the operator/scalar bit will find plenty of things that are actually part of + * strings. When we're done, json_block will fuse the two together by masking out tokens that are + * part of a string. + */ + class json_scanner { + public: + json_scanner() = default; + simdjson_inline json_block next(const simd::simd8x64& in); + // Returns either UNCLOSED_STRING or SUCCESS + simdjson_inline error_code finish(); + + private: + // Whether the last character of the previous iteration is part of a scalar token + // (anything except whitespace or a structural character/'operator'). + uint64_t prev_scalar = 0ULL; + json_string_scanner string_scanner{}; + }; + + + // + // Check if the current character immediately follows a matching character. + // + // For example, this checks for quotes with backslashes in front of them: + // + // const uint64_t backslashed_quote = in.eq('"') & immediately_follows(in.eq('\'), prev_backslash); + // + simdjson_inline uint64_t follows(const uint64_t match, uint64_t& overflow) { + const uint64_t result = match << 1 | overflow; + overflow = match >> 63; + return result; + } + + simdjson_inline json_block json_scanner::next(const simd::simd8x64& in) { + json_string_block strings = string_scanner.next(in); + // identifies the white-space and the structural characters + json_character_block characters = json_character_block::classify(in); + // The term "scalar" refers to anything except structural characters and white space + // (so letters, numbers, quotes). + // We want follows_scalar to mark anything that follows a non-quote scalar (so letters and numbers). + // + // A terminal quote should either be followed by a structural character (comma, brace, bracket, colon) + // or nothing. However, we still want ' "a string"true ' to mark the 't' of 'true' as a potential + // pseudo-structural character just like we would if we had ' "a string" true '; otherwise we + // may need to add an extra check when parsing strings. + // + // Performance: there are many ways to skin this cat. + const uint64_t nonquote_scalar = characters.scalar() & ~strings.quote(); + uint64_t follows_nonquote_scalar = follows(nonquote_scalar, prev_scalar); + // We are returning a function-local object so either we get a move constructor + // or we get copy elision. + return json_block( + strings,// strings is a function-local object so either it moves or the copy is elided. + characters, + follows_nonquote_scalar + ); + } + + simdjson_inline error_code json_scanner::finish() { + return string_scanner.finish(); + } + + } // namespace stage1 + } // unnamed namespace + } // namespace lasx +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE1_JSON_SCANNER_H +/* end file generic/stage1/json_scanner.h for lasx */ + +// All other declarations +/* including generic/stage1/find_next_document_index.h for lasx: #include */ +/* begin file generic/stage1/find_next_document_index.h for lasx */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE1_FIND_NEXT_DOCUMENT_INDEX_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_FIND_NEXT_DOCUMENT_INDEX_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lasx { + namespace { + namespace stage1 { + + /** + * This algorithm is used to quickly identify the last structural position that + * makes up a complete document. + * + * It does this by going backwards and finding the last *document boundary* (a + * place where one value follows another without a comma between them). If the + * last document (the characters after the boundary) has an equal number of + * start and end brackets, it is considered complete. + * + * Simply put, we iterate over the structural characters, starting from + * the end. We consider that we found the end of a JSON document when the + * first element of the pair is NOT one of these characters: '{' '[' ':' ',' + * and when the second element is NOT one of these characters: '}' ']' ':' ','. + * + * This simple comparison works most of the time, but it does not cover cases + * where the batch's structural indexes contain a perfect amount of documents. + * In such a case, we do not have access to the structural index which follows + * the last document, therefore, we do not have access to the second element in + * the pair, and that means we cannot identify the last document. To fix this + * issue, we keep a count of the open and closed curly/square braces we found + * while searching for the pair. When we find a pair AND the count of open and + * closed curly/square braces is the same, we know that we just passed a + * complete document, therefore the last json buffer location is the end of the + * batch. + */ + simdjson_inline uint32_t find_next_document_index(dom_parser_implementation& parser) { + // Variant: do not count separately, just figure out depth + if (parser.n_structural_indexes == 0) { return 0; } + auto arr_cnt = 0; + auto obj_cnt = 0; + for (auto i = parser.n_structural_indexes - 1; i > 0; i--) { + auto idxb = parser.structural_indexes[i]; + switch (parser.buf[idxb]) { + case ':': + case ',': + continue; + case '}': + obj_cnt--; + continue; + case ']': + arr_cnt--; + continue; + case '{': + obj_cnt++; + break; + case '[': + arr_cnt++; + break; + } + auto idxa = parser.structural_indexes[i - 1]; + switch (parser.buf[idxa]) { + case '{': + case '[': + case ':': + case ',': + continue; + } + // Last document is complete, so the next document will appear after! + if (!arr_cnt && !obj_cnt) { + return parser.n_structural_indexes; + } + // Last document is incomplete; mark the document at i + 1 as the next one + return i; + } + // If we made it to the end, we want to finish counting to see if we have a full document. + switch (parser.buf[parser.structural_indexes[0]]) { + case '}': + obj_cnt--; + break; + case ']': + arr_cnt--; + break; + case '{': + obj_cnt++; + break; + case '[': + arr_cnt++; + break; + } + if (!arr_cnt && !obj_cnt) { + // We have a complete document. + return parser.n_structural_indexes; + } + return 0; + } + + } // namespace stage1 + } // unnamed namespace + } // namespace lasx +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE1_FIND_NEXT_DOCUMENT_INDEX_H +/* end file generic/stage1/find_next_document_index.h for lasx */ +/* including generic/stage1/json_minifier.h for lasx: #include */ +/* begin file generic/stage1/json_minifier.h for lasx */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE1_JSON_MINIFIER_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_JSON_MINIFIER_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +// This file contains the common code every implementation uses in stage1 +// It is intended to be included multiple times and compiled multiple times +// We assume the file in which it is included already includes +// "simdjson/stage1.h" (this simplifies amalgation) + +namespace simdjson { + namespace lasx { + namespace { + namespace stage1 { + + class json_minifier { + public: + template + static error_code minify(const uint8_t* buf, size_t len, uint8_t* dst, size_t& dst_len) noexcept; + + private: + simdjson_inline json_minifier(uint8_t* _dst) + : dst{ _dst } + {} + template + simdjson_inline void step(const uint8_t* block_buf, buf_block_reader& reader) noexcept; + simdjson_inline void next(const simd::simd8x64& in, const json_block& block); + simdjson_inline error_code finish(uint8_t* dst_start, size_t& dst_len); + json_scanner scanner{}; + uint8_t* dst; + }; + + simdjson_inline void json_minifier::next(const simd::simd8x64& in, const json_block& block) { + uint64_t mask = block.whitespace(); + dst += in.compress(mask, dst); + } + + simdjson_inline error_code json_minifier::finish(uint8_t* dst_start, size_t& dst_len) { + error_code error = scanner.finish(); + if (error) { dst_len = 0; return error; } + dst_len = dst - dst_start; + return SUCCESS; + } + + template<> + simdjson_inline void json_minifier::step<128>(const uint8_t* block_buf, buf_block_reader<128>& reader) noexcept { + simd::simd8x64 in_1(block_buf); + simd::simd8x64 in_2(block_buf + 64); + json_block block_1 = scanner.next(in_1); + json_block block_2 = scanner.next(in_2); + this->next(in_1, block_1); + this->next(in_2, block_2); + reader.advance(); + } + + template<> + simdjson_inline void json_minifier::step<64>(const uint8_t* block_buf, buf_block_reader<64>& reader) noexcept { + simd::simd8x64 in_1(block_buf); + json_block block_1 = scanner.next(in_1); + this->next(block_buf, block_1); + reader.advance(); + } + + template + error_code json_minifier::minify(const uint8_t* buf, size_t len, uint8_t* dst, size_t& dst_len) noexcept { + buf_block_reader reader(buf, len); + json_minifier minifier(dst); + + // Index the first n-1 blocks + while (reader.has_full_block()) { + minifier.step(reader.full_block(), reader); + } + + // Index the last (remainder) block, padded with spaces + uint8_t block[STEP_SIZE]; + size_t remaining_bytes = reader.get_remainder(block); + if (remaining_bytes > 0) { + // We do not want to write directly to the output stream. Rather, we write + // to a local buffer (for safety). + uint8_t out_block[STEP_SIZE]; + uint8_t* const guarded_dst{ minifier.dst }; + minifier.dst = out_block; + minifier.step(block, reader); + size_t to_write = minifier.dst - out_block; + // In some cases, we could be enticed to consider the padded spaces + // as part of the string. This is fine as long as we do not write more + // than we consumed. + if (to_write > remaining_bytes) { to_write = remaining_bytes; } + memcpy(guarded_dst, out_block, to_write); + minifier.dst = guarded_dst + to_write; + } + return minifier.finish(dst, dst_len); + } + + } // namespace stage1 + } // unnamed namespace + } // namespace lasx +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE1_JSON_MINIFIER_H +/* end file generic/stage1/json_minifier.h for lasx */ +/* including generic/stage1/json_structural_indexer.h for lasx: #include */ +/* begin file generic/stage1/json_structural_indexer.h for lasx */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRUCTURAL_INDEXER_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRUCTURAL_INDEXER_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +// This file contains the common code every implementation uses in stage1 +// It is intended to be included multiple times and compiled multiple times +// We assume the file in which it is included already includes +// "simdjson/stage1.h" (this simplifies amalgation) + +namespace simdjson { + namespace lasx { + namespace { + namespace stage1 { + + class bit_indexer { + public: + uint32_t* tail; + + simdjson_inline bit_indexer(uint32_t* index_buf) : tail(index_buf) {} + +#if SIMDJSON_PREFER_REVERSE_BITS + /** + * ARM lacks a fast trailing zero instruction, but it has a fast + * bit reversal instruction and a fast leading zero instruction. + * Thus it may be profitable to reverse the bits (once) and then + * to rely on a sequence of instructions that call the leading + * zero instruction. + * + * Performance notes: + * The chosen routine is not optimal in terms of data dependency + * since zero_leading_bit might require two instructions. However, + * it tends to minimize the total number of instructions which is + * beneficial. + */ + simdjson_inline void write_index(uint32_t idx, uint64_t& rev_bits, int i) { + int lz = leading_zeroes(rev_bits); + this->tail[i] = static_cast(idx) + lz; + rev_bits = zero_leading_bit(rev_bits, lz); + } +#else + /** + * Under recent x64 systems, we often have both a fast trailing zero + * instruction and a fast 'clear-lower-bit' instruction so the following + * algorithm can be competitive. + */ + + simdjson_inline void write_index(uint32_t idx, uint64_t& bits, int i) { + this->tail[i] = idx + trailing_zeroes(bits); + bits = clear_lowest_bit(bits); + } +#endif // SIMDJSON_PREFER_REVERSE_BITS + + template + simdjson_inline int write_indexes(uint32_t idx, uint64_t& bits) { + write_index(idx, bits, START); + SIMDJSON_IF_CONSTEXPR(N > 1) { + write_indexes<(N - 1 > 0 ? START + 1 : START), (N - 1 >= 0 ? N - 1 : 1)>(idx, bits); + } + return START + N; + } + + template + simdjson_inline int write_indexes_stepped(uint32_t idx, uint64_t& bits, int cnt) { + write_indexes(idx, bits); + SIMDJSON_IF_CONSTEXPR((START + STEP) < END) { + if (simdjson_unlikely((START + STEP) < cnt)) { + write_indexes_stepped<(START + STEP < END ? START + STEP : END), END, STEP>(idx, bits, cnt); + } + } + return ((END - START) % STEP) == 0 ? END : (END - START) - ((END - START) % STEP) + STEP; + } + + // flatten out values in 'bits' assuming that they are are to have values of idx + // plus their position in the bitvector, and store these indexes at + // base_ptr[base] incrementing base as we go + // will potentially store extra values beyond end of valid bits, so base_ptr + // needs to be large enough to handle this + // + // If the kernel sets SIMDJSON_GENERIC_JSON_STRUCTURAL_INDEXER_CUSTOM_BIT_INDEXER, then it + // will provide its own version of the code. +#ifdef SIMDJSON_GENERIC_JSON_STRUCTURAL_INDEXER_CUSTOM_BIT_INDEXER + simdjson_inline void write(uint32_t idx, uint64_t bits); +#else + simdjson_inline void write(uint32_t idx, uint64_t bits) { + // In some instances, the next branch is expensive because it is mispredicted. + // Unfortunately, in other cases, + // it helps tremendously. + if (bits == 0) + return; + + int cnt = static_cast(count_ones(bits)); + +#if SIMDJSON_PREFER_REVERSE_BITS + bits = reverse_bits(bits); +#endif +#ifdef SIMDJSON_STRUCTURAL_INDEXER_STEP + static constexpr const int STEP = SIMDJSON_STRUCTURAL_INDEXER_STEP; +#else + static constexpr const int STEP = 4; +#endif + static constexpr const int STEP_UNTIL = 24; + + write_indexes_stepped<0, STEP_UNTIL, STEP>(idx, bits, cnt); + SIMDJSON_IF_CONSTEXPR(STEP_UNTIL < 64) { + if (simdjson_unlikely(STEP_UNTIL < cnt)) { + for (int i = STEP_UNTIL; i < cnt; i++) { + write_index(idx, bits, i); + } + } + } + + this->tail += cnt; + } +#endif // SIMDJSON_GENERIC_JSON_STRUCTURAL_INDEXER_CUSTOM_BIT_INDEXER + + }; + + class json_structural_indexer { + public: + /** + * Find the important bits of JSON in a 128-byte chunk, and add them to structural_indexes. + * + * @param partial Setting the partial parameter to true allows the find_structural_bits to + * tolerate unclosed strings. The caller should still ensure that the input is valid UTF-8. If + * you are processing substrings, you may want to call on a function like trimmed_length_safe_utf8. + */ + template + static error_code index(const uint8_t* buf, size_t len, dom_parser_implementation& parser, stage1_mode partial) noexcept; + + private: + simdjson_inline json_structural_indexer(uint32_t* structural_indexes); + template + simdjson_inline void step(const uint8_t* block, buf_block_reader& reader) noexcept; + simdjson_inline void next(const simd::simd8x64& in, const json_block& block, size_t idx); + simdjson_inline error_code finish(dom_parser_implementation& parser, size_t idx, size_t len, stage1_mode partial); + + json_scanner scanner{}; + utf8_checker checker{}; + bit_indexer indexer; + uint64_t prev_structurals = 0; + uint64_t unescaped_chars_error = 0; + }; + + simdjson_inline json_structural_indexer::json_structural_indexer(uint32_t* structural_indexes) : indexer{ structural_indexes } {} + + // Skip the last character if it is partial + simdjson_inline size_t trim_partial_utf8(const uint8_t* buf, size_t len) { + if (simdjson_unlikely(len < 3)) { + switch (len) { + case 2: + if (buf[len - 1] >= 0xc0) { return len - 1; } // 2-, 3- and 4-byte characters with only 1 byte left + if (buf[len - 2] >= 0xe0) { return len - 2; } // 3- and 4-byte characters with only 2 bytes left + return len; + case 1: + if (buf[len - 1] >= 0xc0) { return len - 1; } // 2-, 3- and 4-byte characters with only 1 byte left + return len; + case 0: + return len; + } + } + if (buf[len - 1] >= 0xc0) { return len - 1; } // 2-, 3- and 4-byte characters with only 1 byte left + if (buf[len - 2] >= 0xe0) { return len - 2; } // 3- and 4-byte characters with only 1 byte left + if (buf[len - 3] >= 0xf0) { return len - 3; } // 4-byte characters with only 3 bytes left + return len; + } + + // + // PERF NOTES: + // We pipe 2 inputs through these stages: + // 1. Load JSON into registers. This takes a long time and is highly parallelizable, so we load + // 2 inputs' worth at once so that by the time step 2 is looking for them input, it's available. + // 2. Scan the JSON for critical data: strings, scalars and operators. This is the critical path. + // The output of step 1 depends entirely on this information. These functions don't quite use + // up enough CPU: the second half of the functions is highly serial, only using 1 execution core + // at a time. The second input's scans has some dependency on the first ones finishing it, but + // they can make a lot of progress before they need that information. + // 3. Step 1 does not use enough capacity, so we run some extra stuff while we're waiting for that + // to finish: utf-8 checks and generating the output from the last iteration. + // + // The reason we run 2 inputs at a time, is steps 2 and 3 are *still* not enough to soak up all + // available capacity with just one input. Running 2 at a time seems to give the CPU a good enough + // workout. + // + template + error_code json_structural_indexer::index(const uint8_t* buf, size_t len, dom_parser_implementation& parser, stage1_mode partial) noexcept { + if (simdjson_unlikely(len > parser.capacity())) { return CAPACITY; } + // We guard the rest of the code so that we can assume that len > 0 throughout. + if (len == 0) { return EMPTY; } + if (is_streaming(partial)) { + len = trim_partial_utf8(buf, len); + // If you end up with an empty window after trimming + // the partial UTF-8 bytes, then chances are good that you + // have an UTF-8 formatting error. + if (len == 0) { return UTF8_ERROR; } + } + buf_block_reader reader(buf, len); + json_structural_indexer indexer(parser.structural_indexes.get()); + + // Read all but the last block + while (reader.has_full_block()) { + indexer.step(reader.full_block(), reader); + } + // Take care of the last block (will always be there unless file is empty which is + // not supposed to happen.) + uint8_t block[STEP_SIZE]; + if (simdjson_unlikely(reader.get_remainder(block) == 0)) { return UNEXPECTED_ERROR; } + indexer.step(block, reader); + return indexer.finish(parser, reader.block_index(), len, partial); + } + + template<> + simdjson_inline void json_structural_indexer::step<128>(const uint8_t* block, buf_block_reader<128>& reader) noexcept { + simd::simd8x64 in_1(block); + simd::simd8x64 in_2(block + 64); + json_block block_1 = scanner.next(in_1); + json_block block_2 = scanner.next(in_2); + this->next(in_1, block_1, reader.block_index()); + this->next(in_2, block_2, reader.block_index() + 64); + reader.advance(); + } + + template<> + simdjson_inline void json_structural_indexer::step<64>(const uint8_t* block, buf_block_reader<64>& reader) noexcept { + simd::simd8x64 in_1(block); + json_block block_1 = scanner.next(in_1); + this->next(in_1, block_1, reader.block_index()); + reader.advance(); + } + + simdjson_inline void json_structural_indexer::next(const simd::simd8x64& in, const json_block& block, size_t idx) { + uint64_t unescaped = in.lteq(0x1F); +#if SIMDJSON_UTF8VALIDATION + checker.check_next_input(in); +#endif + indexer.write(uint32_t(idx - 64), prev_structurals); // Output *last* iteration's structurals to the parser + prev_structurals = block.structural_start(); + unescaped_chars_error |= block.non_quote_inside_string(unescaped); + } + + simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementation& parser, size_t idx, size_t len, stage1_mode partial) { + // Write out the final iteration's structurals + indexer.write(uint32_t(idx - 64), prev_structurals); + error_code error = scanner.finish(); + // We deliberately break down the next expression so that it is + // human readable. + const bool should_we_exit = is_streaming(partial) ? + ((error != SUCCESS) && (error != UNCLOSED_STRING)) // when partial we tolerate UNCLOSED_STRING + : (error != SUCCESS); // if partial is false, we must have SUCCESS + const bool have_unclosed_string = (error == UNCLOSED_STRING); + if (simdjson_unlikely(should_we_exit)) { return error; } + + if (unescaped_chars_error) { + return UNESCAPED_CHARS; + } + parser.n_structural_indexes = uint32_t(indexer.tail - parser.structural_indexes.get()); + /*** + * The On-Demand API requires special padding. + * + * This is related to https://github.com/simdjson/simdjson/issues/906 + * Basically, we want to make sure that if the parsing continues beyond the last (valid) + * structural character, it quickly stops. + * Only three structural characters can be repeated without triggering an error in JSON: [,] and }. + * We repeat the padding character (at 'len'). We don't know what it is, but if the parsing + * continues, then it must be [,] or }. + * Suppose it is ] or }. We backtrack to the first character, what could it be that would + * not trigger an error? It could be ] or } but no, because you can't start a document that way. + * It can't be a comma, a colon or any simple value. So the only way we could continue is + * if the repeated character is [. But if so, the document must start with [. But if the document + * starts with [, it should end with ]. If we enforce that rule, then we would get + * ][[ which is invalid. + * + * This is illustrated with the test array_iterate_unclosed_error() on the following input: + * R"({ "a": [,,)" + **/ + parser.structural_indexes[parser.n_structural_indexes] = uint32_t(len); // used later in partial == stage1_mode::streaming_final + parser.structural_indexes[parser.n_structural_indexes + 1] = uint32_t(len); + parser.structural_indexes[parser.n_structural_indexes + 2] = 0; + parser.next_structural_index = 0; + // a valid JSON file cannot have zero structural indexes - we should have found something + if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { + return EMPTY; + } + if (simdjson_unlikely(parser.structural_indexes[parser.n_structural_indexes - 1] > len)) { + return UNEXPECTED_ERROR; + } + if (partial == stage1_mode::streaming_partial) { + // If we have an unclosed string, then the last structural + // will be the quote and we want to make sure to omit it. + if (have_unclosed_string) { + parser.n_structural_indexes--; + // a valid JSON file cannot have zero structural indexes - we should have found something + if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return CAPACITY; } + } + // We truncate the input to the end of the last complete document (or zero). + auto new_structural_indexes = find_next_document_index(parser); + if (new_structural_indexes == 0 && parser.n_structural_indexes > 0) { + if (parser.structural_indexes[0] == 0) { + // If the buffer is partial and we started at index 0 but the document is + // incomplete, it's too big to parse. + return CAPACITY; + } + else { + // It is possible that the document could be parsed, we just had a lot + // of white space. + parser.n_structural_indexes = 0; + return EMPTY; + } + } + + parser.n_structural_indexes = new_structural_indexes; + } + else if (partial == stage1_mode::streaming_final) { + if (have_unclosed_string) { parser.n_structural_indexes--; } + // We truncate the input to the end of the last complete document (or zero). + // Because partial == stage1_mode::streaming_final, it means that we may + // silently ignore trailing garbage. Though it sounds bad, we do it + // deliberately because many people who have streams of JSON documents + // will truncate them for processing. E.g., imagine that you are uncompressing + // the data from a size file or receiving it in chunks from the network. You + // may not know where exactly the last document will be. Meanwhile the + // document_stream instances allow people to know the JSON documents they are + // parsing (see the iterator.source() method). + parser.n_structural_indexes = find_next_document_index(parser); + // We store the initial n_structural_indexes so that the client can see + // whether we used truncation. If initial_n_structural_indexes == parser.n_structural_indexes, + // then this will query parser.structural_indexes[parser.n_structural_indexes] which is len, + // otherwise, it will copy some prior index. + parser.structural_indexes[parser.n_structural_indexes + 1] = parser.structural_indexes[parser.n_structural_indexes]; + // This next line is critical, do not change it unless you understand what you are + // doing. + parser.structural_indexes[parser.n_structural_indexes] = uint32_t(len); + if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { + // We tolerate an unclosed string at the very end of the stream. Indeed, users + // often load their data in bulk without being careful and they want us to ignore + // the trailing garbage. + return EMPTY; + } + } + checker.check_eof(); + return checker.errors(); + } + + } // namespace stage1 + } // unnamed namespace + } // namespace lasx +} // namespace simdjson + +// Clear CUSTOM_BIT_INDEXER so other implementations can set it if they need to. +#undef SIMDJSON_GENERIC_JSON_STRUCTURAL_INDEXER_CUSTOM_BIT_INDEXER + +#endif // SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRUCTURAL_INDEXER_H +/* end file generic/stage1/json_structural_indexer.h for lasx */ +/* including generic/stage1/utf8_validator.h for lasx: #include */ +/* begin file generic/stage1/utf8_validator.h for lasx */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE1_UTF8_VALIDATOR_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_UTF8_VALIDATOR_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lasx { + namespace { + namespace stage1 { + + /** + * Validates that the string is actual UTF-8. + */ + template + bool generic_validate_utf8(const uint8_t* input, size_t length) { + checker c{}; + buf_block_reader<64> reader(input, length); + while (reader.has_full_block()) { + simd::simd8x64 in(reader.full_block()); + c.check_next_input(in); + reader.advance(); + } + uint8_t block[64]{}; + reader.get_remainder(block); + simd::simd8x64 in(block); + c.check_next_input(in); + reader.advance(); + c.check_eof(); + return c.errors() == error_code::SUCCESS; + } + + bool generic_validate_utf8(const char* input, size_t length) { + return generic_validate_utf8(reinterpret_cast(input), length); + } + + } // namespace stage1 + } // unnamed namespace + } // namespace lasx +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE1_UTF8_VALIDATOR_H +/* end file generic/stage1/utf8_validator.h for lasx */ +/* end file generic/stage1/amalgamated.h for lasx */ +/* including generic/stage2/amalgamated.h for lasx: #include */ +/* begin file generic/stage2/amalgamated.h for lasx */ +// Stuff other things depend on +/* including generic/stage2/base.h for lasx: #include */ +/* begin file generic/stage2/base.h for lasx */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE2_BASE_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_BASE_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lasx { + namespace { + namespace stage2 { + + class json_iterator; + class structural_iterator; + struct tape_builder; + struct tape_writer; + + } // namespace stage2 + } // unnamed namespace + } // namespace lasx +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE2_BASE_H +/* end file generic/stage2/base.h for lasx */ +/* including generic/stage2/tape_writer.h for lasx: #include */ +/* begin file generic/stage2/tape_writer.h for lasx */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE2_TAPE_WRITER_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_TAPE_WRITER_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +#include + +namespace simdjson { + namespace lasx { + namespace { + namespace stage2 { + + struct tape_writer { + /** The next place to write to tape */ + uint64_t* next_tape_loc; + + /** Write a signed 64-bit value to tape. */ + simdjson_inline void append_s64(int64_t value) noexcept; + + /** Write an unsigned 64-bit value to tape. */ + simdjson_inline void append_u64(uint64_t value) noexcept; + + /** Write a double value to tape. */ + simdjson_inline void append_double(double value) noexcept; + + /** + * Append a tape entry (an 8-bit type,and 56 bits worth of value). + */ + simdjson_inline void append(uint64_t val, internal::tape_type t) noexcept; + + /** + * Skip the current tape entry without writing. + * + * Used to skip the start of the container, since we'll come back later to fill it in when the + * container ends. + */ + simdjson_inline void skip() noexcept; + + /** + * Skip the number of tape entries necessary to write a large u64 or i64. + */ + simdjson_inline void skip_large_integer() noexcept; + + /** + * Skip the number of tape entries necessary to write a double. + */ + simdjson_inline void skip_double() noexcept; + + /** + * Write a value to a known location on tape. + * + * Used to go back and write out the start of a container after the container ends. + */ + simdjson_inline static void write(uint64_t& tape_loc, uint64_t val, internal::tape_type t) noexcept; + + private: + /** + * Append both the tape entry, and a supplementary value following it. Used for types that need + * all 64 bits, such as double and uint64_t. + */ + template + simdjson_inline void append2(uint64_t val, T val2, internal::tape_type t) noexcept; + }; // struct tape_writer + + simdjson_inline void tape_writer::append_s64(int64_t value) noexcept { + append2(0, value, internal::tape_type::INT64); + } + + simdjson_inline void tape_writer::append_u64(uint64_t value) noexcept { + append(0, internal::tape_type::UINT64); + *next_tape_loc = value; + next_tape_loc++; + } + + /** Write a double value to tape. */ + simdjson_inline void tape_writer::append_double(double value) noexcept { + append2(0, value, internal::tape_type::DOUBLE); + } + + simdjson_inline void tape_writer::skip() noexcept { + next_tape_loc++; + } + + simdjson_inline void tape_writer::skip_large_integer() noexcept { + next_tape_loc += 2; + } + + simdjson_inline void tape_writer::skip_double() noexcept { + next_tape_loc += 2; + } + + simdjson_inline void tape_writer::append(uint64_t val, internal::tape_type t) noexcept { + *next_tape_loc = val | ((uint64_t(char(t))) << 56); + next_tape_loc++; + } + + template + simdjson_inline void tape_writer::append2(uint64_t val, T val2, internal::tape_type t) noexcept { + append(val, t); + static_assert(sizeof(val2) == sizeof(*next_tape_loc), "Type is not 64 bits!"); + memcpy(next_tape_loc, &val2, sizeof(val2)); + next_tape_loc++; + } + + simdjson_inline void tape_writer::write(uint64_t& tape_loc, uint64_t val, internal::tape_type t) noexcept { + tape_loc = val | ((uint64_t(char(t))) << 56); + } + + } // namespace stage2 + } // unnamed namespace + } // namespace lasx +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE2_TAPE_WRITER_H +/* end file generic/stage2/tape_writer.h for lasx */ +/* including generic/stage2/logger.h for lasx: #include */ +/* begin file generic/stage2/logger.h for lasx */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE2_LOGGER_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_LOGGER_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +#include + + +// This is for an internal-only stage 2 specific logger. +// Set LOG_ENABLED = true to log what stage 2 is doing! +namespace simdjson { + namespace lasx { + namespace { + namespace logger { + + static constexpr const char* DASHES = "----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------"; + +#if SIMDJSON_VERBOSE_LOGGING + static constexpr const bool LOG_ENABLED = true; +#else + static constexpr const bool LOG_ENABLED = false; +#endif + static constexpr const int LOG_EVENT_LEN = 20; + static constexpr const int LOG_BUFFER_LEN = 30; + static constexpr const int LOG_SMALL_BUFFER_LEN = 10; + static constexpr const int LOG_INDEX_LEN = 5; + + static int log_depth; // Not threadsafe. Log only. + + // Helper to turn unprintable or newline characters into spaces + static simdjson_inline char printable_char(char c) { + if (c >= 0x20) { + return c; + } + else { + return ' '; + } + } + + // Print the header and set up log_start + static simdjson_inline void log_start() { + if (LOG_ENABLED) { + log_depth = 0; + printf("\n"); + printf("| %-*s | %-*s | %-*s | %-*s | Detail |\n", LOG_EVENT_LEN, "Event", LOG_BUFFER_LEN, "Buffer", LOG_SMALL_BUFFER_LEN, "Next", 5, "Next#"); + printf("|%.*s|%.*s|%.*s|%.*s|--------|\n", LOG_EVENT_LEN + 2, DASHES, LOG_BUFFER_LEN + 2, DASHES, LOG_SMALL_BUFFER_LEN + 2, DASHES, 5 + 2, DASHES); + } + } + + simdjson_unused static simdjson_inline void log_string(const char* message) { + if (LOG_ENABLED) { + printf("%s\n", message); + } + } + + // Logs a single line from the stage 2 DOM parser + template + static simdjson_inline void log_line(S& structurals, const char* title_prefix, const char* title, const char* detail) { + if (LOG_ENABLED) { + printf("| %*s%s%-*s ", log_depth * 2, "", title_prefix, LOG_EVENT_LEN - log_depth * 2 - int(strlen(title_prefix)), title); + auto current_index = structurals.at_beginning() ? nullptr : structurals.next_structural - 1; + auto next_index = structurals.next_structural; + auto current = current_index ? &structurals.buf[*current_index] : reinterpret_cast(" "); + auto next = &structurals.buf[*next_index]; + { + // Print the next N characters in the buffer. + printf("| "); + // Otherwise, print the characters starting from the buffer position. + // Print spaces for unprintable or newline characters. + for (int i = 0; i < LOG_BUFFER_LEN; i++) { + printf("%c", printable_char(current[i])); + } + printf(" "); + // Print the next N characters in the buffer. + printf("| "); + // Otherwise, print the characters starting from the buffer position. + // Print spaces for unprintable or newline characters. + for (int i = 0; i < LOG_SMALL_BUFFER_LEN; i++) { + printf("%c", printable_char(next[i])); + } + printf(" "); + } + if (current_index) { + printf("| %*u ", LOG_INDEX_LEN, *current_index); + } + else { + printf("| %-*s ", LOG_INDEX_LEN, ""); + } + // printf("| %*u ", LOG_INDEX_LEN, structurals.next_tape_index()); + printf("| %-s ", detail); + printf("|\n"); + } + } + + } // namespace logger + } // unnamed namespace + } // namespace lasx +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE2_LOGGER_H +/* end file generic/stage2/logger.h for lasx */ + +// All other declarations +/* including generic/stage2/json_iterator.h for lasx: #include */ +/* begin file generic/stage2/json_iterator.h for lasx */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE2_JSON_ITERATOR_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_JSON_ITERATOR_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lasx { + namespace { + namespace stage2 { + + class json_iterator { + public: + const uint8_t* const buf; + uint32_t* next_structural; + dom_parser_implementation& dom_parser; + uint32_t depth{ 0 }; + + /** + * Walk the JSON document. + * + * The visitor receives callbacks when values are encountered. All callbacks pass the iterator as + * the first parameter; some callbacks have other parameters as well: + * + * - visit_document_start() - at the beginning. + * - visit_document_end() - at the end (if things were successful). + * + * - visit_array_start() - at the start `[` of a non-empty array. + * - visit_array_end() - at the end `]` of a non-empty array. + * - visit_empty_array() - when an empty array is encountered. + * + * - visit_object_end() - at the start `]` of a non-empty object. + * - visit_object_start() - at the end `]` of a non-empty object. + * - visit_empty_object() - when an empty object is encountered. + * - visit_key(const uint8_t *key) - when a key in an object field is encountered. key is + * guaranteed to point at the first quote of the string (`"key"`). + * - visit_primitive(const uint8_t *value) - when a value is a string, number, boolean or null. + * - visit_root_primitive(iter, uint8_t *value) - when the top-level value is a string, number, boolean or null. + * + * - increment_count(iter) - each time a value is found in an array or object. + */ + template + simdjson_warn_unused simdjson_inline error_code walk_document(V& visitor) noexcept; + + /** + * Create an iterator capable of walking a JSON document. + * + * The document must have already passed through stage 1. + */ + simdjson_inline json_iterator(dom_parser_implementation& _dom_parser, size_t start_structural_index); + + /** + * Look at the next token. + * + * Tokens can be strings, numbers, booleans, null, or operators (`[{]},:`)). + * + * They may include invalid JSON as well (such as `1.2.3` or `ture`). + */ + simdjson_inline const uint8_t* peek() const noexcept; + /** + * Advance to the next token. + * + * Tokens can be strings, numbers, booleans, null, or operators (`[{]},:`)). + * + * They may include invalid JSON as well (such as `1.2.3` or `ture`). + */ + simdjson_inline const uint8_t* advance() noexcept; + /** + * Get the remaining length of the document, from the start of the current token. + */ + simdjson_inline size_t remaining_len() const noexcept; + /** + * Check if we are at the end of the document. + * + * If this is true, there are no more tokens. + */ + simdjson_inline bool at_eof() const noexcept; + /** + * Check if we are at the beginning of the document. + */ + simdjson_inline bool at_beginning() const noexcept; + simdjson_inline uint8_t last_structural() const noexcept; + + /** + * Log that a value has been found. + * + * Set LOG_ENABLED=true in logger.h to see logging. + */ + simdjson_inline void log_value(const char* type) const noexcept; + /** + * Log the start of a multipart value. + * + * Set LOG_ENABLED=true in logger.h to see logging. + */ + simdjson_inline void log_start_value(const char* type) const noexcept; + /** + * Log the end of a multipart value. + * + * Set LOG_ENABLED=true in logger.h to see logging. + */ + simdjson_inline void log_end_value(const char* type) const noexcept; + /** + * Log an error. + * + * Set LOG_ENABLED=true in logger.h to see logging. + */ + simdjson_inline void log_error(const char* error) const noexcept; + + template + simdjson_warn_unused simdjson_inline error_code visit_root_primitive(V& visitor, const uint8_t* value) noexcept; + template + simdjson_warn_unused simdjson_inline error_code visit_primitive(V& visitor, const uint8_t* value) noexcept; + }; + + template + simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V& visitor) noexcept { + logger::log_start(); + + // + // Start the document + // + if (at_eof()) { return EMPTY; } + log_start_value("document"); + SIMDJSON_TRY(visitor.visit_document_start(*this)); + + // + // Read first value + // + { + auto value = advance(); + + // Make sure the outer object or array is closed before continuing; otherwise, there are ways we + // could get into memory corruption. See https://github.com/simdjson/simdjson/issues/906 + if (!STREAMING) { + switch (*value) { + case '{': if (last_structural() != '}') { log_value("starting brace unmatched"); return TAPE_ERROR; }; break; + case '[': if (last_structural() != ']') { log_value("starting bracket unmatched"); return TAPE_ERROR; }; break; + } + } + + switch (*value) { + case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY(visitor.visit_empty_object(*this)); break; } goto object_begin; + case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY(visitor.visit_empty_array(*this)); break; } goto array_begin; + default: SIMDJSON_TRY(visitor.visit_root_primitive(*this, value)); break; + } + } + goto document_end; + + // + // Object parser states + // + object_begin: + log_start_value("object"); + depth++; + if (depth >= dom_parser.max_depth()) { log_error("Exceeded max depth!"); return DEPTH_ERROR; } + dom_parser.is_array[depth] = false; + SIMDJSON_TRY(visitor.visit_object_start(*this)); + + { + auto key = advance(); + if (*key != '"') { log_error("Object does not start with a key"); return TAPE_ERROR; } + SIMDJSON_TRY(visitor.increment_count(*this)); + SIMDJSON_TRY(visitor.visit_key(*this, key)); + } + + object_field: + if (simdjson_unlikely(*advance() != ':')) { log_error("Missing colon after key in object"); return TAPE_ERROR; } + { + auto value = advance(); + switch (*value) { + case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY(visitor.visit_empty_object(*this)); break; } goto object_begin; + case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY(visitor.visit_empty_array(*this)); break; } goto array_begin; + default: SIMDJSON_TRY(visitor.visit_primitive(*this, value)); break; + } + } + + object_continue: + switch (*advance()) { + case ',': + SIMDJSON_TRY(visitor.increment_count(*this)); + { + auto key = advance(); + if (simdjson_unlikely(*key != '"')) { log_error("Key string missing at beginning of field in object"); return TAPE_ERROR; } + SIMDJSON_TRY(visitor.visit_key(*this, key)); + } + goto object_field; + case '}': log_end_value("object"); SIMDJSON_TRY(visitor.visit_object_end(*this)); goto scope_end; + default: log_error("No comma between object fields"); return TAPE_ERROR; + } + + scope_end: + depth--; + if (depth == 0) { goto document_end; } + if (dom_parser.is_array[depth]) { goto array_continue; } + goto object_continue; + + // + // Array parser states + // + array_begin: + log_start_value("array"); + depth++; + if (depth >= dom_parser.max_depth()) { log_error("Exceeded max depth!"); return DEPTH_ERROR; } + dom_parser.is_array[depth] = true; + SIMDJSON_TRY(visitor.visit_array_start(*this)); + SIMDJSON_TRY(visitor.increment_count(*this)); + + array_value: + { + auto value = advance(); + switch (*value) { + case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY(visitor.visit_empty_object(*this)); break; } goto object_begin; + case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY(visitor.visit_empty_array(*this)); break; } goto array_begin; + default: SIMDJSON_TRY(visitor.visit_primitive(*this, value)); break; + } + } + + array_continue: + switch (*advance()) { + case ',': SIMDJSON_TRY(visitor.increment_count(*this)); goto array_value; + case ']': log_end_value("array"); SIMDJSON_TRY(visitor.visit_array_end(*this)); goto scope_end; + default: log_error("Missing comma between array values"); return TAPE_ERROR; + } + + document_end: + log_end_value("document"); + SIMDJSON_TRY(visitor.visit_document_end(*this)); + + dom_parser.next_structural_index = uint32_t(next_structural - &dom_parser.structural_indexes[0]); + + // If we didn't make it to the end, it's an error + if (!STREAMING && dom_parser.next_structural_index != dom_parser.n_structural_indexes) { + log_error("More than one JSON value at the root of the document, or extra characters at the end of the JSON!"); + return TAPE_ERROR; + } + + return SUCCESS; + + } // walk_document() + + simdjson_inline json_iterator::json_iterator(dom_parser_implementation& _dom_parser, size_t start_structural_index) + : buf{ _dom_parser.buf }, + next_structural{ &_dom_parser.structural_indexes[start_structural_index] }, + dom_parser{ _dom_parser } { + } + + simdjson_inline const uint8_t* json_iterator::peek() const noexcept { + return &buf[*(next_structural)]; + } + simdjson_inline const uint8_t* json_iterator::advance() noexcept { + return &buf[*(next_structural++)]; + } + simdjson_inline size_t json_iterator::remaining_len() const noexcept { + return dom_parser.len - *(next_structural - 1); + } + + simdjson_inline bool json_iterator::at_eof() const noexcept { + return next_structural == &dom_parser.structural_indexes[dom_parser.n_structural_indexes]; + } + simdjson_inline bool json_iterator::at_beginning() const noexcept { + return next_structural == dom_parser.structural_indexes.get(); + } + simdjson_inline uint8_t json_iterator::last_structural() const noexcept { + return buf[dom_parser.structural_indexes[dom_parser.n_structural_indexes - 1]]; + } + + simdjson_inline void json_iterator::log_value(const char* type) const noexcept { + logger::log_line(*this, "", type, ""); + } + + simdjson_inline void json_iterator::log_start_value(const char* type) const noexcept { + logger::log_line(*this, "+", type, ""); + if (logger::LOG_ENABLED) { logger::log_depth++; } + } + + simdjson_inline void json_iterator::log_end_value(const char* type) const noexcept { + if (logger::LOG_ENABLED) { logger::log_depth--; } + logger::log_line(*this, "-", type, ""); + } + + simdjson_inline void json_iterator::log_error(const char* error) const noexcept { + logger::log_line(*this, "", "ERROR", error); + } + + template + simdjson_warn_unused simdjson_inline error_code json_iterator::visit_root_primitive(V& visitor, const uint8_t* value) noexcept { + switch (*value) { + case '"': return visitor.visit_root_string(*this, value); + case 't': return visitor.visit_root_true_atom(*this, value); + case 'f': return visitor.visit_root_false_atom(*this, value); + case 'n': return visitor.visit_root_null_atom(*this, value); + case '-': + case '0': case '1': case '2': case '3': case '4': + case '5': case '6': case '7': case '8': case '9': + return visitor.visit_root_number(*this, value); + default: + log_error("Document starts with a non-value character"); + return TAPE_ERROR; + } + } + template + simdjson_warn_unused simdjson_inline error_code json_iterator::visit_primitive(V& visitor, const uint8_t* value) noexcept { + // Use the fact that most scalars are going to be either strings or numbers. + if (*value == '"') { + return visitor.visit_string(*this, value); + } + else if (((*value - '0') < 10) || (*value == '-')) { + return visitor.visit_number(*this, value); + } + // true, false, null are uncommon. + switch (*value) { + case 't': return visitor.visit_true_atom(*this, value); + case 'f': return visitor.visit_false_atom(*this, value); + case 'n': return visitor.visit_null_atom(*this, value); + default: + log_error("Non-value found when value was expected!"); + return TAPE_ERROR; + } + } + + } // namespace stage2 + } // unnamed namespace + } // namespace lasx +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE2_JSON_ITERATOR_H +/* end file generic/stage2/json_iterator.h for lasx */ +/* including generic/stage2/stringparsing.h for lasx: #include */ +/* begin file generic/stage2/stringparsing.h for lasx */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +// This file contains the common code every implementation uses +// It is intended to be included multiple times and compiled multiple times + +namespace simdjson { + namespace lasx { + namespace { + /// @private + namespace stringparsing { + + // begin copypasta + // These chars yield themselves: " \ / + // b -> backspace, f -> formfeed, n -> newline, r -> cr, t -> horizontal tab + // u not handled in this table as it's complex + static const uint8_t escape_map[256] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x0. + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0x22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x2f, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x4. + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x5c, 0, 0, 0, // 0x5. + 0, 0, 0x08, 0, 0, 0, 0x0c, 0, 0, 0, 0, 0, 0, 0, 0x0a, 0, // 0x6. + 0, 0, 0x0d, 0, 0x09, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x7. + + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + }; + + // handle a unicode codepoint + // write appropriate values into dest + // src will advance 6 bytes or 12 bytes + // dest will advance a variable amount (return via pointer) + // return true if the unicode codepoint was valid + // We work in little-endian then swap at write time + simdjson_warn_unused + simdjson_inline bool handle_unicode_codepoint(const uint8_t** src_ptr, + uint8_t** dst_ptr, bool allow_replacement) { + // Use the default Unicode Character 'REPLACEMENT CHARACTER' (U+FFFD) + constexpr uint32_t substitution_code_point = 0xfffd; + // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the + // conversion is not valid; we defer the check for this to inside the + // multilingual plane check. + uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2); + *src_ptr += 6; + + // If we found a high surrogate, we must + // check for low surrogate for characters + // outside the Basic + // Multilingual Plane. + if (code_point >= 0xd800 && code_point < 0xdc00) { + const uint8_t* src_data = *src_ptr; + /* Compiler optimizations convert this to a single 16-bit load and compare on most platforms */ + if (((src_data[0] << 8) | src_data[1]) != ((static_cast ('\\') << 8) | static_cast ('u'))) { + if (!allow_replacement) { return false; } + code_point = substitution_code_point; + } + else { + uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(src_data + 2); + + // We have already checked that the high surrogate is valid and + // (code_point - 0xd800) < 1024. + // + // Check that code_point_2 is in the range 0xdc00..0xdfff + // and that code_point_2 was parsed from valid hex. + uint32_t low_bit = code_point_2 - 0xdc00; + if (low_bit >> 10) { + if (!allow_replacement) { return false; } + code_point = substitution_code_point; + } + else { + code_point = (((code_point - 0xd800) << 10) | low_bit) + 0x10000; + *src_ptr += 6; + } + + } + } + else if (code_point >= 0xdc00 && code_point <= 0xdfff) { + // If we encounter a low surrogate (not preceded by a high surrogate) + // then we have an error. + if (!allow_replacement) { return false; } + code_point = substitution_code_point; + } + size_t offset = jsoncharutils::codepoint_to_utf8(code_point, *dst_ptr); + *dst_ptr += offset; + return offset > 0; + } + + + // handle a unicode codepoint using the wobbly convention + // https://simonsapin.github.io/wtf-8/ + // write appropriate values into dest + // src will advance 6 bytes or 12 bytes + // dest will advance a variable amount (return via pointer) + // return true if the unicode codepoint was valid + // We work in little-endian then swap at write time + simdjson_warn_unused + simdjson_inline bool handle_unicode_codepoint_wobbly(const uint8_t** src_ptr, + uint8_t** dst_ptr) { + // It is not ideal that this function is nearly identical to handle_unicode_codepoint. + // + // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the + // conversion is not valid; we defer the check for this to inside the + // multilingual plane check. + uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2); + *src_ptr += 6; + // If we found a high surrogate, we must + // check for low surrogate for characters + // outside the Basic + // Multilingual Plane. + if (code_point >= 0xd800 && code_point < 0xdc00) { + const uint8_t* src_data = *src_ptr; + /* Compiler optimizations convert this to a single 16-bit load and compare on most platforms */ + if (((src_data[0] << 8) | src_data[1]) == ((static_cast ('\\') << 8) | static_cast ('u'))) { + uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(src_data + 2); + uint32_t low_bit = code_point_2 - 0xdc00; + if ((low_bit >> 10) == 0) { + code_point = + (((code_point - 0xd800) << 10) | low_bit) + 0x10000; + *src_ptr += 6; + } + } + } + + size_t offset = jsoncharutils::codepoint_to_utf8(code_point, *dst_ptr); + *dst_ptr += offset; + return offset > 0; + } + + + /** + * Unescape a valid UTF-8 string from src to dst, stopping at a final unescaped quote. There + * must be an unescaped quote terminating the string. It returns the final output + * position as pointer. In case of error (e.g., the string has bad escaped codes), + * then null_ptr is returned. It is assumed that the output buffer is large + * enough. E.g., if src points at 'joe"', then dst needs to have four free bytes + + * SIMDJSON_PADDING bytes. + */ + simdjson_warn_unused simdjson_inline uint8_t* parse_string(const uint8_t* src, uint8_t* dst, bool allow_replacement) { + while (1) { + // Copy the next n bytes, and find the backslash and quote in them. + auto bs_quote = backslash_and_quote::copy_and_find(src, dst); + // If the next thing is the end quote, copy and return + if (bs_quote.has_quote_first()) { + // we encountered quotes first. Move dst to point to quotes and exit + return dst + bs_quote.quote_index(); + } + if (bs_quote.has_backslash()) { + /* find out where the backspace is */ + auto bs_dist = bs_quote.backslash_index(); + uint8_t escape_char = src[bs_dist + 1]; + /* we encountered backslash first. Handle backslash */ + if (escape_char == 'u') { + /* move src/dst up to the start; they will be further adjusted + within the unicode codepoint handling code. */ + src += bs_dist; + dst += bs_dist; + if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { + return nullptr; + } + } + else { + /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and + * write bs_dist+1 characters to output + * note this may reach beyond the part of the buffer we've actually + * seen. I think this is ok */ + uint8_t escape_result = escape_map[escape_char]; + if (escape_result == 0u) { + return nullptr; /* bogus escape value is an error */ + } + dst[bs_dist] = escape_result; + src += bs_dist + 2; + dst += bs_dist + 1; + } + } + else { + /* they are the same. Since they can't co-occur, it means we + * encountered neither. */ + src += backslash_and_quote::BYTES_PROCESSED; + dst += backslash_and_quote::BYTES_PROCESSED; + } + } + } + + simdjson_warn_unused simdjson_inline uint8_t* parse_wobbly_string(const uint8_t* src, uint8_t* dst) { + // It is not ideal that this function is nearly identical to parse_string. + while (1) { + // Copy the next n bytes, and find the backslash and quote in them. + auto bs_quote = backslash_and_quote::copy_and_find(src, dst); + // If the next thing is the end quote, copy and return + if (bs_quote.has_quote_first()) { + // we encountered quotes first. Move dst to point to quotes and exit + return dst + bs_quote.quote_index(); + } + if (bs_quote.has_backslash()) { + /* find out where the backspace is */ + auto bs_dist = bs_quote.backslash_index(); + uint8_t escape_char = src[bs_dist + 1]; + /* we encountered backslash first. Handle backslash */ + if (escape_char == 'u') { + /* move src/dst up to the start; they will be further adjusted + within the unicode codepoint handling code. */ + src += bs_dist; + dst += bs_dist; + if (!handle_unicode_codepoint_wobbly(&src, &dst)) { + return nullptr; + } + } + else { + /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and + * write bs_dist+1 characters to output + * note this may reach beyond the part of the buffer we've actually + * seen. I think this is ok */ + uint8_t escape_result = escape_map[escape_char]; + if (escape_result == 0u) { + return nullptr; /* bogus escape value is an error */ + } + dst[bs_dist] = escape_result; + src += bs_dist + 2; + dst += bs_dist + 1; + } + } + else { + /* they are the same. Since they can't co-occur, it means we + * encountered neither. */ + src += backslash_and_quote::BYTES_PROCESSED; + dst += backslash_and_quote::BYTES_PROCESSED; + } + } + } + + } // namespace stringparsing + } // unnamed namespace + } // namespace lasx +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H +/* end file generic/stage2/stringparsing.h for lasx */ +/* including generic/stage2/structural_iterator.h for lasx: #include */ +/* begin file generic/stage2/structural_iterator.h for lasx */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE2_STRUCTURAL_ITERATOR_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_STRUCTURAL_ITERATOR_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lasx { + namespace { + namespace stage2 { + + class structural_iterator { + public: + const uint8_t* const buf; + uint32_t* next_structural; + dom_parser_implementation& dom_parser; + + // Start a structural + simdjson_inline structural_iterator(dom_parser_implementation& _dom_parser, size_t start_structural_index) + : buf{ _dom_parser.buf }, + next_structural{ &_dom_parser.structural_indexes[start_structural_index] }, + dom_parser{ _dom_parser } { + } + // Get the buffer position of the current structural character + simdjson_inline const uint8_t* current() { + return &buf[*(next_structural - 1)]; + } + // Get the current structural character + simdjson_inline char current_char() { + return buf[*(next_structural - 1)]; + } + // Get the next structural character without advancing + simdjson_inline char peek_next_char() { + return buf[*next_structural]; + } + simdjson_inline const uint8_t* peek() { + return &buf[*next_structural]; + } + simdjson_inline const uint8_t* advance() { + return &buf[*(next_structural++)]; + } + simdjson_inline char advance_char() { + return buf[*(next_structural++)]; + } + simdjson_inline size_t remaining_len() { + return dom_parser.len - *(next_structural - 1); + } + + simdjson_inline bool at_end() { + return next_structural == &dom_parser.structural_indexes[dom_parser.n_structural_indexes]; + } + simdjson_inline bool at_beginning() { + return next_structural == dom_parser.structural_indexes.get(); + } + }; + + } // namespace stage2 + } // unnamed namespace + } // namespace lasx +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE2_STRUCTURAL_ITERATOR_H +/* end file generic/stage2/structural_iterator.h for lasx */ +/* including generic/stage2/tape_builder.h for lasx: #include */ +/* begin file generic/stage2/tape_builder.h for lasx */ +#ifndef SIMDJSON_SRC_GENERIC_STAGE2_TAPE_BUILDER_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_TAPE_BUILDER_H */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + + +namespace simdjson { + namespace lasx { + namespace { + namespace stage2 { + + struct tape_builder { + template + simdjson_warn_unused static simdjson_inline error_code parse_document( + dom_parser_implementation& dom_parser, + dom::document& doc) noexcept; + + /** Called when a non-empty document starts. */ + simdjson_warn_unused simdjson_inline error_code visit_document_start(json_iterator& iter) noexcept; + /** Called when a non-empty document ends without error. */ + simdjson_warn_unused simdjson_inline error_code visit_document_end(json_iterator& iter) noexcept; + + /** Called when a non-empty array starts. */ + simdjson_warn_unused simdjson_inline error_code visit_array_start(json_iterator& iter) noexcept; + /** Called when a non-empty array ends. */ + simdjson_warn_unused simdjson_inline error_code visit_array_end(json_iterator& iter) noexcept; + /** Called when an empty array is found. */ + simdjson_warn_unused simdjson_inline error_code visit_empty_array(json_iterator& iter) noexcept; + + /** Called when a non-empty object starts. */ + simdjson_warn_unused simdjson_inline error_code visit_object_start(json_iterator& iter) noexcept; + /** + * Called when a key in a field is encountered. + * + * primitive, visit_object_start, visit_empty_object, visit_array_start, or visit_empty_array + * will be called after this with the field value. + */ + simdjson_warn_unused simdjson_inline error_code visit_key(json_iterator& iter, const uint8_t* key) noexcept; + /** Called when a non-empty object ends. */ + simdjson_warn_unused simdjson_inline error_code visit_object_end(json_iterator& iter) noexcept; + /** Called when an empty object is found. */ + simdjson_warn_unused simdjson_inline error_code visit_empty_object(json_iterator& iter) noexcept; + + /** + * Called when a string, number, boolean or null is found. + */ + simdjson_warn_unused simdjson_inline error_code visit_primitive(json_iterator& iter, const uint8_t* value) noexcept; + /** + * Called when a string, number, boolean or null is found at the top level of a document (i.e. + * when there is no array or object and the entire document is a single string, number, boolean or + * null. + * + * This is separate from primitive() because simdjson's normal primitive parsing routines assume + * there is at least one more token after the value, which is only true in an array or object. + */ + simdjson_warn_unused simdjson_inline error_code visit_root_primitive(json_iterator& iter, const uint8_t* value) noexcept; + + simdjson_warn_unused simdjson_inline error_code visit_string(json_iterator& iter, const uint8_t* value, bool key = false) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_number(json_iterator& iter, const uint8_t* value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_true_atom(json_iterator& iter, const uint8_t* value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_false_atom(json_iterator& iter, const uint8_t* value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_null_atom(json_iterator& iter, const uint8_t* value) noexcept; + + simdjson_warn_unused simdjson_inline error_code visit_root_string(json_iterator& iter, const uint8_t* value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_root_number(json_iterator& iter, const uint8_t* value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_root_true_atom(json_iterator& iter, const uint8_t* value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_root_false_atom(json_iterator& iter, const uint8_t* value) noexcept; + simdjson_warn_unused simdjson_inline error_code visit_root_null_atom(json_iterator& iter, const uint8_t* value) noexcept; + + /** Called each time a new field or element in an array or object is found. */ + simdjson_warn_unused simdjson_inline error_code increment_count(json_iterator& iter) noexcept; + + /** Next location to write to tape */ + tape_writer tape; + private: + /** Next write location in the string buf for stage 2 parsing */ + uint8_t* current_string_buf_loc; + + simdjson_inline tape_builder(dom::document& doc) noexcept; + + simdjson_inline uint32_t next_tape_index(json_iterator& iter) const noexcept; + simdjson_inline void start_container(json_iterator& iter) noexcept; + simdjson_warn_unused simdjson_inline error_code end_container(json_iterator& iter, internal::tape_type start, internal::tape_type end) noexcept; + simdjson_warn_unused simdjson_inline error_code empty_container(json_iterator& iter, internal::tape_type start, internal::tape_type end) noexcept; + simdjson_inline uint8_t* on_start_string(json_iterator& iter) noexcept; + simdjson_inline void on_end_string(uint8_t* dst) noexcept; + }; // struct tape_builder + + template + simdjson_warn_unused simdjson_inline error_code tape_builder::parse_document( + dom_parser_implementation& dom_parser, + dom::document& doc) noexcept { + dom_parser.doc = &doc; + json_iterator iter(dom_parser, STREAMING ? dom_parser.next_structural_index : 0); + tape_builder builder(doc); + return iter.walk_document(builder); + } + + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_primitive(json_iterator& iter, const uint8_t* value) noexcept { + return iter.visit_root_primitive(*this, value); + } + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_primitive(json_iterator& iter, const uint8_t* value) noexcept { + return iter.visit_primitive(*this, value); + } + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_empty_object(json_iterator& iter) noexcept { + return empty_container(iter, internal::tape_type::START_OBJECT, internal::tape_type::END_OBJECT); + } + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_empty_array(json_iterator& iter) noexcept { + return empty_container(iter, internal::tape_type::START_ARRAY, internal::tape_type::END_ARRAY); + } + + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_document_start(json_iterator& iter) noexcept { + start_container(iter); + return SUCCESS; + } + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_object_start(json_iterator& iter) noexcept { + start_container(iter); + return SUCCESS; + } + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_array_start(json_iterator& iter) noexcept { + start_container(iter); + return SUCCESS; + } + + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_object_end(json_iterator& iter) noexcept { + return end_container(iter, internal::tape_type::START_OBJECT, internal::tape_type::END_OBJECT); + } + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_array_end(json_iterator& iter) noexcept { + return end_container(iter, internal::tape_type::START_ARRAY, internal::tape_type::END_ARRAY); + } + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_document_end(json_iterator& iter) noexcept { + constexpr uint32_t start_tape_index = 0; + tape.append(start_tape_index, internal::tape_type::ROOT); + tape_writer::write(iter.dom_parser.doc->tape[start_tape_index], next_tape_index(iter), internal::tape_type::ROOT); + return SUCCESS; + } + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_key(json_iterator& iter, const uint8_t* key) noexcept { + return visit_string(iter, key, true); + } + + simdjson_warn_unused simdjson_inline error_code tape_builder::increment_count(json_iterator& iter) noexcept { + iter.dom_parser.open_containers[iter.depth].count++; // we have a key value pair in the object at parser.dom_parser.depth - 1 + return SUCCESS; + } + + simdjson_inline tape_builder::tape_builder(dom::document& doc) noexcept : tape{ doc.tape.get() }, current_string_buf_loc{ doc.string_buf.get() } {} + + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_string(json_iterator& iter, const uint8_t* value, bool key) noexcept { + iter.log_value(key ? "key" : "string"); + uint8_t* dst = on_start_string(iter); + dst = stringparsing::parse_string(value + 1, dst, false); // We do not allow replacement when the escape characters are invalid. + if (dst == nullptr) { + iter.log_error("Invalid escape in string"); + return STRING_ERROR; + } + on_end_string(dst); + return SUCCESS; + } + + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_string(json_iterator& iter, const uint8_t* value) noexcept { + return visit_string(iter, value); + } + + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_number(json_iterator& iter, const uint8_t* value) noexcept { + iter.log_value("number"); + return numberparsing::parse_number(value, tape); + } + + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_number(json_iterator& iter, const uint8_t* value) noexcept { + // + // We need to make a copy to make sure that the string is space terminated. + // This is not about padding the input, which should already padded up + // to len + SIMDJSON_PADDING. However, we have no control at this stage + // on how the padding was done. What if the input string was padded with nulls? + // It is quite common for an input string to have an extra null character (C string). + // We do not want to allow 9\0 (where \0 is the null character) inside a JSON + // document, but the string "9\0" by itself is fine. So we make a copy and + // pad the input with spaces when we know that there is just one input element. + // This copy is relatively expensive, but it will almost never be called in + // practice unless you are in the strange scenario where you have many JSON + // documents made of single atoms. + // + std::unique_ptrcopy(new (std::nothrow) uint8_t[iter.remaining_len() + SIMDJSON_PADDING]); + if (copy.get() == nullptr) { return MEMALLOC; } + std::memcpy(copy.get(), value, iter.remaining_len()); + std::memset(copy.get() + iter.remaining_len(), ' ', SIMDJSON_PADDING); + error_code error = visit_number(iter, copy.get()); + return error; + } + + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_true_atom(json_iterator& iter, const uint8_t* value) noexcept { + iter.log_value("true"); + if (!atomparsing::is_valid_true_atom(value)) { return T_ATOM_ERROR; } + tape.append(0, internal::tape_type::TRUE_VALUE); + return SUCCESS; + } + + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_true_atom(json_iterator& iter, const uint8_t* value) noexcept { + iter.log_value("true"); + if (!atomparsing::is_valid_true_atom(value, iter.remaining_len())) { return T_ATOM_ERROR; } + tape.append(0, internal::tape_type::TRUE_VALUE); + return SUCCESS; + } + + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_false_atom(json_iterator& iter, const uint8_t* value) noexcept { + iter.log_value("false"); + if (!atomparsing::is_valid_false_atom(value)) { return F_ATOM_ERROR; } + tape.append(0, internal::tape_type::FALSE_VALUE); + return SUCCESS; + } + + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_false_atom(json_iterator& iter, const uint8_t* value) noexcept { + iter.log_value("false"); + if (!atomparsing::is_valid_false_atom(value, iter.remaining_len())) { return F_ATOM_ERROR; } + tape.append(0, internal::tape_type::FALSE_VALUE); + return SUCCESS; + } + + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_null_atom(json_iterator& iter, const uint8_t* value) noexcept { + iter.log_value("null"); + if (!atomparsing::is_valid_null_atom(value)) { return N_ATOM_ERROR; } + tape.append(0, internal::tape_type::NULL_VALUE); + return SUCCESS; + } + + simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_null_atom(json_iterator& iter, const uint8_t* value) noexcept { + iter.log_value("null"); + if (!atomparsing::is_valid_null_atom(value, iter.remaining_len())) { return N_ATOM_ERROR; } + tape.append(0, internal::tape_type::NULL_VALUE); + return SUCCESS; + } + + // private: + + simdjson_inline uint32_t tape_builder::next_tape_index(json_iterator& iter) const noexcept { + return uint32_t(tape.next_tape_loc - iter.dom_parser.doc->tape.get()); + } + + simdjson_warn_unused simdjson_inline error_code tape_builder::empty_container(json_iterator& iter, internal::tape_type start, internal::tape_type end) noexcept { + auto start_index = next_tape_index(iter); + tape.append(start_index + 2, start); + tape.append(start_index, end); + return SUCCESS; + } + + simdjson_inline void tape_builder::start_container(json_iterator& iter) noexcept { + iter.dom_parser.open_containers[iter.depth].tape_index = next_tape_index(iter); + iter.dom_parser.open_containers[iter.depth].count = 0; + tape.skip(); // We don't actually *write* the start element until the end. + } + + simdjson_warn_unused simdjson_inline error_code tape_builder::end_container(json_iterator& iter, internal::tape_type start, internal::tape_type end) noexcept { + // Write the ending tape element, pointing at the start location + const uint32_t start_tape_index = iter.dom_parser.open_containers[iter.depth].tape_index; + tape.append(start_tape_index, end); + // Write the start tape element, pointing at the end location (and including count) + // count can overflow if it exceeds 24 bits... so we saturate + // the convention being that a cnt of 0xffffff or more is undetermined in value (>= 0xffffff). + const uint32_t count = iter.dom_parser.open_containers[iter.depth].count; + const uint32_t cntsat = count > 0xFFFFFF ? 0xFFFFFF : count; + tape_writer::write(iter.dom_parser.doc->tape[start_tape_index], next_tape_index(iter) | (uint64_t(cntsat) << 32), start); + return SUCCESS; + } + + simdjson_inline uint8_t* tape_builder::on_start_string(json_iterator& iter) noexcept { + // we advance the point, accounting for the fact that we have a NULL termination + tape.append(current_string_buf_loc - iter.dom_parser.doc->string_buf.get(), internal::tape_type::STRING); + return current_string_buf_loc + sizeof(uint32_t); + } + + simdjson_inline void tape_builder::on_end_string(uint8_t* dst) noexcept { + uint32_t str_length = uint32_t(dst - (current_string_buf_loc + sizeof(uint32_t))); + // TODO check for overflow in case someone has a crazy string (>=4GB?) + // But only add the overflow check when the document itself exceeds 4GB + // Currently unneeded because we refuse to parse docs larger or equal to 4GB. + memcpy(current_string_buf_loc, &str_length, sizeof(uint32_t)); + // NULL termination is still handy if you expect all your strings to + // be NULL terminated? It comes at a small cost + *dst = 0; + current_string_buf_loc = dst + 1; + } + + } // namespace stage2 + } // unnamed namespace + } // namespace lasx +} // namespace simdjson + +#endif // SIMDJSON_SRC_GENERIC_STAGE2_TAPE_BUILDER_H +/* end file generic/stage2/tape_builder.h for lasx */ +/* end file generic/stage2/amalgamated.h for lasx */ + +// +// Stage 1 +// +namespace simdjson { + namespace lasx { + + simdjson_warn_unused error_code implementation::create_dom_parser_implementation( + size_t capacity, + size_t max_depth, + std::unique_ptr& dst + ) const noexcept { + dst.reset(new (std::nothrow) dom_parser_implementation()); + if (!dst) { return MEMALLOC; } + if (auto err = dst->set_capacity(capacity)) + return err; + if (auto err = dst->set_max_depth(max_depth)) + return err; + return SUCCESS; + } + + namespace { + + using namespace simd; + + simdjson_inline json_character_block json_character_block::classify(const simd::simd8x64& in) { + // Inspired by haswell. + // LASX use low 5 bits as index. For the 6 operators (:,[]{}), the unique-5bits is [6:2]. + // The ASCII white-space and operators have these values: (char, hex, unique-5bits) + // (' ', 20, 00000) ('\t', 09, 01001) ('\n', 0A, 01010) ('\r', 0D, 01101) + // (',', 2C, 01011) (':', 3A, 01110) ('[', 5B, 10110) ('{', 7B, 11110) (']', 5D, 10111) ('}', 7D, 11111) + const simd8 ws_table = simd8::repeat_16( + ' ', 0, 0, 0, 0, 0, 0, 0, 0, '\t', '\n', 0, 0, '\r', 0, 0 + ); + const simd8 op_table_lo = simd8::repeat_16( + 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ',', 0, 0, ':', 0 + ); + const simd8 op_table_hi = simd8::repeat_16( + 0, 0, 0, 0, 0, 0, '[', ']', 0, 0, 0, 0, 0, 0, '{', '}' + ); + uint64_t ws = in.eq({ + in.chunks[0].lookup_16(ws_table), + in.chunks[1].lookup_16(ws_table), + }); + uint64_t op = in.eq({ + __lasx_xvshuf_b(op_table_hi, op_table_lo, in.chunks[0].shr<2>()), + __lasx_xvshuf_b(op_table_hi, op_table_lo, in.chunks[1].shr<2>()), + }); + + return { ws, op }; + } + + simdjson_inline bool is_ascii(const simd8x64& input) { + return input.reduce_or().is_ascii(); + } + + simdjson_inline simd8 must_be_2_3_continuation(const simd8 prev2, const simd8 prev3) { + simd8 is_third_byte = prev2.saturating_sub(0xe0u - 0x80); // Only 111_____ will be >= 0x80 + simd8 is_fourth_byte = prev3.saturating_sub(0xf0u - 0x80); // Only 1111____ will be >= 0x80 + return is_third_byte | is_fourth_byte; + } + + } // unnamed namespace + } // namespace lasx +} // namespace simdjson + +// +// Stage 2 +// + +// +// Implementation-specific overrides +// +namespace simdjson { + namespace lasx { + + simdjson_warn_unused error_code implementation::minify(const uint8_t* buf, size_t len, uint8_t* dst, size_t& dst_len) const noexcept { + return lasx::stage1::json_minifier::minify<64>(buf, len, dst, dst_len); + } + + simdjson_warn_unused error_code dom_parser_implementation::stage1(const uint8_t* _buf, size_t _len, stage1_mode streaming) noexcept { + this->buf = _buf; + this->len = _len; + return lasx::stage1::json_structural_indexer::index<64>(buf, len, *this, streaming); + } + + simdjson_warn_unused bool implementation::validate_utf8(const char* buf, size_t len) const noexcept { + return lasx::stage1::generic_validate_utf8(buf, len); + } + + simdjson_warn_unused error_code dom_parser_implementation::stage2(dom::document& _doc) noexcept { + return stage2::tape_builder::parse_document(*this, _doc); + } + + simdjson_warn_unused error_code dom_parser_implementation::stage2_next(dom::document& _doc) noexcept { + return stage2::tape_builder::parse_document(*this, _doc); + } + + simdjson_warn_unused uint8_t* dom_parser_implementation::parse_string(const uint8_t* src, uint8_t* dst, bool allow_replacement) const noexcept { + return lasx::stringparsing::parse_string(src, dst, allow_replacement); + } + + simdjson_warn_unused uint8_t* dom_parser_implementation::parse_wobbly_string(const uint8_t* src, uint8_t* dst) const noexcept { + return lasx::stringparsing::parse_wobbly_string(src, dst); + } + + simdjson_warn_unused error_code dom_parser_implementation::parse(const uint8_t* _buf, size_t _len, dom::document& _doc) noexcept { + auto error = stage1(_buf, _len, stage1_mode::regular); + if (error) { return error; } + return stage2(_doc); + } + + } // namespace lasx +} // namespace simdjson + +/* including simdjson/lasx/end.h: #include */ +/* begin file simdjson/lasx/end.h */ +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/lasx/base.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +#undef SIMDJSON_SKIP_BACKSLASH_SHORT_CIRCUIT +/* undefining SIMDJSON_IMPLEMENTATION from "lasx" */ +#undef SIMDJSON_IMPLEMENTATION +/* end file simdjson/lasx/end.h */ + +#endif // SIMDJSON_SRC_LASX_CPP +/* end file lasx.cpp */ +#endif +#if SIMDJSON_IMPLEMENTATION_FALLBACK +/* including fallback.cpp: #include */ +/* begin file fallback.cpp */ +#ifndef SIMDJSON_SRC_FALLBACK_CPP +#define SIMDJSON_SRC_FALLBACK_CPP + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +/* including simdjson/fallback.h: #include */ +/* begin file simdjson/fallback.h */ +#ifndef SIMDJSON_FALLBACK_H +#define SIMDJSON_FALLBACK_H + +/* including simdjson/fallback/begin.h: #include "simdjson/fallback/begin.h" */ +/* begin file simdjson/fallback/begin.h */ +/* defining SIMDJSON_IMPLEMENTATION to "fallback" */ +#define SIMDJSON_IMPLEMENTATION fallback +/* including simdjson/fallback/base.h: #include "simdjson/fallback/base.h" */ +/* begin file simdjson/fallback/base.h */ +#ifndef SIMDJSON_FALLBACK_BASE_H +#define SIMDJSON_FALLBACK_BASE_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/base.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + /** + * Fallback implementation (runs on any machine). + */ + namespace fallback { + + class implementation; + + } // namespace fallback +} // namespace simdjson + +#endif // SIMDJSON_FALLBACK_BASE_H +/* end file simdjson/fallback/base.h */ +/* including simdjson/fallback/bitmanipulation.h: #include "simdjson/fallback/bitmanipulation.h" */ +/* begin file simdjson/fallback/bitmanipulation.h */ +#ifndef SIMDJSON_FALLBACK_BITMANIPULATION_H +#define SIMDJSON_FALLBACK_BITMANIPULATION_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/fallback/base.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace fallback { + namespace { + +#if defined(_MSC_VER) && !defined(_M_ARM64) && !defined(_M_X64) + static inline unsigned char _BitScanForward64(unsigned long* ret, uint64_t x) { + unsigned long x0 = (unsigned long)x, top, bottom; + _BitScanForward(&top, (unsigned long)(x >> 32)); + _BitScanForward(&bottom, x0); + *ret = x0 ? bottom : 32 + top; + return x != 0; + } + static unsigned char _BitScanReverse64(unsigned long* ret, uint64_t x) { + unsigned long x1 = (unsigned long)(x >> 32), top, bottom; + _BitScanReverse(&top, x1); + _BitScanReverse(&bottom, (unsigned long)x); + *ret = x1 ? top + 32 : bottom; + return x != 0; + } +#endif + + /* result might be undefined when input_num is zero */ + simdjson_inline int leading_zeroes(uint64_t input_num) { +#ifdef _MSC_VER + unsigned long leading_zero = 0; + // Search the mask data from most significant bit (MSB) + // to least significant bit (LSB) for a set bit (1). + if (_BitScanReverse64(&leading_zero, input_num)) + return (int)(63 - leading_zero); + else + return 64; +#else + return __builtin_clzll(input_num); +#endif// _MSC_VER + } + + } // unnamed namespace + } // namespace fallback +} // namespace simdjson + +#endif // SIMDJSON_FALLBACK_BITMANIPULATION_H +/* end file simdjson/fallback/bitmanipulation.h */ +/* including simdjson/fallback/stringparsing_defs.h: #include "simdjson/fallback/stringparsing_defs.h" */ +/* begin file simdjson/fallback/stringparsing_defs.h */ +#ifndef SIMDJSON_FALLBACK_STRINGPARSING_DEFS_H +#define SIMDJSON_FALLBACK_STRINGPARSING_DEFS_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/fallback/base.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace fallback { + namespace { + + // Holds backslashes and quotes locations. + struct backslash_and_quote { + public: + static constexpr uint32_t BYTES_PROCESSED = 1; + simdjson_inline static backslash_and_quote copy_and_find(const uint8_t* src, uint8_t* dst); + + simdjson_inline bool has_quote_first() { return c == '"'; } + simdjson_inline bool has_backslash() { return c == '\\'; } + simdjson_inline int quote_index() { return c == '"' ? 0 : 1; } + simdjson_inline int backslash_index() { return c == '\\' ? 0 : 1; } + + uint8_t c; + }; // struct backslash_and_quote + + simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8_t* src, uint8_t* dst) { + // store to dest unconditionally - we can overwrite the bits we don't like later + dst[0] = src[0]; + return { src[0] }; + } + + } // unnamed namespace + } // namespace fallback +} // namespace simdjson + +#endif // SIMDJSON_FALLBACK_STRINGPARSING_DEFS_H +/* end file simdjson/fallback/stringparsing_defs.h */ +/* including simdjson/fallback/numberparsing_defs.h: #include "simdjson/fallback/numberparsing_defs.h" */ +/* begin file simdjson/fallback/numberparsing_defs.h */ +#ifndef SIMDJSON_FALLBACK_NUMBERPARSING_DEFS_H +#define SIMDJSON_FALLBACK_NUMBERPARSING_DEFS_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/fallback/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +#include + +#ifdef JSON_TEST_NUMBERS // for unit testing +void found_invalid_number(const uint8_t* buf); +void found_integer(int64_t result, const uint8_t* buf); +void found_unsigned_integer(uint64_t result, const uint8_t* buf); +void found_float(double result, const uint8_t* buf); +#endif + +namespace simdjson { + namespace fallback { + namespace numberparsing { + + // credit: https://johnnylee-sde.github.io/Fast-numeric-string-to-int/ + /** @private */ + static simdjson_inline uint32_t parse_eight_digits_unrolled(const char* chars) { + uint64_t val; + memcpy(&val, chars, sizeof(uint64_t)); + val = (val & 0x0F0F0F0F0F0F0F0F) * 2561 >> 8; + val = (val & 0x00FF00FF00FF00FF) * 6553601 >> 16; + return uint32_t((val & 0x0000FFFF0000FFFF) * 42949672960001 >> 32); + } + + /** @private */ + static simdjson_inline uint32_t parse_eight_digits_unrolled(const uint8_t* chars) { + return parse_eight_digits_unrolled(reinterpret_cast(chars)); + } + +#if SIMDJSON_IS_32BITS // _umul128 for x86, arm + // this is a slow emulation routine for 32-bit + // + static simdjson_inline uint64_t __emulu(uint32_t x, uint32_t y) { + return x * (uint64_t)y; + } + static simdjson_inline uint64_t _umul128(uint64_t ab, uint64_t cd, uint64_t* hi) { + uint64_t ad = __emulu((uint32_t)(ab >> 32), (uint32_t)cd); + uint64_t bd = __emulu((uint32_t)ab, (uint32_t)cd); + uint64_t adbc = ad + __emulu((uint32_t)ab, (uint32_t)(cd >> 32)); + uint64_t adbc_carry = !!(adbc < ad); + uint64_t lo = bd + (adbc << 32); + *hi = __emulu((uint32_t)(ab >> 32), (uint32_t)(cd >> 32)) + (adbc >> 32) + + (adbc_carry << 32) + !!(lo < bd); + return lo; + } +#endif + + /** @private */ + simdjson_inline internal::value128 full_multiplication(uint64_t value1, uint64_t value2) { + internal::value128 answer; +#if SIMDJSON_REGULAR_VISUAL_STUDIO || SIMDJSON_IS_32BITS +#if SIMDJSON_IS_ARM64 + // ARM64 has native support for 64-bit multiplications, no need to emultate + answer.high = __umulh(value1, value2); + answer.low = value1 * value2; +#else + answer.low = _umul128(value1, value2, &answer.high); // _umul128 not available on ARM64 +#endif // SIMDJSON_IS_ARM64 +#else // SIMDJSON_REGULAR_VISUAL_STUDIO || SIMDJSON_IS_32BITS + __uint128_t r = (static_cast<__uint128_t>(value1)) * value2; + answer.low = uint64_t(r); + answer.high = uint64_t(r >> 64); +#endif + return answer; + } + + } // namespace numberparsing + } // namespace fallback +} // namespace simdjson + +#define SIMDJSON_SWAR_NUMBER_PARSING 1 + +#endif // SIMDJSON_FALLBACK_NUMBERPARSING_DEFS_H +/* end file simdjson/fallback/numberparsing_defs.h */ +/* end file simdjson/fallback/begin.h */ +/* including simdjson/generic/amalgamated.h for fallback: #include "simdjson/generic/amalgamated.h" */ +/* begin file simdjson/generic/amalgamated.h for fallback */ +#if defined(SIMDJSON_CONDITIONAL_INCLUDE) && !defined(SIMDJSON_GENERIC_DEPENDENCIES_H) +#error simdjson/generic/dependencies.h must be included before simdjson/generic/amalgamated.h! +#endif + +/* including simdjson/generic/base.h for fallback: #include "simdjson/generic/base.h" */ +/* begin file simdjson/generic/base.h for fallback */ +#ifndef SIMDJSON_GENERIC_BASE_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_BASE_H */ +/* amalgamation skipped (editor-only): #include "simdjson/base.h" */ +/* amalgamation skipped (editor-only): // If we haven't got an implementation yet, we're in the editor, editing a generic file! Just */ +/* amalgamation skipped (editor-only): // use the most advanced one we can so the most possible stuff can be tested. */ +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_IMPLEMENTATION */ +/* amalgamation skipped (editor-only): #include "simdjson/implementation_detection.h" */ +/* amalgamation skipped (editor-only): #if SIMDJSON_IMPLEMENTATION_ICELAKE */ +/* amalgamation skipped (editor-only): #include "simdjson/icelake/begin.h" */ +/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_HASWELL */ +/* amalgamation skipped (editor-only): #include "simdjson/haswell/begin.h" */ +/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_WESTMERE */ +/* amalgamation skipped (editor-only): #include "simdjson/westmere/begin.h" */ +/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_ARM64 */ +/* amalgamation skipped (editor-only): #include "simdjson/arm64/begin.h" */ +/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_PPC64 */ +/* amalgamation skipped (editor-only): #include "simdjson/ppc64/begin.h" */ +/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_LSX */ +/* amalgamation skipped (editor-only): #include "simdjson/lsx/begin.h" */ +/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_LASX */ +/* amalgamation skipped (editor-only): #include "simdjson/lasx/begin.h" */ +/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_FALLBACK */ +/* amalgamation skipped (editor-only): #include "simdjson/fallback/begin.h" */ +/* amalgamation skipped (editor-only): #else */ +/* amalgamation skipped (editor-only): #error "All possible implementations (including fallback) have been disabled! simdjson will not run." */ +/* amalgamation skipped (editor-only): #endif */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_IMPLEMENTATION */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace fallback { + + struct open_container; + class dom_parser_implementation; + + /** + * The type of a JSON number + */ + enum class number_type { + floating_point_number = 1, /// a binary64 number + signed_integer, /// a signed integer that fits in a 64-bit word using two's complement + unsigned_integer, /// a positive integer larger or equal to 1<<63 + big_integer /// a big integer that does not fit in a 64-bit word + }; + + } // namespace fallback +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_BASE_H +/* end file simdjson/generic/base.h for fallback */ +/* including simdjson/generic/jsoncharutils.h for fallback: #include "simdjson/generic/jsoncharutils.h" */ +/* begin file simdjson/generic/jsoncharutils.h for fallback */ +#ifndef SIMDJSON_GENERIC_JSONCHARUTILS_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_JSONCHARUTILS_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/jsoncharutils_tables.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace fallback { + namespace { + namespace jsoncharutils { + + // return non-zero if not a structural or whitespace char + // zero otherwise + simdjson_inline uint32_t is_not_structural_or_whitespace(uint8_t c) { + return internal::structural_or_whitespace_negated[c]; + } + + simdjson_inline uint32_t is_structural_or_whitespace(uint8_t c) { + return internal::structural_or_whitespace[c]; + } + + // returns a value with the high 16 bits set if not valid + // otherwise returns the conversion of the 4 hex digits at src into the bottom + // 16 bits of the 32-bit return register + // + // see + // https://lemire.me/blog/2019/04/17/parsing-short-hexadecimal-strings-efficiently/ + static inline uint32_t hex_to_u32_nocheck( + const uint8_t* src) { // strictly speaking, static inline is a C-ism + uint32_t v1 = internal::digit_to_val32[630 + src[0]]; + uint32_t v2 = internal::digit_to_val32[420 + src[1]]; + uint32_t v3 = internal::digit_to_val32[210 + src[2]]; + uint32_t v4 = internal::digit_to_val32[0 + src[3]]; + return v1 | v2 | v3 | v4; + } + + // given a code point cp, writes to c + // the utf-8 code, outputting the length in + // bytes, if the length is zero, the code point + // is invalid + // + // This can possibly be made faster using pdep + // and clz and table lookups, but JSON documents + // have few escaped code points, and the following + // function looks cheap. + // + // Note: we assume that surrogates are treated separately + // + simdjson_inline size_t codepoint_to_utf8(uint32_t cp, uint8_t* c) { + if (cp <= 0x7F) { + c[0] = uint8_t(cp); + return 1; // ascii + } + if (cp <= 0x7FF) { + c[0] = uint8_t((cp >> 6) + 192); + c[1] = uint8_t((cp & 63) + 128); + return 2; // universal plane + // Surrogates are treated elsewhere... + //} //else if (0xd800 <= cp && cp <= 0xdfff) { + // return 0; // surrogates // could put assert here + } + else if (cp <= 0xFFFF) { + c[0] = uint8_t((cp >> 12) + 224); + c[1] = uint8_t(((cp >> 6) & 63) + 128); + c[2] = uint8_t((cp & 63) + 128); + return 3; + } + else if (cp <= 0x10FFFF) { // if you know you have a valid code point, this + // is not needed + c[0] = uint8_t((cp >> 18) + 240); + c[1] = uint8_t(((cp >> 12) & 63) + 128); + c[2] = uint8_t(((cp >> 6) & 63) + 128); + c[3] = uint8_t((cp & 63) + 128); + return 4; + } + // will return 0 when the code point was too large. + return 0; // bad r + } + +#if SIMDJSON_IS_32BITS // _umul128 for x86, arm + // this is a slow emulation routine for 32-bit + // + static simdjson_inline uint64_t __emulu(uint32_t x, uint32_t y) { + return x * (uint64_t)y; + } + static simdjson_inline uint64_t _umul128(uint64_t ab, uint64_t cd, uint64_t* hi) { + uint64_t ad = __emulu((uint32_t)(ab >> 32), (uint32_t)cd); + uint64_t bd = __emulu((uint32_t)ab, (uint32_t)cd); + uint64_t adbc = ad + __emulu((uint32_t)ab, (uint32_t)(cd >> 32)); + uint64_t adbc_carry = !!(adbc < ad); + uint64_t lo = bd + (adbc << 32); + *hi = __emulu((uint32_t)(ab >> 32), (uint32_t)(cd >> 32)) + (adbc >> 32) + + (adbc_carry << 32) + !!(lo < bd); + return lo; + } +#endif + + } // namespace jsoncharutils + } // unnamed namespace + } // namespace fallback +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_JSONCHARUTILS_H +/* end file simdjson/generic/jsoncharutils.h for fallback */ +/* including simdjson/generic/atomparsing.h for fallback: #include "simdjson/generic/atomparsing.h" */ +/* begin file simdjson/generic/atomparsing.h for fallback */ +#ifndef SIMDJSON_GENERIC_ATOMPARSING_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_ATOMPARSING_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/jsoncharutils.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +#include + +namespace simdjson { + namespace fallback { + namespace { + /// @private + namespace atomparsing { + + // The string_to_uint32 is exclusively used to map literal strings to 32-bit values. + // We use memcpy instead of a pointer cast to avoid undefined behaviors since we cannot + // be certain that the character pointer will be properly aligned. + // You might think that using memcpy makes this function expensive, but you'd be wrong. + // All decent optimizing compilers (GCC, clang, Visual Studio) will compile string_to_uint32("false"); + // to the compile-time constant 1936482662. + simdjson_inline uint32_t string_to_uint32(const char* str) { uint32_t val; std::memcpy(&val, str, sizeof(uint32_t)); return val; } + + + // Again in str4ncmp we use a memcpy to avoid undefined behavior. The memcpy may appear expensive. + // Yet all decent optimizing compilers will compile memcpy to a single instruction, just about. + simdjson_warn_unused + simdjson_inline uint32_t str4ncmp(const uint8_t* src, const char* atom) { + uint32_t srcval; // we want to avoid unaligned 32-bit loads (undefined in C/C++) + static_assert(sizeof(uint32_t) <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be larger than 4 bytes"); + std::memcpy(&srcval, src, sizeof(uint32_t)); + return srcval ^ string_to_uint32(atom); + } + + simdjson_warn_unused + simdjson_inline bool is_valid_true_atom(const uint8_t* src) { + return (str4ncmp(src, "true") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; + } + + simdjson_warn_unused + simdjson_inline bool is_valid_true_atom(const uint8_t* src, size_t len) { + if (len > 4) { return is_valid_true_atom(src); } + else if (len == 4) { return !str4ncmp(src, "true"); } + else { return false; } + } + + simdjson_warn_unused + simdjson_inline bool is_valid_false_atom(const uint8_t* src) { + return (str4ncmp(src + 1, "alse") | jsoncharutils::is_not_structural_or_whitespace(src[5])) == 0; + } + + simdjson_warn_unused + simdjson_inline bool is_valid_false_atom(const uint8_t* src, size_t len) { + if (len > 5) { return is_valid_false_atom(src); } + else if (len == 5) { return !str4ncmp(src + 1, "alse"); } + else { return false; } + } + + simdjson_warn_unused + simdjson_inline bool is_valid_null_atom(const uint8_t* src) { + return (str4ncmp(src, "null") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; + } + + simdjson_warn_unused + simdjson_inline bool is_valid_null_atom(const uint8_t* src, size_t len) { + if (len > 4) { return is_valid_null_atom(src); } + else if (len == 4) { return !str4ncmp(src, "null"); } + else { return false; } + } + + } // namespace atomparsing + } // unnamed namespace + } // namespace fallback +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_ATOMPARSING_H +/* end file simdjson/generic/atomparsing.h for fallback */ +/* including simdjson/generic/dom_parser_implementation.h for fallback: #include "simdjson/generic/dom_parser_implementation.h" */ +/* begin file simdjson/generic/dom_parser_implementation.h for fallback */ +#ifndef SIMDJSON_GENERIC_DOM_PARSER_IMPLEMENTATION_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_DOM_PARSER_IMPLEMENTATION_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/dom_parser_implementation.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace fallback { + + // expectation: sizeof(open_container) = 64/8. + struct open_container { + uint32_t tape_index; // where, on the tape, does the scope ([,{) begins + uint32_t count; // how many elements in the scope + }; // struct open_container + + static_assert(sizeof(open_container) == 64 / 8, "Open container must be 64 bits"); + + class dom_parser_implementation final : public internal::dom_parser_implementation { + public: + /** Tape location of each open { or [ */ + std::unique_ptr open_containers{}; + /** Whether each open container is a [ or { */ + std::unique_ptr is_array{}; + /** Buffer passed to stage 1 */ + const uint8_t* buf{}; + /** Length passed to stage 1 */ + size_t len{ 0 }; + /** Document passed to stage 2 */ + dom::document* doc{}; + + inline dom_parser_implementation() noexcept; + inline dom_parser_implementation(dom_parser_implementation&& other) noexcept; + inline dom_parser_implementation& operator=(dom_parser_implementation&& other) noexcept; + dom_parser_implementation(const dom_parser_implementation&) = delete; + dom_parser_implementation& operator=(const dom_parser_implementation&) = delete; + + simdjson_warn_unused error_code parse(const uint8_t* buf, size_t len, dom::document& doc) noexcept final; + simdjson_warn_unused error_code stage1(const uint8_t* buf, size_t len, stage1_mode partial) noexcept final; + simdjson_warn_unused error_code stage2(dom::document& doc) noexcept final; + simdjson_warn_unused error_code stage2_next(dom::document& doc) noexcept final; + simdjson_warn_unused uint8_t* parse_string(const uint8_t* src, uint8_t* dst, bool allow_replacement) const noexcept final; + simdjson_warn_unused uint8_t* parse_wobbly_string(const uint8_t* src, uint8_t* dst) const noexcept final; + inline simdjson_warn_unused error_code set_capacity(size_t capacity) noexcept final; + inline simdjson_warn_unused error_code set_max_depth(size_t max_depth) noexcept final; + private: + simdjson_inline simdjson_warn_unused error_code set_capacity_stage1(size_t capacity); + + }; + + } // namespace fallback +} // namespace simdjson + +namespace simdjson { + namespace fallback { + + inline dom_parser_implementation::dom_parser_implementation() noexcept = default; + inline dom_parser_implementation::dom_parser_implementation(dom_parser_implementation&& other) noexcept = default; + inline dom_parser_implementation& dom_parser_implementation::operator=(dom_parser_implementation&& other) noexcept = default; + + // Leaving these here so they can be inlined if so desired + inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(size_t capacity) noexcept { + if (capacity > SIMDJSON_MAXSIZE_BYTES) { return CAPACITY; } + // Stage 1 index output + size_t max_structures = SIMDJSON_ROUNDUP_N(capacity, 64) + 2 + 7; + structural_indexes.reset(new (std::nothrow) uint32_t[max_structures]); + if (!structural_indexes) { _capacity = 0; return MEMALLOC; } + structural_indexes[0] = 0; + n_structural_indexes = 0; + + _capacity = capacity; + return SUCCESS; + } + + inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { + // Stage 2 stacks + open_containers.reset(new (std::nothrow) open_container[max_depth]); + is_array.reset(new (std::nothrow) bool[max_depth]); + if (!is_array || !open_containers) { _max_depth = 0; return MEMALLOC; } + + _max_depth = max_depth; + return SUCCESS; + } + + } // namespace fallback +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_DOM_PARSER_IMPLEMENTATION_H +/* end file simdjson/generic/dom_parser_implementation.h for fallback */ +/* including simdjson/generic/implementation_simdjson_result_base.h for fallback: #include "simdjson/generic/implementation_simdjson_result_base.h" */ +/* begin file simdjson/generic/implementation_simdjson_result_base.h for fallback */ +#ifndef SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace fallback { + + // This is a near copy of include/error.h's implementation_simdjson_result_base, except it doesn't use std::pair + // so we can avoid inlining errors + // TODO reconcile these! + /** + * The result of a simdjson operation that could fail. + * + * Gives the option of reading error codes, or throwing an exception by casting to the desired result. + * + * This is a base class for implementations that want to add functions to the result type for + * chaining. + * + * Override like: + * + * struct simdjson_result : public internal::implementation_simdjson_result_base { + * simdjson_result() noexcept : internal::implementation_simdjson_result_base() {} + * simdjson_result(error_code error) noexcept : internal::implementation_simdjson_result_base(error) {} + * simdjson_result(T &&value) noexcept : internal::implementation_simdjson_result_base(std::forward(value)) {} + * simdjson_result(T &&value, error_code error) noexcept : internal::implementation_simdjson_result_base(value, error) {} + * // Your extra methods here + * } + * + * Then any method returning simdjson_result will be chainable with your methods. + */ + template + struct implementation_simdjson_result_base { + + /** + * Create a new empty result with error = UNINITIALIZED. + */ + simdjson_inline implementation_simdjson_result_base() noexcept = default; + + /** + * Create a new error result. + */ + simdjson_inline implementation_simdjson_result_base(error_code error) noexcept; + + /** + * Create a new successful result. + */ + simdjson_inline implementation_simdjson_result_base(T&& value) noexcept; + + /** + * Create a new result with both things (use if you don't want to branch when creating the result). + */ + simdjson_inline implementation_simdjson_result_base(T&& value, error_code error) noexcept; + + /** + * Move the value and the error to the provided variables. + * + * @param value The variable to assign the value to. May not be set if there is an error. + * @param error The variable to assign the error to. Set to SUCCESS if there is no error. + */ + simdjson_inline void tie(T& value, error_code& error) && noexcept; + + /** + * Move the value to the provided variable. + * + * @param value The variable to assign the value to. May not be set if there is an error. + */ + simdjson_inline error_code get(T& value) && noexcept; + + /** + * The error. + */ + simdjson_inline error_code error() const noexcept; + +#if SIMDJSON_EXCEPTIONS + + /** + * Get the result value. + * + * @throw simdjson_error if there was an error. + */ + simdjson_inline T& value() & noexcept(false); + + /** + * Take the result value (move it). + * + * @throw simdjson_error if there was an error. + */ + simdjson_inline T&& value() && noexcept(false); + + /** + * Take the result value (move it). + * + * @throw simdjson_error if there was an error. + */ + simdjson_inline T&& take_value() && noexcept(false); + + /** + * Cast to the value (will throw on error). + * + * @throw simdjson_error if there was an error. + */ + simdjson_inline operator T && () && noexcept(false); + + +#endif // SIMDJSON_EXCEPTIONS + + /** + * Get the result value. This function is safe if and only + * the error() method returns a value that evaluates to false. + */ + simdjson_inline const T& value_unsafe() const& noexcept; + /** + * Get the result value. This function is safe if and only + * the error() method returns a value that evaluates to false. + */ + simdjson_inline T& value_unsafe() & noexcept; + /** + * Take the result value (move it). This function is safe if and only + * the error() method returns a value that evaluates to false. + */ + simdjson_inline T&& value_unsafe() && noexcept; + protected: + /** users should never directly access first and second. **/ + T first{}; /** Users should never directly access 'first'. **/ + error_code second{ UNINITIALIZED }; /** Users should never directly access 'second'. **/ + }; // struct implementation_simdjson_result_base + + } // namespace fallback +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_H +/* end file simdjson/generic/implementation_simdjson_result_base.h for fallback */ +/* including simdjson/generic/numberparsing.h for fallback: #include "simdjson/generic/numberparsing.h" */ +/* begin file simdjson/generic/numberparsing.h for fallback */ +#ifndef SIMDJSON_GENERIC_NUMBERPARSING_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_NUMBERPARSING_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/jsoncharutils.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +#include +#include +#include + +namespace simdjson { + namespace fallback { + namespace numberparsing { + +#ifdef JSON_TEST_NUMBERS +#define INVALID_NUMBER(SRC) (found_invalid_number((SRC)), NUMBER_ERROR) +#define WRITE_INTEGER(VALUE, SRC, WRITER) (found_integer((VALUE), (SRC)), (WRITER).append_s64((VALUE))) +#define WRITE_UNSIGNED(VALUE, SRC, WRITER) (found_unsigned_integer((VALUE), (SRC)), (WRITER).append_u64((VALUE))) +#define WRITE_DOUBLE(VALUE, SRC, WRITER) (found_float((VALUE), (SRC)), (WRITER).append_double((VALUE))) +#define BIGINT_NUMBER(SRC) (found_invalid_number((SRC)), BIGINT_ERROR) +#else +#define INVALID_NUMBER(SRC) (NUMBER_ERROR) +#define WRITE_INTEGER(VALUE, SRC, WRITER) (WRITER).append_s64((VALUE)) +#define WRITE_UNSIGNED(VALUE, SRC, WRITER) (WRITER).append_u64((VALUE)) +#define WRITE_DOUBLE(VALUE, SRC, WRITER) (WRITER).append_double((VALUE)) +#define BIGINT_NUMBER(SRC) (BIGINT_ERROR) +#endif + + namespace { + + // Convert a mantissa, an exponent and a sign bit into an ieee64 double. + // The real_exponent needs to be in [0, 2046] (technically real_exponent = 2047 would be acceptable). + // The mantissa should be in [0,1<<53). The bit at index (1ULL << 52) while be zeroed. + simdjson_inline double to_double(uint64_t mantissa, uint64_t real_exponent, bool negative) { + double d; + mantissa &= ~(1ULL << 52); + mantissa |= real_exponent << 52; + mantissa |= ((static_cast(negative)) << 63); + std::memcpy(&d, &mantissa, sizeof(d)); + return d; + } + + // Attempts to compute i * 10^(power) exactly; and if "negative" is + // true, negate the result. + // This function will only work in some cases, when it does not work, success is + // set to false. This should work *most of the time* (like 99% of the time). + // We assume that power is in the [smallest_power, + // largest_power] interval: the caller is responsible for this check. + simdjson_inline bool compute_float_64(int64_t power, uint64_t i, bool negative, double& d) { + // we start with a fast path + // It was described in + // Clinger WD. How to read floating point numbers accurately. + // ACM SIGPLAN Notices. 1990 +#ifndef FLT_EVAL_METHOD +#error "FLT_EVAL_METHOD should be defined, please include cfloat." +#endif +#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0) + // We cannot be certain that x/y is rounded to nearest. + if (0 <= power && power <= 22 && i <= 9007199254740991) +#else + if (-22 <= power && power <= 22 && i <= 9007199254740991) +#endif + { + // convert the integer into a double. This is lossless since + // 0 <= i <= 2^53 - 1. + d = double(i); + // + // The general idea is as follows. + // If 0 <= s < 2^53 and if 10^0 <= p <= 10^22 then + // 1) Both s and p can be represented exactly as 64-bit floating-point + // values + // (binary64). + // 2) Because s and p can be represented exactly as floating-point values, + // then s * p + // and s / p will produce correctly rounded values. + // + if (power < 0) { + d = d / simdjson::internal::power_of_ten[-power]; + } + else { + d = d * simdjson::internal::power_of_ten[power]; + } + if (negative) { + d = -d; + } + return true; + } + // When 22 < power && power < 22 + 16, we could + // hope for another, secondary fast path. It was + // described by David M. Gay in "Correctly rounded + // binary-decimal and decimal-binary conversions." (1990) + // If you need to compute i * 10^(22 + x) for x < 16, + // first compute i * 10^x, if you know that result is exact + // (e.g., when i * 10^x < 2^53), + // then you can still proceed and do (i * 10^x) * 10^22. + // Is this worth your time? + // You need 22 < power *and* power < 22 + 16 *and* (i * 10^(x-22) < 2^53) + // for this second fast path to work. + // If you you have 22 < power *and* power < 22 + 16, and then you + // optimistically compute "i * 10^(x-22)", there is still a chance that you + // have wasted your time if i * 10^(x-22) >= 2^53. It makes the use cases of + // this optimization maybe less common than we would like. Source: + // http://www.exploringbinary.com/fast-path-decimal-to-floating-point-conversion/ + // also used in RapidJSON: https://rapidjson.org/strtod_8h_source.html + + // The fast path has now failed, so we are failing back on the slower path. + + // In the slow path, we need to adjust i so that it is > 1<<63 which is always + // possible, except if i == 0, so we handle i == 0 separately. + if (i == 0) { + d = negative ? -0.0 : 0.0; + return true; + } + + + // The exponent is 1024 + 63 + power + // + floor(log(5**power)/log(2)). + // The 1024 comes from the ieee64 standard. + // The 63 comes from the fact that we use a 64-bit word. + // + // Computing floor(log(5**power)/log(2)) could be + // slow. Instead we use a fast function. + // + // For power in (-400,350), we have that + // (((152170 + 65536) * power ) >> 16); + // is equal to + // floor(log(5**power)/log(2)) + power when power >= 0 + // and it is equal to + // ceil(log(5**-power)/log(2)) + power when power < 0 + // + // The 65536 is (1<<16) and corresponds to + // (65536 * power) >> 16 ---> power + // + // ((152170 * power ) >> 16) is equal to + // floor(log(5**power)/log(2)) + // + // Note that this is not magic: 152170/(1<<16) is + // approximatively equal to log(5)/log(2). + // The 1<<16 value is a power of two; we could use a + // larger power of 2 if we wanted to. + // + int64_t exponent = (((152170 + 65536) * power) >> 16) + 1024 + 63; + + + // We want the most significant bit of i to be 1. Shift if needed. + int lz = leading_zeroes(i); + i <<= lz; + + + // We are going to need to do some 64-bit arithmetic to get a precise product. + // We use a table lookup approach. + // It is safe because + // power >= smallest_power + // and power <= largest_power + // We recover the mantissa of the power, it has a leading 1. It is always + // rounded down. + // + // We want the most significant 64 bits of the product. We know + // this will be non-zero because the most significant bit of i is + // 1. + const uint32_t index = 2 * uint32_t(power - simdjson::internal::smallest_power); + // Optimization: It may be that materializing the index as a variable might confuse some compilers and prevent effective complex-addressing loads. (Done for code clarity.) + // + // The full_multiplication function computes the 128-bit product of two 64-bit words + // with a returned value of type value128 with a "low component" corresponding to the + // 64-bit least significant bits of the product and with a "high component" corresponding + // to the 64-bit most significant bits of the product. + simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index]); + // Both i and power_of_five_128[index] have their most significant bit set to 1 which + // implies that the either the most or the second most significant bit of the product + // is 1. We pack values in this manner for efficiency reasons: it maximizes the use + // we make of the product. It also makes it easy to reason about the product: there + // is 0 or 1 leading zero in the product. + + // Unless the least significant 9 bits of the high (64-bit) part of the full + // product are all 1s, then we know that the most significant 55 bits are + // exact and no further work is needed. Having 55 bits is necessary because + // we need 53 bits for the mantissa but we have to have one rounding bit and + // we can waste a bit if the most significant bit of the product is zero. + if ((firstproduct.high & 0x1FF) == 0x1FF) { + // We want to compute i * 5^q, but only care about the top 55 bits at most. + // Consider the scenario where q>=0. Then 5^q may not fit in 64-bits. Doing + // the full computation is wasteful. So we do what is called a "truncated + // multiplication". + // We take the most significant 64-bits, and we put them in + // power_of_five_128[index]. Usually, that's good enough to approximate i * 5^q + // to the desired approximation using one multiplication. Sometimes it does not suffice. + // Then we store the next most significant 64 bits in power_of_five_128[index + 1], and + // then we get a better approximation to i * 5^q. + // + // That's for when q>=0. The logic for q<0 is somewhat similar but it is somewhat + // more complicated. + // + // There is an extra layer of complexity in that we need more than 55 bits of + // accuracy in the round-to-even scenario. + // + // The full_multiplication function computes the 128-bit product of two 64-bit words + // with a returned value of type value128 with a "low component" corresponding to the + // 64-bit least significant bits of the product and with a "high component" corresponding + // to the 64-bit most significant bits of the product. + simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index + 1]); + firstproduct.low += secondproduct.high; + if (secondproduct.high > firstproduct.low) { firstproduct.high++; } + // As it has been proven by Noble Mushtak and Daniel Lemire in "Fast Number Parsing Without + // Fallback" (https://arxiv.org/abs/2212.06644), at this point we are sure that the product + // is sufficiently accurate, and more computation is not needed. + } + uint64_t lower = firstproduct.low; + uint64_t upper = firstproduct.high; + // The final mantissa should be 53 bits with a leading 1. + // We shift it so that it occupies 54 bits with a leading 1. + /////// + uint64_t upperbit = upper >> 63; + uint64_t mantissa = upper >> (upperbit + 9); + lz += int(1 ^ upperbit); + + // Here we have mantissa < (1<<54). + int64_t real_exponent = exponent - lz; + if (simdjson_unlikely(real_exponent <= 0)) { // we have a subnormal? + // Here have that real_exponent <= 0 so -real_exponent >= 0 + if (-real_exponent + 1 >= 64) { // if we have more than 64 bits below the minimum exponent, you have a zero for sure. + d = negative ? -0.0 : 0.0; + return true; + } + // next line is safe because -real_exponent + 1 < 0 + mantissa >>= -real_exponent + 1; + // Thankfully, we can't have both "round-to-even" and subnormals because + // "round-to-even" only occurs for powers close to 0. + mantissa += (mantissa & 1); // round up + mantissa >>= 1; + // There is a weird scenario where we don't have a subnormal but just. + // Suppose we start with 2.2250738585072013e-308, we end up + // with 0x3fffffffffffff x 2^-1023-53 which is technically subnormal + // whereas 0x40000000000000 x 2^-1023-53 is normal. Now, we need to round + // up 0x3fffffffffffff x 2^-1023-53 and once we do, we are no longer + // subnormal, but we can only know this after rounding. + // So we only declare a subnormal if we are smaller than the threshold. + real_exponent = (mantissa < (uint64_t(1) << 52)) ? 0 : 1; + d = to_double(mantissa, real_exponent, negative); + return true; + } + // We have to round to even. The "to even" part + // is only a problem when we are right in between two floats + // which we guard against. + // If we have lots of trailing zeros, we may fall right between two + // floating-point values. + // + // The round-to-even cases take the form of a number 2m+1 which is in (2^53,2^54] + // times a power of two. That is, it is right between a number with binary significand + // m and another number with binary significand m+1; and it must be the case + // that it cannot be represented by a float itself. + // + // We must have that w * 10 ^q == (2m+1) * 2^p for some power of two 2^p. + // Recall that 10^q = 5^q * 2^q. + // When q >= 0, we must have that (2m+1) is divible by 5^q, so 5^q <= 2^54. We have that + // 5^23 <= 2^54 and it is the last power of five to qualify, so q <= 23. + // When q<0, we have w >= (2m+1) x 5^{-q}. We must have that w<2^{64} so + // (2m+1) x 5^{-q} < 2^{64}. We have that 2m+1>2^{53}. Hence, we must have + // 2^{53} x 5^{-q} < 2^{64}. + // Hence we have 5^{-q} < 2^{11}$ or q>= -4. + // + // We require lower <= 1 and not lower == 0 because we could not prove that + // that lower == 0 is implied; but we could prove that lower <= 1 is a necessary and sufficient test. + if (simdjson_unlikely((lower <= 1) && (power >= -4) && (power <= 23) && ((mantissa & 3) == 1))) { + if ((mantissa << (upperbit + 64 - 53 - 2)) == upper) { + mantissa &= ~1; // flip it so that we do not round up + } + } + + mantissa += mantissa & 1; + mantissa >>= 1; + + // Here we have mantissa < (1<<53), unless there was an overflow + if (mantissa >= (1ULL << 53)) { + ////////// + // This will happen when parsing values such as 7.2057594037927933e+16 + //////// + mantissa = (1ULL << 52); + real_exponent++; + } + mantissa &= ~(1ULL << 52); + // we have to check that real_exponent is in range, otherwise we bail out + if (simdjson_unlikely(real_exponent > 2046)) { + // We have an infinite value!!! We could actually throw an error here if we could. + return false; + } + d = to_double(mantissa, real_exponent, negative); + return true; + } + + // We call a fallback floating-point parser that might be slow. Note + // it will accept JSON numbers, but the JSON spec. is more restrictive so + // before you call parse_float_fallback, you need to have validated the input + // string with the JSON grammar. + // It will return an error (false) if the parsed number is infinite. + // The string parsing itself always succeeds. We know that there is at least + // one digit. + static bool parse_float_fallback(const uint8_t* ptr, double* outDouble) { + *outDouble = simdjson::internal::from_chars(reinterpret_cast(ptr)); + // We do not accept infinite values. + + // Detecting finite values in a portable manner is ridiculously hard, ideally + // we would want to do: + // return !std::isfinite(*outDouble); + // but that mysteriously fails under legacy/old libc++ libraries, see + // https://github.com/simdjson/simdjson/issues/1286 + // + // Therefore, fall back to this solution (the extra parens are there + // to handle that max may be a macro on windows). + return !(*outDouble > (std::numeric_limits::max)() || *outDouble < std::numeric_limits::lowest()); + } + + static bool parse_float_fallback(const uint8_t* ptr, const uint8_t* end_ptr, double* outDouble) { + *outDouble = simdjson::internal::from_chars(reinterpret_cast(ptr), reinterpret_cast(end_ptr)); + // We do not accept infinite values. + + // Detecting finite values in a portable manner is ridiculously hard, ideally + // we would want to do: + // return !std::isfinite(*outDouble); + // but that mysteriously fails under legacy/old libc++ libraries, see + // https://github.com/simdjson/simdjson/issues/1286 + // + // Therefore, fall back to this solution (the extra parens are there + // to handle that max may be a macro on windows). + return !(*outDouble > (std::numeric_limits::max)() || *outDouble < std::numeric_limits::lowest()); + } + + // check quickly whether the next 8 chars are made of digits + // at a glance, it looks better than Mula's + // http://0x80.pl/articles/swar-digits-validate.html + simdjson_inline bool is_made_of_eight_digits_fast(const uint8_t* chars) { + uint64_t val; + // this can read up to 7 bytes beyond the buffer size, but we require + // SIMDJSON_PADDING of padding + static_assert(7 <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be bigger than 7"); + std::memcpy(&val, chars, 8); + // a branchy method might be faster: + // return (( val & 0xF0F0F0F0F0F0F0F0 ) == 0x3030303030303030) + // && (( (val + 0x0606060606060606) & 0xF0F0F0F0F0F0F0F0 ) == + // 0x3030303030303030); + return (((val & 0xF0F0F0F0F0F0F0F0) | + (((val + 0x0606060606060606) & 0xF0F0F0F0F0F0F0F0) >> 4)) == + 0x3333333333333333); + } + + template + SIMDJSON_NO_SANITIZE_UNDEFINED // We deliberately allow overflow here and check later + simdjson_inline bool parse_digit(const uint8_t c, I& i) { + const uint8_t digit = static_cast(c - '0'); + if (digit > 9) { + return false; + } + // PERF NOTE: multiplication by 10 is cheaper than arbitrary integer multiplication + i = 10 * i + digit; // might overflow, we will handle the overflow later + return true; + } + + simdjson_inline bool is_digit(const uint8_t c) { + return static_cast(c - '0') <= 9; + } + + simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t* const src, const uint8_t*& p, uint64_t& i, int64_t& exponent) { + // we continue with the fiction that we have an integer. If the + // floating point number is representable as x * 10^z for some integer + // z that fits in 53 bits, then we will be able to convert back the + // the integer into a float in a lossless manner. + const uint8_t* const first_after_period = p; + +#ifdef SIMDJSON_SWAR_NUMBER_PARSING +#if SIMDJSON_SWAR_NUMBER_PARSING + // this helps if we have lots of decimals! + // this turns out to be frequent enough. + if (is_made_of_eight_digits_fast(p)) { + i = i * 100000000 + parse_eight_digits_unrolled(p); + p += 8; + } +#endif // SIMDJSON_SWAR_NUMBER_PARSING +#endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING + // Unrolling the first digit makes a small difference on some implementations (e.g. westmere) + if (parse_digit(*p, i)) { ++p; } + while (parse_digit(*p, i)) { p++; } + exponent = first_after_period - p; + // Decimal without digits (123.) is illegal + if (exponent == 0) { + return INVALID_NUMBER(src); + } + return SUCCESS; + } + + simdjson_inline error_code parse_exponent(simdjson_unused const uint8_t* const src, const uint8_t*& p, int64_t& exponent) { + // Exp Sign: -123.456e[-]78 + bool neg_exp = ('-' == *p); + if (neg_exp || '+' == *p) { p++; } // Skip + as well + + // Exponent: -123.456e-[78] + auto start_exp = p; + int64_t exp_number = 0; + while (parse_digit(*p, exp_number)) { ++p; } + // It is possible for parse_digit to overflow. + // In particular, it could overflow to INT64_MIN, and we cannot do - INT64_MIN. + // Thus we *must* check for possible overflow before we negate exp_number. + + // Performance notes: it may seem like combining the two "simdjson_unlikely checks" below into + // a single simdjson_unlikely path would be faster. The reasoning is sound, but the compiler may + // not oblige and may, in fact, generate two distinct paths in any case. It might be + // possible to do uint64_t(p - start_exp - 1) >= 18 but it could end up trading off + // instructions for a simdjson_likely branch, an unconclusive gain. + + // If there were no digits, it's an error. + if (simdjson_unlikely(p == start_exp)) { + return INVALID_NUMBER(src); + } + // We have a valid positive exponent in exp_number at this point, except that + // it may have overflowed. + + // If there were more than 18 digits, we may have overflowed the integer. We have to do + // something!!!! + if (simdjson_unlikely(p > start_exp + 18)) { + // Skip leading zeroes: 1e000000000000000000001 is technically valid and does not overflow + while (*start_exp == '0') { start_exp++; } + // 19 digits could overflow int64_t and is kind of absurd anyway. We don't + // support exponents smaller than -999,999,999,999,999,999 and bigger + // than 999,999,999,999,999,999. + // We can truncate. + // Note that 999999999999999999 is assuredly too large. The maximal ieee64 value before + // infinity is ~1.8e308. The smallest subnormal is ~5e-324. So, actually, we could + // truncate at 324. + // Note that there is no reason to fail per se at this point in time. + // E.g., 0e999999999999999999999 is a fine number. + if (p > start_exp + 18) { exp_number = 999999999999999999; } + } + // At this point, we know that exp_number is a sane, positive, signed integer. + // It is <= 999,999,999,999,999,999. As long as 'exponent' is in + // [-8223372036854775808, 8223372036854775808], we won't overflow. Because 'exponent' + // is bounded in magnitude by the size of the JSON input, we are fine in this universe. + // To sum it up: the next line should never overflow. + exponent += (neg_exp ? -exp_number : exp_number); + return SUCCESS; + } + + simdjson_inline bool check_if_integer(const uint8_t* const src, size_t max_length) { + const uint8_t* const srcend = src + max_length; + bool negative = (*src == '-'); // we can always read at least one character after the '-' + const uint8_t* p = src + uint8_t(negative); + if (p == srcend) { return false; } + if (*p == '0') { + ++p; + if (p == srcend) { return true; } + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return false; } + return true; + } + while (p != srcend && is_digit(*p)) { ++p; } + if (p == srcend) { return true; } + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return false; } + return true; + } + + simdjson_inline size_t significant_digits(const uint8_t* start_digits, size_t digit_count) { + // It is possible that the integer had an overflow. + // We have to handle the case where we have 0.0000somenumber. + const uint8_t* start = start_digits; + while ((*start == '0') || (*start == '.')) { ++start; } + // we over-decrement by one when there is a '.' + return digit_count - size_t(start - start_digits); + } + + } // unnamed namespace + + /** @private */ + static error_code slow_float_parsing(simdjson_unused const uint8_t* src, double* answer) { + if (parse_float_fallback(src, answer)) { + return SUCCESS; + } + return INVALID_NUMBER(src); + } + + /** @private */ + template + simdjson_inline error_code write_float(const uint8_t* const src, bool negative, uint64_t i, const uint8_t* start_digits, size_t digit_count, int64_t exponent, W& writer) { + // If we frequently had to deal with long strings of digits, + // we could extend our code by using a 128-bit integer instead + // of a 64-bit integer. However, this is uncommon in practice. + // + // 9999999999999999999 < 2**64 so we can accommodate 19 digits. + // If we have a decimal separator, then digit_count - 1 is the number of digits, but we + // may not have a decimal separator! + if (simdjson_unlikely(digit_count > 19 && significant_digits(start_digits, digit_count) > 19)) { + // Ok, chances are good that we had an overflow! + // this is almost never going to get called!!! + // we start anew, going slowly!!! + // This will happen in the following examples: + // 10000000000000000000000000000000000000000000e+308 + // 3.1415926535897932384626433832795028841971693993751 + // + // NOTE: We do not pass a reference to the to slow_float_parsing. If we passed our writer + // reference to it, it would force it to be stored in memory, preventing the compiler from + // picking it apart and putting into registers. i.e. if we pass it as reference, + // it gets slow. + double d; + error_code error = slow_float_parsing(src, &d); + writer.append_double(d); + return error; + } + // NOTE: it's weird that the simdjson_unlikely() only wraps half the if, but it seems to get slower any other + // way we've tried: https://github.com/simdjson/simdjson/pull/990#discussion_r448497331 + // To future reader: we'd love if someone found a better way, or at least could explain this result! + if (simdjson_unlikely(exponent < simdjson::internal::smallest_power) || (exponent > simdjson::internal::largest_power)) { + // + // Important: smallest_power is such that it leads to a zero value. + // Observe that 18446744073709551615e-343 == 0, i.e. (2**64 - 1) e -343 is zero + // so something x 10^-343 goes to zero, but not so with something x 10^-342. + static_assert(simdjson::internal::smallest_power <= -342, "smallest_power is not small enough"); + // + if ((exponent < simdjson::internal::smallest_power) || (i == 0)) { + // E.g. Parse "-0.0e-999" into the same value as "-0.0". See https://en.wikipedia.org/wiki/Signed_zero + WRITE_DOUBLE(negative ? -0.0 : 0.0, src, writer); + return SUCCESS; + } + else { // (exponent > largest_power) and (i != 0) + // We have, for sure, an infinite value and simdjson refuses to parse infinite values. + return INVALID_NUMBER(src); + } + } + double d; + if (!compute_float_64(exponent, i, negative, d)) { + // we are almost never going to get here. + if (!parse_float_fallback(src, &d)) { return INVALID_NUMBER(src); } + } + WRITE_DOUBLE(d, src, writer); + return SUCCESS; + } + + // parse the number at src + // define JSON_TEST_NUMBERS for unit testing + // + // It is assumed that the number is followed by a structural ({,},],[) character + // or a white space character. If that is not the case (e.g., when the JSON + // document is made of a single number), then it is necessary to copy the + // content and append a space before calling this function. + // + // Our objective is accurate parsing (ULP of 0) at high speed. + template + simdjson_inline error_code parse_number(const uint8_t* const src, W& writer); + + // for performance analysis, it is sometimes useful to skip parsing +#ifdef SIMDJSON_SKIPNUMBERPARSING + + template + simdjson_inline error_code parse_number(const uint8_t* const, W& writer) { + writer.append_s64(0); // always write zero + return SUCCESS; // always succeeds + } + + simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t* const src) noexcept { return 0; } + simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t* const src) noexcept { return 0; } + simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t* const src) noexcept { return 0; } + simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t* const src) noexcept { return 0; } + simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t* const src) noexcept { return 0; } + simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t* const src) noexcept { return 0; } + simdjson_unused simdjson_inline bool is_negative(const uint8_t* src) noexcept { return false; } + simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t* src) noexcept { return false; } + simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t* src) noexcept { return number_type::signed_integer; } +#else + +// parse the number at src +// define JSON_TEST_NUMBERS for unit testing +// +// It is assumed that the number is followed by a structural ({,},],[) character +// or a white space character. If that is not the case (e.g., when the JSON +// document is made of a single number), then it is necessary to copy the +// content and append a space before calling this function. +// +// Our objective is accurate parsing (ULP of 0) at high speed. + template + simdjson_inline error_code parse_number(const uint8_t* const src, W& writer) { + + // + // Check for minus sign + // + bool negative = (*src == '-'); + const uint8_t* p = src + uint8_t(negative); + + // + // Parse the integer part. + // + // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare + const uint8_t* const start_digits = p; + uint64_t i = 0; + while (parse_digit(*p, i)) { p++; } + + // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. + // Optimization note: size_t is expected to be unsigned. + size_t digit_count = size_t(p - start_digits); + if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { return INVALID_NUMBER(src); } + + // + // Handle floats if there is a . or e (or both) + // + int64_t exponent = 0; + bool is_float = false; + if ('.' == *p) { + is_float = true; + ++p; + SIMDJSON_TRY(parse_decimal_after_separator(src, p, i, exponent)); + digit_count = int(p - start_digits); // used later to guard against overflows + } + if (('e' == *p) || ('E' == *p)) { + is_float = true; + ++p; + SIMDJSON_TRY(parse_exponent(src, p, exponent)); + } + if (is_float) { + const bool dirty_end = jsoncharutils::is_not_structural_or_whitespace(*p); + SIMDJSON_TRY(write_float(src, negative, i, start_digits, digit_count, exponent, writer)); + if (dirty_end) { return INVALID_NUMBER(src); } + return SUCCESS; + } + + // The longest negative 64-bit number is 19 digits. + // The longest positive 64-bit number is 20 digits. + // We do it this way so we don't trigger this branch unless we must. + size_t longest_digit_count = negative ? 19 : 20; + if (digit_count > longest_digit_count) { return BIGINT_NUMBER(src); } + if (digit_count == longest_digit_count) { + if (negative) { + // Anything negative above INT64_MAX+1 is invalid + if (i > uint64_t(INT64_MAX) + 1) { return BIGINT_NUMBER(src); } + WRITE_INTEGER(~i + 1, src, writer); + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return INVALID_NUMBER(src); } + return SUCCESS; + // Positive overflow check: + // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the + // biggest uint64_t. + // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX. + // If we got here, it's a 20 digit number starting with the digit "1". + // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller + // than 1,553,255,926,290,448,384. + // - That is smaller than the smallest possible 20-digit number the user could write: + // 10,000,000,000,000,000,000. + // - Therefore, if the number is positive and lower than that, it's overflow. + // - The value we are looking at is less than or equal to INT64_MAX. + // + } + else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INVALID_NUMBER(src); } + } + + // Write unsigned if it does not fit in a signed integer. + if (i > uint64_t(INT64_MAX)) { + WRITE_UNSIGNED(i, src, writer); + } + else { + WRITE_INTEGER(negative ? (~i + 1) : i, src, writer); + } + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return INVALID_NUMBER(src); } + return SUCCESS; + } + + // Inlineable functions + namespace { + + // This table can be used to characterize the final character of an integer + // string. For JSON structural character and allowable white space characters, + // we return SUCCESS. For 'e', '.' and 'E', we return INCORRECT_TYPE. Otherwise + // we return NUMBER_ERROR. + // Optimization note: we could easily reduce the size of the table by half (to 128) + // at the cost of an extra branch. + // Optimization note: we want the values to use at most 8 bits (not, e.g., 32 bits): + static_assert(error_code(uint8_t(NUMBER_ERROR)) == NUMBER_ERROR, "bad NUMBER_ERROR cast"); + static_assert(error_code(uint8_t(SUCCESS)) == SUCCESS, "bad NUMBER_ERROR cast"); + static_assert(error_code(uint8_t(INCORRECT_TYPE)) == INCORRECT_TYPE, "bad NUMBER_ERROR cast"); + + const uint8_t integer_string_finisher[256] = { + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, + SUCCESS, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, + NUMBER_ERROR, INCORRECT_TYPE, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, INCORRECT_TYPE, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, SUCCESS, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, INCORRECT_TYPE, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, + SUCCESS, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR }; + + // Parse any number from 0 to 18,446,744,073,709,551,615 + simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t* const src) noexcept { + const uint8_t* p = src; + // + // Parse the integer part. + // + // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare + const uint8_t* const start_digits = p; + uint64_t i = 0; + while (parse_digit(*p, i)) { p++; } + + // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. + // Optimization note: size_t is expected to be unsigned. + size_t digit_count = size_t(p - start_digits); + // The longest positive 64-bit number is 20 digits. + // We do it this way so we don't trigger this branch unless we must. + // Optimization note: the compiler can probably merge + // ((digit_count == 0) || (digit_count > 20)) + // into a single branch since digit_count is unsigned. + if ((digit_count == 0) || (digit_count > 20)) { return INCORRECT_TYPE; } + // Here digit_count > 0. + if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } + // We can do the following... + // if (!jsoncharutils::is_structural_or_whitespace(*p)) { + // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; + // } + // as a single table lookup: + if (integer_string_finisher[*p] != SUCCESS) { return error_code(integer_string_finisher[*p]); } + + if (digit_count == 20) { + // Positive overflow check: + // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the + // biggest uint64_t. + // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX. + // If we got here, it's a 20 digit number starting with the digit "1". + // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller + // than 1,553,255,926,290,448,384. + // - That is smaller than the smallest possible 20-digit number the user could write: + // 10,000,000,000,000,000,000. + // - Therefore, if the number is positive and lower than that, it's overflow. + // - The value we are looking at is less than or equal to INT64_MAX. + // + if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INCORRECT_TYPE; } + } + + return i; + } + + + // Parse any number from 0 to 18,446,744,073,709,551,615 + // Never read at src_end or beyond + simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t* const src, const uint8_t* const src_end) noexcept { + const uint8_t* p = src; + // + // Parse the integer part. + // + // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare + const uint8_t* const start_digits = p; + uint64_t i = 0; + while ((p != src_end) && parse_digit(*p, i)) { p++; } + + // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. + // Optimization note: size_t is expected to be unsigned. + size_t digit_count = size_t(p - start_digits); + // The longest positive 64-bit number is 20 digits. + // We do it this way so we don't trigger this branch unless we must. + // Optimization note: the compiler can probably merge + // ((digit_count == 0) || (digit_count > 20)) + // into a single branch since digit_count is unsigned. + if ((digit_count == 0) || (digit_count > 20)) { return INCORRECT_TYPE; } + // Here digit_count > 0. + if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } + // We can do the following... + // if (!jsoncharutils::is_structural_or_whitespace(*p)) { + // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; + // } + // as a single table lookup: + if ((p != src_end) && integer_string_finisher[*p] != SUCCESS) { return error_code(integer_string_finisher[*p]); } + + if (digit_count == 20) { + // Positive overflow check: + // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the + // biggest uint64_t. + // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX. + // If we got here, it's a 20 digit number starting with the digit "1". + // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller + // than 1,553,255,926,290,448,384. + // - That is smaller than the smallest possible 20-digit number the user could write: + // 10,000,000,000,000,000,000. + // - Therefore, if the number is positive and lower than that, it's overflow. + // - The value we are looking at is less than or equal to INT64_MAX. + // + if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INCORRECT_TYPE; } + } + + return i; + } + + // Parse any number from 0 to 18,446,744,073,709,551,615 + simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t* const src) noexcept { + const uint8_t* p = src + 1; + // + // Parse the integer part. + // + // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare + const uint8_t* const start_digits = p; + uint64_t i = 0; + while (parse_digit(*p, i)) { p++; } + + // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. + // Optimization note: size_t is expected to be unsigned. + size_t digit_count = size_t(p - start_digits); + // The longest positive 64-bit number is 20 digits. + // We do it this way so we don't trigger this branch unless we must. + // Optimization note: the compiler can probably merge + // ((digit_count == 0) || (digit_count > 20)) + // into a single branch since digit_count is unsigned. + if ((digit_count == 0) || (digit_count > 20)) { return INCORRECT_TYPE; } + // Here digit_count > 0. + if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } + // We can do the following... + // if (!jsoncharutils::is_structural_or_whitespace(*p)) { + // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; + // } + // as a single table lookup: + if (*p != '"') { return NUMBER_ERROR; } + + if (digit_count == 20) { + // Positive overflow check: + // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the + // biggest uint64_t. + // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX. + // If we got here, it's a 20 digit number starting with the digit "1". + // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller + // than 1,553,255,926,290,448,384. + // - That is smaller than the smallest possible 20-digit number the user could write: + // 10,000,000,000,000,000,000. + // - Therefore, if the number is positive and lower than that, it's overflow. + // - The value we are looking at is less than or equal to INT64_MAX. + // + // Note: we use src[1] and not src[0] because src[0] is the quote character in this + // instance. + if (src[1] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INCORRECT_TYPE; } + } + + return i; + } + + // Parse any number from -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 + simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t* src) noexcept { + // + // Check for minus sign + // + bool negative = (*src == '-'); + const uint8_t* p = src + uint8_t(negative); + + // + // Parse the integer part. + // + // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare + const uint8_t* const start_digits = p; + uint64_t i = 0; + while (parse_digit(*p, i)) { p++; } + + // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. + // Optimization note: size_t is expected to be unsigned. + size_t digit_count = size_t(p - start_digits); + // We go from + // -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 + // so we can never represent numbers that have more than 19 digits. + size_t longest_digit_count = 19; + // Optimization note: the compiler can probably merge + // ((digit_count == 0) || (digit_count > longest_digit_count)) + // into a single branch since digit_count is unsigned. + if ((digit_count == 0) || (digit_count > longest_digit_count)) { return INCORRECT_TYPE; } + // Here digit_count > 0. + if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } + // We can do the following... + // if (!jsoncharutils::is_structural_or_whitespace(*p)) { + // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; + // } + // as a single table lookup: + if (integer_string_finisher[*p] != SUCCESS) { return error_code(integer_string_finisher[*p]); } + // Negative numbers have can go down to - INT64_MAX - 1 whereas positive numbers are limited to INT64_MAX. + // Performance note: This check is only needed when digit_count == longest_digit_count but it is + // so cheap that we might as well always make it. + if (i > uint64_t(INT64_MAX) + uint64_t(negative)) { return INCORRECT_TYPE; } + return negative ? (~i + 1) : i; + } + + // Parse any number from -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 + // Never read at src_end or beyond + simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t* const src, const uint8_t* const src_end) noexcept { + // + // Check for minus sign + // + if (src == src_end) { return NUMBER_ERROR; } + bool negative = (*src == '-'); + const uint8_t* p = src + uint8_t(negative); + + // + // Parse the integer part. + // + // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare + const uint8_t* const start_digits = p; + uint64_t i = 0; + while ((p != src_end) && parse_digit(*p, i)) { p++; } + + // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. + // Optimization note: size_t is expected to be unsigned. + size_t digit_count = size_t(p - start_digits); + // We go from + // -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 + // so we can never represent numbers that have more than 19 digits. + size_t longest_digit_count = 19; + // Optimization note: the compiler can probably merge + // ((digit_count == 0) || (digit_count > longest_digit_count)) + // into a single branch since digit_count is unsigned. + if ((digit_count == 0) || (digit_count > longest_digit_count)) { return INCORRECT_TYPE; } + // Here digit_count > 0. + if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } + // We can do the following... + // if (!jsoncharutils::is_structural_or_whitespace(*p)) { + // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; + // } + // as a single table lookup: + if ((p != src_end) && integer_string_finisher[*p] != SUCCESS) { return error_code(integer_string_finisher[*p]); } + // Negative numbers have can go down to - INT64_MAX - 1 whereas positive numbers are limited to INT64_MAX. + // Performance note: This check is only needed when digit_count == longest_digit_count but it is + // so cheap that we might as well always make it. + if (i > uint64_t(INT64_MAX) + uint64_t(negative)) { return INCORRECT_TYPE; } + return negative ? (~i + 1) : i; + } + + // Parse any number from -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 + simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t* src) noexcept { + // + // Check for minus sign + // + bool negative = (*(src + 1) == '-'); + src += uint8_t(negative) + 1; + + // + // Parse the integer part. + // + // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare + const uint8_t* const start_digits = src; + uint64_t i = 0; + while (parse_digit(*src, i)) { src++; } + + // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. + // Optimization note: size_t is expected to be unsigned. + size_t digit_count = size_t(src - start_digits); + // We go from + // -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 + // so we can never represent numbers that have more than 19 digits. + size_t longest_digit_count = 19; + // Optimization note: the compiler can probably merge + // ((digit_count == 0) || (digit_count > longest_digit_count)) + // into a single branch since digit_count is unsigned. + if ((digit_count == 0) || (digit_count > longest_digit_count)) { return INCORRECT_TYPE; } + // Here digit_count > 0. + if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } + // We can do the following... + // if (!jsoncharutils::is_structural_or_whitespace(*src)) { + // return (*src == '.' || *src == 'e' || *src == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; + // } + // as a single table lookup: + if (*src != '"') { return NUMBER_ERROR; } + // Negative numbers have can go down to - INT64_MAX - 1 whereas positive numbers are limited to INT64_MAX. + // Performance note: This check is only needed when digit_count == longest_digit_count but it is + // so cheap that we might as well always make it. + if (i > uint64_t(INT64_MAX) + uint64_t(negative)) { return INCORRECT_TYPE; } + return negative ? (~i + 1) : i; + } + + simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t* src) noexcept { + // + // Check for minus sign + // + bool negative = (*src == '-'); + src += uint8_t(negative); + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t* p = src; + p += parse_digit(*p, i); + bool leading_zero = (i == 0); + while (parse_digit(*p, i)) { p++; } + // no integer digits, or 0123 (zero must be solo) + if (p == src) { return INCORRECT_TYPE; } + if ((leading_zero && p != src + 1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely(*p == '.')) { + p++; + const uint8_t* start_decimal_digits = p; + if (!parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits + p++; + while (parse_digit(*p, i)) { p++; } + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p - src - 1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t* start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = p - start_digits > 19; + } + } + else { + overflow = p - src > 19; + } + + // + // Parse the exponent + // + if (*p == 'e' || *p == 'E') { + p++; + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t* start_exp_digits = p; + while (parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p - start_exp_digits == 0 || p - start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0 - exp : exp; + } + + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + double d; + if (simdjson_likely(!overflow)) { + if (compute_float_64(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), &d)) { + return NUMBER_ERROR; + } + return d; + } + + simdjson_unused simdjson_inline bool is_negative(const uint8_t* src) noexcept { + return (*src == '-'); + } + + simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t* src) noexcept { + bool negative = (*src == '-'); + src += uint8_t(negative); + const uint8_t* p = src; + while (static_cast(*p - '0') <= 9) { p++; } + if (p == src) { return NUMBER_ERROR; } + if (jsoncharutils::is_structural_or_whitespace(*p)) { return true; } + return false; + } + + simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t* src) noexcept { + bool negative = (*src == '-'); + src += uint8_t(negative); + const uint8_t* p = src; + while (static_cast(*p - '0') <= 9) { p++; } + size_t digit_count = size_t(p - src); + if (p == src) { return NUMBER_ERROR; } + if (jsoncharutils::is_structural_or_whitespace(*p)) { + static const uint8_t* smaller_big_integer = reinterpret_cast("9223372036854775808"); + // We have an integer. + if (simdjson_unlikely(digit_count > 20)) { + return number_type::big_integer; + } + // If the number is negative and valid, it must be a signed integer. + if (negative) { + if (simdjson_unlikely(digit_count > 19)) return number_type::big_integer; + if (simdjson_unlikely(digit_count == 19 && memcmp(src, smaller_big_integer, 19) > 0)) { + return number_type::big_integer; + } + return number_type::signed_integer; + } + // Let us check if we have a big integer (>=2**64). + static const uint8_t* two_to_sixtyfour = reinterpret_cast("18446744073709551616"); + if ((digit_count > 20) || (digit_count == 20 && memcmp(src, two_to_sixtyfour, 20) >= 0)) { + return number_type::big_integer; + } + // The number is positive and smaller than 18446744073709551616 (or 2**64). + // We want values larger or equal to 9223372036854775808 to be unsigned + // integers, and the other values to be signed integers. + if ((digit_count == 20) || (digit_count >= 19 && memcmp(src, smaller_big_integer, 19) >= 0)) { + return number_type::unsigned_integer; + } + return number_type::signed_integer; + } + // Hopefully, we have 'e' or 'E' or '.'. + return number_type::floating_point_number; + } + + // Never read at src_end or beyond + simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t* src, const uint8_t* const src_end) noexcept { + if (src == src_end) { return NUMBER_ERROR; } + // + // Check for minus sign + // + bool negative = (*src == '-'); + src += uint8_t(negative); + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t* p = src; + if (p == src_end) { return NUMBER_ERROR; } + p += parse_digit(*p, i); + bool leading_zero = (i == 0); + while ((p != src_end) && parse_digit(*p, i)) { p++; } + // no integer digits, or 0123 (zero must be solo) + if (p == src) { return INCORRECT_TYPE; } + if ((leading_zero && p != src + 1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely((p != src_end) && (*p == '.'))) { + p++; + const uint8_t* start_decimal_digits = p; + if ((p == src_end) || !parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits + p++; + while ((p != src_end) && parse_digit(*p, i)) { p++; } + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p - src - 1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t* start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = start_digits - src > 19; + } + } + else { + overflow = p - src > 19; + } + + // + // Parse the exponent + // + if ((p != src_end) && (*p == 'e' || *p == 'E')) { + p++; + if (p == src_end) { return NUMBER_ERROR; } + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t* start_exp_digits = p; + while ((p != src_end) && parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p - start_exp_digits == 0 || p - start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0 - exp : exp; + } + + if ((p != src_end) && jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + double d; + if (simdjson_likely(!overflow)) { + if (compute_float_64(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), src_end, &d)) { + return NUMBER_ERROR; + } + return d; + } + + simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t* src) noexcept { + // + // Check for minus sign + // + bool negative = (*(src + 1) == '-'); + src += uint8_t(negative) + 1; + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t* p = src; + p += parse_digit(*p, i); + bool leading_zero = (i == 0); + while (parse_digit(*p, i)) { p++; } + // no integer digits, or 0123 (zero must be solo) + if (p == src) { return INCORRECT_TYPE; } + if ((leading_zero && p != src + 1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely(*p == '.')) { + p++; + const uint8_t* start_decimal_digits = p; + if (!parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits + p++; + while (parse_digit(*p, i)) { p++; } + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p - src - 1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t* start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = p - start_digits > 19; + } + } + else { + overflow = p - src > 19; + } + + // + // Parse the exponent + // + if (*p == 'e' || *p == 'E') { + p++; + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t* start_exp_digits = p; + while (parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p - start_exp_digits == 0 || p - start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0 - exp : exp; + } + + if (*p != '"') { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + double d; + if (simdjson_likely(!overflow)) { + if (compute_float_64(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), &d)) { + return NUMBER_ERROR; + } + return d; + } + + } // unnamed namespace +#endif // SIMDJSON_SKIPNUMBERPARSING + + } // namespace numberparsing + + inline std::ostream& operator<<(std::ostream& out, number_type type) noexcept { + switch (type) { + case number_type::signed_integer: out << "integer in [-9223372036854775808,9223372036854775808)"; break; + case number_type::unsigned_integer: out << "unsigned integer in [9223372036854775808,18446744073709551616)"; break; + case number_type::floating_point_number: out << "floating-point number (binary64)"; break; + case number_type::big_integer: out << "big integer"; break; default: SIMDJSON_UNREACHABLE(); } return out; @@ -16111,13 +54615,13 @@ namespace simdjson { simdjson_inline internal::value128 full_multiplication(uint64_t value1, uint64_t value2) { internal::value128 answer; #if SIMDJSON_REGULAR_VISUAL_STUDIO || SIMDJSON_IS_32BITS -#ifdef _M_ARM64 +#if SIMDJSON_IS_ARM64 // ARM64 has native support for 64-bit multiplications, no need to emultate answer.high = __umulh(value1, value2); answer.low = value1 * value2; #else answer.low = _umul128(value1, value2, &answer.high); // _umul128 not available on ARM64 -#endif // _M_ARM64 +#endif // SIMDJSON_IS_ARM64 #else // SIMDJSON_REGULAR_VISUAL_STUDIO || SIMDJSON_IS_32BITS __uint128_t r = (static_cast<__uint128_t>(value1)) * value2; answer.low = uint64_t(r); @@ -16300,8 +54804,8 @@ namespace simdjson { // Use the default Unicode Character 'REPLACEMENT CHARACTER' (U+FFFD) constexpr uint32_t substitution_code_point = 0xfffd; // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the - // conversion isn't valid; we defer the check for this to inside the - // multilingual plane check + // conversion is not valid; we defer the check for this to inside the + // multilingual plane check. uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2); *src_ptr += 6; @@ -16361,8 +54865,8 @@ namespace simdjson { // It is not ideal that this function is nearly identical to handle_unicode_codepoint. // // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the - // conversion isn't valid; we defer the check for this to inside the - // multilingual plane check + // conversion is not valid; we defer the check for this to inside the + // multilingual plane check. uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2); *src_ptr += 6; // If we found a high surrogate, we must @@ -16393,7 +54897,7 @@ namespace simdjson { * Unescape a valid UTF-8 string from src to dst, stopping at a final unescaped quote. There * must be an unescaped quote terminating the string. It returns the final output * position as pointer. In case of error (e.g., the string has bad escaped codes), - * then null_nullptrptr is returned. It is assumed that the output buffer is large + * then null_ptr is returned. It is assumed that the output buffer is large * enough. E.g., if src points at 'joe"', then dst needs to have four free bytes + * SIMDJSON_PADDING bytes. */ @@ -17766,25897 +56270,6 @@ namespace simdjson { #endif // SIMDJSON_SRC_FALLBACK_CPP /* end file fallback.cpp */ #endif -#if SIMDJSON_IMPLEMENTATION_HASWELL -/* including haswell.cpp: #include */ -/* begin file haswell.cpp */ -#ifndef SIMDJSON_SRC_HASWELL_CPP -#define SIMDJSON_SRC_HASWELL_CPP - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -/* including simdjson/haswell.h: #include */ -/* begin file simdjson/haswell.h */ -#ifndef SIMDJSON_HASWELL_H -#define SIMDJSON_HASWELL_H - -/* including simdjson/haswell/begin.h: #include "simdjson/haswell/begin.h" */ -/* begin file simdjson/haswell/begin.h */ -/* defining SIMDJSON_IMPLEMENTATION to "haswell" */ -#define SIMDJSON_IMPLEMENTATION haswell - -/* including simdjson/haswell/base.h: #include "simdjson/haswell/base.h" */ -/* begin file simdjson/haswell/base.h */ -#ifndef SIMDJSON_HASWELL_BASE_H -#define SIMDJSON_HASWELL_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -// The constructor may be executed on any host, so we take care not to use SIMDJSON_TARGET_HASWELL -namespace simdjson { - /** - * Implementation for Haswell (Intel AVX2). - */ - namespace haswell { - - class implementation; - - namespace { - namespace simd { - template struct simd8; - template struct simd8x64; - } // namespace simd - } // unnamed namespace - - } // namespace haswell -} // namespace simdjson - -#endif // SIMDJSON_HASWELL_BASE_H -/* end file simdjson/haswell/base.h */ -/* including simdjson/haswell/intrinsics.h: #include "simdjson/haswell/intrinsics.h" */ -/* begin file simdjson/haswell/intrinsics.h */ -#ifndef SIMDJSON_HASWELL_INTRINSICS_H -#define SIMDJSON_HASWELL_INTRINSICS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/haswell/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#if SIMDJSON_VISUAL_STUDIO -// under clang within visual studio, this will include -#include // visual studio or clang -#else -#include // elsewhere -#endif // SIMDJSON_VISUAL_STUDIO - -#if SIMDJSON_CLANG_VISUAL_STUDIO -/** - * You are not supposed, normally, to include these - * headers directly. Instead you should either include intrin.h - * or x86intrin.h. However, when compiling with clang - * under Windows (i.e., when _MSC_VER is set), these headers - * only get included *if* the corresponding features are detected - * from macros: - * e.g., if __AVX2__ is set... in turn, we normally set these - * macros by compiling against the corresponding architecture - * (e.g., arch:AVX2, -mavx2, etc.) which compiles the whole - * software with these advanced instructions. In simdjson, we - * want to compile the whole program for a generic target, - * and only target our specific kernels. As a workaround, - * we directly include the needed headers. These headers would - * normally guard against such usage, but we carefully included - * (or ) before, so the headers - * are fooled. - */ -#include // for _blsr_u64 -#include // for __lzcnt64 -#include // for most things (AVX2, AVX512, _popcnt64) -#include -#include -#include -#include -#include // for _mm_clmulepi64_si128 - // unfortunately, we may not get _blsr_u64, but, thankfully, clang - // has it as a macro. -#ifndef _blsr_u64 -// we roll our own -#define _blsr_u64(n) ((n - 1) & n) -#endif // _blsr_u64 -#endif // SIMDJSON_CLANG_VISUAL_STUDIO - -static_assert(sizeof(__m256i) <= simdjson::SIMDJSON_PADDING, "insufficient padding for haswell kernel."); - -#endif // SIMDJSON_HASWELL_INTRINSICS_H -/* end file simdjson/haswell/intrinsics.h */ - -#if !SIMDJSON_CAN_ALWAYS_RUN_HASWELL -SIMDJSON_TARGET_REGION("avx2,bmi,pclmul,lzcnt,popcnt") -#endif - -/* including simdjson/haswell/bitmanipulation.h: #include "simdjson/haswell/bitmanipulation.h" */ -/* begin file simdjson/haswell/bitmanipulation.h */ -#ifndef SIMDJSON_HASWELL_BITMANIPULATION_H -#define SIMDJSON_HASWELL_BITMANIPULATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/haswell/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/haswell/intrinsics.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/haswell/bitmask.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - namespace haswell { - namespace { - - // We sometimes call trailing_zero on inputs that are zero, - // but the algorithms do not end up using the returned value. - // Sadly, sanitizers are not smart enough to figure it out. - SIMDJSON_NO_SANITIZE_UNDEFINED - // This function can be used safely even if not all bytes have been - // initialized. - // See issue https://github.com/simdjson/simdjson/issues/1965 - SIMDJSON_NO_SANITIZE_MEMORY - simdjson_inline int trailing_zeroes(uint64_t input_num) { -#if SIMDJSON_REGULAR_VISUAL_STUDIO - return (int)_tzcnt_u64(input_num); -#else // SIMDJSON_REGULAR_VISUAL_STUDIO - //////// - // You might expect the next line to be equivalent to - // return (int)_tzcnt_u64(input_num); - // but the generated code differs and might be less efficient? - //////// - return __builtin_ctzll(input_num); -#endif // SIMDJSON_REGULAR_VISUAL_STUDIO - } - - /* result might be undefined when input_num is zero */ - simdjson_inline uint64_t clear_lowest_bit(uint64_t input_num) { - return _blsr_u64(input_num); - } - - /* result might be undefined when input_num is zero */ - simdjson_inline int leading_zeroes(uint64_t input_num) { - return int(_lzcnt_u64(input_num)); - } - -#if SIMDJSON_REGULAR_VISUAL_STUDIO - simdjson_inline unsigned __int64 count_ones(uint64_t input_num) { - // note: we do not support legacy 32-bit Windows in this kernel - return __popcnt64(input_num);// Visual Studio wants two underscores - } -#else - simdjson_inline long long int count_ones(uint64_t input_num) { - return _popcnt64(input_num); - } -#endif - - simdjson_inline bool add_overflow(uint64_t value1, uint64_t value2, - uint64_t* result) { -#if SIMDJSON_REGULAR_VISUAL_STUDIO - return _addcarry_u64(0, value1, value2, - reinterpret_cast(result)); -#else - return __builtin_uaddll_overflow(value1, value2, - reinterpret_cast(result)); -#endif - } - - } // unnamed namespace - } // namespace haswell -} // namespace simdjson - -#endif // SIMDJSON_HASWELL_BITMANIPULATION_H -/* end file simdjson/haswell/bitmanipulation.h */ -/* including simdjson/haswell/bitmask.h: #include "simdjson/haswell/bitmask.h" */ -/* begin file simdjson/haswell/bitmask.h */ -#ifndef SIMDJSON_HASWELL_BITMASK_H -#define SIMDJSON_HASWELL_BITMASK_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/haswell/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/haswell/intrinsics.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - namespace haswell { - namespace { - - // - // Perform a "cumulative bitwise xor," flipping bits each time a 1 is encountered. - // - // For example, prefix_xor(00100100) == 00011100 - // - simdjson_inline uint64_t prefix_xor(const uint64_t bitmask) { - // There should be no such thing with a processor supporting avx2 - // but not clmul. - __m128i all_ones = _mm_set1_epi8('\xFF'); - __m128i result = _mm_clmulepi64_si128(_mm_set_epi64x(0ULL, bitmask), all_ones, 0); - return _mm_cvtsi128_si64(result); - } - - } // unnamed namespace - } // namespace haswell -} // namespace simdjson - -#endif // SIMDJSON_HASWELL_BITMASK_H -/* end file simdjson/haswell/bitmask.h */ -/* including simdjson/haswell/numberparsing_defs.h: #include "simdjson/haswell/numberparsing_defs.h" */ -/* begin file simdjson/haswell/numberparsing_defs.h */ -#ifndef SIMDJSON_HASWELL_NUMBERPARSING_DEFS_H -#define SIMDJSON_HASWELL_NUMBERPARSING_DEFS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/haswell/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/haswell/intrinsics.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - namespace haswell { - namespace numberparsing { - - /** @private */ - static simdjson_inline uint32_t parse_eight_digits_unrolled(const uint8_t* chars) { - // this actually computes *16* values so we are being wasteful. - const __m128i ascii0 = _mm_set1_epi8('0'); - const __m128i mul_1_10 = - _mm_setr_epi8(10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1); - const __m128i mul_1_100 = _mm_setr_epi16(100, 1, 100, 1, 100, 1, 100, 1); - const __m128i mul_1_10000 = - _mm_setr_epi16(10000, 1, 10000, 1, 10000, 1, 10000, 1); - const __m128i input = _mm_sub_epi8( - _mm_loadu_si128(reinterpret_cast(chars)), ascii0); - const __m128i t1 = _mm_maddubs_epi16(input, mul_1_10); - const __m128i t2 = _mm_madd_epi16(t1, mul_1_100); - const __m128i t3 = _mm_packus_epi32(t2, t2); - const __m128i t4 = _mm_madd_epi16(t3, mul_1_10000); - return _mm_cvtsi128_si32( - t4); // only captures the sum of the first 8 digits, drop the rest - } - - /** @private */ - simdjson_inline internal::value128 full_multiplication(uint64_t value1, uint64_t value2) { - internal::value128 answer; -#if SIMDJSON_REGULAR_VISUAL_STUDIO || SIMDJSON_IS_32BITS -#ifdef _M_ARM64 - // ARM64 has native support for 64-bit multiplications, no need to emultate - answer.high = __umulh(value1, value2); - answer.low = value1 * value2; -#else - answer.low = _umul128(value1, value2, &answer.high); // _umul128 not available on ARM64 -#endif // _M_ARM64 -#else // SIMDJSON_REGULAR_VISUAL_STUDIO || SIMDJSON_IS_32BITS - __uint128_t r = (static_cast<__uint128_t>(value1)) * value2; - answer.low = uint64_t(r); - answer.high = uint64_t(r >> 64); -#endif - return answer; - } - - } // namespace numberparsing - } // namespace haswell -} // namespace simdjson - -#define SIMDJSON_SWAR_NUMBER_PARSING 1 - -#endif // SIMDJSON_HASWELL_NUMBERPARSING_DEFS_H -/* end file simdjson/haswell/numberparsing_defs.h */ -/* including simdjson/haswell/simd.h: #include "simdjson/haswell/simd.h" */ -/* begin file simdjson/haswell/simd.h */ -#ifndef SIMDJSON_HASWELL_SIMD_H -#define SIMDJSON_HASWELL_SIMD_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/haswell/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/haswell/intrinsics.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/haswell/bitmanipulation.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/simdprune_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - namespace haswell { - namespace { - namespace simd { - - // Forward-declared so they can be used by splat and friends. - template - struct base { - __m256i value; - - // Zero constructor - simdjson_inline base() : value{ __m256i() } {} - - // Conversion from SIMD register - simdjson_inline base(const __m256i _value) : value(_value) {} - - // Conversion to SIMD register - simdjson_inline operator const __m256i& () const { return this->value; } - simdjson_inline operator __m256i& () { return this->value; } - - // Bit operations - simdjson_inline Child operator|(const Child other) const { return _mm256_or_si256(*this, other); } - simdjson_inline Child operator&(const Child other) const { return _mm256_and_si256(*this, other); } - simdjson_inline Child operator^(const Child other) const { return _mm256_xor_si256(*this, other); } - simdjson_inline Child bit_andnot(const Child other) const { return _mm256_andnot_si256(other, *this); } - simdjson_inline Child& operator|=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast | other; return *this_cast; } - simdjson_inline Child& operator&=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast & other; return *this_cast; } - simdjson_inline Child& operator^=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast ^ other; return *this_cast; } - }; - - // Forward-declared so they can be used by splat and friends. - template - struct simd8; - - template> - struct base8 : base> { - typedef uint32_t bitmask_t; - typedef uint64_t bitmask2_t; - - simdjson_inline base8() : base>() {} - simdjson_inline base8(const __m256i _value) : base>(_value) {} - - friend simdjson_really_inline Mask operator==(const simd8 lhs, const simd8 rhs) { return _mm256_cmpeq_epi8(lhs, rhs); } - - static const int SIZE = sizeof(base::value); - - template - simdjson_inline simd8 prev(const simd8 prev_chunk) const { - return _mm256_alignr_epi8(*this, _mm256_permute2x128_si256(prev_chunk, *this, 0x21), 16 - N); - } - }; - - // SIMD byte mask type (returned by things like eq and gt) - template<> - struct simd8 : base8 { - static simdjson_inline simd8 splat(bool _value) { return _mm256_set1_epi8(uint8_t(-(!!_value))); } - - simdjson_inline simd8() : base8() {} - simdjson_inline simd8(const __m256i _value) : base8(_value) {} - // Splat constructor - simdjson_inline simd8(bool _value) : base8(splat(_value)) {} - - simdjson_inline int to_bitmask() const { return _mm256_movemask_epi8(*this); } - simdjson_inline bool any() const { return !_mm256_testz_si256(*this, *this); } - simdjson_inline simd8 operator~() const { return *this ^ true; } - }; - - template - struct base8_numeric : base8 { - static simdjson_inline simd8 splat(T _value) { return _mm256_set1_epi8(_value); } - static simdjson_inline simd8 zero() { return _mm256_setzero_si256(); } - static simdjson_inline simd8 load(const T values[32]) { - return _mm256_loadu_si256(reinterpret_cast(values)); - } - // Repeat 16 values as many times as necessary (usually for lookup tables) - static simdjson_inline simd8 repeat_16( - T v0, T v1, T v2, T v3, T v4, T v5, T v6, T v7, - T v8, T v9, T v10, T v11, T v12, T v13, T v14, T v15 - ) { - return simd8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10, v11, v12, v13, v14, v15, - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10, v11, v12, v13, v14, v15 - ); - } - - simdjson_inline base8_numeric() : base8() {} - simdjson_inline base8_numeric(const __m256i _value) : base8(_value) {} - - // Store to array - simdjson_inline void store(T dst[32]) const { return _mm256_storeu_si256(reinterpret_cast<__m256i*>(dst), *this); } - - // Addition/subtraction are the same for signed and unsigned - simdjson_inline simd8 operator+(const simd8 other) const { return _mm256_add_epi8(*this, other); } - simdjson_inline simd8 operator-(const simd8 other) const { return _mm256_sub_epi8(*this, other); } - simdjson_inline simd8& operator+=(const simd8 other) { *this = *this + other; return *static_cast*>(this); } - simdjson_inline simd8& operator-=(const simd8 other) { *this = *this - other; return *static_cast*>(this); } - - // Override to distinguish from bool version - simdjson_inline simd8 operator~() const { return *this ^ 0xFFu; } - - // Perform a lookup assuming the value is between 0 and 16 (undefined behavior for out of range values) - template - simdjson_inline simd8 lookup_16(simd8 lookup_table) const { - return _mm256_shuffle_epi8(lookup_table, *this); - } - - // Copies to 'output" all bytes corresponding to a 0 in the mask (interpreted as a bitset). - // Passing a 0 value for mask would be equivalent to writing out every byte to output. - // Only the first 32 - count_ones(mask) bytes of the result are significant but 32 bytes - // get written. - // Design consideration: it seems like a function with the - // signature simd8 compress(uint32_t mask) would be - // sensible, but the AVX ISA makes this kind of approach difficult. - template - simdjson_inline void compress(uint32_t mask, L* output) const { - using internal::thintable_epi8; - using internal::BitsSetTable256mul2; - using internal::pshufb_combine_table; - // this particular implementation was inspired by work done by @animetosho - // we do it in four steps, first 8 bytes and then second 8 bytes... - uint8_t mask1 = uint8_t(mask); // least significant 8 bits - uint8_t mask2 = uint8_t(mask >> 8); // second least significant 8 bits - uint8_t mask3 = uint8_t(mask >> 16); // ... - uint8_t mask4 = uint8_t(mask >> 24); // ... - // next line just loads the 64-bit values thintable_epi8[mask1] and - // thintable_epi8[mask2] into a 128-bit register, using only - // two instructions on most compilers. - __m256i shufmask = _mm256_set_epi64x(thintable_epi8[mask4], thintable_epi8[mask3], - thintable_epi8[mask2], thintable_epi8[mask1]); - // we increment by 0x08 the second half of the mask and so forth - shufmask = - _mm256_add_epi8(shufmask, _mm256_set_epi32(0x18181818, 0x18181818, - 0x10101010, 0x10101010, 0x08080808, 0x08080808, 0, 0)); - // this is the version "nearly pruned" - __m256i pruned = _mm256_shuffle_epi8(*this, shufmask); - // we still need to put the pieces back together. - // we compute the popcount of the first words: - int pop1 = BitsSetTable256mul2[mask1]; - int pop3 = BitsSetTable256mul2[mask3]; - - // then load the corresponding mask - // could be done with _mm256_loadu2_m128i but many standard libraries omit this intrinsic. - __m256i v256 = _mm256_castsi128_si256( - _mm_loadu_si128(reinterpret_cast(pshufb_combine_table + pop1 * 8))); - __m256i compactmask = _mm256_insertf128_si256(v256, - _mm_loadu_si128(reinterpret_cast(pshufb_combine_table + pop3 * 8)), 1); - __m256i almostthere = _mm256_shuffle_epi8(pruned, compactmask); - // We just need to write out the result. - // This is the tricky bit that is hard to do - // if we want to return a SIMD register, since there - // is no single-instruction approach to recombine - // the two 128-bit lanes with an offset. - __m128i v128; - v128 = _mm256_castsi256_si128(almostthere); - _mm_storeu_si128(reinterpret_cast<__m128i*>(output), v128); - v128 = _mm256_extractf128_si256(almostthere, 1); - _mm_storeu_si128(reinterpret_cast<__m128i*>(output + 16 - count_ones(mask & 0xFFFF)), v128); - } - - template - simdjson_inline simd8 lookup_16( - L replace0, L replace1, L replace2, L replace3, - L replace4, L replace5, L replace6, L replace7, - L replace8, L replace9, L replace10, L replace11, - L replace12, L replace13, L replace14, L replace15) const { - return lookup_16(simd8::repeat_16( - replace0, replace1, replace2, replace3, - replace4, replace5, replace6, replace7, - replace8, replace9, replace10, replace11, - replace12, replace13, replace14, replace15 - )); - } - }; - - // Signed bytes - template<> - struct simd8 : base8_numeric { - simdjson_inline simd8() : base8_numeric() {} - simdjson_inline simd8(const __m256i _value) : base8_numeric(_value) {} - // Splat constructor - simdjson_inline simd8(int8_t _value) : simd8(splat(_value)) {} - // Array constructor - simdjson_inline simd8(const int8_t values[32]) : simd8(load(values)) {} - // Member-by-member initialization - simdjson_inline simd8( - int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, - int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15, - int8_t v16, int8_t v17, int8_t v18, int8_t v19, int8_t v20, int8_t v21, int8_t v22, int8_t v23, - int8_t v24, int8_t v25, int8_t v26, int8_t v27, int8_t v28, int8_t v29, int8_t v30, int8_t v31 - ) : simd8(_mm256_setr_epi8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10, v11, v12, v13, v14, v15, - v16, v17, v18, v19, v20, v21, v22, v23, - v24, v25, v26, v27, v28, v29, v30, v31 - )) {} - // Repeat 16 values as many times as necessary (usually for lookup tables) - simdjson_inline static simd8 repeat_16( - int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, - int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 - ) { - return simd8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10, v11, v12, v13, v14, v15, - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10, v11, v12, v13, v14, v15 - ); - } - - // Order-sensitive comparisons - simdjson_inline simd8 max_val(const simd8 other) const { return _mm256_max_epi8(*this, other); } - simdjson_inline simd8 min_val(const simd8 other) const { return _mm256_min_epi8(*this, other); } - simdjson_inline simd8 operator>(const simd8 other) const { return _mm256_cmpgt_epi8(*this, other); } - simdjson_inline simd8 operator<(const simd8 other) const { return _mm256_cmpgt_epi8(other, *this); } - }; - - // Unsigned bytes - template<> - struct simd8 : base8_numeric { - simdjson_inline simd8() : base8_numeric() {} - simdjson_inline simd8(const __m256i _value) : base8_numeric(_value) {} - // Splat constructor - simdjson_inline simd8(uint8_t _value) : simd8(splat(_value)) {} - // Array constructor - simdjson_inline simd8(const uint8_t values[32]) : simd8(load(values)) {} - // Member-by-member initialization - simdjson_inline simd8( - uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, - uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15, - uint8_t v16, uint8_t v17, uint8_t v18, uint8_t v19, uint8_t v20, uint8_t v21, uint8_t v22, uint8_t v23, - uint8_t v24, uint8_t v25, uint8_t v26, uint8_t v27, uint8_t v28, uint8_t v29, uint8_t v30, uint8_t v31 - ) : simd8(_mm256_setr_epi8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10, v11, v12, v13, v14, v15, - v16, v17, v18, v19, v20, v21, v22, v23, - v24, v25, v26, v27, v28, v29, v30, v31 - )) {} - // Repeat 16 values as many times as necessary (usually for lookup tables) - simdjson_inline static simd8 repeat_16( - uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, - uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 - ) { - return simd8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10, v11, v12, v13, v14, v15, - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10, v11, v12, v13, v14, v15 - ); - } - - // Saturated math - simdjson_inline simd8 saturating_add(const simd8 other) const { return _mm256_adds_epu8(*this, other); } - simdjson_inline simd8 saturating_sub(const simd8 other) const { return _mm256_subs_epu8(*this, other); } - - // Order-specific operations - simdjson_inline simd8 max_val(const simd8 other) const { return _mm256_max_epu8(*this, other); } - simdjson_inline simd8 min_val(const simd8 other) const { return _mm256_min_epu8(other, *this); } - // Same as >, but only guarantees true is nonzero (< guarantees true = -1) - simdjson_inline simd8 gt_bits(const simd8 other) const { return this->saturating_sub(other); } - // Same as <, but only guarantees true is nonzero (< guarantees true = -1) - simdjson_inline simd8 lt_bits(const simd8 other) const { return other.saturating_sub(*this); } - simdjson_inline simd8 operator<=(const simd8 other) const { return other.max_val(*this) == other; } - simdjson_inline simd8 operator>=(const simd8 other) const { return other.min_val(*this) == other; } - simdjson_inline simd8 operator>(const simd8 other) const { return this->gt_bits(other).any_bits_set(); } - simdjson_inline simd8 operator<(const simd8 other) const { return this->lt_bits(other).any_bits_set(); } - - // Bit-specific operations - simdjson_inline simd8 bits_not_set() const { return *this == uint8_t(0); } - simdjson_inline simd8 bits_not_set(simd8 bits) const { return (*this & bits).bits_not_set(); } - simdjson_inline simd8 any_bits_set() const { return ~this->bits_not_set(); } - simdjson_inline simd8 any_bits_set(simd8 bits) const { return ~this->bits_not_set(bits); } - simdjson_inline bool is_ascii() const { return _mm256_movemask_epi8(*this) == 0; } - simdjson_inline bool bits_not_set_anywhere() const { return _mm256_testz_si256(*this, *this); } - simdjson_inline bool any_bits_set_anywhere() const { return !bits_not_set_anywhere(); } - simdjson_inline bool bits_not_set_anywhere(simd8 bits) const { return _mm256_testz_si256(*this, bits); } - simdjson_inline bool any_bits_set_anywhere(simd8 bits) const { return !bits_not_set_anywhere(bits); } - template - simdjson_inline simd8 shr() const { return simd8(_mm256_srli_epi16(*this, N)) & uint8_t(0xFFu >> N); } - template - simdjson_inline simd8 shl() const { return simd8(_mm256_slli_epi16(*this, N)) & uint8_t(0xFFu << N); } - // Get one of the bits and make a bitmask out of it. - // e.g. value.get_bit<7>() gets the high bit - template - simdjson_inline int get_bit() const { return _mm256_movemask_epi8(_mm256_slli_epi16(*this, 7 - N)); } - }; - - template - struct simd8x64 { - static constexpr int NUM_CHUNKS = 64 / sizeof(simd8); - static_assert(NUM_CHUNKS == 2, "Haswell kernel should use two registers per 64-byte block."); - const simd8 chunks[NUM_CHUNKS]; - - simd8x64(const simd8x64& o) = delete; // no copy allowed - simd8x64& operator=(const simd8& other) = delete; // no assignment allowed - simd8x64() = delete; // no default constructor allowed - - simdjson_inline simd8x64(const simd8 chunk0, const simd8 chunk1) : chunks{ chunk0, chunk1 } {} - simdjson_inline simd8x64(const T ptr[64]) : chunks{ simd8::load(ptr), simd8::load(ptr + 32) } {} - - simdjson_inline uint64_t compress(uint64_t mask, T* output) const { - uint32_t mask1 = uint32_t(mask); - uint32_t mask2 = uint32_t(mask >> 32); - this->chunks[0].compress(mask1, output); - this->chunks[1].compress(mask2, output + 32 - count_ones(mask1)); - return 64 - count_ones(mask); - } - - simdjson_inline void store(T ptr[64]) const { - this->chunks[0].store(ptr + sizeof(simd8) * 0); - this->chunks[1].store(ptr + sizeof(simd8) * 1); - } - - simdjson_inline uint64_t to_bitmask() const { - uint64_t r_lo = uint32_t(this->chunks[0].to_bitmask()); - uint64_t r_hi = this->chunks[1].to_bitmask(); - return r_lo | (r_hi << 32); - } - - simdjson_inline simd8 reduce_or() const { - return this->chunks[0] | this->chunks[1]; - } - - simdjson_inline simd8x64 bit_or(const T m) const { - const simd8 mask = simd8::splat(m); - return simd8x64( - this->chunks[0] | mask, - this->chunks[1] | mask - ); - } - - simdjson_inline uint64_t eq(const T m) const { - const simd8 mask = simd8::splat(m); - return simd8x64( - this->chunks[0] == mask, - this->chunks[1] == mask - ).to_bitmask(); - } - - simdjson_inline uint64_t eq(const simd8x64& other) const { - return simd8x64( - this->chunks[0] == other.chunks[0], - this->chunks[1] == other.chunks[1] - ).to_bitmask(); - } - - simdjson_inline uint64_t lteq(const T m) const { - const simd8 mask = simd8::splat(m); - return simd8x64( - this->chunks[0] <= mask, - this->chunks[1] <= mask - ).to_bitmask(); - } - }; // struct simd8x64 - - } // namespace simd - - } // unnamed namespace - } // namespace haswell -} // namespace simdjson - -#endif // SIMDJSON_HASWELL_SIMD_H -/* end file simdjson/haswell/simd.h */ -/* including simdjson/haswell/stringparsing_defs.h: #include "simdjson/haswell/stringparsing_defs.h" */ -/* begin file simdjson/haswell/stringparsing_defs.h */ -#ifndef SIMDJSON_HASWELL_STRINGPARSING_DEFS_H -#define SIMDJSON_HASWELL_STRINGPARSING_DEFS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/haswell/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/haswell/simd.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/haswell/bitmanipulation.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - namespace haswell { - namespace { - - using namespace simd; - - // Holds backslashes and quotes locations. - struct backslash_and_quote { - public: - static constexpr uint32_t BYTES_PROCESSED = 32; - simdjson_inline static backslash_and_quote copy_and_find(const uint8_t* src, uint8_t* dst); - - simdjson_inline bool has_quote_first() { return ((bs_bits - 1) & quote_bits) != 0; } - simdjson_inline bool has_backslash() { return ((quote_bits - 1) & bs_bits) != 0; } - simdjson_inline int quote_index() { return trailing_zeroes(quote_bits); } - simdjson_inline int backslash_index() { return trailing_zeroes(bs_bits); } - - uint32_t bs_bits; - uint32_t quote_bits; - }; // struct backslash_and_quote - - simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8_t* src, uint8_t* dst) { - // this can read up to 15 bytes beyond the buffer size, but we require - // SIMDJSON_PADDING of padding - static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "backslash and quote finder must process fewer than SIMDJSON_PADDING bytes"); - simd8 v(src); - // store to dest unconditionally - we can overwrite the bits we don't like later - v.store(dst); - return { - static_cast((v == '\\').to_bitmask()), // bs_bits - static_cast((v == '"').to_bitmask()), // quote_bits - }; - } - - } // unnamed namespace - } // namespace haswell -} // namespace simdjson - -#endif // SIMDJSON_HASWELL_STRINGPARSING_DEFS_H -/* end file simdjson/haswell/stringparsing_defs.h */ -/* end file simdjson/haswell/begin.h */ -/* including simdjson/generic/amalgamated.h for haswell: #include "simdjson/generic/amalgamated.h" */ -/* begin file simdjson/generic/amalgamated.h for haswell */ -#if defined(SIMDJSON_CONDITIONAL_INCLUDE) && !defined(SIMDJSON_GENERIC_DEPENDENCIES_H) -#error simdjson/generic/dependencies.h must be included before simdjson/generic/amalgamated.h! -#endif - -/* including simdjson/generic/base.h for haswell: #include "simdjson/generic/base.h" */ -/* begin file simdjson/generic/base.h for haswell */ -#ifndef SIMDJSON_GENERIC_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_BASE_H */ -/* amalgamation skipped (editor-only): #include "simdjson/base.h" */ -/* amalgamation skipped (editor-only): // If we haven't got an implementation yet, we're in the editor, editing a generic file! Just */ -/* amalgamation skipped (editor-only): // use the most advanced one we can so the most possible stuff can be tested. */ -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_IMPLEMENTATION */ -/* amalgamation skipped (editor-only): #include "simdjson/implementation_detection.h" */ -/* amalgamation skipped (editor-only): #if SIMDJSON_IMPLEMENTATION_ICELAKE */ -/* amalgamation skipped (editor-only): #include "simdjson/icelake/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_HASWELL */ -/* amalgamation skipped (editor-only): #include "simdjson/haswell/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_WESTMERE */ -/* amalgamation skipped (editor-only): #include "simdjson/westmere/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_ARM64 */ -/* amalgamation skipped (editor-only): #include "simdjson/arm64/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_PPC64 */ -/* amalgamation skipped (editor-only): #include "simdjson/ppc64/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_FALLBACK */ -/* amalgamation skipped (editor-only): #include "simdjson/fallback/begin.h" */ -/* amalgamation skipped (editor-only): #else */ -/* amalgamation skipped (editor-only): #error "All possible implementations (including fallback) have been disabled! simdjson will not run." */ -/* amalgamation skipped (editor-only): #endif */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_IMPLEMENTATION */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - namespace haswell { - - struct open_container; - class dom_parser_implementation; - - /** - * The type of a JSON number - */ - enum class number_type { - floating_point_number = 1, /// a binary64 number - signed_integer, /// a signed integer that fits in a 64-bit word using two's complement - unsigned_integer /// a positive integer larger or equal to 1<<63 - }; - - } // namespace haswell -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_BASE_H -/* end file simdjson/generic/base.h for haswell */ -/* including simdjson/generic/jsoncharutils.h for haswell: #include "simdjson/generic/jsoncharutils.h" */ -/* begin file simdjson/generic/jsoncharutils.h for haswell */ -#ifndef SIMDJSON_GENERIC_JSONCHARUTILS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_JSONCHARUTILS_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/jsoncharutils_tables.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - namespace haswell { - namespace { - namespace jsoncharutils { - - // return non-zero if not a structural or whitespace char - // zero otherwise - simdjson_inline uint32_t is_not_structural_or_whitespace(uint8_t c) { - return internal::structural_or_whitespace_negated[c]; - } - - simdjson_inline uint32_t is_structural_or_whitespace(uint8_t c) { - return internal::structural_or_whitespace[c]; - } - - // returns a value with the high 16 bits set if not valid - // otherwise returns the conversion of the 4 hex digits at src into the bottom - // 16 bits of the 32-bit return register - // - // see - // https://lemire.me/blog/2019/04/17/parsing-short-hexadecimal-strings-efficiently/ - static inline uint32_t hex_to_u32_nocheck( - const uint8_t* src) { // strictly speaking, static inline is a C-ism - uint32_t v1 = internal::digit_to_val32[630 + src[0]]; - uint32_t v2 = internal::digit_to_val32[420 + src[1]]; - uint32_t v3 = internal::digit_to_val32[210 + src[2]]; - uint32_t v4 = internal::digit_to_val32[0 + src[3]]; - return v1 | v2 | v3 | v4; - } - - // given a code point cp, writes to c - // the utf-8 code, outputting the length in - // bytes, if the length is zero, the code point - // is invalid - // - // This can possibly be made faster using pdep - // and clz and table lookups, but JSON documents - // have few escaped code points, and the following - // function looks cheap. - // - // Note: we assume that surrogates are treated separately - // - simdjson_inline size_t codepoint_to_utf8(uint32_t cp, uint8_t* c) { - if (cp <= 0x7F) { - c[0] = uint8_t(cp); - return 1; // ascii - } - if (cp <= 0x7FF) { - c[0] = uint8_t((cp >> 6) + 192); - c[1] = uint8_t((cp & 63) + 128); - return 2; // universal plane - // Surrogates are treated elsewhere... - //} //else if (0xd800 <= cp && cp <= 0xdfff) { - // return 0; // surrogates // could put assert here - } - else if (cp <= 0xFFFF) { - c[0] = uint8_t((cp >> 12) + 224); - c[1] = uint8_t(((cp >> 6) & 63) + 128); - c[2] = uint8_t((cp & 63) + 128); - return 3; - } - else if (cp <= 0x10FFFF) { // if you know you have a valid code point, this - // is not needed - c[0] = uint8_t((cp >> 18) + 240); - c[1] = uint8_t(((cp >> 12) & 63) + 128); - c[2] = uint8_t(((cp >> 6) & 63) + 128); - c[3] = uint8_t((cp & 63) + 128); - return 4; - } - // will return 0 when the code point was too large. - return 0; // bad r - } - -#if SIMDJSON_IS_32BITS // _umul128 for x86, arm - // this is a slow emulation routine for 32-bit - // - static simdjson_inline uint64_t __emulu(uint32_t x, uint32_t y) { - return x * (uint64_t)y; - } - static simdjson_inline uint64_t _umul128(uint64_t ab, uint64_t cd, uint64_t* hi) { - uint64_t ad = __emulu((uint32_t)(ab >> 32), (uint32_t)cd); - uint64_t bd = __emulu((uint32_t)ab, (uint32_t)cd); - uint64_t adbc = ad + __emulu((uint32_t)ab, (uint32_t)(cd >> 32)); - uint64_t adbc_carry = !!(adbc < ad); - uint64_t lo = bd + (adbc << 32); - *hi = __emulu((uint32_t)(ab >> 32), (uint32_t)(cd >> 32)) + (adbc >> 32) + - (adbc_carry << 32) + !!(lo < bd); - return lo; - } -#endif - - } // namespace jsoncharutils - } // unnamed namespace - } // namespace haswell -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_JSONCHARUTILS_H -/* end file simdjson/generic/jsoncharutils.h for haswell */ -/* including simdjson/generic/atomparsing.h for haswell: #include "simdjson/generic/atomparsing.h" */ -/* begin file simdjson/generic/atomparsing.h for haswell */ -#ifndef SIMDJSON_GENERIC_ATOMPARSING_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_ATOMPARSING_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/jsoncharutils.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include - -namespace simdjson { - namespace haswell { - namespace { - /// @private - namespace atomparsing { - - // The string_to_uint32 is exclusively used to map literal strings to 32-bit values. - // We use memcpy instead of a pointer cast to avoid undefined behaviors since we cannot - // be certain that the character pointer will be properly aligned. - // You might think that using memcpy makes this function expensive, but you'd be wrong. - // All decent optimizing compilers (GCC, clang, Visual Studio) will compile string_to_uint32("false"); - // to the compile-time constant 1936482662. - simdjson_inline uint32_t string_to_uint32(const char* str) { uint32_t val; std::memcpy(&val, str, sizeof(uint32_t)); return val; } - - - // Again in str4ncmp we use a memcpy to avoid undefined behavior. The memcpy may appear expensive. - // Yet all decent optimizing compilers will compile memcpy to a single instruction, just about. - simdjson_warn_unused - simdjson_inline uint32_t str4ncmp(const uint8_t* src, const char* atom) { - uint32_t srcval; // we want to avoid unaligned 32-bit loads (undefined in C/C++) - static_assert(sizeof(uint32_t) <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be larger than 4 bytes"); - std::memcpy(&srcval, src, sizeof(uint32_t)); - return srcval ^ string_to_uint32(atom); - } - - simdjson_warn_unused - simdjson_inline bool is_valid_true_atom(const uint8_t* src) { - return (str4ncmp(src, "true") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; - } - - simdjson_warn_unused - simdjson_inline bool is_valid_true_atom(const uint8_t* src, size_t len) { - if (len > 4) { return is_valid_true_atom(src); } - else if (len == 4) { return !str4ncmp(src, "true"); } - else { return false; } - } - - simdjson_warn_unused - simdjson_inline bool is_valid_false_atom(const uint8_t* src) { - return (str4ncmp(src + 1, "alse") | jsoncharutils::is_not_structural_or_whitespace(src[5])) == 0; - } - - simdjson_warn_unused - simdjson_inline bool is_valid_false_atom(const uint8_t* src, size_t len) { - if (len > 5) { return is_valid_false_atom(src); } - else if (len == 5) { return !str4ncmp(src + 1, "alse"); } - else { return false; } - } - - simdjson_warn_unused - simdjson_inline bool is_valid_null_atom(const uint8_t* src) { - return (str4ncmp(src, "null") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; - } - - simdjson_warn_unused - simdjson_inline bool is_valid_null_atom(const uint8_t* src, size_t len) { - if (len > 4) { return is_valid_null_atom(src); } - else if (len == 4) { return !str4ncmp(src, "null"); } - else { return false; } - } - - } // namespace atomparsing - } // unnamed namespace - } // namespace haswell -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_ATOMPARSING_H -/* end file simdjson/generic/atomparsing.h for haswell */ -/* including simdjson/generic/dom_parser_implementation.h for haswell: #include "simdjson/generic/dom_parser_implementation.h" */ -/* begin file simdjson/generic/dom_parser_implementation.h for haswell */ -#ifndef SIMDJSON_GENERIC_DOM_PARSER_IMPLEMENTATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_DOM_PARSER_IMPLEMENTATION_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/dom_parser_implementation.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - namespace haswell { - - // expectation: sizeof(open_container) = 64/8. - struct open_container { - uint32_t tape_index; // where, on the tape, does the scope ([,{) begins - uint32_t count; // how many elements in the scope - }; // struct open_container - - static_assert(sizeof(open_container) == 64 / 8, "Open container must be 64 bits"); - - class dom_parser_implementation final : public internal::dom_parser_implementation { - public: - /** Tape location of each open { or [ */ - std::unique_ptr open_containers{}; - /** Whether each open container is a [ or { */ - std::unique_ptr is_array{}; - /** Buffer passed to stage 1 */ - const uint8_t* buf{}; - /** Length passed to stage 1 */ - size_t len{ 0 }; - /** Document passed to stage 2 */ - dom::document* doc{}; - - inline dom_parser_implementation() noexcept; - inline dom_parser_implementation(dom_parser_implementation&& other) noexcept; - inline dom_parser_implementation& operator=(dom_parser_implementation&& other) noexcept; - dom_parser_implementation(const dom_parser_implementation&) = delete; - dom_parser_implementation& operator=(const dom_parser_implementation&) = delete; - - simdjson_warn_unused error_code parse(const uint8_t* buf, size_t len, dom::document& doc) noexcept final; - simdjson_warn_unused error_code stage1(const uint8_t* buf, size_t len, stage1_mode partial) noexcept final; - simdjson_warn_unused error_code stage2(dom::document& doc) noexcept final; - simdjson_warn_unused error_code stage2_next(dom::document& doc) noexcept final; - simdjson_warn_unused uint8_t* parse_string(const uint8_t* src, uint8_t* dst, bool allow_replacement) const noexcept final; - simdjson_warn_unused uint8_t* parse_wobbly_string(const uint8_t* src, uint8_t* dst) const noexcept final; - inline simdjson_warn_unused error_code set_capacity(size_t capacity) noexcept final; - inline simdjson_warn_unused error_code set_max_depth(size_t max_depth) noexcept final; - private: - simdjson_inline simdjson_warn_unused error_code set_capacity_stage1(size_t capacity); - - }; - - } // namespace haswell -} // namespace simdjson - -namespace simdjson { - namespace haswell { - - inline dom_parser_implementation::dom_parser_implementation() noexcept = default; - inline dom_parser_implementation::dom_parser_implementation(dom_parser_implementation&& other) noexcept = default; - inline dom_parser_implementation& dom_parser_implementation::operator=(dom_parser_implementation&& other) noexcept = default; - - // Leaving these here so they can be inlined if so desired - inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(size_t capacity) noexcept { - if (capacity > SIMDJSON_MAXSIZE_BYTES) { return CAPACITY; } - // Stage 1 index output - size_t max_structures = SIMDJSON_ROUNDUP_N(capacity, 64) + 2 + 7; - structural_indexes.reset(new (std::nothrow) uint32_t[max_structures]); - if (!structural_indexes) { _capacity = 0; return MEMALLOC; } - structural_indexes[0] = 0; - n_structural_indexes = 0; - - _capacity = capacity; - return SUCCESS; - } - - inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { - // Stage 2 stacks - open_containers.reset(new (std::nothrow) open_container[max_depth]); - is_array.reset(new (std::nothrow) bool[max_depth]); - if (!is_array || !open_containers) { _max_depth = 0; return MEMALLOC; } - - _max_depth = max_depth; - return SUCCESS; - } - - } // namespace haswell -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_DOM_PARSER_IMPLEMENTATION_H -/* end file simdjson/generic/dom_parser_implementation.h for haswell */ -/* including simdjson/generic/implementation_simdjson_result_base.h for haswell: #include "simdjson/generic/implementation_simdjson_result_base.h" */ -/* begin file simdjson/generic/implementation_simdjson_result_base.h for haswell */ -#ifndef SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - namespace haswell { - - // This is a near copy of include/error.h's implementation_simdjson_result_base, except it doesn't use std::pair - // so we can avoid inlining errors - // TODO reconcile these! - /** - * The result of a simdjson operation that could fail. - * - * Gives the option of reading error codes, or throwing an exception by casting to the desired result. - * - * This is a base class for implementations that want to add functions to the result type for - * chaining. - * - * Override like: - * - * struct simdjson_result : public internal::implementation_simdjson_result_base { - * simdjson_result() noexcept : internal::implementation_simdjson_result_base() {} - * simdjson_result(error_code error) noexcept : internal::implementation_simdjson_result_base(error) {} - * simdjson_result(T &&value) noexcept : internal::implementation_simdjson_result_base(std::forward(value)) {} - * simdjson_result(T &&value, error_code error) noexcept : internal::implementation_simdjson_result_base(value, error) {} - * // Your extra methods here - * } - * - * Then any method returning simdjson_result will be chainable with your methods. - */ - template - struct implementation_simdjson_result_base { - - /** - * Create a new empty result with error = UNINITIALIZED. - */ - simdjson_inline implementation_simdjson_result_base() noexcept = default; - - /** - * Create a new error result. - */ - simdjson_inline implementation_simdjson_result_base(error_code error) noexcept; - - /** - * Create a new successful result. - */ - simdjson_inline implementation_simdjson_result_base(T&& value) noexcept; - - /** - * Create a new result with both things (use if you don't want to branch when creating the result). - */ - simdjson_inline implementation_simdjson_result_base(T&& value, error_code error) noexcept; - - /** - * Move the value and the error to the provided variables. - * - * @param value The variable to assign the value to. May not be set if there is an error. - * @param error The variable to assign the error to. Set to SUCCESS if there is no error. - */ - simdjson_inline void tie(T& value, error_code& error) && noexcept; - - /** - * Move the value to the provided variable. - * - * @param value The variable to assign the value to. May not be set if there is an error. - */ - simdjson_inline error_code get(T& value) && noexcept; - - /** - * The error. - */ - simdjson_inline error_code error() const noexcept; - -#if SIMDJSON_EXCEPTIONS - - /** - * Get the result value. - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline T& value() & noexcept(false); - - /** - * Take the result value (move it). - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline T&& value() && noexcept(false); - - /** - * Take the result value (move it). - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline T&& take_value() && noexcept(false); - - /** - * Cast to the value (will throw on error). - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline operator T && () && noexcept(false); - - -#endif // SIMDJSON_EXCEPTIONS - - /** - * Get the result value. This function is safe if and only - * the error() method returns a value that evaluates to false. - */ - simdjson_inline const T& value_unsafe() const& noexcept; - /** - * Get the result value. This function is safe if and only - * the error() method returns a value that evaluates to false. - */ - simdjson_inline T& value_unsafe() & noexcept; - /** - * Take the result value (move it). This function is safe if and only - * the error() method returns a value that evaluates to false. - */ - simdjson_inline T&& value_unsafe() && noexcept; - protected: - /** users should never directly access first and second. **/ - T first{}; /** Users should never directly access 'first'. **/ - error_code second{ UNINITIALIZED }; /** Users should never directly access 'second'. **/ - }; // struct implementation_simdjson_result_base - - } // namespace haswell -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_H -/* end file simdjson/generic/implementation_simdjson_result_base.h for haswell */ -/* including simdjson/generic/numberparsing.h for haswell: #include "simdjson/generic/numberparsing.h" */ -/* begin file simdjson/generic/numberparsing.h for haswell */ -#ifndef SIMDJSON_GENERIC_NUMBERPARSING_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_NUMBERPARSING_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/jsoncharutils.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include -#include -#include - -namespace simdjson { - namespace haswell { - namespace numberparsing { - -#ifdef JSON_TEST_NUMBERS -#define INVALID_NUMBER(SRC) (found_invalid_number((SRC)), NUMBER_ERROR) -#define WRITE_INTEGER(VALUE, SRC, WRITER) (found_integer((VALUE), (SRC)), (WRITER).append_s64((VALUE))) -#define WRITE_UNSIGNED(VALUE, SRC, WRITER) (found_unsigned_integer((VALUE), (SRC)), (WRITER).append_u64((VALUE))) -#define WRITE_DOUBLE(VALUE, SRC, WRITER) (found_float((VALUE), (SRC)), (WRITER).append_double((VALUE))) -#else -#define INVALID_NUMBER(SRC) (NUMBER_ERROR) -#define WRITE_INTEGER(VALUE, SRC, WRITER) (WRITER).append_s64((VALUE)) -#define WRITE_UNSIGNED(VALUE, SRC, WRITER) (WRITER).append_u64((VALUE)) -#define WRITE_DOUBLE(VALUE, SRC, WRITER) (WRITER).append_double((VALUE)) -#endif - - namespace { - - // Convert a mantissa, an exponent and a sign bit into an ieee64 double. - // The real_exponent needs to be in [0, 2046] (technically real_exponent = 2047 would be acceptable). - // The mantissa should be in [0,1<<53). The bit at index (1ULL << 52) while be zeroed. - simdjson_inline double to_double(uint64_t mantissa, uint64_t real_exponent, bool negative) { - double d; - mantissa &= ~(1ULL << 52); - mantissa |= real_exponent << 52; - mantissa |= ((static_cast(negative)) << 63); - std::memcpy(&d, &mantissa, sizeof(d)); - return d; - } - - // Attempts to compute i * 10^(power) exactly; and if "negative" is - // true, negate the result. - // This function will only work in some cases, when it does not work, success is - // set to false. This should work *most of the time* (like 99% of the time). - // We assume that power is in the [smallest_power, - // largest_power] interval: the caller is responsible for this check. - simdjson_inline bool compute_float_64(int64_t power, uint64_t i, bool negative, double& d) { - // we start with a fast path - // It was described in - // Clinger WD. How to read floating point numbers accurately. - // ACM SIGPLAN Notices. 1990 -#ifndef FLT_EVAL_METHOD -#error "FLT_EVAL_METHOD should be defined, please include cfloat." -#endif -#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0) - // We cannot be certain that x/y is rounded to nearest. - if (0 <= power && power <= 22 && i <= 9007199254740991) -#else - if (-22 <= power && power <= 22 && i <= 9007199254740991) -#endif - { - // convert the integer into a double. This is lossless since - // 0 <= i <= 2^53 - 1. - d = double(i); - // - // The general idea is as follows. - // If 0 <= s < 2^53 and if 10^0 <= p <= 10^22 then - // 1) Both s and p can be represented exactly as 64-bit floating-point - // values - // (binary64). - // 2) Because s and p can be represented exactly as floating-point values, - // then s * p - // and s / p will produce correctly rounded values. - // - if (power < 0) { - d = d / simdjson::internal::power_of_ten[-power]; - } - else { - d = d * simdjson::internal::power_of_ten[power]; - } - if (negative) { - d = -d; - } - return true; - } - // When 22 < power && power < 22 + 16, we could - // hope for another, secondary fast path. It was - // described by David M. Gay in "Correctly rounded - // binary-decimal and decimal-binary conversions." (1990) - // If you need to compute i * 10^(22 + x) for x < 16, - // first compute i * 10^x, if you know that result is exact - // (e.g., when i * 10^x < 2^53), - // then you can still proceed and do (i * 10^x) * 10^22. - // Is this worth your time? - // You need 22 < power *and* power < 22 + 16 *and* (i * 10^(x-22) < 2^53) - // for this second fast path to work. - // If you you have 22 < power *and* power < 22 + 16, and then you - // optimistically compute "i * 10^(x-22)", there is still a chance that you - // have wasted your time if i * 10^(x-22) >= 2^53. It makes the use cases of - // this optimization maybe less common than we would like. Source: - // http://www.exploringbinary.com/fast-path-decimal-to-floating-point-conversion/ - // also used in RapidJSON: https://rapidjson.org/strtod_8h_source.html - - // The fast path has now failed, so we are failing back on the slower path. - - // In the slow path, we need to adjust i so that it is > 1<<63 which is always - // possible, except if i == 0, so we handle i == 0 separately. - if (i == 0) { - d = negative ? -0.0 : 0.0; - return true; - } - - - // The exponent is 1024 + 63 + power - // + floor(log(5**power)/log(2)). - // The 1024 comes from the ieee64 standard. - // The 63 comes from the fact that we use a 64-bit word. - // - // Computing floor(log(5**power)/log(2)) could be - // slow. Instead we use a fast function. - // - // For power in (-400,350), we have that - // (((152170 + 65536) * power ) >> 16); - // is equal to - // floor(log(5**power)/log(2)) + power when power >= 0 - // and it is equal to - // ceil(log(5**-power)/log(2)) + power when power < 0 - // - // The 65536 is (1<<16) and corresponds to - // (65536 * power) >> 16 ---> power - // - // ((152170 * power ) >> 16) is equal to - // floor(log(5**power)/log(2)) - // - // Note that this is not magic: 152170/(1<<16) is - // approximatively equal to log(5)/log(2). - // The 1<<16 value is a power of two; we could use a - // larger power of 2 if we wanted to. - // - int64_t exponent = (((152170 + 65536) * power) >> 16) + 1024 + 63; - - - // We want the most significant bit of i to be 1. Shift if needed. - int lz = leading_zeroes(i); - i <<= lz; - - - // We are going to need to do some 64-bit arithmetic to get a precise product. - // We use a table lookup approach. - // It is safe because - // power >= smallest_power - // and power <= largest_power - // We recover the mantissa of the power, it has a leading 1. It is always - // rounded down. - // - // We want the most significant 64 bits of the product. We know - // this will be non-zero because the most significant bit of i is - // 1. - const uint32_t index = 2 * uint32_t(power - simdjson::internal::smallest_power); - // Optimization: It may be that materializing the index as a variable might confuse some compilers and prevent effective complex-addressing loads. (Done for code clarity.) - // - // The full_multiplication function computes the 128-bit product of two 64-bit words - // with a returned value of type value128 with a "low component" corresponding to the - // 64-bit least significant bits of the product and with a "high component" corresponding - // to the 64-bit most significant bits of the product. - simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index]); - // Both i and power_of_five_128[index] have their most significant bit set to 1 which - // implies that the either the most or the second most significant bit of the product - // is 1. We pack values in this manner for efficiency reasons: it maximizes the use - // we make of the product. It also makes it easy to reason about the product: there - // is 0 or 1 leading zero in the product. - - // Unless the least significant 9 bits of the high (64-bit) part of the full - // product are all 1s, then we know that the most significant 55 bits are - // exact and no further work is needed. Having 55 bits is necessary because - // we need 53 bits for the mantissa but we have to have one rounding bit and - // we can waste a bit if the most significant bit of the product is zero. - if ((firstproduct.high & 0x1FF) == 0x1FF) { - // We want to compute i * 5^q, but only care about the top 55 bits at most. - // Consider the scenario where q>=0. Then 5^q may not fit in 64-bits. Doing - // the full computation is wasteful. So we do what is called a "truncated - // multiplication". - // We take the most significant 64-bits, and we put them in - // power_of_five_128[index]. Usually, that's good enough to approximate i * 5^q - // to the desired approximation using one multiplication. Sometimes it does not suffice. - // Then we store the next most significant 64 bits in power_of_five_128[index + 1], and - // then we get a better approximation to i * 5^q. - // - // That's for when q>=0. The logic for q<0 is somewhat similar but it is somewhat - // more complicated. - // - // There is an extra layer of complexity in that we need more than 55 bits of - // accuracy in the round-to-even scenario. - // - // The full_multiplication function computes the 128-bit product of two 64-bit words - // with a returned value of type value128 with a "low component" corresponding to the - // 64-bit least significant bits of the product and with a "high component" corresponding - // to the 64-bit most significant bits of the product. - simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index + 1]); - firstproduct.low += secondproduct.high; - if (secondproduct.high > firstproduct.low) { firstproduct.high++; } - // As it has been proven by Noble Mushtak and Daniel Lemire in "Fast Number Parsing Without - // Fallback" (https://arxiv.org/abs/2212.06644), at this point we are sure that the product - // is sufficiently accurate, and more computation is not needed. - } - uint64_t lower = firstproduct.low; - uint64_t upper = firstproduct.high; - // The final mantissa should be 53 bits with a leading 1. - // We shift it so that it occupies 54 bits with a leading 1. - /////// - uint64_t upperbit = upper >> 63; - uint64_t mantissa = upper >> (upperbit + 9); - lz += int(1 ^ upperbit); - - // Here we have mantissa < (1<<54). - int64_t real_exponent = exponent - lz; - if (simdjson_unlikely(real_exponent <= 0)) { // we have a subnormal? - // Here have that real_exponent <= 0 so -real_exponent >= 0 - if (-real_exponent + 1 >= 64) { // if we have more than 64 bits below the minimum exponent, you have a zero for sure. - d = negative ? -0.0 : 0.0; - return true; - } - // next line is safe because -real_exponent + 1 < 0 - mantissa >>= -real_exponent + 1; - // Thankfully, we can't have both "round-to-even" and subnormals because - // "round-to-even" only occurs for powers close to 0. - mantissa += (mantissa & 1); // round up - mantissa >>= 1; - // There is a weird scenario where we don't have a subnormal but just. - // Suppose we start with 2.2250738585072013e-308, we end up - // with 0x3fffffffffffff x 2^-1023-53 which is technically subnormal - // whereas 0x40000000000000 x 2^-1023-53 is normal. Now, we need to round - // up 0x3fffffffffffff x 2^-1023-53 and once we do, we are no longer - // subnormal, but we can only know this after rounding. - // So we only declare a subnormal if we are smaller than the threshold. - real_exponent = (mantissa < (uint64_t(1) << 52)) ? 0 : 1; - d = to_double(mantissa, real_exponent, negative); - return true; - } - // We have to round to even. The "to even" part - // is only a problem when we are right in between two floats - // which we guard against. - // If we have lots of trailing zeros, we may fall right between two - // floating-point values. - // - // The round-to-even cases take the form of a number 2m+1 which is in (2^53,2^54] - // times a power of two. That is, it is right between a number with binary significand - // m and another number with binary significand m+1; and it must be the case - // that it cannot be represented by a float itself. - // - // We must have that w * 10 ^q == (2m+1) * 2^p for some power of two 2^p. - // Recall that 10^q = 5^q * 2^q. - // When q >= 0, we must have that (2m+1) is divible by 5^q, so 5^q <= 2^54. We have that - // 5^23 <= 2^54 and it is the last power of five to qualify, so q <= 23. - // When q<0, we have w >= (2m+1) x 5^{-q}. We must have that w<2^{64} so - // (2m+1) x 5^{-q} < 2^{64}. We have that 2m+1>2^{53}. Hence, we must have - // 2^{53} x 5^{-q} < 2^{64}. - // Hence we have 5^{-q} < 2^{11}$ or q>= -4. - // - // We require lower <= 1 and not lower == 0 because we could not prove that - // that lower == 0 is implied; but we could prove that lower <= 1 is a necessary and sufficient test. - if (simdjson_unlikely((lower <= 1) && (power >= -4) && (power <= 23) && ((mantissa & 3) == 1))) { - if ((mantissa << (upperbit + 64 - 53 - 2)) == upper) { - mantissa &= ~1; // flip it so that we do not round up - } - } - - mantissa += mantissa & 1; - mantissa >>= 1; - - // Here we have mantissa < (1<<53), unless there was an overflow - if (mantissa >= (1ULL << 53)) { - ////////// - // This will happen when parsing values such as 7.2057594037927933e+16 - //////// - mantissa = (1ULL << 52); - real_exponent++; - } - mantissa &= ~(1ULL << 52); - // we have to check that real_exponent is in range, otherwise we bail out - if (simdjson_unlikely(real_exponent > 2046)) { - // We have an infinite value!!! We could actually throw an error here if we could. - return false; - } - d = to_double(mantissa, real_exponent, negative); - return true; - } - - // We call a fallback floating-point parser that might be slow. Note - // it will accept JSON numbers, but the JSON spec. is more restrictive so - // before you call parse_float_fallback, you need to have validated the input - // string with the JSON grammar. - // It will return an error (false) if the parsed number is infinite. - // The string parsing itself always succeeds. We know that there is at least - // one digit. - static bool parse_float_fallback(const uint8_t* ptr, double* outDouble) { - *outDouble = simdjson::internal::from_chars(reinterpret_cast(ptr)); - // We do not accept infinite values. - - // Detecting finite values in a portable manner is ridiculously hard, ideally - // we would want to do: - // return !std::isfinite(*outDouble); - // but that mysteriously fails under legacy/old libc++ libraries, see - // https://github.com/simdjson/simdjson/issues/1286 - // - // Therefore, fall back to this solution (the extra parens are there - // to handle that max may be a macro on windows). - return !(*outDouble > (std::numeric_limits::max)() || *outDouble < std::numeric_limits::lowest()); - } - - static bool parse_float_fallback(const uint8_t* ptr, const uint8_t* end_ptr, double* outDouble) { - *outDouble = simdjson::internal::from_chars(reinterpret_cast(ptr), reinterpret_cast(end_ptr)); - // We do not accept infinite values. - - // Detecting finite values in a portable manner is ridiculously hard, ideally - // we would want to do: - // return !std::isfinite(*outDouble); - // but that mysteriously fails under legacy/old libc++ libraries, see - // https://github.com/simdjson/simdjson/issues/1286 - // - // Therefore, fall back to this solution (the extra parens are there - // to handle that max may be a macro on windows). - return !(*outDouble > (std::numeric_limits::max)() || *outDouble < std::numeric_limits::lowest()); - } - - // check quickly whether the next 8 chars are made of digits - // at a glance, it looks better than Mula's - // http://0x80.pl/articles/swar-digits-validate.html - simdjson_inline bool is_made_of_eight_digits_fast(const uint8_t* chars) { - uint64_t val; - // this can read up to 7 bytes beyond the buffer size, but we require - // SIMDJSON_PADDING of padding - static_assert(7 <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be bigger than 7"); - std::memcpy(&val, chars, 8); - // a branchy method might be faster: - // return (( val & 0xF0F0F0F0F0F0F0F0 ) == 0x3030303030303030) - // && (( (val + 0x0606060606060606) & 0xF0F0F0F0F0F0F0F0 ) == - // 0x3030303030303030); - return (((val & 0xF0F0F0F0F0F0F0F0) | - (((val + 0x0606060606060606) & 0xF0F0F0F0F0F0F0F0) >> 4)) == - 0x3333333333333333); - } - - template - SIMDJSON_NO_SANITIZE_UNDEFINED // We deliberately allow overflow here and check later - simdjson_inline bool parse_digit(const uint8_t c, I& i) { - const uint8_t digit = static_cast(c - '0'); - if (digit > 9) { - return false; - } - // PERF NOTE: multiplication by 10 is cheaper than arbitrary integer multiplication - i = 10 * i + digit; // might overflow, we will handle the overflow later - return true; - } - - simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t* const src, const uint8_t*& p, uint64_t& i, int64_t& exponent) { - // we continue with the fiction that we have an integer. If the - // floating point number is representable as x * 10^z for some integer - // z that fits in 53 bits, then we will be able to convert back the - // the integer into a float in a lossless manner. - const uint8_t* const first_after_period = p; - -#ifdef SIMDJSON_SWAR_NUMBER_PARSING -#if SIMDJSON_SWAR_NUMBER_PARSING - // this helps if we have lots of decimals! - // this turns out to be frequent enough. - if (is_made_of_eight_digits_fast(p)) { - i = i * 100000000 + parse_eight_digits_unrolled(p); - p += 8; - } -#endif // SIMDJSON_SWAR_NUMBER_PARSING -#endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING - // Unrolling the first digit makes a small difference on some implementations (e.g. westmere) - if (parse_digit(*p, i)) { ++p; } - while (parse_digit(*p, i)) { p++; } - exponent = first_after_period - p; - // Decimal without digits (123.) is illegal - if (exponent == 0) { - return INVALID_NUMBER(src); - } - return SUCCESS; - } - - simdjson_inline error_code parse_exponent(simdjson_unused const uint8_t* const src, const uint8_t*& p, int64_t& exponent) { - // Exp Sign: -123.456e[-]78 - bool neg_exp = ('-' == *p); - if (neg_exp || '+' == *p) { p++; } // Skip + as well - - // Exponent: -123.456e-[78] - auto start_exp = p; - int64_t exp_number = 0; - while (parse_digit(*p, exp_number)) { ++p; } - // It is possible for parse_digit to overflow. - // In particular, it could overflow to INT64_MIN, and we cannot do - INT64_MIN. - // Thus we *must* check for possible overflow before we negate exp_number. - - // Performance notes: it may seem like combining the two "simdjson_unlikely checks" below into - // a single simdjson_unlikely path would be faster. The reasoning is sound, but the compiler may - // not oblige and may, in fact, generate two distinct paths in any case. It might be - // possible to do uint64_t(p - start_exp - 1) >= 18 but it could end up trading off - // instructions for a simdjson_likely branch, an unconclusive gain. - - // If there were no digits, it's an error. - if (simdjson_unlikely(p == start_exp)) { - return INVALID_NUMBER(src); - } - // We have a valid positive exponent in exp_number at this point, except that - // it may have overflowed. - - // If there were more than 18 digits, we may have overflowed the integer. We have to do - // something!!!! - if (simdjson_unlikely(p > start_exp + 18)) { - // Skip leading zeroes: 1e000000000000000000001 is technically valid and doesn't overflow - while (*start_exp == '0') { start_exp++; } - // 19 digits could overflow int64_t and is kind of absurd anyway. We don't - // support exponents smaller than -999,999,999,999,999,999 and bigger - // than 999,999,999,999,999,999. - // We can truncate. - // Note that 999999999999999999 is assuredly too large. The maximal ieee64 value before - // infinity is ~1.8e308. The smallest subnormal is ~5e-324. So, actually, we could - // truncate at 324. - // Note that there is no reason to fail per se at this point in time. - // E.g., 0e999999999999999999999 is a fine number. - if (p > start_exp + 18) { exp_number = 999999999999999999; } - } - // At this point, we know that exp_number is a sane, positive, signed integer. - // It is <= 999,999,999,999,999,999. As long as 'exponent' is in - // [-8223372036854775808, 8223372036854775808], we won't overflow. Because 'exponent' - // is bounded in magnitude by the size of the JSON input, we are fine in this universe. - // To sum it up: the next line should never overflow. - exponent += (neg_exp ? -exp_number : exp_number); - return SUCCESS; - } - - simdjson_inline size_t significant_digits(const uint8_t* start_digits, size_t digit_count) { - // It is possible that the integer had an overflow. - // We have to handle the case where we have 0.0000somenumber. - const uint8_t* start = start_digits; - while ((*start == '0') || (*start == '.')) { ++start; } - // we over-decrement by one when there is a '.' - return digit_count - size_t(start - start_digits); - } - - } // unnamed namespace - - /** @private */ - template - error_code slow_float_parsing(simdjson_unused const uint8_t* src, W writer) { - double d; - if (parse_float_fallback(src, &d)) { - writer.append_double(d); - return SUCCESS; - } - return INVALID_NUMBER(src); - } - - /** @private */ - template - simdjson_inline error_code write_float(const uint8_t* const src, bool negative, uint64_t i, const uint8_t* start_digits, size_t digit_count, int64_t exponent, W& writer) { - // If we frequently had to deal with long strings of digits, - // we could extend our code by using a 128-bit integer instead - // of a 64-bit integer. However, this is uncommon in practice. - // - // 9999999999999999999 < 2**64 so we can accommodate 19 digits. - // If we have a decimal separator, then digit_count - 1 is the number of digits, but we - // may not have a decimal separator! - if (simdjson_unlikely(digit_count > 19 && significant_digits(start_digits, digit_count) > 19)) { - // Ok, chances are good that we had an overflow! - // this is almost never going to get called!!! - // we start anew, going slowly!!! - // This will happen in the following examples: - // 10000000000000000000000000000000000000000000e+308 - // 3.1415926535897932384626433832795028841971693993751 - // - // NOTE: This makes a *copy* of the writer and passes it to slow_float_parsing. This happens - // because slow_float_parsing is a non-inlined function. If we passed our writer reference to - // it, it would force it to be stored in memory, preventing the compiler from picking it apart - // and putting into registers. i.e. if we pass it as reference, it gets slow. - // This is what forces the skip_double, as well. - error_code error = slow_float_parsing(src, writer); - writer.skip_double(); - return error; - } - // NOTE: it's weird that the simdjson_unlikely() only wraps half the if, but it seems to get slower any other - // way we've tried: https://github.com/simdjson/simdjson/pull/990#discussion_r448497331 - // To future reader: we'd love if someone found a better way, or at least could explain this result! - if (simdjson_unlikely(exponent < simdjson::internal::smallest_power) || (exponent > simdjson::internal::largest_power)) { - // - // Important: smallest_power is such that it leads to a zero value. - // Observe that 18446744073709551615e-343 == 0, i.e. (2**64 - 1) e -343 is zero - // so something x 10^-343 goes to zero, but not so with something x 10^-342. - static_assert(simdjson::internal::smallest_power <= -342, "smallest_power is not small enough"); - // - if ((exponent < simdjson::internal::smallest_power) || (i == 0)) { - // E.g. Parse "-0.0e-999" into the same value as "-0.0". See https://en.wikipedia.org/wiki/Signed_zero - WRITE_DOUBLE(negative ? -0.0 : 0.0, src, writer); - return SUCCESS; - } - else { // (exponent > largest_power) and (i != 0) - // We have, for sure, an infinite value and simdjson refuses to parse infinite values. - return INVALID_NUMBER(src); - } - } - double d; - if (!compute_float_64(exponent, i, negative, d)) { - // we are almost never going to get here. - if (!parse_float_fallback(src, &d)) { return INVALID_NUMBER(src); } - } - WRITE_DOUBLE(d, src, writer); - return SUCCESS; - } - - // for performance analysis, it is sometimes useful to skip parsing -#ifdef SIMDJSON_SKIPNUMBERPARSING - - template - simdjson_inline error_code parse_number(const uint8_t* const, W& writer) { - writer.append_s64(0); // always write zero - return SUCCESS; // always succeeds - } - - simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t* const src) noexcept { return 0; } - simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t* const src) noexcept { return 0; } - simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t* const src) noexcept { return 0; } - simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t* const src) noexcept { return 0; } - simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t* const src) noexcept { return 0; } - simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t* const src) noexcept { return 0; } - simdjson_unused simdjson_inline bool is_negative(const uint8_t* src) noexcept { return false; } - simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t* src) noexcept { return false; } - simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t* src) noexcept { return number_type::signed_integer; } -#else - -// parse the number at src -// define JSON_TEST_NUMBERS for unit testing -// -// It is assumed that the number is followed by a structural ({,},],[) character -// or a white space character. If that is not the case (e.g., when the JSON -// document is made of a single number), then it is necessary to copy the -// content and append a space before calling this function. -// -// Our objective is accurate parsing (ULP of 0) at high speed. - template - simdjson_inline error_code parse_number(const uint8_t* const src, W& writer) { - - // - // Check for minus sign - // - bool negative = (*src == '-'); - const uint8_t* p = src + uint8_t(negative); - - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t* const start_digits = p; - uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(p - start_digits); - if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { return INVALID_NUMBER(src); } - - // - // Handle floats if there is a . or e (or both) - // - int64_t exponent = 0; - bool is_float = false; - if ('.' == *p) { - is_float = true; - ++p; - SIMDJSON_TRY(parse_decimal_after_separator(src, p, i, exponent)); - digit_count = int(p - start_digits); // used later to guard against overflows - } - if (('e' == *p) || ('E' == *p)) { - is_float = true; - ++p; - SIMDJSON_TRY(parse_exponent(src, p, exponent)); - } - if (is_float) { - const bool dirty_end = jsoncharutils::is_not_structural_or_whitespace(*p); - SIMDJSON_TRY(write_float(src, negative, i, start_digits, digit_count, exponent, writer)); - if (dirty_end) { return INVALID_NUMBER(src); } - return SUCCESS; - } - - // The longest negative 64-bit number is 19 digits. - // The longest positive 64-bit number is 20 digits. - // We do it this way so we don't trigger this branch unless we must. - size_t longest_digit_count = negative ? 19 : 20; - if (digit_count > longest_digit_count) { return INVALID_NUMBER(src); } - if (digit_count == longest_digit_count) { - if (negative) { - // Anything negative above INT64_MAX+1 is invalid - if (i > uint64_t(INT64_MAX) + 1) { return INVALID_NUMBER(src); } - WRITE_INTEGER(~i + 1, src, writer); - if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return INVALID_NUMBER(src); } - return SUCCESS; - // Positive overflow check: - // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the - // biggest uint64_t. - // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX. - // If we got here, it's a 20 digit number starting with the digit "1". - // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller - // than 1,553,255,926,290,448,384. - // - That is smaller than the smallest possible 20-digit number the user could write: - // 10,000,000,000,000,000,000. - // - Therefore, if the number is positive and lower than that, it's overflow. - // - The value we are looking at is less than or equal to INT64_MAX. - // - } - else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INVALID_NUMBER(src); } - } - - // Write unsigned if it doesn't fit in a signed integer. - if (i > uint64_t(INT64_MAX)) { - WRITE_UNSIGNED(i, src, writer); - } - else { - WRITE_INTEGER(negative ? (~i + 1) : i, src, writer); - } - if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return INVALID_NUMBER(src); } - return SUCCESS; - } - - // Inlineable functions - namespace { - - // This table can be used to characterize the final character of an integer - // string. For JSON structural character and allowable white space characters, - // we return SUCCESS. For 'e', '.' and 'E', we return INCORRECT_TYPE. Otherwise - // we return NUMBER_ERROR. - // Optimization note: we could easily reduce the size of the table by half (to 128) - // at the cost of an extra branch. - // Optimization note: we want the values to use at most 8 bits (not, e.g., 32 bits): - static_assert(error_code(uint8_t(NUMBER_ERROR)) == NUMBER_ERROR, "bad NUMBER_ERROR cast"); - static_assert(error_code(uint8_t(SUCCESS)) == SUCCESS, "bad NUMBER_ERROR cast"); - static_assert(error_code(uint8_t(INCORRECT_TYPE)) == INCORRECT_TYPE, "bad NUMBER_ERROR cast"); - - const uint8_t integer_string_finisher[256] = { - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, - SUCCESS, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, - NUMBER_ERROR, INCORRECT_TYPE, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, INCORRECT_TYPE, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, SUCCESS, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, INCORRECT_TYPE, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, - SUCCESS, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR }; - - // Parse any number from 0 to 18,446,744,073,709,551,615 - simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t* const src) noexcept { - const uint8_t* p = src; - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t* const start_digits = p; - uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(p - start_digits); - // The longest positive 64-bit number is 20 digits. - // We do it this way so we don't trigger this branch unless we must. - // Optimization note: the compiler can probably merge - // ((digit_count == 0) || (digit_count > 20)) - // into a single branch since digit_count is unsigned. - if ((digit_count == 0) || (digit_count > 20)) { return INCORRECT_TYPE; } - // Here digit_count > 0. - if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } - // We can do the following... - // if (!jsoncharutils::is_structural_or_whitespace(*p)) { - // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; - // } - // as a single table lookup: - if (integer_string_finisher[*p] != SUCCESS) { return error_code(integer_string_finisher[*p]); } - - if (digit_count == 20) { - // Positive overflow check: - // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the - // biggest uint64_t. - // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX. - // If we got here, it's a 20 digit number starting with the digit "1". - // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller - // than 1,553,255,926,290,448,384. - // - That is smaller than the smallest possible 20-digit number the user could write: - // 10,000,000,000,000,000,000. - // - Therefore, if the number is positive and lower than that, it's overflow. - // - The value we are looking at is less than or equal to INT64_MAX. - // - if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INCORRECT_TYPE; } - } - - return i; - } - - - // Parse any number from 0 to 18,446,744,073,709,551,615 - // Never read at src_end or beyond - simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t* const src, const uint8_t* const src_end) noexcept { - const uint8_t* p = src; - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t* const start_digits = p; - uint64_t i = 0; - while ((p != src_end) && parse_digit(*p, i)) { p++; } - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(p - start_digits); - // The longest positive 64-bit number is 20 digits. - // We do it this way so we don't trigger this branch unless we must. - // Optimization note: the compiler can probably merge - // ((digit_count == 0) || (digit_count > 20)) - // into a single branch since digit_count is unsigned. - if ((digit_count == 0) || (digit_count > 20)) { return INCORRECT_TYPE; } - // Here digit_count > 0. - if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } - // We can do the following... - // if (!jsoncharutils::is_structural_or_whitespace(*p)) { - // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; - // } - // as a single table lookup: - if ((p != src_end) && integer_string_finisher[*p] != SUCCESS) { return error_code(integer_string_finisher[*p]); } - - if (digit_count == 20) { - // Positive overflow check: - // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the - // biggest uint64_t. - // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX. - // If we got here, it's a 20 digit number starting with the digit "1". - // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller - // than 1,553,255,926,290,448,384. - // - That is smaller than the smallest possible 20-digit number the user could write: - // 10,000,000,000,000,000,000. - // - Therefore, if the number is positive and lower than that, it's overflow. - // - The value we are looking at is less than or equal to INT64_MAX. - // - if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INCORRECT_TYPE; } - } - - return i; - } - - // Parse any number from 0 to 18,446,744,073,709,551,615 - simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t* const src) noexcept { - const uint8_t* p = src + 1; - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t* const start_digits = p; - uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(p - start_digits); - // The longest positive 64-bit number is 20 digits. - // We do it this way so we don't trigger this branch unless we must. - // Optimization note: the compiler can probably merge - // ((digit_count == 0) || (digit_count > 20)) - // into a single branch since digit_count is unsigned. - if ((digit_count == 0) || (digit_count > 20)) { return INCORRECT_TYPE; } - // Here digit_count > 0. - if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } - // We can do the following... - // if (!jsoncharutils::is_structural_or_whitespace(*p)) { - // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; - // } - // as a single table lookup: - if (*p != '"') { return NUMBER_ERROR; } - - if (digit_count == 20) { - // Positive overflow check: - // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the - // biggest uint64_t. - // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX. - // If we got here, it's a 20 digit number starting with the digit "1". - // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller - // than 1,553,255,926,290,448,384. - // - That is smaller than the smallest possible 20-digit number the user could write: - // 10,000,000,000,000,000,000. - // - Therefore, if the number is positive and lower than that, it's overflow. - // - The value we are looking at is less than or equal to INT64_MAX. - // - // Note: we use src[1] and not src[0] because src[0] is the quote character in this - // instance. - if (src[1] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INCORRECT_TYPE; } - } - - return i; - } - - // Parse any number from -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 - simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t* src) noexcept { - // - // Check for minus sign - // - bool negative = (*src == '-'); - const uint8_t* p = src + uint8_t(negative); - - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t* const start_digits = p; - uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(p - start_digits); - // We go from - // -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 - // so we can never represent numbers that have more than 19 digits. - size_t longest_digit_count = 19; - // Optimization note: the compiler can probably merge - // ((digit_count == 0) || (digit_count > longest_digit_count)) - // into a single branch since digit_count is unsigned. - if ((digit_count == 0) || (digit_count > longest_digit_count)) { return INCORRECT_TYPE; } - // Here digit_count > 0. - if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } - // We can do the following... - // if (!jsoncharutils::is_structural_or_whitespace(*p)) { - // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; - // } - // as a single table lookup: - if (integer_string_finisher[*p] != SUCCESS) { return error_code(integer_string_finisher[*p]); } - // Negative numbers have can go down to - INT64_MAX - 1 whereas positive numbers are limited to INT64_MAX. - // Performance note: This check is only needed when digit_count == longest_digit_count but it is - // so cheap that we might as well always make it. - if (i > uint64_t(INT64_MAX) + uint64_t(negative)) { return INCORRECT_TYPE; } - return negative ? (~i + 1) : i; - } - - // Parse any number from -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 - // Never read at src_end or beyond - simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t* const src, const uint8_t* const src_end) noexcept { - // - // Check for minus sign - // - if (src == src_end) { return NUMBER_ERROR; } - bool negative = (*src == '-'); - const uint8_t* p = src + uint8_t(negative); - - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t* const start_digits = p; - uint64_t i = 0; - while ((p != src_end) && parse_digit(*p, i)) { p++; } - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(p - start_digits); - // We go from - // -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 - // so we can never represent numbers that have more than 19 digits. - size_t longest_digit_count = 19; - // Optimization note: the compiler can probably merge - // ((digit_count == 0) || (digit_count > longest_digit_count)) - // into a single branch since digit_count is unsigned. - if ((digit_count == 0) || (digit_count > longest_digit_count)) { return INCORRECT_TYPE; } - // Here digit_count > 0. - if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } - // We can do the following... - // if (!jsoncharutils::is_structural_or_whitespace(*p)) { - // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; - // } - // as a single table lookup: - if ((p != src_end) && integer_string_finisher[*p] != SUCCESS) { return error_code(integer_string_finisher[*p]); } - // Negative numbers have can go down to - INT64_MAX - 1 whereas positive numbers are limited to INT64_MAX. - // Performance note: This check is only needed when digit_count == longest_digit_count but it is - // so cheap that we might as well always make it. - if (i > uint64_t(INT64_MAX) + uint64_t(negative)) { return INCORRECT_TYPE; } - return negative ? (~i + 1) : i; - } - - // Parse any number from -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 - simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t* src) noexcept { - // - // Check for minus sign - // - bool negative = (*(src + 1) == '-'); - src += uint8_t(negative) + 1; - - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t* const start_digits = src; - uint64_t i = 0; - while (parse_digit(*src, i)) { src++; } - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(src - start_digits); - // We go from - // -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 - // so we can never represent numbers that have more than 19 digits. - size_t longest_digit_count = 19; - // Optimization note: the compiler can probably merge - // ((digit_count == 0) || (digit_count > longest_digit_count)) - // into a single branch since digit_count is unsigned. - if ((digit_count == 0) || (digit_count > longest_digit_count)) { return INCORRECT_TYPE; } - // Here digit_count > 0. - if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } - // We can do the following... - // if (!jsoncharutils::is_structural_or_whitespace(*src)) { - // return (*src == '.' || *src == 'e' || *src == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; - // } - // as a single table lookup: - if (*src != '"') { return NUMBER_ERROR; } - // Negative numbers have can go down to - INT64_MAX - 1 whereas positive numbers are limited to INT64_MAX. - // Performance note: This check is only needed when digit_count == longest_digit_count but it is - // so cheap that we might as well always make it. - if (i > uint64_t(INT64_MAX) + uint64_t(negative)) { return INCORRECT_TYPE; } - return negative ? (~i + 1) : i; - } - - simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t* src) noexcept { - // - // Check for minus sign - // - bool negative = (*src == '-'); - src += uint8_t(negative); - - // - // Parse the integer part. - // - uint64_t i = 0; - const uint8_t* p = src; - p += parse_digit(*p, i); - bool leading_zero = (i == 0); - while (parse_digit(*p, i)) { p++; } - // no integer digits, or 0123 (zero must be solo) - if (p == src) { return INCORRECT_TYPE; } - if ((leading_zero && p != src + 1)) { return NUMBER_ERROR; } - - // - // Parse the decimal part. - // - int64_t exponent = 0; - bool overflow; - if (simdjson_likely(*p == '.')) { - p++; - const uint8_t* start_decimal_digits = p; - if (!parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits - p++; - while (parse_digit(*p, i)) { p++; } - exponent = -(p - start_decimal_digits); - - // Overflow check. More than 19 digits (minus the decimal) may be overflow. - overflow = p - src - 1 > 19; - if (simdjson_unlikely(overflow && leading_zero)) { - // Skip leading 0.00000 and see if it still overflows - const uint8_t* start_digits = src + 2; - while (*start_digits == '0') { start_digits++; } - overflow = p - start_digits > 19; - } - } - else { - overflow = p - src > 19; - } - - // - // Parse the exponent - // - if (*p == 'e' || *p == 'E') { - p++; - bool exp_neg = *p == '-'; - p += exp_neg || *p == '+'; - - uint64_t exp = 0; - const uint8_t* start_exp_digits = p; - while (parse_digit(*p, exp)) { p++; } - // no exp digits, or 20+ exp digits - if (p - start_exp_digits == 0 || p - start_exp_digits > 19) { return NUMBER_ERROR; } - - exponent += exp_neg ? 0 - exp : exp; - } - - if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } - - overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; - - // - // Assemble (or slow-parse) the float - // - double d; - if (simdjson_likely(!overflow)) { - if (compute_float_64(exponent, i, negative, d)) { return d; } - } - if (!parse_float_fallback(src - uint8_t(negative), &d)) { - return NUMBER_ERROR; - } - return d; - } - - simdjson_unused simdjson_inline bool is_negative(const uint8_t* src) noexcept { - return (*src == '-'); - } - - simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t* src) noexcept { - bool negative = (*src == '-'); - src += uint8_t(negative); - const uint8_t* p = src; - while (static_cast(*p - '0') <= 9) { p++; } - if (p == src) { return NUMBER_ERROR; } - if (jsoncharutils::is_structural_or_whitespace(*p)) { return true; } - return false; - } - - simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t* src) noexcept { - bool negative = (*src == '-'); - src += uint8_t(negative); - const uint8_t* p = src; - while (static_cast(*p - '0') <= 9) { p++; } - if (p == src) { return NUMBER_ERROR; } - if (jsoncharutils::is_structural_or_whitespace(*p)) { - // We have an integer. - // If the number is negative and valid, it must be a signed integer. - if (negative) { return number_type::signed_integer; } - // We want values larger or equal to 9223372036854775808 to be unsigned - // integers, and the other values to be signed integers. - int digit_count = int(p - src); - if (digit_count >= 19) { - const uint8_t* smaller_big_integer = reinterpret_cast("9223372036854775808"); - if ((digit_count >= 20) || (memcmp(src, smaller_big_integer, 19) >= 0)) { - return number_type::unsigned_integer; - } - } - return number_type::signed_integer; - } - // Hopefully, we have 'e' or 'E' or '.'. - return number_type::floating_point_number; - } - - // Never read at src_end or beyond - simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t* src, const uint8_t* const src_end) noexcept { - if (src == src_end) { return NUMBER_ERROR; } - // - // Check for minus sign - // - bool negative = (*src == '-'); - src += uint8_t(negative); - - // - // Parse the integer part. - // - uint64_t i = 0; - const uint8_t* p = src; - if (p == src_end) { return NUMBER_ERROR; } - p += parse_digit(*p, i); - bool leading_zero = (i == 0); - while ((p != src_end) && parse_digit(*p, i)) { p++; } - // no integer digits, or 0123 (zero must be solo) - if (p == src) { return INCORRECT_TYPE; } - if ((leading_zero && p != src + 1)) { return NUMBER_ERROR; } - - // - // Parse the decimal part. - // - int64_t exponent = 0; - bool overflow; - if (simdjson_likely((p != src_end) && (*p == '.'))) { - p++; - const uint8_t* start_decimal_digits = p; - if ((p == src_end) || !parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits - p++; - while ((p != src_end) && parse_digit(*p, i)) { p++; } - exponent = -(p - start_decimal_digits); - - // Overflow check. More than 19 digits (minus the decimal) may be overflow. - overflow = p - src - 1 > 19; - if (simdjson_unlikely(overflow && leading_zero)) { - // Skip leading 0.00000 and see if it still overflows - const uint8_t* start_digits = src + 2; - while (*start_digits == '0') { start_digits++; } - overflow = start_digits - src > 19; - } - } - else { - overflow = p - src > 19; - } - - // - // Parse the exponent - // - if ((p != src_end) && (*p == 'e' || *p == 'E')) { - p++; - if (p == src_end) { return NUMBER_ERROR; } - bool exp_neg = *p == '-'; - p += exp_neg || *p == '+'; - - uint64_t exp = 0; - const uint8_t* start_exp_digits = p; - while ((p != src_end) && parse_digit(*p, exp)) { p++; } - // no exp digits, or 20+ exp digits - if (p - start_exp_digits == 0 || p - start_exp_digits > 19) { return NUMBER_ERROR; } - - exponent += exp_neg ? 0 - exp : exp; - } - - if ((p != src_end) && jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } - - overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; - - // - // Assemble (or slow-parse) the float - // - double d; - if (simdjson_likely(!overflow)) { - if (compute_float_64(exponent, i, negative, d)) { return d; } - } - if (!parse_float_fallback(src - uint8_t(negative), src_end, &d)) { - return NUMBER_ERROR; - } - return d; - } - - simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t* src) noexcept { - // - // Check for minus sign - // - bool negative = (*(src + 1) == '-'); - src += uint8_t(negative) + 1; - - // - // Parse the integer part. - // - uint64_t i = 0; - const uint8_t* p = src; - p += parse_digit(*p, i); - bool leading_zero = (i == 0); - while (parse_digit(*p, i)) { p++; } - // no integer digits, or 0123 (zero must be solo) - if (p == src) { return INCORRECT_TYPE; } - if ((leading_zero && p != src + 1)) { return NUMBER_ERROR; } - - // - // Parse the decimal part. - // - int64_t exponent = 0; - bool overflow; - if (simdjson_likely(*p == '.')) { - p++; - const uint8_t* start_decimal_digits = p; - if (!parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits - p++; - while (parse_digit(*p, i)) { p++; } - exponent = -(p - start_decimal_digits); - - // Overflow check. More than 19 digits (minus the decimal) may be overflow. - overflow = p - src - 1 > 19; - if (simdjson_unlikely(overflow && leading_zero)) { - // Skip leading 0.00000 and see if it still overflows - const uint8_t* start_digits = src + 2; - while (*start_digits == '0') { start_digits++; } - overflow = p - start_digits > 19; - } - } - else { - overflow = p - src > 19; - } - - // - // Parse the exponent - // - if (*p == 'e' || *p == 'E') { - p++; - bool exp_neg = *p == '-'; - p += exp_neg || *p == '+'; - - uint64_t exp = 0; - const uint8_t* start_exp_digits = p; - while (parse_digit(*p, exp)) { p++; } - // no exp digits, or 20+ exp digits - if (p - start_exp_digits == 0 || p - start_exp_digits > 19) { return NUMBER_ERROR; } - - exponent += exp_neg ? 0 - exp : exp; - } - - if (*p != '"') { return NUMBER_ERROR; } - - overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; - - // - // Assemble (or slow-parse) the float - // - double d; - if (simdjson_likely(!overflow)) { - if (compute_float_64(exponent, i, negative, d)) { return d; } - } - if (!parse_float_fallback(src - uint8_t(negative), &d)) { - return NUMBER_ERROR; - } - return d; - } - - } // unnamed namespace -#endif // SIMDJSON_SKIPNUMBERPARSING - - } // namespace numberparsing - - inline std::ostream& operator<<(std::ostream& out, number_type type) noexcept { - switch (type) { - case number_type::signed_integer: out << "integer in [-9223372036854775808,9223372036854775808)"; break; - case number_type::unsigned_integer: out << "unsigned integer in [9223372036854775808,18446744073709551616)"; break; - case number_type::floating_point_number: out << "floating-point number (binary64)"; break; - default: SIMDJSON_UNREACHABLE(); - } - return out; - } - - } // namespace haswell -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_NUMBERPARSING_H -/* end file simdjson/generic/numberparsing.h for haswell */ - -/* including simdjson/generic/implementation_simdjson_result_base-inl.h for haswell: #include "simdjson/generic/implementation_simdjson_result_base-inl.h" */ -/* begin file simdjson/generic/implementation_simdjson_result_base-inl.h for haswell */ -#ifndef SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_INL_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_INL_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/implementation_simdjson_result_base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - namespace haswell { - - // - // internal::implementation_simdjson_result_base inline implementation - // - - template - simdjson_inline void implementation_simdjson_result_base::tie(T& value, error_code& error) && noexcept { - error = this->second; - if (!error) { - value = std::forward>(*this).first; - } - } - - template - simdjson_warn_unused simdjson_inline error_code implementation_simdjson_result_base::get(T& value) && noexcept { - error_code error; - std::forward>(*this).tie(value, error); - return error; - } - - template - simdjson_inline error_code implementation_simdjson_result_base::error() const noexcept { - return this->second; - } - -#if SIMDJSON_EXCEPTIONS - - template - simdjson_inline T& implementation_simdjson_result_base::value() & noexcept(false) { - if (error()) { throw simdjson_error(error()); } - return this->first; - } - - template - simdjson_inline T&& implementation_simdjson_result_base::value() && noexcept(false) { - return std::forward>(*this).take_value(); - } - - template - simdjson_inline T&& implementation_simdjson_result_base::take_value() && noexcept(false) { - if (error()) { throw simdjson_error(error()); } - return std::forward(this->first); - } - - template - simdjson_inline implementation_simdjson_result_base::operator T && () && noexcept(false) { - return std::forward>(*this).take_value(); - } - -#endif // SIMDJSON_EXCEPTIONS - - template - simdjson_inline const T& implementation_simdjson_result_base::value_unsafe() const& noexcept { - return this->first; - } - - template - simdjson_inline T& implementation_simdjson_result_base::value_unsafe() & noexcept { - return this->first; - } - - template - simdjson_inline T&& implementation_simdjson_result_base::value_unsafe() && noexcept { - return std::forward(this->first); - } - - template - simdjson_inline implementation_simdjson_result_base::implementation_simdjson_result_base(T&& value, error_code error) noexcept - : first{ std::forward(value) }, second{ error } {} - template - simdjson_inline implementation_simdjson_result_base::implementation_simdjson_result_base(error_code error) noexcept - : implementation_simdjson_result_base(T{}, error) {} - template - simdjson_inline implementation_simdjson_result_base::implementation_simdjson_result_base(T&& value) noexcept - : implementation_simdjson_result_base(std::forward(value), SUCCESS) {} - - } // namespace haswell -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_INL_H -/* end file simdjson/generic/implementation_simdjson_result_base-inl.h for haswell */ -/* end file simdjson/generic/amalgamated.h for haswell */ -/* including simdjson/haswell/end.h: #include "simdjson/haswell/end.h" */ -/* begin file simdjson/haswell/end.h */ -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/haswell/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#if !SIMDJSON_CAN_ALWAYS_RUN_HASWELL -SIMDJSON_UNTARGET_REGION -#endif - -/* undefining SIMDJSON_IMPLEMENTATION from "haswell" */ -#undef SIMDJSON_IMPLEMENTATION -/* end file simdjson/haswell/end.h */ - -#endif // SIMDJSON_HASWELL_H -/* end file simdjson/haswell.h */ -/* including simdjson/haswell/implementation.h: #include */ -/* begin file simdjson/haswell/implementation.h */ -#ifndef SIMDJSON_HASWELL_IMPLEMENTATION_H -#define SIMDJSON_HASWELL_IMPLEMENTATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/haswell/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/implementation.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/instruction_set.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -// The constructor may be executed on any host, so we take care not to use SIMDJSON_TARGET_HASWELL -namespace simdjson { - namespace haswell { - - /** - * @private - */ - class implementation final : public simdjson::implementation { - public: - simdjson_inline implementation() : simdjson::implementation( - "haswell", - "Intel/AMD AVX2", - internal::instruction_set::AVX2 | internal::instruction_set::PCLMULQDQ | internal::instruction_set::BMI1 | internal::instruction_set::BMI2 - ) {} - simdjson_warn_unused error_code create_dom_parser_implementation( - size_t capacity, - size_t max_length, - std::unique_ptr& dst - ) const noexcept final; - simdjson_warn_unused error_code minify(const uint8_t* buf, size_t len, uint8_t* dst, size_t& dst_len) const noexcept final; - simdjson_warn_unused bool validate_utf8(const char* buf, size_t len) const noexcept final; - }; - - } // namespace haswell -} // namespace simdjson - -#endif // SIMDJSON_HASWELL_IMPLEMENTATION_H -/* end file simdjson/haswell/implementation.h */ - -/* including simdjson/haswell/begin.h: #include */ -/* begin file simdjson/haswell/begin.h */ -/* defining SIMDJSON_IMPLEMENTATION to "haswell" */ -#define SIMDJSON_IMPLEMENTATION haswell - -/* including simdjson/haswell/base.h: #include "simdjson/haswell/base.h" */ -/* begin file simdjson/haswell/base.h */ -#ifndef SIMDJSON_HASWELL_BASE_H -#define SIMDJSON_HASWELL_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -// The constructor may be executed on any host, so we take care not to use SIMDJSON_TARGET_HASWELL -namespace simdjson { - /** - * Implementation for Haswell (Intel AVX2). - */ - namespace haswell { - - class implementation; - - namespace { - namespace simd { - template struct simd8; - template struct simd8x64; - } // namespace simd - } // unnamed namespace - - } // namespace haswell -} // namespace simdjson - -#endif // SIMDJSON_HASWELL_BASE_H -/* end file simdjson/haswell/base.h */ -/* including simdjson/haswell/intrinsics.h: #include "simdjson/haswell/intrinsics.h" */ -/* begin file simdjson/haswell/intrinsics.h */ -#ifndef SIMDJSON_HASWELL_INTRINSICS_H -#define SIMDJSON_HASWELL_INTRINSICS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/haswell/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#if SIMDJSON_VISUAL_STUDIO -// under clang within visual studio, this will include -#include // visual studio or clang -#else -#include // elsewhere -#endif // SIMDJSON_VISUAL_STUDIO - -#if SIMDJSON_CLANG_VISUAL_STUDIO -/** - * You are not supposed, normally, to include these - * headers directly. Instead you should either include intrin.h - * or x86intrin.h. However, when compiling with clang - * under Windows (i.e., when _MSC_VER is set), these headers - * only get included *if* the corresponding features are detected - * from macros: - * e.g., if __AVX2__ is set... in turn, we normally set these - * macros by compiling against the corresponding architecture - * (e.g., arch:AVX2, -mavx2, etc.) which compiles the whole - * software with these advanced instructions. In simdjson, we - * want to compile the whole program for a generic target, - * and only target our specific kernels. As a workaround, - * we directly include the needed headers. These headers would - * normally guard against such usage, but we carefully included - * (or ) before, so the headers - * are fooled. - */ -#include // for _blsr_u64 -#include // for __lzcnt64 -#include // for most things (AVX2, AVX512, _popcnt64) -#include -#include -#include -#include -#include // for _mm_clmulepi64_si128 - // unfortunately, we may not get _blsr_u64, but, thankfully, clang - // has it as a macro. -#ifndef _blsr_u64 -// we roll our own -#define _blsr_u64(n) ((n - 1) & n) -#endif // _blsr_u64 -#endif // SIMDJSON_CLANG_VISUAL_STUDIO - -static_assert(sizeof(__m256i) <= simdjson::SIMDJSON_PADDING, "insufficient padding for haswell kernel."); - -#endif // SIMDJSON_HASWELL_INTRINSICS_H -/* end file simdjson/haswell/intrinsics.h */ - -#if !SIMDJSON_CAN_ALWAYS_RUN_HASWELL -SIMDJSON_TARGET_REGION("avx2,bmi,pclmul,lzcnt,popcnt") -#endif - -/* including simdjson/haswell/bitmanipulation.h: #include "simdjson/haswell/bitmanipulation.h" */ -/* begin file simdjson/haswell/bitmanipulation.h */ -#ifndef SIMDJSON_HASWELL_BITMANIPULATION_H -#define SIMDJSON_HASWELL_BITMANIPULATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/haswell/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/haswell/intrinsics.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/haswell/bitmask.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - namespace haswell { - namespace { - - // We sometimes call trailing_zero on inputs that are zero, - // but the algorithms do not end up using the returned value. - // Sadly, sanitizers are not smart enough to figure it out. - SIMDJSON_NO_SANITIZE_UNDEFINED - // This function can be used safely even if not all bytes have been - // initialized. - // See issue https://github.com/simdjson/simdjson/issues/1965 - SIMDJSON_NO_SANITIZE_MEMORY - simdjson_inline int trailing_zeroes(uint64_t input_num) { -#if SIMDJSON_REGULAR_VISUAL_STUDIO - return (int)_tzcnt_u64(input_num); -#else // SIMDJSON_REGULAR_VISUAL_STUDIO - //////// - // You might expect the next line to be equivalent to - // return (int)_tzcnt_u64(input_num); - // but the generated code differs and might be less efficient? - //////// - return __builtin_ctzll(input_num); -#endif // SIMDJSON_REGULAR_VISUAL_STUDIO - } - - /* result might be undefined when input_num is zero */ - simdjson_inline uint64_t clear_lowest_bit(uint64_t input_num) { - return _blsr_u64(input_num); - } - - /* result might be undefined when input_num is zero */ - simdjson_inline int leading_zeroes(uint64_t input_num) { - return int(_lzcnt_u64(input_num)); - } - -#if SIMDJSON_REGULAR_VISUAL_STUDIO - simdjson_inline unsigned __int64 count_ones(uint64_t input_num) { - // note: we do not support legacy 32-bit Windows in this kernel - return __popcnt64(input_num);// Visual Studio wants two underscores - } -#else - simdjson_inline long long int count_ones(uint64_t input_num) { - return _popcnt64(input_num); - } -#endif - - simdjson_inline bool add_overflow(uint64_t value1, uint64_t value2, - uint64_t* result) { -#if SIMDJSON_REGULAR_VISUAL_STUDIO - return _addcarry_u64(0, value1, value2, - reinterpret_cast(result)); -#else - return __builtin_uaddll_overflow(value1, value2, - reinterpret_cast(result)); -#endif - } - - } // unnamed namespace - } // namespace haswell -} // namespace simdjson - -#endif // SIMDJSON_HASWELL_BITMANIPULATION_H -/* end file simdjson/haswell/bitmanipulation.h */ -/* including simdjson/haswell/bitmask.h: #include "simdjson/haswell/bitmask.h" */ -/* begin file simdjson/haswell/bitmask.h */ -#ifndef SIMDJSON_HASWELL_BITMASK_H -#define SIMDJSON_HASWELL_BITMASK_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/haswell/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/haswell/intrinsics.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - namespace haswell { - namespace { - - // - // Perform a "cumulative bitwise xor," flipping bits each time a 1 is encountered. - // - // For example, prefix_xor(00100100) == 00011100 - // - simdjson_inline uint64_t prefix_xor(const uint64_t bitmask) { - // There should be no such thing with a processor supporting avx2 - // but not clmul. - __m128i all_ones = _mm_set1_epi8('\xFF'); - __m128i result = _mm_clmulepi64_si128(_mm_set_epi64x(0ULL, bitmask), all_ones, 0); - return _mm_cvtsi128_si64(result); - } - - } // unnamed namespace - } // namespace haswell -} // namespace simdjson - -#endif // SIMDJSON_HASWELL_BITMASK_H -/* end file simdjson/haswell/bitmask.h */ -/* including simdjson/haswell/numberparsing_defs.h: #include "simdjson/haswell/numberparsing_defs.h" */ -/* begin file simdjson/haswell/numberparsing_defs.h */ -#ifndef SIMDJSON_HASWELL_NUMBERPARSING_DEFS_H -#define SIMDJSON_HASWELL_NUMBERPARSING_DEFS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/haswell/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/haswell/intrinsics.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - namespace haswell { - namespace numberparsing { - - /** @private */ - static simdjson_inline uint32_t parse_eight_digits_unrolled(const uint8_t* chars) { - // this actually computes *16* values so we are being wasteful. - const __m128i ascii0 = _mm_set1_epi8('0'); - const __m128i mul_1_10 = - _mm_setr_epi8(10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1); - const __m128i mul_1_100 = _mm_setr_epi16(100, 1, 100, 1, 100, 1, 100, 1); - const __m128i mul_1_10000 = - _mm_setr_epi16(10000, 1, 10000, 1, 10000, 1, 10000, 1); - const __m128i input = _mm_sub_epi8( - _mm_loadu_si128(reinterpret_cast(chars)), ascii0); - const __m128i t1 = _mm_maddubs_epi16(input, mul_1_10); - const __m128i t2 = _mm_madd_epi16(t1, mul_1_100); - const __m128i t3 = _mm_packus_epi32(t2, t2); - const __m128i t4 = _mm_madd_epi16(t3, mul_1_10000); - return _mm_cvtsi128_si32( - t4); // only captures the sum of the first 8 digits, drop the rest - } - - /** @private */ - simdjson_inline internal::value128 full_multiplication(uint64_t value1, uint64_t value2) { - internal::value128 answer; -#if SIMDJSON_REGULAR_VISUAL_STUDIO || SIMDJSON_IS_32BITS -#ifdef _M_ARM64 - // ARM64 has native support for 64-bit multiplications, no need to emultate - answer.high = __umulh(value1, value2); - answer.low = value1 * value2; -#else - answer.low = _umul128(value1, value2, &answer.high); // _umul128 not available on ARM64 -#endif // _M_ARM64 -#else // SIMDJSON_REGULAR_VISUAL_STUDIO || SIMDJSON_IS_32BITS - __uint128_t r = (static_cast<__uint128_t>(value1)) * value2; - answer.low = uint64_t(r); - answer.high = uint64_t(r >> 64); -#endif - return answer; - } - - } // namespace numberparsing - } // namespace haswell -} // namespace simdjson - -#define SIMDJSON_SWAR_NUMBER_PARSING 1 - -#endif // SIMDJSON_HASWELL_NUMBERPARSING_DEFS_H -/* end file simdjson/haswell/numberparsing_defs.h */ -/* including simdjson/haswell/simd.h: #include "simdjson/haswell/simd.h" */ -/* begin file simdjson/haswell/simd.h */ -#ifndef SIMDJSON_HASWELL_SIMD_H -#define SIMDJSON_HASWELL_SIMD_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/haswell/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/haswell/intrinsics.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/haswell/bitmanipulation.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/simdprune_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - namespace haswell { - namespace { - namespace simd { - - // Forward-declared so they can be used by splat and friends. - template - struct base { - __m256i value; - - // Zero constructor - simdjson_inline base() : value{ __m256i() } {} - - // Conversion from SIMD register - simdjson_inline base(const __m256i _value) : value(_value) {} - - // Conversion to SIMD register - simdjson_inline operator const __m256i& () const { return this->value; } - simdjson_inline operator __m256i& () { return this->value; } - - // Bit operations - simdjson_inline Child operator|(const Child other) const { return _mm256_or_si256(*this, other); } - simdjson_inline Child operator&(const Child other) const { return _mm256_and_si256(*this, other); } - simdjson_inline Child operator^(const Child other) const { return _mm256_xor_si256(*this, other); } - simdjson_inline Child bit_andnot(const Child other) const { return _mm256_andnot_si256(other, *this); } - simdjson_inline Child& operator|=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast | other; return *this_cast; } - simdjson_inline Child& operator&=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast & other; return *this_cast; } - simdjson_inline Child& operator^=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast ^ other; return *this_cast; } - }; - - // Forward-declared so they can be used by splat and friends. - template - struct simd8; - - template> - struct base8 : base> { - typedef uint32_t bitmask_t; - typedef uint64_t bitmask2_t; - - simdjson_inline base8() : base>() {} - simdjson_inline base8(const __m256i _value) : base>(_value) {} - - friend simdjson_really_inline Mask operator==(const simd8 lhs, const simd8 rhs) { return _mm256_cmpeq_epi8(lhs, rhs); } - - static const int SIZE = sizeof(base::value); - - template - simdjson_inline simd8 prev(const simd8 prev_chunk) const { - return _mm256_alignr_epi8(*this, _mm256_permute2x128_si256(prev_chunk, *this, 0x21), 16 - N); - } - }; - - // SIMD byte mask type (returned by things like eq and gt) - template<> - struct simd8 : base8 { - static simdjson_inline simd8 splat(bool _value) { return _mm256_set1_epi8(uint8_t(-(!!_value))); } - - simdjson_inline simd8() : base8() {} - simdjson_inline simd8(const __m256i _value) : base8(_value) {} - // Splat constructor - simdjson_inline simd8(bool _value) : base8(splat(_value)) {} - - simdjson_inline int to_bitmask() const { return _mm256_movemask_epi8(*this); } - simdjson_inline bool any() const { return !_mm256_testz_si256(*this, *this); } - simdjson_inline simd8 operator~() const { return *this ^ true; } - }; - - template - struct base8_numeric : base8 { - static simdjson_inline simd8 splat(T _value) { return _mm256_set1_epi8(_value); } - static simdjson_inline simd8 zero() { return _mm256_setzero_si256(); } - static simdjson_inline simd8 load(const T values[32]) { - return _mm256_loadu_si256(reinterpret_cast(values)); - } - // Repeat 16 values as many times as necessary (usually for lookup tables) - static simdjson_inline simd8 repeat_16( - T v0, T v1, T v2, T v3, T v4, T v5, T v6, T v7, - T v8, T v9, T v10, T v11, T v12, T v13, T v14, T v15 - ) { - return simd8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10, v11, v12, v13, v14, v15, - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10, v11, v12, v13, v14, v15 - ); - } - - simdjson_inline base8_numeric() : base8() {} - simdjson_inline base8_numeric(const __m256i _value) : base8(_value) {} - - // Store to array - simdjson_inline void store(T dst[32]) const { return _mm256_storeu_si256(reinterpret_cast<__m256i*>(dst), *this); } - - // Addition/subtraction are the same for signed and unsigned - simdjson_inline simd8 operator+(const simd8 other) const { return _mm256_add_epi8(*this, other); } - simdjson_inline simd8 operator-(const simd8 other) const { return _mm256_sub_epi8(*this, other); } - simdjson_inline simd8& operator+=(const simd8 other) { *this = *this + other; return *static_cast*>(this); } - simdjson_inline simd8& operator-=(const simd8 other) { *this = *this - other; return *static_cast*>(this); } - - // Override to distinguish from bool version - simdjson_inline simd8 operator~() const { return *this ^ 0xFFu; } - - // Perform a lookup assuming the value is between 0 and 16 (undefined behavior for out of range values) - template - simdjson_inline simd8 lookup_16(simd8 lookup_table) const { - return _mm256_shuffle_epi8(lookup_table, *this); - } - - // Copies to 'output" all bytes corresponding to a 0 in the mask (interpreted as a bitset). - // Passing a 0 value for mask would be equivalent to writing out every byte to output. - // Only the first 32 - count_ones(mask) bytes of the result are significant but 32 bytes - // get written. - // Design consideration: it seems like a function with the - // signature simd8 compress(uint32_t mask) would be - // sensible, but the AVX ISA makes this kind of approach difficult. - template - simdjson_inline void compress(uint32_t mask, L* output) const { - using internal::thintable_epi8; - using internal::BitsSetTable256mul2; - using internal::pshufb_combine_table; - // this particular implementation was inspired by work done by @animetosho - // we do it in four steps, first 8 bytes and then second 8 bytes... - uint8_t mask1 = uint8_t(mask); // least significant 8 bits - uint8_t mask2 = uint8_t(mask >> 8); // second least significant 8 bits - uint8_t mask3 = uint8_t(mask >> 16); // ... - uint8_t mask4 = uint8_t(mask >> 24); // ... - // next line just loads the 64-bit values thintable_epi8[mask1] and - // thintable_epi8[mask2] into a 128-bit register, using only - // two instructions on most compilers. - __m256i shufmask = _mm256_set_epi64x(thintable_epi8[mask4], thintable_epi8[mask3], - thintable_epi8[mask2], thintable_epi8[mask1]); - // we increment by 0x08 the second half of the mask and so forth - shufmask = - _mm256_add_epi8(shufmask, _mm256_set_epi32(0x18181818, 0x18181818, - 0x10101010, 0x10101010, 0x08080808, 0x08080808, 0, 0)); - // this is the version "nearly pruned" - __m256i pruned = _mm256_shuffle_epi8(*this, shufmask); - // we still need to put the pieces back together. - // we compute the popcount of the first words: - int pop1 = BitsSetTable256mul2[mask1]; - int pop3 = BitsSetTable256mul2[mask3]; - - // then load the corresponding mask - // could be done with _mm256_loadu2_m128i but many standard libraries omit this intrinsic. - __m256i v256 = _mm256_castsi128_si256( - _mm_loadu_si128(reinterpret_cast(pshufb_combine_table + pop1 * 8))); - __m256i compactmask = _mm256_insertf128_si256(v256, - _mm_loadu_si128(reinterpret_cast(pshufb_combine_table + pop3 * 8)), 1); - __m256i almostthere = _mm256_shuffle_epi8(pruned, compactmask); - // We just need to write out the result. - // This is the tricky bit that is hard to do - // if we want to return a SIMD register, since there - // is no single-instruction approach to recombine - // the two 128-bit lanes with an offset. - __m128i v128; - v128 = _mm256_castsi256_si128(almostthere); - _mm_storeu_si128(reinterpret_cast<__m128i*>(output), v128); - v128 = _mm256_extractf128_si256(almostthere, 1); - _mm_storeu_si128(reinterpret_cast<__m128i*>(output + 16 - count_ones(mask & 0xFFFF)), v128); - } - - template - simdjson_inline simd8 lookup_16( - L replace0, L replace1, L replace2, L replace3, - L replace4, L replace5, L replace6, L replace7, - L replace8, L replace9, L replace10, L replace11, - L replace12, L replace13, L replace14, L replace15) const { - return lookup_16(simd8::repeat_16( - replace0, replace1, replace2, replace3, - replace4, replace5, replace6, replace7, - replace8, replace9, replace10, replace11, - replace12, replace13, replace14, replace15 - )); - } - }; - - // Signed bytes - template<> - struct simd8 : base8_numeric { - simdjson_inline simd8() : base8_numeric() {} - simdjson_inline simd8(const __m256i _value) : base8_numeric(_value) {} - // Splat constructor - simdjson_inline simd8(int8_t _value) : simd8(splat(_value)) {} - // Array constructor - simdjson_inline simd8(const int8_t values[32]) : simd8(load(values)) {} - // Member-by-member initialization - simdjson_inline simd8( - int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, - int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15, - int8_t v16, int8_t v17, int8_t v18, int8_t v19, int8_t v20, int8_t v21, int8_t v22, int8_t v23, - int8_t v24, int8_t v25, int8_t v26, int8_t v27, int8_t v28, int8_t v29, int8_t v30, int8_t v31 - ) : simd8(_mm256_setr_epi8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10, v11, v12, v13, v14, v15, - v16, v17, v18, v19, v20, v21, v22, v23, - v24, v25, v26, v27, v28, v29, v30, v31 - )) {} - // Repeat 16 values as many times as necessary (usually for lookup tables) - simdjson_inline static simd8 repeat_16( - int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, - int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 - ) { - return simd8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10, v11, v12, v13, v14, v15, - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10, v11, v12, v13, v14, v15 - ); - } - - // Order-sensitive comparisons - simdjson_inline simd8 max_val(const simd8 other) const { return _mm256_max_epi8(*this, other); } - simdjson_inline simd8 min_val(const simd8 other) const { return _mm256_min_epi8(*this, other); } - simdjson_inline simd8 operator>(const simd8 other) const { return _mm256_cmpgt_epi8(*this, other); } - simdjson_inline simd8 operator<(const simd8 other) const { return _mm256_cmpgt_epi8(other, *this); } - }; - - // Unsigned bytes - template<> - struct simd8 : base8_numeric { - simdjson_inline simd8() : base8_numeric() {} - simdjson_inline simd8(const __m256i _value) : base8_numeric(_value) {} - // Splat constructor - simdjson_inline simd8(uint8_t _value) : simd8(splat(_value)) {} - // Array constructor - simdjson_inline simd8(const uint8_t values[32]) : simd8(load(values)) {} - // Member-by-member initialization - simdjson_inline simd8( - uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, - uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15, - uint8_t v16, uint8_t v17, uint8_t v18, uint8_t v19, uint8_t v20, uint8_t v21, uint8_t v22, uint8_t v23, - uint8_t v24, uint8_t v25, uint8_t v26, uint8_t v27, uint8_t v28, uint8_t v29, uint8_t v30, uint8_t v31 - ) : simd8(_mm256_setr_epi8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10, v11, v12, v13, v14, v15, - v16, v17, v18, v19, v20, v21, v22, v23, - v24, v25, v26, v27, v28, v29, v30, v31 - )) {} - // Repeat 16 values as many times as necessary (usually for lookup tables) - simdjson_inline static simd8 repeat_16( - uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, - uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 - ) { - return simd8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10, v11, v12, v13, v14, v15, - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10, v11, v12, v13, v14, v15 - ); - } - - // Saturated math - simdjson_inline simd8 saturating_add(const simd8 other) const { return _mm256_adds_epu8(*this, other); } - simdjson_inline simd8 saturating_sub(const simd8 other) const { return _mm256_subs_epu8(*this, other); } - - // Order-specific operations - simdjson_inline simd8 max_val(const simd8 other) const { return _mm256_max_epu8(*this, other); } - simdjson_inline simd8 min_val(const simd8 other) const { return _mm256_min_epu8(other, *this); } - // Same as >, but only guarantees true is nonzero (< guarantees true = -1) - simdjson_inline simd8 gt_bits(const simd8 other) const { return this->saturating_sub(other); } - // Same as <, but only guarantees true is nonzero (< guarantees true = -1) - simdjson_inline simd8 lt_bits(const simd8 other) const { return other.saturating_sub(*this); } - simdjson_inline simd8 operator<=(const simd8 other) const { return other.max_val(*this) == other; } - simdjson_inline simd8 operator>=(const simd8 other) const { return other.min_val(*this) == other; } - simdjson_inline simd8 operator>(const simd8 other) const { return this->gt_bits(other).any_bits_set(); } - simdjson_inline simd8 operator<(const simd8 other) const { return this->lt_bits(other).any_bits_set(); } - - // Bit-specific operations - simdjson_inline simd8 bits_not_set() const { return *this == uint8_t(0); } - simdjson_inline simd8 bits_not_set(simd8 bits) const { return (*this & bits).bits_not_set(); } - simdjson_inline simd8 any_bits_set() const { return ~this->bits_not_set(); } - simdjson_inline simd8 any_bits_set(simd8 bits) const { return ~this->bits_not_set(bits); } - simdjson_inline bool is_ascii() const { return _mm256_movemask_epi8(*this) == 0; } - simdjson_inline bool bits_not_set_anywhere() const { return _mm256_testz_si256(*this, *this); } - simdjson_inline bool any_bits_set_anywhere() const { return !bits_not_set_anywhere(); } - simdjson_inline bool bits_not_set_anywhere(simd8 bits) const { return _mm256_testz_si256(*this, bits); } - simdjson_inline bool any_bits_set_anywhere(simd8 bits) const { return !bits_not_set_anywhere(bits); } - template - simdjson_inline simd8 shr() const { return simd8(_mm256_srli_epi16(*this, N)) & uint8_t(0xFFu >> N); } - template - simdjson_inline simd8 shl() const { return simd8(_mm256_slli_epi16(*this, N)) & uint8_t(0xFFu << N); } - // Get one of the bits and make a bitmask out of it. - // e.g. value.get_bit<7>() gets the high bit - template - simdjson_inline int get_bit() const { return _mm256_movemask_epi8(_mm256_slli_epi16(*this, 7 - N)); } - }; - - template - struct simd8x64 { - static constexpr int NUM_CHUNKS = 64 / sizeof(simd8); - static_assert(NUM_CHUNKS == 2, "Haswell kernel should use two registers per 64-byte block."); - const simd8 chunks[NUM_CHUNKS]; - - simd8x64(const simd8x64& o) = delete; // no copy allowed - simd8x64& operator=(const simd8& other) = delete; // no assignment allowed - simd8x64() = delete; // no default constructor allowed - - simdjson_inline simd8x64(const simd8 chunk0, const simd8 chunk1) : chunks{ chunk0, chunk1 } {} - simdjson_inline simd8x64(const T ptr[64]) : chunks{ simd8::load(ptr), simd8::load(ptr + 32) } {} - - simdjson_inline uint64_t compress(uint64_t mask, T* output) const { - uint32_t mask1 = uint32_t(mask); - uint32_t mask2 = uint32_t(mask >> 32); - this->chunks[0].compress(mask1, output); - this->chunks[1].compress(mask2, output + 32 - count_ones(mask1)); - return 64 - count_ones(mask); - } - - simdjson_inline void store(T ptr[64]) const { - this->chunks[0].store(ptr + sizeof(simd8) * 0); - this->chunks[1].store(ptr + sizeof(simd8) * 1); - } - - simdjson_inline uint64_t to_bitmask() const { - uint64_t r_lo = uint32_t(this->chunks[0].to_bitmask()); - uint64_t r_hi = this->chunks[1].to_bitmask(); - return r_lo | (r_hi << 32); - } - - simdjson_inline simd8 reduce_or() const { - return this->chunks[0] | this->chunks[1]; - } - - simdjson_inline simd8x64 bit_or(const T m) const { - const simd8 mask = simd8::splat(m); - return simd8x64( - this->chunks[0] | mask, - this->chunks[1] | mask - ); - } - - simdjson_inline uint64_t eq(const T m) const { - const simd8 mask = simd8::splat(m); - return simd8x64( - this->chunks[0] == mask, - this->chunks[1] == mask - ).to_bitmask(); - } - - simdjson_inline uint64_t eq(const simd8x64& other) const { - return simd8x64( - this->chunks[0] == other.chunks[0], - this->chunks[1] == other.chunks[1] - ).to_bitmask(); - } - - simdjson_inline uint64_t lteq(const T m) const { - const simd8 mask = simd8::splat(m); - return simd8x64( - this->chunks[0] <= mask, - this->chunks[1] <= mask - ).to_bitmask(); - } - }; // struct simd8x64 - - } // namespace simd - - } // unnamed namespace - } // namespace haswell -} // namespace simdjson - -#endif // SIMDJSON_HASWELL_SIMD_H -/* end file simdjson/haswell/simd.h */ -/* including simdjson/haswell/stringparsing_defs.h: #include "simdjson/haswell/stringparsing_defs.h" */ -/* begin file simdjson/haswell/stringparsing_defs.h */ -#ifndef SIMDJSON_HASWELL_STRINGPARSING_DEFS_H -#define SIMDJSON_HASWELL_STRINGPARSING_DEFS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/haswell/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/haswell/simd.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/haswell/bitmanipulation.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - namespace haswell { - namespace { - - using namespace simd; - - // Holds backslashes and quotes locations. - struct backslash_and_quote { - public: - static constexpr uint32_t BYTES_PROCESSED = 32; - simdjson_inline static backslash_and_quote copy_and_find(const uint8_t* src, uint8_t* dst); - - simdjson_inline bool has_quote_first() { return ((bs_bits - 1) & quote_bits) != 0; } - simdjson_inline bool has_backslash() { return ((quote_bits - 1) & bs_bits) != 0; } - simdjson_inline int quote_index() { return trailing_zeroes(quote_bits); } - simdjson_inline int backslash_index() { return trailing_zeroes(bs_bits); } - - uint32_t bs_bits; - uint32_t quote_bits; - }; // struct backslash_and_quote - - simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8_t* src, uint8_t* dst) { - // this can read up to 15 bytes beyond the buffer size, but we require - // SIMDJSON_PADDING of padding - static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "backslash and quote finder must process fewer than SIMDJSON_PADDING bytes"); - simd8 v(src); - // store to dest unconditionally - we can overwrite the bits we don't like later - v.store(dst); - return { - static_cast((v == '\\').to_bitmask()), // bs_bits - static_cast((v == '"').to_bitmask()), // quote_bits - }; - } - - } // unnamed namespace - } // namespace haswell -} // namespace simdjson - -#endif // SIMDJSON_HASWELL_STRINGPARSING_DEFS_H -/* end file simdjson/haswell/stringparsing_defs.h */ -/* end file simdjson/haswell/begin.h */ -/* including generic/amalgamated.h for haswell: #include */ -/* begin file generic/amalgamated.h for haswell */ -#if defined(SIMDJSON_CONDITIONAL_INCLUDE) && !defined(SIMDJSON_SRC_GENERIC_DEPENDENCIES_H) -#error generic/dependencies.h must be included before generic/amalgamated.h! -#endif - -/* including generic/base.h for haswell: #include */ -/* begin file generic/base.h for haswell */ -#ifndef SIMDJSON_SRC_GENERIC_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_BASE_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - namespace haswell { - namespace { - - struct json_character_block; - - } // unnamed namespace - } // namespace haswell -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_BASE_H -/* end file generic/base.h for haswell */ -/* including generic/dom_parser_implementation.h for haswell: #include */ -/* begin file generic/dom_parser_implementation.h for haswell */ -#ifndef SIMDJSON_SRC_GENERIC_DOM_PARSER_IMPLEMENTATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_DOM_PARSER_IMPLEMENTATION_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -// Interface a dom parser implementation must fulfill -namespace simdjson { - namespace haswell { - namespace { - - simdjson_inline simd8 must_be_2_3_continuation(const simd8 prev2, const simd8 prev3); - simdjson_inline bool is_ascii(const simd8x64& input); - - } // unnamed namespace - } // namespace haswell -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_DOM_PARSER_IMPLEMENTATION_H -/* end file generic/dom_parser_implementation.h for haswell */ -/* including generic/json_character_block.h for haswell: #include */ -/* begin file generic/json_character_block.h for haswell */ -#ifndef SIMDJSON_SRC_GENERIC_JSON_CHARACTER_BLOCK_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_JSON_CHARACTER_BLOCK_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - namespace haswell { - namespace { - - struct json_character_block { - static simdjson_inline json_character_block classify(const simd::simd8x64& in); - - simdjson_inline uint64_t whitespace() const noexcept { return _whitespace; } - simdjson_inline uint64_t op() const noexcept { return _op; } - simdjson_inline uint64_t scalar() const noexcept { return ~(op() | whitespace()); } - - uint64_t _whitespace; - uint64_t _op; - }; - - } // unnamed namespace - } // namespace haswell -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_JSON_CHARACTER_BLOCK_H -/* end file generic/json_character_block.h for haswell */ -/* end file generic/amalgamated.h for haswell */ -/* including generic/stage1/amalgamated.h for haswell: #include */ -/* begin file generic/stage1/amalgamated.h for haswell */ -// Stuff other things depend on -/* including generic/stage1/base.h for haswell: #include */ -/* begin file generic/stage1/base.h for haswell */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_BASE_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - namespace haswell { - namespace { - namespace stage1 { - - class bit_indexer; - template - struct buf_block_reader; - struct json_block; - class json_minifier; - class json_scanner; - struct json_string_block; - class json_string_scanner; - class json_structural_indexer; - - } // namespace stage1 - - namespace utf8_validation { - struct utf8_checker; - } // namespace utf8_validation - - using utf8_validation::utf8_checker; - - } // unnamed namespace - } // namespace haswell -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_BASE_H -/* end file generic/stage1/base.h for haswell */ -/* including generic/stage1/buf_block_reader.h for haswell: #include */ -/* begin file generic/stage1/buf_block_reader.h for haswell */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_BUF_BLOCK_READER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_BUF_BLOCK_READER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include - -namespace simdjson { - namespace haswell { - namespace { - namespace stage1 { - - // Walks through a buffer in block-sized increments, loading the last part with spaces - template - struct buf_block_reader { - public: - simdjson_inline buf_block_reader(const uint8_t* _buf, size_t _len); - simdjson_inline size_t block_index(); - simdjson_inline bool has_full_block() const; - simdjson_inline const uint8_t* full_block() const; - /** - * Get the last block, padded with spaces. - * - * There will always be a last block, with at least 1 byte, unless len == 0 (in which case this - * function fills the buffer with spaces and returns 0. In particular, if len == STEP_SIZE there - * will be 0 full_blocks and 1 remainder block with STEP_SIZE bytes and no spaces for padding. - * - * @return the number of effective characters in the last block. - */ - simdjson_inline size_t get_remainder(uint8_t* dst) const; - simdjson_inline void advance(); - private: - const uint8_t* buf; - const size_t len; - const size_t lenminusstep; - size_t idx; - }; - - // Routines to print masks and text for debugging bitmask operations - simdjson_unused static char* format_input_text_64(const uint8_t* text) { - static char buf[sizeof(simd8x64) + 1]; - for (size_t i = 0; i < sizeof(simd8x64); i++) { - buf[i] = int8_t(text[i]) < ' ' ? '_' : int8_t(text[i]); - } - buf[sizeof(simd8x64)] = '\0'; - return buf; - } - - // Routines to print masks and text for debugging bitmask operations - simdjson_unused static char* format_input_text(const simd8x64& in) { - static char buf[sizeof(simd8x64) + 1]; - in.store(reinterpret_cast(buf)); - for (size_t i = 0; i < sizeof(simd8x64); i++) { - if (buf[i] < ' ') { buf[i] = '_'; } - } - buf[sizeof(simd8x64)] = '\0'; - return buf; - } - - simdjson_unused static char* format_input_text(const simd8x64& in, uint64_t mask) { - static char buf[sizeof(simd8x64) + 1]; - in.store(reinterpret_cast(buf)); - for (size_t i = 0; i < sizeof(simd8x64); i++) { - if (buf[i] <= ' ') { buf[i] = '_'; } - if (!(mask & (size_t(1) << i))) { buf[i] = ' '; } - } - buf[sizeof(simd8x64)] = '\0'; - return buf; - } - - simdjson_unused static char* format_mask(uint64_t mask) { - static char buf[sizeof(simd8x64) + 1]; - for (size_t i = 0; i < 64; i++) { - buf[i] = (mask & (size_t(1) << i)) ? 'X' : ' '; - } - buf[64] = '\0'; - return buf; - } - - template - simdjson_inline buf_block_reader::buf_block_reader(const uint8_t* _buf, size_t _len) : buf{ _buf }, len{ _len }, lenminusstep{ len < STEP_SIZE ? 0 : len - STEP_SIZE }, idx{ 0 } {} - - template - simdjson_inline size_t buf_block_reader::block_index() { return idx; } - - template - simdjson_inline bool buf_block_reader::has_full_block() const { - return idx < lenminusstep; - } - - template - simdjson_inline const uint8_t* buf_block_reader::full_block() const { - return &buf[idx]; - } - - template - simdjson_inline size_t buf_block_reader::get_remainder(uint8_t* dst) const { - if (len == idx) { return 0; } // memcpy(dst, null, 0) will trigger an error with some sanitizers - std::memset(dst, 0x20, STEP_SIZE); // std::memset STEP_SIZE because it's more efficient to write out 8 or 16 bytes at once. - std::memcpy(dst, buf + idx, len - idx); - return len - idx; - } - - template - simdjson_inline void buf_block_reader::advance() { - idx += STEP_SIZE; - } - - } // namespace stage1 - } // unnamed namespace - } // namespace haswell -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_BUF_BLOCK_READER_H -/* end file generic/stage1/buf_block_reader.h for haswell */ -/* including generic/stage1/json_escape_scanner.h for haswell: #include */ -/* begin file generic/stage1/json_escape_scanner.h for haswell */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_JSON_ESCAPE_SCANNER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_JSON_ESCAPE_SCANNER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - namespace haswell { - namespace { - namespace stage1 { - - /** - * Scans for escape characters in JSON, taking care with multiple backslashes (\\n vs. \n). - */ - struct json_escape_scanner { - /** The actual escape characters (the backslashes themselves). */ - uint64_t next_is_escaped = 0ULL; - - struct escaped_and_escape { - /** - * Mask of escaped characters. - * - * ``` - * \n \\n \\\n \\\\n \ - * 0100100010100101000 - * n \ \ n \ \ - * ``` - */ - uint64_t escaped; - /** - * Mask of escape characters. - * - * ``` - * \n \\n \\\n \\\\n \ - * 1001000101001010001 - * \ \ \ \ \ \ \ - * ``` - */ - uint64_t escape; - }; - - /** - * Get a mask of both escape and escaped characters (the characters following a backslash). - * - * @param potential_escape A mask of the character that can escape others (but could be - * escaped itself). e.g. block.eq('\\') - */ - simdjson_really_inline escaped_and_escape next(uint64_t backslash) noexcept { - -#if !SIMDJSON_SKIP_BACKSLASH_SHORT_CIRCUIT - if (!backslash) { return { next_escaped_without_backslashes(), 0 }; } -#endif - - // | | Mask (shows characters instead of 1's) | Depth | Instructions | - // |--------------------------------|----------------------------------------|-------|---------------------| - // | string | `\\n_\\\n___\\\n___\\\\___\\\\__\\\` | | | - // | | ` even odd even odd odd` | | | - // | potential_escape | ` \ \\\ \\\ \\\\ \\\\ \\\` | 1 | 1 (backslash & ~first_is_escaped) - // | escape_and_terminal_code | ` \n \ \n \ \n \ \ \ \ \ \` | 5 | 5 (next_escape_and_terminal_code()) - // | escaped | `\ \ n \ n \ \ \ \ \ ` X | 6 | 7 (escape_and_terminal_code ^ (potential_escape | first_is_escaped)) - // | escape | ` \ \ \ \ \ \ \ \ \ \` | 6 | 8 (escape_and_terminal_code & backslash) - // | first_is_escaped | `\ ` | 7 (*) | 9 (escape >> 63) () - // (*) this is not needed until the next iteration - uint64_t escape_and_terminal_code = next_escape_and_terminal_code(backslash & ~this->next_is_escaped); - uint64_t escaped = escape_and_terminal_code ^ (backslash | this->next_is_escaped); - uint64_t escape = escape_and_terminal_code & backslash; - this->next_is_escaped = escape >> 63; - return { escaped, escape }; - } - - private: - static constexpr const uint64_t ODD_BITS = 0xAAAAAAAAAAAAAAAAULL; - - simdjson_really_inline uint64_t next_escaped_without_backslashes() noexcept { - uint64_t escaped = this->next_is_escaped; - this->next_is_escaped = 0; - return escaped; - } - - /** - * Returns a mask of the next escape characters (masking out escaped backslashes), along with - * any non-backslash escape codes. - * - * \n \\n \\\n \\\\n returns: - * \n \ \ \n \ \ - * 11 100 1011 10100 - * - * You are expected to mask out the first bit yourself if the previous block had a trailing - * escape. - * - * & the result with potential_escape to get just the escape characters. - * ^ the result with (potential_escape | first_is_escaped) to get escaped characters. - */ - static simdjson_really_inline uint64_t next_escape_and_terminal_code(uint64_t potential_escape) noexcept { - // If we were to just shift and mask out any odd bits, we'd actually get a *half* right answer: - // any even-aligned backslash runs would be correct! Odd-aligned backslash runs would be - // inverted (\\\ would be 010 instead of 101). - // - // ``` - // string: | ____\\\\_\\\\_____ | - // maybe_escaped | ODD | \ \ \ \ | - // even-aligned ^^^ ^^^^ odd-aligned - // ``` - // - // Taking that into account, our basic strategy is: - // - // 1. Use subtraction to produce a mask with 1's for even-aligned runs and 0's for - // odd-aligned runs. - // 2. XOR all odd bits, which masks out the odd bits in even-aligned runs, and brings IN the - // odd bits in odd-aligned runs. - // 3. & with backslash to clean up any stray bits. - // runs are set to 0, and then XORing with "odd": - // - // | | Mask (shows characters instead of 1's) | Instructions | - // |--------------------------------|----------------------------------------|---------------------| - // | string | `\\n_\\\n___\\\n___\\\\___\\\\__\\\` | - // | | ` even odd even odd odd` | - // | maybe_escaped | ` n \\n \\n \\\_ \\\_ \\` X | 1 (potential_escape << 1) - // | maybe_escaped_and_odd | ` \n_ \\n _ \\\n_ _ \\\__ _\\\_ \\\` | 1 (maybe_escaped | odd) - // | even_series_codes_and_odd | ` n_\\\ _ n_ _\\\\ _ _ ` | 1 (maybe_escaped_and_odd - potential_escape) - // | escape_and_terminal_code | ` \n \ \n \ \n \ \ \ \ \ \` | 1 (^ odd) - // - - // Escaped characters are characters following an escape. - uint64_t maybe_escaped = potential_escape << 1; - - // To distinguish odd from even escape sequences, therefore, we turn on any *starting* - // escapes that are on an odd byte. (We actually bring in all odd bits, for speed.) - // - Odd runs of backslashes are 0000, and the code at the end ("n" in \n or \\n) is 1. - // - Odd runs of backslashes are 1111, and the code at the end ("n" in \n or \\n) is 0. - // - All other odd bytes are 1, and even bytes are 0. - uint64_t maybe_escaped_and_odd_bits = maybe_escaped | ODD_BITS; - uint64_t even_series_codes_and_odd_bits = maybe_escaped_and_odd_bits - potential_escape; - - // Now we flip all odd bytes back with xor. This: - // - Makes odd runs of backslashes go from 0000 to 1010 - // - Makes even runs of backslashes go from 1111 to 1010 - // - Sets actually-escaped codes to 1 (the n in \n and \\n: \n = 11, \\n = 100) - // - Resets all other bytes to 0 - return even_series_codes_and_odd_bits ^ ODD_BITS; - } - }; - - } // namespace stage1 - } // unnamed namespace - } // namespace haswell -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRING_SCANNER_H -/* end file generic/stage1/json_escape_scanner.h for haswell */ -/* including generic/stage1/json_string_scanner.h for haswell: #include */ -/* begin file generic/stage1/json_string_scanner.h for haswell */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRING_SCANNER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRING_SCANNER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - namespace haswell { - namespace { - namespace stage1 { - - struct json_string_block { - // We spell out the constructors in the hope of resolving inlining issues with Visual Studio 2017 - simdjson_really_inline json_string_block(uint64_t escaped, uint64_t quote, uint64_t in_string) : - _escaped(escaped), _quote(quote), _in_string(in_string) {} - - // Escaped characters (characters following an escape() character) - simdjson_really_inline uint64_t escaped() const { return _escaped; } - // Real (non-backslashed) quotes - simdjson_really_inline uint64_t quote() const { return _quote; } - // Only characters inside the string (not including the quotes) - simdjson_really_inline uint64_t string_content() const { return _in_string & ~_quote; } - // Return a mask of whether the given characters are inside a string (only works on non-quotes) - simdjson_really_inline uint64_t non_quote_inside_string(uint64_t mask) const { return mask & _in_string; } - // Return a mask of whether the given characters are inside a string (only works on non-quotes) - simdjson_really_inline uint64_t non_quote_outside_string(uint64_t mask) const { return mask & ~_in_string; } - // Tail of string (everything except the start quote) - simdjson_really_inline uint64_t string_tail() const { return _in_string ^ _quote; } - - // escaped characters (backslashed--does not include the hex characters after \u) - uint64_t _escaped; - // real quotes (non-escaped ones) - uint64_t _quote; - // string characters (includes start quote but not end quote) - uint64_t _in_string; - }; - - // Scans blocks for string characters, storing the state necessary to do so - class json_string_scanner { - public: - simdjson_really_inline json_string_block next(const simd::simd8x64& in); - // Returns either UNCLOSED_STRING or SUCCESS - simdjson_really_inline error_code finish(); - - private: - // Scans for escape characters - json_escape_scanner escape_scanner{}; - // Whether the last iteration was still inside a string (all 1's = true, all 0's = false). - uint64_t prev_in_string = 0ULL; - }; - - // - // Return a mask of all string characters plus end quotes. - // - // prev_escaped is overflow saying whether the next character is escaped. - // prev_in_string is overflow saying whether we're still in a string. - // - // Backslash sequences outside of quotes will be detected in stage 2. - // - simdjson_really_inline json_string_block json_string_scanner::next(const simd::simd8x64& in) { - const uint64_t backslash = in.eq('\\'); - const uint64_t escaped = escape_scanner.next(backslash).escaped; - const uint64_t quote = in.eq('"') & ~escaped; - - // - // prefix_xor flips on bits inside the string (and flips off the end quote). - // - // Then we xor with prev_in_string: if we were in a string already, its effect is flipped - // (characters inside strings are outside, and characters outside strings are inside). - // - const uint64_t in_string = prefix_xor(quote) ^ prev_in_string; - - // - // Check if we're still in a string at the end of the box so the next block will know - // - prev_in_string = uint64_t(static_cast(in_string) >> 63); - - // Use ^ to turn the beginning quote off, and the end quote on. - - // We are returning a function-local object so either we get a move constructor - // or we get copy elision. - return json_string_block(escaped, quote, in_string); - } - - simdjson_really_inline error_code json_string_scanner::finish() { - if (prev_in_string) { - return UNCLOSED_STRING; - } - return SUCCESS; - } - - } // namespace stage1 - } // unnamed namespace - } // namespace haswell -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRING_SCANNER_H -/* end file generic/stage1/json_string_scanner.h for haswell */ -/* including generic/stage1/utf8_lookup4_algorithm.h for haswell: #include */ -/* begin file generic/stage1/utf8_lookup4_algorithm.h for haswell */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_UTF8_LOOKUP4_ALGORITHM_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_UTF8_LOOKUP4_ALGORITHM_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - namespace haswell { - namespace { - namespace utf8_validation { - - using namespace simd; - - simdjson_inline simd8 check_special_cases(const simd8 input, const simd8 prev1) { - // Bit 0 = Too Short (lead byte/ASCII followed by lead byte/ASCII) - // Bit 1 = Too Long (ASCII followed by continuation) - // Bit 2 = Overlong 3-byte - // Bit 4 = Surrogate - // Bit 5 = Overlong 2-byte - // Bit 7 = Two Continuations - constexpr const uint8_t TOO_SHORT = 1 << 0; // 11______ 0_______ - // 11______ 11______ - constexpr const uint8_t TOO_LONG = 1 << 1; // 0_______ 10______ - constexpr const uint8_t OVERLONG_3 = 1 << 2; // 11100000 100_____ - constexpr const uint8_t SURROGATE = 1 << 4; // 11101101 101_____ - constexpr const uint8_t OVERLONG_2 = 1 << 5; // 1100000_ 10______ - constexpr const uint8_t TWO_CONTS = 1 << 7; // 10______ 10______ - constexpr const uint8_t TOO_LARGE = 1 << 3; // 11110100 1001____ - // 11110100 101_____ - // 11110101 1001____ - // 11110101 101_____ - // 1111011_ 1001____ - // 1111011_ 101_____ - // 11111___ 1001____ - // 11111___ 101_____ - constexpr const uint8_t TOO_LARGE_1000 = 1 << 6; - // 11110101 1000____ - // 1111011_ 1000____ - // 11111___ 1000____ - constexpr const uint8_t OVERLONG_4 = 1 << 6; // 11110000 1000____ - - const simd8 byte_1_high = prev1.shr<4>().lookup_16( - // 0_______ ________ - TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, - TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, - // 10______ ________ - TWO_CONTS, TWO_CONTS, TWO_CONTS, TWO_CONTS, - // 1100____ ________ - TOO_SHORT | OVERLONG_2, - // 1101____ ________ - TOO_SHORT, - // 1110____ ________ - TOO_SHORT | OVERLONG_3 | SURROGATE, - // 1111____ ________ - TOO_SHORT | TOO_LARGE | TOO_LARGE_1000 | OVERLONG_4 - ); - constexpr const uint8_t CARRY = TOO_SHORT | TOO_LONG | TWO_CONTS; // These all have ____ in byte 1 . - const simd8 byte_1_low = (prev1 & 0x0F).lookup_16( - // ____0000 ________ - CARRY | OVERLONG_3 | OVERLONG_2 | OVERLONG_4, - // ____0001 ________ - CARRY | OVERLONG_2, - // ____001_ ________ - CARRY, - CARRY, - - // ____0100 ________ - CARRY | TOO_LARGE, - // ____0101 ________ - CARRY | TOO_LARGE | TOO_LARGE_1000, - // ____011_ ________ - CARRY | TOO_LARGE | TOO_LARGE_1000, - CARRY | TOO_LARGE | TOO_LARGE_1000, - - // ____1___ ________ - CARRY | TOO_LARGE | TOO_LARGE_1000, - CARRY | TOO_LARGE | TOO_LARGE_1000, - CARRY | TOO_LARGE | TOO_LARGE_1000, - CARRY | TOO_LARGE | TOO_LARGE_1000, - CARRY | TOO_LARGE | TOO_LARGE_1000, - // ____1101 ________ - CARRY | TOO_LARGE | TOO_LARGE_1000 | SURROGATE, - CARRY | TOO_LARGE | TOO_LARGE_1000, - CARRY | TOO_LARGE | TOO_LARGE_1000 - ); - const simd8 byte_2_high = input.shr<4>().lookup_16( - // ________ 0_______ - TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT, - TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT, - - // ________ 1000____ - TOO_LONG | OVERLONG_2 | TWO_CONTS | OVERLONG_3 | TOO_LARGE_1000 | OVERLONG_4, - // ________ 1001____ - TOO_LONG | OVERLONG_2 | TWO_CONTS | OVERLONG_3 | TOO_LARGE, - // ________ 101_____ - TOO_LONG | OVERLONG_2 | TWO_CONTS | SURROGATE | TOO_LARGE, - TOO_LONG | OVERLONG_2 | TWO_CONTS | SURROGATE | TOO_LARGE, - - // ________ 11______ - TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT - ); - return (byte_1_high & byte_1_low & byte_2_high); - } - simdjson_inline simd8 check_multibyte_lengths(const simd8 input, - const simd8 prev_input, const simd8 sc) { - simd8 prev2 = input.prev<2>(prev_input); - simd8 prev3 = input.prev<3>(prev_input); - simd8 must23 = simd8(must_be_2_3_continuation(prev2, prev3)); - simd8 must23_80 = must23 & uint8_t(0x80); - return must23_80 ^ sc; - } - - // - // Return nonzero if there are incomplete multibyte characters at the end of the block: - // e.g. if there is a 4-byte character, but it's 3 bytes from the end. - // - simdjson_inline simd8 is_incomplete(const simd8 input) { - // If the previous input's last 3 bytes match this, they're too short (they ended at EOF): - // ... 1111____ 111_____ 11______ -#if SIMDJSON_IMPLEMENTATION_ICELAKE - static const uint8_t max_array[64] = { - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 0xf0u - 1, 0xe0u - 1, 0xc0u - 1 - }; -#else - static const uint8_t max_array[32] = { - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 0xf0u - 1, 0xe0u - 1, 0xc0u - 1 - }; -#endif - const simd8 max_value(&max_array[sizeof(max_array) - sizeof(simd8)]); - return input.gt_bits(max_value); - } - - struct utf8_checker { - // If this is nonzero, there has been a UTF-8 error. - simd8 error; - // The last input we received - simd8 prev_input_block; - // Whether the last input we received was incomplete (used for ASCII fast path) - simd8 prev_incomplete; - - // - // Check whether the current bytes are valid UTF-8. - // - simdjson_inline void check_utf8_bytes(const simd8 input, const simd8 prev_input) { - // Flip prev1...prev3 so we can easily determine if they are 2+, 3+ or 4+ lead bytes - // (2, 3, 4-byte leads become large positive numbers instead of small negative numbers) - simd8 prev1 = input.prev<1>(prev_input); - simd8 sc = check_special_cases(input, prev1); - this->error |= check_multibyte_lengths(input, prev_input, sc); - } - - // The only problem that can happen at EOF is that a multibyte character is too short - // or a byte value too large in the last bytes: check_special_cases only checks for bytes - // too large in the first of two bytes. - simdjson_inline void check_eof() { - // If the previous block had incomplete UTF-8 characters at the end, an ASCII block can't - // possibly finish them. - this->error |= this->prev_incomplete; - } - - simdjson_inline void check_next_input(const simd8x64& input) { - if (simdjson_likely(is_ascii(input))) { - this->error |= this->prev_incomplete; - } - else { - // you might think that a for-loop would work, but under Visual Studio, it is not good enough. - static_assert((simd8x64::NUM_CHUNKS == 1) - || (simd8x64::NUM_CHUNKS == 2) - || (simd8x64::NUM_CHUNKS == 4), - "We support one, two or four chunks per 64-byte block."); - SIMDJSON_IF_CONSTEXPR(simd8x64::NUM_CHUNKS == 1) { - this->check_utf8_bytes(input.chunks[0], this->prev_input_block); - } -else SIMDJSON_IF_CONSTEXPR(simd8x64::NUM_CHUNKS == 2) { - this->check_utf8_bytes(input.chunks[0], this->prev_input_block); - this->check_utf8_bytes(input.chunks[1], input.chunks[0]); - } - else SIMDJSON_IF_CONSTEXPR(simd8x64::NUM_CHUNKS == 4) { - this->check_utf8_bytes(input.chunks[0], this->prev_input_block); - this->check_utf8_bytes(input.chunks[1], input.chunks[0]); - this->check_utf8_bytes(input.chunks[2], input.chunks[1]); - this->check_utf8_bytes(input.chunks[3], input.chunks[2]); - } - this->prev_incomplete = is_incomplete(input.chunks[simd8x64::NUM_CHUNKS - 1]); - this->prev_input_block = input.chunks[simd8x64::NUM_CHUNKS - 1]; - } - } - // do not forget to call check_eof! - simdjson_inline error_code errors() { - return this->error.any_bits_set_anywhere() ? error_code::UTF8_ERROR : error_code::SUCCESS; - } - - }; // struct utf8_checker - } // namespace utf8_validation - - } // unnamed namespace - } // namespace haswell -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_UTF8_LOOKUP4_ALGORITHM_H -/* end file generic/stage1/utf8_lookup4_algorithm.h for haswell */ -/* including generic/stage1/json_scanner.h for haswell: #include */ -/* begin file generic/stage1/json_scanner.h for haswell */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_JSON_SCANNER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_JSON_SCANNER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - namespace haswell { - namespace { - namespace stage1 { - - /** - * A block of scanned json, with information on operators and scalars. - * - * We seek to identify pseudo-structural characters. Anything that is inside - * a string must be omitted (hence & ~_string.string_tail()). - * Otherwise, pseudo-structural characters come in two forms. - * 1. We have the structural characters ([,],{,},:, comma). The - * term 'structural character' is from the JSON RFC. - * 2. We have the 'scalar pseudo-structural characters'. - * Scalars are quotes, and any character except structural characters and white space. - * - * To identify the scalar pseudo-structural characters, we must look at what comes - * before them: it must be a space, a quote or a structural characters. - * Starting with simdjson v0.3, we identify them by - * negation: we identify everything that is followed by a non-quote scalar, - * and we negate that. Whatever remains must be a 'scalar pseudo-structural character'. - */ - struct json_block { - public: - // We spell out the constructors in the hope of resolving inlining issues with Visual Studio 2017 - simdjson_inline json_block(json_string_block&& string, json_character_block characters, uint64_t follows_potential_nonquote_scalar) : - _string(std::move(string)), _characters(characters), _follows_potential_nonquote_scalar(follows_potential_nonquote_scalar) {} - simdjson_inline json_block(json_string_block string, json_character_block characters, uint64_t follows_potential_nonquote_scalar) : - _string(string), _characters(characters), _follows_potential_nonquote_scalar(follows_potential_nonquote_scalar) {} - - /** - * The start of structurals. - * In simdjson prior to v0.3, these were called the pseudo-structural characters. - **/ - simdjson_inline uint64_t structural_start() const noexcept { return potential_structural_start() & ~_string.string_tail(); } - /** All JSON whitespace (i.e. not in a string) */ - simdjson_inline uint64_t whitespace() const noexcept { return non_quote_outside_string(_characters.whitespace()); } - - // Helpers - - /** Whether the given characters are inside a string (only works on non-quotes) */ - simdjson_inline uint64_t non_quote_inside_string(uint64_t mask) const noexcept { return _string.non_quote_inside_string(mask); } - /** Whether the given characters are outside a string (only works on non-quotes) */ - simdjson_inline uint64_t non_quote_outside_string(uint64_t mask) const noexcept { return _string.non_quote_outside_string(mask); } - - // string and escape characters - json_string_block _string; - // whitespace, structural characters ('operators'), scalars - json_character_block _characters; - // whether the previous character was a scalar - uint64_t _follows_potential_nonquote_scalar; - private: - // Potential structurals (i.e. disregarding strings) - - /** - * structural elements ([,],{,},:, comma) plus scalar starts like 123, true and "abc". - * They may reside inside a string. - **/ - simdjson_inline uint64_t potential_structural_start() const noexcept { return _characters.op() | potential_scalar_start(); } - /** - * The start of non-operator runs, like 123, true and "abc". - * It main reside inside a string. - **/ - simdjson_inline uint64_t potential_scalar_start() const noexcept { - // The term "scalar" refers to anything except structural characters and white space - // (so letters, numbers, quotes). - // Whenever it is preceded by something that is not a structural element ({,},[,],:, ") nor a white-space - // then we know that it is irrelevant structurally. - return _characters.scalar() & ~follows_potential_scalar(); - } - /** - * Whether the given character is immediately after a non-operator like 123, true. - * The characters following a quote are not included. - */ - simdjson_inline uint64_t follows_potential_scalar() const noexcept { - // _follows_potential_nonquote_scalar: is defined as marking any character that follows a character - // that is not a structural element ({,},[,],:, comma) nor a quote (") and that is not a - // white space. - // It is understood that within quoted region, anything at all could be marked (irrelevant). - return _follows_potential_nonquote_scalar; - } - }; - - /** - * Scans JSON for important bits: structural characters or 'operators', strings, and scalars. - * - * The scanner starts by calculating two distinct things: - * - string characters (taking \" into account) - * - structural characters or 'operators' ([]{},:, comma) - * and scalars (runs of non-operators like 123, true and "abc") - * - * To minimize data dependency (a key component of the scanner's speed), it finds these in parallel: - * in particular, the operator/scalar bit will find plenty of things that are actually part of - * strings. When we're done, json_block will fuse the two together by masking out tokens that are - * part of a string. - */ - class json_scanner { - public: - json_scanner() = default; - simdjson_inline json_block next(const simd::simd8x64& in); - // Returns either UNCLOSED_STRING or SUCCESS - simdjson_inline error_code finish(); - - private: - // Whether the last character of the previous iteration is part of a scalar token - // (anything except whitespace or a structural character/'operator'). - uint64_t prev_scalar = 0ULL; - json_string_scanner string_scanner{}; - }; - - - // - // Check if the current character immediately follows a matching character. - // - // For example, this checks for quotes with backslashes in front of them: - // - // const uint64_t backslashed_quote = in.eq('"') & immediately_follows(in.eq('\'), prev_backslash); - // - simdjson_inline uint64_t follows(const uint64_t match, uint64_t& overflow) { - const uint64_t result = match << 1 | overflow; - overflow = match >> 63; - return result; - } - - simdjson_inline json_block json_scanner::next(const simd::simd8x64& in) { - json_string_block strings = string_scanner.next(in); - // identifies the white-space and the structural characters - json_character_block characters = json_character_block::classify(in); - // The term "scalar" refers to anything except structural characters and white space - // (so letters, numbers, quotes). - // We want follows_scalar to mark anything that follows a non-quote scalar (so letters and numbers). - // - // A terminal quote should either be followed by a structural character (comma, brace, bracket, colon) - // or nothing. However, we still want ' "a string"true ' to mark the 't' of 'true' as a potential - // pseudo-structural character just like we would if we had ' "a string" true '; otherwise we - // may need to add an extra check when parsing strings. - // - // Performance: there are many ways to skin this cat. - const uint64_t nonquote_scalar = characters.scalar() & ~strings.quote(); - uint64_t follows_nonquote_scalar = follows(nonquote_scalar, prev_scalar); - // We are returning a function-local object so either we get a move constructor - // or we get copy elision. - return json_block( - strings,// strings is a function-local object so either it moves or the copy is elided. - characters, - follows_nonquote_scalar - ); - } - - simdjson_inline error_code json_scanner::finish() { - return string_scanner.finish(); - } - - } // namespace stage1 - } // unnamed namespace - } // namespace haswell -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_JSON_SCANNER_H -/* end file generic/stage1/json_scanner.h for haswell */ - -// All other declarations -/* including generic/stage1/find_next_document_index.h for haswell: #include */ -/* begin file generic/stage1/find_next_document_index.h for haswell */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_FIND_NEXT_DOCUMENT_INDEX_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_FIND_NEXT_DOCUMENT_INDEX_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - namespace haswell { - namespace { - namespace stage1 { - - /** - * This algorithm is used to quickly identify the last structural position that - * makes up a complete document. - * - * It does this by going backwards and finding the last *document boundary* (a - * place where one value follows another without a comma between them). If the - * last document (the characters after the boundary) has an equal number of - * start and end brackets, it is considered complete. - * - * Simply put, we iterate over the structural characters, starting from - * the end. We consider that we found the end of a JSON document when the - * first element of the pair is NOT one of these characters: '{' '[' ':' ',' - * and when the second element is NOT one of these characters: '}' ']' ':' ','. - * - * This simple comparison works most of the time, but it does not cover cases - * where the batch's structural indexes contain a perfect amount of documents. - * In such a case, we do not have access to the structural index which follows - * the last document, therefore, we do not have access to the second element in - * the pair, and that means we cannot identify the last document. To fix this - * issue, we keep a count of the open and closed curly/square braces we found - * while searching for the pair. When we find a pair AND the count of open and - * closed curly/square braces is the same, we know that we just passed a - * complete document, therefore the last json buffer location is the end of the - * batch. - */ - simdjson_inline uint32_t find_next_document_index(dom_parser_implementation& parser) { - // Variant: do not count separately, just figure out depth - if (parser.n_structural_indexes == 0) { return 0; } - auto arr_cnt = 0; - auto obj_cnt = 0; - for (auto i = parser.n_structural_indexes - 1; i > 0; i--) { - auto idxb = parser.structural_indexes[i]; - switch (parser.buf[idxb]) { - case ':': - case ',': - continue; - case '}': - obj_cnt--; - continue; - case ']': - arr_cnt--; - continue; - case '{': - obj_cnt++; - break; - case '[': - arr_cnt++; - break; - } - auto idxa = parser.structural_indexes[i - 1]; - switch (parser.buf[idxa]) { - case '{': - case '[': - case ':': - case ',': - continue; - } - // Last document is complete, so the next document will appear after! - if (!arr_cnt && !obj_cnt) { - return parser.n_structural_indexes; - } - // Last document is incomplete; mark the document at i + 1 as the next one - return i; - } - // If we made it to the end, we want to finish counting to see if we have a full document. - switch (parser.buf[parser.structural_indexes[0]]) { - case '}': - obj_cnt--; - break; - case ']': - arr_cnt--; - break; - case '{': - obj_cnt++; - break; - case '[': - arr_cnt++; - break; - } - if (!arr_cnt && !obj_cnt) { - // We have a complete document. - return parser.n_structural_indexes; - } - return 0; - } - - } // namespace stage1 - } // unnamed namespace - } // namespace haswell -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_FIND_NEXT_DOCUMENT_INDEX_H -/* end file generic/stage1/find_next_document_index.h for haswell */ -/* including generic/stage1/json_minifier.h for haswell: #include */ -/* begin file generic/stage1/json_minifier.h for haswell */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_JSON_MINIFIER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_JSON_MINIFIER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -// This file contains the common code every implementation uses in stage1 -// It is intended to be included multiple times and compiled multiple times -// We assume the file in which it is included already includes -// "simdjson/stage1.h" (this simplifies amalgation) - -namespace simdjson { - namespace haswell { - namespace { - namespace stage1 { - - class json_minifier { - public: - template - static error_code minify(const uint8_t* buf, size_t len, uint8_t* dst, size_t& dst_len) noexcept; - - private: - simdjson_inline json_minifier(uint8_t* _dst) - : dst{ _dst } - {} - template - simdjson_inline void step(const uint8_t* block_buf, buf_block_reader& reader) noexcept; - simdjson_inline void next(const simd::simd8x64& in, const json_block& block); - simdjson_inline error_code finish(uint8_t* dst_start, size_t& dst_len); - json_scanner scanner{}; - uint8_t* dst; - }; - - simdjson_inline void json_minifier::next(const simd::simd8x64& in, const json_block& block) { - uint64_t mask = block.whitespace(); - dst += in.compress(mask, dst); - } - - simdjson_inline error_code json_minifier::finish(uint8_t* dst_start, size_t& dst_len) { - error_code error = scanner.finish(); - if (error) { dst_len = 0; return error; } - dst_len = dst - dst_start; - return SUCCESS; - } - - template<> - simdjson_inline void json_minifier::step<128>(const uint8_t* block_buf, buf_block_reader<128>& reader) noexcept { - simd::simd8x64 in_1(block_buf); - simd::simd8x64 in_2(block_buf + 64); - json_block block_1 = scanner.next(in_1); - json_block block_2 = scanner.next(in_2); - this->next(in_1, block_1); - this->next(in_2, block_2); - reader.advance(); - } - - template<> - simdjson_inline void json_minifier::step<64>(const uint8_t* block_buf, buf_block_reader<64>& reader) noexcept { - simd::simd8x64 in_1(block_buf); - json_block block_1 = scanner.next(in_1); - this->next(block_buf, block_1); - reader.advance(); - } - - template - error_code json_minifier::minify(const uint8_t* buf, size_t len, uint8_t* dst, size_t& dst_len) noexcept { - buf_block_reader reader(buf, len); - json_minifier minifier(dst); - - // Index the first n-1 blocks - while (reader.has_full_block()) { - minifier.step(reader.full_block(), reader); - } - - // Index the last (remainder) block, padded with spaces - uint8_t block[STEP_SIZE]; - size_t remaining_bytes = reader.get_remainder(block); - if (remaining_bytes > 0) { - // We do not want to write directly to the output stream. Rather, we write - // to a local buffer (for safety). - uint8_t out_block[STEP_SIZE]; - uint8_t* const guarded_dst{ minifier.dst }; - minifier.dst = out_block; - minifier.step(block, reader); - size_t to_write = minifier.dst - out_block; - // In some cases, we could be enticed to consider the padded spaces - // as part of the string. This is fine as long as we do not write more - // than we consumed. - if (to_write > remaining_bytes) { to_write = remaining_bytes; } - memcpy(guarded_dst, out_block, to_write); - minifier.dst = guarded_dst + to_write; - } - return minifier.finish(dst, dst_len); - } - - } // namespace stage1 - } // unnamed namespace - } // namespace haswell -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_JSON_MINIFIER_H -/* end file generic/stage1/json_minifier.h for haswell */ -/* including generic/stage1/json_structural_indexer.h for haswell: #include */ -/* begin file generic/stage1/json_structural_indexer.h for haswell */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRUCTURAL_INDEXER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRUCTURAL_INDEXER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -// This file contains the common code every implementation uses in stage1 -// It is intended to be included multiple times and compiled multiple times -// We assume the file in which it is included already includes -// "simdjson/stage1.h" (this simplifies amalgation) - -namespace simdjson { - namespace haswell { - namespace { - namespace stage1 { - - class bit_indexer { - public: - uint32_t* tail; - - simdjson_inline bit_indexer(uint32_t* index_buf) : tail(index_buf) {} - -#if SIMDJSON_PREFER_REVERSE_BITS - /** - * ARM lacks a fast trailing zero instruction, but it has a fast - * bit reversal instruction and a fast leading zero instruction. - * Thus it may be profitable to reverse the bits (once) and then - * to rely on a sequence of instructions that call the leading - * zero instruction. - * - * Performance notes: - * The chosen routine is not optimal in terms of data dependency - * since zero_leading_bit might require two instructions. However, - * it tends to minimize the total number of instructions which is - * beneficial. - */ - simdjson_inline void write_index(uint32_t idx, uint64_t& rev_bits, int i) { - int lz = leading_zeroes(rev_bits); - this->tail[i] = static_cast(idx) + lz; - rev_bits = zero_leading_bit(rev_bits, lz); - } -#else - /** - * Under recent x64 systems, we often have both a fast trailing zero - * instruction and a fast 'clear-lower-bit' instruction so the following - * algorithm can be competitive. - */ - - simdjson_inline void write_index(uint32_t idx, uint64_t& bits, int i) { - this->tail[i] = idx + trailing_zeroes(bits); - bits = clear_lowest_bit(bits); - } -#endif // SIMDJSON_PREFER_REVERSE_BITS - - template - simdjson_inline int write_indexes(uint32_t idx, uint64_t& bits) { - write_index(idx, bits, START); - SIMDJSON_IF_CONSTEXPR(N > 1) { - write_indexes<(N - 1 > 0 ? START + 1 : START), (N - 1 >= 0 ? N - 1 : 1)>(idx, bits); - } - return START + N; - } - - template - simdjson_inline int write_indexes_stepped(uint32_t idx, uint64_t& bits, int cnt) { - write_indexes(idx, bits); - SIMDJSON_IF_CONSTEXPR((START + STEP) < END) { - if (simdjson_unlikely((START + STEP) < cnt)) { - write_indexes_stepped<(START + STEP < END ? START + STEP : END), END, STEP>(idx, bits, cnt); - } - } - return ((END - START) % STEP) == 0 ? END : (END - START) - ((END - START) % STEP) + STEP; - } - - // flatten out values in 'bits' assuming that they are are to have values of idx - // plus their position in the bitvector, and store these indexes at - // base_ptr[base] incrementing base as we go - // will potentially store extra values beyond end of valid bits, so base_ptr - // needs to be large enough to handle this - // - // If the kernel sets SIMDJSON_GENERIC_JSON_STRUCTURAL_INDEXER_CUSTOM_BIT_INDEXER, then it - // will provide its own version of the code. -#ifdef SIMDJSON_GENERIC_JSON_STRUCTURAL_INDEXER_CUSTOM_BIT_INDEXER - simdjson_inline void write(uint32_t idx, uint64_t bits); -#else - simdjson_inline void write(uint32_t idx, uint64_t bits) { - // In some instances, the next branch is expensive because it is mispredicted. - // Unfortunately, in other cases, - // it helps tremendously. - if (bits == 0) - return; - - int cnt = static_cast(count_ones(bits)); - -#if SIMDJSON_PREFER_REVERSE_BITS - bits = reverse_bits(bits); -#endif -#ifdef SIMDJSON_STRUCTURAL_INDEXER_STEP - static constexpr const int STEP = SIMDJSON_STRUCTURAL_INDEXER_STEP; -#else - static constexpr const int STEP = 4; -#endif - static constexpr const int STEP_UNTIL = 24; - - write_indexes_stepped<0, STEP_UNTIL, STEP>(idx, bits, cnt); - SIMDJSON_IF_CONSTEXPR(STEP_UNTIL < 64) { - if (simdjson_unlikely(STEP_UNTIL < cnt)) { - for (int i = STEP_UNTIL; i < cnt; i++) { - write_index(idx, bits, i); - } - } - } - - this->tail += cnt; - } -#endif // SIMDJSON_GENERIC_JSON_STRUCTURAL_INDEXER_CUSTOM_BIT_INDEXER - - }; - - class json_structural_indexer { - public: - /** - * Find the important bits of JSON in a 128-byte chunk, and add them to structural_indexes. - * - * @param partial Setting the partial parameter to true allows the find_structural_bits to - * tolerate unclosed strings. The caller should still ensure that the input is valid UTF-8. If - * you are processing substrings, you may want to call on a function like trimmed_length_safe_utf8. - */ - template - static error_code index(const uint8_t* buf, size_t len, dom_parser_implementation& parser, stage1_mode partial) noexcept; - - private: - simdjson_inline json_structural_indexer(uint32_t* structural_indexes); - template - simdjson_inline void step(const uint8_t* block, buf_block_reader& reader) noexcept; - simdjson_inline void next(const simd::simd8x64& in, const json_block& block, size_t idx); - simdjson_inline error_code finish(dom_parser_implementation& parser, size_t idx, size_t len, stage1_mode partial); - - json_scanner scanner{}; - utf8_checker checker{}; - bit_indexer indexer; - uint64_t prev_structurals = 0; - uint64_t unescaped_chars_error = 0; - }; - - simdjson_inline json_structural_indexer::json_structural_indexer(uint32_t* structural_indexes) : indexer{ structural_indexes } {} - - // Skip the last character if it is partial - simdjson_inline size_t trim_partial_utf8(const uint8_t* buf, size_t len) { - if (simdjson_unlikely(len < 3)) { - switch (len) { - case 2: - if (buf[len - 1] >= 0xc0) { return len - 1; } // 2-, 3- and 4-byte characters with only 1 byte left - if (buf[len - 2] >= 0xe0) { return len - 2; } // 3- and 4-byte characters with only 2 bytes left - return len; - case 1: - if (buf[len - 1] >= 0xc0) { return len - 1; } // 2-, 3- and 4-byte characters with only 1 byte left - return len; - case 0: - return len; - } - } - if (buf[len - 1] >= 0xc0) { return len - 1; } // 2-, 3- and 4-byte characters with only 1 byte left - if (buf[len - 2] >= 0xe0) { return len - 2; } // 3- and 4-byte characters with only 1 byte left - if (buf[len - 3] >= 0xf0) { return len - 3; } // 4-byte characters with only 3 bytes left - return len; - } - - // - // PERF NOTES: - // We pipe 2 inputs through these stages: - // 1. Load JSON into registers. This takes a long time and is highly parallelizable, so we load - // 2 inputs' worth at once so that by the time step 2 is looking for them input, it's available. - // 2. Scan the JSON for critical data: strings, scalars and operators. This is the critical path. - // The output of step 1 depends entirely on this information. These functions don't quite use - // up enough CPU: the second half of the functions is highly serial, only using 1 execution core - // at a time. The second input's scans has some dependency on the first ones finishing it, but - // they can make a lot of progress before they need that information. - // 3. Step 1 doesn't use enough capacity, so we run some extra stuff while we're waiting for that - // to finish: utf-8 checks and generating the output from the last iteration. - // - // The reason we run 2 inputs at a time, is steps 2 and 3 are *still* not enough to soak up all - // available capacity with just one input. Running 2 at a time seems to give the CPU a good enough - // workout. - // - template - error_code json_structural_indexer::index(const uint8_t* buf, size_t len, dom_parser_implementation& parser, stage1_mode partial) noexcept { - if (simdjson_unlikely(len > parser.capacity())) { return CAPACITY; } - // We guard the rest of the code so that we can assume that len > 0 throughout. - if (len == 0) { return EMPTY; } - if (is_streaming(partial)) { - len = trim_partial_utf8(buf, len); - // If you end up with an empty window after trimming - // the partial UTF-8 bytes, then chances are good that you - // have an UTF-8 formatting error. - if (len == 0) { return UTF8_ERROR; } - } - buf_block_reader reader(buf, len); - json_structural_indexer indexer(parser.structural_indexes.get()); - - // Read all but the last block - while (reader.has_full_block()) { - indexer.step(reader.full_block(), reader); - } - // Take care of the last block (will always be there unless file is empty which is - // not supposed to happen.) - uint8_t block[STEP_SIZE]; - if (simdjson_unlikely(reader.get_remainder(block) == 0)) { return UNEXPECTED_ERROR; } - indexer.step(block, reader); - return indexer.finish(parser, reader.block_index(), len, partial); - } - - template<> - simdjson_inline void json_structural_indexer::step<128>(const uint8_t* block, buf_block_reader<128>& reader) noexcept { - simd::simd8x64 in_1(block); - simd::simd8x64 in_2(block + 64); - json_block block_1 = scanner.next(in_1); - json_block block_2 = scanner.next(in_2); - this->next(in_1, block_1, reader.block_index()); - this->next(in_2, block_2, reader.block_index() + 64); - reader.advance(); - } - - template<> - simdjson_inline void json_structural_indexer::step<64>(const uint8_t* block, buf_block_reader<64>& reader) noexcept { - simd::simd8x64 in_1(block); - json_block block_1 = scanner.next(in_1); - this->next(in_1, block_1, reader.block_index()); - reader.advance(); - } - - simdjson_inline void json_structural_indexer::next(const simd::simd8x64& in, const json_block& block, size_t idx) { - uint64_t unescaped = in.lteq(0x1F); -#if SIMDJSON_UTF8VALIDATION - checker.check_next_input(in); -#endif - indexer.write(uint32_t(idx - 64), prev_structurals); // Output *last* iteration's structurals to the parser - prev_structurals = block.structural_start(); - unescaped_chars_error |= block.non_quote_inside_string(unescaped); - } - - simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementation& parser, size_t idx, size_t len, stage1_mode partial) { - // Write out the final iteration's structurals - indexer.write(uint32_t(idx - 64), prev_structurals); - error_code error = scanner.finish(); - // We deliberately break down the next expression so that it is - // human readable. - const bool should_we_exit = is_streaming(partial) ? - ((error != SUCCESS) && (error != UNCLOSED_STRING)) // when partial we tolerate UNCLOSED_STRING - : (error != SUCCESS); // if partial is false, we must have SUCCESS - const bool have_unclosed_string = (error == UNCLOSED_STRING); - if (simdjson_unlikely(should_we_exit)) { return error; } - - if (unescaped_chars_error) { - return UNESCAPED_CHARS; - } - parser.n_structural_indexes = uint32_t(indexer.tail - parser.structural_indexes.get()); - /*** - * The On Demand API requires special padding. - * - * This is related to https://github.com/simdjson/simdjson/issues/906 - * Basically, we want to make sure that if the parsing continues beyond the last (valid) - * structural character, it quickly stops. - * Only three structural characters can be repeated without triggering an error in JSON: [,] and }. - * We repeat the padding character (at 'len'). We don't know what it is, but if the parsing - * continues, then it must be [,] or }. - * Suppose it is ] or }. We backtrack to the first character, what could it be that would - * not trigger an error? It could be ] or } but no, because you can't start a document that way. - * It can't be a comma, a colon or any simple value. So the only way we could continue is - * if the repeated character is [. But if so, the document must start with [. But if the document - * starts with [, it should end with ]. If we enforce that rule, then we would get - * ][[ which is invalid. - * - * This is illustrated with the test array_iterate_unclosed_error() on the following input: - * R"({ "a": [,,)" - **/ - parser.structural_indexes[parser.n_structural_indexes] = uint32_t(len); // used later in partial == stage1_mode::streaming_final - parser.structural_indexes[parser.n_structural_indexes + 1] = uint32_t(len); - parser.structural_indexes[parser.n_structural_indexes + 2] = 0; - parser.next_structural_index = 0; - // a valid JSON file cannot have zero structural indexes - we should have found something - if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { - return EMPTY; - } - if (simdjson_unlikely(parser.structural_indexes[parser.n_structural_indexes - 1] > len)) { - return UNEXPECTED_ERROR; - } - if (partial == stage1_mode::streaming_partial) { - // If we have an unclosed string, then the last structural - // will be the quote and we want to make sure to omit it. - if (have_unclosed_string) { - parser.n_structural_indexes--; - // a valid JSON file cannot have zero structural indexes - we should have found something - if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return CAPACITY; } - } - // We truncate the input to the end of the last complete document (or zero). - auto new_structural_indexes = find_next_document_index(parser); - if (new_structural_indexes == 0 && parser.n_structural_indexes > 0) { - if (parser.structural_indexes[0] == 0) { - // If the buffer is partial and we started at index 0 but the document is - // incomplete, it's too big to parse. - return CAPACITY; - } - else { - // It is possible that the document could be parsed, we just had a lot - // of white space. - parser.n_structural_indexes = 0; - return EMPTY; - } - } - - parser.n_structural_indexes = new_structural_indexes; - } - else if (partial == stage1_mode::streaming_final) { - if (have_unclosed_string) { parser.n_structural_indexes--; } - // We truncate the input to the end of the last complete document (or zero). - // Because partial == stage1_mode::streaming_final, it means that we may - // silently ignore trailing garbage. Though it sounds bad, we do it - // deliberately because many people who have streams of JSON documents - // will truncate them for processing. E.g., imagine that you are uncompressing - // the data from a size file or receiving it in chunks from the network. You - // may not know where exactly the last document will be. Meanwhile the - // document_stream instances allow people to know the JSON documents they are - // parsing (see the iterator.source() method). - parser.n_structural_indexes = find_next_document_index(parser); - // We store the initial n_structural_indexes so that the client can see - // whether we used truncation. If initial_n_structural_indexes == parser.n_structural_indexes, - // then this will query parser.structural_indexes[parser.n_structural_indexes] which is len, - // otherwise, it will copy some prior index. - parser.structural_indexes[parser.n_structural_indexes + 1] = parser.structural_indexes[parser.n_structural_indexes]; - // This next line is critical, do not change it unless you understand what you are - // doing. - parser.structural_indexes[parser.n_structural_indexes] = uint32_t(len); - if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { - // We tolerate an unclosed string at the very end of the stream. Indeed, users - // often load their data in bulk without being careful and they want us to ignore - // the trailing garbage. - return EMPTY; - } - } - checker.check_eof(); - return checker.errors(); - } - - } // namespace stage1 - } // unnamed namespace - } // namespace haswell -} // namespace simdjson - -// Clear CUSTOM_BIT_INDEXER so other implementations can set it if they need to. -#undef SIMDJSON_GENERIC_JSON_STRUCTURAL_INDEXER_CUSTOM_BIT_INDEXER - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRUCTURAL_INDEXER_H -/* end file generic/stage1/json_structural_indexer.h for haswell */ -/* including generic/stage1/utf8_validator.h for haswell: #include */ -/* begin file generic/stage1/utf8_validator.h for haswell */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_UTF8_VALIDATOR_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_UTF8_VALIDATOR_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - namespace haswell { - namespace { - namespace stage1 { - - /** - * Validates that the string is actual UTF-8. - */ - template - bool generic_validate_utf8(const uint8_t* input, size_t length) { - checker c{}; - buf_block_reader<64> reader(input, length); - while (reader.has_full_block()) { - simd::simd8x64 in(reader.full_block()); - c.check_next_input(in); - reader.advance(); - } - uint8_t block[64]{}; - reader.get_remainder(block); - simd::simd8x64 in(block); - c.check_next_input(in); - reader.advance(); - c.check_eof(); - return c.errors() == error_code::SUCCESS; - } - - bool generic_validate_utf8(const char* input, size_t length) { - return generic_validate_utf8(reinterpret_cast(input), length); - } - - } // namespace stage1 - } // unnamed namespace - } // namespace haswell -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_UTF8_VALIDATOR_H -/* end file generic/stage1/utf8_validator.h for haswell */ -/* end file generic/stage1/amalgamated.h for haswell */ -/* including generic/stage2/amalgamated.h for haswell: #include */ -/* begin file generic/stage2/amalgamated.h for haswell */ -// Stuff other things depend on -/* including generic/stage2/base.h for haswell: #include */ -/* begin file generic/stage2/base.h for haswell */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_BASE_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - namespace haswell { - namespace { - namespace stage2 { - - class json_iterator; - class structural_iterator; - struct tape_builder; - struct tape_writer; - - } // namespace stage2 - } // unnamed namespace - } // namespace haswell -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE2_BASE_H -/* end file generic/stage2/base.h for haswell */ -/* including generic/stage2/tape_writer.h for haswell: #include */ -/* begin file generic/stage2/tape_writer.h for haswell */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_TAPE_WRITER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_TAPE_WRITER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include - -namespace simdjson { - namespace haswell { - namespace { - namespace stage2 { - - struct tape_writer { - /** The next place to write to tape */ - uint64_t* next_tape_loc; - - /** Write a signed 64-bit value to tape. */ - simdjson_inline void append_s64(int64_t value) noexcept; - - /** Write an unsigned 64-bit value to tape. */ - simdjson_inline void append_u64(uint64_t value) noexcept; - - /** Write a double value to tape. */ - simdjson_inline void append_double(double value) noexcept; - - /** - * Append a tape entry (an 8-bit type,and 56 bits worth of value). - */ - simdjson_inline void append(uint64_t val, internal::tape_type t) noexcept; - - /** - * Skip the current tape entry without writing. - * - * Used to skip the start of the container, since we'll come back later to fill it in when the - * container ends. - */ - simdjson_inline void skip() noexcept; - - /** - * Skip the number of tape entries necessary to write a large u64 or i64. - */ - simdjson_inline void skip_large_integer() noexcept; - - /** - * Skip the number of tape entries necessary to write a double. - */ - simdjson_inline void skip_double() noexcept; - - /** - * Write a value to a known location on tape. - * - * Used to go back and write out the start of a container after the container ends. - */ - simdjson_inline static void write(uint64_t& tape_loc, uint64_t val, internal::tape_type t) noexcept; - - private: - /** - * Append both the tape entry, and a supplementary value following it. Used for types that need - * all 64 bits, such as double and uint64_t. - */ - template - simdjson_inline void append2(uint64_t val, T val2, internal::tape_type t) noexcept; - }; // struct tape_writer - - simdjson_inline void tape_writer::append_s64(int64_t value) noexcept { - append2(0, value, internal::tape_type::INT64); - } - - simdjson_inline void tape_writer::append_u64(uint64_t value) noexcept { - append(0, internal::tape_type::UINT64); - *next_tape_loc = value; - next_tape_loc++; - } - - /** Write a double value to tape. */ - simdjson_inline void tape_writer::append_double(double value) noexcept { - append2(0, value, internal::tape_type::DOUBLE); - } - - simdjson_inline void tape_writer::skip() noexcept { - next_tape_loc++; - } - - simdjson_inline void tape_writer::skip_large_integer() noexcept { - next_tape_loc += 2; - } - - simdjson_inline void tape_writer::skip_double() noexcept { - next_tape_loc += 2; - } - - simdjson_inline void tape_writer::append(uint64_t val, internal::tape_type t) noexcept { - *next_tape_loc = val | ((uint64_t(char(t))) << 56); - next_tape_loc++; - } - - template - simdjson_inline void tape_writer::append2(uint64_t val, T val2, internal::tape_type t) noexcept { - append(val, t); - static_assert(sizeof(val2) == sizeof(*next_tape_loc), "Type is not 64 bits!"); - memcpy(next_tape_loc, &val2, sizeof(val2)); - next_tape_loc++; - } - - simdjson_inline void tape_writer::write(uint64_t& tape_loc, uint64_t val, internal::tape_type t) noexcept { - tape_loc = val | ((uint64_t(char(t))) << 56); - } - - } // namespace stage2 - } // unnamed namespace - } // namespace haswell -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE2_TAPE_WRITER_H -/* end file generic/stage2/tape_writer.h for haswell */ -/* including generic/stage2/logger.h for haswell: #include */ -/* begin file generic/stage2/logger.h for haswell */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_LOGGER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_LOGGER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include - - -// This is for an internal-only stage 2 specific logger. -// Set LOG_ENABLED = true to log what stage 2 is doing! -namespace simdjson { - namespace haswell { - namespace { - namespace logger { - - static constexpr const char* DASHES = "----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------"; - -#if SIMDJSON_VERBOSE_LOGGING - static constexpr const bool LOG_ENABLED = true; -#else - static constexpr const bool LOG_ENABLED = false; -#endif - static constexpr const int LOG_EVENT_LEN = 20; - static constexpr const int LOG_BUFFER_LEN = 30; - static constexpr const int LOG_SMALL_BUFFER_LEN = 10; - static constexpr const int LOG_INDEX_LEN = 5; - - static int log_depth; // Not threadsafe. Log only. - - // Helper to turn unprintable or newline characters into spaces - static simdjson_inline char printable_char(char c) { - if (c >= 0x20) { - return c; - } - else { - return ' '; - } - } - - // Print the header and set up log_start - static simdjson_inline void log_start() { - if (LOG_ENABLED) { - log_depth = 0; - printf("\n"); - printf("| %-*s | %-*s | %-*s | %-*s | Detail |\n", LOG_EVENT_LEN, "Event", LOG_BUFFER_LEN, "Buffer", LOG_SMALL_BUFFER_LEN, "Next", 5, "Next#"); - printf("|%.*s|%.*s|%.*s|%.*s|--------|\n", LOG_EVENT_LEN + 2, DASHES, LOG_BUFFER_LEN + 2, DASHES, LOG_SMALL_BUFFER_LEN + 2, DASHES, 5 + 2, DASHES); - } - } - - simdjson_unused static simdjson_inline void log_string(const char* message) { - if (LOG_ENABLED) { - printf("%s\n", message); - } - } - - // Logs a single line from the stage 2 DOM parser - template - static simdjson_inline void log_line(S& structurals, const char* title_prefix, const char* title, const char* detail) { - if (LOG_ENABLED) { - printf("| %*s%s%-*s ", log_depth * 2, "", title_prefix, LOG_EVENT_LEN - log_depth * 2 - int(strlen(title_prefix)), title); - auto current_index = structurals.at_beginning() ? nullptr : structurals.next_structural - 1; - auto next_index = structurals.next_structural; - auto current = current_index ? &structurals.buf[*current_index] : reinterpret_cast(" "); - auto next = &structurals.buf[*next_index]; - { - // Print the next N characters in the buffer. - printf("| "); - // Otherwise, print the characters starting from the buffer position. - // Print spaces for unprintable or newline characters. - for (int i = 0; i < LOG_BUFFER_LEN; i++) { - printf("%c", printable_char(current[i])); - } - printf(" "); - // Print the next N characters in the buffer. - printf("| "); - // Otherwise, print the characters starting from the buffer position. - // Print spaces for unprintable or newline characters. - for (int i = 0; i < LOG_SMALL_BUFFER_LEN; i++) { - printf("%c", printable_char(next[i])); - } - printf(" "); - } - if (current_index) { - printf("| %*u ", LOG_INDEX_LEN, *current_index); - } - else { - printf("| %-*s ", LOG_INDEX_LEN, ""); - } - // printf("| %*u ", LOG_INDEX_LEN, structurals.next_tape_index()); - printf("| %-s ", detail); - printf("|\n"); - } - } - - } // namespace logger - } // unnamed namespace - } // namespace haswell -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE2_LOGGER_H -/* end file generic/stage2/logger.h for haswell */ - -// All other declarations -/* including generic/stage2/json_iterator.h for haswell: #include */ -/* begin file generic/stage2/json_iterator.h for haswell */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_JSON_ITERATOR_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_JSON_ITERATOR_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - namespace haswell { - namespace { - namespace stage2 { - - class json_iterator { - public: - const uint8_t* const buf; - uint32_t* next_structural; - dom_parser_implementation& dom_parser; - uint32_t depth{ 0 }; - - /** - * Walk the JSON document. - * - * The visitor receives callbacks when values are encountered. All callbacks pass the iterator as - * the first parameter; some callbacks have other parameters as well: - * - * - visit_document_start() - at the beginning. - * - visit_document_end() - at the end (if things were successful). - * - * - visit_array_start() - at the start `[` of a non-empty array. - * - visit_array_end() - at the end `]` of a non-empty array. - * - visit_empty_array() - when an empty array is encountered. - * - * - visit_object_end() - at the start `]` of a non-empty object. - * - visit_object_start() - at the end `]` of a non-empty object. - * - visit_empty_object() - when an empty object is encountered. - * - visit_key(const uint8_t *key) - when a key in an object field is encountered. key is - * guaranteed to point at the first quote of the string (`"key"`). - * - visit_primitive(const uint8_t *value) - when a value is a string, number, boolean or null. - * - visit_root_primitive(iter, uint8_t *value) - when the top-level value is a string, number, boolean or null. - * - * - increment_count(iter) - each time a value is found in an array or object. - */ - template - simdjson_warn_unused simdjson_inline error_code walk_document(V& visitor) noexcept; - - /** - * Create an iterator capable of walking a JSON document. - * - * The document must have already passed through stage 1. - */ - simdjson_inline json_iterator(dom_parser_implementation& _dom_parser, size_t start_structural_index); - - /** - * Look at the next token. - * - * Tokens can be strings, numbers, booleans, null, or operators (`[{]},:`)). - * - * They may include invalid JSON as well (such as `1.2.3` or `ture`). - */ - simdjson_inline const uint8_t* peek() const noexcept; - /** - * Advance to the next token. - * - * Tokens can be strings, numbers, booleans, null, or operators (`[{]},:`)). - * - * They may include invalid JSON as well (such as `1.2.3` or `ture`). - */ - simdjson_inline const uint8_t* advance() noexcept; - /** - * Get the remaining length of the document, from the start of the current token. - */ - simdjson_inline size_t remaining_len() const noexcept; - /** - * Check if we are at the end of the document. - * - * If this is true, there are no more tokens. - */ - simdjson_inline bool at_eof() const noexcept; - /** - * Check if we are at the beginning of the document. - */ - simdjson_inline bool at_beginning() const noexcept; - simdjson_inline uint8_t last_structural() const noexcept; - - /** - * Log that a value has been found. - * - * Set LOG_ENABLED=true in logger.h to see logging. - */ - simdjson_inline void log_value(const char* type) const noexcept; - /** - * Log the start of a multipart value. - * - * Set LOG_ENABLED=true in logger.h to see logging. - */ - simdjson_inline void log_start_value(const char* type) const noexcept; - /** - * Log the end of a multipart value. - * - * Set LOG_ENABLED=true in logger.h to see logging. - */ - simdjson_inline void log_end_value(const char* type) const noexcept; - /** - * Log an error. - * - * Set LOG_ENABLED=true in logger.h to see logging. - */ - simdjson_inline void log_error(const char* error) const noexcept; - - template - simdjson_warn_unused simdjson_inline error_code visit_root_primitive(V& visitor, const uint8_t* value) noexcept; - template - simdjson_warn_unused simdjson_inline error_code visit_primitive(V& visitor, const uint8_t* value) noexcept; - }; - - template - simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V& visitor) noexcept { - logger::log_start(); - - // - // Start the document - // - if (at_eof()) { return EMPTY; } - log_start_value("document"); - SIMDJSON_TRY(visitor.visit_document_start(*this)); - - // - // Read first value - // - { - auto value = advance(); - - // Make sure the outer object or array is closed before continuing; otherwise, there are ways we - // could get into memory corruption. See https://github.com/simdjson/simdjson/issues/906 - if (!STREAMING) { - switch (*value) { - case '{': if (last_structural() != '}') { log_value("starting brace unmatched"); return TAPE_ERROR; }; break; - case '[': if (last_structural() != ']') { log_value("starting bracket unmatched"); return TAPE_ERROR; }; break; - } - } - - switch (*value) { - case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY(visitor.visit_empty_object(*this)); break; } goto object_begin; - case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY(visitor.visit_empty_array(*this)); break; } goto array_begin; - default: SIMDJSON_TRY(visitor.visit_root_primitive(*this, value)); break; - } - } - goto document_end; - - // - // Object parser states - // - object_begin: - log_start_value("object"); - depth++; - if (depth >= dom_parser.max_depth()) { log_error("Exceeded max depth!"); return DEPTH_ERROR; } - dom_parser.is_array[depth] = false; - SIMDJSON_TRY(visitor.visit_object_start(*this)); - - { - auto key = advance(); - if (*key != '"') { log_error("Object does not start with a key"); return TAPE_ERROR; } - SIMDJSON_TRY(visitor.increment_count(*this)); - SIMDJSON_TRY(visitor.visit_key(*this, key)); - } - - object_field: - if (simdjson_unlikely(*advance() != ':')) { log_error("Missing colon after key in object"); return TAPE_ERROR; } - { - auto value = advance(); - switch (*value) { - case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY(visitor.visit_empty_object(*this)); break; } goto object_begin; - case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY(visitor.visit_empty_array(*this)); break; } goto array_begin; - default: SIMDJSON_TRY(visitor.visit_primitive(*this, value)); break; - } - } - - object_continue: - switch (*advance()) { - case ',': - SIMDJSON_TRY(visitor.increment_count(*this)); - { - auto key = advance(); - if (simdjson_unlikely(*key != '"')) { log_error("Key string missing at beginning of field in object"); return TAPE_ERROR; } - SIMDJSON_TRY(visitor.visit_key(*this, key)); - } - goto object_field; - case '}': log_end_value("object"); SIMDJSON_TRY(visitor.visit_object_end(*this)); goto scope_end; - default: log_error("No comma between object fields"); return TAPE_ERROR; - } - - scope_end: - depth--; - if (depth == 0) { goto document_end; } - if (dom_parser.is_array[depth]) { goto array_continue; } - goto object_continue; - - // - // Array parser states - // - array_begin: - log_start_value("array"); - depth++; - if (depth >= dom_parser.max_depth()) { log_error("Exceeded max depth!"); return DEPTH_ERROR; } - dom_parser.is_array[depth] = true; - SIMDJSON_TRY(visitor.visit_array_start(*this)); - SIMDJSON_TRY(visitor.increment_count(*this)); - - array_value: - { - auto value = advance(); - switch (*value) { - case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY(visitor.visit_empty_object(*this)); break; } goto object_begin; - case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY(visitor.visit_empty_array(*this)); break; } goto array_begin; - default: SIMDJSON_TRY(visitor.visit_primitive(*this, value)); break; - } - } - - array_continue: - switch (*advance()) { - case ',': SIMDJSON_TRY(visitor.increment_count(*this)); goto array_value; - case ']': log_end_value("array"); SIMDJSON_TRY(visitor.visit_array_end(*this)); goto scope_end; - default: log_error("Missing comma between array values"); return TAPE_ERROR; - } - - document_end: - log_end_value("document"); - SIMDJSON_TRY(visitor.visit_document_end(*this)); - - dom_parser.next_structural_index = uint32_t(next_structural - &dom_parser.structural_indexes[0]); - - // If we didn't make it to the end, it's an error - if (!STREAMING && dom_parser.next_structural_index != dom_parser.n_structural_indexes) { - log_error("More than one JSON value at the root of the document, or extra characters at the end of the JSON!"); - return TAPE_ERROR; - } - - return SUCCESS; - - } // walk_document() - - simdjson_inline json_iterator::json_iterator(dom_parser_implementation& _dom_parser, size_t start_structural_index) - : buf{ _dom_parser.buf }, - next_structural{ &_dom_parser.structural_indexes[start_structural_index] }, - dom_parser{ _dom_parser } { - } - - simdjson_inline const uint8_t* json_iterator::peek() const noexcept { - return &buf[*(next_structural)]; - } - simdjson_inline const uint8_t* json_iterator::advance() noexcept { - return &buf[*(next_structural++)]; - } - simdjson_inline size_t json_iterator::remaining_len() const noexcept { - return dom_parser.len - *(next_structural - 1); - } - - simdjson_inline bool json_iterator::at_eof() const noexcept { - return next_structural == &dom_parser.structural_indexes[dom_parser.n_structural_indexes]; - } - simdjson_inline bool json_iterator::at_beginning() const noexcept { - return next_structural == dom_parser.structural_indexes.get(); - } - simdjson_inline uint8_t json_iterator::last_structural() const noexcept { - return buf[dom_parser.structural_indexes[dom_parser.n_structural_indexes - 1]]; - } - - simdjson_inline void json_iterator::log_value(const char* type) const noexcept { - logger::log_line(*this, "", type, ""); - } - - simdjson_inline void json_iterator::log_start_value(const char* type) const noexcept { - logger::log_line(*this, "+", type, ""); - if (logger::LOG_ENABLED) { logger::log_depth++; } - } - - simdjson_inline void json_iterator::log_end_value(const char* type) const noexcept { - if (logger::LOG_ENABLED) { logger::log_depth--; } - logger::log_line(*this, "-", type, ""); - } - - simdjson_inline void json_iterator::log_error(const char* error) const noexcept { - logger::log_line(*this, "", "ERROR", error); - } - - template - simdjson_warn_unused simdjson_inline error_code json_iterator::visit_root_primitive(V& visitor, const uint8_t* value) noexcept { - switch (*value) { - case '"': return visitor.visit_root_string(*this, value); - case 't': return visitor.visit_root_true_atom(*this, value); - case 'f': return visitor.visit_root_false_atom(*this, value); - case 'n': return visitor.visit_root_null_atom(*this, value); - case '-': - case '0': case '1': case '2': case '3': case '4': - case '5': case '6': case '7': case '8': case '9': - return visitor.visit_root_number(*this, value); - default: - log_error("Document starts with a non-value character"); - return TAPE_ERROR; - } - } - template - simdjson_warn_unused simdjson_inline error_code json_iterator::visit_primitive(V& visitor, const uint8_t* value) noexcept { - // Use the fact that most scalars are going to be either strings or numbers. - if (*value == '"') { - return visitor.visit_string(*this, value); - } - else if (((*value - '0') < 10) || (*value == '-')) { - return visitor.visit_number(*this, value); - } - // true, false, null are uncommon. - switch (*value) { - case 't': return visitor.visit_true_atom(*this, value); - case 'f': return visitor.visit_false_atom(*this, value); - case 'n': return visitor.visit_null_atom(*this, value); - default: - log_error("Non-value found when value was expected!"); - return TAPE_ERROR; - } - } - - } // namespace stage2 - } // unnamed namespace - } // namespace haswell -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE2_JSON_ITERATOR_H -/* end file generic/stage2/json_iterator.h for haswell */ -/* including generic/stage2/stringparsing.h for haswell: #include */ -/* begin file generic/stage2/stringparsing.h for haswell */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -// This file contains the common code every implementation uses -// It is intended to be included multiple times and compiled multiple times - -namespace simdjson { - namespace haswell { - namespace { - /// @private - namespace stringparsing { - - // begin copypasta - // These chars yield themselves: " \ / - // b -> backspace, f -> formfeed, n -> newline, r -> cr, t -> horizontal tab - // u not handled in this table as it's complex - static const uint8_t escape_map[256] = { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x0. - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0x22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x2f, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x4. - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x5c, 0, 0, 0, // 0x5. - 0, 0, 0x08, 0, 0, 0, 0x0c, 0, 0, 0, 0, 0, 0, 0, 0x0a, 0, // 0x6. - 0, 0, 0x0d, 0, 0x09, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x7. - - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - }; - - // handle a unicode codepoint - // write appropriate values into dest - // src will advance 6 bytes or 12 bytes - // dest will advance a variable amount (return via pointer) - // return true if the unicode codepoint was valid - // We work in little-endian then swap at write time - simdjson_warn_unused - simdjson_inline bool handle_unicode_codepoint(const uint8_t** src_ptr, - uint8_t** dst_ptr, bool allow_replacement) { - // Use the default Unicode Character 'REPLACEMENT CHARACTER' (U+FFFD) - constexpr uint32_t substitution_code_point = 0xfffd; - // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the - // conversion isn't valid; we defer the check for this to inside the - // multilingual plane check - uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2); - *src_ptr += 6; - - // If we found a high surrogate, we must - // check for low surrogate for characters - // outside the Basic - // Multilingual Plane. - if (code_point >= 0xd800 && code_point < 0xdc00) { - const uint8_t* src_data = *src_ptr; - /* Compiler optimizations convert this to a single 16-bit load and compare on most platforms */ - if (((src_data[0] << 8) | src_data[1]) != ((static_cast ('\\') << 8) | static_cast ('u'))) { - if (!allow_replacement) { return false; } - code_point = substitution_code_point; - } - else { - uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(src_data + 2); - - // We have already checked that the high surrogate is valid and - // (code_point - 0xd800) < 1024. - // - // Check that code_point_2 is in the range 0xdc00..0xdfff - // and that code_point_2 was parsed from valid hex. - uint32_t low_bit = code_point_2 - 0xdc00; - if (low_bit >> 10) { - if (!allow_replacement) { return false; } - code_point = substitution_code_point; - } - else { - code_point = (((code_point - 0xd800) << 10) | low_bit) + 0x10000; - *src_ptr += 6; - } - - } - } - else if (code_point >= 0xdc00 && code_point <= 0xdfff) { - // If we encounter a low surrogate (not preceded by a high surrogate) - // then we have an error. - if (!allow_replacement) { return false; } - code_point = substitution_code_point; - } - size_t offset = jsoncharutils::codepoint_to_utf8(code_point, *dst_ptr); - *dst_ptr += offset; - return offset > 0; - } - - - // handle a unicode codepoint using the wobbly convention - // https://simonsapin.github.io/wtf-8/ - // write appropriate values into dest - // src will advance 6 bytes or 12 bytes - // dest will advance a variable amount (return via pointer) - // return true if the unicode codepoint was valid - // We work in little-endian then swap at write time - simdjson_warn_unused - simdjson_inline bool handle_unicode_codepoint_wobbly(const uint8_t** src_ptr, - uint8_t** dst_ptr) { - // It is not ideal that this function is nearly identical to handle_unicode_codepoint. - // - // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the - // conversion isn't valid; we defer the check for this to inside the - // multilingual plane check - uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2); - *src_ptr += 6; - // If we found a high surrogate, we must - // check for low surrogate for characters - // outside the Basic - // Multilingual Plane. - if (code_point >= 0xd800 && code_point < 0xdc00) { - const uint8_t* src_data = *src_ptr; - /* Compiler optimizations convert this to a single 16-bit load and compare on most platforms */ - if (((src_data[0] << 8) | src_data[1]) == ((static_cast ('\\') << 8) | static_cast ('u'))) { - uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(src_data + 2); - uint32_t low_bit = code_point_2 - 0xdc00; - if ((low_bit >> 10) == 0) { - code_point = - (((code_point - 0xd800) << 10) | low_bit) + 0x10000; - *src_ptr += 6; - } - } - } - - size_t offset = jsoncharutils::codepoint_to_utf8(code_point, *dst_ptr); - *dst_ptr += offset; - return offset > 0; - } - - - /** - * Unescape a valid UTF-8 string from src to dst, stopping at a final unescaped quote. There - * must be an unescaped quote terminating the string. It returns the final output - * position as pointer. In case of error (e.g., the string has bad escaped codes), - * then null_nullptrptr is returned. It is assumed that the output buffer is large - * enough. E.g., if src points at 'joe"', then dst needs to have four free bytes + - * SIMDJSON_PADDING bytes. - */ - simdjson_warn_unused simdjson_inline uint8_t* parse_string(const uint8_t* src, uint8_t* dst, bool allow_replacement) { - while (1) { - // Copy the next n bytes, and find the backslash and quote in them. - auto bs_quote = backslash_and_quote::copy_and_find(src, dst); - // If the next thing is the end quote, copy and return - if (bs_quote.has_quote_first()) { - // we encountered quotes first. Move dst to point to quotes and exit - return dst + bs_quote.quote_index(); - } - if (bs_quote.has_backslash()) { - /* find out where the backspace is */ - auto bs_dist = bs_quote.backslash_index(); - uint8_t escape_char = src[bs_dist + 1]; - /* we encountered backslash first. Handle backslash */ - if (escape_char == 'u') { - /* move src/dst up to the start; they will be further adjusted - within the unicode codepoint handling code. */ - src += bs_dist; - dst += bs_dist; - if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { - return nullptr; - } - } - else { - /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and - * write bs_dist+1 characters to output - * note this may reach beyond the part of the buffer we've actually - * seen. I think this is ok */ - uint8_t escape_result = escape_map[escape_char]; - if (escape_result == 0u) { - return nullptr; /* bogus escape value is an error */ - } - dst[bs_dist] = escape_result; - src += bs_dist + 2; - dst += bs_dist + 1; - } - } - else { - /* they are the same. Since they can't co-occur, it means we - * encountered neither. */ - src += backslash_and_quote::BYTES_PROCESSED; - dst += backslash_and_quote::BYTES_PROCESSED; - } - } - } - - simdjson_warn_unused simdjson_inline uint8_t* parse_wobbly_string(const uint8_t* src, uint8_t* dst) { - // It is not ideal that this function is nearly identical to parse_string. - while (1) { - // Copy the next n bytes, and find the backslash and quote in them. - auto bs_quote = backslash_and_quote::copy_and_find(src, dst); - // If the next thing is the end quote, copy and return - if (bs_quote.has_quote_first()) { - // we encountered quotes first. Move dst to point to quotes and exit - return dst + bs_quote.quote_index(); - } - if (bs_quote.has_backslash()) { - /* find out where the backspace is */ - auto bs_dist = bs_quote.backslash_index(); - uint8_t escape_char = src[bs_dist + 1]; - /* we encountered backslash first. Handle backslash */ - if (escape_char == 'u') { - /* move src/dst up to the start; they will be further adjusted - within the unicode codepoint handling code. */ - src += bs_dist; - dst += bs_dist; - if (!handle_unicode_codepoint_wobbly(&src, &dst)) { - return nullptr; - } - } - else { - /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and - * write bs_dist+1 characters to output - * note this may reach beyond the part of the buffer we've actually - * seen. I think this is ok */ - uint8_t escape_result = escape_map[escape_char]; - if (escape_result == 0u) { - return nullptr; /* bogus escape value is an error */ - } - dst[bs_dist] = escape_result; - src += bs_dist + 2; - dst += bs_dist + 1; - } - } - else { - /* they are the same. Since they can't co-occur, it means we - * encountered neither. */ - src += backslash_and_quote::BYTES_PROCESSED; - dst += backslash_and_quote::BYTES_PROCESSED; - } - } - } - - } // namespace stringparsing - } // unnamed namespace - } // namespace haswell -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H -/* end file generic/stage2/stringparsing.h for haswell */ -/* including generic/stage2/structural_iterator.h for haswell: #include */ -/* begin file generic/stage2/structural_iterator.h for haswell */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_STRUCTURAL_ITERATOR_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_STRUCTURAL_ITERATOR_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - namespace haswell { - namespace { - namespace stage2 { - - class structural_iterator { - public: - const uint8_t* const buf; - uint32_t* next_structural; - dom_parser_implementation& dom_parser; - - // Start a structural - simdjson_inline structural_iterator(dom_parser_implementation& _dom_parser, size_t start_structural_index) - : buf{ _dom_parser.buf }, - next_structural{ &_dom_parser.structural_indexes[start_structural_index] }, - dom_parser{ _dom_parser } { - } - // Get the buffer position of the current structural character - simdjson_inline const uint8_t* current() { - return &buf[*(next_structural - 1)]; - } - // Get the current structural character - simdjson_inline char current_char() { - return buf[*(next_structural - 1)]; - } - // Get the next structural character without advancing - simdjson_inline char peek_next_char() { - return buf[*next_structural]; - } - simdjson_inline const uint8_t* peek() { - return &buf[*next_structural]; - } - simdjson_inline const uint8_t* advance() { - return &buf[*(next_structural++)]; - } - simdjson_inline char advance_char() { - return buf[*(next_structural++)]; - } - simdjson_inline size_t remaining_len() { - return dom_parser.len - *(next_structural - 1); - } - - simdjson_inline bool at_end() { - return next_structural == &dom_parser.structural_indexes[dom_parser.n_structural_indexes]; - } - simdjson_inline bool at_beginning() { - return next_structural == dom_parser.structural_indexes.get(); - } - }; - - } // namespace stage2 - } // unnamed namespace - } // namespace haswell -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE2_STRUCTURAL_ITERATOR_H -/* end file generic/stage2/structural_iterator.h for haswell */ -/* including generic/stage2/tape_builder.h for haswell: #include */ -/* begin file generic/stage2/tape_builder.h for haswell */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_TAPE_BUILDER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_TAPE_BUILDER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - - -namespace simdjson { - namespace haswell { - namespace { - namespace stage2 { - - struct tape_builder { - template - simdjson_warn_unused static simdjson_inline error_code parse_document( - dom_parser_implementation& dom_parser, - dom::document& doc) noexcept; - - /** Called when a non-empty document starts. */ - simdjson_warn_unused simdjson_inline error_code visit_document_start(json_iterator& iter) noexcept; - /** Called when a non-empty document ends without error. */ - simdjson_warn_unused simdjson_inline error_code visit_document_end(json_iterator& iter) noexcept; - - /** Called when a non-empty array starts. */ - simdjson_warn_unused simdjson_inline error_code visit_array_start(json_iterator& iter) noexcept; - /** Called when a non-empty array ends. */ - simdjson_warn_unused simdjson_inline error_code visit_array_end(json_iterator& iter) noexcept; - /** Called when an empty array is found. */ - simdjson_warn_unused simdjson_inline error_code visit_empty_array(json_iterator& iter) noexcept; - - /** Called when a non-empty object starts. */ - simdjson_warn_unused simdjson_inline error_code visit_object_start(json_iterator& iter) noexcept; - /** - * Called when a key in a field is encountered. - * - * primitive, visit_object_start, visit_empty_object, visit_array_start, or visit_empty_array - * will be called after this with the field value. - */ - simdjson_warn_unused simdjson_inline error_code visit_key(json_iterator& iter, const uint8_t* key) noexcept; - /** Called when a non-empty object ends. */ - simdjson_warn_unused simdjson_inline error_code visit_object_end(json_iterator& iter) noexcept; - /** Called when an empty object is found. */ - simdjson_warn_unused simdjson_inline error_code visit_empty_object(json_iterator& iter) noexcept; - - /** - * Called when a string, number, boolean or null is found. - */ - simdjson_warn_unused simdjson_inline error_code visit_primitive(json_iterator& iter, const uint8_t* value) noexcept; - /** - * Called when a string, number, boolean or null is found at the top level of a document (i.e. - * when there is no array or object and the entire document is a single string, number, boolean or - * null. - * - * This is separate from primitive() because simdjson's normal primitive parsing routines assume - * there is at least one more token after the value, which is only true in an array or object. - */ - simdjson_warn_unused simdjson_inline error_code visit_root_primitive(json_iterator& iter, const uint8_t* value) noexcept; - - simdjson_warn_unused simdjson_inline error_code visit_string(json_iterator& iter, const uint8_t* value, bool key = false) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_number(json_iterator& iter, const uint8_t* value) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_true_atom(json_iterator& iter, const uint8_t* value) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_false_atom(json_iterator& iter, const uint8_t* value) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_null_atom(json_iterator& iter, const uint8_t* value) noexcept; - - simdjson_warn_unused simdjson_inline error_code visit_root_string(json_iterator& iter, const uint8_t* value) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_root_number(json_iterator& iter, const uint8_t* value) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_root_true_atom(json_iterator& iter, const uint8_t* value) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_root_false_atom(json_iterator& iter, const uint8_t* value) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_root_null_atom(json_iterator& iter, const uint8_t* value) noexcept; - - /** Called each time a new field or element in an array or object is found. */ - simdjson_warn_unused simdjson_inline error_code increment_count(json_iterator& iter) noexcept; - - /** Next location to write to tape */ - tape_writer tape; - private: - /** Next write location in the string buf for stage 2 parsing */ - uint8_t* current_string_buf_loc; - - simdjson_inline tape_builder(dom::document& doc) noexcept; - - simdjson_inline uint32_t next_tape_index(json_iterator& iter) const noexcept; - simdjson_inline void start_container(json_iterator& iter) noexcept; - simdjson_warn_unused simdjson_inline error_code end_container(json_iterator& iter, internal::tape_type start, internal::tape_type end) noexcept; - simdjson_warn_unused simdjson_inline error_code empty_container(json_iterator& iter, internal::tape_type start, internal::tape_type end) noexcept; - simdjson_inline uint8_t* on_start_string(json_iterator& iter) noexcept; - simdjson_inline void on_end_string(uint8_t* dst) noexcept; - }; // struct tape_builder - - template - simdjson_warn_unused simdjson_inline error_code tape_builder::parse_document( - dom_parser_implementation& dom_parser, - dom::document& doc) noexcept { - dom_parser.doc = &doc; - json_iterator iter(dom_parser, STREAMING ? dom_parser.next_structural_index : 0); - tape_builder builder(doc); - return iter.walk_document(builder); - } - - simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_primitive(json_iterator& iter, const uint8_t* value) noexcept { - return iter.visit_root_primitive(*this, value); - } - simdjson_warn_unused simdjson_inline error_code tape_builder::visit_primitive(json_iterator& iter, const uint8_t* value) noexcept { - return iter.visit_primitive(*this, value); - } - simdjson_warn_unused simdjson_inline error_code tape_builder::visit_empty_object(json_iterator& iter) noexcept { - return empty_container(iter, internal::tape_type::START_OBJECT, internal::tape_type::END_OBJECT); - } - simdjson_warn_unused simdjson_inline error_code tape_builder::visit_empty_array(json_iterator& iter) noexcept { - return empty_container(iter, internal::tape_type::START_ARRAY, internal::tape_type::END_ARRAY); - } - - simdjson_warn_unused simdjson_inline error_code tape_builder::visit_document_start(json_iterator& iter) noexcept { - start_container(iter); - return SUCCESS; - } - simdjson_warn_unused simdjson_inline error_code tape_builder::visit_object_start(json_iterator& iter) noexcept { - start_container(iter); - return SUCCESS; - } - simdjson_warn_unused simdjson_inline error_code tape_builder::visit_array_start(json_iterator& iter) noexcept { - start_container(iter); - return SUCCESS; - } - - simdjson_warn_unused simdjson_inline error_code tape_builder::visit_object_end(json_iterator& iter) noexcept { - return end_container(iter, internal::tape_type::START_OBJECT, internal::tape_type::END_OBJECT); - } - simdjson_warn_unused simdjson_inline error_code tape_builder::visit_array_end(json_iterator& iter) noexcept { - return end_container(iter, internal::tape_type::START_ARRAY, internal::tape_type::END_ARRAY); - } - simdjson_warn_unused simdjson_inline error_code tape_builder::visit_document_end(json_iterator& iter) noexcept { - constexpr uint32_t start_tape_index = 0; - tape.append(start_tape_index, internal::tape_type::ROOT); - tape_writer::write(iter.dom_parser.doc->tape[start_tape_index], next_tape_index(iter), internal::tape_type::ROOT); - return SUCCESS; - } - simdjson_warn_unused simdjson_inline error_code tape_builder::visit_key(json_iterator& iter, const uint8_t* key) noexcept { - return visit_string(iter, key, true); - } - - simdjson_warn_unused simdjson_inline error_code tape_builder::increment_count(json_iterator& iter) noexcept { - iter.dom_parser.open_containers[iter.depth].count++; // we have a key value pair in the object at parser.dom_parser.depth - 1 - return SUCCESS; - } - - simdjson_inline tape_builder::tape_builder(dom::document& doc) noexcept : tape{ doc.tape.get() }, current_string_buf_loc{ doc.string_buf.get() } {} - - simdjson_warn_unused simdjson_inline error_code tape_builder::visit_string(json_iterator& iter, const uint8_t* value, bool key) noexcept { - iter.log_value(key ? "key" : "string"); - uint8_t* dst = on_start_string(iter); - dst = stringparsing::parse_string(value + 1, dst, false); // We do not allow replacement when the escape characters are invalid. - if (dst == nullptr) { - iter.log_error("Invalid escape in string"); - return STRING_ERROR; - } - on_end_string(dst); - return SUCCESS; - } - - simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_string(json_iterator& iter, const uint8_t* value) noexcept { - return visit_string(iter, value); - } - - simdjson_warn_unused simdjson_inline error_code tape_builder::visit_number(json_iterator& iter, const uint8_t* value) noexcept { - iter.log_value("number"); - return numberparsing::parse_number(value, tape); - } - - simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_number(json_iterator& iter, const uint8_t* value) noexcept { - // - // We need to make a copy to make sure that the string is space terminated. - // This is not about padding the input, which should already padded up - // to len + SIMDJSON_PADDING. However, we have no control at this stage - // on how the padding was done. What if the input string was padded with nulls? - // It is quite common for an input string to have an extra null character (C string). - // We do not want to allow 9\0 (where \0 is the null character) inside a JSON - // document, but the string "9\0" by itself is fine. So we make a copy and - // pad the input with spaces when we know that there is just one input element. - // This copy is relatively expensive, but it will almost never be called in - // practice unless you are in the strange scenario where you have many JSON - // documents made of single atoms. - // - std::unique_ptrcopy(new (std::nothrow) uint8_t[iter.remaining_len() + SIMDJSON_PADDING]); - if (copy.get() == nullptr) { return MEMALLOC; } - std::memcpy(copy.get(), value, iter.remaining_len()); - std::memset(copy.get() + iter.remaining_len(), ' ', SIMDJSON_PADDING); - error_code error = visit_number(iter, copy.get()); - return error; - } - - simdjson_warn_unused simdjson_inline error_code tape_builder::visit_true_atom(json_iterator& iter, const uint8_t* value) noexcept { - iter.log_value("true"); - if (!atomparsing::is_valid_true_atom(value)) { return T_ATOM_ERROR; } - tape.append(0, internal::tape_type::TRUE_VALUE); - return SUCCESS; - } - - simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_true_atom(json_iterator& iter, const uint8_t* value) noexcept { - iter.log_value("true"); - if (!atomparsing::is_valid_true_atom(value, iter.remaining_len())) { return T_ATOM_ERROR; } - tape.append(0, internal::tape_type::TRUE_VALUE); - return SUCCESS; - } - - simdjson_warn_unused simdjson_inline error_code tape_builder::visit_false_atom(json_iterator& iter, const uint8_t* value) noexcept { - iter.log_value("false"); - if (!atomparsing::is_valid_false_atom(value)) { return F_ATOM_ERROR; } - tape.append(0, internal::tape_type::FALSE_VALUE); - return SUCCESS; - } - - simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_false_atom(json_iterator& iter, const uint8_t* value) noexcept { - iter.log_value("false"); - if (!atomparsing::is_valid_false_atom(value, iter.remaining_len())) { return F_ATOM_ERROR; } - tape.append(0, internal::tape_type::FALSE_VALUE); - return SUCCESS; - } - - simdjson_warn_unused simdjson_inline error_code tape_builder::visit_null_atom(json_iterator& iter, const uint8_t* value) noexcept { - iter.log_value("null"); - if (!atomparsing::is_valid_null_atom(value)) { return N_ATOM_ERROR; } - tape.append(0, internal::tape_type::NULL_VALUE); - return SUCCESS; - } - - simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_null_atom(json_iterator& iter, const uint8_t* value) noexcept { - iter.log_value("null"); - if (!atomparsing::is_valid_null_atom(value, iter.remaining_len())) { return N_ATOM_ERROR; } - tape.append(0, internal::tape_type::NULL_VALUE); - return SUCCESS; - } - - // private: - - simdjson_inline uint32_t tape_builder::next_tape_index(json_iterator& iter) const noexcept { - return uint32_t(tape.next_tape_loc - iter.dom_parser.doc->tape.get()); - } - - simdjson_warn_unused simdjson_inline error_code tape_builder::empty_container(json_iterator& iter, internal::tape_type start, internal::tape_type end) noexcept { - auto start_index = next_tape_index(iter); - tape.append(start_index + 2, start); - tape.append(start_index, end); - return SUCCESS; - } - - simdjson_inline void tape_builder::start_container(json_iterator& iter) noexcept { - iter.dom_parser.open_containers[iter.depth].tape_index = next_tape_index(iter); - iter.dom_parser.open_containers[iter.depth].count = 0; - tape.skip(); // We don't actually *write* the start element until the end. - } - - simdjson_warn_unused simdjson_inline error_code tape_builder::end_container(json_iterator& iter, internal::tape_type start, internal::tape_type end) noexcept { - // Write the ending tape element, pointing at the start location - const uint32_t start_tape_index = iter.dom_parser.open_containers[iter.depth].tape_index; - tape.append(start_tape_index, end); - // Write the start tape element, pointing at the end location (and including count) - // count can overflow if it exceeds 24 bits... so we saturate - // the convention being that a cnt of 0xffffff or more is undetermined in value (>= 0xffffff). - const uint32_t count = iter.dom_parser.open_containers[iter.depth].count; - const uint32_t cntsat = count > 0xFFFFFF ? 0xFFFFFF : count; - tape_writer::write(iter.dom_parser.doc->tape[start_tape_index], next_tape_index(iter) | (uint64_t(cntsat) << 32), start); - return SUCCESS; - } - - simdjson_inline uint8_t* tape_builder::on_start_string(json_iterator& iter) noexcept { - // we advance the point, accounting for the fact that we have a NULL termination - tape.append(current_string_buf_loc - iter.dom_parser.doc->string_buf.get(), internal::tape_type::STRING); - return current_string_buf_loc + sizeof(uint32_t); - } - - simdjson_inline void tape_builder::on_end_string(uint8_t* dst) noexcept { - uint32_t str_length = uint32_t(dst - (current_string_buf_loc + sizeof(uint32_t))); - // TODO check for overflow in case someone has a crazy string (>=4GB?) - // But only add the overflow check when the document itself exceeds 4GB - // Currently unneeded because we refuse to parse docs larger or equal to 4GB. - memcpy(current_string_buf_loc, &str_length, sizeof(uint32_t)); - // NULL termination is still handy if you expect all your strings to - // be NULL terminated? It comes at a small cost - *dst = 0; - current_string_buf_loc = dst + 1; - } - - } // namespace stage2 - } // unnamed namespace - } // namespace haswell -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE2_TAPE_BUILDER_H -/* end file generic/stage2/tape_builder.h for haswell */ -/* end file generic/stage2/amalgamated.h for haswell */ - -// -// Stage 1 -// - -namespace simdjson { - namespace haswell { - - simdjson_warn_unused error_code implementation::create_dom_parser_implementation( - size_t capacity, - size_t max_depth, - std::unique_ptr& dst - ) const noexcept { - dst.reset(new (std::nothrow) dom_parser_implementation()); - if (!dst) { return MEMALLOC; } - if (auto err = dst->set_capacity(capacity)) - return err; - if (auto err = dst->set_max_depth(max_depth)) - return err; - return SUCCESS; - } - - namespace { - - using namespace simd; - - // This identifies structural characters (comma, colon, braces, brackets), - // and ASCII white-space ('\r','\n','\t',' '). - simdjson_inline json_character_block json_character_block::classify(const simd::simd8x64& in) { - // These lookups rely on the fact that anything < 127 will match the lower 4 bits, which is why - // we can't use the generic lookup_16. - const auto whitespace_table = simd8::repeat_16(' ', 100, 100, 100, 17, 100, 113, 2, 100, '\t', '\n', 112, 100, '\r', 100, 100); - - // The 6 operators (:,[]{}) have these values: - // - // , 2C - // : 3A - // [ 5B - // { 7B - // ] 5D - // } 7D - // - // If you use | 0x20 to turn [ and ] into { and }, the lower 4 bits of each character is unique. - // We exploit this, using a simd 4-bit lookup to tell us which character match against, and then - // match it (against | 0x20). - // - // To prevent recognizing other characters, everything else gets compared with 0, which cannot - // match due to the | 0x20. - // - // NOTE: Due to the | 0x20, this ALSO treats and (control characters 0C and 1A) like , - // and :. This gets caught in stage 2, which checks the actual character to ensure the right - // operators are in the right places. - const auto op_table = simd8::repeat_16( - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, ':', '{', // : = 3A, [ = 5B, { = 7B - ',', '}', 0, 0 // , = 2C, ] = 5D, } = 7D - ); - - // We compute whitespace and op separately. If later code only uses one or the - // other, given the fact that all functions are aggressively inlined, we can - // hope that useless computations will be omitted. This is namely case when - // minifying (we only need whitespace). - - const uint64_t whitespace = in.eq({ - _mm256_shuffle_epi8(whitespace_table, in.chunks[0]), - _mm256_shuffle_epi8(whitespace_table, in.chunks[1]) - }); - // Turn [ and ] into { and } - const simd8x64 curlified{ - in.chunks[0] | 0x20, - in.chunks[1] | 0x20 - }; - const uint64_t op = curlified.eq({ - _mm256_shuffle_epi8(op_table, in.chunks[0]), - _mm256_shuffle_epi8(op_table, in.chunks[1]) - }); - - return { whitespace, op }; - } - - simdjson_inline bool is_ascii(const simd8x64& input) { - return input.reduce_or().is_ascii(); - } - - simdjson_unused simdjson_inline simd8 must_be_continuation(const simd8 prev1, const simd8 prev2, const simd8 prev3) { - simd8 is_second_byte = prev1.saturating_sub(0xc0u - 1); // Only 11______ will be > 0 - simd8 is_third_byte = prev2.saturating_sub(0xe0u - 1); // Only 111_____ will be > 0 - simd8 is_fourth_byte = prev3.saturating_sub(0xf0u - 1); // Only 1111____ will be > 0 - // Caller requires a bool (all 1's). All values resulting from the subtraction will be <= 64, so signed comparison is fine. - return simd8(is_second_byte | is_third_byte | is_fourth_byte) > int8_t(0); - } - - simdjson_inline simd8 must_be_2_3_continuation(const simd8 prev2, const simd8 prev3) { - simd8 is_third_byte = prev2.saturating_sub(0xe0u - 1); // Only 111_____ will be > 0 - simd8 is_fourth_byte = prev3.saturating_sub(0xf0u - 1); // Only 1111____ will be > 0 - // Caller requires a bool (all 1's). All values resulting from the subtraction will be <= 64, so signed comparison is fine. - return simd8(is_third_byte | is_fourth_byte) > int8_t(0); - } - - } // unnamed namespace - } // namespace haswell -} // namespace simdjson - -// -// Stage 2 -// - -// -// Implementation-specific overrides -// -namespace simdjson { - namespace haswell { - - simdjson_warn_unused error_code implementation::minify(const uint8_t* buf, size_t len, uint8_t* dst, size_t& dst_len) const noexcept { - return haswell::stage1::json_minifier::minify<128>(buf, len, dst, dst_len); - } - - simdjson_warn_unused error_code dom_parser_implementation::stage1(const uint8_t* _buf, size_t _len, stage1_mode streaming) noexcept { - this->buf = _buf; - this->len = _len; - return haswell::stage1::json_structural_indexer::index<128>(_buf, _len, *this, streaming); - } - - simdjson_warn_unused bool implementation::validate_utf8(const char* buf, size_t len) const noexcept { - return haswell::stage1::generic_validate_utf8(buf, len); - } - - simdjson_warn_unused error_code dom_parser_implementation::stage2(dom::document& _doc) noexcept { - return stage2::tape_builder::parse_document(*this, _doc); - } - - simdjson_warn_unused error_code dom_parser_implementation::stage2_next(dom::document& _doc) noexcept { - return stage2::tape_builder::parse_document(*this, _doc); - } - - simdjson_warn_unused uint8_t* dom_parser_implementation::parse_string(const uint8_t* src, uint8_t* dst, bool replacement_char) const noexcept { - return haswell::stringparsing::parse_string(src, dst, replacement_char); - } - - simdjson_warn_unused uint8_t* dom_parser_implementation::parse_wobbly_string(const uint8_t* src, uint8_t* dst) const noexcept { - return haswell::stringparsing::parse_wobbly_string(src, dst); - } - - simdjson_warn_unused error_code dom_parser_implementation::parse(const uint8_t* _buf, size_t _len, dom::document& _doc) noexcept { - auto error = stage1(_buf, _len, stage1_mode::regular); - if (error) { return error; } - return stage2(_doc); - } - - } // namespace haswell -} // namespace simdjson - -/* including simdjson/haswell/end.h: #include */ -/* begin file simdjson/haswell/end.h */ -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/haswell/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#if !SIMDJSON_CAN_ALWAYS_RUN_HASWELL -SIMDJSON_UNTARGET_REGION -#endif - -/* undefining SIMDJSON_IMPLEMENTATION from "haswell" */ -#undef SIMDJSON_IMPLEMENTATION -/* end file simdjson/haswell/end.h */ - -#endif // SIMDJSON_SRC_HASWELL_CPP -/* end file haswell.cpp */ -#endif -#if SIMDJSON_IMPLEMENTATION_ICELAKE -/* including icelake.cpp: #include */ -/* begin file icelake.cpp */ -#ifndef SIMDJSON_SRC_ICELAKE_CPP -#define SIMDJSON_SRC_ICELAKE_CPP - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -/* including simdjson/icelake.h: #include */ -/* begin file simdjson/icelake.h */ -#ifndef SIMDJSON_ICELAKE_H -#define SIMDJSON_ICELAKE_H - -/* including simdjson/icelake/begin.h: #include "simdjson/icelake/begin.h" */ -/* begin file simdjson/icelake/begin.h */ -/* defining SIMDJSON_IMPLEMENTATION to "icelake" */ -#define SIMDJSON_IMPLEMENTATION icelake -/* including simdjson/icelake/base.h: #include "simdjson/icelake/base.h" */ -/* begin file simdjson/icelake/base.h */ -#ifndef SIMDJSON_ICELAKE_BASE_H -#define SIMDJSON_ICELAKE_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -// The constructor may be executed on any host, so we take care not to use SIMDJSON_TARGET_ICELAKE -namespace simdjson { - /** - * Implementation for Icelake (Intel AVX512). - */ - namespace icelake { - - class implementation; - - } // namespace icelake -} // namespace simdjson - -#endif // SIMDJSON_ICELAKE_BASE_H -/* end file simdjson/icelake/base.h */ -/* including simdjson/icelake/intrinsics.h: #include "simdjson/icelake/intrinsics.h" */ -/* begin file simdjson/icelake/intrinsics.h */ -#ifndef SIMDJSON_ICELAKE_INTRINSICS_H -#define SIMDJSON_ICELAKE_INTRINSICS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/icelake/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#if SIMDJSON_VISUAL_STUDIO -// under clang within visual studio, this will include -#include // visual studio or clang -#else -#include // elsewhere -#endif // SIMDJSON_VISUAL_STUDIO - -#if SIMDJSON_CLANG_VISUAL_STUDIO -/** - * You are not supposed, normally, to include these - * headers directly. Instead you should either include intrin.h - * or x86intrin.h. However, when compiling with clang - * under Windows (i.e., when _MSC_VER is set), these headers - * only get included *if* the corresponding features are detected - * from macros: - * e.g., if __AVX2__ is set... in turn, we normally set these - * macros by compiling against the corresponding architecture - * (e.g., arch:AVX2, -mavx2, etc.) which compiles the whole - * software with these advanced instructions. In simdjson, we - * want to compile the whole program for a generic target, - * and only target our specific kernels. As a workaround, - * we directly include the needed headers. These headers would - * normally guard against such usage, but we carefully included - * (or ) before, so the headers - * are fooled. - */ -#include // for _blsr_u64 -#include // for __lzcnt64 -#include // for most things (AVX2, AVX512, _popcnt64) -#include -#include -#include -#include -#include // for _mm_clmulepi64_si128 - // Important: we need the AVX-512 headers: -#include -#include -#include -#include -#include -#include -#include -// unfortunately, we may not get _blsr_u64, but, thankfully, clang -// has it as a macro. -#ifndef _blsr_u64 -// we roll our own -#define _blsr_u64(n) ((n - 1) & n) -#endif // _blsr_u64 -#endif // SIMDJSON_CLANG_VISUAL_STUDIO - -static_assert(sizeof(__m512i) <= simdjson::SIMDJSON_PADDING, "insufficient padding for icelake"); - -#endif // SIMDJSON_ICELAKE_INTRINSICS_H -/* end file simdjson/icelake/intrinsics.h */ - -#if !SIMDJSON_CAN_ALWAYS_RUN_ICELAKE -SIMDJSON_TARGET_REGION("avx512f,avx512dq,avx512cd,avx512bw,avx512vbmi,avx512vbmi2,avx512vl,avx2,bmi,pclmul,lzcnt,popcnt") -#endif - -/* including simdjson/icelake/bitmanipulation.h: #include "simdjson/icelake/bitmanipulation.h" */ -/* begin file simdjson/icelake/bitmanipulation.h */ -#ifndef SIMDJSON_ICELAKE_BITMANIPULATION_H -#define SIMDJSON_ICELAKE_BITMANIPULATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/icelake/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/icelake/intrinsics.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - namespace icelake { - namespace { - - // We sometimes call trailing_zero on inputs that are zero, - // but the algorithms do not end up using the returned value. - // Sadly, sanitizers are not smart enough to figure it out. - SIMDJSON_NO_SANITIZE_UNDEFINED - // This function can be used safely even if not all bytes have been - // initialized. - // See issue https://github.com/simdjson/simdjson/issues/1965 - SIMDJSON_NO_SANITIZE_MEMORY - simdjson_inline int trailing_zeroes(uint64_t input_num) { -#if SIMDJSON_REGULAR_VISUAL_STUDIO - return (int)_tzcnt_u64(input_num); -#else // SIMDJSON_REGULAR_VISUAL_STUDIO - //////// - // You might expect the next line to be equivalent to - // return (int)_tzcnt_u64(input_num); - // but the generated code differs and might be less efficient? - //////// - return __builtin_ctzll(input_num); -#endif // SIMDJSON_REGULAR_VISUAL_STUDIO - } - - /* result might be undefined when input_num is zero */ - simdjson_inline uint64_t clear_lowest_bit(uint64_t input_num) { - return _blsr_u64(input_num); - } - - /* result might be undefined when input_num is zero */ - simdjson_inline int leading_zeroes(uint64_t input_num) { - return int(_lzcnt_u64(input_num)); - } - -#if SIMDJSON_REGULAR_VISUAL_STUDIO - simdjson_inline unsigned __int64 count_ones(uint64_t input_num) { - // note: we do not support legacy 32-bit Windows - return __popcnt64(input_num);// Visual Studio wants two underscores - } -#else - simdjson_inline long long int count_ones(uint64_t input_num) { - return _popcnt64(input_num); - } -#endif - - simdjson_inline bool add_overflow(uint64_t value1, uint64_t value2, - uint64_t* result) { -#if SIMDJSON_REGULAR_VISUAL_STUDIO - return _addcarry_u64(0, value1, value2, - reinterpret_cast(result)); -#else - return __builtin_uaddll_overflow(value1, value2, - reinterpret_cast(result)); -#endif - } - - } // unnamed namespace - } // namespace icelake -} // namespace simdjson - -#endif // SIMDJSON_ICELAKE_BITMANIPULATION_H -/* end file simdjson/icelake/bitmanipulation.h */ -/* including simdjson/icelake/bitmask.h: #include "simdjson/icelake/bitmask.h" */ -/* begin file simdjson/icelake/bitmask.h */ -#ifndef SIMDJSON_ICELAKE_BITMASK_H -#define SIMDJSON_ICELAKE_BITMASK_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/icelake/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/icelake/intrinsics.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - namespace icelake { - namespace { - - // - // Perform a "cumulative bitwise xor," flipping bits each time a 1 is encountered. - // - // For example, prefix_xor(00100100) == 00011100 - // - simdjson_inline uint64_t prefix_xor(const uint64_t bitmask) { - // There should be no such thing with a processor supporting avx2 - // but not clmul. - __m128i all_ones = _mm_set1_epi8('\xFF'); - __m128i result = _mm_clmulepi64_si128(_mm_set_epi64x(0ULL, bitmask), all_ones, 0); - return _mm_cvtsi128_si64(result); - } - - } // unnamed namespace - } // namespace icelake -} // namespace simdjson - -#endif // SIMDJSON_ICELAKE_BITMASK_H -/* end file simdjson/icelake/bitmask.h */ -/* including simdjson/icelake/simd.h: #include "simdjson/icelake/simd.h" */ -/* begin file simdjson/icelake/simd.h */ -#ifndef SIMDJSON_ICELAKE_SIMD_H -#define SIMDJSON_ICELAKE_SIMD_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/icelake/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/icelake/intrinsics.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/icelake/bitmanipulation.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/simdprune_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#if defined(__GNUC__) && !defined(__clang__) -#if __GNUC__ == 8 -#define SIMDJSON_GCC8 1 -#endif // __GNUC__ == 8 -#endif // defined(__GNUC__) && !defined(__clang__) - -#if SIMDJSON_GCC8 -/** - * GCC 8 fails to provide _mm512_set_epi8. We roll our own. - */ -inline __m512i _mm512_set_epi8(uint8_t a0, uint8_t a1, uint8_t a2, uint8_t a3, uint8_t a4, uint8_t a5, uint8_t a6, uint8_t a7, uint8_t a8, uint8_t a9, uint8_t a10, uint8_t a11, uint8_t a12, uint8_t a13, uint8_t a14, uint8_t a15, uint8_t a16, uint8_t a17, uint8_t a18, uint8_t a19, uint8_t a20, uint8_t a21, uint8_t a22, uint8_t a23, uint8_t a24, uint8_t a25, uint8_t a26, uint8_t a27, uint8_t a28, uint8_t a29, uint8_t a30, uint8_t a31, uint8_t a32, uint8_t a33, uint8_t a34, uint8_t a35, uint8_t a36, uint8_t a37, uint8_t a38, uint8_t a39, uint8_t a40, uint8_t a41, uint8_t a42, uint8_t a43, uint8_t a44, uint8_t a45, uint8_t a46, uint8_t a47, uint8_t a48, uint8_t a49, uint8_t a50, uint8_t a51, uint8_t a52, uint8_t a53, uint8_t a54, uint8_t a55, uint8_t a56, uint8_t a57, uint8_t a58, uint8_t a59, uint8_t a60, uint8_t a61, uint8_t a62, uint8_t a63) { - return _mm512_set_epi64(uint64_t(a7) + (uint64_t(a6) << 8) + (uint64_t(a5) << 16) + (uint64_t(a4) << 24) + (uint64_t(a3) << 32) + (uint64_t(a2) << 40) + (uint64_t(a1) << 48) + (uint64_t(a0) << 56), - uint64_t(a15) + (uint64_t(a14) << 8) + (uint64_t(a13) << 16) + (uint64_t(a12) << 24) + (uint64_t(a11) << 32) + (uint64_t(a10) << 40) + (uint64_t(a9) << 48) + (uint64_t(a8) << 56), - uint64_t(a23) + (uint64_t(a22) << 8) + (uint64_t(a21) << 16) + (uint64_t(a20) << 24) + (uint64_t(a19) << 32) + (uint64_t(a18) << 40) + (uint64_t(a17) << 48) + (uint64_t(a16) << 56), - uint64_t(a31) + (uint64_t(a30) << 8) + (uint64_t(a29) << 16) + (uint64_t(a28) << 24) + (uint64_t(a27) << 32) + (uint64_t(a26) << 40) + (uint64_t(a25) << 48) + (uint64_t(a24) << 56), - uint64_t(a39) + (uint64_t(a38) << 8) + (uint64_t(a37) << 16) + (uint64_t(a36) << 24) + (uint64_t(a35) << 32) + (uint64_t(a34) << 40) + (uint64_t(a33) << 48) + (uint64_t(a32) << 56), - uint64_t(a47) + (uint64_t(a46) << 8) + (uint64_t(a45) << 16) + (uint64_t(a44) << 24) + (uint64_t(a43) << 32) + (uint64_t(a42) << 40) + (uint64_t(a41) << 48) + (uint64_t(a40) << 56), - uint64_t(a55) + (uint64_t(a54) << 8) + (uint64_t(a53) << 16) + (uint64_t(a52) << 24) + (uint64_t(a51) << 32) + (uint64_t(a50) << 40) + (uint64_t(a49) << 48) + (uint64_t(a48) << 56), - uint64_t(a63) + (uint64_t(a62) << 8) + (uint64_t(a61) << 16) + (uint64_t(a60) << 24) + (uint64_t(a59) << 32) + (uint64_t(a58) << 40) + (uint64_t(a57) << 48) + (uint64_t(a56) << 56)); -} -#endif // SIMDJSON_GCC8 - - - -namespace simdjson { - namespace icelake { - namespace { - namespace simd { - - // Forward-declared so they can be used by splat and friends. - template - struct base { - __m512i value; - - // Zero constructor - simdjson_inline base() : value{ __m512i() } {} - - // Conversion from SIMD register - simdjson_inline base(const __m512i _value) : value(_value) {} - - // Conversion to SIMD register - simdjson_inline operator const __m512i& () const { return this->value; } - simdjson_inline operator __m512i& () { return this->value; } - - // Bit operations - simdjson_inline Child operator|(const Child other) const { return _mm512_or_si512(*this, other); } - simdjson_inline Child operator&(const Child other) const { return _mm512_and_si512(*this, other); } - simdjson_inline Child operator^(const Child other) const { return _mm512_xor_si512(*this, other); } - simdjson_inline Child bit_andnot(const Child other) const { return _mm512_andnot_si512(other, *this); } - simdjson_inline Child& operator|=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast | other; return *this_cast; } - simdjson_inline Child& operator&=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast & other; return *this_cast; } - simdjson_inline Child& operator^=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast ^ other; return *this_cast; } - }; - - // Forward-declared so they can be used by splat and friends. - template - struct simd8; - - template> - struct base8 : base> { - typedef uint32_t bitmask_t; - typedef uint64_t bitmask2_t; - - simdjson_inline base8() : base>() {} - simdjson_inline base8(const __m512i _value) : base>(_value) {} - - friend simdjson_really_inline uint64_t operator==(const simd8 lhs, const simd8 rhs) { - return _mm512_cmpeq_epi8_mask(lhs, rhs); - } - - static const int SIZE = sizeof(base::value); - - template - simdjson_inline simd8 prev(const simd8 prev_chunk) const { - // workaround for compilers unable to figure out that 16 - N is a constant (GCC 8) - constexpr int shift = 16 - N; - return _mm512_alignr_epi8(*this, _mm512_permutex2var_epi64(prev_chunk, _mm512_set_epi64(13, 12, 11, 10, 9, 8, 7, 6), *this), shift); - } - }; - - // SIMD byte mask type (returned by things like eq and gt) - template<> - struct simd8 : base8 { - static simdjson_inline simd8 splat(bool _value) { return _mm512_set1_epi8(uint8_t(-(!!_value))); } - - simdjson_inline simd8() : base8() {} - simdjson_inline simd8(const __m512i _value) : base8(_value) {} - // Splat constructor - simdjson_inline simd8(bool _value) : base8(splat(_value)) {} - simdjson_inline bool any() const { return !!_mm512_test_epi8_mask(*this, *this); } - simdjson_inline simd8 operator~() const { return *this ^ true; } - }; - - template - struct base8_numeric : base8 { - static simdjson_inline simd8 splat(T _value) { return _mm512_set1_epi8(_value); } - static simdjson_inline simd8 zero() { return _mm512_setzero_si512(); } - static simdjson_inline simd8 load(const T values[64]) { - return _mm512_loadu_si512(reinterpret_cast(values)); - } - // Repeat 16 values as many times as necessary (usually for lookup tables) - static simdjson_inline simd8 repeat_16( - T v0, T v1, T v2, T v3, T v4, T v5, T v6, T v7, - T v8, T v9, T v10, T v11, T v12, T v13, T v14, T v15 - ) { - return simd8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10, v11, v12, v13, v14, v15, - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10, v11, v12, v13, v14, v15, - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10, v11, v12, v13, v14, v15, - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10, v11, v12, v13, v14, v15 - ); - } - - simdjson_inline base8_numeric() : base8() {} - simdjson_inline base8_numeric(const __m512i _value) : base8(_value) {} - - // Store to array - simdjson_inline void store(T dst[64]) const { return _mm512_storeu_si512(reinterpret_cast<__m512i*>(dst), *this); } - - // Addition/subtraction are the same for signed and unsigned - simdjson_inline simd8 operator+(const simd8 other) const { return _mm512_add_epi8(*this, other); } - simdjson_inline simd8 operator-(const simd8 other) const { return _mm512_sub_epi8(*this, other); } - simdjson_inline simd8& operator+=(const simd8 other) { *this = *this + other; return *static_cast*>(this); } - simdjson_inline simd8& operator-=(const simd8 other) { *this = *this - other; return *static_cast*>(this); } - - // Override to distinguish from bool version - simdjson_inline simd8 operator~() const { return *this ^ 0xFFu; } - - // Perform a lookup assuming the value is between 0 and 16 (undefined behavior for out of range values) - template - simdjson_inline simd8 lookup_16(simd8 lookup_table) const { - return _mm512_shuffle_epi8(lookup_table, *this); - } - - // Copies to 'output" all bytes corresponding to a 0 in the mask (interpreted as a bitset). - // Passing a 0 value for mask would be equivalent to writing out every byte to output. - // Only the first 32 - count_ones(mask) bytes of the result are significant but 32 bytes - // get written. - // Design consideration: it seems like a function with the - // signature simd8 compress(uint32_t mask) would be - // sensible, but the AVX ISA makes this kind of approach difficult. - template - simdjson_inline void compress(uint64_t mask, L* output) const { - _mm512_mask_compressstoreu_epi8(output, ~mask, *this); - } - - template - simdjson_inline simd8 lookup_16( - L replace0, L replace1, L replace2, L replace3, - L replace4, L replace5, L replace6, L replace7, - L replace8, L replace9, L replace10, L replace11, - L replace12, L replace13, L replace14, L replace15) const { - return lookup_16(simd8::repeat_16( - replace0, replace1, replace2, replace3, - replace4, replace5, replace6, replace7, - replace8, replace9, replace10, replace11, - replace12, replace13, replace14, replace15 - )); - } - }; - - // Signed bytes - template<> - struct simd8 : base8_numeric { - simdjson_inline simd8() : base8_numeric() {} - simdjson_inline simd8(const __m512i _value) : base8_numeric(_value) {} - // Splat constructor - simdjson_inline simd8(int8_t _value) : simd8(splat(_value)) {} - // Array constructor - simdjson_inline simd8(const int8_t values[64]) : simd8(load(values)) {} - // Member-by-member initialization - simdjson_inline simd8( - int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, - int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15, - int8_t v16, int8_t v17, int8_t v18, int8_t v19, int8_t v20, int8_t v21, int8_t v22, int8_t v23, - int8_t v24, int8_t v25, int8_t v26, int8_t v27, int8_t v28, int8_t v29, int8_t v30, int8_t v31, - int8_t v32, int8_t v33, int8_t v34, int8_t v35, int8_t v36, int8_t v37, int8_t v38, int8_t v39, - int8_t v40, int8_t v41, int8_t v42, int8_t v43, int8_t v44, int8_t v45, int8_t v46, int8_t v47, - int8_t v48, int8_t v49, int8_t v50, int8_t v51, int8_t v52, int8_t v53, int8_t v54, int8_t v55, - int8_t v56, int8_t v57, int8_t v58, int8_t v59, int8_t v60, int8_t v61, int8_t v62, int8_t v63 - ) : simd8(_mm512_set_epi8( - v63, v62, v61, v60, v59, v58, v57, v56, - v55, v54, v53, v52, v51, v50, v49, v48, - v47, v46, v45, v44, v43, v42, v41, v40, - v39, v38, v37, v36, v35, v34, v33, v32, - v31, v30, v29, v28, v27, v26, v25, v24, - v23, v22, v21, v20, v19, v18, v17, v16, - v15, v14, v13, v12, v11, v10, v9, v8, - v7, v6, v5, v4, v3, v2, v1, v0 - )) {} - - // Repeat 16 values as many times as necessary (usually for lookup tables) - simdjson_inline static simd8 repeat_16( - int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, - int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 - ) { - return simd8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10, v11, v12, v13, v14, v15, - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10, v11, v12, v13, v14, v15, - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10, v11, v12, v13, v14, v15, - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10, v11, v12, v13, v14, v15 - ); - } - - // Order-sensitive comparisons - simdjson_inline simd8 max_val(const simd8 other) const { return _mm512_max_epi8(*this, other); } - simdjson_inline simd8 min_val(const simd8 other) const { return _mm512_min_epi8(*this, other); } - - simdjson_inline simd8 operator>(const simd8 other) const { return _mm512_maskz_abs_epi8(_mm512_cmpgt_epi8_mask(*this, other), _mm512_set1_epi8(uint8_t(0x80))); } - simdjson_inline simd8 operator<(const simd8 other) const { return _mm512_maskz_abs_epi8(_mm512_cmpgt_epi8_mask(other, *this), _mm512_set1_epi8(uint8_t(0x80))); } - }; - - // Unsigned bytes - template<> - struct simd8 : base8_numeric { - simdjson_inline simd8() : base8_numeric() {} - simdjson_inline simd8(const __m512i _value) : base8_numeric(_value) {} - // Splat constructor - simdjson_inline simd8(uint8_t _value) : simd8(splat(_value)) {} - // Array constructor - simdjson_inline simd8(const uint8_t values[64]) : simd8(load(values)) {} - // Member-by-member initialization - simdjson_inline simd8( - uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, - uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15, - uint8_t v16, uint8_t v17, uint8_t v18, uint8_t v19, uint8_t v20, uint8_t v21, uint8_t v22, uint8_t v23, - uint8_t v24, uint8_t v25, uint8_t v26, uint8_t v27, uint8_t v28, uint8_t v29, uint8_t v30, uint8_t v31, - uint8_t v32, uint8_t v33, uint8_t v34, uint8_t v35, uint8_t v36, uint8_t v37, uint8_t v38, uint8_t v39, - uint8_t v40, uint8_t v41, uint8_t v42, uint8_t v43, uint8_t v44, uint8_t v45, uint8_t v46, uint8_t v47, - uint8_t v48, uint8_t v49, uint8_t v50, uint8_t v51, uint8_t v52, uint8_t v53, uint8_t v54, uint8_t v55, - uint8_t v56, uint8_t v57, uint8_t v58, uint8_t v59, uint8_t v60, uint8_t v61, uint8_t v62, uint8_t v63 - ) : simd8(_mm512_set_epi8( - v63, v62, v61, v60, v59, v58, v57, v56, - v55, v54, v53, v52, v51, v50, v49, v48, - v47, v46, v45, v44, v43, v42, v41, v40, - v39, v38, v37, v36, v35, v34, v33, v32, - v31, v30, v29, v28, v27, v26, v25, v24, - v23, v22, v21, v20, v19, v18, v17, v16, - v15, v14, v13, v12, v11, v10, v9, v8, - v7, v6, v5, v4, v3, v2, v1, v0 - )) {} - - // Repeat 16 values as many times as necessary (usually for lookup tables) - simdjson_inline static simd8 repeat_16( - uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, - uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 - ) { - return simd8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10, v11, v12, v13, v14, v15, - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10, v11, v12, v13, v14, v15, - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10, v11, v12, v13, v14, v15, - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10, v11, v12, v13, v14, v15 - ); - } - - // Saturated math - simdjson_inline simd8 saturating_add(const simd8 other) const { return _mm512_adds_epu8(*this, other); } - simdjson_inline simd8 saturating_sub(const simd8 other) const { return _mm512_subs_epu8(*this, other); } - - // Order-specific operations - simdjson_inline simd8 max_val(const simd8 other) const { return _mm512_max_epu8(*this, other); } - simdjson_inline simd8 min_val(const simd8 other) const { return _mm512_min_epu8(other, *this); } - // Same as >, but only guarantees true is nonzero (< guarantees true = -1) - simdjson_inline simd8 gt_bits(const simd8 other) const { return this->saturating_sub(other); } - // Same as <, but only guarantees true is nonzero (< guarantees true = -1) - simdjson_inline simd8 lt_bits(const simd8 other) const { return other.saturating_sub(*this); } - simdjson_inline uint64_t operator<=(const simd8 other) const { return other.max_val(*this) == other; } - simdjson_inline uint64_t operator>=(const simd8 other) const { return other.min_val(*this) == other; } - simdjson_inline simd8 operator>(const simd8 other) const { return this->gt_bits(other).any_bits_set(); } - simdjson_inline simd8 operator<(const simd8 other) const { return this->lt_bits(other).any_bits_set(); } - - // Bit-specific operations - simdjson_inline simd8 bits_not_set() const { return _mm512_mask_blend_epi8(*this == uint8_t(0), _mm512_set1_epi8(0), _mm512_set1_epi8(-1)); } - simdjson_inline simd8 bits_not_set(simd8 bits) const { return (*this & bits).bits_not_set(); } - simdjson_inline simd8 any_bits_set() const { return ~this->bits_not_set(); } - simdjson_inline simd8 any_bits_set(simd8 bits) const { return ~this->bits_not_set(bits); } - - simdjson_inline bool is_ascii() const { return _mm512_movepi8_mask(*this) == 0; } - simdjson_inline bool bits_not_set_anywhere() const { - return !_mm512_test_epi8_mask(*this, *this); - } - simdjson_inline bool any_bits_set_anywhere() const { return !bits_not_set_anywhere(); } - simdjson_inline bool bits_not_set_anywhere(simd8 bits) const { return !_mm512_test_epi8_mask(*this, bits); } - simdjson_inline bool any_bits_set_anywhere(simd8 bits) const { return !bits_not_set_anywhere(bits); } - template - simdjson_inline simd8 shr() const { return simd8(_mm512_srli_epi16(*this, N)) & uint8_t(0xFFu >> N); } - template - simdjson_inline simd8 shl() const { return simd8(_mm512_slli_epi16(*this, N)) & uint8_t(0xFFu << N); } - // Get one of the bits and make a bitmask out of it. - // e.g. value.get_bit<7>() gets the high bit - template - simdjson_inline uint64_t get_bit() const { return _mm512_movepi8_mask(_mm512_slli_epi16(*this, 7 - N)); } - }; - - template - struct simd8x64 { - static constexpr int NUM_CHUNKS = 64 / sizeof(simd8); - static_assert(NUM_CHUNKS == 1, "Icelake kernel should use one register per 64-byte block."); - const simd8 chunks[NUM_CHUNKS]; - - simd8x64(const simd8x64& o) = delete; // no copy allowed - simd8x64& operator=(const simd8& other) = delete; // no assignment allowed - simd8x64() = delete; // no default constructor allowed - - simdjson_inline simd8x64(const simd8 chunk0, const simd8 chunk1) : chunks{ chunk0, chunk1 } {} - simdjson_inline simd8x64(const simd8 chunk0) : chunks{ chunk0 } {} - simdjson_inline simd8x64(const T ptr[64]) : chunks{ simd8::load(ptr) } {} - - simdjson_inline uint64_t compress(uint64_t mask, T* output) const { - this->chunks[0].compress(mask, output); - return 64 - count_ones(mask); - } - - simdjson_inline void store(T ptr[64]) const { - this->chunks[0].store(ptr + sizeof(simd8) * 0); - } - - simdjson_inline simd8 reduce_or() const { - return this->chunks[0]; - } - - simdjson_inline simd8x64 bit_or(const T m) const { - const simd8 mask = simd8::splat(m); - return simd8x64( - this->chunks[0] | mask - ); - } - - simdjson_inline uint64_t eq(const T m) const { - const simd8 mask = simd8::splat(m); - return this->chunks[0] == mask; - } - - simdjson_inline uint64_t eq(const simd8x64& other) const { - return this->chunks[0] == other.chunks[0]; - } - - simdjson_inline uint64_t lteq(const T m) const { - const simd8 mask = simd8::splat(m); - return this->chunks[0] <= mask; - } - }; // struct simd8x64 - - } // namespace simd - - } // unnamed namespace - } // namespace icelake -} // namespace simdjson - -#endif // SIMDJSON_ICELAKE_SIMD_H -/* end file simdjson/icelake/simd.h */ -/* including simdjson/icelake/stringparsing_defs.h: #include "simdjson/icelake/stringparsing_defs.h" */ -/* begin file simdjson/icelake/stringparsing_defs.h */ -#ifndef SIMDJSON_ICELAKE_STRINGPARSING_DEFS_H -#define SIMDJSON_ICELAKE_STRINGPARSING_DEFS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/icelake/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/icelake/simd.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/icelake/bitmanipulation.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - namespace icelake { - namespace { - - using namespace simd; - - // Holds backslashes and quotes locations. - struct backslash_and_quote { - public: - static constexpr uint32_t BYTES_PROCESSED = 64; - simdjson_inline static backslash_and_quote copy_and_find(const uint8_t* src, uint8_t* dst); - - simdjson_inline bool has_quote_first() { return ((bs_bits - 1) & quote_bits) != 0; } - simdjson_inline bool has_backslash() { return ((quote_bits - 1) & bs_bits) != 0; } - simdjson_inline int quote_index() { return trailing_zeroes(quote_bits); } - simdjson_inline int backslash_index() { return trailing_zeroes(bs_bits); } - - uint64_t bs_bits; - uint64_t quote_bits; - }; // struct backslash_and_quote - - simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8_t* src, uint8_t* dst) { - // this can read up to 15 bytes beyond the buffer size, but we require - // SIMDJSON_PADDING of padding - static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "backslash and quote finder must process fewer than SIMDJSON_PADDING bytes"); - simd8 v(src); - // store to dest unconditionally - we can overwrite the bits we don't like later - v.store(dst); - return { - static_cast(v == '\\'), // bs_bits - static_cast(v == '"'), // quote_bits - }; - } - - } // unnamed namespace - } // namespace icelake -} // namespace simdjson - -#endif // SIMDJSON_ICELAKE_STRINGPARSING_DEFS_H -/* end file simdjson/icelake/stringparsing_defs.h */ -/* including simdjson/icelake/numberparsing_defs.h: #include "simdjson/icelake/numberparsing_defs.h" */ -/* begin file simdjson/icelake/numberparsing_defs.h */ -#ifndef SIMDJSON_ICELAKE_NUMBERPARSING_DEFS_H -#define SIMDJSON_ICELAKE_NUMBERPARSING_DEFS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/icelake/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/icelake/intrinsics.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - namespace icelake { - namespace numberparsing { - - static simdjson_inline uint32_t parse_eight_digits_unrolled(const uint8_t* chars) { - // this actually computes *16* values so we are being wasteful. - const __m128i ascii0 = _mm_set1_epi8('0'); - const __m128i mul_1_10 = - _mm_setr_epi8(10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1); - const __m128i mul_1_100 = _mm_setr_epi16(100, 1, 100, 1, 100, 1, 100, 1); - const __m128i mul_1_10000 = - _mm_setr_epi16(10000, 1, 10000, 1, 10000, 1, 10000, 1); - const __m128i input = _mm_sub_epi8( - _mm_loadu_si128(reinterpret_cast(chars)), ascii0); - const __m128i t1 = _mm_maddubs_epi16(input, mul_1_10); - const __m128i t2 = _mm_madd_epi16(t1, mul_1_100); - const __m128i t3 = _mm_packus_epi32(t2, t2); - const __m128i t4 = _mm_madd_epi16(t3, mul_1_10000); - return _mm_cvtsi128_si32( - t4); // only captures the sum of the first 8 digits, drop the rest - } - - /** @private */ - simdjson_inline internal::value128 full_multiplication(uint64_t value1, uint64_t value2) { - internal::value128 answer; -#if SIMDJSON_REGULAR_VISUAL_STUDIO || SIMDJSON_IS_32BITS -#ifdef _M_ARM64 - // ARM64 has native support for 64-bit multiplications, no need to emultate - answer.high = __umulh(value1, value2); - answer.low = value1 * value2; -#else - answer.low = _umul128(value1, value2, &answer.high); // _umul128 not available on ARM64 -#endif // _M_ARM64 -#else // SIMDJSON_REGULAR_VISUAL_STUDIO || SIMDJSON_IS_32BITS - __uint128_t r = (static_cast<__uint128_t>(value1)) * value2; - answer.low = uint64_t(r); - answer.high = uint64_t(r >> 64); -#endif - return answer; - } - - } // namespace numberparsing - } // namespace icelake -} // namespace simdjson - -#define SIMDJSON_SWAR_NUMBER_PARSING 1 - -#endif // SIMDJSON_ICELAKE_NUMBERPARSING_DEFS_H -/* end file simdjson/icelake/numberparsing_defs.h */ -/* end file simdjson/icelake/begin.h */ -/* including simdjson/generic/amalgamated.h for icelake: #include "simdjson/generic/amalgamated.h" */ -/* begin file simdjson/generic/amalgamated.h for icelake */ -#if defined(SIMDJSON_CONDITIONAL_INCLUDE) && !defined(SIMDJSON_GENERIC_DEPENDENCIES_H) -#error simdjson/generic/dependencies.h must be included before simdjson/generic/amalgamated.h! -#endif - -/* including simdjson/generic/base.h for icelake: #include "simdjson/generic/base.h" */ -/* begin file simdjson/generic/base.h for icelake */ -#ifndef SIMDJSON_GENERIC_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_BASE_H */ -/* amalgamation skipped (editor-only): #include "simdjson/base.h" */ -/* amalgamation skipped (editor-only): // If we haven't got an implementation yet, we're in the editor, editing a generic file! Just */ -/* amalgamation skipped (editor-only): // use the most advanced one we can so the most possible stuff can be tested. */ -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_IMPLEMENTATION */ -/* amalgamation skipped (editor-only): #include "simdjson/implementation_detection.h" */ -/* amalgamation skipped (editor-only): #if SIMDJSON_IMPLEMENTATION_ICELAKE */ -/* amalgamation skipped (editor-only): #include "simdjson/icelake/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_HASWELL */ -/* amalgamation skipped (editor-only): #include "simdjson/haswell/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_WESTMERE */ -/* amalgamation skipped (editor-only): #include "simdjson/westmere/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_ARM64 */ -/* amalgamation skipped (editor-only): #include "simdjson/arm64/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_PPC64 */ -/* amalgamation skipped (editor-only): #include "simdjson/ppc64/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_FALLBACK */ -/* amalgamation skipped (editor-only): #include "simdjson/fallback/begin.h" */ -/* amalgamation skipped (editor-only): #else */ -/* amalgamation skipped (editor-only): #error "All possible implementations (including fallback) have been disabled! simdjson will not run." */ -/* amalgamation skipped (editor-only): #endif */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_IMPLEMENTATION */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - namespace icelake { - - struct open_container; - class dom_parser_implementation; - - /** - * The type of a JSON number - */ - enum class number_type { - floating_point_number = 1, /// a binary64 number - signed_integer, /// a signed integer that fits in a 64-bit word using two's complement - unsigned_integer /// a positive integer larger or equal to 1<<63 - }; - - } // namespace icelake -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_BASE_H -/* end file simdjson/generic/base.h for icelake */ -/* including simdjson/generic/jsoncharutils.h for icelake: #include "simdjson/generic/jsoncharutils.h" */ -/* begin file simdjson/generic/jsoncharutils.h for icelake */ -#ifndef SIMDJSON_GENERIC_JSONCHARUTILS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_JSONCHARUTILS_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/jsoncharutils_tables.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - namespace icelake { - namespace { - namespace jsoncharutils { - - // return non-zero if not a structural or whitespace char - // zero otherwise - simdjson_inline uint32_t is_not_structural_or_whitespace(uint8_t c) { - return internal::structural_or_whitespace_negated[c]; - } - - simdjson_inline uint32_t is_structural_or_whitespace(uint8_t c) { - return internal::structural_or_whitespace[c]; - } - - // returns a value with the high 16 bits set if not valid - // otherwise returns the conversion of the 4 hex digits at src into the bottom - // 16 bits of the 32-bit return register - // - // see - // https://lemire.me/blog/2019/04/17/parsing-short-hexadecimal-strings-efficiently/ - static inline uint32_t hex_to_u32_nocheck( - const uint8_t* src) { // strictly speaking, static inline is a C-ism - uint32_t v1 = internal::digit_to_val32[630 + src[0]]; - uint32_t v2 = internal::digit_to_val32[420 + src[1]]; - uint32_t v3 = internal::digit_to_val32[210 + src[2]]; - uint32_t v4 = internal::digit_to_val32[0 + src[3]]; - return v1 | v2 | v3 | v4; - } - - // given a code point cp, writes to c - // the utf-8 code, outputting the length in - // bytes, if the length is zero, the code point - // is invalid - // - // This can possibly be made faster using pdep - // and clz and table lookups, but JSON documents - // have few escaped code points, and the following - // function looks cheap. - // - // Note: we assume that surrogates are treated separately - // - simdjson_inline size_t codepoint_to_utf8(uint32_t cp, uint8_t* c) { - if (cp <= 0x7F) { - c[0] = uint8_t(cp); - return 1; // ascii - } - if (cp <= 0x7FF) { - c[0] = uint8_t((cp >> 6) + 192); - c[1] = uint8_t((cp & 63) + 128); - return 2; // universal plane - // Surrogates are treated elsewhere... - //} //else if (0xd800 <= cp && cp <= 0xdfff) { - // return 0; // surrogates // could put assert here - } - else if (cp <= 0xFFFF) { - c[0] = uint8_t((cp >> 12) + 224); - c[1] = uint8_t(((cp >> 6) & 63) + 128); - c[2] = uint8_t((cp & 63) + 128); - return 3; - } - else if (cp <= 0x10FFFF) { // if you know you have a valid code point, this - // is not needed - c[0] = uint8_t((cp >> 18) + 240); - c[1] = uint8_t(((cp >> 12) & 63) + 128); - c[2] = uint8_t(((cp >> 6) & 63) + 128); - c[3] = uint8_t((cp & 63) + 128); - return 4; - } - // will return 0 when the code point was too large. - return 0; // bad r - } - -#if SIMDJSON_IS_32BITS // _umul128 for x86, arm - // this is a slow emulation routine for 32-bit - // - static simdjson_inline uint64_t __emulu(uint32_t x, uint32_t y) { - return x * (uint64_t)y; - } - static simdjson_inline uint64_t _umul128(uint64_t ab, uint64_t cd, uint64_t* hi) { - uint64_t ad = __emulu((uint32_t)(ab >> 32), (uint32_t)cd); - uint64_t bd = __emulu((uint32_t)ab, (uint32_t)cd); - uint64_t adbc = ad + __emulu((uint32_t)ab, (uint32_t)(cd >> 32)); - uint64_t adbc_carry = !!(adbc < ad); - uint64_t lo = bd + (adbc << 32); - *hi = __emulu((uint32_t)(ab >> 32), (uint32_t)(cd >> 32)) + (adbc >> 32) + - (adbc_carry << 32) + !!(lo < bd); - return lo; - } -#endif - - } // namespace jsoncharutils - } // unnamed namespace - } // namespace icelake -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_JSONCHARUTILS_H -/* end file simdjson/generic/jsoncharutils.h for icelake */ -/* including simdjson/generic/atomparsing.h for icelake: #include "simdjson/generic/atomparsing.h" */ -/* begin file simdjson/generic/atomparsing.h for icelake */ -#ifndef SIMDJSON_GENERIC_ATOMPARSING_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_ATOMPARSING_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/jsoncharutils.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include - -namespace simdjson { - namespace icelake { - namespace { - /// @private - namespace atomparsing { - - // The string_to_uint32 is exclusively used to map literal strings to 32-bit values. - // We use memcpy instead of a pointer cast to avoid undefined behaviors since we cannot - // be certain that the character pointer will be properly aligned. - // You might think that using memcpy makes this function expensive, but you'd be wrong. - // All decent optimizing compilers (GCC, clang, Visual Studio) will compile string_to_uint32("false"); - // to the compile-time constant 1936482662. - simdjson_inline uint32_t string_to_uint32(const char* str) { uint32_t val; std::memcpy(&val, str, sizeof(uint32_t)); return val; } - - - // Again in str4ncmp we use a memcpy to avoid undefined behavior. The memcpy may appear expensive. - // Yet all decent optimizing compilers will compile memcpy to a single instruction, just about. - simdjson_warn_unused - simdjson_inline uint32_t str4ncmp(const uint8_t* src, const char* atom) { - uint32_t srcval; // we want to avoid unaligned 32-bit loads (undefined in C/C++) - static_assert(sizeof(uint32_t) <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be larger than 4 bytes"); - std::memcpy(&srcval, src, sizeof(uint32_t)); - return srcval ^ string_to_uint32(atom); - } - - simdjson_warn_unused - simdjson_inline bool is_valid_true_atom(const uint8_t* src) { - return (str4ncmp(src, "true") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; - } - - simdjson_warn_unused - simdjson_inline bool is_valid_true_atom(const uint8_t* src, size_t len) { - if (len > 4) { return is_valid_true_atom(src); } - else if (len == 4) { return !str4ncmp(src, "true"); } - else { return false; } - } - - simdjson_warn_unused - simdjson_inline bool is_valid_false_atom(const uint8_t* src) { - return (str4ncmp(src + 1, "alse") | jsoncharutils::is_not_structural_or_whitespace(src[5])) == 0; - } - - simdjson_warn_unused - simdjson_inline bool is_valid_false_atom(const uint8_t* src, size_t len) { - if (len > 5) { return is_valid_false_atom(src); } - else if (len == 5) { return !str4ncmp(src + 1, "alse"); } - else { return false; } - } - - simdjson_warn_unused - simdjson_inline bool is_valid_null_atom(const uint8_t* src) { - return (str4ncmp(src, "null") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; - } - - simdjson_warn_unused - simdjson_inline bool is_valid_null_atom(const uint8_t* src, size_t len) { - if (len > 4) { return is_valid_null_atom(src); } - else if (len == 4) { return !str4ncmp(src, "null"); } - else { return false; } - } - - } // namespace atomparsing - } // unnamed namespace - } // namespace icelake -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_ATOMPARSING_H -/* end file simdjson/generic/atomparsing.h for icelake */ -/* including simdjson/generic/dom_parser_implementation.h for icelake: #include "simdjson/generic/dom_parser_implementation.h" */ -/* begin file simdjson/generic/dom_parser_implementation.h for icelake */ -#ifndef SIMDJSON_GENERIC_DOM_PARSER_IMPLEMENTATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_DOM_PARSER_IMPLEMENTATION_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/dom_parser_implementation.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - namespace icelake { - - // expectation: sizeof(open_container) = 64/8. - struct open_container { - uint32_t tape_index; // where, on the tape, does the scope ([,{) begins - uint32_t count; // how many elements in the scope - }; // struct open_container - - static_assert(sizeof(open_container) == 64 / 8, "Open container must be 64 bits"); - - class dom_parser_implementation final : public internal::dom_parser_implementation { - public: - /** Tape location of each open { or [ */ - std::unique_ptr open_containers{}; - /** Whether each open container is a [ or { */ - std::unique_ptr is_array{}; - /** Buffer passed to stage 1 */ - const uint8_t* buf{}; - /** Length passed to stage 1 */ - size_t len{ 0 }; - /** Document passed to stage 2 */ - dom::document* doc{}; - - inline dom_parser_implementation() noexcept; - inline dom_parser_implementation(dom_parser_implementation&& other) noexcept; - inline dom_parser_implementation& operator=(dom_parser_implementation&& other) noexcept; - dom_parser_implementation(const dom_parser_implementation&) = delete; - dom_parser_implementation& operator=(const dom_parser_implementation&) = delete; - - simdjson_warn_unused error_code parse(const uint8_t* buf, size_t len, dom::document& doc) noexcept final; - simdjson_warn_unused error_code stage1(const uint8_t* buf, size_t len, stage1_mode partial) noexcept final; - simdjson_warn_unused error_code stage2(dom::document& doc) noexcept final; - simdjson_warn_unused error_code stage2_next(dom::document& doc) noexcept final; - simdjson_warn_unused uint8_t* parse_string(const uint8_t* src, uint8_t* dst, bool allow_replacement) const noexcept final; - simdjson_warn_unused uint8_t* parse_wobbly_string(const uint8_t* src, uint8_t* dst) const noexcept final; - inline simdjson_warn_unused error_code set_capacity(size_t capacity) noexcept final; - inline simdjson_warn_unused error_code set_max_depth(size_t max_depth) noexcept final; - private: - simdjson_inline simdjson_warn_unused error_code set_capacity_stage1(size_t capacity); - - }; - - } // namespace icelake -} // namespace simdjson - -namespace simdjson { - namespace icelake { - - inline dom_parser_implementation::dom_parser_implementation() noexcept = default; - inline dom_parser_implementation::dom_parser_implementation(dom_parser_implementation&& other) noexcept = default; - inline dom_parser_implementation& dom_parser_implementation::operator=(dom_parser_implementation&& other) noexcept = default; - - // Leaving these here so they can be inlined if so desired - inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(size_t capacity) noexcept { - if (capacity > SIMDJSON_MAXSIZE_BYTES) { return CAPACITY; } - // Stage 1 index output - size_t max_structures = SIMDJSON_ROUNDUP_N(capacity, 64) + 2 + 7; - structural_indexes.reset(new (std::nothrow) uint32_t[max_structures]); - if (!structural_indexes) { _capacity = 0; return MEMALLOC; } - structural_indexes[0] = 0; - n_structural_indexes = 0; - - _capacity = capacity; - return SUCCESS; - } - - inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { - // Stage 2 stacks - open_containers.reset(new (std::nothrow) open_container[max_depth]); - is_array.reset(new (std::nothrow) bool[max_depth]); - if (!is_array || !open_containers) { _max_depth = 0; return MEMALLOC; } - - _max_depth = max_depth; - return SUCCESS; - } - - } // namespace icelake -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_DOM_PARSER_IMPLEMENTATION_H -/* end file simdjson/generic/dom_parser_implementation.h for icelake */ -/* including simdjson/generic/implementation_simdjson_result_base.h for icelake: #include "simdjson/generic/implementation_simdjson_result_base.h" */ -/* begin file simdjson/generic/implementation_simdjson_result_base.h for icelake */ -#ifndef SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - namespace icelake { - - // This is a near copy of include/error.h's implementation_simdjson_result_base, except it doesn't use std::pair - // so we can avoid inlining errors - // TODO reconcile these! - /** - * The result of a simdjson operation that could fail. - * - * Gives the option of reading error codes, or throwing an exception by casting to the desired result. - * - * This is a base class for implementations that want to add functions to the result type for - * chaining. - * - * Override like: - * - * struct simdjson_result : public internal::implementation_simdjson_result_base { - * simdjson_result() noexcept : internal::implementation_simdjson_result_base() {} - * simdjson_result(error_code error) noexcept : internal::implementation_simdjson_result_base(error) {} - * simdjson_result(T &&value) noexcept : internal::implementation_simdjson_result_base(std::forward(value)) {} - * simdjson_result(T &&value, error_code error) noexcept : internal::implementation_simdjson_result_base(value, error) {} - * // Your extra methods here - * } - * - * Then any method returning simdjson_result will be chainable with your methods. - */ - template - struct implementation_simdjson_result_base { - - /** - * Create a new empty result with error = UNINITIALIZED. - */ - simdjson_inline implementation_simdjson_result_base() noexcept = default; - - /** - * Create a new error result. - */ - simdjson_inline implementation_simdjson_result_base(error_code error) noexcept; - - /** - * Create a new successful result. - */ - simdjson_inline implementation_simdjson_result_base(T&& value) noexcept; - - /** - * Create a new result with both things (use if you don't want to branch when creating the result). - */ - simdjson_inline implementation_simdjson_result_base(T&& value, error_code error) noexcept; - - /** - * Move the value and the error to the provided variables. - * - * @param value The variable to assign the value to. May not be set if there is an error. - * @param error The variable to assign the error to. Set to SUCCESS if there is no error. - */ - simdjson_inline void tie(T& value, error_code& error) && noexcept; - - /** - * Move the value to the provided variable. - * - * @param value The variable to assign the value to. May not be set if there is an error. - */ - simdjson_inline error_code get(T& value) && noexcept; - - /** - * The error. - */ - simdjson_inline error_code error() const noexcept; - -#if SIMDJSON_EXCEPTIONS - - /** - * Get the result value. - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline T& value() & noexcept(false); - - /** - * Take the result value (move it). - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline T&& value() && noexcept(false); - - /** - * Take the result value (move it). - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline T&& take_value() && noexcept(false); - - /** - * Cast to the value (will throw on error). - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline operator T && () && noexcept(false); - - -#endif // SIMDJSON_EXCEPTIONS - - /** - * Get the result value. This function is safe if and only - * the error() method returns a value that evaluates to false. - */ - simdjson_inline const T& value_unsafe() const& noexcept; - /** - * Get the result value. This function is safe if and only - * the error() method returns a value that evaluates to false. - */ - simdjson_inline T& value_unsafe() & noexcept; - /** - * Take the result value (move it). This function is safe if and only - * the error() method returns a value that evaluates to false. - */ - simdjson_inline T&& value_unsafe() && noexcept; - protected: - /** users should never directly access first and second. **/ - T first{}; /** Users should never directly access 'first'. **/ - error_code second{ UNINITIALIZED }; /** Users should never directly access 'second'. **/ - }; // struct implementation_simdjson_result_base - - } // namespace icelake -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_H -/* end file simdjson/generic/implementation_simdjson_result_base.h for icelake */ -/* including simdjson/generic/numberparsing.h for icelake: #include "simdjson/generic/numberparsing.h" */ -/* begin file simdjson/generic/numberparsing.h for icelake */ -#ifndef SIMDJSON_GENERIC_NUMBERPARSING_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_NUMBERPARSING_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/jsoncharutils.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include -#include -#include - -namespace simdjson { - namespace icelake { - namespace numberparsing { - -#ifdef JSON_TEST_NUMBERS -#define INVALID_NUMBER(SRC) (found_invalid_number((SRC)), NUMBER_ERROR) -#define WRITE_INTEGER(VALUE, SRC, WRITER) (found_integer((VALUE), (SRC)), (WRITER).append_s64((VALUE))) -#define WRITE_UNSIGNED(VALUE, SRC, WRITER) (found_unsigned_integer((VALUE), (SRC)), (WRITER).append_u64((VALUE))) -#define WRITE_DOUBLE(VALUE, SRC, WRITER) (found_float((VALUE), (SRC)), (WRITER).append_double((VALUE))) -#else -#define INVALID_NUMBER(SRC) (NUMBER_ERROR) -#define WRITE_INTEGER(VALUE, SRC, WRITER) (WRITER).append_s64((VALUE)) -#define WRITE_UNSIGNED(VALUE, SRC, WRITER) (WRITER).append_u64((VALUE)) -#define WRITE_DOUBLE(VALUE, SRC, WRITER) (WRITER).append_double((VALUE)) -#endif - - namespace { - - // Convert a mantissa, an exponent and a sign bit into an ieee64 double. - // The real_exponent needs to be in [0, 2046] (technically real_exponent = 2047 would be acceptable). - // The mantissa should be in [0,1<<53). The bit at index (1ULL << 52) while be zeroed. - simdjson_inline double to_double(uint64_t mantissa, uint64_t real_exponent, bool negative) { - double d; - mantissa &= ~(1ULL << 52); - mantissa |= real_exponent << 52; - mantissa |= ((static_cast(negative)) << 63); - std::memcpy(&d, &mantissa, sizeof(d)); - return d; - } - - // Attempts to compute i * 10^(power) exactly; and if "negative" is - // true, negate the result. - // This function will only work in some cases, when it does not work, success is - // set to false. This should work *most of the time* (like 99% of the time). - // We assume that power is in the [smallest_power, - // largest_power] interval: the caller is responsible for this check. - simdjson_inline bool compute_float_64(int64_t power, uint64_t i, bool negative, double& d) { - // we start with a fast path - // It was described in - // Clinger WD. How to read floating point numbers accurately. - // ACM SIGPLAN Notices. 1990 -#ifndef FLT_EVAL_METHOD -#error "FLT_EVAL_METHOD should be defined, please include cfloat." -#endif -#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0) - // We cannot be certain that x/y is rounded to nearest. - if (0 <= power && power <= 22 && i <= 9007199254740991) -#else - if (-22 <= power && power <= 22 && i <= 9007199254740991) -#endif - { - // convert the integer into a double. This is lossless since - // 0 <= i <= 2^53 - 1. - d = double(i); - // - // The general idea is as follows. - // If 0 <= s < 2^53 and if 10^0 <= p <= 10^22 then - // 1) Both s and p can be represented exactly as 64-bit floating-point - // values - // (binary64). - // 2) Because s and p can be represented exactly as floating-point values, - // then s * p - // and s / p will produce correctly rounded values. - // - if (power < 0) { - d = d / simdjson::internal::power_of_ten[-power]; - } - else { - d = d * simdjson::internal::power_of_ten[power]; - } - if (negative) { - d = -d; - } - return true; - } - // When 22 < power && power < 22 + 16, we could - // hope for another, secondary fast path. It was - // described by David M. Gay in "Correctly rounded - // binary-decimal and decimal-binary conversions." (1990) - // If you need to compute i * 10^(22 + x) for x < 16, - // first compute i * 10^x, if you know that result is exact - // (e.g., when i * 10^x < 2^53), - // then you can still proceed and do (i * 10^x) * 10^22. - // Is this worth your time? - // You need 22 < power *and* power < 22 + 16 *and* (i * 10^(x-22) < 2^53) - // for this second fast path to work. - // If you you have 22 < power *and* power < 22 + 16, and then you - // optimistically compute "i * 10^(x-22)", there is still a chance that you - // have wasted your time if i * 10^(x-22) >= 2^53. It makes the use cases of - // this optimization maybe less common than we would like. Source: - // http://www.exploringbinary.com/fast-path-decimal-to-floating-point-conversion/ - // also used in RapidJSON: https://rapidjson.org/strtod_8h_source.html - - // The fast path has now failed, so we are failing back on the slower path. - - // In the slow path, we need to adjust i so that it is > 1<<63 which is always - // possible, except if i == 0, so we handle i == 0 separately. - if (i == 0) { - d = negative ? -0.0 : 0.0; - return true; - } - - - // The exponent is 1024 + 63 + power - // + floor(log(5**power)/log(2)). - // The 1024 comes from the ieee64 standard. - // The 63 comes from the fact that we use a 64-bit word. - // - // Computing floor(log(5**power)/log(2)) could be - // slow. Instead we use a fast function. - // - // For power in (-400,350), we have that - // (((152170 + 65536) * power ) >> 16); - // is equal to - // floor(log(5**power)/log(2)) + power when power >= 0 - // and it is equal to - // ceil(log(5**-power)/log(2)) + power when power < 0 - // - // The 65536 is (1<<16) and corresponds to - // (65536 * power) >> 16 ---> power - // - // ((152170 * power ) >> 16) is equal to - // floor(log(5**power)/log(2)) - // - // Note that this is not magic: 152170/(1<<16) is - // approximatively equal to log(5)/log(2). - // The 1<<16 value is a power of two; we could use a - // larger power of 2 if we wanted to. - // - int64_t exponent = (((152170 + 65536) * power) >> 16) + 1024 + 63; - - - // We want the most significant bit of i to be 1. Shift if needed. - int lz = leading_zeroes(i); - i <<= lz; - - - // We are going to need to do some 64-bit arithmetic to get a precise product. - // We use a table lookup approach. - // It is safe because - // power >= smallest_power - // and power <= largest_power - // We recover the mantissa of the power, it has a leading 1. It is always - // rounded down. - // - // We want the most significant 64 bits of the product. We know - // this will be non-zero because the most significant bit of i is - // 1. - const uint32_t index = 2 * uint32_t(power - simdjson::internal::smallest_power); - // Optimization: It may be that materializing the index as a variable might confuse some compilers and prevent effective complex-addressing loads. (Done for code clarity.) - // - // The full_multiplication function computes the 128-bit product of two 64-bit words - // with a returned value of type value128 with a "low component" corresponding to the - // 64-bit least significant bits of the product and with a "high component" corresponding - // to the 64-bit most significant bits of the product. - simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index]); - // Both i and power_of_five_128[index] have their most significant bit set to 1 which - // implies that the either the most or the second most significant bit of the product - // is 1. We pack values in this manner for efficiency reasons: it maximizes the use - // we make of the product. It also makes it easy to reason about the product: there - // is 0 or 1 leading zero in the product. - - // Unless the least significant 9 bits of the high (64-bit) part of the full - // product are all 1s, then we know that the most significant 55 bits are - // exact and no further work is needed. Having 55 bits is necessary because - // we need 53 bits for the mantissa but we have to have one rounding bit and - // we can waste a bit if the most significant bit of the product is zero. - if ((firstproduct.high & 0x1FF) == 0x1FF) { - // We want to compute i * 5^q, but only care about the top 55 bits at most. - // Consider the scenario where q>=0. Then 5^q may not fit in 64-bits. Doing - // the full computation is wasteful. So we do what is called a "truncated - // multiplication". - // We take the most significant 64-bits, and we put them in - // power_of_five_128[index]. Usually, that's good enough to approximate i * 5^q - // to the desired approximation using one multiplication. Sometimes it does not suffice. - // Then we store the next most significant 64 bits in power_of_five_128[index + 1], and - // then we get a better approximation to i * 5^q. - // - // That's for when q>=0. The logic for q<0 is somewhat similar but it is somewhat - // more complicated. - // - // There is an extra layer of complexity in that we need more than 55 bits of - // accuracy in the round-to-even scenario. - // - // The full_multiplication function computes the 128-bit product of two 64-bit words - // with a returned value of type value128 with a "low component" corresponding to the - // 64-bit least significant bits of the product and with a "high component" corresponding - // to the 64-bit most significant bits of the product. - simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index + 1]); - firstproduct.low += secondproduct.high; - if (secondproduct.high > firstproduct.low) { firstproduct.high++; } - // As it has been proven by Noble Mushtak and Daniel Lemire in "Fast Number Parsing Without - // Fallback" (https://arxiv.org/abs/2212.06644), at this point we are sure that the product - // is sufficiently accurate, and more computation is not needed. - } - uint64_t lower = firstproduct.low; - uint64_t upper = firstproduct.high; - // The final mantissa should be 53 bits with a leading 1. - // We shift it so that it occupies 54 bits with a leading 1. - /////// - uint64_t upperbit = upper >> 63; - uint64_t mantissa = upper >> (upperbit + 9); - lz += int(1 ^ upperbit); - - // Here we have mantissa < (1<<54). - int64_t real_exponent = exponent - lz; - if (simdjson_unlikely(real_exponent <= 0)) { // we have a subnormal? - // Here have that real_exponent <= 0 so -real_exponent >= 0 - if (-real_exponent + 1 >= 64) { // if we have more than 64 bits below the minimum exponent, you have a zero for sure. - d = negative ? -0.0 : 0.0; - return true; - } - // next line is safe because -real_exponent + 1 < 0 - mantissa >>= -real_exponent + 1; - // Thankfully, we can't have both "round-to-even" and subnormals because - // "round-to-even" only occurs for powers close to 0. - mantissa += (mantissa & 1); // round up - mantissa >>= 1; - // There is a weird scenario where we don't have a subnormal but just. - // Suppose we start with 2.2250738585072013e-308, we end up - // with 0x3fffffffffffff x 2^-1023-53 which is technically subnormal - // whereas 0x40000000000000 x 2^-1023-53 is normal. Now, we need to round - // up 0x3fffffffffffff x 2^-1023-53 and once we do, we are no longer - // subnormal, but we can only know this after rounding. - // So we only declare a subnormal if we are smaller than the threshold. - real_exponent = (mantissa < (uint64_t(1) << 52)) ? 0 : 1; - d = to_double(mantissa, real_exponent, negative); - return true; - } - // We have to round to even. The "to even" part - // is only a problem when we are right in between two floats - // which we guard against. - // If we have lots of trailing zeros, we may fall right between two - // floating-point values. - // - // The round-to-even cases take the form of a number 2m+1 which is in (2^53,2^54] - // times a power of two. That is, it is right between a number with binary significand - // m and another number with binary significand m+1; and it must be the case - // that it cannot be represented by a float itself. - // - // We must have that w * 10 ^q == (2m+1) * 2^p for some power of two 2^p. - // Recall that 10^q = 5^q * 2^q. - // When q >= 0, we must have that (2m+1) is divible by 5^q, so 5^q <= 2^54. We have that - // 5^23 <= 2^54 and it is the last power of five to qualify, so q <= 23. - // When q<0, we have w >= (2m+1) x 5^{-q}. We must have that w<2^{64} so - // (2m+1) x 5^{-q} < 2^{64}. We have that 2m+1>2^{53}. Hence, we must have - // 2^{53} x 5^{-q} < 2^{64}. - // Hence we have 5^{-q} < 2^{11}$ or q>= -4. - // - // We require lower <= 1 and not lower == 0 because we could not prove that - // that lower == 0 is implied; but we could prove that lower <= 1 is a necessary and sufficient test. - if (simdjson_unlikely((lower <= 1) && (power >= -4) && (power <= 23) && ((mantissa & 3) == 1))) { - if ((mantissa << (upperbit + 64 - 53 - 2)) == upper) { - mantissa &= ~1; // flip it so that we do not round up - } - } - - mantissa += mantissa & 1; - mantissa >>= 1; - - // Here we have mantissa < (1<<53), unless there was an overflow - if (mantissa >= (1ULL << 53)) { - ////////// - // This will happen when parsing values such as 7.2057594037927933e+16 - //////// - mantissa = (1ULL << 52); - real_exponent++; - } - mantissa &= ~(1ULL << 52); - // we have to check that real_exponent is in range, otherwise we bail out - if (simdjson_unlikely(real_exponent > 2046)) { - // We have an infinite value!!! We could actually throw an error here if we could. - return false; - } - d = to_double(mantissa, real_exponent, negative); - return true; - } - - // We call a fallback floating-point parser that might be slow. Note - // it will accept JSON numbers, but the JSON spec. is more restrictive so - // before you call parse_float_fallback, you need to have validated the input - // string with the JSON grammar. - // It will return an error (false) if the parsed number is infinite. - // The string parsing itself always succeeds. We know that there is at least - // one digit. - static bool parse_float_fallback(const uint8_t* ptr, double* outDouble) { - *outDouble = simdjson::internal::from_chars(reinterpret_cast(ptr)); - // We do not accept infinite values. - - // Detecting finite values in a portable manner is ridiculously hard, ideally - // we would want to do: - // return !std::isfinite(*outDouble); - // but that mysteriously fails under legacy/old libc++ libraries, see - // https://github.com/simdjson/simdjson/issues/1286 - // - // Therefore, fall back to this solution (the extra parens are there - // to handle that max may be a macro on windows). - return !(*outDouble > (std::numeric_limits::max)() || *outDouble < std::numeric_limits::lowest()); - } - - static bool parse_float_fallback(const uint8_t* ptr, const uint8_t* end_ptr, double* outDouble) { - *outDouble = simdjson::internal::from_chars(reinterpret_cast(ptr), reinterpret_cast(end_ptr)); - // We do not accept infinite values. - - // Detecting finite values in a portable manner is ridiculously hard, ideally - // we would want to do: - // return !std::isfinite(*outDouble); - // but that mysteriously fails under legacy/old libc++ libraries, see - // https://github.com/simdjson/simdjson/issues/1286 - // - // Therefore, fall back to this solution (the extra parens are there - // to handle that max may be a macro on windows). - return !(*outDouble > (std::numeric_limits::max)() || *outDouble < std::numeric_limits::lowest()); - } - - // check quickly whether the next 8 chars are made of digits - // at a glance, it looks better than Mula's - // http://0x80.pl/articles/swar-digits-validate.html - simdjson_inline bool is_made_of_eight_digits_fast(const uint8_t* chars) { - uint64_t val; - // this can read up to 7 bytes beyond the buffer size, but we require - // SIMDJSON_PADDING of padding - static_assert(7 <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be bigger than 7"); - std::memcpy(&val, chars, 8); - // a branchy method might be faster: - // return (( val & 0xF0F0F0F0F0F0F0F0 ) == 0x3030303030303030) - // && (( (val + 0x0606060606060606) & 0xF0F0F0F0F0F0F0F0 ) == - // 0x3030303030303030); - return (((val & 0xF0F0F0F0F0F0F0F0) | - (((val + 0x0606060606060606) & 0xF0F0F0F0F0F0F0F0) >> 4)) == - 0x3333333333333333); - } - - template - SIMDJSON_NO_SANITIZE_UNDEFINED // We deliberately allow overflow here and check later - simdjson_inline bool parse_digit(const uint8_t c, I& i) { - const uint8_t digit = static_cast(c - '0'); - if (digit > 9) { - return false; - } - // PERF NOTE: multiplication by 10 is cheaper than arbitrary integer multiplication - i = 10 * i + digit; // might overflow, we will handle the overflow later - return true; - } - - simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t* const src, const uint8_t*& p, uint64_t& i, int64_t& exponent) { - // we continue with the fiction that we have an integer. If the - // floating point number is representable as x * 10^z for some integer - // z that fits in 53 bits, then we will be able to convert back the - // the integer into a float in a lossless manner. - const uint8_t* const first_after_period = p; - -#ifdef SIMDJSON_SWAR_NUMBER_PARSING -#if SIMDJSON_SWAR_NUMBER_PARSING - // this helps if we have lots of decimals! - // this turns out to be frequent enough. - if (is_made_of_eight_digits_fast(p)) { - i = i * 100000000 + parse_eight_digits_unrolled(p); - p += 8; - } -#endif // SIMDJSON_SWAR_NUMBER_PARSING -#endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING - // Unrolling the first digit makes a small difference on some implementations (e.g. westmere) - if (parse_digit(*p, i)) { ++p; } - while (parse_digit(*p, i)) { p++; } - exponent = first_after_period - p; - // Decimal without digits (123.) is illegal - if (exponent == 0) { - return INVALID_NUMBER(src); - } - return SUCCESS; - } - - simdjson_inline error_code parse_exponent(simdjson_unused const uint8_t* const src, const uint8_t*& p, int64_t& exponent) { - // Exp Sign: -123.456e[-]78 - bool neg_exp = ('-' == *p); - if (neg_exp || '+' == *p) { p++; } // Skip + as well - - // Exponent: -123.456e-[78] - auto start_exp = p; - int64_t exp_number = 0; - while (parse_digit(*p, exp_number)) { ++p; } - // It is possible for parse_digit to overflow. - // In particular, it could overflow to INT64_MIN, and we cannot do - INT64_MIN. - // Thus we *must* check for possible overflow before we negate exp_number. - - // Performance notes: it may seem like combining the two "simdjson_unlikely checks" below into - // a single simdjson_unlikely path would be faster. The reasoning is sound, but the compiler may - // not oblige and may, in fact, generate two distinct paths in any case. It might be - // possible to do uint64_t(p - start_exp - 1) >= 18 but it could end up trading off - // instructions for a simdjson_likely branch, an unconclusive gain. - - // If there were no digits, it's an error. - if (simdjson_unlikely(p == start_exp)) { - return INVALID_NUMBER(src); - } - // We have a valid positive exponent in exp_number at this point, except that - // it may have overflowed. - - // If there were more than 18 digits, we may have overflowed the integer. We have to do - // something!!!! - if (simdjson_unlikely(p > start_exp + 18)) { - // Skip leading zeroes: 1e000000000000000000001 is technically valid and doesn't overflow - while (*start_exp == '0') { start_exp++; } - // 19 digits could overflow int64_t and is kind of absurd anyway. We don't - // support exponents smaller than -999,999,999,999,999,999 and bigger - // than 999,999,999,999,999,999. - // We can truncate. - // Note that 999999999999999999 is assuredly too large. The maximal ieee64 value before - // infinity is ~1.8e308. The smallest subnormal is ~5e-324. So, actually, we could - // truncate at 324. - // Note that there is no reason to fail per se at this point in time. - // E.g., 0e999999999999999999999 is a fine number. - if (p > start_exp + 18) { exp_number = 999999999999999999; } - } - // At this point, we know that exp_number is a sane, positive, signed integer. - // It is <= 999,999,999,999,999,999. As long as 'exponent' is in - // [-8223372036854775808, 8223372036854775808], we won't overflow. Because 'exponent' - // is bounded in magnitude by the size of the JSON input, we are fine in this universe. - // To sum it up: the next line should never overflow. - exponent += (neg_exp ? -exp_number : exp_number); - return SUCCESS; - } - - simdjson_inline size_t significant_digits(const uint8_t* start_digits, size_t digit_count) { - // It is possible that the integer had an overflow. - // We have to handle the case where we have 0.0000somenumber. - const uint8_t* start = start_digits; - while ((*start == '0') || (*start == '.')) { ++start; } - // we over-decrement by one when there is a '.' - return digit_count - size_t(start - start_digits); - } - - } // unnamed namespace - - /** @private */ - template - error_code slow_float_parsing(simdjson_unused const uint8_t* src, W writer) { - double d; - if (parse_float_fallback(src, &d)) { - writer.append_double(d); - return SUCCESS; - } - return INVALID_NUMBER(src); - } - - /** @private */ - template - simdjson_inline error_code write_float(const uint8_t* const src, bool negative, uint64_t i, const uint8_t* start_digits, size_t digit_count, int64_t exponent, W& writer) { - // If we frequently had to deal with long strings of digits, - // we could extend our code by using a 128-bit integer instead - // of a 64-bit integer. However, this is uncommon in practice. - // - // 9999999999999999999 < 2**64 so we can accommodate 19 digits. - // If we have a decimal separator, then digit_count - 1 is the number of digits, but we - // may not have a decimal separator! - if (simdjson_unlikely(digit_count > 19 && significant_digits(start_digits, digit_count) > 19)) { - // Ok, chances are good that we had an overflow! - // this is almost never going to get called!!! - // we start anew, going slowly!!! - // This will happen in the following examples: - // 10000000000000000000000000000000000000000000e+308 - // 3.1415926535897932384626433832795028841971693993751 - // - // NOTE: This makes a *copy* of the writer and passes it to slow_float_parsing. This happens - // because slow_float_parsing is a non-inlined function. If we passed our writer reference to - // it, it would force it to be stored in memory, preventing the compiler from picking it apart - // and putting into registers. i.e. if we pass it as reference, it gets slow. - // This is what forces the skip_double, as well. - error_code error = slow_float_parsing(src, writer); - writer.skip_double(); - return error; - } - // NOTE: it's weird that the simdjson_unlikely() only wraps half the if, but it seems to get slower any other - // way we've tried: https://github.com/simdjson/simdjson/pull/990#discussion_r448497331 - // To future reader: we'd love if someone found a better way, or at least could explain this result! - if (simdjson_unlikely(exponent < simdjson::internal::smallest_power) || (exponent > simdjson::internal::largest_power)) { - // - // Important: smallest_power is such that it leads to a zero value. - // Observe that 18446744073709551615e-343 == 0, i.e. (2**64 - 1) e -343 is zero - // so something x 10^-343 goes to zero, but not so with something x 10^-342. - static_assert(simdjson::internal::smallest_power <= -342, "smallest_power is not small enough"); - // - if ((exponent < simdjson::internal::smallest_power) || (i == 0)) { - // E.g. Parse "-0.0e-999" into the same value as "-0.0". See https://en.wikipedia.org/wiki/Signed_zero - WRITE_DOUBLE(negative ? -0.0 : 0.0, src, writer); - return SUCCESS; - } - else { // (exponent > largest_power) and (i != 0) - // We have, for sure, an infinite value and simdjson refuses to parse infinite values. - return INVALID_NUMBER(src); - } - } - double d; - if (!compute_float_64(exponent, i, negative, d)) { - // we are almost never going to get here. - if (!parse_float_fallback(src, &d)) { return INVALID_NUMBER(src); } - } - WRITE_DOUBLE(d, src, writer); - return SUCCESS; - } - - // for performance analysis, it is sometimes useful to skip parsing -#ifdef SIMDJSON_SKIPNUMBERPARSING - - template - simdjson_inline error_code parse_number(const uint8_t* const, W& writer) { - writer.append_s64(0); // always write zero - return SUCCESS; // always succeeds - } - - simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t* const src) noexcept { return 0; } - simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t* const src) noexcept { return 0; } - simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t* const src) noexcept { return 0; } - simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t* const src) noexcept { return 0; } - simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t* const src) noexcept { return 0; } - simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t* const src) noexcept { return 0; } - simdjson_unused simdjson_inline bool is_negative(const uint8_t* src) noexcept { return false; } - simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t* src) noexcept { return false; } - simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t* src) noexcept { return number_type::signed_integer; } -#else - -// parse the number at src -// define JSON_TEST_NUMBERS for unit testing -// -// It is assumed that the number is followed by a structural ({,},],[) character -// or a white space character. If that is not the case (e.g., when the JSON -// document is made of a single number), then it is necessary to copy the -// content and append a space before calling this function. -// -// Our objective is accurate parsing (ULP of 0) at high speed. - template - simdjson_inline error_code parse_number(const uint8_t* const src, W& writer) { - - // - // Check for minus sign - // - bool negative = (*src == '-'); - const uint8_t* p = src + uint8_t(negative); - - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t* const start_digits = p; - uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(p - start_digits); - if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { return INVALID_NUMBER(src); } - - // - // Handle floats if there is a . or e (or both) - // - int64_t exponent = 0; - bool is_float = false; - if ('.' == *p) { - is_float = true; - ++p; - SIMDJSON_TRY(parse_decimal_after_separator(src, p, i, exponent)); - digit_count = int(p - start_digits); // used later to guard against overflows - } - if (('e' == *p) || ('E' == *p)) { - is_float = true; - ++p; - SIMDJSON_TRY(parse_exponent(src, p, exponent)); - } - if (is_float) { - const bool dirty_end = jsoncharutils::is_not_structural_or_whitespace(*p); - SIMDJSON_TRY(write_float(src, negative, i, start_digits, digit_count, exponent, writer)); - if (dirty_end) { return INVALID_NUMBER(src); } - return SUCCESS; - } - - // The longest negative 64-bit number is 19 digits. - // The longest positive 64-bit number is 20 digits. - // We do it this way so we don't trigger this branch unless we must. - size_t longest_digit_count = negative ? 19 : 20; - if (digit_count > longest_digit_count) { return INVALID_NUMBER(src); } - if (digit_count == longest_digit_count) { - if (negative) { - // Anything negative above INT64_MAX+1 is invalid - if (i > uint64_t(INT64_MAX) + 1) { return INVALID_NUMBER(src); } - WRITE_INTEGER(~i + 1, src, writer); - if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return INVALID_NUMBER(src); } - return SUCCESS; - // Positive overflow check: - // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the - // biggest uint64_t. - // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX. - // If we got here, it's a 20 digit number starting with the digit "1". - // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller - // than 1,553,255,926,290,448,384. - // - That is smaller than the smallest possible 20-digit number the user could write: - // 10,000,000,000,000,000,000. - // - Therefore, if the number is positive and lower than that, it's overflow. - // - The value we are looking at is less than or equal to INT64_MAX. - // - } - else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INVALID_NUMBER(src); } - } - - // Write unsigned if it doesn't fit in a signed integer. - if (i > uint64_t(INT64_MAX)) { - WRITE_UNSIGNED(i, src, writer); - } - else { - WRITE_INTEGER(negative ? (~i + 1) : i, src, writer); - } - if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return INVALID_NUMBER(src); } - return SUCCESS; - } - - // Inlineable functions - namespace { - - // This table can be used to characterize the final character of an integer - // string. For JSON structural character and allowable white space characters, - // we return SUCCESS. For 'e', '.' and 'E', we return INCORRECT_TYPE. Otherwise - // we return NUMBER_ERROR. - // Optimization note: we could easily reduce the size of the table by half (to 128) - // at the cost of an extra branch. - // Optimization note: we want the values to use at most 8 bits (not, e.g., 32 bits): - static_assert(error_code(uint8_t(NUMBER_ERROR)) == NUMBER_ERROR, "bad NUMBER_ERROR cast"); - static_assert(error_code(uint8_t(SUCCESS)) == SUCCESS, "bad NUMBER_ERROR cast"); - static_assert(error_code(uint8_t(INCORRECT_TYPE)) == INCORRECT_TYPE, "bad NUMBER_ERROR cast"); - - const uint8_t integer_string_finisher[256] = { - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, - SUCCESS, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, - NUMBER_ERROR, INCORRECT_TYPE, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, INCORRECT_TYPE, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, SUCCESS, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, INCORRECT_TYPE, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, - SUCCESS, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR }; - - // Parse any number from 0 to 18,446,744,073,709,551,615 - simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t* const src) noexcept { - const uint8_t* p = src; - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t* const start_digits = p; - uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(p - start_digits); - // The longest positive 64-bit number is 20 digits. - // We do it this way so we don't trigger this branch unless we must. - // Optimization note: the compiler can probably merge - // ((digit_count == 0) || (digit_count > 20)) - // into a single branch since digit_count is unsigned. - if ((digit_count == 0) || (digit_count > 20)) { return INCORRECT_TYPE; } - // Here digit_count > 0. - if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } - // We can do the following... - // if (!jsoncharutils::is_structural_or_whitespace(*p)) { - // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; - // } - // as a single table lookup: - if (integer_string_finisher[*p] != SUCCESS) { return error_code(integer_string_finisher[*p]); } - - if (digit_count == 20) { - // Positive overflow check: - // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the - // biggest uint64_t. - // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX. - // If we got here, it's a 20 digit number starting with the digit "1". - // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller - // than 1,553,255,926,290,448,384. - // - That is smaller than the smallest possible 20-digit number the user could write: - // 10,000,000,000,000,000,000. - // - Therefore, if the number is positive and lower than that, it's overflow. - // - The value we are looking at is less than or equal to INT64_MAX. - // - if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INCORRECT_TYPE; } - } - - return i; - } - - - // Parse any number from 0 to 18,446,744,073,709,551,615 - // Never read at src_end or beyond - simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t* const src, const uint8_t* const src_end) noexcept { - const uint8_t* p = src; - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t* const start_digits = p; - uint64_t i = 0; - while ((p != src_end) && parse_digit(*p, i)) { p++; } - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(p - start_digits); - // The longest positive 64-bit number is 20 digits. - // We do it this way so we don't trigger this branch unless we must. - // Optimization note: the compiler can probably merge - // ((digit_count == 0) || (digit_count > 20)) - // into a single branch since digit_count is unsigned. - if ((digit_count == 0) || (digit_count > 20)) { return INCORRECT_TYPE; } - // Here digit_count > 0. - if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } - // We can do the following... - // if (!jsoncharutils::is_structural_or_whitespace(*p)) { - // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; - // } - // as a single table lookup: - if ((p != src_end) && integer_string_finisher[*p] != SUCCESS) { return error_code(integer_string_finisher[*p]); } - - if (digit_count == 20) { - // Positive overflow check: - // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the - // biggest uint64_t. - // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX. - // If we got here, it's a 20 digit number starting with the digit "1". - // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller - // than 1,553,255,926,290,448,384. - // - That is smaller than the smallest possible 20-digit number the user could write: - // 10,000,000,000,000,000,000. - // - Therefore, if the number is positive and lower than that, it's overflow. - // - The value we are looking at is less than or equal to INT64_MAX. - // - if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INCORRECT_TYPE; } - } - - return i; - } - - // Parse any number from 0 to 18,446,744,073,709,551,615 - simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t* const src) noexcept { - const uint8_t* p = src + 1; - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t* const start_digits = p; - uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(p - start_digits); - // The longest positive 64-bit number is 20 digits. - // We do it this way so we don't trigger this branch unless we must. - // Optimization note: the compiler can probably merge - // ((digit_count == 0) || (digit_count > 20)) - // into a single branch since digit_count is unsigned. - if ((digit_count == 0) || (digit_count > 20)) { return INCORRECT_TYPE; } - // Here digit_count > 0. - if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } - // We can do the following... - // if (!jsoncharutils::is_structural_or_whitespace(*p)) { - // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; - // } - // as a single table lookup: - if (*p != '"') { return NUMBER_ERROR; } - - if (digit_count == 20) { - // Positive overflow check: - // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the - // biggest uint64_t. - // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX. - // If we got here, it's a 20 digit number starting with the digit "1". - // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller - // than 1,553,255,926,290,448,384. - // - That is smaller than the smallest possible 20-digit number the user could write: - // 10,000,000,000,000,000,000. - // - Therefore, if the number is positive and lower than that, it's overflow. - // - The value we are looking at is less than or equal to INT64_MAX. - // - // Note: we use src[1] and not src[0] because src[0] is the quote character in this - // instance. - if (src[1] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INCORRECT_TYPE; } - } - - return i; - } - - // Parse any number from -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 - simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t* src) noexcept { - // - // Check for minus sign - // - bool negative = (*src == '-'); - const uint8_t* p = src + uint8_t(negative); - - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t* const start_digits = p; - uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(p - start_digits); - // We go from - // -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 - // so we can never represent numbers that have more than 19 digits. - size_t longest_digit_count = 19; - // Optimization note: the compiler can probably merge - // ((digit_count == 0) || (digit_count > longest_digit_count)) - // into a single branch since digit_count is unsigned. - if ((digit_count == 0) || (digit_count > longest_digit_count)) { return INCORRECT_TYPE; } - // Here digit_count > 0. - if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } - // We can do the following... - // if (!jsoncharutils::is_structural_or_whitespace(*p)) { - // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; - // } - // as a single table lookup: - if (integer_string_finisher[*p] != SUCCESS) { return error_code(integer_string_finisher[*p]); } - // Negative numbers have can go down to - INT64_MAX - 1 whereas positive numbers are limited to INT64_MAX. - // Performance note: This check is only needed when digit_count == longest_digit_count but it is - // so cheap that we might as well always make it. - if (i > uint64_t(INT64_MAX) + uint64_t(negative)) { return INCORRECT_TYPE; } - return negative ? (~i + 1) : i; - } - - // Parse any number from -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 - // Never read at src_end or beyond - simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t* const src, const uint8_t* const src_end) noexcept { - // - // Check for minus sign - // - if (src == src_end) { return NUMBER_ERROR; } - bool negative = (*src == '-'); - const uint8_t* p = src + uint8_t(negative); - - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t* const start_digits = p; - uint64_t i = 0; - while ((p != src_end) && parse_digit(*p, i)) { p++; } - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(p - start_digits); - // We go from - // -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 - // so we can never represent numbers that have more than 19 digits. - size_t longest_digit_count = 19; - // Optimization note: the compiler can probably merge - // ((digit_count == 0) || (digit_count > longest_digit_count)) - // into a single branch since digit_count is unsigned. - if ((digit_count == 0) || (digit_count > longest_digit_count)) { return INCORRECT_TYPE; } - // Here digit_count > 0. - if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } - // We can do the following... - // if (!jsoncharutils::is_structural_or_whitespace(*p)) { - // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; - // } - // as a single table lookup: - if ((p != src_end) && integer_string_finisher[*p] != SUCCESS) { return error_code(integer_string_finisher[*p]); } - // Negative numbers have can go down to - INT64_MAX - 1 whereas positive numbers are limited to INT64_MAX. - // Performance note: This check is only needed when digit_count == longest_digit_count but it is - // so cheap that we might as well always make it. - if (i > uint64_t(INT64_MAX) + uint64_t(negative)) { return INCORRECT_TYPE; } - return negative ? (~i + 1) : i; - } - - // Parse any number from -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 - simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t* src) noexcept { - // - // Check for minus sign - // - bool negative = (*(src + 1) == '-'); - src += uint8_t(negative) + 1; - - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t* const start_digits = src; - uint64_t i = 0; - while (parse_digit(*src, i)) { src++; } - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(src - start_digits); - // We go from - // -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 - // so we can never represent numbers that have more than 19 digits. - size_t longest_digit_count = 19; - // Optimization note: the compiler can probably merge - // ((digit_count == 0) || (digit_count > longest_digit_count)) - // into a single branch since digit_count is unsigned. - if ((digit_count == 0) || (digit_count > longest_digit_count)) { return INCORRECT_TYPE; } - // Here digit_count > 0. - if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } - // We can do the following... - // if (!jsoncharutils::is_structural_or_whitespace(*src)) { - // return (*src == '.' || *src == 'e' || *src == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; - // } - // as a single table lookup: - if (*src != '"') { return NUMBER_ERROR; } - // Negative numbers have can go down to - INT64_MAX - 1 whereas positive numbers are limited to INT64_MAX. - // Performance note: This check is only needed when digit_count == longest_digit_count but it is - // so cheap that we might as well always make it. - if (i > uint64_t(INT64_MAX) + uint64_t(negative)) { return INCORRECT_TYPE; } - return negative ? (~i + 1) : i; - } - - simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t* src) noexcept { - // - // Check for minus sign - // - bool negative = (*src == '-'); - src += uint8_t(negative); - - // - // Parse the integer part. - // - uint64_t i = 0; - const uint8_t* p = src; - p += parse_digit(*p, i); - bool leading_zero = (i == 0); - while (parse_digit(*p, i)) { p++; } - // no integer digits, or 0123 (zero must be solo) - if (p == src) { return INCORRECT_TYPE; } - if ((leading_zero && p != src + 1)) { return NUMBER_ERROR; } - - // - // Parse the decimal part. - // - int64_t exponent = 0; - bool overflow; - if (simdjson_likely(*p == '.')) { - p++; - const uint8_t* start_decimal_digits = p; - if (!parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits - p++; - while (parse_digit(*p, i)) { p++; } - exponent = -(p - start_decimal_digits); - - // Overflow check. More than 19 digits (minus the decimal) may be overflow. - overflow = p - src - 1 > 19; - if (simdjson_unlikely(overflow && leading_zero)) { - // Skip leading 0.00000 and see if it still overflows - const uint8_t* start_digits = src + 2; - while (*start_digits == '0') { start_digits++; } - overflow = p - start_digits > 19; - } - } - else { - overflow = p - src > 19; - } - - // - // Parse the exponent - // - if (*p == 'e' || *p == 'E') { - p++; - bool exp_neg = *p == '-'; - p += exp_neg || *p == '+'; - - uint64_t exp = 0; - const uint8_t* start_exp_digits = p; - while (parse_digit(*p, exp)) { p++; } - // no exp digits, or 20+ exp digits - if (p - start_exp_digits == 0 || p - start_exp_digits > 19) { return NUMBER_ERROR; } - - exponent += exp_neg ? 0 - exp : exp; - } - - if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } - - overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; - - // - // Assemble (or slow-parse) the float - // - double d; - if (simdjson_likely(!overflow)) { - if (compute_float_64(exponent, i, negative, d)) { return d; } - } - if (!parse_float_fallback(src - uint8_t(negative), &d)) { - return NUMBER_ERROR; - } - return d; - } - - simdjson_unused simdjson_inline bool is_negative(const uint8_t* src) noexcept { - return (*src == '-'); - } - - simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t* src) noexcept { - bool negative = (*src == '-'); - src += uint8_t(negative); - const uint8_t* p = src; - while (static_cast(*p - '0') <= 9) { p++; } - if (p == src) { return NUMBER_ERROR; } - if (jsoncharutils::is_structural_or_whitespace(*p)) { return true; } - return false; - } - - simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t* src) noexcept { - bool negative = (*src == '-'); - src += uint8_t(negative); - const uint8_t* p = src; - while (static_cast(*p - '0') <= 9) { p++; } - if (p == src) { return NUMBER_ERROR; } - if (jsoncharutils::is_structural_or_whitespace(*p)) { - // We have an integer. - // If the number is negative and valid, it must be a signed integer. - if (negative) { return number_type::signed_integer; } - // We want values larger or equal to 9223372036854775808 to be unsigned - // integers, and the other values to be signed integers. - int digit_count = int(p - src); - if (digit_count >= 19) { - const uint8_t* smaller_big_integer = reinterpret_cast("9223372036854775808"); - if ((digit_count >= 20) || (memcmp(src, smaller_big_integer, 19) >= 0)) { - return number_type::unsigned_integer; - } - } - return number_type::signed_integer; - } - // Hopefully, we have 'e' or 'E' or '.'. - return number_type::floating_point_number; - } - - // Never read at src_end or beyond - simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t* src, const uint8_t* const src_end) noexcept { - if (src == src_end) { return NUMBER_ERROR; } - // - // Check for minus sign - // - bool negative = (*src == '-'); - src += uint8_t(negative); - - // - // Parse the integer part. - // - uint64_t i = 0; - const uint8_t* p = src; - if (p == src_end) { return NUMBER_ERROR; } - p += parse_digit(*p, i); - bool leading_zero = (i == 0); - while ((p != src_end) && parse_digit(*p, i)) { p++; } - // no integer digits, or 0123 (zero must be solo) - if (p == src) { return INCORRECT_TYPE; } - if ((leading_zero && p != src + 1)) { return NUMBER_ERROR; } - - // - // Parse the decimal part. - // - int64_t exponent = 0; - bool overflow; - if (simdjson_likely((p != src_end) && (*p == '.'))) { - p++; - const uint8_t* start_decimal_digits = p; - if ((p == src_end) || !parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits - p++; - while ((p != src_end) && parse_digit(*p, i)) { p++; } - exponent = -(p - start_decimal_digits); - - // Overflow check. More than 19 digits (minus the decimal) may be overflow. - overflow = p - src - 1 > 19; - if (simdjson_unlikely(overflow && leading_zero)) { - // Skip leading 0.00000 and see if it still overflows - const uint8_t* start_digits = src + 2; - while (*start_digits == '0') { start_digits++; } - overflow = start_digits - src > 19; - } - } - else { - overflow = p - src > 19; - } - - // - // Parse the exponent - // - if ((p != src_end) && (*p == 'e' || *p == 'E')) { - p++; - if (p == src_end) { return NUMBER_ERROR; } - bool exp_neg = *p == '-'; - p += exp_neg || *p == '+'; - - uint64_t exp = 0; - const uint8_t* start_exp_digits = p; - while ((p != src_end) && parse_digit(*p, exp)) { p++; } - // no exp digits, or 20+ exp digits - if (p - start_exp_digits == 0 || p - start_exp_digits > 19) { return NUMBER_ERROR; } - - exponent += exp_neg ? 0 - exp : exp; - } - - if ((p != src_end) && jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } - - overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; - - // - // Assemble (or slow-parse) the float - // - double d; - if (simdjson_likely(!overflow)) { - if (compute_float_64(exponent, i, negative, d)) { return d; } - } - if (!parse_float_fallback(src - uint8_t(negative), src_end, &d)) { - return NUMBER_ERROR; - } - return d; - } - - simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t* src) noexcept { - // - // Check for minus sign - // - bool negative = (*(src + 1) == '-'); - src += uint8_t(negative) + 1; - - // - // Parse the integer part. - // - uint64_t i = 0; - const uint8_t* p = src; - p += parse_digit(*p, i); - bool leading_zero = (i == 0); - while (parse_digit(*p, i)) { p++; } - // no integer digits, or 0123 (zero must be solo) - if (p == src) { return INCORRECT_TYPE; } - if ((leading_zero && p != src + 1)) { return NUMBER_ERROR; } - - // - // Parse the decimal part. - // - int64_t exponent = 0; - bool overflow; - if (simdjson_likely(*p == '.')) { - p++; - const uint8_t* start_decimal_digits = p; - if (!parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits - p++; - while (parse_digit(*p, i)) { p++; } - exponent = -(p - start_decimal_digits); - - // Overflow check. More than 19 digits (minus the decimal) may be overflow. - overflow = p - src - 1 > 19; - if (simdjson_unlikely(overflow && leading_zero)) { - // Skip leading 0.00000 and see if it still overflows - const uint8_t* start_digits = src + 2; - while (*start_digits == '0') { start_digits++; } - overflow = p - start_digits > 19; - } - } - else { - overflow = p - src > 19; - } - - // - // Parse the exponent - // - if (*p == 'e' || *p == 'E') { - p++; - bool exp_neg = *p == '-'; - p += exp_neg || *p == '+'; - - uint64_t exp = 0; - const uint8_t* start_exp_digits = p; - while (parse_digit(*p, exp)) { p++; } - // no exp digits, or 20+ exp digits - if (p - start_exp_digits == 0 || p - start_exp_digits > 19) { return NUMBER_ERROR; } - - exponent += exp_neg ? 0 - exp : exp; - } - - if (*p != '"') { return NUMBER_ERROR; } - - overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; - - // - // Assemble (or slow-parse) the float - // - double d; - if (simdjson_likely(!overflow)) { - if (compute_float_64(exponent, i, negative, d)) { return d; } - } - if (!parse_float_fallback(src - uint8_t(negative), &d)) { - return NUMBER_ERROR; - } - return d; - } - - } // unnamed namespace -#endif // SIMDJSON_SKIPNUMBERPARSING - - } // namespace numberparsing - - inline std::ostream& operator<<(std::ostream& out, number_type type) noexcept { - switch (type) { - case number_type::signed_integer: out << "integer in [-9223372036854775808,9223372036854775808)"; break; - case number_type::unsigned_integer: out << "unsigned integer in [9223372036854775808,18446744073709551616)"; break; - case number_type::floating_point_number: out << "floating-point number (binary64)"; break; - default: SIMDJSON_UNREACHABLE(); - } - return out; - } - - } // namespace icelake -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_NUMBERPARSING_H -/* end file simdjson/generic/numberparsing.h for icelake */ - -/* including simdjson/generic/implementation_simdjson_result_base-inl.h for icelake: #include "simdjson/generic/implementation_simdjson_result_base-inl.h" */ -/* begin file simdjson/generic/implementation_simdjson_result_base-inl.h for icelake */ -#ifndef SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_INL_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_INL_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/implementation_simdjson_result_base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - namespace icelake { - - // - // internal::implementation_simdjson_result_base inline implementation - // - - template - simdjson_inline void implementation_simdjson_result_base::tie(T& value, error_code& error) && noexcept { - error = this->second; - if (!error) { - value = std::forward>(*this).first; - } - } - - template - simdjson_warn_unused simdjson_inline error_code implementation_simdjson_result_base::get(T& value) && noexcept { - error_code error; - std::forward>(*this).tie(value, error); - return error; - } - - template - simdjson_inline error_code implementation_simdjson_result_base::error() const noexcept { - return this->second; - } - -#if SIMDJSON_EXCEPTIONS - - template - simdjson_inline T& implementation_simdjson_result_base::value() & noexcept(false) { - if (error()) { throw simdjson_error(error()); } - return this->first; - } - - template - simdjson_inline T&& implementation_simdjson_result_base::value() && noexcept(false) { - return std::forward>(*this).take_value(); - } - - template - simdjson_inline T&& implementation_simdjson_result_base::take_value() && noexcept(false) { - if (error()) { throw simdjson_error(error()); } - return std::forward(this->first); - } - - template - simdjson_inline implementation_simdjson_result_base::operator T && () && noexcept(false) { - return std::forward>(*this).take_value(); - } - -#endif // SIMDJSON_EXCEPTIONS - - template - simdjson_inline const T& implementation_simdjson_result_base::value_unsafe() const& noexcept { - return this->first; - } - - template - simdjson_inline T& implementation_simdjson_result_base::value_unsafe() & noexcept { - return this->first; - } - - template - simdjson_inline T&& implementation_simdjson_result_base::value_unsafe() && noexcept { - return std::forward(this->first); - } - - template - simdjson_inline implementation_simdjson_result_base::implementation_simdjson_result_base(T&& value, error_code error) noexcept - : first{ std::forward(value) }, second{ error } {} - template - simdjson_inline implementation_simdjson_result_base::implementation_simdjson_result_base(error_code error) noexcept - : implementation_simdjson_result_base(T{}, error) {} - template - simdjson_inline implementation_simdjson_result_base::implementation_simdjson_result_base(T&& value) noexcept - : implementation_simdjson_result_base(std::forward(value), SUCCESS) {} - - } // namespace icelake -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_INL_H -/* end file simdjson/generic/implementation_simdjson_result_base-inl.h for icelake */ -/* end file simdjson/generic/amalgamated.h for icelake */ -/* including simdjson/icelake/end.h: #include "simdjson/icelake/end.h" */ -/* begin file simdjson/icelake/end.h */ -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/icelake/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#if !SIMDJSON_CAN_ALWAYS_RUN_ICELAKE -SIMDJSON_UNTARGET_REGION -#endif - -/* undefining SIMDJSON_IMPLEMENTATION from "icelake" */ -#undef SIMDJSON_IMPLEMENTATION -/* end file simdjson/icelake/end.h */ - -#endif // SIMDJSON_ICELAKE_H -/* end file simdjson/icelake.h */ -/* including simdjson/icelake/implementation.h: #include */ -/* begin file simdjson/icelake/implementation.h */ -#ifndef SIMDJSON_ICELAKE_IMPLEMENTATION_H -#define SIMDJSON_ICELAKE_IMPLEMENTATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/icelake/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/implementation.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/instruction_set.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -// The constructor may be executed on any host, so we take care not to use SIMDJSON_TARGET_ICELAKE -namespace simdjson { - namespace icelake { - - /** - * @private - */ - class implementation final : public simdjson::implementation { - public: - simdjson_inline implementation() : simdjson::implementation( - "icelake", - "Intel/AMD AVX512", - internal::instruction_set::AVX2 | internal::instruction_set::PCLMULQDQ | internal::instruction_set::BMI1 | internal::instruction_set::BMI2 | internal::instruction_set::AVX512F | internal::instruction_set::AVX512DQ | internal::instruction_set::AVX512CD | internal::instruction_set::AVX512BW | internal::instruction_set::AVX512VL | internal::instruction_set::AVX512VBMI2 - ) {} - simdjson_warn_unused error_code create_dom_parser_implementation( - size_t capacity, - size_t max_length, - std::unique_ptr& dst - ) const noexcept final; - simdjson_warn_unused error_code minify(const uint8_t* buf, size_t len, uint8_t* dst, size_t& dst_len) const noexcept final; - simdjson_warn_unused bool validate_utf8(const char* buf, size_t len) const noexcept final; - }; - - } // namespace icelake -} // namespace simdjson - -#endif // SIMDJSON_ICELAKE_IMPLEMENTATION_H -/* end file simdjson/icelake/implementation.h */ - -// defining SIMDJSON_GENERIC_JSON_STRUCTURAL_INDEXER_CUSTOM_BIT_INDEXER allows us to provide our own bit_indexer::write -#define SIMDJSON_GENERIC_JSON_STRUCTURAL_INDEXER_CUSTOM_BIT_INDEXER - -/* including simdjson/icelake/begin.h: #include */ -/* begin file simdjson/icelake/begin.h */ -/* defining SIMDJSON_IMPLEMENTATION to "icelake" */ -#define SIMDJSON_IMPLEMENTATION icelake -/* including simdjson/icelake/base.h: #include "simdjson/icelake/base.h" */ -/* begin file simdjson/icelake/base.h */ -#ifndef SIMDJSON_ICELAKE_BASE_H -#define SIMDJSON_ICELAKE_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -// The constructor may be executed on any host, so we take care not to use SIMDJSON_TARGET_ICELAKE -namespace simdjson { - /** - * Implementation for Icelake (Intel AVX512). - */ - namespace icelake { - - class implementation; - - } // namespace icelake -} // namespace simdjson - -#endif // SIMDJSON_ICELAKE_BASE_H -/* end file simdjson/icelake/base.h */ -/* including simdjson/icelake/intrinsics.h: #include "simdjson/icelake/intrinsics.h" */ -/* begin file simdjson/icelake/intrinsics.h */ -#ifndef SIMDJSON_ICELAKE_INTRINSICS_H -#define SIMDJSON_ICELAKE_INTRINSICS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/icelake/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#if SIMDJSON_VISUAL_STUDIO -// under clang within visual studio, this will include -#include // visual studio or clang -#else -#include // elsewhere -#endif // SIMDJSON_VISUAL_STUDIO - -#if SIMDJSON_CLANG_VISUAL_STUDIO -/** - * You are not supposed, normally, to include these - * headers directly. Instead you should either include intrin.h - * or x86intrin.h. However, when compiling with clang - * under Windows (i.e., when _MSC_VER is set), these headers - * only get included *if* the corresponding features are detected - * from macros: - * e.g., if __AVX2__ is set... in turn, we normally set these - * macros by compiling against the corresponding architecture - * (e.g., arch:AVX2, -mavx2, etc.) which compiles the whole - * software with these advanced instructions. In simdjson, we - * want to compile the whole program for a generic target, - * and only target our specific kernels. As a workaround, - * we directly include the needed headers. These headers would - * normally guard against such usage, but we carefully included - * (or ) before, so the headers - * are fooled. - */ -#include // for _blsr_u64 -#include // for __lzcnt64 -#include // for most things (AVX2, AVX512, _popcnt64) -#include -#include -#include -#include -#include // for _mm_clmulepi64_si128 - // Important: we need the AVX-512 headers: -#include -#include -#include -#include -#include -#include -#include -// unfortunately, we may not get _blsr_u64, but, thankfully, clang -// has it as a macro. -#ifndef _blsr_u64 -// we roll our own -#define _blsr_u64(n) ((n - 1) & n) -#endif // _blsr_u64 -#endif // SIMDJSON_CLANG_VISUAL_STUDIO - -static_assert(sizeof(__m512i) <= simdjson::SIMDJSON_PADDING, "insufficient padding for icelake"); - -#endif // SIMDJSON_ICELAKE_INTRINSICS_H -/* end file simdjson/icelake/intrinsics.h */ - -#if !SIMDJSON_CAN_ALWAYS_RUN_ICELAKE -SIMDJSON_TARGET_REGION("avx512f,avx512dq,avx512cd,avx512bw,avx512vbmi,avx512vbmi2,avx512vl,avx2,bmi,pclmul,lzcnt,popcnt") -#endif - -/* including simdjson/icelake/bitmanipulation.h: #include "simdjson/icelake/bitmanipulation.h" */ -/* begin file simdjson/icelake/bitmanipulation.h */ -#ifndef SIMDJSON_ICELAKE_BITMANIPULATION_H -#define SIMDJSON_ICELAKE_BITMANIPULATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/icelake/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/icelake/intrinsics.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - namespace icelake { - namespace { - - // We sometimes call trailing_zero on inputs that are zero, - // but the algorithms do not end up using the returned value. - // Sadly, sanitizers are not smart enough to figure it out. - SIMDJSON_NO_SANITIZE_UNDEFINED - // This function can be used safely even if not all bytes have been - // initialized. - // See issue https://github.com/simdjson/simdjson/issues/1965 - SIMDJSON_NO_SANITIZE_MEMORY - simdjson_inline int trailing_zeroes(uint64_t input_num) { -#if SIMDJSON_REGULAR_VISUAL_STUDIO - return (int)_tzcnt_u64(input_num); -#else // SIMDJSON_REGULAR_VISUAL_STUDIO - //////// - // You might expect the next line to be equivalent to - // return (int)_tzcnt_u64(input_num); - // but the generated code differs and might be less efficient? - //////// - return __builtin_ctzll(input_num); -#endif // SIMDJSON_REGULAR_VISUAL_STUDIO - } - - /* result might be undefined when input_num is zero */ - simdjson_inline uint64_t clear_lowest_bit(uint64_t input_num) { - return _blsr_u64(input_num); - } - - /* result might be undefined when input_num is zero */ - simdjson_inline int leading_zeroes(uint64_t input_num) { - return int(_lzcnt_u64(input_num)); - } - -#if SIMDJSON_REGULAR_VISUAL_STUDIO - simdjson_inline unsigned __int64 count_ones(uint64_t input_num) { - // note: we do not support legacy 32-bit Windows - return __popcnt64(input_num);// Visual Studio wants two underscores - } -#else - simdjson_inline long long int count_ones(uint64_t input_num) { - return _popcnt64(input_num); - } -#endif - - simdjson_inline bool add_overflow(uint64_t value1, uint64_t value2, - uint64_t* result) { -#if SIMDJSON_REGULAR_VISUAL_STUDIO - return _addcarry_u64(0, value1, value2, - reinterpret_cast(result)); -#else - return __builtin_uaddll_overflow(value1, value2, - reinterpret_cast(result)); -#endif - } - - } // unnamed namespace - } // namespace icelake -} // namespace simdjson - -#endif // SIMDJSON_ICELAKE_BITMANIPULATION_H -/* end file simdjson/icelake/bitmanipulation.h */ -/* including simdjson/icelake/bitmask.h: #include "simdjson/icelake/bitmask.h" */ -/* begin file simdjson/icelake/bitmask.h */ -#ifndef SIMDJSON_ICELAKE_BITMASK_H -#define SIMDJSON_ICELAKE_BITMASK_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/icelake/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/icelake/intrinsics.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - namespace icelake { - namespace { - - // - // Perform a "cumulative bitwise xor," flipping bits each time a 1 is encountered. - // - // For example, prefix_xor(00100100) == 00011100 - // - simdjson_inline uint64_t prefix_xor(const uint64_t bitmask) { - // There should be no such thing with a processor supporting avx2 - // but not clmul. - __m128i all_ones = _mm_set1_epi8('\xFF'); - __m128i result = _mm_clmulepi64_si128(_mm_set_epi64x(0ULL, bitmask), all_ones, 0); - return _mm_cvtsi128_si64(result); - } - - } // unnamed namespace - } // namespace icelake -} // namespace simdjson - -#endif // SIMDJSON_ICELAKE_BITMASK_H -/* end file simdjson/icelake/bitmask.h */ -/* including simdjson/icelake/simd.h: #include "simdjson/icelake/simd.h" */ -/* begin file simdjson/icelake/simd.h */ -#ifndef SIMDJSON_ICELAKE_SIMD_H -#define SIMDJSON_ICELAKE_SIMD_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/icelake/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/icelake/intrinsics.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/icelake/bitmanipulation.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/simdprune_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#if defined(__GNUC__) && !defined(__clang__) -#if __GNUC__ == 8 -#define SIMDJSON_GCC8 1 -#endif // __GNUC__ == 8 -#endif // defined(__GNUC__) && !defined(__clang__) - -#if SIMDJSON_GCC8 -/** - * GCC 8 fails to provide _mm512_set_epi8. We roll our own. - */ -inline __m512i _mm512_set_epi8(uint8_t a0, uint8_t a1, uint8_t a2, uint8_t a3, uint8_t a4, uint8_t a5, uint8_t a6, uint8_t a7, uint8_t a8, uint8_t a9, uint8_t a10, uint8_t a11, uint8_t a12, uint8_t a13, uint8_t a14, uint8_t a15, uint8_t a16, uint8_t a17, uint8_t a18, uint8_t a19, uint8_t a20, uint8_t a21, uint8_t a22, uint8_t a23, uint8_t a24, uint8_t a25, uint8_t a26, uint8_t a27, uint8_t a28, uint8_t a29, uint8_t a30, uint8_t a31, uint8_t a32, uint8_t a33, uint8_t a34, uint8_t a35, uint8_t a36, uint8_t a37, uint8_t a38, uint8_t a39, uint8_t a40, uint8_t a41, uint8_t a42, uint8_t a43, uint8_t a44, uint8_t a45, uint8_t a46, uint8_t a47, uint8_t a48, uint8_t a49, uint8_t a50, uint8_t a51, uint8_t a52, uint8_t a53, uint8_t a54, uint8_t a55, uint8_t a56, uint8_t a57, uint8_t a58, uint8_t a59, uint8_t a60, uint8_t a61, uint8_t a62, uint8_t a63) { - return _mm512_set_epi64(uint64_t(a7) + (uint64_t(a6) << 8) + (uint64_t(a5) << 16) + (uint64_t(a4) << 24) + (uint64_t(a3) << 32) + (uint64_t(a2) << 40) + (uint64_t(a1) << 48) + (uint64_t(a0) << 56), - uint64_t(a15) + (uint64_t(a14) << 8) + (uint64_t(a13) << 16) + (uint64_t(a12) << 24) + (uint64_t(a11) << 32) + (uint64_t(a10) << 40) + (uint64_t(a9) << 48) + (uint64_t(a8) << 56), - uint64_t(a23) + (uint64_t(a22) << 8) + (uint64_t(a21) << 16) + (uint64_t(a20) << 24) + (uint64_t(a19) << 32) + (uint64_t(a18) << 40) + (uint64_t(a17) << 48) + (uint64_t(a16) << 56), - uint64_t(a31) + (uint64_t(a30) << 8) + (uint64_t(a29) << 16) + (uint64_t(a28) << 24) + (uint64_t(a27) << 32) + (uint64_t(a26) << 40) + (uint64_t(a25) << 48) + (uint64_t(a24) << 56), - uint64_t(a39) + (uint64_t(a38) << 8) + (uint64_t(a37) << 16) + (uint64_t(a36) << 24) + (uint64_t(a35) << 32) + (uint64_t(a34) << 40) + (uint64_t(a33) << 48) + (uint64_t(a32) << 56), - uint64_t(a47) + (uint64_t(a46) << 8) + (uint64_t(a45) << 16) + (uint64_t(a44) << 24) + (uint64_t(a43) << 32) + (uint64_t(a42) << 40) + (uint64_t(a41) << 48) + (uint64_t(a40) << 56), - uint64_t(a55) + (uint64_t(a54) << 8) + (uint64_t(a53) << 16) + (uint64_t(a52) << 24) + (uint64_t(a51) << 32) + (uint64_t(a50) << 40) + (uint64_t(a49) << 48) + (uint64_t(a48) << 56), - uint64_t(a63) + (uint64_t(a62) << 8) + (uint64_t(a61) << 16) + (uint64_t(a60) << 24) + (uint64_t(a59) << 32) + (uint64_t(a58) << 40) + (uint64_t(a57) << 48) + (uint64_t(a56) << 56)); -} -#endif // SIMDJSON_GCC8 - - - -namespace simdjson { - namespace icelake { - namespace { - namespace simd { - - // Forward-declared so they can be used by splat and friends. - template - struct base { - __m512i value; - - // Zero constructor - simdjson_inline base() : value{ __m512i() } {} - - // Conversion from SIMD register - simdjson_inline base(const __m512i _value) : value(_value) {} - - // Conversion to SIMD register - simdjson_inline operator const __m512i& () const { return this->value; } - simdjson_inline operator __m512i& () { return this->value; } - - // Bit operations - simdjson_inline Child operator|(const Child other) const { return _mm512_or_si512(*this, other); } - simdjson_inline Child operator&(const Child other) const { return _mm512_and_si512(*this, other); } - simdjson_inline Child operator^(const Child other) const { return _mm512_xor_si512(*this, other); } - simdjson_inline Child bit_andnot(const Child other) const { return _mm512_andnot_si512(other, *this); } - simdjson_inline Child& operator|=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast | other; return *this_cast; } - simdjson_inline Child& operator&=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast & other; return *this_cast; } - simdjson_inline Child& operator^=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast ^ other; return *this_cast; } - }; - - // Forward-declared so they can be used by splat and friends. - template - struct simd8; - - template> - struct base8 : base> { - typedef uint32_t bitmask_t; - typedef uint64_t bitmask2_t; - - simdjson_inline base8() : base>() {} - simdjson_inline base8(const __m512i _value) : base>(_value) {} - - friend simdjson_really_inline uint64_t operator==(const simd8 lhs, const simd8 rhs) { - return _mm512_cmpeq_epi8_mask(lhs, rhs); - } - - static const int SIZE = sizeof(base::value); - - template - simdjson_inline simd8 prev(const simd8 prev_chunk) const { - // workaround for compilers unable to figure out that 16 - N is a constant (GCC 8) - constexpr int shift = 16 - N; - return _mm512_alignr_epi8(*this, _mm512_permutex2var_epi64(prev_chunk, _mm512_set_epi64(13, 12, 11, 10, 9, 8, 7, 6), *this), shift); - } - }; - - // SIMD byte mask type (returned by things like eq and gt) - template<> - struct simd8 : base8 { - static simdjson_inline simd8 splat(bool _value) { return _mm512_set1_epi8(uint8_t(-(!!_value))); } - - simdjson_inline simd8() : base8() {} - simdjson_inline simd8(const __m512i _value) : base8(_value) {} - // Splat constructor - simdjson_inline simd8(bool _value) : base8(splat(_value)) {} - simdjson_inline bool any() const { return !!_mm512_test_epi8_mask(*this, *this); } - simdjson_inline simd8 operator~() const { return *this ^ true; } - }; - - template - struct base8_numeric : base8 { - static simdjson_inline simd8 splat(T _value) { return _mm512_set1_epi8(_value); } - static simdjson_inline simd8 zero() { return _mm512_setzero_si512(); } - static simdjson_inline simd8 load(const T values[64]) { - return _mm512_loadu_si512(reinterpret_cast(values)); - } - // Repeat 16 values as many times as necessary (usually for lookup tables) - static simdjson_inline simd8 repeat_16( - T v0, T v1, T v2, T v3, T v4, T v5, T v6, T v7, - T v8, T v9, T v10, T v11, T v12, T v13, T v14, T v15 - ) { - return simd8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10, v11, v12, v13, v14, v15, - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10, v11, v12, v13, v14, v15, - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10, v11, v12, v13, v14, v15, - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10, v11, v12, v13, v14, v15 - ); - } - - simdjson_inline base8_numeric() : base8() {} - simdjson_inline base8_numeric(const __m512i _value) : base8(_value) {} - - // Store to array - simdjson_inline void store(T dst[64]) const { return _mm512_storeu_si512(reinterpret_cast<__m512i*>(dst), *this); } - - // Addition/subtraction are the same for signed and unsigned - simdjson_inline simd8 operator+(const simd8 other) const { return _mm512_add_epi8(*this, other); } - simdjson_inline simd8 operator-(const simd8 other) const { return _mm512_sub_epi8(*this, other); } - simdjson_inline simd8& operator+=(const simd8 other) { *this = *this + other; return *static_cast*>(this); } - simdjson_inline simd8& operator-=(const simd8 other) { *this = *this - other; return *static_cast*>(this); } - - // Override to distinguish from bool version - simdjson_inline simd8 operator~() const { return *this ^ 0xFFu; } - - // Perform a lookup assuming the value is between 0 and 16 (undefined behavior for out of range values) - template - simdjson_inline simd8 lookup_16(simd8 lookup_table) const { - return _mm512_shuffle_epi8(lookup_table, *this); - } - - // Copies to 'output" all bytes corresponding to a 0 in the mask (interpreted as a bitset). - // Passing a 0 value for mask would be equivalent to writing out every byte to output. - // Only the first 32 - count_ones(mask) bytes of the result are significant but 32 bytes - // get written. - // Design consideration: it seems like a function with the - // signature simd8 compress(uint32_t mask) would be - // sensible, but the AVX ISA makes this kind of approach difficult. - template - simdjson_inline void compress(uint64_t mask, L* output) const { - _mm512_mask_compressstoreu_epi8(output, ~mask, *this); - } - - template - simdjson_inline simd8 lookup_16( - L replace0, L replace1, L replace2, L replace3, - L replace4, L replace5, L replace6, L replace7, - L replace8, L replace9, L replace10, L replace11, - L replace12, L replace13, L replace14, L replace15) const { - return lookup_16(simd8::repeat_16( - replace0, replace1, replace2, replace3, - replace4, replace5, replace6, replace7, - replace8, replace9, replace10, replace11, - replace12, replace13, replace14, replace15 - )); - } - }; - - // Signed bytes - template<> - struct simd8 : base8_numeric { - simdjson_inline simd8() : base8_numeric() {} - simdjson_inline simd8(const __m512i _value) : base8_numeric(_value) {} - // Splat constructor - simdjson_inline simd8(int8_t _value) : simd8(splat(_value)) {} - // Array constructor - simdjson_inline simd8(const int8_t values[64]) : simd8(load(values)) {} - // Member-by-member initialization - simdjson_inline simd8( - int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, - int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15, - int8_t v16, int8_t v17, int8_t v18, int8_t v19, int8_t v20, int8_t v21, int8_t v22, int8_t v23, - int8_t v24, int8_t v25, int8_t v26, int8_t v27, int8_t v28, int8_t v29, int8_t v30, int8_t v31, - int8_t v32, int8_t v33, int8_t v34, int8_t v35, int8_t v36, int8_t v37, int8_t v38, int8_t v39, - int8_t v40, int8_t v41, int8_t v42, int8_t v43, int8_t v44, int8_t v45, int8_t v46, int8_t v47, - int8_t v48, int8_t v49, int8_t v50, int8_t v51, int8_t v52, int8_t v53, int8_t v54, int8_t v55, - int8_t v56, int8_t v57, int8_t v58, int8_t v59, int8_t v60, int8_t v61, int8_t v62, int8_t v63 - ) : simd8(_mm512_set_epi8( - v63, v62, v61, v60, v59, v58, v57, v56, - v55, v54, v53, v52, v51, v50, v49, v48, - v47, v46, v45, v44, v43, v42, v41, v40, - v39, v38, v37, v36, v35, v34, v33, v32, - v31, v30, v29, v28, v27, v26, v25, v24, - v23, v22, v21, v20, v19, v18, v17, v16, - v15, v14, v13, v12, v11, v10, v9, v8, - v7, v6, v5, v4, v3, v2, v1, v0 - )) {} - - // Repeat 16 values as many times as necessary (usually for lookup tables) - simdjson_inline static simd8 repeat_16( - int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, - int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 - ) { - return simd8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10, v11, v12, v13, v14, v15, - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10, v11, v12, v13, v14, v15, - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10, v11, v12, v13, v14, v15, - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10, v11, v12, v13, v14, v15 - ); - } - - // Order-sensitive comparisons - simdjson_inline simd8 max_val(const simd8 other) const { return _mm512_max_epi8(*this, other); } - simdjson_inline simd8 min_val(const simd8 other) const { return _mm512_min_epi8(*this, other); } - - simdjson_inline simd8 operator>(const simd8 other) const { return _mm512_maskz_abs_epi8(_mm512_cmpgt_epi8_mask(*this, other), _mm512_set1_epi8(uint8_t(0x80))); } - simdjson_inline simd8 operator<(const simd8 other) const { return _mm512_maskz_abs_epi8(_mm512_cmpgt_epi8_mask(other, *this), _mm512_set1_epi8(uint8_t(0x80))); } - }; - - // Unsigned bytes - template<> - struct simd8 : base8_numeric { - simdjson_inline simd8() : base8_numeric() {} - simdjson_inline simd8(const __m512i _value) : base8_numeric(_value) {} - // Splat constructor - simdjson_inline simd8(uint8_t _value) : simd8(splat(_value)) {} - // Array constructor - simdjson_inline simd8(const uint8_t values[64]) : simd8(load(values)) {} - // Member-by-member initialization - simdjson_inline simd8( - uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, - uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15, - uint8_t v16, uint8_t v17, uint8_t v18, uint8_t v19, uint8_t v20, uint8_t v21, uint8_t v22, uint8_t v23, - uint8_t v24, uint8_t v25, uint8_t v26, uint8_t v27, uint8_t v28, uint8_t v29, uint8_t v30, uint8_t v31, - uint8_t v32, uint8_t v33, uint8_t v34, uint8_t v35, uint8_t v36, uint8_t v37, uint8_t v38, uint8_t v39, - uint8_t v40, uint8_t v41, uint8_t v42, uint8_t v43, uint8_t v44, uint8_t v45, uint8_t v46, uint8_t v47, - uint8_t v48, uint8_t v49, uint8_t v50, uint8_t v51, uint8_t v52, uint8_t v53, uint8_t v54, uint8_t v55, - uint8_t v56, uint8_t v57, uint8_t v58, uint8_t v59, uint8_t v60, uint8_t v61, uint8_t v62, uint8_t v63 - ) : simd8(_mm512_set_epi8( - v63, v62, v61, v60, v59, v58, v57, v56, - v55, v54, v53, v52, v51, v50, v49, v48, - v47, v46, v45, v44, v43, v42, v41, v40, - v39, v38, v37, v36, v35, v34, v33, v32, - v31, v30, v29, v28, v27, v26, v25, v24, - v23, v22, v21, v20, v19, v18, v17, v16, - v15, v14, v13, v12, v11, v10, v9, v8, - v7, v6, v5, v4, v3, v2, v1, v0 - )) {} - - // Repeat 16 values as many times as necessary (usually for lookup tables) - simdjson_inline static simd8 repeat_16( - uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, - uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 - ) { - return simd8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10, v11, v12, v13, v14, v15, - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10, v11, v12, v13, v14, v15, - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10, v11, v12, v13, v14, v15, - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10, v11, v12, v13, v14, v15 - ); - } - - // Saturated math - simdjson_inline simd8 saturating_add(const simd8 other) const { return _mm512_adds_epu8(*this, other); } - simdjson_inline simd8 saturating_sub(const simd8 other) const { return _mm512_subs_epu8(*this, other); } - - // Order-specific operations - simdjson_inline simd8 max_val(const simd8 other) const { return _mm512_max_epu8(*this, other); } - simdjson_inline simd8 min_val(const simd8 other) const { return _mm512_min_epu8(other, *this); } - // Same as >, but only guarantees true is nonzero (< guarantees true = -1) - simdjson_inline simd8 gt_bits(const simd8 other) const { return this->saturating_sub(other); } - // Same as <, but only guarantees true is nonzero (< guarantees true = -1) - simdjson_inline simd8 lt_bits(const simd8 other) const { return other.saturating_sub(*this); } - simdjson_inline uint64_t operator<=(const simd8 other) const { return other.max_val(*this) == other; } - simdjson_inline uint64_t operator>=(const simd8 other) const { return other.min_val(*this) == other; } - simdjson_inline simd8 operator>(const simd8 other) const { return this->gt_bits(other).any_bits_set(); } - simdjson_inline simd8 operator<(const simd8 other) const { return this->lt_bits(other).any_bits_set(); } - - // Bit-specific operations - simdjson_inline simd8 bits_not_set() const { return _mm512_mask_blend_epi8(*this == uint8_t(0), _mm512_set1_epi8(0), _mm512_set1_epi8(-1)); } - simdjson_inline simd8 bits_not_set(simd8 bits) const { return (*this & bits).bits_not_set(); } - simdjson_inline simd8 any_bits_set() const { return ~this->bits_not_set(); } - simdjson_inline simd8 any_bits_set(simd8 bits) const { return ~this->bits_not_set(bits); } - - simdjson_inline bool is_ascii() const { return _mm512_movepi8_mask(*this) == 0; } - simdjson_inline bool bits_not_set_anywhere() const { - return !_mm512_test_epi8_mask(*this, *this); - } - simdjson_inline bool any_bits_set_anywhere() const { return !bits_not_set_anywhere(); } - simdjson_inline bool bits_not_set_anywhere(simd8 bits) const { return !_mm512_test_epi8_mask(*this, bits); } - simdjson_inline bool any_bits_set_anywhere(simd8 bits) const { return !bits_not_set_anywhere(bits); } - template - simdjson_inline simd8 shr() const { return simd8(_mm512_srli_epi16(*this, N)) & uint8_t(0xFFu >> N); } - template - simdjson_inline simd8 shl() const { return simd8(_mm512_slli_epi16(*this, N)) & uint8_t(0xFFu << N); } - // Get one of the bits and make a bitmask out of it. - // e.g. value.get_bit<7>() gets the high bit - template - simdjson_inline uint64_t get_bit() const { return _mm512_movepi8_mask(_mm512_slli_epi16(*this, 7 - N)); } - }; - - template - struct simd8x64 { - static constexpr int NUM_CHUNKS = 64 / sizeof(simd8); - static_assert(NUM_CHUNKS == 1, "Icelake kernel should use one register per 64-byte block."); - const simd8 chunks[NUM_CHUNKS]; - - simd8x64(const simd8x64& o) = delete; // no copy allowed - simd8x64& operator=(const simd8& other) = delete; // no assignment allowed - simd8x64() = delete; // no default constructor allowed - - simdjson_inline simd8x64(const simd8 chunk0, const simd8 chunk1) : chunks{ chunk0, chunk1 } {} - simdjson_inline simd8x64(const simd8 chunk0) : chunks{ chunk0 } {} - simdjson_inline simd8x64(const T ptr[64]) : chunks{ simd8::load(ptr) } {} - - simdjson_inline uint64_t compress(uint64_t mask, T* output) const { - this->chunks[0].compress(mask, output); - return 64 - count_ones(mask); - } - - simdjson_inline void store(T ptr[64]) const { - this->chunks[0].store(ptr + sizeof(simd8) * 0); - } - - simdjson_inline simd8 reduce_or() const { - return this->chunks[0]; - } - - simdjson_inline simd8x64 bit_or(const T m) const { - const simd8 mask = simd8::splat(m); - return simd8x64( - this->chunks[0] | mask - ); - } - - simdjson_inline uint64_t eq(const T m) const { - const simd8 mask = simd8::splat(m); - return this->chunks[0] == mask; - } - - simdjson_inline uint64_t eq(const simd8x64& other) const { - return this->chunks[0] == other.chunks[0]; - } - - simdjson_inline uint64_t lteq(const T m) const { - const simd8 mask = simd8::splat(m); - return this->chunks[0] <= mask; - } - }; // struct simd8x64 - - } // namespace simd - - } // unnamed namespace - } // namespace icelake -} // namespace simdjson - -#endif // SIMDJSON_ICELAKE_SIMD_H -/* end file simdjson/icelake/simd.h */ -/* including simdjson/icelake/stringparsing_defs.h: #include "simdjson/icelake/stringparsing_defs.h" */ -/* begin file simdjson/icelake/stringparsing_defs.h */ -#ifndef SIMDJSON_ICELAKE_STRINGPARSING_DEFS_H -#define SIMDJSON_ICELAKE_STRINGPARSING_DEFS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/icelake/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/icelake/simd.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/icelake/bitmanipulation.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - namespace icelake { - namespace { - - using namespace simd; - - // Holds backslashes and quotes locations. - struct backslash_and_quote { - public: - static constexpr uint32_t BYTES_PROCESSED = 64; - simdjson_inline static backslash_and_quote copy_and_find(const uint8_t* src, uint8_t* dst); - - simdjson_inline bool has_quote_first() { return ((bs_bits - 1) & quote_bits) != 0; } - simdjson_inline bool has_backslash() { return ((quote_bits - 1) & bs_bits) != 0; } - simdjson_inline int quote_index() { return trailing_zeroes(quote_bits); } - simdjson_inline int backslash_index() { return trailing_zeroes(bs_bits); } - - uint64_t bs_bits; - uint64_t quote_bits; - }; // struct backslash_and_quote - - simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8_t* src, uint8_t* dst) { - // this can read up to 15 bytes beyond the buffer size, but we require - // SIMDJSON_PADDING of padding - static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "backslash and quote finder must process fewer than SIMDJSON_PADDING bytes"); - simd8 v(src); - // store to dest unconditionally - we can overwrite the bits we don't like later - v.store(dst); - return { - static_cast(v == '\\'), // bs_bits - static_cast(v == '"'), // quote_bits - }; - } - - } // unnamed namespace - } // namespace icelake -} // namespace simdjson - -#endif // SIMDJSON_ICELAKE_STRINGPARSING_DEFS_H -/* end file simdjson/icelake/stringparsing_defs.h */ -/* including simdjson/icelake/numberparsing_defs.h: #include "simdjson/icelake/numberparsing_defs.h" */ -/* begin file simdjson/icelake/numberparsing_defs.h */ -#ifndef SIMDJSON_ICELAKE_NUMBERPARSING_DEFS_H -#define SIMDJSON_ICELAKE_NUMBERPARSING_DEFS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/icelake/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/icelake/intrinsics.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - namespace icelake { - namespace numberparsing { - - static simdjson_inline uint32_t parse_eight_digits_unrolled(const uint8_t* chars) { - // this actually computes *16* values so we are being wasteful. - const __m128i ascii0 = _mm_set1_epi8('0'); - const __m128i mul_1_10 = - _mm_setr_epi8(10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1); - const __m128i mul_1_100 = _mm_setr_epi16(100, 1, 100, 1, 100, 1, 100, 1); - const __m128i mul_1_10000 = - _mm_setr_epi16(10000, 1, 10000, 1, 10000, 1, 10000, 1); - const __m128i input = _mm_sub_epi8( - _mm_loadu_si128(reinterpret_cast(chars)), ascii0); - const __m128i t1 = _mm_maddubs_epi16(input, mul_1_10); - const __m128i t2 = _mm_madd_epi16(t1, mul_1_100); - const __m128i t3 = _mm_packus_epi32(t2, t2); - const __m128i t4 = _mm_madd_epi16(t3, mul_1_10000); - return _mm_cvtsi128_si32( - t4); // only captures the sum of the first 8 digits, drop the rest - } - - /** @private */ - simdjson_inline internal::value128 full_multiplication(uint64_t value1, uint64_t value2) { - internal::value128 answer; -#if SIMDJSON_REGULAR_VISUAL_STUDIO || SIMDJSON_IS_32BITS -#ifdef _M_ARM64 - // ARM64 has native support for 64-bit multiplications, no need to emultate - answer.high = __umulh(value1, value2); - answer.low = value1 * value2; -#else - answer.low = _umul128(value1, value2, &answer.high); // _umul128 not available on ARM64 -#endif // _M_ARM64 -#else // SIMDJSON_REGULAR_VISUAL_STUDIO || SIMDJSON_IS_32BITS - __uint128_t r = (static_cast<__uint128_t>(value1)) * value2; - answer.low = uint64_t(r); - answer.high = uint64_t(r >> 64); -#endif - return answer; - } - - } // namespace numberparsing - } // namespace icelake -} // namespace simdjson - -#define SIMDJSON_SWAR_NUMBER_PARSING 1 - -#endif // SIMDJSON_ICELAKE_NUMBERPARSING_DEFS_H -/* end file simdjson/icelake/numberparsing_defs.h */ -/* end file simdjson/icelake/begin.h */ -/* including generic/amalgamated.h for icelake: #include */ -/* begin file generic/amalgamated.h for icelake */ -#if defined(SIMDJSON_CONDITIONAL_INCLUDE) && !defined(SIMDJSON_SRC_GENERIC_DEPENDENCIES_H) -#error generic/dependencies.h must be included before generic/amalgamated.h! -#endif - -/* including generic/base.h for icelake: #include */ -/* begin file generic/base.h for icelake */ -#ifndef SIMDJSON_SRC_GENERIC_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_BASE_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - namespace icelake { - namespace { - - struct json_character_block; - - } // unnamed namespace - } // namespace icelake -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_BASE_H -/* end file generic/base.h for icelake */ -/* including generic/dom_parser_implementation.h for icelake: #include */ -/* begin file generic/dom_parser_implementation.h for icelake */ -#ifndef SIMDJSON_SRC_GENERIC_DOM_PARSER_IMPLEMENTATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_DOM_PARSER_IMPLEMENTATION_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -// Interface a dom parser implementation must fulfill -namespace simdjson { - namespace icelake { - namespace { - - simdjson_inline simd8 must_be_2_3_continuation(const simd8 prev2, const simd8 prev3); - simdjson_inline bool is_ascii(const simd8x64& input); - - } // unnamed namespace - } // namespace icelake -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_DOM_PARSER_IMPLEMENTATION_H -/* end file generic/dom_parser_implementation.h for icelake */ -/* including generic/json_character_block.h for icelake: #include */ -/* begin file generic/json_character_block.h for icelake */ -#ifndef SIMDJSON_SRC_GENERIC_JSON_CHARACTER_BLOCK_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_JSON_CHARACTER_BLOCK_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - namespace icelake { - namespace { - - struct json_character_block { - static simdjson_inline json_character_block classify(const simd::simd8x64& in); - - simdjson_inline uint64_t whitespace() const noexcept { return _whitespace; } - simdjson_inline uint64_t op() const noexcept { return _op; } - simdjson_inline uint64_t scalar() const noexcept { return ~(op() | whitespace()); } - - uint64_t _whitespace; - uint64_t _op; - }; - - } // unnamed namespace - } // namespace icelake -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_JSON_CHARACTER_BLOCK_H -/* end file generic/json_character_block.h for icelake */ -/* end file generic/amalgamated.h for icelake */ -/* including generic/stage1/amalgamated.h for icelake: #include */ -/* begin file generic/stage1/amalgamated.h for icelake */ -// Stuff other things depend on -/* including generic/stage1/base.h for icelake: #include */ -/* begin file generic/stage1/base.h for icelake */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_BASE_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - namespace icelake { - namespace { - namespace stage1 { - - class bit_indexer; - template - struct buf_block_reader; - struct json_block; - class json_minifier; - class json_scanner; - struct json_string_block; - class json_string_scanner; - class json_structural_indexer; - - } // namespace stage1 - - namespace utf8_validation { - struct utf8_checker; - } // namespace utf8_validation - - using utf8_validation::utf8_checker; - - } // unnamed namespace - } // namespace icelake -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_BASE_H -/* end file generic/stage1/base.h for icelake */ -/* including generic/stage1/buf_block_reader.h for icelake: #include */ -/* begin file generic/stage1/buf_block_reader.h for icelake */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_BUF_BLOCK_READER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_BUF_BLOCK_READER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include - -namespace simdjson { - namespace icelake { - namespace { - namespace stage1 { - - // Walks through a buffer in block-sized increments, loading the last part with spaces - template - struct buf_block_reader { - public: - simdjson_inline buf_block_reader(const uint8_t* _buf, size_t _len); - simdjson_inline size_t block_index(); - simdjson_inline bool has_full_block() const; - simdjson_inline const uint8_t* full_block() const; - /** - * Get the last block, padded with spaces. - * - * There will always be a last block, with at least 1 byte, unless len == 0 (in which case this - * function fills the buffer with spaces and returns 0. In particular, if len == STEP_SIZE there - * will be 0 full_blocks and 1 remainder block with STEP_SIZE bytes and no spaces for padding. - * - * @return the number of effective characters in the last block. - */ - simdjson_inline size_t get_remainder(uint8_t* dst) const; - simdjson_inline void advance(); - private: - const uint8_t* buf; - const size_t len; - const size_t lenminusstep; - size_t idx; - }; - - // Routines to print masks and text for debugging bitmask operations - simdjson_unused static char* format_input_text_64(const uint8_t* text) { - static char buf[sizeof(simd8x64) + 1]; - for (size_t i = 0; i < sizeof(simd8x64); i++) { - buf[i] = int8_t(text[i]) < ' ' ? '_' : int8_t(text[i]); - } - buf[sizeof(simd8x64)] = '\0'; - return buf; - } - - // Routines to print masks and text for debugging bitmask operations - simdjson_unused static char* format_input_text(const simd8x64& in) { - static char buf[sizeof(simd8x64) + 1]; - in.store(reinterpret_cast(buf)); - for (size_t i = 0; i < sizeof(simd8x64); i++) { - if (buf[i] < ' ') { buf[i] = '_'; } - } - buf[sizeof(simd8x64)] = '\0'; - return buf; - } - - simdjson_unused static char* format_input_text(const simd8x64& in, uint64_t mask) { - static char buf[sizeof(simd8x64) + 1]; - in.store(reinterpret_cast(buf)); - for (size_t i = 0; i < sizeof(simd8x64); i++) { - if (buf[i] <= ' ') { buf[i] = '_'; } - if (!(mask & (size_t(1) << i))) { buf[i] = ' '; } - } - buf[sizeof(simd8x64)] = '\0'; - return buf; - } - - simdjson_unused static char* format_mask(uint64_t mask) { - static char buf[sizeof(simd8x64) + 1]; - for (size_t i = 0; i < 64; i++) { - buf[i] = (mask & (size_t(1) << i)) ? 'X' : ' '; - } - buf[64] = '\0'; - return buf; - } - - template - simdjson_inline buf_block_reader::buf_block_reader(const uint8_t* _buf, size_t _len) : buf{ _buf }, len{ _len }, lenminusstep{ len < STEP_SIZE ? 0 : len - STEP_SIZE }, idx{ 0 } {} - - template - simdjson_inline size_t buf_block_reader::block_index() { return idx; } - - template - simdjson_inline bool buf_block_reader::has_full_block() const { - return idx < lenminusstep; - } - - template - simdjson_inline const uint8_t* buf_block_reader::full_block() const { - return &buf[idx]; - } - - template - simdjson_inline size_t buf_block_reader::get_remainder(uint8_t* dst) const { - if (len == idx) { return 0; } // memcpy(dst, null, 0) will trigger an error with some sanitizers - std::memset(dst, 0x20, STEP_SIZE); // std::memset STEP_SIZE because it's more efficient to write out 8 or 16 bytes at once. - std::memcpy(dst, buf + idx, len - idx); - return len - idx; - } - - template - simdjson_inline void buf_block_reader::advance() { - idx += STEP_SIZE; - } - - } // namespace stage1 - } // unnamed namespace - } // namespace icelake -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_BUF_BLOCK_READER_H -/* end file generic/stage1/buf_block_reader.h for icelake */ -/* including generic/stage1/json_escape_scanner.h for icelake: #include */ -/* begin file generic/stage1/json_escape_scanner.h for icelake */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_JSON_ESCAPE_SCANNER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_JSON_ESCAPE_SCANNER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - namespace icelake { - namespace { - namespace stage1 { - - /** - * Scans for escape characters in JSON, taking care with multiple backslashes (\\n vs. \n). - */ - struct json_escape_scanner { - /** The actual escape characters (the backslashes themselves). */ - uint64_t next_is_escaped = 0ULL; - - struct escaped_and_escape { - /** - * Mask of escaped characters. - * - * ``` - * \n \\n \\\n \\\\n \ - * 0100100010100101000 - * n \ \ n \ \ - * ``` - */ - uint64_t escaped; - /** - * Mask of escape characters. - * - * ``` - * \n \\n \\\n \\\\n \ - * 1001000101001010001 - * \ \ \ \ \ \ \ - * ``` - */ - uint64_t escape; - }; - - /** - * Get a mask of both escape and escaped characters (the characters following a backslash). - * - * @param potential_escape A mask of the character that can escape others (but could be - * escaped itself). e.g. block.eq('\\') - */ - simdjson_really_inline escaped_and_escape next(uint64_t backslash) noexcept { - -#if !SIMDJSON_SKIP_BACKSLASH_SHORT_CIRCUIT - if (!backslash) { return { next_escaped_without_backslashes(), 0 }; } -#endif - - // | | Mask (shows characters instead of 1's) | Depth | Instructions | - // |--------------------------------|----------------------------------------|-------|---------------------| - // | string | `\\n_\\\n___\\\n___\\\\___\\\\__\\\` | | | - // | | ` even odd even odd odd` | | | - // | potential_escape | ` \ \\\ \\\ \\\\ \\\\ \\\` | 1 | 1 (backslash & ~first_is_escaped) - // | escape_and_terminal_code | ` \n \ \n \ \n \ \ \ \ \ \` | 5 | 5 (next_escape_and_terminal_code()) - // | escaped | `\ \ n \ n \ \ \ \ \ ` X | 6 | 7 (escape_and_terminal_code ^ (potential_escape | first_is_escaped)) - // | escape | ` \ \ \ \ \ \ \ \ \ \` | 6 | 8 (escape_and_terminal_code & backslash) - // | first_is_escaped | `\ ` | 7 (*) | 9 (escape >> 63) () - // (*) this is not needed until the next iteration - uint64_t escape_and_terminal_code = next_escape_and_terminal_code(backslash & ~this->next_is_escaped); - uint64_t escaped = escape_and_terminal_code ^ (backslash | this->next_is_escaped); - uint64_t escape = escape_and_terminal_code & backslash; - this->next_is_escaped = escape >> 63; - return { escaped, escape }; - } - - private: - static constexpr const uint64_t ODD_BITS = 0xAAAAAAAAAAAAAAAAULL; - - simdjson_really_inline uint64_t next_escaped_without_backslashes() noexcept { - uint64_t escaped = this->next_is_escaped; - this->next_is_escaped = 0; - return escaped; - } - - /** - * Returns a mask of the next escape characters (masking out escaped backslashes), along with - * any non-backslash escape codes. - * - * \n \\n \\\n \\\\n returns: - * \n \ \ \n \ \ - * 11 100 1011 10100 - * - * You are expected to mask out the first bit yourself if the previous block had a trailing - * escape. - * - * & the result with potential_escape to get just the escape characters. - * ^ the result with (potential_escape | first_is_escaped) to get escaped characters. - */ - static simdjson_really_inline uint64_t next_escape_and_terminal_code(uint64_t potential_escape) noexcept { - // If we were to just shift and mask out any odd bits, we'd actually get a *half* right answer: - // any even-aligned backslash runs would be correct! Odd-aligned backslash runs would be - // inverted (\\\ would be 010 instead of 101). - // - // ``` - // string: | ____\\\\_\\\\_____ | - // maybe_escaped | ODD | \ \ \ \ | - // even-aligned ^^^ ^^^^ odd-aligned - // ``` - // - // Taking that into account, our basic strategy is: - // - // 1. Use subtraction to produce a mask with 1's for even-aligned runs and 0's for - // odd-aligned runs. - // 2. XOR all odd bits, which masks out the odd bits in even-aligned runs, and brings IN the - // odd bits in odd-aligned runs. - // 3. & with backslash to clean up any stray bits. - // runs are set to 0, and then XORing with "odd": - // - // | | Mask (shows characters instead of 1's) | Instructions | - // |--------------------------------|----------------------------------------|---------------------| - // | string | `\\n_\\\n___\\\n___\\\\___\\\\__\\\` | - // | | ` even odd even odd odd` | - // | maybe_escaped | ` n \\n \\n \\\_ \\\_ \\` X | 1 (potential_escape << 1) - // | maybe_escaped_and_odd | ` \n_ \\n _ \\\n_ _ \\\__ _\\\_ \\\` | 1 (maybe_escaped | odd) - // | even_series_codes_and_odd | ` n_\\\ _ n_ _\\\\ _ _ ` | 1 (maybe_escaped_and_odd - potential_escape) - // | escape_and_terminal_code | ` \n \ \n \ \n \ \ \ \ \ \` | 1 (^ odd) - // - - // Escaped characters are characters following an escape. - uint64_t maybe_escaped = potential_escape << 1; - - // To distinguish odd from even escape sequences, therefore, we turn on any *starting* - // escapes that are on an odd byte. (We actually bring in all odd bits, for speed.) - // - Odd runs of backslashes are 0000, and the code at the end ("n" in \n or \\n) is 1. - // - Odd runs of backslashes are 1111, and the code at the end ("n" in \n or \\n) is 0. - // - All other odd bytes are 1, and even bytes are 0. - uint64_t maybe_escaped_and_odd_bits = maybe_escaped | ODD_BITS; - uint64_t even_series_codes_and_odd_bits = maybe_escaped_and_odd_bits - potential_escape; - - // Now we flip all odd bytes back with xor. This: - // - Makes odd runs of backslashes go from 0000 to 1010 - // - Makes even runs of backslashes go from 1111 to 1010 - // - Sets actually-escaped codes to 1 (the n in \n and \\n: \n = 11, \\n = 100) - // - Resets all other bytes to 0 - return even_series_codes_and_odd_bits ^ ODD_BITS; - } - }; - - } // namespace stage1 - } // unnamed namespace - } // namespace icelake -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRING_SCANNER_H -/* end file generic/stage1/json_escape_scanner.h for icelake */ -/* including generic/stage1/json_string_scanner.h for icelake: #include */ -/* begin file generic/stage1/json_string_scanner.h for icelake */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRING_SCANNER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRING_SCANNER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - namespace icelake { - namespace { - namespace stage1 { - - struct json_string_block { - // We spell out the constructors in the hope of resolving inlining issues with Visual Studio 2017 - simdjson_really_inline json_string_block(uint64_t escaped, uint64_t quote, uint64_t in_string) : - _escaped(escaped), _quote(quote), _in_string(in_string) {} - - // Escaped characters (characters following an escape() character) - simdjson_really_inline uint64_t escaped() const { return _escaped; } - // Real (non-backslashed) quotes - simdjson_really_inline uint64_t quote() const { return _quote; } - // Only characters inside the string (not including the quotes) - simdjson_really_inline uint64_t string_content() const { return _in_string & ~_quote; } - // Return a mask of whether the given characters are inside a string (only works on non-quotes) - simdjson_really_inline uint64_t non_quote_inside_string(uint64_t mask) const { return mask & _in_string; } - // Return a mask of whether the given characters are inside a string (only works on non-quotes) - simdjson_really_inline uint64_t non_quote_outside_string(uint64_t mask) const { return mask & ~_in_string; } - // Tail of string (everything except the start quote) - simdjson_really_inline uint64_t string_tail() const { return _in_string ^ _quote; } - - // escaped characters (backslashed--does not include the hex characters after \u) - uint64_t _escaped; - // real quotes (non-escaped ones) - uint64_t _quote; - // string characters (includes start quote but not end quote) - uint64_t _in_string; - }; - - // Scans blocks for string characters, storing the state necessary to do so - class json_string_scanner { - public: - simdjson_really_inline json_string_block next(const simd::simd8x64& in); - // Returns either UNCLOSED_STRING or SUCCESS - simdjson_really_inline error_code finish(); - - private: - // Scans for escape characters - json_escape_scanner escape_scanner{}; - // Whether the last iteration was still inside a string (all 1's = true, all 0's = false). - uint64_t prev_in_string = 0ULL; - }; - - // - // Return a mask of all string characters plus end quotes. - // - // prev_escaped is overflow saying whether the next character is escaped. - // prev_in_string is overflow saying whether we're still in a string. - // - // Backslash sequences outside of quotes will be detected in stage 2. - // - simdjson_really_inline json_string_block json_string_scanner::next(const simd::simd8x64& in) { - const uint64_t backslash = in.eq('\\'); - const uint64_t escaped = escape_scanner.next(backslash).escaped; - const uint64_t quote = in.eq('"') & ~escaped; - - // - // prefix_xor flips on bits inside the string (and flips off the end quote). - // - // Then we xor with prev_in_string: if we were in a string already, its effect is flipped - // (characters inside strings are outside, and characters outside strings are inside). - // - const uint64_t in_string = prefix_xor(quote) ^ prev_in_string; - - // - // Check if we're still in a string at the end of the box so the next block will know - // - prev_in_string = uint64_t(static_cast(in_string) >> 63); - - // Use ^ to turn the beginning quote off, and the end quote on. - - // We are returning a function-local object so either we get a move constructor - // or we get copy elision. - return json_string_block(escaped, quote, in_string); - } - - simdjson_really_inline error_code json_string_scanner::finish() { - if (prev_in_string) { - return UNCLOSED_STRING; - } - return SUCCESS; - } - - } // namespace stage1 - } // unnamed namespace - } // namespace icelake -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRING_SCANNER_H -/* end file generic/stage1/json_string_scanner.h for icelake */ -/* including generic/stage1/utf8_lookup4_algorithm.h for icelake: #include */ -/* begin file generic/stage1/utf8_lookup4_algorithm.h for icelake */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_UTF8_LOOKUP4_ALGORITHM_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_UTF8_LOOKUP4_ALGORITHM_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - namespace icelake { - namespace { - namespace utf8_validation { - - using namespace simd; - - simdjson_inline simd8 check_special_cases(const simd8 input, const simd8 prev1) { - // Bit 0 = Too Short (lead byte/ASCII followed by lead byte/ASCII) - // Bit 1 = Too Long (ASCII followed by continuation) - // Bit 2 = Overlong 3-byte - // Bit 4 = Surrogate - // Bit 5 = Overlong 2-byte - // Bit 7 = Two Continuations - constexpr const uint8_t TOO_SHORT = 1 << 0; // 11______ 0_______ - // 11______ 11______ - constexpr const uint8_t TOO_LONG = 1 << 1; // 0_______ 10______ - constexpr const uint8_t OVERLONG_3 = 1 << 2; // 11100000 100_____ - constexpr const uint8_t SURROGATE = 1 << 4; // 11101101 101_____ - constexpr const uint8_t OVERLONG_2 = 1 << 5; // 1100000_ 10______ - constexpr const uint8_t TWO_CONTS = 1 << 7; // 10______ 10______ - constexpr const uint8_t TOO_LARGE = 1 << 3; // 11110100 1001____ - // 11110100 101_____ - // 11110101 1001____ - // 11110101 101_____ - // 1111011_ 1001____ - // 1111011_ 101_____ - // 11111___ 1001____ - // 11111___ 101_____ - constexpr const uint8_t TOO_LARGE_1000 = 1 << 6; - // 11110101 1000____ - // 1111011_ 1000____ - // 11111___ 1000____ - constexpr const uint8_t OVERLONG_4 = 1 << 6; // 11110000 1000____ - - const simd8 byte_1_high = prev1.shr<4>().lookup_16( - // 0_______ ________ - TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, - TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, - // 10______ ________ - TWO_CONTS, TWO_CONTS, TWO_CONTS, TWO_CONTS, - // 1100____ ________ - TOO_SHORT | OVERLONG_2, - // 1101____ ________ - TOO_SHORT, - // 1110____ ________ - TOO_SHORT | OVERLONG_3 | SURROGATE, - // 1111____ ________ - TOO_SHORT | TOO_LARGE | TOO_LARGE_1000 | OVERLONG_4 - ); - constexpr const uint8_t CARRY = TOO_SHORT | TOO_LONG | TWO_CONTS; // These all have ____ in byte 1 . - const simd8 byte_1_low = (prev1 & 0x0F).lookup_16( - // ____0000 ________ - CARRY | OVERLONG_3 | OVERLONG_2 | OVERLONG_4, - // ____0001 ________ - CARRY | OVERLONG_2, - // ____001_ ________ - CARRY, - CARRY, - - // ____0100 ________ - CARRY | TOO_LARGE, - // ____0101 ________ - CARRY | TOO_LARGE | TOO_LARGE_1000, - // ____011_ ________ - CARRY | TOO_LARGE | TOO_LARGE_1000, - CARRY | TOO_LARGE | TOO_LARGE_1000, - - // ____1___ ________ - CARRY | TOO_LARGE | TOO_LARGE_1000, - CARRY | TOO_LARGE | TOO_LARGE_1000, - CARRY | TOO_LARGE | TOO_LARGE_1000, - CARRY | TOO_LARGE | TOO_LARGE_1000, - CARRY | TOO_LARGE | TOO_LARGE_1000, - // ____1101 ________ - CARRY | TOO_LARGE | TOO_LARGE_1000 | SURROGATE, - CARRY | TOO_LARGE | TOO_LARGE_1000, - CARRY | TOO_LARGE | TOO_LARGE_1000 - ); - const simd8 byte_2_high = input.shr<4>().lookup_16( - // ________ 0_______ - TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT, - TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT, - - // ________ 1000____ - TOO_LONG | OVERLONG_2 | TWO_CONTS | OVERLONG_3 | TOO_LARGE_1000 | OVERLONG_4, - // ________ 1001____ - TOO_LONG | OVERLONG_2 | TWO_CONTS | OVERLONG_3 | TOO_LARGE, - // ________ 101_____ - TOO_LONG | OVERLONG_2 | TWO_CONTS | SURROGATE | TOO_LARGE, - TOO_LONG | OVERLONG_2 | TWO_CONTS | SURROGATE | TOO_LARGE, - - // ________ 11______ - TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT - ); - return (byte_1_high & byte_1_low & byte_2_high); - } - simdjson_inline simd8 check_multibyte_lengths(const simd8 input, - const simd8 prev_input, const simd8 sc) { - simd8 prev2 = input.prev<2>(prev_input); - simd8 prev3 = input.prev<3>(prev_input); - simd8 must23 = simd8(must_be_2_3_continuation(prev2, prev3)); - simd8 must23_80 = must23 & uint8_t(0x80); - return must23_80 ^ sc; - } - - // - // Return nonzero if there are incomplete multibyte characters at the end of the block: - // e.g. if there is a 4-byte character, but it's 3 bytes from the end. - // - simdjson_inline simd8 is_incomplete(const simd8 input) { - // If the previous input's last 3 bytes match this, they're too short (they ended at EOF): - // ... 1111____ 111_____ 11______ -#if SIMDJSON_IMPLEMENTATION_ICELAKE - static const uint8_t max_array[64] = { - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 0xf0u - 1, 0xe0u - 1, 0xc0u - 1 - }; -#else - static const uint8_t max_array[32] = { - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 0xf0u - 1, 0xe0u - 1, 0xc0u - 1 - }; -#endif - const simd8 max_value(&max_array[sizeof(max_array) - sizeof(simd8)]); - return input.gt_bits(max_value); - } - - struct utf8_checker { - // If this is nonzero, there has been a UTF-8 error. - simd8 error; - // The last input we received - simd8 prev_input_block; - // Whether the last input we received was incomplete (used for ASCII fast path) - simd8 prev_incomplete; - - // - // Check whether the current bytes are valid UTF-8. - // - simdjson_inline void check_utf8_bytes(const simd8 input, const simd8 prev_input) { - // Flip prev1...prev3 so we can easily determine if they are 2+, 3+ or 4+ lead bytes - // (2, 3, 4-byte leads become large positive numbers instead of small negative numbers) - simd8 prev1 = input.prev<1>(prev_input); - simd8 sc = check_special_cases(input, prev1); - this->error |= check_multibyte_lengths(input, prev_input, sc); - } - - // The only problem that can happen at EOF is that a multibyte character is too short - // or a byte value too large in the last bytes: check_special_cases only checks for bytes - // too large in the first of two bytes. - simdjson_inline void check_eof() { - // If the previous block had incomplete UTF-8 characters at the end, an ASCII block can't - // possibly finish them. - this->error |= this->prev_incomplete; - } - - simdjson_inline void check_next_input(const simd8x64& input) { - if (simdjson_likely(is_ascii(input))) { - this->error |= this->prev_incomplete; - } - else { - // you might think that a for-loop would work, but under Visual Studio, it is not good enough. - static_assert((simd8x64::NUM_CHUNKS == 1) - || (simd8x64::NUM_CHUNKS == 2) - || (simd8x64::NUM_CHUNKS == 4), - "We support one, two or four chunks per 64-byte block."); - SIMDJSON_IF_CONSTEXPR(simd8x64::NUM_CHUNKS == 1) { - this->check_utf8_bytes(input.chunks[0], this->prev_input_block); - } -else SIMDJSON_IF_CONSTEXPR(simd8x64::NUM_CHUNKS == 2) { - this->check_utf8_bytes(input.chunks[0], this->prev_input_block); - this->check_utf8_bytes(input.chunks[1], input.chunks[0]); - } - else SIMDJSON_IF_CONSTEXPR(simd8x64::NUM_CHUNKS == 4) { - this->check_utf8_bytes(input.chunks[0], this->prev_input_block); - this->check_utf8_bytes(input.chunks[1], input.chunks[0]); - this->check_utf8_bytes(input.chunks[2], input.chunks[1]); - this->check_utf8_bytes(input.chunks[3], input.chunks[2]); - } - this->prev_incomplete = is_incomplete(input.chunks[simd8x64::NUM_CHUNKS - 1]); - this->prev_input_block = input.chunks[simd8x64::NUM_CHUNKS - 1]; - } - } - // do not forget to call check_eof! - simdjson_inline error_code errors() { - return this->error.any_bits_set_anywhere() ? error_code::UTF8_ERROR : error_code::SUCCESS; - } - - }; // struct utf8_checker - } // namespace utf8_validation - - } // unnamed namespace - } // namespace icelake -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_UTF8_LOOKUP4_ALGORITHM_H -/* end file generic/stage1/utf8_lookup4_algorithm.h for icelake */ -/* including generic/stage1/json_scanner.h for icelake: #include */ -/* begin file generic/stage1/json_scanner.h for icelake */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_JSON_SCANNER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_JSON_SCANNER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - namespace icelake { - namespace { - namespace stage1 { - - /** - * A block of scanned json, with information on operators and scalars. - * - * We seek to identify pseudo-structural characters. Anything that is inside - * a string must be omitted (hence & ~_string.string_tail()). - * Otherwise, pseudo-structural characters come in two forms. - * 1. We have the structural characters ([,],{,},:, comma). The - * term 'structural character' is from the JSON RFC. - * 2. We have the 'scalar pseudo-structural characters'. - * Scalars are quotes, and any character except structural characters and white space. - * - * To identify the scalar pseudo-structural characters, we must look at what comes - * before them: it must be a space, a quote or a structural characters. - * Starting with simdjson v0.3, we identify them by - * negation: we identify everything that is followed by a non-quote scalar, - * and we negate that. Whatever remains must be a 'scalar pseudo-structural character'. - */ - struct json_block { - public: - // We spell out the constructors in the hope of resolving inlining issues with Visual Studio 2017 - simdjson_inline json_block(json_string_block&& string, json_character_block characters, uint64_t follows_potential_nonquote_scalar) : - _string(std::move(string)), _characters(characters), _follows_potential_nonquote_scalar(follows_potential_nonquote_scalar) {} - simdjson_inline json_block(json_string_block string, json_character_block characters, uint64_t follows_potential_nonquote_scalar) : - _string(string), _characters(characters), _follows_potential_nonquote_scalar(follows_potential_nonquote_scalar) {} - - /** - * The start of structurals. - * In simdjson prior to v0.3, these were called the pseudo-structural characters. - **/ - simdjson_inline uint64_t structural_start() const noexcept { return potential_structural_start() & ~_string.string_tail(); } - /** All JSON whitespace (i.e. not in a string) */ - simdjson_inline uint64_t whitespace() const noexcept { return non_quote_outside_string(_characters.whitespace()); } - - // Helpers - - /** Whether the given characters are inside a string (only works on non-quotes) */ - simdjson_inline uint64_t non_quote_inside_string(uint64_t mask) const noexcept { return _string.non_quote_inside_string(mask); } - /** Whether the given characters are outside a string (only works on non-quotes) */ - simdjson_inline uint64_t non_quote_outside_string(uint64_t mask) const noexcept { return _string.non_quote_outside_string(mask); } - - // string and escape characters - json_string_block _string; - // whitespace, structural characters ('operators'), scalars - json_character_block _characters; - // whether the previous character was a scalar - uint64_t _follows_potential_nonquote_scalar; - private: - // Potential structurals (i.e. disregarding strings) - - /** - * structural elements ([,],{,},:, comma) plus scalar starts like 123, true and "abc". - * They may reside inside a string. - **/ - simdjson_inline uint64_t potential_structural_start() const noexcept { return _characters.op() | potential_scalar_start(); } - /** - * The start of non-operator runs, like 123, true and "abc". - * It main reside inside a string. - **/ - simdjson_inline uint64_t potential_scalar_start() const noexcept { - // The term "scalar" refers to anything except structural characters and white space - // (so letters, numbers, quotes). - // Whenever it is preceded by something that is not a structural element ({,},[,],:, ") nor a white-space - // then we know that it is irrelevant structurally. - return _characters.scalar() & ~follows_potential_scalar(); - } - /** - * Whether the given character is immediately after a non-operator like 123, true. - * The characters following a quote are not included. - */ - simdjson_inline uint64_t follows_potential_scalar() const noexcept { - // _follows_potential_nonquote_scalar: is defined as marking any character that follows a character - // that is not a structural element ({,},[,],:, comma) nor a quote (") and that is not a - // white space. - // It is understood that within quoted region, anything at all could be marked (irrelevant). - return _follows_potential_nonquote_scalar; - } - }; - - /** - * Scans JSON for important bits: structural characters or 'operators', strings, and scalars. - * - * The scanner starts by calculating two distinct things: - * - string characters (taking \" into account) - * - structural characters or 'operators' ([]{},:, comma) - * and scalars (runs of non-operators like 123, true and "abc") - * - * To minimize data dependency (a key component of the scanner's speed), it finds these in parallel: - * in particular, the operator/scalar bit will find plenty of things that are actually part of - * strings. When we're done, json_block will fuse the two together by masking out tokens that are - * part of a string. - */ - class json_scanner { - public: - json_scanner() = default; - simdjson_inline json_block next(const simd::simd8x64& in); - // Returns either UNCLOSED_STRING or SUCCESS - simdjson_inline error_code finish(); - - private: - // Whether the last character of the previous iteration is part of a scalar token - // (anything except whitespace or a structural character/'operator'). - uint64_t prev_scalar = 0ULL; - json_string_scanner string_scanner{}; - }; - - - // - // Check if the current character immediately follows a matching character. - // - // For example, this checks for quotes with backslashes in front of them: - // - // const uint64_t backslashed_quote = in.eq('"') & immediately_follows(in.eq('\'), prev_backslash); - // - simdjson_inline uint64_t follows(const uint64_t match, uint64_t& overflow) { - const uint64_t result = match << 1 | overflow; - overflow = match >> 63; - return result; - } - - simdjson_inline json_block json_scanner::next(const simd::simd8x64& in) { - json_string_block strings = string_scanner.next(in); - // identifies the white-space and the structural characters - json_character_block characters = json_character_block::classify(in); - // The term "scalar" refers to anything except structural characters and white space - // (so letters, numbers, quotes). - // We want follows_scalar to mark anything that follows a non-quote scalar (so letters and numbers). - // - // A terminal quote should either be followed by a structural character (comma, brace, bracket, colon) - // or nothing. However, we still want ' "a string"true ' to mark the 't' of 'true' as a potential - // pseudo-structural character just like we would if we had ' "a string" true '; otherwise we - // may need to add an extra check when parsing strings. - // - // Performance: there are many ways to skin this cat. - const uint64_t nonquote_scalar = characters.scalar() & ~strings.quote(); - uint64_t follows_nonquote_scalar = follows(nonquote_scalar, prev_scalar); - // We are returning a function-local object so either we get a move constructor - // or we get copy elision. - return json_block( - strings,// strings is a function-local object so either it moves or the copy is elided. - characters, - follows_nonquote_scalar - ); - } - - simdjson_inline error_code json_scanner::finish() { - return string_scanner.finish(); - } - - } // namespace stage1 - } // unnamed namespace - } // namespace icelake -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_JSON_SCANNER_H -/* end file generic/stage1/json_scanner.h for icelake */ - -// All other declarations -/* including generic/stage1/find_next_document_index.h for icelake: #include */ -/* begin file generic/stage1/find_next_document_index.h for icelake */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_FIND_NEXT_DOCUMENT_INDEX_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_FIND_NEXT_DOCUMENT_INDEX_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - namespace icelake { - namespace { - namespace stage1 { - - /** - * This algorithm is used to quickly identify the last structural position that - * makes up a complete document. - * - * It does this by going backwards and finding the last *document boundary* (a - * place where one value follows another without a comma between them). If the - * last document (the characters after the boundary) has an equal number of - * start and end brackets, it is considered complete. - * - * Simply put, we iterate over the structural characters, starting from - * the end. We consider that we found the end of a JSON document when the - * first element of the pair is NOT one of these characters: '{' '[' ':' ',' - * and when the second element is NOT one of these characters: '}' ']' ':' ','. - * - * This simple comparison works most of the time, but it does not cover cases - * where the batch's structural indexes contain a perfect amount of documents. - * In such a case, we do not have access to the structural index which follows - * the last document, therefore, we do not have access to the second element in - * the pair, and that means we cannot identify the last document. To fix this - * issue, we keep a count of the open and closed curly/square braces we found - * while searching for the pair. When we find a pair AND the count of open and - * closed curly/square braces is the same, we know that we just passed a - * complete document, therefore the last json buffer location is the end of the - * batch. - */ - simdjson_inline uint32_t find_next_document_index(dom_parser_implementation& parser) { - // Variant: do not count separately, just figure out depth - if (parser.n_structural_indexes == 0) { return 0; } - auto arr_cnt = 0; - auto obj_cnt = 0; - for (auto i = parser.n_structural_indexes - 1; i > 0; i--) { - auto idxb = parser.structural_indexes[i]; - switch (parser.buf[idxb]) { - case ':': - case ',': - continue; - case '}': - obj_cnt--; - continue; - case ']': - arr_cnt--; - continue; - case '{': - obj_cnt++; - break; - case '[': - arr_cnt++; - break; - } - auto idxa = parser.structural_indexes[i - 1]; - switch (parser.buf[idxa]) { - case '{': - case '[': - case ':': - case ',': - continue; - } - // Last document is complete, so the next document will appear after! - if (!arr_cnt && !obj_cnt) { - return parser.n_structural_indexes; - } - // Last document is incomplete; mark the document at i + 1 as the next one - return i; - } - // If we made it to the end, we want to finish counting to see if we have a full document. - switch (parser.buf[parser.structural_indexes[0]]) { - case '}': - obj_cnt--; - break; - case ']': - arr_cnt--; - break; - case '{': - obj_cnt++; - break; - case '[': - arr_cnt++; - break; - } - if (!arr_cnt && !obj_cnt) { - // We have a complete document. - return parser.n_structural_indexes; - } - return 0; - } - - } // namespace stage1 - } // unnamed namespace - } // namespace icelake -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_FIND_NEXT_DOCUMENT_INDEX_H -/* end file generic/stage1/find_next_document_index.h for icelake */ -/* including generic/stage1/json_minifier.h for icelake: #include */ -/* begin file generic/stage1/json_minifier.h for icelake */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_JSON_MINIFIER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_JSON_MINIFIER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -// This file contains the common code every implementation uses in stage1 -// It is intended to be included multiple times and compiled multiple times -// We assume the file in which it is included already includes -// "simdjson/stage1.h" (this simplifies amalgation) - -namespace simdjson { - namespace icelake { - namespace { - namespace stage1 { - - class json_minifier { - public: - template - static error_code minify(const uint8_t* buf, size_t len, uint8_t* dst, size_t& dst_len) noexcept; - - private: - simdjson_inline json_minifier(uint8_t* _dst) - : dst{ _dst } - {} - template - simdjson_inline void step(const uint8_t* block_buf, buf_block_reader& reader) noexcept; - simdjson_inline void next(const simd::simd8x64& in, const json_block& block); - simdjson_inline error_code finish(uint8_t* dst_start, size_t& dst_len); - json_scanner scanner{}; - uint8_t* dst; - }; - - simdjson_inline void json_minifier::next(const simd::simd8x64& in, const json_block& block) { - uint64_t mask = block.whitespace(); - dst += in.compress(mask, dst); - } - - simdjson_inline error_code json_minifier::finish(uint8_t* dst_start, size_t& dst_len) { - error_code error = scanner.finish(); - if (error) { dst_len = 0; return error; } - dst_len = dst - dst_start; - return SUCCESS; - } - - template<> - simdjson_inline void json_minifier::step<128>(const uint8_t* block_buf, buf_block_reader<128>& reader) noexcept { - simd::simd8x64 in_1(block_buf); - simd::simd8x64 in_2(block_buf + 64); - json_block block_1 = scanner.next(in_1); - json_block block_2 = scanner.next(in_2); - this->next(in_1, block_1); - this->next(in_2, block_2); - reader.advance(); - } - - template<> - simdjson_inline void json_minifier::step<64>(const uint8_t* block_buf, buf_block_reader<64>& reader) noexcept { - simd::simd8x64 in_1(block_buf); - json_block block_1 = scanner.next(in_1); - this->next(block_buf, block_1); - reader.advance(); - } - - template - error_code json_minifier::minify(const uint8_t* buf, size_t len, uint8_t* dst, size_t& dst_len) noexcept { - buf_block_reader reader(buf, len); - json_minifier minifier(dst); - - // Index the first n-1 blocks - while (reader.has_full_block()) { - minifier.step(reader.full_block(), reader); - } - - // Index the last (remainder) block, padded with spaces - uint8_t block[STEP_SIZE]; - size_t remaining_bytes = reader.get_remainder(block); - if (remaining_bytes > 0) { - // We do not want to write directly to the output stream. Rather, we write - // to a local buffer (for safety). - uint8_t out_block[STEP_SIZE]; - uint8_t* const guarded_dst{ minifier.dst }; - minifier.dst = out_block; - minifier.step(block, reader); - size_t to_write = minifier.dst - out_block; - // In some cases, we could be enticed to consider the padded spaces - // as part of the string. This is fine as long as we do not write more - // than we consumed. - if (to_write > remaining_bytes) { to_write = remaining_bytes; } - memcpy(guarded_dst, out_block, to_write); - minifier.dst = guarded_dst + to_write; - } - return minifier.finish(dst, dst_len); - } - - } // namespace stage1 - } // unnamed namespace - } // namespace icelake -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_JSON_MINIFIER_H -/* end file generic/stage1/json_minifier.h for icelake */ -/* including generic/stage1/json_structural_indexer.h for icelake: #include */ -/* begin file generic/stage1/json_structural_indexer.h for icelake */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRUCTURAL_INDEXER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRUCTURAL_INDEXER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -// This file contains the common code every implementation uses in stage1 -// It is intended to be included multiple times and compiled multiple times -// We assume the file in which it is included already includes -// "simdjson/stage1.h" (this simplifies amalgation) - -namespace simdjson { - namespace icelake { - namespace { - namespace stage1 { - - class bit_indexer { - public: - uint32_t* tail; - - simdjson_inline bit_indexer(uint32_t* index_buf) : tail(index_buf) {} - -#if SIMDJSON_PREFER_REVERSE_BITS - /** - * ARM lacks a fast trailing zero instruction, but it has a fast - * bit reversal instruction and a fast leading zero instruction. - * Thus it may be profitable to reverse the bits (once) and then - * to rely on a sequence of instructions that call the leading - * zero instruction. - * - * Performance notes: - * The chosen routine is not optimal in terms of data dependency - * since zero_leading_bit might require two instructions. However, - * it tends to minimize the total number of instructions which is - * beneficial. - */ - simdjson_inline void write_index(uint32_t idx, uint64_t& rev_bits, int i) { - int lz = leading_zeroes(rev_bits); - this->tail[i] = static_cast(idx) + lz; - rev_bits = zero_leading_bit(rev_bits, lz); - } -#else - /** - * Under recent x64 systems, we often have both a fast trailing zero - * instruction and a fast 'clear-lower-bit' instruction so the following - * algorithm can be competitive. - */ - - simdjson_inline void write_index(uint32_t idx, uint64_t& bits, int i) { - this->tail[i] = idx + trailing_zeroes(bits); - bits = clear_lowest_bit(bits); - } -#endif // SIMDJSON_PREFER_REVERSE_BITS - - template - simdjson_inline int write_indexes(uint32_t idx, uint64_t& bits) { - write_index(idx, bits, START); - SIMDJSON_IF_CONSTEXPR(N > 1) { - write_indexes<(N - 1 > 0 ? START + 1 : START), (N - 1 >= 0 ? N - 1 : 1)>(idx, bits); - } - return START + N; - } - - template - simdjson_inline int write_indexes_stepped(uint32_t idx, uint64_t& bits, int cnt) { - write_indexes(idx, bits); - SIMDJSON_IF_CONSTEXPR((START + STEP) < END) { - if (simdjson_unlikely((START + STEP) < cnt)) { - write_indexes_stepped<(START + STEP < END ? START + STEP : END), END, STEP>(idx, bits, cnt); - } - } - return ((END - START) % STEP) == 0 ? END : (END - START) - ((END - START) % STEP) + STEP; - } - - // flatten out values in 'bits' assuming that they are are to have values of idx - // plus their position in the bitvector, and store these indexes at - // base_ptr[base] incrementing base as we go - // will potentially store extra values beyond end of valid bits, so base_ptr - // needs to be large enough to handle this - // - // If the kernel sets SIMDJSON_GENERIC_JSON_STRUCTURAL_INDEXER_CUSTOM_BIT_INDEXER, then it - // will provide its own version of the code. -#ifdef SIMDJSON_GENERIC_JSON_STRUCTURAL_INDEXER_CUSTOM_BIT_INDEXER - simdjson_inline void write(uint32_t idx, uint64_t bits); -#else - simdjson_inline void write(uint32_t idx, uint64_t bits) { - // In some instances, the next branch is expensive because it is mispredicted. - // Unfortunately, in other cases, - // it helps tremendously. - if (bits == 0) - return; - - int cnt = static_cast(count_ones(bits)); - -#if SIMDJSON_PREFER_REVERSE_BITS - bits = reverse_bits(bits); -#endif -#ifdef SIMDJSON_STRUCTURAL_INDEXER_STEP - static constexpr const int STEP = SIMDJSON_STRUCTURAL_INDEXER_STEP; -#else - static constexpr const int STEP = 4; -#endif - static constexpr const int STEP_UNTIL = 24; - - write_indexes_stepped<0, STEP_UNTIL, STEP>(idx, bits, cnt); - SIMDJSON_IF_CONSTEXPR(STEP_UNTIL < 64) { - if (simdjson_unlikely(STEP_UNTIL < cnt)) { - for (int i = STEP_UNTIL; i < cnt; i++) { - write_index(idx, bits, i); - } - } - } - - this->tail += cnt; - } -#endif // SIMDJSON_GENERIC_JSON_STRUCTURAL_INDEXER_CUSTOM_BIT_INDEXER - - }; - - class json_structural_indexer { - public: - /** - * Find the important bits of JSON in a 128-byte chunk, and add them to structural_indexes. - * - * @param partial Setting the partial parameter to true allows the find_structural_bits to - * tolerate unclosed strings. The caller should still ensure that the input is valid UTF-8. If - * you are processing substrings, you may want to call on a function like trimmed_length_safe_utf8. - */ - template - static error_code index(const uint8_t* buf, size_t len, dom_parser_implementation& parser, stage1_mode partial) noexcept; - - private: - simdjson_inline json_structural_indexer(uint32_t* structural_indexes); - template - simdjson_inline void step(const uint8_t* block, buf_block_reader& reader) noexcept; - simdjson_inline void next(const simd::simd8x64& in, const json_block& block, size_t idx); - simdjson_inline error_code finish(dom_parser_implementation& parser, size_t idx, size_t len, stage1_mode partial); - - json_scanner scanner{}; - utf8_checker checker{}; - bit_indexer indexer; - uint64_t prev_structurals = 0; - uint64_t unescaped_chars_error = 0; - }; - - simdjson_inline json_structural_indexer::json_structural_indexer(uint32_t* structural_indexes) : indexer{ structural_indexes } {} - - // Skip the last character if it is partial - simdjson_inline size_t trim_partial_utf8(const uint8_t* buf, size_t len) { - if (simdjson_unlikely(len < 3)) { - switch (len) { - case 2: - if (buf[len - 1] >= 0xc0) { return len - 1; } // 2-, 3- and 4-byte characters with only 1 byte left - if (buf[len - 2] >= 0xe0) { return len - 2; } // 3- and 4-byte characters with only 2 bytes left - return len; - case 1: - if (buf[len - 1] >= 0xc0) { return len - 1; } // 2-, 3- and 4-byte characters with only 1 byte left - return len; - case 0: - return len; - } - } - if (buf[len - 1] >= 0xc0) { return len - 1; } // 2-, 3- and 4-byte characters with only 1 byte left - if (buf[len - 2] >= 0xe0) { return len - 2; } // 3- and 4-byte characters with only 1 byte left - if (buf[len - 3] >= 0xf0) { return len - 3; } // 4-byte characters with only 3 bytes left - return len; - } - - // - // PERF NOTES: - // We pipe 2 inputs through these stages: - // 1. Load JSON into registers. This takes a long time and is highly parallelizable, so we load - // 2 inputs' worth at once so that by the time step 2 is looking for them input, it's available. - // 2. Scan the JSON for critical data: strings, scalars and operators. This is the critical path. - // The output of step 1 depends entirely on this information. These functions don't quite use - // up enough CPU: the second half of the functions is highly serial, only using 1 execution core - // at a time. The second input's scans has some dependency on the first ones finishing it, but - // they can make a lot of progress before they need that information. - // 3. Step 1 doesn't use enough capacity, so we run some extra stuff while we're waiting for that - // to finish: utf-8 checks and generating the output from the last iteration. - // - // The reason we run 2 inputs at a time, is steps 2 and 3 are *still* not enough to soak up all - // available capacity with just one input. Running 2 at a time seems to give the CPU a good enough - // workout. - // - template - error_code json_structural_indexer::index(const uint8_t* buf, size_t len, dom_parser_implementation& parser, stage1_mode partial) noexcept { - if (simdjson_unlikely(len > parser.capacity())) { return CAPACITY; } - // We guard the rest of the code so that we can assume that len > 0 throughout. - if (len == 0) { return EMPTY; } - if (is_streaming(partial)) { - len = trim_partial_utf8(buf, len); - // If you end up with an empty window after trimming - // the partial UTF-8 bytes, then chances are good that you - // have an UTF-8 formatting error. - if (len == 0) { return UTF8_ERROR; } - } - buf_block_reader reader(buf, len); - json_structural_indexer indexer(parser.structural_indexes.get()); - - // Read all but the last block - while (reader.has_full_block()) { - indexer.step(reader.full_block(), reader); - } - // Take care of the last block (will always be there unless file is empty which is - // not supposed to happen.) - uint8_t block[STEP_SIZE]; - if (simdjson_unlikely(reader.get_remainder(block) == 0)) { return UNEXPECTED_ERROR; } - indexer.step(block, reader); - return indexer.finish(parser, reader.block_index(), len, partial); - } - - template<> - simdjson_inline void json_structural_indexer::step<128>(const uint8_t* block, buf_block_reader<128>& reader) noexcept { - simd::simd8x64 in_1(block); - simd::simd8x64 in_2(block + 64); - json_block block_1 = scanner.next(in_1); - json_block block_2 = scanner.next(in_2); - this->next(in_1, block_1, reader.block_index()); - this->next(in_2, block_2, reader.block_index() + 64); - reader.advance(); - } - - template<> - simdjson_inline void json_structural_indexer::step<64>(const uint8_t* block, buf_block_reader<64>& reader) noexcept { - simd::simd8x64 in_1(block); - json_block block_1 = scanner.next(in_1); - this->next(in_1, block_1, reader.block_index()); - reader.advance(); - } - - simdjson_inline void json_structural_indexer::next(const simd::simd8x64& in, const json_block& block, size_t idx) { - uint64_t unescaped = in.lteq(0x1F); -#if SIMDJSON_UTF8VALIDATION - checker.check_next_input(in); -#endif - indexer.write(uint32_t(idx - 64), prev_structurals); // Output *last* iteration's structurals to the parser - prev_structurals = block.structural_start(); - unescaped_chars_error |= block.non_quote_inside_string(unescaped); - } - - simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementation& parser, size_t idx, size_t len, stage1_mode partial) { - // Write out the final iteration's structurals - indexer.write(uint32_t(idx - 64), prev_structurals); - error_code error = scanner.finish(); - // We deliberately break down the next expression so that it is - // human readable. - const bool should_we_exit = is_streaming(partial) ? - ((error != SUCCESS) && (error != UNCLOSED_STRING)) // when partial we tolerate UNCLOSED_STRING - : (error != SUCCESS); // if partial is false, we must have SUCCESS - const bool have_unclosed_string = (error == UNCLOSED_STRING); - if (simdjson_unlikely(should_we_exit)) { return error; } - - if (unescaped_chars_error) { - return UNESCAPED_CHARS; - } - parser.n_structural_indexes = uint32_t(indexer.tail - parser.structural_indexes.get()); - /*** - * The On Demand API requires special padding. - * - * This is related to https://github.com/simdjson/simdjson/issues/906 - * Basically, we want to make sure that if the parsing continues beyond the last (valid) - * structural character, it quickly stops. - * Only three structural characters can be repeated without triggering an error in JSON: [,] and }. - * We repeat the padding character (at 'len'). We don't know what it is, but if the parsing - * continues, then it must be [,] or }. - * Suppose it is ] or }. We backtrack to the first character, what could it be that would - * not trigger an error? It could be ] or } but no, because you can't start a document that way. - * It can't be a comma, a colon or any simple value. So the only way we could continue is - * if the repeated character is [. But if so, the document must start with [. But if the document - * starts with [, it should end with ]. If we enforce that rule, then we would get - * ][[ which is invalid. - * - * This is illustrated with the test array_iterate_unclosed_error() on the following input: - * R"({ "a": [,,)" - **/ - parser.structural_indexes[parser.n_structural_indexes] = uint32_t(len); // used later in partial == stage1_mode::streaming_final - parser.structural_indexes[parser.n_structural_indexes + 1] = uint32_t(len); - parser.structural_indexes[parser.n_structural_indexes + 2] = 0; - parser.next_structural_index = 0; - // a valid JSON file cannot have zero structural indexes - we should have found something - if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { - return EMPTY; - } - if (simdjson_unlikely(parser.structural_indexes[parser.n_structural_indexes - 1] > len)) { - return UNEXPECTED_ERROR; - } - if (partial == stage1_mode::streaming_partial) { - // If we have an unclosed string, then the last structural - // will be the quote and we want to make sure to omit it. - if (have_unclosed_string) { - parser.n_structural_indexes--; - // a valid JSON file cannot have zero structural indexes - we should have found something - if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return CAPACITY; } - } - // We truncate the input to the end of the last complete document (or zero). - auto new_structural_indexes = find_next_document_index(parser); - if (new_structural_indexes == 0 && parser.n_structural_indexes > 0) { - if (parser.structural_indexes[0] == 0) { - // If the buffer is partial and we started at index 0 but the document is - // incomplete, it's too big to parse. - return CAPACITY; - } - else { - // It is possible that the document could be parsed, we just had a lot - // of white space. - parser.n_structural_indexes = 0; - return EMPTY; - } - } - - parser.n_structural_indexes = new_structural_indexes; - } - else if (partial == stage1_mode::streaming_final) { - if (have_unclosed_string) { parser.n_structural_indexes--; } - // We truncate the input to the end of the last complete document (or zero). - // Because partial == stage1_mode::streaming_final, it means that we may - // silently ignore trailing garbage. Though it sounds bad, we do it - // deliberately because many people who have streams of JSON documents - // will truncate them for processing. E.g., imagine that you are uncompressing - // the data from a size file or receiving it in chunks from the network. You - // may not know where exactly the last document will be. Meanwhile the - // document_stream instances allow people to know the JSON documents they are - // parsing (see the iterator.source() method). - parser.n_structural_indexes = find_next_document_index(parser); - // We store the initial n_structural_indexes so that the client can see - // whether we used truncation. If initial_n_structural_indexes == parser.n_structural_indexes, - // then this will query parser.structural_indexes[parser.n_structural_indexes] which is len, - // otherwise, it will copy some prior index. - parser.structural_indexes[parser.n_structural_indexes + 1] = parser.structural_indexes[parser.n_structural_indexes]; - // This next line is critical, do not change it unless you understand what you are - // doing. - parser.structural_indexes[parser.n_structural_indexes] = uint32_t(len); - if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { - // We tolerate an unclosed string at the very end of the stream. Indeed, users - // often load their data in bulk without being careful and they want us to ignore - // the trailing garbage. - return EMPTY; - } - } - checker.check_eof(); - return checker.errors(); - } - - } // namespace stage1 - } // unnamed namespace - } // namespace icelake -} // namespace simdjson - -// Clear CUSTOM_BIT_INDEXER so other implementations can set it if they need to. -#undef SIMDJSON_GENERIC_JSON_STRUCTURAL_INDEXER_CUSTOM_BIT_INDEXER - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRUCTURAL_INDEXER_H -/* end file generic/stage1/json_structural_indexer.h for icelake */ -/* including generic/stage1/utf8_validator.h for icelake: #include */ -/* begin file generic/stage1/utf8_validator.h for icelake */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_UTF8_VALIDATOR_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_UTF8_VALIDATOR_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - namespace icelake { - namespace { - namespace stage1 { - - /** - * Validates that the string is actual UTF-8. - */ - template - bool generic_validate_utf8(const uint8_t* input, size_t length) { - checker c{}; - buf_block_reader<64> reader(input, length); - while (reader.has_full_block()) { - simd::simd8x64 in(reader.full_block()); - c.check_next_input(in); - reader.advance(); - } - uint8_t block[64]{}; - reader.get_remainder(block); - simd::simd8x64 in(block); - c.check_next_input(in); - reader.advance(); - c.check_eof(); - return c.errors() == error_code::SUCCESS; - } - - bool generic_validate_utf8(const char* input, size_t length) { - return generic_validate_utf8(reinterpret_cast(input), length); - } - - } // namespace stage1 - } // unnamed namespace - } // namespace icelake -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_UTF8_VALIDATOR_H -/* end file generic/stage1/utf8_validator.h for icelake */ -/* end file generic/stage1/amalgamated.h for icelake */ -/* including generic/stage2/amalgamated.h for icelake: #include */ -/* begin file generic/stage2/amalgamated.h for icelake */ -// Stuff other things depend on -/* including generic/stage2/base.h for icelake: #include */ -/* begin file generic/stage2/base.h for icelake */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_BASE_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - namespace icelake { - namespace { - namespace stage2 { - - class json_iterator; - class structural_iterator; - struct tape_builder; - struct tape_writer; - - } // namespace stage2 - } // unnamed namespace - } // namespace icelake -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE2_BASE_H -/* end file generic/stage2/base.h for icelake */ -/* including generic/stage2/tape_writer.h for icelake: #include */ -/* begin file generic/stage2/tape_writer.h for icelake */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_TAPE_WRITER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_TAPE_WRITER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include - -namespace simdjson { - namespace icelake { - namespace { - namespace stage2 { - - struct tape_writer { - /** The next place to write to tape */ - uint64_t* next_tape_loc; - - /** Write a signed 64-bit value to tape. */ - simdjson_inline void append_s64(int64_t value) noexcept; - - /** Write an unsigned 64-bit value to tape. */ - simdjson_inline void append_u64(uint64_t value) noexcept; - - /** Write a double value to tape. */ - simdjson_inline void append_double(double value) noexcept; - - /** - * Append a tape entry (an 8-bit type,and 56 bits worth of value). - */ - simdjson_inline void append(uint64_t val, internal::tape_type t) noexcept; - - /** - * Skip the current tape entry without writing. - * - * Used to skip the start of the container, since we'll come back later to fill it in when the - * container ends. - */ - simdjson_inline void skip() noexcept; - - /** - * Skip the number of tape entries necessary to write a large u64 or i64. - */ - simdjson_inline void skip_large_integer() noexcept; - - /** - * Skip the number of tape entries necessary to write a double. - */ - simdjson_inline void skip_double() noexcept; - - /** - * Write a value to a known location on tape. - * - * Used to go back and write out the start of a container after the container ends. - */ - simdjson_inline static void write(uint64_t& tape_loc, uint64_t val, internal::tape_type t) noexcept; - - private: - /** - * Append both the tape entry, and a supplementary value following it. Used for types that need - * all 64 bits, such as double and uint64_t. - */ - template - simdjson_inline void append2(uint64_t val, T val2, internal::tape_type t) noexcept; - }; // struct tape_writer - - simdjson_inline void tape_writer::append_s64(int64_t value) noexcept { - append2(0, value, internal::tape_type::INT64); - } - - simdjson_inline void tape_writer::append_u64(uint64_t value) noexcept { - append(0, internal::tape_type::UINT64); - *next_tape_loc = value; - next_tape_loc++; - } - - /** Write a double value to tape. */ - simdjson_inline void tape_writer::append_double(double value) noexcept { - append2(0, value, internal::tape_type::DOUBLE); - } - - simdjson_inline void tape_writer::skip() noexcept { - next_tape_loc++; - } - - simdjson_inline void tape_writer::skip_large_integer() noexcept { - next_tape_loc += 2; - } - - simdjson_inline void tape_writer::skip_double() noexcept { - next_tape_loc += 2; - } - - simdjson_inline void tape_writer::append(uint64_t val, internal::tape_type t) noexcept { - *next_tape_loc = val | ((uint64_t(char(t))) << 56); - next_tape_loc++; - } - - template - simdjson_inline void tape_writer::append2(uint64_t val, T val2, internal::tape_type t) noexcept { - append(val, t); - static_assert(sizeof(val2) == sizeof(*next_tape_loc), "Type is not 64 bits!"); - memcpy(next_tape_loc, &val2, sizeof(val2)); - next_tape_loc++; - } - - simdjson_inline void tape_writer::write(uint64_t& tape_loc, uint64_t val, internal::tape_type t) noexcept { - tape_loc = val | ((uint64_t(char(t))) << 56); - } - - } // namespace stage2 - } // unnamed namespace - } // namespace icelake -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE2_TAPE_WRITER_H -/* end file generic/stage2/tape_writer.h for icelake */ -/* including generic/stage2/logger.h for icelake: #include */ -/* begin file generic/stage2/logger.h for icelake */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_LOGGER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_LOGGER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include - - -// This is for an internal-only stage 2 specific logger. -// Set LOG_ENABLED = true to log what stage 2 is doing! -namespace simdjson { - namespace icelake { - namespace { - namespace logger { - - static constexpr const char* DASHES = "----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------"; - -#if SIMDJSON_VERBOSE_LOGGING - static constexpr const bool LOG_ENABLED = true; -#else - static constexpr const bool LOG_ENABLED = false; -#endif - static constexpr const int LOG_EVENT_LEN = 20; - static constexpr const int LOG_BUFFER_LEN = 30; - static constexpr const int LOG_SMALL_BUFFER_LEN = 10; - static constexpr const int LOG_INDEX_LEN = 5; - - static int log_depth; // Not threadsafe. Log only. - - // Helper to turn unprintable or newline characters into spaces - static simdjson_inline char printable_char(char c) { - if (c >= 0x20) { - return c; - } - else { - return ' '; - } - } - - // Print the header and set up log_start - static simdjson_inline void log_start() { - if (LOG_ENABLED) { - log_depth = 0; - printf("\n"); - printf("| %-*s | %-*s | %-*s | %-*s | Detail |\n", LOG_EVENT_LEN, "Event", LOG_BUFFER_LEN, "Buffer", LOG_SMALL_BUFFER_LEN, "Next", 5, "Next#"); - printf("|%.*s|%.*s|%.*s|%.*s|--------|\n", LOG_EVENT_LEN + 2, DASHES, LOG_BUFFER_LEN + 2, DASHES, LOG_SMALL_BUFFER_LEN + 2, DASHES, 5 + 2, DASHES); - } - } - - simdjson_unused static simdjson_inline void log_string(const char* message) { - if (LOG_ENABLED) { - printf("%s\n", message); - } - } - - // Logs a single line from the stage 2 DOM parser - template - static simdjson_inline void log_line(S& structurals, const char* title_prefix, const char* title, const char* detail) { - if (LOG_ENABLED) { - printf("| %*s%s%-*s ", log_depth * 2, "", title_prefix, LOG_EVENT_LEN - log_depth * 2 - int(strlen(title_prefix)), title); - auto current_index = structurals.at_beginning() ? nullptr : structurals.next_structural - 1; - auto next_index = structurals.next_structural; - auto current = current_index ? &structurals.buf[*current_index] : reinterpret_cast(" "); - auto next = &structurals.buf[*next_index]; - { - // Print the next N characters in the buffer. - printf("| "); - // Otherwise, print the characters starting from the buffer position. - // Print spaces for unprintable or newline characters. - for (int i = 0; i < LOG_BUFFER_LEN; i++) { - printf("%c", printable_char(current[i])); - } - printf(" "); - // Print the next N characters in the buffer. - printf("| "); - // Otherwise, print the characters starting from the buffer position. - // Print spaces for unprintable or newline characters. - for (int i = 0; i < LOG_SMALL_BUFFER_LEN; i++) { - printf("%c", printable_char(next[i])); - } - printf(" "); - } - if (current_index) { - printf("| %*u ", LOG_INDEX_LEN, *current_index); - } - else { - printf("| %-*s ", LOG_INDEX_LEN, ""); - } - // printf("| %*u ", LOG_INDEX_LEN, structurals.next_tape_index()); - printf("| %-s ", detail); - printf("|\n"); - } - } - - } // namespace logger - } // unnamed namespace - } // namespace icelake -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE2_LOGGER_H -/* end file generic/stage2/logger.h for icelake */ - -// All other declarations -/* including generic/stage2/json_iterator.h for icelake: #include */ -/* begin file generic/stage2/json_iterator.h for icelake */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_JSON_ITERATOR_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_JSON_ITERATOR_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - namespace icelake { - namespace { - namespace stage2 { - - class json_iterator { - public: - const uint8_t* const buf; - uint32_t* next_structural; - dom_parser_implementation& dom_parser; - uint32_t depth{ 0 }; - - /** - * Walk the JSON document. - * - * The visitor receives callbacks when values are encountered. All callbacks pass the iterator as - * the first parameter; some callbacks have other parameters as well: - * - * - visit_document_start() - at the beginning. - * - visit_document_end() - at the end (if things were successful). - * - * - visit_array_start() - at the start `[` of a non-empty array. - * - visit_array_end() - at the end `]` of a non-empty array. - * - visit_empty_array() - when an empty array is encountered. - * - * - visit_object_end() - at the start `]` of a non-empty object. - * - visit_object_start() - at the end `]` of a non-empty object. - * - visit_empty_object() - when an empty object is encountered. - * - visit_key(const uint8_t *key) - when a key in an object field is encountered. key is - * guaranteed to point at the first quote of the string (`"key"`). - * - visit_primitive(const uint8_t *value) - when a value is a string, number, boolean or null. - * - visit_root_primitive(iter, uint8_t *value) - when the top-level value is a string, number, boolean or null. - * - * - increment_count(iter) - each time a value is found in an array or object. - */ - template - simdjson_warn_unused simdjson_inline error_code walk_document(V& visitor) noexcept; - - /** - * Create an iterator capable of walking a JSON document. - * - * The document must have already passed through stage 1. - */ - simdjson_inline json_iterator(dom_parser_implementation& _dom_parser, size_t start_structural_index); - - /** - * Look at the next token. - * - * Tokens can be strings, numbers, booleans, null, or operators (`[{]},:`)). - * - * They may include invalid JSON as well (such as `1.2.3` or `ture`). - */ - simdjson_inline const uint8_t* peek() const noexcept; - /** - * Advance to the next token. - * - * Tokens can be strings, numbers, booleans, null, or operators (`[{]},:`)). - * - * They may include invalid JSON as well (such as `1.2.3` or `ture`). - */ - simdjson_inline const uint8_t* advance() noexcept; - /** - * Get the remaining length of the document, from the start of the current token. - */ - simdjson_inline size_t remaining_len() const noexcept; - /** - * Check if we are at the end of the document. - * - * If this is true, there are no more tokens. - */ - simdjson_inline bool at_eof() const noexcept; - /** - * Check if we are at the beginning of the document. - */ - simdjson_inline bool at_beginning() const noexcept; - simdjson_inline uint8_t last_structural() const noexcept; - - /** - * Log that a value has been found. - * - * Set LOG_ENABLED=true in logger.h to see logging. - */ - simdjson_inline void log_value(const char* type) const noexcept; - /** - * Log the start of a multipart value. - * - * Set LOG_ENABLED=true in logger.h to see logging. - */ - simdjson_inline void log_start_value(const char* type) const noexcept; - /** - * Log the end of a multipart value. - * - * Set LOG_ENABLED=true in logger.h to see logging. - */ - simdjson_inline void log_end_value(const char* type) const noexcept; - /** - * Log an error. - * - * Set LOG_ENABLED=true in logger.h to see logging. - */ - simdjson_inline void log_error(const char* error) const noexcept; - - template - simdjson_warn_unused simdjson_inline error_code visit_root_primitive(V& visitor, const uint8_t* value) noexcept; - template - simdjson_warn_unused simdjson_inline error_code visit_primitive(V& visitor, const uint8_t* value) noexcept; - }; - - template - simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V& visitor) noexcept { - logger::log_start(); - - // - // Start the document - // - if (at_eof()) { return EMPTY; } - log_start_value("document"); - SIMDJSON_TRY(visitor.visit_document_start(*this)); - - // - // Read first value - // - { - auto value = advance(); - - // Make sure the outer object or array is closed before continuing; otherwise, there are ways we - // could get into memory corruption. See https://github.com/simdjson/simdjson/issues/906 - if (!STREAMING) { - switch (*value) { - case '{': if (last_structural() != '}') { log_value("starting brace unmatched"); return TAPE_ERROR; }; break; - case '[': if (last_structural() != ']') { log_value("starting bracket unmatched"); return TAPE_ERROR; }; break; - } - } - - switch (*value) { - case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY(visitor.visit_empty_object(*this)); break; } goto object_begin; - case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY(visitor.visit_empty_array(*this)); break; } goto array_begin; - default: SIMDJSON_TRY(visitor.visit_root_primitive(*this, value)); break; - } - } - goto document_end; - - // - // Object parser states - // - object_begin: - log_start_value("object"); - depth++; - if (depth >= dom_parser.max_depth()) { log_error("Exceeded max depth!"); return DEPTH_ERROR; } - dom_parser.is_array[depth] = false; - SIMDJSON_TRY(visitor.visit_object_start(*this)); - - { - auto key = advance(); - if (*key != '"') { log_error("Object does not start with a key"); return TAPE_ERROR; } - SIMDJSON_TRY(visitor.increment_count(*this)); - SIMDJSON_TRY(visitor.visit_key(*this, key)); - } - - object_field: - if (simdjson_unlikely(*advance() != ':')) { log_error("Missing colon after key in object"); return TAPE_ERROR; } - { - auto value = advance(); - switch (*value) { - case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY(visitor.visit_empty_object(*this)); break; } goto object_begin; - case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY(visitor.visit_empty_array(*this)); break; } goto array_begin; - default: SIMDJSON_TRY(visitor.visit_primitive(*this, value)); break; - } - } - - object_continue: - switch (*advance()) { - case ',': - SIMDJSON_TRY(visitor.increment_count(*this)); - { - auto key = advance(); - if (simdjson_unlikely(*key != '"')) { log_error("Key string missing at beginning of field in object"); return TAPE_ERROR; } - SIMDJSON_TRY(visitor.visit_key(*this, key)); - } - goto object_field; - case '}': log_end_value("object"); SIMDJSON_TRY(visitor.visit_object_end(*this)); goto scope_end; - default: log_error("No comma between object fields"); return TAPE_ERROR; - } - - scope_end: - depth--; - if (depth == 0) { goto document_end; } - if (dom_parser.is_array[depth]) { goto array_continue; } - goto object_continue; - - // - // Array parser states - // - array_begin: - log_start_value("array"); - depth++; - if (depth >= dom_parser.max_depth()) { log_error("Exceeded max depth!"); return DEPTH_ERROR; } - dom_parser.is_array[depth] = true; - SIMDJSON_TRY(visitor.visit_array_start(*this)); - SIMDJSON_TRY(visitor.increment_count(*this)); - - array_value: - { - auto value = advance(); - switch (*value) { - case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY(visitor.visit_empty_object(*this)); break; } goto object_begin; - case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY(visitor.visit_empty_array(*this)); break; } goto array_begin; - default: SIMDJSON_TRY(visitor.visit_primitive(*this, value)); break; - } - } - - array_continue: - switch (*advance()) { - case ',': SIMDJSON_TRY(visitor.increment_count(*this)); goto array_value; - case ']': log_end_value("array"); SIMDJSON_TRY(visitor.visit_array_end(*this)); goto scope_end; - default: log_error("Missing comma between array values"); return TAPE_ERROR; - } - - document_end: - log_end_value("document"); - SIMDJSON_TRY(visitor.visit_document_end(*this)); - - dom_parser.next_structural_index = uint32_t(next_structural - &dom_parser.structural_indexes[0]); - - // If we didn't make it to the end, it's an error - if (!STREAMING && dom_parser.next_structural_index != dom_parser.n_structural_indexes) { - log_error("More than one JSON value at the root of the document, or extra characters at the end of the JSON!"); - return TAPE_ERROR; - } - - return SUCCESS; - - } // walk_document() - - simdjson_inline json_iterator::json_iterator(dom_parser_implementation& _dom_parser, size_t start_structural_index) - : buf{ _dom_parser.buf }, - next_structural{ &_dom_parser.structural_indexes[start_structural_index] }, - dom_parser{ _dom_parser } { - } - - simdjson_inline const uint8_t* json_iterator::peek() const noexcept { - return &buf[*(next_structural)]; - } - simdjson_inline const uint8_t* json_iterator::advance() noexcept { - return &buf[*(next_structural++)]; - } - simdjson_inline size_t json_iterator::remaining_len() const noexcept { - return dom_parser.len - *(next_structural - 1); - } - - simdjson_inline bool json_iterator::at_eof() const noexcept { - return next_structural == &dom_parser.structural_indexes[dom_parser.n_structural_indexes]; - } - simdjson_inline bool json_iterator::at_beginning() const noexcept { - return next_structural == dom_parser.structural_indexes.get(); - } - simdjson_inline uint8_t json_iterator::last_structural() const noexcept { - return buf[dom_parser.structural_indexes[dom_parser.n_structural_indexes - 1]]; - } - - simdjson_inline void json_iterator::log_value(const char* type) const noexcept { - logger::log_line(*this, "", type, ""); - } - - simdjson_inline void json_iterator::log_start_value(const char* type) const noexcept { - logger::log_line(*this, "+", type, ""); - if (logger::LOG_ENABLED) { logger::log_depth++; } - } - - simdjson_inline void json_iterator::log_end_value(const char* type) const noexcept { - if (logger::LOG_ENABLED) { logger::log_depth--; } - logger::log_line(*this, "-", type, ""); - } - - simdjson_inline void json_iterator::log_error(const char* error) const noexcept { - logger::log_line(*this, "", "ERROR", error); - } - - template - simdjson_warn_unused simdjson_inline error_code json_iterator::visit_root_primitive(V& visitor, const uint8_t* value) noexcept { - switch (*value) { - case '"': return visitor.visit_root_string(*this, value); - case 't': return visitor.visit_root_true_atom(*this, value); - case 'f': return visitor.visit_root_false_atom(*this, value); - case 'n': return visitor.visit_root_null_atom(*this, value); - case '-': - case '0': case '1': case '2': case '3': case '4': - case '5': case '6': case '7': case '8': case '9': - return visitor.visit_root_number(*this, value); - default: - log_error("Document starts with a non-value character"); - return TAPE_ERROR; - } - } - template - simdjson_warn_unused simdjson_inline error_code json_iterator::visit_primitive(V& visitor, const uint8_t* value) noexcept { - // Use the fact that most scalars are going to be either strings or numbers. - if (*value == '"') { - return visitor.visit_string(*this, value); - } - else if (((*value - '0') < 10) || (*value == '-')) { - return visitor.visit_number(*this, value); - } - // true, false, null are uncommon. - switch (*value) { - case 't': return visitor.visit_true_atom(*this, value); - case 'f': return visitor.visit_false_atom(*this, value); - case 'n': return visitor.visit_null_atom(*this, value); - default: - log_error("Non-value found when value was expected!"); - return TAPE_ERROR; - } - } - - } // namespace stage2 - } // unnamed namespace - } // namespace icelake -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE2_JSON_ITERATOR_H -/* end file generic/stage2/json_iterator.h for icelake */ -/* including generic/stage2/stringparsing.h for icelake: #include */ -/* begin file generic/stage2/stringparsing.h for icelake */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -// This file contains the common code every implementation uses -// It is intended to be included multiple times and compiled multiple times - -namespace simdjson { - namespace icelake { - namespace { - /// @private - namespace stringparsing { - - // begin copypasta - // These chars yield themselves: " \ / - // b -> backspace, f -> formfeed, n -> newline, r -> cr, t -> horizontal tab - // u not handled in this table as it's complex - static const uint8_t escape_map[256] = { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x0. - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0x22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x2f, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x4. - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x5c, 0, 0, 0, // 0x5. - 0, 0, 0x08, 0, 0, 0, 0x0c, 0, 0, 0, 0, 0, 0, 0, 0x0a, 0, // 0x6. - 0, 0, 0x0d, 0, 0x09, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x7. - - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - }; - - // handle a unicode codepoint - // write appropriate values into dest - // src will advance 6 bytes or 12 bytes - // dest will advance a variable amount (return via pointer) - // return true if the unicode codepoint was valid - // We work in little-endian then swap at write time - simdjson_warn_unused - simdjson_inline bool handle_unicode_codepoint(const uint8_t** src_ptr, - uint8_t** dst_ptr, bool allow_replacement) { - // Use the default Unicode Character 'REPLACEMENT CHARACTER' (U+FFFD) - constexpr uint32_t substitution_code_point = 0xfffd; - // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the - // conversion isn't valid; we defer the check for this to inside the - // multilingual plane check - uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2); - *src_ptr += 6; - - // If we found a high surrogate, we must - // check for low surrogate for characters - // outside the Basic - // Multilingual Plane. - if (code_point >= 0xd800 && code_point < 0xdc00) { - const uint8_t* src_data = *src_ptr; - /* Compiler optimizations convert this to a single 16-bit load and compare on most platforms */ - if (((src_data[0] << 8) | src_data[1]) != ((static_cast ('\\') << 8) | static_cast ('u'))) { - if (!allow_replacement) { return false; } - code_point = substitution_code_point; - } - else { - uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(src_data + 2); - - // We have already checked that the high surrogate is valid and - // (code_point - 0xd800) < 1024. - // - // Check that code_point_2 is in the range 0xdc00..0xdfff - // and that code_point_2 was parsed from valid hex. - uint32_t low_bit = code_point_2 - 0xdc00; - if (low_bit >> 10) { - if (!allow_replacement) { return false; } - code_point = substitution_code_point; - } - else { - code_point = (((code_point - 0xd800) << 10) | low_bit) + 0x10000; - *src_ptr += 6; - } - - } - } - else if (code_point >= 0xdc00 && code_point <= 0xdfff) { - // If we encounter a low surrogate (not preceded by a high surrogate) - // then we have an error. - if (!allow_replacement) { return false; } - code_point = substitution_code_point; - } - size_t offset = jsoncharutils::codepoint_to_utf8(code_point, *dst_ptr); - *dst_ptr += offset; - return offset > 0; - } - - - // handle a unicode codepoint using the wobbly convention - // https://simonsapin.github.io/wtf-8/ - // write appropriate values into dest - // src will advance 6 bytes or 12 bytes - // dest will advance a variable amount (return via pointer) - // return true if the unicode codepoint was valid - // We work in little-endian then swap at write time - simdjson_warn_unused - simdjson_inline bool handle_unicode_codepoint_wobbly(const uint8_t** src_ptr, - uint8_t** dst_ptr) { - // It is not ideal that this function is nearly identical to handle_unicode_codepoint. - // - // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the - // conversion isn't valid; we defer the check for this to inside the - // multilingual plane check - uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2); - *src_ptr += 6; - // If we found a high surrogate, we must - // check for low surrogate for characters - // outside the Basic - // Multilingual Plane. - if (code_point >= 0xd800 && code_point < 0xdc00) { - const uint8_t* src_data = *src_ptr; - /* Compiler optimizations convert this to a single 16-bit load and compare on most platforms */ - if (((src_data[0] << 8) | src_data[1]) == ((static_cast ('\\') << 8) | static_cast ('u'))) { - uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(src_data + 2); - uint32_t low_bit = code_point_2 - 0xdc00; - if ((low_bit >> 10) == 0) { - code_point = - (((code_point - 0xd800) << 10) | low_bit) + 0x10000; - *src_ptr += 6; - } - } - } - - size_t offset = jsoncharutils::codepoint_to_utf8(code_point, *dst_ptr); - *dst_ptr += offset; - return offset > 0; - } - - - /** - * Unescape a valid UTF-8 string from src to dst, stopping at a final unescaped quote. There - * must be an unescaped quote terminating the string. It returns the final output - * position as pointer. In case of error (e.g., the string has bad escaped codes), - * then null_nullptrptr is returned. It is assumed that the output buffer is large - * enough. E.g., if src points at 'joe"', then dst needs to have four free bytes + - * SIMDJSON_PADDING bytes. - */ - simdjson_warn_unused simdjson_inline uint8_t* parse_string(const uint8_t* src, uint8_t* dst, bool allow_replacement) { - while (1) { - // Copy the next n bytes, and find the backslash and quote in them. - auto bs_quote = backslash_and_quote::copy_and_find(src, dst); - // If the next thing is the end quote, copy and return - if (bs_quote.has_quote_first()) { - // we encountered quotes first. Move dst to point to quotes and exit - return dst + bs_quote.quote_index(); - } - if (bs_quote.has_backslash()) { - /* find out where the backspace is */ - auto bs_dist = bs_quote.backslash_index(); - uint8_t escape_char = src[bs_dist + 1]; - /* we encountered backslash first. Handle backslash */ - if (escape_char == 'u') { - /* move src/dst up to the start; they will be further adjusted - within the unicode codepoint handling code. */ - src += bs_dist; - dst += bs_dist; - if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { - return nullptr; - } - } - else { - /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and - * write bs_dist+1 characters to output - * note this may reach beyond the part of the buffer we've actually - * seen. I think this is ok */ - uint8_t escape_result = escape_map[escape_char]; - if (escape_result == 0u) { - return nullptr; /* bogus escape value is an error */ - } - dst[bs_dist] = escape_result; - src += bs_dist + 2; - dst += bs_dist + 1; - } - } - else { - /* they are the same. Since they can't co-occur, it means we - * encountered neither. */ - src += backslash_and_quote::BYTES_PROCESSED; - dst += backslash_and_quote::BYTES_PROCESSED; - } - } - } - - simdjson_warn_unused simdjson_inline uint8_t* parse_wobbly_string(const uint8_t* src, uint8_t* dst) { - // It is not ideal that this function is nearly identical to parse_string. - while (1) { - // Copy the next n bytes, and find the backslash and quote in them. - auto bs_quote = backslash_and_quote::copy_and_find(src, dst); - // If the next thing is the end quote, copy and return - if (bs_quote.has_quote_first()) { - // we encountered quotes first. Move dst to point to quotes and exit - return dst + bs_quote.quote_index(); - } - if (bs_quote.has_backslash()) { - /* find out where the backspace is */ - auto bs_dist = bs_quote.backslash_index(); - uint8_t escape_char = src[bs_dist + 1]; - /* we encountered backslash first. Handle backslash */ - if (escape_char == 'u') { - /* move src/dst up to the start; they will be further adjusted - within the unicode codepoint handling code. */ - src += bs_dist; - dst += bs_dist; - if (!handle_unicode_codepoint_wobbly(&src, &dst)) { - return nullptr; - } - } - else { - /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and - * write bs_dist+1 characters to output - * note this may reach beyond the part of the buffer we've actually - * seen. I think this is ok */ - uint8_t escape_result = escape_map[escape_char]; - if (escape_result == 0u) { - return nullptr; /* bogus escape value is an error */ - } - dst[bs_dist] = escape_result; - src += bs_dist + 2; - dst += bs_dist + 1; - } - } - else { - /* they are the same. Since they can't co-occur, it means we - * encountered neither. */ - src += backslash_and_quote::BYTES_PROCESSED; - dst += backslash_and_quote::BYTES_PROCESSED; - } - } - } - - } // namespace stringparsing - } // unnamed namespace - } // namespace icelake -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H -/* end file generic/stage2/stringparsing.h for icelake */ -/* including generic/stage2/structural_iterator.h for icelake: #include */ -/* begin file generic/stage2/structural_iterator.h for icelake */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_STRUCTURAL_ITERATOR_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_STRUCTURAL_ITERATOR_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - namespace icelake { - namespace { - namespace stage2 { - - class structural_iterator { - public: - const uint8_t* const buf; - uint32_t* next_structural; - dom_parser_implementation& dom_parser; - - // Start a structural - simdjson_inline structural_iterator(dom_parser_implementation& _dom_parser, size_t start_structural_index) - : buf{ _dom_parser.buf }, - next_structural{ &_dom_parser.structural_indexes[start_structural_index] }, - dom_parser{ _dom_parser } { - } - // Get the buffer position of the current structural character - simdjson_inline const uint8_t* current() { - return &buf[*(next_structural - 1)]; - } - // Get the current structural character - simdjson_inline char current_char() { - return buf[*(next_structural - 1)]; - } - // Get the next structural character without advancing - simdjson_inline char peek_next_char() { - return buf[*next_structural]; - } - simdjson_inline const uint8_t* peek() { - return &buf[*next_structural]; - } - simdjson_inline const uint8_t* advance() { - return &buf[*(next_structural++)]; - } - simdjson_inline char advance_char() { - return buf[*(next_structural++)]; - } - simdjson_inline size_t remaining_len() { - return dom_parser.len - *(next_structural - 1); - } - - simdjson_inline bool at_end() { - return next_structural == &dom_parser.structural_indexes[dom_parser.n_structural_indexes]; - } - simdjson_inline bool at_beginning() { - return next_structural == dom_parser.structural_indexes.get(); - } - }; - - } // namespace stage2 - } // unnamed namespace - } // namespace icelake -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE2_STRUCTURAL_ITERATOR_H -/* end file generic/stage2/structural_iterator.h for icelake */ -/* including generic/stage2/tape_builder.h for icelake: #include */ -/* begin file generic/stage2/tape_builder.h for icelake */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_TAPE_BUILDER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_TAPE_BUILDER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - - -namespace simdjson { - namespace icelake { - namespace { - namespace stage2 { - - struct tape_builder { - template - simdjson_warn_unused static simdjson_inline error_code parse_document( - dom_parser_implementation& dom_parser, - dom::document& doc) noexcept; - - /** Called when a non-empty document starts. */ - simdjson_warn_unused simdjson_inline error_code visit_document_start(json_iterator& iter) noexcept; - /** Called when a non-empty document ends without error. */ - simdjson_warn_unused simdjson_inline error_code visit_document_end(json_iterator& iter) noexcept; - - /** Called when a non-empty array starts. */ - simdjson_warn_unused simdjson_inline error_code visit_array_start(json_iterator& iter) noexcept; - /** Called when a non-empty array ends. */ - simdjson_warn_unused simdjson_inline error_code visit_array_end(json_iterator& iter) noexcept; - /** Called when an empty array is found. */ - simdjson_warn_unused simdjson_inline error_code visit_empty_array(json_iterator& iter) noexcept; - - /** Called when a non-empty object starts. */ - simdjson_warn_unused simdjson_inline error_code visit_object_start(json_iterator& iter) noexcept; - /** - * Called when a key in a field is encountered. - * - * primitive, visit_object_start, visit_empty_object, visit_array_start, or visit_empty_array - * will be called after this with the field value. - */ - simdjson_warn_unused simdjson_inline error_code visit_key(json_iterator& iter, const uint8_t* key) noexcept; - /** Called when a non-empty object ends. */ - simdjson_warn_unused simdjson_inline error_code visit_object_end(json_iterator& iter) noexcept; - /** Called when an empty object is found. */ - simdjson_warn_unused simdjson_inline error_code visit_empty_object(json_iterator& iter) noexcept; - - /** - * Called when a string, number, boolean or null is found. - */ - simdjson_warn_unused simdjson_inline error_code visit_primitive(json_iterator& iter, const uint8_t* value) noexcept; - /** - * Called when a string, number, boolean or null is found at the top level of a document (i.e. - * when there is no array or object and the entire document is a single string, number, boolean or - * null. - * - * This is separate from primitive() because simdjson's normal primitive parsing routines assume - * there is at least one more token after the value, which is only true in an array or object. - */ - simdjson_warn_unused simdjson_inline error_code visit_root_primitive(json_iterator& iter, const uint8_t* value) noexcept; - - simdjson_warn_unused simdjson_inline error_code visit_string(json_iterator& iter, const uint8_t* value, bool key = false) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_number(json_iterator& iter, const uint8_t* value) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_true_atom(json_iterator& iter, const uint8_t* value) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_false_atom(json_iterator& iter, const uint8_t* value) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_null_atom(json_iterator& iter, const uint8_t* value) noexcept; - - simdjson_warn_unused simdjson_inline error_code visit_root_string(json_iterator& iter, const uint8_t* value) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_root_number(json_iterator& iter, const uint8_t* value) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_root_true_atom(json_iterator& iter, const uint8_t* value) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_root_false_atom(json_iterator& iter, const uint8_t* value) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_root_null_atom(json_iterator& iter, const uint8_t* value) noexcept; - - /** Called each time a new field or element in an array or object is found. */ - simdjson_warn_unused simdjson_inline error_code increment_count(json_iterator& iter) noexcept; - - /** Next location to write to tape */ - tape_writer tape; - private: - /** Next write location in the string buf for stage 2 parsing */ - uint8_t* current_string_buf_loc; - - simdjson_inline tape_builder(dom::document& doc) noexcept; - - simdjson_inline uint32_t next_tape_index(json_iterator& iter) const noexcept; - simdjson_inline void start_container(json_iterator& iter) noexcept; - simdjson_warn_unused simdjson_inline error_code end_container(json_iterator& iter, internal::tape_type start, internal::tape_type end) noexcept; - simdjson_warn_unused simdjson_inline error_code empty_container(json_iterator& iter, internal::tape_type start, internal::tape_type end) noexcept; - simdjson_inline uint8_t* on_start_string(json_iterator& iter) noexcept; - simdjson_inline void on_end_string(uint8_t* dst) noexcept; - }; // struct tape_builder - - template - simdjson_warn_unused simdjson_inline error_code tape_builder::parse_document( - dom_parser_implementation& dom_parser, - dom::document& doc) noexcept { - dom_parser.doc = &doc; - json_iterator iter(dom_parser, STREAMING ? dom_parser.next_structural_index : 0); - tape_builder builder(doc); - return iter.walk_document(builder); - } - - simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_primitive(json_iterator& iter, const uint8_t* value) noexcept { - return iter.visit_root_primitive(*this, value); - } - simdjson_warn_unused simdjson_inline error_code tape_builder::visit_primitive(json_iterator& iter, const uint8_t* value) noexcept { - return iter.visit_primitive(*this, value); - } - simdjson_warn_unused simdjson_inline error_code tape_builder::visit_empty_object(json_iterator& iter) noexcept { - return empty_container(iter, internal::tape_type::START_OBJECT, internal::tape_type::END_OBJECT); - } - simdjson_warn_unused simdjson_inline error_code tape_builder::visit_empty_array(json_iterator& iter) noexcept { - return empty_container(iter, internal::tape_type::START_ARRAY, internal::tape_type::END_ARRAY); - } - - simdjson_warn_unused simdjson_inline error_code tape_builder::visit_document_start(json_iterator& iter) noexcept { - start_container(iter); - return SUCCESS; - } - simdjson_warn_unused simdjson_inline error_code tape_builder::visit_object_start(json_iterator& iter) noexcept { - start_container(iter); - return SUCCESS; - } - simdjson_warn_unused simdjson_inline error_code tape_builder::visit_array_start(json_iterator& iter) noexcept { - start_container(iter); - return SUCCESS; - } - - simdjson_warn_unused simdjson_inline error_code tape_builder::visit_object_end(json_iterator& iter) noexcept { - return end_container(iter, internal::tape_type::START_OBJECT, internal::tape_type::END_OBJECT); - } - simdjson_warn_unused simdjson_inline error_code tape_builder::visit_array_end(json_iterator& iter) noexcept { - return end_container(iter, internal::tape_type::START_ARRAY, internal::tape_type::END_ARRAY); - } - simdjson_warn_unused simdjson_inline error_code tape_builder::visit_document_end(json_iterator& iter) noexcept { - constexpr uint32_t start_tape_index = 0; - tape.append(start_tape_index, internal::tape_type::ROOT); - tape_writer::write(iter.dom_parser.doc->tape[start_tape_index], next_tape_index(iter), internal::tape_type::ROOT); - return SUCCESS; - } - simdjson_warn_unused simdjson_inline error_code tape_builder::visit_key(json_iterator& iter, const uint8_t* key) noexcept { - return visit_string(iter, key, true); - } - - simdjson_warn_unused simdjson_inline error_code tape_builder::increment_count(json_iterator& iter) noexcept { - iter.dom_parser.open_containers[iter.depth].count++; // we have a key value pair in the object at parser.dom_parser.depth - 1 - return SUCCESS; - } - - simdjson_inline tape_builder::tape_builder(dom::document& doc) noexcept : tape{ doc.tape.get() }, current_string_buf_loc{ doc.string_buf.get() } {} - - simdjson_warn_unused simdjson_inline error_code tape_builder::visit_string(json_iterator& iter, const uint8_t* value, bool key) noexcept { - iter.log_value(key ? "key" : "string"); - uint8_t* dst = on_start_string(iter); - dst = stringparsing::parse_string(value + 1, dst, false); // We do not allow replacement when the escape characters are invalid. - if (dst == nullptr) { - iter.log_error("Invalid escape in string"); - return STRING_ERROR; - } - on_end_string(dst); - return SUCCESS; - } - - simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_string(json_iterator& iter, const uint8_t* value) noexcept { - return visit_string(iter, value); - } - - simdjson_warn_unused simdjson_inline error_code tape_builder::visit_number(json_iterator& iter, const uint8_t* value) noexcept { - iter.log_value("number"); - return numberparsing::parse_number(value, tape); - } - - simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_number(json_iterator& iter, const uint8_t* value) noexcept { - // - // We need to make a copy to make sure that the string is space terminated. - // This is not about padding the input, which should already padded up - // to len + SIMDJSON_PADDING. However, we have no control at this stage - // on how the padding was done. What if the input string was padded with nulls? - // It is quite common for an input string to have an extra null character (C string). - // We do not want to allow 9\0 (where \0 is the null character) inside a JSON - // document, but the string "9\0" by itself is fine. So we make a copy and - // pad the input with spaces when we know that there is just one input element. - // This copy is relatively expensive, but it will almost never be called in - // practice unless you are in the strange scenario where you have many JSON - // documents made of single atoms. - // - std::unique_ptrcopy(new (std::nothrow) uint8_t[iter.remaining_len() + SIMDJSON_PADDING]); - if (copy.get() == nullptr) { return MEMALLOC; } - std::memcpy(copy.get(), value, iter.remaining_len()); - std::memset(copy.get() + iter.remaining_len(), ' ', SIMDJSON_PADDING); - error_code error = visit_number(iter, copy.get()); - return error; - } - - simdjson_warn_unused simdjson_inline error_code tape_builder::visit_true_atom(json_iterator& iter, const uint8_t* value) noexcept { - iter.log_value("true"); - if (!atomparsing::is_valid_true_atom(value)) { return T_ATOM_ERROR; } - tape.append(0, internal::tape_type::TRUE_VALUE); - return SUCCESS; - } - - simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_true_atom(json_iterator& iter, const uint8_t* value) noexcept { - iter.log_value("true"); - if (!atomparsing::is_valid_true_atom(value, iter.remaining_len())) { return T_ATOM_ERROR; } - tape.append(0, internal::tape_type::TRUE_VALUE); - return SUCCESS; - } - - simdjson_warn_unused simdjson_inline error_code tape_builder::visit_false_atom(json_iterator& iter, const uint8_t* value) noexcept { - iter.log_value("false"); - if (!atomparsing::is_valid_false_atom(value)) { return F_ATOM_ERROR; } - tape.append(0, internal::tape_type::FALSE_VALUE); - return SUCCESS; - } - - simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_false_atom(json_iterator& iter, const uint8_t* value) noexcept { - iter.log_value("false"); - if (!atomparsing::is_valid_false_atom(value, iter.remaining_len())) { return F_ATOM_ERROR; } - tape.append(0, internal::tape_type::FALSE_VALUE); - return SUCCESS; - } - - simdjson_warn_unused simdjson_inline error_code tape_builder::visit_null_atom(json_iterator& iter, const uint8_t* value) noexcept { - iter.log_value("null"); - if (!atomparsing::is_valid_null_atom(value)) { return N_ATOM_ERROR; } - tape.append(0, internal::tape_type::NULL_VALUE); - return SUCCESS; - } - - simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_null_atom(json_iterator& iter, const uint8_t* value) noexcept { - iter.log_value("null"); - if (!atomparsing::is_valid_null_atom(value, iter.remaining_len())) { return N_ATOM_ERROR; } - tape.append(0, internal::tape_type::NULL_VALUE); - return SUCCESS; - } - - // private: - - simdjson_inline uint32_t tape_builder::next_tape_index(json_iterator& iter) const noexcept { - return uint32_t(tape.next_tape_loc - iter.dom_parser.doc->tape.get()); - } - - simdjson_warn_unused simdjson_inline error_code tape_builder::empty_container(json_iterator& iter, internal::tape_type start, internal::tape_type end) noexcept { - auto start_index = next_tape_index(iter); - tape.append(start_index + 2, start); - tape.append(start_index, end); - return SUCCESS; - } - - simdjson_inline void tape_builder::start_container(json_iterator& iter) noexcept { - iter.dom_parser.open_containers[iter.depth].tape_index = next_tape_index(iter); - iter.dom_parser.open_containers[iter.depth].count = 0; - tape.skip(); // We don't actually *write* the start element until the end. - } - - simdjson_warn_unused simdjson_inline error_code tape_builder::end_container(json_iterator& iter, internal::tape_type start, internal::tape_type end) noexcept { - // Write the ending tape element, pointing at the start location - const uint32_t start_tape_index = iter.dom_parser.open_containers[iter.depth].tape_index; - tape.append(start_tape_index, end); - // Write the start tape element, pointing at the end location (and including count) - // count can overflow if it exceeds 24 bits... so we saturate - // the convention being that a cnt of 0xffffff or more is undetermined in value (>= 0xffffff). - const uint32_t count = iter.dom_parser.open_containers[iter.depth].count; - const uint32_t cntsat = count > 0xFFFFFF ? 0xFFFFFF : count; - tape_writer::write(iter.dom_parser.doc->tape[start_tape_index], next_tape_index(iter) | (uint64_t(cntsat) << 32), start); - return SUCCESS; - } - - simdjson_inline uint8_t* tape_builder::on_start_string(json_iterator& iter) noexcept { - // we advance the point, accounting for the fact that we have a NULL termination - tape.append(current_string_buf_loc - iter.dom_parser.doc->string_buf.get(), internal::tape_type::STRING); - return current_string_buf_loc + sizeof(uint32_t); - } - - simdjson_inline void tape_builder::on_end_string(uint8_t* dst) noexcept { - uint32_t str_length = uint32_t(dst - (current_string_buf_loc + sizeof(uint32_t))); - // TODO check for overflow in case someone has a crazy string (>=4GB?) - // But only add the overflow check when the document itself exceeds 4GB - // Currently unneeded because we refuse to parse docs larger or equal to 4GB. - memcpy(current_string_buf_loc, &str_length, sizeof(uint32_t)); - // NULL termination is still handy if you expect all your strings to - // be NULL terminated? It comes at a small cost - *dst = 0; - current_string_buf_loc = dst + 1; - } - - } // namespace stage2 - } // unnamed namespace - } // namespace icelake -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE2_TAPE_BUILDER_H -/* end file generic/stage2/tape_builder.h for icelake */ -/* end file generic/stage2/amalgamated.h for icelake */ - -#undef SIMDJSON_GENERIC_JSON_STRUCTURAL_INDEXER_CUSTOM_BIT_INDEXER - -// -// Stage 1 -// - -namespace simdjson { - namespace icelake { - - simdjson_warn_unused error_code implementation::create_dom_parser_implementation( - size_t capacity, - size_t max_depth, - std::unique_ptr& dst - ) const noexcept { - dst.reset(new (std::nothrow) dom_parser_implementation()); - if (!dst) { return MEMALLOC; } - if (auto err = dst->set_capacity(capacity)) - return err; - if (auto err = dst->set_max_depth(max_depth)) - return err; - return SUCCESS; - } - - namespace { - - using namespace simd; - - // This identifies structural characters (comma, colon, braces, brackets), - // and ASCII white-space ('\r','\n','\t',' '). - simdjson_inline json_character_block json_character_block::classify(const simd::simd8x64& in) { - // These lookups rely on the fact that anything < 127 will match the lower 4 bits, which is why - // we can't use the generic lookup_16. - const auto whitespace_table = simd8::repeat_16(' ', 100, 100, 100, 17, 100, 113, 2, 100, '\t', '\n', 112, 100, '\r', 100, 100); - - // The 6 operators (:,[]{}) have these values: - // - // , 2C - // : 3A - // [ 5B - // { 7B - // ] 5D - // } 7D - // - // If you use | 0x20 to turn [ and ] into { and }, the lower 4 bits of each character is unique. - // We exploit this, using a simd 4-bit lookup to tell us which character match against, and then - // match it (against | 0x20). - // - // To prevent recognizing other characters, everything else gets compared with 0, which cannot - // match due to the | 0x20. - // - // NOTE: Due to the | 0x20, this ALSO treats and (control characters 0C and 1A) like , - // and :. This gets caught in stage 2, which checks the actual character to ensure the right - // operators are in the right places. - const auto op_table = simd8::repeat_16( - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, ':', '{', // : = 3A, [ = 5B, { = 7B - ',', '}', 0, 0 // , = 2C, ] = 5D, } = 7D - ); - - // We compute whitespace and op separately. If later code only uses one or the - // other, given the fact that all functions are aggressively inlined, we can - // hope that useless computations will be omitted. This is namely case when - // minifying (we only need whitespace). - - const uint64_t whitespace = in.eq({ - _mm512_shuffle_epi8(whitespace_table, in.chunks[0]) - }); - // Turn [ and ] into { and } - const simd8x64 curlified{ - in.chunks[0] | 0x20 - }; - const uint64_t op = curlified.eq({ - _mm512_shuffle_epi8(op_table, in.chunks[0]) - }); - - return { whitespace, op }; - } - - simdjson_inline bool is_ascii(const simd8x64& input) { - return input.reduce_or().is_ascii(); - } - - simdjson_unused simdjson_inline simd8 must_be_continuation(const simd8 prev1, const simd8 prev2, const simd8 prev3) { - simd8 is_second_byte = prev1.saturating_sub(0xc0u - 1); // Only 11______ will be > 0 - simd8 is_third_byte = prev2.saturating_sub(0xe0u - 1); // Only 111_____ will be > 0 - simd8 is_fourth_byte = prev3.saturating_sub(0xf0u - 1); // Only 1111____ will be > 0 - // Caller requires a bool (all 1's). All values resulting from the subtraction will be <= 64, so signed comparison is fine. - return simd8(is_second_byte | is_third_byte | is_fourth_byte) > int8_t(0); - } - - simdjson_inline simd8 must_be_2_3_continuation(const simd8 prev2, const simd8 prev3) { - simd8 is_third_byte = prev2.saturating_sub(0xe0u - 1); // Only 111_____ will be > 0 - simd8 is_fourth_byte = prev3.saturating_sub(0xf0u - 1); // Only 1111____ will be > 0 - // Caller requires a bool (all 1's). All values resulting from the subtraction will be <= 64, so signed comparison is fine. - return simd8(is_third_byte | is_fourth_byte) > int8_t(0); - } - - } // unnamed namespace - } // namespace icelake -} // namespace simdjson - -/** - * We provide a custom version of bit_indexer::write using - * naked intrinsics. - * TODO: make this code more elegant. - */ - // Under GCC 12, the intrinsic _mm512_extracti32x4_epi32 may generate 'maybe uninitialized'. - // as a workaround, we disable warnings within the following function. -SIMDJSON_PUSH_DISABLE_ALL_WARNINGS -namespace simdjson { - namespace icelake { - namespace { - namespace stage1 { - simdjson_inline void bit_indexer::write(uint32_t idx, uint64_t bits) { - // In some instances, the next branch is expensive because it is mispredicted. - // Unfortunately, in other cases, - // it helps tremendously. - if (bits == 0) { return; } - - const __m512i indexes = _mm512_maskz_compress_epi8(bits, _mm512_set_epi32( - 0x3f3e3d3c, 0x3b3a3938, 0x37363534, 0x33323130, - 0x2f2e2d2c, 0x2b2a2928, 0x27262524, 0x23222120, - 0x1f1e1d1c, 0x1b1a1918, 0x17161514, 0x13121110, - 0x0f0e0d0c, 0x0b0a0908, 0x07060504, 0x03020100 - )); - const __m512i start_index = _mm512_set1_epi32(idx); - - const auto count = count_ones(bits); - __m512i t0 = _mm512_cvtepu8_epi32(_mm512_castsi512_si128(indexes)); - _mm512_storeu_si512(this->tail, _mm512_add_epi32(t0, start_index)); - - if (count > 16) { - const __m512i t1 = _mm512_cvtepu8_epi32(_mm512_extracti32x4_epi32(indexes, 1)); - _mm512_storeu_si512(this->tail + 16, _mm512_add_epi32(t1, start_index)); - if (count > 32) { - const __m512i t2 = _mm512_cvtepu8_epi32(_mm512_extracti32x4_epi32(indexes, 2)); - _mm512_storeu_si512(this->tail + 32, _mm512_add_epi32(t2, start_index)); - if (count > 48) { - const __m512i t3 = _mm512_cvtepu8_epi32(_mm512_extracti32x4_epi32(indexes, 3)); - _mm512_storeu_si512(this->tail + 48, _mm512_add_epi32(t3, start_index)); - } - } - } - this->tail += count; - } - } - } - } -} -SIMDJSON_POP_DISABLE_WARNINGS - -// -// Stage 2 -// - -// -// Implementation-specific overrides -// -namespace simdjson { - namespace icelake { - - simdjson_warn_unused error_code implementation::minify(const uint8_t* buf, size_t len, uint8_t* dst, size_t& dst_len) const noexcept { - return icelake::stage1::json_minifier::minify<128>(buf, len, dst, dst_len); - } - - simdjson_warn_unused error_code dom_parser_implementation::stage1(const uint8_t* _buf, size_t _len, stage1_mode streaming) noexcept { - this->buf = _buf; - this->len = _len; - return icelake::stage1::json_structural_indexer::index<128>(_buf, _len, *this, streaming); - } - - simdjson_warn_unused bool implementation::validate_utf8(const char* buf, size_t len) const noexcept { - return icelake::stage1::generic_validate_utf8(buf, len); - } - - simdjson_warn_unused error_code dom_parser_implementation::stage2(dom::document& _doc) noexcept { - return stage2::tape_builder::parse_document(*this, _doc); - } - - simdjson_warn_unused error_code dom_parser_implementation::stage2_next(dom::document& _doc) noexcept { - return stage2::tape_builder::parse_document(*this, _doc); - } - - simdjson_warn_unused uint8_t* dom_parser_implementation::parse_string(const uint8_t* src, uint8_t* dst, bool replacement_char) const noexcept { - return icelake::stringparsing::parse_string(src, dst, replacement_char); - } - - simdjson_warn_unused uint8_t* dom_parser_implementation::parse_wobbly_string(const uint8_t* src, uint8_t* dst) const noexcept { - return icelake::stringparsing::parse_wobbly_string(src, dst); - } - - simdjson_warn_unused error_code dom_parser_implementation::parse(const uint8_t* _buf, size_t _len, dom::document& _doc) noexcept { - auto error = stage1(_buf, _len, stage1_mode::regular); - if (error) { return error; } - return stage2(_doc); - } - - } // namespace icelake -} // namespace simdjson - -/* including simdjson/icelake/end.h: #include */ -/* begin file simdjson/icelake/end.h */ -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/icelake/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#if !SIMDJSON_CAN_ALWAYS_RUN_ICELAKE -SIMDJSON_UNTARGET_REGION -#endif - -/* undefining SIMDJSON_IMPLEMENTATION from "icelake" */ -#undef SIMDJSON_IMPLEMENTATION -/* end file simdjson/icelake/end.h */ - -#endif // SIMDJSON_SRC_ICELAKE_CPP -/* end file icelake.cpp */ -#endif -#if SIMDJSON_IMPLEMENTATION_PPC64 -/* including ppc64.cpp: #include */ -/* begin file ppc64.cpp */ -#ifndef SIMDJSON_SRC_PPC64_CPP -#define SIMDJSON_SRC_PPC64_CPP - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -/* including simdjson/ppc64.h: #include */ -/* begin file simdjson/ppc64.h */ -#ifndef SIMDJSON_PPC64_H -#define SIMDJSON_PPC64_H - -/* including simdjson/ppc64/begin.h: #include "simdjson/ppc64/begin.h" */ -/* begin file simdjson/ppc64/begin.h */ -/* defining SIMDJSON_IMPLEMENTATION to "ppc64" */ -#define SIMDJSON_IMPLEMENTATION ppc64 -/* including simdjson/ppc64/base.h: #include "simdjson/ppc64/base.h" */ -/* begin file simdjson/ppc64/base.h */ -#ifndef SIMDJSON_PPC64_BASE_H -#define SIMDJSON_PPC64_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - /** - * Implementation for ALTIVEC (PPC64). - */ - namespace ppc64 { - - class implementation; - - namespace { - namespace simd { - template struct simd8; - template struct simd8x64; - } // namespace simd - } // unnamed namespace - - } // namespace ppc64 -} // namespace simdjson - -#endif // SIMDJSON_PPC64_BASE_H -/* end file simdjson/ppc64/base.h */ -/* including simdjson/ppc64/intrinsics.h: #include "simdjson/ppc64/intrinsics.h" */ -/* begin file simdjson/ppc64/intrinsics.h */ -#ifndef SIMDJSON_PPC64_INTRINSICS_H -#define SIMDJSON_PPC64_INTRINSICS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/ppc64/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -// This should be the correct header whether -// you use visual studio or other compilers. -#include - -// These are defined by altivec.h in GCC toolchain, it is safe to undef them. -#ifdef bool -#undef bool -#endif - -#ifdef vector -#undef vector -#endif - -static_assert(sizeof(__vector unsigned char) <= simdjson::SIMDJSON_PADDING, "insufficient padding for ppc64"); - -#endif // SIMDJSON_PPC64_INTRINSICS_H -/* end file simdjson/ppc64/intrinsics.h */ -/* including simdjson/ppc64/bitmanipulation.h: #include "simdjson/ppc64/bitmanipulation.h" */ -/* begin file simdjson/ppc64/bitmanipulation.h */ -#ifndef SIMDJSON_PPC64_BITMANIPULATION_H -#define SIMDJSON_PPC64_BITMANIPULATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/ppc64/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - namespace ppc64 { - namespace { - - // We sometimes call trailing_zero on inputs that are zero, - // but the algorithms do not end up using the returned value. - // Sadly, sanitizers are not smart enough to figure it out. - SIMDJSON_NO_SANITIZE_UNDEFINED - // This function can be used safely even if not all bytes have been - // initialized. - // See issue https://github.com/simdjson/simdjson/issues/1965 - SIMDJSON_NO_SANITIZE_MEMORY - simdjson_inline int trailing_zeroes(uint64_t input_num) { -#if SIMDJSON_REGULAR_VISUAL_STUDIO - unsigned long ret; - // Search the mask data from least significant bit (LSB) - // to the most significant bit (MSB) for a set bit (1). - _BitScanForward64(&ret, input_num); - return (int)ret; -#else // SIMDJSON_REGULAR_VISUAL_STUDIO - return __builtin_ctzll(input_num); -#endif // SIMDJSON_REGULAR_VISUAL_STUDIO - } - - /* result might be undefined when input_num is zero */ - simdjson_inline uint64_t clear_lowest_bit(uint64_t input_num) { - return input_num & (input_num - 1); - } - - /* result might be undefined when input_num is zero */ - simdjson_inline int leading_zeroes(uint64_t input_num) { -#if SIMDJSON_REGULAR_VISUAL_STUDIO - unsigned long leading_zero = 0; - // Search the mask data from most significant bit (MSB) - // to least significant bit (LSB) for a set bit (1). - if (_BitScanReverse64(&leading_zero, input_num)) - return (int)(63 - leading_zero); - else - return 64; -#else - return __builtin_clzll(input_num); -#endif // SIMDJSON_REGULAR_VISUAL_STUDIO - } - -#if SIMDJSON_REGULAR_VISUAL_STUDIO - simdjson_inline int count_ones(uint64_t input_num) { - // note: we do not support legacy 32-bit Windows in this kernel - return __popcnt64(input_num); // Visual Studio wants two underscores - } -#else - simdjson_inline int count_ones(uint64_t input_num) { - return __builtin_popcountll(input_num); - } -#endif - - simdjson_inline bool add_overflow(uint64_t value1, uint64_t value2, - uint64_t* result) { -#if SIMDJSON_REGULAR_VISUAL_STUDIO - * result = value1 + value2; - return *result < value1; -#else - return __builtin_uaddll_overflow(value1, value2, - reinterpret_cast(result)); -#endif - } - - } // unnamed namespace - } // namespace ppc64 -} // namespace simdjson - -#endif // SIMDJSON_PPC64_BITMANIPULATION_H -/* end file simdjson/ppc64/bitmanipulation.h */ -/* including simdjson/ppc64/bitmask.h: #include "simdjson/ppc64/bitmask.h" */ -/* begin file simdjson/ppc64/bitmask.h */ -#ifndef SIMDJSON_PPC64_BITMASK_H -#define SIMDJSON_PPC64_BITMASK_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/ppc64/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - namespace ppc64 { - namespace { - - // - // Perform a "cumulative bitwise xor," flipping bits each time a 1 is - // encountered. - // - // For example, prefix_xor(00100100) == 00011100 - // - simdjson_inline uint64_t prefix_xor(uint64_t bitmask) { - // You can use the version below, however gcc sometimes miscompiles - // vec_pmsum_be, it happens somewhere around between 8 and 9th version. - // The performance boost was not noticeable, falling back to a usual - // implementation. - // __vector unsigned long long all_ones = {~0ull, ~0ull}; - // __vector unsigned long long mask = {bitmask, 0}; - // // Clang and GCC return different values for pmsum for ull so cast it to one. - // // Generally it is not specified by ALTIVEC ISA what is returned by - // // vec_pmsum_be. - // #if defined(__LITTLE_ENDIAN__) - // return (uint64_t)(((__vector unsigned long long)vec_pmsum_be(all_ones, mask))[0]); - // #else - // return (uint64_t)(((__vector unsigned long long)vec_pmsum_be(all_ones, mask))[1]); - // #endif - bitmask ^= bitmask << 1; - bitmask ^= bitmask << 2; - bitmask ^= bitmask << 4; - bitmask ^= bitmask << 8; - bitmask ^= bitmask << 16; - bitmask ^= bitmask << 32; - return bitmask; - } - - } // unnamed namespace - } // namespace ppc64 -} // namespace simdjson - -#endif -/* end file simdjson/ppc64/bitmask.h */ -/* including simdjson/ppc64/numberparsing_defs.h: #include "simdjson/ppc64/numberparsing_defs.h" */ -/* begin file simdjson/ppc64/numberparsing_defs.h */ -#ifndef SIMDJSON_PPC64_NUMBERPARSING_DEFS_H -#define SIMDJSON_PPC64_NUMBERPARSING_DEFS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/ppc64/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/ppc64/intrinsics.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include - -#if defined(__linux__) -#include -#elif defined(__FreeBSD__) -#include -#endif - -namespace simdjson { - namespace ppc64 { - namespace numberparsing { - - // we don't have appropriate instructions, so let us use a scalar function - // credit: https://johnnylee-sde.github.io/Fast-numeric-string-to-int/ - /** @private */ - static simdjson_inline uint32_t parse_eight_digits_unrolled(const uint8_t* chars) { - uint64_t val; - std::memcpy(&val, chars, sizeof(uint64_t)); -#ifdef __BIG_ENDIAN__ -#if defined(__linux__) - val = bswap_64(val); -#elif defined(__FreeBSD__) - val = bswap64(val); -#endif -#endif - val = (val & 0x0F0F0F0F0F0F0F0F) * 2561 >> 8; - val = (val & 0x00FF00FF00FF00FF) * 6553601 >> 16; - return uint32_t((val & 0x0000FFFF0000FFFF) * 42949672960001 >> 32); - } - - /** @private */ - simdjson_inline internal::value128 full_multiplication(uint64_t value1, uint64_t value2) { - internal::value128 answer; -#if SIMDJSON_REGULAR_VISUAL_STUDIO || SIMDJSON_IS_32BITS -#ifdef _M_ARM64 - // ARM64 has native support for 64-bit multiplications, no need to emultate - answer.high = __umulh(value1, value2); - answer.low = value1 * value2; -#else - answer.low = _umul128(value1, value2, &answer.high); // _umul128 not available on ARM64 -#endif // _M_ARM64 -#else // SIMDJSON_REGULAR_VISUAL_STUDIO || SIMDJSON_IS_32BITS - __uint128_t r = (static_cast<__uint128_t>(value1)) * value2; - answer.low = uint64_t(r); - answer.high = uint64_t(r >> 64); -#endif - return answer; - } - - } // namespace numberparsing - } // namespace ppc64 -} // namespace simdjson - -#define SIMDJSON_SWAR_NUMBER_PARSING 1 - -#endif // SIMDJSON_PPC64_NUMBERPARSING_DEFS_H -/* end file simdjson/ppc64/numberparsing_defs.h */ -/* including simdjson/ppc64/simd.h: #include "simdjson/ppc64/simd.h" */ -/* begin file simdjson/ppc64/simd.h */ -#ifndef SIMDJSON_PPC64_SIMD_H -#define SIMDJSON_PPC64_SIMD_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/ppc64/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/ppc64/bitmanipulation.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/simdprune_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include - -namespace simdjson { - namespace ppc64 { - namespace { - namespace simd { - - using __m128i = __vector unsigned char; - - template struct base { - __m128i value; - - // Zero constructor - simdjson_inline base() : value{ __m128i() } {} - - // Conversion from SIMD register - simdjson_inline base(const __m128i _value) : value(_value) {} - - // Conversion to SIMD register - simdjson_inline operator const __m128i& () const { - return this->value; - } - simdjson_inline operator __m128i& () { return this->value; } - - // Bit operations - simdjson_inline Child operator|(const Child other) const { - return vec_or(this->value, (__m128i)other); - } - simdjson_inline Child operator&(const Child other) const { - return vec_and(this->value, (__m128i)other); - } - simdjson_inline Child operator^(const Child other) const { - return vec_xor(this->value, (__m128i)other); - } - simdjson_inline Child bit_andnot(const Child other) const { - return vec_andc(this->value, (__m128i)other); - } - simdjson_inline Child& operator|=(const Child other) { - auto this_cast = static_cast(this); - *this_cast = *this_cast | other; - return *this_cast; - } - simdjson_inline Child& operator&=(const Child other) { - auto this_cast = static_cast(this); - *this_cast = *this_cast & other; - return *this_cast; - } - simdjson_inline Child& operator^=(const Child other) { - auto this_cast = static_cast(this); - *this_cast = *this_cast ^ other; - return *this_cast; - } - }; - - template > - struct base8 : base> { - typedef uint16_t bitmask_t; - typedef uint32_t bitmask2_t; - - simdjson_inline base8() : base>() {} - simdjson_inline base8(const __m128i _value) : base>(_value) {} - - friend simdjson_inline Mask operator==(const simd8 lhs, const simd8 rhs) { - return (__m128i)vec_cmpeq(lhs.value, (__m128i)rhs); - } - - static const int SIZE = sizeof(base>::value); - - template - simdjson_inline simd8 prev(simd8 prev_chunk) const { - __m128i chunk = this->value; -#ifdef __LITTLE_ENDIAN__ - chunk = (__m128i)vec_reve(this->value); - prev_chunk = (__m128i)vec_reve((__m128i)prev_chunk); -#endif - chunk = (__m128i)vec_sld((__m128i)prev_chunk, (__m128i)chunk, 16 - N); -#ifdef __LITTLE_ENDIAN__ - chunk = (__m128i)vec_reve((__m128i)chunk); -#endif - return chunk; - } - }; - - // SIMD byte mask type (returned by things like eq and gt) - template <> struct simd8 : base8 { - static simdjson_inline simd8 splat(bool _value) { - return (__m128i)vec_splats((unsigned char)(-(!!_value))); - } - - simdjson_inline simd8() : base8() {} - simdjson_inline simd8(const __m128i _value) - : base8(_value) {} - // Splat constructor - simdjson_inline simd8(bool _value) - : base8(splat(_value)) {} - - simdjson_inline int to_bitmask() const { - __vector unsigned long long result; - const __m128i perm_mask = { 0x78, 0x70, 0x68, 0x60, 0x58, 0x50, 0x48, 0x40, - 0x38, 0x30, 0x28, 0x20, 0x18, 0x10, 0x08, 0x00 }; - - result = ((__vector unsigned long long)vec_vbpermq((__m128i)this->value, - (__m128i)perm_mask)); -#ifdef __LITTLE_ENDIAN__ - return static_cast(result[1]); -#else - return static_cast(result[0]); -#endif - } - simdjson_inline bool any() const { - return !vec_all_eq(this->value, (__m128i)vec_splats(0)); - } - simdjson_inline simd8 operator~() const { - return this->value ^ (__m128i)splat(true); - } - }; - - template struct base8_numeric : base8 { - static simdjson_inline simd8 splat(T value) { - (void)value; - return (__m128i)vec_splats(value); - } - static simdjson_inline simd8 zero() { return splat(0); } - static simdjson_inline simd8 load(const T values[16]) { - return (__m128i)(vec_vsx_ld(0, reinterpret_cast(values))); - } - // Repeat 16 values as many times as necessary (usually for lookup tables) - static simdjson_inline simd8 repeat_16(T v0, T v1, T v2, T v3, T v4, - T v5, T v6, T v7, T v8, T v9, - T v10, T v11, T v12, T v13, - T v14, T v15) { - return simd8(v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, - v14, v15); - } - - simdjson_inline base8_numeric() : base8() {} - simdjson_inline base8_numeric(const __m128i _value) - : base8(_value) {} - - // Store to array - simdjson_inline void store(T dst[16]) const { - vec_vsx_st(this->value, 0, reinterpret_cast<__m128i*>(dst)); - } - - // Override to distinguish from bool version - simdjson_inline simd8 operator~() const { return *this ^ 0xFFu; } - - // Addition/subtraction are the same for signed and unsigned - simdjson_inline simd8 operator+(const simd8 other) const { - return (__m128i)((__m128i)this->value + (__m128i)other); - } - simdjson_inline simd8 operator-(const simd8 other) const { - return (__m128i)((__m128i)this->value - (__m128i)other); - } - simdjson_inline simd8& operator+=(const simd8 other) { - *this = *this + other; - return *static_cast *>(this); - } - simdjson_inline simd8& operator-=(const simd8 other) { - *this = *this - other; - return *static_cast *>(this); - } - - // Perform a lookup assuming the value is between 0 and 16 (undefined behavior - // for out of range values) - template - simdjson_inline simd8 lookup_16(simd8 lookup_table) const { - return (__m128i)vec_perm((__m128i)lookup_table, (__m128i)lookup_table, this->value); - } - - // Copies to 'output" all bytes corresponding to a 0 in the mask (interpreted - // as a bitset). Passing a 0 value for mask would be equivalent to writing out - // every byte to output. Only the first 16 - count_ones(mask) bytes of the - // result are significant but 16 bytes get written. Design consideration: it - // seems like a function with the signature simd8 compress(uint32_t mask) - // would be sensible, but the AVX ISA makes this kind of approach difficult. - template - simdjson_inline void compress(uint16_t mask, L* output) const { - using internal::BitsSetTable256mul2; - using internal::pshufb_combine_table; - using internal::thintable_epi8; - // this particular implementation was inspired by work done by @animetosho - // we do it in two steps, first 8 bytes and then second 8 bytes - uint8_t mask1 = uint8_t(mask); // least significant 8 bits - uint8_t mask2 = uint8_t(mask >> 8); // most significant 8 bits - // next line just loads the 64-bit values thintable_epi8[mask1] and - // thintable_epi8[mask2] into a 128-bit register, using only - // two instructions on most compilers. -#ifdef __LITTLE_ENDIAN__ - __m128i shufmask = (__m128i)(__vector unsigned long long) { - thintable_epi8[mask1], thintable_epi8[mask2] - }; -#else - __m128i shufmask = (__m128i)(__vector unsigned long long) { - thintable_epi8[mask2], thintable_epi8[mask1] - }; - shufmask = (__m128i)vec_reve((__m128i)shufmask); -#endif - // we increment by 0x08 the second half of the mask - shufmask = ((__m128i)shufmask) + - ((__m128i)(__vector int) { 0, 0, 0x08080808, 0x08080808 }); - - // this is the version "nearly pruned" - __m128i pruned = vec_perm(this->value, this->value, shufmask); - // we still need to put the two halves together. - // we compute the popcount of the first half: - int pop1 = BitsSetTable256mul2[mask1]; - // then load the corresponding mask, what it does is to write - // only the first pop1 bytes from the first 8 bytes, and then - // it fills in with the bytes from the second 8 bytes + some filling - // at the end. - __m128i compactmask = - vec_vsx_ld(0, reinterpret_cast(pshufb_combine_table + pop1 * 8)); - __m128i answer = vec_perm(pruned, (__m128i)vec_splats(0), compactmask); - vec_vsx_st(answer, 0, reinterpret_cast<__m128i*>(output)); - } - - template - simdjson_inline simd8 - lookup_16(L replace0, L replace1, L replace2, L replace3, L replace4, - L replace5, L replace6, L replace7, L replace8, L replace9, - L replace10, L replace11, L replace12, L replace13, L replace14, - L replace15) const { - return lookup_16(simd8::repeat_16( - replace0, replace1, replace2, replace3, replace4, replace5, replace6, - replace7, replace8, replace9, replace10, replace11, replace12, - replace13, replace14, replace15)); - } - }; - - // Signed bytes - template <> struct simd8 : base8_numeric { - simdjson_inline simd8() : base8_numeric() {} - simdjson_inline simd8(const __m128i _value) - : base8_numeric(_value) {} - // Splat constructor - simdjson_inline simd8(int8_t _value) : simd8(splat(_value)) {} - // Array constructor - simdjson_inline simd8(const int8_t* values) : simd8(load(values)) {} - // Member-by-member initialization - simdjson_inline simd8(int8_t v0, int8_t v1, int8_t v2, int8_t v3, - int8_t v4, int8_t v5, int8_t v6, int8_t v7, - int8_t v8, int8_t v9, int8_t v10, int8_t v11, - int8_t v12, int8_t v13, int8_t v14, int8_t v15) - : simd8((__m128i)(__vector signed char) { - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10, v11, v12, v13, v14, - v15 - }) {} - // Repeat 16 values as many times as necessary (usually for lookup tables) - simdjson_inline static simd8 - repeat_16(int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, - int8_t v6, int8_t v7, int8_t v8, int8_t v9, int8_t v10, int8_t v11, - int8_t v12, int8_t v13, int8_t v14, int8_t v15) { - return simd8(v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, - v13, v14, v15); - } - - // Order-sensitive comparisons - simdjson_inline simd8 - max_val(const simd8 other) const { - return (__m128i)vec_max((__vector signed char)this->value, - (__vector signed char)(__m128i)other); - } - simdjson_inline simd8 - min_val(const simd8 other) const { - return (__m128i)vec_min((__vector signed char)this->value, - (__vector signed char)(__m128i)other); - } - simdjson_inline simd8 - operator>(const simd8 other) const { - return (__m128i)vec_cmpgt((__vector signed char)this->value, - (__vector signed char)(__m128i)other); - } - simdjson_inline simd8 - operator<(const simd8 other) const { - return (__m128i)vec_cmplt((__vector signed char)this->value, - (__vector signed char)(__m128i)other); - } - }; - - // Unsigned bytes - template <> struct simd8 : base8_numeric { - simdjson_inline simd8() : base8_numeric() {} - simdjson_inline simd8(const __m128i _value) - : base8_numeric(_value) {} - // Splat constructor - simdjson_inline simd8(uint8_t _value) : simd8(splat(_value)) {} - // Array constructor - simdjson_inline simd8(const uint8_t* values) : simd8(load(values)) {} - // Member-by-member initialization - simdjson_inline - simd8(uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, - uint8_t v6, uint8_t v7, uint8_t v8, uint8_t v9, uint8_t v10, - uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15) - : simd8((__m128i) { - v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, - v13, v14, v15 - }) {} - // Repeat 16 values as many times as necessary (usually for lookup tables) - simdjson_inline static simd8 - repeat_16(uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, - uint8_t v5, uint8_t v6, uint8_t v7, uint8_t v8, uint8_t v9, - uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, - uint8_t v15) { - return simd8(v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, - v13, v14, v15); - } - - // Saturated math - simdjson_inline simd8 - saturating_add(const simd8 other) const { - return (__m128i)vec_adds(this->value, (__m128i)other); - } - simdjson_inline simd8 - saturating_sub(const simd8 other) const { - return (__m128i)vec_subs(this->value, (__m128i)other); - } - - // Order-specific operations - simdjson_inline simd8 - max_val(const simd8 other) const { - return (__m128i)vec_max(this->value, (__m128i)other); - } - simdjson_inline simd8 - min_val(const simd8 other) const { - return (__m128i)vec_min(this->value, (__m128i)other); - } - // Same as >, but only guarantees true is nonzero (< guarantees true = -1) - simdjson_inline simd8 - gt_bits(const simd8 other) const { - return this->saturating_sub(other); - } - // Same as <, but only guarantees true is nonzero (< guarantees true = -1) - simdjson_inline simd8 - lt_bits(const simd8 other) const { - return other.saturating_sub(*this); - } - simdjson_inline simd8 - operator<=(const simd8 other) const { - return other.max_val(*this) == other; - } - simdjson_inline simd8 - operator>=(const simd8 other) const { - return other.min_val(*this) == other; - } - simdjson_inline simd8 - operator>(const simd8 other) const { - return this->gt_bits(other).any_bits_set(); - } - simdjson_inline simd8 - operator<(const simd8 other) const { - return this->gt_bits(other).any_bits_set(); - } - - // Bit-specific operations - simdjson_inline simd8 bits_not_set() const { - return (__m128i)vec_cmpeq(this->value, (__m128i)vec_splats(uint8_t(0))); - } - simdjson_inline simd8 bits_not_set(simd8 bits) const { - return (*this & bits).bits_not_set(); - } - simdjson_inline simd8 any_bits_set() const { - return ~this->bits_not_set(); - } - simdjson_inline simd8 any_bits_set(simd8 bits) const { - return ~this->bits_not_set(bits); - } - simdjson_inline bool bits_not_set_anywhere() const { - return vec_all_eq(this->value, (__m128i)vec_splats(0)); - } - simdjson_inline bool any_bits_set_anywhere() const { - return !bits_not_set_anywhere(); - } - simdjson_inline bool bits_not_set_anywhere(simd8 bits) const { - return vec_all_eq(vec_and(this->value, (__m128i)bits), - (__m128i)vec_splats(0)); - } - simdjson_inline bool any_bits_set_anywhere(simd8 bits) const { - return !bits_not_set_anywhere(bits); - } - template simdjson_inline simd8 shr() const { - return simd8( - (__m128i)vec_sr(this->value, (__m128i)vec_splat_u8(N))); - } - template simdjson_inline simd8 shl() const { - return simd8( - (__m128i)vec_sl(this->value, (__m128i)vec_splat_u8(N))); - } - }; - - template struct simd8x64 { - static constexpr int NUM_CHUNKS = 64 / sizeof(simd8); - static_assert(NUM_CHUNKS == 4, - "PPC64 kernel should use four registers per 64-byte block."); - const simd8 chunks[NUM_CHUNKS]; - - simd8x64(const simd8x64& o) = delete; // no copy allowed - simd8x64& - operator=(const simd8& other) = delete; // no assignment allowed - simd8x64() = delete; // no default constructor allowed - - simdjson_inline simd8x64(const simd8 chunk0, const simd8 chunk1, - const simd8 chunk2, const simd8 chunk3) - : chunks{ chunk0, chunk1, chunk2, chunk3 } {} - simdjson_inline simd8x64(const T ptr[64]) - : chunks{ simd8::load(ptr), simd8::load(ptr + 16), - simd8::load(ptr + 32), simd8::load(ptr + 48) } {} - - simdjson_inline void store(T ptr[64]) const { - this->chunks[0].store(ptr + sizeof(simd8) * 0); - this->chunks[1].store(ptr + sizeof(simd8) * 1); - this->chunks[2].store(ptr + sizeof(simd8) * 2); - this->chunks[3].store(ptr + sizeof(simd8) * 3); - } - - simdjson_inline simd8 reduce_or() const { - return (this->chunks[0] | this->chunks[1]) | - (this->chunks[2] | this->chunks[3]); - } - - simdjson_inline uint64_t compress(uint64_t mask, T* output) const { - this->chunks[0].compress(uint16_t(mask), output); - this->chunks[1].compress(uint16_t(mask >> 16), - output + 16 - count_ones(mask & 0xFFFF)); - this->chunks[2].compress(uint16_t(mask >> 32), - output + 32 - count_ones(mask & 0xFFFFFFFF)); - this->chunks[3].compress(uint16_t(mask >> 48), - output + 48 - count_ones(mask & 0xFFFFFFFFFFFF)); - return 64 - count_ones(mask); - } - - simdjson_inline uint64_t to_bitmask() const { - uint64_t r0 = uint32_t(this->chunks[0].to_bitmask()); - uint64_t r1 = this->chunks[1].to_bitmask(); - uint64_t r2 = this->chunks[2].to_bitmask(); - uint64_t r3 = this->chunks[3].to_bitmask(); - return r0 | (r1 << 16) | (r2 << 32) | (r3 << 48); - } - - simdjson_inline uint64_t eq(const T m) const { - const simd8 mask = simd8::splat(m); - return simd8x64(this->chunks[0] == mask, this->chunks[1] == mask, - this->chunks[2] == mask, this->chunks[3] == mask) - .to_bitmask(); - } - - simdjson_inline uint64_t eq(const simd8x64& other) const { - return simd8x64(this->chunks[0] == other.chunks[0], - this->chunks[1] == other.chunks[1], - this->chunks[2] == other.chunks[2], - this->chunks[3] == other.chunks[3]) - .to_bitmask(); - } - - simdjson_inline uint64_t lteq(const T m) const { - const simd8 mask = simd8::splat(m); - return simd8x64(this->chunks[0] <= mask, this->chunks[1] <= mask, - this->chunks[2] <= mask, this->chunks[3] <= mask) - .to_bitmask(); - } - }; // struct simd8x64 - - } // namespace simd - } // unnamed namespace - } // namespace ppc64 -} // namespace simdjson - -#endif // SIMDJSON_PPC64_SIMD_INPUT_H -/* end file simdjson/ppc64/simd.h */ -/* including simdjson/ppc64/stringparsing_defs.h: #include "simdjson/ppc64/stringparsing_defs.h" */ -/* begin file simdjson/ppc64/stringparsing_defs.h */ -#ifndef SIMDJSON_PPC64_STRINGPARSING_DEFS_H -#define SIMDJSON_PPC64_STRINGPARSING_DEFS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/ppc64/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/ppc64/bitmanipulation.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/ppc64/simd.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - namespace ppc64 { - namespace { - - using namespace simd; - - // Holds backslashes and quotes locations. - struct backslash_and_quote { - public: - static constexpr uint32_t BYTES_PROCESSED = 32; - simdjson_inline static backslash_and_quote - copy_and_find(const uint8_t* src, uint8_t* dst); - - simdjson_inline bool has_quote_first() { - return ((bs_bits - 1) & quote_bits) != 0; - } - simdjson_inline bool has_backslash() { return bs_bits != 0; } - simdjson_inline int quote_index() { - return trailing_zeroes(quote_bits); - } - simdjson_inline int backslash_index() { - return trailing_zeroes(bs_bits); - } - - uint32_t bs_bits; - uint32_t quote_bits; - }; // struct backslash_and_quote - - simdjson_inline backslash_and_quote - backslash_and_quote::copy_and_find(const uint8_t* src, uint8_t* dst) { - // this can read up to 31 bytes beyond the buffer size, but we require - // SIMDJSON_PADDING of padding - static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), - "backslash and quote finder must process fewer than " - "SIMDJSON_PADDING bytes"); - simd8 v0(src); - simd8 v1(src + sizeof(v0)); - v0.store(dst); - v1.store(dst + sizeof(v0)); - - // Getting a 64-bit bitmask is much cheaper than multiple 16-bit bitmasks on - // PPC; therefore, we smash them together into a 64-byte mask and get the - // bitmask from there. - uint64_t bs_and_quote = - simd8x64(v0 == '\\', v1 == '\\', v0 == '"', v1 == '"').to_bitmask(); - return { - uint32_t(bs_and_quote), // bs_bits - uint32_t(bs_and_quote >> 32) // quote_bits - }; - } - - } // unnamed namespace - } // namespace ppc64 -} // namespace simdjson - -#endif // SIMDJSON_PPC64_STRINGPARSING_DEFS_H -/* end file simdjson/ppc64/stringparsing_defs.h */ - -#define SIMDJSON_SKIP_BACKSLASH_SHORT_CIRCUIT 1 -/* end file simdjson/ppc64/begin.h */ -/* including simdjson/generic/amalgamated.h for ppc64: #include "simdjson/generic/amalgamated.h" */ -/* begin file simdjson/generic/amalgamated.h for ppc64 */ -#if defined(SIMDJSON_CONDITIONAL_INCLUDE) && !defined(SIMDJSON_GENERIC_DEPENDENCIES_H) -#error simdjson/generic/dependencies.h must be included before simdjson/generic/amalgamated.h! -#endif - -/* including simdjson/generic/base.h for ppc64: #include "simdjson/generic/base.h" */ -/* begin file simdjson/generic/base.h for ppc64 */ -#ifndef SIMDJSON_GENERIC_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_BASE_H */ -/* amalgamation skipped (editor-only): #include "simdjson/base.h" */ -/* amalgamation skipped (editor-only): // If we haven't got an implementation yet, we're in the editor, editing a generic file! Just */ -/* amalgamation skipped (editor-only): // use the most advanced one we can so the most possible stuff can be tested. */ -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_IMPLEMENTATION */ -/* amalgamation skipped (editor-only): #include "simdjson/implementation_detection.h" */ -/* amalgamation skipped (editor-only): #if SIMDJSON_IMPLEMENTATION_ICELAKE */ -/* amalgamation skipped (editor-only): #include "simdjson/icelake/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_HASWELL */ -/* amalgamation skipped (editor-only): #include "simdjson/haswell/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_WESTMERE */ -/* amalgamation skipped (editor-only): #include "simdjson/westmere/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_ARM64 */ -/* amalgamation skipped (editor-only): #include "simdjson/arm64/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_PPC64 */ -/* amalgamation skipped (editor-only): #include "simdjson/ppc64/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_FALLBACK */ -/* amalgamation skipped (editor-only): #include "simdjson/fallback/begin.h" */ -/* amalgamation skipped (editor-only): #else */ -/* amalgamation skipped (editor-only): #error "All possible implementations (including fallback) have been disabled! simdjson will not run." */ -/* amalgamation skipped (editor-only): #endif */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_IMPLEMENTATION */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - namespace ppc64 { - - struct open_container; - class dom_parser_implementation; - - /** - * The type of a JSON number - */ - enum class number_type { - floating_point_number = 1, /// a binary64 number - signed_integer, /// a signed integer that fits in a 64-bit word using two's complement - unsigned_integer /// a positive integer larger or equal to 1<<63 - }; - - } // namespace ppc64 -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_BASE_H -/* end file simdjson/generic/base.h for ppc64 */ -/* including simdjson/generic/jsoncharutils.h for ppc64: #include "simdjson/generic/jsoncharutils.h" */ -/* begin file simdjson/generic/jsoncharutils.h for ppc64 */ -#ifndef SIMDJSON_GENERIC_JSONCHARUTILS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_JSONCHARUTILS_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/jsoncharutils_tables.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - namespace ppc64 { - namespace { - namespace jsoncharutils { - - // return non-zero if not a structural or whitespace char - // zero otherwise - simdjson_inline uint32_t is_not_structural_or_whitespace(uint8_t c) { - return internal::structural_or_whitespace_negated[c]; - } - - simdjson_inline uint32_t is_structural_or_whitespace(uint8_t c) { - return internal::structural_or_whitespace[c]; - } - - // returns a value with the high 16 bits set if not valid - // otherwise returns the conversion of the 4 hex digits at src into the bottom - // 16 bits of the 32-bit return register - // - // see - // https://lemire.me/blog/2019/04/17/parsing-short-hexadecimal-strings-efficiently/ - static inline uint32_t hex_to_u32_nocheck( - const uint8_t* src) { // strictly speaking, static inline is a C-ism - uint32_t v1 = internal::digit_to_val32[630 + src[0]]; - uint32_t v2 = internal::digit_to_val32[420 + src[1]]; - uint32_t v3 = internal::digit_to_val32[210 + src[2]]; - uint32_t v4 = internal::digit_to_val32[0 + src[3]]; - return v1 | v2 | v3 | v4; - } - - // given a code point cp, writes to c - // the utf-8 code, outputting the length in - // bytes, if the length is zero, the code point - // is invalid - // - // This can possibly be made faster using pdep - // and clz and table lookups, but JSON documents - // have few escaped code points, and the following - // function looks cheap. - // - // Note: we assume that surrogates are treated separately - // - simdjson_inline size_t codepoint_to_utf8(uint32_t cp, uint8_t* c) { - if (cp <= 0x7F) { - c[0] = uint8_t(cp); - return 1; // ascii - } - if (cp <= 0x7FF) { - c[0] = uint8_t((cp >> 6) + 192); - c[1] = uint8_t((cp & 63) + 128); - return 2; // universal plane - // Surrogates are treated elsewhere... - //} //else if (0xd800 <= cp && cp <= 0xdfff) { - // return 0; // surrogates // could put assert here - } - else if (cp <= 0xFFFF) { - c[0] = uint8_t((cp >> 12) + 224); - c[1] = uint8_t(((cp >> 6) & 63) + 128); - c[2] = uint8_t((cp & 63) + 128); - return 3; - } - else if (cp <= 0x10FFFF) { // if you know you have a valid code point, this - // is not needed - c[0] = uint8_t((cp >> 18) + 240); - c[1] = uint8_t(((cp >> 12) & 63) + 128); - c[2] = uint8_t(((cp >> 6) & 63) + 128); - c[3] = uint8_t((cp & 63) + 128); - return 4; - } - // will return 0 when the code point was too large. - return 0; // bad r - } - -#if SIMDJSON_IS_32BITS // _umul128 for x86, arm - // this is a slow emulation routine for 32-bit - // - static simdjson_inline uint64_t __emulu(uint32_t x, uint32_t y) { - return x * (uint64_t)y; - } - static simdjson_inline uint64_t _umul128(uint64_t ab, uint64_t cd, uint64_t* hi) { - uint64_t ad = __emulu((uint32_t)(ab >> 32), (uint32_t)cd); - uint64_t bd = __emulu((uint32_t)ab, (uint32_t)cd); - uint64_t adbc = ad + __emulu((uint32_t)ab, (uint32_t)(cd >> 32)); - uint64_t adbc_carry = !!(adbc < ad); - uint64_t lo = bd + (adbc << 32); - *hi = __emulu((uint32_t)(ab >> 32), (uint32_t)(cd >> 32)) + (adbc >> 32) + - (adbc_carry << 32) + !!(lo < bd); - return lo; - } -#endif - - } // namespace jsoncharutils - } // unnamed namespace - } // namespace ppc64 -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_JSONCHARUTILS_H -/* end file simdjson/generic/jsoncharutils.h for ppc64 */ -/* including simdjson/generic/atomparsing.h for ppc64: #include "simdjson/generic/atomparsing.h" */ -/* begin file simdjson/generic/atomparsing.h for ppc64 */ -#ifndef SIMDJSON_GENERIC_ATOMPARSING_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_ATOMPARSING_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/jsoncharutils.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include - -namespace simdjson { - namespace ppc64 { - namespace { - /// @private - namespace atomparsing { - - // The string_to_uint32 is exclusively used to map literal strings to 32-bit values. - // We use memcpy instead of a pointer cast to avoid undefined behaviors since we cannot - // be certain that the character pointer will be properly aligned. - // You might think that using memcpy makes this function expensive, but you'd be wrong. - // All decent optimizing compilers (GCC, clang, Visual Studio) will compile string_to_uint32("false"); - // to the compile-time constant 1936482662. - simdjson_inline uint32_t string_to_uint32(const char* str) { uint32_t val; std::memcpy(&val, str, sizeof(uint32_t)); return val; } - - - // Again in str4ncmp we use a memcpy to avoid undefined behavior. The memcpy may appear expensive. - // Yet all decent optimizing compilers will compile memcpy to a single instruction, just about. - simdjson_warn_unused - simdjson_inline uint32_t str4ncmp(const uint8_t* src, const char* atom) { - uint32_t srcval; // we want to avoid unaligned 32-bit loads (undefined in C/C++) - static_assert(sizeof(uint32_t) <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be larger than 4 bytes"); - std::memcpy(&srcval, src, sizeof(uint32_t)); - return srcval ^ string_to_uint32(atom); - } - - simdjson_warn_unused - simdjson_inline bool is_valid_true_atom(const uint8_t* src) { - return (str4ncmp(src, "true") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; - } - - simdjson_warn_unused - simdjson_inline bool is_valid_true_atom(const uint8_t* src, size_t len) { - if (len > 4) { return is_valid_true_atom(src); } - else if (len == 4) { return !str4ncmp(src, "true"); } - else { return false; } - } - - simdjson_warn_unused - simdjson_inline bool is_valid_false_atom(const uint8_t* src) { - return (str4ncmp(src + 1, "alse") | jsoncharutils::is_not_structural_or_whitespace(src[5])) == 0; - } - - simdjson_warn_unused - simdjson_inline bool is_valid_false_atom(const uint8_t* src, size_t len) { - if (len > 5) { return is_valid_false_atom(src); } - else if (len == 5) { return !str4ncmp(src + 1, "alse"); } - else { return false; } - } - - simdjson_warn_unused - simdjson_inline bool is_valid_null_atom(const uint8_t* src) { - return (str4ncmp(src, "null") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; - } - - simdjson_warn_unused - simdjson_inline bool is_valid_null_atom(const uint8_t* src, size_t len) { - if (len > 4) { return is_valid_null_atom(src); } - else if (len == 4) { return !str4ncmp(src, "null"); } - else { return false; } - } - - } // namespace atomparsing - } // unnamed namespace - } // namespace ppc64 -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_ATOMPARSING_H -/* end file simdjson/generic/atomparsing.h for ppc64 */ -/* including simdjson/generic/dom_parser_implementation.h for ppc64: #include "simdjson/generic/dom_parser_implementation.h" */ -/* begin file simdjson/generic/dom_parser_implementation.h for ppc64 */ -#ifndef SIMDJSON_GENERIC_DOM_PARSER_IMPLEMENTATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_DOM_PARSER_IMPLEMENTATION_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/dom_parser_implementation.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - namespace ppc64 { - - // expectation: sizeof(open_container) = 64/8. - struct open_container { - uint32_t tape_index; // where, on the tape, does the scope ([,{) begins - uint32_t count; // how many elements in the scope - }; // struct open_container - - static_assert(sizeof(open_container) == 64 / 8, "Open container must be 64 bits"); - - class dom_parser_implementation final : public internal::dom_parser_implementation { - public: - /** Tape location of each open { or [ */ - std::unique_ptr open_containers{}; - /** Whether each open container is a [ or { */ - std::unique_ptr is_array{}; - /** Buffer passed to stage 1 */ - const uint8_t* buf{}; - /** Length passed to stage 1 */ - size_t len{ 0 }; - /** Document passed to stage 2 */ - dom::document* doc{}; - - inline dom_parser_implementation() noexcept; - inline dom_parser_implementation(dom_parser_implementation&& other) noexcept; - inline dom_parser_implementation& operator=(dom_parser_implementation&& other) noexcept; - dom_parser_implementation(const dom_parser_implementation&) = delete; - dom_parser_implementation& operator=(const dom_parser_implementation&) = delete; - - simdjson_warn_unused error_code parse(const uint8_t* buf, size_t len, dom::document& doc) noexcept final; - simdjson_warn_unused error_code stage1(const uint8_t* buf, size_t len, stage1_mode partial) noexcept final; - simdjson_warn_unused error_code stage2(dom::document& doc) noexcept final; - simdjson_warn_unused error_code stage2_next(dom::document& doc) noexcept final; - simdjson_warn_unused uint8_t* parse_string(const uint8_t* src, uint8_t* dst, bool allow_replacement) const noexcept final; - simdjson_warn_unused uint8_t* parse_wobbly_string(const uint8_t* src, uint8_t* dst) const noexcept final; - inline simdjson_warn_unused error_code set_capacity(size_t capacity) noexcept final; - inline simdjson_warn_unused error_code set_max_depth(size_t max_depth) noexcept final; - private: - simdjson_inline simdjson_warn_unused error_code set_capacity_stage1(size_t capacity); - - }; - - } // namespace ppc64 -} // namespace simdjson - -namespace simdjson { - namespace ppc64 { - - inline dom_parser_implementation::dom_parser_implementation() noexcept = default; - inline dom_parser_implementation::dom_parser_implementation(dom_parser_implementation&& other) noexcept = default; - inline dom_parser_implementation& dom_parser_implementation::operator=(dom_parser_implementation&& other) noexcept = default; - - // Leaving these here so they can be inlined if so desired - inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(size_t capacity) noexcept { - if (capacity > SIMDJSON_MAXSIZE_BYTES) { return CAPACITY; } - // Stage 1 index output - size_t max_structures = SIMDJSON_ROUNDUP_N(capacity, 64) + 2 + 7; - structural_indexes.reset(new (std::nothrow) uint32_t[max_structures]); - if (!structural_indexes) { _capacity = 0; return MEMALLOC; } - structural_indexes[0] = 0; - n_structural_indexes = 0; - - _capacity = capacity; - return SUCCESS; - } - - inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { - // Stage 2 stacks - open_containers.reset(new (std::nothrow) open_container[max_depth]); - is_array.reset(new (std::nothrow) bool[max_depth]); - if (!is_array || !open_containers) { _max_depth = 0; return MEMALLOC; } - - _max_depth = max_depth; - return SUCCESS; - } - - } // namespace ppc64 -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_DOM_PARSER_IMPLEMENTATION_H -/* end file simdjson/generic/dom_parser_implementation.h for ppc64 */ -/* including simdjson/generic/implementation_simdjson_result_base.h for ppc64: #include "simdjson/generic/implementation_simdjson_result_base.h" */ -/* begin file simdjson/generic/implementation_simdjson_result_base.h for ppc64 */ -#ifndef SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - namespace ppc64 { - - // This is a near copy of include/error.h's implementation_simdjson_result_base, except it doesn't use std::pair - // so we can avoid inlining errors - // TODO reconcile these! - /** - * The result of a simdjson operation that could fail. - * - * Gives the option of reading error codes, or throwing an exception by casting to the desired result. - * - * This is a base class for implementations that want to add functions to the result type for - * chaining. - * - * Override like: - * - * struct simdjson_result : public internal::implementation_simdjson_result_base { - * simdjson_result() noexcept : internal::implementation_simdjson_result_base() {} - * simdjson_result(error_code error) noexcept : internal::implementation_simdjson_result_base(error) {} - * simdjson_result(T &&value) noexcept : internal::implementation_simdjson_result_base(std::forward(value)) {} - * simdjson_result(T &&value, error_code error) noexcept : internal::implementation_simdjson_result_base(value, error) {} - * // Your extra methods here - * } - * - * Then any method returning simdjson_result will be chainable with your methods. - */ - template - struct implementation_simdjson_result_base { - - /** - * Create a new empty result with error = UNINITIALIZED. - */ - simdjson_inline implementation_simdjson_result_base() noexcept = default; - - /** - * Create a new error result. - */ - simdjson_inline implementation_simdjson_result_base(error_code error) noexcept; - - /** - * Create a new successful result. - */ - simdjson_inline implementation_simdjson_result_base(T&& value) noexcept; - - /** - * Create a new result with both things (use if you don't want to branch when creating the result). - */ - simdjson_inline implementation_simdjson_result_base(T&& value, error_code error) noexcept; - - /** - * Move the value and the error to the provided variables. - * - * @param value The variable to assign the value to. May not be set if there is an error. - * @param error The variable to assign the error to. Set to SUCCESS if there is no error. - */ - simdjson_inline void tie(T& value, error_code& error) && noexcept; - - /** - * Move the value to the provided variable. - * - * @param value The variable to assign the value to. May not be set if there is an error. - */ - simdjson_inline error_code get(T& value) && noexcept; - - /** - * The error. - */ - simdjson_inline error_code error() const noexcept; - -#if SIMDJSON_EXCEPTIONS - - /** - * Get the result value. - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline T& value() & noexcept(false); - - /** - * Take the result value (move it). - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline T&& value() && noexcept(false); - - /** - * Take the result value (move it). - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline T&& take_value() && noexcept(false); - - /** - * Cast to the value (will throw on error). - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline operator T && () && noexcept(false); - - -#endif // SIMDJSON_EXCEPTIONS - - /** - * Get the result value. This function is safe if and only - * the error() method returns a value that evaluates to false. - */ - simdjson_inline const T& value_unsafe() const& noexcept; - /** - * Get the result value. This function is safe if and only - * the error() method returns a value that evaluates to false. - */ - simdjson_inline T& value_unsafe() & noexcept; - /** - * Take the result value (move it). This function is safe if and only - * the error() method returns a value that evaluates to false. - */ - simdjson_inline T&& value_unsafe() && noexcept; - protected: - /** users should never directly access first and second. **/ - T first{}; /** Users should never directly access 'first'. **/ - error_code second{ UNINITIALIZED }; /** Users should never directly access 'second'. **/ - }; // struct implementation_simdjson_result_base - - } // namespace ppc64 -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_H -/* end file simdjson/generic/implementation_simdjson_result_base.h for ppc64 */ -/* including simdjson/generic/numberparsing.h for ppc64: #include "simdjson/generic/numberparsing.h" */ -/* begin file simdjson/generic/numberparsing.h for ppc64 */ -#ifndef SIMDJSON_GENERIC_NUMBERPARSING_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_NUMBERPARSING_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/jsoncharutils.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include -#include -#include - -namespace simdjson { - namespace ppc64 { - namespace numberparsing { - -#ifdef JSON_TEST_NUMBERS -#define INVALID_NUMBER(SRC) (found_invalid_number((SRC)), NUMBER_ERROR) -#define WRITE_INTEGER(VALUE, SRC, WRITER) (found_integer((VALUE), (SRC)), (WRITER).append_s64((VALUE))) -#define WRITE_UNSIGNED(VALUE, SRC, WRITER) (found_unsigned_integer((VALUE), (SRC)), (WRITER).append_u64((VALUE))) -#define WRITE_DOUBLE(VALUE, SRC, WRITER) (found_float((VALUE), (SRC)), (WRITER).append_double((VALUE))) -#else -#define INVALID_NUMBER(SRC) (NUMBER_ERROR) -#define WRITE_INTEGER(VALUE, SRC, WRITER) (WRITER).append_s64((VALUE)) -#define WRITE_UNSIGNED(VALUE, SRC, WRITER) (WRITER).append_u64((VALUE)) -#define WRITE_DOUBLE(VALUE, SRC, WRITER) (WRITER).append_double((VALUE)) -#endif - - namespace { - - // Convert a mantissa, an exponent and a sign bit into an ieee64 double. - // The real_exponent needs to be in [0, 2046] (technically real_exponent = 2047 would be acceptable). - // The mantissa should be in [0,1<<53). The bit at index (1ULL << 52) while be zeroed. - simdjson_inline double to_double(uint64_t mantissa, uint64_t real_exponent, bool negative) { - double d; - mantissa &= ~(1ULL << 52); - mantissa |= real_exponent << 52; - mantissa |= ((static_cast(negative)) << 63); - std::memcpy(&d, &mantissa, sizeof(d)); - return d; - } - - // Attempts to compute i * 10^(power) exactly; and if "negative" is - // true, negate the result. - // This function will only work in some cases, when it does not work, success is - // set to false. This should work *most of the time* (like 99% of the time). - // We assume that power is in the [smallest_power, - // largest_power] interval: the caller is responsible for this check. - simdjson_inline bool compute_float_64(int64_t power, uint64_t i, bool negative, double& d) { - // we start with a fast path - // It was described in - // Clinger WD. How to read floating point numbers accurately. - // ACM SIGPLAN Notices. 1990 -#ifndef FLT_EVAL_METHOD -#error "FLT_EVAL_METHOD should be defined, please include cfloat." -#endif -#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0) - // We cannot be certain that x/y is rounded to nearest. - if (0 <= power && power <= 22 && i <= 9007199254740991) -#else - if (-22 <= power && power <= 22 && i <= 9007199254740991) -#endif - { - // convert the integer into a double. This is lossless since - // 0 <= i <= 2^53 - 1. - d = double(i); - // - // The general idea is as follows. - // If 0 <= s < 2^53 and if 10^0 <= p <= 10^22 then - // 1) Both s and p can be represented exactly as 64-bit floating-point - // values - // (binary64). - // 2) Because s and p can be represented exactly as floating-point values, - // then s * p - // and s / p will produce correctly rounded values. - // - if (power < 0) { - d = d / simdjson::internal::power_of_ten[-power]; - } - else { - d = d * simdjson::internal::power_of_ten[power]; - } - if (negative) { - d = -d; - } - return true; - } - // When 22 < power && power < 22 + 16, we could - // hope for another, secondary fast path. It was - // described by David M. Gay in "Correctly rounded - // binary-decimal and decimal-binary conversions." (1990) - // If you need to compute i * 10^(22 + x) for x < 16, - // first compute i * 10^x, if you know that result is exact - // (e.g., when i * 10^x < 2^53), - // then you can still proceed and do (i * 10^x) * 10^22. - // Is this worth your time? - // You need 22 < power *and* power < 22 + 16 *and* (i * 10^(x-22) < 2^53) - // for this second fast path to work. - // If you you have 22 < power *and* power < 22 + 16, and then you - // optimistically compute "i * 10^(x-22)", there is still a chance that you - // have wasted your time if i * 10^(x-22) >= 2^53. It makes the use cases of - // this optimization maybe less common than we would like. Source: - // http://www.exploringbinary.com/fast-path-decimal-to-floating-point-conversion/ - // also used in RapidJSON: https://rapidjson.org/strtod_8h_source.html - - // The fast path has now failed, so we are failing back on the slower path. - - // In the slow path, we need to adjust i so that it is > 1<<63 which is always - // possible, except if i == 0, so we handle i == 0 separately. - if (i == 0) { - d = negative ? -0.0 : 0.0; - return true; - } - - - // The exponent is 1024 + 63 + power - // + floor(log(5**power)/log(2)). - // The 1024 comes from the ieee64 standard. - // The 63 comes from the fact that we use a 64-bit word. - // - // Computing floor(log(5**power)/log(2)) could be - // slow. Instead we use a fast function. - // - // For power in (-400,350), we have that - // (((152170 + 65536) * power ) >> 16); - // is equal to - // floor(log(5**power)/log(2)) + power when power >= 0 - // and it is equal to - // ceil(log(5**-power)/log(2)) + power when power < 0 - // - // The 65536 is (1<<16) and corresponds to - // (65536 * power) >> 16 ---> power - // - // ((152170 * power ) >> 16) is equal to - // floor(log(5**power)/log(2)) - // - // Note that this is not magic: 152170/(1<<16) is - // approximatively equal to log(5)/log(2). - // The 1<<16 value is a power of two; we could use a - // larger power of 2 if we wanted to. - // - int64_t exponent = (((152170 + 65536) * power) >> 16) + 1024 + 63; - - - // We want the most significant bit of i to be 1. Shift if needed. - int lz = leading_zeroes(i); - i <<= lz; - - - // We are going to need to do some 64-bit arithmetic to get a precise product. - // We use a table lookup approach. - // It is safe because - // power >= smallest_power - // and power <= largest_power - // We recover the mantissa of the power, it has a leading 1. It is always - // rounded down. - // - // We want the most significant 64 bits of the product. We know - // this will be non-zero because the most significant bit of i is - // 1. - const uint32_t index = 2 * uint32_t(power - simdjson::internal::smallest_power); - // Optimization: It may be that materializing the index as a variable might confuse some compilers and prevent effective complex-addressing loads. (Done for code clarity.) - // - // The full_multiplication function computes the 128-bit product of two 64-bit words - // with a returned value of type value128 with a "low component" corresponding to the - // 64-bit least significant bits of the product and with a "high component" corresponding - // to the 64-bit most significant bits of the product. - simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index]); - // Both i and power_of_five_128[index] have their most significant bit set to 1 which - // implies that the either the most or the second most significant bit of the product - // is 1. We pack values in this manner for efficiency reasons: it maximizes the use - // we make of the product. It also makes it easy to reason about the product: there - // is 0 or 1 leading zero in the product. - - // Unless the least significant 9 bits of the high (64-bit) part of the full - // product are all 1s, then we know that the most significant 55 bits are - // exact and no further work is needed. Having 55 bits is necessary because - // we need 53 bits for the mantissa but we have to have one rounding bit and - // we can waste a bit if the most significant bit of the product is zero. - if ((firstproduct.high & 0x1FF) == 0x1FF) { - // We want to compute i * 5^q, but only care about the top 55 bits at most. - // Consider the scenario where q>=0. Then 5^q may not fit in 64-bits. Doing - // the full computation is wasteful. So we do what is called a "truncated - // multiplication". - // We take the most significant 64-bits, and we put them in - // power_of_five_128[index]. Usually, that's good enough to approximate i * 5^q - // to the desired approximation using one multiplication. Sometimes it does not suffice. - // Then we store the next most significant 64 bits in power_of_five_128[index + 1], and - // then we get a better approximation to i * 5^q. - // - // That's for when q>=0. The logic for q<0 is somewhat similar but it is somewhat - // more complicated. - // - // There is an extra layer of complexity in that we need more than 55 bits of - // accuracy in the round-to-even scenario. - // - // The full_multiplication function computes the 128-bit product of two 64-bit words - // with a returned value of type value128 with a "low component" corresponding to the - // 64-bit least significant bits of the product and with a "high component" corresponding - // to the 64-bit most significant bits of the product. - simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index + 1]); - firstproduct.low += secondproduct.high; - if (secondproduct.high > firstproduct.low) { firstproduct.high++; } - // As it has been proven by Noble Mushtak and Daniel Lemire in "Fast Number Parsing Without - // Fallback" (https://arxiv.org/abs/2212.06644), at this point we are sure that the product - // is sufficiently accurate, and more computation is not needed. - } - uint64_t lower = firstproduct.low; - uint64_t upper = firstproduct.high; - // The final mantissa should be 53 bits with a leading 1. - // We shift it so that it occupies 54 bits with a leading 1. - /////// - uint64_t upperbit = upper >> 63; - uint64_t mantissa = upper >> (upperbit + 9); - lz += int(1 ^ upperbit); - - // Here we have mantissa < (1<<54). - int64_t real_exponent = exponent - lz; - if (simdjson_unlikely(real_exponent <= 0)) { // we have a subnormal? - // Here have that real_exponent <= 0 so -real_exponent >= 0 - if (-real_exponent + 1 >= 64) { // if we have more than 64 bits below the minimum exponent, you have a zero for sure. - d = negative ? -0.0 : 0.0; - return true; - } - // next line is safe because -real_exponent + 1 < 0 - mantissa >>= -real_exponent + 1; - // Thankfully, we can't have both "round-to-even" and subnormals because - // "round-to-even" only occurs for powers close to 0. - mantissa += (mantissa & 1); // round up - mantissa >>= 1; - // There is a weird scenario where we don't have a subnormal but just. - // Suppose we start with 2.2250738585072013e-308, we end up - // with 0x3fffffffffffff x 2^-1023-53 which is technically subnormal - // whereas 0x40000000000000 x 2^-1023-53 is normal. Now, we need to round - // up 0x3fffffffffffff x 2^-1023-53 and once we do, we are no longer - // subnormal, but we can only know this after rounding. - // So we only declare a subnormal if we are smaller than the threshold. - real_exponent = (mantissa < (uint64_t(1) << 52)) ? 0 : 1; - d = to_double(mantissa, real_exponent, negative); - return true; - } - // We have to round to even. The "to even" part - // is only a problem when we are right in between two floats - // which we guard against. - // If we have lots of trailing zeros, we may fall right between two - // floating-point values. - // - // The round-to-even cases take the form of a number 2m+1 which is in (2^53,2^54] - // times a power of two. That is, it is right between a number with binary significand - // m and another number with binary significand m+1; and it must be the case - // that it cannot be represented by a float itself. - // - // We must have that w * 10 ^q == (2m+1) * 2^p for some power of two 2^p. - // Recall that 10^q = 5^q * 2^q. - // When q >= 0, we must have that (2m+1) is divible by 5^q, so 5^q <= 2^54. We have that - // 5^23 <= 2^54 and it is the last power of five to qualify, so q <= 23. - // When q<0, we have w >= (2m+1) x 5^{-q}. We must have that w<2^{64} so - // (2m+1) x 5^{-q} < 2^{64}. We have that 2m+1>2^{53}. Hence, we must have - // 2^{53} x 5^{-q} < 2^{64}. - // Hence we have 5^{-q} < 2^{11}$ or q>= -4. - // - // We require lower <= 1 and not lower == 0 because we could not prove that - // that lower == 0 is implied; but we could prove that lower <= 1 is a necessary and sufficient test. - if (simdjson_unlikely((lower <= 1) && (power >= -4) && (power <= 23) && ((mantissa & 3) == 1))) { - if ((mantissa << (upperbit + 64 - 53 - 2)) == upper) { - mantissa &= ~1; // flip it so that we do not round up - } - } - - mantissa += mantissa & 1; - mantissa >>= 1; - - // Here we have mantissa < (1<<53), unless there was an overflow - if (mantissa >= (1ULL << 53)) { - ////////// - // This will happen when parsing values such as 7.2057594037927933e+16 - //////// - mantissa = (1ULL << 52); - real_exponent++; - } - mantissa &= ~(1ULL << 52); - // we have to check that real_exponent is in range, otherwise we bail out - if (simdjson_unlikely(real_exponent > 2046)) { - // We have an infinite value!!! We could actually throw an error here if we could. - return false; - } - d = to_double(mantissa, real_exponent, negative); - return true; - } - - // We call a fallback floating-point parser that might be slow. Note - // it will accept JSON numbers, but the JSON spec. is more restrictive so - // before you call parse_float_fallback, you need to have validated the input - // string with the JSON grammar. - // It will return an error (false) if the parsed number is infinite. - // The string parsing itself always succeeds. We know that there is at least - // one digit. - static bool parse_float_fallback(const uint8_t* ptr, double* outDouble) { - *outDouble = simdjson::internal::from_chars(reinterpret_cast(ptr)); - // We do not accept infinite values. - - // Detecting finite values in a portable manner is ridiculously hard, ideally - // we would want to do: - // return !std::isfinite(*outDouble); - // but that mysteriously fails under legacy/old libc++ libraries, see - // https://github.com/simdjson/simdjson/issues/1286 - // - // Therefore, fall back to this solution (the extra parens are there - // to handle that max may be a macro on windows). - return !(*outDouble > (std::numeric_limits::max)() || *outDouble < std::numeric_limits::lowest()); - } - - static bool parse_float_fallback(const uint8_t* ptr, const uint8_t* end_ptr, double* outDouble) { - *outDouble = simdjson::internal::from_chars(reinterpret_cast(ptr), reinterpret_cast(end_ptr)); - // We do not accept infinite values. - - // Detecting finite values in a portable manner is ridiculously hard, ideally - // we would want to do: - // return !std::isfinite(*outDouble); - // but that mysteriously fails under legacy/old libc++ libraries, see - // https://github.com/simdjson/simdjson/issues/1286 - // - // Therefore, fall back to this solution (the extra parens are there - // to handle that max may be a macro on windows). - return !(*outDouble > (std::numeric_limits::max)() || *outDouble < std::numeric_limits::lowest()); - } - - // check quickly whether the next 8 chars are made of digits - // at a glance, it looks better than Mula's - // http://0x80.pl/articles/swar-digits-validate.html - simdjson_inline bool is_made_of_eight_digits_fast(const uint8_t* chars) { - uint64_t val; - // this can read up to 7 bytes beyond the buffer size, but we require - // SIMDJSON_PADDING of padding - static_assert(7 <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be bigger than 7"); - std::memcpy(&val, chars, 8); - // a branchy method might be faster: - // return (( val & 0xF0F0F0F0F0F0F0F0 ) == 0x3030303030303030) - // && (( (val + 0x0606060606060606) & 0xF0F0F0F0F0F0F0F0 ) == - // 0x3030303030303030); - return (((val & 0xF0F0F0F0F0F0F0F0) | - (((val + 0x0606060606060606) & 0xF0F0F0F0F0F0F0F0) >> 4)) == - 0x3333333333333333); - } - - template - SIMDJSON_NO_SANITIZE_UNDEFINED // We deliberately allow overflow here and check later - simdjson_inline bool parse_digit(const uint8_t c, I& i) { - const uint8_t digit = static_cast(c - '0'); - if (digit > 9) { - return false; - } - // PERF NOTE: multiplication by 10 is cheaper than arbitrary integer multiplication - i = 10 * i + digit; // might overflow, we will handle the overflow later - return true; - } - - simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t* const src, const uint8_t*& p, uint64_t& i, int64_t& exponent) { - // we continue with the fiction that we have an integer. If the - // floating point number is representable as x * 10^z for some integer - // z that fits in 53 bits, then we will be able to convert back the - // the integer into a float in a lossless manner. - const uint8_t* const first_after_period = p; - -#ifdef SIMDJSON_SWAR_NUMBER_PARSING -#if SIMDJSON_SWAR_NUMBER_PARSING - // this helps if we have lots of decimals! - // this turns out to be frequent enough. - if (is_made_of_eight_digits_fast(p)) { - i = i * 100000000 + parse_eight_digits_unrolled(p); - p += 8; - } -#endif // SIMDJSON_SWAR_NUMBER_PARSING -#endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING - // Unrolling the first digit makes a small difference on some implementations (e.g. westmere) - if (parse_digit(*p, i)) { ++p; } - while (parse_digit(*p, i)) { p++; } - exponent = first_after_period - p; - // Decimal without digits (123.) is illegal - if (exponent == 0) { - return INVALID_NUMBER(src); - } - return SUCCESS; - } - - simdjson_inline error_code parse_exponent(simdjson_unused const uint8_t* const src, const uint8_t*& p, int64_t& exponent) { - // Exp Sign: -123.456e[-]78 - bool neg_exp = ('-' == *p); - if (neg_exp || '+' == *p) { p++; } // Skip + as well - - // Exponent: -123.456e-[78] - auto start_exp = p; - int64_t exp_number = 0; - while (parse_digit(*p, exp_number)) { ++p; } - // It is possible for parse_digit to overflow. - // In particular, it could overflow to INT64_MIN, and we cannot do - INT64_MIN. - // Thus we *must* check for possible overflow before we negate exp_number. - - // Performance notes: it may seem like combining the two "simdjson_unlikely checks" below into - // a single simdjson_unlikely path would be faster. The reasoning is sound, but the compiler may - // not oblige and may, in fact, generate two distinct paths in any case. It might be - // possible to do uint64_t(p - start_exp - 1) >= 18 but it could end up trading off - // instructions for a simdjson_likely branch, an unconclusive gain. - - // If there were no digits, it's an error. - if (simdjson_unlikely(p == start_exp)) { - return INVALID_NUMBER(src); - } - // We have a valid positive exponent in exp_number at this point, except that - // it may have overflowed. - - // If there were more than 18 digits, we may have overflowed the integer. We have to do - // something!!!! - if (simdjson_unlikely(p > start_exp + 18)) { - // Skip leading zeroes: 1e000000000000000000001 is technically valid and doesn't overflow - while (*start_exp == '0') { start_exp++; } - // 19 digits could overflow int64_t and is kind of absurd anyway. We don't - // support exponents smaller than -999,999,999,999,999,999 and bigger - // than 999,999,999,999,999,999. - // We can truncate. - // Note that 999999999999999999 is assuredly too large. The maximal ieee64 value before - // infinity is ~1.8e308. The smallest subnormal is ~5e-324. So, actually, we could - // truncate at 324. - // Note that there is no reason to fail per se at this point in time. - // E.g., 0e999999999999999999999 is a fine number. - if (p > start_exp + 18) { exp_number = 999999999999999999; } - } - // At this point, we know that exp_number is a sane, positive, signed integer. - // It is <= 999,999,999,999,999,999. As long as 'exponent' is in - // [-8223372036854775808, 8223372036854775808], we won't overflow. Because 'exponent' - // is bounded in magnitude by the size of the JSON input, we are fine in this universe. - // To sum it up: the next line should never overflow. - exponent += (neg_exp ? -exp_number : exp_number); - return SUCCESS; - } - - simdjson_inline size_t significant_digits(const uint8_t* start_digits, size_t digit_count) { - // It is possible that the integer had an overflow. - // We have to handle the case where we have 0.0000somenumber. - const uint8_t* start = start_digits; - while ((*start == '0') || (*start == '.')) { ++start; } - // we over-decrement by one when there is a '.' - return digit_count - size_t(start - start_digits); - } - - } // unnamed namespace - - /** @private */ - template - error_code slow_float_parsing(simdjson_unused const uint8_t* src, W writer) { - double d; - if (parse_float_fallback(src, &d)) { - writer.append_double(d); - return SUCCESS; - } - return INVALID_NUMBER(src); - } - - /** @private */ - template - simdjson_inline error_code write_float(const uint8_t* const src, bool negative, uint64_t i, const uint8_t* start_digits, size_t digit_count, int64_t exponent, W& writer) { - // If we frequently had to deal with long strings of digits, - // we could extend our code by using a 128-bit integer instead - // of a 64-bit integer. However, this is uncommon in practice. - // - // 9999999999999999999 < 2**64 so we can accommodate 19 digits. - // If we have a decimal separator, then digit_count - 1 is the number of digits, but we - // may not have a decimal separator! - if (simdjson_unlikely(digit_count > 19 && significant_digits(start_digits, digit_count) > 19)) { - // Ok, chances are good that we had an overflow! - // this is almost never going to get called!!! - // we start anew, going slowly!!! - // This will happen in the following examples: - // 10000000000000000000000000000000000000000000e+308 - // 3.1415926535897932384626433832795028841971693993751 - // - // NOTE: This makes a *copy* of the writer and passes it to slow_float_parsing. This happens - // because slow_float_parsing is a non-inlined function. If we passed our writer reference to - // it, it would force it to be stored in memory, preventing the compiler from picking it apart - // and putting into registers. i.e. if we pass it as reference, it gets slow. - // This is what forces the skip_double, as well. - error_code error = slow_float_parsing(src, writer); - writer.skip_double(); - return error; - } - // NOTE: it's weird that the simdjson_unlikely() only wraps half the if, but it seems to get slower any other - // way we've tried: https://github.com/simdjson/simdjson/pull/990#discussion_r448497331 - // To future reader: we'd love if someone found a better way, or at least could explain this result! - if (simdjson_unlikely(exponent < simdjson::internal::smallest_power) || (exponent > simdjson::internal::largest_power)) { - // - // Important: smallest_power is such that it leads to a zero value. - // Observe that 18446744073709551615e-343 == 0, i.e. (2**64 - 1) e -343 is zero - // so something x 10^-343 goes to zero, but not so with something x 10^-342. - static_assert(simdjson::internal::smallest_power <= -342, "smallest_power is not small enough"); - // - if ((exponent < simdjson::internal::smallest_power) || (i == 0)) { - // E.g. Parse "-0.0e-999" into the same value as "-0.0". See https://en.wikipedia.org/wiki/Signed_zero - WRITE_DOUBLE(negative ? -0.0 : 0.0, src, writer); - return SUCCESS; - } - else { // (exponent > largest_power) and (i != 0) - // We have, for sure, an infinite value and simdjson refuses to parse infinite values. - return INVALID_NUMBER(src); - } - } - double d; - if (!compute_float_64(exponent, i, negative, d)) { - // we are almost never going to get here. - if (!parse_float_fallback(src, &d)) { return INVALID_NUMBER(src); } - } - WRITE_DOUBLE(d, src, writer); - return SUCCESS; - } - - // for performance analysis, it is sometimes useful to skip parsing -#ifdef SIMDJSON_SKIPNUMBERPARSING - - template - simdjson_inline error_code parse_number(const uint8_t* const, W& writer) { - writer.append_s64(0); // always write zero - return SUCCESS; // always succeeds - } - - simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t* const src) noexcept { return 0; } - simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t* const src) noexcept { return 0; } - simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t* const src) noexcept { return 0; } - simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t* const src) noexcept { return 0; } - simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t* const src) noexcept { return 0; } - simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t* const src) noexcept { return 0; } - simdjson_unused simdjson_inline bool is_negative(const uint8_t* src) noexcept { return false; } - simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t* src) noexcept { return false; } - simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t* src) noexcept { return number_type::signed_integer; } -#else - -// parse the number at src -// define JSON_TEST_NUMBERS for unit testing -// -// It is assumed that the number is followed by a structural ({,},],[) character -// or a white space character. If that is not the case (e.g., when the JSON -// document is made of a single number), then it is necessary to copy the -// content and append a space before calling this function. -// -// Our objective is accurate parsing (ULP of 0) at high speed. - template - simdjson_inline error_code parse_number(const uint8_t* const src, W& writer) { - - // - // Check for minus sign - // - bool negative = (*src == '-'); - const uint8_t* p = src + uint8_t(negative); - - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t* const start_digits = p; - uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(p - start_digits); - if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { return INVALID_NUMBER(src); } - - // - // Handle floats if there is a . or e (or both) - // - int64_t exponent = 0; - bool is_float = false; - if ('.' == *p) { - is_float = true; - ++p; - SIMDJSON_TRY(parse_decimal_after_separator(src, p, i, exponent)); - digit_count = int(p - start_digits); // used later to guard against overflows - } - if (('e' == *p) || ('E' == *p)) { - is_float = true; - ++p; - SIMDJSON_TRY(parse_exponent(src, p, exponent)); - } - if (is_float) { - const bool dirty_end = jsoncharutils::is_not_structural_or_whitespace(*p); - SIMDJSON_TRY(write_float(src, negative, i, start_digits, digit_count, exponent, writer)); - if (dirty_end) { return INVALID_NUMBER(src); } - return SUCCESS; - } - - // The longest negative 64-bit number is 19 digits. - // The longest positive 64-bit number is 20 digits. - // We do it this way so we don't trigger this branch unless we must. - size_t longest_digit_count = negative ? 19 : 20; - if (digit_count > longest_digit_count) { return INVALID_NUMBER(src); } - if (digit_count == longest_digit_count) { - if (negative) { - // Anything negative above INT64_MAX+1 is invalid - if (i > uint64_t(INT64_MAX) + 1) { return INVALID_NUMBER(src); } - WRITE_INTEGER(~i + 1, src, writer); - if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return INVALID_NUMBER(src); } - return SUCCESS; - // Positive overflow check: - // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the - // biggest uint64_t. - // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX. - // If we got here, it's a 20 digit number starting with the digit "1". - // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller - // than 1,553,255,926,290,448,384. - // - That is smaller than the smallest possible 20-digit number the user could write: - // 10,000,000,000,000,000,000. - // - Therefore, if the number is positive and lower than that, it's overflow. - // - The value we are looking at is less than or equal to INT64_MAX. - // - } - else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INVALID_NUMBER(src); } - } - - // Write unsigned if it doesn't fit in a signed integer. - if (i > uint64_t(INT64_MAX)) { - WRITE_UNSIGNED(i, src, writer); - } - else { - WRITE_INTEGER(negative ? (~i + 1) : i, src, writer); - } - if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return INVALID_NUMBER(src); } - return SUCCESS; - } - - // Inlineable functions - namespace { - - // This table can be used to characterize the final character of an integer - // string. For JSON structural character and allowable white space characters, - // we return SUCCESS. For 'e', '.' and 'E', we return INCORRECT_TYPE. Otherwise - // we return NUMBER_ERROR. - // Optimization note: we could easily reduce the size of the table by half (to 128) - // at the cost of an extra branch. - // Optimization note: we want the values to use at most 8 bits (not, e.g., 32 bits): - static_assert(error_code(uint8_t(NUMBER_ERROR)) == NUMBER_ERROR, "bad NUMBER_ERROR cast"); - static_assert(error_code(uint8_t(SUCCESS)) == SUCCESS, "bad NUMBER_ERROR cast"); - static_assert(error_code(uint8_t(INCORRECT_TYPE)) == INCORRECT_TYPE, "bad NUMBER_ERROR cast"); - - const uint8_t integer_string_finisher[256] = { - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, - SUCCESS, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, - NUMBER_ERROR, INCORRECT_TYPE, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, INCORRECT_TYPE, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, SUCCESS, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, INCORRECT_TYPE, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, - SUCCESS, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR }; - - // Parse any number from 0 to 18,446,744,073,709,551,615 - simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t* const src) noexcept { - const uint8_t* p = src; - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t* const start_digits = p; - uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(p - start_digits); - // The longest positive 64-bit number is 20 digits. - // We do it this way so we don't trigger this branch unless we must. - // Optimization note: the compiler can probably merge - // ((digit_count == 0) || (digit_count > 20)) - // into a single branch since digit_count is unsigned. - if ((digit_count == 0) || (digit_count > 20)) { return INCORRECT_TYPE; } - // Here digit_count > 0. - if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } - // We can do the following... - // if (!jsoncharutils::is_structural_or_whitespace(*p)) { - // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; - // } - // as a single table lookup: - if (integer_string_finisher[*p] != SUCCESS) { return error_code(integer_string_finisher[*p]); } - - if (digit_count == 20) { - // Positive overflow check: - // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the - // biggest uint64_t. - // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX. - // If we got here, it's a 20 digit number starting with the digit "1". - // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller - // than 1,553,255,926,290,448,384. - // - That is smaller than the smallest possible 20-digit number the user could write: - // 10,000,000,000,000,000,000. - // - Therefore, if the number is positive and lower than that, it's overflow. - // - The value we are looking at is less than or equal to INT64_MAX. - // - if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INCORRECT_TYPE; } - } - - return i; - } - - - // Parse any number from 0 to 18,446,744,073,709,551,615 - // Never read at src_end or beyond - simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t* const src, const uint8_t* const src_end) noexcept { - const uint8_t* p = src; - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t* const start_digits = p; - uint64_t i = 0; - while ((p != src_end) && parse_digit(*p, i)) { p++; } - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(p - start_digits); - // The longest positive 64-bit number is 20 digits. - // We do it this way so we don't trigger this branch unless we must. - // Optimization note: the compiler can probably merge - // ((digit_count == 0) || (digit_count > 20)) - // into a single branch since digit_count is unsigned. - if ((digit_count == 0) || (digit_count > 20)) { return INCORRECT_TYPE; } - // Here digit_count > 0. - if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } - // We can do the following... - // if (!jsoncharutils::is_structural_or_whitespace(*p)) { - // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; - // } - // as a single table lookup: - if ((p != src_end) && integer_string_finisher[*p] != SUCCESS) { return error_code(integer_string_finisher[*p]); } - - if (digit_count == 20) { - // Positive overflow check: - // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the - // biggest uint64_t. - // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX. - // If we got here, it's a 20 digit number starting with the digit "1". - // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller - // than 1,553,255,926,290,448,384. - // - That is smaller than the smallest possible 20-digit number the user could write: - // 10,000,000,000,000,000,000. - // - Therefore, if the number is positive and lower than that, it's overflow. - // - The value we are looking at is less than or equal to INT64_MAX. - // - if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INCORRECT_TYPE; } - } - - return i; - } - - // Parse any number from 0 to 18,446,744,073,709,551,615 - simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t* const src) noexcept { - const uint8_t* p = src + 1; - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t* const start_digits = p; - uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(p - start_digits); - // The longest positive 64-bit number is 20 digits. - // We do it this way so we don't trigger this branch unless we must. - // Optimization note: the compiler can probably merge - // ((digit_count == 0) || (digit_count > 20)) - // into a single branch since digit_count is unsigned. - if ((digit_count == 0) || (digit_count > 20)) { return INCORRECT_TYPE; } - // Here digit_count > 0. - if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } - // We can do the following... - // if (!jsoncharutils::is_structural_or_whitespace(*p)) { - // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; - // } - // as a single table lookup: - if (*p != '"') { return NUMBER_ERROR; } - - if (digit_count == 20) { - // Positive overflow check: - // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the - // biggest uint64_t. - // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX. - // If we got here, it's a 20 digit number starting with the digit "1". - // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller - // than 1,553,255,926,290,448,384. - // - That is smaller than the smallest possible 20-digit number the user could write: - // 10,000,000,000,000,000,000. - // - Therefore, if the number is positive and lower than that, it's overflow. - // - The value we are looking at is less than or equal to INT64_MAX. - // - // Note: we use src[1] and not src[0] because src[0] is the quote character in this - // instance. - if (src[1] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INCORRECT_TYPE; } - } - - return i; - } - - // Parse any number from -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 - simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t* src) noexcept { - // - // Check for minus sign - // - bool negative = (*src == '-'); - const uint8_t* p = src + uint8_t(negative); - - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t* const start_digits = p; - uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(p - start_digits); - // We go from - // -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 - // so we can never represent numbers that have more than 19 digits. - size_t longest_digit_count = 19; - // Optimization note: the compiler can probably merge - // ((digit_count == 0) || (digit_count > longest_digit_count)) - // into a single branch since digit_count is unsigned. - if ((digit_count == 0) || (digit_count > longest_digit_count)) { return INCORRECT_TYPE; } - // Here digit_count > 0. - if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } - // We can do the following... - // if (!jsoncharutils::is_structural_or_whitespace(*p)) { - // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; - // } - // as a single table lookup: - if (integer_string_finisher[*p] != SUCCESS) { return error_code(integer_string_finisher[*p]); } - // Negative numbers have can go down to - INT64_MAX - 1 whereas positive numbers are limited to INT64_MAX. - // Performance note: This check is only needed when digit_count == longest_digit_count but it is - // so cheap that we might as well always make it. - if (i > uint64_t(INT64_MAX) + uint64_t(negative)) { return INCORRECT_TYPE; } - return negative ? (~i + 1) : i; - } - - // Parse any number from -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 - // Never read at src_end or beyond - simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t* const src, const uint8_t* const src_end) noexcept { - // - // Check for minus sign - // - if (src == src_end) { return NUMBER_ERROR; } - bool negative = (*src == '-'); - const uint8_t* p = src + uint8_t(negative); - - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t* const start_digits = p; - uint64_t i = 0; - while ((p != src_end) && parse_digit(*p, i)) { p++; } - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(p - start_digits); - // We go from - // -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 - // so we can never represent numbers that have more than 19 digits. - size_t longest_digit_count = 19; - // Optimization note: the compiler can probably merge - // ((digit_count == 0) || (digit_count > longest_digit_count)) - // into a single branch since digit_count is unsigned. - if ((digit_count == 0) || (digit_count > longest_digit_count)) { return INCORRECT_TYPE; } - // Here digit_count > 0. - if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } - // We can do the following... - // if (!jsoncharutils::is_structural_or_whitespace(*p)) { - // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; - // } - // as a single table lookup: - if ((p != src_end) && integer_string_finisher[*p] != SUCCESS) { return error_code(integer_string_finisher[*p]); } - // Negative numbers have can go down to - INT64_MAX - 1 whereas positive numbers are limited to INT64_MAX. - // Performance note: This check is only needed when digit_count == longest_digit_count but it is - // so cheap that we might as well always make it. - if (i > uint64_t(INT64_MAX) + uint64_t(negative)) { return INCORRECT_TYPE; } - return negative ? (~i + 1) : i; - } - - // Parse any number from -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 - simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t* src) noexcept { - // - // Check for minus sign - // - bool negative = (*(src + 1) == '-'); - src += uint8_t(negative) + 1; - - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t* const start_digits = src; - uint64_t i = 0; - while (parse_digit(*src, i)) { src++; } - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(src - start_digits); - // We go from - // -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 - // so we can never represent numbers that have more than 19 digits. - size_t longest_digit_count = 19; - // Optimization note: the compiler can probably merge - // ((digit_count == 0) || (digit_count > longest_digit_count)) - // into a single branch since digit_count is unsigned. - if ((digit_count == 0) || (digit_count > longest_digit_count)) { return INCORRECT_TYPE; } - // Here digit_count > 0. - if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } - // We can do the following... - // if (!jsoncharutils::is_structural_or_whitespace(*src)) { - // return (*src == '.' || *src == 'e' || *src == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; - // } - // as a single table lookup: - if (*src != '"') { return NUMBER_ERROR; } - // Negative numbers have can go down to - INT64_MAX - 1 whereas positive numbers are limited to INT64_MAX. - // Performance note: This check is only needed when digit_count == longest_digit_count but it is - // so cheap that we might as well always make it. - if (i > uint64_t(INT64_MAX) + uint64_t(negative)) { return INCORRECT_TYPE; } - return negative ? (~i + 1) : i; - } - - simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t* src) noexcept { - // - // Check for minus sign - // - bool negative = (*src == '-'); - src += uint8_t(negative); - - // - // Parse the integer part. - // - uint64_t i = 0; - const uint8_t* p = src; - p += parse_digit(*p, i); - bool leading_zero = (i == 0); - while (parse_digit(*p, i)) { p++; } - // no integer digits, or 0123 (zero must be solo) - if (p == src) { return INCORRECT_TYPE; } - if ((leading_zero && p != src + 1)) { return NUMBER_ERROR; } - - // - // Parse the decimal part. - // - int64_t exponent = 0; - bool overflow; - if (simdjson_likely(*p == '.')) { - p++; - const uint8_t* start_decimal_digits = p; - if (!parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits - p++; - while (parse_digit(*p, i)) { p++; } - exponent = -(p - start_decimal_digits); - - // Overflow check. More than 19 digits (minus the decimal) may be overflow. - overflow = p - src - 1 > 19; - if (simdjson_unlikely(overflow && leading_zero)) { - // Skip leading 0.00000 and see if it still overflows - const uint8_t* start_digits = src + 2; - while (*start_digits == '0') { start_digits++; } - overflow = p - start_digits > 19; - } - } - else { - overflow = p - src > 19; - } - - // - // Parse the exponent - // - if (*p == 'e' || *p == 'E') { - p++; - bool exp_neg = *p == '-'; - p += exp_neg || *p == '+'; - - uint64_t exp = 0; - const uint8_t* start_exp_digits = p; - while (parse_digit(*p, exp)) { p++; } - // no exp digits, or 20+ exp digits - if (p - start_exp_digits == 0 || p - start_exp_digits > 19) { return NUMBER_ERROR; } - - exponent += exp_neg ? 0 - exp : exp; - } - - if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } - - overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; - - // - // Assemble (or slow-parse) the float - // - double d; - if (simdjson_likely(!overflow)) { - if (compute_float_64(exponent, i, negative, d)) { return d; } - } - if (!parse_float_fallback(src - uint8_t(negative), &d)) { - return NUMBER_ERROR; - } - return d; - } - - simdjson_unused simdjson_inline bool is_negative(const uint8_t* src) noexcept { - return (*src == '-'); - } - - simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t* src) noexcept { - bool negative = (*src == '-'); - src += uint8_t(negative); - const uint8_t* p = src; - while (static_cast(*p - '0') <= 9) { p++; } - if (p == src) { return NUMBER_ERROR; } - if (jsoncharutils::is_structural_or_whitespace(*p)) { return true; } - return false; - } - - simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t* src) noexcept { - bool negative = (*src == '-'); - src += uint8_t(negative); - const uint8_t* p = src; - while (static_cast(*p - '0') <= 9) { p++; } - if (p == src) { return NUMBER_ERROR; } - if (jsoncharutils::is_structural_or_whitespace(*p)) { - // We have an integer. - // If the number is negative and valid, it must be a signed integer. - if (negative) { return number_type::signed_integer; } - // We want values larger or equal to 9223372036854775808 to be unsigned - // integers, and the other values to be signed integers. - int digit_count = int(p - src); - if (digit_count >= 19) { - const uint8_t* smaller_big_integer = reinterpret_cast("9223372036854775808"); - if ((digit_count >= 20) || (memcmp(src, smaller_big_integer, 19) >= 0)) { - return number_type::unsigned_integer; - } - } - return number_type::signed_integer; - } - // Hopefully, we have 'e' or 'E' or '.'. - return number_type::floating_point_number; - } - - // Never read at src_end or beyond - simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t* src, const uint8_t* const src_end) noexcept { - if (src == src_end) { return NUMBER_ERROR; } - // - // Check for minus sign - // - bool negative = (*src == '-'); - src += uint8_t(negative); - - // - // Parse the integer part. - // - uint64_t i = 0; - const uint8_t* p = src; - if (p == src_end) { return NUMBER_ERROR; } - p += parse_digit(*p, i); - bool leading_zero = (i == 0); - while ((p != src_end) && parse_digit(*p, i)) { p++; } - // no integer digits, or 0123 (zero must be solo) - if (p == src) { return INCORRECT_TYPE; } - if ((leading_zero && p != src + 1)) { return NUMBER_ERROR; } - - // - // Parse the decimal part. - // - int64_t exponent = 0; - bool overflow; - if (simdjson_likely((p != src_end) && (*p == '.'))) { - p++; - const uint8_t* start_decimal_digits = p; - if ((p == src_end) || !parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits - p++; - while ((p != src_end) && parse_digit(*p, i)) { p++; } - exponent = -(p - start_decimal_digits); - - // Overflow check. More than 19 digits (minus the decimal) may be overflow. - overflow = p - src - 1 > 19; - if (simdjson_unlikely(overflow && leading_zero)) { - // Skip leading 0.00000 and see if it still overflows - const uint8_t* start_digits = src + 2; - while (*start_digits == '0') { start_digits++; } - overflow = start_digits - src > 19; - } - } - else { - overflow = p - src > 19; - } - - // - // Parse the exponent - // - if ((p != src_end) && (*p == 'e' || *p == 'E')) { - p++; - if (p == src_end) { return NUMBER_ERROR; } - bool exp_neg = *p == '-'; - p += exp_neg || *p == '+'; - - uint64_t exp = 0; - const uint8_t* start_exp_digits = p; - while ((p != src_end) && parse_digit(*p, exp)) { p++; } - // no exp digits, or 20+ exp digits - if (p - start_exp_digits == 0 || p - start_exp_digits > 19) { return NUMBER_ERROR; } - - exponent += exp_neg ? 0 - exp : exp; - } - - if ((p != src_end) && jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } - - overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; - - // - // Assemble (or slow-parse) the float - // - double d; - if (simdjson_likely(!overflow)) { - if (compute_float_64(exponent, i, negative, d)) { return d; } - } - if (!parse_float_fallback(src - uint8_t(negative), src_end, &d)) { - return NUMBER_ERROR; - } - return d; - } - - simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t* src) noexcept { - // - // Check for minus sign - // - bool negative = (*(src + 1) == '-'); - src += uint8_t(negative) + 1; - - // - // Parse the integer part. - // - uint64_t i = 0; - const uint8_t* p = src; - p += parse_digit(*p, i); - bool leading_zero = (i == 0); - while (parse_digit(*p, i)) { p++; } - // no integer digits, or 0123 (zero must be solo) - if (p == src) { return INCORRECT_TYPE; } - if ((leading_zero && p != src + 1)) { return NUMBER_ERROR; } - - // - // Parse the decimal part. - // - int64_t exponent = 0; - bool overflow; - if (simdjson_likely(*p == '.')) { - p++; - const uint8_t* start_decimal_digits = p; - if (!parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits - p++; - while (parse_digit(*p, i)) { p++; } - exponent = -(p - start_decimal_digits); - - // Overflow check. More than 19 digits (minus the decimal) may be overflow. - overflow = p - src - 1 > 19; - if (simdjson_unlikely(overflow && leading_zero)) { - // Skip leading 0.00000 and see if it still overflows - const uint8_t* start_digits = src + 2; - while (*start_digits == '0') { start_digits++; } - overflow = p - start_digits > 19; - } - } - else { - overflow = p - src > 19; - } - - // - // Parse the exponent - // - if (*p == 'e' || *p == 'E') { - p++; - bool exp_neg = *p == '-'; - p += exp_neg || *p == '+'; - - uint64_t exp = 0; - const uint8_t* start_exp_digits = p; - while (parse_digit(*p, exp)) { p++; } - // no exp digits, or 20+ exp digits - if (p - start_exp_digits == 0 || p - start_exp_digits > 19) { return NUMBER_ERROR; } - - exponent += exp_neg ? 0 - exp : exp; - } - - if (*p != '"') { return NUMBER_ERROR; } - - overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; - - // - // Assemble (or slow-parse) the float - // - double d; - if (simdjson_likely(!overflow)) { - if (compute_float_64(exponent, i, negative, d)) { return d; } - } - if (!parse_float_fallback(src - uint8_t(negative), &d)) { - return NUMBER_ERROR; - } - return d; - } - - } // unnamed namespace -#endif // SIMDJSON_SKIPNUMBERPARSING - - } // namespace numberparsing - - inline std::ostream& operator<<(std::ostream& out, number_type type) noexcept { - switch (type) { - case number_type::signed_integer: out << "integer in [-9223372036854775808,9223372036854775808)"; break; - case number_type::unsigned_integer: out << "unsigned integer in [9223372036854775808,18446744073709551616)"; break; - case number_type::floating_point_number: out << "floating-point number (binary64)"; break; - default: SIMDJSON_UNREACHABLE(); - } - return out; - } - - } // namespace ppc64 -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_NUMBERPARSING_H -/* end file simdjson/generic/numberparsing.h for ppc64 */ - -/* including simdjson/generic/implementation_simdjson_result_base-inl.h for ppc64: #include "simdjson/generic/implementation_simdjson_result_base-inl.h" */ -/* begin file simdjson/generic/implementation_simdjson_result_base-inl.h for ppc64 */ -#ifndef SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_INL_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_INL_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/implementation_simdjson_result_base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - namespace ppc64 { - - // - // internal::implementation_simdjson_result_base inline implementation - // - - template - simdjson_inline void implementation_simdjson_result_base::tie(T& value, error_code& error) && noexcept { - error = this->second; - if (!error) { - value = std::forward>(*this).first; - } - } - - template - simdjson_warn_unused simdjson_inline error_code implementation_simdjson_result_base::get(T& value) && noexcept { - error_code error; - std::forward>(*this).tie(value, error); - return error; - } - - template - simdjson_inline error_code implementation_simdjson_result_base::error() const noexcept { - return this->second; - } - -#if SIMDJSON_EXCEPTIONS - - template - simdjson_inline T& implementation_simdjson_result_base::value() & noexcept(false) { - if (error()) { throw simdjson_error(error()); } - return this->first; - } - - template - simdjson_inline T&& implementation_simdjson_result_base::value() && noexcept(false) { - return std::forward>(*this).take_value(); - } - - template - simdjson_inline T&& implementation_simdjson_result_base::take_value() && noexcept(false) { - if (error()) { throw simdjson_error(error()); } - return std::forward(this->first); - } - - template - simdjson_inline implementation_simdjson_result_base::operator T && () && noexcept(false) { - return std::forward>(*this).take_value(); - } - -#endif // SIMDJSON_EXCEPTIONS - - template - simdjson_inline const T& implementation_simdjson_result_base::value_unsafe() const& noexcept { - return this->first; - } - - template - simdjson_inline T& implementation_simdjson_result_base::value_unsafe() & noexcept { - return this->first; - } - - template - simdjson_inline T&& implementation_simdjson_result_base::value_unsafe() && noexcept { - return std::forward(this->first); - } - - template - simdjson_inline implementation_simdjson_result_base::implementation_simdjson_result_base(T&& value, error_code error) noexcept - : first{ std::forward(value) }, second{ error } {} - template - simdjson_inline implementation_simdjson_result_base::implementation_simdjson_result_base(error_code error) noexcept - : implementation_simdjson_result_base(T{}, error) {} - template - simdjson_inline implementation_simdjson_result_base::implementation_simdjson_result_base(T&& value) noexcept - : implementation_simdjson_result_base(std::forward(value), SUCCESS) {} - - } // namespace ppc64 -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_INL_H -/* end file simdjson/generic/implementation_simdjson_result_base-inl.h for ppc64 */ -/* end file simdjson/generic/amalgamated.h for ppc64 */ -/* including simdjson/ppc64/end.h: #include "simdjson/ppc64/end.h" */ -/* begin file simdjson/ppc64/end.h */ -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/ppc64/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#undef SIMDJSON_SKIP_BACKSLASH_SHORT_CIRCUIT -/* undefining SIMDJSON_IMPLEMENTATION from "ppc64" */ -#undef SIMDJSON_IMPLEMENTATION -/* end file simdjson/ppc64/end.h */ - -#endif // SIMDJSON_PPC64_H -/* end file simdjson/ppc64.h */ -/* including simdjson/ppc64/implementation.h: #include */ -/* begin file simdjson/ppc64/implementation.h */ -#ifndef SIMDJSON_PPC64_IMPLEMENTATION_H -#define SIMDJSON_PPC64_IMPLEMENTATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/ppc64/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/implementation.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/instruction_set.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - - /** - * Implementation for ALTIVEC (PPC64). - */ - namespace ppc64 { - - /** - * @private - */ - class implementation final : public simdjson::implementation { - public: - simdjson_inline implementation() - : simdjson::implementation("ppc64", "PPC64 ALTIVEC", - internal::instruction_set::ALTIVEC) {} - - simdjson_warn_unused error_code create_dom_parser_implementation( - size_t capacity, size_t max_length, - std::unique_ptr& dst) - const noexcept final; - simdjson_warn_unused error_code minify(const uint8_t* buf, size_t len, - uint8_t* dst, - size_t& dst_len) const noexcept final; - simdjson_warn_unused bool validate_utf8(const char* buf, - size_t len) const noexcept final; - }; - - } // namespace ppc64 -} // namespace simdjson - -#endif // SIMDJSON_PPC64_IMPLEMENTATION_H -/* end file simdjson/ppc64/implementation.h */ - -/* including simdjson/ppc64/begin.h: #include */ -/* begin file simdjson/ppc64/begin.h */ -/* defining SIMDJSON_IMPLEMENTATION to "ppc64" */ -#define SIMDJSON_IMPLEMENTATION ppc64 -/* including simdjson/ppc64/base.h: #include "simdjson/ppc64/base.h" */ -/* begin file simdjson/ppc64/base.h */ -#ifndef SIMDJSON_PPC64_BASE_H -#define SIMDJSON_PPC64_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - /** - * Implementation for ALTIVEC (PPC64). - */ - namespace ppc64 { - - class implementation; - - namespace { - namespace simd { - template struct simd8; - template struct simd8x64; - } // namespace simd - } // unnamed namespace - - } // namespace ppc64 -} // namespace simdjson - -#endif // SIMDJSON_PPC64_BASE_H -/* end file simdjson/ppc64/base.h */ -/* including simdjson/ppc64/intrinsics.h: #include "simdjson/ppc64/intrinsics.h" */ -/* begin file simdjson/ppc64/intrinsics.h */ -#ifndef SIMDJSON_PPC64_INTRINSICS_H -#define SIMDJSON_PPC64_INTRINSICS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/ppc64/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -// This should be the correct header whether -// you use visual studio or other compilers. -#include - -// These are defined by altivec.h in GCC toolchain, it is safe to undef them. -#ifdef bool -#undef bool -#endif - -#ifdef vector -#undef vector -#endif - -static_assert(sizeof(__vector unsigned char) <= simdjson::SIMDJSON_PADDING, "insufficient padding for ppc64"); - -#endif // SIMDJSON_PPC64_INTRINSICS_H -/* end file simdjson/ppc64/intrinsics.h */ -/* including simdjson/ppc64/bitmanipulation.h: #include "simdjson/ppc64/bitmanipulation.h" */ -/* begin file simdjson/ppc64/bitmanipulation.h */ -#ifndef SIMDJSON_PPC64_BITMANIPULATION_H -#define SIMDJSON_PPC64_BITMANIPULATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/ppc64/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - namespace ppc64 { - namespace { - - // We sometimes call trailing_zero on inputs that are zero, - // but the algorithms do not end up using the returned value. - // Sadly, sanitizers are not smart enough to figure it out. - SIMDJSON_NO_SANITIZE_UNDEFINED - // This function can be used safely even if not all bytes have been - // initialized. - // See issue https://github.com/simdjson/simdjson/issues/1965 - SIMDJSON_NO_SANITIZE_MEMORY - simdjson_inline int trailing_zeroes(uint64_t input_num) { -#if SIMDJSON_REGULAR_VISUAL_STUDIO - unsigned long ret; - // Search the mask data from least significant bit (LSB) - // to the most significant bit (MSB) for a set bit (1). - _BitScanForward64(&ret, input_num); - return (int)ret; -#else // SIMDJSON_REGULAR_VISUAL_STUDIO - return __builtin_ctzll(input_num); -#endif // SIMDJSON_REGULAR_VISUAL_STUDIO - } - - /* result might be undefined when input_num is zero */ - simdjson_inline uint64_t clear_lowest_bit(uint64_t input_num) { - return input_num & (input_num - 1); - } - - /* result might be undefined when input_num is zero */ - simdjson_inline int leading_zeroes(uint64_t input_num) { -#if SIMDJSON_REGULAR_VISUAL_STUDIO - unsigned long leading_zero = 0; - // Search the mask data from most significant bit (MSB) - // to least significant bit (LSB) for a set bit (1). - if (_BitScanReverse64(&leading_zero, input_num)) - return (int)(63 - leading_zero); - else - return 64; -#else - return __builtin_clzll(input_num); -#endif // SIMDJSON_REGULAR_VISUAL_STUDIO - } - -#if SIMDJSON_REGULAR_VISUAL_STUDIO - simdjson_inline int count_ones(uint64_t input_num) { - // note: we do not support legacy 32-bit Windows in this kernel - return __popcnt64(input_num); // Visual Studio wants two underscores - } -#else - simdjson_inline int count_ones(uint64_t input_num) { - return __builtin_popcountll(input_num); - } -#endif - - simdjson_inline bool add_overflow(uint64_t value1, uint64_t value2, - uint64_t* result) { -#if SIMDJSON_REGULAR_VISUAL_STUDIO - * result = value1 + value2; - return *result < value1; -#else - return __builtin_uaddll_overflow(value1, value2, - reinterpret_cast(result)); -#endif - } - - } // unnamed namespace - } // namespace ppc64 -} // namespace simdjson - -#endif // SIMDJSON_PPC64_BITMANIPULATION_H -/* end file simdjson/ppc64/bitmanipulation.h */ -/* including simdjson/ppc64/bitmask.h: #include "simdjson/ppc64/bitmask.h" */ -/* begin file simdjson/ppc64/bitmask.h */ -#ifndef SIMDJSON_PPC64_BITMASK_H -#define SIMDJSON_PPC64_BITMASK_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/ppc64/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - namespace ppc64 { - namespace { - - // - // Perform a "cumulative bitwise xor," flipping bits each time a 1 is - // encountered. - // - // For example, prefix_xor(00100100) == 00011100 - // - simdjson_inline uint64_t prefix_xor(uint64_t bitmask) { - // You can use the version below, however gcc sometimes miscompiles - // vec_pmsum_be, it happens somewhere around between 8 and 9th version. - // The performance boost was not noticeable, falling back to a usual - // implementation. - // __vector unsigned long long all_ones = {~0ull, ~0ull}; - // __vector unsigned long long mask = {bitmask, 0}; - // // Clang and GCC return different values for pmsum for ull so cast it to one. - // // Generally it is not specified by ALTIVEC ISA what is returned by - // // vec_pmsum_be. - // #if defined(__LITTLE_ENDIAN__) - // return (uint64_t)(((__vector unsigned long long)vec_pmsum_be(all_ones, mask))[0]); - // #else - // return (uint64_t)(((__vector unsigned long long)vec_pmsum_be(all_ones, mask))[1]); - // #endif - bitmask ^= bitmask << 1; - bitmask ^= bitmask << 2; - bitmask ^= bitmask << 4; - bitmask ^= bitmask << 8; - bitmask ^= bitmask << 16; - bitmask ^= bitmask << 32; - return bitmask; - } - - } // unnamed namespace - } // namespace ppc64 -} // namespace simdjson - -#endif -/* end file simdjson/ppc64/bitmask.h */ -/* including simdjson/ppc64/numberparsing_defs.h: #include "simdjson/ppc64/numberparsing_defs.h" */ -/* begin file simdjson/ppc64/numberparsing_defs.h */ -#ifndef SIMDJSON_PPC64_NUMBERPARSING_DEFS_H -#define SIMDJSON_PPC64_NUMBERPARSING_DEFS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/ppc64/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/ppc64/intrinsics.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include - -#if defined(__linux__) -#include -#elif defined(__FreeBSD__) -#include -#endif - -namespace simdjson { - namespace ppc64 { - namespace numberparsing { - - // we don't have appropriate instructions, so let us use a scalar function - // credit: https://johnnylee-sde.github.io/Fast-numeric-string-to-int/ - /** @private */ - static simdjson_inline uint32_t parse_eight_digits_unrolled(const uint8_t* chars) { - uint64_t val; - std::memcpy(&val, chars, sizeof(uint64_t)); -#ifdef __BIG_ENDIAN__ -#if defined(__linux__) - val = bswap_64(val); -#elif defined(__FreeBSD__) - val = bswap64(val); -#endif -#endif - val = (val & 0x0F0F0F0F0F0F0F0F) * 2561 >> 8; - val = (val & 0x00FF00FF00FF00FF) * 6553601 >> 16; - return uint32_t((val & 0x0000FFFF0000FFFF) * 42949672960001 >> 32); - } - - /** @private */ - simdjson_inline internal::value128 full_multiplication(uint64_t value1, uint64_t value2) { - internal::value128 answer; -#if SIMDJSON_REGULAR_VISUAL_STUDIO || SIMDJSON_IS_32BITS -#ifdef _M_ARM64 - // ARM64 has native support for 64-bit multiplications, no need to emultate - answer.high = __umulh(value1, value2); - answer.low = value1 * value2; -#else - answer.low = _umul128(value1, value2, &answer.high); // _umul128 not available on ARM64 -#endif // _M_ARM64 -#else // SIMDJSON_REGULAR_VISUAL_STUDIO || SIMDJSON_IS_32BITS - __uint128_t r = (static_cast<__uint128_t>(value1)) * value2; - answer.low = uint64_t(r); - answer.high = uint64_t(r >> 64); -#endif - return answer; - } - - } // namespace numberparsing - } // namespace ppc64 -} // namespace simdjson - -#define SIMDJSON_SWAR_NUMBER_PARSING 1 - -#endif // SIMDJSON_PPC64_NUMBERPARSING_DEFS_H -/* end file simdjson/ppc64/numberparsing_defs.h */ -/* including simdjson/ppc64/simd.h: #include "simdjson/ppc64/simd.h" */ -/* begin file simdjson/ppc64/simd.h */ -#ifndef SIMDJSON_PPC64_SIMD_H -#define SIMDJSON_PPC64_SIMD_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/ppc64/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/ppc64/bitmanipulation.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/simdprune_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include - -namespace simdjson { - namespace ppc64 { - namespace { - namespace simd { - - using __m128i = __vector unsigned char; - - template struct base { - __m128i value; - - // Zero constructor - simdjson_inline base() : value{ __m128i() } {} - - // Conversion from SIMD register - simdjson_inline base(const __m128i _value) : value(_value) {} - - // Conversion to SIMD register - simdjson_inline operator const __m128i& () const { - return this->value; - } - simdjson_inline operator __m128i& () { return this->value; } - - // Bit operations - simdjson_inline Child operator|(const Child other) const { - return vec_or(this->value, (__m128i)other); - } - simdjson_inline Child operator&(const Child other) const { - return vec_and(this->value, (__m128i)other); - } - simdjson_inline Child operator^(const Child other) const { - return vec_xor(this->value, (__m128i)other); - } - simdjson_inline Child bit_andnot(const Child other) const { - return vec_andc(this->value, (__m128i)other); - } - simdjson_inline Child& operator|=(const Child other) { - auto this_cast = static_cast(this); - *this_cast = *this_cast | other; - return *this_cast; - } - simdjson_inline Child& operator&=(const Child other) { - auto this_cast = static_cast(this); - *this_cast = *this_cast & other; - return *this_cast; - } - simdjson_inline Child& operator^=(const Child other) { - auto this_cast = static_cast(this); - *this_cast = *this_cast ^ other; - return *this_cast; - } - }; - - template > - struct base8 : base> { - typedef uint16_t bitmask_t; - typedef uint32_t bitmask2_t; - - simdjson_inline base8() : base>() {} - simdjson_inline base8(const __m128i _value) : base>(_value) {} - - friend simdjson_inline Mask operator==(const simd8 lhs, const simd8 rhs) { - return (__m128i)vec_cmpeq(lhs.value, (__m128i)rhs); - } - - static const int SIZE = sizeof(base>::value); - - template - simdjson_inline simd8 prev(simd8 prev_chunk) const { - __m128i chunk = this->value; -#ifdef __LITTLE_ENDIAN__ - chunk = (__m128i)vec_reve(this->value); - prev_chunk = (__m128i)vec_reve((__m128i)prev_chunk); -#endif - chunk = (__m128i)vec_sld((__m128i)prev_chunk, (__m128i)chunk, 16 - N); -#ifdef __LITTLE_ENDIAN__ - chunk = (__m128i)vec_reve((__m128i)chunk); -#endif - return chunk; - } - }; - - // SIMD byte mask type (returned by things like eq and gt) - template <> struct simd8 : base8 { - static simdjson_inline simd8 splat(bool _value) { - return (__m128i)vec_splats((unsigned char)(-(!!_value))); - } - - simdjson_inline simd8() : base8() {} - simdjson_inline simd8(const __m128i _value) - : base8(_value) {} - // Splat constructor - simdjson_inline simd8(bool _value) - : base8(splat(_value)) {} - - simdjson_inline int to_bitmask() const { - __vector unsigned long long result; - const __m128i perm_mask = { 0x78, 0x70, 0x68, 0x60, 0x58, 0x50, 0x48, 0x40, - 0x38, 0x30, 0x28, 0x20, 0x18, 0x10, 0x08, 0x00 }; - - result = ((__vector unsigned long long)vec_vbpermq((__m128i)this->value, - (__m128i)perm_mask)); -#ifdef __LITTLE_ENDIAN__ - return static_cast(result[1]); -#else - return static_cast(result[0]); -#endif - } - simdjson_inline bool any() const { - return !vec_all_eq(this->value, (__m128i)vec_splats(0)); - } - simdjson_inline simd8 operator~() const { - return this->value ^ (__m128i)splat(true); - } - }; - - template struct base8_numeric : base8 { - static simdjson_inline simd8 splat(T value) { - (void)value; - return (__m128i)vec_splats(value); - } - static simdjson_inline simd8 zero() { return splat(0); } - static simdjson_inline simd8 load(const T values[16]) { - return (__m128i)(vec_vsx_ld(0, reinterpret_cast(values))); - } - // Repeat 16 values as many times as necessary (usually for lookup tables) - static simdjson_inline simd8 repeat_16(T v0, T v1, T v2, T v3, T v4, - T v5, T v6, T v7, T v8, T v9, - T v10, T v11, T v12, T v13, - T v14, T v15) { - return simd8(v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, - v14, v15); - } - - simdjson_inline base8_numeric() : base8() {} - simdjson_inline base8_numeric(const __m128i _value) - : base8(_value) {} - - // Store to array - simdjson_inline void store(T dst[16]) const { - vec_vsx_st(this->value, 0, reinterpret_cast<__m128i*>(dst)); - } - - // Override to distinguish from bool version - simdjson_inline simd8 operator~() const { return *this ^ 0xFFu; } - - // Addition/subtraction are the same for signed and unsigned - simdjson_inline simd8 operator+(const simd8 other) const { - return (__m128i)((__m128i)this->value + (__m128i)other); - } - simdjson_inline simd8 operator-(const simd8 other) const { - return (__m128i)((__m128i)this->value - (__m128i)other); - } - simdjson_inline simd8& operator+=(const simd8 other) { - *this = *this + other; - return *static_cast *>(this); - } - simdjson_inline simd8& operator-=(const simd8 other) { - *this = *this - other; - return *static_cast *>(this); - } - - // Perform a lookup assuming the value is between 0 and 16 (undefined behavior - // for out of range values) - template - simdjson_inline simd8 lookup_16(simd8 lookup_table) const { - return (__m128i)vec_perm((__m128i)lookup_table, (__m128i)lookup_table, this->value); - } - - // Copies to 'output" all bytes corresponding to a 0 in the mask (interpreted - // as a bitset). Passing a 0 value for mask would be equivalent to writing out - // every byte to output. Only the first 16 - count_ones(mask) bytes of the - // result are significant but 16 bytes get written. Design consideration: it - // seems like a function with the signature simd8 compress(uint32_t mask) - // would be sensible, but the AVX ISA makes this kind of approach difficult. - template - simdjson_inline void compress(uint16_t mask, L* output) const { - using internal::BitsSetTable256mul2; - using internal::pshufb_combine_table; - using internal::thintable_epi8; - // this particular implementation was inspired by work done by @animetosho - // we do it in two steps, first 8 bytes and then second 8 bytes - uint8_t mask1 = uint8_t(mask); // least significant 8 bits - uint8_t mask2 = uint8_t(mask >> 8); // most significant 8 bits - // next line just loads the 64-bit values thintable_epi8[mask1] and - // thintable_epi8[mask2] into a 128-bit register, using only - // two instructions on most compilers. -#ifdef __LITTLE_ENDIAN__ - __m128i shufmask = (__m128i)(__vector unsigned long long) { - thintable_epi8[mask1], thintable_epi8[mask2] - }; -#else - __m128i shufmask = (__m128i)(__vector unsigned long long) { - thintable_epi8[mask2], thintable_epi8[mask1] - }; - shufmask = (__m128i)vec_reve((__m128i)shufmask); -#endif - // we increment by 0x08 the second half of the mask - shufmask = ((__m128i)shufmask) + - ((__m128i)(__vector int) { 0, 0, 0x08080808, 0x08080808 }); - - // this is the version "nearly pruned" - __m128i pruned = vec_perm(this->value, this->value, shufmask); - // we still need to put the two halves together. - // we compute the popcount of the first half: - int pop1 = BitsSetTable256mul2[mask1]; - // then load the corresponding mask, what it does is to write - // only the first pop1 bytes from the first 8 bytes, and then - // it fills in with the bytes from the second 8 bytes + some filling - // at the end. - __m128i compactmask = - vec_vsx_ld(0, reinterpret_cast(pshufb_combine_table + pop1 * 8)); - __m128i answer = vec_perm(pruned, (__m128i)vec_splats(0), compactmask); - vec_vsx_st(answer, 0, reinterpret_cast<__m128i*>(output)); - } - - template - simdjson_inline simd8 - lookup_16(L replace0, L replace1, L replace2, L replace3, L replace4, - L replace5, L replace6, L replace7, L replace8, L replace9, - L replace10, L replace11, L replace12, L replace13, L replace14, - L replace15) const { - return lookup_16(simd8::repeat_16( - replace0, replace1, replace2, replace3, replace4, replace5, replace6, - replace7, replace8, replace9, replace10, replace11, replace12, - replace13, replace14, replace15)); - } - }; - - // Signed bytes - template <> struct simd8 : base8_numeric { - simdjson_inline simd8() : base8_numeric() {} - simdjson_inline simd8(const __m128i _value) - : base8_numeric(_value) {} - // Splat constructor - simdjson_inline simd8(int8_t _value) : simd8(splat(_value)) {} - // Array constructor - simdjson_inline simd8(const int8_t* values) : simd8(load(values)) {} - // Member-by-member initialization - simdjson_inline simd8(int8_t v0, int8_t v1, int8_t v2, int8_t v3, - int8_t v4, int8_t v5, int8_t v6, int8_t v7, - int8_t v8, int8_t v9, int8_t v10, int8_t v11, - int8_t v12, int8_t v13, int8_t v14, int8_t v15) - : simd8((__m128i)(__vector signed char) { - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10, v11, v12, v13, v14, - v15 - }) {} - // Repeat 16 values as many times as necessary (usually for lookup tables) - simdjson_inline static simd8 - repeat_16(int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, - int8_t v6, int8_t v7, int8_t v8, int8_t v9, int8_t v10, int8_t v11, - int8_t v12, int8_t v13, int8_t v14, int8_t v15) { - return simd8(v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, - v13, v14, v15); - } - - // Order-sensitive comparisons - simdjson_inline simd8 - max_val(const simd8 other) const { - return (__m128i)vec_max((__vector signed char)this->value, - (__vector signed char)(__m128i)other); - } - simdjson_inline simd8 - min_val(const simd8 other) const { - return (__m128i)vec_min((__vector signed char)this->value, - (__vector signed char)(__m128i)other); - } - simdjson_inline simd8 - operator>(const simd8 other) const { - return (__m128i)vec_cmpgt((__vector signed char)this->value, - (__vector signed char)(__m128i)other); - } - simdjson_inline simd8 - operator<(const simd8 other) const { - return (__m128i)vec_cmplt((__vector signed char)this->value, - (__vector signed char)(__m128i)other); - } - }; - - // Unsigned bytes - template <> struct simd8 : base8_numeric { - simdjson_inline simd8() : base8_numeric() {} - simdjson_inline simd8(const __m128i _value) - : base8_numeric(_value) {} - // Splat constructor - simdjson_inline simd8(uint8_t _value) : simd8(splat(_value)) {} - // Array constructor - simdjson_inline simd8(const uint8_t* values) : simd8(load(values)) {} - // Member-by-member initialization - simdjson_inline - simd8(uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, - uint8_t v6, uint8_t v7, uint8_t v8, uint8_t v9, uint8_t v10, - uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15) - : simd8((__m128i) { - v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, - v13, v14, v15 - }) {} - // Repeat 16 values as many times as necessary (usually for lookup tables) - simdjson_inline static simd8 - repeat_16(uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, - uint8_t v5, uint8_t v6, uint8_t v7, uint8_t v8, uint8_t v9, - uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, - uint8_t v15) { - return simd8(v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, - v13, v14, v15); - } - - // Saturated math - simdjson_inline simd8 - saturating_add(const simd8 other) const { - return (__m128i)vec_adds(this->value, (__m128i)other); - } - simdjson_inline simd8 - saturating_sub(const simd8 other) const { - return (__m128i)vec_subs(this->value, (__m128i)other); - } - - // Order-specific operations - simdjson_inline simd8 - max_val(const simd8 other) const { - return (__m128i)vec_max(this->value, (__m128i)other); - } - simdjson_inline simd8 - min_val(const simd8 other) const { - return (__m128i)vec_min(this->value, (__m128i)other); - } - // Same as >, but only guarantees true is nonzero (< guarantees true = -1) - simdjson_inline simd8 - gt_bits(const simd8 other) const { - return this->saturating_sub(other); - } - // Same as <, but only guarantees true is nonzero (< guarantees true = -1) - simdjson_inline simd8 - lt_bits(const simd8 other) const { - return other.saturating_sub(*this); - } - simdjson_inline simd8 - operator<=(const simd8 other) const { - return other.max_val(*this) == other; - } - simdjson_inline simd8 - operator>=(const simd8 other) const { - return other.min_val(*this) == other; - } - simdjson_inline simd8 - operator>(const simd8 other) const { - return this->gt_bits(other).any_bits_set(); - } - simdjson_inline simd8 - operator<(const simd8 other) const { - return this->gt_bits(other).any_bits_set(); - } - - // Bit-specific operations - simdjson_inline simd8 bits_not_set() const { - return (__m128i)vec_cmpeq(this->value, (__m128i)vec_splats(uint8_t(0))); - } - simdjson_inline simd8 bits_not_set(simd8 bits) const { - return (*this & bits).bits_not_set(); - } - simdjson_inline simd8 any_bits_set() const { - return ~this->bits_not_set(); - } - simdjson_inline simd8 any_bits_set(simd8 bits) const { - return ~this->bits_not_set(bits); - } - simdjson_inline bool bits_not_set_anywhere() const { - return vec_all_eq(this->value, (__m128i)vec_splats(0)); - } - simdjson_inline bool any_bits_set_anywhere() const { - return !bits_not_set_anywhere(); - } - simdjson_inline bool bits_not_set_anywhere(simd8 bits) const { - return vec_all_eq(vec_and(this->value, (__m128i)bits), - (__m128i)vec_splats(0)); - } - simdjson_inline bool any_bits_set_anywhere(simd8 bits) const { - return !bits_not_set_anywhere(bits); - } - template simdjson_inline simd8 shr() const { - return simd8( - (__m128i)vec_sr(this->value, (__m128i)vec_splat_u8(N))); - } - template simdjson_inline simd8 shl() const { - return simd8( - (__m128i)vec_sl(this->value, (__m128i)vec_splat_u8(N))); - } - }; - - template struct simd8x64 { - static constexpr int NUM_CHUNKS = 64 / sizeof(simd8); - static_assert(NUM_CHUNKS == 4, - "PPC64 kernel should use four registers per 64-byte block."); - const simd8 chunks[NUM_CHUNKS]; - - simd8x64(const simd8x64& o) = delete; // no copy allowed - simd8x64& - operator=(const simd8& other) = delete; // no assignment allowed - simd8x64() = delete; // no default constructor allowed - - simdjson_inline simd8x64(const simd8 chunk0, const simd8 chunk1, - const simd8 chunk2, const simd8 chunk3) - : chunks{ chunk0, chunk1, chunk2, chunk3 } {} - simdjson_inline simd8x64(const T ptr[64]) - : chunks{ simd8::load(ptr), simd8::load(ptr + 16), - simd8::load(ptr + 32), simd8::load(ptr + 48) } {} - - simdjson_inline void store(T ptr[64]) const { - this->chunks[0].store(ptr + sizeof(simd8) * 0); - this->chunks[1].store(ptr + sizeof(simd8) * 1); - this->chunks[2].store(ptr + sizeof(simd8) * 2); - this->chunks[3].store(ptr + sizeof(simd8) * 3); - } - - simdjson_inline simd8 reduce_or() const { - return (this->chunks[0] | this->chunks[1]) | - (this->chunks[2] | this->chunks[3]); - } - - simdjson_inline uint64_t compress(uint64_t mask, T* output) const { - this->chunks[0].compress(uint16_t(mask), output); - this->chunks[1].compress(uint16_t(mask >> 16), - output + 16 - count_ones(mask & 0xFFFF)); - this->chunks[2].compress(uint16_t(mask >> 32), - output + 32 - count_ones(mask & 0xFFFFFFFF)); - this->chunks[3].compress(uint16_t(mask >> 48), - output + 48 - count_ones(mask & 0xFFFFFFFFFFFF)); - return 64 - count_ones(mask); - } - - simdjson_inline uint64_t to_bitmask() const { - uint64_t r0 = uint32_t(this->chunks[0].to_bitmask()); - uint64_t r1 = this->chunks[1].to_bitmask(); - uint64_t r2 = this->chunks[2].to_bitmask(); - uint64_t r3 = this->chunks[3].to_bitmask(); - return r0 | (r1 << 16) | (r2 << 32) | (r3 << 48); - } - - simdjson_inline uint64_t eq(const T m) const { - const simd8 mask = simd8::splat(m); - return simd8x64(this->chunks[0] == mask, this->chunks[1] == mask, - this->chunks[2] == mask, this->chunks[3] == mask) - .to_bitmask(); - } - - simdjson_inline uint64_t eq(const simd8x64& other) const { - return simd8x64(this->chunks[0] == other.chunks[0], - this->chunks[1] == other.chunks[1], - this->chunks[2] == other.chunks[2], - this->chunks[3] == other.chunks[3]) - .to_bitmask(); - } - - simdjson_inline uint64_t lteq(const T m) const { - const simd8 mask = simd8::splat(m); - return simd8x64(this->chunks[0] <= mask, this->chunks[1] <= mask, - this->chunks[2] <= mask, this->chunks[3] <= mask) - .to_bitmask(); - } - }; // struct simd8x64 - - } // namespace simd - } // unnamed namespace - } // namespace ppc64 -} // namespace simdjson - -#endif // SIMDJSON_PPC64_SIMD_INPUT_H -/* end file simdjson/ppc64/simd.h */ -/* including simdjson/ppc64/stringparsing_defs.h: #include "simdjson/ppc64/stringparsing_defs.h" */ -/* begin file simdjson/ppc64/stringparsing_defs.h */ -#ifndef SIMDJSON_PPC64_STRINGPARSING_DEFS_H -#define SIMDJSON_PPC64_STRINGPARSING_DEFS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/ppc64/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/ppc64/bitmanipulation.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/ppc64/simd.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - namespace ppc64 { - namespace { - - using namespace simd; - - // Holds backslashes and quotes locations. - struct backslash_and_quote { - public: - static constexpr uint32_t BYTES_PROCESSED = 32; - simdjson_inline static backslash_and_quote - copy_and_find(const uint8_t* src, uint8_t* dst); - - simdjson_inline bool has_quote_first() { - return ((bs_bits - 1) & quote_bits) != 0; - } - simdjson_inline bool has_backslash() { return bs_bits != 0; } - simdjson_inline int quote_index() { - return trailing_zeroes(quote_bits); - } - simdjson_inline int backslash_index() { - return trailing_zeroes(bs_bits); - } - - uint32_t bs_bits; - uint32_t quote_bits; - }; // struct backslash_and_quote - - simdjson_inline backslash_and_quote - backslash_and_quote::copy_and_find(const uint8_t* src, uint8_t* dst) { - // this can read up to 31 bytes beyond the buffer size, but we require - // SIMDJSON_PADDING of padding - static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), - "backslash and quote finder must process fewer than " - "SIMDJSON_PADDING bytes"); - simd8 v0(src); - simd8 v1(src + sizeof(v0)); - v0.store(dst); - v1.store(dst + sizeof(v0)); - - // Getting a 64-bit bitmask is much cheaper than multiple 16-bit bitmasks on - // PPC; therefore, we smash them together into a 64-byte mask and get the - // bitmask from there. - uint64_t bs_and_quote = - simd8x64(v0 == '\\', v1 == '\\', v0 == '"', v1 == '"').to_bitmask(); - return { - uint32_t(bs_and_quote), // bs_bits - uint32_t(bs_and_quote >> 32) // quote_bits - }; - } - - } // unnamed namespace - } // namespace ppc64 -} // namespace simdjson - -#endif // SIMDJSON_PPC64_STRINGPARSING_DEFS_H -/* end file simdjson/ppc64/stringparsing_defs.h */ - -#define SIMDJSON_SKIP_BACKSLASH_SHORT_CIRCUIT 1 -/* end file simdjson/ppc64/begin.h */ -/* including generic/amalgamated.h for ppc64: #include */ -/* begin file generic/amalgamated.h for ppc64 */ -#if defined(SIMDJSON_CONDITIONAL_INCLUDE) && !defined(SIMDJSON_SRC_GENERIC_DEPENDENCIES_H) -#error generic/dependencies.h must be included before generic/amalgamated.h! -#endif - -/* including generic/base.h for ppc64: #include */ -/* begin file generic/base.h for ppc64 */ -#ifndef SIMDJSON_SRC_GENERIC_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_BASE_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - namespace ppc64 { - namespace { - - struct json_character_block; - - } // unnamed namespace - } // namespace ppc64 -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_BASE_H -/* end file generic/base.h for ppc64 */ -/* including generic/dom_parser_implementation.h for ppc64: #include */ -/* begin file generic/dom_parser_implementation.h for ppc64 */ -#ifndef SIMDJSON_SRC_GENERIC_DOM_PARSER_IMPLEMENTATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_DOM_PARSER_IMPLEMENTATION_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -// Interface a dom parser implementation must fulfill -namespace simdjson { - namespace ppc64 { - namespace { - - simdjson_inline simd8 must_be_2_3_continuation(const simd8 prev2, const simd8 prev3); - simdjson_inline bool is_ascii(const simd8x64& input); - - } // unnamed namespace - } // namespace ppc64 -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_DOM_PARSER_IMPLEMENTATION_H -/* end file generic/dom_parser_implementation.h for ppc64 */ -/* including generic/json_character_block.h for ppc64: #include */ -/* begin file generic/json_character_block.h for ppc64 */ -#ifndef SIMDJSON_SRC_GENERIC_JSON_CHARACTER_BLOCK_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_JSON_CHARACTER_BLOCK_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - namespace ppc64 { - namespace { - - struct json_character_block { - static simdjson_inline json_character_block classify(const simd::simd8x64& in); - - simdjson_inline uint64_t whitespace() const noexcept { return _whitespace; } - simdjson_inline uint64_t op() const noexcept { return _op; } - simdjson_inline uint64_t scalar() const noexcept { return ~(op() | whitespace()); } - - uint64_t _whitespace; - uint64_t _op; - }; - - } // unnamed namespace - } // namespace ppc64 -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_JSON_CHARACTER_BLOCK_H -/* end file generic/json_character_block.h for ppc64 */ -/* end file generic/amalgamated.h for ppc64 */ -/* including generic/stage1/amalgamated.h for ppc64: #include */ -/* begin file generic/stage1/amalgamated.h for ppc64 */ -// Stuff other things depend on -/* including generic/stage1/base.h for ppc64: #include */ -/* begin file generic/stage1/base.h for ppc64 */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_BASE_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - namespace ppc64 { - namespace { - namespace stage1 { - - class bit_indexer; - template - struct buf_block_reader; - struct json_block; - class json_minifier; - class json_scanner; - struct json_string_block; - class json_string_scanner; - class json_structural_indexer; - - } // namespace stage1 - - namespace utf8_validation { - struct utf8_checker; - } // namespace utf8_validation - - using utf8_validation::utf8_checker; - - } // unnamed namespace - } // namespace ppc64 -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_BASE_H -/* end file generic/stage1/base.h for ppc64 */ -/* including generic/stage1/buf_block_reader.h for ppc64: #include */ -/* begin file generic/stage1/buf_block_reader.h for ppc64 */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_BUF_BLOCK_READER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_BUF_BLOCK_READER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include - -namespace simdjson { - namespace ppc64 { - namespace { - namespace stage1 { - - // Walks through a buffer in block-sized increments, loading the last part with spaces - template - struct buf_block_reader { - public: - simdjson_inline buf_block_reader(const uint8_t* _buf, size_t _len); - simdjson_inline size_t block_index(); - simdjson_inline bool has_full_block() const; - simdjson_inline const uint8_t* full_block() const; - /** - * Get the last block, padded with spaces. - * - * There will always be a last block, with at least 1 byte, unless len == 0 (in which case this - * function fills the buffer with spaces and returns 0. In particular, if len == STEP_SIZE there - * will be 0 full_blocks and 1 remainder block with STEP_SIZE bytes and no spaces for padding. - * - * @return the number of effective characters in the last block. - */ - simdjson_inline size_t get_remainder(uint8_t* dst) const; - simdjson_inline void advance(); - private: - const uint8_t* buf; - const size_t len; - const size_t lenminusstep; - size_t idx; - }; - - // Routines to print masks and text for debugging bitmask operations - simdjson_unused static char* format_input_text_64(const uint8_t* text) { - static char buf[sizeof(simd8x64) + 1]; - for (size_t i = 0; i < sizeof(simd8x64); i++) { - buf[i] = int8_t(text[i]) < ' ' ? '_' : int8_t(text[i]); - } - buf[sizeof(simd8x64)] = '\0'; - return buf; - } - - // Routines to print masks and text for debugging bitmask operations - simdjson_unused static char* format_input_text(const simd8x64& in) { - static char buf[sizeof(simd8x64) + 1]; - in.store(reinterpret_cast(buf)); - for (size_t i = 0; i < sizeof(simd8x64); i++) { - if (buf[i] < ' ') { buf[i] = '_'; } - } - buf[sizeof(simd8x64)] = '\0'; - return buf; - } - - simdjson_unused static char* format_input_text(const simd8x64& in, uint64_t mask) { - static char buf[sizeof(simd8x64) + 1]; - in.store(reinterpret_cast(buf)); - for (size_t i = 0; i < sizeof(simd8x64); i++) { - if (buf[i] <= ' ') { buf[i] = '_'; } - if (!(mask & (size_t(1) << i))) { buf[i] = ' '; } - } - buf[sizeof(simd8x64)] = '\0'; - return buf; - } - - simdjson_unused static char* format_mask(uint64_t mask) { - static char buf[sizeof(simd8x64) + 1]; - for (size_t i = 0; i < 64; i++) { - buf[i] = (mask & (size_t(1) << i)) ? 'X' : ' '; - } - buf[64] = '\0'; - return buf; - } - - template - simdjson_inline buf_block_reader::buf_block_reader(const uint8_t* _buf, size_t _len) : buf{ _buf }, len{ _len }, lenminusstep{ len < STEP_SIZE ? 0 : len - STEP_SIZE }, idx{ 0 } {} - - template - simdjson_inline size_t buf_block_reader::block_index() { return idx; } - - template - simdjson_inline bool buf_block_reader::has_full_block() const { - return idx < lenminusstep; - } - - template - simdjson_inline const uint8_t* buf_block_reader::full_block() const { - return &buf[idx]; - } - - template - simdjson_inline size_t buf_block_reader::get_remainder(uint8_t* dst) const { - if (len == idx) { return 0; } // memcpy(dst, null, 0) will trigger an error with some sanitizers - std::memset(dst, 0x20, STEP_SIZE); // std::memset STEP_SIZE because it's more efficient to write out 8 or 16 bytes at once. - std::memcpy(dst, buf + idx, len - idx); - return len - idx; - } - - template - simdjson_inline void buf_block_reader::advance() { - idx += STEP_SIZE; - } - - } // namespace stage1 - } // unnamed namespace - } // namespace ppc64 -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_BUF_BLOCK_READER_H -/* end file generic/stage1/buf_block_reader.h for ppc64 */ -/* including generic/stage1/json_escape_scanner.h for ppc64: #include */ -/* begin file generic/stage1/json_escape_scanner.h for ppc64 */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_JSON_ESCAPE_SCANNER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_JSON_ESCAPE_SCANNER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - namespace ppc64 { - namespace { - namespace stage1 { - - /** - * Scans for escape characters in JSON, taking care with multiple backslashes (\\n vs. \n). - */ - struct json_escape_scanner { - /** The actual escape characters (the backslashes themselves). */ - uint64_t next_is_escaped = 0ULL; - - struct escaped_and_escape { - /** - * Mask of escaped characters. - * - * ``` - * \n \\n \\\n \\\\n \ - * 0100100010100101000 - * n \ \ n \ \ - * ``` - */ - uint64_t escaped; - /** - * Mask of escape characters. - * - * ``` - * \n \\n \\\n \\\\n \ - * 1001000101001010001 - * \ \ \ \ \ \ \ - * ``` - */ - uint64_t escape; - }; - - /** - * Get a mask of both escape and escaped characters (the characters following a backslash). - * - * @param potential_escape A mask of the character that can escape others (but could be - * escaped itself). e.g. block.eq('\\') - */ - simdjson_really_inline escaped_and_escape next(uint64_t backslash) noexcept { - -#if !SIMDJSON_SKIP_BACKSLASH_SHORT_CIRCUIT - if (!backslash) { return { next_escaped_without_backslashes(), 0 }; } -#endif - - // | | Mask (shows characters instead of 1's) | Depth | Instructions | - // |--------------------------------|----------------------------------------|-------|---------------------| - // | string | `\\n_\\\n___\\\n___\\\\___\\\\__\\\` | | | - // | | ` even odd even odd odd` | | | - // | potential_escape | ` \ \\\ \\\ \\\\ \\\\ \\\` | 1 | 1 (backslash & ~first_is_escaped) - // | escape_and_terminal_code | ` \n \ \n \ \n \ \ \ \ \ \` | 5 | 5 (next_escape_and_terminal_code()) - // | escaped | `\ \ n \ n \ \ \ \ \ ` X | 6 | 7 (escape_and_terminal_code ^ (potential_escape | first_is_escaped)) - // | escape | ` \ \ \ \ \ \ \ \ \ \` | 6 | 8 (escape_and_terminal_code & backslash) - // | first_is_escaped | `\ ` | 7 (*) | 9 (escape >> 63) () - // (*) this is not needed until the next iteration - uint64_t escape_and_terminal_code = next_escape_and_terminal_code(backslash & ~this->next_is_escaped); - uint64_t escaped = escape_and_terminal_code ^ (backslash | this->next_is_escaped); - uint64_t escape = escape_and_terminal_code & backslash; - this->next_is_escaped = escape >> 63; - return { escaped, escape }; - } - - private: - static constexpr const uint64_t ODD_BITS = 0xAAAAAAAAAAAAAAAAULL; - - simdjson_really_inline uint64_t next_escaped_without_backslashes() noexcept { - uint64_t escaped = this->next_is_escaped; - this->next_is_escaped = 0; - return escaped; - } - - /** - * Returns a mask of the next escape characters (masking out escaped backslashes), along with - * any non-backslash escape codes. - * - * \n \\n \\\n \\\\n returns: - * \n \ \ \n \ \ - * 11 100 1011 10100 - * - * You are expected to mask out the first bit yourself if the previous block had a trailing - * escape. - * - * & the result with potential_escape to get just the escape characters. - * ^ the result with (potential_escape | first_is_escaped) to get escaped characters. - */ - static simdjson_really_inline uint64_t next_escape_and_terminal_code(uint64_t potential_escape) noexcept { - // If we were to just shift and mask out any odd bits, we'd actually get a *half* right answer: - // any even-aligned backslash runs would be correct! Odd-aligned backslash runs would be - // inverted (\\\ would be 010 instead of 101). - // - // ``` - // string: | ____\\\\_\\\\_____ | - // maybe_escaped | ODD | \ \ \ \ | - // even-aligned ^^^ ^^^^ odd-aligned - // ``` - // - // Taking that into account, our basic strategy is: - // - // 1. Use subtraction to produce a mask with 1's for even-aligned runs and 0's for - // odd-aligned runs. - // 2. XOR all odd bits, which masks out the odd bits in even-aligned runs, and brings IN the - // odd bits in odd-aligned runs. - // 3. & with backslash to clean up any stray bits. - // runs are set to 0, and then XORing with "odd": - // - // | | Mask (shows characters instead of 1's) | Instructions | - // |--------------------------------|----------------------------------------|---------------------| - // | string | `\\n_\\\n___\\\n___\\\\___\\\\__\\\` | - // | | ` even odd even odd odd` | - // | maybe_escaped | ` n \\n \\n \\\_ \\\_ \\` X | 1 (potential_escape << 1) - // | maybe_escaped_and_odd | ` \n_ \\n _ \\\n_ _ \\\__ _\\\_ \\\` | 1 (maybe_escaped | odd) - // | even_series_codes_and_odd | ` n_\\\ _ n_ _\\\\ _ _ ` | 1 (maybe_escaped_and_odd - potential_escape) - // | escape_and_terminal_code | ` \n \ \n \ \n \ \ \ \ \ \` | 1 (^ odd) - // - - // Escaped characters are characters following an escape. - uint64_t maybe_escaped = potential_escape << 1; - - // To distinguish odd from even escape sequences, therefore, we turn on any *starting* - // escapes that are on an odd byte. (We actually bring in all odd bits, for speed.) - // - Odd runs of backslashes are 0000, and the code at the end ("n" in \n or \\n) is 1. - // - Odd runs of backslashes are 1111, and the code at the end ("n" in \n or \\n) is 0. - // - All other odd bytes are 1, and even bytes are 0. - uint64_t maybe_escaped_and_odd_bits = maybe_escaped | ODD_BITS; - uint64_t even_series_codes_and_odd_bits = maybe_escaped_and_odd_bits - potential_escape; - - // Now we flip all odd bytes back with xor. This: - // - Makes odd runs of backslashes go from 0000 to 1010 - // - Makes even runs of backslashes go from 1111 to 1010 - // - Sets actually-escaped codes to 1 (the n in \n and \\n: \n = 11, \\n = 100) - // - Resets all other bytes to 0 - return even_series_codes_and_odd_bits ^ ODD_BITS; - } - }; - - } // namespace stage1 - } // unnamed namespace - } // namespace ppc64 -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRING_SCANNER_H -/* end file generic/stage1/json_escape_scanner.h for ppc64 */ -/* including generic/stage1/json_string_scanner.h for ppc64: #include */ -/* begin file generic/stage1/json_string_scanner.h for ppc64 */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRING_SCANNER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRING_SCANNER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - namespace ppc64 { - namespace { - namespace stage1 { - - struct json_string_block { - // We spell out the constructors in the hope of resolving inlining issues with Visual Studio 2017 - simdjson_really_inline json_string_block(uint64_t escaped, uint64_t quote, uint64_t in_string) : - _escaped(escaped), _quote(quote), _in_string(in_string) {} - - // Escaped characters (characters following an escape() character) - simdjson_really_inline uint64_t escaped() const { return _escaped; } - // Real (non-backslashed) quotes - simdjson_really_inline uint64_t quote() const { return _quote; } - // Only characters inside the string (not including the quotes) - simdjson_really_inline uint64_t string_content() const { return _in_string & ~_quote; } - // Return a mask of whether the given characters are inside a string (only works on non-quotes) - simdjson_really_inline uint64_t non_quote_inside_string(uint64_t mask) const { return mask & _in_string; } - // Return a mask of whether the given characters are inside a string (only works on non-quotes) - simdjson_really_inline uint64_t non_quote_outside_string(uint64_t mask) const { return mask & ~_in_string; } - // Tail of string (everything except the start quote) - simdjson_really_inline uint64_t string_tail() const { return _in_string ^ _quote; } - - // escaped characters (backslashed--does not include the hex characters after \u) - uint64_t _escaped; - // real quotes (non-escaped ones) - uint64_t _quote; - // string characters (includes start quote but not end quote) - uint64_t _in_string; - }; - - // Scans blocks for string characters, storing the state necessary to do so - class json_string_scanner { - public: - simdjson_really_inline json_string_block next(const simd::simd8x64& in); - // Returns either UNCLOSED_STRING or SUCCESS - simdjson_really_inline error_code finish(); - - private: - // Scans for escape characters - json_escape_scanner escape_scanner{}; - // Whether the last iteration was still inside a string (all 1's = true, all 0's = false). - uint64_t prev_in_string = 0ULL; - }; - - // - // Return a mask of all string characters plus end quotes. - // - // prev_escaped is overflow saying whether the next character is escaped. - // prev_in_string is overflow saying whether we're still in a string. - // - // Backslash sequences outside of quotes will be detected in stage 2. - // - simdjson_really_inline json_string_block json_string_scanner::next(const simd::simd8x64& in) { - const uint64_t backslash = in.eq('\\'); - const uint64_t escaped = escape_scanner.next(backslash).escaped; - const uint64_t quote = in.eq('"') & ~escaped; - - // - // prefix_xor flips on bits inside the string (and flips off the end quote). - // - // Then we xor with prev_in_string: if we were in a string already, its effect is flipped - // (characters inside strings are outside, and characters outside strings are inside). - // - const uint64_t in_string = prefix_xor(quote) ^ prev_in_string; - - // - // Check if we're still in a string at the end of the box so the next block will know - // - prev_in_string = uint64_t(static_cast(in_string) >> 63); - - // Use ^ to turn the beginning quote off, and the end quote on. - - // We are returning a function-local object so either we get a move constructor - // or we get copy elision. - return json_string_block(escaped, quote, in_string); - } - - simdjson_really_inline error_code json_string_scanner::finish() { - if (prev_in_string) { - return UNCLOSED_STRING; - } - return SUCCESS; - } - - } // namespace stage1 - } // unnamed namespace - } // namespace ppc64 -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRING_SCANNER_H -/* end file generic/stage1/json_string_scanner.h for ppc64 */ -/* including generic/stage1/utf8_lookup4_algorithm.h for ppc64: #include */ -/* begin file generic/stage1/utf8_lookup4_algorithm.h for ppc64 */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_UTF8_LOOKUP4_ALGORITHM_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_UTF8_LOOKUP4_ALGORITHM_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - namespace ppc64 { - namespace { - namespace utf8_validation { - - using namespace simd; - - simdjson_inline simd8 check_special_cases(const simd8 input, const simd8 prev1) { - // Bit 0 = Too Short (lead byte/ASCII followed by lead byte/ASCII) - // Bit 1 = Too Long (ASCII followed by continuation) - // Bit 2 = Overlong 3-byte - // Bit 4 = Surrogate - // Bit 5 = Overlong 2-byte - // Bit 7 = Two Continuations - constexpr const uint8_t TOO_SHORT = 1 << 0; // 11______ 0_______ - // 11______ 11______ - constexpr const uint8_t TOO_LONG = 1 << 1; // 0_______ 10______ - constexpr const uint8_t OVERLONG_3 = 1 << 2; // 11100000 100_____ - constexpr const uint8_t SURROGATE = 1 << 4; // 11101101 101_____ - constexpr const uint8_t OVERLONG_2 = 1 << 5; // 1100000_ 10______ - constexpr const uint8_t TWO_CONTS = 1 << 7; // 10______ 10______ - constexpr const uint8_t TOO_LARGE = 1 << 3; // 11110100 1001____ - // 11110100 101_____ - // 11110101 1001____ - // 11110101 101_____ - // 1111011_ 1001____ - // 1111011_ 101_____ - // 11111___ 1001____ - // 11111___ 101_____ - constexpr const uint8_t TOO_LARGE_1000 = 1 << 6; - // 11110101 1000____ - // 1111011_ 1000____ - // 11111___ 1000____ - constexpr const uint8_t OVERLONG_4 = 1 << 6; // 11110000 1000____ - - const simd8 byte_1_high = prev1.shr<4>().lookup_16( - // 0_______ ________ - TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, - TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, - // 10______ ________ - TWO_CONTS, TWO_CONTS, TWO_CONTS, TWO_CONTS, - // 1100____ ________ - TOO_SHORT | OVERLONG_2, - // 1101____ ________ - TOO_SHORT, - // 1110____ ________ - TOO_SHORT | OVERLONG_3 | SURROGATE, - // 1111____ ________ - TOO_SHORT | TOO_LARGE | TOO_LARGE_1000 | OVERLONG_4 - ); - constexpr const uint8_t CARRY = TOO_SHORT | TOO_LONG | TWO_CONTS; // These all have ____ in byte 1 . - const simd8 byte_1_low = (prev1 & 0x0F).lookup_16( - // ____0000 ________ - CARRY | OVERLONG_3 | OVERLONG_2 | OVERLONG_4, - // ____0001 ________ - CARRY | OVERLONG_2, - // ____001_ ________ - CARRY, - CARRY, - - // ____0100 ________ - CARRY | TOO_LARGE, - // ____0101 ________ - CARRY | TOO_LARGE | TOO_LARGE_1000, - // ____011_ ________ - CARRY | TOO_LARGE | TOO_LARGE_1000, - CARRY | TOO_LARGE | TOO_LARGE_1000, - - // ____1___ ________ - CARRY | TOO_LARGE | TOO_LARGE_1000, - CARRY | TOO_LARGE | TOO_LARGE_1000, - CARRY | TOO_LARGE | TOO_LARGE_1000, - CARRY | TOO_LARGE | TOO_LARGE_1000, - CARRY | TOO_LARGE | TOO_LARGE_1000, - // ____1101 ________ - CARRY | TOO_LARGE | TOO_LARGE_1000 | SURROGATE, - CARRY | TOO_LARGE | TOO_LARGE_1000, - CARRY | TOO_LARGE | TOO_LARGE_1000 - ); - const simd8 byte_2_high = input.shr<4>().lookup_16( - // ________ 0_______ - TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT, - TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT, - - // ________ 1000____ - TOO_LONG | OVERLONG_2 | TWO_CONTS | OVERLONG_3 | TOO_LARGE_1000 | OVERLONG_4, - // ________ 1001____ - TOO_LONG | OVERLONG_2 | TWO_CONTS | OVERLONG_3 | TOO_LARGE, - // ________ 101_____ - TOO_LONG | OVERLONG_2 | TWO_CONTS | SURROGATE | TOO_LARGE, - TOO_LONG | OVERLONG_2 | TWO_CONTS | SURROGATE | TOO_LARGE, - - // ________ 11______ - TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT - ); - return (byte_1_high & byte_1_low & byte_2_high); - } - simdjson_inline simd8 check_multibyte_lengths(const simd8 input, - const simd8 prev_input, const simd8 sc) { - simd8 prev2 = input.prev<2>(prev_input); - simd8 prev3 = input.prev<3>(prev_input); - simd8 must23 = simd8(must_be_2_3_continuation(prev2, prev3)); - simd8 must23_80 = must23 & uint8_t(0x80); - return must23_80 ^ sc; - } - - // - // Return nonzero if there are incomplete multibyte characters at the end of the block: - // e.g. if there is a 4-byte character, but it's 3 bytes from the end. - // - simdjson_inline simd8 is_incomplete(const simd8 input) { - // If the previous input's last 3 bytes match this, they're too short (they ended at EOF): - // ... 1111____ 111_____ 11______ -#if SIMDJSON_IMPLEMENTATION_ICELAKE - static const uint8_t max_array[64] = { - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 0xf0u - 1, 0xe0u - 1, 0xc0u - 1 - }; -#else - static const uint8_t max_array[32] = { - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 0xf0u - 1, 0xe0u - 1, 0xc0u - 1 - }; -#endif - const simd8 max_value(&max_array[sizeof(max_array) - sizeof(simd8)]); - return input.gt_bits(max_value); - } - - struct utf8_checker { - // If this is nonzero, there has been a UTF-8 error. - simd8 error; - // The last input we received - simd8 prev_input_block; - // Whether the last input we received was incomplete (used for ASCII fast path) - simd8 prev_incomplete; - - // - // Check whether the current bytes are valid UTF-8. - // - simdjson_inline void check_utf8_bytes(const simd8 input, const simd8 prev_input) { - // Flip prev1...prev3 so we can easily determine if they are 2+, 3+ or 4+ lead bytes - // (2, 3, 4-byte leads become large positive numbers instead of small negative numbers) - simd8 prev1 = input.prev<1>(prev_input); - simd8 sc = check_special_cases(input, prev1); - this->error |= check_multibyte_lengths(input, prev_input, sc); - } - - // The only problem that can happen at EOF is that a multibyte character is too short - // or a byte value too large in the last bytes: check_special_cases only checks for bytes - // too large in the first of two bytes. - simdjson_inline void check_eof() { - // If the previous block had incomplete UTF-8 characters at the end, an ASCII block can't - // possibly finish them. - this->error |= this->prev_incomplete; - } - - simdjson_inline void check_next_input(const simd8x64& input) { - if (simdjson_likely(is_ascii(input))) { - this->error |= this->prev_incomplete; - } - else { - // you might think that a for-loop would work, but under Visual Studio, it is not good enough. - static_assert((simd8x64::NUM_CHUNKS == 1) - || (simd8x64::NUM_CHUNKS == 2) - || (simd8x64::NUM_CHUNKS == 4), - "We support one, two or four chunks per 64-byte block."); - SIMDJSON_IF_CONSTEXPR(simd8x64::NUM_CHUNKS == 1) { - this->check_utf8_bytes(input.chunks[0], this->prev_input_block); - } -else SIMDJSON_IF_CONSTEXPR(simd8x64::NUM_CHUNKS == 2) { - this->check_utf8_bytes(input.chunks[0], this->prev_input_block); - this->check_utf8_bytes(input.chunks[1], input.chunks[0]); - } - else SIMDJSON_IF_CONSTEXPR(simd8x64::NUM_CHUNKS == 4) { - this->check_utf8_bytes(input.chunks[0], this->prev_input_block); - this->check_utf8_bytes(input.chunks[1], input.chunks[0]); - this->check_utf8_bytes(input.chunks[2], input.chunks[1]); - this->check_utf8_bytes(input.chunks[3], input.chunks[2]); - } - this->prev_incomplete = is_incomplete(input.chunks[simd8x64::NUM_CHUNKS - 1]); - this->prev_input_block = input.chunks[simd8x64::NUM_CHUNKS - 1]; - } - } - // do not forget to call check_eof! - simdjson_inline error_code errors() { - return this->error.any_bits_set_anywhere() ? error_code::UTF8_ERROR : error_code::SUCCESS; - } - - }; // struct utf8_checker - } // namespace utf8_validation - - } // unnamed namespace - } // namespace ppc64 -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_UTF8_LOOKUP4_ALGORITHM_H -/* end file generic/stage1/utf8_lookup4_algorithm.h for ppc64 */ -/* including generic/stage1/json_scanner.h for ppc64: #include */ -/* begin file generic/stage1/json_scanner.h for ppc64 */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_JSON_SCANNER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_JSON_SCANNER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - namespace ppc64 { - namespace { - namespace stage1 { - - /** - * A block of scanned json, with information on operators and scalars. - * - * We seek to identify pseudo-structural characters. Anything that is inside - * a string must be omitted (hence & ~_string.string_tail()). - * Otherwise, pseudo-structural characters come in two forms. - * 1. We have the structural characters ([,],{,},:, comma). The - * term 'structural character' is from the JSON RFC. - * 2. We have the 'scalar pseudo-structural characters'. - * Scalars are quotes, and any character except structural characters and white space. - * - * To identify the scalar pseudo-structural characters, we must look at what comes - * before them: it must be a space, a quote or a structural characters. - * Starting with simdjson v0.3, we identify them by - * negation: we identify everything that is followed by a non-quote scalar, - * and we negate that. Whatever remains must be a 'scalar pseudo-structural character'. - */ - struct json_block { - public: - // We spell out the constructors in the hope of resolving inlining issues with Visual Studio 2017 - simdjson_inline json_block(json_string_block&& string, json_character_block characters, uint64_t follows_potential_nonquote_scalar) : - _string(std::move(string)), _characters(characters), _follows_potential_nonquote_scalar(follows_potential_nonquote_scalar) {} - simdjson_inline json_block(json_string_block string, json_character_block characters, uint64_t follows_potential_nonquote_scalar) : - _string(string), _characters(characters), _follows_potential_nonquote_scalar(follows_potential_nonquote_scalar) {} - - /** - * The start of structurals. - * In simdjson prior to v0.3, these were called the pseudo-structural characters. - **/ - simdjson_inline uint64_t structural_start() const noexcept { return potential_structural_start() & ~_string.string_tail(); } - /** All JSON whitespace (i.e. not in a string) */ - simdjson_inline uint64_t whitespace() const noexcept { return non_quote_outside_string(_characters.whitespace()); } - - // Helpers - - /** Whether the given characters are inside a string (only works on non-quotes) */ - simdjson_inline uint64_t non_quote_inside_string(uint64_t mask) const noexcept { return _string.non_quote_inside_string(mask); } - /** Whether the given characters are outside a string (only works on non-quotes) */ - simdjson_inline uint64_t non_quote_outside_string(uint64_t mask) const noexcept { return _string.non_quote_outside_string(mask); } - - // string and escape characters - json_string_block _string; - // whitespace, structural characters ('operators'), scalars - json_character_block _characters; - // whether the previous character was a scalar - uint64_t _follows_potential_nonquote_scalar; - private: - // Potential structurals (i.e. disregarding strings) - - /** - * structural elements ([,],{,},:, comma) plus scalar starts like 123, true and "abc". - * They may reside inside a string. - **/ - simdjson_inline uint64_t potential_structural_start() const noexcept { return _characters.op() | potential_scalar_start(); } - /** - * The start of non-operator runs, like 123, true and "abc". - * It main reside inside a string. - **/ - simdjson_inline uint64_t potential_scalar_start() const noexcept { - // The term "scalar" refers to anything except structural characters and white space - // (so letters, numbers, quotes). - // Whenever it is preceded by something that is not a structural element ({,},[,],:, ") nor a white-space - // then we know that it is irrelevant structurally. - return _characters.scalar() & ~follows_potential_scalar(); - } - /** - * Whether the given character is immediately after a non-operator like 123, true. - * The characters following a quote are not included. - */ - simdjson_inline uint64_t follows_potential_scalar() const noexcept { - // _follows_potential_nonquote_scalar: is defined as marking any character that follows a character - // that is not a structural element ({,},[,],:, comma) nor a quote (") and that is not a - // white space. - // It is understood that within quoted region, anything at all could be marked (irrelevant). - return _follows_potential_nonquote_scalar; - } - }; - - /** - * Scans JSON for important bits: structural characters or 'operators', strings, and scalars. - * - * The scanner starts by calculating two distinct things: - * - string characters (taking \" into account) - * - structural characters or 'operators' ([]{},:, comma) - * and scalars (runs of non-operators like 123, true and "abc") - * - * To minimize data dependency (a key component of the scanner's speed), it finds these in parallel: - * in particular, the operator/scalar bit will find plenty of things that are actually part of - * strings. When we're done, json_block will fuse the two together by masking out tokens that are - * part of a string. - */ - class json_scanner { - public: - json_scanner() = default; - simdjson_inline json_block next(const simd::simd8x64& in); - // Returns either UNCLOSED_STRING or SUCCESS - simdjson_inline error_code finish(); - - private: - // Whether the last character of the previous iteration is part of a scalar token - // (anything except whitespace or a structural character/'operator'). - uint64_t prev_scalar = 0ULL; - json_string_scanner string_scanner{}; - }; - - - // - // Check if the current character immediately follows a matching character. - // - // For example, this checks for quotes with backslashes in front of them: - // - // const uint64_t backslashed_quote = in.eq('"') & immediately_follows(in.eq('\'), prev_backslash); - // - simdjson_inline uint64_t follows(const uint64_t match, uint64_t& overflow) { - const uint64_t result = match << 1 | overflow; - overflow = match >> 63; - return result; - } - - simdjson_inline json_block json_scanner::next(const simd::simd8x64& in) { - json_string_block strings = string_scanner.next(in); - // identifies the white-space and the structural characters - json_character_block characters = json_character_block::classify(in); - // The term "scalar" refers to anything except structural characters and white space - // (so letters, numbers, quotes). - // We want follows_scalar to mark anything that follows a non-quote scalar (so letters and numbers). - // - // A terminal quote should either be followed by a structural character (comma, brace, bracket, colon) - // or nothing. However, we still want ' "a string"true ' to mark the 't' of 'true' as a potential - // pseudo-structural character just like we would if we had ' "a string" true '; otherwise we - // may need to add an extra check when parsing strings. - // - // Performance: there are many ways to skin this cat. - const uint64_t nonquote_scalar = characters.scalar() & ~strings.quote(); - uint64_t follows_nonquote_scalar = follows(nonquote_scalar, prev_scalar); - // We are returning a function-local object so either we get a move constructor - // or we get copy elision. - return json_block( - strings,// strings is a function-local object so either it moves or the copy is elided. - characters, - follows_nonquote_scalar - ); - } - - simdjson_inline error_code json_scanner::finish() { - return string_scanner.finish(); - } - - } // namespace stage1 - } // unnamed namespace - } // namespace ppc64 -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_JSON_SCANNER_H -/* end file generic/stage1/json_scanner.h for ppc64 */ - -// All other declarations -/* including generic/stage1/find_next_document_index.h for ppc64: #include */ -/* begin file generic/stage1/find_next_document_index.h for ppc64 */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_FIND_NEXT_DOCUMENT_INDEX_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_FIND_NEXT_DOCUMENT_INDEX_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - namespace ppc64 { - namespace { - namespace stage1 { - - /** - * This algorithm is used to quickly identify the last structural position that - * makes up a complete document. - * - * It does this by going backwards and finding the last *document boundary* (a - * place where one value follows another without a comma between them). If the - * last document (the characters after the boundary) has an equal number of - * start and end brackets, it is considered complete. - * - * Simply put, we iterate over the structural characters, starting from - * the end. We consider that we found the end of a JSON document when the - * first element of the pair is NOT one of these characters: '{' '[' ':' ',' - * and when the second element is NOT one of these characters: '}' ']' ':' ','. - * - * This simple comparison works most of the time, but it does not cover cases - * where the batch's structural indexes contain a perfect amount of documents. - * In such a case, we do not have access to the structural index which follows - * the last document, therefore, we do not have access to the second element in - * the pair, and that means we cannot identify the last document. To fix this - * issue, we keep a count of the open and closed curly/square braces we found - * while searching for the pair. When we find a pair AND the count of open and - * closed curly/square braces is the same, we know that we just passed a - * complete document, therefore the last json buffer location is the end of the - * batch. - */ - simdjson_inline uint32_t find_next_document_index(dom_parser_implementation& parser) { - // Variant: do not count separately, just figure out depth - if (parser.n_structural_indexes == 0) { return 0; } - auto arr_cnt = 0; - auto obj_cnt = 0; - for (auto i = parser.n_structural_indexes - 1; i > 0; i--) { - auto idxb = parser.structural_indexes[i]; - switch (parser.buf[idxb]) { - case ':': - case ',': - continue; - case '}': - obj_cnt--; - continue; - case ']': - arr_cnt--; - continue; - case '{': - obj_cnt++; - break; - case '[': - arr_cnt++; - break; - } - auto idxa = parser.structural_indexes[i - 1]; - switch (parser.buf[idxa]) { - case '{': - case '[': - case ':': - case ',': - continue; - } - // Last document is complete, so the next document will appear after! - if (!arr_cnt && !obj_cnt) { - return parser.n_structural_indexes; - } - // Last document is incomplete; mark the document at i + 1 as the next one - return i; - } - // If we made it to the end, we want to finish counting to see if we have a full document. - switch (parser.buf[parser.structural_indexes[0]]) { - case '}': - obj_cnt--; - break; - case ']': - arr_cnt--; - break; - case '{': - obj_cnt++; - break; - case '[': - arr_cnt++; - break; - } - if (!arr_cnt && !obj_cnt) { - // We have a complete document. - return parser.n_structural_indexes; - } - return 0; - } - - } // namespace stage1 - } // unnamed namespace - } // namespace ppc64 -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_FIND_NEXT_DOCUMENT_INDEX_H -/* end file generic/stage1/find_next_document_index.h for ppc64 */ -/* including generic/stage1/json_minifier.h for ppc64: #include */ -/* begin file generic/stage1/json_minifier.h for ppc64 */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_JSON_MINIFIER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_JSON_MINIFIER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -// This file contains the common code every implementation uses in stage1 -// It is intended to be included multiple times and compiled multiple times -// We assume the file in which it is included already includes -// "simdjson/stage1.h" (this simplifies amalgation) - -namespace simdjson { - namespace ppc64 { - namespace { - namespace stage1 { - - class json_minifier { - public: - template - static error_code minify(const uint8_t* buf, size_t len, uint8_t* dst, size_t& dst_len) noexcept; - - private: - simdjson_inline json_minifier(uint8_t* _dst) - : dst{ _dst } - {} - template - simdjson_inline void step(const uint8_t* block_buf, buf_block_reader& reader) noexcept; - simdjson_inline void next(const simd::simd8x64& in, const json_block& block); - simdjson_inline error_code finish(uint8_t* dst_start, size_t& dst_len); - json_scanner scanner{}; - uint8_t* dst; - }; - - simdjson_inline void json_minifier::next(const simd::simd8x64& in, const json_block& block) { - uint64_t mask = block.whitespace(); - dst += in.compress(mask, dst); - } - - simdjson_inline error_code json_minifier::finish(uint8_t* dst_start, size_t& dst_len) { - error_code error = scanner.finish(); - if (error) { dst_len = 0; return error; } - dst_len = dst - dst_start; - return SUCCESS; - } - - template<> - simdjson_inline void json_minifier::step<128>(const uint8_t* block_buf, buf_block_reader<128>& reader) noexcept { - simd::simd8x64 in_1(block_buf); - simd::simd8x64 in_2(block_buf + 64); - json_block block_1 = scanner.next(in_1); - json_block block_2 = scanner.next(in_2); - this->next(in_1, block_1); - this->next(in_2, block_2); - reader.advance(); - } - - template<> - simdjson_inline void json_minifier::step<64>(const uint8_t* block_buf, buf_block_reader<64>& reader) noexcept { - simd::simd8x64 in_1(block_buf); - json_block block_1 = scanner.next(in_1); - this->next(block_buf, block_1); - reader.advance(); - } - - template - error_code json_minifier::minify(const uint8_t* buf, size_t len, uint8_t* dst, size_t& dst_len) noexcept { - buf_block_reader reader(buf, len); - json_minifier minifier(dst); - - // Index the first n-1 blocks - while (reader.has_full_block()) { - minifier.step(reader.full_block(), reader); - } - - // Index the last (remainder) block, padded with spaces - uint8_t block[STEP_SIZE]; - size_t remaining_bytes = reader.get_remainder(block); - if (remaining_bytes > 0) { - // We do not want to write directly to the output stream. Rather, we write - // to a local buffer (for safety). - uint8_t out_block[STEP_SIZE]; - uint8_t* const guarded_dst{ minifier.dst }; - minifier.dst = out_block; - minifier.step(block, reader); - size_t to_write = minifier.dst - out_block; - // In some cases, we could be enticed to consider the padded spaces - // as part of the string. This is fine as long as we do not write more - // than we consumed. - if (to_write > remaining_bytes) { to_write = remaining_bytes; } - memcpy(guarded_dst, out_block, to_write); - minifier.dst = guarded_dst + to_write; - } - return minifier.finish(dst, dst_len); - } - - } // namespace stage1 - } // unnamed namespace - } // namespace ppc64 -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_JSON_MINIFIER_H -/* end file generic/stage1/json_minifier.h for ppc64 */ -/* including generic/stage1/json_structural_indexer.h for ppc64: #include */ -/* begin file generic/stage1/json_structural_indexer.h for ppc64 */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRUCTURAL_INDEXER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRUCTURAL_INDEXER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -// This file contains the common code every implementation uses in stage1 -// It is intended to be included multiple times and compiled multiple times -// We assume the file in which it is included already includes -// "simdjson/stage1.h" (this simplifies amalgation) - -namespace simdjson { - namespace ppc64 { - namespace { - namespace stage1 { - - class bit_indexer { - public: - uint32_t* tail; - - simdjson_inline bit_indexer(uint32_t* index_buf) : tail(index_buf) {} - -#if SIMDJSON_PREFER_REVERSE_BITS - /** - * ARM lacks a fast trailing zero instruction, but it has a fast - * bit reversal instruction and a fast leading zero instruction. - * Thus it may be profitable to reverse the bits (once) and then - * to rely on a sequence of instructions that call the leading - * zero instruction. - * - * Performance notes: - * The chosen routine is not optimal in terms of data dependency - * since zero_leading_bit might require two instructions. However, - * it tends to minimize the total number of instructions which is - * beneficial. - */ - simdjson_inline void write_index(uint32_t idx, uint64_t& rev_bits, int i) { - int lz = leading_zeroes(rev_bits); - this->tail[i] = static_cast(idx) + lz; - rev_bits = zero_leading_bit(rev_bits, lz); - } -#else - /** - * Under recent x64 systems, we often have both a fast trailing zero - * instruction and a fast 'clear-lower-bit' instruction so the following - * algorithm can be competitive. - */ - - simdjson_inline void write_index(uint32_t idx, uint64_t& bits, int i) { - this->tail[i] = idx + trailing_zeroes(bits); - bits = clear_lowest_bit(bits); - } -#endif // SIMDJSON_PREFER_REVERSE_BITS - - template - simdjson_inline int write_indexes(uint32_t idx, uint64_t& bits) { - write_index(idx, bits, START); - SIMDJSON_IF_CONSTEXPR(N > 1) { - write_indexes<(N - 1 > 0 ? START + 1 : START), (N - 1 >= 0 ? N - 1 : 1)>(idx, bits); - } - return START + N; - } - - template - simdjson_inline int write_indexes_stepped(uint32_t idx, uint64_t& bits, int cnt) { - write_indexes(idx, bits); - SIMDJSON_IF_CONSTEXPR((START + STEP) < END) { - if (simdjson_unlikely((START + STEP) < cnt)) { - write_indexes_stepped<(START + STEP < END ? START + STEP : END), END, STEP>(idx, bits, cnt); - } - } - return ((END - START) % STEP) == 0 ? END : (END - START) - ((END - START) % STEP) + STEP; - } - - // flatten out values in 'bits' assuming that they are are to have values of idx - // plus their position in the bitvector, and store these indexes at - // base_ptr[base] incrementing base as we go - // will potentially store extra values beyond end of valid bits, so base_ptr - // needs to be large enough to handle this - // - // If the kernel sets SIMDJSON_GENERIC_JSON_STRUCTURAL_INDEXER_CUSTOM_BIT_INDEXER, then it - // will provide its own version of the code. -#ifdef SIMDJSON_GENERIC_JSON_STRUCTURAL_INDEXER_CUSTOM_BIT_INDEXER - simdjson_inline void write(uint32_t idx, uint64_t bits); -#else - simdjson_inline void write(uint32_t idx, uint64_t bits) { - // In some instances, the next branch is expensive because it is mispredicted. - // Unfortunately, in other cases, - // it helps tremendously. - if (bits == 0) - return; - - int cnt = static_cast(count_ones(bits)); - -#if SIMDJSON_PREFER_REVERSE_BITS - bits = reverse_bits(bits); -#endif -#ifdef SIMDJSON_STRUCTURAL_INDEXER_STEP - static constexpr const int STEP = SIMDJSON_STRUCTURAL_INDEXER_STEP; -#else - static constexpr const int STEP = 4; -#endif - static constexpr const int STEP_UNTIL = 24; - - write_indexes_stepped<0, STEP_UNTIL, STEP>(idx, bits, cnt); - SIMDJSON_IF_CONSTEXPR(STEP_UNTIL < 64) { - if (simdjson_unlikely(STEP_UNTIL < cnt)) { - for (int i = STEP_UNTIL; i < cnt; i++) { - write_index(idx, bits, i); - } - } - } - - this->tail += cnt; - } -#endif // SIMDJSON_GENERIC_JSON_STRUCTURAL_INDEXER_CUSTOM_BIT_INDEXER - - }; - - class json_structural_indexer { - public: - /** - * Find the important bits of JSON in a 128-byte chunk, and add them to structural_indexes. - * - * @param partial Setting the partial parameter to true allows the find_structural_bits to - * tolerate unclosed strings. The caller should still ensure that the input is valid UTF-8. If - * you are processing substrings, you may want to call on a function like trimmed_length_safe_utf8. - */ - template - static error_code index(const uint8_t* buf, size_t len, dom_parser_implementation& parser, stage1_mode partial) noexcept; - - private: - simdjson_inline json_structural_indexer(uint32_t* structural_indexes); - template - simdjson_inline void step(const uint8_t* block, buf_block_reader& reader) noexcept; - simdjson_inline void next(const simd::simd8x64& in, const json_block& block, size_t idx); - simdjson_inline error_code finish(dom_parser_implementation& parser, size_t idx, size_t len, stage1_mode partial); - - json_scanner scanner{}; - utf8_checker checker{}; - bit_indexer indexer; - uint64_t prev_structurals = 0; - uint64_t unescaped_chars_error = 0; - }; - - simdjson_inline json_structural_indexer::json_structural_indexer(uint32_t* structural_indexes) : indexer{ structural_indexes } {} - - // Skip the last character if it is partial - simdjson_inline size_t trim_partial_utf8(const uint8_t* buf, size_t len) { - if (simdjson_unlikely(len < 3)) { - switch (len) { - case 2: - if (buf[len - 1] >= 0xc0) { return len - 1; } // 2-, 3- and 4-byte characters with only 1 byte left - if (buf[len - 2] >= 0xe0) { return len - 2; } // 3- and 4-byte characters with only 2 bytes left - return len; - case 1: - if (buf[len - 1] >= 0xc0) { return len - 1; } // 2-, 3- and 4-byte characters with only 1 byte left - return len; - case 0: - return len; - } - } - if (buf[len - 1] >= 0xc0) { return len - 1; } // 2-, 3- and 4-byte characters with only 1 byte left - if (buf[len - 2] >= 0xe0) { return len - 2; } // 3- and 4-byte characters with only 1 byte left - if (buf[len - 3] >= 0xf0) { return len - 3; } // 4-byte characters with only 3 bytes left - return len; - } - - // - // PERF NOTES: - // We pipe 2 inputs through these stages: - // 1. Load JSON into registers. This takes a long time and is highly parallelizable, so we load - // 2 inputs' worth at once so that by the time step 2 is looking for them input, it's available. - // 2. Scan the JSON for critical data: strings, scalars and operators. This is the critical path. - // The output of step 1 depends entirely on this information. These functions don't quite use - // up enough CPU: the second half of the functions is highly serial, only using 1 execution core - // at a time. The second input's scans has some dependency on the first ones finishing it, but - // they can make a lot of progress before they need that information. - // 3. Step 1 doesn't use enough capacity, so we run some extra stuff while we're waiting for that - // to finish: utf-8 checks and generating the output from the last iteration. - // - // The reason we run 2 inputs at a time, is steps 2 and 3 are *still* not enough to soak up all - // available capacity with just one input. Running 2 at a time seems to give the CPU a good enough - // workout. - // - template - error_code json_structural_indexer::index(const uint8_t* buf, size_t len, dom_parser_implementation& parser, stage1_mode partial) noexcept { - if (simdjson_unlikely(len > parser.capacity())) { return CAPACITY; } - // We guard the rest of the code so that we can assume that len > 0 throughout. - if (len == 0) { return EMPTY; } - if (is_streaming(partial)) { - len = trim_partial_utf8(buf, len); - // If you end up with an empty window after trimming - // the partial UTF-8 bytes, then chances are good that you - // have an UTF-8 formatting error. - if (len == 0) { return UTF8_ERROR; } - } - buf_block_reader reader(buf, len); - json_structural_indexer indexer(parser.structural_indexes.get()); - - // Read all but the last block - while (reader.has_full_block()) { - indexer.step(reader.full_block(), reader); - } - // Take care of the last block (will always be there unless file is empty which is - // not supposed to happen.) - uint8_t block[STEP_SIZE]; - if (simdjson_unlikely(reader.get_remainder(block) == 0)) { return UNEXPECTED_ERROR; } - indexer.step(block, reader); - return indexer.finish(parser, reader.block_index(), len, partial); - } - - template<> - simdjson_inline void json_structural_indexer::step<128>(const uint8_t* block, buf_block_reader<128>& reader) noexcept { - simd::simd8x64 in_1(block); - simd::simd8x64 in_2(block + 64); - json_block block_1 = scanner.next(in_1); - json_block block_2 = scanner.next(in_2); - this->next(in_1, block_1, reader.block_index()); - this->next(in_2, block_2, reader.block_index() + 64); - reader.advance(); - } - - template<> - simdjson_inline void json_structural_indexer::step<64>(const uint8_t* block, buf_block_reader<64>& reader) noexcept { - simd::simd8x64 in_1(block); - json_block block_1 = scanner.next(in_1); - this->next(in_1, block_1, reader.block_index()); - reader.advance(); - } - - simdjson_inline void json_structural_indexer::next(const simd::simd8x64& in, const json_block& block, size_t idx) { - uint64_t unescaped = in.lteq(0x1F); -#if SIMDJSON_UTF8VALIDATION - checker.check_next_input(in); -#endif - indexer.write(uint32_t(idx - 64), prev_structurals); // Output *last* iteration's structurals to the parser - prev_structurals = block.structural_start(); - unescaped_chars_error |= block.non_quote_inside_string(unescaped); - } - - simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementation& parser, size_t idx, size_t len, stage1_mode partial) { - // Write out the final iteration's structurals - indexer.write(uint32_t(idx - 64), prev_structurals); - error_code error = scanner.finish(); - // We deliberately break down the next expression so that it is - // human readable. - const bool should_we_exit = is_streaming(partial) ? - ((error != SUCCESS) && (error != UNCLOSED_STRING)) // when partial we tolerate UNCLOSED_STRING - : (error != SUCCESS); // if partial is false, we must have SUCCESS - const bool have_unclosed_string = (error == UNCLOSED_STRING); - if (simdjson_unlikely(should_we_exit)) { return error; } - - if (unescaped_chars_error) { - return UNESCAPED_CHARS; - } - parser.n_structural_indexes = uint32_t(indexer.tail - parser.structural_indexes.get()); - /*** - * The On Demand API requires special padding. - * - * This is related to https://github.com/simdjson/simdjson/issues/906 - * Basically, we want to make sure that if the parsing continues beyond the last (valid) - * structural character, it quickly stops. - * Only three structural characters can be repeated without triggering an error in JSON: [,] and }. - * We repeat the padding character (at 'len'). We don't know what it is, but if the parsing - * continues, then it must be [,] or }. - * Suppose it is ] or }. We backtrack to the first character, what could it be that would - * not trigger an error? It could be ] or } but no, because you can't start a document that way. - * It can't be a comma, a colon or any simple value. So the only way we could continue is - * if the repeated character is [. But if so, the document must start with [. But if the document - * starts with [, it should end with ]. If we enforce that rule, then we would get - * ][[ which is invalid. - * - * This is illustrated with the test array_iterate_unclosed_error() on the following input: - * R"({ "a": [,,)" - **/ - parser.structural_indexes[parser.n_structural_indexes] = uint32_t(len); // used later in partial == stage1_mode::streaming_final - parser.structural_indexes[parser.n_structural_indexes + 1] = uint32_t(len); - parser.structural_indexes[parser.n_structural_indexes + 2] = 0; - parser.next_structural_index = 0; - // a valid JSON file cannot have zero structural indexes - we should have found something - if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { - return EMPTY; - } - if (simdjson_unlikely(parser.structural_indexes[parser.n_structural_indexes - 1] > len)) { - return UNEXPECTED_ERROR; - } - if (partial == stage1_mode::streaming_partial) { - // If we have an unclosed string, then the last structural - // will be the quote and we want to make sure to omit it. - if (have_unclosed_string) { - parser.n_structural_indexes--; - // a valid JSON file cannot have zero structural indexes - we should have found something - if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return CAPACITY; } - } - // We truncate the input to the end of the last complete document (or zero). - auto new_structural_indexes = find_next_document_index(parser); - if (new_structural_indexes == 0 && parser.n_structural_indexes > 0) { - if (parser.structural_indexes[0] == 0) { - // If the buffer is partial and we started at index 0 but the document is - // incomplete, it's too big to parse. - return CAPACITY; - } - else { - // It is possible that the document could be parsed, we just had a lot - // of white space. - parser.n_structural_indexes = 0; - return EMPTY; - } - } - - parser.n_structural_indexes = new_structural_indexes; - } - else if (partial == stage1_mode::streaming_final) { - if (have_unclosed_string) { parser.n_structural_indexes--; } - // We truncate the input to the end of the last complete document (or zero). - // Because partial == stage1_mode::streaming_final, it means that we may - // silently ignore trailing garbage. Though it sounds bad, we do it - // deliberately because many people who have streams of JSON documents - // will truncate them for processing. E.g., imagine that you are uncompressing - // the data from a size file or receiving it in chunks from the network. You - // may not know where exactly the last document will be. Meanwhile the - // document_stream instances allow people to know the JSON documents they are - // parsing (see the iterator.source() method). - parser.n_structural_indexes = find_next_document_index(parser); - // We store the initial n_structural_indexes so that the client can see - // whether we used truncation. If initial_n_structural_indexes == parser.n_structural_indexes, - // then this will query parser.structural_indexes[parser.n_structural_indexes] which is len, - // otherwise, it will copy some prior index. - parser.structural_indexes[parser.n_structural_indexes + 1] = parser.structural_indexes[parser.n_structural_indexes]; - // This next line is critical, do not change it unless you understand what you are - // doing. - parser.structural_indexes[parser.n_structural_indexes] = uint32_t(len); - if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { - // We tolerate an unclosed string at the very end of the stream. Indeed, users - // often load their data in bulk without being careful and they want us to ignore - // the trailing garbage. - return EMPTY; - } - } - checker.check_eof(); - return checker.errors(); - } - - } // namespace stage1 - } // unnamed namespace - } // namespace ppc64 -} // namespace simdjson - -// Clear CUSTOM_BIT_INDEXER so other implementations can set it if they need to. -#undef SIMDJSON_GENERIC_JSON_STRUCTURAL_INDEXER_CUSTOM_BIT_INDEXER - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRUCTURAL_INDEXER_H -/* end file generic/stage1/json_structural_indexer.h for ppc64 */ -/* including generic/stage1/utf8_validator.h for ppc64: #include */ -/* begin file generic/stage1/utf8_validator.h for ppc64 */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_UTF8_VALIDATOR_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_UTF8_VALIDATOR_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - namespace ppc64 { - namespace { - namespace stage1 { - - /** - * Validates that the string is actual UTF-8. - */ - template - bool generic_validate_utf8(const uint8_t* input, size_t length) { - checker c{}; - buf_block_reader<64> reader(input, length); - while (reader.has_full_block()) { - simd::simd8x64 in(reader.full_block()); - c.check_next_input(in); - reader.advance(); - } - uint8_t block[64]{}; - reader.get_remainder(block); - simd::simd8x64 in(block); - c.check_next_input(in); - reader.advance(); - c.check_eof(); - return c.errors() == error_code::SUCCESS; - } - - bool generic_validate_utf8(const char* input, size_t length) { - return generic_validate_utf8(reinterpret_cast(input), length); - } - - } // namespace stage1 - } // unnamed namespace - } // namespace ppc64 -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_UTF8_VALIDATOR_H -/* end file generic/stage1/utf8_validator.h for ppc64 */ -/* end file generic/stage1/amalgamated.h for ppc64 */ -/* including generic/stage2/amalgamated.h for ppc64: #include */ -/* begin file generic/stage2/amalgamated.h for ppc64 */ -// Stuff other things depend on -/* including generic/stage2/base.h for ppc64: #include */ -/* begin file generic/stage2/base.h for ppc64 */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_BASE_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - namespace ppc64 { - namespace { - namespace stage2 { - - class json_iterator; - class structural_iterator; - struct tape_builder; - struct tape_writer; - - } // namespace stage2 - } // unnamed namespace - } // namespace ppc64 -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE2_BASE_H -/* end file generic/stage2/base.h for ppc64 */ -/* including generic/stage2/tape_writer.h for ppc64: #include */ -/* begin file generic/stage2/tape_writer.h for ppc64 */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_TAPE_WRITER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_TAPE_WRITER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include - -namespace simdjson { - namespace ppc64 { - namespace { - namespace stage2 { - - struct tape_writer { - /** The next place to write to tape */ - uint64_t* next_tape_loc; - - /** Write a signed 64-bit value to tape. */ - simdjson_inline void append_s64(int64_t value) noexcept; - - /** Write an unsigned 64-bit value to tape. */ - simdjson_inline void append_u64(uint64_t value) noexcept; - - /** Write a double value to tape. */ - simdjson_inline void append_double(double value) noexcept; - - /** - * Append a tape entry (an 8-bit type,and 56 bits worth of value). - */ - simdjson_inline void append(uint64_t val, internal::tape_type t) noexcept; - - /** - * Skip the current tape entry without writing. - * - * Used to skip the start of the container, since we'll come back later to fill it in when the - * container ends. - */ - simdjson_inline void skip() noexcept; - - /** - * Skip the number of tape entries necessary to write a large u64 or i64. - */ - simdjson_inline void skip_large_integer() noexcept; - - /** - * Skip the number of tape entries necessary to write a double. - */ - simdjson_inline void skip_double() noexcept; - - /** - * Write a value to a known location on tape. - * - * Used to go back and write out the start of a container after the container ends. - */ - simdjson_inline static void write(uint64_t& tape_loc, uint64_t val, internal::tape_type t) noexcept; - - private: - /** - * Append both the tape entry, and a supplementary value following it. Used for types that need - * all 64 bits, such as double and uint64_t. - */ - template - simdjson_inline void append2(uint64_t val, T val2, internal::tape_type t) noexcept; - }; // struct tape_writer - - simdjson_inline void tape_writer::append_s64(int64_t value) noexcept { - append2(0, value, internal::tape_type::INT64); - } - - simdjson_inline void tape_writer::append_u64(uint64_t value) noexcept { - append(0, internal::tape_type::UINT64); - *next_tape_loc = value; - next_tape_loc++; - } - - /** Write a double value to tape. */ - simdjson_inline void tape_writer::append_double(double value) noexcept { - append2(0, value, internal::tape_type::DOUBLE); - } - - simdjson_inline void tape_writer::skip() noexcept { - next_tape_loc++; - } - - simdjson_inline void tape_writer::skip_large_integer() noexcept { - next_tape_loc += 2; - } - - simdjson_inline void tape_writer::skip_double() noexcept { - next_tape_loc += 2; - } - - simdjson_inline void tape_writer::append(uint64_t val, internal::tape_type t) noexcept { - *next_tape_loc = val | ((uint64_t(char(t))) << 56); - next_tape_loc++; - } - - template - simdjson_inline void tape_writer::append2(uint64_t val, T val2, internal::tape_type t) noexcept { - append(val, t); - static_assert(sizeof(val2) == sizeof(*next_tape_loc), "Type is not 64 bits!"); - memcpy(next_tape_loc, &val2, sizeof(val2)); - next_tape_loc++; - } - - simdjson_inline void tape_writer::write(uint64_t& tape_loc, uint64_t val, internal::tape_type t) noexcept { - tape_loc = val | ((uint64_t(char(t))) << 56); - } - - } // namespace stage2 - } // unnamed namespace - } // namespace ppc64 -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE2_TAPE_WRITER_H -/* end file generic/stage2/tape_writer.h for ppc64 */ -/* including generic/stage2/logger.h for ppc64: #include */ -/* begin file generic/stage2/logger.h for ppc64 */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_LOGGER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_LOGGER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include - - -// This is for an internal-only stage 2 specific logger. -// Set LOG_ENABLED = true to log what stage 2 is doing! -namespace simdjson { - namespace ppc64 { - namespace { - namespace logger { - - static constexpr const char* DASHES = "----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------"; - -#if SIMDJSON_VERBOSE_LOGGING - static constexpr const bool LOG_ENABLED = true; -#else - static constexpr const bool LOG_ENABLED = false; -#endif - static constexpr const int LOG_EVENT_LEN = 20; - static constexpr const int LOG_BUFFER_LEN = 30; - static constexpr const int LOG_SMALL_BUFFER_LEN = 10; - static constexpr const int LOG_INDEX_LEN = 5; - - static int log_depth; // Not threadsafe. Log only. - - // Helper to turn unprintable or newline characters into spaces - static simdjson_inline char printable_char(char c) { - if (c >= 0x20) { - return c; - } - else { - return ' '; - } - } - - // Print the header and set up log_start - static simdjson_inline void log_start() { - if (LOG_ENABLED) { - log_depth = 0; - printf("\n"); - printf("| %-*s | %-*s | %-*s | %-*s | Detail |\n", LOG_EVENT_LEN, "Event", LOG_BUFFER_LEN, "Buffer", LOG_SMALL_BUFFER_LEN, "Next", 5, "Next#"); - printf("|%.*s|%.*s|%.*s|%.*s|--------|\n", LOG_EVENT_LEN + 2, DASHES, LOG_BUFFER_LEN + 2, DASHES, LOG_SMALL_BUFFER_LEN + 2, DASHES, 5 + 2, DASHES); - } - } - - simdjson_unused static simdjson_inline void log_string(const char* message) { - if (LOG_ENABLED) { - printf("%s\n", message); - } - } - - // Logs a single line from the stage 2 DOM parser - template - static simdjson_inline void log_line(S& structurals, const char* title_prefix, const char* title, const char* detail) { - if (LOG_ENABLED) { - printf("| %*s%s%-*s ", log_depth * 2, "", title_prefix, LOG_EVENT_LEN - log_depth * 2 - int(strlen(title_prefix)), title); - auto current_index = structurals.at_beginning() ? nullptr : structurals.next_structural - 1; - auto next_index = structurals.next_structural; - auto current = current_index ? &structurals.buf[*current_index] : reinterpret_cast(" "); - auto next = &structurals.buf[*next_index]; - { - // Print the next N characters in the buffer. - printf("| "); - // Otherwise, print the characters starting from the buffer position. - // Print spaces for unprintable or newline characters. - for (int i = 0; i < LOG_BUFFER_LEN; i++) { - printf("%c", printable_char(current[i])); - } - printf(" "); - // Print the next N characters in the buffer. - printf("| "); - // Otherwise, print the characters starting from the buffer position. - // Print spaces for unprintable or newline characters. - for (int i = 0; i < LOG_SMALL_BUFFER_LEN; i++) { - printf("%c", printable_char(next[i])); - } - printf(" "); - } - if (current_index) { - printf("| %*u ", LOG_INDEX_LEN, *current_index); - } - else { - printf("| %-*s ", LOG_INDEX_LEN, ""); - } - // printf("| %*u ", LOG_INDEX_LEN, structurals.next_tape_index()); - printf("| %-s ", detail); - printf("|\n"); - } - } - - } // namespace logger - } // unnamed namespace - } // namespace ppc64 -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE2_LOGGER_H -/* end file generic/stage2/logger.h for ppc64 */ - -// All other declarations -/* including generic/stage2/json_iterator.h for ppc64: #include */ -/* begin file generic/stage2/json_iterator.h for ppc64 */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_JSON_ITERATOR_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_JSON_ITERATOR_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - namespace ppc64 { - namespace { - namespace stage2 { - - class json_iterator { - public: - const uint8_t* const buf; - uint32_t* next_structural; - dom_parser_implementation& dom_parser; - uint32_t depth{ 0 }; - - /** - * Walk the JSON document. - * - * The visitor receives callbacks when values are encountered. All callbacks pass the iterator as - * the first parameter; some callbacks have other parameters as well: - * - * - visit_document_start() - at the beginning. - * - visit_document_end() - at the end (if things were successful). - * - * - visit_array_start() - at the start `[` of a non-empty array. - * - visit_array_end() - at the end `]` of a non-empty array. - * - visit_empty_array() - when an empty array is encountered. - * - * - visit_object_end() - at the start `]` of a non-empty object. - * - visit_object_start() - at the end `]` of a non-empty object. - * - visit_empty_object() - when an empty object is encountered. - * - visit_key(const uint8_t *key) - when a key in an object field is encountered. key is - * guaranteed to point at the first quote of the string (`"key"`). - * - visit_primitive(const uint8_t *value) - when a value is a string, number, boolean or null. - * - visit_root_primitive(iter, uint8_t *value) - when the top-level value is a string, number, boolean or null. - * - * - increment_count(iter) - each time a value is found in an array or object. - */ - template - simdjson_warn_unused simdjson_inline error_code walk_document(V& visitor) noexcept; - - /** - * Create an iterator capable of walking a JSON document. - * - * The document must have already passed through stage 1. - */ - simdjson_inline json_iterator(dom_parser_implementation& _dom_parser, size_t start_structural_index); - - /** - * Look at the next token. - * - * Tokens can be strings, numbers, booleans, null, or operators (`[{]},:`)). - * - * They may include invalid JSON as well (such as `1.2.3` or `ture`). - */ - simdjson_inline const uint8_t* peek() const noexcept; - /** - * Advance to the next token. - * - * Tokens can be strings, numbers, booleans, null, or operators (`[{]},:`)). - * - * They may include invalid JSON as well (such as `1.2.3` or `ture`). - */ - simdjson_inline const uint8_t* advance() noexcept; - /** - * Get the remaining length of the document, from the start of the current token. - */ - simdjson_inline size_t remaining_len() const noexcept; - /** - * Check if we are at the end of the document. - * - * If this is true, there are no more tokens. - */ - simdjson_inline bool at_eof() const noexcept; - /** - * Check if we are at the beginning of the document. - */ - simdjson_inline bool at_beginning() const noexcept; - simdjson_inline uint8_t last_structural() const noexcept; - - /** - * Log that a value has been found. - * - * Set LOG_ENABLED=true in logger.h to see logging. - */ - simdjson_inline void log_value(const char* type) const noexcept; - /** - * Log the start of a multipart value. - * - * Set LOG_ENABLED=true in logger.h to see logging. - */ - simdjson_inline void log_start_value(const char* type) const noexcept; - /** - * Log the end of a multipart value. - * - * Set LOG_ENABLED=true in logger.h to see logging. - */ - simdjson_inline void log_end_value(const char* type) const noexcept; - /** - * Log an error. - * - * Set LOG_ENABLED=true in logger.h to see logging. - */ - simdjson_inline void log_error(const char* error) const noexcept; - - template - simdjson_warn_unused simdjson_inline error_code visit_root_primitive(V& visitor, const uint8_t* value) noexcept; - template - simdjson_warn_unused simdjson_inline error_code visit_primitive(V& visitor, const uint8_t* value) noexcept; - }; - - template - simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V& visitor) noexcept { - logger::log_start(); - - // - // Start the document - // - if (at_eof()) { return EMPTY; } - log_start_value("document"); - SIMDJSON_TRY(visitor.visit_document_start(*this)); - - // - // Read first value - // - { - auto value = advance(); - - // Make sure the outer object or array is closed before continuing; otherwise, there are ways we - // could get into memory corruption. See https://github.com/simdjson/simdjson/issues/906 - if (!STREAMING) { - switch (*value) { - case '{': if (last_structural() != '}') { log_value("starting brace unmatched"); return TAPE_ERROR; }; break; - case '[': if (last_structural() != ']') { log_value("starting bracket unmatched"); return TAPE_ERROR; }; break; - } - } - - switch (*value) { - case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY(visitor.visit_empty_object(*this)); break; } goto object_begin; - case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY(visitor.visit_empty_array(*this)); break; } goto array_begin; - default: SIMDJSON_TRY(visitor.visit_root_primitive(*this, value)); break; - } - } - goto document_end; - - // - // Object parser states - // - object_begin: - log_start_value("object"); - depth++; - if (depth >= dom_parser.max_depth()) { log_error("Exceeded max depth!"); return DEPTH_ERROR; } - dom_parser.is_array[depth] = false; - SIMDJSON_TRY(visitor.visit_object_start(*this)); - - { - auto key = advance(); - if (*key != '"') { log_error("Object does not start with a key"); return TAPE_ERROR; } - SIMDJSON_TRY(visitor.increment_count(*this)); - SIMDJSON_TRY(visitor.visit_key(*this, key)); - } - - object_field: - if (simdjson_unlikely(*advance() != ':')) { log_error("Missing colon after key in object"); return TAPE_ERROR; } - { - auto value = advance(); - switch (*value) { - case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY(visitor.visit_empty_object(*this)); break; } goto object_begin; - case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY(visitor.visit_empty_array(*this)); break; } goto array_begin; - default: SIMDJSON_TRY(visitor.visit_primitive(*this, value)); break; - } - } - - object_continue: - switch (*advance()) { - case ',': - SIMDJSON_TRY(visitor.increment_count(*this)); - { - auto key = advance(); - if (simdjson_unlikely(*key != '"')) { log_error("Key string missing at beginning of field in object"); return TAPE_ERROR; } - SIMDJSON_TRY(visitor.visit_key(*this, key)); - } - goto object_field; - case '}': log_end_value("object"); SIMDJSON_TRY(visitor.visit_object_end(*this)); goto scope_end; - default: log_error("No comma between object fields"); return TAPE_ERROR; - } - - scope_end: - depth--; - if (depth == 0) { goto document_end; } - if (dom_parser.is_array[depth]) { goto array_continue; } - goto object_continue; - - // - // Array parser states - // - array_begin: - log_start_value("array"); - depth++; - if (depth >= dom_parser.max_depth()) { log_error("Exceeded max depth!"); return DEPTH_ERROR; } - dom_parser.is_array[depth] = true; - SIMDJSON_TRY(visitor.visit_array_start(*this)); - SIMDJSON_TRY(visitor.increment_count(*this)); - - array_value: - { - auto value = advance(); - switch (*value) { - case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY(visitor.visit_empty_object(*this)); break; } goto object_begin; - case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY(visitor.visit_empty_array(*this)); break; } goto array_begin; - default: SIMDJSON_TRY(visitor.visit_primitive(*this, value)); break; - } - } - - array_continue: - switch (*advance()) { - case ',': SIMDJSON_TRY(visitor.increment_count(*this)); goto array_value; - case ']': log_end_value("array"); SIMDJSON_TRY(visitor.visit_array_end(*this)); goto scope_end; - default: log_error("Missing comma between array values"); return TAPE_ERROR; - } - - document_end: - log_end_value("document"); - SIMDJSON_TRY(visitor.visit_document_end(*this)); - - dom_parser.next_structural_index = uint32_t(next_structural - &dom_parser.structural_indexes[0]); - - // If we didn't make it to the end, it's an error - if (!STREAMING && dom_parser.next_structural_index != dom_parser.n_structural_indexes) { - log_error("More than one JSON value at the root of the document, or extra characters at the end of the JSON!"); - return TAPE_ERROR; - } - - return SUCCESS; - - } // walk_document() - - simdjson_inline json_iterator::json_iterator(dom_parser_implementation& _dom_parser, size_t start_structural_index) - : buf{ _dom_parser.buf }, - next_structural{ &_dom_parser.structural_indexes[start_structural_index] }, - dom_parser{ _dom_parser } { - } - - simdjson_inline const uint8_t* json_iterator::peek() const noexcept { - return &buf[*(next_structural)]; - } - simdjson_inline const uint8_t* json_iterator::advance() noexcept { - return &buf[*(next_structural++)]; - } - simdjson_inline size_t json_iterator::remaining_len() const noexcept { - return dom_parser.len - *(next_structural - 1); - } - - simdjson_inline bool json_iterator::at_eof() const noexcept { - return next_structural == &dom_parser.structural_indexes[dom_parser.n_structural_indexes]; - } - simdjson_inline bool json_iterator::at_beginning() const noexcept { - return next_structural == dom_parser.structural_indexes.get(); - } - simdjson_inline uint8_t json_iterator::last_structural() const noexcept { - return buf[dom_parser.structural_indexes[dom_parser.n_structural_indexes - 1]]; - } - - simdjson_inline void json_iterator::log_value(const char* type) const noexcept { - logger::log_line(*this, "", type, ""); - } - - simdjson_inline void json_iterator::log_start_value(const char* type) const noexcept { - logger::log_line(*this, "+", type, ""); - if (logger::LOG_ENABLED) { logger::log_depth++; } - } - - simdjson_inline void json_iterator::log_end_value(const char* type) const noexcept { - if (logger::LOG_ENABLED) { logger::log_depth--; } - logger::log_line(*this, "-", type, ""); - } - - simdjson_inline void json_iterator::log_error(const char* error) const noexcept { - logger::log_line(*this, "", "ERROR", error); - } - - template - simdjson_warn_unused simdjson_inline error_code json_iterator::visit_root_primitive(V& visitor, const uint8_t* value) noexcept { - switch (*value) { - case '"': return visitor.visit_root_string(*this, value); - case 't': return visitor.visit_root_true_atom(*this, value); - case 'f': return visitor.visit_root_false_atom(*this, value); - case 'n': return visitor.visit_root_null_atom(*this, value); - case '-': - case '0': case '1': case '2': case '3': case '4': - case '5': case '6': case '7': case '8': case '9': - return visitor.visit_root_number(*this, value); - default: - log_error("Document starts with a non-value character"); - return TAPE_ERROR; - } - } - template - simdjson_warn_unused simdjson_inline error_code json_iterator::visit_primitive(V& visitor, const uint8_t* value) noexcept { - // Use the fact that most scalars are going to be either strings or numbers. - if (*value == '"') { - return visitor.visit_string(*this, value); - } - else if (((*value - '0') < 10) || (*value == '-')) { - return visitor.visit_number(*this, value); - } - // true, false, null are uncommon. - switch (*value) { - case 't': return visitor.visit_true_atom(*this, value); - case 'f': return visitor.visit_false_atom(*this, value); - case 'n': return visitor.visit_null_atom(*this, value); - default: - log_error("Non-value found when value was expected!"); - return TAPE_ERROR; - } - } - - } // namespace stage2 - } // unnamed namespace - } // namespace ppc64 -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE2_JSON_ITERATOR_H -/* end file generic/stage2/json_iterator.h for ppc64 */ -/* including generic/stage2/stringparsing.h for ppc64: #include */ -/* begin file generic/stage2/stringparsing.h for ppc64 */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -// This file contains the common code every implementation uses -// It is intended to be included multiple times and compiled multiple times - -namespace simdjson { - namespace ppc64 { - namespace { - /// @private - namespace stringparsing { - - // begin copypasta - // These chars yield themselves: " \ / - // b -> backspace, f -> formfeed, n -> newline, r -> cr, t -> horizontal tab - // u not handled in this table as it's complex - static const uint8_t escape_map[256] = { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x0. - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0x22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x2f, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x4. - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x5c, 0, 0, 0, // 0x5. - 0, 0, 0x08, 0, 0, 0, 0x0c, 0, 0, 0, 0, 0, 0, 0, 0x0a, 0, // 0x6. - 0, 0, 0x0d, 0, 0x09, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x7. - - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - }; - - // handle a unicode codepoint - // write appropriate values into dest - // src will advance 6 bytes or 12 bytes - // dest will advance a variable amount (return via pointer) - // return true if the unicode codepoint was valid - // We work in little-endian then swap at write time - simdjson_warn_unused - simdjson_inline bool handle_unicode_codepoint(const uint8_t** src_ptr, - uint8_t** dst_ptr, bool allow_replacement) { - // Use the default Unicode Character 'REPLACEMENT CHARACTER' (U+FFFD) - constexpr uint32_t substitution_code_point = 0xfffd; - // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the - // conversion isn't valid; we defer the check for this to inside the - // multilingual plane check - uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2); - *src_ptr += 6; - - // If we found a high surrogate, we must - // check for low surrogate for characters - // outside the Basic - // Multilingual Plane. - if (code_point >= 0xd800 && code_point < 0xdc00) { - const uint8_t* src_data = *src_ptr; - /* Compiler optimizations convert this to a single 16-bit load and compare on most platforms */ - if (((src_data[0] << 8) | src_data[1]) != ((static_cast ('\\') << 8) | static_cast ('u'))) { - if (!allow_replacement) { return false; } - code_point = substitution_code_point; - } - else { - uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(src_data + 2); - - // We have already checked that the high surrogate is valid and - // (code_point - 0xd800) < 1024. - // - // Check that code_point_2 is in the range 0xdc00..0xdfff - // and that code_point_2 was parsed from valid hex. - uint32_t low_bit = code_point_2 - 0xdc00; - if (low_bit >> 10) { - if (!allow_replacement) { return false; } - code_point = substitution_code_point; - } - else { - code_point = (((code_point - 0xd800) << 10) | low_bit) + 0x10000; - *src_ptr += 6; - } - - } - } - else if (code_point >= 0xdc00 && code_point <= 0xdfff) { - // If we encounter a low surrogate (not preceded by a high surrogate) - // then we have an error. - if (!allow_replacement) { return false; } - code_point = substitution_code_point; - } - size_t offset = jsoncharutils::codepoint_to_utf8(code_point, *dst_ptr); - *dst_ptr += offset; - return offset > 0; - } - - - // handle a unicode codepoint using the wobbly convention - // https://simonsapin.github.io/wtf-8/ - // write appropriate values into dest - // src will advance 6 bytes or 12 bytes - // dest will advance a variable amount (return via pointer) - // return true if the unicode codepoint was valid - // We work in little-endian then swap at write time - simdjson_warn_unused - simdjson_inline bool handle_unicode_codepoint_wobbly(const uint8_t** src_ptr, - uint8_t** dst_ptr) { - // It is not ideal that this function is nearly identical to handle_unicode_codepoint. - // - // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the - // conversion isn't valid; we defer the check for this to inside the - // multilingual plane check - uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2); - *src_ptr += 6; - // If we found a high surrogate, we must - // check for low surrogate for characters - // outside the Basic - // Multilingual Plane. - if (code_point >= 0xd800 && code_point < 0xdc00) { - const uint8_t* src_data = *src_ptr; - /* Compiler optimizations convert this to a single 16-bit load and compare on most platforms */ - if (((src_data[0] << 8) | src_data[1]) == ((static_cast ('\\') << 8) | static_cast ('u'))) { - uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(src_data + 2); - uint32_t low_bit = code_point_2 - 0xdc00; - if ((low_bit >> 10) == 0) { - code_point = - (((code_point - 0xd800) << 10) | low_bit) + 0x10000; - *src_ptr += 6; - } - } - } - - size_t offset = jsoncharutils::codepoint_to_utf8(code_point, *dst_ptr); - *dst_ptr += offset; - return offset > 0; - } - - - /** - * Unescape a valid UTF-8 string from src to dst, stopping at a final unescaped quote. There - * must be an unescaped quote terminating the string. It returns the final output - * position as pointer. In case of error (e.g., the string has bad escaped codes), - * then null_nullptrptr is returned. It is assumed that the output buffer is large - * enough. E.g., if src points at 'joe"', then dst needs to have four free bytes + - * SIMDJSON_PADDING bytes. - */ - simdjson_warn_unused simdjson_inline uint8_t* parse_string(const uint8_t* src, uint8_t* dst, bool allow_replacement) { - while (1) { - // Copy the next n bytes, and find the backslash and quote in them. - auto bs_quote = backslash_and_quote::copy_and_find(src, dst); - // If the next thing is the end quote, copy and return - if (bs_quote.has_quote_first()) { - // we encountered quotes first. Move dst to point to quotes and exit - return dst + bs_quote.quote_index(); - } - if (bs_quote.has_backslash()) { - /* find out where the backspace is */ - auto bs_dist = bs_quote.backslash_index(); - uint8_t escape_char = src[bs_dist + 1]; - /* we encountered backslash first. Handle backslash */ - if (escape_char == 'u') { - /* move src/dst up to the start; they will be further adjusted - within the unicode codepoint handling code. */ - src += bs_dist; - dst += bs_dist; - if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { - return nullptr; - } - } - else { - /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and - * write bs_dist+1 characters to output - * note this may reach beyond the part of the buffer we've actually - * seen. I think this is ok */ - uint8_t escape_result = escape_map[escape_char]; - if (escape_result == 0u) { - return nullptr; /* bogus escape value is an error */ - } - dst[bs_dist] = escape_result; - src += bs_dist + 2; - dst += bs_dist + 1; - } - } - else { - /* they are the same. Since they can't co-occur, it means we - * encountered neither. */ - src += backslash_and_quote::BYTES_PROCESSED; - dst += backslash_and_quote::BYTES_PROCESSED; - } - } - } - - simdjson_warn_unused simdjson_inline uint8_t* parse_wobbly_string(const uint8_t* src, uint8_t* dst) { - // It is not ideal that this function is nearly identical to parse_string. - while (1) { - // Copy the next n bytes, and find the backslash and quote in them. - auto bs_quote = backslash_and_quote::copy_and_find(src, dst); - // If the next thing is the end quote, copy and return - if (bs_quote.has_quote_first()) { - // we encountered quotes first. Move dst to point to quotes and exit - return dst + bs_quote.quote_index(); - } - if (bs_quote.has_backslash()) { - /* find out where the backspace is */ - auto bs_dist = bs_quote.backslash_index(); - uint8_t escape_char = src[bs_dist + 1]; - /* we encountered backslash first. Handle backslash */ - if (escape_char == 'u') { - /* move src/dst up to the start; they will be further adjusted - within the unicode codepoint handling code. */ - src += bs_dist; - dst += bs_dist; - if (!handle_unicode_codepoint_wobbly(&src, &dst)) { - return nullptr; - } - } - else { - /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and - * write bs_dist+1 characters to output - * note this may reach beyond the part of the buffer we've actually - * seen. I think this is ok */ - uint8_t escape_result = escape_map[escape_char]; - if (escape_result == 0u) { - return nullptr; /* bogus escape value is an error */ - } - dst[bs_dist] = escape_result; - src += bs_dist + 2; - dst += bs_dist + 1; - } - } - else { - /* they are the same. Since they can't co-occur, it means we - * encountered neither. */ - src += backslash_and_quote::BYTES_PROCESSED; - dst += backslash_and_quote::BYTES_PROCESSED; - } - } - } - - } // namespace stringparsing - } // unnamed namespace - } // namespace ppc64 -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H -/* end file generic/stage2/stringparsing.h for ppc64 */ -/* including generic/stage2/structural_iterator.h for ppc64: #include */ -/* begin file generic/stage2/structural_iterator.h for ppc64 */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_STRUCTURAL_ITERATOR_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_STRUCTURAL_ITERATOR_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - namespace ppc64 { - namespace { - namespace stage2 { - - class structural_iterator { - public: - const uint8_t* const buf; - uint32_t* next_structural; - dom_parser_implementation& dom_parser; - - // Start a structural - simdjson_inline structural_iterator(dom_parser_implementation& _dom_parser, size_t start_structural_index) - : buf{ _dom_parser.buf }, - next_structural{ &_dom_parser.structural_indexes[start_structural_index] }, - dom_parser{ _dom_parser } { - } - // Get the buffer position of the current structural character - simdjson_inline const uint8_t* current() { - return &buf[*(next_structural - 1)]; - } - // Get the current structural character - simdjson_inline char current_char() { - return buf[*(next_structural - 1)]; - } - // Get the next structural character without advancing - simdjson_inline char peek_next_char() { - return buf[*next_structural]; - } - simdjson_inline const uint8_t* peek() { - return &buf[*next_structural]; - } - simdjson_inline const uint8_t* advance() { - return &buf[*(next_structural++)]; - } - simdjson_inline char advance_char() { - return buf[*(next_structural++)]; - } - simdjson_inline size_t remaining_len() { - return dom_parser.len - *(next_structural - 1); - } - - simdjson_inline bool at_end() { - return next_structural == &dom_parser.structural_indexes[dom_parser.n_structural_indexes]; - } - simdjson_inline bool at_beginning() { - return next_structural == dom_parser.structural_indexes.get(); - } - }; - - } // namespace stage2 - } // unnamed namespace - } // namespace ppc64 -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE2_STRUCTURAL_ITERATOR_H -/* end file generic/stage2/structural_iterator.h for ppc64 */ -/* including generic/stage2/tape_builder.h for ppc64: #include */ -/* begin file generic/stage2/tape_builder.h for ppc64 */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_TAPE_BUILDER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_TAPE_BUILDER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - - -namespace simdjson { - namespace ppc64 { - namespace { - namespace stage2 { - - struct tape_builder { - template - simdjson_warn_unused static simdjson_inline error_code parse_document( - dom_parser_implementation& dom_parser, - dom::document& doc) noexcept; - - /** Called when a non-empty document starts. */ - simdjson_warn_unused simdjson_inline error_code visit_document_start(json_iterator& iter) noexcept; - /** Called when a non-empty document ends without error. */ - simdjson_warn_unused simdjson_inline error_code visit_document_end(json_iterator& iter) noexcept; - - /** Called when a non-empty array starts. */ - simdjson_warn_unused simdjson_inline error_code visit_array_start(json_iterator& iter) noexcept; - /** Called when a non-empty array ends. */ - simdjson_warn_unused simdjson_inline error_code visit_array_end(json_iterator& iter) noexcept; - /** Called when an empty array is found. */ - simdjson_warn_unused simdjson_inline error_code visit_empty_array(json_iterator& iter) noexcept; - - /** Called when a non-empty object starts. */ - simdjson_warn_unused simdjson_inline error_code visit_object_start(json_iterator& iter) noexcept; - /** - * Called when a key in a field is encountered. - * - * primitive, visit_object_start, visit_empty_object, visit_array_start, or visit_empty_array - * will be called after this with the field value. - */ - simdjson_warn_unused simdjson_inline error_code visit_key(json_iterator& iter, const uint8_t* key) noexcept; - /** Called when a non-empty object ends. */ - simdjson_warn_unused simdjson_inline error_code visit_object_end(json_iterator& iter) noexcept; - /** Called when an empty object is found. */ - simdjson_warn_unused simdjson_inline error_code visit_empty_object(json_iterator& iter) noexcept; - - /** - * Called when a string, number, boolean or null is found. - */ - simdjson_warn_unused simdjson_inline error_code visit_primitive(json_iterator& iter, const uint8_t* value) noexcept; - /** - * Called when a string, number, boolean or null is found at the top level of a document (i.e. - * when there is no array or object and the entire document is a single string, number, boolean or - * null. - * - * This is separate from primitive() because simdjson's normal primitive parsing routines assume - * there is at least one more token after the value, which is only true in an array or object. - */ - simdjson_warn_unused simdjson_inline error_code visit_root_primitive(json_iterator& iter, const uint8_t* value) noexcept; - - simdjson_warn_unused simdjson_inline error_code visit_string(json_iterator& iter, const uint8_t* value, bool key = false) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_number(json_iterator& iter, const uint8_t* value) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_true_atom(json_iterator& iter, const uint8_t* value) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_false_atom(json_iterator& iter, const uint8_t* value) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_null_atom(json_iterator& iter, const uint8_t* value) noexcept; - - simdjson_warn_unused simdjson_inline error_code visit_root_string(json_iterator& iter, const uint8_t* value) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_root_number(json_iterator& iter, const uint8_t* value) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_root_true_atom(json_iterator& iter, const uint8_t* value) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_root_false_atom(json_iterator& iter, const uint8_t* value) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_root_null_atom(json_iterator& iter, const uint8_t* value) noexcept; - - /** Called each time a new field or element in an array or object is found. */ - simdjson_warn_unused simdjson_inline error_code increment_count(json_iterator& iter) noexcept; - - /** Next location to write to tape */ - tape_writer tape; - private: - /** Next write location in the string buf for stage 2 parsing */ - uint8_t* current_string_buf_loc; - - simdjson_inline tape_builder(dom::document& doc) noexcept; - - simdjson_inline uint32_t next_tape_index(json_iterator& iter) const noexcept; - simdjson_inline void start_container(json_iterator& iter) noexcept; - simdjson_warn_unused simdjson_inline error_code end_container(json_iterator& iter, internal::tape_type start, internal::tape_type end) noexcept; - simdjson_warn_unused simdjson_inline error_code empty_container(json_iterator& iter, internal::tape_type start, internal::tape_type end) noexcept; - simdjson_inline uint8_t* on_start_string(json_iterator& iter) noexcept; - simdjson_inline void on_end_string(uint8_t* dst) noexcept; - }; // struct tape_builder - - template - simdjson_warn_unused simdjson_inline error_code tape_builder::parse_document( - dom_parser_implementation& dom_parser, - dom::document& doc) noexcept { - dom_parser.doc = &doc; - json_iterator iter(dom_parser, STREAMING ? dom_parser.next_structural_index : 0); - tape_builder builder(doc); - return iter.walk_document(builder); - } - - simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_primitive(json_iterator& iter, const uint8_t* value) noexcept { - return iter.visit_root_primitive(*this, value); - } - simdjson_warn_unused simdjson_inline error_code tape_builder::visit_primitive(json_iterator& iter, const uint8_t* value) noexcept { - return iter.visit_primitive(*this, value); - } - simdjson_warn_unused simdjson_inline error_code tape_builder::visit_empty_object(json_iterator& iter) noexcept { - return empty_container(iter, internal::tape_type::START_OBJECT, internal::tape_type::END_OBJECT); - } - simdjson_warn_unused simdjson_inline error_code tape_builder::visit_empty_array(json_iterator& iter) noexcept { - return empty_container(iter, internal::tape_type::START_ARRAY, internal::tape_type::END_ARRAY); - } - - simdjson_warn_unused simdjson_inline error_code tape_builder::visit_document_start(json_iterator& iter) noexcept { - start_container(iter); - return SUCCESS; - } - simdjson_warn_unused simdjson_inline error_code tape_builder::visit_object_start(json_iterator& iter) noexcept { - start_container(iter); - return SUCCESS; - } - simdjson_warn_unused simdjson_inline error_code tape_builder::visit_array_start(json_iterator& iter) noexcept { - start_container(iter); - return SUCCESS; - } - - simdjson_warn_unused simdjson_inline error_code tape_builder::visit_object_end(json_iterator& iter) noexcept { - return end_container(iter, internal::tape_type::START_OBJECT, internal::tape_type::END_OBJECT); - } - simdjson_warn_unused simdjson_inline error_code tape_builder::visit_array_end(json_iterator& iter) noexcept { - return end_container(iter, internal::tape_type::START_ARRAY, internal::tape_type::END_ARRAY); - } - simdjson_warn_unused simdjson_inline error_code tape_builder::visit_document_end(json_iterator& iter) noexcept { - constexpr uint32_t start_tape_index = 0; - tape.append(start_tape_index, internal::tape_type::ROOT); - tape_writer::write(iter.dom_parser.doc->tape[start_tape_index], next_tape_index(iter), internal::tape_type::ROOT); - return SUCCESS; - } - simdjson_warn_unused simdjson_inline error_code tape_builder::visit_key(json_iterator& iter, const uint8_t* key) noexcept { - return visit_string(iter, key, true); - } - - simdjson_warn_unused simdjson_inline error_code tape_builder::increment_count(json_iterator& iter) noexcept { - iter.dom_parser.open_containers[iter.depth].count++; // we have a key value pair in the object at parser.dom_parser.depth - 1 - return SUCCESS; - } - - simdjson_inline tape_builder::tape_builder(dom::document& doc) noexcept : tape{ doc.tape.get() }, current_string_buf_loc{ doc.string_buf.get() } {} - - simdjson_warn_unused simdjson_inline error_code tape_builder::visit_string(json_iterator& iter, const uint8_t* value, bool key) noexcept { - iter.log_value(key ? "key" : "string"); - uint8_t* dst = on_start_string(iter); - dst = stringparsing::parse_string(value + 1, dst, false); // We do not allow replacement when the escape characters are invalid. - if (dst == nullptr) { - iter.log_error("Invalid escape in string"); - return STRING_ERROR; - } - on_end_string(dst); - return SUCCESS; - } - - simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_string(json_iterator& iter, const uint8_t* value) noexcept { - return visit_string(iter, value); - } - - simdjson_warn_unused simdjson_inline error_code tape_builder::visit_number(json_iterator& iter, const uint8_t* value) noexcept { - iter.log_value("number"); - return numberparsing::parse_number(value, tape); - } - - simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_number(json_iterator& iter, const uint8_t* value) noexcept { - // - // We need to make a copy to make sure that the string is space terminated. - // This is not about padding the input, which should already padded up - // to len + SIMDJSON_PADDING. However, we have no control at this stage - // on how the padding was done. What if the input string was padded with nulls? - // It is quite common for an input string to have an extra null character (C string). - // We do not want to allow 9\0 (where \0 is the null character) inside a JSON - // document, but the string "9\0" by itself is fine. So we make a copy and - // pad the input with spaces when we know that there is just one input element. - // This copy is relatively expensive, but it will almost never be called in - // practice unless you are in the strange scenario where you have many JSON - // documents made of single atoms. - // - std::unique_ptrcopy(new (std::nothrow) uint8_t[iter.remaining_len() + SIMDJSON_PADDING]); - if (copy.get() == nullptr) { return MEMALLOC; } - std::memcpy(copy.get(), value, iter.remaining_len()); - std::memset(copy.get() + iter.remaining_len(), ' ', SIMDJSON_PADDING); - error_code error = visit_number(iter, copy.get()); - return error; - } - - simdjson_warn_unused simdjson_inline error_code tape_builder::visit_true_atom(json_iterator& iter, const uint8_t* value) noexcept { - iter.log_value("true"); - if (!atomparsing::is_valid_true_atom(value)) { return T_ATOM_ERROR; } - tape.append(0, internal::tape_type::TRUE_VALUE); - return SUCCESS; - } - - simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_true_atom(json_iterator& iter, const uint8_t* value) noexcept { - iter.log_value("true"); - if (!atomparsing::is_valid_true_atom(value, iter.remaining_len())) { return T_ATOM_ERROR; } - tape.append(0, internal::tape_type::TRUE_VALUE); - return SUCCESS; - } - - simdjson_warn_unused simdjson_inline error_code tape_builder::visit_false_atom(json_iterator& iter, const uint8_t* value) noexcept { - iter.log_value("false"); - if (!atomparsing::is_valid_false_atom(value)) { return F_ATOM_ERROR; } - tape.append(0, internal::tape_type::FALSE_VALUE); - return SUCCESS; - } - - simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_false_atom(json_iterator& iter, const uint8_t* value) noexcept { - iter.log_value("false"); - if (!atomparsing::is_valid_false_atom(value, iter.remaining_len())) { return F_ATOM_ERROR; } - tape.append(0, internal::tape_type::FALSE_VALUE); - return SUCCESS; - } - - simdjson_warn_unused simdjson_inline error_code tape_builder::visit_null_atom(json_iterator& iter, const uint8_t* value) noexcept { - iter.log_value("null"); - if (!atomparsing::is_valid_null_atom(value)) { return N_ATOM_ERROR; } - tape.append(0, internal::tape_type::NULL_VALUE); - return SUCCESS; - } - - simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_null_atom(json_iterator& iter, const uint8_t* value) noexcept { - iter.log_value("null"); - if (!atomparsing::is_valid_null_atom(value, iter.remaining_len())) { return N_ATOM_ERROR; } - tape.append(0, internal::tape_type::NULL_VALUE); - return SUCCESS; - } - - // private: - - simdjson_inline uint32_t tape_builder::next_tape_index(json_iterator& iter) const noexcept { - return uint32_t(tape.next_tape_loc - iter.dom_parser.doc->tape.get()); - } - - simdjson_warn_unused simdjson_inline error_code tape_builder::empty_container(json_iterator& iter, internal::tape_type start, internal::tape_type end) noexcept { - auto start_index = next_tape_index(iter); - tape.append(start_index + 2, start); - tape.append(start_index, end); - return SUCCESS; - } - - simdjson_inline void tape_builder::start_container(json_iterator& iter) noexcept { - iter.dom_parser.open_containers[iter.depth].tape_index = next_tape_index(iter); - iter.dom_parser.open_containers[iter.depth].count = 0; - tape.skip(); // We don't actually *write* the start element until the end. - } - - simdjson_warn_unused simdjson_inline error_code tape_builder::end_container(json_iterator& iter, internal::tape_type start, internal::tape_type end) noexcept { - // Write the ending tape element, pointing at the start location - const uint32_t start_tape_index = iter.dom_parser.open_containers[iter.depth].tape_index; - tape.append(start_tape_index, end); - // Write the start tape element, pointing at the end location (and including count) - // count can overflow if it exceeds 24 bits... so we saturate - // the convention being that a cnt of 0xffffff or more is undetermined in value (>= 0xffffff). - const uint32_t count = iter.dom_parser.open_containers[iter.depth].count; - const uint32_t cntsat = count > 0xFFFFFF ? 0xFFFFFF : count; - tape_writer::write(iter.dom_parser.doc->tape[start_tape_index], next_tape_index(iter) | (uint64_t(cntsat) << 32), start); - return SUCCESS; - } - - simdjson_inline uint8_t* tape_builder::on_start_string(json_iterator& iter) noexcept { - // we advance the point, accounting for the fact that we have a NULL termination - tape.append(current_string_buf_loc - iter.dom_parser.doc->string_buf.get(), internal::tape_type::STRING); - return current_string_buf_loc + sizeof(uint32_t); - } - - simdjson_inline void tape_builder::on_end_string(uint8_t* dst) noexcept { - uint32_t str_length = uint32_t(dst - (current_string_buf_loc + sizeof(uint32_t))); - // TODO check for overflow in case someone has a crazy string (>=4GB?) - // But only add the overflow check when the document itself exceeds 4GB - // Currently unneeded because we refuse to parse docs larger or equal to 4GB. - memcpy(current_string_buf_loc, &str_length, sizeof(uint32_t)); - // NULL termination is still handy if you expect all your strings to - // be NULL terminated? It comes at a small cost - *dst = 0; - current_string_buf_loc = dst + 1; - } - - } // namespace stage2 - } // unnamed namespace - } // namespace ppc64 -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE2_TAPE_BUILDER_H -/* end file generic/stage2/tape_builder.h for ppc64 */ -/* end file generic/stage2/amalgamated.h for ppc64 */ - -// -// Stage 1 -// -namespace simdjson { - namespace ppc64 { - - simdjson_warn_unused error_code implementation::create_dom_parser_implementation( - size_t capacity, - size_t max_depth, - std::unique_ptr& dst - ) const noexcept { - dst.reset(new (std::nothrow) dom_parser_implementation()); - if (!dst) { return MEMALLOC; } - if (auto err = dst->set_capacity(capacity)) - return err; - if (auto err = dst->set_max_depth(max_depth)) - return err; - return SUCCESS; - } - - namespace { - - using namespace simd; - - simdjson_inline json_character_block json_character_block::classify(const simd::simd8x64& in) { - const simd8 table1(16, 0, 0, 0, 0, 0, 0, 0, 0, 8, 12, 1, 2, 9, 0, 0); - const simd8 table2(8, 0, 18, 4, 0, 1, 0, 1, 0, 0, 0, 3, 2, 1, 0, 0); - - simd8x64 v( - (in.chunks[0] & 0xf).lookup_16(table1) & (in.chunks[0].shr<4>()).lookup_16(table2), - (in.chunks[1] & 0xf).lookup_16(table1) & (in.chunks[1].shr<4>()).lookup_16(table2), - (in.chunks[2] & 0xf).lookup_16(table1) & (in.chunks[2].shr<4>()).lookup_16(table2), - (in.chunks[3] & 0xf).lookup_16(table1) & (in.chunks[3].shr<4>()).lookup_16(table2) - ); - - uint64_t op = simd8x64( - v.chunks[0].any_bits_set(0x7), - v.chunks[1].any_bits_set(0x7), - v.chunks[2].any_bits_set(0x7), - v.chunks[3].any_bits_set(0x7) - ).to_bitmask(); - - uint64_t whitespace = simd8x64( - v.chunks[0].any_bits_set(0x18), - v.chunks[1].any_bits_set(0x18), - v.chunks[2].any_bits_set(0x18), - v.chunks[3].any_bits_set(0x18) - ).to_bitmask(); - - return { whitespace, op }; - } - - simdjson_inline bool is_ascii(const simd8x64& input) { - // careful: 0x80 is not ascii. - return input.reduce_or().saturating_sub(0x7fu).bits_not_set_anywhere(); - } - - simdjson_unused simdjson_inline simd8 must_be_continuation(const simd8 prev1, const simd8 prev2, const simd8 prev3) { - simd8 is_second_byte = prev1.saturating_sub(0xc0u - 1); // Only 11______ will be > 0 - simd8 is_third_byte = prev2.saturating_sub(0xe0u - 1); // Only 111_____ will be > 0 - simd8 is_fourth_byte = prev3.saturating_sub(0xf0u - 1); // Only 1111____ will be > 0 - // Caller requires a bool (all 1's). All values resulting from the subtraction will be <= 64, so signed comparison is fine. - return simd8(is_second_byte | is_third_byte | is_fourth_byte) > int8_t(0); - } - - simdjson_inline simd8 must_be_2_3_continuation(const simd8 prev2, const simd8 prev3) { - simd8 is_third_byte = prev2.saturating_sub(0xe0u - 1); // Only 111_____ will be > 0 - simd8 is_fourth_byte = prev3.saturating_sub(0xf0u - 1); // Only 1111____ will be > 0 - // Caller requires a bool (all 1's). All values resulting from the subtraction will be <= 64, so signed comparison is fine. - return simd8(is_third_byte | is_fourth_byte) > int8_t(0); - } - - } // unnamed namespace - } // namespace ppc64 -} // namespace simdjson - -// -// Stage 2 -// - -// -// Implementation-specific overrides -// -namespace simdjson { - namespace ppc64 { - - simdjson_warn_unused error_code implementation::minify(const uint8_t* buf, size_t len, uint8_t* dst, size_t& dst_len) const noexcept { - return ppc64::stage1::json_minifier::minify<64>(buf, len, dst, dst_len); - } - - simdjson_warn_unused error_code dom_parser_implementation::stage1(const uint8_t* _buf, size_t _len, stage1_mode streaming) noexcept { - this->buf = _buf; - this->len = _len; - return ppc64::stage1::json_structural_indexer::index<64>(buf, len, *this, streaming); - } - - simdjson_warn_unused bool implementation::validate_utf8(const char* buf, size_t len) const noexcept { - return ppc64::stage1::generic_validate_utf8(buf, len); - } - - simdjson_warn_unused error_code dom_parser_implementation::stage2(dom::document& _doc) noexcept { - return stage2::tape_builder::parse_document(*this, _doc); - } - - simdjson_warn_unused error_code dom_parser_implementation::stage2_next(dom::document& _doc) noexcept { - return stage2::tape_builder::parse_document(*this, _doc); - } - - simdjson_warn_unused uint8_t* dom_parser_implementation::parse_string(const uint8_t* src, uint8_t* dst, bool replacement_char) const noexcept { - return ppc64::stringparsing::parse_string(src, dst, replacement_char); - } - - simdjson_warn_unused uint8_t* dom_parser_implementation::parse_wobbly_string(const uint8_t* src, uint8_t* dst) const noexcept { - return ppc64::stringparsing::parse_wobbly_string(src, dst); - } - - simdjson_warn_unused error_code dom_parser_implementation::parse(const uint8_t* _buf, size_t _len, dom::document& _doc) noexcept { - auto error = stage1(_buf, _len, stage1_mode::regular); - if (error) { return error; } - return stage2(_doc); - } - - } // namespace ppc64 -} // namespace simdjson - -/* including simdjson/ppc64/end.h: #include */ -/* begin file simdjson/ppc64/end.h */ -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/ppc64/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#undef SIMDJSON_SKIP_BACKSLASH_SHORT_CIRCUIT -/* undefining SIMDJSON_IMPLEMENTATION from "ppc64" */ -#undef SIMDJSON_IMPLEMENTATION -/* end file simdjson/ppc64/end.h */ - -#endif // SIMDJSON_SRC_PPC64_CPP -/* end file ppc64.cpp */ -#endif -#if SIMDJSON_IMPLEMENTATION_WESTMERE -/* including westmere.cpp: #include */ -/* begin file westmere.cpp */ -#ifndef SIMDJSON_SRC_WESTMERE_CPP -#define SIMDJSON_SRC_WESTMERE_CPP - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -/* including simdjson/westmere.h: #include */ -/* begin file simdjson/westmere.h */ -#ifndef SIMDJSON_WESTMERE_H -#define SIMDJSON_WESTMERE_H - -/* including simdjson/westmere/begin.h: #include "simdjson/westmere/begin.h" */ -/* begin file simdjson/westmere/begin.h */ -/* defining SIMDJSON_IMPLEMENTATION to "westmere" */ -#define SIMDJSON_IMPLEMENTATION westmere -/* including simdjson/westmere/base.h: #include "simdjson/westmere/base.h" */ -/* begin file simdjson/westmere/base.h */ -#ifndef SIMDJSON_WESTMERE_BASE_H -#define SIMDJSON_WESTMERE_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -// The constructor may be executed on any host, so we take care not to use SIMDJSON_TARGET_WESTMERE -namespace simdjson { - /** - * Implementation for Westmere (Intel SSE4.2). - */ - namespace westmere { - - class implementation; - - namespace { - namespace simd { - - template struct simd8; - template struct simd8x64; - - } // namespace simd - } // unnamed namespace - - } // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_WESTMERE_BASE_H -/* end file simdjson/westmere/base.h */ -/* including simdjson/westmere/intrinsics.h: #include "simdjson/westmere/intrinsics.h" */ -/* begin file simdjson/westmere/intrinsics.h */ -#ifndef SIMDJSON_WESTMERE_INTRINSICS_H -#define SIMDJSON_WESTMERE_INTRINSICS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/westmere/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#if SIMDJSON_VISUAL_STUDIO -// under clang within visual studio, this will include -#include // visual studio or clang -#else -#include // elsewhere -#endif // SIMDJSON_VISUAL_STUDIO - - -#if SIMDJSON_CLANG_VISUAL_STUDIO -/** - * You are not supposed, normally, to include these - * headers directly. Instead you should either include intrin.h - * or x86intrin.h. However, when compiling with clang - * under Windows (i.e., when _MSC_VER is set), these headers - * only get included *if* the corresponding features are detected - * from macros: - */ -#include // for _mm_alignr_epi8 -#include // for _mm_clmulepi64_si128 -#endif - -static_assert(sizeof(__m128i) <= simdjson::SIMDJSON_PADDING, "insufficient padding for westmere"); - -#endif // SIMDJSON_WESTMERE_INTRINSICS_H -/* end file simdjson/westmere/intrinsics.h */ - -#if !SIMDJSON_CAN_ALWAYS_RUN_WESTMERE -SIMDJSON_TARGET_REGION("sse4.2,pclmul,popcnt") -#endif - -/* including simdjson/westmere/bitmanipulation.h: #include "simdjson/westmere/bitmanipulation.h" */ -/* begin file simdjson/westmere/bitmanipulation.h */ -#ifndef SIMDJSON_WESTMERE_BITMANIPULATION_H -#define SIMDJSON_WESTMERE_BITMANIPULATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/westmere/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/westmere/intrinsics.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - namespace westmere { - namespace { - - // We sometimes call trailing_zero on inputs that are zero, - // but the algorithms do not end up using the returned value. - // Sadly, sanitizers are not smart enough to figure it out. - SIMDJSON_NO_SANITIZE_UNDEFINED - // This function can be used safely even if not all bytes have been - // initialized. - // See issue https://github.com/simdjson/simdjson/issues/1965 - SIMDJSON_NO_SANITIZE_MEMORY - simdjson_inline int trailing_zeroes(uint64_t input_num) { -#if SIMDJSON_REGULAR_VISUAL_STUDIO - unsigned long ret; - // Search the mask data from least significant bit (LSB) - // to the most significant bit (MSB) for a set bit (1). - _BitScanForward64(&ret, input_num); - return (int)ret; -#else // SIMDJSON_REGULAR_VISUAL_STUDIO - return __builtin_ctzll(input_num); -#endif // SIMDJSON_REGULAR_VISUAL_STUDIO - } - - /* result might be undefined when input_num is zero */ - simdjson_inline uint64_t clear_lowest_bit(uint64_t input_num) { - return input_num & (input_num - 1); - } - - /* result might be undefined when input_num is zero */ - simdjson_inline int leading_zeroes(uint64_t input_num) { -#if SIMDJSON_REGULAR_VISUAL_STUDIO - unsigned long leading_zero = 0; - // Search the mask data from most significant bit (MSB) - // to least significant bit (LSB) for a set bit (1). - if (_BitScanReverse64(&leading_zero, input_num)) - return (int)(63 - leading_zero); - else - return 64; -#else - return __builtin_clzll(input_num); -#endif// SIMDJSON_REGULAR_VISUAL_STUDIO - } - -#if SIMDJSON_REGULAR_VISUAL_STUDIO - simdjson_inline unsigned __int64 count_ones(uint64_t input_num) { - // note: we do not support legacy 32-bit Windows in this kernel - return __popcnt64(input_num);// Visual Studio wants two underscores - } -#else - simdjson_inline long long int count_ones(uint64_t input_num) { - return _popcnt64(input_num); - } -#endif - - simdjson_inline bool add_overflow(uint64_t value1, uint64_t value2, - uint64_t* result) { -#if SIMDJSON_REGULAR_VISUAL_STUDIO - return _addcarry_u64(0, value1, value2, - reinterpret_cast(result)); -#else - return __builtin_uaddll_overflow(value1, value2, - reinterpret_cast(result)); -#endif - } - - } // unnamed namespace - } // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_WESTMERE_BITMANIPULATION_H -/* end file simdjson/westmere/bitmanipulation.h */ -/* including simdjson/westmere/bitmask.h: #include "simdjson/westmere/bitmask.h" */ -/* begin file simdjson/westmere/bitmask.h */ -#ifndef SIMDJSON_WESTMERE_BITMASK_H -#define SIMDJSON_WESTMERE_BITMASK_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/westmere/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/westmere/intrinsics.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - namespace westmere { - namespace { - - // - // Perform a "cumulative bitwise xor," flipping bits each time a 1 is encountered. - // - // For example, prefix_xor(00100100) == 00011100 - // - simdjson_inline uint64_t prefix_xor(const uint64_t bitmask) { - // There should be no such thing with a processing supporting avx2 - // but not clmul. - __m128i all_ones = _mm_set1_epi8('\xFF'); - __m128i result = _mm_clmulepi64_si128(_mm_set_epi64x(0ULL, bitmask), all_ones, 0); - return _mm_cvtsi128_si64(result); - } - - } // unnamed namespace - } // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_WESTMERE_BITMASK_H -/* end file simdjson/westmere/bitmask.h */ -/* including simdjson/westmere/numberparsing_defs.h: #include "simdjson/westmere/numberparsing_defs.h" */ -/* begin file simdjson/westmere/numberparsing_defs.h */ -#ifndef SIMDJSON_WESTMERE_NUMBERPARSING_DEFS_H -#define SIMDJSON_WESTMERE_NUMBERPARSING_DEFS_H - -/* including simdjson/westmere/base.h: #include "simdjson/westmere/base.h" */ -/* begin file simdjson/westmere/base.h */ -#ifndef SIMDJSON_WESTMERE_BASE_H -#define SIMDJSON_WESTMERE_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -// The constructor may be executed on any host, so we take care not to use SIMDJSON_TARGET_WESTMERE -namespace simdjson { - /** - * Implementation for Westmere (Intel SSE4.2). - */ - namespace westmere { - - class implementation; - - namespace { - namespace simd { - - template struct simd8; - template struct simd8x64; - - } // namespace simd - } // unnamed namespace - - } // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_WESTMERE_BASE_H -/* end file simdjson/westmere/base.h */ -/* including simdjson/westmere/intrinsics.h: #include "simdjson/westmere/intrinsics.h" */ -/* begin file simdjson/westmere/intrinsics.h */ -#ifndef SIMDJSON_WESTMERE_INTRINSICS_H -#define SIMDJSON_WESTMERE_INTRINSICS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/westmere/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#if SIMDJSON_VISUAL_STUDIO -// under clang within visual studio, this will include -#include // visual studio or clang -#else -#include // elsewhere -#endif // SIMDJSON_VISUAL_STUDIO - - -#if SIMDJSON_CLANG_VISUAL_STUDIO -/** - * You are not supposed, normally, to include these - * headers directly. Instead you should either include intrin.h - * or x86intrin.h. However, when compiling with clang - * under Windows (i.e., when _MSC_VER is set), these headers - * only get included *if* the corresponding features are detected - * from macros: - */ -#include // for _mm_alignr_epi8 -#include // for _mm_clmulepi64_si128 -#endif - -static_assert(sizeof(__m128i) <= simdjson::SIMDJSON_PADDING, "insufficient padding for westmere"); - -#endif // SIMDJSON_WESTMERE_INTRINSICS_H -/* end file simdjson/westmere/intrinsics.h */ - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - namespace westmere { - namespace numberparsing { - - /** @private */ - static simdjson_inline uint32_t parse_eight_digits_unrolled(const uint8_t* chars) { - // this actually computes *16* values so we are being wasteful. - const __m128i ascii0 = _mm_set1_epi8('0'); - const __m128i mul_1_10 = - _mm_setr_epi8(10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1); - const __m128i mul_1_100 = _mm_setr_epi16(100, 1, 100, 1, 100, 1, 100, 1); - const __m128i mul_1_10000 = - _mm_setr_epi16(10000, 1, 10000, 1, 10000, 1, 10000, 1); - const __m128i input = _mm_sub_epi8( - _mm_loadu_si128(reinterpret_cast(chars)), ascii0); - const __m128i t1 = _mm_maddubs_epi16(input, mul_1_10); - const __m128i t2 = _mm_madd_epi16(t1, mul_1_100); - const __m128i t3 = _mm_packus_epi32(t2, t2); - const __m128i t4 = _mm_madd_epi16(t3, mul_1_10000); - return _mm_cvtsi128_si32( - t4); // only captures the sum of the first 8 digits, drop the rest - } - - /** @private */ - simdjson_inline internal::value128 full_multiplication(uint64_t value1, uint64_t value2) { - internal::value128 answer; -#if SIMDJSON_REGULAR_VISUAL_STUDIO || SIMDJSON_IS_32BITS -#ifdef _M_ARM64 - // ARM64 has native support for 64-bit multiplications, no need to emultate - answer.high = __umulh(value1, value2); - answer.low = value1 * value2; -#else - answer.low = _umul128(value1, value2, &answer.high); // _umul128 not available on ARM64 -#endif // _M_ARM64 -#else // SIMDJSON_REGULAR_VISUAL_STUDIO || SIMDJSON_IS_32BITS - __uint128_t r = (static_cast<__uint128_t>(value1)) * value2; - answer.low = uint64_t(r); - answer.high = uint64_t(r >> 64); -#endif - return answer; - } - - } // namespace numberparsing - } // namespace westmere -} // namespace simdjson - -#define SIMDJSON_SWAR_NUMBER_PARSING 1 - -#endif // SIMDJSON_WESTMERE_NUMBERPARSING_DEFS_H -/* end file simdjson/westmere/numberparsing_defs.h */ -/* including simdjson/westmere/simd.h: #include "simdjson/westmere/simd.h" */ -/* begin file simdjson/westmere/simd.h */ -#ifndef SIMDJSON_WESTMERE_SIMD_H -#define SIMDJSON_WESTMERE_SIMD_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/westmere/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/westmere/bitmanipulation.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/simdprune_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - namespace westmere { - namespace { - namespace simd { - - template - struct base { - __m128i value; - - // Zero constructor - simdjson_inline base() : value{ __m128i() } {} - - // Conversion from SIMD register - simdjson_inline base(const __m128i _value) : value(_value) {} - - // Conversion to SIMD register - simdjson_inline operator const __m128i& () const { return this->value; } - simdjson_inline operator __m128i& () { return this->value; } - - // Bit operations - simdjson_inline Child operator|(const Child other) const { return _mm_or_si128(*this, other); } - simdjson_inline Child operator&(const Child other) const { return _mm_and_si128(*this, other); } - simdjson_inline Child operator^(const Child other) const { return _mm_xor_si128(*this, other); } - simdjson_inline Child bit_andnot(const Child other) const { return _mm_andnot_si128(other, *this); } - simdjson_inline Child& operator|=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast | other; return *this_cast; } - simdjson_inline Child& operator&=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast & other; return *this_cast; } - simdjson_inline Child& operator^=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast ^ other; return *this_cast; } - }; - - template> - struct base8 : base> { - typedef uint16_t bitmask_t; - typedef uint32_t bitmask2_t; - - simdjson_inline base8() : base>() {} - simdjson_inline base8(const __m128i _value) : base>(_value) {} - - friend simdjson_inline Mask operator==(const simd8 lhs, const simd8 rhs) { return _mm_cmpeq_epi8(lhs, rhs); } - - static const int SIZE = sizeof(base>::value); - - template - simdjson_inline simd8 prev(const simd8 prev_chunk) const { - return _mm_alignr_epi8(*this, prev_chunk, 16 - N); - } - }; - - // SIMD byte mask type (returned by things like eq and gt) - template<> - struct simd8 : base8 { - static simdjson_inline simd8 splat(bool _value) { return _mm_set1_epi8(uint8_t(-(!!_value))); } - - simdjson_inline simd8() : base8() {} - simdjson_inline simd8(const __m128i _value) : base8(_value) {} - // Splat constructor - simdjson_inline simd8(bool _value) : base8(splat(_value)) {} - - simdjson_inline int to_bitmask() const { return _mm_movemask_epi8(*this); } - simdjson_inline bool any() const { return !_mm_testz_si128(*this, *this); } - simdjson_inline simd8 operator~() const { return *this ^ true; } - }; - - template - struct base8_numeric : base8 { - static simdjson_inline simd8 splat(T _value) { return _mm_set1_epi8(_value); } - static simdjson_inline simd8 zero() { return _mm_setzero_si128(); } - static simdjson_inline simd8 load(const T values[16]) { - return _mm_loadu_si128(reinterpret_cast(values)); - } - // Repeat 16 values as many times as necessary (usually for lookup tables) - static simdjson_inline simd8 repeat_16( - T v0, T v1, T v2, T v3, T v4, T v5, T v6, T v7, - T v8, T v9, T v10, T v11, T v12, T v13, T v14, T v15 - ) { - return simd8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10, v11, v12, v13, v14, v15 - ); - } - - simdjson_inline base8_numeric() : base8() {} - simdjson_inline base8_numeric(const __m128i _value) : base8(_value) {} - - // Store to array - simdjson_inline void store(T dst[16]) const { return _mm_storeu_si128(reinterpret_cast<__m128i*>(dst), *this); } - - // Override to distinguish from bool version - simdjson_inline simd8 operator~() const { return *this ^ 0xFFu; } - - // Addition/subtraction are the same for signed and unsigned - simdjson_inline simd8 operator+(const simd8 other) const { return _mm_add_epi8(*this, other); } - simdjson_inline simd8 operator-(const simd8 other) const { return _mm_sub_epi8(*this, other); } - simdjson_inline simd8& operator+=(const simd8 other) { *this = *this + other; return *static_cast*>(this); } - simdjson_inline simd8& operator-=(const simd8 other) { *this = *this - other; return *static_cast*>(this); } - - // Perform a lookup assuming the value is between 0 and 16 (undefined behavior for out of range values) - template - simdjson_inline simd8 lookup_16(simd8 lookup_table) const { - return _mm_shuffle_epi8(lookup_table, *this); - } - - // Copies to 'output" all bytes corresponding to a 0 in the mask (interpreted as a bitset). - // Passing a 0 value for mask would be equivalent to writing out every byte to output. - // Only the first 16 - count_ones(mask) bytes of the result are significant but 16 bytes - // get written. - // Design consideration: it seems like a function with the - // signature simd8 compress(uint32_t mask) would be - // sensible, but the AVX ISA makes this kind of approach difficult. - template - simdjson_inline void compress(uint16_t mask, L* output) const { - using internal::thintable_epi8; - using internal::BitsSetTable256mul2; - using internal::pshufb_combine_table; - // this particular implementation was inspired by work done by @animetosho - // we do it in two steps, first 8 bytes and then second 8 bytes - uint8_t mask1 = uint8_t(mask); // least significant 8 bits - uint8_t mask2 = uint8_t(mask >> 8); // most significant 8 bits - // next line just loads the 64-bit values thintable_epi8[mask1] and - // thintable_epi8[mask2] into a 128-bit register, using only - // two instructions on most compilers. - __m128i shufmask = _mm_set_epi64x(thintable_epi8[mask2], thintable_epi8[mask1]); - // we increment by 0x08 the second half of the mask - shufmask = - _mm_add_epi8(shufmask, _mm_set_epi32(0x08080808, 0x08080808, 0, 0)); - // this is the version "nearly pruned" - __m128i pruned = _mm_shuffle_epi8(*this, shufmask); - // we still need to put the two halves together. - // we compute the popcount of the first half: - int pop1 = BitsSetTable256mul2[mask1]; - // then load the corresponding mask, what it does is to write - // only the first pop1 bytes from the first 8 bytes, and then - // it fills in with the bytes from the second 8 bytes + some filling - // at the end. - __m128i compactmask = - _mm_loadu_si128(reinterpret_cast(pshufb_combine_table + pop1 * 8)); - __m128i answer = _mm_shuffle_epi8(pruned, compactmask); - _mm_storeu_si128(reinterpret_cast<__m128i*>(output), answer); - } - - template - simdjson_inline simd8 lookup_16( - L replace0, L replace1, L replace2, L replace3, - L replace4, L replace5, L replace6, L replace7, - L replace8, L replace9, L replace10, L replace11, - L replace12, L replace13, L replace14, L replace15) const { - return lookup_16(simd8::repeat_16( - replace0, replace1, replace2, replace3, - replace4, replace5, replace6, replace7, - replace8, replace9, replace10, replace11, - replace12, replace13, replace14, replace15 - )); - } - }; - - // Signed bytes - template<> - struct simd8 : base8_numeric { - simdjson_inline simd8() : base8_numeric() {} - simdjson_inline simd8(const __m128i _value) : base8_numeric(_value) {} - // Splat constructor - simdjson_inline simd8(int8_t _value) : simd8(splat(_value)) {} - // Array constructor - simdjson_inline simd8(const int8_t* values) : simd8(load(values)) {} - // Member-by-member initialization - simdjson_inline simd8( - int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, - int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 - ) : simd8(_mm_setr_epi8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10, v11, v12, v13, v14, v15 - )) {} - // Repeat 16 values as many times as necessary (usually for lookup tables) - simdjson_inline static simd8 repeat_16( - int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, - int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 - ) { - return simd8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10, v11, v12, v13, v14, v15 - ); - } - - // Order-sensitive comparisons - simdjson_inline simd8 max_val(const simd8 other) const { return _mm_max_epi8(*this, other); } - simdjson_inline simd8 min_val(const simd8 other) const { return _mm_min_epi8(*this, other); } - simdjson_inline simd8 operator>(const simd8 other) const { return _mm_cmpgt_epi8(*this, other); } - simdjson_inline simd8 operator<(const simd8 other) const { return _mm_cmpgt_epi8(other, *this); } - }; - - // Unsigned bytes - template<> - struct simd8 : base8_numeric { - simdjson_inline simd8() : base8_numeric() {} - simdjson_inline simd8(const __m128i _value) : base8_numeric(_value) {} - // Splat constructor - simdjson_inline simd8(uint8_t _value) : simd8(splat(_value)) {} - // Array constructor - simdjson_inline simd8(const uint8_t* values) : simd8(load(values)) {} - // Member-by-member initialization - simdjson_inline simd8( - uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, - uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 - ) : simd8(_mm_setr_epi8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10, v11, v12, v13, v14, v15 - )) {} - // Repeat 16 values as many times as necessary (usually for lookup tables) - simdjson_inline static simd8 repeat_16( - uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, - uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 - ) { - return simd8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10, v11, v12, v13, v14, v15 - ); - } - - // Saturated math - simdjson_inline simd8 saturating_add(const simd8 other) const { return _mm_adds_epu8(*this, other); } - simdjson_inline simd8 saturating_sub(const simd8 other) const { return _mm_subs_epu8(*this, other); } - - // Order-specific operations - simdjson_inline simd8 max_val(const simd8 other) const { return _mm_max_epu8(*this, other); } - simdjson_inline simd8 min_val(const simd8 other) const { return _mm_min_epu8(*this, other); } - // Same as >, but only guarantees true is nonzero (< guarantees true = -1) - simdjson_inline simd8 gt_bits(const simd8 other) const { return this->saturating_sub(other); } - // Same as <, but only guarantees true is nonzero (< guarantees true = -1) - simdjson_inline simd8 lt_bits(const simd8 other) const { return other.saturating_sub(*this); } - simdjson_inline simd8 operator<=(const simd8 other) const { return other.max_val(*this) == other; } - simdjson_inline simd8 operator>=(const simd8 other) const { return other.min_val(*this) == other; } - simdjson_inline simd8 operator>(const simd8 other) const { return this->gt_bits(other).any_bits_set(); } - simdjson_inline simd8 operator<(const simd8 other) const { return this->gt_bits(other).any_bits_set(); } - - // Bit-specific operations - simdjson_inline simd8 bits_not_set() const { return *this == uint8_t(0); } - simdjson_inline simd8 bits_not_set(simd8 bits) const { return (*this & bits).bits_not_set(); } - simdjson_inline simd8 any_bits_set() const { return ~this->bits_not_set(); } - simdjson_inline simd8 any_bits_set(simd8 bits) const { return ~this->bits_not_set(bits); } - simdjson_inline bool is_ascii() const { return _mm_movemask_epi8(*this) == 0; } - simdjson_inline bool bits_not_set_anywhere() const { return _mm_testz_si128(*this, *this); } - simdjson_inline bool any_bits_set_anywhere() const { return !bits_not_set_anywhere(); } - simdjson_inline bool bits_not_set_anywhere(simd8 bits) const { return _mm_testz_si128(*this, bits); } - simdjson_inline bool any_bits_set_anywhere(simd8 bits) const { return !bits_not_set_anywhere(bits); } - template - simdjson_inline simd8 shr() const { return simd8(_mm_srli_epi16(*this, N)) & uint8_t(0xFFu >> N); } - template - simdjson_inline simd8 shl() const { return simd8(_mm_slli_epi16(*this, N)) & uint8_t(0xFFu << N); } - // Get one of the bits and make a bitmask out of it. - // e.g. value.get_bit<7>() gets the high bit - template - simdjson_inline int get_bit() const { return _mm_movemask_epi8(_mm_slli_epi16(*this, 7 - N)); } - }; - - template - struct simd8x64 { - static constexpr int NUM_CHUNKS = 64 / sizeof(simd8); - static_assert(NUM_CHUNKS == 4, "Westmere kernel should use four registers per 64-byte block."); - const simd8 chunks[NUM_CHUNKS]; - - simd8x64(const simd8x64& o) = delete; // no copy allowed - simd8x64& operator=(const simd8& other) = delete; // no assignment allowed - simd8x64() = delete; // no default constructor allowed - - simdjson_inline simd8x64(const simd8 chunk0, const simd8 chunk1, const simd8 chunk2, const simd8 chunk3) : chunks{ chunk0, chunk1, chunk2, chunk3 } {} - simdjson_inline simd8x64(const T ptr[64]) : chunks{ simd8::load(ptr), simd8::load(ptr + 16), simd8::load(ptr + 32), simd8::load(ptr + 48) } {} - - simdjson_inline void store(T ptr[64]) const { - this->chunks[0].store(ptr + sizeof(simd8) * 0); - this->chunks[1].store(ptr + sizeof(simd8) * 1); - this->chunks[2].store(ptr + sizeof(simd8) * 2); - this->chunks[3].store(ptr + sizeof(simd8) * 3); - } - - simdjson_inline simd8 reduce_or() const { - return (this->chunks[0] | this->chunks[1]) | (this->chunks[2] | this->chunks[3]); - } - - simdjson_inline uint64_t compress(uint64_t mask, T* output) const { - this->chunks[0].compress(uint16_t(mask), output); - this->chunks[1].compress(uint16_t(mask >> 16), output + 16 - count_ones(mask & 0xFFFF)); - this->chunks[2].compress(uint16_t(mask >> 32), output + 32 - count_ones(mask & 0xFFFFFFFF)); - this->chunks[3].compress(uint16_t(mask >> 48), output + 48 - count_ones(mask & 0xFFFFFFFFFFFF)); - return 64 - count_ones(mask); - } - - simdjson_inline uint64_t to_bitmask() const { - uint64_t r0 = uint32_t(this->chunks[0].to_bitmask()); - uint64_t r1 = this->chunks[1].to_bitmask(); - uint64_t r2 = this->chunks[2].to_bitmask(); - uint64_t r3 = this->chunks[3].to_bitmask(); - return r0 | (r1 << 16) | (r2 << 32) | (r3 << 48); - } - - simdjson_inline uint64_t eq(const T m) const { - const simd8 mask = simd8::splat(m); - return simd8x64( - this->chunks[0] == mask, - this->chunks[1] == mask, - this->chunks[2] == mask, - this->chunks[3] == mask - ).to_bitmask(); - } - - simdjson_inline uint64_t eq(const simd8x64& other) const { - return simd8x64( - this->chunks[0] == other.chunks[0], - this->chunks[1] == other.chunks[1], - this->chunks[2] == other.chunks[2], - this->chunks[3] == other.chunks[3] - ).to_bitmask(); - } - - simdjson_inline uint64_t lteq(const T m) const { - const simd8 mask = simd8::splat(m); - return simd8x64( - this->chunks[0] <= mask, - this->chunks[1] <= mask, - this->chunks[2] <= mask, - this->chunks[3] <= mask - ).to_bitmask(); - } - }; // struct simd8x64 - - } // namespace simd - } // unnamed namespace - } // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_WESTMERE_SIMD_INPUT_H -/* end file simdjson/westmere/simd.h */ -/* including simdjson/westmere/stringparsing_defs.h: #include "simdjson/westmere/stringparsing_defs.h" */ -/* begin file simdjson/westmere/stringparsing_defs.h */ -#ifndef SIMDJSON_WESTMERE_STRINGPARSING_DEFS_H -#define SIMDJSON_WESTMERE_STRINGPARSING_DEFS_H - -/* including simdjson/westmere/bitmanipulation.h: #include "simdjson/westmere/bitmanipulation.h" */ -/* begin file simdjson/westmere/bitmanipulation.h */ -#ifndef SIMDJSON_WESTMERE_BITMANIPULATION_H -#define SIMDJSON_WESTMERE_BITMANIPULATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/westmere/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/westmere/intrinsics.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - namespace westmere { - namespace { - - // We sometimes call trailing_zero on inputs that are zero, - // but the algorithms do not end up using the returned value. - // Sadly, sanitizers are not smart enough to figure it out. - SIMDJSON_NO_SANITIZE_UNDEFINED - // This function can be used safely even if not all bytes have been - // initialized. - // See issue https://github.com/simdjson/simdjson/issues/1965 - SIMDJSON_NO_SANITIZE_MEMORY - simdjson_inline int trailing_zeroes(uint64_t input_num) { -#if SIMDJSON_REGULAR_VISUAL_STUDIO - unsigned long ret; - // Search the mask data from least significant bit (LSB) - // to the most significant bit (MSB) for a set bit (1). - _BitScanForward64(&ret, input_num); - return (int)ret; -#else // SIMDJSON_REGULAR_VISUAL_STUDIO - return __builtin_ctzll(input_num); -#endif // SIMDJSON_REGULAR_VISUAL_STUDIO - } - - /* result might be undefined when input_num is zero */ - simdjson_inline uint64_t clear_lowest_bit(uint64_t input_num) { - return input_num & (input_num - 1); - } - - /* result might be undefined when input_num is zero */ - simdjson_inline int leading_zeroes(uint64_t input_num) { -#if SIMDJSON_REGULAR_VISUAL_STUDIO - unsigned long leading_zero = 0; - // Search the mask data from most significant bit (MSB) - // to least significant bit (LSB) for a set bit (1). - if (_BitScanReverse64(&leading_zero, input_num)) - return (int)(63 - leading_zero); - else - return 64; -#else - return __builtin_clzll(input_num); -#endif// SIMDJSON_REGULAR_VISUAL_STUDIO - } - -#if SIMDJSON_REGULAR_VISUAL_STUDIO - simdjson_inline unsigned __int64 count_ones(uint64_t input_num) { - // note: we do not support legacy 32-bit Windows in this kernel - return __popcnt64(input_num);// Visual Studio wants two underscores - } -#else - simdjson_inline long long int count_ones(uint64_t input_num) { - return _popcnt64(input_num); - } -#endif - - simdjson_inline bool add_overflow(uint64_t value1, uint64_t value2, - uint64_t* result) { -#if SIMDJSON_REGULAR_VISUAL_STUDIO - return _addcarry_u64(0, value1, value2, - reinterpret_cast(result)); -#else - return __builtin_uaddll_overflow(value1, value2, - reinterpret_cast(result)); -#endif - } - - } // unnamed namespace - } // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_WESTMERE_BITMANIPULATION_H -/* end file simdjson/westmere/bitmanipulation.h */ -/* including simdjson/westmere/simd.h: #include "simdjson/westmere/simd.h" */ -/* begin file simdjson/westmere/simd.h */ -#ifndef SIMDJSON_WESTMERE_SIMD_H -#define SIMDJSON_WESTMERE_SIMD_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/westmere/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/westmere/bitmanipulation.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/simdprune_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - namespace westmere { - namespace { - namespace simd { - - template - struct base { - __m128i value; - - // Zero constructor - simdjson_inline base() : value{ __m128i() } {} - - // Conversion from SIMD register - simdjson_inline base(const __m128i _value) : value(_value) {} - - // Conversion to SIMD register - simdjson_inline operator const __m128i& () const { return this->value; } - simdjson_inline operator __m128i& () { return this->value; } - - // Bit operations - simdjson_inline Child operator|(const Child other) const { return _mm_or_si128(*this, other); } - simdjson_inline Child operator&(const Child other) const { return _mm_and_si128(*this, other); } - simdjson_inline Child operator^(const Child other) const { return _mm_xor_si128(*this, other); } - simdjson_inline Child bit_andnot(const Child other) const { return _mm_andnot_si128(other, *this); } - simdjson_inline Child& operator|=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast | other; return *this_cast; } - simdjson_inline Child& operator&=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast & other; return *this_cast; } - simdjson_inline Child& operator^=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast ^ other; return *this_cast; } - }; - - template> - struct base8 : base> { - typedef uint16_t bitmask_t; - typedef uint32_t bitmask2_t; - - simdjson_inline base8() : base>() {} - simdjson_inline base8(const __m128i _value) : base>(_value) {} - - friend simdjson_inline Mask operator==(const simd8 lhs, const simd8 rhs) { return _mm_cmpeq_epi8(lhs, rhs); } - - static const int SIZE = sizeof(base>::value); - - template - simdjson_inline simd8 prev(const simd8 prev_chunk) const { - return _mm_alignr_epi8(*this, prev_chunk, 16 - N); - } - }; - - // SIMD byte mask type (returned by things like eq and gt) - template<> - struct simd8 : base8 { - static simdjson_inline simd8 splat(bool _value) { return _mm_set1_epi8(uint8_t(-(!!_value))); } - - simdjson_inline simd8() : base8() {} - simdjson_inline simd8(const __m128i _value) : base8(_value) {} - // Splat constructor - simdjson_inline simd8(bool _value) : base8(splat(_value)) {} - - simdjson_inline int to_bitmask() const { return _mm_movemask_epi8(*this); } - simdjson_inline bool any() const { return !_mm_testz_si128(*this, *this); } - simdjson_inline simd8 operator~() const { return *this ^ true; } - }; - - template - struct base8_numeric : base8 { - static simdjson_inline simd8 splat(T _value) { return _mm_set1_epi8(_value); } - static simdjson_inline simd8 zero() { return _mm_setzero_si128(); } - static simdjson_inline simd8 load(const T values[16]) { - return _mm_loadu_si128(reinterpret_cast(values)); - } - // Repeat 16 values as many times as necessary (usually for lookup tables) - static simdjson_inline simd8 repeat_16( - T v0, T v1, T v2, T v3, T v4, T v5, T v6, T v7, - T v8, T v9, T v10, T v11, T v12, T v13, T v14, T v15 - ) { - return simd8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10, v11, v12, v13, v14, v15 - ); - } - - simdjson_inline base8_numeric() : base8() {} - simdjson_inline base8_numeric(const __m128i _value) : base8(_value) {} - - // Store to array - simdjson_inline void store(T dst[16]) const { return _mm_storeu_si128(reinterpret_cast<__m128i*>(dst), *this); } - - // Override to distinguish from bool version - simdjson_inline simd8 operator~() const { return *this ^ 0xFFu; } - - // Addition/subtraction are the same for signed and unsigned - simdjson_inline simd8 operator+(const simd8 other) const { return _mm_add_epi8(*this, other); } - simdjson_inline simd8 operator-(const simd8 other) const { return _mm_sub_epi8(*this, other); } - simdjson_inline simd8& operator+=(const simd8 other) { *this = *this + other; return *static_cast*>(this); } - simdjson_inline simd8& operator-=(const simd8 other) { *this = *this - other; return *static_cast*>(this); } - - // Perform a lookup assuming the value is between 0 and 16 (undefined behavior for out of range values) - template - simdjson_inline simd8 lookup_16(simd8 lookup_table) const { - return _mm_shuffle_epi8(lookup_table, *this); - } - - // Copies to 'output" all bytes corresponding to a 0 in the mask (interpreted as a bitset). - // Passing a 0 value for mask would be equivalent to writing out every byte to output. - // Only the first 16 - count_ones(mask) bytes of the result are significant but 16 bytes - // get written. - // Design consideration: it seems like a function with the - // signature simd8 compress(uint32_t mask) would be - // sensible, but the AVX ISA makes this kind of approach difficult. - template - simdjson_inline void compress(uint16_t mask, L* output) const { - using internal::thintable_epi8; - using internal::BitsSetTable256mul2; - using internal::pshufb_combine_table; - // this particular implementation was inspired by work done by @animetosho - // we do it in two steps, first 8 bytes and then second 8 bytes - uint8_t mask1 = uint8_t(mask); // least significant 8 bits - uint8_t mask2 = uint8_t(mask >> 8); // most significant 8 bits - // next line just loads the 64-bit values thintable_epi8[mask1] and - // thintable_epi8[mask2] into a 128-bit register, using only - // two instructions on most compilers. - __m128i shufmask = _mm_set_epi64x(thintable_epi8[mask2], thintable_epi8[mask1]); - // we increment by 0x08 the second half of the mask - shufmask = - _mm_add_epi8(shufmask, _mm_set_epi32(0x08080808, 0x08080808, 0, 0)); - // this is the version "nearly pruned" - __m128i pruned = _mm_shuffle_epi8(*this, shufmask); - // we still need to put the two halves together. - // we compute the popcount of the first half: - int pop1 = BitsSetTable256mul2[mask1]; - // then load the corresponding mask, what it does is to write - // only the first pop1 bytes from the first 8 bytes, and then - // it fills in with the bytes from the second 8 bytes + some filling - // at the end. - __m128i compactmask = - _mm_loadu_si128(reinterpret_cast(pshufb_combine_table + pop1 * 8)); - __m128i answer = _mm_shuffle_epi8(pruned, compactmask); - _mm_storeu_si128(reinterpret_cast<__m128i*>(output), answer); - } - - template - simdjson_inline simd8 lookup_16( - L replace0, L replace1, L replace2, L replace3, - L replace4, L replace5, L replace6, L replace7, - L replace8, L replace9, L replace10, L replace11, - L replace12, L replace13, L replace14, L replace15) const { - return lookup_16(simd8::repeat_16( - replace0, replace1, replace2, replace3, - replace4, replace5, replace6, replace7, - replace8, replace9, replace10, replace11, - replace12, replace13, replace14, replace15 - )); - } - }; - - // Signed bytes - template<> - struct simd8 : base8_numeric { - simdjson_inline simd8() : base8_numeric() {} - simdjson_inline simd8(const __m128i _value) : base8_numeric(_value) {} - // Splat constructor - simdjson_inline simd8(int8_t _value) : simd8(splat(_value)) {} - // Array constructor - simdjson_inline simd8(const int8_t* values) : simd8(load(values)) {} - // Member-by-member initialization - simdjson_inline simd8( - int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, - int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 - ) : simd8(_mm_setr_epi8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10, v11, v12, v13, v14, v15 - )) {} - // Repeat 16 values as many times as necessary (usually for lookup tables) - simdjson_inline static simd8 repeat_16( - int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, - int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 - ) { - return simd8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10, v11, v12, v13, v14, v15 - ); - } - - // Order-sensitive comparisons - simdjson_inline simd8 max_val(const simd8 other) const { return _mm_max_epi8(*this, other); } - simdjson_inline simd8 min_val(const simd8 other) const { return _mm_min_epi8(*this, other); } - simdjson_inline simd8 operator>(const simd8 other) const { return _mm_cmpgt_epi8(*this, other); } - simdjson_inline simd8 operator<(const simd8 other) const { return _mm_cmpgt_epi8(other, *this); } - }; - - // Unsigned bytes - template<> - struct simd8 : base8_numeric { - simdjson_inline simd8() : base8_numeric() {} - simdjson_inline simd8(const __m128i _value) : base8_numeric(_value) {} - // Splat constructor - simdjson_inline simd8(uint8_t _value) : simd8(splat(_value)) {} - // Array constructor - simdjson_inline simd8(const uint8_t* values) : simd8(load(values)) {} - // Member-by-member initialization - simdjson_inline simd8( - uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, - uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 - ) : simd8(_mm_setr_epi8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10, v11, v12, v13, v14, v15 - )) {} - // Repeat 16 values as many times as necessary (usually for lookup tables) - simdjson_inline static simd8 repeat_16( - uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, - uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 - ) { - return simd8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10, v11, v12, v13, v14, v15 - ); - } - - // Saturated math - simdjson_inline simd8 saturating_add(const simd8 other) const { return _mm_adds_epu8(*this, other); } - simdjson_inline simd8 saturating_sub(const simd8 other) const { return _mm_subs_epu8(*this, other); } - - // Order-specific operations - simdjson_inline simd8 max_val(const simd8 other) const { return _mm_max_epu8(*this, other); } - simdjson_inline simd8 min_val(const simd8 other) const { return _mm_min_epu8(*this, other); } - // Same as >, but only guarantees true is nonzero (< guarantees true = -1) - simdjson_inline simd8 gt_bits(const simd8 other) const { return this->saturating_sub(other); } - // Same as <, but only guarantees true is nonzero (< guarantees true = -1) - simdjson_inline simd8 lt_bits(const simd8 other) const { return other.saturating_sub(*this); } - simdjson_inline simd8 operator<=(const simd8 other) const { return other.max_val(*this) == other; } - simdjson_inline simd8 operator>=(const simd8 other) const { return other.min_val(*this) == other; } - simdjson_inline simd8 operator>(const simd8 other) const { return this->gt_bits(other).any_bits_set(); } - simdjson_inline simd8 operator<(const simd8 other) const { return this->gt_bits(other).any_bits_set(); } - - // Bit-specific operations - simdjson_inline simd8 bits_not_set() const { return *this == uint8_t(0); } - simdjson_inline simd8 bits_not_set(simd8 bits) const { return (*this & bits).bits_not_set(); } - simdjson_inline simd8 any_bits_set() const { return ~this->bits_not_set(); } - simdjson_inline simd8 any_bits_set(simd8 bits) const { return ~this->bits_not_set(bits); } - simdjson_inline bool is_ascii() const { return _mm_movemask_epi8(*this) == 0; } - simdjson_inline bool bits_not_set_anywhere() const { return _mm_testz_si128(*this, *this); } - simdjson_inline bool any_bits_set_anywhere() const { return !bits_not_set_anywhere(); } - simdjson_inline bool bits_not_set_anywhere(simd8 bits) const { return _mm_testz_si128(*this, bits); } - simdjson_inline bool any_bits_set_anywhere(simd8 bits) const { return !bits_not_set_anywhere(bits); } - template - simdjson_inline simd8 shr() const { return simd8(_mm_srli_epi16(*this, N)) & uint8_t(0xFFu >> N); } - template - simdjson_inline simd8 shl() const { return simd8(_mm_slli_epi16(*this, N)) & uint8_t(0xFFu << N); } - // Get one of the bits and make a bitmask out of it. - // e.g. value.get_bit<7>() gets the high bit - template - simdjson_inline int get_bit() const { return _mm_movemask_epi8(_mm_slli_epi16(*this, 7 - N)); } - }; - - template - struct simd8x64 { - static constexpr int NUM_CHUNKS = 64 / sizeof(simd8); - static_assert(NUM_CHUNKS == 4, "Westmere kernel should use four registers per 64-byte block."); - const simd8 chunks[NUM_CHUNKS]; - - simd8x64(const simd8x64& o) = delete; // no copy allowed - simd8x64& operator=(const simd8& other) = delete; // no assignment allowed - simd8x64() = delete; // no default constructor allowed - - simdjson_inline simd8x64(const simd8 chunk0, const simd8 chunk1, const simd8 chunk2, const simd8 chunk3) : chunks{ chunk0, chunk1, chunk2, chunk3 } {} - simdjson_inline simd8x64(const T ptr[64]) : chunks{ simd8::load(ptr), simd8::load(ptr + 16), simd8::load(ptr + 32), simd8::load(ptr + 48) } {} - - simdjson_inline void store(T ptr[64]) const { - this->chunks[0].store(ptr + sizeof(simd8) * 0); - this->chunks[1].store(ptr + sizeof(simd8) * 1); - this->chunks[2].store(ptr + sizeof(simd8) * 2); - this->chunks[3].store(ptr + sizeof(simd8) * 3); - } - - simdjson_inline simd8 reduce_or() const { - return (this->chunks[0] | this->chunks[1]) | (this->chunks[2] | this->chunks[3]); - } - - simdjson_inline uint64_t compress(uint64_t mask, T* output) const { - this->chunks[0].compress(uint16_t(mask), output); - this->chunks[1].compress(uint16_t(mask >> 16), output + 16 - count_ones(mask & 0xFFFF)); - this->chunks[2].compress(uint16_t(mask >> 32), output + 32 - count_ones(mask & 0xFFFFFFFF)); - this->chunks[3].compress(uint16_t(mask >> 48), output + 48 - count_ones(mask & 0xFFFFFFFFFFFF)); - return 64 - count_ones(mask); - } - - simdjson_inline uint64_t to_bitmask() const { - uint64_t r0 = uint32_t(this->chunks[0].to_bitmask()); - uint64_t r1 = this->chunks[1].to_bitmask(); - uint64_t r2 = this->chunks[2].to_bitmask(); - uint64_t r3 = this->chunks[3].to_bitmask(); - return r0 | (r1 << 16) | (r2 << 32) | (r3 << 48); - } - - simdjson_inline uint64_t eq(const T m) const { - const simd8 mask = simd8::splat(m); - return simd8x64( - this->chunks[0] == mask, - this->chunks[1] == mask, - this->chunks[2] == mask, - this->chunks[3] == mask - ).to_bitmask(); - } - - simdjson_inline uint64_t eq(const simd8x64& other) const { - return simd8x64( - this->chunks[0] == other.chunks[0], - this->chunks[1] == other.chunks[1], - this->chunks[2] == other.chunks[2], - this->chunks[3] == other.chunks[3] - ).to_bitmask(); - } - - simdjson_inline uint64_t lteq(const T m) const { - const simd8 mask = simd8::splat(m); - return simd8x64( - this->chunks[0] <= mask, - this->chunks[1] <= mask, - this->chunks[2] <= mask, - this->chunks[3] <= mask - ).to_bitmask(); - } - }; // struct simd8x64 - - } // namespace simd - } // unnamed namespace - } // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_WESTMERE_SIMD_INPUT_H -/* end file simdjson/westmere/simd.h */ - -namespace simdjson { - namespace westmere { - namespace { - - using namespace simd; - - // Holds backslashes and quotes locations. - struct backslash_and_quote { - public: - static constexpr uint32_t BYTES_PROCESSED = 32; - simdjson_inline static backslash_and_quote copy_and_find(const uint8_t* src, uint8_t* dst); - - simdjson_inline bool has_quote_first() { return ((bs_bits - 1) & quote_bits) != 0; } - simdjson_inline bool has_backslash() { return bs_bits != 0; } - simdjson_inline int quote_index() { return trailing_zeroes(quote_bits); } - simdjson_inline int backslash_index() { return trailing_zeroes(bs_bits); } - - uint32_t bs_bits; - uint32_t quote_bits; - }; // struct backslash_and_quote - - simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8_t* src, uint8_t* dst) { - // this can read up to 31 bytes beyond the buffer size, but we require - // SIMDJSON_PADDING of padding - static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "backslash and quote finder must process fewer than SIMDJSON_PADDING bytes"); - simd8 v0(src); - simd8 v1(src + 16); - v0.store(dst); - v1.store(dst + 16); - uint64_t bs_and_quote = simd8x64(v0 == '\\', v1 == '\\', v0 == '"', v1 == '"').to_bitmask(); - return { - uint32_t(bs_and_quote), // bs_bits - uint32_t(bs_and_quote >> 32) // quote_bits - }; - } - - } // unnamed namespace - } // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_WESTMERE_STRINGPARSING_DEFS_H -/* end file simdjson/westmere/stringparsing_defs.h */ -/* end file simdjson/westmere/begin.h */ -/* including simdjson/generic/amalgamated.h for westmere: #include "simdjson/generic/amalgamated.h" */ -/* begin file simdjson/generic/amalgamated.h for westmere */ -#if defined(SIMDJSON_CONDITIONAL_INCLUDE) && !defined(SIMDJSON_GENERIC_DEPENDENCIES_H) -#error simdjson/generic/dependencies.h must be included before simdjson/generic/amalgamated.h! -#endif - -/* including simdjson/generic/base.h for westmere: #include "simdjson/generic/base.h" */ -/* begin file simdjson/generic/base.h for westmere */ -#ifndef SIMDJSON_GENERIC_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_BASE_H */ -/* amalgamation skipped (editor-only): #include "simdjson/base.h" */ -/* amalgamation skipped (editor-only): // If we haven't got an implementation yet, we're in the editor, editing a generic file! Just */ -/* amalgamation skipped (editor-only): // use the most advanced one we can so the most possible stuff can be tested. */ -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_IMPLEMENTATION */ -/* amalgamation skipped (editor-only): #include "simdjson/implementation_detection.h" */ -/* amalgamation skipped (editor-only): #if SIMDJSON_IMPLEMENTATION_ICELAKE */ -/* amalgamation skipped (editor-only): #include "simdjson/icelake/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_HASWELL */ -/* amalgamation skipped (editor-only): #include "simdjson/haswell/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_WESTMERE */ -/* amalgamation skipped (editor-only): #include "simdjson/westmere/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_ARM64 */ -/* amalgamation skipped (editor-only): #include "simdjson/arm64/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_PPC64 */ -/* amalgamation skipped (editor-only): #include "simdjson/ppc64/begin.h" */ -/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_FALLBACK */ -/* amalgamation skipped (editor-only): #include "simdjson/fallback/begin.h" */ -/* amalgamation skipped (editor-only): #else */ -/* amalgamation skipped (editor-only): #error "All possible implementations (including fallback) have been disabled! simdjson will not run." */ -/* amalgamation skipped (editor-only): #endif */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_IMPLEMENTATION */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - namespace westmere { - - struct open_container; - class dom_parser_implementation; - - /** - * The type of a JSON number - */ - enum class number_type { - floating_point_number = 1, /// a binary64 number - signed_integer, /// a signed integer that fits in a 64-bit word using two's complement - unsigned_integer /// a positive integer larger or equal to 1<<63 - }; - - } // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_BASE_H -/* end file simdjson/generic/base.h for westmere */ -/* including simdjson/generic/jsoncharutils.h for westmere: #include "simdjson/generic/jsoncharutils.h" */ -/* begin file simdjson/generic/jsoncharutils.h for westmere */ -#ifndef SIMDJSON_GENERIC_JSONCHARUTILS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_JSONCHARUTILS_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/jsoncharutils_tables.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - namespace westmere { - namespace { - namespace jsoncharutils { - - // return non-zero if not a structural or whitespace char - // zero otherwise - simdjson_inline uint32_t is_not_structural_or_whitespace(uint8_t c) { - return internal::structural_or_whitespace_negated[c]; - } - - simdjson_inline uint32_t is_structural_or_whitespace(uint8_t c) { - return internal::structural_or_whitespace[c]; - } - - // returns a value with the high 16 bits set if not valid - // otherwise returns the conversion of the 4 hex digits at src into the bottom - // 16 bits of the 32-bit return register - // - // see - // https://lemire.me/blog/2019/04/17/parsing-short-hexadecimal-strings-efficiently/ - static inline uint32_t hex_to_u32_nocheck( - const uint8_t* src) { // strictly speaking, static inline is a C-ism - uint32_t v1 = internal::digit_to_val32[630 + src[0]]; - uint32_t v2 = internal::digit_to_val32[420 + src[1]]; - uint32_t v3 = internal::digit_to_val32[210 + src[2]]; - uint32_t v4 = internal::digit_to_val32[0 + src[3]]; - return v1 | v2 | v3 | v4; - } - - // given a code point cp, writes to c - // the utf-8 code, outputting the length in - // bytes, if the length is zero, the code point - // is invalid - // - // This can possibly be made faster using pdep - // and clz and table lookups, but JSON documents - // have few escaped code points, and the following - // function looks cheap. - // - // Note: we assume that surrogates are treated separately - // - simdjson_inline size_t codepoint_to_utf8(uint32_t cp, uint8_t* c) { - if (cp <= 0x7F) { - c[0] = uint8_t(cp); - return 1; // ascii - } - if (cp <= 0x7FF) { - c[0] = uint8_t((cp >> 6) + 192); - c[1] = uint8_t((cp & 63) + 128); - return 2; // universal plane - // Surrogates are treated elsewhere... - //} //else if (0xd800 <= cp && cp <= 0xdfff) { - // return 0; // surrogates // could put assert here - } - else if (cp <= 0xFFFF) { - c[0] = uint8_t((cp >> 12) + 224); - c[1] = uint8_t(((cp >> 6) & 63) + 128); - c[2] = uint8_t((cp & 63) + 128); - return 3; - } - else if (cp <= 0x10FFFF) { // if you know you have a valid code point, this - // is not needed - c[0] = uint8_t((cp >> 18) + 240); - c[1] = uint8_t(((cp >> 12) & 63) + 128); - c[2] = uint8_t(((cp >> 6) & 63) + 128); - c[3] = uint8_t((cp & 63) + 128); - return 4; - } - // will return 0 when the code point was too large. - return 0; // bad r - } - -#if SIMDJSON_IS_32BITS // _umul128 for x86, arm - // this is a slow emulation routine for 32-bit - // - static simdjson_inline uint64_t __emulu(uint32_t x, uint32_t y) { - return x * (uint64_t)y; - } - static simdjson_inline uint64_t _umul128(uint64_t ab, uint64_t cd, uint64_t* hi) { - uint64_t ad = __emulu((uint32_t)(ab >> 32), (uint32_t)cd); - uint64_t bd = __emulu((uint32_t)ab, (uint32_t)cd); - uint64_t adbc = ad + __emulu((uint32_t)ab, (uint32_t)(cd >> 32)); - uint64_t adbc_carry = !!(adbc < ad); - uint64_t lo = bd + (adbc << 32); - *hi = __emulu((uint32_t)(ab >> 32), (uint32_t)(cd >> 32)) + (adbc >> 32) + - (adbc_carry << 32) + !!(lo < bd); - return lo; - } -#endif - - } // namespace jsoncharutils - } // unnamed namespace - } // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_JSONCHARUTILS_H -/* end file simdjson/generic/jsoncharutils.h for westmere */ -/* including simdjson/generic/atomparsing.h for westmere: #include "simdjson/generic/atomparsing.h" */ -/* begin file simdjson/generic/atomparsing.h for westmere */ -#ifndef SIMDJSON_GENERIC_ATOMPARSING_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_ATOMPARSING_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/jsoncharutils.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include - -namespace simdjson { - namespace westmere { - namespace { - /// @private - namespace atomparsing { - - // The string_to_uint32 is exclusively used to map literal strings to 32-bit values. - // We use memcpy instead of a pointer cast to avoid undefined behaviors since we cannot - // be certain that the character pointer will be properly aligned. - // You might think that using memcpy makes this function expensive, but you'd be wrong. - // All decent optimizing compilers (GCC, clang, Visual Studio) will compile string_to_uint32("false"); - // to the compile-time constant 1936482662. - simdjson_inline uint32_t string_to_uint32(const char* str) { uint32_t val; std::memcpy(&val, str, sizeof(uint32_t)); return val; } - - - // Again in str4ncmp we use a memcpy to avoid undefined behavior. The memcpy may appear expensive. - // Yet all decent optimizing compilers will compile memcpy to a single instruction, just about. - simdjson_warn_unused - simdjson_inline uint32_t str4ncmp(const uint8_t* src, const char* atom) { - uint32_t srcval; // we want to avoid unaligned 32-bit loads (undefined in C/C++) - static_assert(sizeof(uint32_t) <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be larger than 4 bytes"); - std::memcpy(&srcval, src, sizeof(uint32_t)); - return srcval ^ string_to_uint32(atom); - } - - simdjson_warn_unused - simdjson_inline bool is_valid_true_atom(const uint8_t* src) { - return (str4ncmp(src, "true") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; - } - - simdjson_warn_unused - simdjson_inline bool is_valid_true_atom(const uint8_t* src, size_t len) { - if (len > 4) { return is_valid_true_atom(src); } - else if (len == 4) { return !str4ncmp(src, "true"); } - else { return false; } - } - - simdjson_warn_unused - simdjson_inline bool is_valid_false_atom(const uint8_t* src) { - return (str4ncmp(src + 1, "alse") | jsoncharutils::is_not_structural_or_whitespace(src[5])) == 0; - } - - simdjson_warn_unused - simdjson_inline bool is_valid_false_atom(const uint8_t* src, size_t len) { - if (len > 5) { return is_valid_false_atom(src); } - else if (len == 5) { return !str4ncmp(src + 1, "alse"); } - else { return false; } - } - - simdjson_warn_unused - simdjson_inline bool is_valid_null_atom(const uint8_t* src) { - return (str4ncmp(src, "null") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; - } - - simdjson_warn_unused - simdjson_inline bool is_valid_null_atom(const uint8_t* src, size_t len) { - if (len > 4) { return is_valid_null_atom(src); } - else if (len == 4) { return !str4ncmp(src, "null"); } - else { return false; } - } - - } // namespace atomparsing - } // unnamed namespace - } // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_ATOMPARSING_H -/* end file simdjson/generic/atomparsing.h for westmere */ -/* including simdjson/generic/dom_parser_implementation.h for westmere: #include "simdjson/generic/dom_parser_implementation.h" */ -/* begin file simdjson/generic/dom_parser_implementation.h for westmere */ -#ifndef SIMDJSON_GENERIC_DOM_PARSER_IMPLEMENTATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_DOM_PARSER_IMPLEMENTATION_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/dom_parser_implementation.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - namespace westmere { - - // expectation: sizeof(open_container) = 64/8. - struct open_container { - uint32_t tape_index; // where, on the tape, does the scope ([,{) begins - uint32_t count; // how many elements in the scope - }; // struct open_container - - static_assert(sizeof(open_container) == 64 / 8, "Open container must be 64 bits"); - - class dom_parser_implementation final : public internal::dom_parser_implementation { - public: - /** Tape location of each open { or [ */ - std::unique_ptr open_containers{}; - /** Whether each open container is a [ or { */ - std::unique_ptr is_array{}; - /** Buffer passed to stage 1 */ - const uint8_t* buf{}; - /** Length passed to stage 1 */ - size_t len{ 0 }; - /** Document passed to stage 2 */ - dom::document* doc{}; - - inline dom_parser_implementation() noexcept; - inline dom_parser_implementation(dom_parser_implementation&& other) noexcept; - inline dom_parser_implementation& operator=(dom_parser_implementation&& other) noexcept; - dom_parser_implementation(const dom_parser_implementation&) = delete; - dom_parser_implementation& operator=(const dom_parser_implementation&) = delete; - - simdjson_warn_unused error_code parse(const uint8_t* buf, size_t len, dom::document& doc) noexcept final; - simdjson_warn_unused error_code stage1(const uint8_t* buf, size_t len, stage1_mode partial) noexcept final; - simdjson_warn_unused error_code stage2(dom::document& doc) noexcept final; - simdjson_warn_unused error_code stage2_next(dom::document& doc) noexcept final; - simdjson_warn_unused uint8_t* parse_string(const uint8_t* src, uint8_t* dst, bool allow_replacement) const noexcept final; - simdjson_warn_unused uint8_t* parse_wobbly_string(const uint8_t* src, uint8_t* dst) const noexcept final; - inline simdjson_warn_unused error_code set_capacity(size_t capacity) noexcept final; - inline simdjson_warn_unused error_code set_max_depth(size_t max_depth) noexcept final; - private: - simdjson_inline simdjson_warn_unused error_code set_capacity_stage1(size_t capacity); - - }; - - } // namespace westmere -} // namespace simdjson - -namespace simdjson { - namespace westmere { - - inline dom_parser_implementation::dom_parser_implementation() noexcept = default; - inline dom_parser_implementation::dom_parser_implementation(dom_parser_implementation&& other) noexcept = default; - inline dom_parser_implementation& dom_parser_implementation::operator=(dom_parser_implementation&& other) noexcept = default; - - // Leaving these here so they can be inlined if so desired - inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(size_t capacity) noexcept { - if (capacity > SIMDJSON_MAXSIZE_BYTES) { return CAPACITY; } - // Stage 1 index output - size_t max_structures = SIMDJSON_ROUNDUP_N(capacity, 64) + 2 + 7; - structural_indexes.reset(new (std::nothrow) uint32_t[max_structures]); - if (!structural_indexes) { _capacity = 0; return MEMALLOC; } - structural_indexes[0] = 0; - n_structural_indexes = 0; - - _capacity = capacity; - return SUCCESS; - } - - inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { - // Stage 2 stacks - open_containers.reset(new (std::nothrow) open_container[max_depth]); - is_array.reset(new (std::nothrow) bool[max_depth]); - if (!is_array || !open_containers) { _max_depth = 0; return MEMALLOC; } - - _max_depth = max_depth; - return SUCCESS; - } - - } // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_DOM_PARSER_IMPLEMENTATION_H -/* end file simdjson/generic/dom_parser_implementation.h for westmere */ -/* including simdjson/generic/implementation_simdjson_result_base.h for westmere: #include "simdjson/generic/implementation_simdjson_result_base.h" */ -/* begin file simdjson/generic/implementation_simdjson_result_base.h for westmere */ -#ifndef SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - namespace westmere { - - // This is a near copy of include/error.h's implementation_simdjson_result_base, except it doesn't use std::pair - // so we can avoid inlining errors - // TODO reconcile these! - /** - * The result of a simdjson operation that could fail. - * - * Gives the option of reading error codes, or throwing an exception by casting to the desired result. - * - * This is a base class for implementations that want to add functions to the result type for - * chaining. - * - * Override like: - * - * struct simdjson_result : public internal::implementation_simdjson_result_base { - * simdjson_result() noexcept : internal::implementation_simdjson_result_base() {} - * simdjson_result(error_code error) noexcept : internal::implementation_simdjson_result_base(error) {} - * simdjson_result(T &&value) noexcept : internal::implementation_simdjson_result_base(std::forward(value)) {} - * simdjson_result(T &&value, error_code error) noexcept : internal::implementation_simdjson_result_base(value, error) {} - * // Your extra methods here - * } - * - * Then any method returning simdjson_result will be chainable with your methods. - */ - template - struct implementation_simdjson_result_base { - - /** - * Create a new empty result with error = UNINITIALIZED. - */ - simdjson_inline implementation_simdjson_result_base() noexcept = default; - - /** - * Create a new error result. - */ - simdjson_inline implementation_simdjson_result_base(error_code error) noexcept; - - /** - * Create a new successful result. - */ - simdjson_inline implementation_simdjson_result_base(T&& value) noexcept; - - /** - * Create a new result with both things (use if you don't want to branch when creating the result). - */ - simdjson_inline implementation_simdjson_result_base(T&& value, error_code error) noexcept; - - /** - * Move the value and the error to the provided variables. - * - * @param value The variable to assign the value to. May not be set if there is an error. - * @param error The variable to assign the error to. Set to SUCCESS if there is no error. - */ - simdjson_inline void tie(T& value, error_code& error) && noexcept; - - /** - * Move the value to the provided variable. - * - * @param value The variable to assign the value to. May not be set if there is an error. - */ - simdjson_inline error_code get(T& value) && noexcept; - - /** - * The error. - */ - simdjson_inline error_code error() const noexcept; - -#if SIMDJSON_EXCEPTIONS - - /** - * Get the result value. - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline T& value() & noexcept(false); - - /** - * Take the result value (move it). - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline T&& value() && noexcept(false); - - /** - * Take the result value (move it). - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline T&& take_value() && noexcept(false); - - /** - * Cast to the value (will throw on error). - * - * @throw simdjson_error if there was an error. - */ - simdjson_inline operator T && () && noexcept(false); - - -#endif // SIMDJSON_EXCEPTIONS - - /** - * Get the result value. This function is safe if and only - * the error() method returns a value that evaluates to false. - */ - simdjson_inline const T& value_unsafe() const& noexcept; - /** - * Get the result value. This function is safe if and only - * the error() method returns a value that evaluates to false. - */ - simdjson_inline T& value_unsafe() & noexcept; - /** - * Take the result value (move it). This function is safe if and only - * the error() method returns a value that evaluates to false. - */ - simdjson_inline T&& value_unsafe() && noexcept; - protected: - /** users should never directly access first and second. **/ - T first{}; /** Users should never directly access 'first'. **/ - error_code second{ UNINITIALIZED }; /** Users should never directly access 'second'. **/ - }; // struct implementation_simdjson_result_base - - } // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_H -/* end file simdjson/generic/implementation_simdjson_result_base.h for westmere */ -/* including simdjson/generic/numberparsing.h for westmere: #include "simdjson/generic/numberparsing.h" */ -/* begin file simdjson/generic/numberparsing.h for westmere */ -#ifndef SIMDJSON_GENERIC_NUMBERPARSING_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_NUMBERPARSING_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/jsoncharutils.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include -#include -#include - -namespace simdjson { - namespace westmere { - namespace numberparsing { - -#ifdef JSON_TEST_NUMBERS -#define INVALID_NUMBER(SRC) (found_invalid_number((SRC)), NUMBER_ERROR) -#define WRITE_INTEGER(VALUE, SRC, WRITER) (found_integer((VALUE), (SRC)), (WRITER).append_s64((VALUE))) -#define WRITE_UNSIGNED(VALUE, SRC, WRITER) (found_unsigned_integer((VALUE), (SRC)), (WRITER).append_u64((VALUE))) -#define WRITE_DOUBLE(VALUE, SRC, WRITER) (found_float((VALUE), (SRC)), (WRITER).append_double((VALUE))) -#else -#define INVALID_NUMBER(SRC) (NUMBER_ERROR) -#define WRITE_INTEGER(VALUE, SRC, WRITER) (WRITER).append_s64((VALUE)) -#define WRITE_UNSIGNED(VALUE, SRC, WRITER) (WRITER).append_u64((VALUE)) -#define WRITE_DOUBLE(VALUE, SRC, WRITER) (WRITER).append_double((VALUE)) -#endif - - namespace { - - // Convert a mantissa, an exponent and a sign bit into an ieee64 double. - // The real_exponent needs to be in [0, 2046] (technically real_exponent = 2047 would be acceptable). - // The mantissa should be in [0,1<<53). The bit at index (1ULL << 52) while be zeroed. - simdjson_inline double to_double(uint64_t mantissa, uint64_t real_exponent, bool negative) { - double d; - mantissa &= ~(1ULL << 52); - mantissa |= real_exponent << 52; - mantissa |= ((static_cast(negative)) << 63); - std::memcpy(&d, &mantissa, sizeof(d)); - return d; - } - - // Attempts to compute i * 10^(power) exactly; and if "negative" is - // true, negate the result. - // This function will only work in some cases, when it does not work, success is - // set to false. This should work *most of the time* (like 99% of the time). - // We assume that power is in the [smallest_power, - // largest_power] interval: the caller is responsible for this check. - simdjson_inline bool compute_float_64(int64_t power, uint64_t i, bool negative, double& d) { - // we start with a fast path - // It was described in - // Clinger WD. How to read floating point numbers accurately. - // ACM SIGPLAN Notices. 1990 -#ifndef FLT_EVAL_METHOD -#error "FLT_EVAL_METHOD should be defined, please include cfloat." -#endif -#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0) - // We cannot be certain that x/y is rounded to nearest. - if (0 <= power && power <= 22 && i <= 9007199254740991) -#else - if (-22 <= power && power <= 22 && i <= 9007199254740991) -#endif - { - // convert the integer into a double. This is lossless since - // 0 <= i <= 2^53 - 1. - d = double(i); - // - // The general idea is as follows. - // If 0 <= s < 2^53 and if 10^0 <= p <= 10^22 then - // 1) Both s and p can be represented exactly as 64-bit floating-point - // values - // (binary64). - // 2) Because s and p can be represented exactly as floating-point values, - // then s * p - // and s / p will produce correctly rounded values. - // - if (power < 0) { - d = d / simdjson::internal::power_of_ten[-power]; - } - else { - d = d * simdjson::internal::power_of_ten[power]; - } - if (negative) { - d = -d; - } - return true; - } - // When 22 < power && power < 22 + 16, we could - // hope for another, secondary fast path. It was - // described by David M. Gay in "Correctly rounded - // binary-decimal and decimal-binary conversions." (1990) - // If you need to compute i * 10^(22 + x) for x < 16, - // first compute i * 10^x, if you know that result is exact - // (e.g., when i * 10^x < 2^53), - // then you can still proceed and do (i * 10^x) * 10^22. - // Is this worth your time? - // You need 22 < power *and* power < 22 + 16 *and* (i * 10^(x-22) < 2^53) - // for this second fast path to work. - // If you you have 22 < power *and* power < 22 + 16, and then you - // optimistically compute "i * 10^(x-22)", there is still a chance that you - // have wasted your time if i * 10^(x-22) >= 2^53. It makes the use cases of - // this optimization maybe less common than we would like. Source: - // http://www.exploringbinary.com/fast-path-decimal-to-floating-point-conversion/ - // also used in RapidJSON: https://rapidjson.org/strtod_8h_source.html - - // The fast path has now failed, so we are failing back on the slower path. - - // In the slow path, we need to adjust i so that it is > 1<<63 which is always - // possible, except if i == 0, so we handle i == 0 separately. - if (i == 0) { - d = negative ? -0.0 : 0.0; - return true; - } - - - // The exponent is 1024 + 63 + power - // + floor(log(5**power)/log(2)). - // The 1024 comes from the ieee64 standard. - // The 63 comes from the fact that we use a 64-bit word. - // - // Computing floor(log(5**power)/log(2)) could be - // slow. Instead we use a fast function. - // - // For power in (-400,350), we have that - // (((152170 + 65536) * power ) >> 16); - // is equal to - // floor(log(5**power)/log(2)) + power when power >= 0 - // and it is equal to - // ceil(log(5**-power)/log(2)) + power when power < 0 - // - // The 65536 is (1<<16) and corresponds to - // (65536 * power) >> 16 ---> power - // - // ((152170 * power ) >> 16) is equal to - // floor(log(5**power)/log(2)) - // - // Note that this is not magic: 152170/(1<<16) is - // approximatively equal to log(5)/log(2). - // The 1<<16 value is a power of two; we could use a - // larger power of 2 if we wanted to. - // - int64_t exponent = (((152170 + 65536) * power) >> 16) + 1024 + 63; - - - // We want the most significant bit of i to be 1. Shift if needed. - int lz = leading_zeroes(i); - i <<= lz; - - - // We are going to need to do some 64-bit arithmetic to get a precise product. - // We use a table lookup approach. - // It is safe because - // power >= smallest_power - // and power <= largest_power - // We recover the mantissa of the power, it has a leading 1. It is always - // rounded down. - // - // We want the most significant 64 bits of the product. We know - // this will be non-zero because the most significant bit of i is - // 1. - const uint32_t index = 2 * uint32_t(power - simdjson::internal::smallest_power); - // Optimization: It may be that materializing the index as a variable might confuse some compilers and prevent effective complex-addressing loads. (Done for code clarity.) - // - // The full_multiplication function computes the 128-bit product of two 64-bit words - // with a returned value of type value128 with a "low component" corresponding to the - // 64-bit least significant bits of the product and with a "high component" corresponding - // to the 64-bit most significant bits of the product. - simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index]); - // Both i and power_of_five_128[index] have their most significant bit set to 1 which - // implies that the either the most or the second most significant bit of the product - // is 1. We pack values in this manner for efficiency reasons: it maximizes the use - // we make of the product. It also makes it easy to reason about the product: there - // is 0 or 1 leading zero in the product. - - // Unless the least significant 9 bits of the high (64-bit) part of the full - // product are all 1s, then we know that the most significant 55 bits are - // exact and no further work is needed. Having 55 bits is necessary because - // we need 53 bits for the mantissa but we have to have one rounding bit and - // we can waste a bit if the most significant bit of the product is zero. - if ((firstproduct.high & 0x1FF) == 0x1FF) { - // We want to compute i * 5^q, but only care about the top 55 bits at most. - // Consider the scenario where q>=0. Then 5^q may not fit in 64-bits. Doing - // the full computation is wasteful. So we do what is called a "truncated - // multiplication". - // We take the most significant 64-bits, and we put them in - // power_of_five_128[index]. Usually, that's good enough to approximate i * 5^q - // to the desired approximation using one multiplication. Sometimes it does not suffice. - // Then we store the next most significant 64 bits in power_of_five_128[index + 1], and - // then we get a better approximation to i * 5^q. - // - // That's for when q>=0. The logic for q<0 is somewhat similar but it is somewhat - // more complicated. - // - // There is an extra layer of complexity in that we need more than 55 bits of - // accuracy in the round-to-even scenario. - // - // The full_multiplication function computes the 128-bit product of two 64-bit words - // with a returned value of type value128 with a "low component" corresponding to the - // 64-bit least significant bits of the product and with a "high component" corresponding - // to the 64-bit most significant bits of the product. - simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index + 1]); - firstproduct.low += secondproduct.high; - if (secondproduct.high > firstproduct.low) { firstproduct.high++; } - // As it has been proven by Noble Mushtak and Daniel Lemire in "Fast Number Parsing Without - // Fallback" (https://arxiv.org/abs/2212.06644), at this point we are sure that the product - // is sufficiently accurate, and more computation is not needed. - } - uint64_t lower = firstproduct.low; - uint64_t upper = firstproduct.high; - // The final mantissa should be 53 bits with a leading 1. - // We shift it so that it occupies 54 bits with a leading 1. - /////// - uint64_t upperbit = upper >> 63; - uint64_t mantissa = upper >> (upperbit + 9); - lz += int(1 ^ upperbit); - - // Here we have mantissa < (1<<54). - int64_t real_exponent = exponent - lz; - if (simdjson_unlikely(real_exponent <= 0)) { // we have a subnormal? - // Here have that real_exponent <= 0 so -real_exponent >= 0 - if (-real_exponent + 1 >= 64) { // if we have more than 64 bits below the minimum exponent, you have a zero for sure. - d = negative ? -0.0 : 0.0; - return true; - } - // next line is safe because -real_exponent + 1 < 0 - mantissa >>= -real_exponent + 1; - // Thankfully, we can't have both "round-to-even" and subnormals because - // "round-to-even" only occurs for powers close to 0. - mantissa += (mantissa & 1); // round up - mantissa >>= 1; - // There is a weird scenario where we don't have a subnormal but just. - // Suppose we start with 2.2250738585072013e-308, we end up - // with 0x3fffffffffffff x 2^-1023-53 which is technically subnormal - // whereas 0x40000000000000 x 2^-1023-53 is normal. Now, we need to round - // up 0x3fffffffffffff x 2^-1023-53 and once we do, we are no longer - // subnormal, but we can only know this after rounding. - // So we only declare a subnormal if we are smaller than the threshold. - real_exponent = (mantissa < (uint64_t(1) << 52)) ? 0 : 1; - d = to_double(mantissa, real_exponent, negative); - return true; - } - // We have to round to even. The "to even" part - // is only a problem when we are right in between two floats - // which we guard against. - // If we have lots of trailing zeros, we may fall right between two - // floating-point values. - // - // The round-to-even cases take the form of a number 2m+1 which is in (2^53,2^54] - // times a power of two. That is, it is right between a number with binary significand - // m and another number with binary significand m+1; and it must be the case - // that it cannot be represented by a float itself. - // - // We must have that w * 10 ^q == (2m+1) * 2^p for some power of two 2^p. - // Recall that 10^q = 5^q * 2^q. - // When q >= 0, we must have that (2m+1) is divible by 5^q, so 5^q <= 2^54. We have that - // 5^23 <= 2^54 and it is the last power of five to qualify, so q <= 23. - // When q<0, we have w >= (2m+1) x 5^{-q}. We must have that w<2^{64} so - // (2m+1) x 5^{-q} < 2^{64}. We have that 2m+1>2^{53}. Hence, we must have - // 2^{53} x 5^{-q} < 2^{64}. - // Hence we have 5^{-q} < 2^{11}$ or q>= -4. - // - // We require lower <= 1 and not lower == 0 because we could not prove that - // that lower == 0 is implied; but we could prove that lower <= 1 is a necessary and sufficient test. - if (simdjson_unlikely((lower <= 1) && (power >= -4) && (power <= 23) && ((mantissa & 3) == 1))) { - if ((mantissa << (upperbit + 64 - 53 - 2)) == upper) { - mantissa &= ~1; // flip it so that we do not round up - } - } - - mantissa += mantissa & 1; - mantissa >>= 1; - - // Here we have mantissa < (1<<53), unless there was an overflow - if (mantissa >= (1ULL << 53)) { - ////////// - // This will happen when parsing values such as 7.2057594037927933e+16 - //////// - mantissa = (1ULL << 52); - real_exponent++; - } - mantissa &= ~(1ULL << 52); - // we have to check that real_exponent is in range, otherwise we bail out - if (simdjson_unlikely(real_exponent > 2046)) { - // We have an infinite value!!! We could actually throw an error here if we could. - return false; - } - d = to_double(mantissa, real_exponent, negative); - return true; - } - - // We call a fallback floating-point parser that might be slow. Note - // it will accept JSON numbers, but the JSON spec. is more restrictive so - // before you call parse_float_fallback, you need to have validated the input - // string with the JSON grammar. - // It will return an error (false) if the parsed number is infinite. - // The string parsing itself always succeeds. We know that there is at least - // one digit. - static bool parse_float_fallback(const uint8_t* ptr, double* outDouble) { - *outDouble = simdjson::internal::from_chars(reinterpret_cast(ptr)); - // We do not accept infinite values. - - // Detecting finite values in a portable manner is ridiculously hard, ideally - // we would want to do: - // return !std::isfinite(*outDouble); - // but that mysteriously fails under legacy/old libc++ libraries, see - // https://github.com/simdjson/simdjson/issues/1286 - // - // Therefore, fall back to this solution (the extra parens are there - // to handle that max may be a macro on windows). - return !(*outDouble > (std::numeric_limits::max)() || *outDouble < std::numeric_limits::lowest()); - } - - static bool parse_float_fallback(const uint8_t* ptr, const uint8_t* end_ptr, double* outDouble) { - *outDouble = simdjson::internal::from_chars(reinterpret_cast(ptr), reinterpret_cast(end_ptr)); - // We do not accept infinite values. - - // Detecting finite values in a portable manner is ridiculously hard, ideally - // we would want to do: - // return !std::isfinite(*outDouble); - // but that mysteriously fails under legacy/old libc++ libraries, see - // https://github.com/simdjson/simdjson/issues/1286 - // - // Therefore, fall back to this solution (the extra parens are there - // to handle that max may be a macro on windows). - return !(*outDouble > (std::numeric_limits::max)() || *outDouble < std::numeric_limits::lowest()); - } - - // check quickly whether the next 8 chars are made of digits - // at a glance, it looks better than Mula's - // http://0x80.pl/articles/swar-digits-validate.html - simdjson_inline bool is_made_of_eight_digits_fast(const uint8_t* chars) { - uint64_t val; - // this can read up to 7 bytes beyond the buffer size, but we require - // SIMDJSON_PADDING of padding - static_assert(7 <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be bigger than 7"); - std::memcpy(&val, chars, 8); - // a branchy method might be faster: - // return (( val & 0xF0F0F0F0F0F0F0F0 ) == 0x3030303030303030) - // && (( (val + 0x0606060606060606) & 0xF0F0F0F0F0F0F0F0 ) == - // 0x3030303030303030); - return (((val & 0xF0F0F0F0F0F0F0F0) | - (((val + 0x0606060606060606) & 0xF0F0F0F0F0F0F0F0) >> 4)) == - 0x3333333333333333); - } - - template - SIMDJSON_NO_SANITIZE_UNDEFINED // We deliberately allow overflow here and check later - simdjson_inline bool parse_digit(const uint8_t c, I& i) { - const uint8_t digit = static_cast(c - '0'); - if (digit > 9) { - return false; - } - // PERF NOTE: multiplication by 10 is cheaper than arbitrary integer multiplication - i = 10 * i + digit; // might overflow, we will handle the overflow later - return true; - } - - simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t* const src, const uint8_t*& p, uint64_t& i, int64_t& exponent) { - // we continue with the fiction that we have an integer. If the - // floating point number is representable as x * 10^z for some integer - // z that fits in 53 bits, then we will be able to convert back the - // the integer into a float in a lossless manner. - const uint8_t* const first_after_period = p; - -#ifdef SIMDJSON_SWAR_NUMBER_PARSING -#if SIMDJSON_SWAR_NUMBER_PARSING - // this helps if we have lots of decimals! - // this turns out to be frequent enough. - if (is_made_of_eight_digits_fast(p)) { - i = i * 100000000 + parse_eight_digits_unrolled(p); - p += 8; - } -#endif // SIMDJSON_SWAR_NUMBER_PARSING -#endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING - // Unrolling the first digit makes a small difference on some implementations (e.g. westmere) - if (parse_digit(*p, i)) { ++p; } - while (parse_digit(*p, i)) { p++; } - exponent = first_after_period - p; - // Decimal without digits (123.) is illegal - if (exponent == 0) { - return INVALID_NUMBER(src); - } - return SUCCESS; - } - - simdjson_inline error_code parse_exponent(simdjson_unused const uint8_t* const src, const uint8_t*& p, int64_t& exponent) { - // Exp Sign: -123.456e[-]78 - bool neg_exp = ('-' == *p); - if (neg_exp || '+' == *p) { p++; } // Skip + as well - - // Exponent: -123.456e-[78] - auto start_exp = p; - int64_t exp_number = 0; - while (parse_digit(*p, exp_number)) { ++p; } - // It is possible for parse_digit to overflow. - // In particular, it could overflow to INT64_MIN, and we cannot do - INT64_MIN. - // Thus we *must* check for possible overflow before we negate exp_number. - - // Performance notes: it may seem like combining the two "simdjson_unlikely checks" below into - // a single simdjson_unlikely path would be faster. The reasoning is sound, but the compiler may - // not oblige and may, in fact, generate two distinct paths in any case. It might be - // possible to do uint64_t(p - start_exp - 1) >= 18 but it could end up trading off - // instructions for a simdjson_likely branch, an unconclusive gain. - - // If there were no digits, it's an error. - if (simdjson_unlikely(p == start_exp)) { - return INVALID_NUMBER(src); - } - // We have a valid positive exponent in exp_number at this point, except that - // it may have overflowed. - - // If there were more than 18 digits, we may have overflowed the integer. We have to do - // something!!!! - if (simdjson_unlikely(p > start_exp + 18)) { - // Skip leading zeroes: 1e000000000000000000001 is technically valid and doesn't overflow - while (*start_exp == '0') { start_exp++; } - // 19 digits could overflow int64_t and is kind of absurd anyway. We don't - // support exponents smaller than -999,999,999,999,999,999 and bigger - // than 999,999,999,999,999,999. - // We can truncate. - // Note that 999999999999999999 is assuredly too large. The maximal ieee64 value before - // infinity is ~1.8e308. The smallest subnormal is ~5e-324. So, actually, we could - // truncate at 324. - // Note that there is no reason to fail per se at this point in time. - // E.g., 0e999999999999999999999 is a fine number. - if (p > start_exp + 18) { exp_number = 999999999999999999; } - } - // At this point, we know that exp_number is a sane, positive, signed integer. - // It is <= 999,999,999,999,999,999. As long as 'exponent' is in - // [-8223372036854775808, 8223372036854775808], we won't overflow. Because 'exponent' - // is bounded in magnitude by the size of the JSON input, we are fine in this universe. - // To sum it up: the next line should never overflow. - exponent += (neg_exp ? -exp_number : exp_number); - return SUCCESS; - } - - simdjson_inline size_t significant_digits(const uint8_t* start_digits, size_t digit_count) { - // It is possible that the integer had an overflow. - // We have to handle the case where we have 0.0000somenumber. - const uint8_t* start = start_digits; - while ((*start == '0') || (*start == '.')) { ++start; } - // we over-decrement by one when there is a '.' - return digit_count - size_t(start - start_digits); - } - - } // unnamed namespace - - /** @private */ - template - error_code slow_float_parsing(simdjson_unused const uint8_t* src, W writer) { - double d; - if (parse_float_fallback(src, &d)) { - writer.append_double(d); - return SUCCESS; - } - return INVALID_NUMBER(src); - } - - /** @private */ - template - simdjson_inline error_code write_float(const uint8_t* const src, bool negative, uint64_t i, const uint8_t* start_digits, size_t digit_count, int64_t exponent, W& writer) { - // If we frequently had to deal with long strings of digits, - // we could extend our code by using a 128-bit integer instead - // of a 64-bit integer. However, this is uncommon in practice. - // - // 9999999999999999999 < 2**64 so we can accommodate 19 digits. - // If we have a decimal separator, then digit_count - 1 is the number of digits, but we - // may not have a decimal separator! - if (simdjson_unlikely(digit_count > 19 && significant_digits(start_digits, digit_count) > 19)) { - // Ok, chances are good that we had an overflow! - // this is almost never going to get called!!! - // we start anew, going slowly!!! - // This will happen in the following examples: - // 10000000000000000000000000000000000000000000e+308 - // 3.1415926535897932384626433832795028841971693993751 - // - // NOTE: This makes a *copy* of the writer and passes it to slow_float_parsing. This happens - // because slow_float_parsing is a non-inlined function. If we passed our writer reference to - // it, it would force it to be stored in memory, preventing the compiler from picking it apart - // and putting into registers. i.e. if we pass it as reference, it gets slow. - // This is what forces the skip_double, as well. - error_code error = slow_float_parsing(src, writer); - writer.skip_double(); - return error; - } - // NOTE: it's weird that the simdjson_unlikely() only wraps half the if, but it seems to get slower any other - // way we've tried: https://github.com/simdjson/simdjson/pull/990#discussion_r448497331 - // To future reader: we'd love if someone found a better way, or at least could explain this result! - if (simdjson_unlikely(exponent < simdjson::internal::smallest_power) || (exponent > simdjson::internal::largest_power)) { - // - // Important: smallest_power is such that it leads to a zero value. - // Observe that 18446744073709551615e-343 == 0, i.e. (2**64 - 1) e -343 is zero - // so something x 10^-343 goes to zero, but not so with something x 10^-342. - static_assert(simdjson::internal::smallest_power <= -342, "smallest_power is not small enough"); - // - if ((exponent < simdjson::internal::smallest_power) || (i == 0)) { - // E.g. Parse "-0.0e-999" into the same value as "-0.0". See https://en.wikipedia.org/wiki/Signed_zero - WRITE_DOUBLE(negative ? -0.0 : 0.0, src, writer); - return SUCCESS; - } - else { // (exponent > largest_power) and (i != 0) - // We have, for sure, an infinite value and simdjson refuses to parse infinite values. - return INVALID_NUMBER(src); - } - } - double d; - if (!compute_float_64(exponent, i, negative, d)) { - // we are almost never going to get here. - if (!parse_float_fallback(src, &d)) { return INVALID_NUMBER(src); } - } - WRITE_DOUBLE(d, src, writer); - return SUCCESS; - } - - // for performance analysis, it is sometimes useful to skip parsing -#ifdef SIMDJSON_SKIPNUMBERPARSING - - template - simdjson_inline error_code parse_number(const uint8_t* const, W& writer) { - writer.append_s64(0); // always write zero - return SUCCESS; // always succeeds - } - - simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t* const src) noexcept { return 0; } - simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t* const src) noexcept { return 0; } - simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t* const src) noexcept { return 0; } - simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t* const src) noexcept { return 0; } - simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t* const src) noexcept { return 0; } - simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t* const src) noexcept { return 0; } - simdjson_unused simdjson_inline bool is_negative(const uint8_t* src) noexcept { return false; } - simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t* src) noexcept { return false; } - simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t* src) noexcept { return number_type::signed_integer; } -#else - -// parse the number at src -// define JSON_TEST_NUMBERS for unit testing -// -// It is assumed that the number is followed by a structural ({,},],[) character -// or a white space character. If that is not the case (e.g., when the JSON -// document is made of a single number), then it is necessary to copy the -// content and append a space before calling this function. -// -// Our objective is accurate parsing (ULP of 0) at high speed. - template - simdjson_inline error_code parse_number(const uint8_t* const src, W& writer) { - - // - // Check for minus sign - // - bool negative = (*src == '-'); - const uint8_t* p = src + uint8_t(negative); - - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t* const start_digits = p; - uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(p - start_digits); - if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { return INVALID_NUMBER(src); } - - // - // Handle floats if there is a . or e (or both) - // - int64_t exponent = 0; - bool is_float = false; - if ('.' == *p) { - is_float = true; - ++p; - SIMDJSON_TRY(parse_decimal_after_separator(src, p, i, exponent)); - digit_count = int(p - start_digits); // used later to guard against overflows - } - if (('e' == *p) || ('E' == *p)) { - is_float = true; - ++p; - SIMDJSON_TRY(parse_exponent(src, p, exponent)); - } - if (is_float) { - const bool dirty_end = jsoncharutils::is_not_structural_or_whitespace(*p); - SIMDJSON_TRY(write_float(src, negative, i, start_digits, digit_count, exponent, writer)); - if (dirty_end) { return INVALID_NUMBER(src); } - return SUCCESS; - } - - // The longest negative 64-bit number is 19 digits. - // The longest positive 64-bit number is 20 digits. - // We do it this way so we don't trigger this branch unless we must. - size_t longest_digit_count = negative ? 19 : 20; - if (digit_count > longest_digit_count) { return INVALID_NUMBER(src); } - if (digit_count == longest_digit_count) { - if (negative) { - // Anything negative above INT64_MAX+1 is invalid - if (i > uint64_t(INT64_MAX) + 1) { return INVALID_NUMBER(src); } - WRITE_INTEGER(~i + 1, src, writer); - if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return INVALID_NUMBER(src); } - return SUCCESS; - // Positive overflow check: - // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the - // biggest uint64_t. - // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX. - // If we got here, it's a 20 digit number starting with the digit "1". - // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller - // than 1,553,255,926,290,448,384. - // - That is smaller than the smallest possible 20-digit number the user could write: - // 10,000,000,000,000,000,000. - // - Therefore, if the number is positive and lower than that, it's overflow. - // - The value we are looking at is less than or equal to INT64_MAX. - // - } - else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INVALID_NUMBER(src); } - } - - // Write unsigned if it doesn't fit in a signed integer. - if (i > uint64_t(INT64_MAX)) { - WRITE_UNSIGNED(i, src, writer); - } - else { - WRITE_INTEGER(negative ? (~i + 1) : i, src, writer); - } - if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return INVALID_NUMBER(src); } - return SUCCESS; - } - - // Inlineable functions - namespace { - - // This table can be used to characterize the final character of an integer - // string. For JSON structural character and allowable white space characters, - // we return SUCCESS. For 'e', '.' and 'E', we return INCORRECT_TYPE. Otherwise - // we return NUMBER_ERROR. - // Optimization note: we could easily reduce the size of the table by half (to 128) - // at the cost of an extra branch. - // Optimization note: we want the values to use at most 8 bits (not, e.g., 32 bits): - static_assert(error_code(uint8_t(NUMBER_ERROR)) == NUMBER_ERROR, "bad NUMBER_ERROR cast"); - static_assert(error_code(uint8_t(SUCCESS)) == SUCCESS, "bad NUMBER_ERROR cast"); - static_assert(error_code(uint8_t(INCORRECT_TYPE)) == INCORRECT_TYPE, "bad NUMBER_ERROR cast"); - - const uint8_t integer_string_finisher[256] = { - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, - SUCCESS, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, - NUMBER_ERROR, INCORRECT_TYPE, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, INCORRECT_TYPE, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, SUCCESS, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, INCORRECT_TYPE, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, - SUCCESS, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, - NUMBER_ERROR }; - - // Parse any number from 0 to 18,446,744,073,709,551,615 - simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t* const src) noexcept { - const uint8_t* p = src; - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t* const start_digits = p; - uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(p - start_digits); - // The longest positive 64-bit number is 20 digits. - // We do it this way so we don't trigger this branch unless we must. - // Optimization note: the compiler can probably merge - // ((digit_count == 0) || (digit_count > 20)) - // into a single branch since digit_count is unsigned. - if ((digit_count == 0) || (digit_count > 20)) { return INCORRECT_TYPE; } - // Here digit_count > 0. - if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } - // We can do the following... - // if (!jsoncharutils::is_structural_or_whitespace(*p)) { - // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; - // } - // as a single table lookup: - if (integer_string_finisher[*p] != SUCCESS) { return error_code(integer_string_finisher[*p]); } - - if (digit_count == 20) { - // Positive overflow check: - // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the - // biggest uint64_t. - // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX. - // If we got here, it's a 20 digit number starting with the digit "1". - // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller - // than 1,553,255,926,290,448,384. - // - That is smaller than the smallest possible 20-digit number the user could write: - // 10,000,000,000,000,000,000. - // - Therefore, if the number is positive and lower than that, it's overflow. - // - The value we are looking at is less than or equal to INT64_MAX. - // - if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INCORRECT_TYPE; } - } - - return i; - } - - - // Parse any number from 0 to 18,446,744,073,709,551,615 - // Never read at src_end or beyond - simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t* const src, const uint8_t* const src_end) noexcept { - const uint8_t* p = src; - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t* const start_digits = p; - uint64_t i = 0; - while ((p != src_end) && parse_digit(*p, i)) { p++; } - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(p - start_digits); - // The longest positive 64-bit number is 20 digits. - // We do it this way so we don't trigger this branch unless we must. - // Optimization note: the compiler can probably merge - // ((digit_count == 0) || (digit_count > 20)) - // into a single branch since digit_count is unsigned. - if ((digit_count == 0) || (digit_count > 20)) { return INCORRECT_TYPE; } - // Here digit_count > 0. - if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } - // We can do the following... - // if (!jsoncharutils::is_structural_or_whitespace(*p)) { - // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; - // } - // as a single table lookup: - if ((p != src_end) && integer_string_finisher[*p] != SUCCESS) { return error_code(integer_string_finisher[*p]); } - - if (digit_count == 20) { - // Positive overflow check: - // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the - // biggest uint64_t. - // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX. - // If we got here, it's a 20 digit number starting with the digit "1". - // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller - // than 1,553,255,926,290,448,384. - // - That is smaller than the smallest possible 20-digit number the user could write: - // 10,000,000,000,000,000,000. - // - Therefore, if the number is positive and lower than that, it's overflow. - // - The value we are looking at is less than or equal to INT64_MAX. - // - if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INCORRECT_TYPE; } - } - - return i; - } - - // Parse any number from 0 to 18,446,744,073,709,551,615 - simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t* const src) noexcept { - const uint8_t* p = src + 1; - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t* const start_digits = p; - uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(p - start_digits); - // The longest positive 64-bit number is 20 digits. - // We do it this way so we don't trigger this branch unless we must. - // Optimization note: the compiler can probably merge - // ((digit_count == 0) || (digit_count > 20)) - // into a single branch since digit_count is unsigned. - if ((digit_count == 0) || (digit_count > 20)) { return INCORRECT_TYPE; } - // Here digit_count > 0. - if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } - // We can do the following... - // if (!jsoncharutils::is_structural_or_whitespace(*p)) { - // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; - // } - // as a single table lookup: - if (*p != '"') { return NUMBER_ERROR; } - - if (digit_count == 20) { - // Positive overflow check: - // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the - // biggest uint64_t. - // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX. - // If we got here, it's a 20 digit number starting with the digit "1". - // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller - // than 1,553,255,926,290,448,384. - // - That is smaller than the smallest possible 20-digit number the user could write: - // 10,000,000,000,000,000,000. - // - Therefore, if the number is positive and lower than that, it's overflow. - // - The value we are looking at is less than or equal to INT64_MAX. - // - // Note: we use src[1] and not src[0] because src[0] is the quote character in this - // instance. - if (src[1] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INCORRECT_TYPE; } - } - - return i; - } - - // Parse any number from -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 - simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t* src) noexcept { - // - // Check for minus sign - // - bool negative = (*src == '-'); - const uint8_t* p = src + uint8_t(negative); - - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t* const start_digits = p; - uint64_t i = 0; - while (parse_digit(*p, i)) { p++; } - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(p - start_digits); - // We go from - // -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 - // so we can never represent numbers that have more than 19 digits. - size_t longest_digit_count = 19; - // Optimization note: the compiler can probably merge - // ((digit_count == 0) || (digit_count > longest_digit_count)) - // into a single branch since digit_count is unsigned. - if ((digit_count == 0) || (digit_count > longest_digit_count)) { return INCORRECT_TYPE; } - // Here digit_count > 0. - if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } - // We can do the following... - // if (!jsoncharutils::is_structural_or_whitespace(*p)) { - // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; - // } - // as a single table lookup: - if (integer_string_finisher[*p] != SUCCESS) { return error_code(integer_string_finisher[*p]); } - // Negative numbers have can go down to - INT64_MAX - 1 whereas positive numbers are limited to INT64_MAX. - // Performance note: This check is only needed when digit_count == longest_digit_count but it is - // so cheap that we might as well always make it. - if (i > uint64_t(INT64_MAX) + uint64_t(negative)) { return INCORRECT_TYPE; } - return negative ? (~i + 1) : i; - } - - // Parse any number from -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 - // Never read at src_end or beyond - simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t* const src, const uint8_t* const src_end) noexcept { - // - // Check for minus sign - // - if (src == src_end) { return NUMBER_ERROR; } - bool negative = (*src == '-'); - const uint8_t* p = src + uint8_t(negative); - - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t* const start_digits = p; - uint64_t i = 0; - while ((p != src_end) && parse_digit(*p, i)) { p++; } - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(p - start_digits); - // We go from - // -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 - // so we can never represent numbers that have more than 19 digits. - size_t longest_digit_count = 19; - // Optimization note: the compiler can probably merge - // ((digit_count == 0) || (digit_count > longest_digit_count)) - // into a single branch since digit_count is unsigned. - if ((digit_count == 0) || (digit_count > longest_digit_count)) { return INCORRECT_TYPE; } - // Here digit_count > 0. - if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } - // We can do the following... - // if (!jsoncharutils::is_structural_or_whitespace(*p)) { - // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; - // } - // as a single table lookup: - if ((p != src_end) && integer_string_finisher[*p] != SUCCESS) { return error_code(integer_string_finisher[*p]); } - // Negative numbers have can go down to - INT64_MAX - 1 whereas positive numbers are limited to INT64_MAX. - // Performance note: This check is only needed when digit_count == longest_digit_count but it is - // so cheap that we might as well always make it. - if (i > uint64_t(INT64_MAX) + uint64_t(negative)) { return INCORRECT_TYPE; } - return negative ? (~i + 1) : i; - } - - // Parse any number from -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 - simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t* src) noexcept { - // - // Check for minus sign - // - bool negative = (*(src + 1) == '-'); - src += uint8_t(negative) + 1; - - // - // Parse the integer part. - // - // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare - const uint8_t* const start_digits = src; - uint64_t i = 0; - while (parse_digit(*src, i)) { src++; } - - // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. - // Optimization note: size_t is expected to be unsigned. - size_t digit_count = size_t(src - start_digits); - // We go from - // -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 - // so we can never represent numbers that have more than 19 digits. - size_t longest_digit_count = 19; - // Optimization note: the compiler can probably merge - // ((digit_count == 0) || (digit_count > longest_digit_count)) - // into a single branch since digit_count is unsigned. - if ((digit_count == 0) || (digit_count > longest_digit_count)) { return INCORRECT_TYPE; } - // Here digit_count > 0. - if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } - // We can do the following... - // if (!jsoncharutils::is_structural_or_whitespace(*src)) { - // return (*src == '.' || *src == 'e' || *src == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; - // } - // as a single table lookup: - if (*src != '"') { return NUMBER_ERROR; } - // Negative numbers have can go down to - INT64_MAX - 1 whereas positive numbers are limited to INT64_MAX. - // Performance note: This check is only needed when digit_count == longest_digit_count but it is - // so cheap that we might as well always make it. - if (i > uint64_t(INT64_MAX) + uint64_t(negative)) { return INCORRECT_TYPE; } - return negative ? (~i + 1) : i; - } - - simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t* src) noexcept { - // - // Check for minus sign - // - bool negative = (*src == '-'); - src += uint8_t(negative); - - // - // Parse the integer part. - // - uint64_t i = 0; - const uint8_t* p = src; - p += parse_digit(*p, i); - bool leading_zero = (i == 0); - while (parse_digit(*p, i)) { p++; } - // no integer digits, or 0123 (zero must be solo) - if (p == src) { return INCORRECT_TYPE; } - if ((leading_zero && p != src + 1)) { return NUMBER_ERROR; } - - // - // Parse the decimal part. - // - int64_t exponent = 0; - bool overflow; - if (simdjson_likely(*p == '.')) { - p++; - const uint8_t* start_decimal_digits = p; - if (!parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits - p++; - while (parse_digit(*p, i)) { p++; } - exponent = -(p - start_decimal_digits); - - // Overflow check. More than 19 digits (minus the decimal) may be overflow. - overflow = p - src - 1 > 19; - if (simdjson_unlikely(overflow && leading_zero)) { - // Skip leading 0.00000 and see if it still overflows - const uint8_t* start_digits = src + 2; - while (*start_digits == '0') { start_digits++; } - overflow = p - start_digits > 19; - } - } - else { - overflow = p - src > 19; - } - - // - // Parse the exponent - // - if (*p == 'e' || *p == 'E') { - p++; - bool exp_neg = *p == '-'; - p += exp_neg || *p == '+'; - - uint64_t exp = 0; - const uint8_t* start_exp_digits = p; - while (parse_digit(*p, exp)) { p++; } - // no exp digits, or 20+ exp digits - if (p - start_exp_digits == 0 || p - start_exp_digits > 19) { return NUMBER_ERROR; } - - exponent += exp_neg ? 0 - exp : exp; - } - - if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } - - overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; - - // - // Assemble (or slow-parse) the float - // - double d; - if (simdjson_likely(!overflow)) { - if (compute_float_64(exponent, i, negative, d)) { return d; } - } - if (!parse_float_fallback(src - uint8_t(negative), &d)) { - return NUMBER_ERROR; - } - return d; - } - - simdjson_unused simdjson_inline bool is_negative(const uint8_t* src) noexcept { - return (*src == '-'); - } - - simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t* src) noexcept { - bool negative = (*src == '-'); - src += uint8_t(negative); - const uint8_t* p = src; - while (static_cast(*p - '0') <= 9) { p++; } - if (p == src) { return NUMBER_ERROR; } - if (jsoncharutils::is_structural_or_whitespace(*p)) { return true; } - return false; - } - - simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t* src) noexcept { - bool negative = (*src == '-'); - src += uint8_t(negative); - const uint8_t* p = src; - while (static_cast(*p - '0') <= 9) { p++; } - if (p == src) { return NUMBER_ERROR; } - if (jsoncharutils::is_structural_or_whitespace(*p)) { - // We have an integer. - // If the number is negative and valid, it must be a signed integer. - if (negative) { return number_type::signed_integer; } - // We want values larger or equal to 9223372036854775808 to be unsigned - // integers, and the other values to be signed integers. - int digit_count = int(p - src); - if (digit_count >= 19) { - const uint8_t* smaller_big_integer = reinterpret_cast("9223372036854775808"); - if ((digit_count >= 20) || (memcmp(src, smaller_big_integer, 19) >= 0)) { - return number_type::unsigned_integer; - } - } - return number_type::signed_integer; - } - // Hopefully, we have 'e' or 'E' or '.'. - return number_type::floating_point_number; - } - - // Never read at src_end or beyond - simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t* src, const uint8_t* const src_end) noexcept { - if (src == src_end) { return NUMBER_ERROR; } - // - // Check for minus sign - // - bool negative = (*src == '-'); - src += uint8_t(negative); - - // - // Parse the integer part. - // - uint64_t i = 0; - const uint8_t* p = src; - if (p == src_end) { return NUMBER_ERROR; } - p += parse_digit(*p, i); - bool leading_zero = (i == 0); - while ((p != src_end) && parse_digit(*p, i)) { p++; } - // no integer digits, or 0123 (zero must be solo) - if (p == src) { return INCORRECT_TYPE; } - if ((leading_zero && p != src + 1)) { return NUMBER_ERROR; } - - // - // Parse the decimal part. - // - int64_t exponent = 0; - bool overflow; - if (simdjson_likely((p != src_end) && (*p == '.'))) { - p++; - const uint8_t* start_decimal_digits = p; - if ((p == src_end) || !parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits - p++; - while ((p != src_end) && parse_digit(*p, i)) { p++; } - exponent = -(p - start_decimal_digits); - - // Overflow check. More than 19 digits (minus the decimal) may be overflow. - overflow = p - src - 1 > 19; - if (simdjson_unlikely(overflow && leading_zero)) { - // Skip leading 0.00000 and see if it still overflows - const uint8_t* start_digits = src + 2; - while (*start_digits == '0') { start_digits++; } - overflow = start_digits - src > 19; - } - } - else { - overflow = p - src > 19; - } - - // - // Parse the exponent - // - if ((p != src_end) && (*p == 'e' || *p == 'E')) { - p++; - if (p == src_end) { return NUMBER_ERROR; } - bool exp_neg = *p == '-'; - p += exp_neg || *p == '+'; - - uint64_t exp = 0; - const uint8_t* start_exp_digits = p; - while ((p != src_end) && parse_digit(*p, exp)) { p++; } - // no exp digits, or 20+ exp digits - if (p - start_exp_digits == 0 || p - start_exp_digits > 19) { return NUMBER_ERROR; } - - exponent += exp_neg ? 0 - exp : exp; - } - - if ((p != src_end) && jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } - - overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; - - // - // Assemble (or slow-parse) the float - // - double d; - if (simdjson_likely(!overflow)) { - if (compute_float_64(exponent, i, negative, d)) { return d; } - } - if (!parse_float_fallback(src - uint8_t(negative), src_end, &d)) { - return NUMBER_ERROR; - } - return d; - } - - simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t* src) noexcept { - // - // Check for minus sign - // - bool negative = (*(src + 1) == '-'); - src += uint8_t(negative) + 1; - - // - // Parse the integer part. - // - uint64_t i = 0; - const uint8_t* p = src; - p += parse_digit(*p, i); - bool leading_zero = (i == 0); - while (parse_digit(*p, i)) { p++; } - // no integer digits, or 0123 (zero must be solo) - if (p == src) { return INCORRECT_TYPE; } - if ((leading_zero && p != src + 1)) { return NUMBER_ERROR; } - - // - // Parse the decimal part. - // - int64_t exponent = 0; - bool overflow; - if (simdjson_likely(*p == '.')) { - p++; - const uint8_t* start_decimal_digits = p; - if (!parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits - p++; - while (parse_digit(*p, i)) { p++; } - exponent = -(p - start_decimal_digits); - - // Overflow check. More than 19 digits (minus the decimal) may be overflow. - overflow = p - src - 1 > 19; - if (simdjson_unlikely(overflow && leading_zero)) { - // Skip leading 0.00000 and see if it still overflows - const uint8_t* start_digits = src + 2; - while (*start_digits == '0') { start_digits++; } - overflow = p - start_digits > 19; - } - } - else { - overflow = p - src > 19; - } - - // - // Parse the exponent - // - if (*p == 'e' || *p == 'E') { - p++; - bool exp_neg = *p == '-'; - p += exp_neg || *p == '+'; - - uint64_t exp = 0; - const uint8_t* start_exp_digits = p; - while (parse_digit(*p, exp)) { p++; } - // no exp digits, or 20+ exp digits - if (p - start_exp_digits == 0 || p - start_exp_digits > 19) { return NUMBER_ERROR; } - - exponent += exp_neg ? 0 - exp : exp; - } - - if (*p != '"') { return NUMBER_ERROR; } - - overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; - - // - // Assemble (or slow-parse) the float - // - double d; - if (simdjson_likely(!overflow)) { - if (compute_float_64(exponent, i, negative, d)) { return d; } - } - if (!parse_float_fallback(src - uint8_t(negative), &d)) { - return NUMBER_ERROR; - } - return d; - } - - } // unnamed namespace -#endif // SIMDJSON_SKIPNUMBERPARSING - - } // namespace numberparsing - - inline std::ostream& operator<<(std::ostream& out, number_type type) noexcept { - switch (type) { - case number_type::signed_integer: out << "integer in [-9223372036854775808,9223372036854775808)"; break; - case number_type::unsigned_integer: out << "unsigned integer in [9223372036854775808,18446744073709551616)"; break; - case number_type::floating_point_number: out << "floating-point number (binary64)"; break; - default: SIMDJSON_UNREACHABLE(); - } - return out; - } - - } // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_NUMBERPARSING_H -/* end file simdjson/generic/numberparsing.h for westmere */ - -/* including simdjson/generic/implementation_simdjson_result_base-inl.h for westmere: #include "simdjson/generic/implementation_simdjson_result_base-inl.h" */ -/* begin file simdjson/generic/implementation_simdjson_result_base-inl.h for westmere */ -#ifndef SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_INL_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_INL_H */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/implementation_simdjson_result_base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - namespace westmere { - - // - // internal::implementation_simdjson_result_base inline implementation - // - - template - simdjson_inline void implementation_simdjson_result_base::tie(T& value, error_code& error) && noexcept { - error = this->second; - if (!error) { - value = std::forward>(*this).first; - } - } - - template - simdjson_warn_unused simdjson_inline error_code implementation_simdjson_result_base::get(T& value) && noexcept { - error_code error; - std::forward>(*this).tie(value, error); - return error; - } - - template - simdjson_inline error_code implementation_simdjson_result_base::error() const noexcept { - return this->second; - } - -#if SIMDJSON_EXCEPTIONS - - template - simdjson_inline T& implementation_simdjson_result_base::value() & noexcept(false) { - if (error()) { throw simdjson_error(error()); } - return this->first; - } - - template - simdjson_inline T&& implementation_simdjson_result_base::value() && noexcept(false) { - return std::forward>(*this).take_value(); - } - - template - simdjson_inline T&& implementation_simdjson_result_base::take_value() && noexcept(false) { - if (error()) { throw simdjson_error(error()); } - return std::forward(this->first); - } - - template - simdjson_inline implementation_simdjson_result_base::operator T && () && noexcept(false) { - return std::forward>(*this).take_value(); - } - -#endif // SIMDJSON_EXCEPTIONS - - template - simdjson_inline const T& implementation_simdjson_result_base::value_unsafe() const& noexcept { - return this->first; - } - - template - simdjson_inline T& implementation_simdjson_result_base::value_unsafe() & noexcept { - return this->first; - } - - template - simdjson_inline T&& implementation_simdjson_result_base::value_unsafe() && noexcept { - return std::forward(this->first); - } - - template - simdjson_inline implementation_simdjson_result_base::implementation_simdjson_result_base(T&& value, error_code error) noexcept - : first{ std::forward(value) }, second{ error } {} - template - simdjson_inline implementation_simdjson_result_base::implementation_simdjson_result_base(error_code error) noexcept - : implementation_simdjson_result_base(T{}, error) {} - template - simdjson_inline implementation_simdjson_result_base::implementation_simdjson_result_base(T&& value) noexcept - : implementation_simdjson_result_base(std::forward(value), SUCCESS) {} - - } // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_INL_H -/* end file simdjson/generic/implementation_simdjson_result_base-inl.h for westmere */ -/* end file simdjson/generic/amalgamated.h for westmere */ -/* including simdjson/westmere/end.h: #include "simdjson/westmere/end.h" */ -/* begin file simdjson/westmere/end.h */ -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/westmere/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#if !SIMDJSON_CAN_ALWAYS_RUN_WESTMERE -SIMDJSON_UNTARGET_REGION -#endif - -/* undefining SIMDJSON_IMPLEMENTATION from "westmere" */ -#undef SIMDJSON_IMPLEMENTATION -/* end file simdjson/westmere/end.h */ - -#endif // SIMDJSON_WESTMERE_H -/* end file simdjson/westmere.h */ -/* including simdjson/westmere/implementation.h: #include */ -/* begin file simdjson/westmere/implementation.h */ -#ifndef SIMDJSON_WESTMERE_IMPLEMENTATION_H -#define SIMDJSON_WESTMERE_IMPLEMENTATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/westmere/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/implementation.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/instruction_set.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -// The constructor may be executed on any host, so we take care not to use SIMDJSON_TARGET_WESTMERE -namespace simdjson { - namespace westmere { - - /** - * @private - */ - class implementation final : public simdjson::implementation { - public: - simdjson_inline implementation() : simdjson::implementation("westmere", "Intel/AMD SSE4.2", internal::instruction_set::SSE42 | internal::instruction_set::PCLMULQDQ) {} - simdjson_warn_unused error_code create_dom_parser_implementation( - size_t capacity, - size_t max_length, - std::unique_ptr& dst - ) const noexcept final; - simdjson_warn_unused error_code minify(const uint8_t* buf, size_t len, uint8_t* dst, size_t& dst_len) const noexcept final; - simdjson_warn_unused bool validate_utf8(const char* buf, size_t len) const noexcept final; - }; - - } // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_WESTMERE_IMPLEMENTATION_H -/* end file simdjson/westmere/implementation.h */ - -/* including simdjson/westmere/begin.h: #include */ -/* begin file simdjson/westmere/begin.h */ -/* defining SIMDJSON_IMPLEMENTATION to "westmere" */ -#define SIMDJSON_IMPLEMENTATION westmere -/* including simdjson/westmere/base.h: #include "simdjson/westmere/base.h" */ -/* begin file simdjson/westmere/base.h */ -#ifndef SIMDJSON_WESTMERE_BASE_H -#define SIMDJSON_WESTMERE_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -// The constructor may be executed on any host, so we take care not to use SIMDJSON_TARGET_WESTMERE -namespace simdjson { - /** - * Implementation for Westmere (Intel SSE4.2). - */ - namespace westmere { - - class implementation; - - namespace { - namespace simd { - - template struct simd8; - template struct simd8x64; - - } // namespace simd - } // unnamed namespace - - } // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_WESTMERE_BASE_H -/* end file simdjson/westmere/base.h */ -/* including simdjson/westmere/intrinsics.h: #include "simdjson/westmere/intrinsics.h" */ -/* begin file simdjson/westmere/intrinsics.h */ -#ifndef SIMDJSON_WESTMERE_INTRINSICS_H -#define SIMDJSON_WESTMERE_INTRINSICS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/westmere/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#if SIMDJSON_VISUAL_STUDIO -// under clang within visual studio, this will include -#include // visual studio or clang -#else -#include // elsewhere -#endif // SIMDJSON_VISUAL_STUDIO - - -#if SIMDJSON_CLANG_VISUAL_STUDIO -/** - * You are not supposed, normally, to include these - * headers directly. Instead you should either include intrin.h - * or x86intrin.h. However, when compiling with clang - * under Windows (i.e., when _MSC_VER is set), these headers - * only get included *if* the corresponding features are detected - * from macros: - */ -#include // for _mm_alignr_epi8 -#include // for _mm_clmulepi64_si128 -#endif - -static_assert(sizeof(__m128i) <= simdjson::SIMDJSON_PADDING, "insufficient padding for westmere"); - -#endif // SIMDJSON_WESTMERE_INTRINSICS_H -/* end file simdjson/westmere/intrinsics.h */ - -#if !SIMDJSON_CAN_ALWAYS_RUN_WESTMERE -SIMDJSON_TARGET_REGION("sse4.2,pclmul,popcnt") -#endif - -/* including simdjson/westmere/bitmanipulation.h: #include "simdjson/westmere/bitmanipulation.h" */ -/* begin file simdjson/westmere/bitmanipulation.h */ -#ifndef SIMDJSON_WESTMERE_BITMANIPULATION_H -#define SIMDJSON_WESTMERE_BITMANIPULATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/westmere/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/westmere/intrinsics.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - namespace westmere { - namespace { - - // We sometimes call trailing_zero on inputs that are zero, - // but the algorithms do not end up using the returned value. - // Sadly, sanitizers are not smart enough to figure it out. - SIMDJSON_NO_SANITIZE_UNDEFINED - // This function can be used safely even if not all bytes have been - // initialized. - // See issue https://github.com/simdjson/simdjson/issues/1965 - SIMDJSON_NO_SANITIZE_MEMORY - simdjson_inline int trailing_zeroes(uint64_t input_num) { -#if SIMDJSON_REGULAR_VISUAL_STUDIO - unsigned long ret; - // Search the mask data from least significant bit (LSB) - // to the most significant bit (MSB) for a set bit (1). - _BitScanForward64(&ret, input_num); - return (int)ret; -#else // SIMDJSON_REGULAR_VISUAL_STUDIO - return __builtin_ctzll(input_num); -#endif // SIMDJSON_REGULAR_VISUAL_STUDIO - } - - /* result might be undefined when input_num is zero */ - simdjson_inline uint64_t clear_lowest_bit(uint64_t input_num) { - return input_num & (input_num - 1); - } - - /* result might be undefined when input_num is zero */ - simdjson_inline int leading_zeroes(uint64_t input_num) { -#if SIMDJSON_REGULAR_VISUAL_STUDIO - unsigned long leading_zero = 0; - // Search the mask data from most significant bit (MSB) - // to least significant bit (LSB) for a set bit (1). - if (_BitScanReverse64(&leading_zero, input_num)) - return (int)(63 - leading_zero); - else - return 64; -#else - return __builtin_clzll(input_num); -#endif// SIMDJSON_REGULAR_VISUAL_STUDIO - } - -#if SIMDJSON_REGULAR_VISUAL_STUDIO - simdjson_inline unsigned __int64 count_ones(uint64_t input_num) { - // note: we do not support legacy 32-bit Windows in this kernel - return __popcnt64(input_num);// Visual Studio wants two underscores - } -#else - simdjson_inline long long int count_ones(uint64_t input_num) { - return _popcnt64(input_num); - } -#endif - - simdjson_inline bool add_overflow(uint64_t value1, uint64_t value2, - uint64_t* result) { -#if SIMDJSON_REGULAR_VISUAL_STUDIO - return _addcarry_u64(0, value1, value2, - reinterpret_cast(result)); -#else - return __builtin_uaddll_overflow(value1, value2, - reinterpret_cast(result)); -#endif - } - - } // unnamed namespace - } // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_WESTMERE_BITMANIPULATION_H -/* end file simdjson/westmere/bitmanipulation.h */ -/* including simdjson/westmere/bitmask.h: #include "simdjson/westmere/bitmask.h" */ -/* begin file simdjson/westmere/bitmask.h */ -#ifndef SIMDJSON_WESTMERE_BITMASK_H -#define SIMDJSON_WESTMERE_BITMASK_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/westmere/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/westmere/intrinsics.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - namespace westmere { - namespace { - - // - // Perform a "cumulative bitwise xor," flipping bits each time a 1 is encountered. - // - // For example, prefix_xor(00100100) == 00011100 - // - simdjson_inline uint64_t prefix_xor(const uint64_t bitmask) { - // There should be no such thing with a processing supporting avx2 - // but not clmul. - __m128i all_ones = _mm_set1_epi8('\xFF'); - __m128i result = _mm_clmulepi64_si128(_mm_set_epi64x(0ULL, bitmask), all_ones, 0); - return _mm_cvtsi128_si64(result); - } - - } // unnamed namespace - } // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_WESTMERE_BITMASK_H -/* end file simdjson/westmere/bitmask.h */ -/* including simdjson/westmere/numberparsing_defs.h: #include "simdjson/westmere/numberparsing_defs.h" */ -/* begin file simdjson/westmere/numberparsing_defs.h */ -#ifndef SIMDJSON_WESTMERE_NUMBERPARSING_DEFS_H -#define SIMDJSON_WESTMERE_NUMBERPARSING_DEFS_H - -/* including simdjson/westmere/base.h: #include "simdjson/westmere/base.h" */ -/* begin file simdjson/westmere/base.h */ -#ifndef SIMDJSON_WESTMERE_BASE_H -#define SIMDJSON_WESTMERE_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -// The constructor may be executed on any host, so we take care not to use SIMDJSON_TARGET_WESTMERE -namespace simdjson { - /** - * Implementation for Westmere (Intel SSE4.2). - */ - namespace westmere { - - class implementation; - - namespace { - namespace simd { - - template struct simd8; - template struct simd8x64; - - } // namespace simd - } // unnamed namespace - - } // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_WESTMERE_BASE_H -/* end file simdjson/westmere/base.h */ -/* including simdjson/westmere/intrinsics.h: #include "simdjson/westmere/intrinsics.h" */ -/* begin file simdjson/westmere/intrinsics.h */ -#ifndef SIMDJSON_WESTMERE_INTRINSICS_H -#define SIMDJSON_WESTMERE_INTRINSICS_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/westmere/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#if SIMDJSON_VISUAL_STUDIO -// under clang within visual studio, this will include -#include // visual studio or clang -#else -#include // elsewhere -#endif // SIMDJSON_VISUAL_STUDIO - - -#if SIMDJSON_CLANG_VISUAL_STUDIO -/** - * You are not supposed, normally, to include these - * headers directly. Instead you should either include intrin.h - * or x86intrin.h. However, when compiling with clang - * under Windows (i.e., when _MSC_VER is set), these headers - * only get included *if* the corresponding features are detected - * from macros: - */ -#include // for _mm_alignr_epi8 -#include // for _mm_clmulepi64_si128 -#endif - -static_assert(sizeof(__m128i) <= simdjson::SIMDJSON_PADDING, "insufficient padding for westmere"); - -#endif // SIMDJSON_WESTMERE_INTRINSICS_H -/* end file simdjson/westmere/intrinsics.h */ - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - namespace westmere { - namespace numberparsing { - - /** @private */ - static simdjson_inline uint32_t parse_eight_digits_unrolled(const uint8_t* chars) { - // this actually computes *16* values so we are being wasteful. - const __m128i ascii0 = _mm_set1_epi8('0'); - const __m128i mul_1_10 = - _mm_setr_epi8(10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1, 10, 1); - const __m128i mul_1_100 = _mm_setr_epi16(100, 1, 100, 1, 100, 1, 100, 1); - const __m128i mul_1_10000 = - _mm_setr_epi16(10000, 1, 10000, 1, 10000, 1, 10000, 1); - const __m128i input = _mm_sub_epi8( - _mm_loadu_si128(reinterpret_cast(chars)), ascii0); - const __m128i t1 = _mm_maddubs_epi16(input, mul_1_10); - const __m128i t2 = _mm_madd_epi16(t1, mul_1_100); - const __m128i t3 = _mm_packus_epi32(t2, t2); - const __m128i t4 = _mm_madd_epi16(t3, mul_1_10000); - return _mm_cvtsi128_si32( - t4); // only captures the sum of the first 8 digits, drop the rest - } - - /** @private */ - simdjson_inline internal::value128 full_multiplication(uint64_t value1, uint64_t value2) { - internal::value128 answer; -#if SIMDJSON_REGULAR_VISUAL_STUDIO || SIMDJSON_IS_32BITS -#ifdef _M_ARM64 - // ARM64 has native support for 64-bit multiplications, no need to emultate - answer.high = __umulh(value1, value2); - answer.low = value1 * value2; -#else - answer.low = _umul128(value1, value2, &answer.high); // _umul128 not available on ARM64 -#endif // _M_ARM64 -#else // SIMDJSON_REGULAR_VISUAL_STUDIO || SIMDJSON_IS_32BITS - __uint128_t r = (static_cast<__uint128_t>(value1)) * value2; - answer.low = uint64_t(r); - answer.high = uint64_t(r >> 64); -#endif - return answer; - } - - } // namespace numberparsing - } // namespace westmere -} // namespace simdjson - -#define SIMDJSON_SWAR_NUMBER_PARSING 1 - -#endif // SIMDJSON_WESTMERE_NUMBERPARSING_DEFS_H -/* end file simdjson/westmere/numberparsing_defs.h */ -/* including simdjson/westmere/simd.h: #include "simdjson/westmere/simd.h" */ -/* begin file simdjson/westmere/simd.h */ -#ifndef SIMDJSON_WESTMERE_SIMD_H -#define SIMDJSON_WESTMERE_SIMD_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/westmere/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/westmere/bitmanipulation.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/simdprune_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - namespace westmere { - namespace { - namespace simd { - - template - struct base { - __m128i value; - - // Zero constructor - simdjson_inline base() : value{ __m128i() } {} - - // Conversion from SIMD register - simdjson_inline base(const __m128i _value) : value(_value) {} - - // Conversion to SIMD register - simdjson_inline operator const __m128i& () const { return this->value; } - simdjson_inline operator __m128i& () { return this->value; } - - // Bit operations - simdjson_inline Child operator|(const Child other) const { return _mm_or_si128(*this, other); } - simdjson_inline Child operator&(const Child other) const { return _mm_and_si128(*this, other); } - simdjson_inline Child operator^(const Child other) const { return _mm_xor_si128(*this, other); } - simdjson_inline Child bit_andnot(const Child other) const { return _mm_andnot_si128(other, *this); } - simdjson_inline Child& operator|=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast | other; return *this_cast; } - simdjson_inline Child& operator&=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast & other; return *this_cast; } - simdjson_inline Child& operator^=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast ^ other; return *this_cast; } - }; - - template> - struct base8 : base> { - typedef uint16_t bitmask_t; - typedef uint32_t bitmask2_t; - - simdjson_inline base8() : base>() {} - simdjson_inline base8(const __m128i _value) : base>(_value) {} - - friend simdjson_inline Mask operator==(const simd8 lhs, const simd8 rhs) { return _mm_cmpeq_epi8(lhs, rhs); } - - static const int SIZE = sizeof(base>::value); - - template - simdjson_inline simd8 prev(const simd8 prev_chunk) const { - return _mm_alignr_epi8(*this, prev_chunk, 16 - N); - } - }; - - // SIMD byte mask type (returned by things like eq and gt) - template<> - struct simd8 : base8 { - static simdjson_inline simd8 splat(bool _value) { return _mm_set1_epi8(uint8_t(-(!!_value))); } - - simdjson_inline simd8() : base8() {} - simdjson_inline simd8(const __m128i _value) : base8(_value) {} - // Splat constructor - simdjson_inline simd8(bool _value) : base8(splat(_value)) {} - - simdjson_inline int to_bitmask() const { return _mm_movemask_epi8(*this); } - simdjson_inline bool any() const { return !_mm_testz_si128(*this, *this); } - simdjson_inline simd8 operator~() const { return *this ^ true; } - }; - - template - struct base8_numeric : base8 { - static simdjson_inline simd8 splat(T _value) { return _mm_set1_epi8(_value); } - static simdjson_inline simd8 zero() { return _mm_setzero_si128(); } - static simdjson_inline simd8 load(const T values[16]) { - return _mm_loadu_si128(reinterpret_cast(values)); - } - // Repeat 16 values as many times as necessary (usually for lookup tables) - static simdjson_inline simd8 repeat_16( - T v0, T v1, T v2, T v3, T v4, T v5, T v6, T v7, - T v8, T v9, T v10, T v11, T v12, T v13, T v14, T v15 - ) { - return simd8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10, v11, v12, v13, v14, v15 - ); - } - - simdjson_inline base8_numeric() : base8() {} - simdjson_inline base8_numeric(const __m128i _value) : base8(_value) {} - - // Store to array - simdjson_inline void store(T dst[16]) const { return _mm_storeu_si128(reinterpret_cast<__m128i*>(dst), *this); } - - // Override to distinguish from bool version - simdjson_inline simd8 operator~() const { return *this ^ 0xFFu; } - - // Addition/subtraction are the same for signed and unsigned - simdjson_inline simd8 operator+(const simd8 other) const { return _mm_add_epi8(*this, other); } - simdjson_inline simd8 operator-(const simd8 other) const { return _mm_sub_epi8(*this, other); } - simdjson_inline simd8& operator+=(const simd8 other) { *this = *this + other; return *static_cast*>(this); } - simdjson_inline simd8& operator-=(const simd8 other) { *this = *this - other; return *static_cast*>(this); } - - // Perform a lookup assuming the value is between 0 and 16 (undefined behavior for out of range values) - template - simdjson_inline simd8 lookup_16(simd8 lookup_table) const { - return _mm_shuffle_epi8(lookup_table, *this); - } - - // Copies to 'output" all bytes corresponding to a 0 in the mask (interpreted as a bitset). - // Passing a 0 value for mask would be equivalent to writing out every byte to output. - // Only the first 16 - count_ones(mask) bytes of the result are significant but 16 bytes - // get written. - // Design consideration: it seems like a function with the - // signature simd8 compress(uint32_t mask) would be - // sensible, but the AVX ISA makes this kind of approach difficult. - template - simdjson_inline void compress(uint16_t mask, L* output) const { - using internal::thintable_epi8; - using internal::BitsSetTable256mul2; - using internal::pshufb_combine_table; - // this particular implementation was inspired by work done by @animetosho - // we do it in two steps, first 8 bytes and then second 8 bytes - uint8_t mask1 = uint8_t(mask); // least significant 8 bits - uint8_t mask2 = uint8_t(mask >> 8); // most significant 8 bits - // next line just loads the 64-bit values thintable_epi8[mask1] and - // thintable_epi8[mask2] into a 128-bit register, using only - // two instructions on most compilers. - __m128i shufmask = _mm_set_epi64x(thintable_epi8[mask2], thintable_epi8[mask1]); - // we increment by 0x08 the second half of the mask - shufmask = - _mm_add_epi8(shufmask, _mm_set_epi32(0x08080808, 0x08080808, 0, 0)); - // this is the version "nearly pruned" - __m128i pruned = _mm_shuffle_epi8(*this, shufmask); - // we still need to put the two halves together. - // we compute the popcount of the first half: - int pop1 = BitsSetTable256mul2[mask1]; - // then load the corresponding mask, what it does is to write - // only the first pop1 bytes from the first 8 bytes, and then - // it fills in with the bytes from the second 8 bytes + some filling - // at the end. - __m128i compactmask = - _mm_loadu_si128(reinterpret_cast(pshufb_combine_table + pop1 * 8)); - __m128i answer = _mm_shuffle_epi8(pruned, compactmask); - _mm_storeu_si128(reinterpret_cast<__m128i*>(output), answer); - } - - template - simdjson_inline simd8 lookup_16( - L replace0, L replace1, L replace2, L replace3, - L replace4, L replace5, L replace6, L replace7, - L replace8, L replace9, L replace10, L replace11, - L replace12, L replace13, L replace14, L replace15) const { - return lookup_16(simd8::repeat_16( - replace0, replace1, replace2, replace3, - replace4, replace5, replace6, replace7, - replace8, replace9, replace10, replace11, - replace12, replace13, replace14, replace15 - )); - } - }; - - // Signed bytes - template<> - struct simd8 : base8_numeric { - simdjson_inline simd8() : base8_numeric() {} - simdjson_inline simd8(const __m128i _value) : base8_numeric(_value) {} - // Splat constructor - simdjson_inline simd8(int8_t _value) : simd8(splat(_value)) {} - // Array constructor - simdjson_inline simd8(const int8_t* values) : simd8(load(values)) {} - // Member-by-member initialization - simdjson_inline simd8( - int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, - int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 - ) : simd8(_mm_setr_epi8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10, v11, v12, v13, v14, v15 - )) {} - // Repeat 16 values as many times as necessary (usually for lookup tables) - simdjson_inline static simd8 repeat_16( - int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, - int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 - ) { - return simd8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10, v11, v12, v13, v14, v15 - ); - } - - // Order-sensitive comparisons - simdjson_inline simd8 max_val(const simd8 other) const { return _mm_max_epi8(*this, other); } - simdjson_inline simd8 min_val(const simd8 other) const { return _mm_min_epi8(*this, other); } - simdjson_inline simd8 operator>(const simd8 other) const { return _mm_cmpgt_epi8(*this, other); } - simdjson_inline simd8 operator<(const simd8 other) const { return _mm_cmpgt_epi8(other, *this); } - }; - - // Unsigned bytes - template<> - struct simd8 : base8_numeric { - simdjson_inline simd8() : base8_numeric() {} - simdjson_inline simd8(const __m128i _value) : base8_numeric(_value) {} - // Splat constructor - simdjson_inline simd8(uint8_t _value) : simd8(splat(_value)) {} - // Array constructor - simdjson_inline simd8(const uint8_t* values) : simd8(load(values)) {} - // Member-by-member initialization - simdjson_inline simd8( - uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, - uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 - ) : simd8(_mm_setr_epi8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10, v11, v12, v13, v14, v15 - )) {} - // Repeat 16 values as many times as necessary (usually for lookup tables) - simdjson_inline static simd8 repeat_16( - uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, - uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 - ) { - return simd8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10, v11, v12, v13, v14, v15 - ); - } - - // Saturated math - simdjson_inline simd8 saturating_add(const simd8 other) const { return _mm_adds_epu8(*this, other); } - simdjson_inline simd8 saturating_sub(const simd8 other) const { return _mm_subs_epu8(*this, other); } - - // Order-specific operations - simdjson_inline simd8 max_val(const simd8 other) const { return _mm_max_epu8(*this, other); } - simdjson_inline simd8 min_val(const simd8 other) const { return _mm_min_epu8(*this, other); } - // Same as >, but only guarantees true is nonzero (< guarantees true = -1) - simdjson_inline simd8 gt_bits(const simd8 other) const { return this->saturating_sub(other); } - // Same as <, but only guarantees true is nonzero (< guarantees true = -1) - simdjson_inline simd8 lt_bits(const simd8 other) const { return other.saturating_sub(*this); } - simdjson_inline simd8 operator<=(const simd8 other) const { return other.max_val(*this) == other; } - simdjson_inline simd8 operator>=(const simd8 other) const { return other.min_val(*this) == other; } - simdjson_inline simd8 operator>(const simd8 other) const { return this->gt_bits(other).any_bits_set(); } - simdjson_inline simd8 operator<(const simd8 other) const { return this->gt_bits(other).any_bits_set(); } - - // Bit-specific operations - simdjson_inline simd8 bits_not_set() const { return *this == uint8_t(0); } - simdjson_inline simd8 bits_not_set(simd8 bits) const { return (*this & bits).bits_not_set(); } - simdjson_inline simd8 any_bits_set() const { return ~this->bits_not_set(); } - simdjson_inline simd8 any_bits_set(simd8 bits) const { return ~this->bits_not_set(bits); } - simdjson_inline bool is_ascii() const { return _mm_movemask_epi8(*this) == 0; } - simdjson_inline bool bits_not_set_anywhere() const { return _mm_testz_si128(*this, *this); } - simdjson_inline bool any_bits_set_anywhere() const { return !bits_not_set_anywhere(); } - simdjson_inline bool bits_not_set_anywhere(simd8 bits) const { return _mm_testz_si128(*this, bits); } - simdjson_inline bool any_bits_set_anywhere(simd8 bits) const { return !bits_not_set_anywhere(bits); } - template - simdjson_inline simd8 shr() const { return simd8(_mm_srli_epi16(*this, N)) & uint8_t(0xFFu >> N); } - template - simdjson_inline simd8 shl() const { return simd8(_mm_slli_epi16(*this, N)) & uint8_t(0xFFu << N); } - // Get one of the bits and make a bitmask out of it. - // e.g. value.get_bit<7>() gets the high bit - template - simdjson_inline int get_bit() const { return _mm_movemask_epi8(_mm_slli_epi16(*this, 7 - N)); } - }; - - template - struct simd8x64 { - static constexpr int NUM_CHUNKS = 64 / sizeof(simd8); - static_assert(NUM_CHUNKS == 4, "Westmere kernel should use four registers per 64-byte block."); - const simd8 chunks[NUM_CHUNKS]; - - simd8x64(const simd8x64& o) = delete; // no copy allowed - simd8x64& operator=(const simd8& other) = delete; // no assignment allowed - simd8x64() = delete; // no default constructor allowed - - simdjson_inline simd8x64(const simd8 chunk0, const simd8 chunk1, const simd8 chunk2, const simd8 chunk3) : chunks{ chunk0, chunk1, chunk2, chunk3 } {} - simdjson_inline simd8x64(const T ptr[64]) : chunks{ simd8::load(ptr), simd8::load(ptr + 16), simd8::load(ptr + 32), simd8::load(ptr + 48) } {} - - simdjson_inline void store(T ptr[64]) const { - this->chunks[0].store(ptr + sizeof(simd8) * 0); - this->chunks[1].store(ptr + sizeof(simd8) * 1); - this->chunks[2].store(ptr + sizeof(simd8) * 2); - this->chunks[3].store(ptr + sizeof(simd8) * 3); - } - - simdjson_inline simd8 reduce_or() const { - return (this->chunks[0] | this->chunks[1]) | (this->chunks[2] | this->chunks[3]); - } - - simdjson_inline uint64_t compress(uint64_t mask, T* output) const { - this->chunks[0].compress(uint16_t(mask), output); - this->chunks[1].compress(uint16_t(mask >> 16), output + 16 - count_ones(mask & 0xFFFF)); - this->chunks[2].compress(uint16_t(mask >> 32), output + 32 - count_ones(mask & 0xFFFFFFFF)); - this->chunks[3].compress(uint16_t(mask >> 48), output + 48 - count_ones(mask & 0xFFFFFFFFFFFF)); - return 64 - count_ones(mask); - } - - simdjson_inline uint64_t to_bitmask() const { - uint64_t r0 = uint32_t(this->chunks[0].to_bitmask()); - uint64_t r1 = this->chunks[1].to_bitmask(); - uint64_t r2 = this->chunks[2].to_bitmask(); - uint64_t r3 = this->chunks[3].to_bitmask(); - return r0 | (r1 << 16) | (r2 << 32) | (r3 << 48); - } - - simdjson_inline uint64_t eq(const T m) const { - const simd8 mask = simd8::splat(m); - return simd8x64( - this->chunks[0] == mask, - this->chunks[1] == mask, - this->chunks[2] == mask, - this->chunks[3] == mask - ).to_bitmask(); - } - - simdjson_inline uint64_t eq(const simd8x64& other) const { - return simd8x64( - this->chunks[0] == other.chunks[0], - this->chunks[1] == other.chunks[1], - this->chunks[2] == other.chunks[2], - this->chunks[3] == other.chunks[3] - ).to_bitmask(); - } - - simdjson_inline uint64_t lteq(const T m) const { - const simd8 mask = simd8::splat(m); - return simd8x64( - this->chunks[0] <= mask, - this->chunks[1] <= mask, - this->chunks[2] <= mask, - this->chunks[3] <= mask - ).to_bitmask(); - } - }; // struct simd8x64 - - } // namespace simd - } // unnamed namespace - } // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_WESTMERE_SIMD_INPUT_H -/* end file simdjson/westmere/simd.h */ -/* including simdjson/westmere/stringparsing_defs.h: #include "simdjson/westmere/stringparsing_defs.h" */ -/* begin file simdjson/westmere/stringparsing_defs.h */ -#ifndef SIMDJSON_WESTMERE_STRINGPARSING_DEFS_H -#define SIMDJSON_WESTMERE_STRINGPARSING_DEFS_H - -/* including simdjson/westmere/bitmanipulation.h: #include "simdjson/westmere/bitmanipulation.h" */ -/* begin file simdjson/westmere/bitmanipulation.h */ -#ifndef SIMDJSON_WESTMERE_BITMANIPULATION_H -#define SIMDJSON_WESTMERE_BITMANIPULATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/westmere/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/westmere/intrinsics.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - namespace westmere { - namespace { - - // We sometimes call trailing_zero on inputs that are zero, - // but the algorithms do not end up using the returned value. - // Sadly, sanitizers are not smart enough to figure it out. - SIMDJSON_NO_SANITIZE_UNDEFINED - // This function can be used safely even if not all bytes have been - // initialized. - // See issue https://github.com/simdjson/simdjson/issues/1965 - SIMDJSON_NO_SANITIZE_MEMORY - simdjson_inline int trailing_zeroes(uint64_t input_num) { -#if SIMDJSON_REGULAR_VISUAL_STUDIO - unsigned long ret; - // Search the mask data from least significant bit (LSB) - // to the most significant bit (MSB) for a set bit (1). - _BitScanForward64(&ret, input_num); - return (int)ret; -#else // SIMDJSON_REGULAR_VISUAL_STUDIO - return __builtin_ctzll(input_num); -#endif // SIMDJSON_REGULAR_VISUAL_STUDIO - } - - /* result might be undefined when input_num is zero */ - simdjson_inline uint64_t clear_lowest_bit(uint64_t input_num) { - return input_num & (input_num - 1); - } - - /* result might be undefined when input_num is zero */ - simdjson_inline int leading_zeroes(uint64_t input_num) { -#if SIMDJSON_REGULAR_VISUAL_STUDIO - unsigned long leading_zero = 0; - // Search the mask data from most significant bit (MSB) - // to least significant bit (LSB) for a set bit (1). - if (_BitScanReverse64(&leading_zero, input_num)) - return (int)(63 - leading_zero); - else - return 64; -#else - return __builtin_clzll(input_num); -#endif// SIMDJSON_REGULAR_VISUAL_STUDIO - } - -#if SIMDJSON_REGULAR_VISUAL_STUDIO - simdjson_inline unsigned __int64 count_ones(uint64_t input_num) { - // note: we do not support legacy 32-bit Windows in this kernel - return __popcnt64(input_num);// Visual Studio wants two underscores - } -#else - simdjson_inline long long int count_ones(uint64_t input_num) { - return _popcnt64(input_num); - } -#endif - - simdjson_inline bool add_overflow(uint64_t value1, uint64_t value2, - uint64_t* result) { -#if SIMDJSON_REGULAR_VISUAL_STUDIO - return _addcarry_u64(0, value1, value2, - reinterpret_cast(result)); -#else - return __builtin_uaddll_overflow(value1, value2, - reinterpret_cast(result)); -#endif - } - - } // unnamed namespace - } // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_WESTMERE_BITMANIPULATION_H -/* end file simdjson/westmere/bitmanipulation.h */ -/* including simdjson/westmere/simd.h: #include "simdjson/westmere/simd.h" */ -/* begin file simdjson/westmere/simd.h */ -#ifndef SIMDJSON_WESTMERE_SIMD_H -#define SIMDJSON_WESTMERE_SIMD_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/westmere/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/westmere/bitmanipulation.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/internal/simdprune_tables.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - namespace westmere { - namespace { - namespace simd { - - template - struct base { - __m128i value; - - // Zero constructor - simdjson_inline base() : value{ __m128i() } {} - - // Conversion from SIMD register - simdjson_inline base(const __m128i _value) : value(_value) {} - - // Conversion to SIMD register - simdjson_inline operator const __m128i& () const { return this->value; } - simdjson_inline operator __m128i& () { return this->value; } - - // Bit operations - simdjson_inline Child operator|(const Child other) const { return _mm_or_si128(*this, other); } - simdjson_inline Child operator&(const Child other) const { return _mm_and_si128(*this, other); } - simdjson_inline Child operator^(const Child other) const { return _mm_xor_si128(*this, other); } - simdjson_inline Child bit_andnot(const Child other) const { return _mm_andnot_si128(other, *this); } - simdjson_inline Child& operator|=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast | other; return *this_cast; } - simdjson_inline Child& operator&=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast & other; return *this_cast; } - simdjson_inline Child& operator^=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast ^ other; return *this_cast; } - }; - - template> - struct base8 : base> { - typedef uint16_t bitmask_t; - typedef uint32_t bitmask2_t; - - simdjson_inline base8() : base>() {} - simdjson_inline base8(const __m128i _value) : base>(_value) {} - - friend simdjson_inline Mask operator==(const simd8 lhs, const simd8 rhs) { return _mm_cmpeq_epi8(lhs, rhs); } - - static const int SIZE = sizeof(base>::value); - - template - simdjson_inline simd8 prev(const simd8 prev_chunk) const { - return _mm_alignr_epi8(*this, prev_chunk, 16 - N); - } - }; - - // SIMD byte mask type (returned by things like eq and gt) - template<> - struct simd8 : base8 { - static simdjson_inline simd8 splat(bool _value) { return _mm_set1_epi8(uint8_t(-(!!_value))); } - - simdjson_inline simd8() : base8() {} - simdjson_inline simd8(const __m128i _value) : base8(_value) {} - // Splat constructor - simdjson_inline simd8(bool _value) : base8(splat(_value)) {} - - simdjson_inline int to_bitmask() const { return _mm_movemask_epi8(*this); } - simdjson_inline bool any() const { return !_mm_testz_si128(*this, *this); } - simdjson_inline simd8 operator~() const { return *this ^ true; } - }; - - template - struct base8_numeric : base8 { - static simdjson_inline simd8 splat(T _value) { return _mm_set1_epi8(_value); } - static simdjson_inline simd8 zero() { return _mm_setzero_si128(); } - static simdjson_inline simd8 load(const T values[16]) { - return _mm_loadu_si128(reinterpret_cast(values)); - } - // Repeat 16 values as many times as necessary (usually for lookup tables) - static simdjson_inline simd8 repeat_16( - T v0, T v1, T v2, T v3, T v4, T v5, T v6, T v7, - T v8, T v9, T v10, T v11, T v12, T v13, T v14, T v15 - ) { - return simd8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10, v11, v12, v13, v14, v15 - ); - } - - simdjson_inline base8_numeric() : base8() {} - simdjson_inline base8_numeric(const __m128i _value) : base8(_value) {} - - // Store to array - simdjson_inline void store(T dst[16]) const { return _mm_storeu_si128(reinterpret_cast<__m128i*>(dst), *this); } - - // Override to distinguish from bool version - simdjson_inline simd8 operator~() const { return *this ^ 0xFFu; } - - // Addition/subtraction are the same for signed and unsigned - simdjson_inline simd8 operator+(const simd8 other) const { return _mm_add_epi8(*this, other); } - simdjson_inline simd8 operator-(const simd8 other) const { return _mm_sub_epi8(*this, other); } - simdjson_inline simd8& operator+=(const simd8 other) { *this = *this + other; return *static_cast*>(this); } - simdjson_inline simd8& operator-=(const simd8 other) { *this = *this - other; return *static_cast*>(this); } - - // Perform a lookup assuming the value is between 0 and 16 (undefined behavior for out of range values) - template - simdjson_inline simd8 lookup_16(simd8 lookup_table) const { - return _mm_shuffle_epi8(lookup_table, *this); - } - - // Copies to 'output" all bytes corresponding to a 0 in the mask (interpreted as a bitset). - // Passing a 0 value for mask would be equivalent to writing out every byte to output. - // Only the first 16 - count_ones(mask) bytes of the result are significant but 16 bytes - // get written. - // Design consideration: it seems like a function with the - // signature simd8 compress(uint32_t mask) would be - // sensible, but the AVX ISA makes this kind of approach difficult. - template - simdjson_inline void compress(uint16_t mask, L* output) const { - using internal::thintable_epi8; - using internal::BitsSetTable256mul2; - using internal::pshufb_combine_table; - // this particular implementation was inspired by work done by @animetosho - // we do it in two steps, first 8 bytes and then second 8 bytes - uint8_t mask1 = uint8_t(mask); // least significant 8 bits - uint8_t mask2 = uint8_t(mask >> 8); // most significant 8 bits - // next line just loads the 64-bit values thintable_epi8[mask1] and - // thintable_epi8[mask2] into a 128-bit register, using only - // two instructions on most compilers. - __m128i shufmask = _mm_set_epi64x(thintable_epi8[mask2], thintable_epi8[mask1]); - // we increment by 0x08 the second half of the mask - shufmask = - _mm_add_epi8(shufmask, _mm_set_epi32(0x08080808, 0x08080808, 0, 0)); - // this is the version "nearly pruned" - __m128i pruned = _mm_shuffle_epi8(*this, shufmask); - // we still need to put the two halves together. - // we compute the popcount of the first half: - int pop1 = BitsSetTable256mul2[mask1]; - // then load the corresponding mask, what it does is to write - // only the first pop1 bytes from the first 8 bytes, and then - // it fills in with the bytes from the second 8 bytes + some filling - // at the end. - __m128i compactmask = - _mm_loadu_si128(reinterpret_cast(pshufb_combine_table + pop1 * 8)); - __m128i answer = _mm_shuffle_epi8(pruned, compactmask); - _mm_storeu_si128(reinterpret_cast<__m128i*>(output), answer); - } - - template - simdjson_inline simd8 lookup_16( - L replace0, L replace1, L replace2, L replace3, - L replace4, L replace5, L replace6, L replace7, - L replace8, L replace9, L replace10, L replace11, - L replace12, L replace13, L replace14, L replace15) const { - return lookup_16(simd8::repeat_16( - replace0, replace1, replace2, replace3, - replace4, replace5, replace6, replace7, - replace8, replace9, replace10, replace11, - replace12, replace13, replace14, replace15 - )); - } - }; - - // Signed bytes - template<> - struct simd8 : base8_numeric { - simdjson_inline simd8() : base8_numeric() {} - simdjson_inline simd8(const __m128i _value) : base8_numeric(_value) {} - // Splat constructor - simdjson_inline simd8(int8_t _value) : simd8(splat(_value)) {} - // Array constructor - simdjson_inline simd8(const int8_t* values) : simd8(load(values)) {} - // Member-by-member initialization - simdjson_inline simd8( - int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, - int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 - ) : simd8(_mm_setr_epi8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10, v11, v12, v13, v14, v15 - )) {} - // Repeat 16 values as many times as necessary (usually for lookup tables) - simdjson_inline static simd8 repeat_16( - int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, - int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 - ) { - return simd8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10, v11, v12, v13, v14, v15 - ); - } - - // Order-sensitive comparisons - simdjson_inline simd8 max_val(const simd8 other) const { return _mm_max_epi8(*this, other); } - simdjson_inline simd8 min_val(const simd8 other) const { return _mm_min_epi8(*this, other); } - simdjson_inline simd8 operator>(const simd8 other) const { return _mm_cmpgt_epi8(*this, other); } - simdjson_inline simd8 operator<(const simd8 other) const { return _mm_cmpgt_epi8(other, *this); } - }; - - // Unsigned bytes - template<> - struct simd8 : base8_numeric { - simdjson_inline simd8() : base8_numeric() {} - simdjson_inline simd8(const __m128i _value) : base8_numeric(_value) {} - // Splat constructor - simdjson_inline simd8(uint8_t _value) : simd8(splat(_value)) {} - // Array constructor - simdjson_inline simd8(const uint8_t* values) : simd8(load(values)) {} - // Member-by-member initialization - simdjson_inline simd8( - uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, - uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 - ) : simd8(_mm_setr_epi8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10, v11, v12, v13, v14, v15 - )) {} - // Repeat 16 values as many times as necessary (usually for lookup tables) - simdjson_inline static simd8 repeat_16( - uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, - uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 - ) { - return simd8( - v0, v1, v2, v3, v4, v5, v6, v7, - v8, v9, v10, v11, v12, v13, v14, v15 - ); - } - - // Saturated math - simdjson_inline simd8 saturating_add(const simd8 other) const { return _mm_adds_epu8(*this, other); } - simdjson_inline simd8 saturating_sub(const simd8 other) const { return _mm_subs_epu8(*this, other); } - - // Order-specific operations - simdjson_inline simd8 max_val(const simd8 other) const { return _mm_max_epu8(*this, other); } - simdjson_inline simd8 min_val(const simd8 other) const { return _mm_min_epu8(*this, other); } - // Same as >, but only guarantees true is nonzero (< guarantees true = -1) - simdjson_inline simd8 gt_bits(const simd8 other) const { return this->saturating_sub(other); } - // Same as <, but only guarantees true is nonzero (< guarantees true = -1) - simdjson_inline simd8 lt_bits(const simd8 other) const { return other.saturating_sub(*this); } - simdjson_inline simd8 operator<=(const simd8 other) const { return other.max_val(*this) == other; } - simdjson_inline simd8 operator>=(const simd8 other) const { return other.min_val(*this) == other; } - simdjson_inline simd8 operator>(const simd8 other) const { return this->gt_bits(other).any_bits_set(); } - simdjson_inline simd8 operator<(const simd8 other) const { return this->gt_bits(other).any_bits_set(); } - - // Bit-specific operations - simdjson_inline simd8 bits_not_set() const { return *this == uint8_t(0); } - simdjson_inline simd8 bits_not_set(simd8 bits) const { return (*this & bits).bits_not_set(); } - simdjson_inline simd8 any_bits_set() const { return ~this->bits_not_set(); } - simdjson_inline simd8 any_bits_set(simd8 bits) const { return ~this->bits_not_set(bits); } - simdjson_inline bool is_ascii() const { return _mm_movemask_epi8(*this) == 0; } - simdjson_inline bool bits_not_set_anywhere() const { return _mm_testz_si128(*this, *this); } - simdjson_inline bool any_bits_set_anywhere() const { return !bits_not_set_anywhere(); } - simdjson_inline bool bits_not_set_anywhere(simd8 bits) const { return _mm_testz_si128(*this, bits); } - simdjson_inline bool any_bits_set_anywhere(simd8 bits) const { return !bits_not_set_anywhere(bits); } - template - simdjson_inline simd8 shr() const { return simd8(_mm_srli_epi16(*this, N)) & uint8_t(0xFFu >> N); } - template - simdjson_inline simd8 shl() const { return simd8(_mm_slli_epi16(*this, N)) & uint8_t(0xFFu << N); } - // Get one of the bits and make a bitmask out of it. - // e.g. value.get_bit<7>() gets the high bit - template - simdjson_inline int get_bit() const { return _mm_movemask_epi8(_mm_slli_epi16(*this, 7 - N)); } - }; - - template - struct simd8x64 { - static constexpr int NUM_CHUNKS = 64 / sizeof(simd8); - static_assert(NUM_CHUNKS == 4, "Westmere kernel should use four registers per 64-byte block."); - const simd8 chunks[NUM_CHUNKS]; - - simd8x64(const simd8x64& o) = delete; // no copy allowed - simd8x64& operator=(const simd8& other) = delete; // no assignment allowed - simd8x64() = delete; // no default constructor allowed - - simdjson_inline simd8x64(const simd8 chunk0, const simd8 chunk1, const simd8 chunk2, const simd8 chunk3) : chunks{ chunk0, chunk1, chunk2, chunk3 } {} - simdjson_inline simd8x64(const T ptr[64]) : chunks{ simd8::load(ptr), simd8::load(ptr + 16), simd8::load(ptr + 32), simd8::load(ptr + 48) } {} - - simdjson_inline void store(T ptr[64]) const { - this->chunks[0].store(ptr + sizeof(simd8) * 0); - this->chunks[1].store(ptr + sizeof(simd8) * 1); - this->chunks[2].store(ptr + sizeof(simd8) * 2); - this->chunks[3].store(ptr + sizeof(simd8) * 3); - } - - simdjson_inline simd8 reduce_or() const { - return (this->chunks[0] | this->chunks[1]) | (this->chunks[2] | this->chunks[3]); - } - - simdjson_inline uint64_t compress(uint64_t mask, T* output) const { - this->chunks[0].compress(uint16_t(mask), output); - this->chunks[1].compress(uint16_t(mask >> 16), output + 16 - count_ones(mask & 0xFFFF)); - this->chunks[2].compress(uint16_t(mask >> 32), output + 32 - count_ones(mask & 0xFFFFFFFF)); - this->chunks[3].compress(uint16_t(mask >> 48), output + 48 - count_ones(mask & 0xFFFFFFFFFFFF)); - return 64 - count_ones(mask); - } - - simdjson_inline uint64_t to_bitmask() const { - uint64_t r0 = uint32_t(this->chunks[0].to_bitmask()); - uint64_t r1 = this->chunks[1].to_bitmask(); - uint64_t r2 = this->chunks[2].to_bitmask(); - uint64_t r3 = this->chunks[3].to_bitmask(); - return r0 | (r1 << 16) | (r2 << 32) | (r3 << 48); - } - - simdjson_inline uint64_t eq(const T m) const { - const simd8 mask = simd8::splat(m); - return simd8x64( - this->chunks[0] == mask, - this->chunks[1] == mask, - this->chunks[2] == mask, - this->chunks[3] == mask - ).to_bitmask(); - } - - simdjson_inline uint64_t eq(const simd8x64& other) const { - return simd8x64( - this->chunks[0] == other.chunks[0], - this->chunks[1] == other.chunks[1], - this->chunks[2] == other.chunks[2], - this->chunks[3] == other.chunks[3] - ).to_bitmask(); - } - - simdjson_inline uint64_t lteq(const T m) const { - const simd8 mask = simd8::splat(m); - return simd8x64( - this->chunks[0] <= mask, - this->chunks[1] <= mask, - this->chunks[2] <= mask, - this->chunks[3] <= mask - ).to_bitmask(); - } - }; // struct simd8x64 - - } // namespace simd - } // unnamed namespace - } // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_WESTMERE_SIMD_INPUT_H -/* end file simdjson/westmere/simd.h */ - -namespace simdjson { - namespace westmere { - namespace { - - using namespace simd; - - // Holds backslashes and quotes locations. - struct backslash_and_quote { - public: - static constexpr uint32_t BYTES_PROCESSED = 32; - simdjson_inline static backslash_and_quote copy_and_find(const uint8_t* src, uint8_t* dst); - - simdjson_inline bool has_quote_first() { return ((bs_bits - 1) & quote_bits) != 0; } - simdjson_inline bool has_backslash() { return bs_bits != 0; } - simdjson_inline int quote_index() { return trailing_zeroes(quote_bits); } - simdjson_inline int backslash_index() { return trailing_zeroes(bs_bits); } - - uint32_t bs_bits; - uint32_t quote_bits; - }; // struct backslash_and_quote - - simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8_t* src, uint8_t* dst) { - // this can read up to 31 bytes beyond the buffer size, but we require - // SIMDJSON_PADDING of padding - static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "backslash and quote finder must process fewer than SIMDJSON_PADDING bytes"); - simd8 v0(src); - simd8 v1(src + 16); - v0.store(dst); - v1.store(dst + 16); - uint64_t bs_and_quote = simd8x64(v0 == '\\', v1 == '\\', v0 == '"', v1 == '"').to_bitmask(); - return { - uint32_t(bs_and_quote), // bs_bits - uint32_t(bs_and_quote >> 32) // quote_bits - }; - } - - } // unnamed namespace - } // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_WESTMERE_STRINGPARSING_DEFS_H -/* end file simdjson/westmere/stringparsing_defs.h */ -/* end file simdjson/westmere/begin.h */ -/* including generic/amalgamated.h for westmere: #include */ -/* begin file generic/amalgamated.h for westmere */ -#if defined(SIMDJSON_CONDITIONAL_INCLUDE) && !defined(SIMDJSON_SRC_GENERIC_DEPENDENCIES_H) -#error generic/dependencies.h must be included before generic/amalgamated.h! -#endif - -/* including generic/base.h for westmere: #include */ -/* begin file generic/base.h for westmere */ -#ifndef SIMDJSON_SRC_GENERIC_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_BASE_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - namespace westmere { - namespace { - - struct json_character_block; - - } // unnamed namespace - } // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_BASE_H -/* end file generic/base.h for westmere */ -/* including generic/dom_parser_implementation.h for westmere: #include */ -/* begin file generic/dom_parser_implementation.h for westmere */ -#ifndef SIMDJSON_SRC_GENERIC_DOM_PARSER_IMPLEMENTATION_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_DOM_PARSER_IMPLEMENTATION_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -// Interface a dom parser implementation must fulfill -namespace simdjson { - namespace westmere { - namespace { - - simdjson_inline simd8 must_be_2_3_continuation(const simd8 prev2, const simd8 prev3); - simdjson_inline bool is_ascii(const simd8x64& input); - - } // unnamed namespace - } // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_DOM_PARSER_IMPLEMENTATION_H -/* end file generic/dom_parser_implementation.h for westmere */ -/* including generic/json_character_block.h for westmere: #include */ -/* begin file generic/json_character_block.h for westmere */ -#ifndef SIMDJSON_SRC_GENERIC_JSON_CHARACTER_BLOCK_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_JSON_CHARACTER_BLOCK_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - namespace westmere { - namespace { - - struct json_character_block { - static simdjson_inline json_character_block classify(const simd::simd8x64& in); - - simdjson_inline uint64_t whitespace() const noexcept { return _whitespace; } - simdjson_inline uint64_t op() const noexcept { return _op; } - simdjson_inline uint64_t scalar() const noexcept { return ~(op() | whitespace()); } - - uint64_t _whitespace; - uint64_t _op; - }; - - } // unnamed namespace - } // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_JSON_CHARACTER_BLOCK_H -/* end file generic/json_character_block.h for westmere */ -/* end file generic/amalgamated.h for westmere */ -/* including generic/stage1/amalgamated.h for westmere: #include */ -/* begin file generic/stage1/amalgamated.h for westmere */ -// Stuff other things depend on -/* including generic/stage1/base.h for westmere: #include */ -/* begin file generic/stage1/base.h for westmere */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_BASE_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - namespace westmere { - namespace { - namespace stage1 { - - class bit_indexer; - template - struct buf_block_reader; - struct json_block; - class json_minifier; - class json_scanner; - struct json_string_block; - class json_string_scanner; - class json_structural_indexer; - - } // namespace stage1 - - namespace utf8_validation { - struct utf8_checker; - } // namespace utf8_validation - - using utf8_validation::utf8_checker; - - } // unnamed namespace - } // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_BASE_H -/* end file generic/stage1/base.h for westmere */ -/* including generic/stage1/buf_block_reader.h for westmere: #include */ -/* begin file generic/stage1/buf_block_reader.h for westmere */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_BUF_BLOCK_READER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_BUF_BLOCK_READER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include - -namespace simdjson { - namespace westmere { - namespace { - namespace stage1 { - - // Walks through a buffer in block-sized increments, loading the last part with spaces - template - struct buf_block_reader { - public: - simdjson_inline buf_block_reader(const uint8_t* _buf, size_t _len); - simdjson_inline size_t block_index(); - simdjson_inline bool has_full_block() const; - simdjson_inline const uint8_t* full_block() const; - /** - * Get the last block, padded with spaces. - * - * There will always be a last block, with at least 1 byte, unless len == 0 (in which case this - * function fills the buffer with spaces and returns 0. In particular, if len == STEP_SIZE there - * will be 0 full_blocks and 1 remainder block with STEP_SIZE bytes and no spaces for padding. - * - * @return the number of effective characters in the last block. - */ - simdjson_inline size_t get_remainder(uint8_t* dst) const; - simdjson_inline void advance(); - private: - const uint8_t* buf; - const size_t len; - const size_t lenminusstep; - size_t idx; - }; - - // Routines to print masks and text for debugging bitmask operations - simdjson_unused static char* format_input_text_64(const uint8_t* text) { - static char buf[sizeof(simd8x64) + 1]; - for (size_t i = 0; i < sizeof(simd8x64); i++) { - buf[i] = int8_t(text[i]) < ' ' ? '_' : int8_t(text[i]); - } - buf[sizeof(simd8x64)] = '\0'; - return buf; - } - - // Routines to print masks and text for debugging bitmask operations - simdjson_unused static char* format_input_text(const simd8x64& in) { - static char buf[sizeof(simd8x64) + 1]; - in.store(reinterpret_cast(buf)); - for (size_t i = 0; i < sizeof(simd8x64); i++) { - if (buf[i] < ' ') { buf[i] = '_'; } - } - buf[sizeof(simd8x64)] = '\0'; - return buf; - } - - simdjson_unused static char* format_input_text(const simd8x64& in, uint64_t mask) { - static char buf[sizeof(simd8x64) + 1]; - in.store(reinterpret_cast(buf)); - for (size_t i = 0; i < sizeof(simd8x64); i++) { - if (buf[i] <= ' ') { buf[i] = '_'; } - if (!(mask & (size_t(1) << i))) { buf[i] = ' '; } - } - buf[sizeof(simd8x64)] = '\0'; - return buf; - } - - simdjson_unused static char* format_mask(uint64_t mask) { - static char buf[sizeof(simd8x64) + 1]; - for (size_t i = 0; i < 64; i++) { - buf[i] = (mask & (size_t(1) << i)) ? 'X' : ' '; - } - buf[64] = '\0'; - return buf; - } - - template - simdjson_inline buf_block_reader::buf_block_reader(const uint8_t* _buf, size_t _len) : buf{ _buf }, len{ _len }, lenminusstep{ len < STEP_SIZE ? 0 : len - STEP_SIZE }, idx{ 0 } {} - - template - simdjson_inline size_t buf_block_reader::block_index() { return idx; } - - template - simdjson_inline bool buf_block_reader::has_full_block() const { - return idx < lenminusstep; - } - - template - simdjson_inline const uint8_t* buf_block_reader::full_block() const { - return &buf[idx]; - } - - template - simdjson_inline size_t buf_block_reader::get_remainder(uint8_t* dst) const { - if (len == idx) { return 0; } // memcpy(dst, null, 0) will trigger an error with some sanitizers - std::memset(dst, 0x20, STEP_SIZE); // std::memset STEP_SIZE because it's more efficient to write out 8 or 16 bytes at once. - std::memcpy(dst, buf + idx, len - idx); - return len - idx; - } - - template - simdjson_inline void buf_block_reader::advance() { - idx += STEP_SIZE; - } - - } // namespace stage1 - } // unnamed namespace - } // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_BUF_BLOCK_READER_H -/* end file generic/stage1/buf_block_reader.h for westmere */ -/* including generic/stage1/json_escape_scanner.h for westmere: #include */ -/* begin file generic/stage1/json_escape_scanner.h for westmere */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_JSON_ESCAPE_SCANNER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_JSON_ESCAPE_SCANNER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - namespace westmere { - namespace { - namespace stage1 { - - /** - * Scans for escape characters in JSON, taking care with multiple backslashes (\\n vs. \n). - */ - struct json_escape_scanner { - /** The actual escape characters (the backslashes themselves). */ - uint64_t next_is_escaped = 0ULL; - - struct escaped_and_escape { - /** - * Mask of escaped characters. - * - * ``` - * \n \\n \\\n \\\\n \ - * 0100100010100101000 - * n \ \ n \ \ - * ``` - */ - uint64_t escaped; - /** - * Mask of escape characters. - * - * ``` - * \n \\n \\\n \\\\n \ - * 1001000101001010001 - * \ \ \ \ \ \ \ - * ``` - */ - uint64_t escape; - }; - - /** - * Get a mask of both escape and escaped characters (the characters following a backslash). - * - * @param potential_escape A mask of the character that can escape others (but could be - * escaped itself). e.g. block.eq('\\') - */ - simdjson_really_inline escaped_and_escape next(uint64_t backslash) noexcept { - -#if !SIMDJSON_SKIP_BACKSLASH_SHORT_CIRCUIT - if (!backslash) { return { next_escaped_without_backslashes(), 0 }; } -#endif - - // | | Mask (shows characters instead of 1's) | Depth | Instructions | - // |--------------------------------|----------------------------------------|-------|---------------------| - // | string | `\\n_\\\n___\\\n___\\\\___\\\\__\\\` | | | - // | | ` even odd even odd odd` | | | - // | potential_escape | ` \ \\\ \\\ \\\\ \\\\ \\\` | 1 | 1 (backslash & ~first_is_escaped) - // | escape_and_terminal_code | ` \n \ \n \ \n \ \ \ \ \ \` | 5 | 5 (next_escape_and_terminal_code()) - // | escaped | `\ \ n \ n \ \ \ \ \ ` X | 6 | 7 (escape_and_terminal_code ^ (potential_escape | first_is_escaped)) - // | escape | ` \ \ \ \ \ \ \ \ \ \` | 6 | 8 (escape_and_terminal_code & backslash) - // | first_is_escaped | `\ ` | 7 (*) | 9 (escape >> 63) () - // (*) this is not needed until the next iteration - uint64_t escape_and_terminal_code = next_escape_and_terminal_code(backslash & ~this->next_is_escaped); - uint64_t escaped = escape_and_terminal_code ^ (backslash | this->next_is_escaped); - uint64_t escape = escape_and_terminal_code & backslash; - this->next_is_escaped = escape >> 63; - return { escaped, escape }; - } - - private: - static constexpr const uint64_t ODD_BITS = 0xAAAAAAAAAAAAAAAAULL; - - simdjson_really_inline uint64_t next_escaped_without_backslashes() noexcept { - uint64_t escaped = this->next_is_escaped; - this->next_is_escaped = 0; - return escaped; - } - - /** - * Returns a mask of the next escape characters (masking out escaped backslashes), along with - * any non-backslash escape codes. - * - * \n \\n \\\n \\\\n returns: - * \n \ \ \n \ \ - * 11 100 1011 10100 - * - * You are expected to mask out the first bit yourself if the previous block had a trailing - * escape. - * - * & the result with potential_escape to get just the escape characters. - * ^ the result with (potential_escape | first_is_escaped) to get escaped characters. - */ - static simdjson_really_inline uint64_t next_escape_and_terminal_code(uint64_t potential_escape) noexcept { - // If we were to just shift and mask out any odd bits, we'd actually get a *half* right answer: - // any even-aligned backslash runs would be correct! Odd-aligned backslash runs would be - // inverted (\\\ would be 010 instead of 101). - // - // ``` - // string: | ____\\\\_\\\\_____ | - // maybe_escaped | ODD | \ \ \ \ | - // even-aligned ^^^ ^^^^ odd-aligned - // ``` - // - // Taking that into account, our basic strategy is: - // - // 1. Use subtraction to produce a mask with 1's for even-aligned runs and 0's for - // odd-aligned runs. - // 2. XOR all odd bits, which masks out the odd bits in even-aligned runs, and brings IN the - // odd bits in odd-aligned runs. - // 3. & with backslash to clean up any stray bits. - // runs are set to 0, and then XORing with "odd": - // - // | | Mask (shows characters instead of 1's) | Instructions | - // |--------------------------------|----------------------------------------|---------------------| - // | string | `\\n_\\\n___\\\n___\\\\___\\\\__\\\` | - // | | ` even odd even odd odd` | - // | maybe_escaped | ` n \\n \\n \\\_ \\\_ \\` X | 1 (potential_escape << 1) - // | maybe_escaped_and_odd | ` \n_ \\n _ \\\n_ _ \\\__ _\\\_ \\\` | 1 (maybe_escaped | odd) - // | even_series_codes_and_odd | ` n_\\\ _ n_ _\\\\ _ _ ` | 1 (maybe_escaped_and_odd - potential_escape) - // | escape_and_terminal_code | ` \n \ \n \ \n \ \ \ \ \ \` | 1 (^ odd) - // - - // Escaped characters are characters following an escape. - uint64_t maybe_escaped = potential_escape << 1; - - // To distinguish odd from even escape sequences, therefore, we turn on any *starting* - // escapes that are on an odd byte. (We actually bring in all odd bits, for speed.) - // - Odd runs of backslashes are 0000, and the code at the end ("n" in \n or \\n) is 1. - // - Odd runs of backslashes are 1111, and the code at the end ("n" in \n or \\n) is 0. - // - All other odd bytes are 1, and even bytes are 0. - uint64_t maybe_escaped_and_odd_bits = maybe_escaped | ODD_BITS; - uint64_t even_series_codes_and_odd_bits = maybe_escaped_and_odd_bits - potential_escape; - - // Now we flip all odd bytes back with xor. This: - // - Makes odd runs of backslashes go from 0000 to 1010 - // - Makes even runs of backslashes go from 1111 to 1010 - // - Sets actually-escaped codes to 1 (the n in \n and \\n: \n = 11, \\n = 100) - // - Resets all other bytes to 0 - return even_series_codes_and_odd_bits ^ ODD_BITS; - } - }; - - } // namespace stage1 - } // unnamed namespace - } // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRING_SCANNER_H -/* end file generic/stage1/json_escape_scanner.h for westmere */ -/* including generic/stage1/json_string_scanner.h for westmere: #include */ -/* begin file generic/stage1/json_string_scanner.h for westmere */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRING_SCANNER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRING_SCANNER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - namespace westmere { - namespace { - namespace stage1 { - - struct json_string_block { - // We spell out the constructors in the hope of resolving inlining issues with Visual Studio 2017 - simdjson_really_inline json_string_block(uint64_t escaped, uint64_t quote, uint64_t in_string) : - _escaped(escaped), _quote(quote), _in_string(in_string) {} - - // Escaped characters (characters following an escape() character) - simdjson_really_inline uint64_t escaped() const { return _escaped; } - // Real (non-backslashed) quotes - simdjson_really_inline uint64_t quote() const { return _quote; } - // Only characters inside the string (not including the quotes) - simdjson_really_inline uint64_t string_content() const { return _in_string & ~_quote; } - // Return a mask of whether the given characters are inside a string (only works on non-quotes) - simdjson_really_inline uint64_t non_quote_inside_string(uint64_t mask) const { return mask & _in_string; } - // Return a mask of whether the given characters are inside a string (only works on non-quotes) - simdjson_really_inline uint64_t non_quote_outside_string(uint64_t mask) const { return mask & ~_in_string; } - // Tail of string (everything except the start quote) - simdjson_really_inline uint64_t string_tail() const { return _in_string ^ _quote; } - - // escaped characters (backslashed--does not include the hex characters after \u) - uint64_t _escaped; - // real quotes (non-escaped ones) - uint64_t _quote; - // string characters (includes start quote but not end quote) - uint64_t _in_string; - }; - - // Scans blocks for string characters, storing the state necessary to do so - class json_string_scanner { - public: - simdjson_really_inline json_string_block next(const simd::simd8x64& in); - // Returns either UNCLOSED_STRING or SUCCESS - simdjson_really_inline error_code finish(); - - private: - // Scans for escape characters - json_escape_scanner escape_scanner{}; - // Whether the last iteration was still inside a string (all 1's = true, all 0's = false). - uint64_t prev_in_string = 0ULL; - }; - - // - // Return a mask of all string characters plus end quotes. - // - // prev_escaped is overflow saying whether the next character is escaped. - // prev_in_string is overflow saying whether we're still in a string. - // - // Backslash sequences outside of quotes will be detected in stage 2. - // - simdjson_really_inline json_string_block json_string_scanner::next(const simd::simd8x64& in) { - const uint64_t backslash = in.eq('\\'); - const uint64_t escaped = escape_scanner.next(backslash).escaped; - const uint64_t quote = in.eq('"') & ~escaped; - - // - // prefix_xor flips on bits inside the string (and flips off the end quote). - // - // Then we xor with prev_in_string: if we were in a string already, its effect is flipped - // (characters inside strings are outside, and characters outside strings are inside). - // - const uint64_t in_string = prefix_xor(quote) ^ prev_in_string; - - // - // Check if we're still in a string at the end of the box so the next block will know - // - prev_in_string = uint64_t(static_cast(in_string) >> 63); - - // Use ^ to turn the beginning quote off, and the end quote on. - - // We are returning a function-local object so either we get a move constructor - // or we get copy elision. - return json_string_block(escaped, quote, in_string); - } - - simdjson_really_inline error_code json_string_scanner::finish() { - if (prev_in_string) { - return UNCLOSED_STRING; - } - return SUCCESS; - } - - } // namespace stage1 - } // unnamed namespace - } // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRING_SCANNER_H -/* end file generic/stage1/json_string_scanner.h for westmere */ -/* including generic/stage1/utf8_lookup4_algorithm.h for westmere: #include */ -/* begin file generic/stage1/utf8_lookup4_algorithm.h for westmere */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_UTF8_LOOKUP4_ALGORITHM_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_UTF8_LOOKUP4_ALGORITHM_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - namespace westmere { - namespace { - namespace utf8_validation { - - using namespace simd; - - simdjson_inline simd8 check_special_cases(const simd8 input, const simd8 prev1) { - // Bit 0 = Too Short (lead byte/ASCII followed by lead byte/ASCII) - // Bit 1 = Too Long (ASCII followed by continuation) - // Bit 2 = Overlong 3-byte - // Bit 4 = Surrogate - // Bit 5 = Overlong 2-byte - // Bit 7 = Two Continuations - constexpr const uint8_t TOO_SHORT = 1 << 0; // 11______ 0_______ - // 11______ 11______ - constexpr const uint8_t TOO_LONG = 1 << 1; // 0_______ 10______ - constexpr const uint8_t OVERLONG_3 = 1 << 2; // 11100000 100_____ - constexpr const uint8_t SURROGATE = 1 << 4; // 11101101 101_____ - constexpr const uint8_t OVERLONG_2 = 1 << 5; // 1100000_ 10______ - constexpr const uint8_t TWO_CONTS = 1 << 7; // 10______ 10______ - constexpr const uint8_t TOO_LARGE = 1 << 3; // 11110100 1001____ - // 11110100 101_____ - // 11110101 1001____ - // 11110101 101_____ - // 1111011_ 1001____ - // 1111011_ 101_____ - // 11111___ 1001____ - // 11111___ 101_____ - constexpr const uint8_t TOO_LARGE_1000 = 1 << 6; - // 11110101 1000____ - // 1111011_ 1000____ - // 11111___ 1000____ - constexpr const uint8_t OVERLONG_4 = 1 << 6; // 11110000 1000____ - - const simd8 byte_1_high = prev1.shr<4>().lookup_16( - // 0_______ ________ - TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, - TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, - // 10______ ________ - TWO_CONTS, TWO_CONTS, TWO_CONTS, TWO_CONTS, - // 1100____ ________ - TOO_SHORT | OVERLONG_2, - // 1101____ ________ - TOO_SHORT, - // 1110____ ________ - TOO_SHORT | OVERLONG_3 | SURROGATE, - // 1111____ ________ - TOO_SHORT | TOO_LARGE | TOO_LARGE_1000 | OVERLONG_4 - ); - constexpr const uint8_t CARRY = TOO_SHORT | TOO_LONG | TWO_CONTS; // These all have ____ in byte 1 . - const simd8 byte_1_low = (prev1 & 0x0F).lookup_16( - // ____0000 ________ - CARRY | OVERLONG_3 | OVERLONG_2 | OVERLONG_4, - // ____0001 ________ - CARRY | OVERLONG_2, - // ____001_ ________ - CARRY, - CARRY, - - // ____0100 ________ - CARRY | TOO_LARGE, - // ____0101 ________ - CARRY | TOO_LARGE | TOO_LARGE_1000, - // ____011_ ________ - CARRY | TOO_LARGE | TOO_LARGE_1000, - CARRY | TOO_LARGE | TOO_LARGE_1000, - - // ____1___ ________ - CARRY | TOO_LARGE | TOO_LARGE_1000, - CARRY | TOO_LARGE | TOO_LARGE_1000, - CARRY | TOO_LARGE | TOO_LARGE_1000, - CARRY | TOO_LARGE | TOO_LARGE_1000, - CARRY | TOO_LARGE | TOO_LARGE_1000, - // ____1101 ________ - CARRY | TOO_LARGE | TOO_LARGE_1000 | SURROGATE, - CARRY | TOO_LARGE | TOO_LARGE_1000, - CARRY | TOO_LARGE | TOO_LARGE_1000 - ); - const simd8 byte_2_high = input.shr<4>().lookup_16( - // ________ 0_______ - TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT, - TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT, - - // ________ 1000____ - TOO_LONG | OVERLONG_2 | TWO_CONTS | OVERLONG_3 | TOO_LARGE_1000 | OVERLONG_4, - // ________ 1001____ - TOO_LONG | OVERLONG_2 | TWO_CONTS | OVERLONG_3 | TOO_LARGE, - // ________ 101_____ - TOO_LONG | OVERLONG_2 | TWO_CONTS | SURROGATE | TOO_LARGE, - TOO_LONG | OVERLONG_2 | TWO_CONTS | SURROGATE | TOO_LARGE, - - // ________ 11______ - TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT - ); - return (byte_1_high & byte_1_low & byte_2_high); - } - simdjson_inline simd8 check_multibyte_lengths(const simd8 input, - const simd8 prev_input, const simd8 sc) { - simd8 prev2 = input.prev<2>(prev_input); - simd8 prev3 = input.prev<3>(prev_input); - simd8 must23 = simd8(must_be_2_3_continuation(prev2, prev3)); - simd8 must23_80 = must23 & uint8_t(0x80); - return must23_80 ^ sc; - } - - // - // Return nonzero if there are incomplete multibyte characters at the end of the block: - // e.g. if there is a 4-byte character, but it's 3 bytes from the end. - // - simdjson_inline simd8 is_incomplete(const simd8 input) { - // If the previous input's last 3 bytes match this, they're too short (they ended at EOF): - // ... 1111____ 111_____ 11______ -#if SIMDJSON_IMPLEMENTATION_ICELAKE - static const uint8_t max_array[64] = { - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 0xf0u - 1, 0xe0u - 1, 0xc0u - 1 - }; -#else - static const uint8_t max_array[32] = { - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 0xf0u - 1, 0xe0u - 1, 0xc0u - 1 - }; -#endif - const simd8 max_value(&max_array[sizeof(max_array) - sizeof(simd8)]); - return input.gt_bits(max_value); - } - - struct utf8_checker { - // If this is nonzero, there has been a UTF-8 error. - simd8 error; - // The last input we received - simd8 prev_input_block; - // Whether the last input we received was incomplete (used for ASCII fast path) - simd8 prev_incomplete; - - // - // Check whether the current bytes are valid UTF-8. - // - simdjson_inline void check_utf8_bytes(const simd8 input, const simd8 prev_input) { - // Flip prev1...prev3 so we can easily determine if they are 2+, 3+ or 4+ lead bytes - // (2, 3, 4-byte leads become large positive numbers instead of small negative numbers) - simd8 prev1 = input.prev<1>(prev_input); - simd8 sc = check_special_cases(input, prev1); - this->error |= check_multibyte_lengths(input, prev_input, sc); - } - - // The only problem that can happen at EOF is that a multibyte character is too short - // or a byte value too large in the last bytes: check_special_cases only checks for bytes - // too large in the first of two bytes. - simdjson_inline void check_eof() { - // If the previous block had incomplete UTF-8 characters at the end, an ASCII block can't - // possibly finish them. - this->error |= this->prev_incomplete; - } - - simdjson_inline void check_next_input(const simd8x64& input) { - if (simdjson_likely(is_ascii(input))) { - this->error |= this->prev_incomplete; - } - else { - // you might think that a for-loop would work, but under Visual Studio, it is not good enough. - static_assert((simd8x64::NUM_CHUNKS == 1) - || (simd8x64::NUM_CHUNKS == 2) - || (simd8x64::NUM_CHUNKS == 4), - "We support one, two or four chunks per 64-byte block."); - SIMDJSON_IF_CONSTEXPR(simd8x64::NUM_CHUNKS == 1) { - this->check_utf8_bytes(input.chunks[0], this->prev_input_block); - } -else SIMDJSON_IF_CONSTEXPR(simd8x64::NUM_CHUNKS == 2) { - this->check_utf8_bytes(input.chunks[0], this->prev_input_block); - this->check_utf8_bytes(input.chunks[1], input.chunks[0]); - } - else SIMDJSON_IF_CONSTEXPR(simd8x64::NUM_CHUNKS == 4) { - this->check_utf8_bytes(input.chunks[0], this->prev_input_block); - this->check_utf8_bytes(input.chunks[1], input.chunks[0]); - this->check_utf8_bytes(input.chunks[2], input.chunks[1]); - this->check_utf8_bytes(input.chunks[3], input.chunks[2]); - } - this->prev_incomplete = is_incomplete(input.chunks[simd8x64::NUM_CHUNKS - 1]); - this->prev_input_block = input.chunks[simd8x64::NUM_CHUNKS - 1]; - } - } - // do not forget to call check_eof! - simdjson_inline error_code errors() { - return this->error.any_bits_set_anywhere() ? error_code::UTF8_ERROR : error_code::SUCCESS; - } - - }; // struct utf8_checker - } // namespace utf8_validation - - } // unnamed namespace - } // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_UTF8_LOOKUP4_ALGORITHM_H -/* end file generic/stage1/utf8_lookup4_algorithm.h for westmere */ -/* including generic/stage1/json_scanner.h for westmere: #include */ -/* begin file generic/stage1/json_scanner.h for westmere */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_JSON_SCANNER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_JSON_SCANNER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - namespace westmere { - namespace { - namespace stage1 { - - /** - * A block of scanned json, with information on operators and scalars. - * - * We seek to identify pseudo-structural characters. Anything that is inside - * a string must be omitted (hence & ~_string.string_tail()). - * Otherwise, pseudo-structural characters come in two forms. - * 1. We have the structural characters ([,],{,},:, comma). The - * term 'structural character' is from the JSON RFC. - * 2. We have the 'scalar pseudo-structural characters'. - * Scalars are quotes, and any character except structural characters and white space. - * - * To identify the scalar pseudo-structural characters, we must look at what comes - * before them: it must be a space, a quote or a structural characters. - * Starting with simdjson v0.3, we identify them by - * negation: we identify everything that is followed by a non-quote scalar, - * and we negate that. Whatever remains must be a 'scalar pseudo-structural character'. - */ - struct json_block { - public: - // We spell out the constructors in the hope of resolving inlining issues with Visual Studio 2017 - simdjson_inline json_block(json_string_block&& string, json_character_block characters, uint64_t follows_potential_nonquote_scalar) : - _string(std::move(string)), _characters(characters), _follows_potential_nonquote_scalar(follows_potential_nonquote_scalar) {} - simdjson_inline json_block(json_string_block string, json_character_block characters, uint64_t follows_potential_nonquote_scalar) : - _string(string), _characters(characters), _follows_potential_nonquote_scalar(follows_potential_nonquote_scalar) {} - - /** - * The start of structurals. - * In simdjson prior to v0.3, these were called the pseudo-structural characters. - **/ - simdjson_inline uint64_t structural_start() const noexcept { return potential_structural_start() & ~_string.string_tail(); } - /** All JSON whitespace (i.e. not in a string) */ - simdjson_inline uint64_t whitespace() const noexcept { return non_quote_outside_string(_characters.whitespace()); } - - // Helpers - - /** Whether the given characters are inside a string (only works on non-quotes) */ - simdjson_inline uint64_t non_quote_inside_string(uint64_t mask) const noexcept { return _string.non_quote_inside_string(mask); } - /** Whether the given characters are outside a string (only works on non-quotes) */ - simdjson_inline uint64_t non_quote_outside_string(uint64_t mask) const noexcept { return _string.non_quote_outside_string(mask); } - - // string and escape characters - json_string_block _string; - // whitespace, structural characters ('operators'), scalars - json_character_block _characters; - // whether the previous character was a scalar - uint64_t _follows_potential_nonquote_scalar; - private: - // Potential structurals (i.e. disregarding strings) - - /** - * structural elements ([,],{,},:, comma) plus scalar starts like 123, true and "abc". - * They may reside inside a string. - **/ - simdjson_inline uint64_t potential_structural_start() const noexcept { return _characters.op() | potential_scalar_start(); } - /** - * The start of non-operator runs, like 123, true and "abc". - * It main reside inside a string. - **/ - simdjson_inline uint64_t potential_scalar_start() const noexcept { - // The term "scalar" refers to anything except structural characters and white space - // (so letters, numbers, quotes). - // Whenever it is preceded by something that is not a structural element ({,},[,],:, ") nor a white-space - // then we know that it is irrelevant structurally. - return _characters.scalar() & ~follows_potential_scalar(); - } - /** - * Whether the given character is immediately after a non-operator like 123, true. - * The characters following a quote are not included. - */ - simdjson_inline uint64_t follows_potential_scalar() const noexcept { - // _follows_potential_nonquote_scalar: is defined as marking any character that follows a character - // that is not a structural element ({,},[,],:, comma) nor a quote (") and that is not a - // white space. - // It is understood that within quoted region, anything at all could be marked (irrelevant). - return _follows_potential_nonquote_scalar; - } - }; - - /** - * Scans JSON for important bits: structural characters or 'operators', strings, and scalars. - * - * The scanner starts by calculating two distinct things: - * - string characters (taking \" into account) - * - structural characters or 'operators' ([]{},:, comma) - * and scalars (runs of non-operators like 123, true and "abc") - * - * To minimize data dependency (a key component of the scanner's speed), it finds these in parallel: - * in particular, the operator/scalar bit will find plenty of things that are actually part of - * strings. When we're done, json_block will fuse the two together by masking out tokens that are - * part of a string. - */ - class json_scanner { - public: - json_scanner() = default; - simdjson_inline json_block next(const simd::simd8x64& in); - // Returns either UNCLOSED_STRING or SUCCESS - simdjson_inline error_code finish(); - - private: - // Whether the last character of the previous iteration is part of a scalar token - // (anything except whitespace or a structural character/'operator'). - uint64_t prev_scalar = 0ULL; - json_string_scanner string_scanner{}; - }; - - - // - // Check if the current character immediately follows a matching character. - // - // For example, this checks for quotes with backslashes in front of them: - // - // const uint64_t backslashed_quote = in.eq('"') & immediately_follows(in.eq('\'), prev_backslash); - // - simdjson_inline uint64_t follows(const uint64_t match, uint64_t& overflow) { - const uint64_t result = match << 1 | overflow; - overflow = match >> 63; - return result; - } - - simdjson_inline json_block json_scanner::next(const simd::simd8x64& in) { - json_string_block strings = string_scanner.next(in); - // identifies the white-space and the structural characters - json_character_block characters = json_character_block::classify(in); - // The term "scalar" refers to anything except structural characters and white space - // (so letters, numbers, quotes). - // We want follows_scalar to mark anything that follows a non-quote scalar (so letters and numbers). - // - // A terminal quote should either be followed by a structural character (comma, brace, bracket, colon) - // or nothing. However, we still want ' "a string"true ' to mark the 't' of 'true' as a potential - // pseudo-structural character just like we would if we had ' "a string" true '; otherwise we - // may need to add an extra check when parsing strings. - // - // Performance: there are many ways to skin this cat. - const uint64_t nonquote_scalar = characters.scalar() & ~strings.quote(); - uint64_t follows_nonquote_scalar = follows(nonquote_scalar, prev_scalar); - // We are returning a function-local object so either we get a move constructor - // or we get copy elision. - return json_block( - strings,// strings is a function-local object so either it moves or the copy is elided. - characters, - follows_nonquote_scalar - ); - } - - simdjson_inline error_code json_scanner::finish() { - return string_scanner.finish(); - } - - } // namespace stage1 - } // unnamed namespace - } // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_JSON_SCANNER_H -/* end file generic/stage1/json_scanner.h for westmere */ - -// All other declarations -/* including generic/stage1/find_next_document_index.h for westmere: #include */ -/* begin file generic/stage1/find_next_document_index.h for westmere */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_FIND_NEXT_DOCUMENT_INDEX_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_FIND_NEXT_DOCUMENT_INDEX_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - namespace westmere { - namespace { - namespace stage1 { - - /** - * This algorithm is used to quickly identify the last structural position that - * makes up a complete document. - * - * It does this by going backwards and finding the last *document boundary* (a - * place where one value follows another without a comma between them). If the - * last document (the characters after the boundary) has an equal number of - * start and end brackets, it is considered complete. - * - * Simply put, we iterate over the structural characters, starting from - * the end. We consider that we found the end of a JSON document when the - * first element of the pair is NOT one of these characters: '{' '[' ':' ',' - * and when the second element is NOT one of these characters: '}' ']' ':' ','. - * - * This simple comparison works most of the time, but it does not cover cases - * where the batch's structural indexes contain a perfect amount of documents. - * In such a case, we do not have access to the structural index which follows - * the last document, therefore, we do not have access to the second element in - * the pair, and that means we cannot identify the last document. To fix this - * issue, we keep a count of the open and closed curly/square braces we found - * while searching for the pair. When we find a pair AND the count of open and - * closed curly/square braces is the same, we know that we just passed a - * complete document, therefore the last json buffer location is the end of the - * batch. - */ - simdjson_inline uint32_t find_next_document_index(dom_parser_implementation& parser) { - // Variant: do not count separately, just figure out depth - if (parser.n_structural_indexes == 0) { return 0; } - auto arr_cnt = 0; - auto obj_cnt = 0; - for (auto i = parser.n_structural_indexes - 1; i > 0; i--) { - auto idxb = parser.structural_indexes[i]; - switch (parser.buf[idxb]) { - case ':': - case ',': - continue; - case '}': - obj_cnt--; - continue; - case ']': - arr_cnt--; - continue; - case '{': - obj_cnt++; - break; - case '[': - arr_cnt++; - break; - } - auto idxa = parser.structural_indexes[i - 1]; - switch (parser.buf[idxa]) { - case '{': - case '[': - case ':': - case ',': - continue; - } - // Last document is complete, so the next document will appear after! - if (!arr_cnt && !obj_cnt) { - return parser.n_structural_indexes; - } - // Last document is incomplete; mark the document at i + 1 as the next one - return i; - } - // If we made it to the end, we want to finish counting to see if we have a full document. - switch (parser.buf[parser.structural_indexes[0]]) { - case '}': - obj_cnt--; - break; - case ']': - arr_cnt--; - break; - case '{': - obj_cnt++; - break; - case '[': - arr_cnt++; - break; - } - if (!arr_cnt && !obj_cnt) { - // We have a complete document. - return parser.n_structural_indexes; - } - return 0; - } - - } // namespace stage1 - } // unnamed namespace - } // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_FIND_NEXT_DOCUMENT_INDEX_H -/* end file generic/stage1/find_next_document_index.h for westmere */ -/* including generic/stage1/json_minifier.h for westmere: #include */ -/* begin file generic/stage1/json_minifier.h for westmere */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_JSON_MINIFIER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_JSON_MINIFIER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -// This file contains the common code every implementation uses in stage1 -// It is intended to be included multiple times and compiled multiple times -// We assume the file in which it is included already includes -// "simdjson/stage1.h" (this simplifies amalgation) - -namespace simdjson { - namespace westmere { - namespace { - namespace stage1 { - - class json_minifier { - public: - template - static error_code minify(const uint8_t* buf, size_t len, uint8_t* dst, size_t& dst_len) noexcept; - - private: - simdjson_inline json_minifier(uint8_t* _dst) - : dst{ _dst } - {} - template - simdjson_inline void step(const uint8_t* block_buf, buf_block_reader& reader) noexcept; - simdjson_inline void next(const simd::simd8x64& in, const json_block& block); - simdjson_inline error_code finish(uint8_t* dst_start, size_t& dst_len); - json_scanner scanner{}; - uint8_t* dst; - }; - - simdjson_inline void json_minifier::next(const simd::simd8x64& in, const json_block& block) { - uint64_t mask = block.whitespace(); - dst += in.compress(mask, dst); - } - - simdjson_inline error_code json_minifier::finish(uint8_t* dst_start, size_t& dst_len) { - error_code error = scanner.finish(); - if (error) { dst_len = 0; return error; } - dst_len = dst - dst_start; - return SUCCESS; - } - - template<> - simdjson_inline void json_minifier::step<128>(const uint8_t* block_buf, buf_block_reader<128>& reader) noexcept { - simd::simd8x64 in_1(block_buf); - simd::simd8x64 in_2(block_buf + 64); - json_block block_1 = scanner.next(in_1); - json_block block_2 = scanner.next(in_2); - this->next(in_1, block_1); - this->next(in_2, block_2); - reader.advance(); - } - - template<> - simdjson_inline void json_minifier::step<64>(const uint8_t* block_buf, buf_block_reader<64>& reader) noexcept { - simd::simd8x64 in_1(block_buf); - json_block block_1 = scanner.next(in_1); - this->next(block_buf, block_1); - reader.advance(); - } - - template - error_code json_minifier::minify(const uint8_t* buf, size_t len, uint8_t* dst, size_t& dst_len) noexcept { - buf_block_reader reader(buf, len); - json_minifier minifier(dst); - - // Index the first n-1 blocks - while (reader.has_full_block()) { - minifier.step(reader.full_block(), reader); - } - - // Index the last (remainder) block, padded with spaces - uint8_t block[STEP_SIZE]; - size_t remaining_bytes = reader.get_remainder(block); - if (remaining_bytes > 0) { - // We do not want to write directly to the output stream. Rather, we write - // to a local buffer (for safety). - uint8_t out_block[STEP_SIZE]; - uint8_t* const guarded_dst{ minifier.dst }; - minifier.dst = out_block; - minifier.step(block, reader); - size_t to_write = minifier.dst - out_block; - // In some cases, we could be enticed to consider the padded spaces - // as part of the string. This is fine as long as we do not write more - // than we consumed. - if (to_write > remaining_bytes) { to_write = remaining_bytes; } - memcpy(guarded_dst, out_block, to_write); - minifier.dst = guarded_dst + to_write; - } - return minifier.finish(dst, dst_len); - } - - } // namespace stage1 - } // unnamed namespace - } // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_JSON_MINIFIER_H -/* end file generic/stage1/json_minifier.h for westmere */ -/* including generic/stage1/json_structural_indexer.h for westmere: #include */ -/* begin file generic/stage1/json_structural_indexer.h for westmere */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRUCTURAL_INDEXER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRUCTURAL_INDEXER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -// This file contains the common code every implementation uses in stage1 -// It is intended to be included multiple times and compiled multiple times -// We assume the file in which it is included already includes -// "simdjson/stage1.h" (this simplifies amalgation) - -namespace simdjson { - namespace westmere { - namespace { - namespace stage1 { - - class bit_indexer { - public: - uint32_t* tail; - - simdjson_inline bit_indexer(uint32_t* index_buf) : tail(index_buf) {} - -#if SIMDJSON_PREFER_REVERSE_BITS - /** - * ARM lacks a fast trailing zero instruction, but it has a fast - * bit reversal instruction and a fast leading zero instruction. - * Thus it may be profitable to reverse the bits (once) and then - * to rely on a sequence of instructions that call the leading - * zero instruction. - * - * Performance notes: - * The chosen routine is not optimal in terms of data dependency - * since zero_leading_bit might require two instructions. However, - * it tends to minimize the total number of instructions which is - * beneficial. - */ - simdjson_inline void write_index(uint32_t idx, uint64_t& rev_bits, int i) { - int lz = leading_zeroes(rev_bits); - this->tail[i] = static_cast(idx) + lz; - rev_bits = zero_leading_bit(rev_bits, lz); - } -#else - /** - * Under recent x64 systems, we often have both a fast trailing zero - * instruction and a fast 'clear-lower-bit' instruction so the following - * algorithm can be competitive. - */ - - simdjson_inline void write_index(uint32_t idx, uint64_t& bits, int i) { - this->tail[i] = idx + trailing_zeroes(bits); - bits = clear_lowest_bit(bits); - } -#endif // SIMDJSON_PREFER_REVERSE_BITS - - template - simdjson_inline int write_indexes(uint32_t idx, uint64_t& bits) { - write_index(idx, bits, START); - SIMDJSON_IF_CONSTEXPR(N > 1) { - write_indexes<(N - 1 > 0 ? START + 1 : START), (N - 1 >= 0 ? N - 1 : 1)>(idx, bits); - } - return START + N; - } - - template - simdjson_inline int write_indexes_stepped(uint32_t idx, uint64_t& bits, int cnt) { - write_indexes(idx, bits); - SIMDJSON_IF_CONSTEXPR((START + STEP) < END) { - if (simdjson_unlikely((START + STEP) < cnt)) { - write_indexes_stepped<(START + STEP < END ? START + STEP : END), END, STEP>(idx, bits, cnt); - } - } - return ((END - START) % STEP) == 0 ? END : (END - START) - ((END - START) % STEP) + STEP; - } - - // flatten out values in 'bits' assuming that they are are to have values of idx - // plus their position in the bitvector, and store these indexes at - // base_ptr[base] incrementing base as we go - // will potentially store extra values beyond end of valid bits, so base_ptr - // needs to be large enough to handle this - // - // If the kernel sets SIMDJSON_GENERIC_JSON_STRUCTURAL_INDEXER_CUSTOM_BIT_INDEXER, then it - // will provide its own version of the code. -#ifdef SIMDJSON_GENERIC_JSON_STRUCTURAL_INDEXER_CUSTOM_BIT_INDEXER - simdjson_inline void write(uint32_t idx, uint64_t bits); -#else - simdjson_inline void write(uint32_t idx, uint64_t bits) { - // In some instances, the next branch is expensive because it is mispredicted. - // Unfortunately, in other cases, - // it helps tremendously. - if (bits == 0) - return; - - int cnt = static_cast(count_ones(bits)); - -#if SIMDJSON_PREFER_REVERSE_BITS - bits = reverse_bits(bits); -#endif -#ifdef SIMDJSON_STRUCTURAL_INDEXER_STEP - static constexpr const int STEP = SIMDJSON_STRUCTURAL_INDEXER_STEP; -#else - static constexpr const int STEP = 4; -#endif - static constexpr const int STEP_UNTIL = 24; - - write_indexes_stepped<0, STEP_UNTIL, STEP>(idx, bits, cnt); - SIMDJSON_IF_CONSTEXPR(STEP_UNTIL < 64) { - if (simdjson_unlikely(STEP_UNTIL < cnt)) { - for (int i = STEP_UNTIL; i < cnt; i++) { - write_index(idx, bits, i); - } - } - } - - this->tail += cnt; - } -#endif // SIMDJSON_GENERIC_JSON_STRUCTURAL_INDEXER_CUSTOM_BIT_INDEXER - - }; - - class json_structural_indexer { - public: - /** - * Find the important bits of JSON in a 128-byte chunk, and add them to structural_indexes. - * - * @param partial Setting the partial parameter to true allows the find_structural_bits to - * tolerate unclosed strings. The caller should still ensure that the input is valid UTF-8. If - * you are processing substrings, you may want to call on a function like trimmed_length_safe_utf8. - */ - template - static error_code index(const uint8_t* buf, size_t len, dom_parser_implementation& parser, stage1_mode partial) noexcept; - - private: - simdjson_inline json_structural_indexer(uint32_t* structural_indexes); - template - simdjson_inline void step(const uint8_t* block, buf_block_reader& reader) noexcept; - simdjson_inline void next(const simd::simd8x64& in, const json_block& block, size_t idx); - simdjson_inline error_code finish(dom_parser_implementation& parser, size_t idx, size_t len, stage1_mode partial); - - json_scanner scanner{}; - utf8_checker checker{}; - bit_indexer indexer; - uint64_t prev_structurals = 0; - uint64_t unescaped_chars_error = 0; - }; - - simdjson_inline json_structural_indexer::json_structural_indexer(uint32_t* structural_indexes) : indexer{ structural_indexes } {} - - // Skip the last character if it is partial - simdjson_inline size_t trim_partial_utf8(const uint8_t* buf, size_t len) { - if (simdjson_unlikely(len < 3)) { - switch (len) { - case 2: - if (buf[len - 1] >= 0xc0) { return len - 1; } // 2-, 3- and 4-byte characters with only 1 byte left - if (buf[len - 2] >= 0xe0) { return len - 2; } // 3- and 4-byte characters with only 2 bytes left - return len; - case 1: - if (buf[len - 1] >= 0xc0) { return len - 1; } // 2-, 3- and 4-byte characters with only 1 byte left - return len; - case 0: - return len; - } - } - if (buf[len - 1] >= 0xc0) { return len - 1; } // 2-, 3- and 4-byte characters with only 1 byte left - if (buf[len - 2] >= 0xe0) { return len - 2; } // 3- and 4-byte characters with only 1 byte left - if (buf[len - 3] >= 0xf0) { return len - 3; } // 4-byte characters with only 3 bytes left - return len; - } - - // - // PERF NOTES: - // We pipe 2 inputs through these stages: - // 1. Load JSON into registers. This takes a long time and is highly parallelizable, so we load - // 2 inputs' worth at once so that by the time step 2 is looking for them input, it's available. - // 2. Scan the JSON for critical data: strings, scalars and operators. This is the critical path. - // The output of step 1 depends entirely on this information. These functions don't quite use - // up enough CPU: the second half of the functions is highly serial, only using 1 execution core - // at a time. The second input's scans has some dependency on the first ones finishing it, but - // they can make a lot of progress before they need that information. - // 3. Step 1 doesn't use enough capacity, so we run some extra stuff while we're waiting for that - // to finish: utf-8 checks and generating the output from the last iteration. - // - // The reason we run 2 inputs at a time, is steps 2 and 3 are *still* not enough to soak up all - // available capacity with just one input. Running 2 at a time seems to give the CPU a good enough - // workout. - // - template - error_code json_structural_indexer::index(const uint8_t* buf, size_t len, dom_parser_implementation& parser, stage1_mode partial) noexcept { - if (simdjson_unlikely(len > parser.capacity())) { return CAPACITY; } - // We guard the rest of the code so that we can assume that len > 0 throughout. - if (len == 0) { return EMPTY; } - if (is_streaming(partial)) { - len = trim_partial_utf8(buf, len); - // If you end up with an empty window after trimming - // the partial UTF-8 bytes, then chances are good that you - // have an UTF-8 formatting error. - if (len == 0) { return UTF8_ERROR; } - } - buf_block_reader reader(buf, len); - json_structural_indexer indexer(parser.structural_indexes.get()); - - // Read all but the last block - while (reader.has_full_block()) { - indexer.step(reader.full_block(), reader); - } - // Take care of the last block (will always be there unless file is empty which is - // not supposed to happen.) - uint8_t block[STEP_SIZE]; - if (simdjson_unlikely(reader.get_remainder(block) == 0)) { return UNEXPECTED_ERROR; } - indexer.step(block, reader); - return indexer.finish(parser, reader.block_index(), len, partial); - } - - template<> - simdjson_inline void json_structural_indexer::step<128>(const uint8_t* block, buf_block_reader<128>& reader) noexcept { - simd::simd8x64 in_1(block); - simd::simd8x64 in_2(block + 64); - json_block block_1 = scanner.next(in_1); - json_block block_2 = scanner.next(in_2); - this->next(in_1, block_1, reader.block_index()); - this->next(in_2, block_2, reader.block_index() + 64); - reader.advance(); - } - - template<> - simdjson_inline void json_structural_indexer::step<64>(const uint8_t* block, buf_block_reader<64>& reader) noexcept { - simd::simd8x64 in_1(block); - json_block block_1 = scanner.next(in_1); - this->next(in_1, block_1, reader.block_index()); - reader.advance(); - } - - simdjson_inline void json_structural_indexer::next(const simd::simd8x64& in, const json_block& block, size_t idx) { - uint64_t unescaped = in.lteq(0x1F); -#if SIMDJSON_UTF8VALIDATION - checker.check_next_input(in); -#endif - indexer.write(uint32_t(idx - 64), prev_structurals); // Output *last* iteration's structurals to the parser - prev_structurals = block.structural_start(); - unescaped_chars_error |= block.non_quote_inside_string(unescaped); - } - - simdjson_inline error_code json_structural_indexer::finish(dom_parser_implementation& parser, size_t idx, size_t len, stage1_mode partial) { - // Write out the final iteration's structurals - indexer.write(uint32_t(idx - 64), prev_structurals); - error_code error = scanner.finish(); - // We deliberately break down the next expression so that it is - // human readable. - const bool should_we_exit = is_streaming(partial) ? - ((error != SUCCESS) && (error != UNCLOSED_STRING)) // when partial we tolerate UNCLOSED_STRING - : (error != SUCCESS); // if partial is false, we must have SUCCESS - const bool have_unclosed_string = (error == UNCLOSED_STRING); - if (simdjson_unlikely(should_we_exit)) { return error; } - - if (unescaped_chars_error) { - return UNESCAPED_CHARS; - } - parser.n_structural_indexes = uint32_t(indexer.tail - parser.structural_indexes.get()); - /*** - * The On Demand API requires special padding. - * - * This is related to https://github.com/simdjson/simdjson/issues/906 - * Basically, we want to make sure that if the parsing continues beyond the last (valid) - * structural character, it quickly stops. - * Only three structural characters can be repeated without triggering an error in JSON: [,] and }. - * We repeat the padding character (at 'len'). We don't know what it is, but if the parsing - * continues, then it must be [,] or }. - * Suppose it is ] or }. We backtrack to the first character, what could it be that would - * not trigger an error? It could be ] or } but no, because you can't start a document that way. - * It can't be a comma, a colon or any simple value. So the only way we could continue is - * if the repeated character is [. But if so, the document must start with [. But if the document - * starts with [, it should end with ]. If we enforce that rule, then we would get - * ][[ which is invalid. - * - * This is illustrated with the test array_iterate_unclosed_error() on the following input: - * R"({ "a": [,,)" - **/ - parser.structural_indexes[parser.n_structural_indexes] = uint32_t(len); // used later in partial == stage1_mode::streaming_final - parser.structural_indexes[parser.n_structural_indexes + 1] = uint32_t(len); - parser.structural_indexes[parser.n_structural_indexes + 2] = 0; - parser.next_structural_index = 0; - // a valid JSON file cannot have zero structural indexes - we should have found something - if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { - return EMPTY; - } - if (simdjson_unlikely(parser.structural_indexes[parser.n_structural_indexes - 1] > len)) { - return UNEXPECTED_ERROR; - } - if (partial == stage1_mode::streaming_partial) { - // If we have an unclosed string, then the last structural - // will be the quote and we want to make sure to omit it. - if (have_unclosed_string) { - parser.n_structural_indexes--; - // a valid JSON file cannot have zero structural indexes - we should have found something - if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { return CAPACITY; } - } - // We truncate the input to the end of the last complete document (or zero). - auto new_structural_indexes = find_next_document_index(parser); - if (new_structural_indexes == 0 && parser.n_structural_indexes > 0) { - if (parser.structural_indexes[0] == 0) { - // If the buffer is partial and we started at index 0 but the document is - // incomplete, it's too big to parse. - return CAPACITY; - } - else { - // It is possible that the document could be parsed, we just had a lot - // of white space. - parser.n_structural_indexes = 0; - return EMPTY; - } - } - - parser.n_structural_indexes = new_structural_indexes; - } - else if (partial == stage1_mode::streaming_final) { - if (have_unclosed_string) { parser.n_structural_indexes--; } - // We truncate the input to the end of the last complete document (or zero). - // Because partial == stage1_mode::streaming_final, it means that we may - // silently ignore trailing garbage. Though it sounds bad, we do it - // deliberately because many people who have streams of JSON documents - // will truncate them for processing. E.g., imagine that you are uncompressing - // the data from a size file or receiving it in chunks from the network. You - // may not know where exactly the last document will be. Meanwhile the - // document_stream instances allow people to know the JSON documents they are - // parsing (see the iterator.source() method). - parser.n_structural_indexes = find_next_document_index(parser); - // We store the initial n_structural_indexes so that the client can see - // whether we used truncation. If initial_n_structural_indexes == parser.n_structural_indexes, - // then this will query parser.structural_indexes[parser.n_structural_indexes] which is len, - // otherwise, it will copy some prior index. - parser.structural_indexes[parser.n_structural_indexes + 1] = parser.structural_indexes[parser.n_structural_indexes]; - // This next line is critical, do not change it unless you understand what you are - // doing. - parser.structural_indexes[parser.n_structural_indexes] = uint32_t(len); - if (simdjson_unlikely(parser.n_structural_indexes == 0u)) { - // We tolerate an unclosed string at the very end of the stream. Indeed, users - // often load their data in bulk without being careful and they want us to ignore - // the trailing garbage. - return EMPTY; - } - } - checker.check_eof(); - return checker.errors(); - } - - } // namespace stage1 - } // unnamed namespace - } // namespace westmere -} // namespace simdjson - -// Clear CUSTOM_BIT_INDEXER so other implementations can set it if they need to. -#undef SIMDJSON_GENERIC_JSON_STRUCTURAL_INDEXER_CUSTOM_BIT_INDEXER - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_JSON_STRUCTURAL_INDEXER_H -/* end file generic/stage1/json_structural_indexer.h for westmere */ -/* including generic/stage1/utf8_validator.h for westmere: #include */ -/* begin file generic/stage1/utf8_validator.h for westmere */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE1_UTF8_VALIDATOR_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE1_UTF8_VALIDATOR_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - namespace westmere { - namespace { - namespace stage1 { - - /** - * Validates that the string is actual UTF-8. - */ - template - bool generic_validate_utf8(const uint8_t* input, size_t length) { - checker c{}; - buf_block_reader<64> reader(input, length); - while (reader.has_full_block()) { - simd::simd8x64 in(reader.full_block()); - c.check_next_input(in); - reader.advance(); - } - uint8_t block[64]{}; - reader.get_remainder(block); - simd::simd8x64 in(block); - c.check_next_input(in); - reader.advance(); - c.check_eof(); - return c.errors() == error_code::SUCCESS; - } - - bool generic_validate_utf8(const char* input, size_t length) { - return generic_validate_utf8(reinterpret_cast(input), length); - } - - } // namespace stage1 - } // unnamed namespace - } // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE1_UTF8_VALIDATOR_H -/* end file generic/stage1/utf8_validator.h for westmere */ -/* end file generic/stage1/amalgamated.h for westmere */ -/* including generic/stage2/amalgamated.h for westmere: #include */ -/* begin file generic/stage2/amalgamated.h for westmere */ -// Stuff other things depend on -/* including generic/stage2/base.h for westmere: #include */ -/* begin file generic/stage2/base.h for westmere */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_BASE_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_BASE_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - namespace westmere { - namespace { - namespace stage2 { - - class json_iterator; - class structural_iterator; - struct tape_builder; - struct tape_writer; - - } // namespace stage2 - } // unnamed namespace - } // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE2_BASE_H -/* end file generic/stage2/base.h for westmere */ -/* including generic/stage2/tape_writer.h for westmere: #include */ -/* begin file generic/stage2/tape_writer.h for westmere */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_TAPE_WRITER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_TAPE_WRITER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include - -namespace simdjson { - namespace westmere { - namespace { - namespace stage2 { - - struct tape_writer { - /** The next place to write to tape */ - uint64_t* next_tape_loc; - - /** Write a signed 64-bit value to tape. */ - simdjson_inline void append_s64(int64_t value) noexcept; - - /** Write an unsigned 64-bit value to tape. */ - simdjson_inline void append_u64(uint64_t value) noexcept; - - /** Write a double value to tape. */ - simdjson_inline void append_double(double value) noexcept; - - /** - * Append a tape entry (an 8-bit type,and 56 bits worth of value). - */ - simdjson_inline void append(uint64_t val, internal::tape_type t) noexcept; - - /** - * Skip the current tape entry without writing. - * - * Used to skip the start of the container, since we'll come back later to fill it in when the - * container ends. - */ - simdjson_inline void skip() noexcept; - - /** - * Skip the number of tape entries necessary to write a large u64 or i64. - */ - simdjson_inline void skip_large_integer() noexcept; - - /** - * Skip the number of tape entries necessary to write a double. - */ - simdjson_inline void skip_double() noexcept; - - /** - * Write a value to a known location on tape. - * - * Used to go back and write out the start of a container after the container ends. - */ - simdjson_inline static void write(uint64_t& tape_loc, uint64_t val, internal::tape_type t) noexcept; - - private: - /** - * Append both the tape entry, and a supplementary value following it. Used for types that need - * all 64 bits, such as double and uint64_t. - */ - template - simdjson_inline void append2(uint64_t val, T val2, internal::tape_type t) noexcept; - }; // struct tape_writer - - simdjson_inline void tape_writer::append_s64(int64_t value) noexcept { - append2(0, value, internal::tape_type::INT64); - } - - simdjson_inline void tape_writer::append_u64(uint64_t value) noexcept { - append(0, internal::tape_type::UINT64); - *next_tape_loc = value; - next_tape_loc++; - } - - /** Write a double value to tape. */ - simdjson_inline void tape_writer::append_double(double value) noexcept { - append2(0, value, internal::tape_type::DOUBLE); - } - - simdjson_inline void tape_writer::skip() noexcept { - next_tape_loc++; - } - - simdjson_inline void tape_writer::skip_large_integer() noexcept { - next_tape_loc += 2; - } - - simdjson_inline void tape_writer::skip_double() noexcept { - next_tape_loc += 2; - } - - simdjson_inline void tape_writer::append(uint64_t val, internal::tape_type t) noexcept { - *next_tape_loc = val | ((uint64_t(char(t))) << 56); - next_tape_loc++; - } - - template - simdjson_inline void tape_writer::append2(uint64_t val, T val2, internal::tape_type t) noexcept { - append(val, t); - static_assert(sizeof(val2) == sizeof(*next_tape_loc), "Type is not 64 bits!"); - memcpy(next_tape_loc, &val2, sizeof(val2)); - next_tape_loc++; - } - - simdjson_inline void tape_writer::write(uint64_t& tape_loc, uint64_t val, internal::tape_type t) noexcept { - tape_loc = val | ((uint64_t(char(t))) << 56); - } - - } // namespace stage2 - } // unnamed namespace - } // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE2_TAPE_WRITER_H -/* end file generic/stage2/tape_writer.h for westmere */ -/* including generic/stage2/logger.h for westmere: #include */ -/* begin file generic/stage2/logger.h for westmere */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_LOGGER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_LOGGER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#include - - -// This is for an internal-only stage 2 specific logger. -// Set LOG_ENABLED = true to log what stage 2 is doing! -namespace simdjson { - namespace westmere { - namespace { - namespace logger { - - static constexpr const char* DASHES = "----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------"; - -#if SIMDJSON_VERBOSE_LOGGING - static constexpr const bool LOG_ENABLED = true; -#else - static constexpr const bool LOG_ENABLED = false; -#endif - static constexpr const int LOG_EVENT_LEN = 20; - static constexpr const int LOG_BUFFER_LEN = 30; - static constexpr const int LOG_SMALL_BUFFER_LEN = 10; - static constexpr const int LOG_INDEX_LEN = 5; - - static int log_depth; // Not threadsafe. Log only. - - // Helper to turn unprintable or newline characters into spaces - static simdjson_inline char printable_char(char c) { - if (c >= 0x20) { - return c; - } - else { - return ' '; - } - } - - // Print the header and set up log_start - static simdjson_inline void log_start() { - if (LOG_ENABLED) { - log_depth = 0; - printf("\n"); - printf("| %-*s | %-*s | %-*s | %-*s | Detail |\n", LOG_EVENT_LEN, "Event", LOG_BUFFER_LEN, "Buffer", LOG_SMALL_BUFFER_LEN, "Next", 5, "Next#"); - printf("|%.*s|%.*s|%.*s|%.*s|--------|\n", LOG_EVENT_LEN + 2, DASHES, LOG_BUFFER_LEN + 2, DASHES, LOG_SMALL_BUFFER_LEN + 2, DASHES, 5 + 2, DASHES); - } - } - - simdjson_unused static simdjson_inline void log_string(const char* message) { - if (LOG_ENABLED) { - printf("%s\n", message); - } - } - - // Logs a single line from the stage 2 DOM parser - template - static simdjson_inline void log_line(S& structurals, const char* title_prefix, const char* title, const char* detail) { - if (LOG_ENABLED) { - printf("| %*s%s%-*s ", log_depth * 2, "", title_prefix, LOG_EVENT_LEN - log_depth * 2 - int(strlen(title_prefix)), title); - auto current_index = structurals.at_beginning() ? nullptr : structurals.next_structural - 1; - auto next_index = structurals.next_structural; - auto current = current_index ? &structurals.buf[*current_index] : reinterpret_cast(" "); - auto next = &structurals.buf[*next_index]; - { - // Print the next N characters in the buffer. - printf("| "); - // Otherwise, print the characters starting from the buffer position. - // Print spaces for unprintable or newline characters. - for (int i = 0; i < LOG_BUFFER_LEN; i++) { - printf("%c", printable_char(current[i])); - } - printf(" "); - // Print the next N characters in the buffer. - printf("| "); - // Otherwise, print the characters starting from the buffer position. - // Print spaces for unprintable or newline characters. - for (int i = 0; i < LOG_SMALL_BUFFER_LEN; i++) { - printf("%c", printable_char(next[i])); - } - printf(" "); - } - if (current_index) { - printf("| %*u ", LOG_INDEX_LEN, *current_index); - } - else { - printf("| %-*s ", LOG_INDEX_LEN, ""); - } - // printf("| %*u ", LOG_INDEX_LEN, structurals.next_tape_index()); - printf("| %-s ", detail); - printf("|\n"); - } - } - - } // namespace logger - } // unnamed namespace - } // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE2_LOGGER_H -/* end file generic/stage2/logger.h for westmere */ - -// All other declarations -/* including generic/stage2/json_iterator.h for westmere: #include */ -/* begin file generic/stage2/json_iterator.h for westmere */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_JSON_ITERATOR_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_JSON_ITERATOR_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - namespace westmere { - namespace { - namespace stage2 { - - class json_iterator { - public: - const uint8_t* const buf; - uint32_t* next_structural; - dom_parser_implementation& dom_parser; - uint32_t depth{ 0 }; - - /** - * Walk the JSON document. - * - * The visitor receives callbacks when values are encountered. All callbacks pass the iterator as - * the first parameter; some callbacks have other parameters as well: - * - * - visit_document_start() - at the beginning. - * - visit_document_end() - at the end (if things were successful). - * - * - visit_array_start() - at the start `[` of a non-empty array. - * - visit_array_end() - at the end `]` of a non-empty array. - * - visit_empty_array() - when an empty array is encountered. - * - * - visit_object_end() - at the start `]` of a non-empty object. - * - visit_object_start() - at the end `]` of a non-empty object. - * - visit_empty_object() - when an empty object is encountered. - * - visit_key(const uint8_t *key) - when a key in an object field is encountered. key is - * guaranteed to point at the first quote of the string (`"key"`). - * - visit_primitive(const uint8_t *value) - when a value is a string, number, boolean or null. - * - visit_root_primitive(iter, uint8_t *value) - when the top-level value is a string, number, boolean or null. - * - * - increment_count(iter) - each time a value is found in an array or object. - */ - template - simdjson_warn_unused simdjson_inline error_code walk_document(V& visitor) noexcept; - - /** - * Create an iterator capable of walking a JSON document. - * - * The document must have already passed through stage 1. - */ - simdjson_inline json_iterator(dom_parser_implementation& _dom_parser, size_t start_structural_index); - - /** - * Look at the next token. - * - * Tokens can be strings, numbers, booleans, null, or operators (`[{]},:`)). - * - * They may include invalid JSON as well (such as `1.2.3` or `ture`). - */ - simdjson_inline const uint8_t* peek() const noexcept; - /** - * Advance to the next token. - * - * Tokens can be strings, numbers, booleans, null, or operators (`[{]},:`)). - * - * They may include invalid JSON as well (such as `1.2.3` or `ture`). - */ - simdjson_inline const uint8_t* advance() noexcept; - /** - * Get the remaining length of the document, from the start of the current token. - */ - simdjson_inline size_t remaining_len() const noexcept; - /** - * Check if we are at the end of the document. - * - * If this is true, there are no more tokens. - */ - simdjson_inline bool at_eof() const noexcept; - /** - * Check if we are at the beginning of the document. - */ - simdjson_inline bool at_beginning() const noexcept; - simdjson_inline uint8_t last_structural() const noexcept; - - /** - * Log that a value has been found. - * - * Set LOG_ENABLED=true in logger.h to see logging. - */ - simdjson_inline void log_value(const char* type) const noexcept; - /** - * Log the start of a multipart value. - * - * Set LOG_ENABLED=true in logger.h to see logging. - */ - simdjson_inline void log_start_value(const char* type) const noexcept; - /** - * Log the end of a multipart value. - * - * Set LOG_ENABLED=true in logger.h to see logging. - */ - simdjson_inline void log_end_value(const char* type) const noexcept; - /** - * Log an error. - * - * Set LOG_ENABLED=true in logger.h to see logging. - */ - simdjson_inline void log_error(const char* error) const noexcept; - - template - simdjson_warn_unused simdjson_inline error_code visit_root_primitive(V& visitor, const uint8_t* value) noexcept; - template - simdjson_warn_unused simdjson_inline error_code visit_primitive(V& visitor, const uint8_t* value) noexcept; - }; - - template - simdjson_warn_unused simdjson_inline error_code json_iterator::walk_document(V& visitor) noexcept { - logger::log_start(); - - // - // Start the document - // - if (at_eof()) { return EMPTY; } - log_start_value("document"); - SIMDJSON_TRY(visitor.visit_document_start(*this)); - - // - // Read first value - // - { - auto value = advance(); - - // Make sure the outer object or array is closed before continuing; otherwise, there are ways we - // could get into memory corruption. See https://github.com/simdjson/simdjson/issues/906 - if (!STREAMING) { - switch (*value) { - case '{': if (last_structural() != '}') { log_value("starting brace unmatched"); return TAPE_ERROR; }; break; - case '[': if (last_structural() != ']') { log_value("starting bracket unmatched"); return TAPE_ERROR; }; break; - } - } - - switch (*value) { - case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY(visitor.visit_empty_object(*this)); break; } goto object_begin; - case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY(visitor.visit_empty_array(*this)); break; } goto array_begin; - default: SIMDJSON_TRY(visitor.visit_root_primitive(*this, value)); break; - } - } - goto document_end; - - // - // Object parser states - // - object_begin: - log_start_value("object"); - depth++; - if (depth >= dom_parser.max_depth()) { log_error("Exceeded max depth!"); return DEPTH_ERROR; } - dom_parser.is_array[depth] = false; - SIMDJSON_TRY(visitor.visit_object_start(*this)); - - { - auto key = advance(); - if (*key != '"') { log_error("Object does not start with a key"); return TAPE_ERROR; } - SIMDJSON_TRY(visitor.increment_count(*this)); - SIMDJSON_TRY(visitor.visit_key(*this, key)); - } - - object_field: - if (simdjson_unlikely(*advance() != ':')) { log_error("Missing colon after key in object"); return TAPE_ERROR; } - { - auto value = advance(); - switch (*value) { - case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY(visitor.visit_empty_object(*this)); break; } goto object_begin; - case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY(visitor.visit_empty_array(*this)); break; } goto array_begin; - default: SIMDJSON_TRY(visitor.visit_primitive(*this, value)); break; - } - } - - object_continue: - switch (*advance()) { - case ',': - SIMDJSON_TRY(visitor.increment_count(*this)); - { - auto key = advance(); - if (simdjson_unlikely(*key != '"')) { log_error("Key string missing at beginning of field in object"); return TAPE_ERROR; } - SIMDJSON_TRY(visitor.visit_key(*this, key)); - } - goto object_field; - case '}': log_end_value("object"); SIMDJSON_TRY(visitor.visit_object_end(*this)); goto scope_end; - default: log_error("No comma between object fields"); return TAPE_ERROR; - } - - scope_end: - depth--; - if (depth == 0) { goto document_end; } - if (dom_parser.is_array[depth]) { goto array_continue; } - goto object_continue; - - // - // Array parser states - // - array_begin: - log_start_value("array"); - depth++; - if (depth >= dom_parser.max_depth()) { log_error("Exceeded max depth!"); return DEPTH_ERROR; } - dom_parser.is_array[depth] = true; - SIMDJSON_TRY(visitor.visit_array_start(*this)); - SIMDJSON_TRY(visitor.increment_count(*this)); - - array_value: - { - auto value = advance(); - switch (*value) { - case '{': if (*peek() == '}') { advance(); log_value("empty object"); SIMDJSON_TRY(visitor.visit_empty_object(*this)); break; } goto object_begin; - case '[': if (*peek() == ']') { advance(); log_value("empty array"); SIMDJSON_TRY(visitor.visit_empty_array(*this)); break; } goto array_begin; - default: SIMDJSON_TRY(visitor.visit_primitive(*this, value)); break; - } - } - - array_continue: - switch (*advance()) { - case ',': SIMDJSON_TRY(visitor.increment_count(*this)); goto array_value; - case ']': log_end_value("array"); SIMDJSON_TRY(visitor.visit_array_end(*this)); goto scope_end; - default: log_error("Missing comma between array values"); return TAPE_ERROR; - } - - document_end: - log_end_value("document"); - SIMDJSON_TRY(visitor.visit_document_end(*this)); - - dom_parser.next_structural_index = uint32_t(next_structural - &dom_parser.structural_indexes[0]); - - // If we didn't make it to the end, it's an error - if (!STREAMING && dom_parser.next_structural_index != dom_parser.n_structural_indexes) { - log_error("More than one JSON value at the root of the document, or extra characters at the end of the JSON!"); - return TAPE_ERROR; - } - - return SUCCESS; - - } // walk_document() - - simdjson_inline json_iterator::json_iterator(dom_parser_implementation& _dom_parser, size_t start_structural_index) - : buf{ _dom_parser.buf }, - next_structural{ &_dom_parser.structural_indexes[start_structural_index] }, - dom_parser{ _dom_parser } { - } - - simdjson_inline const uint8_t* json_iterator::peek() const noexcept { - return &buf[*(next_structural)]; - } - simdjson_inline const uint8_t* json_iterator::advance() noexcept { - return &buf[*(next_structural++)]; - } - simdjson_inline size_t json_iterator::remaining_len() const noexcept { - return dom_parser.len - *(next_structural - 1); - } - - simdjson_inline bool json_iterator::at_eof() const noexcept { - return next_structural == &dom_parser.structural_indexes[dom_parser.n_structural_indexes]; - } - simdjson_inline bool json_iterator::at_beginning() const noexcept { - return next_structural == dom_parser.structural_indexes.get(); - } - simdjson_inline uint8_t json_iterator::last_structural() const noexcept { - return buf[dom_parser.structural_indexes[dom_parser.n_structural_indexes - 1]]; - } - - simdjson_inline void json_iterator::log_value(const char* type) const noexcept { - logger::log_line(*this, "", type, ""); - } - - simdjson_inline void json_iterator::log_start_value(const char* type) const noexcept { - logger::log_line(*this, "+", type, ""); - if (logger::LOG_ENABLED) { logger::log_depth++; } - } - - simdjson_inline void json_iterator::log_end_value(const char* type) const noexcept { - if (logger::LOG_ENABLED) { logger::log_depth--; } - logger::log_line(*this, "-", type, ""); - } - - simdjson_inline void json_iterator::log_error(const char* error) const noexcept { - logger::log_line(*this, "", "ERROR", error); - } - - template - simdjson_warn_unused simdjson_inline error_code json_iterator::visit_root_primitive(V& visitor, const uint8_t* value) noexcept { - switch (*value) { - case '"': return visitor.visit_root_string(*this, value); - case 't': return visitor.visit_root_true_atom(*this, value); - case 'f': return visitor.visit_root_false_atom(*this, value); - case 'n': return visitor.visit_root_null_atom(*this, value); - case '-': - case '0': case '1': case '2': case '3': case '4': - case '5': case '6': case '7': case '8': case '9': - return visitor.visit_root_number(*this, value); - default: - log_error("Document starts with a non-value character"); - return TAPE_ERROR; - } - } - template - simdjson_warn_unused simdjson_inline error_code json_iterator::visit_primitive(V& visitor, const uint8_t* value) noexcept { - // Use the fact that most scalars are going to be either strings or numbers. - if (*value == '"') { - return visitor.visit_string(*this, value); - } - else if (((*value - '0') < 10) || (*value == '-')) { - return visitor.visit_number(*this, value); - } - // true, false, null are uncommon. - switch (*value) { - case 't': return visitor.visit_true_atom(*this, value); - case 'f': return visitor.visit_false_atom(*this, value); - case 'n': return visitor.visit_null_atom(*this, value); - default: - log_error("Non-value found when value was expected!"); - return TAPE_ERROR; - } - } - - } // namespace stage2 - } // unnamed namespace - } // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE2_JSON_ITERATOR_H -/* end file generic/stage2/json_iterator.h for westmere */ -/* including generic/stage2/stringparsing.h for westmere: #include */ -/* begin file generic/stage2/stringparsing.h for westmere */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -// This file contains the common code every implementation uses -// It is intended to be included multiple times and compiled multiple times - -namespace simdjson { - namespace westmere { - namespace { - /// @private - namespace stringparsing { - - // begin copypasta - // These chars yield themselves: " \ / - // b -> backspace, f -> formfeed, n -> newline, r -> cr, t -> horizontal tab - // u not handled in this table as it's complex - static const uint8_t escape_map[256] = { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x0. - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0x22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x2f, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x4. - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x5c, 0, 0, 0, // 0x5. - 0, 0, 0x08, 0, 0, 0, 0x0c, 0, 0, 0, 0, 0, 0, 0, 0x0a, 0, // 0x6. - 0, 0, 0x0d, 0, 0x09, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 0x7. - - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - }; - - // handle a unicode codepoint - // write appropriate values into dest - // src will advance 6 bytes or 12 bytes - // dest will advance a variable amount (return via pointer) - // return true if the unicode codepoint was valid - // We work in little-endian then swap at write time - simdjson_warn_unused - simdjson_inline bool handle_unicode_codepoint(const uint8_t** src_ptr, - uint8_t** dst_ptr, bool allow_replacement) { - // Use the default Unicode Character 'REPLACEMENT CHARACTER' (U+FFFD) - constexpr uint32_t substitution_code_point = 0xfffd; - // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the - // conversion isn't valid; we defer the check for this to inside the - // multilingual plane check - uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2); - *src_ptr += 6; - - // If we found a high surrogate, we must - // check for low surrogate for characters - // outside the Basic - // Multilingual Plane. - if (code_point >= 0xd800 && code_point < 0xdc00) { - const uint8_t* src_data = *src_ptr; - /* Compiler optimizations convert this to a single 16-bit load and compare on most platforms */ - if (((src_data[0] << 8) | src_data[1]) != ((static_cast ('\\') << 8) | static_cast ('u'))) { - if (!allow_replacement) { return false; } - code_point = substitution_code_point; - } - else { - uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(src_data + 2); - - // We have already checked that the high surrogate is valid and - // (code_point - 0xd800) < 1024. - // - // Check that code_point_2 is in the range 0xdc00..0xdfff - // and that code_point_2 was parsed from valid hex. - uint32_t low_bit = code_point_2 - 0xdc00; - if (low_bit >> 10) { - if (!allow_replacement) { return false; } - code_point = substitution_code_point; - } - else { - code_point = (((code_point - 0xd800) << 10) | low_bit) + 0x10000; - *src_ptr += 6; - } - - } - } - else if (code_point >= 0xdc00 && code_point <= 0xdfff) { - // If we encounter a low surrogate (not preceded by a high surrogate) - // then we have an error. - if (!allow_replacement) { return false; } - code_point = substitution_code_point; - } - size_t offset = jsoncharutils::codepoint_to_utf8(code_point, *dst_ptr); - *dst_ptr += offset; - return offset > 0; - } - - - // handle a unicode codepoint using the wobbly convention - // https://simonsapin.github.io/wtf-8/ - // write appropriate values into dest - // src will advance 6 bytes or 12 bytes - // dest will advance a variable amount (return via pointer) - // return true if the unicode codepoint was valid - // We work in little-endian then swap at write time - simdjson_warn_unused - simdjson_inline bool handle_unicode_codepoint_wobbly(const uint8_t** src_ptr, - uint8_t** dst_ptr) { - // It is not ideal that this function is nearly identical to handle_unicode_codepoint. - // - // jsoncharutils::hex_to_u32_nocheck fills high 16 bits of the return value with 1s if the - // conversion isn't valid; we defer the check for this to inside the - // multilingual plane check - uint32_t code_point = jsoncharutils::hex_to_u32_nocheck(*src_ptr + 2); - *src_ptr += 6; - // If we found a high surrogate, we must - // check for low surrogate for characters - // outside the Basic - // Multilingual Plane. - if (code_point >= 0xd800 && code_point < 0xdc00) { - const uint8_t* src_data = *src_ptr; - /* Compiler optimizations convert this to a single 16-bit load and compare on most platforms */ - if (((src_data[0] << 8) | src_data[1]) == ((static_cast ('\\') << 8) | static_cast ('u'))) { - uint32_t code_point_2 = jsoncharutils::hex_to_u32_nocheck(src_data + 2); - uint32_t low_bit = code_point_2 - 0xdc00; - if ((low_bit >> 10) == 0) { - code_point = - (((code_point - 0xd800) << 10) | low_bit) + 0x10000; - *src_ptr += 6; - } - } - } - - size_t offset = jsoncharutils::codepoint_to_utf8(code_point, *dst_ptr); - *dst_ptr += offset; - return offset > 0; - } - - - /** - * Unescape a valid UTF-8 string from src to dst, stopping at a final unescaped quote. There - * must be an unescaped quote terminating the string. It returns the final output - * position as pointer. In case of error (e.g., the string has bad escaped codes), - * then null_nullptrptr is returned. It is assumed that the output buffer is large - * enough. E.g., if src points at 'joe"', then dst needs to have four free bytes + - * SIMDJSON_PADDING bytes. - */ - simdjson_warn_unused simdjson_inline uint8_t* parse_string(const uint8_t* src, uint8_t* dst, bool allow_replacement) { - while (1) { - // Copy the next n bytes, and find the backslash and quote in them. - auto bs_quote = backslash_and_quote::copy_and_find(src, dst); - // If the next thing is the end quote, copy and return - if (bs_quote.has_quote_first()) { - // we encountered quotes first. Move dst to point to quotes and exit - return dst + bs_quote.quote_index(); - } - if (bs_quote.has_backslash()) { - /* find out where the backspace is */ - auto bs_dist = bs_quote.backslash_index(); - uint8_t escape_char = src[bs_dist + 1]; - /* we encountered backslash first. Handle backslash */ - if (escape_char == 'u') { - /* move src/dst up to the start; they will be further adjusted - within the unicode codepoint handling code. */ - src += bs_dist; - dst += bs_dist; - if (!handle_unicode_codepoint(&src, &dst, allow_replacement)) { - return nullptr; - } - } - else { - /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and - * write bs_dist+1 characters to output - * note this may reach beyond the part of the buffer we've actually - * seen. I think this is ok */ - uint8_t escape_result = escape_map[escape_char]; - if (escape_result == 0u) { - return nullptr; /* bogus escape value is an error */ - } - dst[bs_dist] = escape_result; - src += bs_dist + 2; - dst += bs_dist + 1; - } - } - else { - /* they are the same. Since they can't co-occur, it means we - * encountered neither. */ - src += backslash_and_quote::BYTES_PROCESSED; - dst += backslash_and_quote::BYTES_PROCESSED; - } - } - } - - simdjson_warn_unused simdjson_inline uint8_t* parse_wobbly_string(const uint8_t* src, uint8_t* dst) { - // It is not ideal that this function is nearly identical to parse_string. - while (1) { - // Copy the next n bytes, and find the backslash and quote in them. - auto bs_quote = backslash_and_quote::copy_and_find(src, dst); - // If the next thing is the end quote, copy and return - if (bs_quote.has_quote_first()) { - // we encountered quotes first. Move dst to point to quotes and exit - return dst + bs_quote.quote_index(); - } - if (bs_quote.has_backslash()) { - /* find out where the backspace is */ - auto bs_dist = bs_quote.backslash_index(); - uint8_t escape_char = src[bs_dist + 1]; - /* we encountered backslash first. Handle backslash */ - if (escape_char == 'u') { - /* move src/dst up to the start; they will be further adjusted - within the unicode codepoint handling code. */ - src += bs_dist; - dst += bs_dist; - if (!handle_unicode_codepoint_wobbly(&src, &dst)) { - return nullptr; - } - } - else { - /* simple 1:1 conversion. Will eat bs_dist+2 characters in input and - * write bs_dist+1 characters to output - * note this may reach beyond the part of the buffer we've actually - * seen. I think this is ok */ - uint8_t escape_result = escape_map[escape_char]; - if (escape_result == 0u) { - return nullptr; /* bogus escape value is an error */ - } - dst[bs_dist] = escape_result; - src += bs_dist + 2; - dst += bs_dist + 1; - } - } - else { - /* they are the same. Since they can't co-occur, it means we - * encountered neither. */ - src += backslash_and_quote::BYTES_PROCESSED; - dst += backslash_and_quote::BYTES_PROCESSED; - } - } - } - - } // namespace stringparsing - } // unnamed namespace - } // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE2_STRINGPARSING_H -/* end file generic/stage2/stringparsing.h for westmere */ -/* including generic/stage2/structural_iterator.h for westmere: #include */ -/* begin file generic/stage2/structural_iterator.h for westmere */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_STRUCTURAL_ITERATOR_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_STRUCTURAL_ITERATOR_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -namespace simdjson { - namespace westmere { - namespace { - namespace stage2 { - - class structural_iterator { - public: - const uint8_t* const buf; - uint32_t* next_structural; - dom_parser_implementation& dom_parser; - - // Start a structural - simdjson_inline structural_iterator(dom_parser_implementation& _dom_parser, size_t start_structural_index) - : buf{ _dom_parser.buf }, - next_structural{ &_dom_parser.structural_indexes[start_structural_index] }, - dom_parser{ _dom_parser } { - } - // Get the buffer position of the current structural character - simdjson_inline const uint8_t* current() { - return &buf[*(next_structural - 1)]; - } - // Get the current structural character - simdjson_inline char current_char() { - return buf[*(next_structural - 1)]; - } - // Get the next structural character without advancing - simdjson_inline char peek_next_char() { - return buf[*next_structural]; - } - simdjson_inline const uint8_t* peek() { - return &buf[*next_structural]; - } - simdjson_inline const uint8_t* advance() { - return &buf[*(next_structural++)]; - } - simdjson_inline char advance_char() { - return buf[*(next_structural++)]; - } - simdjson_inline size_t remaining_len() { - return dom_parser.len - *(next_structural - 1); - } - - simdjson_inline bool at_end() { - return next_structural == &dom_parser.structural_indexes[dom_parser.n_structural_indexes]; - } - simdjson_inline bool at_beginning() { - return next_structural == dom_parser.structural_indexes.get(); - } - }; - - } // namespace stage2 - } // unnamed namespace - } // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE2_STRUCTURAL_ITERATOR_H -/* end file generic/stage2/structural_iterator.h for westmere */ -/* including generic/stage2/tape_builder.h for westmere: #include */ -/* begin file generic/stage2/tape_builder.h for westmere */ -#ifndef SIMDJSON_SRC_GENERIC_STAGE2_TAPE_BUILDER_H - -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #define SIMDJSON_SRC_GENERIC_STAGE2_TAPE_BUILDER_H */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #include */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - - -namespace simdjson { - namespace westmere { - namespace { - namespace stage2 { - - struct tape_builder { - template - simdjson_warn_unused static simdjson_inline error_code parse_document( - dom_parser_implementation& dom_parser, - dom::document& doc) noexcept; - - /** Called when a non-empty document starts. */ - simdjson_warn_unused simdjson_inline error_code visit_document_start(json_iterator& iter) noexcept; - /** Called when a non-empty document ends without error. */ - simdjson_warn_unused simdjson_inline error_code visit_document_end(json_iterator& iter) noexcept; - - /** Called when a non-empty array starts. */ - simdjson_warn_unused simdjson_inline error_code visit_array_start(json_iterator& iter) noexcept; - /** Called when a non-empty array ends. */ - simdjson_warn_unused simdjson_inline error_code visit_array_end(json_iterator& iter) noexcept; - /** Called when an empty array is found. */ - simdjson_warn_unused simdjson_inline error_code visit_empty_array(json_iterator& iter) noexcept; - - /** Called when a non-empty object starts. */ - simdjson_warn_unused simdjson_inline error_code visit_object_start(json_iterator& iter) noexcept; - /** - * Called when a key in a field is encountered. - * - * primitive, visit_object_start, visit_empty_object, visit_array_start, or visit_empty_array - * will be called after this with the field value. - */ - simdjson_warn_unused simdjson_inline error_code visit_key(json_iterator& iter, const uint8_t* key) noexcept; - /** Called when a non-empty object ends. */ - simdjson_warn_unused simdjson_inline error_code visit_object_end(json_iterator& iter) noexcept; - /** Called when an empty object is found. */ - simdjson_warn_unused simdjson_inline error_code visit_empty_object(json_iterator& iter) noexcept; - - /** - * Called when a string, number, boolean or null is found. - */ - simdjson_warn_unused simdjson_inline error_code visit_primitive(json_iterator& iter, const uint8_t* value) noexcept; - /** - * Called when a string, number, boolean or null is found at the top level of a document (i.e. - * when there is no array or object and the entire document is a single string, number, boolean or - * null. - * - * This is separate from primitive() because simdjson's normal primitive parsing routines assume - * there is at least one more token after the value, which is only true in an array or object. - */ - simdjson_warn_unused simdjson_inline error_code visit_root_primitive(json_iterator& iter, const uint8_t* value) noexcept; - - simdjson_warn_unused simdjson_inline error_code visit_string(json_iterator& iter, const uint8_t* value, bool key = false) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_number(json_iterator& iter, const uint8_t* value) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_true_atom(json_iterator& iter, const uint8_t* value) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_false_atom(json_iterator& iter, const uint8_t* value) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_null_atom(json_iterator& iter, const uint8_t* value) noexcept; - - simdjson_warn_unused simdjson_inline error_code visit_root_string(json_iterator& iter, const uint8_t* value) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_root_number(json_iterator& iter, const uint8_t* value) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_root_true_atom(json_iterator& iter, const uint8_t* value) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_root_false_atom(json_iterator& iter, const uint8_t* value) noexcept; - simdjson_warn_unused simdjson_inline error_code visit_root_null_atom(json_iterator& iter, const uint8_t* value) noexcept; - - /** Called each time a new field or element in an array or object is found. */ - simdjson_warn_unused simdjson_inline error_code increment_count(json_iterator& iter) noexcept; - - /** Next location to write to tape */ - tape_writer tape; - private: - /** Next write location in the string buf for stage 2 parsing */ - uint8_t* current_string_buf_loc; - - simdjson_inline tape_builder(dom::document& doc) noexcept; - - simdjson_inline uint32_t next_tape_index(json_iterator& iter) const noexcept; - simdjson_inline void start_container(json_iterator& iter) noexcept; - simdjson_warn_unused simdjson_inline error_code end_container(json_iterator& iter, internal::tape_type start, internal::tape_type end) noexcept; - simdjson_warn_unused simdjson_inline error_code empty_container(json_iterator& iter, internal::tape_type start, internal::tape_type end) noexcept; - simdjson_inline uint8_t* on_start_string(json_iterator& iter) noexcept; - simdjson_inline void on_end_string(uint8_t* dst) noexcept; - }; // struct tape_builder - - template - simdjson_warn_unused simdjson_inline error_code tape_builder::parse_document( - dom_parser_implementation& dom_parser, - dom::document& doc) noexcept { - dom_parser.doc = &doc; - json_iterator iter(dom_parser, STREAMING ? dom_parser.next_structural_index : 0); - tape_builder builder(doc); - return iter.walk_document(builder); - } - - simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_primitive(json_iterator& iter, const uint8_t* value) noexcept { - return iter.visit_root_primitive(*this, value); - } - simdjson_warn_unused simdjson_inline error_code tape_builder::visit_primitive(json_iterator& iter, const uint8_t* value) noexcept { - return iter.visit_primitive(*this, value); - } - simdjson_warn_unused simdjson_inline error_code tape_builder::visit_empty_object(json_iterator& iter) noexcept { - return empty_container(iter, internal::tape_type::START_OBJECT, internal::tape_type::END_OBJECT); - } - simdjson_warn_unused simdjson_inline error_code tape_builder::visit_empty_array(json_iterator& iter) noexcept { - return empty_container(iter, internal::tape_type::START_ARRAY, internal::tape_type::END_ARRAY); - } - - simdjson_warn_unused simdjson_inline error_code tape_builder::visit_document_start(json_iterator& iter) noexcept { - start_container(iter); - return SUCCESS; - } - simdjson_warn_unused simdjson_inline error_code tape_builder::visit_object_start(json_iterator& iter) noexcept { - start_container(iter); - return SUCCESS; - } - simdjson_warn_unused simdjson_inline error_code tape_builder::visit_array_start(json_iterator& iter) noexcept { - start_container(iter); - return SUCCESS; - } - - simdjson_warn_unused simdjson_inline error_code tape_builder::visit_object_end(json_iterator& iter) noexcept { - return end_container(iter, internal::tape_type::START_OBJECT, internal::tape_type::END_OBJECT); - } - simdjson_warn_unused simdjson_inline error_code tape_builder::visit_array_end(json_iterator& iter) noexcept { - return end_container(iter, internal::tape_type::START_ARRAY, internal::tape_type::END_ARRAY); - } - simdjson_warn_unused simdjson_inline error_code tape_builder::visit_document_end(json_iterator& iter) noexcept { - constexpr uint32_t start_tape_index = 0; - tape.append(start_tape_index, internal::tape_type::ROOT); - tape_writer::write(iter.dom_parser.doc->tape[start_tape_index], next_tape_index(iter), internal::tape_type::ROOT); - return SUCCESS; - } - simdjson_warn_unused simdjson_inline error_code tape_builder::visit_key(json_iterator& iter, const uint8_t* key) noexcept { - return visit_string(iter, key, true); - } - - simdjson_warn_unused simdjson_inline error_code tape_builder::increment_count(json_iterator& iter) noexcept { - iter.dom_parser.open_containers[iter.depth].count++; // we have a key value pair in the object at parser.dom_parser.depth - 1 - return SUCCESS; - } - - simdjson_inline tape_builder::tape_builder(dom::document& doc) noexcept : tape{ doc.tape.get() }, current_string_buf_loc{ doc.string_buf.get() } {} - - simdjson_warn_unused simdjson_inline error_code tape_builder::visit_string(json_iterator& iter, const uint8_t* value, bool key) noexcept { - iter.log_value(key ? "key" : "string"); - uint8_t* dst = on_start_string(iter); - dst = stringparsing::parse_string(value + 1, dst, false); // We do not allow replacement when the escape characters are invalid. - if (dst == nullptr) { - iter.log_error("Invalid escape in string"); - return STRING_ERROR; - } - on_end_string(dst); - return SUCCESS; - } - - simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_string(json_iterator& iter, const uint8_t* value) noexcept { - return visit_string(iter, value); - } - - simdjson_warn_unused simdjson_inline error_code tape_builder::visit_number(json_iterator& iter, const uint8_t* value) noexcept { - iter.log_value("number"); - return numberparsing::parse_number(value, tape); - } - - simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_number(json_iterator& iter, const uint8_t* value) noexcept { - // - // We need to make a copy to make sure that the string is space terminated. - // This is not about padding the input, which should already padded up - // to len + SIMDJSON_PADDING. However, we have no control at this stage - // on how the padding was done. What if the input string was padded with nulls? - // It is quite common for an input string to have an extra null character (C string). - // We do not want to allow 9\0 (where \0 is the null character) inside a JSON - // document, but the string "9\0" by itself is fine. So we make a copy and - // pad the input with spaces when we know that there is just one input element. - // This copy is relatively expensive, but it will almost never be called in - // practice unless you are in the strange scenario where you have many JSON - // documents made of single atoms. - // - std::unique_ptrcopy(new (std::nothrow) uint8_t[iter.remaining_len() + SIMDJSON_PADDING]); - if (copy.get() == nullptr) { return MEMALLOC; } - std::memcpy(copy.get(), value, iter.remaining_len()); - std::memset(copy.get() + iter.remaining_len(), ' ', SIMDJSON_PADDING); - error_code error = visit_number(iter, copy.get()); - return error; - } - - simdjson_warn_unused simdjson_inline error_code tape_builder::visit_true_atom(json_iterator& iter, const uint8_t* value) noexcept { - iter.log_value("true"); - if (!atomparsing::is_valid_true_atom(value)) { return T_ATOM_ERROR; } - tape.append(0, internal::tape_type::TRUE_VALUE); - return SUCCESS; - } - - simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_true_atom(json_iterator& iter, const uint8_t* value) noexcept { - iter.log_value("true"); - if (!atomparsing::is_valid_true_atom(value, iter.remaining_len())) { return T_ATOM_ERROR; } - tape.append(0, internal::tape_type::TRUE_VALUE); - return SUCCESS; - } - - simdjson_warn_unused simdjson_inline error_code tape_builder::visit_false_atom(json_iterator& iter, const uint8_t* value) noexcept { - iter.log_value("false"); - if (!atomparsing::is_valid_false_atom(value)) { return F_ATOM_ERROR; } - tape.append(0, internal::tape_type::FALSE_VALUE); - return SUCCESS; - } - - simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_false_atom(json_iterator& iter, const uint8_t* value) noexcept { - iter.log_value("false"); - if (!atomparsing::is_valid_false_atom(value, iter.remaining_len())) { return F_ATOM_ERROR; } - tape.append(0, internal::tape_type::FALSE_VALUE); - return SUCCESS; - } - - simdjson_warn_unused simdjson_inline error_code tape_builder::visit_null_atom(json_iterator& iter, const uint8_t* value) noexcept { - iter.log_value("null"); - if (!atomparsing::is_valid_null_atom(value)) { return N_ATOM_ERROR; } - tape.append(0, internal::tape_type::NULL_VALUE); - return SUCCESS; - } - - simdjson_warn_unused simdjson_inline error_code tape_builder::visit_root_null_atom(json_iterator& iter, const uint8_t* value) noexcept { - iter.log_value("null"); - if (!atomparsing::is_valid_null_atom(value, iter.remaining_len())) { return N_ATOM_ERROR; } - tape.append(0, internal::tape_type::NULL_VALUE); - return SUCCESS; - } - - // private: - - simdjson_inline uint32_t tape_builder::next_tape_index(json_iterator& iter) const noexcept { - return uint32_t(tape.next_tape_loc - iter.dom_parser.doc->tape.get()); - } - - simdjson_warn_unused simdjson_inline error_code tape_builder::empty_container(json_iterator& iter, internal::tape_type start, internal::tape_type end) noexcept { - auto start_index = next_tape_index(iter); - tape.append(start_index + 2, start); - tape.append(start_index, end); - return SUCCESS; - } - - simdjson_inline void tape_builder::start_container(json_iterator& iter) noexcept { - iter.dom_parser.open_containers[iter.depth].tape_index = next_tape_index(iter); - iter.dom_parser.open_containers[iter.depth].count = 0; - tape.skip(); // We don't actually *write* the start element until the end. - } - - simdjson_warn_unused simdjson_inline error_code tape_builder::end_container(json_iterator& iter, internal::tape_type start, internal::tape_type end) noexcept { - // Write the ending tape element, pointing at the start location - const uint32_t start_tape_index = iter.dom_parser.open_containers[iter.depth].tape_index; - tape.append(start_tape_index, end); - // Write the start tape element, pointing at the end location (and including count) - // count can overflow if it exceeds 24 bits... so we saturate - // the convention being that a cnt of 0xffffff or more is undetermined in value (>= 0xffffff). - const uint32_t count = iter.dom_parser.open_containers[iter.depth].count; - const uint32_t cntsat = count > 0xFFFFFF ? 0xFFFFFF : count; - tape_writer::write(iter.dom_parser.doc->tape[start_tape_index], next_tape_index(iter) | (uint64_t(cntsat) << 32), start); - return SUCCESS; - } - - simdjson_inline uint8_t* tape_builder::on_start_string(json_iterator& iter) noexcept { - // we advance the point, accounting for the fact that we have a NULL termination - tape.append(current_string_buf_loc - iter.dom_parser.doc->string_buf.get(), internal::tape_type::STRING); - return current_string_buf_loc + sizeof(uint32_t); - } - - simdjson_inline void tape_builder::on_end_string(uint8_t* dst) noexcept { - uint32_t str_length = uint32_t(dst - (current_string_buf_loc + sizeof(uint32_t))); - // TODO check for overflow in case someone has a crazy string (>=4GB?) - // But only add the overflow check when the document itself exceeds 4GB - // Currently unneeded because we refuse to parse docs larger or equal to 4GB. - memcpy(current_string_buf_loc, &str_length, sizeof(uint32_t)); - // NULL termination is still handy if you expect all your strings to - // be NULL terminated? It comes at a small cost - *dst = 0; - current_string_buf_loc = dst + 1; - } - - } // namespace stage2 - } // unnamed namespace - } // namespace westmere -} // namespace simdjson - -#endif // SIMDJSON_SRC_GENERIC_STAGE2_TAPE_BUILDER_H -/* end file generic/stage2/tape_builder.h for westmere */ -/* end file generic/stage2/amalgamated.h for westmere */ - -// -// Stage 1 -// - -namespace simdjson { - namespace westmere { - - simdjson_warn_unused error_code implementation::create_dom_parser_implementation( - size_t capacity, - size_t max_depth, - std::unique_ptr& dst - ) const noexcept { - dst.reset(new (std::nothrow) dom_parser_implementation()); - if (!dst) { return MEMALLOC; } - if (auto err = dst->set_capacity(capacity)) - return err; - if (auto err = dst->set_max_depth(max_depth)) - return err; - return SUCCESS; - } - - namespace { - - using namespace simd; - - simdjson_inline json_character_block json_character_block::classify(const simd::simd8x64& in) { - // These lookups rely on the fact that anything < 127 will match the lower 4 bits, which is why - // we can't use the generic lookup_16. - auto whitespace_table = simd8::repeat_16(' ', 100, 100, 100, 17, 100, 113, 2, 100, '\t', '\n', 112, 100, '\r', 100, 100); - - // The 6 operators (:,[]{}) have these values: - // - // , 2C - // : 3A - // [ 5B - // { 7B - // ] 5D - // } 7D - // - // If you use | 0x20 to turn [ and ] into { and }, the lower 4 bits of each character is unique. - // We exploit this, using a simd 4-bit lookup to tell us which character match against, and then - // match it (against | 0x20). - // - // To prevent recognizing other characters, everything else gets compared with 0, which cannot - // match due to the | 0x20. - // - // NOTE: Due to the | 0x20, this ALSO treats and (control characters 0C and 1A) like , - // and :. This gets caught in stage 2, which checks the actual character to ensure the right - // operators are in the right places. - const auto op_table = simd8::repeat_16( - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 0, ':', '{', // : = 3A, [ = 5B, { = 7B - ',', '}', 0, 0 // , = 2C, ] = 5D, } = 7D - ); - - // We compute whitespace and op separately. If the code later only use one or the - // other, given the fact that all functions are aggressively inlined, we can - // hope that useless computations will be omitted. This is namely case when - // minifying (we only need whitespace). - - - const uint64_t whitespace = in.eq({ - _mm_shuffle_epi8(whitespace_table, in.chunks[0]), - _mm_shuffle_epi8(whitespace_table, in.chunks[1]), - _mm_shuffle_epi8(whitespace_table, in.chunks[2]), - _mm_shuffle_epi8(whitespace_table, in.chunks[3]) - }); - // Turn [ and ] into { and } - const simd8x64 curlified{ - in.chunks[0] | 0x20, - in.chunks[1] | 0x20, - in.chunks[2] | 0x20, - in.chunks[3] | 0x20 - }; - const uint64_t op = curlified.eq({ - _mm_shuffle_epi8(op_table, in.chunks[0]), - _mm_shuffle_epi8(op_table, in.chunks[1]), - _mm_shuffle_epi8(op_table, in.chunks[2]), - _mm_shuffle_epi8(op_table, in.chunks[3]) - }); - return { whitespace, op }; - } - - simdjson_inline bool is_ascii(const simd8x64& input) { - return input.reduce_or().is_ascii(); - } - - simdjson_unused simdjson_inline simd8 must_be_continuation(const simd8 prev1, const simd8 prev2, const simd8 prev3) { - simd8 is_second_byte = prev1.saturating_sub(0xc0u - 1); // Only 11______ will be > 0 - simd8 is_third_byte = prev2.saturating_sub(0xe0u - 1); // Only 111_____ will be > 0 - simd8 is_fourth_byte = prev3.saturating_sub(0xf0u - 1); // Only 1111____ will be > 0 - // Caller requires a bool (all 1's). All values resulting from the subtraction will be <= 64, so signed comparison is fine. - return simd8(is_second_byte | is_third_byte | is_fourth_byte) > int8_t(0); - } - - simdjson_inline simd8 must_be_2_3_continuation(const simd8 prev2, const simd8 prev3) { - simd8 is_third_byte = prev2.saturating_sub(0xe0u - 1); // Only 111_____ will be > 0 - simd8 is_fourth_byte = prev3.saturating_sub(0xf0u - 1); // Only 1111____ will be > 0 - // Caller requires a bool (all 1's). All values resulting from the subtraction will be <= 64, so signed comparison is fine. - return simd8(is_third_byte | is_fourth_byte) > int8_t(0); - } - - } // unnamed namespace - } // namespace westmere -} // namespace simdjson - -// -// Stage 2 -// - -// -// Implementation-specific overrides -// - -namespace simdjson { - namespace westmere { - - simdjson_warn_unused error_code implementation::minify(const uint8_t* buf, size_t len, uint8_t* dst, size_t& dst_len) const noexcept { - return westmere::stage1::json_minifier::minify<64>(buf, len, dst, dst_len); - } - - simdjson_warn_unused error_code dom_parser_implementation::stage1(const uint8_t* _buf, size_t _len, stage1_mode streaming) noexcept { - this->buf = _buf; - this->len = _len; - return westmere::stage1::json_structural_indexer::index<64>(_buf, _len, *this, streaming); - } - - simdjson_warn_unused bool implementation::validate_utf8(const char* buf, size_t len) const noexcept { - return westmere::stage1::generic_validate_utf8(buf, len); - } - - simdjson_warn_unused error_code dom_parser_implementation::stage2(dom::document& _doc) noexcept { - return stage2::tape_builder::parse_document(*this, _doc); - } - - simdjson_warn_unused error_code dom_parser_implementation::stage2_next(dom::document& _doc) noexcept { - return stage2::tape_builder::parse_document(*this, _doc); - } - - simdjson_warn_unused uint8_t* dom_parser_implementation::parse_string(const uint8_t* src, uint8_t* dst, bool replacement_char) const noexcept { - return westmere::stringparsing::parse_string(src, dst, replacement_char); - } - - simdjson_warn_unused uint8_t* dom_parser_implementation::parse_wobbly_string(const uint8_t* src, uint8_t* dst) const noexcept { - return westmere::stringparsing::parse_wobbly_string(src, dst); - } - - simdjson_warn_unused error_code dom_parser_implementation::parse(const uint8_t* _buf, size_t _len, dom::document& _doc) noexcept { - auto error = stage1(_buf, _len, stage1_mode::regular); - if (error) { return error; } - return stage2(_doc); - } - - } // namespace westmere -} // namespace simdjson - -/* including simdjson/westmere/end.h: #include */ -/* begin file simdjson/westmere/end.h */ -/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ -/* amalgamation skipped (editor-only): #include "simdjson/westmere/base.h" */ -/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ - -#if !SIMDJSON_CAN_ALWAYS_RUN_WESTMERE -SIMDJSON_UNTARGET_REGION -#endif - -/* undefining SIMDJSON_IMPLEMENTATION from "westmere" */ -#undef SIMDJSON_IMPLEMENTATION -/* end file simdjson/westmere/end.h */ - -#endif // SIMDJSON_SRC_WESTMERE_CPP -/* end file westmere.cpp */ -#endif - /* undefining SIMDJSON_CONDITIONAL_INCLUDE */ #undef SIMDJSON_CONDITIONAL_INCLUDE diff --git a/M6502/HarteTest_6502/simdjson/simdjson.h b/M6502/HarteTest_6502/simdjson/simdjson.h index cf5dce4..ab22449 100644 --- a/M6502/HarteTest_6502/simdjson/simdjson.h +++ b/M6502/HarteTest_6502/simdjson/simdjson.h @@ -1,4 +1,4 @@ -/* auto-generated on 2023-12-07 12:42:28 -0500. Do not edit! */ +/* auto-generated on 2024-08-26 09:37:03 -0400. Do not edit! */ /* including simdjson.h: */ /* begin file simdjson.h */ #ifndef SIMDJSON_H @@ -60,6 +60,16 @@ #endif #endif +// C++ 23 +#if !defined(SIMDJSON_CPLUSPLUS23) && (SIMDJSON_CPLUSPLUS >= 202302L) +#define SIMDJSON_CPLUSPLUS23 1 +#endif + +// C++ 20 +#if !defined(SIMDJSON_CPLUSPLUS20) && (SIMDJSON_CPLUSPLUS >= 202002L) +#define SIMDJSON_CPLUSPLUS20 1 +#endif + // C++ 17 #if !defined(SIMDJSON_CPLUSPLUS17) && (SIMDJSON_CPLUSPLUS >= 201703L) #define SIMDJSON_CPLUSPLUS17 1 @@ -104,6 +114,9 @@ #include #endif +// We are using size_t without namespace std:: throughout the project +using std::size_t; + #ifdef _MSC_VER #define SIMDJSON_VISUAL_STUDIO 1 /** @@ -125,12 +138,14 @@ #endif // __clang__ #endif // _MSC_VER -#if defined(__x86_64__) || defined(_M_AMD64) +#if (defined(__x86_64__) || defined(_M_AMD64)) && !defined(_M_ARM64EC) #define SIMDJSON_IS_X86_64 1 -#elif defined(__aarch64__) || defined(_M_ARM64) +#elif defined(__aarch64__) || defined(_M_ARM64) || defined(_M_ARM64EC) #define SIMDJSON_IS_ARM64 1 #elif defined(__riscv) && __riscv_xlen == 64 #define SIMDJSON_IS_RISCV64 1 +#elif defined(__loongarch_lp64) +#define SIMDJSON_IS_LOONGARCH64 1 #elif defined(__PPC64__) || defined(_M_PPC64) #if defined(__ALTIVEC__) #define SIMDJSON_IS_PPC64_VMX 1 @@ -239,6 +254,11 @@ #define SIMDJSON_NO_SANITIZE_UNDEFINED #endif +#if defined(__clang__) || defined(__GNUC__) +#define simdjson_pure [[gnu::pure]] +#else +#define simdjson_pure +#endif #if defined(__clang__) || defined(__GNUC__) #if defined(__has_feature) @@ -332,6 +352,8 @@ namespace simdjson { #define SIMDJSON_ISALIGNED_N(ptr, n) (((uintptr_t)(ptr) & ((n)-1)) == 0) #if SIMDJSON_REGULAR_VISUAL_STUDIO + // We could use [[deprecated]] but it requires C++14 +#define simdjson_deprecated __declspec(deprecated) #define simdjson_really_inline __forceinline #define simdjson_never_inline __declspec(noinline) @@ -370,6 +392,8 @@ namespace simdjson { #define SIMDJSON_POP_DISABLE_UNUSED_WARNINGS #else // SIMDJSON_REGULAR_VISUAL_STUDIO + // We could use [[deprecated]] but it requires C++14 +#define simdjson_deprecated __attribute__((deprecated)) #define simdjson_really_inline inline __attribute__((always_inline)) #define simdjson_never_inline inline __attribute__((noinline)) @@ -551,7 +575,7 @@ SIMDJSON_PUSH_DISABLE_ALL_WARNINGS #define NONSTD_SV_LITE_H_INCLUDED #define string_view_lite_MAJOR 1 -#define string_view_lite_MINOR 7 +#define string_view_lite_MINOR 8 #define string_view_lite_PATCH 0 #define string_view_lite_VERSION nssv_STRINGIFY(string_view_lite_MAJOR) "." nssv_STRINGIFY(string_view_lite_MINOR) "." nssv_STRINGIFY(string_view_lite_PATCH) @@ -608,6 +632,10 @@ SIMDJSON_PUSH_DISABLE_ALL_WARNINGS # define nssv_CONFIG_NO_STREAM_INSERTION 0 #endif +#ifndef nssv_CONFIG_CONSTEXPR11_STD_SEARCH +# define nssv_CONFIG_CONSTEXPR11_STD_SEARCH 1 +#endif + // Control presence of exception handling (try and auto discover): #ifndef nssv_CONFIG_NO_EXCEPTIONS @@ -669,6 +697,8 @@ SIMDJSON_PUSH_DISABLE_ALL_WARNINGS #if nssv_CONFIG_CONVERSION_STD_STRING_FREE_FUNCTIONS +#include + namespace nonstd { template< class CharT, class Traits, class Allocator = std::allocator > @@ -807,7 +837,7 @@ namespace nonstd { # define nssv_HAS_CPP0X 0 #endif -// Unless defined otherwise below, consider VC14 as C++11 for variant-lite: +// Unless defined otherwise below, consider VC14 as C++11 for string-view-lite: #if nssv_COMPILER_MSVC_VER >= 1900 # undef nssv_CPP11_OR_GREATER @@ -974,9 +1004,9 @@ namespace nonstd { # pragma clang diagnostic ignored "-Wreserved-user-defined-literal" # pragma clang diagnostic push # pragma clang diagnostic ignored "-Wuser-defined-literals" -#elif defined(__GNUC__) -# pragma GCC diagnostic push -# pragma GCC diagnostic ignored "-Wliteral-suffix" +#elif nssv_COMPILER_GNUC_VERSION >= 480 +# pragma GCC diagnostic push +# pragma GCC diagnostic ignored "-Wliteral-suffix" #endif // __clang__ #if nssv_COMPILER_MSVC_VERSION >= 140 @@ -991,8 +1021,8 @@ namespace nonstd { #if defined(__clang__) # define nssv_RESTORE_WARNINGS() _Pragma("clang diagnostic pop") -#elif defined(__GNUC__) -# define nssv_RESTORE_WARNINGS() _Pragma("GCC diagnostic pop") +#elif nssv_COMPILER_GNUC_VERSION >= 480 +# define nssv_RESTORE_WARNINGS() _Pragma("GCC diagnostic pop") #elif nssv_COMPILER_MSVC_VERSION >= 140 # define nssv_RESTORE_WARNINGS() __pragma(warning(pop )) #else @@ -1108,12 +1138,31 @@ namespace nonstd { // non-recursive: +#if nssv_CONFIG_CONSTEXPR11_STD_SEARCH + template< class CharT, class Traits = std::char_traits > constexpr const CharT* search(basic_string_view haystack, basic_string_view needle) { return std::search(haystack.begin(), haystack.end(), needle.begin(), needle.end()); } +#else // nssv_CONFIG_CONSTEXPR11_STD_SEARCH + + template< class CharT, class Traits = std::char_traits > + nssv_constexpr14 const CharT* search(basic_string_view haystack, basic_string_view needle) + { + while (needle.size() <= haystack.size()) + { + if (haystack.starts_with(needle)) + { + return haystack.cbegin(); + } + haystack = basic_string_view{ haystack.begin() + 1, haystack.size() - 1U }; + } + return haystack.cend(); + } +#endif // nssv_CONFIG_CONSTEXPR11_STD_SEARCH + #endif // OPTIMIZE #endif // nssv_CPP11_OR_GREATER && ! nssv_CPP17_OR_GREATER @@ -1386,7 +1435,7 @@ namespace nonstd { // find(), 4x: - nssv_constexpr size_type find(basic_string_view v, size_type pos = 0) const nssv_noexcept // (1) + nssv_constexpr14 size_type find(basic_string_view v, size_type pos = 0) const nssv_noexcept // (1) { return assert(v.size() == 0 || v.data() != nssv_nullptr) , pos >= size() @@ -2411,7 +2460,7 @@ namespace std { #define SIMDJSON_SIMDJSON_VERSION_H /** The version of simdjson being used (major.minor.revision) */ -#define SIMDJSON_VERSION "3.6.3" +#define SIMDJSON_VERSION "3.10.1" namespace simdjson { enum { @@ -2422,11 +2471,11 @@ namespace simdjson { /** * The minor version (major.MINOR.revision) of simdjson being used. */ - SIMDJSON_VERSION_MINOR = 6, + SIMDJSON_VERSION_MINOR = 10, /** * The revision (major.minor.REVISION) of simdjson being used. */ - SIMDJSON_VERSION_REVISION = 3 + SIMDJSON_VERSION_REVISION = 1 }; } // namespace simdjson @@ -2475,6 +2524,7 @@ namespace simdjson { F_ATOM_ERROR, ///< Problem while parsing an atom starting with the letter 'f' N_ATOM_ERROR, ///< Problem while parsing an atom starting with the letter 'n' NUMBER_ERROR, ///< Problem while parsing a number + BIGINT_ERROR, ///< The integer value exceeds 64 bits UTF8_ERROR, ///< the input is not valid UTF-8 UNINITIALIZED, ///< unknown error, or uninitialized document EMPTY, ///< no structural element found @@ -2486,7 +2536,7 @@ namespace simdjson { INDEX_OUT_OF_BOUNDS, ///< JSON array index too large NO_SUCH_FIELD, ///< JSON field not found in object IO_ERROR, ///< Error reading a file - INVALID_JSON_POINTER, ///< Invalid JSON pointer reference + INVALID_JSON_POINTER, ///< Invalid JSON pointer syntax INVALID_URI_FRAGMENT, ///< Invalid URI fragment UNEXPECTED_ERROR, ///< indicative of a bug in simdjson PARSER_IN_USE, ///< parser is already in use. @@ -3153,7 +3203,7 @@ namespace simdjson { * Unescape a valid UTF-8 string from src to dst, stopping at a final unescaped quote. There * must be an unescaped quote terminating the string. It returns the final output * position as pointer. In case of error (e.g., the string has bad escaped codes), - * then null_nullptrptr is returned. It is assumed that the output buffer is large + * then null_ptr is returned. It is assumed that the output buffer is large * enough. E.g., if src points at 'joe"', then dst needs to have four free bytes + * SIMDJSON_PADDING bytes. * @@ -3170,7 +3220,7 @@ namespace simdjson { * Unescape a NON-valid UTF-8 string from src to dst, stopping at a final unescaped quote. There * must be an unescaped quote terminating the string. It returns the final output * position as pointer. In case of error (e.g., the string has bad escaped codes), - * then null_nullptrptr is returned. It is assumed that the output buffer is large + * then null_ptr is returned. It is assumed that the output buffer is large * enough. E.g., if src points at 'joe"', then dst needs to have four free bytes + * SIMDJSON_PADDING bytes. * @@ -3224,14 +3274,14 @@ namespace simdjson { * * @return Current capacity, in bytes. */ - simdjson_inline size_t capacity() const noexcept; + simdjson_pure simdjson_inline size_t capacity() const noexcept; /** * The maximum level of nested object and arrays supported by this parser. * * @return Maximum depth, in bytes. */ - simdjson_inline size_t max_depth() const noexcept; + simdjson_pure simdjson_inline size_t max_depth() const noexcept; /** * Ensure this parser has enough memory to process JSON documents up to `capacity` bytes in length @@ -3272,11 +3322,11 @@ namespace simdjson { simdjson_inline dom_parser_implementation::dom_parser_implementation(dom_parser_implementation&& other) noexcept = default; simdjson_inline dom_parser_implementation& dom_parser_implementation::operator=(dom_parser_implementation&& other) noexcept = default; - simdjson_inline size_t dom_parser_implementation::capacity() const noexcept { + simdjson_pure simdjson_inline size_t dom_parser_implementation::capacity() const noexcept { return _capacity; } - simdjson_inline size_t dom_parser_implementation::max_depth() const noexcept { + simdjson_pure simdjson_inline size_t dom_parser_implementation::max_depth() const noexcept { return _max_depth; } @@ -3348,7 +3398,7 @@ namespace simdjson { * * @return the name of the implementation, e.g. "haswell", "westmere", "arm64". */ - virtual const std::string& name() const { return _name; } + virtual std::string name() const { return std::string(_name); } /** * The description of this implementation. @@ -3358,7 +3408,7 @@ namespace simdjson { * * @return the description of the implementation, e.g. "Intel/AMD AVX2", "Intel/AMD SSE4.2", "ARM NEON". */ - virtual const std::string& description() const { return _description; } + virtual std::string description() const { return std::string(_description); } /** * The instruction sets this implementation is compiled against @@ -3434,18 +3484,19 @@ namespace simdjson { _required_instruction_sets(required_instruction_sets) { } - virtual ~implementation() = default; + protected: + ~implementation() = default; private: /** * The name of this implementation. */ - const std::string _name; + std::string_view _name; /** * The description of this implementation. */ - const std::string _description; + std::string_view _description; /** * Instruction sets required for this implementation. @@ -3572,6 +3623,9 @@ namespace simdjson { * @param length the number of bytes to copy */ explicit inline padded_string(const char* data, size_t length) noexcept; +#ifdef __cpp_char8_t + explicit inline padded_string(const char8_t* data, size_t length) noexcept; +#endif /** * Create a new padded string by copying the given input. * @@ -3641,6 +3695,15 @@ namespace simdjson { /** * Load this padded string from a file. * + * ## Windows and Unicode + * + * Windows users who need to read files with non-ANSI characters in the + * name should set their code page to UTF-8 (65001) before calling this + * function. This should be the default with Windows 11 and better. + * Further, they may use the AreFileApisANSI function to determine whether + * the filename is interpreted using the ANSI or the system default OEM + * codepage, and they may call SetFileApisToOEM accordingly. + * * @return IO_ERROR on error. Be mindful that on some 32-bit systems, * the file size might be limited to 2 GB. * @@ -3683,6 +3746,9 @@ namespace simdjson { // This is deliberately outside of simdjson so that people get it without having to use the namespace inline simdjson::padded_string operator "" _padded(const char* str, size_t len); +#ifdef __cpp_char8_t +inline simdjson::padded_string operator "" _padded(const char8_t* str, size_t len); +#endif namespace simdjson { namespace internal { @@ -3765,12 +3831,15 @@ namespace simdjson { * * @param s The string. * @param len The length of the string (not including padding). - * @param capacity The allocated length of the string, including padding. + * @param capacity The allocated length of the string, including padding. If the capacity is less + * than the length, the capacity will be set to the length. */ explicit inline padded_string_view(const char* s, size_t len, size_t capacity) noexcept; /** overload explicit inline padded_string_view(const char* s, size_t len) noexcept */ explicit inline padded_string_view(const uint8_t* s, size_t len, size_t capacity) noexcept; - +#ifdef __cpp_char8_t + explicit inline padded_string_view(const char8_t* s, size_t len, size_t capacity) noexcept; +#endif /** * Promise the given string has at least SIMDJSON_PADDING extra bytes allocated to it. * @@ -3784,7 +3853,8 @@ namespace simdjson { * Promise the given string_view has at least SIMDJSON_PADDING extra bytes allocated to it. * * @param s The string. - * @param capacity The allocated length of the string, including padding. + * @param capacity The allocated length of the string, including padding. If the capacity is less + * than the length, the capacity will be set to the length. */ explicit inline padded_string_view(std::string_view s, size_t capacity) noexcept; @@ -3837,13 +3907,19 @@ namespace simdjson { inline padded_string_view::padded_string_view(const char* s, size_t len, size_t capacity) noexcept : std::string_view(s, len), _capacity(capacity) { + if (_capacity < len) { _capacity = len; } } inline padded_string_view::padded_string_view(const uint8_t* s, size_t len, size_t capacity) noexcept : padded_string_view(reinterpret_cast(s), len, capacity) { } - +#ifdef __cpp_char8_t + inline padded_string_view::padded_string_view(const char8_t* s, size_t len, size_t capacity) noexcept + : padded_string_view(reinterpret_cast(s), len, capacity) + { + } +#endif inline padded_string_view::padded_string_view(const std::string& s) noexcept : std::string_view(s), _capacity(s.capacity()) { @@ -3852,6 +3928,7 @@ namespace simdjson { inline padded_string_view::padded_string_view(std::string_view s, size_t capacity) noexcept : std::string_view(s), _capacity(capacity) { + if (_capacity < s.length()) { _capacity = s.length(); } } inline size_t padded_string_view::capacity() const noexcept { return _capacity; } @@ -3925,6 +4002,14 @@ namespace simdjson { std::memcpy(data_ptr, data, length); } } +#ifdef __cpp_char8_t + inline padded_string::padded_string(const char8_t* data, size_t length) noexcept + : viable_size(length), data_ptr(internal::allocate_padded_buffer(length)) { + if ((data != nullptr) && (data_ptr != nullptr)) { + std::memcpy(data_ptr, reinterpret_cast(data), length); + } + } +#endif // note: do not pass std::string arguments by value inline padded_string::padded_string(const std::string& str_) noexcept : viable_size(str_.size()), data_ptr(internal::allocate_padded_buffer(str_.size())) { @@ -3947,6 +4032,7 @@ namespace simdjson { inline padded_string::padded_string(padded_string&& o) noexcept : viable_size(o.viable_size), data_ptr(o.data_ptr) { o.data_ptr = nullptr; // we take ownership + o.viable_size = 0; } inline padded_string& padded_string::operator=(padded_string&& o) noexcept { @@ -4044,7 +4130,11 @@ namespace simdjson { inline simdjson::padded_string operator "" _padded(const char* str, size_t len) { return simdjson::padded_string(str, len); } - +#ifdef __cpp_char8_t +inline simdjson::padded_string operator "" _padded(const char8_t* str, size_t len) { + return simdjson::padded_string(reinterpret_cast(str), len); +} +#endif #endif // SIMDJSON_PADDED_STRING_INL_H /* end file simdjson/padded_string-inl.h */ /* skipped duplicate #include "simdjson/padded_string_view.h" */ @@ -4186,11 +4276,14 @@ namespace simdjson { public: using value_type = element; using difference_type = std::ptrdiff_t; + using pointer = void; + using reference = value_type; + using iterator_category = std::forward_iterator_tag; /** * Get the actual value */ - inline value_type operator*() const noexcept; + inline reference operator*() const noexcept; /** * Get the next value. * @@ -4290,6 +4383,11 @@ namespace simdjson { */ inline simdjson_result at(size_t index) const noexcept; + /** + * Implicitly convert object to element + */ + inline operator element() const noexcept; + private: simdjson_inline array(const internal::tape_ref& tape) noexcept; internal::tape_ref tape; @@ -4527,6 +4625,15 @@ namespace simdjson { * If the parser's current capacity is less than the file length, it will allocate enough capacity * to handle it (up to max_capacity). * + * ## Windows and Unicode + * + * Windows users who need to read files with non-ANSI characters in the + * name should set their code page to UTF-8 (65001) before calling this + * function. This should be the default with Windows 11 and better. + * Further, they may use the AreFileApisANSI function to determine whether + * the filename is interpreted using the ANSI or the system default OEM + * codepage, and they may call SetFileApisToOEM accordingly. + * * @param path The path to load. * @return The document, or an error: * - IO_ERROR if there was an error opening or reading the file. @@ -4539,6 +4646,53 @@ namespace simdjson { */ inline simdjson_result load(const std::string& path) & noexcept; inline simdjson_result load(const std::string& path) && = delete; + + /** + * Load a JSON document from a file into a provide document instance and return a temporary reference to it. + * It is similar to the function `load` except that instead of parsing into the internal + * `document` instance associated with the parser, it allows the user to provide a document + * instance. + * + * dom::parser parser; + * dom::document doc; + * element doc_root = parser.load_into_document(doc, "jsonexamples/twitter.json"); + * + * The function is eager: the file's content is loaded in memory inside the parser instance + * and immediately parsed. The file can be deleted after the `parser.load_into_document` call. + * + * ### IMPORTANT: Document Lifetime + * + * After the call to load_into_document, the parser is no longer needed. + * + * The JSON document lives in the document instance: you must keep the document + * instance alive while you navigate through it (i.e., used the returned value from + * load_into_document). You are encourage to reuse the document instance + * many times with new data to avoid reallocations: + * + * dom::document doc; + * element doc_root1 = parser.load_into_document(doc, "jsonexamples/twitter.json"); + * //... doc_root1 is a pointer inside doc + * element doc_root2 = parser.load_into_document(doc, "jsonexamples/twitter.json"); + * //... doc_root2 is a pointer inside doc + * // at this point doc_root1 is no longer safe + * + * Moving the document instance is safe, but it invalidates the element instances. After + * moving a document, you can recover safe access to the document root with its `root()` method. + * + * @param doc The document instance where the parsed data will be stored (on success). + * @param path The path to load. + * @return The document, or an error: + * - IO_ERROR if there was an error opening or reading the file. + * Be mindful that on some 32-bit systems, + * the file size might be limited to 2 GB. + * - MEMALLOC if the parser does not have enough capacity and memory allocation fails. + * - CAPACITY if the parser does not have enough capacity and len > max_capacity. + * - other json errors if parsing fails. You should not rely on these errors to always the same for the + * same document: they may vary under runtime dispatch (so they may vary depending on your system and hardware). + */ + inline simdjson_result load_into_document(document& doc, const std::string& path) & noexcept; + inline simdjson_result load_into_document(document& doc, const std::string& path) && = delete; + /** * Parse a JSON document and return a temporary reference to it. * @@ -4915,7 +5069,7 @@ namespace simdjson { * * @return Maximum depth, in bytes. */ - simdjson_inline size_t max_depth() const noexcept; + simdjson_pure simdjson_inline size_t max_depth() const noexcept; /** * Set max_capacity. This is the largest document this parser can automatically support. @@ -5574,7 +5728,7 @@ namespace simdjson { static_assert(!sizeof(T), "The get method with given type is not implemented by the simdjson library. " "The supported types are Boolean (bool), numbers (double, uint64_t, int64_t), " "strings (std::string_view, const char *), arrays (dom::array) and objects (dom::object). " - "We recommand you use get_double(), get_bool(), get_uint64(), get_int64(), " + "We recommend you use get_double(), get_bool(), get_uint64(), get_int64(), " "get_object(), get_array() or get_string() instead of the get template."); } @@ -5653,7 +5807,7 @@ namespace simdjson { * * @return The integer value. * @exception simdjson_error(INCORRECT_TYPE) if the JSON element is not an integer - * @exception simdjson_error(NUMBER_OUT_OF_RANGE) if the integer doesn't fit in 64 bits or is negative + * @exception simdjson_error(NUMBER_OUT_OF_RANGE) if the integer does not fit in 64 bits or is negative */ inline operator uint64_t() const noexcept(false); /** @@ -5661,7 +5815,7 @@ namespace simdjson { * * @return The integer value. * @exception simdjson_error(INCORRECT_TYPE) if the JSON element is not an integer - * @exception simdjson_error(NUMBER_OUT_OF_RANGE) if the integer doesn't fit in 64 bits + * @exception simdjson_error(NUMBER_OUT_OF_RANGE) if the integer does not fit in 64 bits */ inline operator int64_t() const noexcept(false); /** @@ -5669,7 +5823,6 @@ namespace simdjson { * * @return The double value. * @exception simdjson_error(INCORRECT_TYPE) if the JSON element is not a number - * @exception simdjson_error(NUMBER_OUT_OF_RANGE) if the integer doesn't fit in 64 bits or is negative */ inline operator double() const noexcept(false); /** @@ -5934,13 +6087,16 @@ namespace simdjson { class iterator { public: - using value_type = key_value_pair; + using value_type = const key_value_pair; using difference_type = std::ptrdiff_t; + using pointer = void; + using reference = value_type; + using iterator_category = std::forward_iterator_tag; /** * Get the actual key/value pair */ - inline const value_type operator*() const noexcept; + inline reference operator*() const noexcept; /** * Get the next key/value pair. * @@ -6113,6 +6269,11 @@ namespace simdjson { */ inline simdjson_result at_key_case_insensitive(std::string_view key) const noexcept; + /** + * Implicitly convert object to element + */ + inline operator element() const noexcept; + private: simdjson_inline object(const internal::tape_ref& tape) noexcept; @@ -6243,15 +6404,15 @@ namespace simdjson { simdjson_inline void one_char(char c); simdjson_inline void call_print_newline() { - this->print_newline(); + static_cast(this)->print_newline(); } simdjson_inline void call_print_indents(size_t depth) { - this->print_indents(depth); + static_cast(this)->print_indents(depth); } simdjson_inline void call_print_space() { - this->print_space(); + static_cast(this)->print_space(); } protected: @@ -6446,46 +6607,25 @@ namespace simdjson { #endif /* end file simdjson/dom/serialization.h */ -// Deprecated API -/* including simdjson/dom/jsonparser.h: #include "simdjson/dom/jsonparser.h" */ -/* begin file simdjson/dom/jsonparser.h */ -// TODO Remove this -- deprecated API and files +// Inline functions +/* including simdjson/dom/array-inl.h: #include "simdjson/dom/array-inl.h" */ +/* begin file simdjson/dom/array-inl.h */ +#ifndef SIMDJSON_ARRAY_INL_H +#define SIMDJSON_ARRAY_INL_H -#ifndef SIMDJSON_DOM_JSONPARSER_H -#define SIMDJSON_DOM_JSONPARSER_H +#include /* skipped duplicate #include "simdjson/dom/base.h" */ -/* skipped duplicate #include "simdjson/dom/parser.h" */ +/* skipped duplicate #include "simdjson/dom/array.h" */ /* skipped duplicate #include "simdjson/dom/element.h" */ - -/* including simdjson/dom/parser-inl.h: #include "simdjson/dom/parser-inl.h" */ -/* begin file simdjson/dom/parser-inl.h */ -#ifndef SIMDJSON_PARSER_INL_H -#define SIMDJSON_PARSER_INL_H - -/* skipped duplicate #include "simdjson/dom/base.h" */ -/* skipped duplicate #include "simdjson/dom/document_stream.h" */ -/* skipped duplicate #include "simdjson/implementation.h" */ -/* skipped duplicate #include "simdjson/internal/dom_parser_implementation.h" */ - /* skipped duplicate #include "simdjson/error-inl.h" */ -/* skipped duplicate #include "simdjson/padded_string-inl.h" */ -/* including simdjson/dom/document_stream-inl.h: #include "simdjson/dom/document_stream-inl.h" */ -/* begin file simdjson/dom/document_stream-inl.h */ -#ifndef SIMDJSON_DOCUMENT_STREAM_INL_H -#define SIMDJSON_DOCUMENT_STREAM_INL_H +/* including simdjson/internal/tape_ref-inl.h: #include "simdjson/internal/tape_ref-inl.h" */ +/* begin file simdjson/internal/tape_ref-inl.h */ +#ifndef SIMDJSON_TAPE_REF_INL_H +#define SIMDJSON_TAPE_REF_INL_H -/* skipped duplicate #include "simdjson/dom/base.h" */ -/* skipped duplicate #include "simdjson/dom/document_stream.h" */ -/* including simdjson/dom/element-inl.h: #include "simdjson/dom/element-inl.h" */ -/* begin file simdjson/dom/element-inl.h */ -#ifndef SIMDJSON_ELEMENT_INL_H -#define SIMDJSON_ELEMENT_INL_H - -/* skipped duplicate #include "simdjson/dom/base.h" */ -/* skipped duplicate #include "simdjson/dom/element.h" */ /* skipped duplicate #include "simdjson/dom/document.h" */ -/* skipped duplicate #include "simdjson/dom/object.h" */ +/* skipped duplicate #include "simdjson/internal/tape_ref.h" */ /* including simdjson/internal/tape_type.h: #include "simdjson/internal/tape_type.h" */ /* begin file simdjson/internal/tape_type.h */ #ifndef SIMDJSON_INTERNAL_TAPE_TYPE_H @@ -6518,6 +6658,289 @@ namespace simdjson { #endif // SIMDJSON_INTERNAL_TAPE_TYPE_H /* end file simdjson/internal/tape_type.h */ +#include + +namespace simdjson { + namespace internal { + + constexpr const uint64_t JSON_VALUE_MASK = 0x00FFFFFFFFFFFFFF; + constexpr const uint32_t JSON_COUNT_MASK = 0xFFFFFF; + + // + // tape_ref inline implementation + // + simdjson_inline tape_ref::tape_ref() noexcept : doc{ nullptr }, json_index{ 0 } {} + simdjson_inline tape_ref::tape_ref(const dom::document* _doc, size_t _json_index) noexcept : doc{ _doc }, json_index{ _json_index } {} + + + simdjson_inline bool tape_ref::is_document_root() const noexcept { + return json_index == 1; // should we ever change the structure of the tape, this should get updated. + } + simdjson_inline bool tape_ref::usable() const noexcept { + return doc != nullptr; // when the document pointer is null, this tape_ref is uninitialized (should not be accessed). + } + // Some value types have a specific on-tape word value. It can be faster + // to check the type by doing a word-to-word comparison instead of extracting the + // most significant 8 bits. + + simdjson_inline bool tape_ref::is_double() const noexcept { + constexpr uint64_t tape_double = uint64_t(tape_type::DOUBLE) << 56; + return doc->tape[json_index] == tape_double; + } + simdjson_inline bool tape_ref::is_int64() const noexcept { + constexpr uint64_t tape_int64 = uint64_t(tape_type::INT64) << 56; + return doc->tape[json_index] == tape_int64; + } + simdjson_inline bool tape_ref::is_uint64() const noexcept { + constexpr uint64_t tape_uint64 = uint64_t(tape_type::UINT64) << 56; + return doc->tape[json_index] == tape_uint64; + } + simdjson_inline bool tape_ref::is_false() const noexcept { + constexpr uint64_t tape_false = uint64_t(tape_type::FALSE_VALUE) << 56; + return doc->tape[json_index] == tape_false; + } + simdjson_inline bool tape_ref::is_true() const noexcept { + constexpr uint64_t tape_true = uint64_t(tape_type::TRUE_VALUE) << 56; + return doc->tape[json_index] == tape_true; + } + simdjson_inline bool tape_ref::is_null_on_tape() const noexcept { + constexpr uint64_t tape_null = uint64_t(tape_type::NULL_VALUE) << 56; + return doc->tape[json_index] == tape_null; + } + + inline size_t tape_ref::after_element() const noexcept { + switch (tape_ref_type()) { + case tape_type::START_ARRAY: + case tape_type::START_OBJECT: + return matching_brace_index(); + case tape_type::UINT64: + case tape_type::INT64: + case tape_type::DOUBLE: + return json_index + 2; + default: + return json_index + 1; + } + } + simdjson_inline tape_type tape_ref::tape_ref_type() const noexcept { + return static_cast(doc->tape[json_index] >> 56); + } + simdjson_inline uint64_t internal::tape_ref::tape_value() const noexcept { + return doc->tape[json_index] & internal::JSON_VALUE_MASK; + } + simdjson_inline uint32_t internal::tape_ref::matching_brace_index() const noexcept { + return uint32_t(doc->tape[json_index]); + } + simdjson_inline uint32_t internal::tape_ref::scope_count() const noexcept { + return uint32_t((doc->tape[json_index] >> 32) & internal::JSON_COUNT_MASK); + } + + template + simdjson_inline T tape_ref::next_tape_value() const noexcept { + static_assert(sizeof(T) == sizeof(uint64_t), "next_tape_value() template parameter must be 64-bit"); + // Though the following is tempting... + // return *reinterpret_cast(&doc->tape[json_index + 1]); + // It is not generally safe. It is safer, and often faster to rely + // on memcpy. Yes, it is uglier, but it is also encapsulated. + T x; + std::memcpy(&x, &doc->tape[json_index + 1], sizeof(uint64_t)); + return x; + } + + simdjson_inline uint32_t internal::tape_ref::get_string_length() const noexcept { + size_t string_buf_index = size_t(tape_value()); + uint32_t len; + std::memcpy(&len, &doc->string_buf[string_buf_index], sizeof(len)); + return len; + } + + simdjson_inline const char* internal::tape_ref::get_c_str() const noexcept { + size_t string_buf_index = size_t(tape_value()); + return reinterpret_cast(&doc->string_buf[string_buf_index + sizeof(uint32_t)]); + } + + inline std::string_view internal::tape_ref::get_string_view() const noexcept { + return std::string_view( + get_c_str(), + get_string_length() + ); + } + + } // namespace internal +} // namespace simdjson + +#endif // SIMDJSON_TAPE_REF_INL_H +/* end file simdjson/internal/tape_ref-inl.h */ + +#include + +namespace simdjson { + + // + // simdjson_result inline implementation + // + simdjson_inline simdjson_result::simdjson_result() noexcept + : internal::simdjson_result_base() {} + simdjson_inline simdjson_result::simdjson_result(dom::array value) noexcept + : internal::simdjson_result_base(std::forward(value)) {} + simdjson_inline simdjson_result::simdjson_result(error_code error) noexcept + : internal::simdjson_result_base(error) {} + +#if SIMDJSON_EXCEPTIONS + + inline dom::array::iterator simdjson_result::begin() const noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return first.begin(); + } + inline dom::array::iterator simdjson_result::end() const noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return first.end(); + } + inline size_t simdjson_result::size() const noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return first.size(); + } + +#endif // SIMDJSON_EXCEPTIONS + + inline simdjson_result simdjson_result::at_pointer(std::string_view json_pointer) const noexcept { + if (error()) { return error(); } + return first.at_pointer(json_pointer); + } + inline simdjson_result simdjson_result::at(size_t index) const noexcept { + if (error()) { return error(); } + return first.at(index); + } + + namespace dom { + + // + // array inline implementation + // + simdjson_inline array::array() noexcept : tape{} {} + simdjson_inline array::array(const internal::tape_ref& _tape) noexcept : tape{ _tape } {} + inline array::iterator array::begin() const noexcept { + SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 + return internal::tape_ref(tape.doc, tape.json_index + 1); + } + inline array::iterator array::end() const noexcept { + SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 + return internal::tape_ref(tape.doc, tape.after_element() - 1); + } + inline size_t array::size() const noexcept { + SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 + return tape.scope_count(); + } + inline size_t array::number_of_slots() const noexcept { + SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 + return tape.matching_brace_index() - tape.json_index; + } + inline simdjson_result array::at_pointer(std::string_view json_pointer) const noexcept { + SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 + if (json_pointer.empty()) { // an empty string means that we return the current node + return element(this->tape); // copy the current node + } + else if (json_pointer[0] != '/') { // otherwise there is an error + return INVALID_JSON_POINTER; + } + json_pointer = json_pointer.substr(1); + // - means "the append position" or "the element after the end of the array" + // We don't support this, because we're returning a real element, not a position. + if (json_pointer == "-") { return INDEX_OUT_OF_BOUNDS; } + + // Read the array index + size_t array_index = 0; + size_t i; + for (i = 0; i < json_pointer.length() && json_pointer[i] != '/'; i++) { + uint8_t digit = uint8_t(json_pointer[i] - '0'); + // Check for non-digit in array index. If it's there, we're trying to get a field in an object + if (digit > 9) { return INCORRECT_TYPE; } + array_index = array_index * 10 + digit; + } + + // 0 followed by other digits is invalid + if (i > 1 && json_pointer[0] == '0') { return INVALID_JSON_POINTER; } // "JSON pointer array index has other characters after 0" + + // Empty string is invalid; so is a "/" with no digits before it + if (i == 0) { return INVALID_JSON_POINTER; } // "Empty string in JSON pointer array index" + + // Get the child + auto child = array(tape).at(array_index); + // If there is an error, it ends here + if (child.error()) { + return child; + } + // If there is a /, we're not done yet, call recursively. + if (i < json_pointer.length()) { + child = child.at_pointer(json_pointer.substr(i)); + } + return child; + } + + inline simdjson_result array::at(size_t index) const noexcept { + SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 + size_t i = 0; + for (auto element : *this) { + if (i == index) { return element; } + i++; + } + return INDEX_OUT_OF_BOUNDS; + } + + inline array::operator element() const noexcept { + return element(tape); + } + + // + // array::iterator inline implementation + // + simdjson_inline array::iterator::iterator(const internal::tape_ref& _tape) noexcept : tape{ _tape } { } + inline element array::iterator::operator*() const noexcept { + return element(tape); + } + inline array::iterator& array::iterator::operator++() noexcept { + tape.json_index = tape.after_element(); + return *this; + } + inline array::iterator array::iterator::operator++(int) noexcept { + array::iterator out = *this; + ++*this; + return out; + } + inline bool array::iterator::operator!=(const array::iterator& other) const noexcept { + return tape.json_index != other.tape.json_index; + } + inline bool array::iterator::operator==(const array::iterator& other) const noexcept { + return tape.json_index == other.tape.json_index; + } + inline bool array::iterator::operator<(const array::iterator& other) const noexcept { + return tape.json_index < other.tape.json_index; + } + inline bool array::iterator::operator<=(const array::iterator& other) const noexcept { + return tape.json_index <= other.tape.json_index; + } + inline bool array::iterator::operator>=(const array::iterator& other) const noexcept { + return tape.json_index >= other.tape.json_index; + } + inline bool array::iterator::operator>(const array::iterator& other) const noexcept { + return tape.json_index > other.tape.json_index; + } + + } // namespace dom + + +} // namespace simdjson + +/* including simdjson/dom/element-inl.h: #include "simdjson/dom/element-inl.h" */ +/* begin file simdjson/dom/element-inl.h */ +#ifndef SIMDJSON_ELEMENT_INL_H +#define SIMDJSON_ELEMENT_INL_H + +/* skipped duplicate #include "simdjson/dom/base.h" */ +/* skipped duplicate #include "simdjson/dom/element.h" */ +/* skipped duplicate #include "simdjson/dom/document.h" */ +/* skipped duplicate #include "simdjson/dom/object.h" */ +/* skipped duplicate #include "simdjson/internal/tape_type.h" */ + /* including simdjson/dom/object-inl.h: #include "simdjson/dom/object-inl.h" */ /* begin file simdjson/dom/object-inl.h */ #ifndef SIMDJSON_OBJECT_INL_H @@ -6677,6 +7100,10 @@ namespace simdjson { return NO_SUCH_FIELD; } + inline object::operator element() const noexcept { + return element(tape); + } + // // object::iterator inline implementation // @@ -7151,6 +7578,23 @@ namespace simdjson { return at_key(key); } + inline bool is_pointer_well_formed(std::string_view json_pointer) noexcept { + if (simdjson_unlikely(json_pointer[0] != '/')) { + return false; + } + size_t escape = json_pointer.find('~'); + if (escape == std::string_view::npos) { + return true; + } + if (escape == json_pointer.size() - 1) { + return false; + } + if (json_pointer[escape + 1] != '0' && json_pointer[escape + 1] != '1') { + return false; + } + return true; + } + inline simdjson_result element::at_pointer(std::string_view json_pointer) const noexcept { SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 switch (tape.tape_ref_type()) { @@ -7159,7 +7603,10 @@ namespace simdjson { case internal::tape_type::START_ARRAY: return array(tape).at_pointer(json_pointer); default: { - if (!json_pointer.empty()) { // a non-empty string is invalid on an atom + if (!json_pointer.empty()) { // a non-empty string can be invalid, or accessing a primitive (issue 2154) + if (is_pointer_well_formed(json_pointer)) { + return NO_SUCH_FIELD; + } return INVALID_JSON_POINTER; } // an empty string means that we return the current node @@ -7228,7 +7675,289 @@ namespace simdjson { #endif // SIMDJSON_ELEMENT_INL_H /* end file simdjson/dom/element-inl.h */ -/* skipped duplicate #include "simdjson/dom/parser-inl.h" */ + +#if defined(__cpp_lib_ranges) +static_assert(std::ranges::view); +static_assert(std::ranges::sized_range); +#if SIMDJSON_EXCEPTIONS +static_assert(std::ranges::view>); +static_assert(std::ranges::sized_range>); +#endif // SIMDJSON_EXCEPTIONS +#endif // defined(__cpp_lib_ranges) + +#endif // SIMDJSON_ARRAY_INL_H +/* end file simdjson/dom/array-inl.h */ +/* including simdjson/dom/document_stream-inl.h: #include "simdjson/dom/document_stream-inl.h" */ +/* begin file simdjson/dom/document_stream-inl.h */ +#ifndef SIMDJSON_DOCUMENT_STREAM_INL_H +#define SIMDJSON_DOCUMENT_STREAM_INL_H + +/* skipped duplicate #include "simdjson/dom/base.h" */ +/* skipped duplicate #include "simdjson/dom/document_stream.h" */ +/* skipped duplicate #include "simdjson/dom/element-inl.h" */ +/* including simdjson/dom/parser-inl.h: #include "simdjson/dom/parser-inl.h" */ +/* begin file simdjson/dom/parser-inl.h */ +#ifndef SIMDJSON_PARSER_INL_H +#define SIMDJSON_PARSER_INL_H + +/* skipped duplicate #include "simdjson/dom/base.h" */ +/* skipped duplicate #include "simdjson/dom/document_stream.h" */ +/* skipped duplicate #include "simdjson/implementation.h" */ +/* skipped duplicate #include "simdjson/internal/dom_parser_implementation.h" */ + +/* skipped duplicate #include "simdjson/error-inl.h" */ +/* skipped duplicate #include "simdjson/padded_string-inl.h" */ +/* skipped duplicate #include "simdjson/dom/document_stream-inl.h" */ +/* skipped duplicate #include "simdjson/dom/element-inl.h" */ + +#include +#include /* memcmp */ + +namespace simdjson { + namespace dom { + + // + // parser inline implementation + // + simdjson_inline parser::parser(size_t max_capacity) noexcept + : _max_capacity{ max_capacity }, + loaded_bytes(nullptr) { + } + simdjson_inline parser::parser(parser&& other) noexcept = default; + simdjson_inline parser& parser::operator=(parser&& other) noexcept = default; + + inline bool parser::is_valid() const noexcept { return valid; } + inline int parser::get_error_code() const noexcept { return error; } + inline std::string parser::get_error_message() const noexcept { return error_message(error); } + + inline bool parser::dump_raw_tape(std::ostream& os) const noexcept { + return valid ? doc.dump_raw_tape(os) : false; + } + + inline simdjson_result parser::read_file(const std::string& path) noexcept { + // Open the file + SIMDJSON_PUSH_DISABLE_WARNINGS + SIMDJSON_DISABLE_DEPRECATED_WARNING // Disable CRT_SECURE warning on MSVC: manually verified this is safe + std::FILE* fp = std::fopen(path.c_str(), "rb"); + SIMDJSON_POP_DISABLE_WARNINGS + + if (fp == nullptr) { + return IO_ERROR; + } + + // Get the file size + int ret; +#if SIMDJSON_VISUAL_STUDIO && !SIMDJSON_IS_32BITS + ret = _fseeki64(fp, 0, SEEK_END); +#else + ret = std::fseek(fp, 0, SEEK_END); +#endif // _WIN64 + if (ret < 0) { + std::fclose(fp); + return IO_ERROR; + } +#if SIMDJSON_VISUAL_STUDIO && !SIMDJSON_IS_32BITS + __int64 len = _ftelli64(fp); + if (len == -1L) { + std::fclose(fp); + return IO_ERROR; + } +#else + long len = std::ftell(fp); + if ((len < 0) || (len == LONG_MAX)) { + std::fclose(fp); + return IO_ERROR; + } +#endif + + // Make sure we have enough capacity to load the file + if (_loaded_bytes_capacity < size_t(len)) { + loaded_bytes.reset(internal::allocate_padded_buffer(len)); + if (!loaded_bytes) { + std::fclose(fp); + return MEMALLOC; + } + _loaded_bytes_capacity = len; + } + + // Read the string + std::rewind(fp); + size_t bytes_read = std::fread(loaded_bytes.get(), 1, len, fp); + if (std::fclose(fp) != 0 || bytes_read != size_t(len)) { + return IO_ERROR; + } + + return bytes_read; + } + + inline simdjson_result parser::load(const std::string& path) & noexcept { + return load_into_document(doc, path); + } + + inline simdjson_result parser::load_into_document(document& provided_doc, const std::string& path) & noexcept { + size_t len; + auto _error = read_file(path).get(len); + if (_error) { return _error; } + return parse_into_document(provided_doc, loaded_bytes.get(), len, false); + } + + inline simdjson_result parser::load_many(const std::string& path, size_t batch_size) noexcept { + size_t len; + auto _error = read_file(path).get(len); + if (_error) { return _error; } + if (batch_size < MINIMAL_BATCH_SIZE) { batch_size = MINIMAL_BATCH_SIZE; } + return document_stream(*this, reinterpret_cast(loaded_bytes.get()), len, batch_size); + } + + inline simdjson_result parser::parse_into_document(document& provided_doc, const uint8_t* buf, size_t len, bool realloc_if_needed) & noexcept { + // Important: we need to ensure that document has enough capacity. + // Important: It is possible that provided_doc is actually the internal 'doc' within the parser!!! + error_code _error = ensure_capacity(provided_doc, len); + if (_error) { return _error; } + if (realloc_if_needed) { + // Make sure we have enough capacity to copy len bytes + if (!loaded_bytes || _loaded_bytes_capacity < len) { + loaded_bytes.reset(internal::allocate_padded_buffer(len)); + if (!loaded_bytes) { + return MEMALLOC; + } + _loaded_bytes_capacity = len; + } + std::memcpy(static_cast(loaded_bytes.get()), buf, len); + buf = reinterpret_cast(loaded_bytes.get()); + } + + if ((len >= 3) && (std::memcmp(buf, "\xEF\xBB\xBF", 3) == 0)) { + buf += 3; + len -= 3; + } + _error = implementation->parse(buf, len, provided_doc); + + if (_error) { return _error; } + + return provided_doc.root(); + } + + simdjson_inline simdjson_result parser::parse_into_document(document& provided_doc, const char* buf, size_t len, bool realloc_if_needed) & noexcept { + return parse_into_document(provided_doc, reinterpret_cast(buf), len, realloc_if_needed); + } + simdjson_inline simdjson_result parser::parse_into_document(document& provided_doc, const std::string& s) & noexcept { + return parse_into_document(provided_doc, s.data(), s.length(), s.capacity() - s.length() < SIMDJSON_PADDING); + } + simdjson_inline simdjson_result parser::parse_into_document(document& provided_doc, const padded_string& s) & noexcept { + return parse_into_document(provided_doc, s.data(), s.length(), false); + } + + + inline simdjson_result parser::parse(const uint8_t* buf, size_t len, bool realloc_if_needed) & noexcept { + return parse_into_document(doc, buf, len, realloc_if_needed); + } + + simdjson_inline simdjson_result parser::parse(const char* buf, size_t len, bool realloc_if_needed) & noexcept { + return parse(reinterpret_cast(buf), len, realloc_if_needed); + } + simdjson_inline simdjson_result parser::parse(const std::string& s) & noexcept { + return parse(s.data(), s.length(), s.capacity() - s.length() < SIMDJSON_PADDING); + } + simdjson_inline simdjson_result parser::parse(const padded_string& s) & noexcept { + return parse(s.data(), s.length(), false); + } + simdjson_inline simdjson_result parser::parse(const padded_string_view& v) & noexcept { + return parse(v.data(), v.length(), false); + } + + inline simdjson_result parser::parse_many(const uint8_t* buf, size_t len, size_t batch_size) noexcept { + if (batch_size < MINIMAL_BATCH_SIZE) { batch_size = MINIMAL_BATCH_SIZE; } + if ((len >= 3) && (std::memcmp(buf, "\xEF\xBB\xBF", 3) == 0)) { + buf += 3; + len -= 3; + } + return document_stream(*this, buf, len, batch_size); + } + inline simdjson_result parser::parse_many(const char* buf, size_t len, size_t batch_size) noexcept { + return parse_many(reinterpret_cast(buf), len, batch_size); + } + inline simdjson_result parser::parse_many(const std::string& s, size_t batch_size) noexcept { + return parse_many(s.data(), s.length(), batch_size); + } + inline simdjson_result parser::parse_many(const padded_string& s, size_t batch_size) noexcept { + return parse_many(s.data(), s.length(), batch_size); + } + + simdjson_inline size_t parser::capacity() const noexcept { + return implementation ? implementation->capacity() : 0; + } + simdjson_inline size_t parser::max_capacity() const noexcept { + return _max_capacity; + } + simdjson_pure simdjson_inline size_t parser::max_depth() const noexcept { + return implementation ? implementation->max_depth() : DEFAULT_MAX_DEPTH; + } + + simdjson_warn_unused + inline error_code parser::allocate(size_t capacity, size_t max_depth) noexcept { + // + // Reallocate implementation if needed + // + error_code err; + if (implementation) { + err = implementation->allocate(capacity, max_depth); + } + else { + err = simdjson::get_active_implementation()->create_dom_parser_implementation(capacity, max_depth, implementation); + } + if (err) { return err; } + return SUCCESS; + } + +#ifndef SIMDJSON_DISABLE_DEPRECATED_API + simdjson_warn_unused + inline bool parser::allocate_capacity(size_t capacity, size_t max_depth) noexcept { + return !allocate(capacity, max_depth); + } +#endif // SIMDJSON_DISABLE_DEPRECATED_API + + inline error_code parser::ensure_capacity(size_t desired_capacity) noexcept { + return ensure_capacity(doc, desired_capacity); + } + + + inline error_code parser::ensure_capacity(document& target_document, size_t desired_capacity) noexcept { + // 1. It is wasteful to allocate a document and a parser for documents spanning less than MINIMAL_DOCUMENT_CAPACITY bytes. + // 2. If we allow desired_capacity = 0 then it is possible to exit this function with implementation == nullptr. + if (desired_capacity < MINIMAL_DOCUMENT_CAPACITY) { desired_capacity = MINIMAL_DOCUMENT_CAPACITY; } + // If we don't have enough capacity, (try to) automatically bump it. + // If the document needs allocation, do it too. + // Both in one if statement to minimize unlikely branching. + // + // Note: we must make sure that this function is called if capacity() == 0. We do so because we + // ensure that desired_capacity > 0. + if (simdjson_unlikely(capacity() < desired_capacity || target_document.capacity() < desired_capacity)) { + if (desired_capacity > max_capacity()) { + return error = CAPACITY; + } + error_code err1 = target_document.capacity() < desired_capacity ? target_document.allocate(desired_capacity) : SUCCESS; + error_code err2 = capacity() < desired_capacity ? allocate(desired_capacity, max_depth()) : SUCCESS; + if (err1 != SUCCESS) { return error = err1; } + if (err2 != SUCCESS) { return error = err2; } + } + return SUCCESS; + } + + simdjson_inline void parser::set_max_capacity(size_t max_capacity) noexcept { + if (max_capacity > MINIMAL_DOCUMENT_CAPACITY) { + _max_capacity = max_capacity; + } + else { + _max_capacity = MINIMAL_DOCUMENT_CAPACITY; + } + } + + } // namespace dom +} // namespace simdjson + +#endif // SIMDJSON_PARSER_INL_H +/* end file simdjson/dom/parser-inl.h */ /* skipped duplicate #include "simdjson/error-inl.h" */ /* skipped duplicate #include "simdjson/internal/dom_parser_implementation.h" */ @@ -7448,7 +8177,11 @@ namespace simdjson { } else { size_t next_doc_index = stream->batch_start + stream->parser->implementation->structural_indexes[stream->parser->implementation->next_structural_index]; - return std::string_view(reinterpret_cast(stream->buf) + current_index(), next_doc_index - current_index() - 1); + size_t svlen = next_doc_index - current_index(); + while (svlen > 1 && (std::isspace(start[svlen - 1]) || start[svlen - 1] == '\0')) { + svlen--; + } + return std::string_view(start, svlen); } } @@ -7570,922 +8303,6 @@ namespace simdjson { } // namespace simdjson #endif // SIMDJSON_DOCUMENT_STREAM_INL_H /* end file simdjson/dom/document_stream-inl.h */ -/* skipped duplicate #include "simdjson/dom/element-inl.h" */ - -#include -#include /* memcmp */ - -namespace simdjson { - namespace dom { - - // - // parser inline implementation - // - simdjson_inline parser::parser(size_t max_capacity) noexcept - : _max_capacity{ max_capacity }, - loaded_bytes(nullptr) { - } - simdjson_inline parser::parser(parser&& other) noexcept = default; - simdjson_inline parser& parser::operator=(parser&& other) noexcept = default; - - inline bool parser::is_valid() const noexcept { return valid; } - inline int parser::get_error_code() const noexcept { return error; } - inline std::string parser::get_error_message() const noexcept { return error_message(error); } - - inline bool parser::dump_raw_tape(std::ostream& os) const noexcept { - return valid ? doc.dump_raw_tape(os) : false; - } - - inline simdjson_result parser::read_file(const std::string& path) noexcept { - // Open the file - SIMDJSON_PUSH_DISABLE_WARNINGS - SIMDJSON_DISABLE_DEPRECATED_WARNING // Disable CRT_SECURE warning on MSVC: manually verified this is safe - std::FILE* fp = std::fopen(path.c_str(), "rb"); - SIMDJSON_POP_DISABLE_WARNINGS - - if (fp == nullptr) { - return IO_ERROR; - } - - // Get the file size - int ret; -#if SIMDJSON_VISUAL_STUDIO && !SIMDJSON_IS_32BITS - ret = _fseeki64(fp, 0, SEEK_END); -#else - ret = std::fseek(fp, 0, SEEK_END); -#endif // _WIN64 - if (ret < 0) { - std::fclose(fp); - return IO_ERROR; - } -#if SIMDJSON_VISUAL_STUDIO && !SIMDJSON_IS_32BITS - __int64 len = _ftelli64(fp); - if (len == -1L) { - std::fclose(fp); - return IO_ERROR; - } -#else - long len = std::ftell(fp); - if ((len < 0) || (len == LONG_MAX)) { - std::fclose(fp); - return IO_ERROR; - } -#endif - - // Make sure we have enough capacity to load the file - if (_loaded_bytes_capacity < size_t(len)) { - loaded_bytes.reset(internal::allocate_padded_buffer(len)); - if (!loaded_bytes) { - std::fclose(fp); - return MEMALLOC; - } - _loaded_bytes_capacity = len; - } - - // Read the string - std::rewind(fp); - size_t bytes_read = std::fread(loaded_bytes.get(), 1, len, fp); - if (std::fclose(fp) != 0 || bytes_read != size_t(len)) { - return IO_ERROR; - } - - return bytes_read; - } - - inline simdjson_result parser::load(const std::string& path) & noexcept { - size_t len; - auto _error = read_file(path).get(len); - if (_error) { return _error; } - return parse(loaded_bytes.get(), len, false); - } - - inline simdjson_result parser::load_many(const std::string& path, size_t batch_size) noexcept { - size_t len; - auto _error = read_file(path).get(len); - if (_error) { return _error; } - if (batch_size < MINIMAL_BATCH_SIZE) { batch_size = MINIMAL_BATCH_SIZE; } - return document_stream(*this, reinterpret_cast(loaded_bytes.get()), len, batch_size); - } - - inline simdjson_result parser::parse_into_document(document& provided_doc, const uint8_t* buf, size_t len, bool realloc_if_needed) & noexcept { - // Important: we need to ensure that document has enough capacity. - // Important: It is possible that provided_doc is actually the internal 'doc' within the parser!!! - error_code _error = ensure_capacity(provided_doc, len); - if (_error) { return _error; } - if (realloc_if_needed) { - // Make sure we have enough capacity to copy len bytes - if (!loaded_bytes || _loaded_bytes_capacity < len) { - loaded_bytes.reset(internal::allocate_padded_buffer(len)); - if (!loaded_bytes) { - return MEMALLOC; - } - _loaded_bytes_capacity = len; - } - std::memcpy(static_cast(loaded_bytes.get()), buf, len); - buf = reinterpret_cast(loaded_bytes.get()); - } - - if ((len >= 3) && (std::memcmp(buf, "\xEF\xBB\xBF", 3) == 0)) { - buf += 3; - len -= 3; - } - _error = implementation->parse(buf, len, provided_doc); - - if (_error) { return _error; } - - return provided_doc.root(); - } - - simdjson_inline simdjson_result parser::parse_into_document(document& provided_doc, const char* buf, size_t len, bool realloc_if_needed) & noexcept { - return parse_into_document(provided_doc, reinterpret_cast(buf), len, realloc_if_needed); - } - simdjson_inline simdjson_result parser::parse_into_document(document& provided_doc, const std::string& s) & noexcept { - return parse_into_document(provided_doc, s.data(), s.length(), s.capacity() - s.length() < SIMDJSON_PADDING); - } - simdjson_inline simdjson_result parser::parse_into_document(document& provided_doc, const padded_string& s) & noexcept { - return parse_into_document(provided_doc, s.data(), s.length(), false); - } - - - inline simdjson_result parser::parse(const uint8_t* buf, size_t len, bool realloc_if_needed) & noexcept { - return parse_into_document(doc, buf, len, realloc_if_needed); - } - - simdjson_inline simdjson_result parser::parse(const char* buf, size_t len, bool realloc_if_needed) & noexcept { - return parse(reinterpret_cast(buf), len, realloc_if_needed); - } - simdjson_inline simdjson_result parser::parse(const std::string& s) & noexcept { - return parse(s.data(), s.length(), s.capacity() - s.length() < SIMDJSON_PADDING); - } - simdjson_inline simdjson_result parser::parse(const padded_string& s) & noexcept { - return parse(s.data(), s.length(), false); - } - simdjson_inline simdjson_result parser::parse(const padded_string_view& v) & noexcept { - return parse(v.data(), v.length(), false); - } - - inline simdjson_result parser::parse_many(const uint8_t* buf, size_t len, size_t batch_size) noexcept { - if (batch_size < MINIMAL_BATCH_SIZE) { batch_size = MINIMAL_BATCH_SIZE; } - if ((len >= 3) && (std::memcmp(buf, "\xEF\xBB\xBF", 3) == 0)) { - buf += 3; - len -= 3; - } - return document_stream(*this, buf, len, batch_size); - } - inline simdjson_result parser::parse_many(const char* buf, size_t len, size_t batch_size) noexcept { - return parse_many(reinterpret_cast(buf), len, batch_size); - } - inline simdjson_result parser::parse_many(const std::string& s, size_t batch_size) noexcept { - return parse_many(s.data(), s.length(), batch_size); - } - inline simdjson_result parser::parse_many(const padded_string& s, size_t batch_size) noexcept { - return parse_many(s.data(), s.length(), batch_size); - } - - simdjson_inline size_t parser::capacity() const noexcept { - return implementation ? implementation->capacity() : 0; - } - simdjson_inline size_t parser::max_capacity() const noexcept { - return _max_capacity; - } - simdjson_inline size_t parser::max_depth() const noexcept { - return implementation ? implementation->max_depth() : DEFAULT_MAX_DEPTH; - } - - simdjson_warn_unused - inline error_code parser::allocate(size_t capacity, size_t max_depth) noexcept { - // - // Reallocate implementation if needed - // - error_code err; - if (implementation) { - err = implementation->allocate(capacity, max_depth); - } - else { - err = simdjson::get_active_implementation()->create_dom_parser_implementation(capacity, max_depth, implementation); - } - if (err) { return err; } - return SUCCESS; - } - -#ifndef SIMDJSON_DISABLE_DEPRECATED_API - simdjson_warn_unused - inline bool parser::allocate_capacity(size_t capacity, size_t max_depth) noexcept { - return !allocate(capacity, max_depth); - } -#endif // SIMDJSON_DISABLE_DEPRECATED_API - - inline error_code parser::ensure_capacity(size_t desired_capacity) noexcept { - return ensure_capacity(doc, desired_capacity); - } - - - inline error_code parser::ensure_capacity(document& target_document, size_t desired_capacity) noexcept { - // 1. It is wasteful to allocate a document and a parser for documents spanning less than MINIMAL_DOCUMENT_CAPACITY bytes. - // 2. If we allow desired_capacity = 0 then it is possible to exit this function with implementation == nullptr. - if (desired_capacity < MINIMAL_DOCUMENT_CAPACITY) { desired_capacity = MINIMAL_DOCUMENT_CAPACITY; } - // If we don't have enough capacity, (try to) automatically bump it. - // If the document needs allocation, do it too. - // Both in one if statement to minimize unlikely branching. - // - // Note: we must make sure that this function is called if capacity() == 0. We do so because we - // ensure that desired_capacity > 0. - if (simdjson_unlikely(capacity() < desired_capacity || target_document.capacity() < desired_capacity)) { - if (desired_capacity > max_capacity()) { - return error = CAPACITY; - } - error_code err1 = target_document.capacity() < desired_capacity ? target_document.allocate(desired_capacity) : SUCCESS; - error_code err2 = capacity() < desired_capacity ? allocate(desired_capacity, max_depth()) : SUCCESS; - if (err1 != SUCCESS) { return error = err1; } - if (err2 != SUCCESS) { return error = err2; } - } - return SUCCESS; - } - - simdjson_inline void parser::set_max_capacity(size_t max_capacity) noexcept { - if (max_capacity > MINIMAL_DOCUMENT_CAPACITY) { - _max_capacity = max_capacity; - } - else { - _max_capacity = MINIMAL_DOCUMENT_CAPACITY; - } - } - - } // namespace dom -} // namespace simdjson - -#endif // SIMDJSON_PARSER_INL_H -/* end file simdjson/dom/parser-inl.h */ - -namespace simdjson { - - // - // C API (json_parse and build_parsed_json) declarations - // - -#ifndef SIMDJSON_DISABLE_DEPRECATED_API - [[deprecated("Use parser.parse() instead")]] - inline int json_parse(const uint8_t* buf, size_t len, dom::parser& parser, bool realloc_if_needed = true) noexcept { - error_code code = parser.parse(buf, len, realloc_if_needed).error(); - // The deprecated json_parse API is a signal that the user plans to *use* the error code / valid - // bits in the parser instead of heeding the result code. The normal parser unsets those in - // anticipation of making the error code ephemeral. - // Here we put the code back into the parser, until we've removed this method. - parser.valid = code == SUCCESS; - parser.error = code; - return code; - } - [[deprecated("Use parser.parse() instead")]] - inline int json_parse(const char* buf, size_t len, dom::parser& parser, bool realloc_if_needed = true) noexcept { - error_code code = parser.parse(buf, len, realloc_if_needed).error(); - // The deprecated json_parse API is a signal that the user plans to *use* the error code / valid - // bits in the parser instead of heeding the result code. The normal parser unsets those in - // anticipation of making the error code ephemeral. - // Here we put the code back into the parser, until we've removed this method. - parser.valid = code == SUCCESS; - parser.error = code; - return code; - } - [[deprecated("Use parser.parse() instead")]] - inline int json_parse(const std::string& s, dom::parser& parser, bool realloc_if_needed = true) noexcept { - error_code code = parser.parse(s.data(), s.length(), realloc_if_needed).error(); - // The deprecated json_parse API is a signal that the user plans to *use* the error code / valid - // bits in the parser instead of heeding the result code. The normal parser unsets those in - // anticipation of making the error code ephemeral. - // Here we put the code back into the parser, until we've removed this method. - parser.valid = code == SUCCESS; - parser.error = code; - return code; - } - [[deprecated("Use parser.parse() instead")]] - inline int json_parse(const padded_string& s, dom::parser& parser) noexcept { - error_code code = parser.parse(s).error(); - // The deprecated json_parse API is a signal that the user plans to *use* the error code / valid - // bits in the parser instead of heeding the result code. The normal parser unsets those in - // anticipation of making the error code ephemeral. - // Here we put the code back into the parser, until we've removed this method. - parser.valid = code == SUCCESS; - parser.error = code; - return code; - } - - [[deprecated("Use parser.parse() instead")]] - simdjson_warn_unused inline dom::parser build_parsed_json(const uint8_t* buf, size_t len, bool realloc_if_needed = true) noexcept { - dom::parser parser; - error_code code = parser.parse(buf, len, realloc_if_needed).error(); - // The deprecated json_parse API is a signal that the user plans to *use* the error code / valid - // bits in the parser instead of heeding the result code. The normal parser unsets those in - // anticipation of making the error code ephemeral. - // Here we put the code back into the parser, until we've removed this method. - parser.valid = code == SUCCESS; - parser.error = code; - return parser; - } - [[deprecated("Use parser.parse() instead")]] - simdjson_warn_unused inline dom::parser build_parsed_json(const char* buf, size_t len, bool realloc_if_needed = true) noexcept { - dom::parser parser; - error_code code = parser.parse(buf, len, realloc_if_needed).error(); - // The deprecated json_parse API is a signal that the user plans to *use* the error code / valid - // bits in the parser instead of heeding the result code. The normal parser unsets those in - // anticipation of making the error code ephemeral. - // Here we put the code back into the parser, until we've removed this method. - parser.valid = code == SUCCESS; - parser.error = code; - return parser; - } - [[deprecated("Use parser.parse() instead")]] - simdjson_warn_unused inline dom::parser build_parsed_json(const std::string& s, bool realloc_if_needed = true) noexcept { - dom::parser parser; - error_code code = parser.parse(s.data(), s.length(), realloc_if_needed).error(); - // The deprecated json_parse API is a signal that the user plans to *use* the error code / valid - // bits in the parser instead of heeding the result code. The normal parser unsets those in - // anticipation of making the error code ephemeral. - // Here we put the code back into the parser, until we've removed this method. - parser.valid = code == SUCCESS; - parser.error = code; - return parser; - } - [[deprecated("Use parser.parse() instead")]] - simdjson_warn_unused inline dom::parser build_parsed_json(const padded_string& s) noexcept { - dom::parser parser; - error_code code = parser.parse(s).error(); - // The deprecated json_parse API is a signal that the user plans to *use* the error code / valid - // bits in the parser instead of heeding the result code. The normal parser unsets those in - // anticipation of making the error code ephemeral. - // Here we put the code back into the parser, until we've removed this method. - parser.valid = code == SUCCESS; - parser.error = code; - return parser; - } -#endif // SIMDJSON_DISABLE_DEPRECATED_API - - /** @private We do not want to allow implicit conversion from C string to std::string. */ - int json_parse(const char* buf, dom::parser& parser) noexcept = delete; - /** @private We do not want to allow implicit conversion from C string to std::string. */ - dom::parser build_parsed_json(const char* buf) noexcept = delete; - -} // namespace simdjson - -#endif // SIMDJSON_DOM_JSONPARSER_H -/* end file simdjson/dom/jsonparser.h */ -/* including simdjson/dom/parsedjson.h: #include "simdjson/dom/parsedjson.h" */ -/* begin file simdjson/dom/parsedjson.h */ -// TODO Remove this -- deprecated API and files - -#ifndef SIMDJSON_DOM_PARSEDJSON_H -#define SIMDJSON_DOM_PARSEDJSON_H - -/* skipped duplicate #include "simdjson/dom/base.h" */ - -namespace simdjson { - - /** - * @deprecated Use `dom::parser` instead. - */ - using ParsedJson [[deprecated("Use dom::parser instead")]] = dom::parser; - -} // namespace simdjson - -#endif // SIMDJSON_DOM_PARSEDJSON_H -/* end file simdjson/dom/parsedjson.h */ -/* including simdjson/dom/parsedjson_iterator.h: #include "simdjson/dom/parsedjson_iterator.h" */ -/* begin file simdjson/dom/parsedjson_iterator.h */ -// TODO Remove this -- deprecated API and files - -#ifndef SIMDJSON_DOM_PARSEDJSON_ITERATOR_H -#define SIMDJSON_DOM_PARSEDJSON_ITERATOR_H - -/* skipped duplicate #include "simdjson/dom/base.h" */ -/* skipped duplicate #include "simdjson/dom/parser.h" */ - -#ifndef SIMDJSON_DISABLE_DEPRECATED_API - -namespace simdjson { - /** @private **/ - class [[deprecated("Use the new DOM navigation API instead (see doc/basics.md)")]] dom::parser::Iterator { - public: - inline Iterator(const dom::parser& parser) noexcept(false); - inline Iterator(const Iterator& o) noexcept; - inline ~Iterator() noexcept; - - inline Iterator& operator=(const Iterator&) = delete; - - inline bool is_ok() const; - - // useful for debugging purposes - inline size_t get_tape_location() const; - - // useful for debugging purposes - inline size_t get_tape_length() const; - - // returns the current depth (start at 1 with 0 reserved for the fictitious - // root node) - inline size_t get_depth() const; - - // A scope is a series of nodes at the same depth, typically it is either an - // object ({) or an array ([). The root node has type 'r'. - inline uint8_t get_scope_type() const; - - // move forward in document order - inline bool move_forward(); - - // retrieve the character code of what we're looking at: - // [{"slutfn are the possibilities - inline uint8_t get_type() const { - return current_type; // short functions should be inlined! - } - - // get the int64_t value at this node; valid only if get_type is "l" - inline int64_t get_integer() const; - - // get the value as uint64; valid only if if get_type is "u" - inline uint64_t get_unsigned_integer() const; - - // get the string value at this node (NULL ended); valid only if get_type is " - // note that tabs, and line endings are escaped in the returned value (see - // print_with_escapes) return value is valid UTF-8, it may contain NULL chars - // within the string: get_string_length determines the true string length. - inline const char* get_string() const; - - // return the length of the string in bytes - inline uint32_t get_string_length() const; - - // get the double value at this node; valid only if - // get_type() is "d" - inline double get_double() const; - - inline bool is_object_or_array() const { return is_object() || is_array(); } - - inline bool is_object() const { return get_type() == '{'; } - - inline bool is_array() const { return get_type() == '['; } - - inline bool is_string() const { return get_type() == '"'; } - - // Returns true if the current type of the node is an signed integer. - // You can get its value with `get_integer()`. - inline bool is_integer() const { return get_type() == 'l'; } - - // Returns true if the current type of the node is an unsigned integer. - // You can get its value with `get_unsigned_integer()`. - // - // NOTE: - // Only a large value, which is out of range of a 64-bit signed integer, is - // represented internally as an unsigned node. On the other hand, a typical - // positive integer, such as 1, 42, or 1000000, is as a signed node. - // Be aware this function returns false for a signed node. - inline bool is_unsigned_integer() const { return get_type() == 'u'; } - // Returns true if the current type of the node is a double floating-point number. - inline bool is_double() const { return get_type() == 'd'; } - // Returns true if the current type of the node is a number (integer or floating-point). - inline bool is_number() const { - return is_integer() || is_unsigned_integer() || is_double(); - } - // Returns true if the current type of the node is a bool with true value. - inline bool is_true() const { return get_type() == 't'; } - // Returns true if the current type of the node is a bool with false value. - inline bool is_false() const { return get_type() == 'f'; } - // Returns true if the current type of the node is null. - inline bool is_null() const { return get_type() == 'n'; } - // Returns true if the type byte represents an object of an array - static bool is_object_or_array(uint8_t type) { - return ((type == '[') || (type == '{')); - } - - // when at {, go one level deep, looking for a given key - // if successful, we are left pointing at the value, - // if not, we are still pointing at the object ({) - // (in case of repeated keys, this only finds the first one). - // We seek the key using C's strcmp so if your JSON strings contain - // NULL chars, this would trigger a false positive: if you expect that - // to be the case, take extra precautions. - // Furthermore, we do the comparison character-by-character - // without taking into account Unicode equivalence. - inline bool move_to_key(const char* key); - - // as above, but case insensitive lookup (strcmpi instead of strcmp) - inline bool move_to_key_insensitive(const char* key); - - // when at {, go one level deep, looking for a given key - // if successful, we are left pointing at the value, - // if not, we are still pointing at the object ({) - // (in case of repeated keys, this only finds the first one). - // The string we search for can contain NULL values. - // Furthermore, we do the comparison character-by-character - // without taking into account Unicode equivalence. - inline bool move_to_key(const char* key, uint32_t length); - - // when at a key location within an object, this moves to the accompanying - // value (located next to it). This is equivalent but much faster than - // calling "next()". - inline void move_to_value(); - - // when at [, go one level deep, and advance to the given index. - // if successful, we are left pointing at the value, - // if not, we are still pointing at the array ([) - inline bool move_to_index(uint32_t index); - - // Moves the iterator to the value corresponding to the json pointer. - // Always search from the root of the document. - // if successful, we are left pointing at the value, - // if not, we are still pointing the same value we were pointing before the - // call. The json pointer follows the rfc6901 standard's syntax: - // https://tools.ietf.org/html/rfc6901 However, the standard says "If a - // referenced member name is not unique in an object, the member that is - // referenced is undefined, and evaluation fails". Here we just return the - // first corresponding value. The length parameter is the length of the - // jsonpointer string ('pointer'). - inline bool move_to(const char* pointer, uint32_t length); - - // Moves the iterator to the value corresponding to the json pointer. - // Always search from the root of the document. - // if successful, we are left pointing at the value, - // if not, we are still pointing the same value we were pointing before the - // call. The json pointer implementation follows the rfc6901 standard's - // syntax: https://tools.ietf.org/html/rfc6901 However, the standard says - // "If a referenced member name is not unique in an object, the member that - // is referenced is undefined, and evaluation fails". Here we just return - // the first corresponding value. - inline bool move_to(const std::string& pointer); - - private: - // Almost the same as move_to(), except it searches from the current - // position. The pointer's syntax is identical, though that case is not - // handled by the rfc6901 standard. The '/' is still required at the - // beginning. However, contrary to move_to(), the URI Fragment Identifier - // Representation is not supported here. Also, in case of failure, we are - // left pointing at the closest value it could reach. For these reasons it - // is private. It exists because it is used by move_to(). - inline bool relative_move_to(const char* pointer, uint32_t length); - - public: - // throughout return true if we can do the navigation, false - // otherwise - - // Within a given scope (series of nodes at the same depth within either an - // array or an object), we move forward. - // Thus, given [true, null, {"a":1}, [1,2]], we would visit true, null, { - // and [. At the object ({) or at the array ([), you can issue a "down" to - // visit their content. valid if we're not at the end of a scope (returns - // true). - inline bool next(); - - // Within a given scope (series of nodes at the same depth within either an - // array or an object), we move backward. - // Thus, given [true, null, {"a":1}, [1,2]], we would visit ], }, null, true - // when starting at the end of the scope. At the object ({) or at the array - // ([), you can issue a "down" to visit their content. - // Performance warning: This function is implemented by starting again - // from the beginning of the scope and scanning forward. You should expect - // it to be relatively slow. - inline bool prev(); - - // Moves back to either the containing array or object (type { or [) from - // within a contained scope. - // Valid unless we are at the first level of the document - inline bool up(); - - // Valid if we're at a [ or { and it starts a non-empty scope; moves us to - // start of that deeper scope if it not empty. Thus, given [true, null, - // {"a":1}, [1,2]], if we are at the { node, we would move to the "a" node. - inline bool down(); - - // move us to the start of our current scope, - // a scope is a series of nodes at the same level - inline void to_start_scope(); - - inline void rewind(); - - - - // print the node we are currently pointing at - inline bool print(std::ostream& os, bool escape_strings = true) const; - - private: - const document& doc; - size_t max_depth{}; - size_t depth{}; - size_t location{}; // our current location on a tape - size_t tape_length{}; - uint8_t current_type{}; - uint64_t current_val{}; - typedef struct { - size_t start_of_scope; - uint8_t scope_type; - } scopeindex_t; - - scopeindex_t* depth_index{}; - }; - -} // namespace simdjson -#endif // SIMDJSON_DISABLE_DEPRECATED_API - -#endif // SIMDJSON_DOM_PARSEDJSON_ITERATOR_H -/* end file simdjson/dom/parsedjson_iterator.h */ - -// Inline functions -/* including simdjson/dom/array-inl.h: #include "simdjson/dom/array-inl.h" */ -/* begin file simdjson/dom/array-inl.h */ -#ifndef SIMDJSON_ARRAY_INL_H -#define SIMDJSON_ARRAY_INL_H - -#include - -/* skipped duplicate #include "simdjson/dom/base.h" */ -/* skipped duplicate #include "simdjson/dom/array.h" */ -/* skipped duplicate #include "simdjson/dom/element.h" */ -/* skipped duplicate #include "simdjson/error-inl.h" */ -/* including simdjson/internal/tape_ref-inl.h: #include "simdjson/internal/tape_ref-inl.h" */ -/* begin file simdjson/internal/tape_ref-inl.h */ -#ifndef SIMDJSON_TAPE_REF_INL_H -#define SIMDJSON_TAPE_REF_INL_H - -/* skipped duplicate #include "simdjson/dom/document.h" */ -/* skipped duplicate #include "simdjson/internal/tape_ref.h" */ -/* skipped duplicate #include "simdjson/internal/tape_type.h" */ - -#include - -namespace simdjson { - namespace internal { - - constexpr const uint64_t JSON_VALUE_MASK = 0x00FFFFFFFFFFFFFF; - constexpr const uint32_t JSON_COUNT_MASK = 0xFFFFFF; - - // - // tape_ref inline implementation - // - simdjson_inline tape_ref::tape_ref() noexcept : doc{ nullptr }, json_index{ 0 } {} - simdjson_inline tape_ref::tape_ref(const dom::document* _doc, size_t _json_index) noexcept : doc{ _doc }, json_index{ _json_index } {} - - - simdjson_inline bool tape_ref::is_document_root() const noexcept { - return json_index == 1; // should we ever change the structure of the tape, this should get updated. - } - simdjson_inline bool tape_ref::usable() const noexcept { - return doc != nullptr; // when the document pointer is null, this tape_ref is uninitialized (should not be accessed). - } - // Some value types have a specific on-tape word value. It can be faster - // to check the type by doing a word-to-word comparison instead of extracting the - // most significant 8 bits. - - simdjson_inline bool tape_ref::is_double() const noexcept { - constexpr uint64_t tape_double = uint64_t(tape_type::DOUBLE) << 56; - return doc->tape[json_index] == tape_double; - } - simdjson_inline bool tape_ref::is_int64() const noexcept { - constexpr uint64_t tape_int64 = uint64_t(tape_type::INT64) << 56; - return doc->tape[json_index] == tape_int64; - } - simdjson_inline bool tape_ref::is_uint64() const noexcept { - constexpr uint64_t tape_uint64 = uint64_t(tape_type::UINT64) << 56; - return doc->tape[json_index] == tape_uint64; - } - simdjson_inline bool tape_ref::is_false() const noexcept { - constexpr uint64_t tape_false = uint64_t(tape_type::FALSE_VALUE) << 56; - return doc->tape[json_index] == tape_false; - } - simdjson_inline bool tape_ref::is_true() const noexcept { - constexpr uint64_t tape_true = uint64_t(tape_type::TRUE_VALUE) << 56; - return doc->tape[json_index] == tape_true; - } - simdjson_inline bool tape_ref::is_null_on_tape() const noexcept { - constexpr uint64_t tape_null = uint64_t(tape_type::NULL_VALUE) << 56; - return doc->tape[json_index] == tape_null; - } - - inline size_t tape_ref::after_element() const noexcept { - switch (tape_ref_type()) { - case tape_type::START_ARRAY: - case tape_type::START_OBJECT: - return matching_brace_index(); - case tape_type::UINT64: - case tape_type::INT64: - case tape_type::DOUBLE: - return json_index + 2; - default: - return json_index + 1; - } - } - simdjson_inline tape_type tape_ref::tape_ref_type() const noexcept { - return static_cast(doc->tape[json_index] >> 56); - } - simdjson_inline uint64_t internal::tape_ref::tape_value() const noexcept { - return doc->tape[json_index] & internal::JSON_VALUE_MASK; - } - simdjson_inline uint32_t internal::tape_ref::matching_brace_index() const noexcept { - return uint32_t(doc->tape[json_index]); - } - simdjson_inline uint32_t internal::tape_ref::scope_count() const noexcept { - return uint32_t((doc->tape[json_index] >> 32) & internal::JSON_COUNT_MASK); - } - - template - simdjson_inline T tape_ref::next_tape_value() const noexcept { - static_assert(sizeof(T) == sizeof(uint64_t), "next_tape_value() template parameter must be 64-bit"); - // Though the following is tempting... - // return *reinterpret_cast(&doc->tape[json_index + 1]); - // It is not generally safe. It is safer, and often faster to rely - // on memcpy. Yes, it is uglier, but it is also encapsulated. - T x; - std::memcpy(&x, &doc->tape[json_index + 1], sizeof(uint64_t)); - return x; - } - - simdjson_inline uint32_t internal::tape_ref::get_string_length() const noexcept { - size_t string_buf_index = size_t(tape_value()); - uint32_t len; - std::memcpy(&len, &doc->string_buf[string_buf_index], sizeof(len)); - return len; - } - - simdjson_inline const char* internal::tape_ref::get_c_str() const noexcept { - size_t string_buf_index = size_t(tape_value()); - return reinterpret_cast(&doc->string_buf[string_buf_index + sizeof(uint32_t)]); - } - - inline std::string_view internal::tape_ref::get_string_view() const noexcept { - return std::string_view( - get_c_str(), - get_string_length() - ); - } - - } // namespace internal -} // namespace simdjson - -#endif // SIMDJSON_TAPE_REF_INL_H -/* end file simdjson/internal/tape_ref-inl.h */ - -#include - -namespace simdjson { - - // - // simdjson_result inline implementation - // - simdjson_inline simdjson_result::simdjson_result() noexcept - : internal::simdjson_result_base() {} - simdjson_inline simdjson_result::simdjson_result(dom::array value) noexcept - : internal::simdjson_result_base(std::forward(value)) {} - simdjson_inline simdjson_result::simdjson_result(error_code error) noexcept - : internal::simdjson_result_base(error) {} - -#if SIMDJSON_EXCEPTIONS - - inline dom::array::iterator simdjson_result::begin() const noexcept(false) { - if (error()) { throw simdjson_error(error()); } - return first.begin(); - } - inline dom::array::iterator simdjson_result::end() const noexcept(false) { - if (error()) { throw simdjson_error(error()); } - return first.end(); - } - inline size_t simdjson_result::size() const noexcept(false) { - if (error()) { throw simdjson_error(error()); } - return first.size(); - } - -#endif // SIMDJSON_EXCEPTIONS - - inline simdjson_result simdjson_result::at_pointer(std::string_view json_pointer) const noexcept { - if (error()) { return error(); } - return first.at_pointer(json_pointer); - } - inline simdjson_result simdjson_result::at(size_t index) const noexcept { - if (error()) { return error(); } - return first.at(index); - } - - namespace dom { - - // - // array inline implementation - // - simdjson_inline array::array() noexcept : tape{} {} - simdjson_inline array::array(const internal::tape_ref& _tape) noexcept : tape{ _tape } {} - inline array::iterator array::begin() const noexcept { - SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 - return internal::tape_ref(tape.doc, tape.json_index + 1); - } - inline array::iterator array::end() const noexcept { - SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 - return internal::tape_ref(tape.doc, tape.after_element() - 1); - } - inline size_t array::size() const noexcept { - SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 - return tape.scope_count(); - } - inline size_t array::number_of_slots() const noexcept { - SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 - return tape.matching_brace_index() - tape.json_index; - } - inline simdjson_result array::at_pointer(std::string_view json_pointer) const noexcept { - SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 - if (json_pointer.empty()) { // an empty string means that we return the current node - return element(this->tape); // copy the current node - } - else if (json_pointer[0] != '/') { // otherwise there is an error - return INVALID_JSON_POINTER; - } - json_pointer = json_pointer.substr(1); - // - means "the append position" or "the element after the end of the array" - // We don't support this, because we're returning a real element, not a position. - if (json_pointer == "-") { return INDEX_OUT_OF_BOUNDS; } - - // Read the array index - size_t array_index = 0; - size_t i; - for (i = 0; i < json_pointer.length() && json_pointer[i] != '/'; i++) { - uint8_t digit = uint8_t(json_pointer[i] - '0'); - // Check for non-digit in array index. If it's there, we're trying to get a field in an object - if (digit > 9) { return INCORRECT_TYPE; } - array_index = array_index * 10 + digit; - } - - // 0 followed by other digits is invalid - if (i > 1 && json_pointer[0] == '0') { return INVALID_JSON_POINTER; } // "JSON pointer array index has other characters after 0" - - // Empty string is invalid; so is a "/" with no digits before it - if (i == 0) { return INVALID_JSON_POINTER; } // "Empty string in JSON pointer array index" - - // Get the child - auto child = array(tape).at(array_index); - // If there is an error, it ends here - if (child.error()) { - return child; - } - // If there is a /, we're not done yet, call recursively. - if (i < json_pointer.length()) { - child = child.at_pointer(json_pointer.substr(i)); - } - return child; - } - - inline simdjson_result array::at(size_t index) const noexcept { - SIMDJSON_DEVELOPMENT_ASSERT(tape.usable()); // https://github.com/simdjson/simdjson/issues/1914 - size_t i = 0; - for (auto element : *this) { - if (i == index) { return element; } - i++; - } - return INDEX_OUT_OF_BOUNDS; - } - - // - // array::iterator inline implementation - // - simdjson_inline array::iterator::iterator(const internal::tape_ref& _tape) noexcept : tape{ _tape } { } - inline element array::iterator::operator*() const noexcept { - return element(tape); - } - inline array::iterator& array::iterator::operator++() noexcept { - tape.json_index = tape.after_element(); - return *this; - } - inline array::iterator array::iterator::operator++(int) noexcept { - array::iterator out = *this; - ++*this; - return out; - } - inline bool array::iterator::operator!=(const array::iterator& other) const noexcept { - return tape.json_index != other.tape.json_index; - } - inline bool array::iterator::operator==(const array::iterator& other) const noexcept { - return tape.json_index == other.tape.json_index; - } - inline bool array::iterator::operator<(const array::iterator& other) const noexcept { - return tape.json_index < other.tape.json_index; - } - inline bool array::iterator::operator<=(const array::iterator& other) const noexcept { - return tape.json_index <= other.tape.json_index; - } - inline bool array::iterator::operator>=(const array::iterator& other) const noexcept { - return tape.json_index >= other.tape.json_index; - } - inline bool array::iterator::operator>(const array::iterator& other) const noexcept { - return tape.json_index > other.tape.json_index; - } - - } // namespace dom - - -} // namespace simdjson - -/* skipped duplicate #include "simdjson/dom/element-inl.h" */ - -#if defined(__cpp_lib_ranges) -static_assert(std::ranges::view); -static_assert(std::ranges::sized_range); -#if SIMDJSON_EXCEPTIONS -static_assert(std::ranges::view>); -static_assert(std::ranges::sized_range>); -#endif // SIMDJSON_EXCEPTIONS -#endif // defined(__cpp_lib_ranges) - -#endif // SIMDJSON_ARRAY_INL_H -/* end file simdjson/dom/array-inl.h */ -/* skipped duplicate #include "simdjson/dom/document_stream-inl.h" */ /* including simdjson/dom/document-inl.h: #include "simdjson/dom/document-inl.h" */ /* begin file simdjson/dom/document-inl.h */ #ifndef SIMDJSON_DOCUMENT_INL_H @@ -8718,564 +8535,6 @@ namespace simdjson { /* end file simdjson/dom/document-inl.h */ /* skipped duplicate #include "simdjson/dom/element-inl.h" */ /* skipped duplicate #include "simdjson/dom/object-inl.h" */ -/* including simdjson/dom/parsedjson_iterator-inl.h: #include "simdjson/dom/parsedjson_iterator-inl.h" */ -/* begin file simdjson/dom/parsedjson_iterator-inl.h */ -#ifndef SIMDJSON_PARSEDJSON_ITERATOR_INL_H -#define SIMDJSON_PARSEDJSON_ITERATOR_INL_H - -/* skipped duplicate #include "simdjson/dom/base.h" */ -/* skipped duplicate #include "simdjson/dom/parsedjson_iterator.h" */ -/* skipped duplicate #include "simdjson/internal/jsonformatutils.h" */ - -/* skipped duplicate #include "simdjson/dom/parser-inl.h" */ -/* skipped duplicate #include "simdjson/internal/tape_ref-inl.h" */ - -#include -#include -#include -#include - -#ifndef SIMDJSON_DISABLE_DEPRECATED_API - -namespace simdjson { - - // VS2017 reports deprecated warnings when you define a deprecated class's methods. - SIMDJSON_PUSH_DISABLE_WARNINGS - SIMDJSON_DISABLE_DEPRECATED_WARNING - - // Because of template weirdness, the actual class definition is inline in the document class - simdjson_warn_unused bool dom::parser::Iterator::is_ok() const { - return location < tape_length; - } - - // useful for debugging purposes - size_t dom::parser::Iterator::get_tape_location() const { - return location; - } - - // useful for debugging purposes - size_t dom::parser::Iterator::get_tape_length() const { - return tape_length; - } - - // returns the current depth (start at 1 with 0 reserved for the fictitious root - // node) - size_t dom::parser::Iterator::get_depth() const { - return depth; - } - - // A scope is a series of nodes at the same depth, typically it is either an - // object ({) or an array ([). The root node has type 'r'. - uint8_t dom::parser::Iterator::get_scope_type() const { - return depth_index[depth].scope_type; - } - - bool dom::parser::Iterator::move_forward() { - if (location + 1 >= tape_length) { - return false; // we are at the end! - } - - if ((current_type == '[') || (current_type == '{')) { - // We are entering a new scope - depth++; - assert(depth < max_depth); - depth_index[depth].start_of_scope = location; - depth_index[depth].scope_type = current_type; - } - else if ((current_type == ']') || (current_type == '}')) { - // Leaving a scope. - depth--; - } - else if (is_number()) { - // these types use 2 locations on the tape, not just one. - location += 1; - } - - location += 1; - current_val = doc.tape[location]; - current_type = uint8_t(current_val >> 56); - return true; - } - - void dom::parser::Iterator::move_to_value() { - // assume that we are on a key, so move by 1. - location += 1; - current_val = doc.tape[location]; - current_type = uint8_t(current_val >> 56); - } - - bool dom::parser::Iterator::move_to_key(const char* key) { - if (down()) { - do { - const bool right_key = (strcmp(get_string(), key) == 0); - move_to_value(); - if (right_key) { - return true; - } - } while (next()); - up(); - } - return false; - } - - bool dom::parser::Iterator::move_to_key_insensitive( - const char* key) { - if (down()) { - do { - const bool right_key = (simdjson_strcasecmp(get_string(), key) == 0); - move_to_value(); - if (right_key) { - return true; - } - } while (next()); - up(); - } - return false; - } - - bool dom::parser::Iterator::move_to_key(const char* key, - uint32_t length) { - if (down()) { - do { - bool right_key = ((get_string_length() == length) && - (memcmp(get_string(), key, length) == 0)); - move_to_value(); - if (right_key) { - return true; - } - } while (next()); - up(); - } - return false; - } - - bool dom::parser::Iterator::move_to_index(uint32_t index) { - if (down()) { - uint32_t i = 0; - for (; i < index; i++) { - if (!next()) { - break; - } - } - if (i == index) { - return true; - } - up(); - } - return false; - } - - bool dom::parser::Iterator::prev() { - size_t target_location = location; - to_start_scope(); - size_t npos = location; - if (target_location == npos) { - return false; // we were already at the start - } - size_t oldnpos; - // we have that npos < target_location here - do { - oldnpos = npos; - if ((current_type == '[') || (current_type == '{')) { - // we need to jump - npos = uint32_t(current_val); - } - else { - npos = npos + ((current_type == 'd' || current_type == 'l') ? 2 : 1); - } - } while (npos < target_location); - location = oldnpos; - current_val = doc.tape[location]; - current_type = uint8_t(current_val >> 56); - return true; - } - - bool dom::parser::Iterator::up() { - if (depth == 1) { - return false; // don't allow moving back to root - } - to_start_scope(); - // next we just move to the previous value - depth--; - location -= 1; - current_val = doc.tape[location]; - current_type = uint8_t(current_val >> 56); - return true; - } - - bool dom::parser::Iterator::down() { - if (location + 1 >= tape_length) { - return false; - } - if ((current_type == '[') || (current_type == '{')) { - size_t npos = uint32_t(current_val); - if (npos == location + 2) { - return false; // we have an empty scope - } - depth++; - assert(depth < max_depth); - location = location + 1; - depth_index[depth].start_of_scope = location; - depth_index[depth].scope_type = current_type; - current_val = doc.tape[location]; - current_type = uint8_t(current_val >> 56); - return true; - } - return false; - } - - void dom::parser::Iterator::to_start_scope() { - location = depth_index[depth].start_of_scope; - current_val = doc.tape[location]; - current_type = uint8_t(current_val >> 56); - } - - inline void dom::parser::Iterator::rewind() { - while (up()) - ; - } - - - bool dom::parser::Iterator::next() { - size_t npos; - if ((current_type == '[') || (current_type == '{')) { - // we need to jump - npos = uint32_t(current_val); - } - else { - npos = location + (is_number() ? 2 : 1); - } - uint64_t next_val = doc.tape[npos]; - uint8_t next_type = uint8_t(next_val >> 56); - if ((next_type == ']') || (next_type == '}')) { - return false; // we reached the end of the scope - } - location = npos; - current_val = next_val; - current_type = next_type; - return true; - } - dom::parser::Iterator::Iterator(const dom::parser& pj) noexcept(false) - : doc(pj.doc) - { -#if SIMDJSON_EXCEPTIONS - if (!pj.valid) { throw simdjson_error(pj.error); } -#else - if (!pj.valid) { return; } // abort() usage is forbidden in the library -#endif - - max_depth = pj.max_depth(); - depth_index = new scopeindex_t[max_depth + 1]; - depth_index[0].start_of_scope = location; - current_val = doc.tape[location++]; - current_type = uint8_t(current_val >> 56); - depth_index[0].scope_type = current_type; - tape_length = size_t(current_val & internal::JSON_VALUE_MASK); - if (location < tape_length) { - // If we make it here, then depth_capacity must >=2, but the compiler - // may not know this. - current_val = doc.tape[location]; - current_type = uint8_t(current_val >> 56); - depth++; - assert(depth < max_depth); - depth_index[depth].start_of_scope = location; - depth_index[depth].scope_type = current_type; - } - } - dom::parser::Iterator::Iterator( - const dom::parser::Iterator& o) noexcept - : doc(o.doc), - max_depth(o.depth), - depth(o.depth), - location(o.location), - tape_length(o.tape_length), - current_type(o.current_type), - current_val(o.current_val) - { - depth_index = new scopeindex_t[max_depth + 1]; - std::memcpy(depth_index, o.depth_index, (depth + 1) * sizeof(depth_index[0])); - } - - dom::parser::Iterator::~Iterator() noexcept { - if (depth_index) { delete[] depth_index; } - } - - bool dom::parser::Iterator::print(std::ostream& os, bool escape_strings) const { - if (!is_ok()) { - return false; - } - switch (current_type) { - case '"': // we have a string - os << '"'; - if (escape_strings) { - os << internal::escape_json_string(std::string_view(get_string(), get_string_length())); - } - else { - // was: os << get_string();, but given that we can include null chars, we - // have to do something crazier: - std::copy(get_string(), get_string() + get_string_length(), std::ostream_iterator(os)); - } - os << '"'; - break; - case 'l': // we have a long int - os << get_integer(); - break; - case 'u': - os << get_unsigned_integer(); - break; - case 'd': - os << get_double(); - break; - case 'n': // we have a null - os << "null"; - break; - case 't': // we have a true - os << "true"; - break; - case 'f': // we have a false - os << "false"; - break; - case '{': // we have an object - case '}': // we end an object - case '[': // we start an array - case ']': // we end an array - os << char(current_type); - break; - default: - return false; - } - return true; - } - - bool dom::parser::Iterator::move_to(const char* pointer, - uint32_t length) { - char* new_pointer = nullptr; - if (pointer[0] == '#') { - // Converting fragment representation to string representation - new_pointer = new char[length]; - uint32_t new_length = 0; - for (uint32_t i = 1; i < length; i++) { - if (pointer[i] == '%' && pointer[i + 1] == 'x') { -#if __cpp_exceptions - try { -#endif - int fragment = - std::stoi(std::string(&pointer[i + 2], 2), nullptr, 16); - if (fragment == '\\' || fragment == '"' || (fragment <= 0x1F)) { - // escaping the character - new_pointer[new_length] = '\\'; - new_length++; - } - new_pointer[new_length] = char(fragment); - i += 3; -#if __cpp_exceptions - } - catch (std::invalid_argument&) { - delete[] new_pointer; - return false; // the fragment is invalid - } -#endif - } - else { - new_pointer[new_length] = pointer[i]; - } - new_length++; - } - length = new_length; - pointer = new_pointer; - } - - // saving the current state - size_t depth_s = depth; - size_t location_s = location; - uint8_t current_type_s = current_type; - uint64_t current_val_s = current_val; - - rewind(); // The json pointer is used from the root of the document. - - bool found = relative_move_to(pointer, length); - delete[] new_pointer; - - if (!found) { - // since the pointer has found nothing, we get back to the original - // position. - depth = depth_s; - location = location_s; - current_type = current_type_s; - current_val = current_val_s; - } - - return found; - } - - inline bool dom::parser::Iterator::move_to(const std::string& pointer) { - return move_to(pointer.c_str(), uint32_t(pointer.length())); - } - - inline int64_t dom::parser::Iterator::get_integer() const { - if (location + 1 >= tape_length) { - return 0; // default value in case of error - } - return static_cast(doc.tape[location + 1]); - } - - inline uint64_t dom::parser::Iterator::get_unsigned_integer() const { - if (location + 1 >= tape_length) { - return 0; // default value in case of error - } - return doc.tape[location + 1]; - } - - inline const char* dom::parser::Iterator::get_string() const { - return reinterpret_cast( - doc.string_buf.get() + (current_val & internal::JSON_VALUE_MASK) + sizeof(uint32_t)); - } - - inline uint32_t dom::parser::Iterator::get_string_length() const { - uint32_t answer; - std::memcpy(&answer, - reinterpret_cast(doc.string_buf.get() + - (current_val & internal::JSON_VALUE_MASK)), - sizeof(uint32_t)); - return answer; - } - - inline double dom::parser::Iterator::get_double() const { - if (location + 1 >= tape_length) { - return std::numeric_limits::quiet_NaN(); // default value in - // case of error - } - double answer; - std::memcpy(&answer, &doc.tape[location + 1], sizeof(answer)); - return answer; - } - - bool dom::parser::Iterator::relative_move_to(const char* pointer, - uint32_t length) { - if (length == 0) { - // returns the whole document - return true; - } - - if (pointer[0] != '/') { - // '/' must be the first character - return false; - } - - // finding the key in an object or the index in an array - std::string key_or_index; - uint32_t offset = 1; - - // checking for the "-" case - if (is_array() && pointer[1] == '-') { - if (length != 2) { - // the pointer must be exactly "/-" - // there can't be anything more after '-' as an index - return false; - } - key_or_index = '-'; - offset = length; // will skip the loop coming right after - } - - // We either transform the first reference token to a valid json key - // or we make sure it is a valid index in an array. - for (; offset < length; offset++) { - if (pointer[offset] == '/') { - // beginning of the next key or index - break; - } - if (is_array() && (pointer[offset] < '0' || pointer[offset] > '9')) { - // the index of an array must be an integer - // we also make sure std::stoi won't discard whitespaces later - return false; - } - if (pointer[offset] == '~') { - // "~1" represents "/" - if (pointer[offset + 1] == '1') { - key_or_index += '/'; - offset++; - continue; - } - // "~0" represents "~" - if (pointer[offset + 1] == '0') { - key_or_index += '~'; - offset++; - continue; - } - } - if (pointer[offset] == '\\') { - if (pointer[offset + 1] == '\\' || pointer[offset + 1] == '"' || - (pointer[offset + 1] <= 0x1F)) { - key_or_index += pointer[offset + 1]; - offset++; - continue; - } - return false; // invalid escaped character - } - if (pointer[offset] == '\"') { - // unescaped quote character. this is an invalid case. - // lets do nothing and assume most pointers will be valid. - // it won't find any corresponding json key anyway. - // return false; - } - key_or_index += pointer[offset]; - } - - bool found = false; - if (is_object()) { - if (move_to_key(key_or_index.c_str(), uint32_t(key_or_index.length()))) { - found = relative_move_to(pointer + offset, length - offset); - } - } - else if (is_array()) { - if (key_or_index == "-") { // handling "-" case first - if (down()) { - while (next()) - ; // moving to the end of the array - // moving to the nonexistent value right after... - size_t npos; - if ((current_type == '[') || (current_type == '{')) { - // we need to jump - npos = uint32_t(current_val); - } - else { - npos = - location + ((current_type == 'd' || current_type == 'l') ? 2 : 1); - } - location = npos; - current_val = doc.tape[npos]; - current_type = uint8_t(current_val >> 56); - return true; // how could it fail ? - } - } - else { // regular numeric index - // The index can't have a leading '0' - if (key_or_index[0] == '0' && key_or_index.length() > 1) { - return false; - } - // it cannot be empty - if (key_or_index.length() == 0) { - return false; - } - // we already checked the index contains only valid digits - uint32_t index = std::stoi(key_or_index); - if (move_to_index(index)) { - found = relative_move_to(pointer + offset, length - offset); - } - } - } - - return found; - } - - SIMDJSON_POP_DISABLE_WARNINGS -} // namespace simdjson - -#endif // SIMDJSON_DISABLE_DEPRECATED_API - - -#endif // SIMDJSON_PARSEDJSON_ITERATOR_INL_H -/* end file simdjson/dom/parsedjson_iterator-inl.h */ /* skipped duplicate #include "simdjson/dom/parser-inl.h" */ /* skipped duplicate #include "simdjson/internal/tape_ref-inl.h" */ /* including simdjson/dom/serialization-inl.h: #include "simdjson/dom/serialization-inl.h" */ @@ -9860,6 +9119,8 @@ namespace simdjson { #define SIMDJSON_IMPLEMENTATION_ID_icelake 4 #define SIMDJSON_IMPLEMENTATION_ID_ppc64 5 #define SIMDJSON_IMPLEMENTATION_ID_westmere 6 +#define SIMDJSON_IMPLEMENTATION_ID_lsx 7 +#define SIMDJSON_IMPLEMENTATION_ID_lasx 8 #define SIMDJSON_IMPLEMENTATION_ID_FOR(IMPL) SIMDJSON_CAT(SIMDJSON_IMPLEMENTATION_ID_, IMPL) #define SIMDJSON_IMPLEMENTATION_ID SIMDJSON_IMPLEMENTATION_ID_FOR(SIMDJSON_IMPLEMENTATION) @@ -9874,7 +9135,11 @@ namespace simdjson { #ifndef SIMDJSON_IMPLEMENTATION_ARM64 #define SIMDJSON_IMPLEMENTATION_ARM64 (SIMDJSON_IS_ARM64) #endif -#define SIMDJSON_CAN_ALWAYS_RUN_ARM64 SIMDJSON_IMPLEMENTATION_ARM64 && SIMDJSON_IS_ARM64 +#if SIMDJSON_IMPLEMENTATION_ARM64 && SIMDJSON_IS_ARM64 +#define SIMDJSON_CAN_ALWAYS_RUN_ARM64 1 +#else +#define SIMDJSON_CAN_ALWAYS_RUN_ARM64 0 +#endif // Default Icelake to on if this is x86-64. Even if we're not compiled for it, it could be selected // at runtime. @@ -9885,9 +9150,20 @@ namespace simdjson { #ifdef _MSC_VER // To see why (__BMI__) && (__PCLMUL__) && (__LZCNT__) are not part of this next line, see // https://github.com/simdjson/simdjson/issues/1247 -#define SIMDJSON_CAN_ALWAYS_RUN_ICELAKE ((SIMDJSON_IMPLEMENTATION_ICELAKE) && (__AVX2__) && (__AVX512F__) && (__AVX512DQ__) && (__AVX512CD__) && (__AVX512BW__) && (__AVX512VL__) && (__AVX512VBMI2__)) +#if ((SIMDJSON_IMPLEMENTATION_ICELAKE) && (__AVX2__) && (__AVX512F__) && (__AVX512DQ__) && (__AVX512CD__) && (__AVX512BW__) && (__AVX512VL__) && (__AVX512VBMI2__)) +#define SIMDJSON_CAN_ALWAYS_RUN_ICELAKE 1 #else -#define SIMDJSON_CAN_ALWAYS_RUN_ICELAKE ((SIMDJSON_IMPLEMENTATION_ICELAKE) && (__AVX2__) && (__BMI__) && (__PCLMUL__) && (__LZCNT__) && (__AVX512F__) && (__AVX512DQ__) && (__AVX512CD__) && (__AVX512BW__) && (__AVX512VL__) && (__AVX512VBMI2__)) +#define SIMDJSON_CAN_ALWAYS_RUN_ICELAKE 0 +#endif + +#else + +#if ((SIMDJSON_IMPLEMENTATION_ICELAKE) && (__AVX2__) && (__BMI__) && (__PCLMUL__) && (__LZCNT__) && (__AVX512F__) && (__AVX512DQ__) && (__AVX512CD__) && (__AVX512BW__) && (__AVX512VL__) && (__AVX512VBMI2__)) +#define SIMDJSON_CAN_ALWAYS_RUN_ICELAKE 1 +#else +#define SIMDJSON_CAN_ALWAYS_RUN_ICELAKE 0 +#endif + #endif // Default Haswell to on if this is x86-64. Even if we're not compiled for it, it could be selected @@ -9903,9 +9179,20 @@ namespace simdjson { #ifdef _MSC_VER // To see why (__BMI__) && (__PCLMUL__) && (__LZCNT__) are not part of this next line, see // https://github.com/simdjson/simdjson/issues/1247 -#define SIMDJSON_CAN_ALWAYS_RUN_HASWELL ((SIMDJSON_IMPLEMENTATION_HASWELL) && (SIMDJSON_IS_X86_64) && (__AVX2__)) +#if ((SIMDJSON_IMPLEMENTATION_HASWELL) && (SIMDJSON_IS_X86_64) && (__AVX2__)) +#define SIMDJSON_CAN_ALWAYS_RUN_HASWELL 1 #else -#define SIMDJSON_CAN_ALWAYS_RUN_HASWELL ((SIMDJSON_IMPLEMENTATION_HASWELL) && (SIMDJSON_IS_X86_64) && (__AVX2__) && (__BMI__) && (__PCLMUL__) && (__LZCNT__)) +#define SIMDJSON_CAN_ALWAYS_RUN_HASWELL 0 +#endif + +#else + +#if ((SIMDJSON_IMPLEMENTATION_HASWELL) && (SIMDJSON_IS_X86_64) && (__AVX2__) && (__BMI__) && (__PCLMUL__) && (__LZCNT__)) +#define SIMDJSON_CAN_ALWAYS_RUN_HASWELL 1 +#else +#define SIMDJSON_CAN_ALWAYS_RUN_HASWELL 0 +#endif + #endif // Default Westmere to on if this is x86-64. @@ -9917,16 +9204,40 @@ namespace simdjson { #define SIMDJSON_IMPLEMENTATION_WESTMERE SIMDJSON_IS_X86_64 #endif #endif -#define SIMDJSON_CAN_ALWAYS_RUN_WESTMERE (SIMDJSON_IMPLEMENTATION_WESTMERE && SIMDJSON_IS_X86_64 && __SSE4_2__ && __PCLMUL__) + +#if (SIMDJSON_IMPLEMENTATION_WESTMERE && SIMDJSON_IS_X86_64 && __SSE4_2__ && __PCLMUL__) +#define SIMDJSON_CAN_ALWAYS_RUN_WESTMERE 1 +#else +#define SIMDJSON_CAN_ALWAYS_RUN_WESTMERE 0 +#endif + #ifndef SIMDJSON_IMPLEMENTATION_PPC64 #define SIMDJSON_IMPLEMENTATION_PPC64 (SIMDJSON_IS_PPC64 && SIMDJSON_IS_PPC64_VMX) #endif -#define SIMDJSON_CAN_ALWAYS_RUN_PPC64 SIMDJSON_IMPLEMENTATION_PPC64 && SIMDJSON_IS_PPC64 && SIMDJSON_IS_PPC64_VMX +#if SIMDJSON_IMPLEMENTATION_PPC64 && SIMDJSON_IS_PPC64 && SIMDJSON_IS_PPC64_VMX +#define SIMDJSON_CAN_ALWAYS_RUN_PPC64 1 +#else +#define SIMDJSON_CAN_ALWAYS_RUN_PPC64 0 +#endif + +#ifndef SIMDJSON_IMPLEMENTATION_LASX +#define SIMDJSON_IMPLEMENTATION_LASX (SIMDJSON_IS_LOONGARCH64 && __loongarch_asx) +#endif +#define SIMDJSON_CAN_ALWAYS_RUN_LASX (SIMDJSON_IMPLEMENTATION_LASX) + +#ifndef SIMDJSON_IMPLEMENTATION_LSX +#if SIMDJSON_CAN_ALWAYS_RUN_LASX +#define SIMDJSON_IMPLEMENTATION_LSX 0 +#else +#define SIMDJSON_IMPLEMENTATION_LSX (SIMDJSON_IS_LOONGARCH64 && __loongarch_sx) +#endif +#endif +#define SIMDJSON_CAN_ALWAYS_RUN_LSX (SIMDJSON_IMPLEMENTATION_LSX) // Default Fallback to on unless a builtin implementation has already been selected. #ifndef SIMDJSON_IMPLEMENTATION_FALLBACK -#if SIMDJSON_CAN_ALWAYS_RUN_ARM64 || SIMDJSON_CAN_ALWAYS_RUN_ICELAKE || SIMDJSON_CAN_ALWAYS_RUN_HASWELL || SIMDJSON_CAN_ALWAYS_RUN_WESTMERE || SIMDJSON_CAN_ALWAYS_RUN_PPC64 +#if SIMDJSON_CAN_ALWAYS_RUN_ARM64 || SIMDJSON_CAN_ALWAYS_RUN_ICELAKE || SIMDJSON_CAN_ALWAYS_RUN_HASWELL || SIMDJSON_CAN_ALWAYS_RUN_WESTMERE || SIMDJSON_CAN_ALWAYS_RUN_PPC64 || SIMDJSON_CAN_ALWAYS_RUN_LSX || SIMDJSON_CAN_ALWAYS_RUN_LASX // if anything at all except fallback can always run, then disable fallback. #define SIMDJSON_IMPLEMENTATION_FALLBACK 0 #else @@ -9948,6 +9259,10 @@ namespace simdjson { #define SIMDJSON_BUILTIN_IMPLEMENTATION arm64 #elif SIMDJSON_CAN_ALWAYS_RUN_PPC64 #define SIMDJSON_BUILTIN_IMPLEMENTATION ppc64 +#elif SIMDJSON_CAN_ALWAYS_RUN_LSX +#define SIMDJSON_BUILTIN_IMPLEMENTATION lsx +#elif SIMDJSON_CAN_ALWAYS_RUN_LASX +#define SIMDJSON_BUILTIN_IMPLEMENTATION lasx #elif SIMDJSON_CAN_ALWAYS_RUN_FALLBACK #define SIMDJSON_BUILTIN_IMPLEMENTATION fallback #else @@ -9975,6 +9290,10 @@ namespace simdjson { namespace ppc64 {} #elif SIMDJSON_BUILTIN_IMPLEMENTATION_IS(westmere) namespace westmere {} +#elif SIMDJSON_BUILTIN_IMPLEMENTATION_IS(lsx) + namespace lsx {} +#elif SIMDJSON_BUILTIN_IMPLEMENTATION_IS(lasx) + namespace lasx {} #else #error Unknown SIMDJSON_BUILTIN_IMPLEMENTATION #endif @@ -10086,7 +9405,9 @@ namespace simdjson { AVX512CD = 0x2000, AVX512BW = 0x4000, AVX512VL = 0x8000, - AVX512VBMI2 = 0x10000 + AVX512VBMI2 = 0x10000, + LSX = 0x20000, + LASX = 0x40000, }; } // namespace internal @@ -10450,6 +9771,76 @@ namespace simdjson { #endif // SIMDJSON_WESTMERE_IMPLEMENTATION_H /* end file simdjson/westmere/implementation.h */ +#elif SIMDJSON_BUILTIN_IMPLEMENTATION_IS(lsx) +/* including simdjson/lsx/implementation.h: #include "simdjson/lsx/implementation.h" */ +/* begin file simdjson/lsx/implementation.h */ +#ifndef SIMDJSON_LSX_IMPLEMENTATION_H +#define SIMDJSON_LSX_IMPLEMENTATION_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/implementation.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/instruction_set.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lsx { + + /** + * @private + */ + class implementation final : public simdjson::implementation { + public: + simdjson_inline implementation() : simdjson::implementation("lsx", "LoongArch SX", internal::instruction_set::LSX) {} + simdjson_warn_unused error_code create_dom_parser_implementation( + size_t capacity, + size_t max_length, + std::unique_ptr& dst + ) const noexcept final; + simdjson_warn_unused error_code minify(const uint8_t* buf, size_t len, uint8_t* dst, size_t& dst_len) const noexcept final; + simdjson_warn_unused bool validate_utf8(const char* buf, size_t len) const noexcept final; + }; + + } // namespace lsx +} // namespace simdjson + +#endif // SIMDJSON_LSX_IMPLEMENTATION_H +/* end file simdjson/lsx/implementation.h */ +#elif SIMDJSON_BUILTIN_IMPLEMENTATION_IS(lasx) +/* including simdjson/lasx/implementation.h: #include "simdjson/lasx/implementation.h" */ +/* begin file simdjson/lasx/implementation.h */ +#ifndef SIMDJSON_LASX_IMPLEMENTATION_H +#define SIMDJSON_LASX_IMPLEMENTATION_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/implementation.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/instruction_set.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lasx { + + /** + * @private + */ + class implementation final : public simdjson::implementation { + public: + simdjson_inline implementation() : simdjson::implementation("lasx", "LoongArch ASX", internal::instruction_set::LASX) {} + simdjson_warn_unused error_code create_dom_parser_implementation( + size_t capacity, + size_t max_length, + std::unique_ptr& dst + ) const noexcept final; + simdjson_warn_unused error_code minify(const uint8_t* buf, size_t len, uint8_t* dst, size_t& dst_len) const noexcept final; + simdjson_warn_unused bool validate_utf8(const char* buf, size_t len) const noexcept final; + }; + + } // namespace lasx +} // namespace simdjson + +#endif // SIMDJSON_LASX_IMPLEMENTATION_H +/* end file simdjson/lasx/implementation.h */ #else #error Unknown SIMDJSON_BUILTIN_IMPLEMENTATION #endif @@ -10700,10 +10091,10 @@ namespace simdjson { #include -#if _M_ARM64 +#if SIMDJSON_REGULAR_VISUAL_STUDIO && SIMDJSON_IS_ARM64 // __umulh requires intrin.h #include -#endif // _M_ARM64 +#endif // SIMDJSON_REGULAR_VISUAL_STUDIO && SIMDJSON_IS_ARM64 namespace simdjson { namespace arm64 { @@ -10723,13 +10114,13 @@ namespace simdjson { simdjson_inline internal::value128 full_multiplication(uint64_t value1, uint64_t value2) { internal::value128 answer; #if SIMDJSON_REGULAR_VISUAL_STUDIO || SIMDJSON_IS_32BITS -#ifdef _M_ARM64 +#if SIMDJSON_IS_ARM64 // ARM64 has native support for 64-bit multiplications, no need to emultate answer.high = __umulh(value1, value2); answer.low = value1 * value2; #else answer.low = _umul128(value1, value2, &answer.high); // _umul128 not available on ARM64 -#endif // _M_ARM64 +#endif // SIMDJSON_IS_ARM64 #else // SIMDJSON_REGULAR_VISUAL_STUDIO || SIMDJSON_IS_32BITS __uint128_t r = (static_cast<__uint128_t>(value1)) * value2; answer.low = uint64_t(r); @@ -10884,7 +10275,7 @@ namespace simdjson { tmp = vpaddq_u8(tmp, tmp); return vgetq_lane_u16(vreinterpretq_u16_u8(tmp), 0); } - simdjson_inline bool any() const { return vmaxvq_u8(*this) != 0; } + simdjson_inline bool any() const { return vmaxvq_u32(vreinterpretq_u32_u8(*this)) != 0; } }; // Unsigned bytes @@ -11332,6 +10723,10 @@ namespace simdjson { /* amalgamation skipped (editor-only): #include "simdjson/arm64/begin.h" */ /* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_PPC64 */ /* amalgamation skipped (editor-only): #include "simdjson/ppc64/begin.h" */ +/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_LSX */ +/* amalgamation skipped (editor-only): #include "simdjson/lsx/begin.h" */ +/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_LASX */ +/* amalgamation skipped (editor-only): #include "simdjson/lasx/begin.h" */ /* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_FALLBACK */ /* amalgamation skipped (editor-only): #include "simdjson/fallback/begin.h" */ /* amalgamation skipped (editor-only): #else */ @@ -11352,7 +10747,8 @@ namespace simdjson { enum class number_type { floating_point_number = 1, /// a binary64 number signed_integer, /// a signed integer that fits in a 64-bit word using two's complement - unsigned_integer /// a positive integer larger or equal to 1<<63 + unsigned_integer, /// a positive integer larger or equal to 1<<63 + big_integer /// a big integer that does not fit in a 64-bit word }; } // namespace arm64 @@ -11802,11 +11198,13 @@ namespace simdjson { #define WRITE_INTEGER(VALUE, SRC, WRITER) (found_integer((VALUE), (SRC)), (WRITER).append_s64((VALUE))) #define WRITE_UNSIGNED(VALUE, SRC, WRITER) (found_unsigned_integer((VALUE), (SRC)), (WRITER).append_u64((VALUE))) #define WRITE_DOUBLE(VALUE, SRC, WRITER) (found_float((VALUE), (SRC)), (WRITER).append_double((VALUE))) +#define BIGINT_NUMBER(SRC) (found_invalid_number((SRC)), BIGINT_ERROR) #else #define INVALID_NUMBER(SRC) (NUMBER_ERROR) #define WRITE_INTEGER(VALUE, SRC, WRITER) (WRITER).append_s64((VALUE)) #define WRITE_UNSIGNED(VALUE, SRC, WRITER) (WRITER).append_u64((VALUE)) #define WRITE_DOUBLE(VALUE, SRC, WRITER) (WRITER).append_double((VALUE)) +#define BIGINT_NUMBER(SRC) (BIGINT_ERROR) #endif namespace { @@ -12138,6 +11536,10 @@ namespace simdjson { return true; } + simdjson_inline bool is_digit(const uint8_t c) { + return static_cast(c - '0') <= 9; + } + simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t* const src, const uint8_t*& p, uint64_t& i, int64_t& exponent) { // we continue with the fiction that we have an integer. If the // floating point number is representable as x * 10^z for some integer @@ -12195,7 +11597,7 @@ namespace simdjson { // If there were more than 18 digits, we may have overflowed the integer. We have to do // something!!!! if (simdjson_unlikely(p > start_exp + 18)) { - // Skip leading zeroes: 1e000000000000000000001 is technically valid and doesn't overflow + // Skip leading zeroes: 1e000000000000000000001 is technically valid and does not overflow while (*start_exp == '0') { start_exp++; } // 19 digits could overflow int64_t and is kind of absurd anyway. We don't // support exponents smaller than -999,999,999,999,999,999 and bigger @@ -12217,6 +11619,23 @@ namespace simdjson { return SUCCESS; } + simdjson_inline bool check_if_integer(const uint8_t* const src, size_t max_length) { + const uint8_t* const srcend = src + max_length; + bool negative = (*src == '-'); // we can always read at least one character after the '-' + const uint8_t* p = src + uint8_t(negative); + if (p == srcend) { return false; } + if (*p == '0') { + ++p; + if (p == srcend) { return true; } + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return false; } + return true; + } + while (p != srcend && is_digit(*p)) { ++p; } + if (p == srcend) { return true; } + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return false; } + return true; + } + simdjson_inline size_t significant_digits(const uint8_t* start_digits, size_t digit_count) { // It is possible that the integer had an overflow. // We have to handle the case where we have 0.0000somenumber. @@ -12229,11 +11648,8 @@ namespace simdjson { } // unnamed namespace /** @private */ - template - error_code slow_float_parsing(simdjson_unused const uint8_t* src, W writer) { - double d; - if (parse_float_fallback(src, &d)) { - writer.append_double(d); + static error_code slow_float_parsing(simdjson_unused const uint8_t* src, double* answer) { + if (parse_float_fallback(src, answer)) { return SUCCESS; } return INVALID_NUMBER(src); @@ -12257,13 +11673,13 @@ namespace simdjson { // 10000000000000000000000000000000000000000000e+308 // 3.1415926535897932384626433832795028841971693993751 // - // NOTE: This makes a *copy* of the writer and passes it to slow_float_parsing. This happens - // because slow_float_parsing is a non-inlined function. If we passed our writer reference to - // it, it would force it to be stored in memory, preventing the compiler from picking it apart - // and putting into registers. i.e. if we pass it as reference, it gets slow. - // This is what forces the skip_double, as well. - error_code error = slow_float_parsing(src, writer); - writer.skip_double(); + // NOTE: We do not pass a reference to the to slow_float_parsing. If we passed our writer + // reference to it, it would force it to be stored in memory, preventing the compiler from + // picking it apart and putting into registers. i.e. if we pass it as reference, + // it gets slow. + double d; + error_code error = slow_float_parsing(src, &d); + writer.append_double(d); return error; } // NOTE: it's weird that the simdjson_unlikely() only wraps half the if, but it seems to get slower any other @@ -12295,6 +11711,18 @@ namespace simdjson { return SUCCESS; } + // parse the number at src + // define JSON_TEST_NUMBERS for unit testing + // + // It is assumed that the number is followed by a structural ({,},],[) character + // or a white space character. If that is not the case (e.g., when the JSON + // document is made of a single number), then it is necessary to copy the + // content and append a space before calling this function. + // + // Our objective is accurate parsing (ULP of 0) at high speed. + template + simdjson_inline error_code parse_number(const uint8_t* const src, W& writer); + // for performance analysis, it is sometimes useful to skip parsing #ifdef SIMDJSON_SKIPNUMBERPARSING @@ -12373,11 +11801,11 @@ namespace simdjson { // The longest positive 64-bit number is 20 digits. // We do it this way so we don't trigger this branch unless we must. size_t longest_digit_count = negative ? 19 : 20; - if (digit_count > longest_digit_count) { return INVALID_NUMBER(src); } + if (digit_count > longest_digit_count) { return BIGINT_NUMBER(src); } if (digit_count == longest_digit_count) { if (negative) { // Anything negative above INT64_MAX+1 is invalid - if (i > uint64_t(INT64_MAX) + 1) { return INVALID_NUMBER(src); } + if (i > uint64_t(INT64_MAX) + 1) { return BIGINT_NUMBER(src); } WRITE_INTEGER(~i + 1, src, writer); if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return INVALID_NUMBER(src); } return SUCCESS; @@ -12397,7 +11825,7 @@ namespace simdjson { else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INVALID_NUMBER(src); } } - // Write unsigned if it doesn't fit in a signed integer. + // Write unsigned if it does not fit in a signed integer. if (i > uint64_t(INT64_MAX)) { WRITE_UNSIGNED(i, src, writer); } @@ -12850,19 +12278,32 @@ namespace simdjson { src += uint8_t(negative); const uint8_t* p = src; while (static_cast(*p - '0') <= 9) { p++; } + size_t digit_count = size_t(p - src); if (p == src) { return NUMBER_ERROR; } if (jsoncharutils::is_structural_or_whitespace(*p)) { + static const uint8_t* smaller_big_integer = reinterpret_cast("9223372036854775808"); // We have an integer. + if (simdjson_unlikely(digit_count > 20)) { + return number_type::big_integer; + } // If the number is negative and valid, it must be a signed integer. - if (negative) { return number_type::signed_integer; } + if (negative) { + if (simdjson_unlikely(digit_count > 19)) return number_type::big_integer; + if (simdjson_unlikely(digit_count == 19 && memcmp(src, smaller_big_integer, 19) > 0)) { + return number_type::big_integer; + } + return number_type::signed_integer; + } + // Let us check if we have a big integer (>=2**64). + static const uint8_t* two_to_sixtyfour = reinterpret_cast("18446744073709551616"); + if ((digit_count > 20) || (digit_count == 20 && memcmp(src, two_to_sixtyfour, 20) >= 0)) { + return number_type::big_integer; + } + // The number is positive and smaller than 18446744073709551616 (or 2**64). // We want values larger or equal to 9223372036854775808 to be unsigned // integers, and the other values to be signed integers. - int digit_count = int(p - src); - if (digit_count >= 19) { - const uint8_t* smaller_big_integer = reinterpret_cast("9223372036854775808"); - if ((digit_count >= 20) || (memcmp(src, smaller_big_integer, 19) >= 0)) { - return number_type::unsigned_integer; - } + if ((digit_count == 20) || (digit_count >= 19 && memcmp(src, smaller_big_integer, 19) >= 0)) { + return number_type::unsigned_integer; } return number_type::signed_integer; } @@ -13042,6 +12483,7 @@ namespace simdjson { case number_type::signed_integer: out << "integer in [-9223372036854775808,9223372036854775808)"; break; case number_type::unsigned_integer: out << "unsigned integer in [9223372036854775808,18446744073709551616)"; break; case number_type::floating_point_number: out << "floating-point number (binary64)"; break; + case number_type::big_integer: out << "big integer"; break; default: SIMDJSON_UNREACHABLE(); } return out; @@ -13342,13 +12784,13 @@ namespace simdjson { simdjson_inline internal::value128 full_multiplication(uint64_t value1, uint64_t value2) { internal::value128 answer; #if SIMDJSON_REGULAR_VISUAL_STUDIO || SIMDJSON_IS_32BITS -#ifdef _M_ARM64 +#if SIMDJSON_IS_ARM64 // ARM64 has native support for 64-bit multiplications, no need to emultate answer.high = __umulh(value1, value2); answer.low = value1 * value2; #else answer.low = _umul128(value1, value2, &answer.high); // _umul128 not available on ARM64 -#endif // _M_ARM64 +#endif // SIMDJSON_IS_ARM64 #else // SIMDJSON_REGULAR_VISUAL_STUDIO || SIMDJSON_IS_32BITS __uint128_t r = (static_cast<__uint128_t>(value1)) * value2; answer.low = uint64_t(r); @@ -13393,6 +12835,10 @@ namespace simdjson { /* amalgamation skipped (editor-only): #include "simdjson/arm64/begin.h" */ /* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_PPC64 */ /* amalgamation skipped (editor-only): #include "simdjson/ppc64/begin.h" */ +/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_LSX */ +/* amalgamation skipped (editor-only): #include "simdjson/lsx/begin.h" */ +/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_LASX */ +/* amalgamation skipped (editor-only): #include "simdjson/lasx/begin.h" */ /* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_FALLBACK */ /* amalgamation skipped (editor-only): #include "simdjson/fallback/begin.h" */ /* amalgamation skipped (editor-only): #else */ @@ -13413,7 +12859,8 @@ namespace simdjson { enum class number_type { floating_point_number = 1, /// a binary64 number signed_integer, /// a signed integer that fits in a 64-bit word using two's complement - unsigned_integer /// a positive integer larger or equal to 1<<63 + unsigned_integer, /// a positive integer larger or equal to 1<<63 + big_integer /// a big integer that does not fit in a 64-bit word }; } // namespace fallback @@ -13863,11 +13310,13 @@ namespace simdjson { #define WRITE_INTEGER(VALUE, SRC, WRITER) (found_integer((VALUE), (SRC)), (WRITER).append_s64((VALUE))) #define WRITE_UNSIGNED(VALUE, SRC, WRITER) (found_unsigned_integer((VALUE), (SRC)), (WRITER).append_u64((VALUE))) #define WRITE_DOUBLE(VALUE, SRC, WRITER) (found_float((VALUE), (SRC)), (WRITER).append_double((VALUE))) +#define BIGINT_NUMBER(SRC) (found_invalid_number((SRC)), BIGINT_ERROR) #else #define INVALID_NUMBER(SRC) (NUMBER_ERROR) #define WRITE_INTEGER(VALUE, SRC, WRITER) (WRITER).append_s64((VALUE)) #define WRITE_UNSIGNED(VALUE, SRC, WRITER) (WRITER).append_u64((VALUE)) #define WRITE_DOUBLE(VALUE, SRC, WRITER) (WRITER).append_double((VALUE)) +#define BIGINT_NUMBER(SRC) (BIGINT_ERROR) #endif namespace { @@ -14199,6 +13648,10 @@ namespace simdjson { return true; } + simdjson_inline bool is_digit(const uint8_t c) { + return static_cast(c - '0') <= 9; + } + simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t* const src, const uint8_t*& p, uint64_t& i, int64_t& exponent) { // we continue with the fiction that we have an integer. If the // floating point number is representable as x * 10^z for some integer @@ -14256,7 +13709,7 @@ namespace simdjson { // If there were more than 18 digits, we may have overflowed the integer. We have to do // something!!!! if (simdjson_unlikely(p > start_exp + 18)) { - // Skip leading zeroes: 1e000000000000000000001 is technically valid and doesn't overflow + // Skip leading zeroes: 1e000000000000000000001 is technically valid and does not overflow while (*start_exp == '0') { start_exp++; } // 19 digits could overflow int64_t and is kind of absurd anyway. We don't // support exponents smaller than -999,999,999,999,999,999 and bigger @@ -14278,6 +13731,23 @@ namespace simdjson { return SUCCESS; } + simdjson_inline bool check_if_integer(const uint8_t* const src, size_t max_length) { + const uint8_t* const srcend = src + max_length; + bool negative = (*src == '-'); // we can always read at least one character after the '-' + const uint8_t* p = src + uint8_t(negative); + if (p == srcend) { return false; } + if (*p == '0') { + ++p; + if (p == srcend) { return true; } + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return false; } + return true; + } + while (p != srcend && is_digit(*p)) { ++p; } + if (p == srcend) { return true; } + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return false; } + return true; + } + simdjson_inline size_t significant_digits(const uint8_t* start_digits, size_t digit_count) { // It is possible that the integer had an overflow. // We have to handle the case where we have 0.0000somenumber. @@ -14290,11 +13760,8 @@ namespace simdjson { } // unnamed namespace /** @private */ - template - error_code slow_float_parsing(simdjson_unused const uint8_t* src, W writer) { - double d; - if (parse_float_fallback(src, &d)) { - writer.append_double(d); + static error_code slow_float_parsing(simdjson_unused const uint8_t* src, double* answer) { + if (parse_float_fallback(src, answer)) { return SUCCESS; } return INVALID_NUMBER(src); @@ -14318,13 +13785,13 @@ namespace simdjson { // 10000000000000000000000000000000000000000000e+308 // 3.1415926535897932384626433832795028841971693993751 // - // NOTE: This makes a *copy* of the writer and passes it to slow_float_parsing. This happens - // because slow_float_parsing is a non-inlined function. If we passed our writer reference to - // it, it would force it to be stored in memory, preventing the compiler from picking it apart - // and putting into registers. i.e. if we pass it as reference, it gets slow. - // This is what forces the skip_double, as well. - error_code error = slow_float_parsing(src, writer); - writer.skip_double(); + // NOTE: We do not pass a reference to the to slow_float_parsing. If we passed our writer + // reference to it, it would force it to be stored in memory, preventing the compiler from + // picking it apart and putting into registers. i.e. if we pass it as reference, + // it gets slow. + double d; + error_code error = slow_float_parsing(src, &d); + writer.append_double(d); return error; } // NOTE: it's weird that the simdjson_unlikely() only wraps half the if, but it seems to get slower any other @@ -14356,6 +13823,18 @@ namespace simdjson { return SUCCESS; } + // parse the number at src + // define JSON_TEST_NUMBERS for unit testing + // + // It is assumed that the number is followed by a structural ({,},],[) character + // or a white space character. If that is not the case (e.g., when the JSON + // document is made of a single number), then it is necessary to copy the + // content and append a space before calling this function. + // + // Our objective is accurate parsing (ULP of 0) at high speed. + template + simdjson_inline error_code parse_number(const uint8_t* const src, W& writer); + // for performance analysis, it is sometimes useful to skip parsing #ifdef SIMDJSON_SKIPNUMBERPARSING @@ -14434,11 +13913,11 @@ namespace simdjson { // The longest positive 64-bit number is 20 digits. // We do it this way so we don't trigger this branch unless we must. size_t longest_digit_count = negative ? 19 : 20; - if (digit_count > longest_digit_count) { return INVALID_NUMBER(src); } + if (digit_count > longest_digit_count) { return BIGINT_NUMBER(src); } if (digit_count == longest_digit_count) { if (negative) { // Anything negative above INT64_MAX+1 is invalid - if (i > uint64_t(INT64_MAX) + 1) { return INVALID_NUMBER(src); } + if (i > uint64_t(INT64_MAX) + 1) { return BIGINT_NUMBER(src); } WRITE_INTEGER(~i + 1, src, writer); if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return INVALID_NUMBER(src); } return SUCCESS; @@ -14458,7 +13937,7 @@ namespace simdjson { else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INVALID_NUMBER(src); } } - // Write unsigned if it doesn't fit in a signed integer. + // Write unsigned if it does not fit in a signed integer. if (i > uint64_t(INT64_MAX)) { WRITE_UNSIGNED(i, src, writer); } @@ -14911,19 +14390,32 @@ namespace simdjson { src += uint8_t(negative); const uint8_t* p = src; while (static_cast(*p - '0') <= 9) { p++; } + size_t digit_count = size_t(p - src); if (p == src) { return NUMBER_ERROR; } if (jsoncharutils::is_structural_or_whitespace(*p)) { + static const uint8_t* smaller_big_integer = reinterpret_cast("9223372036854775808"); // We have an integer. + if (simdjson_unlikely(digit_count > 20)) { + return number_type::big_integer; + } // If the number is negative and valid, it must be a signed integer. - if (negative) { return number_type::signed_integer; } + if (negative) { + if (simdjson_unlikely(digit_count > 19)) return number_type::big_integer; + if (simdjson_unlikely(digit_count == 19 && memcmp(src, smaller_big_integer, 19) > 0)) { + return number_type::big_integer; + } + return number_type::signed_integer; + } + // Let us check if we have a big integer (>=2**64). + static const uint8_t* two_to_sixtyfour = reinterpret_cast("18446744073709551616"); + if ((digit_count > 20) || (digit_count == 20 && memcmp(src, two_to_sixtyfour, 20) >= 0)) { + return number_type::big_integer; + } + // The number is positive and smaller than 18446744073709551616 (or 2**64). // We want values larger or equal to 9223372036854775808 to be unsigned // integers, and the other values to be signed integers. - int digit_count = int(p - src); - if (digit_count >= 19) { - const uint8_t* smaller_big_integer = reinterpret_cast("9223372036854775808"); - if ((digit_count >= 20) || (memcmp(src, smaller_big_integer, 19) >= 0)) { - return number_type::unsigned_integer; - } + if ((digit_count == 20) || (digit_count >= 19 && memcmp(src, smaller_big_integer, 19) >= 0)) { + return number_type::unsigned_integer; } return number_type::signed_integer; } @@ -15103,6 +14595,7 @@ namespace simdjson { case number_type::signed_integer: out << "integer in [-9223372036854775808,9223372036854775808)"; break; case number_type::unsigned_integer: out << "unsigned integer in [9223372036854775808,18446744073709551616)"; break; case number_type::floating_point_number: out << "floating-point number (binary64)"; break; + case number_type::big_integer: out << "big integer"; break; default: SIMDJSON_UNREACHABLE(); } return out; @@ -15469,13 +14962,13 @@ namespace simdjson { simdjson_inline internal::value128 full_multiplication(uint64_t value1, uint64_t value2) { internal::value128 answer; #if SIMDJSON_REGULAR_VISUAL_STUDIO || SIMDJSON_IS_32BITS -#ifdef _M_ARM64 +#if SIMDJSON_IS_ARM64 // ARM64 has native support for 64-bit multiplications, no need to emultate answer.high = __umulh(value1, value2); answer.low = value1 * value2; #else answer.low = _umul128(value1, value2, &answer.high); // _umul128 not available on ARM64 -#endif // _M_ARM64 +#endif // SIMDJSON_IS_ARM64 #else // SIMDJSON_REGULAR_VISUAL_STUDIO || SIMDJSON_IS_32BITS __uint128_t r = (static_cast<__uint128_t>(value1)) * value2; answer.low = uint64_t(r); @@ -15561,10 +15054,10 @@ namespace simdjson { struct simd8 : base8 { static simdjson_inline simd8 splat(bool _value) { return _mm256_set1_epi8(uint8_t(-(!!_value))); } - simdjson_inline simd8() : base8() {} - simdjson_inline simd8(const __m256i _value) : base8(_value) {} + simdjson_inline simd8() : base8() {} + simdjson_inline simd8(const __m256i _value) : base8(_value) {} // Splat constructor - simdjson_inline simd8(bool _value) : base8(splat(_value)) {} + simdjson_inline simd8(bool _value) : base8(splat(_value)) {} simdjson_inline int to_bitmask() const { return _mm256_movemask_epi8(*this); } simdjson_inline bool any() const { return !_mm256_testz_si256(*this, *this); } @@ -15946,6 +15439,10 @@ namespace simdjson { /* amalgamation skipped (editor-only): #include "simdjson/arm64/begin.h" */ /* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_PPC64 */ /* amalgamation skipped (editor-only): #include "simdjson/ppc64/begin.h" */ +/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_LSX */ +/* amalgamation skipped (editor-only): #include "simdjson/lsx/begin.h" */ +/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_LASX */ +/* amalgamation skipped (editor-only): #include "simdjson/lasx/begin.h" */ /* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_FALLBACK */ /* amalgamation skipped (editor-only): #include "simdjson/fallback/begin.h" */ /* amalgamation skipped (editor-only): #else */ @@ -15966,7 +15463,8 @@ namespace simdjson { enum class number_type { floating_point_number = 1, /// a binary64 number signed_integer, /// a signed integer that fits in a 64-bit word using two's complement - unsigned_integer /// a positive integer larger or equal to 1<<63 + unsigned_integer, /// a positive integer larger or equal to 1<<63 + big_integer /// a big integer that does not fit in a 64-bit word }; } // namespace haswell @@ -16416,11 +15914,13 @@ namespace simdjson { #define WRITE_INTEGER(VALUE, SRC, WRITER) (found_integer((VALUE), (SRC)), (WRITER).append_s64((VALUE))) #define WRITE_UNSIGNED(VALUE, SRC, WRITER) (found_unsigned_integer((VALUE), (SRC)), (WRITER).append_u64((VALUE))) #define WRITE_DOUBLE(VALUE, SRC, WRITER) (found_float((VALUE), (SRC)), (WRITER).append_double((VALUE))) +#define BIGINT_NUMBER(SRC) (found_invalid_number((SRC)), BIGINT_ERROR) #else #define INVALID_NUMBER(SRC) (NUMBER_ERROR) #define WRITE_INTEGER(VALUE, SRC, WRITER) (WRITER).append_s64((VALUE)) #define WRITE_UNSIGNED(VALUE, SRC, WRITER) (WRITER).append_u64((VALUE)) #define WRITE_DOUBLE(VALUE, SRC, WRITER) (WRITER).append_double((VALUE)) +#define BIGINT_NUMBER(SRC) (BIGINT_ERROR) #endif namespace { @@ -16752,6 +16252,10 @@ namespace simdjson { return true; } + simdjson_inline bool is_digit(const uint8_t c) { + return static_cast(c - '0') <= 9; + } + simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t* const src, const uint8_t*& p, uint64_t& i, int64_t& exponent) { // we continue with the fiction that we have an integer. If the // floating point number is representable as x * 10^z for some integer @@ -16809,7 +16313,7 @@ namespace simdjson { // If there were more than 18 digits, we may have overflowed the integer. We have to do // something!!!! if (simdjson_unlikely(p > start_exp + 18)) { - // Skip leading zeroes: 1e000000000000000000001 is technically valid and doesn't overflow + // Skip leading zeroes: 1e000000000000000000001 is technically valid and does not overflow while (*start_exp == '0') { start_exp++; } // 19 digits could overflow int64_t and is kind of absurd anyway. We don't // support exponents smaller than -999,999,999,999,999,999 and bigger @@ -16831,6 +16335,23 @@ namespace simdjson { return SUCCESS; } + simdjson_inline bool check_if_integer(const uint8_t* const src, size_t max_length) { + const uint8_t* const srcend = src + max_length; + bool negative = (*src == '-'); // we can always read at least one character after the '-' + const uint8_t* p = src + uint8_t(negative); + if (p == srcend) { return false; } + if (*p == '0') { + ++p; + if (p == srcend) { return true; } + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return false; } + return true; + } + while (p != srcend && is_digit(*p)) { ++p; } + if (p == srcend) { return true; } + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return false; } + return true; + } + simdjson_inline size_t significant_digits(const uint8_t* start_digits, size_t digit_count) { // It is possible that the integer had an overflow. // We have to handle the case where we have 0.0000somenumber. @@ -16843,11 +16364,8 @@ namespace simdjson { } // unnamed namespace /** @private */ - template - error_code slow_float_parsing(simdjson_unused const uint8_t* src, W writer) { - double d; - if (parse_float_fallback(src, &d)) { - writer.append_double(d); + static error_code slow_float_parsing(simdjson_unused const uint8_t* src, double* answer) { + if (parse_float_fallback(src, answer)) { return SUCCESS; } return INVALID_NUMBER(src); @@ -16871,13 +16389,13 @@ namespace simdjson { // 10000000000000000000000000000000000000000000e+308 // 3.1415926535897932384626433832795028841971693993751 // - // NOTE: This makes a *copy* of the writer and passes it to slow_float_parsing. This happens - // because slow_float_parsing is a non-inlined function. If we passed our writer reference to - // it, it would force it to be stored in memory, preventing the compiler from picking it apart - // and putting into registers. i.e. if we pass it as reference, it gets slow. - // This is what forces the skip_double, as well. - error_code error = slow_float_parsing(src, writer); - writer.skip_double(); + // NOTE: We do not pass a reference to the to slow_float_parsing. If we passed our writer + // reference to it, it would force it to be stored in memory, preventing the compiler from + // picking it apart and putting into registers. i.e. if we pass it as reference, + // it gets slow. + double d; + error_code error = slow_float_parsing(src, &d); + writer.append_double(d); return error; } // NOTE: it's weird that the simdjson_unlikely() only wraps half the if, but it seems to get slower any other @@ -16909,6 +16427,18 @@ namespace simdjson { return SUCCESS; } + // parse the number at src + // define JSON_TEST_NUMBERS for unit testing + // + // It is assumed that the number is followed by a structural ({,},],[) character + // or a white space character. If that is not the case (e.g., when the JSON + // document is made of a single number), then it is necessary to copy the + // content and append a space before calling this function. + // + // Our objective is accurate parsing (ULP of 0) at high speed. + template + simdjson_inline error_code parse_number(const uint8_t* const src, W& writer); + // for performance analysis, it is sometimes useful to skip parsing #ifdef SIMDJSON_SKIPNUMBERPARSING @@ -16987,11 +16517,11 @@ namespace simdjson { // The longest positive 64-bit number is 20 digits. // We do it this way so we don't trigger this branch unless we must. size_t longest_digit_count = negative ? 19 : 20; - if (digit_count > longest_digit_count) { return INVALID_NUMBER(src); } + if (digit_count > longest_digit_count) { return BIGINT_NUMBER(src); } if (digit_count == longest_digit_count) { if (negative) { // Anything negative above INT64_MAX+1 is invalid - if (i > uint64_t(INT64_MAX) + 1) { return INVALID_NUMBER(src); } + if (i > uint64_t(INT64_MAX) + 1) { return BIGINT_NUMBER(src); } WRITE_INTEGER(~i + 1, src, writer); if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return INVALID_NUMBER(src); } return SUCCESS; @@ -17011,7 +16541,7 @@ namespace simdjson { else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INVALID_NUMBER(src); } } - // Write unsigned if it doesn't fit in a signed integer. + // Write unsigned if it does not fit in a signed integer. if (i > uint64_t(INT64_MAX)) { WRITE_UNSIGNED(i, src, writer); } @@ -17464,19 +16994,32 @@ namespace simdjson { src += uint8_t(negative); const uint8_t* p = src; while (static_cast(*p - '0') <= 9) { p++; } + size_t digit_count = size_t(p - src); if (p == src) { return NUMBER_ERROR; } if (jsoncharutils::is_structural_or_whitespace(*p)) { + static const uint8_t* smaller_big_integer = reinterpret_cast("9223372036854775808"); // We have an integer. + if (simdjson_unlikely(digit_count > 20)) { + return number_type::big_integer; + } // If the number is negative and valid, it must be a signed integer. - if (negative) { return number_type::signed_integer; } + if (negative) { + if (simdjson_unlikely(digit_count > 19)) return number_type::big_integer; + if (simdjson_unlikely(digit_count == 19 && memcmp(src, smaller_big_integer, 19) > 0)) { + return number_type::big_integer; + } + return number_type::signed_integer; + } + // Let us check if we have a big integer (>=2**64). + static const uint8_t* two_to_sixtyfour = reinterpret_cast("18446744073709551616"); + if ((digit_count > 20) || (digit_count == 20 && memcmp(src, two_to_sixtyfour, 20) >= 0)) { + return number_type::big_integer; + } + // The number is positive and smaller than 18446744073709551616 (or 2**64). // We want values larger or equal to 9223372036854775808 to be unsigned // integers, and the other values to be signed integers. - int digit_count = int(p - src); - if (digit_count >= 19) { - const uint8_t* smaller_big_integer = reinterpret_cast("9223372036854775808"); - if ((digit_count >= 20) || (memcmp(src, smaller_big_integer, 19) >= 0)) { - return number_type::unsigned_integer; - } + if ((digit_count == 20) || (digit_count >= 19 && memcmp(src, smaller_big_integer, 19) >= 0)) { + return number_type::unsigned_integer; } return number_type::signed_integer; } @@ -17656,6 +17199,7 @@ namespace simdjson { case number_type::signed_integer: out << "integer in [-9223372036854775808,9223372036854775808)"; break; case number_type::unsigned_integer: out << "unsigned integer in [9223372036854775808,18446744073709551616)"; break; case number_type::floating_point_number: out << "floating-point number (binary64)"; break; + case number_type::big_integer: out << "big integer"; break; default: SIMDJSON_UNREACHABLE(); } return out; @@ -18081,10 +17625,10 @@ namespace simdjson { struct simd8 : base8 { static simdjson_inline simd8 splat(bool _value) { return _mm512_set1_epi8(uint8_t(-(!!_value))); } - simdjson_inline simd8() : base8() {} - simdjson_inline simd8(const __m512i _value) : base8(_value) {} + simdjson_inline simd8() : base8() {} + simdjson_inline simd8(const __m512i _value) : base8(_value) {} // Splat constructor - simdjson_inline simd8(bool _value) : base8(splat(_value)) {} + simdjson_inline simd8(bool _value) : base8(splat(_value)) {} simdjson_inline bool any() const { return !!_mm512_test_epi8_mask(*this, *this); } simdjson_inline simd8 operator~() const { return *this ^ true; } }; @@ -18447,13 +17991,13 @@ namespace simdjson { simdjson_inline internal::value128 full_multiplication(uint64_t value1, uint64_t value2) { internal::value128 answer; #if SIMDJSON_REGULAR_VISUAL_STUDIO || SIMDJSON_IS_32BITS -#ifdef _M_ARM64 +#if SIMDJSON_IS_ARM64 // ARM64 has native support for 64-bit multiplications, no need to emultate answer.high = __umulh(value1, value2); answer.low = value1 * value2; #else answer.low = _umul128(value1, value2, &answer.high); // _umul128 not available on ARM64 -#endif // _M_ARM64 +#endif // SIMDJSON_IS_ARM64 #else // SIMDJSON_REGULAR_VISUAL_STUDIO || SIMDJSON_IS_32BITS __uint128_t r = (static_cast<__uint128_t>(value1)) * value2; answer.low = uint64_t(r); @@ -18498,6 +18042,10 @@ namespace simdjson { /* amalgamation skipped (editor-only): #include "simdjson/arm64/begin.h" */ /* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_PPC64 */ /* amalgamation skipped (editor-only): #include "simdjson/ppc64/begin.h" */ +/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_LSX */ +/* amalgamation skipped (editor-only): #include "simdjson/lsx/begin.h" */ +/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_LASX */ +/* amalgamation skipped (editor-only): #include "simdjson/lasx/begin.h" */ /* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_FALLBACK */ /* amalgamation skipped (editor-only): #include "simdjson/fallback/begin.h" */ /* amalgamation skipped (editor-only): #else */ @@ -18518,7 +18066,8 @@ namespace simdjson { enum class number_type { floating_point_number = 1, /// a binary64 number signed_integer, /// a signed integer that fits in a 64-bit word using two's complement - unsigned_integer /// a positive integer larger or equal to 1<<63 + unsigned_integer, /// a positive integer larger or equal to 1<<63 + big_integer /// a big integer that does not fit in a 64-bit word }; } // namespace icelake @@ -18968,11 +18517,13 @@ namespace simdjson { #define WRITE_INTEGER(VALUE, SRC, WRITER) (found_integer((VALUE), (SRC)), (WRITER).append_s64((VALUE))) #define WRITE_UNSIGNED(VALUE, SRC, WRITER) (found_unsigned_integer((VALUE), (SRC)), (WRITER).append_u64((VALUE))) #define WRITE_DOUBLE(VALUE, SRC, WRITER) (found_float((VALUE), (SRC)), (WRITER).append_double((VALUE))) +#define BIGINT_NUMBER(SRC) (found_invalid_number((SRC)), BIGINT_ERROR) #else #define INVALID_NUMBER(SRC) (NUMBER_ERROR) #define WRITE_INTEGER(VALUE, SRC, WRITER) (WRITER).append_s64((VALUE)) #define WRITE_UNSIGNED(VALUE, SRC, WRITER) (WRITER).append_u64((VALUE)) #define WRITE_DOUBLE(VALUE, SRC, WRITER) (WRITER).append_double((VALUE)) +#define BIGINT_NUMBER(SRC) (BIGINT_ERROR) #endif namespace { @@ -19304,6 +18855,10 @@ namespace simdjson { return true; } + simdjson_inline bool is_digit(const uint8_t c) { + return static_cast(c - '0') <= 9; + } + simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t* const src, const uint8_t*& p, uint64_t& i, int64_t& exponent) { // we continue with the fiction that we have an integer. If the // floating point number is representable as x * 10^z for some integer @@ -19361,7 +18916,7 @@ namespace simdjson { // If there were more than 18 digits, we may have overflowed the integer. We have to do // something!!!! if (simdjson_unlikely(p > start_exp + 18)) { - // Skip leading zeroes: 1e000000000000000000001 is technically valid and doesn't overflow + // Skip leading zeroes: 1e000000000000000000001 is technically valid and does not overflow while (*start_exp == '0') { start_exp++; } // 19 digits could overflow int64_t and is kind of absurd anyway. We don't // support exponents smaller than -999,999,999,999,999,999 and bigger @@ -19383,6 +18938,23 @@ namespace simdjson { return SUCCESS; } + simdjson_inline bool check_if_integer(const uint8_t* const src, size_t max_length) { + const uint8_t* const srcend = src + max_length; + bool negative = (*src == '-'); // we can always read at least one character after the '-' + const uint8_t* p = src + uint8_t(negative); + if (p == srcend) { return false; } + if (*p == '0') { + ++p; + if (p == srcend) { return true; } + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return false; } + return true; + } + while (p != srcend && is_digit(*p)) { ++p; } + if (p == srcend) { return true; } + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return false; } + return true; + } + simdjson_inline size_t significant_digits(const uint8_t* start_digits, size_t digit_count) { // It is possible that the integer had an overflow. // We have to handle the case where we have 0.0000somenumber. @@ -19395,11 +18967,8 @@ namespace simdjson { } // unnamed namespace /** @private */ - template - error_code slow_float_parsing(simdjson_unused const uint8_t* src, W writer) { - double d; - if (parse_float_fallback(src, &d)) { - writer.append_double(d); + static error_code slow_float_parsing(simdjson_unused const uint8_t* src, double* answer) { + if (parse_float_fallback(src, answer)) { return SUCCESS; } return INVALID_NUMBER(src); @@ -19423,13 +18992,13 @@ namespace simdjson { // 10000000000000000000000000000000000000000000e+308 // 3.1415926535897932384626433832795028841971693993751 // - // NOTE: This makes a *copy* of the writer and passes it to slow_float_parsing. This happens - // because slow_float_parsing is a non-inlined function. If we passed our writer reference to - // it, it would force it to be stored in memory, preventing the compiler from picking it apart - // and putting into registers. i.e. if we pass it as reference, it gets slow. - // This is what forces the skip_double, as well. - error_code error = slow_float_parsing(src, writer); - writer.skip_double(); + // NOTE: We do not pass a reference to the to slow_float_parsing. If we passed our writer + // reference to it, it would force it to be stored in memory, preventing the compiler from + // picking it apart and putting into registers. i.e. if we pass it as reference, + // it gets slow. + double d; + error_code error = slow_float_parsing(src, &d); + writer.append_double(d); return error; } // NOTE: it's weird that the simdjson_unlikely() only wraps half the if, but it seems to get slower any other @@ -19461,6 +19030,18 @@ namespace simdjson { return SUCCESS; } + // parse the number at src + // define JSON_TEST_NUMBERS for unit testing + // + // It is assumed that the number is followed by a structural ({,},],[) character + // or a white space character. If that is not the case (e.g., when the JSON + // document is made of a single number), then it is necessary to copy the + // content and append a space before calling this function. + // + // Our objective is accurate parsing (ULP of 0) at high speed. + template + simdjson_inline error_code parse_number(const uint8_t* const src, W& writer); + // for performance analysis, it is sometimes useful to skip parsing #ifdef SIMDJSON_SKIPNUMBERPARSING @@ -19539,11 +19120,11 @@ namespace simdjson { // The longest positive 64-bit number is 20 digits. // We do it this way so we don't trigger this branch unless we must. size_t longest_digit_count = negative ? 19 : 20; - if (digit_count > longest_digit_count) { return INVALID_NUMBER(src); } + if (digit_count > longest_digit_count) { return BIGINT_NUMBER(src); } if (digit_count == longest_digit_count) { if (negative) { // Anything negative above INT64_MAX+1 is invalid - if (i > uint64_t(INT64_MAX) + 1) { return INVALID_NUMBER(src); } + if (i > uint64_t(INT64_MAX) + 1) { return BIGINT_NUMBER(src); } WRITE_INTEGER(~i + 1, src, writer); if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return INVALID_NUMBER(src); } return SUCCESS; @@ -19563,7 +19144,7 @@ namespace simdjson { else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INVALID_NUMBER(src); } } - // Write unsigned if it doesn't fit in a signed integer. + // Write unsigned if it does not fit in a signed integer. if (i > uint64_t(INT64_MAX)) { WRITE_UNSIGNED(i, src, writer); } @@ -20016,19 +19597,32 @@ namespace simdjson { src += uint8_t(negative); const uint8_t* p = src; while (static_cast(*p - '0') <= 9) { p++; } + size_t digit_count = size_t(p - src); if (p == src) { return NUMBER_ERROR; } if (jsoncharutils::is_structural_or_whitespace(*p)) { + static const uint8_t* smaller_big_integer = reinterpret_cast("9223372036854775808"); // We have an integer. + if (simdjson_unlikely(digit_count > 20)) { + return number_type::big_integer; + } // If the number is negative and valid, it must be a signed integer. - if (negative) { return number_type::signed_integer; } + if (negative) { + if (simdjson_unlikely(digit_count > 19)) return number_type::big_integer; + if (simdjson_unlikely(digit_count == 19 && memcmp(src, smaller_big_integer, 19) > 0)) { + return number_type::big_integer; + } + return number_type::signed_integer; + } + // Let us check if we have a big integer (>=2**64). + static const uint8_t* two_to_sixtyfour = reinterpret_cast("18446744073709551616"); + if ((digit_count > 20) || (digit_count == 20 && memcmp(src, two_to_sixtyfour, 20) >= 0)) { + return number_type::big_integer; + } + // The number is positive and smaller than 18446744073709551616 (or 2**64). // We want values larger or equal to 9223372036854775808 to be unsigned // integers, and the other values to be signed integers. - int digit_count = int(p - src); - if (digit_count >= 19) { - const uint8_t* smaller_big_integer = reinterpret_cast("9223372036854775808"); - if ((digit_count >= 20) || (memcmp(src, smaller_big_integer, 19) >= 0)) { - return number_type::unsigned_integer; - } + if ((digit_count == 20) || (digit_count >= 19 && memcmp(src, smaller_big_integer, 19) >= 0)) { + return number_type::unsigned_integer; } return number_type::signed_integer; } @@ -20208,6 +19802,7 @@ namespace simdjson { case number_type::signed_integer: out << "integer in [-9223372036854775808,9223372036854775808)"; break; case number_type::unsigned_integer: out << "unsigned integer in [9223372036854775808,18446744073709551616)"; break; case number_type::floating_point_number: out << "floating-point number (binary64)"; break; + case number_type::big_integer: out << "big integer"; break; default: SIMDJSON_UNREACHABLE(); } return out; @@ -20569,13 +20164,13 @@ namespace simdjson { simdjson_inline internal::value128 full_multiplication(uint64_t value1, uint64_t value2) { internal::value128 answer; #if SIMDJSON_REGULAR_VISUAL_STUDIO || SIMDJSON_IS_32BITS -#ifdef _M_ARM64 +#if SIMDJSON_IS_ARM64 // ARM64 has native support for 64-bit multiplications, no need to emultate answer.high = __umulh(value1, value2); answer.low = value1 * value2; #else answer.low = _umul128(value1, value2, &answer.high); // _umul128 not available on ARM64 -#endif // _M_ARM64 +#endif // SIMDJSON_IS_ARM64 #else // SIMDJSON_REGULAR_VISUAL_STUDIO || SIMDJSON_IS_32BITS __uint128_t r = (static_cast<__uint128_t>(value1)) * value2; answer.low = uint64_t(r); @@ -20692,11 +20287,11 @@ namespace simdjson { return (__m128i)vec_splats((unsigned char)(-(!!_value))); } - simdjson_inline simd8() : base8() {} - simdjson_inline simd8(const __m128i _value) + simdjson_inline simd8() : base8() {} + simdjson_inline simd8(const __m128i _value) : base8(_value) {} // Splat constructor - simdjson_inline simd8(bool _value) + simdjson_inline simd8(bool _value) : base8(splat(_value)) {} simdjson_inline int to_bitmask() const { @@ -21171,6 +20766,10 @@ namespace simdjson { /* amalgamation skipped (editor-only): #include "simdjson/arm64/begin.h" */ /* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_PPC64 */ /* amalgamation skipped (editor-only): #include "simdjson/ppc64/begin.h" */ +/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_LSX */ +/* amalgamation skipped (editor-only): #include "simdjson/lsx/begin.h" */ +/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_LASX */ +/* amalgamation skipped (editor-only): #include "simdjson/lasx/begin.h" */ /* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_FALLBACK */ /* amalgamation skipped (editor-only): #include "simdjson/fallback/begin.h" */ /* amalgamation skipped (editor-only): #else */ @@ -21191,7 +20790,8 @@ namespace simdjson { enum class number_type { floating_point_number = 1, /// a binary64 number signed_integer, /// a signed integer that fits in a 64-bit word using two's complement - unsigned_integer /// a positive integer larger or equal to 1<<63 + unsigned_integer, /// a positive integer larger or equal to 1<<63 + big_integer /// a big integer that does not fit in a 64-bit word }; } // namespace ppc64 @@ -21641,11 +21241,13 @@ namespace simdjson { #define WRITE_INTEGER(VALUE, SRC, WRITER) (found_integer((VALUE), (SRC)), (WRITER).append_s64((VALUE))) #define WRITE_UNSIGNED(VALUE, SRC, WRITER) (found_unsigned_integer((VALUE), (SRC)), (WRITER).append_u64((VALUE))) #define WRITE_DOUBLE(VALUE, SRC, WRITER) (found_float((VALUE), (SRC)), (WRITER).append_double((VALUE))) +#define BIGINT_NUMBER(SRC) (found_invalid_number((SRC)), BIGINT_ERROR) #else #define INVALID_NUMBER(SRC) (NUMBER_ERROR) #define WRITE_INTEGER(VALUE, SRC, WRITER) (WRITER).append_s64((VALUE)) #define WRITE_UNSIGNED(VALUE, SRC, WRITER) (WRITER).append_u64((VALUE)) #define WRITE_DOUBLE(VALUE, SRC, WRITER) (WRITER).append_double((VALUE)) +#define BIGINT_NUMBER(SRC) (BIGINT_ERROR) #endif namespace { @@ -21977,6 +21579,10 @@ namespace simdjson { return true; } + simdjson_inline bool is_digit(const uint8_t c) { + return static_cast(c - '0') <= 9; + } + simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t* const src, const uint8_t*& p, uint64_t& i, int64_t& exponent) { // we continue with the fiction that we have an integer. If the // floating point number is representable as x * 10^z for some integer @@ -22034,7 +21640,7 @@ namespace simdjson { // If there were more than 18 digits, we may have overflowed the integer. We have to do // something!!!! if (simdjson_unlikely(p > start_exp + 18)) { - // Skip leading zeroes: 1e000000000000000000001 is technically valid and doesn't overflow + // Skip leading zeroes: 1e000000000000000000001 is technically valid and does not overflow while (*start_exp == '0') { start_exp++; } // 19 digits could overflow int64_t and is kind of absurd anyway. We don't // support exponents smaller than -999,999,999,999,999,999 and bigger @@ -22056,6 +21662,23 @@ namespace simdjson { return SUCCESS; } + simdjson_inline bool check_if_integer(const uint8_t* const src, size_t max_length) { + const uint8_t* const srcend = src + max_length; + bool negative = (*src == '-'); // we can always read at least one character after the '-' + const uint8_t* p = src + uint8_t(negative); + if (p == srcend) { return false; } + if (*p == '0') { + ++p; + if (p == srcend) { return true; } + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return false; } + return true; + } + while (p != srcend && is_digit(*p)) { ++p; } + if (p == srcend) { return true; } + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return false; } + return true; + } + simdjson_inline size_t significant_digits(const uint8_t* start_digits, size_t digit_count) { // It is possible that the integer had an overflow. // We have to handle the case where we have 0.0000somenumber. @@ -22068,11 +21691,8 @@ namespace simdjson { } // unnamed namespace /** @private */ - template - error_code slow_float_parsing(simdjson_unused const uint8_t* src, W writer) { - double d; - if (parse_float_fallback(src, &d)) { - writer.append_double(d); + static error_code slow_float_parsing(simdjson_unused const uint8_t* src, double* answer) { + if (parse_float_fallback(src, answer)) { return SUCCESS; } return INVALID_NUMBER(src); @@ -22096,13 +21716,13 @@ namespace simdjson { // 10000000000000000000000000000000000000000000e+308 // 3.1415926535897932384626433832795028841971693993751 // - // NOTE: This makes a *copy* of the writer and passes it to slow_float_parsing. This happens - // because slow_float_parsing is a non-inlined function. If we passed our writer reference to - // it, it would force it to be stored in memory, preventing the compiler from picking it apart - // and putting into registers. i.e. if we pass it as reference, it gets slow. - // This is what forces the skip_double, as well. - error_code error = slow_float_parsing(src, writer); - writer.skip_double(); + // NOTE: We do not pass a reference to the to slow_float_parsing. If we passed our writer + // reference to it, it would force it to be stored in memory, preventing the compiler from + // picking it apart and putting into registers. i.e. if we pass it as reference, + // it gets slow. + double d; + error_code error = slow_float_parsing(src, &d); + writer.append_double(d); return error; } // NOTE: it's weird that the simdjson_unlikely() only wraps half the if, but it seems to get slower any other @@ -22134,6 +21754,18 @@ namespace simdjson { return SUCCESS; } + // parse the number at src + // define JSON_TEST_NUMBERS for unit testing + // + // It is assumed that the number is followed by a structural ({,},],[) character + // or a white space character. If that is not the case (e.g., when the JSON + // document is made of a single number), then it is necessary to copy the + // content and append a space before calling this function. + // + // Our objective is accurate parsing (ULP of 0) at high speed. + template + simdjson_inline error_code parse_number(const uint8_t* const src, W& writer); + // for performance analysis, it is sometimes useful to skip parsing #ifdef SIMDJSON_SKIPNUMBERPARSING @@ -22212,11 +21844,11 @@ namespace simdjson { // The longest positive 64-bit number is 20 digits. // We do it this way so we don't trigger this branch unless we must. size_t longest_digit_count = negative ? 19 : 20; - if (digit_count > longest_digit_count) { return INVALID_NUMBER(src); } + if (digit_count > longest_digit_count) { return BIGINT_NUMBER(src); } if (digit_count == longest_digit_count) { if (negative) { // Anything negative above INT64_MAX+1 is invalid - if (i > uint64_t(INT64_MAX) + 1) { return INVALID_NUMBER(src); } + if (i > uint64_t(INT64_MAX) + 1) { return BIGINT_NUMBER(src); } WRITE_INTEGER(~i + 1, src, writer); if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return INVALID_NUMBER(src); } return SUCCESS; @@ -22236,7 +21868,7 @@ namespace simdjson { else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INVALID_NUMBER(src); } } - // Write unsigned if it doesn't fit in a signed integer. + // Write unsigned if it does not fit in a signed integer. if (i > uint64_t(INT64_MAX)) { WRITE_UNSIGNED(i, src, writer); } @@ -22689,19 +22321,32 @@ namespace simdjson { src += uint8_t(negative); const uint8_t* p = src; while (static_cast(*p - '0') <= 9) { p++; } + size_t digit_count = size_t(p - src); if (p == src) { return NUMBER_ERROR; } if (jsoncharutils::is_structural_or_whitespace(*p)) { + static const uint8_t* smaller_big_integer = reinterpret_cast("9223372036854775808"); // We have an integer. + if (simdjson_unlikely(digit_count > 20)) { + return number_type::big_integer; + } // If the number is negative and valid, it must be a signed integer. - if (negative) { return number_type::signed_integer; } + if (negative) { + if (simdjson_unlikely(digit_count > 19)) return number_type::big_integer; + if (simdjson_unlikely(digit_count == 19 && memcmp(src, smaller_big_integer, 19) > 0)) { + return number_type::big_integer; + } + return number_type::signed_integer; + } + // Let us check if we have a big integer (>=2**64). + static const uint8_t* two_to_sixtyfour = reinterpret_cast("18446744073709551616"); + if ((digit_count > 20) || (digit_count == 20 && memcmp(src, two_to_sixtyfour, 20) >= 0)) { + return number_type::big_integer; + } + // The number is positive and smaller than 18446744073709551616 (or 2**64). // We want values larger or equal to 9223372036854775808 to be unsigned // integers, and the other values to be signed integers. - int digit_count = int(p - src); - if (digit_count >= 19) { - const uint8_t* smaller_big_integer = reinterpret_cast("9223372036854775808"); - if ((digit_count >= 20) || (memcmp(src, smaller_big_integer, 19) >= 0)) { - return number_type::unsigned_integer; - } + if ((digit_count == 20) || (digit_count >= 19 && memcmp(src, smaller_big_integer, 19) >= 0)) { + return number_type::unsigned_integer; } return number_type::signed_integer; } @@ -22881,6 +22526,7 @@ namespace simdjson { case number_type::signed_integer: out << "integer in [-9223372036854775808,9223372036854775808)"; break; case number_type::unsigned_integer: out << "unsigned integer in [9223372036854775808,18446744073709551616)"; break; case number_type::floating_point_number: out << "floating-point number (binary64)"; break; + case number_type::big_integer: out << "big integer"; break; default: SIMDJSON_UNREACHABLE(); } return out; @@ -23298,13 +22944,13 @@ namespace simdjson { simdjson_inline internal::value128 full_multiplication(uint64_t value1, uint64_t value2) { internal::value128 answer; #if SIMDJSON_REGULAR_VISUAL_STUDIO || SIMDJSON_IS_32BITS -#ifdef _M_ARM64 +#if SIMDJSON_IS_ARM64 // ARM64 has native support for 64-bit multiplications, no need to emultate answer.high = __umulh(value1, value2); answer.low = value1 * value2; #else answer.low = _umul128(value1, value2, &answer.high); // _umul128 not available on ARM64 -#endif // _M_ARM64 +#endif // SIMDJSON_IS_ARM64 #else // SIMDJSON_REGULAR_VISUAL_STUDIO || SIMDJSON_IS_32BITS __uint128_t r = (static_cast<__uint128_t>(value1)) * value2; answer.low = uint64_t(r); @@ -23384,10 +23030,10 @@ namespace simdjson { struct simd8 : base8 { static simdjson_inline simd8 splat(bool _value) { return _mm_set1_epi8(uint8_t(-(!!_value))); } - simdjson_inline simd8() : base8() {} - simdjson_inline simd8(const __m128i _value) : base8(_value) {} + simdjson_inline simd8() : base8() {} + simdjson_inline simd8(const __m128i _value) : base8(_value) {} // Splat constructor - simdjson_inline simd8(bool _value) : base8(splat(_value)) {} + simdjson_inline simd8(bool _value) : base8(splat(_value)) {} simdjson_inline int to_bitmask() const { return _mm_movemask_epi8(*this); } simdjson_inline bool any() const { return !_mm_testz_si128(*this, *this); } @@ -23812,10 +23458,10 @@ namespace simdjson { struct simd8 : base8 { static simdjson_inline simd8 splat(bool _value) { return _mm_set1_epi8(uint8_t(-(!!_value))); } - simdjson_inline simd8() : base8() {} - simdjson_inline simd8(const __m128i _value) : base8(_value) {} + simdjson_inline simd8() : base8() {} + simdjson_inline simd8(const __m128i _value) : base8(_value) {} // Splat constructor - simdjson_inline simd8(bool _value) : base8(splat(_value)) {} + simdjson_inline simd8(bool _value) : base8(splat(_value)) {} simdjson_inline int to_bitmask() const { return _mm_movemask_epi8(*this); } simdjson_inline bool any() const { return !_mm_testz_si128(*this, *this); } @@ -24161,6 +23807,10 @@ namespace simdjson { /* amalgamation skipped (editor-only): #include "simdjson/arm64/begin.h" */ /* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_PPC64 */ /* amalgamation skipped (editor-only): #include "simdjson/ppc64/begin.h" */ +/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_LSX */ +/* amalgamation skipped (editor-only): #include "simdjson/lsx/begin.h" */ +/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_LASX */ +/* amalgamation skipped (editor-only): #include "simdjson/lasx/begin.h" */ /* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_FALLBACK */ /* amalgamation skipped (editor-only): #include "simdjson/fallback/begin.h" */ /* amalgamation skipped (editor-only): #else */ @@ -24181,7 +23831,8 @@ namespace simdjson { enum class number_type { floating_point_number = 1, /// a binary64 number signed_integer, /// a signed integer that fits in a 64-bit word using two's complement - unsigned_integer /// a positive integer larger or equal to 1<<63 + unsigned_integer, /// a positive integer larger or equal to 1<<63 + big_integer /// a big integer that does not fit in a 64-bit word }; } // namespace westmere @@ -24631,11 +24282,13 @@ namespace simdjson { #define WRITE_INTEGER(VALUE, SRC, WRITER) (found_integer((VALUE), (SRC)), (WRITER).append_s64((VALUE))) #define WRITE_UNSIGNED(VALUE, SRC, WRITER) (found_unsigned_integer((VALUE), (SRC)), (WRITER).append_u64((VALUE))) #define WRITE_DOUBLE(VALUE, SRC, WRITER) (found_float((VALUE), (SRC)), (WRITER).append_double((VALUE))) +#define BIGINT_NUMBER(SRC) (found_invalid_number((SRC)), BIGINT_ERROR) #else #define INVALID_NUMBER(SRC) (NUMBER_ERROR) #define WRITE_INTEGER(VALUE, SRC, WRITER) (WRITER).append_s64((VALUE)) #define WRITE_UNSIGNED(VALUE, SRC, WRITER) (WRITER).append_u64((VALUE)) #define WRITE_DOUBLE(VALUE, SRC, WRITER) (WRITER).append_double((VALUE)) +#define BIGINT_NUMBER(SRC) (BIGINT_ERROR) #endif namespace { @@ -24967,6 +24620,10 @@ namespace simdjson { return true; } + simdjson_inline bool is_digit(const uint8_t c) { + return static_cast(c - '0') <= 9; + } + simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t* const src, const uint8_t*& p, uint64_t& i, int64_t& exponent) { // we continue with the fiction that we have an integer. If the // floating point number is representable as x * 10^z for some integer @@ -25024,7 +24681,7 @@ namespace simdjson { // If there were more than 18 digits, we may have overflowed the integer. We have to do // something!!!! if (simdjson_unlikely(p > start_exp + 18)) { - // Skip leading zeroes: 1e000000000000000000001 is technically valid and doesn't overflow + // Skip leading zeroes: 1e000000000000000000001 is technically valid and does not overflow while (*start_exp == '0') { start_exp++; } // 19 digits could overflow int64_t and is kind of absurd anyway. We don't // support exponents smaller than -999,999,999,999,999,999 and bigger @@ -25046,6 +24703,23 @@ namespace simdjson { return SUCCESS; } + simdjson_inline bool check_if_integer(const uint8_t* const src, size_t max_length) { + const uint8_t* const srcend = src + max_length; + bool negative = (*src == '-'); // we can always read at least one character after the '-' + const uint8_t* p = src + uint8_t(negative); + if (p == srcend) { return false; } + if (*p == '0') { + ++p; + if (p == srcend) { return true; } + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return false; } + return true; + } + while (p != srcend && is_digit(*p)) { ++p; } + if (p == srcend) { return true; } + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return false; } + return true; + } + simdjson_inline size_t significant_digits(const uint8_t* start_digits, size_t digit_count) { // It is possible that the integer had an overflow. // We have to handle the case where we have 0.0000somenumber. @@ -25058,11 +24732,8 @@ namespace simdjson { } // unnamed namespace /** @private */ - template - error_code slow_float_parsing(simdjson_unused const uint8_t* src, W writer) { - double d; - if (parse_float_fallback(src, &d)) { - writer.append_double(d); + static error_code slow_float_parsing(simdjson_unused const uint8_t* src, double* answer) { + if (parse_float_fallback(src, answer)) { return SUCCESS; } return INVALID_NUMBER(src); @@ -25086,13 +24757,13 @@ namespace simdjson { // 10000000000000000000000000000000000000000000e+308 // 3.1415926535897932384626433832795028841971693993751 // - // NOTE: This makes a *copy* of the writer and passes it to slow_float_parsing. This happens - // because slow_float_parsing is a non-inlined function. If we passed our writer reference to - // it, it would force it to be stored in memory, preventing the compiler from picking it apart - // and putting into registers. i.e. if we pass it as reference, it gets slow. - // This is what forces the skip_double, as well. - error_code error = slow_float_parsing(src, writer); - writer.skip_double(); + // NOTE: We do not pass a reference to the to slow_float_parsing. If we passed our writer + // reference to it, it would force it to be stored in memory, preventing the compiler from + // picking it apart and putting into registers. i.e. if we pass it as reference, + // it gets slow. + double d; + error_code error = slow_float_parsing(src, &d); + writer.append_double(d); return error; } // NOTE: it's weird that the simdjson_unlikely() only wraps half the if, but it seems to get slower any other @@ -25124,6 +24795,18 @@ namespace simdjson { return SUCCESS; } + // parse the number at src + // define JSON_TEST_NUMBERS for unit testing + // + // It is assumed that the number is followed by a structural ({,},],[) character + // or a white space character. If that is not the case (e.g., when the JSON + // document is made of a single number), then it is necessary to copy the + // content and append a space before calling this function. + // + // Our objective is accurate parsing (ULP of 0) at high speed. + template + simdjson_inline error_code parse_number(const uint8_t* const src, W& writer); + // for performance analysis, it is sometimes useful to skip parsing #ifdef SIMDJSON_SKIPNUMBERPARSING @@ -25202,11 +24885,11 @@ namespace simdjson { // The longest positive 64-bit number is 20 digits. // We do it this way so we don't trigger this branch unless we must. size_t longest_digit_count = negative ? 19 : 20; - if (digit_count > longest_digit_count) { return INVALID_NUMBER(src); } + if (digit_count > longest_digit_count) { return BIGINT_NUMBER(src); } if (digit_count == longest_digit_count) { if (negative) { // Anything negative above INT64_MAX+1 is invalid - if (i > uint64_t(INT64_MAX) + 1) { return INVALID_NUMBER(src); } + if (i > uint64_t(INT64_MAX) + 1) { return BIGINT_NUMBER(src); } WRITE_INTEGER(~i + 1, src, writer); if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return INVALID_NUMBER(src); } return SUCCESS; @@ -25226,7 +24909,7 @@ namespace simdjson { else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INVALID_NUMBER(src); } } - // Write unsigned if it doesn't fit in a signed integer. + // Write unsigned if it does not fit in a signed integer. if (i > uint64_t(INT64_MAX)) { WRITE_UNSIGNED(i, src, writer); } @@ -25679,19 +25362,32 @@ namespace simdjson { src += uint8_t(negative); const uint8_t* p = src; while (static_cast(*p - '0') <= 9) { p++; } + size_t digit_count = size_t(p - src); if (p == src) { return NUMBER_ERROR; } if (jsoncharutils::is_structural_or_whitespace(*p)) { + static const uint8_t* smaller_big_integer = reinterpret_cast("9223372036854775808"); // We have an integer. + if (simdjson_unlikely(digit_count > 20)) { + return number_type::big_integer; + } // If the number is negative and valid, it must be a signed integer. - if (negative) { return number_type::signed_integer; } + if (negative) { + if (simdjson_unlikely(digit_count > 19)) return number_type::big_integer; + if (simdjson_unlikely(digit_count == 19 && memcmp(src, smaller_big_integer, 19) > 0)) { + return number_type::big_integer; + } + return number_type::signed_integer; + } + // Let us check if we have a big integer (>=2**64). + static const uint8_t* two_to_sixtyfour = reinterpret_cast("18446744073709551616"); + if ((digit_count > 20) || (digit_count == 20 && memcmp(src, two_to_sixtyfour, 20) >= 0)) { + return number_type::big_integer; + } + // The number is positive and smaller than 18446744073709551616 (or 2**64). // We want values larger or equal to 9223372036854775808 to be unsigned // integers, and the other values to be signed integers. - int digit_count = int(p - src); - if (digit_count >= 19) { - const uint8_t* smaller_big_integer = reinterpret_cast("9223372036854775808"); - if ((digit_count >= 20) || (memcmp(src, smaller_big_integer, 19) >= 0)) { - return number_type::unsigned_integer; - } + if ((digit_count == 20) || (digit_count >= 19 && memcmp(src, smaller_big_integer, 19) >= 0)) { + return number_type::unsigned_integer; } return number_type::signed_integer; } @@ -25871,6 +25567,7 @@ namespace simdjson { case number_type::signed_integer: out << "integer in [-9223372036854775808,9223372036854775808)"; break; case number_type::unsigned_integer: out << "unsigned integer in [9223372036854775808,18446744073709551616)"; break; case number_type::floating_point_number: out << "floating-point number (binary64)"; break; + case number_type::big_integer: out << "big integer"; break; default: SIMDJSON_UNREACHABLE(); } return out; @@ -25992,6 +25689,5040 @@ SIMDJSON_UNTARGET_REGION #endif // SIMDJSON_WESTMERE_H /* end file simdjson/westmere.h */ +#elif SIMDJSON_BUILTIN_IMPLEMENTATION_IS(lsx) +/* including simdjson/lsx.h: #include "simdjson/lsx.h" */ +/* begin file simdjson/lsx.h */ +#ifndef SIMDJSON_LSX_H +#define SIMDJSON_LSX_H + +/* including simdjson/lsx/begin.h: #include "simdjson/lsx/begin.h" */ +/* begin file simdjson/lsx/begin.h */ +/* defining SIMDJSON_IMPLEMENTATION to "lsx" */ +#define SIMDJSON_IMPLEMENTATION lsx +/* including simdjson/lsx/base.h: #include "simdjson/lsx/base.h" */ +/* begin file simdjson/lsx/base.h */ +#ifndef SIMDJSON_LSX_BASE_H +#define SIMDJSON_LSX_BASE_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/base.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + /** + * Implementation for LSX. + */ + namespace lsx { + + class implementation; + + namespace { + namespace simd { + template struct simd8; + template struct simd8x64; + } // namespace simd + } // unnamed namespace + + } // namespace lsx +} // namespace simdjson + +#endif // SIMDJSON_LSX_BASE_H +/* end file simdjson/lsx/base.h */ +/* including simdjson/lsx/intrinsics.h: #include "simdjson/lsx/intrinsics.h" */ +/* begin file simdjson/lsx/intrinsics.h */ +#ifndef SIMDJSON_LSX_INTRINSICS_H +#define SIMDJSON_LSX_INTRINSICS_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/lsx/base.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +// This should be the correct header whether +// you use visual studio or other compilers. +#include + +static_assert(sizeof(__m128i) <= simdjson::SIMDJSON_PADDING, "insufficient padding for LoongArch SX"); + +#endif // SIMDJSON_LSX_INTRINSICS_H +/* end file simdjson/lsx/intrinsics.h */ +/* including simdjson/lsx/bitmanipulation.h: #include "simdjson/lsx/bitmanipulation.h" */ +/* begin file simdjson/lsx/bitmanipulation.h */ +#ifndef SIMDJSON_LSX_BITMANIPULATION_H +#define SIMDJSON_LSX_BITMANIPULATION_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/lsx/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/lsx/intrinsics.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/lsx/bitmask.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lsx { + namespace { + + // We sometimes call trailing_zero on inputs that are zero, + // but the algorithms do not end up using the returned value. + // Sadly, sanitizers are not smart enough to figure it out. + SIMDJSON_NO_SANITIZE_UNDEFINED + // This function can be used safely even if not all bytes have been + // initialized. + // See issue https://github.com/simdjson/simdjson/issues/1965 + SIMDJSON_NO_SANITIZE_MEMORY + simdjson_inline int trailing_zeroes(uint64_t input_num) { + return __builtin_ctzll(input_num); + } + + /* result might be undefined when input_num is zero */ + simdjson_inline uint64_t clear_lowest_bit(uint64_t input_num) { + return input_num & (input_num - 1); + } + + /* result might be undefined when input_num is zero */ + simdjson_inline int leading_zeroes(uint64_t input_num) { + return __builtin_clzll(input_num); + } + + /* result might be undefined when input_num is zero */ + simdjson_inline int count_ones(uint64_t input_num) { + return __lsx_vpickve2gr_w(__lsx_vpcnt_d(__m128i(v2u64{ input_num, 0 })), 0); + } + + simdjson_inline bool add_overflow(uint64_t value1, uint64_t value2, uint64_t* result) { + return __builtin_uaddll_overflow(value1, value2, + reinterpret_cast(result)); + } + + } // unnamed namespace + } // namespace lsx +} // namespace simdjson + +#endif // SIMDJSON_LSX_BITMANIPULATION_H +/* end file simdjson/lsx/bitmanipulation.h */ +/* including simdjson/lsx/bitmask.h: #include "simdjson/lsx/bitmask.h" */ +/* begin file simdjson/lsx/bitmask.h */ +#ifndef SIMDJSON_LSX_BITMASK_H +#define SIMDJSON_LSX_BITMASK_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/lsx/base.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lsx { + namespace { + + // + // Perform a "cumulative bitwise xor," flipping bits each time a 1 is encountered. + // + // For example, prefix_xor(00100100) == 00011100 + // + simdjson_inline uint64_t prefix_xor(uint64_t bitmask) { + bitmask ^= bitmask << 1; + bitmask ^= bitmask << 2; + bitmask ^= bitmask << 4; + bitmask ^= bitmask << 8; + bitmask ^= bitmask << 16; + bitmask ^= bitmask << 32; + return bitmask; + } + + } // unnamed namespace + } // namespace lsx +} // namespace simdjson + +#endif +/* end file simdjson/lsx/bitmask.h */ +/* including simdjson/lsx/numberparsing_defs.h: #include "simdjson/lsx/numberparsing_defs.h" */ +/* begin file simdjson/lsx/numberparsing_defs.h */ +#ifndef SIMDJSON_LSX_NUMBERPARSING_DEFS_H +#define SIMDJSON_LSX_NUMBERPARSING_DEFS_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/lsx/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/lsx/intrinsics.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +#include + +namespace simdjson { + namespace lsx { + namespace numberparsing { + + // we don't have appropriate instructions, so let us use a scalar function + // credit: https://johnnylee-sde.github.io/Fast-numeric-string-to-int/ + /** @private */ + static simdjson_inline uint32_t parse_eight_digits_unrolled(const uint8_t* chars) { + uint64_t val; + std::memcpy(&val, chars, sizeof(uint64_t)); + val = (val & 0x0F0F0F0F0F0F0F0F) * 2561 >> 8; + val = (val & 0x00FF00FF00FF00FF) * 6553601 >> 16; + return uint32_t((val & 0x0000FFFF0000FFFF) * 42949672960001 >> 32); + } + + simdjson_inline internal::value128 full_multiplication(uint64_t value1, uint64_t value2) { + internal::value128 answer; + __uint128_t r = (static_cast<__uint128_t>(value1)) * value2; + answer.low = uint64_t(r); + answer.high = uint64_t(r >> 64); + return answer; + } + + } // namespace numberparsing + } // namespace lsx +} // namespace simdjson + +#define SIMDJSON_SWAR_NUMBER_PARSING 1 + +#endif // SIMDJSON_LSX_NUMBERPARSING_DEFS_H +/* end file simdjson/lsx/numberparsing_defs.h */ +/* including simdjson/lsx/simd.h: #include "simdjson/lsx/simd.h" */ +/* begin file simdjson/lsx/simd.h */ +#ifndef SIMDJSON_LSX_SIMD_H +#define SIMDJSON_LSX_SIMD_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/lsx/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/lsx/bitmanipulation.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/simdprune_tables.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lsx { + namespace { + namespace simd { + + // Forward-declared so they can be used by splat and friends. + template + struct base { + __m128i value; + + // Zero constructor + simdjson_inline base() : value{ __m128i() } {} + + // Conversion from SIMD register + simdjson_inline base(const __m128i _value) : value(_value) {} + + // Conversion to SIMD register + simdjson_inline operator const __m128i& () const { return this->value; } + simdjson_inline operator __m128i& () { return this->value; } + simdjson_inline operator const v16i8& () const { return (v16i8&)this->value; } + simdjson_inline operator v16i8& () { return (v16i8&)this->value; } + + // Bit operations + simdjson_inline Child operator|(const Child other) const { return __lsx_vor_v(*this, other); } + simdjson_inline Child operator&(const Child other) const { return __lsx_vand_v(*this, other); } + simdjson_inline Child operator^(const Child other) const { return __lsx_vxor_v(*this, other); } + simdjson_inline Child bit_andnot(const Child other) const { return __lsx_vandn_v(other, *this); } + simdjson_inline Child& operator|=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast | other; return *this_cast; } + simdjson_inline Child& operator&=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast & other; return *this_cast; } + simdjson_inline Child& operator^=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast ^ other; return *this_cast; } + }; + + // Forward-declared so they can be used by splat and friends. + template + struct simd8; + + template> + struct base8 : base> { + simdjson_inline base8() : base>() {} + simdjson_inline base8(const __m128i _value) : base>(_value) {} + + friend simdjson_really_inline Mask operator==(const simd8 lhs, const simd8 rhs) { return __lsx_vseq_b(lhs, rhs); } + + static const int SIZE = sizeof(base>::value); + + template + simdjson_inline simd8 prev(const simd8 prev_chunk) const { + return __lsx_vor_v(__lsx_vbsll_v(*this, N), __lsx_vbsrl_v(prev_chunk, 16 - N)); + } + }; + + // SIMD byte mask type (returned by things like eq and gt) + template<> + struct simd8 : base8 { + static simdjson_inline simd8 splat(bool _value) { + return __lsx_vreplgr2vr_b(uint8_t(-(!!_value))); + } + + simdjson_inline simd8() : base8() {} + simdjson_inline simd8(const __m128i _value) : base8(_value) {} + // Splat constructor + simdjson_inline simd8(bool _value) : base8(splat(_value)) {} + + simdjson_inline int to_bitmask() const { return __lsx_vpickve2gr_w(__lsx_vmskltz_b(*this), 0); } + simdjson_inline bool any() const { return 0 == __lsx_vpickve2gr_hu(__lsx_vmsknz_b(*this), 0); } + simdjson_inline simd8 operator~() const { return *this ^ true; } + }; + + template + struct base8_numeric : base8 { + static simdjson_inline simd8 splat(T _value) { return __lsx_vreplgr2vr_b(_value); } + static simdjson_inline simd8 zero() { return __lsx_vldi(0); } + static simdjson_inline simd8 load(const T values[16]) { + return __lsx_vld(reinterpret_cast(values), 0); + } + // Repeat 16 values as many times as necessary (usually for lookup tables) + static simdjson_inline simd8 repeat_16( + T v0, T v1, T v2, T v3, T v4, T v5, T v6, T v7, + T v8, T v9, T v10, T v11, T v12, T v13, T v14, T v15 + ) { + return simd8( + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15 + ); + } + + simdjson_inline base8_numeric() : base8() {} + simdjson_inline base8_numeric(const __m128i _value) : base8(_value) {} + + // Store to array + simdjson_inline void store(T dst[16]) const { + return __lsx_vst(*this, reinterpret_cast<__m128i*>(dst), 0); + } + + // Addition/subtraction are the same for signed and unsigned + simdjson_inline simd8 operator+(const simd8 other) const { return __lsx_vadd_b(*this, other); } + simdjson_inline simd8 operator-(const simd8 other) const { return __lsx_vsub_b(*this, other); } + simdjson_inline simd8& operator+=(const simd8 other) { *this = *this + other; return *static_cast*>(this); } + simdjson_inline simd8& operator-=(const simd8 other) { *this = *this - other; return *static_cast*>(this); } + + // Override to distinguish from bool version + simdjson_inline simd8 operator~() const { return *this ^ 0xFFu; } + + // Perform a lookup assuming the value is between 0 and 16 (undefined behavior for out of range values) + template + simdjson_inline simd8 lookup_16(simd8 lookup_table) const { + return __lsx_vshuf_b(lookup_table, lookup_table, *this); + } + + // Copies to 'output" all bytes corresponding to a 0 in the mask (interpreted as a bitset). + // Passing a 0 value for mask would be equivalent to writing out every byte to output. + // Only the first 16 - count_ones(mask) bytes of the result are significant but 16 bytes + // get written. + template + simdjson_inline void compress(uint16_t mask, L* output) const { + using internal::thintable_epi8; + using internal::BitsSetTable256mul2; + using internal::pshufb_combine_table; + // this particular implementation was inspired by haswell + // lsx do it in 2 steps, first 8 bytes and then second 8 bytes... + uint8_t mask1 = uint8_t(mask); // least significant 8 bits + uint8_t mask2 = uint8_t(mask >> 8); // second least significant 8 bits + // next line just loads the 64-bit values thintable_epi8[mask1] and + // thintable_epi8[mask2] into a 128-bit register. + __m128i shufmask = { int64_t(thintable_epi8[mask1]), int64_t(thintable_epi8[mask2]) + 0x0808080808080808 }; + // this is the version "nearly pruned" + __m128i pruned = __lsx_vshuf_b(*this, *this, shufmask); + // we still need to put the pieces back together. + // we compute the popcount of the first words: + int pop1 = BitsSetTable256mul2[mask1]; + // then load the corresponding mask + __m128i compactmask = __lsx_vldx(reinterpret_cast(reinterpret_cast(pshufb_combine_table)), pop1 * 8); + __m128i answer = __lsx_vshuf_b(pruned, pruned, compactmask); + __lsx_vst(answer, reinterpret_cast(output), 0); + } + + template + simdjson_inline simd8 lookup_16( + L replace0, L replace1, L replace2, L replace3, + L replace4, L replace5, L replace6, L replace7, + L replace8, L replace9, L replace10, L replace11, + L replace12, L replace13, L replace14, L replace15) const { + return lookup_16(simd8::repeat_16( + replace0, replace1, replace2, replace3, + replace4, replace5, replace6, replace7, + replace8, replace9, replace10, replace11, + replace12, replace13, replace14, replace15 + )); + } + }; + + // Signed bytes + template<> + struct simd8 : base8_numeric { + simdjson_inline simd8() : base8_numeric() {} + simdjson_inline simd8(const __m128i _value) : base8_numeric(_value) {} + // Splat constructor + simdjson_inline simd8(int8_t _value) : simd8(splat(_value)) {} + // Array constructor + simdjson_inline simd8(const int8_t values[16]) : simd8(load(values)) {} + // Member-by-member initialization + simdjson_inline simd8( + int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, + int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 + ) : simd8({ + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10,v11,v12,v13,v14,v15 + }) {} + // Repeat 16 values as many times as necessary (usually for lookup tables) + simdjson_inline static simd8 repeat_16( + int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, + int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 + ) { + return simd8( + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15 + ); + } + + // Order-sensitive comparisons + simdjson_inline simd8 max_val(const simd8 other) const { return __lsx_vmax_b(*this, other); } + simdjson_inline simd8 min_val(const simd8 other) const { return __lsx_vmin_b(*this, other); } + simdjson_inline simd8 operator>(const simd8 other) const { return __lsx_vslt_b(other, *this); } + simdjson_inline simd8 operator<(const simd8 other) const { return __lsx_vslt_b(*this, other); } + }; + + // Unsigned bytes + template<> + struct simd8 : base8_numeric { + simdjson_inline simd8() : base8_numeric() {} + simdjson_inline simd8(const __m128i _value) : base8_numeric(_value) {} + // Splat constructor + simdjson_inline simd8(uint8_t _value) : simd8(splat(_value)) {} + // Array constructor + simdjson_inline simd8(const uint8_t values[16]) : simd8(load(values)) {} + // Member-by-member initialization + simdjson_inline simd8( + uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, + uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 + ) : simd8(__m128i(v16u8{ + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10,v11,v12,v13,v14,v15 + })) {} + // Repeat 16 values as many times as necessary (usually for lookup tables) + simdjson_inline static simd8 repeat_16( + uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, + uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 + ) { + return simd8( + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15 + ); + } + + // Saturated math + simdjson_inline simd8 saturating_add(const simd8 other) const { return __lsx_vsadd_bu(*this, other); } + simdjson_inline simd8 saturating_sub(const simd8 other) const { return __lsx_vssub_bu(*this, other); } + + // Order-specific operations + simdjson_inline simd8 max_val(const simd8 other) const { return __lsx_vmax_bu(*this, other); } + simdjson_inline simd8 min_val(const simd8 other) const { return __lsx_vmin_bu(other, *this); } + // Same as >, but only guarantees true is nonzero (< guarantees true = -1) + simdjson_inline simd8 gt_bits(const simd8 other) const { return this->saturating_sub(other); } + // Same as <, but only guarantees true is nonzero (< guarantees true = -1) + simdjson_inline simd8 lt_bits(const simd8 other) const { return other.saturating_sub(*this); } + simdjson_inline simd8 operator<=(const simd8 other) const { return other.max_val(*this) == other; } + simdjson_inline simd8 operator>=(const simd8 other) const { return other.min_val(*this) == other; } + simdjson_inline simd8 operator>(const simd8 other) const { return this->gt_bits(other).any_bits_set(); } + simdjson_inline simd8 operator<(const simd8 other) const { return this->lt_bits(other).any_bits_set(); } + + // Bit-specific operations + simdjson_inline simd8 bits_not_set() const { return *this == uint8_t(0); } + simdjson_inline simd8 bits_not_set(simd8 bits) const { return (*this & bits).bits_not_set(); } + simdjson_inline simd8 any_bits_set() const { return ~this->bits_not_set(); } + simdjson_inline simd8 any_bits_set(simd8 bits) const { return ~this->bits_not_set(bits); } + simdjson_inline bool is_ascii() const { return 0 == __lsx_vpickve2gr_w(__lsx_vmskltz_b(*this), 0); } + simdjson_inline bool bits_not_set_anywhere() const { return 0 == __lsx_vpickve2gr_hu(__lsx_vmsknz_b(*this), 0); } + simdjson_inline bool any_bits_set_anywhere() const { return !bits_not_set_anywhere(); } + simdjson_inline bool bits_not_set_anywhere(simd8 bits) const { + return 0 == __lsx_vpickve2gr_hu(__lsx_vmsknz_b(__lsx_vand_v(*this, bits)), 0); + } + simdjson_inline bool any_bits_set_anywhere(simd8 bits) const { return !bits_not_set_anywhere(bits); } + template + simdjson_inline simd8 shr() const { return simd8(__lsx_vsrli_b(*this, N)); } + template + simdjson_inline simd8 shl() const { return simd8(__lsx_vslli_b(*this, N)); } + }; + + template + struct simd8x64 { + static constexpr int NUM_CHUNKS = 64 / sizeof(simd8); + static_assert(NUM_CHUNKS == 4, "LSX kernel should use four registers per 64-byte block."); + const simd8 chunks[NUM_CHUNKS]; + + simd8x64(const simd8x64& o) = delete; // no copy allowed + simd8x64& operator=(const simd8& other) = delete; // no assignment allowed + simd8x64() = delete; // no default constructor allowed + + simdjson_inline simd8x64(const simd8 chunk0, const simd8 chunk1, const simd8 chunk2, const simd8 chunk3) : chunks{ chunk0, chunk1, chunk2, chunk3 } {} + simdjson_inline simd8x64(const T ptr[64]) : chunks{ simd8::load(ptr), simd8::load(ptr + 16), simd8::load(ptr + 32), simd8::load(ptr + 48) } {} + + simdjson_inline uint64_t compress(uint64_t mask, T* output) const { + uint16_t mask1 = uint16_t(mask); + uint16_t mask2 = uint16_t(mask >> 16); + uint16_t mask3 = uint16_t(mask >> 32); + uint16_t mask4 = uint16_t(mask >> 48); + __m128i zcnt = __lsx_vpcnt_h(__m128i(v2u64{ ~mask, 0 })); + uint64_t zcnt1 = __lsx_vpickve2gr_hu(zcnt, 0); + uint64_t zcnt2 = __lsx_vpickve2gr_hu(zcnt, 1); + uint64_t zcnt3 = __lsx_vpickve2gr_hu(zcnt, 2); + uint64_t zcnt4 = __lsx_vpickve2gr_hu(zcnt, 3); + uint8_t* voutput = reinterpret_cast(output); + // There should be a critical value which processes in scaler is faster. + if (zcnt1) + this->chunks[0].compress(mask1, reinterpret_cast(voutput)); + voutput += zcnt1; + if (zcnt2) + this->chunks[1].compress(mask2, reinterpret_cast(voutput)); + voutput += zcnt2; + if (zcnt3) + this->chunks[2].compress(mask3, reinterpret_cast(voutput)); + voutput += zcnt3; + if (zcnt4) + this->chunks[3].compress(mask4, reinterpret_cast(voutput)); + voutput += zcnt4; + return reinterpret_cast(voutput) - reinterpret_cast(output); + } + + simdjson_inline void store(T ptr[64]) const { + this->chunks[0].store(ptr + sizeof(simd8) * 0); + this->chunks[1].store(ptr + sizeof(simd8) * 1); + this->chunks[2].store(ptr + sizeof(simd8) * 2); + this->chunks[3].store(ptr + sizeof(simd8) * 3); + } + + simdjson_inline uint64_t to_bitmask() const { + __m128i mask1 = __lsx_vmskltz_b(this->chunks[0]); + __m128i mask2 = __lsx_vmskltz_b(this->chunks[1]); + __m128i mask3 = __lsx_vmskltz_b(this->chunks[2]); + __m128i mask4 = __lsx_vmskltz_b(this->chunks[3]); + mask1 = __lsx_vilvl_h(mask2, mask1); + mask2 = __lsx_vilvl_h(mask4, mask3); + return __lsx_vpickve2gr_du(__lsx_vilvl_w(mask2, mask1), 0); + } + + simdjson_inline simd8 reduce_or() const { + return (this->chunks[0] | this->chunks[1]) | (this->chunks[2] | this->chunks[3]); + } + + simdjson_inline uint64_t eq(const T m) const { + const simd8 mask = simd8::splat(m); + return simd8x64( + this->chunks[0] == mask, + this->chunks[1] == mask, + this->chunks[2] == mask, + this->chunks[3] == mask + ).to_bitmask(); + } + + simdjson_inline uint64_t eq(const simd8x64& other) const { + return simd8x64( + this->chunks[0] == other.chunks[0], + this->chunks[1] == other.chunks[1], + this->chunks[2] == other.chunks[2], + this->chunks[3] == other.chunks[3] + ).to_bitmask(); + } + + simdjson_inline uint64_t lteq(const T m) const { + const simd8 mask = simd8::splat(m); + return simd8x64( + this->chunks[0] <= mask, + this->chunks[1] <= mask, + this->chunks[2] <= mask, + this->chunks[3] <= mask + ).to_bitmask(); + } + }; // struct simd8x64 + + } // namespace simd + } // unnamed namespace + } // namespace lsx +} // namespace simdjson + +#endif // SIMDJSON_LSX_SIMD_H +/* end file simdjson/lsx/simd.h */ +/* including simdjson/lsx/stringparsing_defs.h: #include "simdjson/lsx/stringparsing_defs.h" */ +/* begin file simdjson/lsx/stringparsing_defs.h */ +#ifndef SIMDJSON_LSX_STRINGPARSING_DEFS_H +#define SIMDJSON_LSX_STRINGPARSING_DEFS_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/lsx/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/lsx/simd.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/lsx/bitmanipulation.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lsx { + namespace { + + using namespace simd; + + // Holds backslashes and quotes locations. + struct backslash_and_quote { + public: + static constexpr uint32_t BYTES_PROCESSED = 32; + simdjson_inline static backslash_and_quote copy_and_find(const uint8_t* src, uint8_t* dst); + + simdjson_inline bool has_quote_first() { return ((bs_bits - 1) & quote_bits) != 0; } + simdjson_inline bool has_backslash() { return bs_bits != 0; } + simdjson_inline int quote_index() { return trailing_zeroes(quote_bits); } + simdjson_inline int backslash_index() { return trailing_zeroes(bs_bits); } + + uint32_t bs_bits; + uint32_t quote_bits; + }; // struct backslash_and_quote + + simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8_t* src, uint8_t* dst) { + // this can read up to 31 bytes beyond the buffer size, but we require + // SIMDJSON_PADDING of padding + static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "backslash and quote finder must process fewer than SIMDJSON_PADDING bytes"); + simd8 v0(src); + simd8 v1(src + sizeof(v0)); + v0.store(dst); + v1.store(dst + sizeof(v0)); + + // Getting a 64-bit bitmask is much cheaper than multiple 16-bit bitmasks on LSX; therefore, we + // smash them together into a 64-byte mask and get the bitmask from there. + uint64_t bs_and_quote = simd8x64(v0 == '\\', v1 == '\\', v0 == '"', v1 == '"').to_bitmask(); + return { + uint32_t(bs_and_quote), // bs_bits + uint32_t(bs_and_quote >> 32) // quote_bits + }; + } + + } // unnamed namespace + } // namespace lsx +} // namespace simdjson + +#endif // SIMDJSON_LSX_STRINGPARSING_DEFS_H +/* end file simdjson/lsx/stringparsing_defs.h */ + +#define SIMDJSON_SKIP_BACKSLASH_SHORT_CIRCUIT 1 +/* end file simdjson/lsx/begin.h */ +/* including simdjson/generic/amalgamated.h for lsx: #include "simdjson/generic/amalgamated.h" */ +/* begin file simdjson/generic/amalgamated.h for lsx */ +#if defined(SIMDJSON_CONDITIONAL_INCLUDE) && !defined(SIMDJSON_GENERIC_DEPENDENCIES_H) +#error simdjson/generic/dependencies.h must be included before simdjson/generic/amalgamated.h! +#endif + +/* including simdjson/generic/base.h for lsx: #include "simdjson/generic/base.h" */ +/* begin file simdjson/generic/base.h for lsx */ +#ifndef SIMDJSON_GENERIC_BASE_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_BASE_H */ +/* amalgamation skipped (editor-only): #include "simdjson/base.h" */ +/* amalgamation skipped (editor-only): // If we haven't got an implementation yet, we're in the editor, editing a generic file! Just */ +/* amalgamation skipped (editor-only): // use the most advanced one we can so the most possible stuff can be tested. */ +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_IMPLEMENTATION */ +/* amalgamation skipped (editor-only): #include "simdjson/implementation_detection.h" */ +/* amalgamation skipped (editor-only): #if SIMDJSON_IMPLEMENTATION_ICELAKE */ +/* amalgamation skipped (editor-only): #include "simdjson/icelake/begin.h" */ +/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_HASWELL */ +/* amalgamation skipped (editor-only): #include "simdjson/haswell/begin.h" */ +/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_WESTMERE */ +/* amalgamation skipped (editor-only): #include "simdjson/westmere/begin.h" */ +/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_ARM64 */ +/* amalgamation skipped (editor-only): #include "simdjson/arm64/begin.h" */ +/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_PPC64 */ +/* amalgamation skipped (editor-only): #include "simdjson/ppc64/begin.h" */ +/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_LSX */ +/* amalgamation skipped (editor-only): #include "simdjson/lsx/begin.h" */ +/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_LASX */ +/* amalgamation skipped (editor-only): #include "simdjson/lasx/begin.h" */ +/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_FALLBACK */ +/* amalgamation skipped (editor-only): #include "simdjson/fallback/begin.h" */ +/* amalgamation skipped (editor-only): #else */ +/* amalgamation skipped (editor-only): #error "All possible implementations (including fallback) have been disabled! simdjson will not run." */ +/* amalgamation skipped (editor-only): #endif */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_IMPLEMENTATION */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lsx { + + struct open_container; + class dom_parser_implementation; + + /** + * The type of a JSON number + */ + enum class number_type { + floating_point_number = 1, /// a binary64 number + signed_integer, /// a signed integer that fits in a 64-bit word using two's complement + unsigned_integer, /// a positive integer larger or equal to 1<<63 + big_integer /// a big integer that does not fit in a 64-bit word + }; + + } // namespace lsx +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_BASE_H +/* end file simdjson/generic/base.h for lsx */ +/* including simdjson/generic/jsoncharutils.h for lsx: #include "simdjson/generic/jsoncharutils.h" */ +/* begin file simdjson/generic/jsoncharutils.h for lsx */ +#ifndef SIMDJSON_GENERIC_JSONCHARUTILS_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_JSONCHARUTILS_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/jsoncharutils_tables.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lsx { + namespace { + namespace jsoncharutils { + + // return non-zero if not a structural or whitespace char + // zero otherwise + simdjson_inline uint32_t is_not_structural_or_whitespace(uint8_t c) { + return internal::structural_or_whitespace_negated[c]; + } + + simdjson_inline uint32_t is_structural_or_whitespace(uint8_t c) { + return internal::structural_or_whitespace[c]; + } + + // returns a value with the high 16 bits set if not valid + // otherwise returns the conversion of the 4 hex digits at src into the bottom + // 16 bits of the 32-bit return register + // + // see + // https://lemire.me/blog/2019/04/17/parsing-short-hexadecimal-strings-efficiently/ + static inline uint32_t hex_to_u32_nocheck( + const uint8_t* src) { // strictly speaking, static inline is a C-ism + uint32_t v1 = internal::digit_to_val32[630 + src[0]]; + uint32_t v2 = internal::digit_to_val32[420 + src[1]]; + uint32_t v3 = internal::digit_to_val32[210 + src[2]]; + uint32_t v4 = internal::digit_to_val32[0 + src[3]]; + return v1 | v2 | v3 | v4; + } + + // given a code point cp, writes to c + // the utf-8 code, outputting the length in + // bytes, if the length is zero, the code point + // is invalid + // + // This can possibly be made faster using pdep + // and clz and table lookups, but JSON documents + // have few escaped code points, and the following + // function looks cheap. + // + // Note: we assume that surrogates are treated separately + // + simdjson_inline size_t codepoint_to_utf8(uint32_t cp, uint8_t* c) { + if (cp <= 0x7F) { + c[0] = uint8_t(cp); + return 1; // ascii + } + if (cp <= 0x7FF) { + c[0] = uint8_t((cp >> 6) + 192); + c[1] = uint8_t((cp & 63) + 128); + return 2; // universal plane + // Surrogates are treated elsewhere... + //} //else if (0xd800 <= cp && cp <= 0xdfff) { + // return 0; // surrogates // could put assert here + } + else if (cp <= 0xFFFF) { + c[0] = uint8_t((cp >> 12) + 224); + c[1] = uint8_t(((cp >> 6) & 63) + 128); + c[2] = uint8_t((cp & 63) + 128); + return 3; + } + else if (cp <= 0x10FFFF) { // if you know you have a valid code point, this + // is not needed + c[0] = uint8_t((cp >> 18) + 240); + c[1] = uint8_t(((cp >> 12) & 63) + 128); + c[2] = uint8_t(((cp >> 6) & 63) + 128); + c[3] = uint8_t((cp & 63) + 128); + return 4; + } + // will return 0 when the code point was too large. + return 0; // bad r + } + +#if SIMDJSON_IS_32BITS // _umul128 for x86, arm + // this is a slow emulation routine for 32-bit + // + static simdjson_inline uint64_t __emulu(uint32_t x, uint32_t y) { + return x * (uint64_t)y; + } + static simdjson_inline uint64_t _umul128(uint64_t ab, uint64_t cd, uint64_t* hi) { + uint64_t ad = __emulu((uint32_t)(ab >> 32), (uint32_t)cd); + uint64_t bd = __emulu((uint32_t)ab, (uint32_t)cd); + uint64_t adbc = ad + __emulu((uint32_t)ab, (uint32_t)(cd >> 32)); + uint64_t adbc_carry = !!(adbc < ad); + uint64_t lo = bd + (adbc << 32); + *hi = __emulu((uint32_t)(ab >> 32), (uint32_t)(cd >> 32)) + (adbc >> 32) + + (adbc_carry << 32) + !!(lo < bd); + return lo; + } +#endif + + } // namespace jsoncharutils + } // unnamed namespace + } // namespace lsx +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_JSONCHARUTILS_H +/* end file simdjson/generic/jsoncharutils.h for lsx */ +/* including simdjson/generic/atomparsing.h for lsx: #include "simdjson/generic/atomparsing.h" */ +/* begin file simdjson/generic/atomparsing.h for lsx */ +#ifndef SIMDJSON_GENERIC_ATOMPARSING_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_ATOMPARSING_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/jsoncharutils.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +#include + +namespace simdjson { + namespace lsx { + namespace { + /// @private + namespace atomparsing { + + // The string_to_uint32 is exclusively used to map literal strings to 32-bit values. + // We use memcpy instead of a pointer cast to avoid undefined behaviors since we cannot + // be certain that the character pointer will be properly aligned. + // You might think that using memcpy makes this function expensive, but you'd be wrong. + // All decent optimizing compilers (GCC, clang, Visual Studio) will compile string_to_uint32("false"); + // to the compile-time constant 1936482662. + simdjson_inline uint32_t string_to_uint32(const char* str) { uint32_t val; std::memcpy(&val, str, sizeof(uint32_t)); return val; } + + + // Again in str4ncmp we use a memcpy to avoid undefined behavior. The memcpy may appear expensive. + // Yet all decent optimizing compilers will compile memcpy to a single instruction, just about. + simdjson_warn_unused + simdjson_inline uint32_t str4ncmp(const uint8_t* src, const char* atom) { + uint32_t srcval; // we want to avoid unaligned 32-bit loads (undefined in C/C++) + static_assert(sizeof(uint32_t) <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be larger than 4 bytes"); + std::memcpy(&srcval, src, sizeof(uint32_t)); + return srcval ^ string_to_uint32(atom); + } + + simdjson_warn_unused + simdjson_inline bool is_valid_true_atom(const uint8_t* src) { + return (str4ncmp(src, "true") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; + } + + simdjson_warn_unused + simdjson_inline bool is_valid_true_atom(const uint8_t* src, size_t len) { + if (len > 4) { return is_valid_true_atom(src); } + else if (len == 4) { return !str4ncmp(src, "true"); } + else { return false; } + } + + simdjson_warn_unused + simdjson_inline bool is_valid_false_atom(const uint8_t* src) { + return (str4ncmp(src + 1, "alse") | jsoncharutils::is_not_structural_or_whitespace(src[5])) == 0; + } + + simdjson_warn_unused + simdjson_inline bool is_valid_false_atom(const uint8_t* src, size_t len) { + if (len > 5) { return is_valid_false_atom(src); } + else if (len == 5) { return !str4ncmp(src + 1, "alse"); } + else { return false; } + } + + simdjson_warn_unused + simdjson_inline bool is_valid_null_atom(const uint8_t* src) { + return (str4ncmp(src, "null") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; + } + + simdjson_warn_unused + simdjson_inline bool is_valid_null_atom(const uint8_t* src, size_t len) { + if (len > 4) { return is_valid_null_atom(src); } + else if (len == 4) { return !str4ncmp(src, "null"); } + else { return false; } + } + + } // namespace atomparsing + } // unnamed namespace + } // namespace lsx +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_ATOMPARSING_H +/* end file simdjson/generic/atomparsing.h for lsx */ +/* including simdjson/generic/dom_parser_implementation.h for lsx: #include "simdjson/generic/dom_parser_implementation.h" */ +/* begin file simdjson/generic/dom_parser_implementation.h for lsx */ +#ifndef SIMDJSON_GENERIC_DOM_PARSER_IMPLEMENTATION_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_DOM_PARSER_IMPLEMENTATION_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/dom_parser_implementation.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lsx { + + // expectation: sizeof(open_container) = 64/8. + struct open_container { + uint32_t tape_index; // where, on the tape, does the scope ([,{) begins + uint32_t count; // how many elements in the scope + }; // struct open_container + + static_assert(sizeof(open_container) == 64 / 8, "Open container must be 64 bits"); + + class dom_parser_implementation final : public internal::dom_parser_implementation { + public: + /** Tape location of each open { or [ */ + std::unique_ptr open_containers{}; + /** Whether each open container is a [ or { */ + std::unique_ptr is_array{}; + /** Buffer passed to stage 1 */ + const uint8_t* buf{}; + /** Length passed to stage 1 */ + size_t len{ 0 }; + /** Document passed to stage 2 */ + dom::document* doc{}; + + inline dom_parser_implementation() noexcept; + inline dom_parser_implementation(dom_parser_implementation&& other) noexcept; + inline dom_parser_implementation& operator=(dom_parser_implementation&& other) noexcept; + dom_parser_implementation(const dom_parser_implementation&) = delete; + dom_parser_implementation& operator=(const dom_parser_implementation&) = delete; + + simdjson_warn_unused error_code parse(const uint8_t* buf, size_t len, dom::document& doc) noexcept final; + simdjson_warn_unused error_code stage1(const uint8_t* buf, size_t len, stage1_mode partial) noexcept final; + simdjson_warn_unused error_code stage2(dom::document& doc) noexcept final; + simdjson_warn_unused error_code stage2_next(dom::document& doc) noexcept final; + simdjson_warn_unused uint8_t* parse_string(const uint8_t* src, uint8_t* dst, bool allow_replacement) const noexcept final; + simdjson_warn_unused uint8_t* parse_wobbly_string(const uint8_t* src, uint8_t* dst) const noexcept final; + inline simdjson_warn_unused error_code set_capacity(size_t capacity) noexcept final; + inline simdjson_warn_unused error_code set_max_depth(size_t max_depth) noexcept final; + private: + simdjson_inline simdjson_warn_unused error_code set_capacity_stage1(size_t capacity); + + }; + + } // namespace lsx +} // namespace simdjson + +namespace simdjson { + namespace lsx { + + inline dom_parser_implementation::dom_parser_implementation() noexcept = default; + inline dom_parser_implementation::dom_parser_implementation(dom_parser_implementation&& other) noexcept = default; + inline dom_parser_implementation& dom_parser_implementation::operator=(dom_parser_implementation&& other) noexcept = default; + + // Leaving these here so they can be inlined if so desired + inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(size_t capacity) noexcept { + if (capacity > SIMDJSON_MAXSIZE_BYTES) { return CAPACITY; } + // Stage 1 index output + size_t max_structures = SIMDJSON_ROUNDUP_N(capacity, 64) + 2 + 7; + structural_indexes.reset(new (std::nothrow) uint32_t[max_structures]); + if (!structural_indexes) { _capacity = 0; return MEMALLOC; } + structural_indexes[0] = 0; + n_structural_indexes = 0; + + _capacity = capacity; + return SUCCESS; + } + + inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { + // Stage 2 stacks + open_containers.reset(new (std::nothrow) open_container[max_depth]); + is_array.reset(new (std::nothrow) bool[max_depth]); + if (!is_array || !open_containers) { _max_depth = 0; return MEMALLOC; } + + _max_depth = max_depth; + return SUCCESS; + } + + } // namespace lsx +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_DOM_PARSER_IMPLEMENTATION_H +/* end file simdjson/generic/dom_parser_implementation.h for lsx */ +/* including simdjson/generic/implementation_simdjson_result_base.h for lsx: #include "simdjson/generic/implementation_simdjson_result_base.h" */ +/* begin file simdjson/generic/implementation_simdjson_result_base.h for lsx */ +#ifndef SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lsx { + + // This is a near copy of include/error.h's implementation_simdjson_result_base, except it doesn't use std::pair + // so we can avoid inlining errors + // TODO reconcile these! + /** + * The result of a simdjson operation that could fail. + * + * Gives the option of reading error codes, or throwing an exception by casting to the desired result. + * + * This is a base class for implementations that want to add functions to the result type for + * chaining. + * + * Override like: + * + * struct simdjson_result : public internal::implementation_simdjson_result_base { + * simdjson_result() noexcept : internal::implementation_simdjson_result_base() {} + * simdjson_result(error_code error) noexcept : internal::implementation_simdjson_result_base(error) {} + * simdjson_result(T &&value) noexcept : internal::implementation_simdjson_result_base(std::forward(value)) {} + * simdjson_result(T &&value, error_code error) noexcept : internal::implementation_simdjson_result_base(value, error) {} + * // Your extra methods here + * } + * + * Then any method returning simdjson_result will be chainable with your methods. + */ + template + struct implementation_simdjson_result_base { + + /** + * Create a new empty result with error = UNINITIALIZED. + */ + simdjson_inline implementation_simdjson_result_base() noexcept = default; + + /** + * Create a new error result. + */ + simdjson_inline implementation_simdjson_result_base(error_code error) noexcept; + + /** + * Create a new successful result. + */ + simdjson_inline implementation_simdjson_result_base(T&& value) noexcept; + + /** + * Create a new result with both things (use if you don't want to branch when creating the result). + */ + simdjson_inline implementation_simdjson_result_base(T&& value, error_code error) noexcept; + + /** + * Move the value and the error to the provided variables. + * + * @param value The variable to assign the value to. May not be set if there is an error. + * @param error The variable to assign the error to. Set to SUCCESS if there is no error. + */ + simdjson_inline void tie(T& value, error_code& error) && noexcept; + + /** + * Move the value to the provided variable. + * + * @param value The variable to assign the value to. May not be set if there is an error. + */ + simdjson_inline error_code get(T& value) && noexcept; + + /** + * The error. + */ + simdjson_inline error_code error() const noexcept; + +#if SIMDJSON_EXCEPTIONS + + /** + * Get the result value. + * + * @throw simdjson_error if there was an error. + */ + simdjson_inline T& value() & noexcept(false); + + /** + * Take the result value (move it). + * + * @throw simdjson_error if there was an error. + */ + simdjson_inline T&& value() && noexcept(false); + + /** + * Take the result value (move it). + * + * @throw simdjson_error if there was an error. + */ + simdjson_inline T&& take_value() && noexcept(false); + + /** + * Cast to the value (will throw on error). + * + * @throw simdjson_error if there was an error. + */ + simdjson_inline operator T && () && noexcept(false); + + +#endif // SIMDJSON_EXCEPTIONS + + /** + * Get the result value. This function is safe if and only + * the error() method returns a value that evaluates to false. + */ + simdjson_inline const T& value_unsafe() const& noexcept; + /** + * Get the result value. This function is safe if and only + * the error() method returns a value that evaluates to false. + */ + simdjson_inline T& value_unsafe() & noexcept; + /** + * Take the result value (move it). This function is safe if and only + * the error() method returns a value that evaluates to false. + */ + simdjson_inline T&& value_unsafe() && noexcept; + protected: + /** users should never directly access first and second. **/ + T first{}; /** Users should never directly access 'first'. **/ + error_code second{ UNINITIALIZED }; /** Users should never directly access 'second'. **/ + }; // struct implementation_simdjson_result_base + + } // namespace lsx +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_H +/* end file simdjson/generic/implementation_simdjson_result_base.h for lsx */ +/* including simdjson/generic/numberparsing.h for lsx: #include "simdjson/generic/numberparsing.h" */ +/* begin file simdjson/generic/numberparsing.h for lsx */ +#ifndef SIMDJSON_GENERIC_NUMBERPARSING_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_NUMBERPARSING_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/jsoncharutils.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +#include +#include +#include + +namespace simdjson { + namespace lsx { + namespace numberparsing { + +#ifdef JSON_TEST_NUMBERS +#define INVALID_NUMBER(SRC) (found_invalid_number((SRC)), NUMBER_ERROR) +#define WRITE_INTEGER(VALUE, SRC, WRITER) (found_integer((VALUE), (SRC)), (WRITER).append_s64((VALUE))) +#define WRITE_UNSIGNED(VALUE, SRC, WRITER) (found_unsigned_integer((VALUE), (SRC)), (WRITER).append_u64((VALUE))) +#define WRITE_DOUBLE(VALUE, SRC, WRITER) (found_float((VALUE), (SRC)), (WRITER).append_double((VALUE))) +#define BIGINT_NUMBER(SRC) (found_invalid_number((SRC)), BIGINT_ERROR) +#else +#define INVALID_NUMBER(SRC) (NUMBER_ERROR) +#define WRITE_INTEGER(VALUE, SRC, WRITER) (WRITER).append_s64((VALUE)) +#define WRITE_UNSIGNED(VALUE, SRC, WRITER) (WRITER).append_u64((VALUE)) +#define WRITE_DOUBLE(VALUE, SRC, WRITER) (WRITER).append_double((VALUE)) +#define BIGINT_NUMBER(SRC) (BIGINT_ERROR) +#endif + + namespace { + + // Convert a mantissa, an exponent and a sign bit into an ieee64 double. + // The real_exponent needs to be in [0, 2046] (technically real_exponent = 2047 would be acceptable). + // The mantissa should be in [0,1<<53). The bit at index (1ULL << 52) while be zeroed. + simdjson_inline double to_double(uint64_t mantissa, uint64_t real_exponent, bool negative) { + double d; + mantissa &= ~(1ULL << 52); + mantissa |= real_exponent << 52; + mantissa |= ((static_cast(negative)) << 63); + std::memcpy(&d, &mantissa, sizeof(d)); + return d; + } + + // Attempts to compute i * 10^(power) exactly; and if "negative" is + // true, negate the result. + // This function will only work in some cases, when it does not work, success is + // set to false. This should work *most of the time* (like 99% of the time). + // We assume that power is in the [smallest_power, + // largest_power] interval: the caller is responsible for this check. + simdjson_inline bool compute_float_64(int64_t power, uint64_t i, bool negative, double& d) { + // we start with a fast path + // It was described in + // Clinger WD. How to read floating point numbers accurately. + // ACM SIGPLAN Notices. 1990 +#ifndef FLT_EVAL_METHOD +#error "FLT_EVAL_METHOD should be defined, please include cfloat." +#endif +#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0) + // We cannot be certain that x/y is rounded to nearest. + if (0 <= power && power <= 22 && i <= 9007199254740991) +#else + if (-22 <= power && power <= 22 && i <= 9007199254740991) +#endif + { + // convert the integer into a double. This is lossless since + // 0 <= i <= 2^53 - 1. + d = double(i); + // + // The general idea is as follows. + // If 0 <= s < 2^53 and if 10^0 <= p <= 10^22 then + // 1) Both s and p can be represented exactly as 64-bit floating-point + // values + // (binary64). + // 2) Because s and p can be represented exactly as floating-point values, + // then s * p + // and s / p will produce correctly rounded values. + // + if (power < 0) { + d = d / simdjson::internal::power_of_ten[-power]; + } + else { + d = d * simdjson::internal::power_of_ten[power]; + } + if (negative) { + d = -d; + } + return true; + } + // When 22 < power && power < 22 + 16, we could + // hope for another, secondary fast path. It was + // described by David M. Gay in "Correctly rounded + // binary-decimal and decimal-binary conversions." (1990) + // If you need to compute i * 10^(22 + x) for x < 16, + // first compute i * 10^x, if you know that result is exact + // (e.g., when i * 10^x < 2^53), + // then you can still proceed and do (i * 10^x) * 10^22. + // Is this worth your time? + // You need 22 < power *and* power < 22 + 16 *and* (i * 10^(x-22) < 2^53) + // for this second fast path to work. + // If you you have 22 < power *and* power < 22 + 16, and then you + // optimistically compute "i * 10^(x-22)", there is still a chance that you + // have wasted your time if i * 10^(x-22) >= 2^53. It makes the use cases of + // this optimization maybe less common than we would like. Source: + // http://www.exploringbinary.com/fast-path-decimal-to-floating-point-conversion/ + // also used in RapidJSON: https://rapidjson.org/strtod_8h_source.html + + // The fast path has now failed, so we are failing back on the slower path. + + // In the slow path, we need to adjust i so that it is > 1<<63 which is always + // possible, except if i == 0, so we handle i == 0 separately. + if (i == 0) { + d = negative ? -0.0 : 0.0; + return true; + } + + + // The exponent is 1024 + 63 + power + // + floor(log(5**power)/log(2)). + // The 1024 comes from the ieee64 standard. + // The 63 comes from the fact that we use a 64-bit word. + // + // Computing floor(log(5**power)/log(2)) could be + // slow. Instead we use a fast function. + // + // For power in (-400,350), we have that + // (((152170 + 65536) * power ) >> 16); + // is equal to + // floor(log(5**power)/log(2)) + power when power >= 0 + // and it is equal to + // ceil(log(5**-power)/log(2)) + power when power < 0 + // + // The 65536 is (1<<16) and corresponds to + // (65536 * power) >> 16 ---> power + // + // ((152170 * power ) >> 16) is equal to + // floor(log(5**power)/log(2)) + // + // Note that this is not magic: 152170/(1<<16) is + // approximatively equal to log(5)/log(2). + // The 1<<16 value is a power of two; we could use a + // larger power of 2 if we wanted to. + // + int64_t exponent = (((152170 + 65536) * power) >> 16) + 1024 + 63; + + + // We want the most significant bit of i to be 1. Shift if needed. + int lz = leading_zeroes(i); + i <<= lz; + + + // We are going to need to do some 64-bit arithmetic to get a precise product. + // We use a table lookup approach. + // It is safe because + // power >= smallest_power + // and power <= largest_power + // We recover the mantissa of the power, it has a leading 1. It is always + // rounded down. + // + // We want the most significant 64 bits of the product. We know + // this will be non-zero because the most significant bit of i is + // 1. + const uint32_t index = 2 * uint32_t(power - simdjson::internal::smallest_power); + // Optimization: It may be that materializing the index as a variable might confuse some compilers and prevent effective complex-addressing loads. (Done for code clarity.) + // + // The full_multiplication function computes the 128-bit product of two 64-bit words + // with a returned value of type value128 with a "low component" corresponding to the + // 64-bit least significant bits of the product and with a "high component" corresponding + // to the 64-bit most significant bits of the product. + simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index]); + // Both i and power_of_five_128[index] have their most significant bit set to 1 which + // implies that the either the most or the second most significant bit of the product + // is 1. We pack values in this manner for efficiency reasons: it maximizes the use + // we make of the product. It also makes it easy to reason about the product: there + // is 0 or 1 leading zero in the product. + + // Unless the least significant 9 bits of the high (64-bit) part of the full + // product are all 1s, then we know that the most significant 55 bits are + // exact and no further work is needed. Having 55 bits is necessary because + // we need 53 bits for the mantissa but we have to have one rounding bit and + // we can waste a bit if the most significant bit of the product is zero. + if ((firstproduct.high & 0x1FF) == 0x1FF) { + // We want to compute i * 5^q, but only care about the top 55 bits at most. + // Consider the scenario where q>=0. Then 5^q may not fit in 64-bits. Doing + // the full computation is wasteful. So we do what is called a "truncated + // multiplication". + // We take the most significant 64-bits, and we put them in + // power_of_five_128[index]. Usually, that's good enough to approximate i * 5^q + // to the desired approximation using one multiplication. Sometimes it does not suffice. + // Then we store the next most significant 64 bits in power_of_five_128[index + 1], and + // then we get a better approximation to i * 5^q. + // + // That's for when q>=0. The logic for q<0 is somewhat similar but it is somewhat + // more complicated. + // + // There is an extra layer of complexity in that we need more than 55 bits of + // accuracy in the round-to-even scenario. + // + // The full_multiplication function computes the 128-bit product of two 64-bit words + // with a returned value of type value128 with a "low component" corresponding to the + // 64-bit least significant bits of the product and with a "high component" corresponding + // to the 64-bit most significant bits of the product. + simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index + 1]); + firstproduct.low += secondproduct.high; + if (secondproduct.high > firstproduct.low) { firstproduct.high++; } + // As it has been proven by Noble Mushtak and Daniel Lemire in "Fast Number Parsing Without + // Fallback" (https://arxiv.org/abs/2212.06644), at this point we are sure that the product + // is sufficiently accurate, and more computation is not needed. + } + uint64_t lower = firstproduct.low; + uint64_t upper = firstproduct.high; + // The final mantissa should be 53 bits with a leading 1. + // We shift it so that it occupies 54 bits with a leading 1. + /////// + uint64_t upperbit = upper >> 63; + uint64_t mantissa = upper >> (upperbit + 9); + lz += int(1 ^ upperbit); + + // Here we have mantissa < (1<<54). + int64_t real_exponent = exponent - lz; + if (simdjson_unlikely(real_exponent <= 0)) { // we have a subnormal? + // Here have that real_exponent <= 0 so -real_exponent >= 0 + if (-real_exponent + 1 >= 64) { // if we have more than 64 bits below the minimum exponent, you have a zero for sure. + d = negative ? -0.0 : 0.0; + return true; + } + // next line is safe because -real_exponent + 1 < 0 + mantissa >>= -real_exponent + 1; + // Thankfully, we can't have both "round-to-even" and subnormals because + // "round-to-even" only occurs for powers close to 0. + mantissa += (mantissa & 1); // round up + mantissa >>= 1; + // There is a weird scenario where we don't have a subnormal but just. + // Suppose we start with 2.2250738585072013e-308, we end up + // with 0x3fffffffffffff x 2^-1023-53 which is technically subnormal + // whereas 0x40000000000000 x 2^-1023-53 is normal. Now, we need to round + // up 0x3fffffffffffff x 2^-1023-53 and once we do, we are no longer + // subnormal, but we can only know this after rounding. + // So we only declare a subnormal if we are smaller than the threshold. + real_exponent = (mantissa < (uint64_t(1) << 52)) ? 0 : 1; + d = to_double(mantissa, real_exponent, negative); + return true; + } + // We have to round to even. The "to even" part + // is only a problem when we are right in between two floats + // which we guard against. + // If we have lots of trailing zeros, we may fall right between two + // floating-point values. + // + // The round-to-even cases take the form of a number 2m+1 which is in (2^53,2^54] + // times a power of two. That is, it is right between a number with binary significand + // m and another number with binary significand m+1; and it must be the case + // that it cannot be represented by a float itself. + // + // We must have that w * 10 ^q == (2m+1) * 2^p for some power of two 2^p. + // Recall that 10^q = 5^q * 2^q. + // When q >= 0, we must have that (2m+1) is divible by 5^q, so 5^q <= 2^54. We have that + // 5^23 <= 2^54 and it is the last power of five to qualify, so q <= 23. + // When q<0, we have w >= (2m+1) x 5^{-q}. We must have that w<2^{64} so + // (2m+1) x 5^{-q} < 2^{64}. We have that 2m+1>2^{53}. Hence, we must have + // 2^{53} x 5^{-q} < 2^{64}. + // Hence we have 5^{-q} < 2^{11}$ or q>= -4. + // + // We require lower <= 1 and not lower == 0 because we could not prove that + // that lower == 0 is implied; but we could prove that lower <= 1 is a necessary and sufficient test. + if (simdjson_unlikely((lower <= 1) && (power >= -4) && (power <= 23) && ((mantissa & 3) == 1))) { + if ((mantissa << (upperbit + 64 - 53 - 2)) == upper) { + mantissa &= ~1; // flip it so that we do not round up + } + } + + mantissa += mantissa & 1; + mantissa >>= 1; + + // Here we have mantissa < (1<<53), unless there was an overflow + if (mantissa >= (1ULL << 53)) { + ////////// + // This will happen when parsing values such as 7.2057594037927933e+16 + //////// + mantissa = (1ULL << 52); + real_exponent++; + } + mantissa &= ~(1ULL << 52); + // we have to check that real_exponent is in range, otherwise we bail out + if (simdjson_unlikely(real_exponent > 2046)) { + // We have an infinite value!!! We could actually throw an error here if we could. + return false; + } + d = to_double(mantissa, real_exponent, negative); + return true; + } + + // We call a fallback floating-point parser that might be slow. Note + // it will accept JSON numbers, but the JSON spec. is more restrictive so + // before you call parse_float_fallback, you need to have validated the input + // string with the JSON grammar. + // It will return an error (false) if the parsed number is infinite. + // The string parsing itself always succeeds. We know that there is at least + // one digit. + static bool parse_float_fallback(const uint8_t* ptr, double* outDouble) { + *outDouble = simdjson::internal::from_chars(reinterpret_cast(ptr)); + // We do not accept infinite values. + + // Detecting finite values in a portable manner is ridiculously hard, ideally + // we would want to do: + // return !std::isfinite(*outDouble); + // but that mysteriously fails under legacy/old libc++ libraries, see + // https://github.com/simdjson/simdjson/issues/1286 + // + // Therefore, fall back to this solution (the extra parens are there + // to handle that max may be a macro on windows). + return !(*outDouble > (std::numeric_limits::max)() || *outDouble < std::numeric_limits::lowest()); + } + + static bool parse_float_fallback(const uint8_t* ptr, const uint8_t* end_ptr, double* outDouble) { + *outDouble = simdjson::internal::from_chars(reinterpret_cast(ptr), reinterpret_cast(end_ptr)); + // We do not accept infinite values. + + // Detecting finite values in a portable manner is ridiculously hard, ideally + // we would want to do: + // return !std::isfinite(*outDouble); + // but that mysteriously fails under legacy/old libc++ libraries, see + // https://github.com/simdjson/simdjson/issues/1286 + // + // Therefore, fall back to this solution (the extra parens are there + // to handle that max may be a macro on windows). + return !(*outDouble > (std::numeric_limits::max)() || *outDouble < std::numeric_limits::lowest()); + } + + // check quickly whether the next 8 chars are made of digits + // at a glance, it looks better than Mula's + // http://0x80.pl/articles/swar-digits-validate.html + simdjson_inline bool is_made_of_eight_digits_fast(const uint8_t* chars) { + uint64_t val; + // this can read up to 7 bytes beyond the buffer size, but we require + // SIMDJSON_PADDING of padding + static_assert(7 <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be bigger than 7"); + std::memcpy(&val, chars, 8); + // a branchy method might be faster: + // return (( val & 0xF0F0F0F0F0F0F0F0 ) == 0x3030303030303030) + // && (( (val + 0x0606060606060606) & 0xF0F0F0F0F0F0F0F0 ) == + // 0x3030303030303030); + return (((val & 0xF0F0F0F0F0F0F0F0) | + (((val + 0x0606060606060606) & 0xF0F0F0F0F0F0F0F0) >> 4)) == + 0x3333333333333333); + } + + template + SIMDJSON_NO_SANITIZE_UNDEFINED // We deliberately allow overflow here and check later + simdjson_inline bool parse_digit(const uint8_t c, I& i) { + const uint8_t digit = static_cast(c - '0'); + if (digit > 9) { + return false; + } + // PERF NOTE: multiplication by 10 is cheaper than arbitrary integer multiplication + i = 10 * i + digit; // might overflow, we will handle the overflow later + return true; + } + + simdjson_inline bool is_digit(const uint8_t c) { + return static_cast(c - '0') <= 9; + } + + simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t* const src, const uint8_t*& p, uint64_t& i, int64_t& exponent) { + // we continue with the fiction that we have an integer. If the + // floating point number is representable as x * 10^z for some integer + // z that fits in 53 bits, then we will be able to convert back the + // the integer into a float in a lossless manner. + const uint8_t* const first_after_period = p; + +#ifdef SIMDJSON_SWAR_NUMBER_PARSING +#if SIMDJSON_SWAR_NUMBER_PARSING + // this helps if we have lots of decimals! + // this turns out to be frequent enough. + if (is_made_of_eight_digits_fast(p)) { + i = i * 100000000 + parse_eight_digits_unrolled(p); + p += 8; + } +#endif // SIMDJSON_SWAR_NUMBER_PARSING +#endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING + // Unrolling the first digit makes a small difference on some implementations (e.g. westmere) + if (parse_digit(*p, i)) { ++p; } + while (parse_digit(*p, i)) { p++; } + exponent = first_after_period - p; + // Decimal without digits (123.) is illegal + if (exponent == 0) { + return INVALID_NUMBER(src); + } + return SUCCESS; + } + + simdjson_inline error_code parse_exponent(simdjson_unused const uint8_t* const src, const uint8_t*& p, int64_t& exponent) { + // Exp Sign: -123.456e[-]78 + bool neg_exp = ('-' == *p); + if (neg_exp || '+' == *p) { p++; } // Skip + as well + + // Exponent: -123.456e-[78] + auto start_exp = p; + int64_t exp_number = 0; + while (parse_digit(*p, exp_number)) { ++p; } + // It is possible for parse_digit to overflow. + // In particular, it could overflow to INT64_MIN, and we cannot do - INT64_MIN. + // Thus we *must* check for possible overflow before we negate exp_number. + + // Performance notes: it may seem like combining the two "simdjson_unlikely checks" below into + // a single simdjson_unlikely path would be faster. The reasoning is sound, but the compiler may + // not oblige and may, in fact, generate two distinct paths in any case. It might be + // possible to do uint64_t(p - start_exp - 1) >= 18 but it could end up trading off + // instructions for a simdjson_likely branch, an unconclusive gain. + + // If there were no digits, it's an error. + if (simdjson_unlikely(p == start_exp)) { + return INVALID_NUMBER(src); + } + // We have a valid positive exponent in exp_number at this point, except that + // it may have overflowed. + + // If there were more than 18 digits, we may have overflowed the integer. We have to do + // something!!!! + if (simdjson_unlikely(p > start_exp + 18)) { + // Skip leading zeroes: 1e000000000000000000001 is technically valid and does not overflow + while (*start_exp == '0') { start_exp++; } + // 19 digits could overflow int64_t and is kind of absurd anyway. We don't + // support exponents smaller than -999,999,999,999,999,999 and bigger + // than 999,999,999,999,999,999. + // We can truncate. + // Note that 999999999999999999 is assuredly too large. The maximal ieee64 value before + // infinity is ~1.8e308. The smallest subnormal is ~5e-324. So, actually, we could + // truncate at 324. + // Note that there is no reason to fail per se at this point in time. + // E.g., 0e999999999999999999999 is a fine number. + if (p > start_exp + 18) { exp_number = 999999999999999999; } + } + // At this point, we know that exp_number is a sane, positive, signed integer. + // It is <= 999,999,999,999,999,999. As long as 'exponent' is in + // [-8223372036854775808, 8223372036854775808], we won't overflow. Because 'exponent' + // is bounded in magnitude by the size of the JSON input, we are fine in this universe. + // To sum it up: the next line should never overflow. + exponent += (neg_exp ? -exp_number : exp_number); + return SUCCESS; + } + + simdjson_inline bool check_if_integer(const uint8_t* const src, size_t max_length) { + const uint8_t* const srcend = src + max_length; + bool negative = (*src == '-'); // we can always read at least one character after the '-' + const uint8_t* p = src + uint8_t(negative); + if (p == srcend) { return false; } + if (*p == '0') { + ++p; + if (p == srcend) { return true; } + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return false; } + return true; + } + while (p != srcend && is_digit(*p)) { ++p; } + if (p == srcend) { return true; } + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return false; } + return true; + } + + simdjson_inline size_t significant_digits(const uint8_t* start_digits, size_t digit_count) { + // It is possible that the integer had an overflow. + // We have to handle the case where we have 0.0000somenumber. + const uint8_t* start = start_digits; + while ((*start == '0') || (*start == '.')) { ++start; } + // we over-decrement by one when there is a '.' + return digit_count - size_t(start - start_digits); + } + + } // unnamed namespace + + /** @private */ + static error_code slow_float_parsing(simdjson_unused const uint8_t* src, double* answer) { + if (parse_float_fallback(src, answer)) { + return SUCCESS; + } + return INVALID_NUMBER(src); + } + + /** @private */ + template + simdjson_inline error_code write_float(const uint8_t* const src, bool negative, uint64_t i, const uint8_t* start_digits, size_t digit_count, int64_t exponent, W& writer) { + // If we frequently had to deal with long strings of digits, + // we could extend our code by using a 128-bit integer instead + // of a 64-bit integer. However, this is uncommon in practice. + // + // 9999999999999999999 < 2**64 so we can accommodate 19 digits. + // If we have a decimal separator, then digit_count - 1 is the number of digits, but we + // may not have a decimal separator! + if (simdjson_unlikely(digit_count > 19 && significant_digits(start_digits, digit_count) > 19)) { + // Ok, chances are good that we had an overflow! + // this is almost never going to get called!!! + // we start anew, going slowly!!! + // This will happen in the following examples: + // 10000000000000000000000000000000000000000000e+308 + // 3.1415926535897932384626433832795028841971693993751 + // + // NOTE: We do not pass a reference to the to slow_float_parsing. If we passed our writer + // reference to it, it would force it to be stored in memory, preventing the compiler from + // picking it apart and putting into registers. i.e. if we pass it as reference, + // it gets slow. + double d; + error_code error = slow_float_parsing(src, &d); + writer.append_double(d); + return error; + } + // NOTE: it's weird that the simdjson_unlikely() only wraps half the if, but it seems to get slower any other + // way we've tried: https://github.com/simdjson/simdjson/pull/990#discussion_r448497331 + // To future reader: we'd love if someone found a better way, or at least could explain this result! + if (simdjson_unlikely(exponent < simdjson::internal::smallest_power) || (exponent > simdjson::internal::largest_power)) { + // + // Important: smallest_power is such that it leads to a zero value. + // Observe that 18446744073709551615e-343 == 0, i.e. (2**64 - 1) e -343 is zero + // so something x 10^-343 goes to zero, but not so with something x 10^-342. + static_assert(simdjson::internal::smallest_power <= -342, "smallest_power is not small enough"); + // + if ((exponent < simdjson::internal::smallest_power) || (i == 0)) { + // E.g. Parse "-0.0e-999" into the same value as "-0.0". See https://en.wikipedia.org/wiki/Signed_zero + WRITE_DOUBLE(negative ? -0.0 : 0.0, src, writer); + return SUCCESS; + } + else { // (exponent > largest_power) and (i != 0) + // We have, for sure, an infinite value and simdjson refuses to parse infinite values. + return INVALID_NUMBER(src); + } + } + double d; + if (!compute_float_64(exponent, i, negative, d)) { + // we are almost never going to get here. + if (!parse_float_fallback(src, &d)) { return INVALID_NUMBER(src); } + } + WRITE_DOUBLE(d, src, writer); + return SUCCESS; + } + + // parse the number at src + // define JSON_TEST_NUMBERS for unit testing + // + // It is assumed that the number is followed by a structural ({,},],[) character + // or a white space character. If that is not the case (e.g., when the JSON + // document is made of a single number), then it is necessary to copy the + // content and append a space before calling this function. + // + // Our objective is accurate parsing (ULP of 0) at high speed. + template + simdjson_inline error_code parse_number(const uint8_t* const src, W& writer); + + // for performance analysis, it is sometimes useful to skip parsing +#ifdef SIMDJSON_SKIPNUMBERPARSING + + template + simdjson_inline error_code parse_number(const uint8_t* const, W& writer) { + writer.append_s64(0); // always write zero + return SUCCESS; // always succeeds + } + + simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t* const src) noexcept { return 0; } + simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t* const src) noexcept { return 0; } + simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t* const src) noexcept { return 0; } + simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t* const src) noexcept { return 0; } + simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t* const src) noexcept { return 0; } + simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t* const src) noexcept { return 0; } + simdjson_unused simdjson_inline bool is_negative(const uint8_t* src) noexcept { return false; } + simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t* src) noexcept { return false; } + simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t* src) noexcept { return number_type::signed_integer; } +#else + +// parse the number at src +// define JSON_TEST_NUMBERS for unit testing +// +// It is assumed that the number is followed by a structural ({,},],[) character +// or a white space character. If that is not the case (e.g., when the JSON +// document is made of a single number), then it is necessary to copy the +// content and append a space before calling this function. +// +// Our objective is accurate parsing (ULP of 0) at high speed. + template + simdjson_inline error_code parse_number(const uint8_t* const src, W& writer) { + + // + // Check for minus sign + // + bool negative = (*src == '-'); + const uint8_t* p = src + uint8_t(negative); + + // + // Parse the integer part. + // + // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare + const uint8_t* const start_digits = p; + uint64_t i = 0; + while (parse_digit(*p, i)) { p++; } + + // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. + // Optimization note: size_t is expected to be unsigned. + size_t digit_count = size_t(p - start_digits); + if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { return INVALID_NUMBER(src); } + + // + // Handle floats if there is a . or e (or both) + // + int64_t exponent = 0; + bool is_float = false; + if ('.' == *p) { + is_float = true; + ++p; + SIMDJSON_TRY(parse_decimal_after_separator(src, p, i, exponent)); + digit_count = int(p - start_digits); // used later to guard against overflows + } + if (('e' == *p) || ('E' == *p)) { + is_float = true; + ++p; + SIMDJSON_TRY(parse_exponent(src, p, exponent)); + } + if (is_float) { + const bool dirty_end = jsoncharutils::is_not_structural_or_whitespace(*p); + SIMDJSON_TRY(write_float(src, negative, i, start_digits, digit_count, exponent, writer)); + if (dirty_end) { return INVALID_NUMBER(src); } + return SUCCESS; + } + + // The longest negative 64-bit number is 19 digits. + // The longest positive 64-bit number is 20 digits. + // We do it this way so we don't trigger this branch unless we must. + size_t longest_digit_count = negative ? 19 : 20; + if (digit_count > longest_digit_count) { return BIGINT_NUMBER(src); } + if (digit_count == longest_digit_count) { + if (negative) { + // Anything negative above INT64_MAX+1 is invalid + if (i > uint64_t(INT64_MAX) + 1) { return BIGINT_NUMBER(src); } + WRITE_INTEGER(~i + 1, src, writer); + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return INVALID_NUMBER(src); } + return SUCCESS; + // Positive overflow check: + // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the + // biggest uint64_t. + // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX. + // If we got here, it's a 20 digit number starting with the digit "1". + // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller + // than 1,553,255,926,290,448,384. + // - That is smaller than the smallest possible 20-digit number the user could write: + // 10,000,000,000,000,000,000. + // - Therefore, if the number is positive and lower than that, it's overflow. + // - The value we are looking at is less than or equal to INT64_MAX. + // + } + else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INVALID_NUMBER(src); } + } + + // Write unsigned if it does not fit in a signed integer. + if (i > uint64_t(INT64_MAX)) { + WRITE_UNSIGNED(i, src, writer); + } + else { + WRITE_INTEGER(negative ? (~i + 1) : i, src, writer); + } + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return INVALID_NUMBER(src); } + return SUCCESS; + } + + // Inlineable functions + namespace { + + // This table can be used to characterize the final character of an integer + // string. For JSON structural character and allowable white space characters, + // we return SUCCESS. For 'e', '.' and 'E', we return INCORRECT_TYPE. Otherwise + // we return NUMBER_ERROR. + // Optimization note: we could easily reduce the size of the table by half (to 128) + // at the cost of an extra branch. + // Optimization note: we want the values to use at most 8 bits (not, e.g., 32 bits): + static_assert(error_code(uint8_t(NUMBER_ERROR)) == NUMBER_ERROR, "bad NUMBER_ERROR cast"); + static_assert(error_code(uint8_t(SUCCESS)) == SUCCESS, "bad NUMBER_ERROR cast"); + static_assert(error_code(uint8_t(INCORRECT_TYPE)) == INCORRECT_TYPE, "bad NUMBER_ERROR cast"); + + const uint8_t integer_string_finisher[256] = { + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, + SUCCESS, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, + NUMBER_ERROR, INCORRECT_TYPE, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, INCORRECT_TYPE, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, SUCCESS, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, INCORRECT_TYPE, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, + SUCCESS, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR }; + + // Parse any number from 0 to 18,446,744,073,709,551,615 + simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t* const src) noexcept { + const uint8_t* p = src; + // + // Parse the integer part. + // + // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare + const uint8_t* const start_digits = p; + uint64_t i = 0; + while (parse_digit(*p, i)) { p++; } + + // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. + // Optimization note: size_t is expected to be unsigned. + size_t digit_count = size_t(p - start_digits); + // The longest positive 64-bit number is 20 digits. + // We do it this way so we don't trigger this branch unless we must. + // Optimization note: the compiler can probably merge + // ((digit_count == 0) || (digit_count > 20)) + // into a single branch since digit_count is unsigned. + if ((digit_count == 0) || (digit_count > 20)) { return INCORRECT_TYPE; } + // Here digit_count > 0. + if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } + // We can do the following... + // if (!jsoncharutils::is_structural_or_whitespace(*p)) { + // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; + // } + // as a single table lookup: + if (integer_string_finisher[*p] != SUCCESS) { return error_code(integer_string_finisher[*p]); } + + if (digit_count == 20) { + // Positive overflow check: + // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the + // biggest uint64_t. + // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX. + // If we got here, it's a 20 digit number starting with the digit "1". + // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller + // than 1,553,255,926,290,448,384. + // - That is smaller than the smallest possible 20-digit number the user could write: + // 10,000,000,000,000,000,000. + // - Therefore, if the number is positive and lower than that, it's overflow. + // - The value we are looking at is less than or equal to INT64_MAX. + // + if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INCORRECT_TYPE; } + } + + return i; + } + + + // Parse any number from 0 to 18,446,744,073,709,551,615 + // Never read at src_end or beyond + simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t* const src, const uint8_t* const src_end) noexcept { + const uint8_t* p = src; + // + // Parse the integer part. + // + // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare + const uint8_t* const start_digits = p; + uint64_t i = 0; + while ((p != src_end) && parse_digit(*p, i)) { p++; } + + // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. + // Optimization note: size_t is expected to be unsigned. + size_t digit_count = size_t(p - start_digits); + // The longest positive 64-bit number is 20 digits. + // We do it this way so we don't trigger this branch unless we must. + // Optimization note: the compiler can probably merge + // ((digit_count == 0) || (digit_count > 20)) + // into a single branch since digit_count is unsigned. + if ((digit_count == 0) || (digit_count > 20)) { return INCORRECT_TYPE; } + // Here digit_count > 0. + if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } + // We can do the following... + // if (!jsoncharutils::is_structural_or_whitespace(*p)) { + // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; + // } + // as a single table lookup: + if ((p != src_end) && integer_string_finisher[*p] != SUCCESS) { return error_code(integer_string_finisher[*p]); } + + if (digit_count == 20) { + // Positive overflow check: + // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the + // biggest uint64_t. + // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX. + // If we got here, it's a 20 digit number starting with the digit "1". + // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller + // than 1,553,255,926,290,448,384. + // - That is smaller than the smallest possible 20-digit number the user could write: + // 10,000,000,000,000,000,000. + // - Therefore, if the number is positive and lower than that, it's overflow. + // - The value we are looking at is less than or equal to INT64_MAX. + // + if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INCORRECT_TYPE; } + } + + return i; + } + + // Parse any number from 0 to 18,446,744,073,709,551,615 + simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t* const src) noexcept { + const uint8_t* p = src + 1; + // + // Parse the integer part. + // + // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare + const uint8_t* const start_digits = p; + uint64_t i = 0; + while (parse_digit(*p, i)) { p++; } + + // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. + // Optimization note: size_t is expected to be unsigned. + size_t digit_count = size_t(p - start_digits); + // The longest positive 64-bit number is 20 digits. + // We do it this way so we don't trigger this branch unless we must. + // Optimization note: the compiler can probably merge + // ((digit_count == 0) || (digit_count > 20)) + // into a single branch since digit_count is unsigned. + if ((digit_count == 0) || (digit_count > 20)) { return INCORRECT_TYPE; } + // Here digit_count > 0. + if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } + // We can do the following... + // if (!jsoncharutils::is_structural_or_whitespace(*p)) { + // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; + // } + // as a single table lookup: + if (*p != '"') { return NUMBER_ERROR; } + + if (digit_count == 20) { + // Positive overflow check: + // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the + // biggest uint64_t. + // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX. + // If we got here, it's a 20 digit number starting with the digit "1". + // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller + // than 1,553,255,926,290,448,384. + // - That is smaller than the smallest possible 20-digit number the user could write: + // 10,000,000,000,000,000,000. + // - Therefore, if the number is positive and lower than that, it's overflow. + // - The value we are looking at is less than or equal to INT64_MAX. + // + // Note: we use src[1] and not src[0] because src[0] is the quote character in this + // instance. + if (src[1] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INCORRECT_TYPE; } + } + + return i; + } + + // Parse any number from -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 + simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t* src) noexcept { + // + // Check for minus sign + // + bool negative = (*src == '-'); + const uint8_t* p = src + uint8_t(negative); + + // + // Parse the integer part. + // + // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare + const uint8_t* const start_digits = p; + uint64_t i = 0; + while (parse_digit(*p, i)) { p++; } + + // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. + // Optimization note: size_t is expected to be unsigned. + size_t digit_count = size_t(p - start_digits); + // We go from + // -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 + // so we can never represent numbers that have more than 19 digits. + size_t longest_digit_count = 19; + // Optimization note: the compiler can probably merge + // ((digit_count == 0) || (digit_count > longest_digit_count)) + // into a single branch since digit_count is unsigned. + if ((digit_count == 0) || (digit_count > longest_digit_count)) { return INCORRECT_TYPE; } + // Here digit_count > 0. + if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } + // We can do the following... + // if (!jsoncharutils::is_structural_or_whitespace(*p)) { + // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; + // } + // as a single table lookup: + if (integer_string_finisher[*p] != SUCCESS) { return error_code(integer_string_finisher[*p]); } + // Negative numbers have can go down to - INT64_MAX - 1 whereas positive numbers are limited to INT64_MAX. + // Performance note: This check is only needed when digit_count == longest_digit_count but it is + // so cheap that we might as well always make it. + if (i > uint64_t(INT64_MAX) + uint64_t(negative)) { return INCORRECT_TYPE; } + return negative ? (~i + 1) : i; + } + + // Parse any number from -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 + // Never read at src_end or beyond + simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t* const src, const uint8_t* const src_end) noexcept { + // + // Check for minus sign + // + if (src == src_end) { return NUMBER_ERROR; } + bool negative = (*src == '-'); + const uint8_t* p = src + uint8_t(negative); + + // + // Parse the integer part. + // + // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare + const uint8_t* const start_digits = p; + uint64_t i = 0; + while ((p != src_end) && parse_digit(*p, i)) { p++; } + + // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. + // Optimization note: size_t is expected to be unsigned. + size_t digit_count = size_t(p - start_digits); + // We go from + // -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 + // so we can never represent numbers that have more than 19 digits. + size_t longest_digit_count = 19; + // Optimization note: the compiler can probably merge + // ((digit_count == 0) || (digit_count > longest_digit_count)) + // into a single branch since digit_count is unsigned. + if ((digit_count == 0) || (digit_count > longest_digit_count)) { return INCORRECT_TYPE; } + // Here digit_count > 0. + if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } + // We can do the following... + // if (!jsoncharutils::is_structural_or_whitespace(*p)) { + // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; + // } + // as a single table lookup: + if ((p != src_end) && integer_string_finisher[*p] != SUCCESS) { return error_code(integer_string_finisher[*p]); } + // Negative numbers have can go down to - INT64_MAX - 1 whereas positive numbers are limited to INT64_MAX. + // Performance note: This check is only needed when digit_count == longest_digit_count but it is + // so cheap that we might as well always make it. + if (i > uint64_t(INT64_MAX) + uint64_t(negative)) { return INCORRECT_TYPE; } + return negative ? (~i + 1) : i; + } + + // Parse any number from -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 + simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t* src) noexcept { + // + // Check for minus sign + // + bool negative = (*(src + 1) == '-'); + src += uint8_t(negative) + 1; + + // + // Parse the integer part. + // + // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare + const uint8_t* const start_digits = src; + uint64_t i = 0; + while (parse_digit(*src, i)) { src++; } + + // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. + // Optimization note: size_t is expected to be unsigned. + size_t digit_count = size_t(src - start_digits); + // We go from + // -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 + // so we can never represent numbers that have more than 19 digits. + size_t longest_digit_count = 19; + // Optimization note: the compiler can probably merge + // ((digit_count == 0) || (digit_count > longest_digit_count)) + // into a single branch since digit_count is unsigned. + if ((digit_count == 0) || (digit_count > longest_digit_count)) { return INCORRECT_TYPE; } + // Here digit_count > 0. + if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } + // We can do the following... + // if (!jsoncharutils::is_structural_or_whitespace(*src)) { + // return (*src == '.' || *src == 'e' || *src == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; + // } + // as a single table lookup: + if (*src != '"') { return NUMBER_ERROR; } + // Negative numbers have can go down to - INT64_MAX - 1 whereas positive numbers are limited to INT64_MAX. + // Performance note: This check is only needed when digit_count == longest_digit_count but it is + // so cheap that we might as well always make it. + if (i > uint64_t(INT64_MAX) + uint64_t(negative)) { return INCORRECT_TYPE; } + return negative ? (~i + 1) : i; + } + + simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t* src) noexcept { + // + // Check for minus sign + // + bool negative = (*src == '-'); + src += uint8_t(negative); + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t* p = src; + p += parse_digit(*p, i); + bool leading_zero = (i == 0); + while (parse_digit(*p, i)) { p++; } + // no integer digits, or 0123 (zero must be solo) + if (p == src) { return INCORRECT_TYPE; } + if ((leading_zero && p != src + 1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely(*p == '.')) { + p++; + const uint8_t* start_decimal_digits = p; + if (!parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits + p++; + while (parse_digit(*p, i)) { p++; } + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p - src - 1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t* start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = p - start_digits > 19; + } + } + else { + overflow = p - src > 19; + } + + // + // Parse the exponent + // + if (*p == 'e' || *p == 'E') { + p++; + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t* start_exp_digits = p; + while (parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p - start_exp_digits == 0 || p - start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0 - exp : exp; + } + + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + double d; + if (simdjson_likely(!overflow)) { + if (compute_float_64(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), &d)) { + return NUMBER_ERROR; + } + return d; + } + + simdjson_unused simdjson_inline bool is_negative(const uint8_t* src) noexcept { + return (*src == '-'); + } + + simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t* src) noexcept { + bool negative = (*src == '-'); + src += uint8_t(negative); + const uint8_t* p = src; + while (static_cast(*p - '0') <= 9) { p++; } + if (p == src) { return NUMBER_ERROR; } + if (jsoncharutils::is_structural_or_whitespace(*p)) { return true; } + return false; + } + + simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t* src) noexcept { + bool negative = (*src == '-'); + src += uint8_t(negative); + const uint8_t* p = src; + while (static_cast(*p - '0') <= 9) { p++; } + size_t digit_count = size_t(p - src); + if (p == src) { return NUMBER_ERROR; } + if (jsoncharutils::is_structural_or_whitespace(*p)) { + static const uint8_t* smaller_big_integer = reinterpret_cast("9223372036854775808"); + // We have an integer. + if (simdjson_unlikely(digit_count > 20)) { + return number_type::big_integer; + } + // If the number is negative and valid, it must be a signed integer. + if (negative) { + if (simdjson_unlikely(digit_count > 19)) return number_type::big_integer; + if (simdjson_unlikely(digit_count == 19 && memcmp(src, smaller_big_integer, 19) > 0)) { + return number_type::big_integer; + } + return number_type::signed_integer; + } + // Let us check if we have a big integer (>=2**64). + static const uint8_t* two_to_sixtyfour = reinterpret_cast("18446744073709551616"); + if ((digit_count > 20) || (digit_count == 20 && memcmp(src, two_to_sixtyfour, 20) >= 0)) { + return number_type::big_integer; + } + // The number is positive and smaller than 18446744073709551616 (or 2**64). + // We want values larger or equal to 9223372036854775808 to be unsigned + // integers, and the other values to be signed integers. + if ((digit_count == 20) || (digit_count >= 19 && memcmp(src, smaller_big_integer, 19) >= 0)) { + return number_type::unsigned_integer; + } + return number_type::signed_integer; + } + // Hopefully, we have 'e' or 'E' or '.'. + return number_type::floating_point_number; + } + + // Never read at src_end or beyond + simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t* src, const uint8_t* const src_end) noexcept { + if (src == src_end) { return NUMBER_ERROR; } + // + // Check for minus sign + // + bool negative = (*src == '-'); + src += uint8_t(negative); + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t* p = src; + if (p == src_end) { return NUMBER_ERROR; } + p += parse_digit(*p, i); + bool leading_zero = (i == 0); + while ((p != src_end) && parse_digit(*p, i)) { p++; } + // no integer digits, or 0123 (zero must be solo) + if (p == src) { return INCORRECT_TYPE; } + if ((leading_zero && p != src + 1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely((p != src_end) && (*p == '.'))) { + p++; + const uint8_t* start_decimal_digits = p; + if ((p == src_end) || !parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits + p++; + while ((p != src_end) && parse_digit(*p, i)) { p++; } + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p - src - 1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t* start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = start_digits - src > 19; + } + } + else { + overflow = p - src > 19; + } + + // + // Parse the exponent + // + if ((p != src_end) && (*p == 'e' || *p == 'E')) { + p++; + if (p == src_end) { return NUMBER_ERROR; } + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t* start_exp_digits = p; + while ((p != src_end) && parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p - start_exp_digits == 0 || p - start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0 - exp : exp; + } + + if ((p != src_end) && jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + double d; + if (simdjson_likely(!overflow)) { + if (compute_float_64(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), src_end, &d)) { + return NUMBER_ERROR; + } + return d; + } + + simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t* src) noexcept { + // + // Check for minus sign + // + bool negative = (*(src + 1) == '-'); + src += uint8_t(negative) + 1; + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t* p = src; + p += parse_digit(*p, i); + bool leading_zero = (i == 0); + while (parse_digit(*p, i)) { p++; } + // no integer digits, or 0123 (zero must be solo) + if (p == src) { return INCORRECT_TYPE; } + if ((leading_zero && p != src + 1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely(*p == '.')) { + p++; + const uint8_t* start_decimal_digits = p; + if (!parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits + p++; + while (parse_digit(*p, i)) { p++; } + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p - src - 1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t* start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = p - start_digits > 19; + } + } + else { + overflow = p - src > 19; + } + + // + // Parse the exponent + // + if (*p == 'e' || *p == 'E') { + p++; + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t* start_exp_digits = p; + while (parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p - start_exp_digits == 0 || p - start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0 - exp : exp; + } + + if (*p != '"') { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + double d; + if (simdjson_likely(!overflow)) { + if (compute_float_64(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), &d)) { + return NUMBER_ERROR; + } + return d; + } + + } // unnamed namespace +#endif // SIMDJSON_SKIPNUMBERPARSING + + } // namespace numberparsing + + inline std::ostream& operator<<(std::ostream& out, number_type type) noexcept { + switch (type) { + case number_type::signed_integer: out << "integer in [-9223372036854775808,9223372036854775808)"; break; + case number_type::unsigned_integer: out << "unsigned integer in [9223372036854775808,18446744073709551616)"; break; + case number_type::floating_point_number: out << "floating-point number (binary64)"; break; + case number_type::big_integer: out << "big integer"; break; + default: SIMDJSON_UNREACHABLE(); + } + return out; + } + + } // namespace lsx +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_NUMBERPARSING_H +/* end file simdjson/generic/numberparsing.h for lsx */ + +/* including simdjson/generic/implementation_simdjson_result_base-inl.h for lsx: #include "simdjson/generic/implementation_simdjson_result_base-inl.h" */ +/* begin file simdjson/generic/implementation_simdjson_result_base-inl.h for lsx */ +#ifndef SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_INL_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_INL_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/implementation_simdjson_result_base.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lsx { + + // + // internal::implementation_simdjson_result_base inline implementation + // + + template + simdjson_inline void implementation_simdjson_result_base::tie(T& value, error_code& error) && noexcept { + error = this->second; + if (!error) { + value = std::forward>(*this).first; + } + } + + template + simdjson_warn_unused simdjson_inline error_code implementation_simdjson_result_base::get(T& value) && noexcept { + error_code error; + std::forward>(*this).tie(value, error); + return error; + } + + template + simdjson_inline error_code implementation_simdjson_result_base::error() const noexcept { + return this->second; + } + +#if SIMDJSON_EXCEPTIONS + + template + simdjson_inline T& implementation_simdjson_result_base::value() & noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return this->first; + } + + template + simdjson_inline T&& implementation_simdjson_result_base::value() && noexcept(false) { + return std::forward>(*this).take_value(); + } + + template + simdjson_inline T&& implementation_simdjson_result_base::take_value() && noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return std::forward(this->first); + } + + template + simdjson_inline implementation_simdjson_result_base::operator T && () && noexcept(false) { + return std::forward>(*this).take_value(); + } + +#endif // SIMDJSON_EXCEPTIONS + + template + simdjson_inline const T& implementation_simdjson_result_base::value_unsafe() const& noexcept { + return this->first; + } + + template + simdjson_inline T& implementation_simdjson_result_base::value_unsafe() & noexcept { + return this->first; + } + + template + simdjson_inline T&& implementation_simdjson_result_base::value_unsafe() && noexcept { + return std::forward(this->first); + } + + template + simdjson_inline implementation_simdjson_result_base::implementation_simdjson_result_base(T&& value, error_code error) noexcept + : first{ std::forward(value) }, second{ error } {} + template + simdjson_inline implementation_simdjson_result_base::implementation_simdjson_result_base(error_code error) noexcept + : implementation_simdjson_result_base(T{}, error) {} + template + simdjson_inline implementation_simdjson_result_base::implementation_simdjson_result_base(T&& value) noexcept + : implementation_simdjson_result_base(std::forward(value), SUCCESS) {} + + } // namespace lsx +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_INL_H +/* end file simdjson/generic/implementation_simdjson_result_base-inl.h for lsx */ +/* end file simdjson/generic/amalgamated.h for lsx */ +/* including simdjson/lsx/end.h: #include "simdjson/lsx/end.h" */ +/* begin file simdjson/lsx/end.h */ +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/lsx/base.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +#undef SIMDJSON_SKIP_BACKSLASH_SHORT_CIRCUIT +/* undefining SIMDJSON_IMPLEMENTATION from "lsx" */ +#undef SIMDJSON_IMPLEMENTATION +/* end file simdjson/lsx/end.h */ + +#endif // SIMDJSON_LSX_H +/* end file simdjson/lsx.h */ +#elif SIMDJSON_BUILTIN_IMPLEMENTATION_IS(lasx) +/* including simdjson/lasx.h: #include "simdjson/lasx.h" */ +/* begin file simdjson/lasx.h */ +#ifndef SIMDJSON_LASX_H +#define SIMDJSON_LASX_H + +/* including simdjson/lasx/begin.h: #include "simdjson/lasx/begin.h" */ +/* begin file simdjson/lasx/begin.h */ +/* defining SIMDJSON_IMPLEMENTATION to "lasx" */ +#define SIMDJSON_IMPLEMENTATION lasx +/* including simdjson/lasx/base.h: #include "simdjson/lasx/base.h" */ +/* begin file simdjson/lasx/base.h */ +#ifndef SIMDJSON_LASX_BASE_H +#define SIMDJSON_LASX_BASE_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/base.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + /** + * Implementation for LASX. + */ + namespace lasx { + + class implementation; + + namespace { + namespace simd { + template struct simd8; + template struct simd8x64; + } // namespace simd + } // unnamed namespace + + } // namespace lasx +} // namespace simdjson + +#endif // SIMDJSON_LASX_BASE_H +/* end file simdjson/lasx/base.h */ +/* including simdjson/lasx/intrinsics.h: #include "simdjson/lasx/intrinsics.h" */ +/* begin file simdjson/lasx/intrinsics.h */ +#ifndef SIMDJSON_LASX_INTRINSICS_H +#define SIMDJSON_LASX_INTRINSICS_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/lasx/base.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +// This should be the correct header whether +// you use visual studio or other compilers. +#include + +static_assert(sizeof(__m256i) <= simdjson::SIMDJSON_PADDING, "insufficient padding for LoongArch ASX"); + +#endif // SIMDJSON_LASX_INTRINSICS_H +/* end file simdjson/lasx/intrinsics.h */ +/* including simdjson/lasx/bitmanipulation.h: #include "simdjson/lasx/bitmanipulation.h" */ +/* begin file simdjson/lasx/bitmanipulation.h */ +#ifndef SIMDJSON_LASX_BITMANIPULATION_H +#define SIMDJSON_LASX_BITMANIPULATION_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/lasx/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/lasx/intrinsics.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/lasx/bitmask.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lasx { + namespace { + + // We sometimes call trailing_zero on inputs that are zero, + // but the algorithms do not end up using the returned value. + // Sadly, sanitizers are not smart enough to figure it out. + SIMDJSON_NO_SANITIZE_UNDEFINED + // This function can be used safely even if not all bytes have been + // initialized. + // See issue https://github.com/simdjson/simdjson/issues/1965 + SIMDJSON_NO_SANITIZE_MEMORY + simdjson_inline int trailing_zeroes(uint64_t input_num) { + return __builtin_ctzll(input_num); + } + + /* result might be undefined when input_num is zero */ + simdjson_inline uint64_t clear_lowest_bit(uint64_t input_num) { + return input_num & (input_num - 1); + } + + /* result might be undefined when input_num is zero */ + simdjson_inline int leading_zeroes(uint64_t input_num) { + return __builtin_clzll(input_num); + } + + /* result might be undefined when input_num is zero */ + simdjson_inline int count_ones(uint64_t input_num) { + return __lasx_xvpickve2gr_w(__lasx_xvpcnt_d(__m256i(v4u64{ input_num, 0, 0, 0 })), 0); + } + + simdjson_inline bool add_overflow(uint64_t value1, uint64_t value2, uint64_t* result) { + return __builtin_uaddll_overflow(value1, value2, + reinterpret_cast(result)); + } + + } // unnamed namespace + } // namespace lasx +} // namespace simdjson + +#endif // SIMDJSON_LASX_BITMANIPULATION_H +/* end file simdjson/lasx/bitmanipulation.h */ +/* including simdjson/lasx/bitmask.h: #include "simdjson/lasx/bitmask.h" */ +/* begin file simdjson/lasx/bitmask.h */ +#ifndef SIMDJSON_LASX_BITMASK_H +#define SIMDJSON_LASX_BITMASK_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/lasx/base.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lasx { + namespace { + + // + // Perform a "cumulative bitwise xor," flipping bits each time a 1 is encountered. + // + // For example, prefix_xor(00100100) == 00011100 + // + simdjson_inline uint64_t prefix_xor(uint64_t bitmask) { + bitmask ^= bitmask << 1; + bitmask ^= bitmask << 2; + bitmask ^= bitmask << 4; + bitmask ^= bitmask << 8; + bitmask ^= bitmask << 16; + bitmask ^= bitmask << 32; + return bitmask; + } + + } // unnamed namespace + } // namespace lasx +} // namespace simdjson + +#endif +/* end file simdjson/lasx/bitmask.h */ +/* including simdjson/lasx/numberparsing_defs.h: #include "simdjson/lasx/numberparsing_defs.h" */ +/* begin file simdjson/lasx/numberparsing_defs.h */ +#ifndef SIMDJSON_LASX_NUMBERPARSING_DEFS_H +#define SIMDJSON_LASX_NUMBERPARSING_DEFS_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/lasx/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/lasx/intrinsics.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +#include + +namespace simdjson { + namespace lasx { + namespace numberparsing { + + // we don't have appropriate instructions, so let us use a scalar function + // credit: https://johnnylee-sde.github.io/Fast-numeric-string-to-int/ + /** @private */ + static simdjson_inline uint32_t parse_eight_digits_unrolled(const uint8_t* chars) { + uint64_t val; + std::memcpy(&val, chars, sizeof(uint64_t)); + val = (val & 0x0F0F0F0F0F0F0F0F) * 2561 >> 8; + val = (val & 0x00FF00FF00FF00FF) * 6553601 >> 16; + return uint32_t((val & 0x0000FFFF0000FFFF) * 42949672960001 >> 32); + } + + simdjson_inline internal::value128 full_multiplication(uint64_t value1, uint64_t value2) { + internal::value128 answer; + __uint128_t r = (static_cast<__uint128_t>(value1)) * value2; + answer.low = uint64_t(r); + answer.high = uint64_t(r >> 64); + return answer; + } + + } // namespace numberparsing + } // namespace lasx +} // namespace simdjson + +#define SIMDJSON_SWAR_NUMBER_PARSING 1 + +#endif // SIMDJSON_LASX_NUMBERPARSING_DEFS_H +/* end file simdjson/lasx/numberparsing_defs.h */ +/* including simdjson/lasx/simd.h: #include "simdjson/lasx/simd.h" */ +/* begin file simdjson/lasx/simd.h */ +#ifndef SIMDJSON_LASX_SIMD_H +#define SIMDJSON_LASX_SIMD_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/lasx/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/lasx/bitmanipulation.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/simdprune_tables.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lasx { + namespace { + namespace simd { + + // Forward-declared so they can be used by splat and friends. + template + struct base { + __m256i value; + + // Zero constructor + simdjson_inline base() : value{ __m256i() } {} + + // Conversion from SIMD register + simdjson_inline base(const __m256i _value) : value(_value) {} + + // Conversion to SIMD register + simdjson_inline operator const __m256i& () const { return this->value; } + simdjson_inline operator __m256i& () { return this->value; } + simdjson_inline operator const v32i8& () const { return (v32i8&)this->value; } + simdjson_inline operator v32i8& () { return (v32i8&)this->value; } + + // Bit operations + simdjson_inline Child operator|(const Child other) const { return __lasx_xvor_v(*this, other); } + simdjson_inline Child operator&(const Child other) const { return __lasx_xvand_v(*this, other); } + simdjson_inline Child operator^(const Child other) const { return __lasx_xvxor_v(*this, other); } + simdjson_inline Child bit_andnot(const Child other) const { return __lasx_xvandn_v(other, *this); } + simdjson_inline Child& operator|=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast | other; return *this_cast; } + simdjson_inline Child& operator&=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast & other; return *this_cast; } + simdjson_inline Child& operator^=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast ^ other; return *this_cast; } + }; + + // Forward-declared so they can be used by splat and friends. + template + struct simd8; + + template> + struct base8 : base> { + simdjson_inline base8() : base>() {} + simdjson_inline base8(const __m256i _value) : base>(_value) {} + + friend simdjson_really_inline Mask operator==(const simd8 lhs, const simd8 rhs) { return __lasx_xvseq_b(lhs, rhs); } + + static const int SIZE = sizeof(base>::value); + + template + simdjson_inline simd8 prev(const simd8 prev_chunk) const { + __m256i hi = __lasx_xvbsll_v(*this, N); + __m256i lo = __lasx_xvbsrl_v(*this, 16 - N); + __m256i tmp = __lasx_xvbsrl_v(prev_chunk, 16 - N); + lo = __lasx_xvpermi_q(lo, tmp, 0x21); + return __lasx_xvor_v(hi, lo); + } + }; + + // SIMD byte mask type (returned by things like eq and gt) + template<> + struct simd8 : base8 { + static simdjson_inline simd8 splat(bool _value) { return __lasx_xvreplgr2vr_b(uint8_t(-(!!_value))); } + + simdjson_inline simd8() : base8() {} + simdjson_inline simd8(const __m256i _value) : base8(_value) {} + // Splat constructor + simdjson_inline simd8(bool _value) : base8(splat(_value)) {} + + simdjson_inline int to_bitmask() const { + __m256i mask = __lasx_xvmskltz_b(*this); + return (__lasx_xvpickve2gr_w(mask, 4) << 16) | (__lasx_xvpickve2gr_w(mask, 0)); + } + simdjson_inline bool any() const { + __m256i v = __lasx_xvmsknz_b(*this); + return (0 == __lasx_xvpickve2gr_w(v, 0)) && (0 == __lasx_xvpickve2gr_w(v, 4)); + } + simdjson_inline simd8 operator~() const { return *this ^ true; } + }; + + template + struct base8_numeric : base8 { + static simdjson_inline simd8 splat(T _value) { + return __lasx_xvreplgr2vr_b(_value); + } + static simdjson_inline simd8 zero() { return __lasx_xvldi(0); } + static simdjson_inline simd8 load(const T values[32]) { + return __lasx_xvld(reinterpret_cast(values), 0); + } + // Repeat 16 values as many times as necessary (usually for lookup tables) + static simdjson_inline simd8 repeat_16( + T v0, T v1, T v2, T v3, T v4, T v5, T v6, T v7, + T v8, T v9, T v10, T v11, T v12, T v13, T v14, T v15 + ) { + return simd8( + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15, + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15 + ); + } + + simdjson_inline base8_numeric() : base8() {} + simdjson_inline base8_numeric(const __m256i _value) : base8(_value) {} + + // Store to array + simdjson_inline void store(T dst[32]) const { + return __lasx_xvst(*this, reinterpret_cast<__m256i*>(dst), 0); + } + + // Addition/subtraction are the same for signed and unsigned + simdjson_inline simd8 operator+(const simd8 other) const { return __lasx_xvadd_b(*this, other); } + simdjson_inline simd8 operator-(const simd8 other) const { return __lasx_xvsub_b(*this, other); } + simdjson_inline simd8& operator+=(const simd8 other) { *this = *this + other; return *static_cast*>(this); } + simdjson_inline simd8& operator-=(const simd8 other) { *this = *this - other; return *static_cast*>(this); } + + // Override to distinguish from bool version + simdjson_inline simd8 operator~() const { return *this ^ 0xFFu; } + + // Perform a lookup assuming the value is between 0 and 16 (undefined behavior for out of range values) + template + simdjson_inline simd8 lookup_16(simd8 lookup_table) const { + return __lasx_xvshuf_b(lookup_table, lookup_table, *this); + } + + // Copies to 'output" all bytes corresponding to a 0 in the mask (interpreted as a bitset). + // Passing a 0 value for mask would be equivalent to writing out every byte to output. + // Only the first 16 - count_ones(mask) bytes of the result are significant but 16 bytes + // get written. + template + simdjson_inline void compress(uint32_t mask, L* output) const { + using internal::thintable_epi8; + using internal::BitsSetTable256mul2; + using internal::pshufb_combine_table; + // this particular implementation was inspired by haswell + // lasx do it in 4 steps, first 8 bytes and then second 8 bytes... + uint8_t mask1 = uint8_t(mask); // least significant 8 bits + uint8_t mask2 = uint8_t(mask >> 8); // second significant 8 bits + uint8_t mask3 = uint8_t(mask >> 16); // ... + uint8_t mask4 = uint8_t(mask >> 24); // ... + // next line just loads the 64-bit values thintable_epi8[mask{1,2,3,4}] + // into a 256-bit register. + __m256i shufmask = { int64_t(thintable_epi8[mask1]), int64_t(thintable_epi8[mask2]) + 0x0808080808080808, int64_t(thintable_epi8[mask3]), int64_t(thintable_epi8[mask4]) + 0x0808080808080808 }; + // this is the version "nearly pruned" + __m256i pruned = __lasx_xvshuf_b(*this, *this, shufmask); + // we still need to put the pieces back together. + // we compute the popcount of the first words: + int pop1 = BitsSetTable256mul2[mask1]; + int pop2 = BitsSetTable256mul2[mask2]; + int pop3 = BitsSetTable256mul2[mask3]; + + // then load the corresponding mask + __m256i masklo = __lasx_xvldx(reinterpret_cast(reinterpret_cast(pshufb_combine_table)), pop1 * 8); + __m256i maskhi = __lasx_xvldx(reinterpret_cast(reinterpret_cast(pshufb_combine_table)), pop3 * 8); + __m256i compactmask = __lasx_xvpermi_q(maskhi, masklo, 0x20); + __m256i answer = __lasx_xvshuf_b(pruned, pruned, compactmask); + __lasx_xvst(answer, reinterpret_cast(output), 0); + uint64_t value3 = __lasx_xvpickve2gr_du(answer, 2); + uint64_t value4 = __lasx_xvpickve2gr_du(answer, 3); + uint64_t* pos = reinterpret_cast(reinterpret_cast(output) + 16 - (pop1 + pop2) / 2); + pos[0] = value3; + pos[1] = value4; + } + + template + simdjson_inline simd8 lookup_16( + L replace0, L replace1, L replace2, L replace3, + L replace4, L replace5, L replace6, L replace7, + L replace8, L replace9, L replace10, L replace11, + L replace12, L replace13, L replace14, L replace15) const { + return lookup_16(simd8::repeat_16( + replace0, replace1, replace2, replace3, + replace4, replace5, replace6, replace7, + replace8, replace9, replace10, replace11, + replace12, replace13, replace14, replace15 + )); + } + }; + + // Signed bytes + template<> + struct simd8 : base8_numeric { + simdjson_inline simd8() : base8_numeric() {} + simdjson_inline simd8(const __m256i _value) : base8_numeric(_value) {} + // Splat constructor + simdjson_inline simd8(int8_t _value) : simd8(splat(_value)) {} + // Array constructor + simdjson_inline simd8(const int8_t values[32]) : simd8(load(values)) {} + // Member-by-member initialization + simdjson_inline simd8( + int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, + int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15, + int8_t v16, int8_t v17, int8_t v18, int8_t v19, int8_t v20, int8_t v21, int8_t v22, int8_t v23, + int8_t v24, int8_t v25, int8_t v26, int8_t v27, int8_t v28, int8_t v29, int8_t v30, int8_t v31 + ) : simd8({ + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10,v11,v12,v13,v14,v15, + v16,v17,v18,v19,v20,v21,v22,v23, + v24,v25,v26,v27,v28,v29,v30,v31 + }) {} + // Repeat 16 values as many times as necessary (usually for lookup tables) + simdjson_inline static simd8 repeat_16( + int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, + int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 + ) { + return simd8( + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15, + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15 + ); + } + + // Order-sensitive comparisons + simdjson_inline simd8 max_val(const simd8 other) const { return __lasx_xvmax_b(*this, other); } + simdjson_inline simd8 min_val(const simd8 other) const { return __lasx_xvmin_b(*this, other); } + simdjson_inline simd8 operator>(const simd8 other) const { return __lasx_xvslt_b(other, *this); } + simdjson_inline simd8 operator<(const simd8 other) const { return __lasx_xvslt_b(*this, other); } + }; + + // Unsigned bytes + template<> + struct simd8 : base8_numeric { + simdjson_inline simd8() : base8_numeric() {} + simdjson_inline simd8(const __m256i _value) : base8_numeric(_value) {} + // Splat constructor + simdjson_inline simd8(uint8_t _value) : simd8(splat(_value)) {} + // Array constructor + simdjson_inline simd8(const uint8_t values[32]) : simd8(load(values)) {} + // Member-by-member initialization + simdjson_inline simd8( + uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, + uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15, + uint8_t v16, uint8_t v17, uint8_t v18, uint8_t v19, uint8_t v20, uint8_t v21, uint8_t v22, uint8_t v23, + uint8_t v24, uint8_t v25, uint8_t v26, uint8_t v27, uint8_t v28, uint8_t v29, uint8_t v30, uint8_t v31 + ) : simd8(__m256i(v32u8{ + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10,v11,v12,v13,v14,v15, + v16,v17,v18,v19,v20,v21,v22,v23, + v24,v25,v26,v27,v28,v29,v30,v31 + })) {} + // Repeat 16 values as many times as necessary (usually for lookup tables) + simdjson_inline static simd8 repeat_16( + uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, + uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 + ) { + return simd8( + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15, + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15 + ); + } + + // Saturated math + simdjson_inline simd8 saturating_add(const simd8 other) const { return __lasx_xvsadd_bu(*this, other); } + simdjson_inline simd8 saturating_sub(const simd8 other) const { return __lasx_xvssub_bu(*this, other); } + + // Order-specific operations + simdjson_inline simd8 max_val(const simd8 other) const { return __lasx_xvmax_bu(*this, other); } + simdjson_inline simd8 min_val(const simd8 other) const { return __lasx_xvmin_bu(other, *this); } + // Same as >, but only guarantees true is nonzero (< guarantees true = -1) + simdjson_inline simd8 gt_bits(const simd8 other) const { return this->saturating_sub(other); } + // Same as <, but only guarantees true is nonzero (< guarantees true = -1) + simdjson_inline simd8 lt_bits(const simd8 other) const { return other.saturating_sub(*this); } + simdjson_inline simd8 operator<=(const simd8 other) const { return other.max_val(*this) == other; } + simdjson_inline simd8 operator>=(const simd8 other) const { return other.min_val(*this) == other; } + simdjson_inline simd8 operator>(const simd8 other) const { return this->gt_bits(other).any_bits_set(); } + simdjson_inline simd8 operator<(const simd8 other) const { return this->lt_bits(other).any_bits_set(); } + + // Bit-specific operations + simdjson_inline simd8 bits_not_set() const { return *this == uint8_t(0); } + simdjson_inline simd8 bits_not_set(simd8 bits) const { return (*this & bits).bits_not_set(); } + simdjson_inline simd8 any_bits_set() const { return ~this->bits_not_set(); } + simdjson_inline simd8 any_bits_set(simd8 bits) const { return ~this->bits_not_set(bits); } + simdjson_inline bool is_ascii() const { + __m256i mask = __lasx_xvmskltz_b(*this); + return (0 == __lasx_xvpickve2gr_w(mask, 0)) && (0 == __lasx_xvpickve2gr_w(mask, 4)); + } + simdjson_inline bool bits_not_set_anywhere() const { + __m256i v = __lasx_xvmsknz_b(*this); + return (0 == __lasx_xvpickve2gr_w(v, 0)) && (0 == __lasx_xvpickve2gr_w(v, 4)); + } + simdjson_inline bool any_bits_set_anywhere() const { return !bits_not_set_anywhere(); } + simdjson_inline bool bits_not_set_anywhere(simd8 bits) const { + __m256i v = __lasx_xvmsknz_b(__lasx_xvand_v(*this, bits)); + return (0 == __lasx_xvpickve2gr_w(v, 0)) && (0 == __lasx_xvpickve2gr_w(v, 4)); + } + simdjson_inline bool any_bits_set_anywhere(simd8 bits) const { return !bits_not_set_anywhere(bits); } + template + simdjson_inline simd8 shr() const { return simd8(__lasx_xvsrli_b(*this, N)); } + template + simdjson_inline simd8 shl() const { return simd8(__lasx_xvslli_b(*this, N)); } + }; + + template + struct simd8x64 { + static constexpr int NUM_CHUNKS = 64 / sizeof(simd8); + static_assert(NUM_CHUNKS == 2, "LASX kernel should use two registers per 64-byte block."); + const simd8 chunks[NUM_CHUNKS]; + + simd8x64(const simd8x64& o) = delete; // no copy allowed + simd8x64& operator=(const simd8& other) = delete; // no assignment allowed + simd8x64() = delete; // no default constructor allowed + + simdjson_inline simd8x64(const simd8 chunk0, const simd8 chunk1) : chunks{ chunk0, chunk1 } {} + simdjson_inline simd8x64(const T ptr[64]) : chunks{ simd8::load(ptr), simd8::load(ptr + 32) } {} + + simdjson_inline uint64_t compress(uint64_t mask, T* output) const { + uint32_t mask1 = uint32_t(mask); + uint32_t mask2 = uint32_t(mask >> 32); + __m256i zcnt = __lasx_xvpcnt_w(__m256i(v4u64{ ~mask, 0, 0, 0 })); + uint64_t zcnt1 = __lasx_xvpickve2gr_wu(zcnt, 0); + uint64_t zcnt2 = __lasx_xvpickve2gr_wu(zcnt, 1); + // There should be a critical value which processes in scaler is faster. + if (zcnt1) + this->chunks[0].compress(mask1, output); + if (zcnt2) + this->chunks[1].compress(mask2, output + zcnt1); + return zcnt1 + zcnt2; + } + + simdjson_inline void store(T ptr[64]) const { + this->chunks[0].store(ptr + sizeof(simd8) * 0); + this->chunks[1].store(ptr + sizeof(simd8) * 1); + } + + simdjson_inline uint64_t to_bitmask() const { + __m256i mask0 = __lasx_xvmskltz_b(this->chunks[0]); + __m256i mask1 = __lasx_xvmskltz_b(this->chunks[1]); + __m256i mask_tmp = __lasx_xvpickve_w(mask0, 4); + __m256i tmp = __lasx_xvpickve_w(mask1, 4); + mask0 = __lasx_xvinsve0_w(mask0, mask1, 1); + mask_tmp = __lasx_xvinsve0_w(mask_tmp, tmp, 1); + return __lasx_xvpickve2gr_du(__lasx_xvpackev_h(mask_tmp, mask0), 0); + } + + simdjson_inline simd8 reduce_or() const { + return this->chunks[0] | this->chunks[1]; + } + + simdjson_inline uint64_t eq(const T m) const { + const simd8 mask = simd8::splat(m); + return simd8x64( + this->chunks[0] == mask, + this->chunks[1] == mask + ).to_bitmask(); + } + + simdjson_inline uint64_t eq(const simd8x64& other) const { + return simd8x64( + this->chunks[0] == other.chunks[0], + this->chunks[1] == other.chunks[1] + ).to_bitmask(); + } + + simdjson_inline uint64_t lteq(const T m) const { + const simd8 mask = simd8::splat(m); + return simd8x64( + this->chunks[0] <= mask, + this->chunks[1] <= mask + ).to_bitmask(); + } + }; // struct simd8x64 + + } // namespace simd + } // unnamed namespace + } // namespace lasx +} // namespace simdjson + +#endif // SIMDJSON_LASX_SIMD_H +/* end file simdjson/lasx/simd.h */ +/* including simdjson/lasx/stringparsing_defs.h: #include "simdjson/lasx/stringparsing_defs.h" */ +/* begin file simdjson/lasx/stringparsing_defs.h */ +#ifndef SIMDJSON_LASX_STRINGPARSING_DEFS_H +#define SIMDJSON_LASX_STRINGPARSING_DEFS_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/lasx/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/lasx/simd.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/lasx/bitmanipulation.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lasx { + namespace { + + using namespace simd; + + // Holds backslashes and quotes locations. + struct backslash_and_quote { + public: + static constexpr uint32_t BYTES_PROCESSED = 32; + simdjson_inline static backslash_and_quote copy_and_find(const uint8_t* src, uint8_t* dst); + + simdjson_inline bool has_quote_first() { return ((bs_bits - 1) & quote_bits) != 0; } + simdjson_inline bool has_backslash() { return bs_bits != 0; } + simdjson_inline int quote_index() { return trailing_zeroes(quote_bits); } + simdjson_inline int backslash_index() { return trailing_zeroes(bs_bits); } + + uint32_t bs_bits; + uint32_t quote_bits; + }; // struct backslash_and_quote + + simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8_t* src, uint8_t* dst) { + // this can read up to 31 bytes beyond the buffer size, but we require + // SIMDJSON_PADDING of padding + static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "backslash and quote finder must process fewer than SIMDJSON_PADDING bytes"); + simd8 v(src); + v.store(dst); + return { + static_cast((v == '\\').to_bitmask()), // bs_bits + static_cast((v == '"').to_bitmask()), // quote_bits + }; + } + + } // unnamed namespace + } // namespace lasx +} // namespace simdjson + +#endif // SIMDJSON_LASX_STRINGPARSING_DEFS_H +/* end file simdjson/lasx/stringparsing_defs.h */ + +#define SIMDJSON_SKIP_BACKSLASH_SHORT_CIRCUIT 1 +/* end file simdjson/lasx/begin.h */ +/* including simdjson/generic/amalgamated.h for lasx: #include "simdjson/generic/amalgamated.h" */ +/* begin file simdjson/generic/amalgamated.h for lasx */ +#if defined(SIMDJSON_CONDITIONAL_INCLUDE) && !defined(SIMDJSON_GENERIC_DEPENDENCIES_H) +#error simdjson/generic/dependencies.h must be included before simdjson/generic/amalgamated.h! +#endif + +/* including simdjson/generic/base.h for lasx: #include "simdjson/generic/base.h" */ +/* begin file simdjson/generic/base.h for lasx */ +#ifndef SIMDJSON_GENERIC_BASE_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_BASE_H */ +/* amalgamation skipped (editor-only): #include "simdjson/base.h" */ +/* amalgamation skipped (editor-only): // If we haven't got an implementation yet, we're in the editor, editing a generic file! Just */ +/* amalgamation skipped (editor-only): // use the most advanced one we can so the most possible stuff can be tested. */ +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_IMPLEMENTATION */ +/* amalgamation skipped (editor-only): #include "simdjson/implementation_detection.h" */ +/* amalgamation skipped (editor-only): #if SIMDJSON_IMPLEMENTATION_ICELAKE */ +/* amalgamation skipped (editor-only): #include "simdjson/icelake/begin.h" */ +/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_HASWELL */ +/* amalgamation skipped (editor-only): #include "simdjson/haswell/begin.h" */ +/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_WESTMERE */ +/* amalgamation skipped (editor-only): #include "simdjson/westmere/begin.h" */ +/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_ARM64 */ +/* amalgamation skipped (editor-only): #include "simdjson/arm64/begin.h" */ +/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_PPC64 */ +/* amalgamation skipped (editor-only): #include "simdjson/ppc64/begin.h" */ +/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_LSX */ +/* amalgamation skipped (editor-only): #include "simdjson/lsx/begin.h" */ +/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_LASX */ +/* amalgamation skipped (editor-only): #include "simdjson/lasx/begin.h" */ +/* amalgamation skipped (editor-only): #elif SIMDJSON_IMPLEMENTATION_FALLBACK */ +/* amalgamation skipped (editor-only): #include "simdjson/fallback/begin.h" */ +/* amalgamation skipped (editor-only): #else */ +/* amalgamation skipped (editor-only): #error "All possible implementations (including fallback) have been disabled! simdjson will not run." */ +/* amalgamation skipped (editor-only): #endif */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_IMPLEMENTATION */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lasx { + + struct open_container; + class dom_parser_implementation; + + /** + * The type of a JSON number + */ + enum class number_type { + floating_point_number = 1, /// a binary64 number + signed_integer, /// a signed integer that fits in a 64-bit word using two's complement + unsigned_integer, /// a positive integer larger or equal to 1<<63 + big_integer /// a big integer that does not fit in a 64-bit word + }; + + } // namespace lasx +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_BASE_H +/* end file simdjson/generic/base.h for lasx */ +/* including simdjson/generic/jsoncharutils.h for lasx: #include "simdjson/generic/jsoncharutils.h" */ +/* begin file simdjson/generic/jsoncharutils.h for lasx */ +#ifndef SIMDJSON_GENERIC_JSONCHARUTILS_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_JSONCHARUTILS_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/jsoncharutils_tables.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lasx { + namespace { + namespace jsoncharutils { + + // return non-zero if not a structural or whitespace char + // zero otherwise + simdjson_inline uint32_t is_not_structural_or_whitespace(uint8_t c) { + return internal::structural_or_whitespace_negated[c]; + } + + simdjson_inline uint32_t is_structural_or_whitespace(uint8_t c) { + return internal::structural_or_whitespace[c]; + } + + // returns a value with the high 16 bits set if not valid + // otherwise returns the conversion of the 4 hex digits at src into the bottom + // 16 bits of the 32-bit return register + // + // see + // https://lemire.me/blog/2019/04/17/parsing-short-hexadecimal-strings-efficiently/ + static inline uint32_t hex_to_u32_nocheck( + const uint8_t* src) { // strictly speaking, static inline is a C-ism + uint32_t v1 = internal::digit_to_val32[630 + src[0]]; + uint32_t v2 = internal::digit_to_val32[420 + src[1]]; + uint32_t v3 = internal::digit_to_val32[210 + src[2]]; + uint32_t v4 = internal::digit_to_val32[0 + src[3]]; + return v1 | v2 | v3 | v4; + } + + // given a code point cp, writes to c + // the utf-8 code, outputting the length in + // bytes, if the length is zero, the code point + // is invalid + // + // This can possibly be made faster using pdep + // and clz and table lookups, but JSON documents + // have few escaped code points, and the following + // function looks cheap. + // + // Note: we assume that surrogates are treated separately + // + simdjson_inline size_t codepoint_to_utf8(uint32_t cp, uint8_t* c) { + if (cp <= 0x7F) { + c[0] = uint8_t(cp); + return 1; // ascii + } + if (cp <= 0x7FF) { + c[0] = uint8_t((cp >> 6) + 192); + c[1] = uint8_t((cp & 63) + 128); + return 2; // universal plane + // Surrogates are treated elsewhere... + //} //else if (0xd800 <= cp && cp <= 0xdfff) { + // return 0; // surrogates // could put assert here + } + else if (cp <= 0xFFFF) { + c[0] = uint8_t((cp >> 12) + 224); + c[1] = uint8_t(((cp >> 6) & 63) + 128); + c[2] = uint8_t((cp & 63) + 128); + return 3; + } + else if (cp <= 0x10FFFF) { // if you know you have a valid code point, this + // is not needed + c[0] = uint8_t((cp >> 18) + 240); + c[1] = uint8_t(((cp >> 12) & 63) + 128); + c[2] = uint8_t(((cp >> 6) & 63) + 128); + c[3] = uint8_t((cp & 63) + 128); + return 4; + } + // will return 0 when the code point was too large. + return 0; // bad r + } + +#if SIMDJSON_IS_32BITS // _umul128 for x86, arm + // this is a slow emulation routine for 32-bit + // + static simdjson_inline uint64_t __emulu(uint32_t x, uint32_t y) { + return x * (uint64_t)y; + } + static simdjson_inline uint64_t _umul128(uint64_t ab, uint64_t cd, uint64_t* hi) { + uint64_t ad = __emulu((uint32_t)(ab >> 32), (uint32_t)cd); + uint64_t bd = __emulu((uint32_t)ab, (uint32_t)cd); + uint64_t adbc = ad + __emulu((uint32_t)ab, (uint32_t)(cd >> 32)); + uint64_t adbc_carry = !!(adbc < ad); + uint64_t lo = bd + (adbc << 32); + *hi = __emulu((uint32_t)(ab >> 32), (uint32_t)(cd >> 32)) + (adbc >> 32) + + (adbc_carry << 32) + !!(lo < bd); + return lo; + } +#endif + + } // namespace jsoncharutils + } // unnamed namespace + } // namespace lasx +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_JSONCHARUTILS_H +/* end file simdjson/generic/jsoncharutils.h for lasx */ +/* including simdjson/generic/atomparsing.h for lasx: #include "simdjson/generic/atomparsing.h" */ +/* begin file simdjson/generic/atomparsing.h for lasx */ +#ifndef SIMDJSON_GENERIC_ATOMPARSING_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_ATOMPARSING_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/jsoncharutils.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +#include + +namespace simdjson { + namespace lasx { + namespace { + /// @private + namespace atomparsing { + + // The string_to_uint32 is exclusively used to map literal strings to 32-bit values. + // We use memcpy instead of a pointer cast to avoid undefined behaviors since we cannot + // be certain that the character pointer will be properly aligned. + // You might think that using memcpy makes this function expensive, but you'd be wrong. + // All decent optimizing compilers (GCC, clang, Visual Studio) will compile string_to_uint32("false"); + // to the compile-time constant 1936482662. + simdjson_inline uint32_t string_to_uint32(const char* str) { uint32_t val; std::memcpy(&val, str, sizeof(uint32_t)); return val; } + + + // Again in str4ncmp we use a memcpy to avoid undefined behavior. The memcpy may appear expensive. + // Yet all decent optimizing compilers will compile memcpy to a single instruction, just about. + simdjson_warn_unused + simdjson_inline uint32_t str4ncmp(const uint8_t* src, const char* atom) { + uint32_t srcval; // we want to avoid unaligned 32-bit loads (undefined in C/C++) + static_assert(sizeof(uint32_t) <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be larger than 4 bytes"); + std::memcpy(&srcval, src, sizeof(uint32_t)); + return srcval ^ string_to_uint32(atom); + } + + simdjson_warn_unused + simdjson_inline bool is_valid_true_atom(const uint8_t* src) { + return (str4ncmp(src, "true") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; + } + + simdjson_warn_unused + simdjson_inline bool is_valid_true_atom(const uint8_t* src, size_t len) { + if (len > 4) { return is_valid_true_atom(src); } + else if (len == 4) { return !str4ncmp(src, "true"); } + else { return false; } + } + + simdjson_warn_unused + simdjson_inline bool is_valid_false_atom(const uint8_t* src) { + return (str4ncmp(src + 1, "alse") | jsoncharutils::is_not_structural_or_whitespace(src[5])) == 0; + } + + simdjson_warn_unused + simdjson_inline bool is_valid_false_atom(const uint8_t* src, size_t len) { + if (len > 5) { return is_valid_false_atom(src); } + else if (len == 5) { return !str4ncmp(src + 1, "alse"); } + else { return false; } + } + + simdjson_warn_unused + simdjson_inline bool is_valid_null_atom(const uint8_t* src) { + return (str4ncmp(src, "null") | jsoncharutils::is_not_structural_or_whitespace(src[4])) == 0; + } + + simdjson_warn_unused + simdjson_inline bool is_valid_null_atom(const uint8_t* src, size_t len) { + if (len > 4) { return is_valid_null_atom(src); } + else if (len == 4) { return !str4ncmp(src, "null"); } + else { return false; } + } + + } // namespace atomparsing + } // unnamed namespace + } // namespace lasx +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_ATOMPARSING_H +/* end file simdjson/generic/atomparsing.h for lasx */ +/* including simdjson/generic/dom_parser_implementation.h for lasx: #include "simdjson/generic/dom_parser_implementation.h" */ +/* begin file simdjson/generic/dom_parser_implementation.h for lasx */ +#ifndef SIMDJSON_GENERIC_DOM_PARSER_IMPLEMENTATION_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_DOM_PARSER_IMPLEMENTATION_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/dom_parser_implementation.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lasx { + + // expectation: sizeof(open_container) = 64/8. + struct open_container { + uint32_t tape_index; // where, on the tape, does the scope ([,{) begins + uint32_t count; // how many elements in the scope + }; // struct open_container + + static_assert(sizeof(open_container) == 64 / 8, "Open container must be 64 bits"); + + class dom_parser_implementation final : public internal::dom_parser_implementation { + public: + /** Tape location of each open { or [ */ + std::unique_ptr open_containers{}; + /** Whether each open container is a [ or { */ + std::unique_ptr is_array{}; + /** Buffer passed to stage 1 */ + const uint8_t* buf{}; + /** Length passed to stage 1 */ + size_t len{ 0 }; + /** Document passed to stage 2 */ + dom::document* doc{}; + + inline dom_parser_implementation() noexcept; + inline dom_parser_implementation(dom_parser_implementation&& other) noexcept; + inline dom_parser_implementation& operator=(dom_parser_implementation&& other) noexcept; + dom_parser_implementation(const dom_parser_implementation&) = delete; + dom_parser_implementation& operator=(const dom_parser_implementation&) = delete; + + simdjson_warn_unused error_code parse(const uint8_t* buf, size_t len, dom::document& doc) noexcept final; + simdjson_warn_unused error_code stage1(const uint8_t* buf, size_t len, stage1_mode partial) noexcept final; + simdjson_warn_unused error_code stage2(dom::document& doc) noexcept final; + simdjson_warn_unused error_code stage2_next(dom::document& doc) noexcept final; + simdjson_warn_unused uint8_t* parse_string(const uint8_t* src, uint8_t* dst, bool allow_replacement) const noexcept final; + simdjson_warn_unused uint8_t* parse_wobbly_string(const uint8_t* src, uint8_t* dst) const noexcept final; + inline simdjson_warn_unused error_code set_capacity(size_t capacity) noexcept final; + inline simdjson_warn_unused error_code set_max_depth(size_t max_depth) noexcept final; + private: + simdjson_inline simdjson_warn_unused error_code set_capacity_stage1(size_t capacity); + + }; + + } // namespace lasx +} // namespace simdjson + +namespace simdjson { + namespace lasx { + + inline dom_parser_implementation::dom_parser_implementation() noexcept = default; + inline dom_parser_implementation::dom_parser_implementation(dom_parser_implementation&& other) noexcept = default; + inline dom_parser_implementation& dom_parser_implementation::operator=(dom_parser_implementation&& other) noexcept = default; + + // Leaving these here so they can be inlined if so desired + inline simdjson_warn_unused error_code dom_parser_implementation::set_capacity(size_t capacity) noexcept { + if (capacity > SIMDJSON_MAXSIZE_BYTES) { return CAPACITY; } + // Stage 1 index output + size_t max_structures = SIMDJSON_ROUNDUP_N(capacity, 64) + 2 + 7; + structural_indexes.reset(new (std::nothrow) uint32_t[max_structures]); + if (!structural_indexes) { _capacity = 0; return MEMALLOC; } + structural_indexes[0] = 0; + n_structural_indexes = 0; + + _capacity = capacity; + return SUCCESS; + } + + inline simdjson_warn_unused error_code dom_parser_implementation::set_max_depth(size_t max_depth) noexcept { + // Stage 2 stacks + open_containers.reset(new (std::nothrow) open_container[max_depth]); + is_array.reset(new (std::nothrow) bool[max_depth]); + if (!is_array || !open_containers) { _max_depth = 0; return MEMALLOC; } + + _max_depth = max_depth; + return SUCCESS; + } + + } // namespace lasx +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_DOM_PARSER_IMPLEMENTATION_H +/* end file simdjson/generic/dom_parser_implementation.h for lasx */ +/* including simdjson/generic/implementation_simdjson_result_base.h for lasx: #include "simdjson/generic/implementation_simdjson_result_base.h" */ +/* begin file simdjson/generic/implementation_simdjson_result_base.h for lasx */ +#ifndef SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lasx { + + // This is a near copy of include/error.h's implementation_simdjson_result_base, except it doesn't use std::pair + // so we can avoid inlining errors + // TODO reconcile these! + /** + * The result of a simdjson operation that could fail. + * + * Gives the option of reading error codes, or throwing an exception by casting to the desired result. + * + * This is a base class for implementations that want to add functions to the result type for + * chaining. + * + * Override like: + * + * struct simdjson_result : public internal::implementation_simdjson_result_base { + * simdjson_result() noexcept : internal::implementation_simdjson_result_base() {} + * simdjson_result(error_code error) noexcept : internal::implementation_simdjson_result_base(error) {} + * simdjson_result(T &&value) noexcept : internal::implementation_simdjson_result_base(std::forward(value)) {} + * simdjson_result(T &&value, error_code error) noexcept : internal::implementation_simdjson_result_base(value, error) {} + * // Your extra methods here + * } + * + * Then any method returning simdjson_result will be chainable with your methods. + */ + template + struct implementation_simdjson_result_base { + + /** + * Create a new empty result with error = UNINITIALIZED. + */ + simdjson_inline implementation_simdjson_result_base() noexcept = default; + + /** + * Create a new error result. + */ + simdjson_inline implementation_simdjson_result_base(error_code error) noexcept; + + /** + * Create a new successful result. + */ + simdjson_inline implementation_simdjson_result_base(T&& value) noexcept; + + /** + * Create a new result with both things (use if you don't want to branch when creating the result). + */ + simdjson_inline implementation_simdjson_result_base(T&& value, error_code error) noexcept; + + /** + * Move the value and the error to the provided variables. + * + * @param value The variable to assign the value to. May not be set if there is an error. + * @param error The variable to assign the error to. Set to SUCCESS if there is no error. + */ + simdjson_inline void tie(T& value, error_code& error) && noexcept; + + /** + * Move the value to the provided variable. + * + * @param value The variable to assign the value to. May not be set if there is an error. + */ + simdjson_inline error_code get(T& value) && noexcept; + + /** + * The error. + */ + simdjson_inline error_code error() const noexcept; + +#if SIMDJSON_EXCEPTIONS + + /** + * Get the result value. + * + * @throw simdjson_error if there was an error. + */ + simdjson_inline T& value() & noexcept(false); + + /** + * Take the result value (move it). + * + * @throw simdjson_error if there was an error. + */ + simdjson_inline T&& value() && noexcept(false); + + /** + * Take the result value (move it). + * + * @throw simdjson_error if there was an error. + */ + simdjson_inline T&& take_value() && noexcept(false); + + /** + * Cast to the value (will throw on error). + * + * @throw simdjson_error if there was an error. + */ + simdjson_inline operator T && () && noexcept(false); + + +#endif // SIMDJSON_EXCEPTIONS + + /** + * Get the result value. This function is safe if and only + * the error() method returns a value that evaluates to false. + */ + simdjson_inline const T& value_unsafe() const& noexcept; + /** + * Get the result value. This function is safe if and only + * the error() method returns a value that evaluates to false. + */ + simdjson_inline T& value_unsafe() & noexcept; + /** + * Take the result value (move it). This function is safe if and only + * the error() method returns a value that evaluates to false. + */ + simdjson_inline T&& value_unsafe() && noexcept; + protected: + /** users should never directly access first and second. **/ + T first{}; /** Users should never directly access 'first'. **/ + error_code second{ UNINITIALIZED }; /** Users should never directly access 'second'. **/ + }; // struct implementation_simdjson_result_base + + } // namespace lasx +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_H +/* end file simdjson/generic/implementation_simdjson_result_base.h for lasx */ +/* including simdjson/generic/numberparsing.h for lasx: #include "simdjson/generic/numberparsing.h" */ +/* begin file simdjson/generic/numberparsing.h for lasx */ +#ifndef SIMDJSON_GENERIC_NUMBERPARSING_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_NUMBERPARSING_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/jsoncharutils.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +#include +#include +#include + +namespace simdjson { + namespace lasx { + namespace numberparsing { + +#ifdef JSON_TEST_NUMBERS +#define INVALID_NUMBER(SRC) (found_invalid_number((SRC)), NUMBER_ERROR) +#define WRITE_INTEGER(VALUE, SRC, WRITER) (found_integer((VALUE), (SRC)), (WRITER).append_s64((VALUE))) +#define WRITE_UNSIGNED(VALUE, SRC, WRITER) (found_unsigned_integer((VALUE), (SRC)), (WRITER).append_u64((VALUE))) +#define WRITE_DOUBLE(VALUE, SRC, WRITER) (found_float((VALUE), (SRC)), (WRITER).append_double((VALUE))) +#define BIGINT_NUMBER(SRC) (found_invalid_number((SRC)), BIGINT_ERROR) +#else +#define INVALID_NUMBER(SRC) (NUMBER_ERROR) +#define WRITE_INTEGER(VALUE, SRC, WRITER) (WRITER).append_s64((VALUE)) +#define WRITE_UNSIGNED(VALUE, SRC, WRITER) (WRITER).append_u64((VALUE)) +#define WRITE_DOUBLE(VALUE, SRC, WRITER) (WRITER).append_double((VALUE)) +#define BIGINT_NUMBER(SRC) (BIGINT_ERROR) +#endif + + namespace { + + // Convert a mantissa, an exponent and a sign bit into an ieee64 double. + // The real_exponent needs to be in [0, 2046] (technically real_exponent = 2047 would be acceptable). + // The mantissa should be in [0,1<<53). The bit at index (1ULL << 52) while be zeroed. + simdjson_inline double to_double(uint64_t mantissa, uint64_t real_exponent, bool negative) { + double d; + mantissa &= ~(1ULL << 52); + mantissa |= real_exponent << 52; + mantissa |= ((static_cast(negative)) << 63); + std::memcpy(&d, &mantissa, sizeof(d)); + return d; + } + + // Attempts to compute i * 10^(power) exactly; and if "negative" is + // true, negate the result. + // This function will only work in some cases, when it does not work, success is + // set to false. This should work *most of the time* (like 99% of the time). + // We assume that power is in the [smallest_power, + // largest_power] interval: the caller is responsible for this check. + simdjson_inline bool compute_float_64(int64_t power, uint64_t i, bool negative, double& d) { + // we start with a fast path + // It was described in + // Clinger WD. How to read floating point numbers accurately. + // ACM SIGPLAN Notices. 1990 +#ifndef FLT_EVAL_METHOD +#error "FLT_EVAL_METHOD should be defined, please include cfloat." +#endif +#if (FLT_EVAL_METHOD != 1) && (FLT_EVAL_METHOD != 0) + // We cannot be certain that x/y is rounded to nearest. + if (0 <= power && power <= 22 && i <= 9007199254740991) +#else + if (-22 <= power && power <= 22 && i <= 9007199254740991) +#endif + { + // convert the integer into a double. This is lossless since + // 0 <= i <= 2^53 - 1. + d = double(i); + // + // The general idea is as follows. + // If 0 <= s < 2^53 and if 10^0 <= p <= 10^22 then + // 1) Both s and p can be represented exactly as 64-bit floating-point + // values + // (binary64). + // 2) Because s and p can be represented exactly as floating-point values, + // then s * p + // and s / p will produce correctly rounded values. + // + if (power < 0) { + d = d / simdjson::internal::power_of_ten[-power]; + } + else { + d = d * simdjson::internal::power_of_ten[power]; + } + if (negative) { + d = -d; + } + return true; + } + // When 22 < power && power < 22 + 16, we could + // hope for another, secondary fast path. It was + // described by David M. Gay in "Correctly rounded + // binary-decimal and decimal-binary conversions." (1990) + // If you need to compute i * 10^(22 + x) for x < 16, + // first compute i * 10^x, if you know that result is exact + // (e.g., when i * 10^x < 2^53), + // then you can still proceed and do (i * 10^x) * 10^22. + // Is this worth your time? + // You need 22 < power *and* power < 22 + 16 *and* (i * 10^(x-22) < 2^53) + // for this second fast path to work. + // If you you have 22 < power *and* power < 22 + 16, and then you + // optimistically compute "i * 10^(x-22)", there is still a chance that you + // have wasted your time if i * 10^(x-22) >= 2^53. It makes the use cases of + // this optimization maybe less common than we would like. Source: + // http://www.exploringbinary.com/fast-path-decimal-to-floating-point-conversion/ + // also used in RapidJSON: https://rapidjson.org/strtod_8h_source.html + + // The fast path has now failed, so we are failing back on the slower path. + + // In the slow path, we need to adjust i so that it is > 1<<63 which is always + // possible, except if i == 0, so we handle i == 0 separately. + if (i == 0) { + d = negative ? -0.0 : 0.0; + return true; + } + + + // The exponent is 1024 + 63 + power + // + floor(log(5**power)/log(2)). + // The 1024 comes from the ieee64 standard. + // The 63 comes from the fact that we use a 64-bit word. + // + // Computing floor(log(5**power)/log(2)) could be + // slow. Instead we use a fast function. + // + // For power in (-400,350), we have that + // (((152170 + 65536) * power ) >> 16); + // is equal to + // floor(log(5**power)/log(2)) + power when power >= 0 + // and it is equal to + // ceil(log(5**-power)/log(2)) + power when power < 0 + // + // The 65536 is (1<<16) and corresponds to + // (65536 * power) >> 16 ---> power + // + // ((152170 * power ) >> 16) is equal to + // floor(log(5**power)/log(2)) + // + // Note that this is not magic: 152170/(1<<16) is + // approximatively equal to log(5)/log(2). + // The 1<<16 value is a power of two; we could use a + // larger power of 2 if we wanted to. + // + int64_t exponent = (((152170 + 65536) * power) >> 16) + 1024 + 63; + + + // We want the most significant bit of i to be 1. Shift if needed. + int lz = leading_zeroes(i); + i <<= lz; + + + // We are going to need to do some 64-bit arithmetic to get a precise product. + // We use a table lookup approach. + // It is safe because + // power >= smallest_power + // and power <= largest_power + // We recover the mantissa of the power, it has a leading 1. It is always + // rounded down. + // + // We want the most significant 64 bits of the product. We know + // this will be non-zero because the most significant bit of i is + // 1. + const uint32_t index = 2 * uint32_t(power - simdjson::internal::smallest_power); + // Optimization: It may be that materializing the index as a variable might confuse some compilers and prevent effective complex-addressing loads. (Done for code clarity.) + // + // The full_multiplication function computes the 128-bit product of two 64-bit words + // with a returned value of type value128 with a "low component" corresponding to the + // 64-bit least significant bits of the product and with a "high component" corresponding + // to the 64-bit most significant bits of the product. + simdjson::internal::value128 firstproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index]); + // Both i and power_of_five_128[index] have their most significant bit set to 1 which + // implies that the either the most or the second most significant bit of the product + // is 1. We pack values in this manner for efficiency reasons: it maximizes the use + // we make of the product. It also makes it easy to reason about the product: there + // is 0 or 1 leading zero in the product. + + // Unless the least significant 9 bits of the high (64-bit) part of the full + // product are all 1s, then we know that the most significant 55 bits are + // exact and no further work is needed. Having 55 bits is necessary because + // we need 53 bits for the mantissa but we have to have one rounding bit and + // we can waste a bit if the most significant bit of the product is zero. + if ((firstproduct.high & 0x1FF) == 0x1FF) { + // We want to compute i * 5^q, but only care about the top 55 bits at most. + // Consider the scenario where q>=0. Then 5^q may not fit in 64-bits. Doing + // the full computation is wasteful. So we do what is called a "truncated + // multiplication". + // We take the most significant 64-bits, and we put them in + // power_of_five_128[index]. Usually, that's good enough to approximate i * 5^q + // to the desired approximation using one multiplication. Sometimes it does not suffice. + // Then we store the next most significant 64 bits in power_of_five_128[index + 1], and + // then we get a better approximation to i * 5^q. + // + // That's for when q>=0. The logic for q<0 is somewhat similar but it is somewhat + // more complicated. + // + // There is an extra layer of complexity in that we need more than 55 bits of + // accuracy in the round-to-even scenario. + // + // The full_multiplication function computes the 128-bit product of two 64-bit words + // with a returned value of type value128 with a "low component" corresponding to the + // 64-bit least significant bits of the product and with a "high component" corresponding + // to the 64-bit most significant bits of the product. + simdjson::internal::value128 secondproduct = full_multiplication(i, simdjson::internal::power_of_five_128[index + 1]); + firstproduct.low += secondproduct.high; + if (secondproduct.high > firstproduct.low) { firstproduct.high++; } + // As it has been proven by Noble Mushtak and Daniel Lemire in "Fast Number Parsing Without + // Fallback" (https://arxiv.org/abs/2212.06644), at this point we are sure that the product + // is sufficiently accurate, and more computation is not needed. + } + uint64_t lower = firstproduct.low; + uint64_t upper = firstproduct.high; + // The final mantissa should be 53 bits with a leading 1. + // We shift it so that it occupies 54 bits with a leading 1. + /////// + uint64_t upperbit = upper >> 63; + uint64_t mantissa = upper >> (upperbit + 9); + lz += int(1 ^ upperbit); + + // Here we have mantissa < (1<<54). + int64_t real_exponent = exponent - lz; + if (simdjson_unlikely(real_exponent <= 0)) { // we have a subnormal? + // Here have that real_exponent <= 0 so -real_exponent >= 0 + if (-real_exponent + 1 >= 64) { // if we have more than 64 bits below the minimum exponent, you have a zero for sure. + d = negative ? -0.0 : 0.0; + return true; + } + // next line is safe because -real_exponent + 1 < 0 + mantissa >>= -real_exponent + 1; + // Thankfully, we can't have both "round-to-even" and subnormals because + // "round-to-even" only occurs for powers close to 0. + mantissa += (mantissa & 1); // round up + mantissa >>= 1; + // There is a weird scenario where we don't have a subnormal but just. + // Suppose we start with 2.2250738585072013e-308, we end up + // with 0x3fffffffffffff x 2^-1023-53 which is technically subnormal + // whereas 0x40000000000000 x 2^-1023-53 is normal. Now, we need to round + // up 0x3fffffffffffff x 2^-1023-53 and once we do, we are no longer + // subnormal, but we can only know this after rounding. + // So we only declare a subnormal if we are smaller than the threshold. + real_exponent = (mantissa < (uint64_t(1) << 52)) ? 0 : 1; + d = to_double(mantissa, real_exponent, negative); + return true; + } + // We have to round to even. The "to even" part + // is only a problem when we are right in between two floats + // which we guard against. + // If we have lots of trailing zeros, we may fall right between two + // floating-point values. + // + // The round-to-even cases take the form of a number 2m+1 which is in (2^53,2^54] + // times a power of two. That is, it is right between a number with binary significand + // m and another number with binary significand m+1; and it must be the case + // that it cannot be represented by a float itself. + // + // We must have that w * 10 ^q == (2m+1) * 2^p for some power of two 2^p. + // Recall that 10^q = 5^q * 2^q. + // When q >= 0, we must have that (2m+1) is divible by 5^q, so 5^q <= 2^54. We have that + // 5^23 <= 2^54 and it is the last power of five to qualify, so q <= 23. + // When q<0, we have w >= (2m+1) x 5^{-q}. We must have that w<2^{64} so + // (2m+1) x 5^{-q} < 2^{64}. We have that 2m+1>2^{53}. Hence, we must have + // 2^{53} x 5^{-q} < 2^{64}. + // Hence we have 5^{-q} < 2^{11}$ or q>= -4. + // + // We require lower <= 1 and not lower == 0 because we could not prove that + // that lower == 0 is implied; but we could prove that lower <= 1 is a necessary and sufficient test. + if (simdjson_unlikely((lower <= 1) && (power >= -4) && (power <= 23) && ((mantissa & 3) == 1))) { + if ((mantissa << (upperbit + 64 - 53 - 2)) == upper) { + mantissa &= ~1; // flip it so that we do not round up + } + } + + mantissa += mantissa & 1; + mantissa >>= 1; + + // Here we have mantissa < (1<<53), unless there was an overflow + if (mantissa >= (1ULL << 53)) { + ////////// + // This will happen when parsing values such as 7.2057594037927933e+16 + //////// + mantissa = (1ULL << 52); + real_exponent++; + } + mantissa &= ~(1ULL << 52); + // we have to check that real_exponent is in range, otherwise we bail out + if (simdjson_unlikely(real_exponent > 2046)) { + // We have an infinite value!!! We could actually throw an error here if we could. + return false; + } + d = to_double(mantissa, real_exponent, negative); + return true; + } + + // We call a fallback floating-point parser that might be slow. Note + // it will accept JSON numbers, but the JSON spec. is more restrictive so + // before you call parse_float_fallback, you need to have validated the input + // string with the JSON grammar. + // It will return an error (false) if the parsed number is infinite. + // The string parsing itself always succeeds. We know that there is at least + // one digit. + static bool parse_float_fallback(const uint8_t* ptr, double* outDouble) { + *outDouble = simdjson::internal::from_chars(reinterpret_cast(ptr)); + // We do not accept infinite values. + + // Detecting finite values in a portable manner is ridiculously hard, ideally + // we would want to do: + // return !std::isfinite(*outDouble); + // but that mysteriously fails under legacy/old libc++ libraries, see + // https://github.com/simdjson/simdjson/issues/1286 + // + // Therefore, fall back to this solution (the extra parens are there + // to handle that max may be a macro on windows). + return !(*outDouble > (std::numeric_limits::max)() || *outDouble < std::numeric_limits::lowest()); + } + + static bool parse_float_fallback(const uint8_t* ptr, const uint8_t* end_ptr, double* outDouble) { + *outDouble = simdjson::internal::from_chars(reinterpret_cast(ptr), reinterpret_cast(end_ptr)); + // We do not accept infinite values. + + // Detecting finite values in a portable manner is ridiculously hard, ideally + // we would want to do: + // return !std::isfinite(*outDouble); + // but that mysteriously fails under legacy/old libc++ libraries, see + // https://github.com/simdjson/simdjson/issues/1286 + // + // Therefore, fall back to this solution (the extra parens are there + // to handle that max may be a macro on windows). + return !(*outDouble > (std::numeric_limits::max)() || *outDouble < std::numeric_limits::lowest()); + } + + // check quickly whether the next 8 chars are made of digits + // at a glance, it looks better than Mula's + // http://0x80.pl/articles/swar-digits-validate.html + simdjson_inline bool is_made_of_eight_digits_fast(const uint8_t* chars) { + uint64_t val; + // this can read up to 7 bytes beyond the buffer size, but we require + // SIMDJSON_PADDING of padding + static_assert(7 <= SIMDJSON_PADDING, "SIMDJSON_PADDING must be bigger than 7"); + std::memcpy(&val, chars, 8); + // a branchy method might be faster: + // return (( val & 0xF0F0F0F0F0F0F0F0 ) == 0x3030303030303030) + // && (( (val + 0x0606060606060606) & 0xF0F0F0F0F0F0F0F0 ) == + // 0x3030303030303030); + return (((val & 0xF0F0F0F0F0F0F0F0) | + (((val + 0x0606060606060606) & 0xF0F0F0F0F0F0F0F0) >> 4)) == + 0x3333333333333333); + } + + template + SIMDJSON_NO_SANITIZE_UNDEFINED // We deliberately allow overflow here and check later + simdjson_inline bool parse_digit(const uint8_t c, I& i) { + const uint8_t digit = static_cast(c - '0'); + if (digit > 9) { + return false; + } + // PERF NOTE: multiplication by 10 is cheaper than arbitrary integer multiplication + i = 10 * i + digit; // might overflow, we will handle the overflow later + return true; + } + + simdjson_inline bool is_digit(const uint8_t c) { + return static_cast(c - '0') <= 9; + } + + simdjson_inline error_code parse_decimal_after_separator(simdjson_unused const uint8_t* const src, const uint8_t*& p, uint64_t& i, int64_t& exponent) { + // we continue with the fiction that we have an integer. If the + // floating point number is representable as x * 10^z for some integer + // z that fits in 53 bits, then we will be able to convert back the + // the integer into a float in a lossless manner. + const uint8_t* const first_after_period = p; + +#ifdef SIMDJSON_SWAR_NUMBER_PARSING +#if SIMDJSON_SWAR_NUMBER_PARSING + // this helps if we have lots of decimals! + // this turns out to be frequent enough. + if (is_made_of_eight_digits_fast(p)) { + i = i * 100000000 + parse_eight_digits_unrolled(p); + p += 8; + } +#endif // SIMDJSON_SWAR_NUMBER_PARSING +#endif // #ifdef SIMDJSON_SWAR_NUMBER_PARSING + // Unrolling the first digit makes a small difference on some implementations (e.g. westmere) + if (parse_digit(*p, i)) { ++p; } + while (parse_digit(*p, i)) { p++; } + exponent = first_after_period - p; + // Decimal without digits (123.) is illegal + if (exponent == 0) { + return INVALID_NUMBER(src); + } + return SUCCESS; + } + + simdjson_inline error_code parse_exponent(simdjson_unused const uint8_t* const src, const uint8_t*& p, int64_t& exponent) { + // Exp Sign: -123.456e[-]78 + bool neg_exp = ('-' == *p); + if (neg_exp || '+' == *p) { p++; } // Skip + as well + + // Exponent: -123.456e-[78] + auto start_exp = p; + int64_t exp_number = 0; + while (parse_digit(*p, exp_number)) { ++p; } + // It is possible for parse_digit to overflow. + // In particular, it could overflow to INT64_MIN, and we cannot do - INT64_MIN. + // Thus we *must* check for possible overflow before we negate exp_number. + + // Performance notes: it may seem like combining the two "simdjson_unlikely checks" below into + // a single simdjson_unlikely path would be faster. The reasoning is sound, but the compiler may + // not oblige and may, in fact, generate two distinct paths in any case. It might be + // possible to do uint64_t(p - start_exp - 1) >= 18 but it could end up trading off + // instructions for a simdjson_likely branch, an unconclusive gain. + + // If there were no digits, it's an error. + if (simdjson_unlikely(p == start_exp)) { + return INVALID_NUMBER(src); + } + // We have a valid positive exponent in exp_number at this point, except that + // it may have overflowed. + + // If there were more than 18 digits, we may have overflowed the integer. We have to do + // something!!!! + if (simdjson_unlikely(p > start_exp + 18)) { + // Skip leading zeroes: 1e000000000000000000001 is technically valid and does not overflow + while (*start_exp == '0') { start_exp++; } + // 19 digits could overflow int64_t and is kind of absurd anyway. We don't + // support exponents smaller than -999,999,999,999,999,999 and bigger + // than 999,999,999,999,999,999. + // We can truncate. + // Note that 999999999999999999 is assuredly too large. The maximal ieee64 value before + // infinity is ~1.8e308. The smallest subnormal is ~5e-324. So, actually, we could + // truncate at 324. + // Note that there is no reason to fail per se at this point in time. + // E.g., 0e999999999999999999999 is a fine number. + if (p > start_exp + 18) { exp_number = 999999999999999999; } + } + // At this point, we know that exp_number is a sane, positive, signed integer. + // It is <= 999,999,999,999,999,999. As long as 'exponent' is in + // [-8223372036854775808, 8223372036854775808], we won't overflow. Because 'exponent' + // is bounded in magnitude by the size of the JSON input, we are fine in this universe. + // To sum it up: the next line should never overflow. + exponent += (neg_exp ? -exp_number : exp_number); + return SUCCESS; + } + + simdjson_inline bool check_if_integer(const uint8_t* const src, size_t max_length) { + const uint8_t* const srcend = src + max_length; + bool negative = (*src == '-'); // we can always read at least one character after the '-' + const uint8_t* p = src + uint8_t(negative); + if (p == srcend) { return false; } + if (*p == '0') { + ++p; + if (p == srcend) { return true; } + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return false; } + return true; + } + while (p != srcend && is_digit(*p)) { ++p; } + if (p == srcend) { return true; } + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return false; } + return true; + } + + simdjson_inline size_t significant_digits(const uint8_t* start_digits, size_t digit_count) { + // It is possible that the integer had an overflow. + // We have to handle the case where we have 0.0000somenumber. + const uint8_t* start = start_digits; + while ((*start == '0') || (*start == '.')) { ++start; } + // we over-decrement by one when there is a '.' + return digit_count - size_t(start - start_digits); + } + + } // unnamed namespace + + /** @private */ + static error_code slow_float_parsing(simdjson_unused const uint8_t* src, double* answer) { + if (parse_float_fallback(src, answer)) { + return SUCCESS; + } + return INVALID_NUMBER(src); + } + + /** @private */ + template + simdjson_inline error_code write_float(const uint8_t* const src, bool negative, uint64_t i, const uint8_t* start_digits, size_t digit_count, int64_t exponent, W& writer) { + // If we frequently had to deal with long strings of digits, + // we could extend our code by using a 128-bit integer instead + // of a 64-bit integer. However, this is uncommon in practice. + // + // 9999999999999999999 < 2**64 so we can accommodate 19 digits. + // If we have a decimal separator, then digit_count - 1 is the number of digits, but we + // may not have a decimal separator! + if (simdjson_unlikely(digit_count > 19 && significant_digits(start_digits, digit_count) > 19)) { + // Ok, chances are good that we had an overflow! + // this is almost never going to get called!!! + // we start anew, going slowly!!! + // This will happen in the following examples: + // 10000000000000000000000000000000000000000000e+308 + // 3.1415926535897932384626433832795028841971693993751 + // + // NOTE: We do not pass a reference to the to slow_float_parsing. If we passed our writer + // reference to it, it would force it to be stored in memory, preventing the compiler from + // picking it apart and putting into registers. i.e. if we pass it as reference, + // it gets slow. + double d; + error_code error = slow_float_parsing(src, &d); + writer.append_double(d); + return error; + } + // NOTE: it's weird that the simdjson_unlikely() only wraps half the if, but it seems to get slower any other + // way we've tried: https://github.com/simdjson/simdjson/pull/990#discussion_r448497331 + // To future reader: we'd love if someone found a better way, or at least could explain this result! + if (simdjson_unlikely(exponent < simdjson::internal::smallest_power) || (exponent > simdjson::internal::largest_power)) { + // + // Important: smallest_power is such that it leads to a zero value. + // Observe that 18446744073709551615e-343 == 0, i.e. (2**64 - 1) e -343 is zero + // so something x 10^-343 goes to zero, but not so with something x 10^-342. + static_assert(simdjson::internal::smallest_power <= -342, "smallest_power is not small enough"); + // + if ((exponent < simdjson::internal::smallest_power) || (i == 0)) { + // E.g. Parse "-0.0e-999" into the same value as "-0.0". See https://en.wikipedia.org/wiki/Signed_zero + WRITE_DOUBLE(negative ? -0.0 : 0.0, src, writer); + return SUCCESS; + } + else { // (exponent > largest_power) and (i != 0) + // We have, for sure, an infinite value and simdjson refuses to parse infinite values. + return INVALID_NUMBER(src); + } + } + double d; + if (!compute_float_64(exponent, i, negative, d)) { + // we are almost never going to get here. + if (!parse_float_fallback(src, &d)) { return INVALID_NUMBER(src); } + } + WRITE_DOUBLE(d, src, writer); + return SUCCESS; + } + + // parse the number at src + // define JSON_TEST_NUMBERS for unit testing + // + // It is assumed that the number is followed by a structural ({,},],[) character + // or a white space character. If that is not the case (e.g., when the JSON + // document is made of a single number), then it is necessary to copy the + // content and append a space before calling this function. + // + // Our objective is accurate parsing (ULP of 0) at high speed. + template + simdjson_inline error_code parse_number(const uint8_t* const src, W& writer); + + // for performance analysis, it is sometimes useful to skip parsing +#ifdef SIMDJSON_SKIPNUMBERPARSING + + template + simdjson_inline error_code parse_number(const uint8_t* const, W& writer) { + writer.append_s64(0); // always write zero + return SUCCESS; // always succeeds + } + + simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t* const src) noexcept { return 0; } + simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t* const src) noexcept { return 0; } + simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t* const src) noexcept { return 0; } + simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t* const src) noexcept { return 0; } + simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t* const src) noexcept { return 0; } + simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t* const src) noexcept { return 0; } + simdjson_unused simdjson_inline bool is_negative(const uint8_t* src) noexcept { return false; } + simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t* src) noexcept { return false; } + simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t* src) noexcept { return number_type::signed_integer; } +#else + +// parse the number at src +// define JSON_TEST_NUMBERS for unit testing +// +// It is assumed that the number is followed by a structural ({,},],[) character +// or a white space character. If that is not the case (e.g., when the JSON +// document is made of a single number), then it is necessary to copy the +// content and append a space before calling this function. +// +// Our objective is accurate parsing (ULP of 0) at high speed. + template + simdjson_inline error_code parse_number(const uint8_t* const src, W& writer) { + + // + // Check for minus sign + // + bool negative = (*src == '-'); + const uint8_t* p = src + uint8_t(negative); + + // + // Parse the integer part. + // + // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare + const uint8_t* const start_digits = p; + uint64_t i = 0; + while (parse_digit(*p, i)) { p++; } + + // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. + // Optimization note: size_t is expected to be unsigned. + size_t digit_count = size_t(p - start_digits); + if (digit_count == 0 || ('0' == *start_digits && digit_count > 1)) { return INVALID_NUMBER(src); } + + // + // Handle floats if there is a . or e (or both) + // + int64_t exponent = 0; + bool is_float = false; + if ('.' == *p) { + is_float = true; + ++p; + SIMDJSON_TRY(parse_decimal_after_separator(src, p, i, exponent)); + digit_count = int(p - start_digits); // used later to guard against overflows + } + if (('e' == *p) || ('E' == *p)) { + is_float = true; + ++p; + SIMDJSON_TRY(parse_exponent(src, p, exponent)); + } + if (is_float) { + const bool dirty_end = jsoncharutils::is_not_structural_or_whitespace(*p); + SIMDJSON_TRY(write_float(src, negative, i, start_digits, digit_count, exponent, writer)); + if (dirty_end) { return INVALID_NUMBER(src); } + return SUCCESS; + } + + // The longest negative 64-bit number is 19 digits. + // The longest positive 64-bit number is 20 digits. + // We do it this way so we don't trigger this branch unless we must. + size_t longest_digit_count = negative ? 19 : 20; + if (digit_count > longest_digit_count) { return BIGINT_NUMBER(src); } + if (digit_count == longest_digit_count) { + if (negative) { + // Anything negative above INT64_MAX+1 is invalid + if (i > uint64_t(INT64_MAX) + 1) { return BIGINT_NUMBER(src); } + WRITE_INTEGER(~i + 1, src, writer); + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return INVALID_NUMBER(src); } + return SUCCESS; + // Positive overflow check: + // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the + // biggest uint64_t. + // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX. + // If we got here, it's a 20 digit number starting with the digit "1". + // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller + // than 1,553,255,926,290,448,384. + // - That is smaller than the smallest possible 20-digit number the user could write: + // 10,000,000,000,000,000,000. + // - Therefore, if the number is positive and lower than that, it's overflow. + // - The value we are looking at is less than or equal to INT64_MAX. + // + } + else if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INVALID_NUMBER(src); } + } + + // Write unsigned if it does not fit in a signed integer. + if (i > uint64_t(INT64_MAX)) { + WRITE_UNSIGNED(i, src, writer); + } + else { + WRITE_INTEGER(negative ? (~i + 1) : i, src, writer); + } + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return INVALID_NUMBER(src); } + return SUCCESS; + } + + // Inlineable functions + namespace { + + // This table can be used to characterize the final character of an integer + // string. For JSON structural character and allowable white space characters, + // we return SUCCESS. For 'e', '.' and 'E', we return INCORRECT_TYPE. Otherwise + // we return NUMBER_ERROR. + // Optimization note: we could easily reduce the size of the table by half (to 128) + // at the cost of an extra branch. + // Optimization note: we want the values to use at most 8 bits (not, e.g., 32 bits): + static_assert(error_code(uint8_t(NUMBER_ERROR)) == NUMBER_ERROR, "bad NUMBER_ERROR cast"); + static_assert(error_code(uint8_t(SUCCESS)) == SUCCESS, "bad NUMBER_ERROR cast"); + static_assert(error_code(uint8_t(INCORRECT_TYPE)) == INCORRECT_TYPE, "bad NUMBER_ERROR cast"); + + const uint8_t integer_string_finisher[256] = { + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, + SUCCESS, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, + NUMBER_ERROR, INCORRECT_TYPE, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, INCORRECT_TYPE, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, SUCCESS, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, INCORRECT_TYPE, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, SUCCESS, NUMBER_ERROR, + SUCCESS, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, NUMBER_ERROR, + NUMBER_ERROR }; + + // Parse any number from 0 to 18,446,744,073,709,551,615 + simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t* const src) noexcept { + const uint8_t* p = src; + // + // Parse the integer part. + // + // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare + const uint8_t* const start_digits = p; + uint64_t i = 0; + while (parse_digit(*p, i)) { p++; } + + // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. + // Optimization note: size_t is expected to be unsigned. + size_t digit_count = size_t(p - start_digits); + // The longest positive 64-bit number is 20 digits. + // We do it this way so we don't trigger this branch unless we must. + // Optimization note: the compiler can probably merge + // ((digit_count == 0) || (digit_count > 20)) + // into a single branch since digit_count is unsigned. + if ((digit_count == 0) || (digit_count > 20)) { return INCORRECT_TYPE; } + // Here digit_count > 0. + if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } + // We can do the following... + // if (!jsoncharutils::is_structural_or_whitespace(*p)) { + // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; + // } + // as a single table lookup: + if (integer_string_finisher[*p] != SUCCESS) { return error_code(integer_string_finisher[*p]); } + + if (digit_count == 20) { + // Positive overflow check: + // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the + // biggest uint64_t. + // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX. + // If we got here, it's a 20 digit number starting with the digit "1". + // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller + // than 1,553,255,926,290,448,384. + // - That is smaller than the smallest possible 20-digit number the user could write: + // 10,000,000,000,000,000,000. + // - Therefore, if the number is positive and lower than that, it's overflow. + // - The value we are looking at is less than or equal to INT64_MAX. + // + if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INCORRECT_TYPE; } + } + + return i; + } + + + // Parse any number from 0 to 18,446,744,073,709,551,615 + // Never read at src_end or beyond + simdjson_unused simdjson_inline simdjson_result parse_unsigned(const uint8_t* const src, const uint8_t* const src_end) noexcept { + const uint8_t* p = src; + // + // Parse the integer part. + // + // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare + const uint8_t* const start_digits = p; + uint64_t i = 0; + while ((p != src_end) && parse_digit(*p, i)) { p++; } + + // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. + // Optimization note: size_t is expected to be unsigned. + size_t digit_count = size_t(p - start_digits); + // The longest positive 64-bit number is 20 digits. + // We do it this way so we don't trigger this branch unless we must. + // Optimization note: the compiler can probably merge + // ((digit_count == 0) || (digit_count > 20)) + // into a single branch since digit_count is unsigned. + if ((digit_count == 0) || (digit_count > 20)) { return INCORRECT_TYPE; } + // Here digit_count > 0. + if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } + // We can do the following... + // if (!jsoncharutils::is_structural_or_whitespace(*p)) { + // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; + // } + // as a single table lookup: + if ((p != src_end) && integer_string_finisher[*p] != SUCCESS) { return error_code(integer_string_finisher[*p]); } + + if (digit_count == 20) { + // Positive overflow check: + // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the + // biggest uint64_t. + // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX. + // If we got here, it's a 20 digit number starting with the digit "1". + // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller + // than 1,553,255,926,290,448,384. + // - That is smaller than the smallest possible 20-digit number the user could write: + // 10,000,000,000,000,000,000. + // - Therefore, if the number is positive and lower than that, it's overflow. + // - The value we are looking at is less than or equal to INT64_MAX. + // + if (src[0] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INCORRECT_TYPE; } + } + + return i; + } + + // Parse any number from 0 to 18,446,744,073,709,551,615 + simdjson_unused simdjson_inline simdjson_result parse_unsigned_in_string(const uint8_t* const src) noexcept { + const uint8_t* p = src + 1; + // + // Parse the integer part. + // + // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare + const uint8_t* const start_digits = p; + uint64_t i = 0; + while (parse_digit(*p, i)) { p++; } + + // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. + // Optimization note: size_t is expected to be unsigned. + size_t digit_count = size_t(p - start_digits); + // The longest positive 64-bit number is 20 digits. + // We do it this way so we don't trigger this branch unless we must. + // Optimization note: the compiler can probably merge + // ((digit_count == 0) || (digit_count > 20)) + // into a single branch since digit_count is unsigned. + if ((digit_count == 0) || (digit_count > 20)) { return INCORRECT_TYPE; } + // Here digit_count > 0. + if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } + // We can do the following... + // if (!jsoncharutils::is_structural_or_whitespace(*p)) { + // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; + // } + // as a single table lookup: + if (*p != '"') { return NUMBER_ERROR; } + + if (digit_count == 20) { + // Positive overflow check: + // - A 20 digit number starting with 2-9 is overflow, because 18,446,744,073,709,551,615 is the + // biggest uint64_t. + // - A 20 digit number starting with 1 is overflow if it is less than INT64_MAX. + // If we got here, it's a 20 digit number starting with the digit "1". + // - If a 20 digit number starting with 1 overflowed (i*10+digit), the result will be smaller + // than 1,553,255,926,290,448,384. + // - That is smaller than the smallest possible 20-digit number the user could write: + // 10,000,000,000,000,000,000. + // - Therefore, if the number is positive and lower than that, it's overflow. + // - The value we are looking at is less than or equal to INT64_MAX. + // + // Note: we use src[1] and not src[0] because src[0] is the quote character in this + // instance. + if (src[1] != uint8_t('1') || i <= uint64_t(INT64_MAX)) { return INCORRECT_TYPE; } + } + + return i; + } + + // Parse any number from -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 + simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t* src) noexcept { + // + // Check for minus sign + // + bool negative = (*src == '-'); + const uint8_t* p = src + uint8_t(negative); + + // + // Parse the integer part. + // + // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare + const uint8_t* const start_digits = p; + uint64_t i = 0; + while (parse_digit(*p, i)) { p++; } + + // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. + // Optimization note: size_t is expected to be unsigned. + size_t digit_count = size_t(p - start_digits); + // We go from + // -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 + // so we can never represent numbers that have more than 19 digits. + size_t longest_digit_count = 19; + // Optimization note: the compiler can probably merge + // ((digit_count == 0) || (digit_count > longest_digit_count)) + // into a single branch since digit_count is unsigned. + if ((digit_count == 0) || (digit_count > longest_digit_count)) { return INCORRECT_TYPE; } + // Here digit_count > 0. + if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } + // We can do the following... + // if (!jsoncharutils::is_structural_or_whitespace(*p)) { + // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; + // } + // as a single table lookup: + if (integer_string_finisher[*p] != SUCCESS) { return error_code(integer_string_finisher[*p]); } + // Negative numbers have can go down to - INT64_MAX - 1 whereas positive numbers are limited to INT64_MAX. + // Performance note: This check is only needed when digit_count == longest_digit_count but it is + // so cheap that we might as well always make it. + if (i > uint64_t(INT64_MAX) + uint64_t(negative)) { return INCORRECT_TYPE; } + return negative ? (~i + 1) : i; + } + + // Parse any number from -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 + // Never read at src_end or beyond + simdjson_unused simdjson_inline simdjson_result parse_integer(const uint8_t* const src, const uint8_t* const src_end) noexcept { + // + // Check for minus sign + // + if (src == src_end) { return NUMBER_ERROR; } + bool negative = (*src == '-'); + const uint8_t* p = src + uint8_t(negative); + + // + // Parse the integer part. + // + // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare + const uint8_t* const start_digits = p; + uint64_t i = 0; + while ((p != src_end) && parse_digit(*p, i)) { p++; } + + // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. + // Optimization note: size_t is expected to be unsigned. + size_t digit_count = size_t(p - start_digits); + // We go from + // -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 + // so we can never represent numbers that have more than 19 digits. + size_t longest_digit_count = 19; + // Optimization note: the compiler can probably merge + // ((digit_count == 0) || (digit_count > longest_digit_count)) + // into a single branch since digit_count is unsigned. + if ((digit_count == 0) || (digit_count > longest_digit_count)) { return INCORRECT_TYPE; } + // Here digit_count > 0. + if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } + // We can do the following... + // if (!jsoncharutils::is_structural_or_whitespace(*p)) { + // return (*p == '.' || *p == 'e' || *p == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; + // } + // as a single table lookup: + if ((p != src_end) && integer_string_finisher[*p] != SUCCESS) { return error_code(integer_string_finisher[*p]); } + // Negative numbers have can go down to - INT64_MAX - 1 whereas positive numbers are limited to INT64_MAX. + // Performance note: This check is only needed when digit_count == longest_digit_count but it is + // so cheap that we might as well always make it. + if (i > uint64_t(INT64_MAX) + uint64_t(negative)) { return INCORRECT_TYPE; } + return negative ? (~i + 1) : i; + } + + // Parse any number from -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 + simdjson_unused simdjson_inline simdjson_result parse_integer_in_string(const uint8_t* src) noexcept { + // + // Check for minus sign + // + bool negative = (*(src + 1) == '-'); + src += uint8_t(negative) + 1; + + // + // Parse the integer part. + // + // PERF NOTE: we don't use is_made_of_eight_digits_fast because large integers like 123456789 are rare + const uint8_t* const start_digits = src; + uint64_t i = 0; + while (parse_digit(*src, i)) { src++; } + + // If there were no digits, or if the integer starts with 0 and has more than one digit, it's an error. + // Optimization note: size_t is expected to be unsigned. + size_t digit_count = size_t(src - start_digits); + // We go from + // -9,223,372,036,854,775,808 to 9,223,372,036,854,775,807 + // so we can never represent numbers that have more than 19 digits. + size_t longest_digit_count = 19; + // Optimization note: the compiler can probably merge + // ((digit_count == 0) || (digit_count > longest_digit_count)) + // into a single branch since digit_count is unsigned. + if ((digit_count == 0) || (digit_count > longest_digit_count)) { return INCORRECT_TYPE; } + // Here digit_count > 0. + if (('0' == *start_digits) && (digit_count > 1)) { return NUMBER_ERROR; } + // We can do the following... + // if (!jsoncharutils::is_structural_or_whitespace(*src)) { + // return (*src == '.' || *src == 'e' || *src == 'E') ? INCORRECT_TYPE : NUMBER_ERROR; + // } + // as a single table lookup: + if (*src != '"') { return NUMBER_ERROR; } + // Negative numbers have can go down to - INT64_MAX - 1 whereas positive numbers are limited to INT64_MAX. + // Performance note: This check is only needed when digit_count == longest_digit_count but it is + // so cheap that we might as well always make it. + if (i > uint64_t(INT64_MAX) + uint64_t(negative)) { return INCORRECT_TYPE; } + return negative ? (~i + 1) : i; + } + + simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t* src) noexcept { + // + // Check for minus sign + // + bool negative = (*src == '-'); + src += uint8_t(negative); + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t* p = src; + p += parse_digit(*p, i); + bool leading_zero = (i == 0); + while (parse_digit(*p, i)) { p++; } + // no integer digits, or 0123 (zero must be solo) + if (p == src) { return INCORRECT_TYPE; } + if ((leading_zero && p != src + 1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely(*p == '.')) { + p++; + const uint8_t* start_decimal_digits = p; + if (!parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits + p++; + while (parse_digit(*p, i)) { p++; } + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p - src - 1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t* start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = p - start_digits > 19; + } + } + else { + overflow = p - src > 19; + } + + // + // Parse the exponent + // + if (*p == 'e' || *p == 'E') { + p++; + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t* start_exp_digits = p; + while (parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p - start_exp_digits == 0 || p - start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0 - exp : exp; + } + + if (jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + double d; + if (simdjson_likely(!overflow)) { + if (compute_float_64(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), &d)) { + return NUMBER_ERROR; + } + return d; + } + + simdjson_unused simdjson_inline bool is_negative(const uint8_t* src) noexcept { + return (*src == '-'); + } + + simdjson_unused simdjson_inline simdjson_result is_integer(const uint8_t* src) noexcept { + bool negative = (*src == '-'); + src += uint8_t(negative); + const uint8_t* p = src; + while (static_cast(*p - '0') <= 9) { p++; } + if (p == src) { return NUMBER_ERROR; } + if (jsoncharutils::is_structural_or_whitespace(*p)) { return true; } + return false; + } + + simdjson_unused simdjson_inline simdjson_result get_number_type(const uint8_t* src) noexcept { + bool negative = (*src == '-'); + src += uint8_t(negative); + const uint8_t* p = src; + while (static_cast(*p - '0') <= 9) { p++; } + size_t digit_count = size_t(p - src); + if (p == src) { return NUMBER_ERROR; } + if (jsoncharutils::is_structural_or_whitespace(*p)) { + static const uint8_t* smaller_big_integer = reinterpret_cast("9223372036854775808"); + // We have an integer. + if (simdjson_unlikely(digit_count > 20)) { + return number_type::big_integer; + } + // If the number is negative and valid, it must be a signed integer. + if (negative) { + if (simdjson_unlikely(digit_count > 19)) return number_type::big_integer; + if (simdjson_unlikely(digit_count == 19 && memcmp(src, smaller_big_integer, 19) > 0)) { + return number_type::big_integer; + } + return number_type::signed_integer; + } + // Let us check if we have a big integer (>=2**64). + static const uint8_t* two_to_sixtyfour = reinterpret_cast("18446744073709551616"); + if ((digit_count > 20) || (digit_count == 20 && memcmp(src, two_to_sixtyfour, 20) >= 0)) { + return number_type::big_integer; + } + // The number is positive and smaller than 18446744073709551616 (or 2**64). + // We want values larger or equal to 9223372036854775808 to be unsigned + // integers, and the other values to be signed integers. + if ((digit_count == 20) || (digit_count >= 19 && memcmp(src, smaller_big_integer, 19) >= 0)) { + return number_type::unsigned_integer; + } + return number_type::signed_integer; + } + // Hopefully, we have 'e' or 'E' or '.'. + return number_type::floating_point_number; + } + + // Never read at src_end or beyond + simdjson_unused simdjson_inline simdjson_result parse_double(const uint8_t* src, const uint8_t* const src_end) noexcept { + if (src == src_end) { return NUMBER_ERROR; } + // + // Check for minus sign + // + bool negative = (*src == '-'); + src += uint8_t(negative); + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t* p = src; + if (p == src_end) { return NUMBER_ERROR; } + p += parse_digit(*p, i); + bool leading_zero = (i == 0); + while ((p != src_end) && parse_digit(*p, i)) { p++; } + // no integer digits, or 0123 (zero must be solo) + if (p == src) { return INCORRECT_TYPE; } + if ((leading_zero && p != src + 1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely((p != src_end) && (*p == '.'))) { + p++; + const uint8_t* start_decimal_digits = p; + if ((p == src_end) || !parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits + p++; + while ((p != src_end) && parse_digit(*p, i)) { p++; } + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p - src - 1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t* start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = start_digits - src > 19; + } + } + else { + overflow = p - src > 19; + } + + // + // Parse the exponent + // + if ((p != src_end) && (*p == 'e' || *p == 'E')) { + p++; + if (p == src_end) { return NUMBER_ERROR; } + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t* start_exp_digits = p; + while ((p != src_end) && parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p - start_exp_digits == 0 || p - start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0 - exp : exp; + } + + if ((p != src_end) && jsoncharutils::is_not_structural_or_whitespace(*p)) { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + double d; + if (simdjson_likely(!overflow)) { + if (compute_float_64(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), src_end, &d)) { + return NUMBER_ERROR; + } + return d; + } + + simdjson_unused simdjson_inline simdjson_result parse_double_in_string(const uint8_t* src) noexcept { + // + // Check for minus sign + // + bool negative = (*(src + 1) == '-'); + src += uint8_t(negative) + 1; + + // + // Parse the integer part. + // + uint64_t i = 0; + const uint8_t* p = src; + p += parse_digit(*p, i); + bool leading_zero = (i == 0); + while (parse_digit(*p, i)) { p++; } + // no integer digits, or 0123 (zero must be solo) + if (p == src) { return INCORRECT_TYPE; } + if ((leading_zero && p != src + 1)) { return NUMBER_ERROR; } + + // + // Parse the decimal part. + // + int64_t exponent = 0; + bool overflow; + if (simdjson_likely(*p == '.')) { + p++; + const uint8_t* start_decimal_digits = p; + if (!parse_digit(*p, i)) { return NUMBER_ERROR; } // no decimal digits + p++; + while (parse_digit(*p, i)) { p++; } + exponent = -(p - start_decimal_digits); + + // Overflow check. More than 19 digits (minus the decimal) may be overflow. + overflow = p - src - 1 > 19; + if (simdjson_unlikely(overflow && leading_zero)) { + // Skip leading 0.00000 and see if it still overflows + const uint8_t* start_digits = src + 2; + while (*start_digits == '0') { start_digits++; } + overflow = p - start_digits > 19; + } + } + else { + overflow = p - src > 19; + } + + // + // Parse the exponent + // + if (*p == 'e' || *p == 'E') { + p++; + bool exp_neg = *p == '-'; + p += exp_neg || *p == '+'; + + uint64_t exp = 0; + const uint8_t* start_exp_digits = p; + while (parse_digit(*p, exp)) { p++; } + // no exp digits, or 20+ exp digits + if (p - start_exp_digits == 0 || p - start_exp_digits > 19) { return NUMBER_ERROR; } + + exponent += exp_neg ? 0 - exp : exp; + } + + if (*p != '"') { return NUMBER_ERROR; } + + overflow = overflow || exponent < simdjson::internal::smallest_power || exponent > simdjson::internal::largest_power; + + // + // Assemble (or slow-parse) the float + // + double d; + if (simdjson_likely(!overflow)) { + if (compute_float_64(exponent, i, negative, d)) { return d; } + } + if (!parse_float_fallback(src - uint8_t(negative), &d)) { + return NUMBER_ERROR; + } + return d; + } + + } // unnamed namespace +#endif // SIMDJSON_SKIPNUMBERPARSING + + } // namespace numberparsing + + inline std::ostream& operator<<(std::ostream& out, number_type type) noexcept { + switch (type) { + case number_type::signed_integer: out << "integer in [-9223372036854775808,9223372036854775808)"; break; + case number_type::unsigned_integer: out << "unsigned integer in [9223372036854775808,18446744073709551616)"; break; + case number_type::floating_point_number: out << "floating-point number (binary64)"; break; + case number_type::big_integer: out << "big integer"; break; + default: SIMDJSON_UNREACHABLE(); + } + return out; + } + + } // namespace lasx +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_NUMBERPARSING_H +/* end file simdjson/generic/numberparsing.h for lasx */ + +/* including simdjson/generic/implementation_simdjson_result_base-inl.h for lasx: #include "simdjson/generic/implementation_simdjson_result_base-inl.h" */ +/* begin file simdjson/generic/implementation_simdjson_result_base-inl.h for lasx */ +#ifndef SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_INL_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_INL_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/implementation_simdjson_result_base.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lasx { + + // + // internal::implementation_simdjson_result_base inline implementation + // + + template + simdjson_inline void implementation_simdjson_result_base::tie(T& value, error_code& error) && noexcept { + error = this->second; + if (!error) { + value = std::forward>(*this).first; + } + } + + template + simdjson_warn_unused simdjson_inline error_code implementation_simdjson_result_base::get(T& value) && noexcept { + error_code error; + std::forward>(*this).tie(value, error); + return error; + } + + template + simdjson_inline error_code implementation_simdjson_result_base::error() const noexcept { + return this->second; + } + +#if SIMDJSON_EXCEPTIONS + + template + simdjson_inline T& implementation_simdjson_result_base::value() & noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return this->first; + } + + template + simdjson_inline T&& implementation_simdjson_result_base::value() && noexcept(false) { + return std::forward>(*this).take_value(); + } + + template + simdjson_inline T&& implementation_simdjson_result_base::take_value() && noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return std::forward(this->first); + } + + template + simdjson_inline implementation_simdjson_result_base::operator T && () && noexcept(false) { + return std::forward>(*this).take_value(); + } + +#endif // SIMDJSON_EXCEPTIONS + + template + simdjson_inline const T& implementation_simdjson_result_base::value_unsafe() const& noexcept { + return this->first; + } + + template + simdjson_inline T& implementation_simdjson_result_base::value_unsafe() & noexcept { + return this->first; + } + + template + simdjson_inline T&& implementation_simdjson_result_base::value_unsafe() && noexcept { + return std::forward(this->first); + } + + template + simdjson_inline implementation_simdjson_result_base::implementation_simdjson_result_base(T&& value, error_code error) noexcept + : first{ std::forward(value) }, second{ error } {} + template + simdjson_inline implementation_simdjson_result_base::implementation_simdjson_result_base(error_code error) noexcept + : implementation_simdjson_result_base(T{}, error) {} + template + simdjson_inline implementation_simdjson_result_base::implementation_simdjson_result_base(T&& value) noexcept + : implementation_simdjson_result_base(std::forward(value), SUCCESS) {} + + } // namespace lasx +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_IMPLEMENTATION_SIMDJSON_RESULT_BASE_INL_H +/* end file simdjson/generic/implementation_simdjson_result_base-inl.h for lasx */ +/* end file simdjson/generic/amalgamated.h for lasx */ +/* including simdjson/lasx/end.h: #include "simdjson/lasx/end.h" */ +/* begin file simdjson/lasx/end.h */ +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/lasx/base.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +#undef SIMDJSON_SKIP_BACKSLASH_SHORT_CIRCUIT +/* undefining SIMDJSON_IMPLEMENTATION from "lasx" */ +#undef SIMDJSON_IMPLEMENTATION +/* end file simdjson/lasx/end.h */ + +#endif // SIMDJSON_LASX_H +/* end file simdjson/lasx.h */ #else #error Unknown SIMDJSON_BUILTIN_IMPLEMENTATION #endif @@ -26252,10 +30983,10 @@ namespace simdjson { #include -#if _M_ARM64 +#if SIMDJSON_REGULAR_VISUAL_STUDIO && SIMDJSON_IS_ARM64 // __umulh requires intrin.h #include -#endif // _M_ARM64 +#endif // SIMDJSON_REGULAR_VISUAL_STUDIO && SIMDJSON_IS_ARM64 namespace simdjson { namespace arm64 { @@ -26275,13 +31006,13 @@ namespace simdjson { simdjson_inline internal::value128 full_multiplication(uint64_t value1, uint64_t value2) { internal::value128 answer; #if SIMDJSON_REGULAR_VISUAL_STUDIO || SIMDJSON_IS_32BITS -#ifdef _M_ARM64 +#if SIMDJSON_IS_ARM64 // ARM64 has native support for 64-bit multiplications, no need to emultate answer.high = __umulh(value1, value2); answer.low = value1 * value2; #else answer.low = _umul128(value1, value2, &answer.high); // _umul128 not available on ARM64 -#endif // _M_ARM64 +#endif // SIMDJSON_IS_ARM64 #else // SIMDJSON_REGULAR_VISUAL_STUDIO || SIMDJSON_IS_32BITS __uint128_t r = (static_cast<__uint128_t>(value1)) * value2; answer.low = uint64_t(r); @@ -26436,7 +31167,7 @@ namespace simdjson { tmp = vpaddq_u8(tmp, tmp); return vgetq_lane_u16(vreinterpretq_u16_u8(tmp), 0); } - simdjson_inline bool any() const { return vmaxvq_u8(*this) != 0; } + simdjson_inline bool any() const { return vmaxvq_u32(vreinterpretq_u32_u8(*this)) != 0; } }; // Unsigned bytes @@ -27293,6 +32024,7 @@ namespace simdjson { simdjson_inline simdjson_result parse_bool(const uint8_t* json) const noexcept; simdjson_inline const uint8_t* peek_start() const noexcept; simdjson_inline uint32_t peek_start_length() const noexcept; + simdjson_inline uint32_t peek_root_length() const noexcept; /** * The general idea of the advance_... methods and the peek_* methods @@ -27387,6 +32119,7 @@ namespace simdjson { friend class object; friend class array; friend class value; + friend class field; }; // value_iterator } // namespace ondemand @@ -27447,12 +32180,13 @@ namespace simdjson { * @returns INCORRECT_TYPE If the JSON value is not the given type. */ template simdjson_inline simdjson_result get() noexcept { - // Unless the simdjson library provides an inline implementation, calling this method should + // Unless the simdjson library or the user provides an inline implementation, calling this method should // immediately fail. static_assert(!sizeof(T), "The get method with given type is not implemented by the simdjson library. " "The supported types are ondemand::object, ondemand::array, raw_json_string, std::string_view, uint64_t, " "int64_t, double, and bool. We recommend you use get_double(), get_bool(), get_uint64(), get_int64(), " - " get_object(), get_array(), get_raw_json_string(), or get_string() instead of the get template."); + " get_object(), get_array(), get_raw_json_string(), or get_string() instead of the get template." + " You may also add support for custom types, see our documentation."); } /** @@ -27605,6 +32339,17 @@ namespace simdjson { simdjson_inline simdjson_result is_null() noexcept; #if SIMDJSON_EXCEPTIONS + /** + * Cast this JSON value to an instance of type T. The programmer is responsible for + * providing an implementation of get for the type T, if T is not one of the types + * supported by the library (object, array, raw_json_string, string_view, uint64_t, etc.). + * + * See https://github.com/simdjson/simdjson/blob/master/doc/basics.md#adding-support-for-custom-types + * + * @returns An instance of type T + */ + template + explicit simdjson_inline operator T() noexcept(false); /** * Cast this JSON value to an array. * @@ -27768,7 +32513,7 @@ namespace simdjson { * that only one field is returned. * * Use find_field() if you are sure fields will be in order (or are willing to treat it as if the - * field wasn't there when they aren't). + * field as not there when they are not in order). * * @param key The key to look up. * @returns The value of the field, or NO_SUCH_FIELD if the field is not in the object. @@ -27804,6 +32549,13 @@ namespace simdjson { * @error TAPE_ERROR when the JSON value is a bad token like "}" "," or "alse". */ simdjson_inline simdjson_result is_scalar() noexcept; + /** + * Checks whether the value is a string. + * + * @returns true if the type is string + * @error TAPE_ERROR when the JSON value is a bad token like "}" "," or "alse". + */ + simdjson_inline simdjson_result is_string() noexcept; /** * Checks whether the value is a negative number. @@ -27836,10 +32588,12 @@ namespace simdjson { * get_number().get_number_type(). * * get_number_type() is number_type::unsigned_integer if we have - * an integer greater or equal to 9223372036854775808 + * an integer greater or equal to 9223372036854775808. * get_number_type() is number_type::signed_integer if we have an - * integer that is less than 9223372036854775808 - * Otherwise, get_number_type() has value number_type::floating_point_number + * integer that is less than 9223372036854775808. + * get_number_type() is number_type::big_integer for integers that do not fit in 64 bits, + * in which case the digit_count is set to the length of the big integer string. + * Otherwise, get_number_type() has value number_type::floating_point_number. * * This function requires processing the number string, but it is expected * to be faster than get_number().get_number_type() because it is does not @@ -27866,6 +32620,8 @@ namespace simdjson { * You can recover the value by calling number.get_uint64() and you * have that number.is_uint64() is true. * + * For integers that do not fit in 64 bits, the function returns BIGINT_ERROR error code. + * * Otherwise, number.get_number_type() has value number_type::floating_point_number * and we have a binary64 number. * You can recover the value by calling number.get_double() and you @@ -27881,7 +32637,6 @@ namespace simdjson { */ simdjson_warn_unused simdjson_inline simdjson_result get_number() noexcept; - /** * Get the raw JSON for this token. * @@ -27979,6 +32734,20 @@ namespace simdjson { */ simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; + /** + * Get the value associated with the given JSONPath expression. We only support + * JSONPath queries that trivially convertible to JSON Pointer queries: key + * names and array indices. + * + * @return The value associated with the given JSONPath expression, or: + * - INVALID_JSON_POINTER if the JSONPath to JSON Pointer conversion fails + * - NO_SUCH_FIELD if a field does not exist in an object + * - INDEX_OUT_OF_BOUNDS if an array index is larger than an array length + * - INCORRECT_TYPE if a non-integer is used to access an array + */ + simdjson_inline simdjson_result at_path(std::string_view at_path) noexcept; + + protected: /** * Create a value. @@ -28018,6 +32787,7 @@ namespace simdjson { friend class object; friend struct simdjson_result; friend struct simdjson_result; + friend class field; }; } // namespace ondemand @@ -28055,6 +32825,8 @@ namespace simdjson { template simdjson_inline error_code get(T& out) noexcept; #if SIMDJSON_EXCEPTIONS + template + explicit simdjson_inline operator T() noexcept(false); simdjson_inline operator arm64::ondemand::array() noexcept(false); simdjson_inline operator arm64::ondemand::object() noexcept(false); simdjson_inline operator uint64_t() noexcept(false); @@ -28108,7 +32880,7 @@ namespace simdjson { * APIs assume this. Therefore, you must be explicit if you want to treat objects as out of order. * * Use find_field() if you are sure fields will be in order (or are willing to treat it as if the - * field wasn't there when they aren't). + * field as not there when they are not in order). * * @param key The key to look up. * @returns The value of the field, or NO_SUCH_FIELD if the field is not in the object. @@ -28130,6 +32902,7 @@ namespace simdjson { */ simdjson_inline simdjson_result type() noexcept; simdjson_inline simdjson_result is_scalar() noexcept; + simdjson_inline simdjson_result is_string() noexcept; simdjson_inline simdjson_result is_negative() noexcept; simdjson_inline simdjson_result is_integer() noexcept; simdjson_inline simdjson_result get_number_type() noexcept; @@ -28144,6 +32917,7 @@ namespace simdjson { /** @copydoc simdjson_inline int32_t current_depth() const noexcept */ simdjson_inline simdjson_result current_depth() const noexcept; simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; + simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; }; } // namespace simdjson @@ -28262,7 +33036,7 @@ namespace simdjson { * 1 = next token, -1 = prev token. * * TODO consider a string_view, assuming the length will get stripped out by the optimizer when - * it isn't used ... + * it is not used... */ simdjson_inline const uint8_t* peek(int32_t delta = 0) const noexcept; /** @@ -28292,7 +33066,14 @@ namespace simdjson { * @param position The position of the token. */ simdjson_inline uint32_t peek_length(token_position position) const noexcept; - + /** + * Get the maximum length of the JSON text for a root token. + * + * The length will include any whitespace at the end of the token. + * + * @param position The position of the token (start of the document). + */ + simdjson_inline uint32_t peek_root_length(token_position position) const noexcept; /** * Return the current index. */ @@ -28515,7 +33296,7 @@ namespace simdjson { * @param delta The relative position of the token to retrieve. e.g. 0 = next token, -1 = prev token. * * TODO consider a string_view, assuming the length will get stripped out by the optimizer when - * it isn't used ... + * it is not used ... */ simdjson_inline const uint8_t* peek(int32_t delta = 0) const noexcept; /** @@ -28543,7 +33324,7 @@ namespace simdjson { * @param position The position of the token to retrieve. * * TODO consider a string_view, assuming the length will get stripped out by the optimizer when - * it isn't used ... + * it is not used ... */ simdjson_inline const uint8_t* peek(token_position position) const noexcept; /** @@ -28554,13 +33335,21 @@ namespace simdjson { * @param position The position of the token to retrieve. */ simdjson_inline uint32_t peek_length(token_position position) const noexcept; + /** + * Get the maximum length of the JSON text for the current root token. + * + * The length will include any whitespace at the end of the token. + * + * @param position The position of the token to retrieve. + */ + simdjson_inline uint32_t peek_root_length(token_position position) const noexcept; /** * Get the JSON text for the last token in the document. * * This is not null-terminated; it is a view into the JSON. * * TODO consider a string_view, assuming the length will get stripped out by the optimizer when - * it isn't used ... + * it is not used ... */ simdjson_inline const uint8_t* peek_last() const noexcept; @@ -28654,6 +33443,9 @@ namespace simdjson { inline bool balanced() const noexcept; protected: simdjson_inline json_iterator(const uint8_t* buf, ondemand::parser* parser) noexcept; +#ifdef SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON + simdjson_inline json_iterator(const uint8_t* buf, ondemand::parser* parser, bool streaming) noexcept; +#endif // SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON /// The last token before the end simdjson_inline token_position last_position() const noexcept; /// The token *at* the end. This points at gibberish and should only be used for comparison. @@ -28669,6 +33461,7 @@ namespace simdjson { friend class raw_json_string; friend class parser; friend class value_iterator; + friend class field; template friend simdjson_inline void logger::log_line(const json_iterator& iter, const char* title_prefix, const char* title, std::string_view detail, int delta, int depth_delta, logger::log_level level, Args&&... args) noexcept; template @@ -28790,8 +33583,6 @@ namespace simdjson { */ friend class value_iterator; template - friend error_code numberparsing::slow_float_parsing(simdjson_unused const uint8_t* src, W writer); - template friend error_code numberparsing::write_float(const uint8_t* const src, bool negative, uint64_t i, const uint8_t* start_digits, size_t digit_count, int64_t exponent, W& writer); template friend error_code numberparsing::parse_number(const uint8_t* const src, W& writer); @@ -29085,8 +33876,8 @@ namespace simdjson { namespace ondemand { /** - * The default batch size for document_stream instances for this On Demand kernel. - * Note that different On Demand kernel may use a different DEFAULT_BATCH_SIZE value + * The default batch size for document_stream instances for this On-Demand kernel. + * Note that different On-Demand kernel may use a different DEFAULT_BATCH_SIZE value * in the future. */ static constexpr size_t DEFAULT_BATCH_SIZE = 1000000; @@ -29170,6 +33961,9 @@ namespace simdjson { * - UNCLOSED_STRING if there is an unclosed string in the document. */ simdjson_warn_unused simdjson_result iterate(padded_string_view json) & noexcept; +#ifdef SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON + simdjson_warn_unused simdjson_result iterate_allow_incomplete_json(padded_string_view json) & noexcept; +#endif // SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON /** @overload simdjson_result iterate(padded_string_view json) & noexcept */ simdjson_warn_unused simdjson_result iterate(const char* json, size_t len, size_t capacity) & noexcept; /** @overload simdjson_result iterate(padded_string_view json) & noexcept */ @@ -29314,9 +34108,9 @@ namespace simdjson { simdjson_result iterate_many(const char* buf, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept = delete; /** The capacity of this parser (the largest document it can process). */ - simdjson_inline size_t capacity() const noexcept; + simdjson_pure simdjson_inline size_t capacity() const noexcept; /** The maximum capacity of this parser (the largest document it is allowed to process). */ - simdjson_inline size_t max_capacity() const noexcept; + simdjson_pure simdjson_inline size_t max_capacity() const noexcept; simdjson_inline void set_max_capacity(size_t max_capacity) noexcept; /** * The maximum depth of this parser (the most deeply nested objects and arrays it can process). @@ -29324,7 +34118,7 @@ namespace simdjson { * The document's instance current_depth() method should be used to monitor the parsing * depth and limit it if desired. */ - simdjson_inline size_t max_depth() const noexcept; + simdjson_pure simdjson_inline size_t max_depth() const noexcept; /** * Ensure this parser has enough memory to process JSON documents up to `capacity` bytes in length @@ -29396,9 +34190,20 @@ namespace simdjson { */ simdjson_inline simdjson_result unescape_wobbly(raw_json_string in, uint8_t*& dst) const noexcept; +#if SIMDJSON_DEVELOPMENT_CHECKS + /** + * Returns true if string_buf_loc is outside of the allocated range for the + * the string buffer. When true, it indicates that the string buffer has overflowed. + * This is a development-time check that is not needed in production. It can be + * used to detect buffer overflows in the string buffer and usafe usage of the + * string buffer. + */ + bool string_buffer_overflow(const uint8_t* string_buf_loc) const noexcept; +#endif + private: /** @private [for benchmarking access] The implementation to use */ - std::unique_ptr implementation{}; + std::unique_ptr implementation{}; size_t _capacity{ 0 }; size_t _max_capacity; size_t _max_depth{ DEFAULT_MAX_DEPTH }; @@ -29535,6 +34340,22 @@ namespace simdjson { * - INVALID_JSON_POINTER if the JSON pointer is invalid and cannot be parsed */ inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; + + /** + * Get the value associated with the given JSONPath expression. We only support + * JSONPath queries that trivially convertible to JSON Pointer queries: key + * names and array indices. + * + * https://datatracker.ietf.org/doc/html/draft-normington-jsonpath-00 + * + * @return The value associated with the given JSONPath expression, or: + * - INVALID_JSON_POINTER if the JSONPath to JSON Pointer conversion fails + * - NO_SUCH_FIELD if a field does not exist in an object + * - INDEX_OUT_OF_BOUNDS if an array index is larger than an array length + * - INCORRECT_TYPE if a non-integer is used to access an array + */ + inline simdjson_result at_path(std::string_view json_path) noexcept; + /** * Consumes the array and returns a string_view instance corresponding to the * array as represented in JSON. It points inside the original document. @@ -29625,6 +34446,7 @@ namespace simdjson { inline simdjson_result reset() & noexcept; simdjson_inline simdjson_result at(size_t index) noexcept; simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; + simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; simdjson_inline simdjson_result raw_json() noexcept; }; @@ -29915,21 +34737,23 @@ namespace simdjson { * @returns INCORRECT_TYPE If the JSON value is not the given type. */ template simdjson_inline simdjson_result get() & noexcept { - // Unless the simdjson library provides an inline implementation, calling this method should + // Unless the simdjson library or the user provides an inline implementation, calling this method should // immediately fail. static_assert(!sizeof(T), "The get method with given type is not implemented by the simdjson library. " "The supported types are ondemand::object, ondemand::array, raw_json_string, std::string_view, uint64_t, " "int64_t, double, and bool. We recommend you use get_double(), get_bool(), get_uint64(), get_int64(), " - " get_object(), get_array(), get_raw_json_string(), or get_string() instead of the get template."); + " get_object(), get_array(), get_raw_json_string(), or get_string() instead of the get template." + " You may also add support for custom types, see our documentation."); } /** @overload template simdjson_result get() & noexcept */ - template simdjson_inline simdjson_result get() && noexcept { - // Unless the simdjson library provides an inline implementation, calling this method should + template simdjson_deprecated simdjson_inline simdjson_result get() && noexcept { + // Unless the simdjson library or the user provides an inline implementation, calling this method should // immediately fail. static_assert(!sizeof(T), "The get method with given type is not implemented by the simdjson library. " "The supported types are ondemand::object, ondemand::array, raw_json_string, std::string_view, uint64_t, " "int64_t, double, and bool. We recommend you use get_double(), get_bool(), get_uint64(), get_int64(), " - " get_object(), get_array(), get_raw_json_string(), or get_string() instead of the get template."); + " get_object(), get_array(), get_raw_json_string(), or get_string() instead of the get template." + " You may also add support for custom types, see our documentation."); } /** @@ -29945,9 +34769,23 @@ namespace simdjson { */ template simdjson_inline error_code get(T& out) & noexcept; /** @overload template error_code get(T &out) & noexcept */ - template simdjson_inline error_code get(T& out) && noexcept; + template simdjson_deprecated simdjson_inline error_code get(T& out) && noexcept; #if SIMDJSON_EXCEPTIONS + /** + * Cast this JSON value to an instance of type T. The programmer is responsible for + * providing an implementation of get for the type T, if T is not one of the types + * supported by the library (object, array, raw_json_string, string_view, uint64_t, etc.) + * + * See https://github.com/simdjson/simdjson/blob/master/doc/basics.md#adding-support-for-custom-types + * + * @returns An instance of type T + */ + template + explicit simdjson_inline operator T() & noexcept(false); + template + explicit simdjson_deprecated simdjson_inline operator T() && noexcept(false); + /** * Cast this JSON value to an array. * @@ -30120,7 +34958,7 @@ namespace simdjson { * APIs assume this. Therefore, you must be explicit if you want to treat objects as out of order. * * Use find_field() if you are sure fields will be in order (or are willing to treat it as if the - * field wasn't there when they aren't). + * field was not there when they are not in order). * * You must consume the fields on an object one at a time. A request for a new key * invalidates previous field values: it makes them unsafe. E.g., the array @@ -30166,6 +35004,14 @@ namespace simdjson { */ simdjson_inline simdjson_result is_scalar() noexcept; + /** + * Checks whether the document is a string. + * + * @returns true if the type is string + * @error TAPE_ERROR when the JSON value is a bad token like "}" "," or "alse". + */ + simdjson_inline simdjson_result is_string() noexcept; + /** * Checks whether the document is a negative number. * @@ -30193,9 +35039,11 @@ namespace simdjson { * get_number().get_number_type(). * * get_number_type() is number_type::unsigned_integer if we have - * an integer greater or equal to 9223372036854775808 + * an integer greater or equal to 9223372036854775808 and no larger than 18446744073709551615. * get_number_type() is number_type::signed_integer if we have an - * integer that is less than 9223372036854775808 + * integer that is less than 9223372036854775808 and greater or equal to -9223372036854775808. + * get_number_type() is number_type::big_integer if we have an integer outside + * of those ranges (either larger than 18446744073709551615 or smaller than -9223372036854775808). * Otherwise, get_number_type() has value number_type::floating_point_number * * This function requires processing the number string, but it is expected @@ -30313,6 +35161,14 @@ namespace simdjson { * auto doc = parser.iterate(json); * doc.at_pointer("//a/1") == 20 * + * Key values are matched exactly, without unescaping or Unicode normalization. + * We do a byte-by-byte comparison. E.g. + * + * const padded_string json = "{\"\\u00E9\":123}"_padded; + * auto doc = parser.iterate(json); + * doc.at_pointer("/\\u00E9") == 123 + * doc.at_pointer((const char*)u8"/\u00E9") returns an error (NO_SUCH_FIELD) + * * Note that at_pointer() automatically calls rewind between each call. Thus * all values, objects and arrays that you have created so far (including unescaped strings) * are invalidated. After calling at_pointer, you need to consume the result: string values @@ -30329,6 +35185,30 @@ namespace simdjson { * - SCALAR_DOCUMENT_AS_VALUE if the json_pointer is empty and the document is not a scalar (see is_scalar() function). */ simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; + + /** + * Get the value associated with the given JSONPath expression. We only support + * JSONPath queries that trivially convertible to JSON Pointer queries: key + * names and array indices. + * + * https://datatracker.ietf.org/doc/html/draft-normington-jsonpath-00 + * + * Key values are matched exactly, without unescaping or Unicode normalization. + * We do a byte-by-byte comparison. E.g. + * + * const padded_string json = "{\"\\u00E9\":123}"_padded; + * auto doc = parser.iterate(json); + * doc.at_path(".\\u00E9") == 123 + * doc.at_path((const char*)u8".\u00E9") returns an error (NO_SUCH_FIELD) + * + * @return The value associated with the given JSONPath expression, or: + * - INVALID_JSON_POINTER if the JSONPath to JSON Pointer conversion fails + * - NO_SUCH_FIELD if a field does not exist in an object + * - INDEX_OUT_OF_BOUNDS if an array index is larger than an array length + * - INCORRECT_TYPE if a non-integer is used to access an array + */ + simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; + /** * Consumes the document and returns a string_view instance corresponding to the * document as represented in JSON. It points inside the original byte array containing @@ -30394,10 +35274,12 @@ namespace simdjson { simdjson_inline simdjson_result get_value() noexcept; simdjson_inline simdjson_result is_null() noexcept; + template simdjson_inline simdjson_result get() & noexcept; simdjson_inline simdjson_result raw_json() noexcept; simdjson_inline operator document& () const noexcept; - #if SIMDJSON_EXCEPTIONS + template + explicit simdjson_inline operator T() noexcept(false); simdjson_inline operator array() & noexcept(false); simdjson_inline operator object() & noexcept(false); simdjson_inline operator uint64_t() noexcept(false); @@ -30422,6 +35304,7 @@ namespace simdjson { simdjson_inline simdjson_result type() noexcept; simdjson_inline simdjson_result is_scalar() noexcept; + simdjson_inline simdjson_result is_string() noexcept; simdjson_inline simdjson_result current_location() noexcept; simdjson_inline int32_t current_depth() const noexcept; @@ -30431,6 +35314,8 @@ namespace simdjson { simdjson_inline simdjson_result get_number() noexcept; simdjson_inline simdjson_result raw_json_token() noexcept; simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; + simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; + private: document* doc{ nullptr }; }; @@ -30466,12 +35351,13 @@ namespace simdjson { simdjson_inline simdjson_result is_null() noexcept; template simdjson_inline simdjson_result get() & noexcept; - template simdjson_inline simdjson_result get() && noexcept; + template simdjson_deprecated simdjson_inline simdjson_result get() && noexcept; template simdjson_inline error_code get(T& out) & noexcept; template simdjson_inline error_code get(T& out) && noexcept; - #if SIMDJSON_EXCEPTIONS + template ::value == false>::type> + explicit simdjson_inline operator T() noexcept(false); simdjson_inline operator arm64::ondemand::array() & noexcept(false); simdjson_inline operator arm64::ondemand::object() & noexcept(false); simdjson_inline operator uint64_t() noexcept(false); @@ -30495,6 +35381,7 @@ namespace simdjson { simdjson_inline simdjson_result find_field_unordered(const char* key) & noexcept; simdjson_inline simdjson_result type() noexcept; simdjson_inline simdjson_result is_scalar() noexcept; + simdjson_inline simdjson_result is_string() noexcept; simdjson_inline simdjson_result current_location() noexcept; simdjson_inline int32_t current_depth() const noexcept; simdjson_inline bool at_end() const noexcept; @@ -30506,6 +35393,7 @@ namespace simdjson { simdjson_inline simdjson_result raw_json_token() noexcept; simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; + simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; }; @@ -30538,8 +35426,9 @@ namespace simdjson { simdjson_inline simdjson_result get_bool() noexcept; simdjson_inline simdjson_result get_value() noexcept; simdjson_inline simdjson_result is_null() noexcept; - #if SIMDJSON_EXCEPTIONS + template ::value == false>::type> + explicit simdjson_inline operator T() noexcept(false); simdjson_inline operator arm64::ondemand::array() & noexcept(false); simdjson_inline operator arm64::ondemand::object() & noexcept(false); simdjson_inline operator uint64_t() noexcept(false); @@ -30563,6 +35452,7 @@ namespace simdjson { simdjson_inline simdjson_result find_field_unordered(const char* key) & noexcept; simdjson_inline simdjson_result type() noexcept; simdjson_inline simdjson_result is_scalar() noexcept; + simdjson_inline simdjson_result is_string() noexcept; simdjson_inline simdjson_result current_location() noexcept; simdjson_inline simdjson_result current_depth() const noexcept; simdjson_inline simdjson_result is_negative() noexcept; @@ -30573,6 +35463,7 @@ namespace simdjson { simdjson_inline simdjson_result raw_json_token() noexcept; simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; + simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; }; @@ -30706,10 +35597,9 @@ namespace simdjson { class iterator { public: using value_type = simdjson_result; - using reference = value_type; - + using reference = simdjson_result; + using pointer = void; using difference_type = std::ptrdiff_t; - using iterator_category = std::input_iterator_tag; /** @@ -30719,7 +35609,7 @@ namespace simdjson { /** * Get the current document (or error). */ - simdjson_inline simdjson_result operator*() noexcept; + simdjson_inline reference operator*() noexcept; /** * Advance to the next document (prefix). */ @@ -30901,7 +35791,7 @@ namespace simdjson { friend class document; friend class json_iterator; friend struct simdjson_result; - friend struct internal::simdjson_result_base; + friend struct simdjson::internal::simdjson_result_base; }; // document_stream } // namespace ondemand @@ -30961,12 +35851,39 @@ namespace simdjson { * This consumes the key: once you have called unescaped_key(), you cannot * call it again nor can you call key(). */ - simdjson_inline simdjson_warn_unused simdjson_result unescaped_key(bool allow_replacement) noexcept; + simdjson_inline simdjson_warn_unused simdjson_result unescaped_key(bool allow_replacement = false) noexcept; + /** + * Get the key as a string_view (for higher speed, consider raw_key). + * We deliberately use a more cumbersome name (unescaped_key) to force users + * to think twice about using it. The content is stored in the receiver. + * + * This consumes the key: once you have called unescaped_key(), you cannot + * call it again nor can you call key(). + */ + template + simdjson_inline simdjson_warn_unused error_code unescaped_key(string_type& receiver, bool allow_replacement = false) noexcept; /** * Get the key as a raw_json_string. Can be used for direct comparison with - * an unescaped C string: e.g., key() == "test". + * an unescaped C string: e.g., key() == "test". This does not count as + * consumption of the content: you can safely call it repeatedly. + * See escaped_key() for a similar function which returns + * a more convenient std::string_view result. */ simdjson_inline raw_json_string key() const noexcept; + /** + * Get the unprocessed key as a string_view. This includes the quotes and may include + * some spaces after the last quote. This does not count as + * consumption of the content: you can safely call it repeatedly. + * See escaped_key(). + */ + simdjson_inline std::string_view key_raw_json_token() const noexcept; + /** + * Get the key as a string_view. This does not include the quotes and + * the string is unprocessed key so it may contain escape characters + * (e.g., \uXXXX or \n). It does not count as a consumption of the content: + * you can safely call it repeatedly. Use unescaped_key() to get the unescaped key. + */ + simdjson_inline std::string_view escaped_key() const noexcept; /** * Get the field value. */ @@ -30998,7 +35915,11 @@ namespace simdjson { simdjson_inline simdjson_result() noexcept = default; simdjson_inline simdjson_result unescaped_key(bool allow_replacement = false) noexcept; + template + simdjson_inline error_code unescaped_key(string_type& receiver, bool allow_replacement = false) noexcept; simdjson_inline simdjson_result key() noexcept; + simdjson_inline simdjson_result key_raw_json_token() noexcept; + simdjson_inline simdjson_result escaped_key() noexcept; simdjson_inline simdjson_result value() noexcept; }; @@ -31066,6 +35987,8 @@ namespace simdjson { * key a single time. Doing object["mykey"].to_string() and then again object["mykey"].to_string() * is an error. * + * If you expect to have keys with escape characters, please review our documentation. + * * @param key The key to look up. * @returns The value of the field, or NO_SUCH_FIELD if the field is not in the object. */ @@ -31087,7 +36010,7 @@ namespace simdjson { * APIs assume this. Therefore, you must be explicit if you want to treat objects as out of order. * * Use find_field() if you are sure fields will be in order (or are willing to treat it as if the - * field wasn't there when they aren't). + * field was not there when they are not in order). * * If you have multiple fields with a matching key ({"x": 1, "x": 1}) be mindful * that only one field is returned. @@ -31103,6 +36026,8 @@ namespace simdjson { * You are expected to access keys only once. You should access the value corresponding to a key * a single time. Doing object["mykey"].to_string() and then again object["mykey"].to_string() is an error. * + * If you expect to have keys with escape characters, please review our documentation. + * * @param key The key to look up. * @returns The value of the field, or NO_SUCH_FIELD if the field is not in the object. */ @@ -31150,11 +36075,24 @@ namespace simdjson { */ inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; + /** + * Get the value associated with the given JSONPath expression. We only support + * JSONPath queries that trivially convertible to JSON Pointer queries: key + * names and array indices. + * + * @return The value associated with the given JSONPath expression, or: + * - INVALID_JSON_POINTER if the JSONPath to JSON Pointer conversion fails + * - NO_SUCH_FIELD if a field does not exist in an object + * - INDEX_OUT_OF_BOUNDS if an array index is larger than an array length + * - INCORRECT_TYPE if a non-integer is used to access an array + */ + inline simdjson_result at_path(std::string_view json_path) noexcept; + /** * Reset the iterator so that we are pointing back at the * beginning of the object. You should still consume values only once even if you - * can iterate through the object more than once. If you unescape a string within - * the object more than once, you have unsafe code. Note that rewinding an object + * can iterate through the object more than once. If you unescape a string or a key + * within the object more than once, you have unsafe code. Note that rewinding an object * means that you may need to reparse it anew: it is not a free operation. * * @returns true if the object contains some elements (not empty) @@ -31237,6 +36175,8 @@ namespace simdjson { simdjson_inline simdjson_result operator[](std::string_view key) & noexcept; simdjson_inline simdjson_result operator[](std::string_view key) && noexcept; simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; + simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; + inline simdjson_result reset() noexcept; inline simdjson_result is_empty() noexcept; inline simdjson_result count_fields() & noexcept; @@ -31610,6 +36550,65 @@ namespace simdjson { return child; } + inline std::string json_path_to_pointer_conversion(std::string_view json_path) { + if (json_path.empty() || (json_path.front() != '.' && + json_path.front() != '[')) { + return "-1"; // This is just a sentinel value, the caller should check for this and return an error. + } + + std::string result; + // Reserve space to reduce allocations, adjusting for potential increases due + // to escaping. + result.reserve(json_path.size() * 2); + + size_t i = 0; + + while (i < json_path.length()) { + if (json_path[i] == '.') { + result += '/'; + } + else if (json_path[i] == '[') { + result += '/'; + ++i; // Move past the '[' + while (i < json_path.length() && json_path[i] != ']') { + if (json_path[i] == '~') { + result += "~0"; + } + else if (json_path[i] == '/') { + result += "~1"; + } + else { + result += json_path[i]; + } + ++i; + } + if (i == json_path.length() || json_path[i] != ']') { + return "-1"; // Using sentinel value that will be handled as an error by the caller. + } + } + else { + if (json_path[i] == '~') { + result += "~0"; + } + else if (json_path[i] == '/') { + result += "~1"; + } + else { + result += json_path[i]; + } + } + ++i; + } + + return result; + } + + inline simdjson_result array::at_path(std::string_view json_path) noexcept { + auto json_pointer = json_path_to_pointer_conversion(json_path); + if (json_pointer == "-1") { return INVALID_JSON_POINTER; } + return at_pointer(json_pointer); + } + simdjson_inline simdjson_result array::at(size_t index) noexcept { size_t i = 0; for (auto value : *this) { @@ -31664,6 +36663,10 @@ namespace simdjson { if (error()) { return error(); } return first.at_pointer(json_pointer); } + simdjson_inline simdjson_result simdjson_result::at_path(std::string_view json_path) noexcept { + if (error()) { return error(); } + return first.at_path(json_path); + } simdjson_inline simdjson_result simdjson_result::raw_json() noexcept { if (error()) { return error(); } return first.raw_json(); @@ -31760,14 +36763,14 @@ namespace simdjson { /* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ /* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_ONDEMAND_DOCUMENT_INL_H */ /* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/array-inl.h" */ /* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/array_iterator.h" */ /* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/document.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/json_iterator-inl.h" */ /* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/json_type.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/object-inl.h" */ /* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/raw_json_string.h" */ /* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/value.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/array-inl.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/json_iterator-inl.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/object-inl.h" */ /* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/value_iterator-inl.h" */ /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ @@ -31923,22 +36926,26 @@ namespace simdjson { template<> simdjson_inline simdjson_result document::get() & noexcept { return get_bool(); } template<> simdjson_inline simdjson_result document::get() & noexcept { return get_value(); } - template<> simdjson_inline simdjson_result document::get() && noexcept { return get_raw_json_string(); } - template<> simdjson_inline simdjson_result document::get() && noexcept { return get_string(false); } - template<> simdjson_inline simdjson_result document::get() && noexcept { return std::forward(*this).get_double(); } - template<> simdjson_inline simdjson_result document::get() && noexcept { return std::forward(*this).get_uint64(); } - template<> simdjson_inline simdjson_result document::get() && noexcept { return std::forward(*this).get_int64(); } - template<> simdjson_inline simdjson_result document::get() && noexcept { return std::forward(*this).get_bool(); } - template<> simdjson_inline simdjson_result document::get() && noexcept { return get_value(); } + template<> simdjson_deprecated simdjson_inline simdjson_result document::get() && noexcept { return get_raw_json_string(); } + template<> simdjson_deprecated simdjson_inline simdjson_result document::get() && noexcept { return get_string(false); } + template<> simdjson_deprecated simdjson_inline simdjson_result document::get() && noexcept { return std::forward(*this).get_double(); } + template<> simdjson_deprecated simdjson_inline simdjson_result document::get() && noexcept { return std::forward(*this).get_uint64(); } + template<> simdjson_deprecated simdjson_inline simdjson_result document::get() && noexcept { return std::forward(*this).get_int64(); } + template<> simdjson_deprecated simdjson_inline simdjson_result document::get() && noexcept { return std::forward(*this).get_bool(); } + template<> simdjson_deprecated simdjson_inline simdjson_result document::get() && noexcept { return get_value(); } template simdjson_inline error_code document::get(T& out) & noexcept { return get().get(out); } - template simdjson_inline error_code document::get(T& out) && noexcept { + template simdjson_deprecated simdjson_inline error_code document::get(T& out) && noexcept { return std::forward(*this).get().get(out); } #if SIMDJSON_EXCEPTIONS + template + simdjson_deprecated simdjson_inline document::operator T() && noexcept(false) { return get(); } + template + simdjson_inline document::operator T() & noexcept(false) { return get(); } simdjson_inline document::operator array() & noexcept(false) { return get_array(); } simdjson_inline document::operator object() & noexcept(false) { return get_object(); } simdjson_inline document::operator uint64_t() noexcept(false) { return get_uint64(); } @@ -32023,6 +37030,13 @@ namespace simdjson { return !((this_type == json_type::array) || (this_type == json_type::object)); } + simdjson_inline simdjson_result document::is_string() noexcept { + json_type this_type; + auto error = type().get(this_type); + if (error) { return error; } + return (this_type == json_type::string); + } + simdjson_inline bool document::is_negative() noexcept { return get_root_value_iterator().is_root_negative(); } @@ -32042,7 +37056,7 @@ namespace simdjson { simdjson_inline simdjson_result document::raw_json_token() noexcept { auto _iter = get_root_value_iterator(); - return std::string_view(reinterpret_cast(_iter.peek_start()), _iter.peek_start_length()); + return std::string_view(reinterpret_cast(_iter.peek_start()), _iter.peek_root_length()); } simdjson_inline simdjson_result document::at_pointer(std::string_view json_pointer) noexcept { @@ -32063,6 +37077,23 @@ namespace simdjson { } } + simdjson_inline simdjson_result document::at_path(std::string_view json_path) noexcept { + rewind(); // Rewind the document each time at_pointer is called + if (json_path.empty()) { + return this->get_value(); + } + json_type t; + SIMDJSON_TRY(type().get(t)); + switch (t) { + case json_type::array: + return (*this).get_array().at_path(json_path); + case json_type::object: + return (*this).get_object().at_path(json_path); + default: + return INVALID_JSON_POINTER; + } + } + } // namespace ondemand } // namespace arm64 } // namespace simdjson @@ -32201,7 +37232,7 @@ namespace simdjson { return first.get(); } template - simdjson_inline simdjson_result simdjson_result::get() && noexcept { + simdjson_deprecated simdjson_inline simdjson_result simdjson_result::get() && noexcept { if (error()) { return error(); } return std::forward(first).get(); } @@ -32217,7 +37248,7 @@ namespace simdjson { } template<> simdjson_inline simdjson_result simdjson_result::get() & noexcept = delete; - template<> simdjson_inline simdjson_result simdjson_result::get() && noexcept { + template<> simdjson_deprecated simdjson_inline simdjson_result simdjson_result::get() && noexcept { if (error()) { return error(); } return std::forward(first); } @@ -32238,6 +37269,10 @@ namespace simdjson { return first.is_scalar(); } + simdjson_inline simdjson_result simdjson_result::is_string() noexcept { + if (error()) { return error(); } + return first.is_string(); + } simdjson_inline bool simdjson_result::is_negative() noexcept { if (error()) { return error(); } @@ -32261,6 +37296,11 @@ namespace simdjson { #if SIMDJSON_EXCEPTIONS + template ::value == false>::type> + simdjson_inline simdjson_result::operator T() noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return first; + } simdjson_inline simdjson_result::operator arm64::ondemand::array() & noexcept(false) { if (error()) { throw simdjson_error(error()); } return first; @@ -32326,6 +37366,10 @@ namespace simdjson { return first.at_pointer(json_pointer); } + simdjson_inline simdjson_result simdjson_result::at_path(std::string_view json_path) noexcept { + if (error()) { return error(); } + return first.at_path(json_path); + } } // namespace simdjson @@ -32367,15 +37411,25 @@ namespace simdjson { simdjson_inline simdjson_result document_reference::get_bool() noexcept { return doc->get_root_value_iterator().get_root_bool(false); } simdjson_inline simdjson_result document_reference::get_value() noexcept { return doc->get_value(); } simdjson_inline simdjson_result document_reference::is_null() noexcept { return doc->get_root_value_iterator().is_root_null(false); } - + template<> simdjson_inline simdjson_result document_reference::get() & noexcept { return get_array(); } + template<> simdjson_inline simdjson_result document_reference::get() & noexcept { return get_object(); } + template<> simdjson_inline simdjson_result document_reference::get() & noexcept { return get_raw_json_string(); } + template<> simdjson_inline simdjson_result document_reference::get() & noexcept { return get_string(false); } + template<> simdjson_inline simdjson_result document_reference::get() & noexcept { return get_double(); } + template<> simdjson_inline simdjson_result document_reference::get() & noexcept { return get_uint64(); } + template<> simdjson_inline simdjson_result document_reference::get() & noexcept { return get_int64(); } + template<> simdjson_inline simdjson_result document_reference::get() & noexcept { return get_bool(); } + template<> simdjson_inline simdjson_result document_reference::get() & noexcept { return get_value(); } #if SIMDJSON_EXCEPTIONS + template + simdjson_inline document_reference::operator T() noexcept(false) { return get(); } simdjson_inline document_reference::operator array() & noexcept(false) { return array(*doc); } simdjson_inline document_reference::operator object() & noexcept(false) { return object(*doc); } simdjson_inline document_reference::operator uint64_t() noexcept(false) { return get_uint64(); } simdjson_inline document_reference::operator int64_t() noexcept(false) { return get_int64(); } simdjson_inline document_reference::operator double() noexcept(false) { return get_double(); } simdjson_inline document_reference::operator std::string_view() noexcept(false) { return std::string_view(*doc); } - simdjson_inline document_reference::operator raw_json_string() noexcept(false) { return raw_json_string(*doc); } + simdjson_inline document_reference::operator raw_json_string() noexcept(false) { return get_raw_json_string(); } simdjson_inline document_reference::operator bool() noexcept(false) { return get_bool(); } simdjson_inline document_reference::operator value() noexcept(false) { return value(*doc); } #endif @@ -32392,6 +37446,7 @@ namespace simdjson { simdjson_inline simdjson_result document_reference::find_field_unordered(const char* key) & noexcept { return doc->find_field_unordered(key); } simdjson_inline simdjson_result document_reference::type() noexcept { return doc->type(); } simdjson_inline simdjson_result document_reference::is_scalar() noexcept { return doc->is_scalar(); } + simdjson_inline simdjson_result document_reference::is_string() noexcept { return doc->is_string(); } simdjson_inline simdjson_result document_reference::current_location() noexcept { return doc->current_location(); } simdjson_inline int32_t document_reference::current_depth() const noexcept { return doc->current_depth(); } simdjson_inline bool document_reference::is_negative() noexcept { return doc->is_negative(); } @@ -32400,6 +37455,7 @@ namespace simdjson { simdjson_inline simdjson_result document_reference::get_number() noexcept { return doc->get_root_value_iterator().get_root_number(false); } simdjson_inline simdjson_result document_reference::raw_json_token() noexcept { return doc->raw_json_token(); } simdjson_inline simdjson_result document_reference::at_pointer(std::string_view json_pointer) noexcept { return doc->at_pointer(json_pointer); } + simdjson_inline simdjson_result document_reference::at_path(std::string_view json_path) noexcept { return doc->at_path(json_path); } simdjson_inline simdjson_result document_reference::raw_json() noexcept { return doc->raw_json(); } simdjson_inline document_reference::operator document& () const noexcept { return *doc; } @@ -32531,6 +37587,10 @@ namespace simdjson { if (error()) { return error(); } return first.is_scalar(); } + simdjson_inline simdjson_result simdjson_result::is_string() noexcept { + if (error()) { return error(); } + return first.is_string(); + } simdjson_inline simdjson_result simdjson_result::is_negative() noexcept { if (error()) { return error(); } return first.is_negative(); @@ -32548,6 +37608,11 @@ namespace simdjson { return first.get_number(); } #if SIMDJSON_EXCEPTIONS + template ::value == false>::type> + simdjson_inline simdjson_result::operator T() noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return first; + } simdjson_inline simdjson_result::operator arm64::ondemand::array() & noexcept(false) { if (error()) { throw simdjson_error(error()); } return first; @@ -32601,6 +37666,12 @@ namespace simdjson { return first.at_pointer(json_pointer); } + simdjson_inline simdjson_result simdjson_result::at_path(std::string_view json_path) noexcept { + if (error()) { + return error(); + } + return first.at_path(json_path); + } } // namespace simdjson @@ -32954,9 +38025,18 @@ namespace simdjson { depth--; break; default: // Scalar value document - // TODO: Remove any trailing whitespaces + // TODO: We could remove trailing whitespaces // This returns a string spanning from start of value to the beginning of the next document (excluded) - return std::string_view(reinterpret_cast(stream->buf) + current_index(), stream->parser->implementation->structural_indexes[++cur_struct_index] - current_index() - 1); + { + auto next_index = stream->parser->implementation->structural_indexes[++cur_struct_index]; + // normally the length would be next_index - current_index() - 1, except for the last document + size_t svlen = next_index - current_index(); + const char* start = reinterpret_cast(stream->buf) + current_index(); + while (svlen > 1 && (std::isspace(start[svlen - 1]) || start[svlen - 1] == '\0')) { + svlen--; + } + return std::string_view(start, svlen); + } } cur_struct_index++; } @@ -33051,7 +38131,7 @@ namespace simdjson { namespace arm64 { namespace ondemand { - // clang 6 doesn't think the default constructor can be noexcept, so we make it explicit + // clang 6 does not think the default constructor can be noexcept, so we make it explicit simdjson_inline field::field() noexcept : std::pair() {} simdjson_inline field::field(raw_json_string key, ondemand::value&& value) noexcept @@ -33077,11 +38157,32 @@ namespace simdjson { return answer; } + template + simdjson_inline simdjson_warn_unused error_code field::unescaped_key(string_type& receiver, bool allow_replacement) noexcept { + std::string_view key; + SIMDJSON_TRY(unescaped_key(allow_replacement).get(key)); + receiver = key; + return SUCCESS; + } + simdjson_inline raw_json_string field::key() const noexcept { SIMDJSON_ASSUME(first.buf != nullptr); // We would like to call .alive() by Visual Studio won't let us. return first; } + + simdjson_inline std::string_view field::key_raw_json_token() const noexcept { + SIMDJSON_ASSUME(first.buf != nullptr); // We would like to call .alive() by Visual Studio won't let us. + return std::string_view(reinterpret_cast(first.buf - 1), second.iter._json_iter->token.peek(-1) - first.buf + 1); + } + + simdjson_inline std::string_view field::escaped_key() const noexcept { + SIMDJSON_ASSUME(first.buf != nullptr); // We would like to call .alive() by Visual Studio won't let us. + auto end_quote = second.iter._json_iter->token.peek(-1); + while (*end_quote != '"') end_quote--; + return std::string_view(reinterpret_cast(first.buf), end_quote - first.buf); + } + simdjson_inline value& field::value() & noexcept { return second; } @@ -33115,10 +38216,28 @@ namespace simdjson { if (error()) { return error(); } return first.key(); } + + simdjson_inline simdjson_result simdjson_result::key_raw_json_token() noexcept { + if (error()) { return error(); } + return first.key_raw_json_token(); + } + + simdjson_inline simdjson_result simdjson_result::escaped_key() noexcept { + if (error()) { return error(); } + return first.escaped_key(); + } + simdjson_inline simdjson_result simdjson_result::unescaped_key(bool allow_replacement) noexcept { if (error()) { return error(); } return first.unescaped_key(allow_replacement); } + + template + simdjson_inline error_code simdjson_result::unescaped_key(string_type& receiver, bool allow_replacement) noexcept { + if (error()) { return error(); } + return first.unescaped_key(receiver, allow_replacement); + } + simdjson_inline simdjson_result simdjson_result::value() noexcept { if (error()) { return error(); } return std::move(first.value()); @@ -33186,6 +38305,23 @@ namespace simdjson { #endif } +#ifdef SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON + simdjson_inline json_iterator::json_iterator(const uint8_t* buf, ondemand::parser* _parser, bool streaming) noexcept + : token(buf, &_parser->implementation->structural_indexes[0]), + parser{ _parser }, + _string_buf_loc{ parser->string_buf.get() }, + _depth{ 1 }, + _root{ parser->implementation->structural_indexes.get() }, + _streaming{ streaming } + + { + logger::log_headers(); +#if SIMDJSON_CHECK_EOF + assert_more_tokens(); +#endif + } +#endif // SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON + inline void json_iterator::rewind() noexcept { token.set_position(root_position()); logger::log_headers(); // We start again @@ -33419,6 +38555,12 @@ namespace simdjson { #endif // SIMDJSON_CHECK_EOF return token.peek_length(position); } + simdjson_inline uint32_t json_iterator::peek_root_length(token_position position) const noexcept { +#if SIMDJSON_CHECK_EOF + assert_valid_position(position); +#endif // SIMDJSON_CHECK_EOF + return token.peek_root_length(position); + } simdjson_inline token_position json_iterator::last_position() const noexcept { // The following line fails under some compilers... @@ -33464,11 +38606,23 @@ namespace simdjson { } simdjson_inline simdjson_result json_iterator::unescape(raw_json_string in, bool allow_replacement) noexcept { +#if SIMDJSON_DEVELOPMENT_CHECKS + auto result = parser->unescape(in, _string_buf_loc, allow_replacement); + SIMDJSON_ASSUME(!parser->string_buffer_overflow(_string_buf_loc)); + return result; +#else return parser->unescape(in, _string_buf_loc, allow_replacement); +#endif } simdjson_inline simdjson_result json_iterator::unescape_wobbly(raw_json_string in) noexcept { +#if SIMDJSON_DEVELOPMENT_CHECKS + auto result = parser->unescape_wobbly(in, _string_buf_loc); + SIMDJSON_ASSUME(!parser->string_buffer_overflow(_string_buf_loc)); + return result; +#else return parser->unescape_wobbly(in, _string_buf_loc); +#endif } simdjson_inline void json_iterator::reenter_child(token_position position, depth_t child_depth) noexcept { @@ -33593,7 +38747,6 @@ namespace simdjson { return get_uint64(); } - simdjson_inline bool number::is_int64() const noexcept { return get_number_type() == number_type::signed_integer; } @@ -34061,6 +39214,14 @@ namespace simdjson { return child; } + inline simdjson_result object::at_path(std::string_view json_path) noexcept { + auto json_pointer = json_path_to_pointer_conversion(json_path); + if (json_pointer == "-1") { + return INVALID_JSON_POINTER; + } + return at_pointer(json_pointer); + } + simdjson_inline simdjson_result object::count_fields() & noexcept { size_t count{ 0 }; // Important: we do not consume any of the values. @@ -34133,6 +39294,14 @@ namespace simdjson { return first.at_pointer(json_pointer); } + simdjson_inline simdjson_result simdjson_result::at_path( + std::string_view json_path) noexcept { + if (error()) { + return error(); + } + return first.at_path(json_path); + } + inline simdjson_result simdjson_result::reset() noexcept { if (error()) { return error(); } return first.reset(); @@ -34344,6 +39513,11 @@ namespace simdjson { _max_depth = new_max_depth; return SUCCESS; } +#if SIMDJSON_DEVELOPMENT_CHECKS + simdjson_inline simdjson_warn_unused bool parser::string_buffer_overflow(const uint8_t* string_buf_loc) const noexcept { + return (string_buf_loc < string_buf.get()) || (size_t(string_buf_loc - string_buf.get()) >= capacity()); + } +#endif simdjson_warn_unused simdjson_inline simdjson_result parser::iterate(padded_string_view json) & noexcept { if (json.padding() < SIMDJSON_PADDING) { return INSUFFICIENT_PADDING; } @@ -34360,6 +39534,27 @@ namespace simdjson { return document::start({ reinterpret_cast(json.data()), this }); } +#ifdef SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON + simdjson_warn_unused simdjson_inline simdjson_result parser::iterate_allow_incomplete_json(padded_string_view json) & noexcept { + if (json.padding() < SIMDJSON_PADDING) { return INSUFFICIENT_PADDING; } + + json.remove_utf8_bom(); + + // Allocate if needed + if (capacity() < json.length() || !string_buf) { + SIMDJSON_TRY(allocate(json.length(), max_depth())); + } + + // Run stage 1. + const simdjson::error_code err = implementation->stage1(reinterpret_cast(json.data()), json.length(), stage1_mode::regular); + if (err) { + if (err != UNCLOSED_STRING) + return err; + } + return document::start({ reinterpret_cast(json.data()), this, true }); + } +#endif // SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON + simdjson_warn_unused simdjson_inline simdjson_result parser::iterate(const char* json, size_t len, size_t allocated) & noexcept { return iterate(padded_string_view(json, len, allocated)); } @@ -34431,13 +39626,13 @@ namespace simdjson { return iterate_many(s.data(), s.length(), batch_size, allow_comma_separated); } - simdjson_inline size_t parser::capacity() const noexcept { + simdjson_pure simdjson_inline size_t parser::capacity() const noexcept { return _capacity; } - simdjson_inline size_t parser::max_capacity() const noexcept { + simdjson_pure simdjson_inline size_t parser::max_capacity() const noexcept { return _max_capacity; } - simdjson_inline size_t parser::max_depth() const noexcept { + simdjson_pure simdjson_inline size_t parser::max_depth() const noexcept { return _max_depth; } @@ -34986,6 +40181,11 @@ namespace simdjson { return *(position + 1) - *position; } + simdjson_inline uint32_t token_iterator::peek_root_length(token_position position) const noexcept { + return *(position + 2) - *(position) > *(position + 1) - *(position) ? + *(position + 1) - *(position) + : *(position + 2) - *(position); + } simdjson_inline const uint8_t* token_iterator::peek(int32_t delta) const noexcept { return &buf[*(_position + delta)]; } @@ -35135,6 +40335,10 @@ namespace simdjson { } #if SIMDJSON_EXCEPTIONS + template + simdjson_inline value::operator T() noexcept(false) { + return get(); + } simdjson_inline value::operator array() noexcept(false) { return get_array(); } @@ -35221,6 +40425,14 @@ namespace simdjson { return !((this_type == json_type::array) || (this_type == json_type::object)); } + simdjson_inline simdjson_result value::is_string() noexcept { + json_type this_type; + auto error = type().get(this_type); + if (error) { return error; } + return (this_type == json_type::string); + } + + simdjson_inline bool value::is_negative() noexcept { return iter.is_negative(); } @@ -35267,6 +40479,26 @@ namespace simdjson { return iter.json_iter().depth(); } + inline bool is_pointer_well_formed(std::string_view json_pointer) noexcept { + if (simdjson_unlikely(json_pointer.empty())) { // can't be + return false; + } + if (simdjson_unlikely(json_pointer[0] != '/')) { + return false; + } + size_t escape = json_pointer.find('~'); + if (escape == std::string_view::npos) { + return true; + } + if (escape == json_pointer.size() - 1) { + return false; + } + if (json_pointer[escape + 1] != '0' && json_pointer[escape + 1] != '1') { + return false; + } + return true; + } + simdjson_inline simdjson_result value::at_pointer(std::string_view json_pointer) noexcept { json_type t; SIMDJSON_TRY(type().get(t)); @@ -35276,6 +40508,23 @@ namespace simdjson { return (*this).get_array().at_pointer(json_pointer); case json_type::object: return (*this).get_object().at_pointer(json_pointer); + default: + // a non-empty string can be invalid, or accessing a primitive (issue 2154) + if (is_pointer_well_formed(json_pointer)) { + return NO_SUCH_FIELD; + } + return INVALID_JSON_POINTER; + } + } + + simdjson_inline simdjson_result value::at_path(std::string_view json_path) noexcept { + json_type t; + SIMDJSON_TRY(type().get(t)); + switch (t) { + case json_type::array: + return (*this).get_array().at_path(json_path); + case json_type::object: + return (*this).get_object().at_path(json_path); default: return INVALID_JSON_POINTER; } @@ -35434,6 +40683,10 @@ namespace simdjson { if (error()) { return error(); } return first.is_scalar(); } + simdjson_inline simdjson_result simdjson_result::is_string() noexcept { + if (error()) { return error(); } + return first.is_string(); + } simdjson_inline simdjson_result simdjson_result::is_negative() noexcept { if (error()) { return error(); } return first.is_negative(); @@ -35451,6 +40704,11 @@ namespace simdjson { return first.get_number(); } #if SIMDJSON_EXCEPTIONS + template + simdjson_inline simdjson_result::operator T() noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return static_cast(first); + } simdjson_inline simdjson_result::operator arm64::ondemand::array() noexcept(false) { if (error()) { throw simdjson_error(error()); } return first; @@ -35505,11 +40763,22 @@ namespace simdjson { return first.current_depth(); } - simdjson_inline simdjson_result simdjson_result::at_pointer(std::string_view json_pointer) noexcept { - if (error()) { return error(); } + simdjson_inline simdjson_result simdjson_result::at_pointer( + std::string_view json_pointer) noexcept { + if (error()) { + return error(); + } return first.at_pointer(json_pointer); } + simdjson_inline simdjson_result simdjson_result::at_path( + std::string_view json_path) noexcept { + if (error()) { + return error(); + } + return first.at_path(json_path); + } + } // namespace simdjson #endif // SIMDJSON_GENERIC_ONDEMAND_VALUE_INL_H @@ -35524,9 +40793,9 @@ namespace simdjson { /* amalgamation skipped (editor-only): #include "simdjson/generic/atomparsing.h" */ /* amalgamation skipped (editor-only): #include "simdjson/generic/numberparsing.h" */ /* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/json_iterator.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/value_iterator.h" */ /* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/json_type-inl.h" */ /* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/raw_json_string-inl.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/value_iterator.h" */ /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ namespace simdjson { @@ -35656,7 +40925,7 @@ namespace simdjson { else if (!is_open()) { #if SIMDJSON_DEVELOPMENT_CHECKS // If we're past the end of the object, we're being iterated out of order. - // Note: this isn't perfect detection. It's possible the user is inside some other object; if so, + // Note: this is not perfect detection. It's possible the user is inside some other object; if so, // this object iterator will blithely scan that object for fields. if (_json_iter->depth() < depth() - 1) { return OUT_OF_ORDER_ITERATION; } #endif @@ -35770,7 +41039,7 @@ namespace simdjson { #if SIMDJSON_DEVELOPMENT_CHECKS // If we're past the end of the object, we're being iterated out of order. - // Note: this isn't perfect detection. It's possible the user is inside some other object; if so, + // Note: this is not perfect detection. It's possible the user is inside some other object; if so, // this object iterator will blithely scan that object for fields. if (_json_iter->depth() < depth() - 1) { return OUT_OF_ORDER_ITERATION; } #endif @@ -36113,7 +41382,7 @@ namespace simdjson { } simdjson_inline simdjson_result value_iterator::is_root_integer(bool check_trailing) noexcept { - auto max_len = peek_start_length(); + auto max_len = peek_root_length(); auto json = peek_root_scalar("is_root_integer"); uint8_t tmpbuf[20 + 1 + 1]{}; // <20 digits> is the longest possible unsigned integer tmpbuf[20 + 1] = '\0'; // make sure that buffer is always null terminated. @@ -36129,7 +41398,7 @@ namespace simdjson { } simdjson_inline simdjson_result value_iterator::get_root_number_type(bool check_trailing) noexcept { - auto max_len = peek_start_length(); + auto max_len = peek_root_length(); auto json = peek_root_scalar("number"); // Per https://www.exploringbinary.com/maximum-number-of-decimal-digits-in-binary-floating-point-numbers/, // 1074 is the maximum number of significant fractional digits. Add 8 more digits for the biggest @@ -36137,7 +41406,12 @@ namespace simdjson { uint8_t tmpbuf[1074 + 8 + 1 + 1]; tmpbuf[1074 + 8 + 1] = '\0'; // make sure that buffer is always null terminated. if (!_json_iter->copy_to_buffer(json, max_len, tmpbuf, 1074 + 8 + 1)) { - logger::log_error(*_json_iter, start_position(), depth(), "Root number more than 1082 characters"); + if (numberparsing::check_if_integer(json, max_len)) { + if (check_trailing && !_json_iter->is_single_token()) { return TRAILING_CONTENT; } + logger::log_error(*_json_iter, start_position(), depth(), "Found big integer"); + return number_type::big_integer; + } + logger::log_error(*_json_iter, start_position(), depth(), "Root number more than 1082 characters and not a big integer"); return NUMBER_ERROR; } auto answer = numberparsing::get_number_type(tmpbuf); @@ -36145,15 +41419,21 @@ namespace simdjson { return answer; } simdjson_inline simdjson_result value_iterator::get_root_number(bool check_trailing) noexcept { - auto max_len = peek_start_length(); + auto max_len = peek_root_length(); auto json = peek_root_scalar("number"); // Per https://www.exploringbinary.com/maximum-number-of-decimal-digits-in-binary-floating-point-numbers/, // 1074 is the maximum number of significant fractional digits. Add 8 more digits for the biggest // number: -0.e-308. + // NOTE: the current approach doesn't work for very big integer numbers containing more than 1074 digits. uint8_t tmpbuf[1074 + 8 + 1 + 1]; tmpbuf[1074 + 8 + 1] = '\0'; // make sure that buffer is always null terminated. if (!_json_iter->copy_to_buffer(json, max_len, tmpbuf, 1074 + 8 + 1)) { - logger::log_error(*_json_iter, start_position(), depth(), "Root number more than 1082 characters"); + if (numberparsing::check_if_integer(json, max_len)) { + if (check_trailing && !_json_iter->is_single_token()) { return TRAILING_CONTENT; } + logger::log_error(*_json_iter, start_position(), depth(), "Found big integer"); + return BIGINT_ERROR; + } + logger::log_error(*_json_iter, start_position(), depth(), "Root number more than 1082 characters and not a big integer"); return NUMBER_ERROR; } number num; @@ -36185,7 +41465,7 @@ namespace simdjson { return raw_json_string(json + 1); } simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_root_uint64(bool check_trailing) noexcept { - auto max_len = peek_start_length(); + auto max_len = peek_root_length(); auto json = peek_root_scalar("uint64"); uint8_t tmpbuf[20 + 1 + 1]{}; // <20 digits> is the longest possible unsigned integer tmpbuf[20 + 1] = '\0'; // make sure that buffer is always null terminated. @@ -36201,7 +41481,7 @@ namespace simdjson { return result; } simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_root_uint64_in_string(bool check_trailing) noexcept { - auto max_len = peek_start_length(); + auto max_len = peek_root_length(); auto json = peek_root_scalar("uint64"); uint8_t tmpbuf[20 + 1 + 1]{}; // <20 digits> is the longest possible unsigned integer tmpbuf[20 + 1] = '\0'; // make sure that buffer is always null terminated. @@ -36217,7 +41497,7 @@ namespace simdjson { return result; } simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_root_int64(bool check_trailing) noexcept { - auto max_len = peek_start_length(); + auto max_len = peek_root_length(); auto json = peek_root_scalar("int64"); uint8_t tmpbuf[20 + 1 + 1]; // -<19 digits> is the longest possible integer tmpbuf[20 + 1] = '\0'; // make sure that buffer is always null terminated. @@ -36234,7 +41514,7 @@ namespace simdjson { return result; } simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_root_int64_in_string(bool check_trailing) noexcept { - auto max_len = peek_start_length(); + auto max_len = peek_root_length(); auto json = peek_root_scalar("int64"); uint8_t tmpbuf[20 + 1 + 1]; // -<19 digits> is the longest possible integer tmpbuf[20 + 1] = '\0'; // make sure that buffer is always null terminated. @@ -36251,7 +41531,7 @@ namespace simdjson { return result; } simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_root_double(bool check_trailing) noexcept { - auto max_len = peek_start_length(); + auto max_len = peek_root_length(); auto json = peek_root_scalar("double"); // Per https://www.exploringbinary.com/maximum-number-of-decimal-digits-in-binary-floating-point-numbers/, // 1074 is the maximum number of significant fractional digits. Add 8 more digits for the biggest @@ -36271,7 +41551,7 @@ namespace simdjson { } simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_root_double_in_string(bool check_trailing) noexcept { - auto max_len = peek_start_length(); + auto max_len = peek_root_length(); auto json = peek_root_scalar("double"); // Per https://www.exploringbinary.com/maximum-number-of-decimal-digits-in-binary-floating-point-numbers/, // 1074 is the maximum number of significant fractional digits. Add 8 more digits for the biggest @@ -36290,7 +41570,7 @@ namespace simdjson { return result; } simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_root_bool(bool check_trailing) noexcept { - auto max_len = peek_start_length(); + auto max_len = peek_root_length(); auto json = peek_root_scalar("bool"); uint8_t tmpbuf[5 + 1 + 1]; // +1 for null termination tmpbuf[5 + 1] = '\0'; // make sure that buffer is always null terminated. @@ -36303,7 +41583,7 @@ namespace simdjson { return result; } simdjson_inline simdjson_result value_iterator::is_root_null(bool check_trailing) noexcept { - auto max_len = peek_start_length(); + auto max_len = peek_root_length(); auto json = peek_root_scalar("null"); bool result = (max_len >= 4 && !atomparsing::str4ncmp(json, "null") && (max_len == 4 || jsoncharutils::is_structural_or_whitespace(json[4]))); @@ -36375,6 +41655,9 @@ namespace simdjson { simdjson_inline uint32_t value_iterator::peek_start_length() const noexcept { return _json_iter->peek_length(start_position()); } + simdjson_inline uint32_t value_iterator::peek_root_length() const noexcept { + return _json_iter->peek_root_length(start_position()); + } simdjson_inline const uint8_t* value_iterator::peek_scalar(const char* type) noexcept { logger::log_value(*_json_iter, start_position(), depth(), type); @@ -36795,13 +42078,13 @@ namespace simdjson { simdjson_inline internal::value128 full_multiplication(uint64_t value1, uint64_t value2) { internal::value128 answer; #if SIMDJSON_REGULAR_VISUAL_STUDIO || SIMDJSON_IS_32BITS -#ifdef _M_ARM64 +#if SIMDJSON_IS_ARM64 // ARM64 has native support for 64-bit multiplications, no need to emultate answer.high = __umulh(value1, value2); answer.low = value1 * value2; #else answer.low = _umul128(value1, value2, &answer.high); // _umul128 not available on ARM64 -#endif // _M_ARM64 +#endif // SIMDJSON_IS_ARM64 #else // SIMDJSON_REGULAR_VISUAL_STUDIO || SIMDJSON_IS_32BITS __uint128_t r = (static_cast<__uint128_t>(value1)) * value2; answer.low = uint64_t(r); @@ -37255,6 +42538,7 @@ namespace simdjson { simdjson_inline simdjson_result parse_bool(const uint8_t* json) const noexcept; simdjson_inline const uint8_t* peek_start() const noexcept; simdjson_inline uint32_t peek_start_length() const noexcept; + simdjson_inline uint32_t peek_root_length() const noexcept; /** * The general idea of the advance_... methods and the peek_* methods @@ -37349,6 +42633,7 @@ namespace simdjson { friend class object; friend class array; friend class value; + friend class field; }; // value_iterator } // namespace ondemand @@ -37409,12 +42694,13 @@ namespace simdjson { * @returns INCORRECT_TYPE If the JSON value is not the given type. */ template simdjson_inline simdjson_result get() noexcept { - // Unless the simdjson library provides an inline implementation, calling this method should + // Unless the simdjson library or the user provides an inline implementation, calling this method should // immediately fail. static_assert(!sizeof(T), "The get method with given type is not implemented by the simdjson library. " "The supported types are ondemand::object, ondemand::array, raw_json_string, std::string_view, uint64_t, " "int64_t, double, and bool. We recommend you use get_double(), get_bool(), get_uint64(), get_int64(), " - " get_object(), get_array(), get_raw_json_string(), or get_string() instead of the get template."); + " get_object(), get_array(), get_raw_json_string(), or get_string() instead of the get template." + " You may also add support for custom types, see our documentation."); } /** @@ -37567,6 +42853,17 @@ namespace simdjson { simdjson_inline simdjson_result is_null() noexcept; #if SIMDJSON_EXCEPTIONS + /** + * Cast this JSON value to an instance of type T. The programmer is responsible for + * providing an implementation of get for the type T, if T is not one of the types + * supported by the library (object, array, raw_json_string, string_view, uint64_t, etc.). + * + * See https://github.com/simdjson/simdjson/blob/master/doc/basics.md#adding-support-for-custom-types + * + * @returns An instance of type T + */ + template + explicit simdjson_inline operator T() noexcept(false); /** * Cast this JSON value to an array. * @@ -37730,7 +43027,7 @@ namespace simdjson { * that only one field is returned. * * Use find_field() if you are sure fields will be in order (or are willing to treat it as if the - * field wasn't there when they aren't). + * field as not there when they are not in order). * * @param key The key to look up. * @returns The value of the field, or NO_SUCH_FIELD if the field is not in the object. @@ -37766,6 +43063,13 @@ namespace simdjson { * @error TAPE_ERROR when the JSON value is a bad token like "}" "," or "alse". */ simdjson_inline simdjson_result is_scalar() noexcept; + /** + * Checks whether the value is a string. + * + * @returns true if the type is string + * @error TAPE_ERROR when the JSON value is a bad token like "}" "," or "alse". + */ + simdjson_inline simdjson_result is_string() noexcept; /** * Checks whether the value is a negative number. @@ -37798,10 +43102,12 @@ namespace simdjson { * get_number().get_number_type(). * * get_number_type() is number_type::unsigned_integer if we have - * an integer greater or equal to 9223372036854775808 + * an integer greater or equal to 9223372036854775808. * get_number_type() is number_type::signed_integer if we have an - * integer that is less than 9223372036854775808 - * Otherwise, get_number_type() has value number_type::floating_point_number + * integer that is less than 9223372036854775808. + * get_number_type() is number_type::big_integer for integers that do not fit in 64 bits, + * in which case the digit_count is set to the length of the big integer string. + * Otherwise, get_number_type() has value number_type::floating_point_number. * * This function requires processing the number string, but it is expected * to be faster than get_number().get_number_type() because it is does not @@ -37828,6 +43134,8 @@ namespace simdjson { * You can recover the value by calling number.get_uint64() and you * have that number.is_uint64() is true. * + * For integers that do not fit in 64 bits, the function returns BIGINT_ERROR error code. + * * Otherwise, number.get_number_type() has value number_type::floating_point_number * and we have a binary64 number. * You can recover the value by calling number.get_double() and you @@ -37843,7 +43151,6 @@ namespace simdjson { */ simdjson_warn_unused simdjson_inline simdjson_result get_number() noexcept; - /** * Get the raw JSON for this token. * @@ -37941,6 +43248,20 @@ namespace simdjson { */ simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; + /** + * Get the value associated with the given JSONPath expression. We only support + * JSONPath queries that trivially convertible to JSON Pointer queries: key + * names and array indices. + * + * @return The value associated with the given JSONPath expression, or: + * - INVALID_JSON_POINTER if the JSONPath to JSON Pointer conversion fails + * - NO_SUCH_FIELD if a field does not exist in an object + * - INDEX_OUT_OF_BOUNDS if an array index is larger than an array length + * - INCORRECT_TYPE if a non-integer is used to access an array + */ + simdjson_inline simdjson_result at_path(std::string_view at_path) noexcept; + + protected: /** * Create a value. @@ -37980,6 +43301,7 @@ namespace simdjson { friend class object; friend struct simdjson_result; friend struct simdjson_result; + friend class field; }; } // namespace ondemand @@ -38017,6 +43339,8 @@ namespace simdjson { template simdjson_inline error_code get(T& out) noexcept; #if SIMDJSON_EXCEPTIONS + template + explicit simdjson_inline operator T() noexcept(false); simdjson_inline operator fallback::ondemand::array() noexcept(false); simdjson_inline operator fallback::ondemand::object() noexcept(false); simdjson_inline operator uint64_t() noexcept(false); @@ -38070,7 +43394,7 @@ namespace simdjson { * APIs assume this. Therefore, you must be explicit if you want to treat objects as out of order. * * Use find_field() if you are sure fields will be in order (or are willing to treat it as if the - * field wasn't there when they aren't). + * field as not there when they are not in order). * * @param key The key to look up. * @returns The value of the field, or NO_SUCH_FIELD if the field is not in the object. @@ -38092,6 +43416,7 @@ namespace simdjson { */ simdjson_inline simdjson_result type() noexcept; simdjson_inline simdjson_result is_scalar() noexcept; + simdjson_inline simdjson_result is_string() noexcept; simdjson_inline simdjson_result is_negative() noexcept; simdjson_inline simdjson_result is_integer() noexcept; simdjson_inline simdjson_result get_number_type() noexcept; @@ -38106,6 +43431,7 @@ namespace simdjson { /** @copydoc simdjson_inline int32_t current_depth() const noexcept */ simdjson_inline simdjson_result current_depth() const noexcept; simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; + simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; }; } // namespace simdjson @@ -38224,7 +43550,7 @@ namespace simdjson { * 1 = next token, -1 = prev token. * * TODO consider a string_view, assuming the length will get stripped out by the optimizer when - * it isn't used ... + * it is not used... */ simdjson_inline const uint8_t* peek(int32_t delta = 0) const noexcept; /** @@ -38254,7 +43580,14 @@ namespace simdjson { * @param position The position of the token. */ simdjson_inline uint32_t peek_length(token_position position) const noexcept; - + /** + * Get the maximum length of the JSON text for a root token. + * + * The length will include any whitespace at the end of the token. + * + * @param position The position of the token (start of the document). + */ + simdjson_inline uint32_t peek_root_length(token_position position) const noexcept; /** * Return the current index. */ @@ -38477,7 +43810,7 @@ namespace simdjson { * @param delta The relative position of the token to retrieve. e.g. 0 = next token, -1 = prev token. * * TODO consider a string_view, assuming the length will get stripped out by the optimizer when - * it isn't used ... + * it is not used ... */ simdjson_inline const uint8_t* peek(int32_t delta = 0) const noexcept; /** @@ -38505,7 +43838,7 @@ namespace simdjson { * @param position The position of the token to retrieve. * * TODO consider a string_view, assuming the length will get stripped out by the optimizer when - * it isn't used ... + * it is not used ... */ simdjson_inline const uint8_t* peek(token_position position) const noexcept; /** @@ -38516,13 +43849,21 @@ namespace simdjson { * @param position The position of the token to retrieve. */ simdjson_inline uint32_t peek_length(token_position position) const noexcept; + /** + * Get the maximum length of the JSON text for the current root token. + * + * The length will include any whitespace at the end of the token. + * + * @param position The position of the token to retrieve. + */ + simdjson_inline uint32_t peek_root_length(token_position position) const noexcept; /** * Get the JSON text for the last token in the document. * * This is not null-terminated; it is a view into the JSON. * * TODO consider a string_view, assuming the length will get stripped out by the optimizer when - * it isn't used ... + * it is not used ... */ simdjson_inline const uint8_t* peek_last() const noexcept; @@ -38616,6 +43957,9 @@ namespace simdjson { inline bool balanced() const noexcept; protected: simdjson_inline json_iterator(const uint8_t* buf, ondemand::parser* parser) noexcept; +#ifdef SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON + simdjson_inline json_iterator(const uint8_t* buf, ondemand::parser* parser, bool streaming) noexcept; +#endif // SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON /// The last token before the end simdjson_inline token_position last_position() const noexcept; /// The token *at* the end. This points at gibberish and should only be used for comparison. @@ -38631,6 +43975,7 @@ namespace simdjson { friend class raw_json_string; friend class parser; friend class value_iterator; + friend class field; template friend simdjson_inline void logger::log_line(const json_iterator& iter, const char* title_prefix, const char* title, std::string_view detail, int delta, int depth_delta, logger::log_level level, Args&&... args) noexcept; template @@ -38752,8 +44097,6 @@ namespace simdjson { */ friend class value_iterator; template - friend error_code numberparsing::slow_float_parsing(simdjson_unused const uint8_t* src, W writer); - template friend error_code numberparsing::write_float(const uint8_t* const src, bool negative, uint64_t i, const uint8_t* start_digits, size_t digit_count, int64_t exponent, W& writer); template friend error_code numberparsing::parse_number(const uint8_t* const src, W& writer); @@ -39047,8 +44390,8 @@ namespace simdjson { namespace ondemand { /** - * The default batch size for document_stream instances for this On Demand kernel. - * Note that different On Demand kernel may use a different DEFAULT_BATCH_SIZE value + * The default batch size for document_stream instances for this On-Demand kernel. + * Note that different On-Demand kernel may use a different DEFAULT_BATCH_SIZE value * in the future. */ static constexpr size_t DEFAULT_BATCH_SIZE = 1000000; @@ -39132,6 +44475,9 @@ namespace simdjson { * - UNCLOSED_STRING if there is an unclosed string in the document. */ simdjson_warn_unused simdjson_result iterate(padded_string_view json) & noexcept; +#ifdef SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON + simdjson_warn_unused simdjson_result iterate_allow_incomplete_json(padded_string_view json) & noexcept; +#endif // SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON /** @overload simdjson_result iterate(padded_string_view json) & noexcept */ simdjson_warn_unused simdjson_result iterate(const char* json, size_t len, size_t capacity) & noexcept; /** @overload simdjson_result iterate(padded_string_view json) & noexcept */ @@ -39276,9 +44622,9 @@ namespace simdjson { simdjson_result iterate_many(const char* buf, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept = delete; /** The capacity of this parser (the largest document it can process). */ - simdjson_inline size_t capacity() const noexcept; + simdjson_pure simdjson_inline size_t capacity() const noexcept; /** The maximum capacity of this parser (the largest document it is allowed to process). */ - simdjson_inline size_t max_capacity() const noexcept; + simdjson_pure simdjson_inline size_t max_capacity() const noexcept; simdjson_inline void set_max_capacity(size_t max_capacity) noexcept; /** * The maximum depth of this parser (the most deeply nested objects and arrays it can process). @@ -39286,7 +44632,7 @@ namespace simdjson { * The document's instance current_depth() method should be used to monitor the parsing * depth and limit it if desired. */ - simdjson_inline size_t max_depth() const noexcept; + simdjson_pure simdjson_inline size_t max_depth() const noexcept; /** * Ensure this parser has enough memory to process JSON documents up to `capacity` bytes in length @@ -39358,9 +44704,20 @@ namespace simdjson { */ simdjson_inline simdjson_result unescape_wobbly(raw_json_string in, uint8_t*& dst) const noexcept; +#if SIMDJSON_DEVELOPMENT_CHECKS + /** + * Returns true if string_buf_loc is outside of the allocated range for the + * the string buffer. When true, it indicates that the string buffer has overflowed. + * This is a development-time check that is not needed in production. It can be + * used to detect buffer overflows in the string buffer and usafe usage of the + * string buffer. + */ + bool string_buffer_overflow(const uint8_t* string_buf_loc) const noexcept; +#endif + private: /** @private [for benchmarking access] The implementation to use */ - std::unique_ptr implementation{}; + std::unique_ptr implementation{}; size_t _capacity{ 0 }; size_t _max_capacity; size_t _max_depth{ DEFAULT_MAX_DEPTH }; @@ -39497,6 +44854,22 @@ namespace simdjson { * - INVALID_JSON_POINTER if the JSON pointer is invalid and cannot be parsed */ inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; + + /** + * Get the value associated with the given JSONPath expression. We only support + * JSONPath queries that trivially convertible to JSON Pointer queries: key + * names and array indices. + * + * https://datatracker.ietf.org/doc/html/draft-normington-jsonpath-00 + * + * @return The value associated with the given JSONPath expression, or: + * - INVALID_JSON_POINTER if the JSONPath to JSON Pointer conversion fails + * - NO_SUCH_FIELD if a field does not exist in an object + * - INDEX_OUT_OF_BOUNDS if an array index is larger than an array length + * - INCORRECT_TYPE if a non-integer is used to access an array + */ + inline simdjson_result at_path(std::string_view json_path) noexcept; + /** * Consumes the array and returns a string_view instance corresponding to the * array as represented in JSON. It points inside the original document. @@ -39587,6 +44960,7 @@ namespace simdjson { inline simdjson_result reset() & noexcept; simdjson_inline simdjson_result at(size_t index) noexcept; simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; + simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; simdjson_inline simdjson_result raw_json() noexcept; }; @@ -39877,21 +45251,23 @@ namespace simdjson { * @returns INCORRECT_TYPE If the JSON value is not the given type. */ template simdjson_inline simdjson_result get() & noexcept { - // Unless the simdjson library provides an inline implementation, calling this method should + // Unless the simdjson library or the user provides an inline implementation, calling this method should // immediately fail. static_assert(!sizeof(T), "The get method with given type is not implemented by the simdjson library. " "The supported types are ondemand::object, ondemand::array, raw_json_string, std::string_view, uint64_t, " "int64_t, double, and bool. We recommend you use get_double(), get_bool(), get_uint64(), get_int64(), " - " get_object(), get_array(), get_raw_json_string(), or get_string() instead of the get template."); + " get_object(), get_array(), get_raw_json_string(), or get_string() instead of the get template." + " You may also add support for custom types, see our documentation."); } /** @overload template simdjson_result get() & noexcept */ - template simdjson_inline simdjson_result get() && noexcept { - // Unless the simdjson library provides an inline implementation, calling this method should + template simdjson_deprecated simdjson_inline simdjson_result get() && noexcept { + // Unless the simdjson library or the user provides an inline implementation, calling this method should // immediately fail. static_assert(!sizeof(T), "The get method with given type is not implemented by the simdjson library. " "The supported types are ondemand::object, ondemand::array, raw_json_string, std::string_view, uint64_t, " "int64_t, double, and bool. We recommend you use get_double(), get_bool(), get_uint64(), get_int64(), " - " get_object(), get_array(), get_raw_json_string(), or get_string() instead of the get template."); + " get_object(), get_array(), get_raw_json_string(), or get_string() instead of the get template." + " You may also add support for custom types, see our documentation."); } /** @@ -39907,9 +45283,23 @@ namespace simdjson { */ template simdjson_inline error_code get(T& out) & noexcept; /** @overload template error_code get(T &out) & noexcept */ - template simdjson_inline error_code get(T& out) && noexcept; + template simdjson_deprecated simdjson_inline error_code get(T& out) && noexcept; #if SIMDJSON_EXCEPTIONS + /** + * Cast this JSON value to an instance of type T. The programmer is responsible for + * providing an implementation of get for the type T, if T is not one of the types + * supported by the library (object, array, raw_json_string, string_view, uint64_t, etc.) + * + * See https://github.com/simdjson/simdjson/blob/master/doc/basics.md#adding-support-for-custom-types + * + * @returns An instance of type T + */ + template + explicit simdjson_inline operator T() & noexcept(false); + template + explicit simdjson_deprecated simdjson_inline operator T() && noexcept(false); + /** * Cast this JSON value to an array. * @@ -40082,7 +45472,7 @@ namespace simdjson { * APIs assume this. Therefore, you must be explicit if you want to treat objects as out of order. * * Use find_field() if you are sure fields will be in order (or are willing to treat it as if the - * field wasn't there when they aren't). + * field was not there when they are not in order). * * You must consume the fields on an object one at a time. A request for a new key * invalidates previous field values: it makes them unsafe. E.g., the array @@ -40128,6 +45518,14 @@ namespace simdjson { */ simdjson_inline simdjson_result is_scalar() noexcept; + /** + * Checks whether the document is a string. + * + * @returns true if the type is string + * @error TAPE_ERROR when the JSON value is a bad token like "}" "," or "alse". + */ + simdjson_inline simdjson_result is_string() noexcept; + /** * Checks whether the document is a negative number. * @@ -40155,9 +45553,11 @@ namespace simdjson { * get_number().get_number_type(). * * get_number_type() is number_type::unsigned_integer if we have - * an integer greater or equal to 9223372036854775808 + * an integer greater or equal to 9223372036854775808 and no larger than 18446744073709551615. * get_number_type() is number_type::signed_integer if we have an - * integer that is less than 9223372036854775808 + * integer that is less than 9223372036854775808 and greater or equal to -9223372036854775808. + * get_number_type() is number_type::big_integer if we have an integer outside + * of those ranges (either larger than 18446744073709551615 or smaller than -9223372036854775808). * Otherwise, get_number_type() has value number_type::floating_point_number * * This function requires processing the number string, but it is expected @@ -40275,6 +45675,14 @@ namespace simdjson { * auto doc = parser.iterate(json); * doc.at_pointer("//a/1") == 20 * + * Key values are matched exactly, without unescaping or Unicode normalization. + * We do a byte-by-byte comparison. E.g. + * + * const padded_string json = "{\"\\u00E9\":123}"_padded; + * auto doc = parser.iterate(json); + * doc.at_pointer("/\\u00E9") == 123 + * doc.at_pointer((const char*)u8"/\u00E9") returns an error (NO_SUCH_FIELD) + * * Note that at_pointer() automatically calls rewind between each call. Thus * all values, objects and arrays that you have created so far (including unescaped strings) * are invalidated. After calling at_pointer, you need to consume the result: string values @@ -40291,6 +45699,30 @@ namespace simdjson { * - SCALAR_DOCUMENT_AS_VALUE if the json_pointer is empty and the document is not a scalar (see is_scalar() function). */ simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; + + /** + * Get the value associated with the given JSONPath expression. We only support + * JSONPath queries that trivially convertible to JSON Pointer queries: key + * names and array indices. + * + * https://datatracker.ietf.org/doc/html/draft-normington-jsonpath-00 + * + * Key values are matched exactly, without unescaping or Unicode normalization. + * We do a byte-by-byte comparison. E.g. + * + * const padded_string json = "{\"\\u00E9\":123}"_padded; + * auto doc = parser.iterate(json); + * doc.at_path(".\\u00E9") == 123 + * doc.at_path((const char*)u8".\u00E9") returns an error (NO_SUCH_FIELD) + * + * @return The value associated with the given JSONPath expression, or: + * - INVALID_JSON_POINTER if the JSONPath to JSON Pointer conversion fails + * - NO_SUCH_FIELD if a field does not exist in an object + * - INDEX_OUT_OF_BOUNDS if an array index is larger than an array length + * - INCORRECT_TYPE if a non-integer is used to access an array + */ + simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; + /** * Consumes the document and returns a string_view instance corresponding to the * document as represented in JSON. It points inside the original byte array containing @@ -40356,10 +45788,12 @@ namespace simdjson { simdjson_inline simdjson_result get_value() noexcept; simdjson_inline simdjson_result is_null() noexcept; + template simdjson_inline simdjson_result get() & noexcept; simdjson_inline simdjson_result raw_json() noexcept; simdjson_inline operator document& () const noexcept; - #if SIMDJSON_EXCEPTIONS + template + explicit simdjson_inline operator T() noexcept(false); simdjson_inline operator array() & noexcept(false); simdjson_inline operator object() & noexcept(false); simdjson_inline operator uint64_t() noexcept(false); @@ -40384,6 +45818,7 @@ namespace simdjson { simdjson_inline simdjson_result type() noexcept; simdjson_inline simdjson_result is_scalar() noexcept; + simdjson_inline simdjson_result is_string() noexcept; simdjson_inline simdjson_result current_location() noexcept; simdjson_inline int32_t current_depth() const noexcept; @@ -40393,6 +45828,8 @@ namespace simdjson { simdjson_inline simdjson_result get_number() noexcept; simdjson_inline simdjson_result raw_json_token() noexcept; simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; + simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; + private: document* doc{ nullptr }; }; @@ -40428,12 +45865,13 @@ namespace simdjson { simdjson_inline simdjson_result is_null() noexcept; template simdjson_inline simdjson_result get() & noexcept; - template simdjson_inline simdjson_result get() && noexcept; + template simdjson_deprecated simdjson_inline simdjson_result get() && noexcept; template simdjson_inline error_code get(T& out) & noexcept; template simdjson_inline error_code get(T& out) && noexcept; - #if SIMDJSON_EXCEPTIONS + template ::value == false>::type> + explicit simdjson_inline operator T() noexcept(false); simdjson_inline operator fallback::ondemand::array() & noexcept(false); simdjson_inline operator fallback::ondemand::object() & noexcept(false); simdjson_inline operator uint64_t() noexcept(false); @@ -40457,6 +45895,7 @@ namespace simdjson { simdjson_inline simdjson_result find_field_unordered(const char* key) & noexcept; simdjson_inline simdjson_result type() noexcept; simdjson_inline simdjson_result is_scalar() noexcept; + simdjson_inline simdjson_result is_string() noexcept; simdjson_inline simdjson_result current_location() noexcept; simdjson_inline int32_t current_depth() const noexcept; simdjson_inline bool at_end() const noexcept; @@ -40468,6 +45907,7 @@ namespace simdjson { simdjson_inline simdjson_result raw_json_token() noexcept; simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; + simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; }; @@ -40500,8 +45940,9 @@ namespace simdjson { simdjson_inline simdjson_result get_bool() noexcept; simdjson_inline simdjson_result get_value() noexcept; simdjson_inline simdjson_result is_null() noexcept; - #if SIMDJSON_EXCEPTIONS + template ::value == false>::type> + explicit simdjson_inline operator T() noexcept(false); simdjson_inline operator fallback::ondemand::array() & noexcept(false); simdjson_inline operator fallback::ondemand::object() & noexcept(false); simdjson_inline operator uint64_t() noexcept(false); @@ -40525,6 +45966,7 @@ namespace simdjson { simdjson_inline simdjson_result find_field_unordered(const char* key) & noexcept; simdjson_inline simdjson_result type() noexcept; simdjson_inline simdjson_result is_scalar() noexcept; + simdjson_inline simdjson_result is_string() noexcept; simdjson_inline simdjson_result current_location() noexcept; simdjson_inline simdjson_result current_depth() const noexcept; simdjson_inline simdjson_result is_negative() noexcept; @@ -40535,6 +45977,7 @@ namespace simdjson { simdjson_inline simdjson_result raw_json_token() noexcept; simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; + simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; }; @@ -40668,10 +46111,9 @@ namespace simdjson { class iterator { public: using value_type = simdjson_result; - using reference = value_type; - + using reference = simdjson_result; + using pointer = void; using difference_type = std::ptrdiff_t; - using iterator_category = std::input_iterator_tag; /** @@ -40681,7 +46123,7 @@ namespace simdjson { /** * Get the current document (or error). */ - simdjson_inline simdjson_result operator*() noexcept; + simdjson_inline reference operator*() noexcept; /** * Advance to the next document (prefix). */ @@ -40863,7 +46305,7 @@ namespace simdjson { friend class document; friend class json_iterator; friend struct simdjson_result; - friend struct internal::simdjson_result_base; + friend struct simdjson::internal::simdjson_result_base; }; // document_stream } // namespace ondemand @@ -40923,12 +46365,39 @@ namespace simdjson { * This consumes the key: once you have called unescaped_key(), you cannot * call it again nor can you call key(). */ - simdjson_inline simdjson_warn_unused simdjson_result unescaped_key(bool allow_replacement) noexcept; + simdjson_inline simdjson_warn_unused simdjson_result unescaped_key(bool allow_replacement = false) noexcept; + /** + * Get the key as a string_view (for higher speed, consider raw_key). + * We deliberately use a more cumbersome name (unescaped_key) to force users + * to think twice about using it. The content is stored in the receiver. + * + * This consumes the key: once you have called unescaped_key(), you cannot + * call it again nor can you call key(). + */ + template + simdjson_inline simdjson_warn_unused error_code unescaped_key(string_type& receiver, bool allow_replacement = false) noexcept; /** * Get the key as a raw_json_string. Can be used for direct comparison with - * an unescaped C string: e.g., key() == "test". + * an unescaped C string: e.g., key() == "test". This does not count as + * consumption of the content: you can safely call it repeatedly. + * See escaped_key() for a similar function which returns + * a more convenient std::string_view result. */ simdjson_inline raw_json_string key() const noexcept; + /** + * Get the unprocessed key as a string_view. This includes the quotes and may include + * some spaces after the last quote. This does not count as + * consumption of the content: you can safely call it repeatedly. + * See escaped_key(). + */ + simdjson_inline std::string_view key_raw_json_token() const noexcept; + /** + * Get the key as a string_view. This does not include the quotes and + * the string is unprocessed key so it may contain escape characters + * (e.g., \uXXXX or \n). It does not count as a consumption of the content: + * you can safely call it repeatedly. Use unescaped_key() to get the unescaped key. + */ + simdjson_inline std::string_view escaped_key() const noexcept; /** * Get the field value. */ @@ -40960,7 +46429,11 @@ namespace simdjson { simdjson_inline simdjson_result() noexcept = default; simdjson_inline simdjson_result unescaped_key(bool allow_replacement = false) noexcept; + template + simdjson_inline error_code unescaped_key(string_type& receiver, bool allow_replacement = false) noexcept; simdjson_inline simdjson_result key() noexcept; + simdjson_inline simdjson_result key_raw_json_token() noexcept; + simdjson_inline simdjson_result escaped_key() noexcept; simdjson_inline simdjson_result value() noexcept; }; @@ -41028,6 +46501,8 @@ namespace simdjson { * key a single time. Doing object["mykey"].to_string() and then again object["mykey"].to_string() * is an error. * + * If you expect to have keys with escape characters, please review our documentation. + * * @param key The key to look up. * @returns The value of the field, or NO_SUCH_FIELD if the field is not in the object. */ @@ -41049,7 +46524,7 @@ namespace simdjson { * APIs assume this. Therefore, you must be explicit if you want to treat objects as out of order. * * Use find_field() if you are sure fields will be in order (or are willing to treat it as if the - * field wasn't there when they aren't). + * field was not there when they are not in order). * * If you have multiple fields with a matching key ({"x": 1, "x": 1}) be mindful * that only one field is returned. @@ -41065,6 +46540,8 @@ namespace simdjson { * You are expected to access keys only once. You should access the value corresponding to a key * a single time. Doing object["mykey"].to_string() and then again object["mykey"].to_string() is an error. * + * If you expect to have keys with escape characters, please review our documentation. + * * @param key The key to look up. * @returns The value of the field, or NO_SUCH_FIELD if the field is not in the object. */ @@ -41112,11 +46589,24 @@ namespace simdjson { */ inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; + /** + * Get the value associated with the given JSONPath expression. We only support + * JSONPath queries that trivially convertible to JSON Pointer queries: key + * names and array indices. + * + * @return The value associated with the given JSONPath expression, or: + * - INVALID_JSON_POINTER if the JSONPath to JSON Pointer conversion fails + * - NO_SUCH_FIELD if a field does not exist in an object + * - INDEX_OUT_OF_BOUNDS if an array index is larger than an array length + * - INCORRECT_TYPE if a non-integer is used to access an array + */ + inline simdjson_result at_path(std::string_view json_path) noexcept; + /** * Reset the iterator so that we are pointing back at the * beginning of the object. You should still consume values only once even if you - * can iterate through the object more than once. If you unescape a string within - * the object more than once, you have unsafe code. Note that rewinding an object + * can iterate through the object more than once. If you unescape a string or a key + * within the object more than once, you have unsafe code. Note that rewinding an object * means that you may need to reparse it anew: it is not a free operation. * * @returns true if the object contains some elements (not empty) @@ -41199,6 +46689,8 @@ namespace simdjson { simdjson_inline simdjson_result operator[](std::string_view key) & noexcept; simdjson_inline simdjson_result operator[](std::string_view key) && noexcept; simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; + simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; + inline simdjson_result reset() noexcept; inline simdjson_result is_empty() noexcept; inline simdjson_result count_fields() & noexcept; @@ -41572,6 +47064,65 @@ namespace simdjson { return child; } + inline std::string json_path_to_pointer_conversion(std::string_view json_path) { + if (json_path.empty() || (json_path.front() != '.' && + json_path.front() != '[')) { + return "-1"; // This is just a sentinel value, the caller should check for this and return an error. + } + + std::string result; + // Reserve space to reduce allocations, adjusting for potential increases due + // to escaping. + result.reserve(json_path.size() * 2); + + size_t i = 0; + + while (i < json_path.length()) { + if (json_path[i] == '.') { + result += '/'; + } + else if (json_path[i] == '[') { + result += '/'; + ++i; // Move past the '[' + while (i < json_path.length() && json_path[i] != ']') { + if (json_path[i] == '~') { + result += "~0"; + } + else if (json_path[i] == '/') { + result += "~1"; + } + else { + result += json_path[i]; + } + ++i; + } + if (i == json_path.length() || json_path[i] != ']') { + return "-1"; // Using sentinel value that will be handled as an error by the caller. + } + } + else { + if (json_path[i] == '~') { + result += "~0"; + } + else if (json_path[i] == '/') { + result += "~1"; + } + else { + result += json_path[i]; + } + } + ++i; + } + + return result; + } + + inline simdjson_result array::at_path(std::string_view json_path) noexcept { + auto json_pointer = json_path_to_pointer_conversion(json_path); + if (json_pointer == "-1") { return INVALID_JSON_POINTER; } + return at_pointer(json_pointer); + } + simdjson_inline simdjson_result array::at(size_t index) noexcept { size_t i = 0; for (auto value : *this) { @@ -41626,6 +47177,10 @@ namespace simdjson { if (error()) { return error(); } return first.at_pointer(json_pointer); } + simdjson_inline simdjson_result simdjson_result::at_path(std::string_view json_path) noexcept { + if (error()) { return error(); } + return first.at_path(json_path); + } simdjson_inline simdjson_result simdjson_result::raw_json() noexcept { if (error()) { return error(); } return first.raw_json(); @@ -41722,14 +47277,14 @@ namespace simdjson { /* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ /* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_ONDEMAND_DOCUMENT_INL_H */ /* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/array-inl.h" */ /* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/array_iterator.h" */ /* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/document.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/json_iterator-inl.h" */ /* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/json_type.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/object-inl.h" */ /* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/raw_json_string.h" */ /* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/value.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/array-inl.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/json_iterator-inl.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/object-inl.h" */ /* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/value_iterator-inl.h" */ /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ @@ -41885,22 +47440,26 @@ namespace simdjson { template<> simdjson_inline simdjson_result document::get() & noexcept { return get_bool(); } template<> simdjson_inline simdjson_result document::get() & noexcept { return get_value(); } - template<> simdjson_inline simdjson_result document::get() && noexcept { return get_raw_json_string(); } - template<> simdjson_inline simdjson_result document::get() && noexcept { return get_string(false); } - template<> simdjson_inline simdjson_result document::get() && noexcept { return std::forward(*this).get_double(); } - template<> simdjson_inline simdjson_result document::get() && noexcept { return std::forward(*this).get_uint64(); } - template<> simdjson_inline simdjson_result document::get() && noexcept { return std::forward(*this).get_int64(); } - template<> simdjson_inline simdjson_result document::get() && noexcept { return std::forward(*this).get_bool(); } - template<> simdjson_inline simdjson_result document::get() && noexcept { return get_value(); } + template<> simdjson_deprecated simdjson_inline simdjson_result document::get() && noexcept { return get_raw_json_string(); } + template<> simdjson_deprecated simdjson_inline simdjson_result document::get() && noexcept { return get_string(false); } + template<> simdjson_deprecated simdjson_inline simdjson_result document::get() && noexcept { return std::forward(*this).get_double(); } + template<> simdjson_deprecated simdjson_inline simdjson_result document::get() && noexcept { return std::forward(*this).get_uint64(); } + template<> simdjson_deprecated simdjson_inline simdjson_result document::get() && noexcept { return std::forward(*this).get_int64(); } + template<> simdjson_deprecated simdjson_inline simdjson_result document::get() && noexcept { return std::forward(*this).get_bool(); } + template<> simdjson_deprecated simdjson_inline simdjson_result document::get() && noexcept { return get_value(); } template simdjson_inline error_code document::get(T& out) & noexcept { return get().get(out); } - template simdjson_inline error_code document::get(T& out) && noexcept { + template simdjson_deprecated simdjson_inline error_code document::get(T& out) && noexcept { return std::forward(*this).get().get(out); } #if SIMDJSON_EXCEPTIONS + template + simdjson_deprecated simdjson_inline document::operator T() && noexcept(false) { return get(); } + template + simdjson_inline document::operator T() & noexcept(false) { return get(); } simdjson_inline document::operator array() & noexcept(false) { return get_array(); } simdjson_inline document::operator object() & noexcept(false) { return get_object(); } simdjson_inline document::operator uint64_t() noexcept(false) { return get_uint64(); } @@ -41985,6 +47544,13 @@ namespace simdjson { return !((this_type == json_type::array) || (this_type == json_type::object)); } + simdjson_inline simdjson_result document::is_string() noexcept { + json_type this_type; + auto error = type().get(this_type); + if (error) { return error; } + return (this_type == json_type::string); + } + simdjson_inline bool document::is_negative() noexcept { return get_root_value_iterator().is_root_negative(); } @@ -42004,7 +47570,7 @@ namespace simdjson { simdjson_inline simdjson_result document::raw_json_token() noexcept { auto _iter = get_root_value_iterator(); - return std::string_view(reinterpret_cast(_iter.peek_start()), _iter.peek_start_length()); + return std::string_view(reinterpret_cast(_iter.peek_start()), _iter.peek_root_length()); } simdjson_inline simdjson_result document::at_pointer(std::string_view json_pointer) noexcept { @@ -42025,6 +47591,23 @@ namespace simdjson { } } + simdjson_inline simdjson_result document::at_path(std::string_view json_path) noexcept { + rewind(); // Rewind the document each time at_pointer is called + if (json_path.empty()) { + return this->get_value(); + } + json_type t; + SIMDJSON_TRY(type().get(t)); + switch (t) { + case json_type::array: + return (*this).get_array().at_path(json_path); + case json_type::object: + return (*this).get_object().at_path(json_path); + default: + return INVALID_JSON_POINTER; + } + } + } // namespace ondemand } // namespace fallback } // namespace simdjson @@ -42163,7 +47746,7 @@ namespace simdjson { return first.get(); } template - simdjson_inline simdjson_result simdjson_result::get() && noexcept { + simdjson_deprecated simdjson_inline simdjson_result simdjson_result::get() && noexcept { if (error()) { return error(); } return std::forward(first).get(); } @@ -42179,7 +47762,7 @@ namespace simdjson { } template<> simdjson_inline simdjson_result simdjson_result::get() & noexcept = delete; - template<> simdjson_inline simdjson_result simdjson_result::get() && noexcept { + template<> simdjson_deprecated simdjson_inline simdjson_result simdjson_result::get() && noexcept { if (error()) { return error(); } return std::forward(first); } @@ -42200,6 +47783,10 @@ namespace simdjson { return first.is_scalar(); } + simdjson_inline simdjson_result simdjson_result::is_string() noexcept { + if (error()) { return error(); } + return first.is_string(); + } simdjson_inline bool simdjson_result::is_negative() noexcept { if (error()) { return error(); } @@ -42223,6 +47810,11 @@ namespace simdjson { #if SIMDJSON_EXCEPTIONS + template ::value == false>::type> + simdjson_inline simdjson_result::operator T() noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return first; + } simdjson_inline simdjson_result::operator fallback::ondemand::array() & noexcept(false) { if (error()) { throw simdjson_error(error()); } return first; @@ -42288,6 +47880,10 @@ namespace simdjson { return first.at_pointer(json_pointer); } + simdjson_inline simdjson_result simdjson_result::at_path(std::string_view json_path) noexcept { + if (error()) { return error(); } + return first.at_path(json_path); + } } // namespace simdjson @@ -42329,15 +47925,25 @@ namespace simdjson { simdjson_inline simdjson_result document_reference::get_bool() noexcept { return doc->get_root_value_iterator().get_root_bool(false); } simdjson_inline simdjson_result document_reference::get_value() noexcept { return doc->get_value(); } simdjson_inline simdjson_result document_reference::is_null() noexcept { return doc->get_root_value_iterator().is_root_null(false); } - + template<> simdjson_inline simdjson_result document_reference::get() & noexcept { return get_array(); } + template<> simdjson_inline simdjson_result document_reference::get() & noexcept { return get_object(); } + template<> simdjson_inline simdjson_result document_reference::get() & noexcept { return get_raw_json_string(); } + template<> simdjson_inline simdjson_result document_reference::get() & noexcept { return get_string(false); } + template<> simdjson_inline simdjson_result document_reference::get() & noexcept { return get_double(); } + template<> simdjson_inline simdjson_result document_reference::get() & noexcept { return get_uint64(); } + template<> simdjson_inline simdjson_result document_reference::get() & noexcept { return get_int64(); } + template<> simdjson_inline simdjson_result document_reference::get() & noexcept { return get_bool(); } + template<> simdjson_inline simdjson_result document_reference::get() & noexcept { return get_value(); } #if SIMDJSON_EXCEPTIONS + template + simdjson_inline document_reference::operator T() noexcept(false) { return get(); } simdjson_inline document_reference::operator array() & noexcept(false) { return array(*doc); } simdjson_inline document_reference::operator object() & noexcept(false) { return object(*doc); } simdjson_inline document_reference::operator uint64_t() noexcept(false) { return get_uint64(); } simdjson_inline document_reference::operator int64_t() noexcept(false) { return get_int64(); } simdjson_inline document_reference::operator double() noexcept(false) { return get_double(); } simdjson_inline document_reference::operator std::string_view() noexcept(false) { return std::string_view(*doc); } - simdjson_inline document_reference::operator raw_json_string() noexcept(false) { return raw_json_string(*doc); } + simdjson_inline document_reference::operator raw_json_string() noexcept(false) { return get_raw_json_string(); } simdjson_inline document_reference::operator bool() noexcept(false) { return get_bool(); } simdjson_inline document_reference::operator value() noexcept(false) { return value(*doc); } #endif @@ -42354,6 +47960,7 @@ namespace simdjson { simdjson_inline simdjson_result document_reference::find_field_unordered(const char* key) & noexcept { return doc->find_field_unordered(key); } simdjson_inline simdjson_result document_reference::type() noexcept { return doc->type(); } simdjson_inline simdjson_result document_reference::is_scalar() noexcept { return doc->is_scalar(); } + simdjson_inline simdjson_result document_reference::is_string() noexcept { return doc->is_string(); } simdjson_inline simdjson_result document_reference::current_location() noexcept { return doc->current_location(); } simdjson_inline int32_t document_reference::current_depth() const noexcept { return doc->current_depth(); } simdjson_inline bool document_reference::is_negative() noexcept { return doc->is_negative(); } @@ -42362,6 +47969,7 @@ namespace simdjson { simdjson_inline simdjson_result document_reference::get_number() noexcept { return doc->get_root_value_iterator().get_root_number(false); } simdjson_inline simdjson_result document_reference::raw_json_token() noexcept { return doc->raw_json_token(); } simdjson_inline simdjson_result document_reference::at_pointer(std::string_view json_pointer) noexcept { return doc->at_pointer(json_pointer); } + simdjson_inline simdjson_result document_reference::at_path(std::string_view json_path) noexcept { return doc->at_path(json_path); } simdjson_inline simdjson_result document_reference::raw_json() noexcept { return doc->raw_json(); } simdjson_inline document_reference::operator document& () const noexcept { return *doc; } @@ -42493,6 +48101,10 @@ namespace simdjson { if (error()) { return error(); } return first.is_scalar(); } + simdjson_inline simdjson_result simdjson_result::is_string() noexcept { + if (error()) { return error(); } + return first.is_string(); + } simdjson_inline simdjson_result simdjson_result::is_negative() noexcept { if (error()) { return error(); } return first.is_negative(); @@ -42510,6 +48122,11 @@ namespace simdjson { return first.get_number(); } #if SIMDJSON_EXCEPTIONS + template ::value == false>::type> + simdjson_inline simdjson_result::operator T() noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return first; + } simdjson_inline simdjson_result::operator fallback::ondemand::array() & noexcept(false) { if (error()) { throw simdjson_error(error()); } return first; @@ -42563,6 +48180,12 @@ namespace simdjson { return first.at_pointer(json_pointer); } + simdjson_inline simdjson_result simdjson_result::at_path(std::string_view json_path) noexcept { + if (error()) { + return error(); + } + return first.at_path(json_path); + } } // namespace simdjson @@ -42916,9 +48539,18 @@ namespace simdjson { depth--; break; default: // Scalar value document - // TODO: Remove any trailing whitespaces + // TODO: We could remove trailing whitespaces // This returns a string spanning from start of value to the beginning of the next document (excluded) - return std::string_view(reinterpret_cast(stream->buf) + current_index(), stream->parser->implementation->structural_indexes[++cur_struct_index] - current_index() - 1); + { + auto next_index = stream->parser->implementation->structural_indexes[++cur_struct_index]; + // normally the length would be next_index - current_index() - 1, except for the last document + size_t svlen = next_index - current_index(); + const char* start = reinterpret_cast(stream->buf) + current_index(); + while (svlen > 1 && (std::isspace(start[svlen - 1]) || start[svlen - 1] == '\0')) { + svlen--; + } + return std::string_view(start, svlen); + } } cur_struct_index++; } @@ -43013,7 +48645,7 @@ namespace simdjson { namespace fallback { namespace ondemand { - // clang 6 doesn't think the default constructor can be noexcept, so we make it explicit + // clang 6 does not think the default constructor can be noexcept, so we make it explicit simdjson_inline field::field() noexcept : std::pair() {} simdjson_inline field::field(raw_json_string key, ondemand::value&& value) noexcept @@ -43039,11 +48671,32 @@ namespace simdjson { return answer; } + template + simdjson_inline simdjson_warn_unused error_code field::unescaped_key(string_type& receiver, bool allow_replacement) noexcept { + std::string_view key; + SIMDJSON_TRY(unescaped_key(allow_replacement).get(key)); + receiver = key; + return SUCCESS; + } + simdjson_inline raw_json_string field::key() const noexcept { SIMDJSON_ASSUME(first.buf != nullptr); // We would like to call .alive() by Visual Studio won't let us. return first; } + + simdjson_inline std::string_view field::key_raw_json_token() const noexcept { + SIMDJSON_ASSUME(first.buf != nullptr); // We would like to call .alive() by Visual Studio won't let us. + return std::string_view(reinterpret_cast(first.buf - 1), second.iter._json_iter->token.peek(-1) - first.buf + 1); + } + + simdjson_inline std::string_view field::escaped_key() const noexcept { + SIMDJSON_ASSUME(first.buf != nullptr); // We would like to call .alive() by Visual Studio won't let us. + auto end_quote = second.iter._json_iter->token.peek(-1); + while (*end_quote != '"') end_quote--; + return std::string_view(reinterpret_cast(first.buf), end_quote - first.buf); + } + simdjson_inline value& field::value() & noexcept { return second; } @@ -43077,10 +48730,28 @@ namespace simdjson { if (error()) { return error(); } return first.key(); } + + simdjson_inline simdjson_result simdjson_result::key_raw_json_token() noexcept { + if (error()) { return error(); } + return first.key_raw_json_token(); + } + + simdjson_inline simdjson_result simdjson_result::escaped_key() noexcept { + if (error()) { return error(); } + return first.escaped_key(); + } + simdjson_inline simdjson_result simdjson_result::unescaped_key(bool allow_replacement) noexcept { if (error()) { return error(); } return first.unescaped_key(allow_replacement); } + + template + simdjson_inline error_code simdjson_result::unescaped_key(string_type& receiver, bool allow_replacement) noexcept { + if (error()) { return error(); } + return first.unescaped_key(receiver, allow_replacement); + } + simdjson_inline simdjson_result simdjson_result::value() noexcept { if (error()) { return error(); } return std::move(first.value()); @@ -43148,6 +48819,23 @@ namespace simdjson { #endif } +#ifdef SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON + simdjson_inline json_iterator::json_iterator(const uint8_t* buf, ondemand::parser* _parser, bool streaming) noexcept + : token(buf, &_parser->implementation->structural_indexes[0]), + parser{ _parser }, + _string_buf_loc{ parser->string_buf.get() }, + _depth{ 1 }, + _root{ parser->implementation->structural_indexes.get() }, + _streaming{ streaming } + + { + logger::log_headers(); +#if SIMDJSON_CHECK_EOF + assert_more_tokens(); +#endif + } +#endif // SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON + inline void json_iterator::rewind() noexcept { token.set_position(root_position()); logger::log_headers(); // We start again @@ -43381,6 +49069,12 @@ namespace simdjson { #endif // SIMDJSON_CHECK_EOF return token.peek_length(position); } + simdjson_inline uint32_t json_iterator::peek_root_length(token_position position) const noexcept { +#if SIMDJSON_CHECK_EOF + assert_valid_position(position); +#endif // SIMDJSON_CHECK_EOF + return token.peek_root_length(position); + } simdjson_inline token_position json_iterator::last_position() const noexcept { // The following line fails under some compilers... @@ -43426,11 +49120,23 @@ namespace simdjson { } simdjson_inline simdjson_result json_iterator::unescape(raw_json_string in, bool allow_replacement) noexcept { +#if SIMDJSON_DEVELOPMENT_CHECKS + auto result = parser->unescape(in, _string_buf_loc, allow_replacement); + SIMDJSON_ASSUME(!parser->string_buffer_overflow(_string_buf_loc)); + return result; +#else return parser->unescape(in, _string_buf_loc, allow_replacement); +#endif } simdjson_inline simdjson_result json_iterator::unescape_wobbly(raw_json_string in) noexcept { +#if SIMDJSON_DEVELOPMENT_CHECKS + auto result = parser->unescape_wobbly(in, _string_buf_loc); + SIMDJSON_ASSUME(!parser->string_buffer_overflow(_string_buf_loc)); + return result; +#else return parser->unescape_wobbly(in, _string_buf_loc); +#endif } simdjson_inline void json_iterator::reenter_child(token_position position, depth_t child_depth) noexcept { @@ -43555,7 +49261,6 @@ namespace simdjson { return get_uint64(); } - simdjson_inline bool number::is_int64() const noexcept { return get_number_type() == number_type::signed_integer; } @@ -44023,6 +49728,14 @@ namespace simdjson { return child; } + inline simdjson_result object::at_path(std::string_view json_path) noexcept { + auto json_pointer = json_path_to_pointer_conversion(json_path); + if (json_pointer == "-1") { + return INVALID_JSON_POINTER; + } + return at_pointer(json_pointer); + } + simdjson_inline simdjson_result object::count_fields() & noexcept { size_t count{ 0 }; // Important: we do not consume any of the values. @@ -44095,6 +49808,14 @@ namespace simdjson { return first.at_pointer(json_pointer); } + simdjson_inline simdjson_result simdjson_result::at_path( + std::string_view json_path) noexcept { + if (error()) { + return error(); + } + return first.at_path(json_path); + } + inline simdjson_result simdjson_result::reset() noexcept { if (error()) { return error(); } return first.reset(); @@ -44306,6 +50027,11 @@ namespace simdjson { _max_depth = new_max_depth; return SUCCESS; } +#if SIMDJSON_DEVELOPMENT_CHECKS + simdjson_inline simdjson_warn_unused bool parser::string_buffer_overflow(const uint8_t* string_buf_loc) const noexcept { + return (string_buf_loc < string_buf.get()) || (size_t(string_buf_loc - string_buf.get()) >= capacity()); + } +#endif simdjson_warn_unused simdjson_inline simdjson_result parser::iterate(padded_string_view json) & noexcept { if (json.padding() < SIMDJSON_PADDING) { return INSUFFICIENT_PADDING; } @@ -44322,6 +50048,27 @@ namespace simdjson { return document::start({ reinterpret_cast(json.data()), this }); } +#ifdef SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON + simdjson_warn_unused simdjson_inline simdjson_result parser::iterate_allow_incomplete_json(padded_string_view json) & noexcept { + if (json.padding() < SIMDJSON_PADDING) { return INSUFFICIENT_PADDING; } + + json.remove_utf8_bom(); + + // Allocate if needed + if (capacity() < json.length() || !string_buf) { + SIMDJSON_TRY(allocate(json.length(), max_depth())); + } + + // Run stage 1. + const simdjson::error_code err = implementation->stage1(reinterpret_cast(json.data()), json.length(), stage1_mode::regular); + if (err) { + if (err != UNCLOSED_STRING) + return err; + } + return document::start({ reinterpret_cast(json.data()), this, true }); + } +#endif // SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON + simdjson_warn_unused simdjson_inline simdjson_result parser::iterate(const char* json, size_t len, size_t allocated) & noexcept { return iterate(padded_string_view(json, len, allocated)); } @@ -44393,13 +50140,13 @@ namespace simdjson { return iterate_many(s.data(), s.length(), batch_size, allow_comma_separated); } - simdjson_inline size_t parser::capacity() const noexcept { + simdjson_pure simdjson_inline size_t parser::capacity() const noexcept { return _capacity; } - simdjson_inline size_t parser::max_capacity() const noexcept { + simdjson_pure simdjson_inline size_t parser::max_capacity() const noexcept { return _max_capacity; } - simdjson_inline size_t parser::max_depth() const noexcept { + simdjson_pure simdjson_inline size_t parser::max_depth() const noexcept { return _max_depth; } @@ -44948,6 +50695,11 @@ namespace simdjson { return *(position + 1) - *position; } + simdjson_inline uint32_t token_iterator::peek_root_length(token_position position) const noexcept { + return *(position + 2) - *(position) > *(position + 1) - *(position) ? + *(position + 1) - *(position) + : *(position + 2) - *(position); + } simdjson_inline const uint8_t* token_iterator::peek(int32_t delta) const noexcept { return &buf[*(_position + delta)]; } @@ -45097,6 +50849,10 @@ namespace simdjson { } #if SIMDJSON_EXCEPTIONS + template + simdjson_inline value::operator T() noexcept(false) { + return get(); + } simdjson_inline value::operator array() noexcept(false) { return get_array(); } @@ -45183,6 +50939,14 @@ namespace simdjson { return !((this_type == json_type::array) || (this_type == json_type::object)); } + simdjson_inline simdjson_result value::is_string() noexcept { + json_type this_type; + auto error = type().get(this_type); + if (error) { return error; } + return (this_type == json_type::string); + } + + simdjson_inline bool value::is_negative() noexcept { return iter.is_negative(); } @@ -45229,6 +50993,26 @@ namespace simdjson { return iter.json_iter().depth(); } + inline bool is_pointer_well_formed(std::string_view json_pointer) noexcept { + if (simdjson_unlikely(json_pointer.empty())) { // can't be + return false; + } + if (simdjson_unlikely(json_pointer[0] != '/')) { + return false; + } + size_t escape = json_pointer.find('~'); + if (escape == std::string_view::npos) { + return true; + } + if (escape == json_pointer.size() - 1) { + return false; + } + if (json_pointer[escape + 1] != '0' && json_pointer[escape + 1] != '1') { + return false; + } + return true; + } + simdjson_inline simdjson_result value::at_pointer(std::string_view json_pointer) noexcept { json_type t; SIMDJSON_TRY(type().get(t)); @@ -45238,6 +51022,23 @@ namespace simdjson { return (*this).get_array().at_pointer(json_pointer); case json_type::object: return (*this).get_object().at_pointer(json_pointer); + default: + // a non-empty string can be invalid, or accessing a primitive (issue 2154) + if (is_pointer_well_formed(json_pointer)) { + return NO_SUCH_FIELD; + } + return INVALID_JSON_POINTER; + } + } + + simdjson_inline simdjson_result value::at_path(std::string_view json_path) noexcept { + json_type t; + SIMDJSON_TRY(type().get(t)); + switch (t) { + case json_type::array: + return (*this).get_array().at_path(json_path); + case json_type::object: + return (*this).get_object().at_path(json_path); default: return INVALID_JSON_POINTER; } @@ -45396,6 +51197,10 @@ namespace simdjson { if (error()) { return error(); } return first.is_scalar(); } + simdjson_inline simdjson_result simdjson_result::is_string() noexcept { + if (error()) { return error(); } + return first.is_string(); + } simdjson_inline simdjson_result simdjson_result::is_negative() noexcept { if (error()) { return error(); } return first.is_negative(); @@ -45413,6 +51218,11 @@ namespace simdjson { return first.get_number(); } #if SIMDJSON_EXCEPTIONS + template + simdjson_inline simdjson_result::operator T() noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return static_cast(first); + } simdjson_inline simdjson_result::operator fallback::ondemand::array() noexcept(false) { if (error()) { throw simdjson_error(error()); } return first; @@ -45467,11 +51277,22 @@ namespace simdjson { return first.current_depth(); } - simdjson_inline simdjson_result simdjson_result::at_pointer(std::string_view json_pointer) noexcept { - if (error()) { return error(); } + simdjson_inline simdjson_result simdjson_result::at_pointer( + std::string_view json_pointer) noexcept { + if (error()) { + return error(); + } return first.at_pointer(json_pointer); } + simdjson_inline simdjson_result simdjson_result::at_path( + std::string_view json_path) noexcept { + if (error()) { + return error(); + } + return first.at_path(json_path); + } + } // namespace simdjson #endif // SIMDJSON_GENERIC_ONDEMAND_VALUE_INL_H @@ -45486,9 +51307,9 @@ namespace simdjson { /* amalgamation skipped (editor-only): #include "simdjson/generic/atomparsing.h" */ /* amalgamation skipped (editor-only): #include "simdjson/generic/numberparsing.h" */ /* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/json_iterator.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/value_iterator.h" */ /* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/json_type-inl.h" */ /* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/raw_json_string-inl.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/value_iterator.h" */ /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ namespace simdjson { @@ -45618,7 +51439,7 @@ namespace simdjson { else if (!is_open()) { #if SIMDJSON_DEVELOPMENT_CHECKS // If we're past the end of the object, we're being iterated out of order. - // Note: this isn't perfect detection. It's possible the user is inside some other object; if so, + // Note: this is not perfect detection. It's possible the user is inside some other object; if so, // this object iterator will blithely scan that object for fields. if (_json_iter->depth() < depth() - 1) { return OUT_OF_ORDER_ITERATION; } #endif @@ -45732,7 +51553,7 @@ namespace simdjson { #if SIMDJSON_DEVELOPMENT_CHECKS // If we're past the end of the object, we're being iterated out of order. - // Note: this isn't perfect detection. It's possible the user is inside some other object; if so, + // Note: this is not perfect detection. It's possible the user is inside some other object; if so, // this object iterator will blithely scan that object for fields. if (_json_iter->depth() < depth() - 1) { return OUT_OF_ORDER_ITERATION; } #endif @@ -46075,7 +51896,7 @@ namespace simdjson { } simdjson_inline simdjson_result value_iterator::is_root_integer(bool check_trailing) noexcept { - auto max_len = peek_start_length(); + auto max_len = peek_root_length(); auto json = peek_root_scalar("is_root_integer"); uint8_t tmpbuf[20 + 1 + 1]{}; // <20 digits> is the longest possible unsigned integer tmpbuf[20 + 1] = '\0'; // make sure that buffer is always null terminated. @@ -46091,7 +51912,7 @@ namespace simdjson { } simdjson_inline simdjson_result value_iterator::get_root_number_type(bool check_trailing) noexcept { - auto max_len = peek_start_length(); + auto max_len = peek_root_length(); auto json = peek_root_scalar("number"); // Per https://www.exploringbinary.com/maximum-number-of-decimal-digits-in-binary-floating-point-numbers/, // 1074 is the maximum number of significant fractional digits. Add 8 more digits for the biggest @@ -46099,7 +51920,12 @@ namespace simdjson { uint8_t tmpbuf[1074 + 8 + 1 + 1]; tmpbuf[1074 + 8 + 1] = '\0'; // make sure that buffer is always null terminated. if (!_json_iter->copy_to_buffer(json, max_len, tmpbuf, 1074 + 8 + 1)) { - logger::log_error(*_json_iter, start_position(), depth(), "Root number more than 1082 characters"); + if (numberparsing::check_if_integer(json, max_len)) { + if (check_trailing && !_json_iter->is_single_token()) { return TRAILING_CONTENT; } + logger::log_error(*_json_iter, start_position(), depth(), "Found big integer"); + return number_type::big_integer; + } + logger::log_error(*_json_iter, start_position(), depth(), "Root number more than 1082 characters and not a big integer"); return NUMBER_ERROR; } auto answer = numberparsing::get_number_type(tmpbuf); @@ -46107,15 +51933,21 @@ namespace simdjson { return answer; } simdjson_inline simdjson_result value_iterator::get_root_number(bool check_trailing) noexcept { - auto max_len = peek_start_length(); + auto max_len = peek_root_length(); auto json = peek_root_scalar("number"); // Per https://www.exploringbinary.com/maximum-number-of-decimal-digits-in-binary-floating-point-numbers/, // 1074 is the maximum number of significant fractional digits. Add 8 more digits for the biggest // number: -0.e-308. + // NOTE: the current approach doesn't work for very big integer numbers containing more than 1074 digits. uint8_t tmpbuf[1074 + 8 + 1 + 1]; tmpbuf[1074 + 8 + 1] = '\0'; // make sure that buffer is always null terminated. if (!_json_iter->copy_to_buffer(json, max_len, tmpbuf, 1074 + 8 + 1)) { - logger::log_error(*_json_iter, start_position(), depth(), "Root number more than 1082 characters"); + if (numberparsing::check_if_integer(json, max_len)) { + if (check_trailing && !_json_iter->is_single_token()) { return TRAILING_CONTENT; } + logger::log_error(*_json_iter, start_position(), depth(), "Found big integer"); + return BIGINT_ERROR; + } + logger::log_error(*_json_iter, start_position(), depth(), "Root number more than 1082 characters and not a big integer"); return NUMBER_ERROR; } number num; @@ -46147,7 +51979,7 @@ namespace simdjson { return raw_json_string(json + 1); } simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_root_uint64(bool check_trailing) noexcept { - auto max_len = peek_start_length(); + auto max_len = peek_root_length(); auto json = peek_root_scalar("uint64"); uint8_t tmpbuf[20 + 1 + 1]{}; // <20 digits> is the longest possible unsigned integer tmpbuf[20 + 1] = '\0'; // make sure that buffer is always null terminated. @@ -46163,7 +51995,7 @@ namespace simdjson { return result; } simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_root_uint64_in_string(bool check_trailing) noexcept { - auto max_len = peek_start_length(); + auto max_len = peek_root_length(); auto json = peek_root_scalar("uint64"); uint8_t tmpbuf[20 + 1 + 1]{}; // <20 digits> is the longest possible unsigned integer tmpbuf[20 + 1] = '\0'; // make sure that buffer is always null terminated. @@ -46179,7 +52011,7 @@ namespace simdjson { return result; } simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_root_int64(bool check_trailing) noexcept { - auto max_len = peek_start_length(); + auto max_len = peek_root_length(); auto json = peek_root_scalar("int64"); uint8_t tmpbuf[20 + 1 + 1]; // -<19 digits> is the longest possible integer tmpbuf[20 + 1] = '\0'; // make sure that buffer is always null terminated. @@ -46196,7 +52028,7 @@ namespace simdjson { return result; } simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_root_int64_in_string(bool check_trailing) noexcept { - auto max_len = peek_start_length(); + auto max_len = peek_root_length(); auto json = peek_root_scalar("int64"); uint8_t tmpbuf[20 + 1 + 1]; // -<19 digits> is the longest possible integer tmpbuf[20 + 1] = '\0'; // make sure that buffer is always null terminated. @@ -46213,7 +52045,7 @@ namespace simdjson { return result; } simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_root_double(bool check_trailing) noexcept { - auto max_len = peek_start_length(); + auto max_len = peek_root_length(); auto json = peek_root_scalar("double"); // Per https://www.exploringbinary.com/maximum-number-of-decimal-digits-in-binary-floating-point-numbers/, // 1074 is the maximum number of significant fractional digits. Add 8 more digits for the biggest @@ -46233,7 +52065,7 @@ namespace simdjson { } simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_root_double_in_string(bool check_trailing) noexcept { - auto max_len = peek_start_length(); + auto max_len = peek_root_length(); auto json = peek_root_scalar("double"); // Per https://www.exploringbinary.com/maximum-number-of-decimal-digits-in-binary-floating-point-numbers/, // 1074 is the maximum number of significant fractional digits. Add 8 more digits for the biggest @@ -46252,7 +52084,7 @@ namespace simdjson { return result; } simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_root_bool(bool check_trailing) noexcept { - auto max_len = peek_start_length(); + auto max_len = peek_root_length(); auto json = peek_root_scalar("bool"); uint8_t tmpbuf[5 + 1 + 1]; // +1 for null termination tmpbuf[5 + 1] = '\0'; // make sure that buffer is always null terminated. @@ -46265,7 +52097,7 @@ namespace simdjson { return result; } simdjson_inline simdjson_result value_iterator::is_root_null(bool check_trailing) noexcept { - auto max_len = peek_start_length(); + auto max_len = peek_root_length(); auto json = peek_root_scalar("null"); bool result = (max_len >= 4 && !atomparsing::str4ncmp(json, "null") && (max_len == 4 || jsoncharutils::is_structural_or_whitespace(json[4]))); @@ -46337,6 +52169,9 @@ namespace simdjson { simdjson_inline uint32_t value_iterator::peek_start_length() const noexcept { return _json_iter->peek_length(start_position()); } + simdjson_inline uint32_t value_iterator::peek_root_length() const noexcept { + return _json_iter->peek_root_length(start_position()); + } simdjson_inline const uint8_t* value_iterator::peek_scalar(const char* type) noexcept { logger::log_value(*_json_iter, start_position(), depth(), type); @@ -46823,13 +52658,13 @@ namespace simdjson { simdjson_inline internal::value128 full_multiplication(uint64_t value1, uint64_t value2) { internal::value128 answer; #if SIMDJSON_REGULAR_VISUAL_STUDIO || SIMDJSON_IS_32BITS -#ifdef _M_ARM64 +#if SIMDJSON_IS_ARM64 // ARM64 has native support for 64-bit multiplications, no need to emultate answer.high = __umulh(value1, value2); answer.low = value1 * value2; #else answer.low = _umul128(value1, value2, &answer.high); // _umul128 not available on ARM64 -#endif // _M_ARM64 +#endif // SIMDJSON_IS_ARM64 #else // SIMDJSON_REGULAR_VISUAL_STUDIO || SIMDJSON_IS_32BITS __uint128_t r = (static_cast<__uint128_t>(value1)) * value2; answer.low = uint64_t(r); @@ -46915,10 +52750,10 @@ namespace simdjson { struct simd8 : base8 { static simdjson_inline simd8 splat(bool _value) { return _mm256_set1_epi8(uint8_t(-(!!_value))); } - simdjson_inline simd8() : base8() {} - simdjson_inline simd8(const __m256i _value) : base8(_value) {} + simdjson_inline simd8() : base8() {} + simdjson_inline simd8(const __m256i _value) : base8(_value) {} // Splat constructor - simdjson_inline simd8(bool _value) : base8(splat(_value)) {} + simdjson_inline simd8(bool _value) : base8(splat(_value)) {} simdjson_inline int to_bitmask() const { return _mm256_movemask_epi8(*this); } simdjson_inline bool any() const { return !_mm256_testz_si256(*this, *this); } @@ -47709,6 +53544,7 @@ namespace simdjson { simdjson_inline simdjson_result parse_bool(const uint8_t* json) const noexcept; simdjson_inline const uint8_t* peek_start() const noexcept; simdjson_inline uint32_t peek_start_length() const noexcept; + simdjson_inline uint32_t peek_root_length() const noexcept; /** * The general idea of the advance_... methods and the peek_* methods @@ -47803,6 +53639,7 @@ namespace simdjson { friend class object; friend class array; friend class value; + friend class field; }; // value_iterator } // namespace ondemand @@ -47863,12 +53700,13 @@ namespace simdjson { * @returns INCORRECT_TYPE If the JSON value is not the given type. */ template simdjson_inline simdjson_result get() noexcept { - // Unless the simdjson library provides an inline implementation, calling this method should + // Unless the simdjson library or the user provides an inline implementation, calling this method should // immediately fail. static_assert(!sizeof(T), "The get method with given type is not implemented by the simdjson library. " "The supported types are ondemand::object, ondemand::array, raw_json_string, std::string_view, uint64_t, " "int64_t, double, and bool. We recommend you use get_double(), get_bool(), get_uint64(), get_int64(), " - " get_object(), get_array(), get_raw_json_string(), or get_string() instead of the get template."); + " get_object(), get_array(), get_raw_json_string(), or get_string() instead of the get template." + " You may also add support for custom types, see our documentation."); } /** @@ -48021,6 +53859,17 @@ namespace simdjson { simdjson_inline simdjson_result is_null() noexcept; #if SIMDJSON_EXCEPTIONS + /** + * Cast this JSON value to an instance of type T. The programmer is responsible for + * providing an implementation of get for the type T, if T is not one of the types + * supported by the library (object, array, raw_json_string, string_view, uint64_t, etc.). + * + * See https://github.com/simdjson/simdjson/blob/master/doc/basics.md#adding-support-for-custom-types + * + * @returns An instance of type T + */ + template + explicit simdjson_inline operator T() noexcept(false); /** * Cast this JSON value to an array. * @@ -48184,7 +54033,7 @@ namespace simdjson { * that only one field is returned. * * Use find_field() if you are sure fields will be in order (or are willing to treat it as if the - * field wasn't there when they aren't). + * field as not there when they are not in order). * * @param key The key to look up. * @returns The value of the field, or NO_SUCH_FIELD if the field is not in the object. @@ -48220,6 +54069,13 @@ namespace simdjson { * @error TAPE_ERROR when the JSON value is a bad token like "}" "," or "alse". */ simdjson_inline simdjson_result is_scalar() noexcept; + /** + * Checks whether the value is a string. + * + * @returns true if the type is string + * @error TAPE_ERROR when the JSON value is a bad token like "}" "," or "alse". + */ + simdjson_inline simdjson_result is_string() noexcept; /** * Checks whether the value is a negative number. @@ -48252,10 +54108,12 @@ namespace simdjson { * get_number().get_number_type(). * * get_number_type() is number_type::unsigned_integer if we have - * an integer greater or equal to 9223372036854775808 + * an integer greater or equal to 9223372036854775808. * get_number_type() is number_type::signed_integer if we have an - * integer that is less than 9223372036854775808 - * Otherwise, get_number_type() has value number_type::floating_point_number + * integer that is less than 9223372036854775808. + * get_number_type() is number_type::big_integer for integers that do not fit in 64 bits, + * in which case the digit_count is set to the length of the big integer string. + * Otherwise, get_number_type() has value number_type::floating_point_number. * * This function requires processing the number string, but it is expected * to be faster than get_number().get_number_type() because it is does not @@ -48282,6 +54140,8 @@ namespace simdjson { * You can recover the value by calling number.get_uint64() and you * have that number.is_uint64() is true. * + * For integers that do not fit in 64 bits, the function returns BIGINT_ERROR error code. + * * Otherwise, number.get_number_type() has value number_type::floating_point_number * and we have a binary64 number. * You can recover the value by calling number.get_double() and you @@ -48297,7 +54157,6 @@ namespace simdjson { */ simdjson_warn_unused simdjson_inline simdjson_result get_number() noexcept; - /** * Get the raw JSON for this token. * @@ -48395,6 +54254,20 @@ namespace simdjson { */ simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; + /** + * Get the value associated with the given JSONPath expression. We only support + * JSONPath queries that trivially convertible to JSON Pointer queries: key + * names and array indices. + * + * @return The value associated with the given JSONPath expression, or: + * - INVALID_JSON_POINTER if the JSONPath to JSON Pointer conversion fails + * - NO_SUCH_FIELD if a field does not exist in an object + * - INDEX_OUT_OF_BOUNDS if an array index is larger than an array length + * - INCORRECT_TYPE if a non-integer is used to access an array + */ + simdjson_inline simdjson_result at_path(std::string_view at_path) noexcept; + + protected: /** * Create a value. @@ -48434,6 +54307,7 @@ namespace simdjson { friend class object; friend struct simdjson_result; friend struct simdjson_result; + friend class field; }; } // namespace ondemand @@ -48471,6 +54345,8 @@ namespace simdjson { template simdjson_inline error_code get(T& out) noexcept; #if SIMDJSON_EXCEPTIONS + template + explicit simdjson_inline operator T() noexcept(false); simdjson_inline operator haswell::ondemand::array() noexcept(false); simdjson_inline operator haswell::ondemand::object() noexcept(false); simdjson_inline operator uint64_t() noexcept(false); @@ -48524,7 +54400,7 @@ namespace simdjson { * APIs assume this. Therefore, you must be explicit if you want to treat objects as out of order. * * Use find_field() if you are sure fields will be in order (or are willing to treat it as if the - * field wasn't there when they aren't). + * field as not there when they are not in order). * * @param key The key to look up. * @returns The value of the field, or NO_SUCH_FIELD if the field is not in the object. @@ -48546,6 +54422,7 @@ namespace simdjson { */ simdjson_inline simdjson_result type() noexcept; simdjson_inline simdjson_result is_scalar() noexcept; + simdjson_inline simdjson_result is_string() noexcept; simdjson_inline simdjson_result is_negative() noexcept; simdjson_inline simdjson_result is_integer() noexcept; simdjson_inline simdjson_result get_number_type() noexcept; @@ -48560,6 +54437,7 @@ namespace simdjson { /** @copydoc simdjson_inline int32_t current_depth() const noexcept */ simdjson_inline simdjson_result current_depth() const noexcept; simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; + simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; }; } // namespace simdjson @@ -48678,7 +54556,7 @@ namespace simdjson { * 1 = next token, -1 = prev token. * * TODO consider a string_view, assuming the length will get stripped out by the optimizer when - * it isn't used ... + * it is not used... */ simdjson_inline const uint8_t* peek(int32_t delta = 0) const noexcept; /** @@ -48708,7 +54586,14 @@ namespace simdjson { * @param position The position of the token. */ simdjson_inline uint32_t peek_length(token_position position) const noexcept; - + /** + * Get the maximum length of the JSON text for a root token. + * + * The length will include any whitespace at the end of the token. + * + * @param position The position of the token (start of the document). + */ + simdjson_inline uint32_t peek_root_length(token_position position) const noexcept; /** * Return the current index. */ @@ -48931,7 +54816,7 @@ namespace simdjson { * @param delta The relative position of the token to retrieve. e.g. 0 = next token, -1 = prev token. * * TODO consider a string_view, assuming the length will get stripped out by the optimizer when - * it isn't used ... + * it is not used ... */ simdjson_inline const uint8_t* peek(int32_t delta = 0) const noexcept; /** @@ -48959,7 +54844,7 @@ namespace simdjson { * @param position The position of the token to retrieve. * * TODO consider a string_view, assuming the length will get stripped out by the optimizer when - * it isn't used ... + * it is not used ... */ simdjson_inline const uint8_t* peek(token_position position) const noexcept; /** @@ -48970,13 +54855,21 @@ namespace simdjson { * @param position The position of the token to retrieve. */ simdjson_inline uint32_t peek_length(token_position position) const noexcept; + /** + * Get the maximum length of the JSON text for the current root token. + * + * The length will include any whitespace at the end of the token. + * + * @param position The position of the token to retrieve. + */ + simdjson_inline uint32_t peek_root_length(token_position position) const noexcept; /** * Get the JSON text for the last token in the document. * * This is not null-terminated; it is a view into the JSON. * * TODO consider a string_view, assuming the length will get stripped out by the optimizer when - * it isn't used ... + * it is not used ... */ simdjson_inline const uint8_t* peek_last() const noexcept; @@ -49070,6 +54963,9 @@ namespace simdjson { inline bool balanced() const noexcept; protected: simdjson_inline json_iterator(const uint8_t* buf, ondemand::parser* parser) noexcept; +#ifdef SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON + simdjson_inline json_iterator(const uint8_t* buf, ondemand::parser* parser, bool streaming) noexcept; +#endif // SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON /// The last token before the end simdjson_inline token_position last_position() const noexcept; /// The token *at* the end. This points at gibberish and should only be used for comparison. @@ -49085,6 +54981,7 @@ namespace simdjson { friend class raw_json_string; friend class parser; friend class value_iterator; + friend class field; template friend simdjson_inline void logger::log_line(const json_iterator& iter, const char* title_prefix, const char* title, std::string_view detail, int delta, int depth_delta, logger::log_level level, Args&&... args) noexcept; template @@ -49206,8 +55103,6 @@ namespace simdjson { */ friend class value_iterator; template - friend error_code numberparsing::slow_float_parsing(simdjson_unused const uint8_t* src, W writer); - template friend error_code numberparsing::write_float(const uint8_t* const src, bool negative, uint64_t i, const uint8_t* start_digits, size_t digit_count, int64_t exponent, W& writer); template friend error_code numberparsing::parse_number(const uint8_t* const src, W& writer); @@ -49501,8 +55396,8 @@ namespace simdjson { namespace ondemand { /** - * The default batch size for document_stream instances for this On Demand kernel. - * Note that different On Demand kernel may use a different DEFAULT_BATCH_SIZE value + * The default batch size for document_stream instances for this On-Demand kernel. + * Note that different On-Demand kernel may use a different DEFAULT_BATCH_SIZE value * in the future. */ static constexpr size_t DEFAULT_BATCH_SIZE = 1000000; @@ -49586,6 +55481,9 @@ namespace simdjson { * - UNCLOSED_STRING if there is an unclosed string in the document. */ simdjson_warn_unused simdjson_result iterate(padded_string_view json) & noexcept; +#ifdef SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON + simdjson_warn_unused simdjson_result iterate_allow_incomplete_json(padded_string_view json) & noexcept; +#endif // SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON /** @overload simdjson_result iterate(padded_string_view json) & noexcept */ simdjson_warn_unused simdjson_result iterate(const char* json, size_t len, size_t capacity) & noexcept; /** @overload simdjson_result iterate(padded_string_view json) & noexcept */ @@ -49730,9 +55628,9 @@ namespace simdjson { simdjson_result iterate_many(const char* buf, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept = delete; /** The capacity of this parser (the largest document it can process). */ - simdjson_inline size_t capacity() const noexcept; + simdjson_pure simdjson_inline size_t capacity() const noexcept; /** The maximum capacity of this parser (the largest document it is allowed to process). */ - simdjson_inline size_t max_capacity() const noexcept; + simdjson_pure simdjson_inline size_t max_capacity() const noexcept; simdjson_inline void set_max_capacity(size_t max_capacity) noexcept; /** * The maximum depth of this parser (the most deeply nested objects and arrays it can process). @@ -49740,7 +55638,7 @@ namespace simdjson { * The document's instance current_depth() method should be used to monitor the parsing * depth and limit it if desired. */ - simdjson_inline size_t max_depth() const noexcept; + simdjson_pure simdjson_inline size_t max_depth() const noexcept; /** * Ensure this parser has enough memory to process JSON documents up to `capacity` bytes in length @@ -49812,9 +55710,20 @@ namespace simdjson { */ simdjson_inline simdjson_result unescape_wobbly(raw_json_string in, uint8_t*& dst) const noexcept; +#if SIMDJSON_DEVELOPMENT_CHECKS + /** + * Returns true if string_buf_loc is outside of the allocated range for the + * the string buffer. When true, it indicates that the string buffer has overflowed. + * This is a development-time check that is not needed in production. It can be + * used to detect buffer overflows in the string buffer and usafe usage of the + * string buffer. + */ + bool string_buffer_overflow(const uint8_t* string_buf_loc) const noexcept; +#endif + private: /** @private [for benchmarking access] The implementation to use */ - std::unique_ptr implementation{}; + std::unique_ptr implementation{}; size_t _capacity{ 0 }; size_t _max_capacity; size_t _max_depth{ DEFAULT_MAX_DEPTH }; @@ -49951,6 +55860,22 @@ namespace simdjson { * - INVALID_JSON_POINTER if the JSON pointer is invalid and cannot be parsed */ inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; + + /** + * Get the value associated with the given JSONPath expression. We only support + * JSONPath queries that trivially convertible to JSON Pointer queries: key + * names and array indices. + * + * https://datatracker.ietf.org/doc/html/draft-normington-jsonpath-00 + * + * @return The value associated with the given JSONPath expression, or: + * - INVALID_JSON_POINTER if the JSONPath to JSON Pointer conversion fails + * - NO_SUCH_FIELD if a field does not exist in an object + * - INDEX_OUT_OF_BOUNDS if an array index is larger than an array length + * - INCORRECT_TYPE if a non-integer is used to access an array + */ + inline simdjson_result at_path(std::string_view json_path) noexcept; + /** * Consumes the array and returns a string_view instance corresponding to the * array as represented in JSON. It points inside the original document. @@ -50041,6 +55966,7 @@ namespace simdjson { inline simdjson_result reset() & noexcept; simdjson_inline simdjson_result at(size_t index) noexcept; simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; + simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; simdjson_inline simdjson_result raw_json() noexcept; }; @@ -50331,21 +56257,23 @@ namespace simdjson { * @returns INCORRECT_TYPE If the JSON value is not the given type. */ template simdjson_inline simdjson_result get() & noexcept { - // Unless the simdjson library provides an inline implementation, calling this method should + // Unless the simdjson library or the user provides an inline implementation, calling this method should // immediately fail. static_assert(!sizeof(T), "The get method with given type is not implemented by the simdjson library. " "The supported types are ondemand::object, ondemand::array, raw_json_string, std::string_view, uint64_t, " "int64_t, double, and bool. We recommend you use get_double(), get_bool(), get_uint64(), get_int64(), " - " get_object(), get_array(), get_raw_json_string(), or get_string() instead of the get template."); + " get_object(), get_array(), get_raw_json_string(), or get_string() instead of the get template." + " You may also add support for custom types, see our documentation."); } /** @overload template simdjson_result get() & noexcept */ - template simdjson_inline simdjson_result get() && noexcept { - // Unless the simdjson library provides an inline implementation, calling this method should + template simdjson_deprecated simdjson_inline simdjson_result get() && noexcept { + // Unless the simdjson library or the user provides an inline implementation, calling this method should // immediately fail. static_assert(!sizeof(T), "The get method with given type is not implemented by the simdjson library. " "The supported types are ondemand::object, ondemand::array, raw_json_string, std::string_view, uint64_t, " "int64_t, double, and bool. We recommend you use get_double(), get_bool(), get_uint64(), get_int64(), " - " get_object(), get_array(), get_raw_json_string(), or get_string() instead of the get template."); + " get_object(), get_array(), get_raw_json_string(), or get_string() instead of the get template." + " You may also add support for custom types, see our documentation."); } /** @@ -50361,9 +56289,23 @@ namespace simdjson { */ template simdjson_inline error_code get(T& out) & noexcept; /** @overload template error_code get(T &out) & noexcept */ - template simdjson_inline error_code get(T& out) && noexcept; + template simdjson_deprecated simdjson_inline error_code get(T& out) && noexcept; #if SIMDJSON_EXCEPTIONS + /** + * Cast this JSON value to an instance of type T. The programmer is responsible for + * providing an implementation of get for the type T, if T is not one of the types + * supported by the library (object, array, raw_json_string, string_view, uint64_t, etc.) + * + * See https://github.com/simdjson/simdjson/blob/master/doc/basics.md#adding-support-for-custom-types + * + * @returns An instance of type T + */ + template + explicit simdjson_inline operator T() & noexcept(false); + template + explicit simdjson_deprecated simdjson_inline operator T() && noexcept(false); + /** * Cast this JSON value to an array. * @@ -50536,7 +56478,7 @@ namespace simdjson { * APIs assume this. Therefore, you must be explicit if you want to treat objects as out of order. * * Use find_field() if you are sure fields will be in order (or are willing to treat it as if the - * field wasn't there when they aren't). + * field was not there when they are not in order). * * You must consume the fields on an object one at a time. A request for a new key * invalidates previous field values: it makes them unsafe. E.g., the array @@ -50582,6 +56524,14 @@ namespace simdjson { */ simdjson_inline simdjson_result is_scalar() noexcept; + /** + * Checks whether the document is a string. + * + * @returns true if the type is string + * @error TAPE_ERROR when the JSON value is a bad token like "}" "," or "alse". + */ + simdjson_inline simdjson_result is_string() noexcept; + /** * Checks whether the document is a negative number. * @@ -50609,9 +56559,11 @@ namespace simdjson { * get_number().get_number_type(). * * get_number_type() is number_type::unsigned_integer if we have - * an integer greater or equal to 9223372036854775808 + * an integer greater or equal to 9223372036854775808 and no larger than 18446744073709551615. * get_number_type() is number_type::signed_integer if we have an - * integer that is less than 9223372036854775808 + * integer that is less than 9223372036854775808 and greater or equal to -9223372036854775808. + * get_number_type() is number_type::big_integer if we have an integer outside + * of those ranges (either larger than 18446744073709551615 or smaller than -9223372036854775808). * Otherwise, get_number_type() has value number_type::floating_point_number * * This function requires processing the number string, but it is expected @@ -50729,6 +56681,14 @@ namespace simdjson { * auto doc = parser.iterate(json); * doc.at_pointer("//a/1") == 20 * + * Key values are matched exactly, without unescaping or Unicode normalization. + * We do a byte-by-byte comparison. E.g. + * + * const padded_string json = "{\"\\u00E9\":123}"_padded; + * auto doc = parser.iterate(json); + * doc.at_pointer("/\\u00E9") == 123 + * doc.at_pointer((const char*)u8"/\u00E9") returns an error (NO_SUCH_FIELD) + * * Note that at_pointer() automatically calls rewind between each call. Thus * all values, objects and arrays that you have created so far (including unescaped strings) * are invalidated. After calling at_pointer, you need to consume the result: string values @@ -50745,6 +56705,30 @@ namespace simdjson { * - SCALAR_DOCUMENT_AS_VALUE if the json_pointer is empty and the document is not a scalar (see is_scalar() function). */ simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; + + /** + * Get the value associated with the given JSONPath expression. We only support + * JSONPath queries that trivially convertible to JSON Pointer queries: key + * names and array indices. + * + * https://datatracker.ietf.org/doc/html/draft-normington-jsonpath-00 + * + * Key values are matched exactly, without unescaping or Unicode normalization. + * We do a byte-by-byte comparison. E.g. + * + * const padded_string json = "{\"\\u00E9\":123}"_padded; + * auto doc = parser.iterate(json); + * doc.at_path(".\\u00E9") == 123 + * doc.at_path((const char*)u8".\u00E9") returns an error (NO_SUCH_FIELD) + * + * @return The value associated with the given JSONPath expression, or: + * - INVALID_JSON_POINTER if the JSONPath to JSON Pointer conversion fails + * - NO_SUCH_FIELD if a field does not exist in an object + * - INDEX_OUT_OF_BOUNDS if an array index is larger than an array length + * - INCORRECT_TYPE if a non-integer is used to access an array + */ + simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; + /** * Consumes the document and returns a string_view instance corresponding to the * document as represented in JSON. It points inside the original byte array containing @@ -50810,10 +56794,12 @@ namespace simdjson { simdjson_inline simdjson_result get_value() noexcept; simdjson_inline simdjson_result is_null() noexcept; + template simdjson_inline simdjson_result get() & noexcept; simdjson_inline simdjson_result raw_json() noexcept; simdjson_inline operator document& () const noexcept; - #if SIMDJSON_EXCEPTIONS + template + explicit simdjson_inline operator T() noexcept(false); simdjson_inline operator array() & noexcept(false); simdjson_inline operator object() & noexcept(false); simdjson_inline operator uint64_t() noexcept(false); @@ -50838,6 +56824,7 @@ namespace simdjson { simdjson_inline simdjson_result type() noexcept; simdjson_inline simdjson_result is_scalar() noexcept; + simdjson_inline simdjson_result is_string() noexcept; simdjson_inline simdjson_result current_location() noexcept; simdjson_inline int32_t current_depth() const noexcept; @@ -50847,6 +56834,8 @@ namespace simdjson { simdjson_inline simdjson_result get_number() noexcept; simdjson_inline simdjson_result raw_json_token() noexcept; simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; + simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; + private: document* doc{ nullptr }; }; @@ -50882,12 +56871,13 @@ namespace simdjson { simdjson_inline simdjson_result is_null() noexcept; template simdjson_inline simdjson_result get() & noexcept; - template simdjson_inline simdjson_result get() && noexcept; + template simdjson_deprecated simdjson_inline simdjson_result get() && noexcept; template simdjson_inline error_code get(T& out) & noexcept; template simdjson_inline error_code get(T& out) && noexcept; - #if SIMDJSON_EXCEPTIONS + template ::value == false>::type> + explicit simdjson_inline operator T() noexcept(false); simdjson_inline operator haswell::ondemand::array() & noexcept(false); simdjson_inline operator haswell::ondemand::object() & noexcept(false); simdjson_inline operator uint64_t() noexcept(false); @@ -50911,6 +56901,7 @@ namespace simdjson { simdjson_inline simdjson_result find_field_unordered(const char* key) & noexcept; simdjson_inline simdjson_result type() noexcept; simdjson_inline simdjson_result is_scalar() noexcept; + simdjson_inline simdjson_result is_string() noexcept; simdjson_inline simdjson_result current_location() noexcept; simdjson_inline int32_t current_depth() const noexcept; simdjson_inline bool at_end() const noexcept; @@ -50922,6 +56913,7 @@ namespace simdjson { simdjson_inline simdjson_result raw_json_token() noexcept; simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; + simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; }; @@ -50954,8 +56946,9 @@ namespace simdjson { simdjson_inline simdjson_result get_bool() noexcept; simdjson_inline simdjson_result get_value() noexcept; simdjson_inline simdjson_result is_null() noexcept; - #if SIMDJSON_EXCEPTIONS + template ::value == false>::type> + explicit simdjson_inline operator T() noexcept(false); simdjson_inline operator haswell::ondemand::array() & noexcept(false); simdjson_inline operator haswell::ondemand::object() & noexcept(false); simdjson_inline operator uint64_t() noexcept(false); @@ -50979,6 +56972,7 @@ namespace simdjson { simdjson_inline simdjson_result find_field_unordered(const char* key) & noexcept; simdjson_inline simdjson_result type() noexcept; simdjson_inline simdjson_result is_scalar() noexcept; + simdjson_inline simdjson_result is_string() noexcept; simdjson_inline simdjson_result current_location() noexcept; simdjson_inline simdjson_result current_depth() const noexcept; simdjson_inline simdjson_result is_negative() noexcept; @@ -50989,6 +56983,7 @@ namespace simdjson { simdjson_inline simdjson_result raw_json_token() noexcept; simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; + simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; }; @@ -51122,10 +57117,9 @@ namespace simdjson { class iterator { public: using value_type = simdjson_result; - using reference = value_type; - + using reference = simdjson_result; + using pointer = void; using difference_type = std::ptrdiff_t; - using iterator_category = std::input_iterator_tag; /** @@ -51135,7 +57129,7 @@ namespace simdjson { /** * Get the current document (or error). */ - simdjson_inline simdjson_result operator*() noexcept; + simdjson_inline reference operator*() noexcept; /** * Advance to the next document (prefix). */ @@ -51317,7 +57311,7 @@ namespace simdjson { friend class document; friend class json_iterator; friend struct simdjson_result; - friend struct internal::simdjson_result_base; + friend struct simdjson::internal::simdjson_result_base; }; // document_stream } // namespace ondemand @@ -51377,12 +57371,39 @@ namespace simdjson { * This consumes the key: once you have called unescaped_key(), you cannot * call it again nor can you call key(). */ - simdjson_inline simdjson_warn_unused simdjson_result unescaped_key(bool allow_replacement) noexcept; + simdjson_inline simdjson_warn_unused simdjson_result unescaped_key(bool allow_replacement = false) noexcept; + /** + * Get the key as a string_view (for higher speed, consider raw_key). + * We deliberately use a more cumbersome name (unescaped_key) to force users + * to think twice about using it. The content is stored in the receiver. + * + * This consumes the key: once you have called unescaped_key(), you cannot + * call it again nor can you call key(). + */ + template + simdjson_inline simdjson_warn_unused error_code unescaped_key(string_type& receiver, bool allow_replacement = false) noexcept; /** * Get the key as a raw_json_string. Can be used for direct comparison with - * an unescaped C string: e.g., key() == "test". + * an unescaped C string: e.g., key() == "test". This does not count as + * consumption of the content: you can safely call it repeatedly. + * See escaped_key() for a similar function which returns + * a more convenient std::string_view result. */ simdjson_inline raw_json_string key() const noexcept; + /** + * Get the unprocessed key as a string_view. This includes the quotes and may include + * some spaces after the last quote. This does not count as + * consumption of the content: you can safely call it repeatedly. + * See escaped_key(). + */ + simdjson_inline std::string_view key_raw_json_token() const noexcept; + /** + * Get the key as a string_view. This does not include the quotes and + * the string is unprocessed key so it may contain escape characters + * (e.g., \uXXXX or \n). It does not count as a consumption of the content: + * you can safely call it repeatedly. Use unescaped_key() to get the unescaped key. + */ + simdjson_inline std::string_view escaped_key() const noexcept; /** * Get the field value. */ @@ -51414,7 +57435,11 @@ namespace simdjson { simdjson_inline simdjson_result() noexcept = default; simdjson_inline simdjson_result unescaped_key(bool allow_replacement = false) noexcept; + template + simdjson_inline error_code unescaped_key(string_type& receiver, bool allow_replacement = false) noexcept; simdjson_inline simdjson_result key() noexcept; + simdjson_inline simdjson_result key_raw_json_token() noexcept; + simdjson_inline simdjson_result escaped_key() noexcept; simdjson_inline simdjson_result value() noexcept; }; @@ -51482,6 +57507,8 @@ namespace simdjson { * key a single time. Doing object["mykey"].to_string() and then again object["mykey"].to_string() * is an error. * + * If you expect to have keys with escape characters, please review our documentation. + * * @param key The key to look up. * @returns The value of the field, or NO_SUCH_FIELD if the field is not in the object. */ @@ -51503,7 +57530,7 @@ namespace simdjson { * APIs assume this. Therefore, you must be explicit if you want to treat objects as out of order. * * Use find_field() if you are sure fields will be in order (or are willing to treat it as if the - * field wasn't there when they aren't). + * field was not there when they are not in order). * * If you have multiple fields with a matching key ({"x": 1, "x": 1}) be mindful * that only one field is returned. @@ -51519,6 +57546,8 @@ namespace simdjson { * You are expected to access keys only once. You should access the value corresponding to a key * a single time. Doing object["mykey"].to_string() and then again object["mykey"].to_string() is an error. * + * If you expect to have keys with escape characters, please review our documentation. + * * @param key The key to look up. * @returns The value of the field, or NO_SUCH_FIELD if the field is not in the object. */ @@ -51566,11 +57595,24 @@ namespace simdjson { */ inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; + /** + * Get the value associated with the given JSONPath expression. We only support + * JSONPath queries that trivially convertible to JSON Pointer queries: key + * names and array indices. + * + * @return The value associated with the given JSONPath expression, or: + * - INVALID_JSON_POINTER if the JSONPath to JSON Pointer conversion fails + * - NO_SUCH_FIELD if a field does not exist in an object + * - INDEX_OUT_OF_BOUNDS if an array index is larger than an array length + * - INCORRECT_TYPE if a non-integer is used to access an array + */ + inline simdjson_result at_path(std::string_view json_path) noexcept; + /** * Reset the iterator so that we are pointing back at the * beginning of the object. You should still consume values only once even if you - * can iterate through the object more than once. If you unescape a string within - * the object more than once, you have unsafe code. Note that rewinding an object + * can iterate through the object more than once. If you unescape a string or a key + * within the object more than once, you have unsafe code. Note that rewinding an object * means that you may need to reparse it anew: it is not a free operation. * * @returns true if the object contains some elements (not empty) @@ -51653,6 +57695,8 @@ namespace simdjson { simdjson_inline simdjson_result operator[](std::string_view key) & noexcept; simdjson_inline simdjson_result operator[](std::string_view key) && noexcept; simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; + simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; + inline simdjson_result reset() noexcept; inline simdjson_result is_empty() noexcept; inline simdjson_result count_fields() & noexcept; @@ -52026,6 +58070,65 @@ namespace simdjson { return child; } + inline std::string json_path_to_pointer_conversion(std::string_view json_path) { + if (json_path.empty() || (json_path.front() != '.' && + json_path.front() != '[')) { + return "-1"; // This is just a sentinel value, the caller should check for this and return an error. + } + + std::string result; + // Reserve space to reduce allocations, adjusting for potential increases due + // to escaping. + result.reserve(json_path.size() * 2); + + size_t i = 0; + + while (i < json_path.length()) { + if (json_path[i] == '.') { + result += '/'; + } + else if (json_path[i] == '[') { + result += '/'; + ++i; // Move past the '[' + while (i < json_path.length() && json_path[i] != ']') { + if (json_path[i] == '~') { + result += "~0"; + } + else if (json_path[i] == '/') { + result += "~1"; + } + else { + result += json_path[i]; + } + ++i; + } + if (i == json_path.length() || json_path[i] != ']') { + return "-1"; // Using sentinel value that will be handled as an error by the caller. + } + } + else { + if (json_path[i] == '~') { + result += "~0"; + } + else if (json_path[i] == '/') { + result += "~1"; + } + else { + result += json_path[i]; + } + } + ++i; + } + + return result; + } + + inline simdjson_result array::at_path(std::string_view json_path) noexcept { + auto json_pointer = json_path_to_pointer_conversion(json_path); + if (json_pointer == "-1") { return INVALID_JSON_POINTER; } + return at_pointer(json_pointer); + } + simdjson_inline simdjson_result array::at(size_t index) noexcept { size_t i = 0; for (auto value : *this) { @@ -52080,6 +58183,10 @@ namespace simdjson { if (error()) { return error(); } return first.at_pointer(json_pointer); } + simdjson_inline simdjson_result simdjson_result::at_path(std::string_view json_path) noexcept { + if (error()) { return error(); } + return first.at_path(json_path); + } simdjson_inline simdjson_result simdjson_result::raw_json() noexcept { if (error()) { return error(); } return first.raw_json(); @@ -52176,14 +58283,14 @@ namespace simdjson { /* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ /* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_ONDEMAND_DOCUMENT_INL_H */ /* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/array-inl.h" */ /* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/array_iterator.h" */ /* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/document.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/json_iterator-inl.h" */ /* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/json_type.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/object-inl.h" */ /* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/raw_json_string.h" */ /* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/value.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/array-inl.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/json_iterator-inl.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/object-inl.h" */ /* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/value_iterator-inl.h" */ /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ @@ -52339,22 +58446,26 @@ namespace simdjson { template<> simdjson_inline simdjson_result document::get() & noexcept { return get_bool(); } template<> simdjson_inline simdjson_result document::get() & noexcept { return get_value(); } - template<> simdjson_inline simdjson_result document::get() && noexcept { return get_raw_json_string(); } - template<> simdjson_inline simdjson_result document::get() && noexcept { return get_string(false); } - template<> simdjson_inline simdjson_result document::get() && noexcept { return std::forward(*this).get_double(); } - template<> simdjson_inline simdjson_result document::get() && noexcept { return std::forward(*this).get_uint64(); } - template<> simdjson_inline simdjson_result document::get() && noexcept { return std::forward(*this).get_int64(); } - template<> simdjson_inline simdjson_result document::get() && noexcept { return std::forward(*this).get_bool(); } - template<> simdjson_inline simdjson_result document::get() && noexcept { return get_value(); } + template<> simdjson_deprecated simdjson_inline simdjson_result document::get() && noexcept { return get_raw_json_string(); } + template<> simdjson_deprecated simdjson_inline simdjson_result document::get() && noexcept { return get_string(false); } + template<> simdjson_deprecated simdjson_inline simdjson_result document::get() && noexcept { return std::forward(*this).get_double(); } + template<> simdjson_deprecated simdjson_inline simdjson_result document::get() && noexcept { return std::forward(*this).get_uint64(); } + template<> simdjson_deprecated simdjson_inline simdjson_result document::get() && noexcept { return std::forward(*this).get_int64(); } + template<> simdjson_deprecated simdjson_inline simdjson_result document::get() && noexcept { return std::forward(*this).get_bool(); } + template<> simdjson_deprecated simdjson_inline simdjson_result document::get() && noexcept { return get_value(); } template simdjson_inline error_code document::get(T& out) & noexcept { return get().get(out); } - template simdjson_inline error_code document::get(T& out) && noexcept { + template simdjson_deprecated simdjson_inline error_code document::get(T& out) && noexcept { return std::forward(*this).get().get(out); } #if SIMDJSON_EXCEPTIONS + template + simdjson_deprecated simdjson_inline document::operator T() && noexcept(false) { return get(); } + template + simdjson_inline document::operator T() & noexcept(false) { return get(); } simdjson_inline document::operator array() & noexcept(false) { return get_array(); } simdjson_inline document::operator object() & noexcept(false) { return get_object(); } simdjson_inline document::operator uint64_t() noexcept(false) { return get_uint64(); } @@ -52439,6 +58550,13 @@ namespace simdjson { return !((this_type == json_type::array) || (this_type == json_type::object)); } + simdjson_inline simdjson_result document::is_string() noexcept { + json_type this_type; + auto error = type().get(this_type); + if (error) { return error; } + return (this_type == json_type::string); + } + simdjson_inline bool document::is_negative() noexcept { return get_root_value_iterator().is_root_negative(); } @@ -52458,7 +58576,7 @@ namespace simdjson { simdjson_inline simdjson_result document::raw_json_token() noexcept { auto _iter = get_root_value_iterator(); - return std::string_view(reinterpret_cast(_iter.peek_start()), _iter.peek_start_length()); + return std::string_view(reinterpret_cast(_iter.peek_start()), _iter.peek_root_length()); } simdjson_inline simdjson_result document::at_pointer(std::string_view json_pointer) noexcept { @@ -52479,6 +58597,23 @@ namespace simdjson { } } + simdjson_inline simdjson_result document::at_path(std::string_view json_path) noexcept { + rewind(); // Rewind the document each time at_pointer is called + if (json_path.empty()) { + return this->get_value(); + } + json_type t; + SIMDJSON_TRY(type().get(t)); + switch (t) { + case json_type::array: + return (*this).get_array().at_path(json_path); + case json_type::object: + return (*this).get_object().at_path(json_path); + default: + return INVALID_JSON_POINTER; + } + } + } // namespace ondemand } // namespace haswell } // namespace simdjson @@ -52617,7 +58752,7 @@ namespace simdjson { return first.get(); } template - simdjson_inline simdjson_result simdjson_result::get() && noexcept { + simdjson_deprecated simdjson_inline simdjson_result simdjson_result::get() && noexcept { if (error()) { return error(); } return std::forward(first).get(); } @@ -52633,7 +58768,7 @@ namespace simdjson { } template<> simdjson_inline simdjson_result simdjson_result::get() & noexcept = delete; - template<> simdjson_inline simdjson_result simdjson_result::get() && noexcept { + template<> simdjson_deprecated simdjson_inline simdjson_result simdjson_result::get() && noexcept { if (error()) { return error(); } return std::forward(first); } @@ -52654,6 +58789,10 @@ namespace simdjson { return first.is_scalar(); } + simdjson_inline simdjson_result simdjson_result::is_string() noexcept { + if (error()) { return error(); } + return first.is_string(); + } simdjson_inline bool simdjson_result::is_negative() noexcept { if (error()) { return error(); } @@ -52677,6 +58816,11 @@ namespace simdjson { #if SIMDJSON_EXCEPTIONS + template ::value == false>::type> + simdjson_inline simdjson_result::operator T() noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return first; + } simdjson_inline simdjson_result::operator haswell::ondemand::array() & noexcept(false) { if (error()) { throw simdjson_error(error()); } return first; @@ -52742,6 +58886,10 @@ namespace simdjson { return first.at_pointer(json_pointer); } + simdjson_inline simdjson_result simdjson_result::at_path(std::string_view json_path) noexcept { + if (error()) { return error(); } + return first.at_path(json_path); + } } // namespace simdjson @@ -52783,15 +58931,25 @@ namespace simdjson { simdjson_inline simdjson_result document_reference::get_bool() noexcept { return doc->get_root_value_iterator().get_root_bool(false); } simdjson_inline simdjson_result document_reference::get_value() noexcept { return doc->get_value(); } simdjson_inline simdjson_result document_reference::is_null() noexcept { return doc->get_root_value_iterator().is_root_null(false); } - + template<> simdjson_inline simdjson_result document_reference::get() & noexcept { return get_array(); } + template<> simdjson_inline simdjson_result document_reference::get() & noexcept { return get_object(); } + template<> simdjson_inline simdjson_result document_reference::get() & noexcept { return get_raw_json_string(); } + template<> simdjson_inline simdjson_result document_reference::get() & noexcept { return get_string(false); } + template<> simdjson_inline simdjson_result document_reference::get() & noexcept { return get_double(); } + template<> simdjson_inline simdjson_result document_reference::get() & noexcept { return get_uint64(); } + template<> simdjson_inline simdjson_result document_reference::get() & noexcept { return get_int64(); } + template<> simdjson_inline simdjson_result document_reference::get() & noexcept { return get_bool(); } + template<> simdjson_inline simdjson_result document_reference::get() & noexcept { return get_value(); } #if SIMDJSON_EXCEPTIONS + template + simdjson_inline document_reference::operator T() noexcept(false) { return get(); } simdjson_inline document_reference::operator array() & noexcept(false) { return array(*doc); } simdjson_inline document_reference::operator object() & noexcept(false) { return object(*doc); } simdjson_inline document_reference::operator uint64_t() noexcept(false) { return get_uint64(); } simdjson_inline document_reference::operator int64_t() noexcept(false) { return get_int64(); } simdjson_inline document_reference::operator double() noexcept(false) { return get_double(); } simdjson_inline document_reference::operator std::string_view() noexcept(false) { return std::string_view(*doc); } - simdjson_inline document_reference::operator raw_json_string() noexcept(false) { return raw_json_string(*doc); } + simdjson_inline document_reference::operator raw_json_string() noexcept(false) { return get_raw_json_string(); } simdjson_inline document_reference::operator bool() noexcept(false) { return get_bool(); } simdjson_inline document_reference::operator value() noexcept(false) { return value(*doc); } #endif @@ -52808,6 +58966,7 @@ namespace simdjson { simdjson_inline simdjson_result document_reference::find_field_unordered(const char* key) & noexcept { return doc->find_field_unordered(key); } simdjson_inline simdjson_result document_reference::type() noexcept { return doc->type(); } simdjson_inline simdjson_result document_reference::is_scalar() noexcept { return doc->is_scalar(); } + simdjson_inline simdjson_result document_reference::is_string() noexcept { return doc->is_string(); } simdjson_inline simdjson_result document_reference::current_location() noexcept { return doc->current_location(); } simdjson_inline int32_t document_reference::current_depth() const noexcept { return doc->current_depth(); } simdjson_inline bool document_reference::is_negative() noexcept { return doc->is_negative(); } @@ -52816,6 +58975,7 @@ namespace simdjson { simdjson_inline simdjson_result document_reference::get_number() noexcept { return doc->get_root_value_iterator().get_root_number(false); } simdjson_inline simdjson_result document_reference::raw_json_token() noexcept { return doc->raw_json_token(); } simdjson_inline simdjson_result document_reference::at_pointer(std::string_view json_pointer) noexcept { return doc->at_pointer(json_pointer); } + simdjson_inline simdjson_result document_reference::at_path(std::string_view json_path) noexcept { return doc->at_path(json_path); } simdjson_inline simdjson_result document_reference::raw_json() noexcept { return doc->raw_json(); } simdjson_inline document_reference::operator document& () const noexcept { return *doc; } @@ -52947,6 +59107,10 @@ namespace simdjson { if (error()) { return error(); } return first.is_scalar(); } + simdjson_inline simdjson_result simdjson_result::is_string() noexcept { + if (error()) { return error(); } + return first.is_string(); + } simdjson_inline simdjson_result simdjson_result::is_negative() noexcept { if (error()) { return error(); } return first.is_negative(); @@ -52964,6 +59128,11 @@ namespace simdjson { return first.get_number(); } #if SIMDJSON_EXCEPTIONS + template ::value == false>::type> + simdjson_inline simdjson_result::operator T() noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return first; + } simdjson_inline simdjson_result::operator haswell::ondemand::array() & noexcept(false) { if (error()) { throw simdjson_error(error()); } return first; @@ -53017,6 +59186,12 @@ namespace simdjson { return first.at_pointer(json_pointer); } + simdjson_inline simdjson_result simdjson_result::at_path(std::string_view json_path) noexcept { + if (error()) { + return error(); + } + return first.at_path(json_path); + } } // namespace simdjson @@ -53370,9 +59545,18 @@ namespace simdjson { depth--; break; default: // Scalar value document - // TODO: Remove any trailing whitespaces + // TODO: We could remove trailing whitespaces // This returns a string spanning from start of value to the beginning of the next document (excluded) - return std::string_view(reinterpret_cast(stream->buf) + current_index(), stream->parser->implementation->structural_indexes[++cur_struct_index] - current_index() - 1); + { + auto next_index = stream->parser->implementation->structural_indexes[++cur_struct_index]; + // normally the length would be next_index - current_index() - 1, except for the last document + size_t svlen = next_index - current_index(); + const char* start = reinterpret_cast(stream->buf) + current_index(); + while (svlen > 1 && (std::isspace(start[svlen - 1]) || start[svlen - 1] == '\0')) { + svlen--; + } + return std::string_view(start, svlen); + } } cur_struct_index++; } @@ -53467,7 +59651,7 @@ namespace simdjson { namespace haswell { namespace ondemand { - // clang 6 doesn't think the default constructor can be noexcept, so we make it explicit + // clang 6 does not think the default constructor can be noexcept, so we make it explicit simdjson_inline field::field() noexcept : std::pair() {} simdjson_inline field::field(raw_json_string key, ondemand::value&& value) noexcept @@ -53493,11 +59677,32 @@ namespace simdjson { return answer; } + template + simdjson_inline simdjson_warn_unused error_code field::unescaped_key(string_type& receiver, bool allow_replacement) noexcept { + std::string_view key; + SIMDJSON_TRY(unescaped_key(allow_replacement).get(key)); + receiver = key; + return SUCCESS; + } + simdjson_inline raw_json_string field::key() const noexcept { SIMDJSON_ASSUME(first.buf != nullptr); // We would like to call .alive() by Visual Studio won't let us. return first; } + + simdjson_inline std::string_view field::key_raw_json_token() const noexcept { + SIMDJSON_ASSUME(first.buf != nullptr); // We would like to call .alive() by Visual Studio won't let us. + return std::string_view(reinterpret_cast(first.buf - 1), second.iter._json_iter->token.peek(-1) - first.buf + 1); + } + + simdjson_inline std::string_view field::escaped_key() const noexcept { + SIMDJSON_ASSUME(first.buf != nullptr); // We would like to call .alive() by Visual Studio won't let us. + auto end_quote = second.iter._json_iter->token.peek(-1); + while (*end_quote != '"') end_quote--; + return std::string_view(reinterpret_cast(first.buf), end_quote - first.buf); + } + simdjson_inline value& field::value() & noexcept { return second; } @@ -53531,10 +59736,28 @@ namespace simdjson { if (error()) { return error(); } return first.key(); } + + simdjson_inline simdjson_result simdjson_result::key_raw_json_token() noexcept { + if (error()) { return error(); } + return first.key_raw_json_token(); + } + + simdjson_inline simdjson_result simdjson_result::escaped_key() noexcept { + if (error()) { return error(); } + return first.escaped_key(); + } + simdjson_inline simdjson_result simdjson_result::unescaped_key(bool allow_replacement) noexcept { if (error()) { return error(); } return first.unescaped_key(allow_replacement); } + + template + simdjson_inline error_code simdjson_result::unescaped_key(string_type& receiver, bool allow_replacement) noexcept { + if (error()) { return error(); } + return first.unescaped_key(receiver, allow_replacement); + } + simdjson_inline simdjson_result simdjson_result::value() noexcept { if (error()) { return error(); } return std::move(first.value()); @@ -53602,6 +59825,23 @@ namespace simdjson { #endif } +#ifdef SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON + simdjson_inline json_iterator::json_iterator(const uint8_t* buf, ondemand::parser* _parser, bool streaming) noexcept + : token(buf, &_parser->implementation->structural_indexes[0]), + parser{ _parser }, + _string_buf_loc{ parser->string_buf.get() }, + _depth{ 1 }, + _root{ parser->implementation->structural_indexes.get() }, + _streaming{ streaming } + + { + logger::log_headers(); +#if SIMDJSON_CHECK_EOF + assert_more_tokens(); +#endif + } +#endif // SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON + inline void json_iterator::rewind() noexcept { token.set_position(root_position()); logger::log_headers(); // We start again @@ -53835,6 +60075,12 @@ namespace simdjson { #endif // SIMDJSON_CHECK_EOF return token.peek_length(position); } + simdjson_inline uint32_t json_iterator::peek_root_length(token_position position) const noexcept { +#if SIMDJSON_CHECK_EOF + assert_valid_position(position); +#endif // SIMDJSON_CHECK_EOF + return token.peek_root_length(position); + } simdjson_inline token_position json_iterator::last_position() const noexcept { // The following line fails under some compilers... @@ -53880,11 +60126,23 @@ namespace simdjson { } simdjson_inline simdjson_result json_iterator::unescape(raw_json_string in, bool allow_replacement) noexcept { +#if SIMDJSON_DEVELOPMENT_CHECKS + auto result = parser->unescape(in, _string_buf_loc, allow_replacement); + SIMDJSON_ASSUME(!parser->string_buffer_overflow(_string_buf_loc)); + return result; +#else return parser->unescape(in, _string_buf_loc, allow_replacement); +#endif } simdjson_inline simdjson_result json_iterator::unescape_wobbly(raw_json_string in) noexcept { +#if SIMDJSON_DEVELOPMENT_CHECKS + auto result = parser->unescape_wobbly(in, _string_buf_loc); + SIMDJSON_ASSUME(!parser->string_buffer_overflow(_string_buf_loc)); + return result; +#else return parser->unescape_wobbly(in, _string_buf_loc); +#endif } simdjson_inline void json_iterator::reenter_child(token_position position, depth_t child_depth) noexcept { @@ -54009,7 +60267,6 @@ namespace simdjson { return get_uint64(); } - simdjson_inline bool number::is_int64() const noexcept { return get_number_type() == number_type::signed_integer; } @@ -54477,6 +60734,14 @@ namespace simdjson { return child; } + inline simdjson_result object::at_path(std::string_view json_path) noexcept { + auto json_pointer = json_path_to_pointer_conversion(json_path); + if (json_pointer == "-1") { + return INVALID_JSON_POINTER; + } + return at_pointer(json_pointer); + } + simdjson_inline simdjson_result object::count_fields() & noexcept { size_t count{ 0 }; // Important: we do not consume any of the values. @@ -54549,6 +60814,14 @@ namespace simdjson { return first.at_pointer(json_pointer); } + simdjson_inline simdjson_result simdjson_result::at_path( + std::string_view json_path) noexcept { + if (error()) { + return error(); + } + return first.at_path(json_path); + } + inline simdjson_result simdjson_result::reset() noexcept { if (error()) { return error(); } return first.reset(); @@ -54760,6 +61033,11 @@ namespace simdjson { _max_depth = new_max_depth; return SUCCESS; } +#if SIMDJSON_DEVELOPMENT_CHECKS + simdjson_inline simdjson_warn_unused bool parser::string_buffer_overflow(const uint8_t* string_buf_loc) const noexcept { + return (string_buf_loc < string_buf.get()) || (size_t(string_buf_loc - string_buf.get()) >= capacity()); + } +#endif simdjson_warn_unused simdjson_inline simdjson_result parser::iterate(padded_string_view json) & noexcept { if (json.padding() < SIMDJSON_PADDING) { return INSUFFICIENT_PADDING; } @@ -54776,6 +61054,27 @@ namespace simdjson { return document::start({ reinterpret_cast(json.data()), this }); } +#ifdef SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON + simdjson_warn_unused simdjson_inline simdjson_result parser::iterate_allow_incomplete_json(padded_string_view json) & noexcept { + if (json.padding() < SIMDJSON_PADDING) { return INSUFFICIENT_PADDING; } + + json.remove_utf8_bom(); + + // Allocate if needed + if (capacity() < json.length() || !string_buf) { + SIMDJSON_TRY(allocate(json.length(), max_depth())); + } + + // Run stage 1. + const simdjson::error_code err = implementation->stage1(reinterpret_cast(json.data()), json.length(), stage1_mode::regular); + if (err) { + if (err != UNCLOSED_STRING) + return err; + } + return document::start({ reinterpret_cast(json.data()), this, true }); + } +#endif // SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON + simdjson_warn_unused simdjson_inline simdjson_result parser::iterate(const char* json, size_t len, size_t allocated) & noexcept { return iterate(padded_string_view(json, len, allocated)); } @@ -54847,13 +61146,13 @@ namespace simdjson { return iterate_many(s.data(), s.length(), batch_size, allow_comma_separated); } - simdjson_inline size_t parser::capacity() const noexcept { + simdjson_pure simdjson_inline size_t parser::capacity() const noexcept { return _capacity; } - simdjson_inline size_t parser::max_capacity() const noexcept { + simdjson_pure simdjson_inline size_t parser::max_capacity() const noexcept { return _max_capacity; } - simdjson_inline size_t parser::max_depth() const noexcept { + simdjson_pure simdjson_inline size_t parser::max_depth() const noexcept { return _max_depth; } @@ -55402,6 +61701,11 @@ namespace simdjson { return *(position + 1) - *position; } + simdjson_inline uint32_t token_iterator::peek_root_length(token_position position) const noexcept { + return *(position + 2) - *(position) > *(position + 1) - *(position) ? + *(position + 1) - *(position) + : *(position + 2) - *(position); + } simdjson_inline const uint8_t* token_iterator::peek(int32_t delta) const noexcept { return &buf[*(_position + delta)]; } @@ -55551,6 +61855,10 @@ namespace simdjson { } #if SIMDJSON_EXCEPTIONS + template + simdjson_inline value::operator T() noexcept(false) { + return get(); + } simdjson_inline value::operator array() noexcept(false) { return get_array(); } @@ -55637,6 +61945,14 @@ namespace simdjson { return !((this_type == json_type::array) || (this_type == json_type::object)); } + simdjson_inline simdjson_result value::is_string() noexcept { + json_type this_type; + auto error = type().get(this_type); + if (error) { return error; } + return (this_type == json_type::string); + } + + simdjson_inline bool value::is_negative() noexcept { return iter.is_negative(); } @@ -55683,6 +61999,26 @@ namespace simdjson { return iter.json_iter().depth(); } + inline bool is_pointer_well_formed(std::string_view json_pointer) noexcept { + if (simdjson_unlikely(json_pointer.empty())) { // can't be + return false; + } + if (simdjson_unlikely(json_pointer[0] != '/')) { + return false; + } + size_t escape = json_pointer.find('~'); + if (escape == std::string_view::npos) { + return true; + } + if (escape == json_pointer.size() - 1) { + return false; + } + if (json_pointer[escape + 1] != '0' && json_pointer[escape + 1] != '1') { + return false; + } + return true; + } + simdjson_inline simdjson_result value::at_pointer(std::string_view json_pointer) noexcept { json_type t; SIMDJSON_TRY(type().get(t)); @@ -55692,6 +62028,23 @@ namespace simdjson { return (*this).get_array().at_pointer(json_pointer); case json_type::object: return (*this).get_object().at_pointer(json_pointer); + default: + // a non-empty string can be invalid, or accessing a primitive (issue 2154) + if (is_pointer_well_formed(json_pointer)) { + return NO_SUCH_FIELD; + } + return INVALID_JSON_POINTER; + } + } + + simdjson_inline simdjson_result value::at_path(std::string_view json_path) noexcept { + json_type t; + SIMDJSON_TRY(type().get(t)); + switch (t) { + case json_type::array: + return (*this).get_array().at_path(json_path); + case json_type::object: + return (*this).get_object().at_path(json_path); default: return INVALID_JSON_POINTER; } @@ -55850,6 +62203,10 @@ namespace simdjson { if (error()) { return error(); } return first.is_scalar(); } + simdjson_inline simdjson_result simdjson_result::is_string() noexcept { + if (error()) { return error(); } + return first.is_string(); + } simdjson_inline simdjson_result simdjson_result::is_negative() noexcept { if (error()) { return error(); } return first.is_negative(); @@ -55867,6 +62224,11 @@ namespace simdjson { return first.get_number(); } #if SIMDJSON_EXCEPTIONS + template + simdjson_inline simdjson_result::operator T() noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return static_cast(first); + } simdjson_inline simdjson_result::operator haswell::ondemand::array() noexcept(false) { if (error()) { throw simdjson_error(error()); } return first; @@ -55921,11 +62283,22 @@ namespace simdjson { return first.current_depth(); } - simdjson_inline simdjson_result simdjson_result::at_pointer(std::string_view json_pointer) noexcept { - if (error()) { return error(); } + simdjson_inline simdjson_result simdjson_result::at_pointer( + std::string_view json_pointer) noexcept { + if (error()) { + return error(); + } return first.at_pointer(json_pointer); } + simdjson_inline simdjson_result simdjson_result::at_path( + std::string_view json_path) noexcept { + if (error()) { + return error(); + } + return first.at_path(json_path); + } + } // namespace simdjson #endif // SIMDJSON_GENERIC_ONDEMAND_VALUE_INL_H @@ -55940,9 +62313,9 @@ namespace simdjson { /* amalgamation skipped (editor-only): #include "simdjson/generic/atomparsing.h" */ /* amalgamation skipped (editor-only): #include "simdjson/generic/numberparsing.h" */ /* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/json_iterator.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/value_iterator.h" */ /* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/json_type-inl.h" */ /* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/raw_json_string-inl.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/value_iterator.h" */ /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ namespace simdjson { @@ -56072,7 +62445,7 @@ namespace simdjson { else if (!is_open()) { #if SIMDJSON_DEVELOPMENT_CHECKS // If we're past the end of the object, we're being iterated out of order. - // Note: this isn't perfect detection. It's possible the user is inside some other object; if so, + // Note: this is not perfect detection. It's possible the user is inside some other object; if so, // this object iterator will blithely scan that object for fields. if (_json_iter->depth() < depth() - 1) { return OUT_OF_ORDER_ITERATION; } #endif @@ -56186,7 +62559,7 @@ namespace simdjson { #if SIMDJSON_DEVELOPMENT_CHECKS // If we're past the end of the object, we're being iterated out of order. - // Note: this isn't perfect detection. It's possible the user is inside some other object; if so, + // Note: this is not perfect detection. It's possible the user is inside some other object; if so, // this object iterator will blithely scan that object for fields. if (_json_iter->depth() < depth() - 1) { return OUT_OF_ORDER_ITERATION; } #endif @@ -56529,7 +62902,7 @@ namespace simdjson { } simdjson_inline simdjson_result value_iterator::is_root_integer(bool check_trailing) noexcept { - auto max_len = peek_start_length(); + auto max_len = peek_root_length(); auto json = peek_root_scalar("is_root_integer"); uint8_t tmpbuf[20 + 1 + 1]{}; // <20 digits> is the longest possible unsigned integer tmpbuf[20 + 1] = '\0'; // make sure that buffer is always null terminated. @@ -56545,7 +62918,7 @@ namespace simdjson { } simdjson_inline simdjson_result value_iterator::get_root_number_type(bool check_trailing) noexcept { - auto max_len = peek_start_length(); + auto max_len = peek_root_length(); auto json = peek_root_scalar("number"); // Per https://www.exploringbinary.com/maximum-number-of-decimal-digits-in-binary-floating-point-numbers/, // 1074 is the maximum number of significant fractional digits. Add 8 more digits for the biggest @@ -56553,7 +62926,12 @@ namespace simdjson { uint8_t tmpbuf[1074 + 8 + 1 + 1]; tmpbuf[1074 + 8 + 1] = '\0'; // make sure that buffer is always null terminated. if (!_json_iter->copy_to_buffer(json, max_len, tmpbuf, 1074 + 8 + 1)) { - logger::log_error(*_json_iter, start_position(), depth(), "Root number more than 1082 characters"); + if (numberparsing::check_if_integer(json, max_len)) { + if (check_trailing && !_json_iter->is_single_token()) { return TRAILING_CONTENT; } + logger::log_error(*_json_iter, start_position(), depth(), "Found big integer"); + return number_type::big_integer; + } + logger::log_error(*_json_iter, start_position(), depth(), "Root number more than 1082 characters and not a big integer"); return NUMBER_ERROR; } auto answer = numberparsing::get_number_type(tmpbuf); @@ -56561,15 +62939,21 @@ namespace simdjson { return answer; } simdjson_inline simdjson_result value_iterator::get_root_number(bool check_trailing) noexcept { - auto max_len = peek_start_length(); + auto max_len = peek_root_length(); auto json = peek_root_scalar("number"); // Per https://www.exploringbinary.com/maximum-number-of-decimal-digits-in-binary-floating-point-numbers/, // 1074 is the maximum number of significant fractional digits. Add 8 more digits for the biggest // number: -0.e-308. + // NOTE: the current approach doesn't work for very big integer numbers containing more than 1074 digits. uint8_t tmpbuf[1074 + 8 + 1 + 1]; tmpbuf[1074 + 8 + 1] = '\0'; // make sure that buffer is always null terminated. if (!_json_iter->copy_to_buffer(json, max_len, tmpbuf, 1074 + 8 + 1)) { - logger::log_error(*_json_iter, start_position(), depth(), "Root number more than 1082 characters"); + if (numberparsing::check_if_integer(json, max_len)) { + if (check_trailing && !_json_iter->is_single_token()) { return TRAILING_CONTENT; } + logger::log_error(*_json_iter, start_position(), depth(), "Found big integer"); + return BIGINT_ERROR; + } + logger::log_error(*_json_iter, start_position(), depth(), "Root number more than 1082 characters and not a big integer"); return NUMBER_ERROR; } number num; @@ -56601,7 +62985,7 @@ namespace simdjson { return raw_json_string(json + 1); } simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_root_uint64(bool check_trailing) noexcept { - auto max_len = peek_start_length(); + auto max_len = peek_root_length(); auto json = peek_root_scalar("uint64"); uint8_t tmpbuf[20 + 1 + 1]{}; // <20 digits> is the longest possible unsigned integer tmpbuf[20 + 1] = '\0'; // make sure that buffer is always null terminated. @@ -56617,7 +63001,7 @@ namespace simdjson { return result; } simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_root_uint64_in_string(bool check_trailing) noexcept { - auto max_len = peek_start_length(); + auto max_len = peek_root_length(); auto json = peek_root_scalar("uint64"); uint8_t tmpbuf[20 + 1 + 1]{}; // <20 digits> is the longest possible unsigned integer tmpbuf[20 + 1] = '\0'; // make sure that buffer is always null terminated. @@ -56633,7 +63017,7 @@ namespace simdjson { return result; } simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_root_int64(bool check_trailing) noexcept { - auto max_len = peek_start_length(); + auto max_len = peek_root_length(); auto json = peek_root_scalar("int64"); uint8_t tmpbuf[20 + 1 + 1]; // -<19 digits> is the longest possible integer tmpbuf[20 + 1] = '\0'; // make sure that buffer is always null terminated. @@ -56650,7 +63034,7 @@ namespace simdjson { return result; } simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_root_int64_in_string(bool check_trailing) noexcept { - auto max_len = peek_start_length(); + auto max_len = peek_root_length(); auto json = peek_root_scalar("int64"); uint8_t tmpbuf[20 + 1 + 1]; // -<19 digits> is the longest possible integer tmpbuf[20 + 1] = '\0'; // make sure that buffer is always null terminated. @@ -56667,7 +63051,7 @@ namespace simdjson { return result; } simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_root_double(bool check_trailing) noexcept { - auto max_len = peek_start_length(); + auto max_len = peek_root_length(); auto json = peek_root_scalar("double"); // Per https://www.exploringbinary.com/maximum-number-of-decimal-digits-in-binary-floating-point-numbers/, // 1074 is the maximum number of significant fractional digits. Add 8 more digits for the biggest @@ -56687,7 +63071,7 @@ namespace simdjson { } simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_root_double_in_string(bool check_trailing) noexcept { - auto max_len = peek_start_length(); + auto max_len = peek_root_length(); auto json = peek_root_scalar("double"); // Per https://www.exploringbinary.com/maximum-number-of-decimal-digits-in-binary-floating-point-numbers/, // 1074 is the maximum number of significant fractional digits. Add 8 more digits for the biggest @@ -56706,7 +63090,7 @@ namespace simdjson { return result; } simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_root_bool(bool check_trailing) noexcept { - auto max_len = peek_start_length(); + auto max_len = peek_root_length(); auto json = peek_root_scalar("bool"); uint8_t tmpbuf[5 + 1 + 1]; // +1 for null termination tmpbuf[5 + 1] = '\0'; // make sure that buffer is always null terminated. @@ -56719,7 +63103,7 @@ namespace simdjson { return result; } simdjson_inline simdjson_result value_iterator::is_root_null(bool check_trailing) noexcept { - auto max_len = peek_start_length(); + auto max_len = peek_root_length(); auto json = peek_root_scalar("null"); bool result = (max_len >= 4 && !atomparsing::str4ncmp(json, "null") && (max_len == 4 || jsoncharutils::is_structural_or_whitespace(json[4]))); @@ -56791,6 +63175,9 @@ namespace simdjson { simdjson_inline uint32_t value_iterator::peek_start_length() const noexcept { return _json_iter->peek_length(start_position()); } + simdjson_inline uint32_t value_iterator::peek_root_length() const noexcept { + return _json_iter->peek_root_length(start_position()); + } simdjson_inline const uint8_t* value_iterator::peek_scalar(const char* type) noexcept { logger::log_value(*_json_iter, start_position(), depth(), type); @@ -57336,10 +63723,10 @@ namespace simdjson { struct simd8 : base8 { static simdjson_inline simd8 splat(bool _value) { return _mm512_set1_epi8(uint8_t(-(!!_value))); } - simdjson_inline simd8() : base8() {} - simdjson_inline simd8(const __m512i _value) : base8(_value) {} + simdjson_inline simd8() : base8() {} + simdjson_inline simd8(const __m512i _value) : base8(_value) {} // Splat constructor - simdjson_inline simd8(bool _value) : base8(splat(_value)) {} + simdjson_inline simd8(bool _value) : base8(splat(_value)) {} simdjson_inline bool any() const { return !!_mm512_test_epi8_mask(*this, *this); } simdjson_inline simd8 operator~() const { return *this ^ true; } }; @@ -57702,13 +64089,13 @@ namespace simdjson { simdjson_inline internal::value128 full_multiplication(uint64_t value1, uint64_t value2) { internal::value128 answer; #if SIMDJSON_REGULAR_VISUAL_STUDIO || SIMDJSON_IS_32BITS -#ifdef _M_ARM64 +#if SIMDJSON_IS_ARM64 // ARM64 has native support for 64-bit multiplications, no need to emultate answer.high = __umulh(value1, value2); answer.low = value1 * value2; #else answer.low = _umul128(value1, value2, &answer.high); // _umul128 not available on ARM64 -#endif // _M_ARM64 +#endif // SIMDJSON_IS_ARM64 #else // SIMDJSON_REGULAR_VISUAL_STUDIO || SIMDJSON_IS_32BITS __uint128_t r = (static_cast<__uint128_t>(value1)) * value2; answer.low = uint64_t(r); @@ -58162,6 +64549,7 @@ namespace simdjson { simdjson_inline simdjson_result parse_bool(const uint8_t* json) const noexcept; simdjson_inline const uint8_t* peek_start() const noexcept; simdjson_inline uint32_t peek_start_length() const noexcept; + simdjson_inline uint32_t peek_root_length() const noexcept; /** * The general idea of the advance_... methods and the peek_* methods @@ -58256,6 +64644,7 @@ namespace simdjson { friend class object; friend class array; friend class value; + friend class field; }; // value_iterator } // namespace ondemand @@ -58316,12 +64705,13 @@ namespace simdjson { * @returns INCORRECT_TYPE If the JSON value is not the given type. */ template simdjson_inline simdjson_result get() noexcept { - // Unless the simdjson library provides an inline implementation, calling this method should + // Unless the simdjson library or the user provides an inline implementation, calling this method should // immediately fail. static_assert(!sizeof(T), "The get method with given type is not implemented by the simdjson library. " "The supported types are ondemand::object, ondemand::array, raw_json_string, std::string_view, uint64_t, " "int64_t, double, and bool. We recommend you use get_double(), get_bool(), get_uint64(), get_int64(), " - " get_object(), get_array(), get_raw_json_string(), or get_string() instead of the get template."); + " get_object(), get_array(), get_raw_json_string(), or get_string() instead of the get template." + " You may also add support for custom types, see our documentation."); } /** @@ -58474,6 +64864,17 @@ namespace simdjson { simdjson_inline simdjson_result is_null() noexcept; #if SIMDJSON_EXCEPTIONS + /** + * Cast this JSON value to an instance of type T. The programmer is responsible for + * providing an implementation of get for the type T, if T is not one of the types + * supported by the library (object, array, raw_json_string, string_view, uint64_t, etc.). + * + * See https://github.com/simdjson/simdjson/blob/master/doc/basics.md#adding-support-for-custom-types + * + * @returns An instance of type T + */ + template + explicit simdjson_inline operator T() noexcept(false); /** * Cast this JSON value to an array. * @@ -58637,7 +65038,7 @@ namespace simdjson { * that only one field is returned. * * Use find_field() if you are sure fields will be in order (or are willing to treat it as if the - * field wasn't there when they aren't). + * field as not there when they are not in order). * * @param key The key to look up. * @returns The value of the field, or NO_SUCH_FIELD if the field is not in the object. @@ -58673,6 +65074,13 @@ namespace simdjson { * @error TAPE_ERROR when the JSON value is a bad token like "}" "," or "alse". */ simdjson_inline simdjson_result is_scalar() noexcept; + /** + * Checks whether the value is a string. + * + * @returns true if the type is string + * @error TAPE_ERROR when the JSON value is a bad token like "}" "," or "alse". + */ + simdjson_inline simdjson_result is_string() noexcept; /** * Checks whether the value is a negative number. @@ -58705,10 +65113,12 @@ namespace simdjson { * get_number().get_number_type(). * * get_number_type() is number_type::unsigned_integer if we have - * an integer greater or equal to 9223372036854775808 + * an integer greater or equal to 9223372036854775808. * get_number_type() is number_type::signed_integer if we have an - * integer that is less than 9223372036854775808 - * Otherwise, get_number_type() has value number_type::floating_point_number + * integer that is less than 9223372036854775808. + * get_number_type() is number_type::big_integer for integers that do not fit in 64 bits, + * in which case the digit_count is set to the length of the big integer string. + * Otherwise, get_number_type() has value number_type::floating_point_number. * * This function requires processing the number string, but it is expected * to be faster than get_number().get_number_type() because it is does not @@ -58735,6 +65145,8 @@ namespace simdjson { * You can recover the value by calling number.get_uint64() and you * have that number.is_uint64() is true. * + * For integers that do not fit in 64 bits, the function returns BIGINT_ERROR error code. + * * Otherwise, number.get_number_type() has value number_type::floating_point_number * and we have a binary64 number. * You can recover the value by calling number.get_double() and you @@ -58750,7 +65162,6 @@ namespace simdjson { */ simdjson_warn_unused simdjson_inline simdjson_result get_number() noexcept; - /** * Get the raw JSON for this token. * @@ -58848,6 +65259,20 @@ namespace simdjson { */ simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; + /** + * Get the value associated with the given JSONPath expression. We only support + * JSONPath queries that trivially convertible to JSON Pointer queries: key + * names and array indices. + * + * @return The value associated with the given JSONPath expression, or: + * - INVALID_JSON_POINTER if the JSONPath to JSON Pointer conversion fails + * - NO_SUCH_FIELD if a field does not exist in an object + * - INDEX_OUT_OF_BOUNDS if an array index is larger than an array length + * - INCORRECT_TYPE if a non-integer is used to access an array + */ + simdjson_inline simdjson_result at_path(std::string_view at_path) noexcept; + + protected: /** * Create a value. @@ -58887,6 +65312,7 @@ namespace simdjson { friend class object; friend struct simdjson_result; friend struct simdjson_result; + friend class field; }; } // namespace ondemand @@ -58924,6 +65350,8 @@ namespace simdjson { template simdjson_inline error_code get(T& out) noexcept; #if SIMDJSON_EXCEPTIONS + template + explicit simdjson_inline operator T() noexcept(false); simdjson_inline operator icelake::ondemand::array() noexcept(false); simdjson_inline operator icelake::ondemand::object() noexcept(false); simdjson_inline operator uint64_t() noexcept(false); @@ -58977,7 +65405,7 @@ namespace simdjson { * APIs assume this. Therefore, you must be explicit if you want to treat objects as out of order. * * Use find_field() if you are sure fields will be in order (or are willing to treat it as if the - * field wasn't there when they aren't). + * field as not there when they are not in order). * * @param key The key to look up. * @returns The value of the field, or NO_SUCH_FIELD if the field is not in the object. @@ -58999,6 +65427,7 @@ namespace simdjson { */ simdjson_inline simdjson_result type() noexcept; simdjson_inline simdjson_result is_scalar() noexcept; + simdjson_inline simdjson_result is_string() noexcept; simdjson_inline simdjson_result is_negative() noexcept; simdjson_inline simdjson_result is_integer() noexcept; simdjson_inline simdjson_result get_number_type() noexcept; @@ -59013,6 +65442,7 @@ namespace simdjson { /** @copydoc simdjson_inline int32_t current_depth() const noexcept */ simdjson_inline simdjson_result current_depth() const noexcept; simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; + simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; }; } // namespace simdjson @@ -59131,7 +65561,7 @@ namespace simdjson { * 1 = next token, -1 = prev token. * * TODO consider a string_view, assuming the length will get stripped out by the optimizer when - * it isn't used ... + * it is not used... */ simdjson_inline const uint8_t* peek(int32_t delta = 0) const noexcept; /** @@ -59161,7 +65591,14 @@ namespace simdjson { * @param position The position of the token. */ simdjson_inline uint32_t peek_length(token_position position) const noexcept; - + /** + * Get the maximum length of the JSON text for a root token. + * + * The length will include any whitespace at the end of the token. + * + * @param position The position of the token (start of the document). + */ + simdjson_inline uint32_t peek_root_length(token_position position) const noexcept; /** * Return the current index. */ @@ -59384,7 +65821,7 @@ namespace simdjson { * @param delta The relative position of the token to retrieve. e.g. 0 = next token, -1 = prev token. * * TODO consider a string_view, assuming the length will get stripped out by the optimizer when - * it isn't used ... + * it is not used ... */ simdjson_inline const uint8_t* peek(int32_t delta = 0) const noexcept; /** @@ -59412,7 +65849,7 @@ namespace simdjson { * @param position The position of the token to retrieve. * * TODO consider a string_view, assuming the length will get stripped out by the optimizer when - * it isn't used ... + * it is not used ... */ simdjson_inline const uint8_t* peek(token_position position) const noexcept; /** @@ -59423,13 +65860,21 @@ namespace simdjson { * @param position The position of the token to retrieve. */ simdjson_inline uint32_t peek_length(token_position position) const noexcept; + /** + * Get the maximum length of the JSON text for the current root token. + * + * The length will include any whitespace at the end of the token. + * + * @param position The position of the token to retrieve. + */ + simdjson_inline uint32_t peek_root_length(token_position position) const noexcept; /** * Get the JSON text for the last token in the document. * * This is not null-terminated; it is a view into the JSON. * * TODO consider a string_view, assuming the length will get stripped out by the optimizer when - * it isn't used ... + * it is not used ... */ simdjson_inline const uint8_t* peek_last() const noexcept; @@ -59523,6 +65968,9 @@ namespace simdjson { inline bool balanced() const noexcept; protected: simdjson_inline json_iterator(const uint8_t* buf, ondemand::parser* parser) noexcept; +#ifdef SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON + simdjson_inline json_iterator(const uint8_t* buf, ondemand::parser* parser, bool streaming) noexcept; +#endif // SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON /// The last token before the end simdjson_inline token_position last_position() const noexcept; /// The token *at* the end. This points at gibberish and should only be used for comparison. @@ -59538,6 +65986,7 @@ namespace simdjson { friend class raw_json_string; friend class parser; friend class value_iterator; + friend class field; template friend simdjson_inline void logger::log_line(const json_iterator& iter, const char* title_prefix, const char* title, std::string_view detail, int delta, int depth_delta, logger::log_level level, Args&&... args) noexcept; template @@ -59659,8 +66108,6 @@ namespace simdjson { */ friend class value_iterator; template - friend error_code numberparsing::slow_float_parsing(simdjson_unused const uint8_t* src, W writer); - template friend error_code numberparsing::write_float(const uint8_t* const src, bool negative, uint64_t i, const uint8_t* start_digits, size_t digit_count, int64_t exponent, W& writer); template friend error_code numberparsing::parse_number(const uint8_t* const src, W& writer); @@ -59954,8 +66401,8 @@ namespace simdjson { namespace ondemand { /** - * The default batch size for document_stream instances for this On Demand kernel. - * Note that different On Demand kernel may use a different DEFAULT_BATCH_SIZE value + * The default batch size for document_stream instances for this On-Demand kernel. + * Note that different On-Demand kernel may use a different DEFAULT_BATCH_SIZE value * in the future. */ static constexpr size_t DEFAULT_BATCH_SIZE = 1000000; @@ -60039,6 +66486,9 @@ namespace simdjson { * - UNCLOSED_STRING if there is an unclosed string in the document. */ simdjson_warn_unused simdjson_result iterate(padded_string_view json) & noexcept; +#ifdef SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON + simdjson_warn_unused simdjson_result iterate_allow_incomplete_json(padded_string_view json) & noexcept; +#endif // SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON /** @overload simdjson_result iterate(padded_string_view json) & noexcept */ simdjson_warn_unused simdjson_result iterate(const char* json, size_t len, size_t capacity) & noexcept; /** @overload simdjson_result iterate(padded_string_view json) & noexcept */ @@ -60183,9 +66633,9 @@ namespace simdjson { simdjson_result iterate_many(const char* buf, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept = delete; /** The capacity of this parser (the largest document it can process). */ - simdjson_inline size_t capacity() const noexcept; + simdjson_pure simdjson_inline size_t capacity() const noexcept; /** The maximum capacity of this parser (the largest document it is allowed to process). */ - simdjson_inline size_t max_capacity() const noexcept; + simdjson_pure simdjson_inline size_t max_capacity() const noexcept; simdjson_inline void set_max_capacity(size_t max_capacity) noexcept; /** * The maximum depth of this parser (the most deeply nested objects and arrays it can process). @@ -60193,7 +66643,7 @@ namespace simdjson { * The document's instance current_depth() method should be used to monitor the parsing * depth and limit it if desired. */ - simdjson_inline size_t max_depth() const noexcept; + simdjson_pure simdjson_inline size_t max_depth() const noexcept; /** * Ensure this parser has enough memory to process JSON documents up to `capacity` bytes in length @@ -60265,9 +66715,20 @@ namespace simdjson { */ simdjson_inline simdjson_result unescape_wobbly(raw_json_string in, uint8_t*& dst) const noexcept; +#if SIMDJSON_DEVELOPMENT_CHECKS + /** + * Returns true if string_buf_loc is outside of the allocated range for the + * the string buffer. When true, it indicates that the string buffer has overflowed. + * This is a development-time check that is not needed in production. It can be + * used to detect buffer overflows in the string buffer and usafe usage of the + * string buffer. + */ + bool string_buffer_overflow(const uint8_t* string_buf_loc) const noexcept; +#endif + private: /** @private [for benchmarking access] The implementation to use */ - std::unique_ptr implementation{}; + std::unique_ptr implementation{}; size_t _capacity{ 0 }; size_t _max_capacity; size_t _max_depth{ DEFAULT_MAX_DEPTH }; @@ -60404,6 +66865,22 @@ namespace simdjson { * - INVALID_JSON_POINTER if the JSON pointer is invalid and cannot be parsed */ inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; + + /** + * Get the value associated with the given JSONPath expression. We only support + * JSONPath queries that trivially convertible to JSON Pointer queries: key + * names and array indices. + * + * https://datatracker.ietf.org/doc/html/draft-normington-jsonpath-00 + * + * @return The value associated with the given JSONPath expression, or: + * - INVALID_JSON_POINTER if the JSONPath to JSON Pointer conversion fails + * - NO_SUCH_FIELD if a field does not exist in an object + * - INDEX_OUT_OF_BOUNDS if an array index is larger than an array length + * - INCORRECT_TYPE if a non-integer is used to access an array + */ + inline simdjson_result at_path(std::string_view json_path) noexcept; + /** * Consumes the array and returns a string_view instance corresponding to the * array as represented in JSON. It points inside the original document. @@ -60494,6 +66971,7 @@ namespace simdjson { inline simdjson_result reset() & noexcept; simdjson_inline simdjson_result at(size_t index) noexcept; simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; + simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; simdjson_inline simdjson_result raw_json() noexcept; }; @@ -60784,21 +67262,23 @@ namespace simdjson { * @returns INCORRECT_TYPE If the JSON value is not the given type. */ template simdjson_inline simdjson_result get() & noexcept { - // Unless the simdjson library provides an inline implementation, calling this method should + // Unless the simdjson library or the user provides an inline implementation, calling this method should // immediately fail. static_assert(!sizeof(T), "The get method with given type is not implemented by the simdjson library. " "The supported types are ondemand::object, ondemand::array, raw_json_string, std::string_view, uint64_t, " "int64_t, double, and bool. We recommend you use get_double(), get_bool(), get_uint64(), get_int64(), " - " get_object(), get_array(), get_raw_json_string(), or get_string() instead of the get template."); + " get_object(), get_array(), get_raw_json_string(), or get_string() instead of the get template." + " You may also add support for custom types, see our documentation."); } /** @overload template simdjson_result get() & noexcept */ - template simdjson_inline simdjson_result get() && noexcept { - // Unless the simdjson library provides an inline implementation, calling this method should + template simdjson_deprecated simdjson_inline simdjson_result get() && noexcept { + // Unless the simdjson library or the user provides an inline implementation, calling this method should // immediately fail. static_assert(!sizeof(T), "The get method with given type is not implemented by the simdjson library. " "The supported types are ondemand::object, ondemand::array, raw_json_string, std::string_view, uint64_t, " "int64_t, double, and bool. We recommend you use get_double(), get_bool(), get_uint64(), get_int64(), " - " get_object(), get_array(), get_raw_json_string(), or get_string() instead of the get template."); + " get_object(), get_array(), get_raw_json_string(), or get_string() instead of the get template." + " You may also add support for custom types, see our documentation."); } /** @@ -60814,9 +67294,23 @@ namespace simdjson { */ template simdjson_inline error_code get(T& out) & noexcept; /** @overload template error_code get(T &out) & noexcept */ - template simdjson_inline error_code get(T& out) && noexcept; + template simdjson_deprecated simdjson_inline error_code get(T& out) && noexcept; #if SIMDJSON_EXCEPTIONS + /** + * Cast this JSON value to an instance of type T. The programmer is responsible for + * providing an implementation of get for the type T, if T is not one of the types + * supported by the library (object, array, raw_json_string, string_view, uint64_t, etc.) + * + * See https://github.com/simdjson/simdjson/blob/master/doc/basics.md#adding-support-for-custom-types + * + * @returns An instance of type T + */ + template + explicit simdjson_inline operator T() & noexcept(false); + template + explicit simdjson_deprecated simdjson_inline operator T() && noexcept(false); + /** * Cast this JSON value to an array. * @@ -60989,7 +67483,7 @@ namespace simdjson { * APIs assume this. Therefore, you must be explicit if you want to treat objects as out of order. * * Use find_field() if you are sure fields will be in order (or are willing to treat it as if the - * field wasn't there when they aren't). + * field was not there when they are not in order). * * You must consume the fields on an object one at a time. A request for a new key * invalidates previous field values: it makes them unsafe. E.g., the array @@ -61035,6 +67529,14 @@ namespace simdjson { */ simdjson_inline simdjson_result is_scalar() noexcept; + /** + * Checks whether the document is a string. + * + * @returns true if the type is string + * @error TAPE_ERROR when the JSON value is a bad token like "}" "," or "alse". + */ + simdjson_inline simdjson_result is_string() noexcept; + /** * Checks whether the document is a negative number. * @@ -61062,9 +67564,11 @@ namespace simdjson { * get_number().get_number_type(). * * get_number_type() is number_type::unsigned_integer if we have - * an integer greater or equal to 9223372036854775808 + * an integer greater or equal to 9223372036854775808 and no larger than 18446744073709551615. * get_number_type() is number_type::signed_integer if we have an - * integer that is less than 9223372036854775808 + * integer that is less than 9223372036854775808 and greater or equal to -9223372036854775808. + * get_number_type() is number_type::big_integer if we have an integer outside + * of those ranges (either larger than 18446744073709551615 or smaller than -9223372036854775808). * Otherwise, get_number_type() has value number_type::floating_point_number * * This function requires processing the number string, but it is expected @@ -61182,6 +67686,14 @@ namespace simdjson { * auto doc = parser.iterate(json); * doc.at_pointer("//a/1") == 20 * + * Key values are matched exactly, without unescaping or Unicode normalization. + * We do a byte-by-byte comparison. E.g. + * + * const padded_string json = "{\"\\u00E9\":123}"_padded; + * auto doc = parser.iterate(json); + * doc.at_pointer("/\\u00E9") == 123 + * doc.at_pointer((const char*)u8"/\u00E9") returns an error (NO_SUCH_FIELD) + * * Note that at_pointer() automatically calls rewind between each call. Thus * all values, objects and arrays that you have created so far (including unescaped strings) * are invalidated. After calling at_pointer, you need to consume the result: string values @@ -61198,6 +67710,30 @@ namespace simdjson { * - SCALAR_DOCUMENT_AS_VALUE if the json_pointer is empty and the document is not a scalar (see is_scalar() function). */ simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; + + /** + * Get the value associated with the given JSONPath expression. We only support + * JSONPath queries that trivially convertible to JSON Pointer queries: key + * names and array indices. + * + * https://datatracker.ietf.org/doc/html/draft-normington-jsonpath-00 + * + * Key values are matched exactly, without unescaping or Unicode normalization. + * We do a byte-by-byte comparison. E.g. + * + * const padded_string json = "{\"\\u00E9\":123}"_padded; + * auto doc = parser.iterate(json); + * doc.at_path(".\\u00E9") == 123 + * doc.at_path((const char*)u8".\u00E9") returns an error (NO_SUCH_FIELD) + * + * @return The value associated with the given JSONPath expression, or: + * - INVALID_JSON_POINTER if the JSONPath to JSON Pointer conversion fails + * - NO_SUCH_FIELD if a field does not exist in an object + * - INDEX_OUT_OF_BOUNDS if an array index is larger than an array length + * - INCORRECT_TYPE if a non-integer is used to access an array + */ + simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; + /** * Consumes the document and returns a string_view instance corresponding to the * document as represented in JSON. It points inside the original byte array containing @@ -61263,10 +67799,12 @@ namespace simdjson { simdjson_inline simdjson_result get_value() noexcept; simdjson_inline simdjson_result is_null() noexcept; + template simdjson_inline simdjson_result get() & noexcept; simdjson_inline simdjson_result raw_json() noexcept; simdjson_inline operator document& () const noexcept; - #if SIMDJSON_EXCEPTIONS + template + explicit simdjson_inline operator T() noexcept(false); simdjson_inline operator array() & noexcept(false); simdjson_inline operator object() & noexcept(false); simdjson_inline operator uint64_t() noexcept(false); @@ -61291,6 +67829,7 @@ namespace simdjson { simdjson_inline simdjson_result type() noexcept; simdjson_inline simdjson_result is_scalar() noexcept; + simdjson_inline simdjson_result is_string() noexcept; simdjson_inline simdjson_result current_location() noexcept; simdjson_inline int32_t current_depth() const noexcept; @@ -61300,6 +67839,8 @@ namespace simdjson { simdjson_inline simdjson_result get_number() noexcept; simdjson_inline simdjson_result raw_json_token() noexcept; simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; + simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; + private: document* doc{ nullptr }; }; @@ -61335,12 +67876,13 @@ namespace simdjson { simdjson_inline simdjson_result is_null() noexcept; template simdjson_inline simdjson_result get() & noexcept; - template simdjson_inline simdjson_result get() && noexcept; + template simdjson_deprecated simdjson_inline simdjson_result get() && noexcept; template simdjson_inline error_code get(T& out) & noexcept; template simdjson_inline error_code get(T& out) && noexcept; - #if SIMDJSON_EXCEPTIONS + template ::value == false>::type> + explicit simdjson_inline operator T() noexcept(false); simdjson_inline operator icelake::ondemand::array() & noexcept(false); simdjson_inline operator icelake::ondemand::object() & noexcept(false); simdjson_inline operator uint64_t() noexcept(false); @@ -61364,6 +67906,7 @@ namespace simdjson { simdjson_inline simdjson_result find_field_unordered(const char* key) & noexcept; simdjson_inline simdjson_result type() noexcept; simdjson_inline simdjson_result is_scalar() noexcept; + simdjson_inline simdjson_result is_string() noexcept; simdjson_inline simdjson_result current_location() noexcept; simdjson_inline int32_t current_depth() const noexcept; simdjson_inline bool at_end() const noexcept; @@ -61375,6 +67918,7 @@ namespace simdjson { simdjson_inline simdjson_result raw_json_token() noexcept; simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; + simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; }; @@ -61407,8 +67951,9 @@ namespace simdjson { simdjson_inline simdjson_result get_bool() noexcept; simdjson_inline simdjson_result get_value() noexcept; simdjson_inline simdjson_result is_null() noexcept; - #if SIMDJSON_EXCEPTIONS + template ::value == false>::type> + explicit simdjson_inline operator T() noexcept(false); simdjson_inline operator icelake::ondemand::array() & noexcept(false); simdjson_inline operator icelake::ondemand::object() & noexcept(false); simdjson_inline operator uint64_t() noexcept(false); @@ -61432,6 +67977,7 @@ namespace simdjson { simdjson_inline simdjson_result find_field_unordered(const char* key) & noexcept; simdjson_inline simdjson_result type() noexcept; simdjson_inline simdjson_result is_scalar() noexcept; + simdjson_inline simdjson_result is_string() noexcept; simdjson_inline simdjson_result current_location() noexcept; simdjson_inline simdjson_result current_depth() const noexcept; simdjson_inline simdjson_result is_negative() noexcept; @@ -61442,6 +67988,7 @@ namespace simdjson { simdjson_inline simdjson_result raw_json_token() noexcept; simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; + simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; }; @@ -61575,10 +68122,9 @@ namespace simdjson { class iterator { public: using value_type = simdjson_result; - using reference = value_type; - + using reference = simdjson_result; + using pointer = void; using difference_type = std::ptrdiff_t; - using iterator_category = std::input_iterator_tag; /** @@ -61588,7 +68134,7 @@ namespace simdjson { /** * Get the current document (or error). */ - simdjson_inline simdjson_result operator*() noexcept; + simdjson_inline reference operator*() noexcept; /** * Advance to the next document (prefix). */ @@ -61770,7 +68316,7 @@ namespace simdjson { friend class document; friend class json_iterator; friend struct simdjson_result; - friend struct internal::simdjson_result_base; + friend struct simdjson::internal::simdjson_result_base; }; // document_stream } // namespace ondemand @@ -61830,12 +68376,39 @@ namespace simdjson { * This consumes the key: once you have called unescaped_key(), you cannot * call it again nor can you call key(). */ - simdjson_inline simdjson_warn_unused simdjson_result unescaped_key(bool allow_replacement) noexcept; + simdjson_inline simdjson_warn_unused simdjson_result unescaped_key(bool allow_replacement = false) noexcept; + /** + * Get the key as a string_view (for higher speed, consider raw_key). + * We deliberately use a more cumbersome name (unescaped_key) to force users + * to think twice about using it. The content is stored in the receiver. + * + * This consumes the key: once you have called unescaped_key(), you cannot + * call it again nor can you call key(). + */ + template + simdjson_inline simdjson_warn_unused error_code unescaped_key(string_type& receiver, bool allow_replacement = false) noexcept; /** * Get the key as a raw_json_string. Can be used for direct comparison with - * an unescaped C string: e.g., key() == "test". + * an unescaped C string: e.g., key() == "test". This does not count as + * consumption of the content: you can safely call it repeatedly. + * See escaped_key() for a similar function which returns + * a more convenient std::string_view result. */ simdjson_inline raw_json_string key() const noexcept; + /** + * Get the unprocessed key as a string_view. This includes the quotes and may include + * some spaces after the last quote. This does not count as + * consumption of the content: you can safely call it repeatedly. + * See escaped_key(). + */ + simdjson_inline std::string_view key_raw_json_token() const noexcept; + /** + * Get the key as a string_view. This does not include the quotes and + * the string is unprocessed key so it may contain escape characters + * (e.g., \uXXXX or \n). It does not count as a consumption of the content: + * you can safely call it repeatedly. Use unescaped_key() to get the unescaped key. + */ + simdjson_inline std::string_view escaped_key() const noexcept; /** * Get the field value. */ @@ -61867,7 +68440,11 @@ namespace simdjson { simdjson_inline simdjson_result() noexcept = default; simdjson_inline simdjson_result unescaped_key(bool allow_replacement = false) noexcept; + template + simdjson_inline error_code unescaped_key(string_type& receiver, bool allow_replacement = false) noexcept; simdjson_inline simdjson_result key() noexcept; + simdjson_inline simdjson_result key_raw_json_token() noexcept; + simdjson_inline simdjson_result escaped_key() noexcept; simdjson_inline simdjson_result value() noexcept; }; @@ -61935,6 +68512,8 @@ namespace simdjson { * key a single time. Doing object["mykey"].to_string() and then again object["mykey"].to_string() * is an error. * + * If you expect to have keys with escape characters, please review our documentation. + * * @param key The key to look up. * @returns The value of the field, or NO_SUCH_FIELD if the field is not in the object. */ @@ -61956,7 +68535,7 @@ namespace simdjson { * APIs assume this. Therefore, you must be explicit if you want to treat objects as out of order. * * Use find_field() if you are sure fields will be in order (or are willing to treat it as if the - * field wasn't there when they aren't). + * field was not there when they are not in order). * * If you have multiple fields with a matching key ({"x": 1, "x": 1}) be mindful * that only one field is returned. @@ -61972,6 +68551,8 @@ namespace simdjson { * You are expected to access keys only once. You should access the value corresponding to a key * a single time. Doing object["mykey"].to_string() and then again object["mykey"].to_string() is an error. * + * If you expect to have keys with escape characters, please review our documentation. + * * @param key The key to look up. * @returns The value of the field, or NO_SUCH_FIELD if the field is not in the object. */ @@ -62019,11 +68600,24 @@ namespace simdjson { */ inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; + /** + * Get the value associated with the given JSONPath expression. We only support + * JSONPath queries that trivially convertible to JSON Pointer queries: key + * names and array indices. + * + * @return The value associated with the given JSONPath expression, or: + * - INVALID_JSON_POINTER if the JSONPath to JSON Pointer conversion fails + * - NO_SUCH_FIELD if a field does not exist in an object + * - INDEX_OUT_OF_BOUNDS if an array index is larger than an array length + * - INCORRECT_TYPE if a non-integer is used to access an array + */ + inline simdjson_result at_path(std::string_view json_path) noexcept; + /** * Reset the iterator so that we are pointing back at the * beginning of the object. You should still consume values only once even if you - * can iterate through the object more than once. If you unescape a string within - * the object more than once, you have unsafe code. Note that rewinding an object + * can iterate through the object more than once. If you unescape a string or a key + * within the object more than once, you have unsafe code. Note that rewinding an object * means that you may need to reparse it anew: it is not a free operation. * * @returns true if the object contains some elements (not empty) @@ -62106,6 +68700,8 @@ namespace simdjson { simdjson_inline simdjson_result operator[](std::string_view key) & noexcept; simdjson_inline simdjson_result operator[](std::string_view key) && noexcept; simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; + simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; + inline simdjson_result reset() noexcept; inline simdjson_result is_empty() noexcept; inline simdjson_result count_fields() & noexcept; @@ -62479,6 +69075,65 @@ namespace simdjson { return child; } + inline std::string json_path_to_pointer_conversion(std::string_view json_path) { + if (json_path.empty() || (json_path.front() != '.' && + json_path.front() != '[')) { + return "-1"; // This is just a sentinel value, the caller should check for this and return an error. + } + + std::string result; + // Reserve space to reduce allocations, adjusting for potential increases due + // to escaping. + result.reserve(json_path.size() * 2); + + size_t i = 0; + + while (i < json_path.length()) { + if (json_path[i] == '.') { + result += '/'; + } + else if (json_path[i] == '[') { + result += '/'; + ++i; // Move past the '[' + while (i < json_path.length() && json_path[i] != ']') { + if (json_path[i] == '~') { + result += "~0"; + } + else if (json_path[i] == '/') { + result += "~1"; + } + else { + result += json_path[i]; + } + ++i; + } + if (i == json_path.length() || json_path[i] != ']') { + return "-1"; // Using sentinel value that will be handled as an error by the caller. + } + } + else { + if (json_path[i] == '~') { + result += "~0"; + } + else if (json_path[i] == '/') { + result += "~1"; + } + else { + result += json_path[i]; + } + } + ++i; + } + + return result; + } + + inline simdjson_result array::at_path(std::string_view json_path) noexcept { + auto json_pointer = json_path_to_pointer_conversion(json_path); + if (json_pointer == "-1") { return INVALID_JSON_POINTER; } + return at_pointer(json_pointer); + } + simdjson_inline simdjson_result array::at(size_t index) noexcept { size_t i = 0; for (auto value : *this) { @@ -62533,6 +69188,10 @@ namespace simdjson { if (error()) { return error(); } return first.at_pointer(json_pointer); } + simdjson_inline simdjson_result simdjson_result::at_path(std::string_view json_path) noexcept { + if (error()) { return error(); } + return first.at_path(json_path); + } simdjson_inline simdjson_result simdjson_result::raw_json() noexcept { if (error()) { return error(); } return first.raw_json(); @@ -62629,14 +69288,14 @@ namespace simdjson { /* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ /* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_ONDEMAND_DOCUMENT_INL_H */ /* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/array-inl.h" */ /* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/array_iterator.h" */ /* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/document.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/json_iterator-inl.h" */ /* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/json_type.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/object-inl.h" */ /* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/raw_json_string.h" */ /* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/value.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/array-inl.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/json_iterator-inl.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/object-inl.h" */ /* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/value_iterator-inl.h" */ /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ @@ -62792,22 +69451,26 @@ namespace simdjson { template<> simdjson_inline simdjson_result document::get() & noexcept { return get_bool(); } template<> simdjson_inline simdjson_result document::get() & noexcept { return get_value(); } - template<> simdjson_inline simdjson_result document::get() && noexcept { return get_raw_json_string(); } - template<> simdjson_inline simdjson_result document::get() && noexcept { return get_string(false); } - template<> simdjson_inline simdjson_result document::get() && noexcept { return std::forward(*this).get_double(); } - template<> simdjson_inline simdjson_result document::get() && noexcept { return std::forward(*this).get_uint64(); } - template<> simdjson_inline simdjson_result document::get() && noexcept { return std::forward(*this).get_int64(); } - template<> simdjson_inline simdjson_result document::get() && noexcept { return std::forward(*this).get_bool(); } - template<> simdjson_inline simdjson_result document::get() && noexcept { return get_value(); } + template<> simdjson_deprecated simdjson_inline simdjson_result document::get() && noexcept { return get_raw_json_string(); } + template<> simdjson_deprecated simdjson_inline simdjson_result document::get() && noexcept { return get_string(false); } + template<> simdjson_deprecated simdjson_inline simdjson_result document::get() && noexcept { return std::forward(*this).get_double(); } + template<> simdjson_deprecated simdjson_inline simdjson_result document::get() && noexcept { return std::forward(*this).get_uint64(); } + template<> simdjson_deprecated simdjson_inline simdjson_result document::get() && noexcept { return std::forward(*this).get_int64(); } + template<> simdjson_deprecated simdjson_inline simdjson_result document::get() && noexcept { return std::forward(*this).get_bool(); } + template<> simdjson_deprecated simdjson_inline simdjson_result document::get() && noexcept { return get_value(); } template simdjson_inline error_code document::get(T& out) & noexcept { return get().get(out); } - template simdjson_inline error_code document::get(T& out) && noexcept { + template simdjson_deprecated simdjson_inline error_code document::get(T& out) && noexcept { return std::forward(*this).get().get(out); } #if SIMDJSON_EXCEPTIONS + template + simdjson_deprecated simdjson_inline document::operator T() && noexcept(false) { return get(); } + template + simdjson_inline document::operator T() & noexcept(false) { return get(); } simdjson_inline document::operator array() & noexcept(false) { return get_array(); } simdjson_inline document::operator object() & noexcept(false) { return get_object(); } simdjson_inline document::operator uint64_t() noexcept(false) { return get_uint64(); } @@ -62892,6 +69555,13 @@ namespace simdjson { return !((this_type == json_type::array) || (this_type == json_type::object)); } + simdjson_inline simdjson_result document::is_string() noexcept { + json_type this_type; + auto error = type().get(this_type); + if (error) { return error; } + return (this_type == json_type::string); + } + simdjson_inline bool document::is_negative() noexcept { return get_root_value_iterator().is_root_negative(); } @@ -62911,7 +69581,7 @@ namespace simdjson { simdjson_inline simdjson_result document::raw_json_token() noexcept { auto _iter = get_root_value_iterator(); - return std::string_view(reinterpret_cast(_iter.peek_start()), _iter.peek_start_length()); + return std::string_view(reinterpret_cast(_iter.peek_start()), _iter.peek_root_length()); } simdjson_inline simdjson_result document::at_pointer(std::string_view json_pointer) noexcept { @@ -62932,6 +69602,23 @@ namespace simdjson { } } + simdjson_inline simdjson_result document::at_path(std::string_view json_path) noexcept { + rewind(); // Rewind the document each time at_pointer is called + if (json_path.empty()) { + return this->get_value(); + } + json_type t; + SIMDJSON_TRY(type().get(t)); + switch (t) { + case json_type::array: + return (*this).get_array().at_path(json_path); + case json_type::object: + return (*this).get_object().at_path(json_path); + default: + return INVALID_JSON_POINTER; + } + } + } // namespace ondemand } // namespace icelake } // namespace simdjson @@ -63070,7 +69757,7 @@ namespace simdjson { return first.get(); } template - simdjson_inline simdjson_result simdjson_result::get() && noexcept { + simdjson_deprecated simdjson_inline simdjson_result simdjson_result::get() && noexcept { if (error()) { return error(); } return std::forward(first).get(); } @@ -63086,7 +69773,7 @@ namespace simdjson { } template<> simdjson_inline simdjson_result simdjson_result::get() & noexcept = delete; - template<> simdjson_inline simdjson_result simdjson_result::get() && noexcept { + template<> simdjson_deprecated simdjson_inline simdjson_result simdjson_result::get() && noexcept { if (error()) { return error(); } return std::forward(first); } @@ -63107,6 +69794,10 @@ namespace simdjson { return first.is_scalar(); } + simdjson_inline simdjson_result simdjson_result::is_string() noexcept { + if (error()) { return error(); } + return first.is_string(); + } simdjson_inline bool simdjson_result::is_negative() noexcept { if (error()) { return error(); } @@ -63130,6 +69821,11 @@ namespace simdjson { #if SIMDJSON_EXCEPTIONS + template ::value == false>::type> + simdjson_inline simdjson_result::operator T() noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return first; + } simdjson_inline simdjson_result::operator icelake::ondemand::array() & noexcept(false) { if (error()) { throw simdjson_error(error()); } return first; @@ -63195,6 +69891,10 @@ namespace simdjson { return first.at_pointer(json_pointer); } + simdjson_inline simdjson_result simdjson_result::at_path(std::string_view json_path) noexcept { + if (error()) { return error(); } + return first.at_path(json_path); + } } // namespace simdjson @@ -63236,15 +69936,25 @@ namespace simdjson { simdjson_inline simdjson_result document_reference::get_bool() noexcept { return doc->get_root_value_iterator().get_root_bool(false); } simdjson_inline simdjson_result document_reference::get_value() noexcept { return doc->get_value(); } simdjson_inline simdjson_result document_reference::is_null() noexcept { return doc->get_root_value_iterator().is_root_null(false); } - + template<> simdjson_inline simdjson_result document_reference::get() & noexcept { return get_array(); } + template<> simdjson_inline simdjson_result document_reference::get() & noexcept { return get_object(); } + template<> simdjson_inline simdjson_result document_reference::get() & noexcept { return get_raw_json_string(); } + template<> simdjson_inline simdjson_result document_reference::get() & noexcept { return get_string(false); } + template<> simdjson_inline simdjson_result document_reference::get() & noexcept { return get_double(); } + template<> simdjson_inline simdjson_result document_reference::get() & noexcept { return get_uint64(); } + template<> simdjson_inline simdjson_result document_reference::get() & noexcept { return get_int64(); } + template<> simdjson_inline simdjson_result document_reference::get() & noexcept { return get_bool(); } + template<> simdjson_inline simdjson_result document_reference::get() & noexcept { return get_value(); } #if SIMDJSON_EXCEPTIONS + template + simdjson_inline document_reference::operator T() noexcept(false) { return get(); } simdjson_inline document_reference::operator array() & noexcept(false) { return array(*doc); } simdjson_inline document_reference::operator object() & noexcept(false) { return object(*doc); } simdjson_inline document_reference::operator uint64_t() noexcept(false) { return get_uint64(); } simdjson_inline document_reference::operator int64_t() noexcept(false) { return get_int64(); } simdjson_inline document_reference::operator double() noexcept(false) { return get_double(); } simdjson_inline document_reference::operator std::string_view() noexcept(false) { return std::string_view(*doc); } - simdjson_inline document_reference::operator raw_json_string() noexcept(false) { return raw_json_string(*doc); } + simdjson_inline document_reference::operator raw_json_string() noexcept(false) { return get_raw_json_string(); } simdjson_inline document_reference::operator bool() noexcept(false) { return get_bool(); } simdjson_inline document_reference::operator value() noexcept(false) { return value(*doc); } #endif @@ -63261,6 +69971,7 @@ namespace simdjson { simdjson_inline simdjson_result document_reference::find_field_unordered(const char* key) & noexcept { return doc->find_field_unordered(key); } simdjson_inline simdjson_result document_reference::type() noexcept { return doc->type(); } simdjson_inline simdjson_result document_reference::is_scalar() noexcept { return doc->is_scalar(); } + simdjson_inline simdjson_result document_reference::is_string() noexcept { return doc->is_string(); } simdjson_inline simdjson_result document_reference::current_location() noexcept { return doc->current_location(); } simdjson_inline int32_t document_reference::current_depth() const noexcept { return doc->current_depth(); } simdjson_inline bool document_reference::is_negative() noexcept { return doc->is_negative(); } @@ -63269,6 +69980,7 @@ namespace simdjson { simdjson_inline simdjson_result document_reference::get_number() noexcept { return doc->get_root_value_iterator().get_root_number(false); } simdjson_inline simdjson_result document_reference::raw_json_token() noexcept { return doc->raw_json_token(); } simdjson_inline simdjson_result document_reference::at_pointer(std::string_view json_pointer) noexcept { return doc->at_pointer(json_pointer); } + simdjson_inline simdjson_result document_reference::at_path(std::string_view json_path) noexcept { return doc->at_path(json_path); } simdjson_inline simdjson_result document_reference::raw_json() noexcept { return doc->raw_json(); } simdjson_inline document_reference::operator document& () const noexcept { return *doc; } @@ -63400,6 +70112,10 @@ namespace simdjson { if (error()) { return error(); } return first.is_scalar(); } + simdjson_inline simdjson_result simdjson_result::is_string() noexcept { + if (error()) { return error(); } + return first.is_string(); + } simdjson_inline simdjson_result simdjson_result::is_negative() noexcept { if (error()) { return error(); } return first.is_negative(); @@ -63417,6 +70133,11 @@ namespace simdjson { return first.get_number(); } #if SIMDJSON_EXCEPTIONS + template ::value == false>::type> + simdjson_inline simdjson_result::operator T() noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return first; + } simdjson_inline simdjson_result::operator icelake::ondemand::array() & noexcept(false) { if (error()) { throw simdjson_error(error()); } return first; @@ -63470,6 +70191,12 @@ namespace simdjson { return first.at_pointer(json_pointer); } + simdjson_inline simdjson_result simdjson_result::at_path(std::string_view json_path) noexcept { + if (error()) { + return error(); + } + return first.at_path(json_path); + } } // namespace simdjson @@ -63823,9 +70550,18 @@ namespace simdjson { depth--; break; default: // Scalar value document - // TODO: Remove any trailing whitespaces + // TODO: We could remove trailing whitespaces // This returns a string spanning from start of value to the beginning of the next document (excluded) - return std::string_view(reinterpret_cast(stream->buf) + current_index(), stream->parser->implementation->structural_indexes[++cur_struct_index] - current_index() - 1); + { + auto next_index = stream->parser->implementation->structural_indexes[++cur_struct_index]; + // normally the length would be next_index - current_index() - 1, except for the last document + size_t svlen = next_index - current_index(); + const char* start = reinterpret_cast(stream->buf) + current_index(); + while (svlen > 1 && (std::isspace(start[svlen - 1]) || start[svlen - 1] == '\0')) { + svlen--; + } + return std::string_view(start, svlen); + } } cur_struct_index++; } @@ -63920,7 +70656,7 @@ namespace simdjson { namespace icelake { namespace ondemand { - // clang 6 doesn't think the default constructor can be noexcept, so we make it explicit + // clang 6 does not think the default constructor can be noexcept, so we make it explicit simdjson_inline field::field() noexcept : std::pair() {} simdjson_inline field::field(raw_json_string key, ondemand::value&& value) noexcept @@ -63946,11 +70682,32 @@ namespace simdjson { return answer; } + template + simdjson_inline simdjson_warn_unused error_code field::unescaped_key(string_type& receiver, bool allow_replacement) noexcept { + std::string_view key; + SIMDJSON_TRY(unescaped_key(allow_replacement).get(key)); + receiver = key; + return SUCCESS; + } + simdjson_inline raw_json_string field::key() const noexcept { SIMDJSON_ASSUME(first.buf != nullptr); // We would like to call .alive() by Visual Studio won't let us. return first; } + + simdjson_inline std::string_view field::key_raw_json_token() const noexcept { + SIMDJSON_ASSUME(first.buf != nullptr); // We would like to call .alive() by Visual Studio won't let us. + return std::string_view(reinterpret_cast(first.buf - 1), second.iter._json_iter->token.peek(-1) - first.buf + 1); + } + + simdjson_inline std::string_view field::escaped_key() const noexcept { + SIMDJSON_ASSUME(first.buf != nullptr); // We would like to call .alive() by Visual Studio won't let us. + auto end_quote = second.iter._json_iter->token.peek(-1); + while (*end_quote != '"') end_quote--; + return std::string_view(reinterpret_cast(first.buf), end_quote - first.buf); + } + simdjson_inline value& field::value() & noexcept { return second; } @@ -63984,10 +70741,28 @@ namespace simdjson { if (error()) { return error(); } return first.key(); } + + simdjson_inline simdjson_result simdjson_result::key_raw_json_token() noexcept { + if (error()) { return error(); } + return first.key_raw_json_token(); + } + + simdjson_inline simdjson_result simdjson_result::escaped_key() noexcept { + if (error()) { return error(); } + return first.escaped_key(); + } + simdjson_inline simdjson_result simdjson_result::unescaped_key(bool allow_replacement) noexcept { if (error()) { return error(); } return first.unescaped_key(allow_replacement); } + + template + simdjson_inline error_code simdjson_result::unescaped_key(string_type& receiver, bool allow_replacement) noexcept { + if (error()) { return error(); } + return first.unescaped_key(receiver, allow_replacement); + } + simdjson_inline simdjson_result simdjson_result::value() noexcept { if (error()) { return error(); } return std::move(first.value()); @@ -64055,6 +70830,23 @@ namespace simdjson { #endif } +#ifdef SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON + simdjson_inline json_iterator::json_iterator(const uint8_t* buf, ondemand::parser* _parser, bool streaming) noexcept + : token(buf, &_parser->implementation->structural_indexes[0]), + parser{ _parser }, + _string_buf_loc{ parser->string_buf.get() }, + _depth{ 1 }, + _root{ parser->implementation->structural_indexes.get() }, + _streaming{ streaming } + + { + logger::log_headers(); +#if SIMDJSON_CHECK_EOF + assert_more_tokens(); +#endif + } +#endif // SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON + inline void json_iterator::rewind() noexcept { token.set_position(root_position()); logger::log_headers(); // We start again @@ -64288,6 +71080,12 @@ namespace simdjson { #endif // SIMDJSON_CHECK_EOF return token.peek_length(position); } + simdjson_inline uint32_t json_iterator::peek_root_length(token_position position) const noexcept { +#if SIMDJSON_CHECK_EOF + assert_valid_position(position); +#endif // SIMDJSON_CHECK_EOF + return token.peek_root_length(position); + } simdjson_inline token_position json_iterator::last_position() const noexcept { // The following line fails under some compilers... @@ -64333,11 +71131,23 @@ namespace simdjson { } simdjson_inline simdjson_result json_iterator::unescape(raw_json_string in, bool allow_replacement) noexcept { +#if SIMDJSON_DEVELOPMENT_CHECKS + auto result = parser->unescape(in, _string_buf_loc, allow_replacement); + SIMDJSON_ASSUME(!parser->string_buffer_overflow(_string_buf_loc)); + return result; +#else return parser->unescape(in, _string_buf_loc, allow_replacement); +#endif } simdjson_inline simdjson_result json_iterator::unescape_wobbly(raw_json_string in) noexcept { +#if SIMDJSON_DEVELOPMENT_CHECKS + auto result = parser->unescape_wobbly(in, _string_buf_loc); + SIMDJSON_ASSUME(!parser->string_buffer_overflow(_string_buf_loc)); + return result; +#else return parser->unescape_wobbly(in, _string_buf_loc); +#endif } simdjson_inline void json_iterator::reenter_child(token_position position, depth_t child_depth) noexcept { @@ -64462,7 +71272,6 @@ namespace simdjson { return get_uint64(); } - simdjson_inline bool number::is_int64() const noexcept { return get_number_type() == number_type::signed_integer; } @@ -64930,6 +71739,14 @@ namespace simdjson { return child; } + inline simdjson_result object::at_path(std::string_view json_path) noexcept { + auto json_pointer = json_path_to_pointer_conversion(json_path); + if (json_pointer == "-1") { + return INVALID_JSON_POINTER; + } + return at_pointer(json_pointer); + } + simdjson_inline simdjson_result object::count_fields() & noexcept { size_t count{ 0 }; // Important: we do not consume any of the values. @@ -65002,6 +71819,14 @@ namespace simdjson { return first.at_pointer(json_pointer); } + simdjson_inline simdjson_result simdjson_result::at_path( + std::string_view json_path) noexcept { + if (error()) { + return error(); + } + return first.at_path(json_path); + } + inline simdjson_result simdjson_result::reset() noexcept { if (error()) { return error(); } return first.reset(); @@ -65213,6 +72038,11 @@ namespace simdjson { _max_depth = new_max_depth; return SUCCESS; } +#if SIMDJSON_DEVELOPMENT_CHECKS + simdjson_inline simdjson_warn_unused bool parser::string_buffer_overflow(const uint8_t* string_buf_loc) const noexcept { + return (string_buf_loc < string_buf.get()) || (size_t(string_buf_loc - string_buf.get()) >= capacity()); + } +#endif simdjson_warn_unused simdjson_inline simdjson_result parser::iterate(padded_string_view json) & noexcept { if (json.padding() < SIMDJSON_PADDING) { return INSUFFICIENT_PADDING; } @@ -65229,6 +72059,27 @@ namespace simdjson { return document::start({ reinterpret_cast(json.data()), this }); } +#ifdef SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON + simdjson_warn_unused simdjson_inline simdjson_result parser::iterate_allow_incomplete_json(padded_string_view json) & noexcept { + if (json.padding() < SIMDJSON_PADDING) { return INSUFFICIENT_PADDING; } + + json.remove_utf8_bom(); + + // Allocate if needed + if (capacity() < json.length() || !string_buf) { + SIMDJSON_TRY(allocate(json.length(), max_depth())); + } + + // Run stage 1. + const simdjson::error_code err = implementation->stage1(reinterpret_cast(json.data()), json.length(), stage1_mode::regular); + if (err) { + if (err != UNCLOSED_STRING) + return err; + } + return document::start({ reinterpret_cast(json.data()), this, true }); + } +#endif // SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON + simdjson_warn_unused simdjson_inline simdjson_result parser::iterate(const char* json, size_t len, size_t allocated) & noexcept { return iterate(padded_string_view(json, len, allocated)); } @@ -65300,13 +72151,13 @@ namespace simdjson { return iterate_many(s.data(), s.length(), batch_size, allow_comma_separated); } - simdjson_inline size_t parser::capacity() const noexcept { + simdjson_pure simdjson_inline size_t parser::capacity() const noexcept { return _capacity; } - simdjson_inline size_t parser::max_capacity() const noexcept { + simdjson_pure simdjson_inline size_t parser::max_capacity() const noexcept { return _max_capacity; } - simdjson_inline size_t parser::max_depth() const noexcept { + simdjson_pure simdjson_inline size_t parser::max_depth() const noexcept { return _max_depth; } @@ -65855,6 +72706,11 @@ namespace simdjson { return *(position + 1) - *position; } + simdjson_inline uint32_t token_iterator::peek_root_length(token_position position) const noexcept { + return *(position + 2) - *(position) > *(position + 1) - *(position) ? + *(position + 1) - *(position) + : *(position + 2) - *(position); + } simdjson_inline const uint8_t* token_iterator::peek(int32_t delta) const noexcept { return &buf[*(_position + delta)]; } @@ -66004,6 +72860,10 @@ namespace simdjson { } #if SIMDJSON_EXCEPTIONS + template + simdjson_inline value::operator T() noexcept(false) { + return get(); + } simdjson_inline value::operator array() noexcept(false) { return get_array(); } @@ -66090,6 +72950,14 @@ namespace simdjson { return !((this_type == json_type::array) || (this_type == json_type::object)); } + simdjson_inline simdjson_result value::is_string() noexcept { + json_type this_type; + auto error = type().get(this_type); + if (error) { return error; } + return (this_type == json_type::string); + } + + simdjson_inline bool value::is_negative() noexcept { return iter.is_negative(); } @@ -66136,6 +73004,26 @@ namespace simdjson { return iter.json_iter().depth(); } + inline bool is_pointer_well_formed(std::string_view json_pointer) noexcept { + if (simdjson_unlikely(json_pointer.empty())) { // can't be + return false; + } + if (simdjson_unlikely(json_pointer[0] != '/')) { + return false; + } + size_t escape = json_pointer.find('~'); + if (escape == std::string_view::npos) { + return true; + } + if (escape == json_pointer.size() - 1) { + return false; + } + if (json_pointer[escape + 1] != '0' && json_pointer[escape + 1] != '1') { + return false; + } + return true; + } + simdjson_inline simdjson_result value::at_pointer(std::string_view json_pointer) noexcept { json_type t; SIMDJSON_TRY(type().get(t)); @@ -66145,6 +73033,23 @@ namespace simdjson { return (*this).get_array().at_pointer(json_pointer); case json_type::object: return (*this).get_object().at_pointer(json_pointer); + default: + // a non-empty string can be invalid, or accessing a primitive (issue 2154) + if (is_pointer_well_formed(json_pointer)) { + return NO_SUCH_FIELD; + } + return INVALID_JSON_POINTER; + } + } + + simdjson_inline simdjson_result value::at_path(std::string_view json_path) noexcept { + json_type t; + SIMDJSON_TRY(type().get(t)); + switch (t) { + case json_type::array: + return (*this).get_array().at_path(json_path); + case json_type::object: + return (*this).get_object().at_path(json_path); default: return INVALID_JSON_POINTER; } @@ -66303,6 +73208,10 @@ namespace simdjson { if (error()) { return error(); } return first.is_scalar(); } + simdjson_inline simdjson_result simdjson_result::is_string() noexcept { + if (error()) { return error(); } + return first.is_string(); + } simdjson_inline simdjson_result simdjson_result::is_negative() noexcept { if (error()) { return error(); } return first.is_negative(); @@ -66320,6 +73229,11 @@ namespace simdjson { return first.get_number(); } #if SIMDJSON_EXCEPTIONS + template + simdjson_inline simdjson_result::operator T() noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return static_cast(first); + } simdjson_inline simdjson_result::operator icelake::ondemand::array() noexcept(false) { if (error()) { throw simdjson_error(error()); } return first; @@ -66374,11 +73288,22 @@ namespace simdjson { return first.current_depth(); } - simdjson_inline simdjson_result simdjson_result::at_pointer(std::string_view json_pointer) noexcept { - if (error()) { return error(); } + simdjson_inline simdjson_result simdjson_result::at_pointer( + std::string_view json_pointer) noexcept { + if (error()) { + return error(); + } return first.at_pointer(json_pointer); } + simdjson_inline simdjson_result simdjson_result::at_path( + std::string_view json_path) noexcept { + if (error()) { + return error(); + } + return first.at_path(json_path); + } + } // namespace simdjson #endif // SIMDJSON_GENERIC_ONDEMAND_VALUE_INL_H @@ -66393,9 +73318,9 @@ namespace simdjson { /* amalgamation skipped (editor-only): #include "simdjson/generic/atomparsing.h" */ /* amalgamation skipped (editor-only): #include "simdjson/generic/numberparsing.h" */ /* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/json_iterator.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/value_iterator.h" */ /* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/json_type-inl.h" */ /* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/raw_json_string-inl.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/value_iterator.h" */ /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ namespace simdjson { @@ -66525,7 +73450,7 @@ namespace simdjson { else if (!is_open()) { #if SIMDJSON_DEVELOPMENT_CHECKS // If we're past the end of the object, we're being iterated out of order. - // Note: this isn't perfect detection. It's possible the user is inside some other object; if so, + // Note: this is not perfect detection. It's possible the user is inside some other object; if so, // this object iterator will blithely scan that object for fields. if (_json_iter->depth() < depth() - 1) { return OUT_OF_ORDER_ITERATION; } #endif @@ -66639,7 +73564,7 @@ namespace simdjson { #if SIMDJSON_DEVELOPMENT_CHECKS // If we're past the end of the object, we're being iterated out of order. - // Note: this isn't perfect detection. It's possible the user is inside some other object; if so, + // Note: this is not perfect detection. It's possible the user is inside some other object; if so, // this object iterator will blithely scan that object for fields. if (_json_iter->depth() < depth() - 1) { return OUT_OF_ORDER_ITERATION; } #endif @@ -66982,7 +73907,7 @@ namespace simdjson { } simdjson_inline simdjson_result value_iterator::is_root_integer(bool check_trailing) noexcept { - auto max_len = peek_start_length(); + auto max_len = peek_root_length(); auto json = peek_root_scalar("is_root_integer"); uint8_t tmpbuf[20 + 1 + 1]{}; // <20 digits> is the longest possible unsigned integer tmpbuf[20 + 1] = '\0'; // make sure that buffer is always null terminated. @@ -66998,7 +73923,7 @@ namespace simdjson { } simdjson_inline simdjson_result value_iterator::get_root_number_type(bool check_trailing) noexcept { - auto max_len = peek_start_length(); + auto max_len = peek_root_length(); auto json = peek_root_scalar("number"); // Per https://www.exploringbinary.com/maximum-number-of-decimal-digits-in-binary-floating-point-numbers/, // 1074 is the maximum number of significant fractional digits. Add 8 more digits for the biggest @@ -67006,7 +73931,12 @@ namespace simdjson { uint8_t tmpbuf[1074 + 8 + 1 + 1]; tmpbuf[1074 + 8 + 1] = '\0'; // make sure that buffer is always null terminated. if (!_json_iter->copy_to_buffer(json, max_len, tmpbuf, 1074 + 8 + 1)) { - logger::log_error(*_json_iter, start_position(), depth(), "Root number more than 1082 characters"); + if (numberparsing::check_if_integer(json, max_len)) { + if (check_trailing && !_json_iter->is_single_token()) { return TRAILING_CONTENT; } + logger::log_error(*_json_iter, start_position(), depth(), "Found big integer"); + return number_type::big_integer; + } + logger::log_error(*_json_iter, start_position(), depth(), "Root number more than 1082 characters and not a big integer"); return NUMBER_ERROR; } auto answer = numberparsing::get_number_type(tmpbuf); @@ -67014,15 +73944,21 @@ namespace simdjson { return answer; } simdjson_inline simdjson_result value_iterator::get_root_number(bool check_trailing) noexcept { - auto max_len = peek_start_length(); + auto max_len = peek_root_length(); auto json = peek_root_scalar("number"); // Per https://www.exploringbinary.com/maximum-number-of-decimal-digits-in-binary-floating-point-numbers/, // 1074 is the maximum number of significant fractional digits. Add 8 more digits for the biggest // number: -0.e-308. + // NOTE: the current approach doesn't work for very big integer numbers containing more than 1074 digits. uint8_t tmpbuf[1074 + 8 + 1 + 1]; tmpbuf[1074 + 8 + 1] = '\0'; // make sure that buffer is always null terminated. if (!_json_iter->copy_to_buffer(json, max_len, tmpbuf, 1074 + 8 + 1)) { - logger::log_error(*_json_iter, start_position(), depth(), "Root number more than 1082 characters"); + if (numberparsing::check_if_integer(json, max_len)) { + if (check_trailing && !_json_iter->is_single_token()) { return TRAILING_CONTENT; } + logger::log_error(*_json_iter, start_position(), depth(), "Found big integer"); + return BIGINT_ERROR; + } + logger::log_error(*_json_iter, start_position(), depth(), "Root number more than 1082 characters and not a big integer"); return NUMBER_ERROR; } number num; @@ -67054,7 +73990,7 @@ namespace simdjson { return raw_json_string(json + 1); } simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_root_uint64(bool check_trailing) noexcept { - auto max_len = peek_start_length(); + auto max_len = peek_root_length(); auto json = peek_root_scalar("uint64"); uint8_t tmpbuf[20 + 1 + 1]{}; // <20 digits> is the longest possible unsigned integer tmpbuf[20 + 1] = '\0'; // make sure that buffer is always null terminated. @@ -67070,7 +74006,7 @@ namespace simdjson { return result; } simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_root_uint64_in_string(bool check_trailing) noexcept { - auto max_len = peek_start_length(); + auto max_len = peek_root_length(); auto json = peek_root_scalar("uint64"); uint8_t tmpbuf[20 + 1 + 1]{}; // <20 digits> is the longest possible unsigned integer tmpbuf[20 + 1] = '\0'; // make sure that buffer is always null terminated. @@ -67086,7 +74022,7 @@ namespace simdjson { return result; } simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_root_int64(bool check_trailing) noexcept { - auto max_len = peek_start_length(); + auto max_len = peek_root_length(); auto json = peek_root_scalar("int64"); uint8_t tmpbuf[20 + 1 + 1]; // -<19 digits> is the longest possible integer tmpbuf[20 + 1] = '\0'; // make sure that buffer is always null terminated. @@ -67103,7 +74039,7 @@ namespace simdjson { return result; } simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_root_int64_in_string(bool check_trailing) noexcept { - auto max_len = peek_start_length(); + auto max_len = peek_root_length(); auto json = peek_root_scalar("int64"); uint8_t tmpbuf[20 + 1 + 1]; // -<19 digits> is the longest possible integer tmpbuf[20 + 1] = '\0'; // make sure that buffer is always null terminated. @@ -67120,7 +74056,7 @@ namespace simdjson { return result; } simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_root_double(bool check_trailing) noexcept { - auto max_len = peek_start_length(); + auto max_len = peek_root_length(); auto json = peek_root_scalar("double"); // Per https://www.exploringbinary.com/maximum-number-of-decimal-digits-in-binary-floating-point-numbers/, // 1074 is the maximum number of significant fractional digits. Add 8 more digits for the biggest @@ -67140,7 +74076,7 @@ namespace simdjson { } simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_root_double_in_string(bool check_trailing) noexcept { - auto max_len = peek_start_length(); + auto max_len = peek_root_length(); auto json = peek_root_scalar("double"); // Per https://www.exploringbinary.com/maximum-number-of-decimal-digits-in-binary-floating-point-numbers/, // 1074 is the maximum number of significant fractional digits. Add 8 more digits for the biggest @@ -67159,7 +74095,7 @@ namespace simdjson { return result; } simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_root_bool(bool check_trailing) noexcept { - auto max_len = peek_start_length(); + auto max_len = peek_root_length(); auto json = peek_root_scalar("bool"); uint8_t tmpbuf[5 + 1 + 1]; // +1 for null termination tmpbuf[5 + 1] = '\0'; // make sure that buffer is always null terminated. @@ -67172,7 +74108,7 @@ namespace simdjson { return result; } simdjson_inline simdjson_result value_iterator::is_root_null(bool check_trailing) noexcept { - auto max_len = peek_start_length(); + auto max_len = peek_root_length(); auto json = peek_root_scalar("null"); bool result = (max_len >= 4 && !atomparsing::str4ncmp(json, "null") && (max_len == 4 || jsoncharutils::is_structural_or_whitespace(json[4]))); @@ -67244,6 +74180,9 @@ namespace simdjson { simdjson_inline uint32_t value_iterator::peek_start_length() const noexcept { return _json_iter->peek_length(start_position()); } + simdjson_inline uint32_t value_iterator::peek_root_length() const noexcept { + return _json_iter->peek_root_length(start_position()); + } simdjson_inline const uint8_t* value_iterator::peek_scalar(const char* type) noexcept { logger::log_value(*_json_iter, start_position(), depth(), type); @@ -67725,13 +74664,13 @@ namespace simdjson { simdjson_inline internal::value128 full_multiplication(uint64_t value1, uint64_t value2) { internal::value128 answer; #if SIMDJSON_REGULAR_VISUAL_STUDIO || SIMDJSON_IS_32BITS -#ifdef _M_ARM64 +#if SIMDJSON_IS_ARM64 // ARM64 has native support for 64-bit multiplications, no need to emultate answer.high = __umulh(value1, value2); answer.low = value1 * value2; #else answer.low = _umul128(value1, value2, &answer.high); // _umul128 not available on ARM64 -#endif // _M_ARM64 +#endif // SIMDJSON_IS_ARM64 #else // SIMDJSON_REGULAR_VISUAL_STUDIO || SIMDJSON_IS_32BITS __uint128_t r = (static_cast<__uint128_t>(value1)) * value2; answer.low = uint64_t(r); @@ -67848,11 +74787,11 @@ namespace simdjson { return (__m128i)vec_splats((unsigned char)(-(!!_value))); } - simdjson_inline simd8() : base8() {} - simdjson_inline simd8(const __m128i _value) + simdjson_inline simd8() : base8() {} + simdjson_inline simd8(const __m128i _value) : base8(_value) {} // Splat constructor - simdjson_inline simd8(bool _value) + simdjson_inline simd8(bool _value) : base8(splat(_value)) {} simdjson_inline int to_bitmask() const { @@ -68736,6 +75675,7 @@ namespace simdjson { simdjson_inline simdjson_result parse_bool(const uint8_t* json) const noexcept; simdjson_inline const uint8_t* peek_start() const noexcept; simdjson_inline uint32_t peek_start_length() const noexcept; + simdjson_inline uint32_t peek_root_length() const noexcept; /** * The general idea of the advance_... methods and the peek_* methods @@ -68830,6 +75770,7 @@ namespace simdjson { friend class object; friend class array; friend class value; + friend class field; }; // value_iterator } // namespace ondemand @@ -68890,12 +75831,13 @@ namespace simdjson { * @returns INCORRECT_TYPE If the JSON value is not the given type. */ template simdjson_inline simdjson_result get() noexcept { - // Unless the simdjson library provides an inline implementation, calling this method should + // Unless the simdjson library or the user provides an inline implementation, calling this method should // immediately fail. static_assert(!sizeof(T), "The get method with given type is not implemented by the simdjson library. " "The supported types are ondemand::object, ondemand::array, raw_json_string, std::string_view, uint64_t, " "int64_t, double, and bool. We recommend you use get_double(), get_bool(), get_uint64(), get_int64(), " - " get_object(), get_array(), get_raw_json_string(), or get_string() instead of the get template."); + " get_object(), get_array(), get_raw_json_string(), or get_string() instead of the get template." + " You may also add support for custom types, see our documentation."); } /** @@ -69048,6 +75990,17 @@ namespace simdjson { simdjson_inline simdjson_result is_null() noexcept; #if SIMDJSON_EXCEPTIONS + /** + * Cast this JSON value to an instance of type T. The programmer is responsible for + * providing an implementation of get for the type T, if T is not one of the types + * supported by the library (object, array, raw_json_string, string_view, uint64_t, etc.). + * + * See https://github.com/simdjson/simdjson/blob/master/doc/basics.md#adding-support-for-custom-types + * + * @returns An instance of type T + */ + template + explicit simdjson_inline operator T() noexcept(false); /** * Cast this JSON value to an array. * @@ -69211,7 +76164,7 @@ namespace simdjson { * that only one field is returned. * * Use find_field() if you are sure fields will be in order (or are willing to treat it as if the - * field wasn't there when they aren't). + * field as not there when they are not in order). * * @param key The key to look up. * @returns The value of the field, or NO_SUCH_FIELD if the field is not in the object. @@ -69247,6 +76200,13 @@ namespace simdjson { * @error TAPE_ERROR when the JSON value is a bad token like "}" "," or "alse". */ simdjson_inline simdjson_result is_scalar() noexcept; + /** + * Checks whether the value is a string. + * + * @returns true if the type is string + * @error TAPE_ERROR when the JSON value is a bad token like "}" "," or "alse". + */ + simdjson_inline simdjson_result is_string() noexcept; /** * Checks whether the value is a negative number. @@ -69279,10 +76239,12 @@ namespace simdjson { * get_number().get_number_type(). * * get_number_type() is number_type::unsigned_integer if we have - * an integer greater or equal to 9223372036854775808 + * an integer greater or equal to 9223372036854775808. * get_number_type() is number_type::signed_integer if we have an - * integer that is less than 9223372036854775808 - * Otherwise, get_number_type() has value number_type::floating_point_number + * integer that is less than 9223372036854775808. + * get_number_type() is number_type::big_integer for integers that do not fit in 64 bits, + * in which case the digit_count is set to the length of the big integer string. + * Otherwise, get_number_type() has value number_type::floating_point_number. * * This function requires processing the number string, but it is expected * to be faster than get_number().get_number_type() because it is does not @@ -69309,6 +76271,8 @@ namespace simdjson { * You can recover the value by calling number.get_uint64() and you * have that number.is_uint64() is true. * + * For integers that do not fit in 64 bits, the function returns BIGINT_ERROR error code. + * * Otherwise, number.get_number_type() has value number_type::floating_point_number * and we have a binary64 number. * You can recover the value by calling number.get_double() and you @@ -69324,7 +76288,6 @@ namespace simdjson { */ simdjson_warn_unused simdjson_inline simdjson_result get_number() noexcept; - /** * Get the raw JSON for this token. * @@ -69422,6 +76385,20 @@ namespace simdjson { */ simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; + /** + * Get the value associated with the given JSONPath expression. We only support + * JSONPath queries that trivially convertible to JSON Pointer queries: key + * names and array indices. + * + * @return The value associated with the given JSONPath expression, or: + * - INVALID_JSON_POINTER if the JSONPath to JSON Pointer conversion fails + * - NO_SUCH_FIELD if a field does not exist in an object + * - INDEX_OUT_OF_BOUNDS if an array index is larger than an array length + * - INCORRECT_TYPE if a non-integer is used to access an array + */ + simdjson_inline simdjson_result at_path(std::string_view at_path) noexcept; + + protected: /** * Create a value. @@ -69461,6 +76438,7 @@ namespace simdjson { friend class object; friend struct simdjson_result; friend struct simdjson_result; + friend class field; }; } // namespace ondemand @@ -69498,6 +76476,8 @@ namespace simdjson { template simdjson_inline error_code get(T& out) noexcept; #if SIMDJSON_EXCEPTIONS + template + explicit simdjson_inline operator T() noexcept(false); simdjson_inline operator ppc64::ondemand::array() noexcept(false); simdjson_inline operator ppc64::ondemand::object() noexcept(false); simdjson_inline operator uint64_t() noexcept(false); @@ -69551,7 +76531,7 @@ namespace simdjson { * APIs assume this. Therefore, you must be explicit if you want to treat objects as out of order. * * Use find_field() if you are sure fields will be in order (or are willing to treat it as if the - * field wasn't there when they aren't). + * field as not there when they are not in order). * * @param key The key to look up. * @returns The value of the field, or NO_SUCH_FIELD if the field is not in the object. @@ -69573,6 +76553,7 @@ namespace simdjson { */ simdjson_inline simdjson_result type() noexcept; simdjson_inline simdjson_result is_scalar() noexcept; + simdjson_inline simdjson_result is_string() noexcept; simdjson_inline simdjson_result is_negative() noexcept; simdjson_inline simdjson_result is_integer() noexcept; simdjson_inline simdjson_result get_number_type() noexcept; @@ -69587,6 +76568,7 @@ namespace simdjson { /** @copydoc simdjson_inline int32_t current_depth() const noexcept */ simdjson_inline simdjson_result current_depth() const noexcept; simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; + simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; }; } // namespace simdjson @@ -69705,7 +76687,7 @@ namespace simdjson { * 1 = next token, -1 = prev token. * * TODO consider a string_view, assuming the length will get stripped out by the optimizer when - * it isn't used ... + * it is not used... */ simdjson_inline const uint8_t* peek(int32_t delta = 0) const noexcept; /** @@ -69735,7 +76717,14 @@ namespace simdjson { * @param position The position of the token. */ simdjson_inline uint32_t peek_length(token_position position) const noexcept; - + /** + * Get the maximum length of the JSON text for a root token. + * + * The length will include any whitespace at the end of the token. + * + * @param position The position of the token (start of the document). + */ + simdjson_inline uint32_t peek_root_length(token_position position) const noexcept; /** * Return the current index. */ @@ -69958,7 +76947,7 @@ namespace simdjson { * @param delta The relative position of the token to retrieve. e.g. 0 = next token, -1 = prev token. * * TODO consider a string_view, assuming the length will get stripped out by the optimizer when - * it isn't used ... + * it is not used ... */ simdjson_inline const uint8_t* peek(int32_t delta = 0) const noexcept; /** @@ -69986,7 +76975,7 @@ namespace simdjson { * @param position The position of the token to retrieve. * * TODO consider a string_view, assuming the length will get stripped out by the optimizer when - * it isn't used ... + * it is not used ... */ simdjson_inline const uint8_t* peek(token_position position) const noexcept; /** @@ -69997,13 +76986,21 @@ namespace simdjson { * @param position The position of the token to retrieve. */ simdjson_inline uint32_t peek_length(token_position position) const noexcept; + /** + * Get the maximum length of the JSON text for the current root token. + * + * The length will include any whitespace at the end of the token. + * + * @param position The position of the token to retrieve. + */ + simdjson_inline uint32_t peek_root_length(token_position position) const noexcept; /** * Get the JSON text for the last token in the document. * * This is not null-terminated; it is a view into the JSON. * * TODO consider a string_view, assuming the length will get stripped out by the optimizer when - * it isn't used ... + * it is not used ... */ simdjson_inline const uint8_t* peek_last() const noexcept; @@ -70097,6 +77094,9 @@ namespace simdjson { inline bool balanced() const noexcept; protected: simdjson_inline json_iterator(const uint8_t* buf, ondemand::parser* parser) noexcept; +#ifdef SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON + simdjson_inline json_iterator(const uint8_t* buf, ondemand::parser* parser, bool streaming) noexcept; +#endif // SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON /// The last token before the end simdjson_inline token_position last_position() const noexcept; /// The token *at* the end. This points at gibberish and should only be used for comparison. @@ -70112,6 +77112,7 @@ namespace simdjson { friend class raw_json_string; friend class parser; friend class value_iterator; + friend class field; template friend simdjson_inline void logger::log_line(const json_iterator& iter, const char* title_prefix, const char* title, std::string_view detail, int delta, int depth_delta, logger::log_level level, Args&&... args) noexcept; template @@ -70233,8 +77234,6 @@ namespace simdjson { */ friend class value_iterator; template - friend error_code numberparsing::slow_float_parsing(simdjson_unused const uint8_t* src, W writer); - template friend error_code numberparsing::write_float(const uint8_t* const src, bool negative, uint64_t i, const uint8_t* start_digits, size_t digit_count, int64_t exponent, W& writer); template friend error_code numberparsing::parse_number(const uint8_t* const src, W& writer); @@ -70528,8 +77527,8 @@ namespace simdjson { namespace ondemand { /** - * The default batch size for document_stream instances for this On Demand kernel. - * Note that different On Demand kernel may use a different DEFAULT_BATCH_SIZE value + * The default batch size for document_stream instances for this On-Demand kernel. + * Note that different On-Demand kernel may use a different DEFAULT_BATCH_SIZE value * in the future. */ static constexpr size_t DEFAULT_BATCH_SIZE = 1000000; @@ -70613,6 +77612,9 @@ namespace simdjson { * - UNCLOSED_STRING if there is an unclosed string in the document. */ simdjson_warn_unused simdjson_result iterate(padded_string_view json) & noexcept; +#ifdef SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON + simdjson_warn_unused simdjson_result iterate_allow_incomplete_json(padded_string_view json) & noexcept; +#endif // SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON /** @overload simdjson_result iterate(padded_string_view json) & noexcept */ simdjson_warn_unused simdjson_result iterate(const char* json, size_t len, size_t capacity) & noexcept; /** @overload simdjson_result iterate(padded_string_view json) & noexcept */ @@ -70757,9 +77759,9 @@ namespace simdjson { simdjson_result iterate_many(const char* buf, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept = delete; /** The capacity of this parser (the largest document it can process). */ - simdjson_inline size_t capacity() const noexcept; + simdjson_pure simdjson_inline size_t capacity() const noexcept; /** The maximum capacity of this parser (the largest document it is allowed to process). */ - simdjson_inline size_t max_capacity() const noexcept; + simdjson_pure simdjson_inline size_t max_capacity() const noexcept; simdjson_inline void set_max_capacity(size_t max_capacity) noexcept; /** * The maximum depth of this parser (the most deeply nested objects and arrays it can process). @@ -70767,7 +77769,7 @@ namespace simdjson { * The document's instance current_depth() method should be used to monitor the parsing * depth and limit it if desired. */ - simdjson_inline size_t max_depth() const noexcept; + simdjson_pure simdjson_inline size_t max_depth() const noexcept; /** * Ensure this parser has enough memory to process JSON documents up to `capacity` bytes in length @@ -70839,9 +77841,20 @@ namespace simdjson { */ simdjson_inline simdjson_result unescape_wobbly(raw_json_string in, uint8_t*& dst) const noexcept; +#if SIMDJSON_DEVELOPMENT_CHECKS + /** + * Returns true if string_buf_loc is outside of the allocated range for the + * the string buffer. When true, it indicates that the string buffer has overflowed. + * This is a development-time check that is not needed in production. It can be + * used to detect buffer overflows in the string buffer and usafe usage of the + * string buffer. + */ + bool string_buffer_overflow(const uint8_t* string_buf_loc) const noexcept; +#endif + private: /** @private [for benchmarking access] The implementation to use */ - std::unique_ptr implementation{}; + std::unique_ptr implementation{}; size_t _capacity{ 0 }; size_t _max_capacity; size_t _max_depth{ DEFAULT_MAX_DEPTH }; @@ -70978,6 +77991,22 @@ namespace simdjson { * - INVALID_JSON_POINTER if the JSON pointer is invalid and cannot be parsed */ inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; + + /** + * Get the value associated with the given JSONPath expression. We only support + * JSONPath queries that trivially convertible to JSON Pointer queries: key + * names and array indices. + * + * https://datatracker.ietf.org/doc/html/draft-normington-jsonpath-00 + * + * @return The value associated with the given JSONPath expression, or: + * - INVALID_JSON_POINTER if the JSONPath to JSON Pointer conversion fails + * - NO_SUCH_FIELD if a field does not exist in an object + * - INDEX_OUT_OF_BOUNDS if an array index is larger than an array length + * - INCORRECT_TYPE if a non-integer is used to access an array + */ + inline simdjson_result at_path(std::string_view json_path) noexcept; + /** * Consumes the array and returns a string_view instance corresponding to the * array as represented in JSON. It points inside the original document. @@ -71068,6 +78097,7 @@ namespace simdjson { inline simdjson_result reset() & noexcept; simdjson_inline simdjson_result at(size_t index) noexcept; simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; + simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; simdjson_inline simdjson_result raw_json() noexcept; }; @@ -71358,21 +78388,23 @@ namespace simdjson { * @returns INCORRECT_TYPE If the JSON value is not the given type. */ template simdjson_inline simdjson_result get() & noexcept { - // Unless the simdjson library provides an inline implementation, calling this method should + // Unless the simdjson library or the user provides an inline implementation, calling this method should // immediately fail. static_assert(!sizeof(T), "The get method with given type is not implemented by the simdjson library. " "The supported types are ondemand::object, ondemand::array, raw_json_string, std::string_view, uint64_t, " "int64_t, double, and bool. We recommend you use get_double(), get_bool(), get_uint64(), get_int64(), " - " get_object(), get_array(), get_raw_json_string(), or get_string() instead of the get template."); + " get_object(), get_array(), get_raw_json_string(), or get_string() instead of the get template." + " You may also add support for custom types, see our documentation."); } /** @overload template simdjson_result get() & noexcept */ - template simdjson_inline simdjson_result get() && noexcept { - // Unless the simdjson library provides an inline implementation, calling this method should + template simdjson_deprecated simdjson_inline simdjson_result get() && noexcept { + // Unless the simdjson library or the user provides an inline implementation, calling this method should // immediately fail. static_assert(!sizeof(T), "The get method with given type is not implemented by the simdjson library. " "The supported types are ondemand::object, ondemand::array, raw_json_string, std::string_view, uint64_t, " "int64_t, double, and bool. We recommend you use get_double(), get_bool(), get_uint64(), get_int64(), " - " get_object(), get_array(), get_raw_json_string(), or get_string() instead of the get template."); + " get_object(), get_array(), get_raw_json_string(), or get_string() instead of the get template." + " You may also add support for custom types, see our documentation."); } /** @@ -71388,9 +78420,23 @@ namespace simdjson { */ template simdjson_inline error_code get(T& out) & noexcept; /** @overload template error_code get(T &out) & noexcept */ - template simdjson_inline error_code get(T& out) && noexcept; + template simdjson_deprecated simdjson_inline error_code get(T& out) && noexcept; #if SIMDJSON_EXCEPTIONS + /** + * Cast this JSON value to an instance of type T. The programmer is responsible for + * providing an implementation of get for the type T, if T is not one of the types + * supported by the library (object, array, raw_json_string, string_view, uint64_t, etc.) + * + * See https://github.com/simdjson/simdjson/blob/master/doc/basics.md#adding-support-for-custom-types + * + * @returns An instance of type T + */ + template + explicit simdjson_inline operator T() & noexcept(false); + template + explicit simdjson_deprecated simdjson_inline operator T() && noexcept(false); + /** * Cast this JSON value to an array. * @@ -71563,7 +78609,7 @@ namespace simdjson { * APIs assume this. Therefore, you must be explicit if you want to treat objects as out of order. * * Use find_field() if you are sure fields will be in order (or are willing to treat it as if the - * field wasn't there when they aren't). + * field was not there when they are not in order). * * You must consume the fields on an object one at a time. A request for a new key * invalidates previous field values: it makes them unsafe. E.g., the array @@ -71609,6 +78655,14 @@ namespace simdjson { */ simdjson_inline simdjson_result is_scalar() noexcept; + /** + * Checks whether the document is a string. + * + * @returns true if the type is string + * @error TAPE_ERROR when the JSON value is a bad token like "}" "," or "alse". + */ + simdjson_inline simdjson_result is_string() noexcept; + /** * Checks whether the document is a negative number. * @@ -71636,9 +78690,11 @@ namespace simdjson { * get_number().get_number_type(). * * get_number_type() is number_type::unsigned_integer if we have - * an integer greater or equal to 9223372036854775808 + * an integer greater or equal to 9223372036854775808 and no larger than 18446744073709551615. * get_number_type() is number_type::signed_integer if we have an - * integer that is less than 9223372036854775808 + * integer that is less than 9223372036854775808 and greater or equal to -9223372036854775808. + * get_number_type() is number_type::big_integer if we have an integer outside + * of those ranges (either larger than 18446744073709551615 or smaller than -9223372036854775808). * Otherwise, get_number_type() has value number_type::floating_point_number * * This function requires processing the number string, but it is expected @@ -71756,6 +78812,14 @@ namespace simdjson { * auto doc = parser.iterate(json); * doc.at_pointer("//a/1") == 20 * + * Key values are matched exactly, without unescaping or Unicode normalization. + * We do a byte-by-byte comparison. E.g. + * + * const padded_string json = "{\"\\u00E9\":123}"_padded; + * auto doc = parser.iterate(json); + * doc.at_pointer("/\\u00E9") == 123 + * doc.at_pointer((const char*)u8"/\u00E9") returns an error (NO_SUCH_FIELD) + * * Note that at_pointer() automatically calls rewind between each call. Thus * all values, objects and arrays that you have created so far (including unescaped strings) * are invalidated. After calling at_pointer, you need to consume the result: string values @@ -71772,6 +78836,30 @@ namespace simdjson { * - SCALAR_DOCUMENT_AS_VALUE if the json_pointer is empty and the document is not a scalar (see is_scalar() function). */ simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; + + /** + * Get the value associated with the given JSONPath expression. We only support + * JSONPath queries that trivially convertible to JSON Pointer queries: key + * names and array indices. + * + * https://datatracker.ietf.org/doc/html/draft-normington-jsonpath-00 + * + * Key values are matched exactly, without unescaping or Unicode normalization. + * We do a byte-by-byte comparison. E.g. + * + * const padded_string json = "{\"\\u00E9\":123}"_padded; + * auto doc = parser.iterate(json); + * doc.at_path(".\\u00E9") == 123 + * doc.at_path((const char*)u8".\u00E9") returns an error (NO_SUCH_FIELD) + * + * @return The value associated with the given JSONPath expression, or: + * - INVALID_JSON_POINTER if the JSONPath to JSON Pointer conversion fails + * - NO_SUCH_FIELD if a field does not exist in an object + * - INDEX_OUT_OF_BOUNDS if an array index is larger than an array length + * - INCORRECT_TYPE if a non-integer is used to access an array + */ + simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; + /** * Consumes the document and returns a string_view instance corresponding to the * document as represented in JSON. It points inside the original byte array containing @@ -71837,10 +78925,12 @@ namespace simdjson { simdjson_inline simdjson_result get_value() noexcept; simdjson_inline simdjson_result is_null() noexcept; + template simdjson_inline simdjson_result get() & noexcept; simdjson_inline simdjson_result raw_json() noexcept; simdjson_inline operator document& () const noexcept; - #if SIMDJSON_EXCEPTIONS + template + explicit simdjson_inline operator T() noexcept(false); simdjson_inline operator array() & noexcept(false); simdjson_inline operator object() & noexcept(false); simdjson_inline operator uint64_t() noexcept(false); @@ -71865,6 +78955,7 @@ namespace simdjson { simdjson_inline simdjson_result type() noexcept; simdjson_inline simdjson_result is_scalar() noexcept; + simdjson_inline simdjson_result is_string() noexcept; simdjson_inline simdjson_result current_location() noexcept; simdjson_inline int32_t current_depth() const noexcept; @@ -71874,6 +78965,8 @@ namespace simdjson { simdjson_inline simdjson_result get_number() noexcept; simdjson_inline simdjson_result raw_json_token() noexcept; simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; + simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; + private: document* doc{ nullptr }; }; @@ -71909,12 +79002,13 @@ namespace simdjson { simdjson_inline simdjson_result is_null() noexcept; template simdjson_inline simdjson_result get() & noexcept; - template simdjson_inline simdjson_result get() && noexcept; + template simdjson_deprecated simdjson_inline simdjson_result get() && noexcept; template simdjson_inline error_code get(T& out) & noexcept; template simdjson_inline error_code get(T& out) && noexcept; - #if SIMDJSON_EXCEPTIONS + template ::value == false>::type> + explicit simdjson_inline operator T() noexcept(false); simdjson_inline operator ppc64::ondemand::array() & noexcept(false); simdjson_inline operator ppc64::ondemand::object() & noexcept(false); simdjson_inline operator uint64_t() noexcept(false); @@ -71938,6 +79032,7 @@ namespace simdjson { simdjson_inline simdjson_result find_field_unordered(const char* key) & noexcept; simdjson_inline simdjson_result type() noexcept; simdjson_inline simdjson_result is_scalar() noexcept; + simdjson_inline simdjson_result is_string() noexcept; simdjson_inline simdjson_result current_location() noexcept; simdjson_inline int32_t current_depth() const noexcept; simdjson_inline bool at_end() const noexcept; @@ -71949,6 +79044,7 @@ namespace simdjson { simdjson_inline simdjson_result raw_json_token() noexcept; simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; + simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; }; @@ -71981,8 +79077,9 @@ namespace simdjson { simdjson_inline simdjson_result get_bool() noexcept; simdjson_inline simdjson_result get_value() noexcept; simdjson_inline simdjson_result is_null() noexcept; - #if SIMDJSON_EXCEPTIONS + template ::value == false>::type> + explicit simdjson_inline operator T() noexcept(false); simdjson_inline operator ppc64::ondemand::array() & noexcept(false); simdjson_inline operator ppc64::ondemand::object() & noexcept(false); simdjson_inline operator uint64_t() noexcept(false); @@ -72006,6 +79103,7 @@ namespace simdjson { simdjson_inline simdjson_result find_field_unordered(const char* key) & noexcept; simdjson_inline simdjson_result type() noexcept; simdjson_inline simdjson_result is_scalar() noexcept; + simdjson_inline simdjson_result is_string() noexcept; simdjson_inline simdjson_result current_location() noexcept; simdjson_inline simdjson_result current_depth() const noexcept; simdjson_inline simdjson_result is_negative() noexcept; @@ -72016,6 +79114,7 @@ namespace simdjson { simdjson_inline simdjson_result raw_json_token() noexcept; simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; + simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; }; @@ -72149,10 +79248,9 @@ namespace simdjson { class iterator { public: using value_type = simdjson_result; - using reference = value_type; - + using reference = simdjson_result; + using pointer = void; using difference_type = std::ptrdiff_t; - using iterator_category = std::input_iterator_tag; /** @@ -72162,7 +79260,7 @@ namespace simdjson { /** * Get the current document (or error). */ - simdjson_inline simdjson_result operator*() noexcept; + simdjson_inline reference operator*() noexcept; /** * Advance to the next document (prefix). */ @@ -72344,7 +79442,7 @@ namespace simdjson { friend class document; friend class json_iterator; friend struct simdjson_result; - friend struct internal::simdjson_result_base; + friend struct simdjson::internal::simdjson_result_base; }; // document_stream } // namespace ondemand @@ -72404,12 +79502,39 @@ namespace simdjson { * This consumes the key: once you have called unescaped_key(), you cannot * call it again nor can you call key(). */ - simdjson_inline simdjson_warn_unused simdjson_result unescaped_key(bool allow_replacement) noexcept; + simdjson_inline simdjson_warn_unused simdjson_result unescaped_key(bool allow_replacement = false) noexcept; + /** + * Get the key as a string_view (for higher speed, consider raw_key). + * We deliberately use a more cumbersome name (unescaped_key) to force users + * to think twice about using it. The content is stored in the receiver. + * + * This consumes the key: once you have called unescaped_key(), you cannot + * call it again nor can you call key(). + */ + template + simdjson_inline simdjson_warn_unused error_code unescaped_key(string_type& receiver, bool allow_replacement = false) noexcept; /** * Get the key as a raw_json_string. Can be used for direct comparison with - * an unescaped C string: e.g., key() == "test". + * an unescaped C string: e.g., key() == "test". This does not count as + * consumption of the content: you can safely call it repeatedly. + * See escaped_key() for a similar function which returns + * a more convenient std::string_view result. */ simdjson_inline raw_json_string key() const noexcept; + /** + * Get the unprocessed key as a string_view. This includes the quotes and may include + * some spaces after the last quote. This does not count as + * consumption of the content: you can safely call it repeatedly. + * See escaped_key(). + */ + simdjson_inline std::string_view key_raw_json_token() const noexcept; + /** + * Get the key as a string_view. This does not include the quotes and + * the string is unprocessed key so it may contain escape characters + * (e.g., \uXXXX or \n). It does not count as a consumption of the content: + * you can safely call it repeatedly. Use unescaped_key() to get the unescaped key. + */ + simdjson_inline std::string_view escaped_key() const noexcept; /** * Get the field value. */ @@ -72441,7 +79566,11 @@ namespace simdjson { simdjson_inline simdjson_result() noexcept = default; simdjson_inline simdjson_result unescaped_key(bool allow_replacement = false) noexcept; + template + simdjson_inline error_code unescaped_key(string_type& receiver, bool allow_replacement = false) noexcept; simdjson_inline simdjson_result key() noexcept; + simdjson_inline simdjson_result key_raw_json_token() noexcept; + simdjson_inline simdjson_result escaped_key() noexcept; simdjson_inline simdjson_result value() noexcept; }; @@ -72509,6 +79638,8 @@ namespace simdjson { * key a single time. Doing object["mykey"].to_string() and then again object["mykey"].to_string() * is an error. * + * If you expect to have keys with escape characters, please review our documentation. + * * @param key The key to look up. * @returns The value of the field, or NO_SUCH_FIELD if the field is not in the object. */ @@ -72530,7 +79661,7 @@ namespace simdjson { * APIs assume this. Therefore, you must be explicit if you want to treat objects as out of order. * * Use find_field() if you are sure fields will be in order (or are willing to treat it as if the - * field wasn't there when they aren't). + * field was not there when they are not in order). * * If you have multiple fields with a matching key ({"x": 1, "x": 1}) be mindful * that only one field is returned. @@ -72546,6 +79677,8 @@ namespace simdjson { * You are expected to access keys only once. You should access the value corresponding to a key * a single time. Doing object["mykey"].to_string() and then again object["mykey"].to_string() is an error. * + * If you expect to have keys with escape characters, please review our documentation. + * * @param key The key to look up. * @returns The value of the field, or NO_SUCH_FIELD if the field is not in the object. */ @@ -72593,11 +79726,24 @@ namespace simdjson { */ inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; + /** + * Get the value associated with the given JSONPath expression. We only support + * JSONPath queries that trivially convertible to JSON Pointer queries: key + * names and array indices. + * + * @return The value associated with the given JSONPath expression, or: + * - INVALID_JSON_POINTER if the JSONPath to JSON Pointer conversion fails + * - NO_SUCH_FIELD if a field does not exist in an object + * - INDEX_OUT_OF_BOUNDS if an array index is larger than an array length + * - INCORRECT_TYPE if a non-integer is used to access an array + */ + inline simdjson_result at_path(std::string_view json_path) noexcept; + /** * Reset the iterator so that we are pointing back at the * beginning of the object. You should still consume values only once even if you - * can iterate through the object more than once. If you unescape a string within - * the object more than once, you have unsafe code. Note that rewinding an object + * can iterate through the object more than once. If you unescape a string or a key + * within the object more than once, you have unsafe code. Note that rewinding an object * means that you may need to reparse it anew: it is not a free operation. * * @returns true if the object contains some elements (not empty) @@ -72680,6 +79826,8 @@ namespace simdjson { simdjson_inline simdjson_result operator[](std::string_view key) & noexcept; simdjson_inline simdjson_result operator[](std::string_view key) && noexcept; simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; + simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; + inline simdjson_result reset() noexcept; inline simdjson_result is_empty() noexcept; inline simdjson_result count_fields() & noexcept; @@ -73053,6 +80201,65 @@ namespace simdjson { return child; } + inline std::string json_path_to_pointer_conversion(std::string_view json_path) { + if (json_path.empty() || (json_path.front() != '.' && + json_path.front() != '[')) { + return "-1"; // This is just a sentinel value, the caller should check for this and return an error. + } + + std::string result; + // Reserve space to reduce allocations, adjusting for potential increases due + // to escaping. + result.reserve(json_path.size() * 2); + + size_t i = 0; + + while (i < json_path.length()) { + if (json_path[i] == '.') { + result += '/'; + } + else if (json_path[i] == '[') { + result += '/'; + ++i; // Move past the '[' + while (i < json_path.length() && json_path[i] != ']') { + if (json_path[i] == '~') { + result += "~0"; + } + else if (json_path[i] == '/') { + result += "~1"; + } + else { + result += json_path[i]; + } + ++i; + } + if (i == json_path.length() || json_path[i] != ']') { + return "-1"; // Using sentinel value that will be handled as an error by the caller. + } + } + else { + if (json_path[i] == '~') { + result += "~0"; + } + else if (json_path[i] == '/') { + result += "~1"; + } + else { + result += json_path[i]; + } + } + ++i; + } + + return result; + } + + inline simdjson_result array::at_path(std::string_view json_path) noexcept { + auto json_pointer = json_path_to_pointer_conversion(json_path); + if (json_pointer == "-1") { return INVALID_JSON_POINTER; } + return at_pointer(json_pointer); + } + simdjson_inline simdjson_result array::at(size_t index) noexcept { size_t i = 0; for (auto value : *this) { @@ -73107,6 +80314,10 @@ namespace simdjson { if (error()) { return error(); } return first.at_pointer(json_pointer); } + simdjson_inline simdjson_result simdjson_result::at_path(std::string_view json_path) noexcept { + if (error()) { return error(); } + return first.at_path(json_path); + } simdjson_inline simdjson_result simdjson_result::raw_json() noexcept { if (error()) { return error(); } return first.raw_json(); @@ -73203,14 +80414,14 @@ namespace simdjson { /* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ /* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_ONDEMAND_DOCUMENT_INL_H */ /* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/array-inl.h" */ /* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/array_iterator.h" */ /* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/document.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/json_iterator-inl.h" */ /* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/json_type.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/object-inl.h" */ /* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/raw_json_string.h" */ /* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/value.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/array-inl.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/json_iterator-inl.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/object-inl.h" */ /* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/value_iterator-inl.h" */ /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ @@ -73366,22 +80577,26 @@ namespace simdjson { template<> simdjson_inline simdjson_result document::get() & noexcept { return get_bool(); } template<> simdjson_inline simdjson_result document::get() & noexcept { return get_value(); } - template<> simdjson_inline simdjson_result document::get() && noexcept { return get_raw_json_string(); } - template<> simdjson_inline simdjson_result document::get() && noexcept { return get_string(false); } - template<> simdjson_inline simdjson_result document::get() && noexcept { return std::forward(*this).get_double(); } - template<> simdjson_inline simdjson_result document::get() && noexcept { return std::forward(*this).get_uint64(); } - template<> simdjson_inline simdjson_result document::get() && noexcept { return std::forward(*this).get_int64(); } - template<> simdjson_inline simdjson_result document::get() && noexcept { return std::forward(*this).get_bool(); } - template<> simdjson_inline simdjson_result document::get() && noexcept { return get_value(); } + template<> simdjson_deprecated simdjson_inline simdjson_result document::get() && noexcept { return get_raw_json_string(); } + template<> simdjson_deprecated simdjson_inline simdjson_result document::get() && noexcept { return get_string(false); } + template<> simdjson_deprecated simdjson_inline simdjson_result document::get() && noexcept { return std::forward(*this).get_double(); } + template<> simdjson_deprecated simdjson_inline simdjson_result document::get() && noexcept { return std::forward(*this).get_uint64(); } + template<> simdjson_deprecated simdjson_inline simdjson_result document::get() && noexcept { return std::forward(*this).get_int64(); } + template<> simdjson_deprecated simdjson_inline simdjson_result document::get() && noexcept { return std::forward(*this).get_bool(); } + template<> simdjson_deprecated simdjson_inline simdjson_result document::get() && noexcept { return get_value(); } template simdjson_inline error_code document::get(T& out) & noexcept { return get().get(out); } - template simdjson_inline error_code document::get(T& out) && noexcept { + template simdjson_deprecated simdjson_inline error_code document::get(T& out) && noexcept { return std::forward(*this).get().get(out); } #if SIMDJSON_EXCEPTIONS + template + simdjson_deprecated simdjson_inline document::operator T() && noexcept(false) { return get(); } + template + simdjson_inline document::operator T() & noexcept(false) { return get(); } simdjson_inline document::operator array() & noexcept(false) { return get_array(); } simdjson_inline document::operator object() & noexcept(false) { return get_object(); } simdjson_inline document::operator uint64_t() noexcept(false) { return get_uint64(); } @@ -73466,6 +80681,13 @@ namespace simdjson { return !((this_type == json_type::array) || (this_type == json_type::object)); } + simdjson_inline simdjson_result document::is_string() noexcept { + json_type this_type; + auto error = type().get(this_type); + if (error) { return error; } + return (this_type == json_type::string); + } + simdjson_inline bool document::is_negative() noexcept { return get_root_value_iterator().is_root_negative(); } @@ -73485,7 +80707,7 @@ namespace simdjson { simdjson_inline simdjson_result document::raw_json_token() noexcept { auto _iter = get_root_value_iterator(); - return std::string_view(reinterpret_cast(_iter.peek_start()), _iter.peek_start_length()); + return std::string_view(reinterpret_cast(_iter.peek_start()), _iter.peek_root_length()); } simdjson_inline simdjson_result document::at_pointer(std::string_view json_pointer) noexcept { @@ -73506,6 +80728,23 @@ namespace simdjson { } } + simdjson_inline simdjson_result document::at_path(std::string_view json_path) noexcept { + rewind(); // Rewind the document each time at_pointer is called + if (json_path.empty()) { + return this->get_value(); + } + json_type t; + SIMDJSON_TRY(type().get(t)); + switch (t) { + case json_type::array: + return (*this).get_array().at_path(json_path); + case json_type::object: + return (*this).get_object().at_path(json_path); + default: + return INVALID_JSON_POINTER; + } + } + } // namespace ondemand } // namespace ppc64 } // namespace simdjson @@ -73644,7 +80883,7 @@ namespace simdjson { return first.get(); } template - simdjson_inline simdjson_result simdjson_result::get() && noexcept { + simdjson_deprecated simdjson_inline simdjson_result simdjson_result::get() && noexcept { if (error()) { return error(); } return std::forward(first).get(); } @@ -73660,7 +80899,7 @@ namespace simdjson { } template<> simdjson_inline simdjson_result simdjson_result::get() & noexcept = delete; - template<> simdjson_inline simdjson_result simdjson_result::get() && noexcept { + template<> simdjson_deprecated simdjson_inline simdjson_result simdjson_result::get() && noexcept { if (error()) { return error(); } return std::forward(first); } @@ -73681,6 +80920,10 @@ namespace simdjson { return first.is_scalar(); } + simdjson_inline simdjson_result simdjson_result::is_string() noexcept { + if (error()) { return error(); } + return first.is_string(); + } simdjson_inline bool simdjson_result::is_negative() noexcept { if (error()) { return error(); } @@ -73704,6 +80947,11 @@ namespace simdjson { #if SIMDJSON_EXCEPTIONS + template ::value == false>::type> + simdjson_inline simdjson_result::operator T() noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return first; + } simdjson_inline simdjson_result::operator ppc64::ondemand::array() & noexcept(false) { if (error()) { throw simdjson_error(error()); } return first; @@ -73769,6 +81017,10 @@ namespace simdjson { return first.at_pointer(json_pointer); } + simdjson_inline simdjson_result simdjson_result::at_path(std::string_view json_path) noexcept { + if (error()) { return error(); } + return first.at_path(json_path); + } } // namespace simdjson @@ -73810,15 +81062,25 @@ namespace simdjson { simdjson_inline simdjson_result document_reference::get_bool() noexcept { return doc->get_root_value_iterator().get_root_bool(false); } simdjson_inline simdjson_result document_reference::get_value() noexcept { return doc->get_value(); } simdjson_inline simdjson_result document_reference::is_null() noexcept { return doc->get_root_value_iterator().is_root_null(false); } - + template<> simdjson_inline simdjson_result document_reference::get() & noexcept { return get_array(); } + template<> simdjson_inline simdjson_result document_reference::get() & noexcept { return get_object(); } + template<> simdjson_inline simdjson_result document_reference::get() & noexcept { return get_raw_json_string(); } + template<> simdjson_inline simdjson_result document_reference::get() & noexcept { return get_string(false); } + template<> simdjson_inline simdjson_result document_reference::get() & noexcept { return get_double(); } + template<> simdjson_inline simdjson_result document_reference::get() & noexcept { return get_uint64(); } + template<> simdjson_inline simdjson_result document_reference::get() & noexcept { return get_int64(); } + template<> simdjson_inline simdjson_result document_reference::get() & noexcept { return get_bool(); } + template<> simdjson_inline simdjson_result document_reference::get() & noexcept { return get_value(); } #if SIMDJSON_EXCEPTIONS + template + simdjson_inline document_reference::operator T() noexcept(false) { return get(); } simdjson_inline document_reference::operator array() & noexcept(false) { return array(*doc); } simdjson_inline document_reference::operator object() & noexcept(false) { return object(*doc); } simdjson_inline document_reference::operator uint64_t() noexcept(false) { return get_uint64(); } simdjson_inline document_reference::operator int64_t() noexcept(false) { return get_int64(); } simdjson_inline document_reference::operator double() noexcept(false) { return get_double(); } simdjson_inline document_reference::operator std::string_view() noexcept(false) { return std::string_view(*doc); } - simdjson_inline document_reference::operator raw_json_string() noexcept(false) { return raw_json_string(*doc); } + simdjson_inline document_reference::operator raw_json_string() noexcept(false) { return get_raw_json_string(); } simdjson_inline document_reference::operator bool() noexcept(false) { return get_bool(); } simdjson_inline document_reference::operator value() noexcept(false) { return value(*doc); } #endif @@ -73835,6 +81097,7 @@ namespace simdjson { simdjson_inline simdjson_result document_reference::find_field_unordered(const char* key) & noexcept { return doc->find_field_unordered(key); } simdjson_inline simdjson_result document_reference::type() noexcept { return doc->type(); } simdjson_inline simdjson_result document_reference::is_scalar() noexcept { return doc->is_scalar(); } + simdjson_inline simdjson_result document_reference::is_string() noexcept { return doc->is_string(); } simdjson_inline simdjson_result document_reference::current_location() noexcept { return doc->current_location(); } simdjson_inline int32_t document_reference::current_depth() const noexcept { return doc->current_depth(); } simdjson_inline bool document_reference::is_negative() noexcept { return doc->is_negative(); } @@ -73843,6 +81106,7 @@ namespace simdjson { simdjson_inline simdjson_result document_reference::get_number() noexcept { return doc->get_root_value_iterator().get_root_number(false); } simdjson_inline simdjson_result document_reference::raw_json_token() noexcept { return doc->raw_json_token(); } simdjson_inline simdjson_result document_reference::at_pointer(std::string_view json_pointer) noexcept { return doc->at_pointer(json_pointer); } + simdjson_inline simdjson_result document_reference::at_path(std::string_view json_path) noexcept { return doc->at_path(json_path); } simdjson_inline simdjson_result document_reference::raw_json() noexcept { return doc->raw_json(); } simdjson_inline document_reference::operator document& () const noexcept { return *doc; } @@ -73974,6 +81238,10 @@ namespace simdjson { if (error()) { return error(); } return first.is_scalar(); } + simdjson_inline simdjson_result simdjson_result::is_string() noexcept { + if (error()) { return error(); } + return first.is_string(); + } simdjson_inline simdjson_result simdjson_result::is_negative() noexcept { if (error()) { return error(); } return first.is_negative(); @@ -73991,6 +81259,11 @@ namespace simdjson { return first.get_number(); } #if SIMDJSON_EXCEPTIONS + template ::value == false>::type> + simdjson_inline simdjson_result::operator T() noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return first; + } simdjson_inline simdjson_result::operator ppc64::ondemand::array() & noexcept(false) { if (error()) { throw simdjson_error(error()); } return first; @@ -74044,6 +81317,12 @@ namespace simdjson { return first.at_pointer(json_pointer); } + simdjson_inline simdjson_result simdjson_result::at_path(std::string_view json_path) noexcept { + if (error()) { + return error(); + } + return first.at_path(json_path); + } } // namespace simdjson @@ -74397,9 +81676,18 @@ namespace simdjson { depth--; break; default: // Scalar value document - // TODO: Remove any trailing whitespaces + // TODO: We could remove trailing whitespaces // This returns a string spanning from start of value to the beginning of the next document (excluded) - return std::string_view(reinterpret_cast(stream->buf) + current_index(), stream->parser->implementation->structural_indexes[++cur_struct_index] - current_index() - 1); + { + auto next_index = stream->parser->implementation->structural_indexes[++cur_struct_index]; + // normally the length would be next_index - current_index() - 1, except for the last document + size_t svlen = next_index - current_index(); + const char* start = reinterpret_cast(stream->buf) + current_index(); + while (svlen > 1 && (std::isspace(start[svlen - 1]) || start[svlen - 1] == '\0')) { + svlen--; + } + return std::string_view(start, svlen); + } } cur_struct_index++; } @@ -74494,7 +81782,7 @@ namespace simdjson { namespace ppc64 { namespace ondemand { - // clang 6 doesn't think the default constructor can be noexcept, so we make it explicit + // clang 6 does not think the default constructor can be noexcept, so we make it explicit simdjson_inline field::field() noexcept : std::pair() {} simdjson_inline field::field(raw_json_string key, ondemand::value&& value) noexcept @@ -74520,11 +81808,32 @@ namespace simdjson { return answer; } + template + simdjson_inline simdjson_warn_unused error_code field::unescaped_key(string_type& receiver, bool allow_replacement) noexcept { + std::string_view key; + SIMDJSON_TRY(unescaped_key(allow_replacement).get(key)); + receiver = key; + return SUCCESS; + } + simdjson_inline raw_json_string field::key() const noexcept { SIMDJSON_ASSUME(first.buf != nullptr); // We would like to call .alive() by Visual Studio won't let us. return first; } + + simdjson_inline std::string_view field::key_raw_json_token() const noexcept { + SIMDJSON_ASSUME(first.buf != nullptr); // We would like to call .alive() by Visual Studio won't let us. + return std::string_view(reinterpret_cast(first.buf - 1), second.iter._json_iter->token.peek(-1) - first.buf + 1); + } + + simdjson_inline std::string_view field::escaped_key() const noexcept { + SIMDJSON_ASSUME(first.buf != nullptr); // We would like to call .alive() by Visual Studio won't let us. + auto end_quote = second.iter._json_iter->token.peek(-1); + while (*end_quote != '"') end_quote--; + return std::string_view(reinterpret_cast(first.buf), end_quote - first.buf); + } + simdjson_inline value& field::value() & noexcept { return second; } @@ -74558,10 +81867,28 @@ namespace simdjson { if (error()) { return error(); } return first.key(); } + + simdjson_inline simdjson_result simdjson_result::key_raw_json_token() noexcept { + if (error()) { return error(); } + return first.key_raw_json_token(); + } + + simdjson_inline simdjson_result simdjson_result::escaped_key() noexcept { + if (error()) { return error(); } + return first.escaped_key(); + } + simdjson_inline simdjson_result simdjson_result::unescaped_key(bool allow_replacement) noexcept { if (error()) { return error(); } return first.unescaped_key(allow_replacement); } + + template + simdjson_inline error_code simdjson_result::unescaped_key(string_type& receiver, bool allow_replacement) noexcept { + if (error()) { return error(); } + return first.unescaped_key(receiver, allow_replacement); + } + simdjson_inline simdjson_result simdjson_result::value() noexcept { if (error()) { return error(); } return std::move(first.value()); @@ -74629,6 +81956,23 @@ namespace simdjson { #endif } +#ifdef SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON + simdjson_inline json_iterator::json_iterator(const uint8_t* buf, ondemand::parser* _parser, bool streaming) noexcept + : token(buf, &_parser->implementation->structural_indexes[0]), + parser{ _parser }, + _string_buf_loc{ parser->string_buf.get() }, + _depth{ 1 }, + _root{ parser->implementation->structural_indexes.get() }, + _streaming{ streaming } + + { + logger::log_headers(); +#if SIMDJSON_CHECK_EOF + assert_more_tokens(); +#endif + } +#endif // SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON + inline void json_iterator::rewind() noexcept { token.set_position(root_position()); logger::log_headers(); // We start again @@ -74862,6 +82206,12 @@ namespace simdjson { #endif // SIMDJSON_CHECK_EOF return token.peek_length(position); } + simdjson_inline uint32_t json_iterator::peek_root_length(token_position position) const noexcept { +#if SIMDJSON_CHECK_EOF + assert_valid_position(position); +#endif // SIMDJSON_CHECK_EOF + return token.peek_root_length(position); + } simdjson_inline token_position json_iterator::last_position() const noexcept { // The following line fails under some compilers... @@ -74907,11 +82257,23 @@ namespace simdjson { } simdjson_inline simdjson_result json_iterator::unescape(raw_json_string in, bool allow_replacement) noexcept { +#if SIMDJSON_DEVELOPMENT_CHECKS + auto result = parser->unescape(in, _string_buf_loc, allow_replacement); + SIMDJSON_ASSUME(!parser->string_buffer_overflow(_string_buf_loc)); + return result; +#else return parser->unescape(in, _string_buf_loc, allow_replacement); +#endif } simdjson_inline simdjson_result json_iterator::unescape_wobbly(raw_json_string in) noexcept { +#if SIMDJSON_DEVELOPMENT_CHECKS + auto result = parser->unescape_wobbly(in, _string_buf_loc); + SIMDJSON_ASSUME(!parser->string_buffer_overflow(_string_buf_loc)); + return result; +#else return parser->unescape_wobbly(in, _string_buf_loc); +#endif } simdjson_inline void json_iterator::reenter_child(token_position position, depth_t child_depth) noexcept { @@ -75036,7 +82398,6 @@ namespace simdjson { return get_uint64(); } - simdjson_inline bool number::is_int64() const noexcept { return get_number_type() == number_type::signed_integer; } @@ -75504,6 +82865,14 @@ namespace simdjson { return child; } + inline simdjson_result object::at_path(std::string_view json_path) noexcept { + auto json_pointer = json_path_to_pointer_conversion(json_path); + if (json_pointer == "-1") { + return INVALID_JSON_POINTER; + } + return at_pointer(json_pointer); + } + simdjson_inline simdjson_result object::count_fields() & noexcept { size_t count{ 0 }; // Important: we do not consume any of the values. @@ -75576,6 +82945,14 @@ namespace simdjson { return first.at_pointer(json_pointer); } + simdjson_inline simdjson_result simdjson_result::at_path( + std::string_view json_path) noexcept { + if (error()) { + return error(); + } + return first.at_path(json_path); + } + inline simdjson_result simdjson_result::reset() noexcept { if (error()) { return error(); } return first.reset(); @@ -75787,6 +83164,11 @@ namespace simdjson { _max_depth = new_max_depth; return SUCCESS; } +#if SIMDJSON_DEVELOPMENT_CHECKS + simdjson_inline simdjson_warn_unused bool parser::string_buffer_overflow(const uint8_t* string_buf_loc) const noexcept { + return (string_buf_loc < string_buf.get()) || (size_t(string_buf_loc - string_buf.get()) >= capacity()); + } +#endif simdjson_warn_unused simdjson_inline simdjson_result parser::iterate(padded_string_view json) & noexcept { if (json.padding() < SIMDJSON_PADDING) { return INSUFFICIENT_PADDING; } @@ -75803,6 +83185,27 @@ namespace simdjson { return document::start({ reinterpret_cast(json.data()), this }); } +#ifdef SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON + simdjson_warn_unused simdjson_inline simdjson_result parser::iterate_allow_incomplete_json(padded_string_view json) & noexcept { + if (json.padding() < SIMDJSON_PADDING) { return INSUFFICIENT_PADDING; } + + json.remove_utf8_bom(); + + // Allocate if needed + if (capacity() < json.length() || !string_buf) { + SIMDJSON_TRY(allocate(json.length(), max_depth())); + } + + // Run stage 1. + const simdjson::error_code err = implementation->stage1(reinterpret_cast(json.data()), json.length(), stage1_mode::regular); + if (err) { + if (err != UNCLOSED_STRING) + return err; + } + return document::start({ reinterpret_cast(json.data()), this, true }); + } +#endif // SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON + simdjson_warn_unused simdjson_inline simdjson_result parser::iterate(const char* json, size_t len, size_t allocated) & noexcept { return iterate(padded_string_view(json, len, allocated)); } @@ -75874,13 +83277,13 @@ namespace simdjson { return iterate_many(s.data(), s.length(), batch_size, allow_comma_separated); } - simdjson_inline size_t parser::capacity() const noexcept { + simdjson_pure simdjson_inline size_t parser::capacity() const noexcept { return _capacity; } - simdjson_inline size_t parser::max_capacity() const noexcept { + simdjson_pure simdjson_inline size_t parser::max_capacity() const noexcept { return _max_capacity; } - simdjson_inline size_t parser::max_depth() const noexcept { + simdjson_pure simdjson_inline size_t parser::max_depth() const noexcept { return _max_depth; } @@ -76429,6 +83832,11 @@ namespace simdjson { return *(position + 1) - *position; } + simdjson_inline uint32_t token_iterator::peek_root_length(token_position position) const noexcept { + return *(position + 2) - *(position) > *(position + 1) - *(position) ? + *(position + 1) - *(position) + : *(position + 2) - *(position); + } simdjson_inline const uint8_t* token_iterator::peek(int32_t delta) const noexcept { return &buf[*(_position + delta)]; } @@ -76578,6 +83986,10 @@ namespace simdjson { } #if SIMDJSON_EXCEPTIONS + template + simdjson_inline value::operator T() noexcept(false) { + return get(); + } simdjson_inline value::operator array() noexcept(false) { return get_array(); } @@ -76664,6 +84076,14 @@ namespace simdjson { return !((this_type == json_type::array) || (this_type == json_type::object)); } + simdjson_inline simdjson_result value::is_string() noexcept { + json_type this_type; + auto error = type().get(this_type); + if (error) { return error; } + return (this_type == json_type::string); + } + + simdjson_inline bool value::is_negative() noexcept { return iter.is_negative(); } @@ -76710,6 +84130,26 @@ namespace simdjson { return iter.json_iter().depth(); } + inline bool is_pointer_well_formed(std::string_view json_pointer) noexcept { + if (simdjson_unlikely(json_pointer.empty())) { // can't be + return false; + } + if (simdjson_unlikely(json_pointer[0] != '/')) { + return false; + } + size_t escape = json_pointer.find('~'); + if (escape == std::string_view::npos) { + return true; + } + if (escape == json_pointer.size() - 1) { + return false; + } + if (json_pointer[escape + 1] != '0' && json_pointer[escape + 1] != '1') { + return false; + } + return true; + } + simdjson_inline simdjson_result value::at_pointer(std::string_view json_pointer) noexcept { json_type t; SIMDJSON_TRY(type().get(t)); @@ -76719,6 +84159,23 @@ namespace simdjson { return (*this).get_array().at_pointer(json_pointer); case json_type::object: return (*this).get_object().at_pointer(json_pointer); + default: + // a non-empty string can be invalid, or accessing a primitive (issue 2154) + if (is_pointer_well_formed(json_pointer)) { + return NO_SUCH_FIELD; + } + return INVALID_JSON_POINTER; + } + } + + simdjson_inline simdjson_result value::at_path(std::string_view json_path) noexcept { + json_type t; + SIMDJSON_TRY(type().get(t)); + switch (t) { + case json_type::array: + return (*this).get_array().at_path(json_path); + case json_type::object: + return (*this).get_object().at_path(json_path); default: return INVALID_JSON_POINTER; } @@ -76877,6 +84334,10 @@ namespace simdjson { if (error()) { return error(); } return first.is_scalar(); } + simdjson_inline simdjson_result simdjson_result::is_string() noexcept { + if (error()) { return error(); } + return first.is_string(); + } simdjson_inline simdjson_result simdjson_result::is_negative() noexcept { if (error()) { return error(); } return first.is_negative(); @@ -76894,6 +84355,11 @@ namespace simdjson { return first.get_number(); } #if SIMDJSON_EXCEPTIONS + template + simdjson_inline simdjson_result::operator T() noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return static_cast(first); + } simdjson_inline simdjson_result::operator ppc64::ondemand::array() noexcept(false) { if (error()) { throw simdjson_error(error()); } return first; @@ -76948,11 +84414,22 @@ namespace simdjson { return first.current_depth(); } - simdjson_inline simdjson_result simdjson_result::at_pointer(std::string_view json_pointer) noexcept { - if (error()) { return error(); } + simdjson_inline simdjson_result simdjson_result::at_pointer( + std::string_view json_pointer) noexcept { + if (error()) { + return error(); + } return first.at_pointer(json_pointer); } + simdjson_inline simdjson_result simdjson_result::at_path( + std::string_view json_path) noexcept { + if (error()) { + return error(); + } + return first.at_path(json_path); + } + } // namespace simdjson #endif // SIMDJSON_GENERIC_ONDEMAND_VALUE_INL_H @@ -76967,9 +84444,9 @@ namespace simdjson { /* amalgamation skipped (editor-only): #include "simdjson/generic/atomparsing.h" */ /* amalgamation skipped (editor-only): #include "simdjson/generic/numberparsing.h" */ /* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/json_iterator.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/value_iterator.h" */ /* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/json_type-inl.h" */ /* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/raw_json_string-inl.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/value_iterator.h" */ /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ namespace simdjson { @@ -77099,7 +84576,7 @@ namespace simdjson { else if (!is_open()) { #if SIMDJSON_DEVELOPMENT_CHECKS // If we're past the end of the object, we're being iterated out of order. - // Note: this isn't perfect detection. It's possible the user is inside some other object; if so, + // Note: this is not perfect detection. It's possible the user is inside some other object; if so, // this object iterator will blithely scan that object for fields. if (_json_iter->depth() < depth() - 1) { return OUT_OF_ORDER_ITERATION; } #endif @@ -77213,7 +84690,7 @@ namespace simdjson { #if SIMDJSON_DEVELOPMENT_CHECKS // If we're past the end of the object, we're being iterated out of order. - // Note: this isn't perfect detection. It's possible the user is inside some other object; if so, + // Note: this is not perfect detection. It's possible the user is inside some other object; if so, // this object iterator will blithely scan that object for fields. if (_json_iter->depth() < depth() - 1) { return OUT_OF_ORDER_ITERATION; } #endif @@ -77556,7 +85033,7 @@ namespace simdjson { } simdjson_inline simdjson_result value_iterator::is_root_integer(bool check_trailing) noexcept { - auto max_len = peek_start_length(); + auto max_len = peek_root_length(); auto json = peek_root_scalar("is_root_integer"); uint8_t tmpbuf[20 + 1 + 1]{}; // <20 digits> is the longest possible unsigned integer tmpbuf[20 + 1] = '\0'; // make sure that buffer is always null terminated. @@ -77572,7 +85049,7 @@ namespace simdjson { } simdjson_inline simdjson_result value_iterator::get_root_number_type(bool check_trailing) noexcept { - auto max_len = peek_start_length(); + auto max_len = peek_root_length(); auto json = peek_root_scalar("number"); // Per https://www.exploringbinary.com/maximum-number-of-decimal-digits-in-binary-floating-point-numbers/, // 1074 is the maximum number of significant fractional digits. Add 8 more digits for the biggest @@ -77580,7 +85057,12 @@ namespace simdjson { uint8_t tmpbuf[1074 + 8 + 1 + 1]; tmpbuf[1074 + 8 + 1] = '\0'; // make sure that buffer is always null terminated. if (!_json_iter->copy_to_buffer(json, max_len, tmpbuf, 1074 + 8 + 1)) { - logger::log_error(*_json_iter, start_position(), depth(), "Root number more than 1082 characters"); + if (numberparsing::check_if_integer(json, max_len)) { + if (check_trailing && !_json_iter->is_single_token()) { return TRAILING_CONTENT; } + logger::log_error(*_json_iter, start_position(), depth(), "Found big integer"); + return number_type::big_integer; + } + logger::log_error(*_json_iter, start_position(), depth(), "Root number more than 1082 characters and not a big integer"); return NUMBER_ERROR; } auto answer = numberparsing::get_number_type(tmpbuf); @@ -77588,15 +85070,21 @@ namespace simdjson { return answer; } simdjson_inline simdjson_result value_iterator::get_root_number(bool check_trailing) noexcept { - auto max_len = peek_start_length(); + auto max_len = peek_root_length(); auto json = peek_root_scalar("number"); // Per https://www.exploringbinary.com/maximum-number-of-decimal-digits-in-binary-floating-point-numbers/, // 1074 is the maximum number of significant fractional digits. Add 8 more digits for the biggest // number: -0.e-308. + // NOTE: the current approach doesn't work for very big integer numbers containing more than 1074 digits. uint8_t tmpbuf[1074 + 8 + 1 + 1]; tmpbuf[1074 + 8 + 1] = '\0'; // make sure that buffer is always null terminated. if (!_json_iter->copy_to_buffer(json, max_len, tmpbuf, 1074 + 8 + 1)) { - logger::log_error(*_json_iter, start_position(), depth(), "Root number more than 1082 characters"); + if (numberparsing::check_if_integer(json, max_len)) { + if (check_trailing && !_json_iter->is_single_token()) { return TRAILING_CONTENT; } + logger::log_error(*_json_iter, start_position(), depth(), "Found big integer"); + return BIGINT_ERROR; + } + logger::log_error(*_json_iter, start_position(), depth(), "Root number more than 1082 characters and not a big integer"); return NUMBER_ERROR; } number num; @@ -77628,7 +85116,7 @@ namespace simdjson { return raw_json_string(json + 1); } simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_root_uint64(bool check_trailing) noexcept { - auto max_len = peek_start_length(); + auto max_len = peek_root_length(); auto json = peek_root_scalar("uint64"); uint8_t tmpbuf[20 + 1 + 1]{}; // <20 digits> is the longest possible unsigned integer tmpbuf[20 + 1] = '\0'; // make sure that buffer is always null terminated. @@ -77644,7 +85132,7 @@ namespace simdjson { return result; } simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_root_uint64_in_string(bool check_trailing) noexcept { - auto max_len = peek_start_length(); + auto max_len = peek_root_length(); auto json = peek_root_scalar("uint64"); uint8_t tmpbuf[20 + 1 + 1]{}; // <20 digits> is the longest possible unsigned integer tmpbuf[20 + 1] = '\0'; // make sure that buffer is always null terminated. @@ -77660,7 +85148,7 @@ namespace simdjson { return result; } simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_root_int64(bool check_trailing) noexcept { - auto max_len = peek_start_length(); + auto max_len = peek_root_length(); auto json = peek_root_scalar("int64"); uint8_t tmpbuf[20 + 1 + 1]; // -<19 digits> is the longest possible integer tmpbuf[20 + 1] = '\0'; // make sure that buffer is always null terminated. @@ -77677,7 +85165,7 @@ namespace simdjson { return result; } simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_root_int64_in_string(bool check_trailing) noexcept { - auto max_len = peek_start_length(); + auto max_len = peek_root_length(); auto json = peek_root_scalar("int64"); uint8_t tmpbuf[20 + 1 + 1]; // -<19 digits> is the longest possible integer tmpbuf[20 + 1] = '\0'; // make sure that buffer is always null terminated. @@ -77694,7 +85182,7 @@ namespace simdjson { return result; } simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_root_double(bool check_trailing) noexcept { - auto max_len = peek_start_length(); + auto max_len = peek_root_length(); auto json = peek_root_scalar("double"); // Per https://www.exploringbinary.com/maximum-number-of-decimal-digits-in-binary-floating-point-numbers/, // 1074 is the maximum number of significant fractional digits. Add 8 more digits for the biggest @@ -77714,7 +85202,7 @@ namespace simdjson { } simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_root_double_in_string(bool check_trailing) noexcept { - auto max_len = peek_start_length(); + auto max_len = peek_root_length(); auto json = peek_root_scalar("double"); // Per https://www.exploringbinary.com/maximum-number-of-decimal-digits-in-binary-floating-point-numbers/, // 1074 is the maximum number of significant fractional digits. Add 8 more digits for the biggest @@ -77733,7 +85221,7 @@ namespace simdjson { return result; } simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_root_bool(bool check_trailing) noexcept { - auto max_len = peek_start_length(); + auto max_len = peek_root_length(); auto json = peek_root_scalar("bool"); uint8_t tmpbuf[5 + 1 + 1]; // +1 for null termination tmpbuf[5 + 1] = '\0'; // make sure that buffer is always null terminated. @@ -77746,7 +85234,7 @@ namespace simdjson { return result; } simdjson_inline simdjson_result value_iterator::is_root_null(bool check_trailing) noexcept { - auto max_len = peek_start_length(); + auto max_len = peek_root_length(); auto json = peek_root_scalar("null"); bool result = (max_len >= 4 && !atomparsing::str4ncmp(json, "null") && (max_len == 4 || jsoncharutils::is_structural_or_whitespace(json[4]))); @@ -77818,6 +85306,9 @@ namespace simdjson { simdjson_inline uint32_t value_iterator::peek_start_length() const noexcept { return _json_iter->peek_length(start_position()); } + simdjson_inline uint32_t value_iterator::peek_root_length() const noexcept { + return _json_iter->peek_root_length(start_position()); + } simdjson_inline const uint8_t* value_iterator::peek_scalar(const char* type) noexcept { logger::log_value(*_json_iter, start_position(), depth(), type); @@ -78355,13 +85846,13 @@ namespace simdjson { simdjson_inline internal::value128 full_multiplication(uint64_t value1, uint64_t value2) { internal::value128 answer; #if SIMDJSON_REGULAR_VISUAL_STUDIO || SIMDJSON_IS_32BITS -#ifdef _M_ARM64 +#if SIMDJSON_IS_ARM64 // ARM64 has native support for 64-bit multiplications, no need to emultate answer.high = __umulh(value1, value2); answer.low = value1 * value2; #else answer.low = _umul128(value1, value2, &answer.high); // _umul128 not available on ARM64 -#endif // _M_ARM64 +#endif // SIMDJSON_IS_ARM64 #else // SIMDJSON_REGULAR_VISUAL_STUDIO || SIMDJSON_IS_32BITS __uint128_t r = (static_cast<__uint128_t>(value1)) * value2; answer.low = uint64_t(r); @@ -78441,10 +85932,10 @@ namespace simdjson { struct simd8 : base8 { static simdjson_inline simd8 splat(bool _value) { return _mm_set1_epi8(uint8_t(-(!!_value))); } - simdjson_inline simd8() : base8() {} - simdjson_inline simd8(const __m128i _value) : base8(_value) {} + simdjson_inline simd8() : base8() {} + simdjson_inline simd8(const __m128i _value) : base8(_value) {} // Splat constructor - simdjson_inline simd8(bool _value) : base8(splat(_value)) {} + simdjson_inline simd8(bool _value) : base8(splat(_value)) {} simdjson_inline int to_bitmask() const { return _mm_movemask_epi8(*this); } simdjson_inline bool any() const { return !_mm_testz_si128(*this, *this); } @@ -78869,10 +86360,10 @@ namespace simdjson { struct simd8 : base8 { static simdjson_inline simd8 splat(bool _value) { return _mm_set1_epi8(uint8_t(-(!!_value))); } - simdjson_inline simd8() : base8() {} - simdjson_inline simd8(const __m128i _value) : base8(_value) {} + simdjson_inline simd8() : base8() {} + simdjson_inline simd8(const __m128i _value) : base8(_value) {} // Splat constructor - simdjson_inline simd8(bool _value) : base8(splat(_value)) {} + simdjson_inline simd8(bool _value) : base8(splat(_value)) {} simdjson_inline int to_bitmask() const { return _mm_movemask_epi8(*this); } simdjson_inline bool any() const { return !_mm_testz_si128(*this, *this); } @@ -79627,6 +87118,7 @@ namespace simdjson { simdjson_inline simdjson_result parse_bool(const uint8_t* json) const noexcept; simdjson_inline const uint8_t* peek_start() const noexcept; simdjson_inline uint32_t peek_start_length() const noexcept; + simdjson_inline uint32_t peek_root_length() const noexcept; /** * The general idea of the advance_... methods and the peek_* methods @@ -79721,6 +87213,7 @@ namespace simdjson { friend class object; friend class array; friend class value; + friend class field; }; // value_iterator } // namespace ondemand @@ -79781,12 +87274,13 @@ namespace simdjson { * @returns INCORRECT_TYPE If the JSON value is not the given type. */ template simdjson_inline simdjson_result get() noexcept { - // Unless the simdjson library provides an inline implementation, calling this method should + // Unless the simdjson library or the user provides an inline implementation, calling this method should // immediately fail. static_assert(!sizeof(T), "The get method with given type is not implemented by the simdjson library. " "The supported types are ondemand::object, ondemand::array, raw_json_string, std::string_view, uint64_t, " "int64_t, double, and bool. We recommend you use get_double(), get_bool(), get_uint64(), get_int64(), " - " get_object(), get_array(), get_raw_json_string(), or get_string() instead of the get template."); + " get_object(), get_array(), get_raw_json_string(), or get_string() instead of the get template." + " You may also add support for custom types, see our documentation."); } /** @@ -79939,6 +87433,17 @@ namespace simdjson { simdjson_inline simdjson_result is_null() noexcept; #if SIMDJSON_EXCEPTIONS + /** + * Cast this JSON value to an instance of type T. The programmer is responsible for + * providing an implementation of get for the type T, if T is not one of the types + * supported by the library (object, array, raw_json_string, string_view, uint64_t, etc.). + * + * See https://github.com/simdjson/simdjson/blob/master/doc/basics.md#adding-support-for-custom-types + * + * @returns An instance of type T + */ + template + explicit simdjson_inline operator T() noexcept(false); /** * Cast this JSON value to an array. * @@ -80102,7 +87607,7 @@ namespace simdjson { * that only one field is returned. * * Use find_field() if you are sure fields will be in order (or are willing to treat it as if the - * field wasn't there when they aren't). + * field as not there when they are not in order). * * @param key The key to look up. * @returns The value of the field, or NO_SUCH_FIELD if the field is not in the object. @@ -80138,6 +87643,13 @@ namespace simdjson { * @error TAPE_ERROR when the JSON value is a bad token like "}" "," or "alse". */ simdjson_inline simdjson_result is_scalar() noexcept; + /** + * Checks whether the value is a string. + * + * @returns true if the type is string + * @error TAPE_ERROR when the JSON value is a bad token like "}" "," or "alse". + */ + simdjson_inline simdjson_result is_string() noexcept; /** * Checks whether the value is a negative number. @@ -80170,10 +87682,12 @@ namespace simdjson { * get_number().get_number_type(). * * get_number_type() is number_type::unsigned_integer if we have - * an integer greater or equal to 9223372036854775808 + * an integer greater or equal to 9223372036854775808. * get_number_type() is number_type::signed_integer if we have an - * integer that is less than 9223372036854775808 - * Otherwise, get_number_type() has value number_type::floating_point_number + * integer that is less than 9223372036854775808. + * get_number_type() is number_type::big_integer for integers that do not fit in 64 bits, + * in which case the digit_count is set to the length of the big integer string. + * Otherwise, get_number_type() has value number_type::floating_point_number. * * This function requires processing the number string, but it is expected * to be faster than get_number().get_number_type() because it is does not @@ -80200,6 +87714,8 @@ namespace simdjson { * You can recover the value by calling number.get_uint64() and you * have that number.is_uint64() is true. * + * For integers that do not fit in 64 bits, the function returns BIGINT_ERROR error code. + * * Otherwise, number.get_number_type() has value number_type::floating_point_number * and we have a binary64 number. * You can recover the value by calling number.get_double() and you @@ -80215,7 +87731,6 @@ namespace simdjson { */ simdjson_warn_unused simdjson_inline simdjson_result get_number() noexcept; - /** * Get the raw JSON for this token. * @@ -80313,6 +87828,20 @@ namespace simdjson { */ simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; + /** + * Get the value associated with the given JSONPath expression. We only support + * JSONPath queries that trivially convertible to JSON Pointer queries: key + * names and array indices. + * + * @return The value associated with the given JSONPath expression, or: + * - INVALID_JSON_POINTER if the JSONPath to JSON Pointer conversion fails + * - NO_SUCH_FIELD if a field does not exist in an object + * - INDEX_OUT_OF_BOUNDS if an array index is larger than an array length + * - INCORRECT_TYPE if a non-integer is used to access an array + */ + simdjson_inline simdjson_result at_path(std::string_view at_path) noexcept; + + protected: /** * Create a value. @@ -80352,6 +87881,7 @@ namespace simdjson { friend class object; friend struct simdjson_result; friend struct simdjson_result; + friend class field; }; } // namespace ondemand @@ -80389,6 +87919,8 @@ namespace simdjson { template simdjson_inline error_code get(T& out) noexcept; #if SIMDJSON_EXCEPTIONS + template + explicit simdjson_inline operator T() noexcept(false); simdjson_inline operator westmere::ondemand::array() noexcept(false); simdjson_inline operator westmere::ondemand::object() noexcept(false); simdjson_inline operator uint64_t() noexcept(false); @@ -80442,7 +87974,7 @@ namespace simdjson { * APIs assume this. Therefore, you must be explicit if you want to treat objects as out of order. * * Use find_field() if you are sure fields will be in order (or are willing to treat it as if the - * field wasn't there when they aren't). + * field as not there when they are not in order). * * @param key The key to look up. * @returns The value of the field, or NO_SUCH_FIELD if the field is not in the object. @@ -80464,6 +87996,7 @@ namespace simdjson { */ simdjson_inline simdjson_result type() noexcept; simdjson_inline simdjson_result is_scalar() noexcept; + simdjson_inline simdjson_result is_string() noexcept; simdjson_inline simdjson_result is_negative() noexcept; simdjson_inline simdjson_result is_integer() noexcept; simdjson_inline simdjson_result get_number_type() noexcept; @@ -80478,6 +88011,7 @@ namespace simdjson { /** @copydoc simdjson_inline int32_t current_depth() const noexcept */ simdjson_inline simdjson_result current_depth() const noexcept; simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; + simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; }; } // namespace simdjson @@ -80596,7 +88130,7 @@ namespace simdjson { * 1 = next token, -1 = prev token. * * TODO consider a string_view, assuming the length will get stripped out by the optimizer when - * it isn't used ... + * it is not used... */ simdjson_inline const uint8_t* peek(int32_t delta = 0) const noexcept; /** @@ -80626,7 +88160,14 @@ namespace simdjson { * @param position The position of the token. */ simdjson_inline uint32_t peek_length(token_position position) const noexcept; - + /** + * Get the maximum length of the JSON text for a root token. + * + * The length will include any whitespace at the end of the token. + * + * @param position The position of the token (start of the document). + */ + simdjson_inline uint32_t peek_root_length(token_position position) const noexcept; /** * Return the current index. */ @@ -80849,7 +88390,7 @@ namespace simdjson { * @param delta The relative position of the token to retrieve. e.g. 0 = next token, -1 = prev token. * * TODO consider a string_view, assuming the length will get stripped out by the optimizer when - * it isn't used ... + * it is not used ... */ simdjson_inline const uint8_t* peek(int32_t delta = 0) const noexcept; /** @@ -80877,7 +88418,7 @@ namespace simdjson { * @param position The position of the token to retrieve. * * TODO consider a string_view, assuming the length will get stripped out by the optimizer when - * it isn't used ... + * it is not used ... */ simdjson_inline const uint8_t* peek(token_position position) const noexcept; /** @@ -80888,13 +88429,21 @@ namespace simdjson { * @param position The position of the token to retrieve. */ simdjson_inline uint32_t peek_length(token_position position) const noexcept; + /** + * Get the maximum length of the JSON text for the current root token. + * + * The length will include any whitespace at the end of the token. + * + * @param position The position of the token to retrieve. + */ + simdjson_inline uint32_t peek_root_length(token_position position) const noexcept; /** * Get the JSON text for the last token in the document. * * This is not null-terminated; it is a view into the JSON. * * TODO consider a string_view, assuming the length will get stripped out by the optimizer when - * it isn't used ... + * it is not used ... */ simdjson_inline const uint8_t* peek_last() const noexcept; @@ -80988,6 +88537,9 @@ namespace simdjson { inline bool balanced() const noexcept; protected: simdjson_inline json_iterator(const uint8_t* buf, ondemand::parser* parser) noexcept; +#ifdef SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON + simdjson_inline json_iterator(const uint8_t* buf, ondemand::parser* parser, bool streaming) noexcept; +#endif // SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON /// The last token before the end simdjson_inline token_position last_position() const noexcept; /// The token *at* the end. This points at gibberish and should only be used for comparison. @@ -81003,6 +88555,7 @@ namespace simdjson { friend class raw_json_string; friend class parser; friend class value_iterator; + friend class field; template friend simdjson_inline void logger::log_line(const json_iterator& iter, const char* title_prefix, const char* title, std::string_view detail, int delta, int depth_delta, logger::log_level level, Args&&... args) noexcept; template @@ -81124,8 +88677,6 @@ namespace simdjson { */ friend class value_iterator; template - friend error_code numberparsing::slow_float_parsing(simdjson_unused const uint8_t* src, W writer); - template friend error_code numberparsing::write_float(const uint8_t* const src, bool negative, uint64_t i, const uint8_t* start_digits, size_t digit_count, int64_t exponent, W& writer); template friend error_code numberparsing::parse_number(const uint8_t* const src, W& writer); @@ -81419,8 +88970,8 @@ namespace simdjson { namespace ondemand { /** - * The default batch size for document_stream instances for this On Demand kernel. - * Note that different On Demand kernel may use a different DEFAULT_BATCH_SIZE value + * The default batch size for document_stream instances for this On-Demand kernel. + * Note that different On-Demand kernel may use a different DEFAULT_BATCH_SIZE value * in the future. */ static constexpr size_t DEFAULT_BATCH_SIZE = 1000000; @@ -81504,6 +89055,9 @@ namespace simdjson { * - UNCLOSED_STRING if there is an unclosed string in the document. */ simdjson_warn_unused simdjson_result iterate(padded_string_view json) & noexcept; +#ifdef SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON + simdjson_warn_unused simdjson_result iterate_allow_incomplete_json(padded_string_view json) & noexcept; +#endif // SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON /** @overload simdjson_result iterate(padded_string_view json) & noexcept */ simdjson_warn_unused simdjson_result iterate(const char* json, size_t len, size_t capacity) & noexcept; /** @overload simdjson_result iterate(padded_string_view json) & noexcept */ @@ -81648,9 +89202,9 @@ namespace simdjson { simdjson_result iterate_many(const char* buf, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept = delete; /** The capacity of this parser (the largest document it can process). */ - simdjson_inline size_t capacity() const noexcept; + simdjson_pure simdjson_inline size_t capacity() const noexcept; /** The maximum capacity of this parser (the largest document it is allowed to process). */ - simdjson_inline size_t max_capacity() const noexcept; + simdjson_pure simdjson_inline size_t max_capacity() const noexcept; simdjson_inline void set_max_capacity(size_t max_capacity) noexcept; /** * The maximum depth of this parser (the most deeply nested objects and arrays it can process). @@ -81658,7 +89212,7 @@ namespace simdjson { * The document's instance current_depth() method should be used to monitor the parsing * depth and limit it if desired. */ - simdjson_inline size_t max_depth() const noexcept; + simdjson_pure simdjson_inline size_t max_depth() const noexcept; /** * Ensure this parser has enough memory to process JSON documents up to `capacity` bytes in length @@ -81730,9 +89284,20 @@ namespace simdjson { */ simdjson_inline simdjson_result unescape_wobbly(raw_json_string in, uint8_t*& dst) const noexcept; +#if SIMDJSON_DEVELOPMENT_CHECKS + /** + * Returns true if string_buf_loc is outside of the allocated range for the + * the string buffer. When true, it indicates that the string buffer has overflowed. + * This is a development-time check that is not needed in production. It can be + * used to detect buffer overflows in the string buffer and usafe usage of the + * string buffer. + */ + bool string_buffer_overflow(const uint8_t* string_buf_loc) const noexcept; +#endif + private: /** @private [for benchmarking access] The implementation to use */ - std::unique_ptr implementation{}; + std::unique_ptr implementation{}; size_t _capacity{ 0 }; size_t _max_capacity; size_t _max_depth{ DEFAULT_MAX_DEPTH }; @@ -81869,6 +89434,22 @@ namespace simdjson { * - INVALID_JSON_POINTER if the JSON pointer is invalid and cannot be parsed */ inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; + + /** + * Get the value associated with the given JSONPath expression. We only support + * JSONPath queries that trivially convertible to JSON Pointer queries: key + * names and array indices. + * + * https://datatracker.ietf.org/doc/html/draft-normington-jsonpath-00 + * + * @return The value associated with the given JSONPath expression, or: + * - INVALID_JSON_POINTER if the JSONPath to JSON Pointer conversion fails + * - NO_SUCH_FIELD if a field does not exist in an object + * - INDEX_OUT_OF_BOUNDS if an array index is larger than an array length + * - INCORRECT_TYPE if a non-integer is used to access an array + */ + inline simdjson_result at_path(std::string_view json_path) noexcept; + /** * Consumes the array and returns a string_view instance corresponding to the * array as represented in JSON. It points inside the original document. @@ -81959,6 +89540,7 @@ namespace simdjson { inline simdjson_result reset() & noexcept; simdjson_inline simdjson_result at(size_t index) noexcept; simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; + simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; simdjson_inline simdjson_result raw_json() noexcept; }; @@ -82249,21 +89831,23 @@ namespace simdjson { * @returns INCORRECT_TYPE If the JSON value is not the given type. */ template simdjson_inline simdjson_result get() & noexcept { - // Unless the simdjson library provides an inline implementation, calling this method should + // Unless the simdjson library or the user provides an inline implementation, calling this method should // immediately fail. static_assert(!sizeof(T), "The get method with given type is not implemented by the simdjson library. " "The supported types are ondemand::object, ondemand::array, raw_json_string, std::string_view, uint64_t, " "int64_t, double, and bool. We recommend you use get_double(), get_bool(), get_uint64(), get_int64(), " - " get_object(), get_array(), get_raw_json_string(), or get_string() instead of the get template."); + " get_object(), get_array(), get_raw_json_string(), or get_string() instead of the get template." + " You may also add support for custom types, see our documentation."); } /** @overload template simdjson_result get() & noexcept */ - template simdjson_inline simdjson_result get() && noexcept { - // Unless the simdjson library provides an inline implementation, calling this method should + template simdjson_deprecated simdjson_inline simdjson_result get() && noexcept { + // Unless the simdjson library or the user provides an inline implementation, calling this method should // immediately fail. static_assert(!sizeof(T), "The get method with given type is not implemented by the simdjson library. " "The supported types are ondemand::object, ondemand::array, raw_json_string, std::string_view, uint64_t, " "int64_t, double, and bool. We recommend you use get_double(), get_bool(), get_uint64(), get_int64(), " - " get_object(), get_array(), get_raw_json_string(), or get_string() instead of the get template."); + " get_object(), get_array(), get_raw_json_string(), or get_string() instead of the get template." + " You may also add support for custom types, see our documentation."); } /** @@ -82279,9 +89863,23 @@ namespace simdjson { */ template simdjson_inline error_code get(T& out) & noexcept; /** @overload template error_code get(T &out) & noexcept */ - template simdjson_inline error_code get(T& out) && noexcept; + template simdjson_deprecated simdjson_inline error_code get(T& out) && noexcept; #if SIMDJSON_EXCEPTIONS + /** + * Cast this JSON value to an instance of type T. The programmer is responsible for + * providing an implementation of get for the type T, if T is not one of the types + * supported by the library (object, array, raw_json_string, string_view, uint64_t, etc.) + * + * See https://github.com/simdjson/simdjson/blob/master/doc/basics.md#adding-support-for-custom-types + * + * @returns An instance of type T + */ + template + explicit simdjson_inline operator T() & noexcept(false); + template + explicit simdjson_deprecated simdjson_inline operator T() && noexcept(false); + /** * Cast this JSON value to an array. * @@ -82454,7 +90052,7 @@ namespace simdjson { * APIs assume this. Therefore, you must be explicit if you want to treat objects as out of order. * * Use find_field() if you are sure fields will be in order (or are willing to treat it as if the - * field wasn't there when they aren't). + * field was not there when they are not in order). * * You must consume the fields on an object one at a time. A request for a new key * invalidates previous field values: it makes them unsafe. E.g., the array @@ -82500,6 +90098,14 @@ namespace simdjson { */ simdjson_inline simdjson_result is_scalar() noexcept; + /** + * Checks whether the document is a string. + * + * @returns true if the type is string + * @error TAPE_ERROR when the JSON value is a bad token like "}" "," or "alse". + */ + simdjson_inline simdjson_result is_string() noexcept; + /** * Checks whether the document is a negative number. * @@ -82527,9 +90133,11 @@ namespace simdjson { * get_number().get_number_type(). * * get_number_type() is number_type::unsigned_integer if we have - * an integer greater or equal to 9223372036854775808 + * an integer greater or equal to 9223372036854775808 and no larger than 18446744073709551615. * get_number_type() is number_type::signed_integer if we have an - * integer that is less than 9223372036854775808 + * integer that is less than 9223372036854775808 and greater or equal to -9223372036854775808. + * get_number_type() is number_type::big_integer if we have an integer outside + * of those ranges (either larger than 18446744073709551615 or smaller than -9223372036854775808). * Otherwise, get_number_type() has value number_type::floating_point_number * * This function requires processing the number string, but it is expected @@ -82647,6 +90255,14 @@ namespace simdjson { * auto doc = parser.iterate(json); * doc.at_pointer("//a/1") == 20 * + * Key values are matched exactly, without unescaping or Unicode normalization. + * We do a byte-by-byte comparison. E.g. + * + * const padded_string json = "{\"\\u00E9\":123}"_padded; + * auto doc = parser.iterate(json); + * doc.at_pointer("/\\u00E9") == 123 + * doc.at_pointer((const char*)u8"/\u00E9") returns an error (NO_SUCH_FIELD) + * * Note that at_pointer() automatically calls rewind between each call. Thus * all values, objects and arrays that you have created so far (including unescaped strings) * are invalidated. After calling at_pointer, you need to consume the result: string values @@ -82663,6 +90279,30 @@ namespace simdjson { * - SCALAR_DOCUMENT_AS_VALUE if the json_pointer is empty and the document is not a scalar (see is_scalar() function). */ simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; + + /** + * Get the value associated with the given JSONPath expression. We only support + * JSONPath queries that trivially convertible to JSON Pointer queries: key + * names and array indices. + * + * https://datatracker.ietf.org/doc/html/draft-normington-jsonpath-00 + * + * Key values are matched exactly, without unescaping or Unicode normalization. + * We do a byte-by-byte comparison. E.g. + * + * const padded_string json = "{\"\\u00E9\":123}"_padded; + * auto doc = parser.iterate(json); + * doc.at_path(".\\u00E9") == 123 + * doc.at_path((const char*)u8".\u00E9") returns an error (NO_SUCH_FIELD) + * + * @return The value associated with the given JSONPath expression, or: + * - INVALID_JSON_POINTER if the JSONPath to JSON Pointer conversion fails + * - NO_SUCH_FIELD if a field does not exist in an object + * - INDEX_OUT_OF_BOUNDS if an array index is larger than an array length + * - INCORRECT_TYPE if a non-integer is used to access an array + */ + simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; + /** * Consumes the document and returns a string_view instance corresponding to the * document as represented in JSON. It points inside the original byte array containing @@ -82728,10 +90368,12 @@ namespace simdjson { simdjson_inline simdjson_result get_value() noexcept; simdjson_inline simdjson_result is_null() noexcept; + template simdjson_inline simdjson_result get() & noexcept; simdjson_inline simdjson_result raw_json() noexcept; simdjson_inline operator document& () const noexcept; - #if SIMDJSON_EXCEPTIONS + template + explicit simdjson_inline operator T() noexcept(false); simdjson_inline operator array() & noexcept(false); simdjson_inline operator object() & noexcept(false); simdjson_inline operator uint64_t() noexcept(false); @@ -82756,6 +90398,7 @@ namespace simdjson { simdjson_inline simdjson_result type() noexcept; simdjson_inline simdjson_result is_scalar() noexcept; + simdjson_inline simdjson_result is_string() noexcept; simdjson_inline simdjson_result current_location() noexcept; simdjson_inline int32_t current_depth() const noexcept; @@ -82765,6 +90408,8 @@ namespace simdjson { simdjson_inline simdjson_result get_number() noexcept; simdjson_inline simdjson_result raw_json_token() noexcept; simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; + simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; + private: document* doc{ nullptr }; }; @@ -82800,12 +90445,13 @@ namespace simdjson { simdjson_inline simdjson_result is_null() noexcept; template simdjson_inline simdjson_result get() & noexcept; - template simdjson_inline simdjson_result get() && noexcept; + template simdjson_deprecated simdjson_inline simdjson_result get() && noexcept; template simdjson_inline error_code get(T& out) & noexcept; template simdjson_inline error_code get(T& out) && noexcept; - #if SIMDJSON_EXCEPTIONS + template ::value == false>::type> + explicit simdjson_inline operator T() noexcept(false); simdjson_inline operator westmere::ondemand::array() & noexcept(false); simdjson_inline operator westmere::ondemand::object() & noexcept(false); simdjson_inline operator uint64_t() noexcept(false); @@ -82829,6 +90475,7 @@ namespace simdjson { simdjson_inline simdjson_result find_field_unordered(const char* key) & noexcept; simdjson_inline simdjson_result type() noexcept; simdjson_inline simdjson_result is_scalar() noexcept; + simdjson_inline simdjson_result is_string() noexcept; simdjson_inline simdjson_result current_location() noexcept; simdjson_inline int32_t current_depth() const noexcept; simdjson_inline bool at_end() const noexcept; @@ -82840,6 +90487,7 @@ namespace simdjson { simdjson_inline simdjson_result raw_json_token() noexcept; simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; + simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; }; @@ -82872,8 +90520,9 @@ namespace simdjson { simdjson_inline simdjson_result get_bool() noexcept; simdjson_inline simdjson_result get_value() noexcept; simdjson_inline simdjson_result is_null() noexcept; - #if SIMDJSON_EXCEPTIONS + template ::value == false>::type> + explicit simdjson_inline operator T() noexcept(false); simdjson_inline operator westmere::ondemand::array() & noexcept(false); simdjson_inline operator westmere::ondemand::object() & noexcept(false); simdjson_inline operator uint64_t() noexcept(false); @@ -82897,6 +90546,7 @@ namespace simdjson { simdjson_inline simdjson_result find_field_unordered(const char* key) & noexcept; simdjson_inline simdjson_result type() noexcept; simdjson_inline simdjson_result is_scalar() noexcept; + simdjson_inline simdjson_result is_string() noexcept; simdjson_inline simdjson_result current_location() noexcept; simdjson_inline simdjson_result current_depth() const noexcept; simdjson_inline simdjson_result is_negative() noexcept; @@ -82907,6 +90557,7 @@ namespace simdjson { simdjson_inline simdjson_result raw_json_token() noexcept; simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; + simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; }; @@ -83040,10 +90691,9 @@ namespace simdjson { class iterator { public: using value_type = simdjson_result; - using reference = value_type; - + using reference = simdjson_result; + using pointer = void; using difference_type = std::ptrdiff_t; - using iterator_category = std::input_iterator_tag; /** @@ -83053,7 +90703,7 @@ namespace simdjson { /** * Get the current document (or error). */ - simdjson_inline simdjson_result operator*() noexcept; + simdjson_inline reference operator*() noexcept; /** * Advance to the next document (prefix). */ @@ -83235,7 +90885,7 @@ namespace simdjson { friend class document; friend class json_iterator; friend struct simdjson_result; - friend struct internal::simdjson_result_base; + friend struct simdjson::internal::simdjson_result_base; }; // document_stream } // namespace ondemand @@ -83295,12 +90945,39 @@ namespace simdjson { * This consumes the key: once you have called unescaped_key(), you cannot * call it again nor can you call key(). */ - simdjson_inline simdjson_warn_unused simdjson_result unescaped_key(bool allow_replacement) noexcept; + simdjson_inline simdjson_warn_unused simdjson_result unescaped_key(bool allow_replacement = false) noexcept; + /** + * Get the key as a string_view (for higher speed, consider raw_key). + * We deliberately use a more cumbersome name (unescaped_key) to force users + * to think twice about using it. The content is stored in the receiver. + * + * This consumes the key: once you have called unescaped_key(), you cannot + * call it again nor can you call key(). + */ + template + simdjson_inline simdjson_warn_unused error_code unescaped_key(string_type& receiver, bool allow_replacement = false) noexcept; /** * Get the key as a raw_json_string. Can be used for direct comparison with - * an unescaped C string: e.g., key() == "test". + * an unescaped C string: e.g., key() == "test". This does not count as + * consumption of the content: you can safely call it repeatedly. + * See escaped_key() for a similar function which returns + * a more convenient std::string_view result. */ simdjson_inline raw_json_string key() const noexcept; + /** + * Get the unprocessed key as a string_view. This includes the quotes and may include + * some spaces after the last quote. This does not count as + * consumption of the content: you can safely call it repeatedly. + * See escaped_key(). + */ + simdjson_inline std::string_view key_raw_json_token() const noexcept; + /** + * Get the key as a string_view. This does not include the quotes and + * the string is unprocessed key so it may contain escape characters + * (e.g., \uXXXX or \n). It does not count as a consumption of the content: + * you can safely call it repeatedly. Use unescaped_key() to get the unescaped key. + */ + simdjson_inline std::string_view escaped_key() const noexcept; /** * Get the field value. */ @@ -83332,7 +91009,11 @@ namespace simdjson { simdjson_inline simdjson_result() noexcept = default; simdjson_inline simdjson_result unescaped_key(bool allow_replacement = false) noexcept; + template + simdjson_inline error_code unescaped_key(string_type& receiver, bool allow_replacement = false) noexcept; simdjson_inline simdjson_result key() noexcept; + simdjson_inline simdjson_result key_raw_json_token() noexcept; + simdjson_inline simdjson_result escaped_key() noexcept; simdjson_inline simdjson_result value() noexcept; }; @@ -83400,6 +91081,8 @@ namespace simdjson { * key a single time. Doing object["mykey"].to_string() and then again object["mykey"].to_string() * is an error. * + * If you expect to have keys with escape characters, please review our documentation. + * * @param key The key to look up. * @returns The value of the field, or NO_SUCH_FIELD if the field is not in the object. */ @@ -83421,7 +91104,7 @@ namespace simdjson { * APIs assume this. Therefore, you must be explicit if you want to treat objects as out of order. * * Use find_field() if you are sure fields will be in order (or are willing to treat it as if the - * field wasn't there when they aren't). + * field was not there when they are not in order). * * If you have multiple fields with a matching key ({"x": 1, "x": 1}) be mindful * that only one field is returned. @@ -83437,6 +91120,8 @@ namespace simdjson { * You are expected to access keys only once. You should access the value corresponding to a key * a single time. Doing object["mykey"].to_string() and then again object["mykey"].to_string() is an error. * + * If you expect to have keys with escape characters, please review our documentation. + * * @param key The key to look up. * @returns The value of the field, or NO_SUCH_FIELD if the field is not in the object. */ @@ -83484,11 +91169,24 @@ namespace simdjson { */ inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; + /** + * Get the value associated with the given JSONPath expression. We only support + * JSONPath queries that trivially convertible to JSON Pointer queries: key + * names and array indices. + * + * @return The value associated with the given JSONPath expression, or: + * - INVALID_JSON_POINTER if the JSONPath to JSON Pointer conversion fails + * - NO_SUCH_FIELD if a field does not exist in an object + * - INDEX_OUT_OF_BOUNDS if an array index is larger than an array length + * - INCORRECT_TYPE if a non-integer is used to access an array + */ + inline simdjson_result at_path(std::string_view json_path) noexcept; + /** * Reset the iterator so that we are pointing back at the * beginning of the object. You should still consume values only once even if you - * can iterate through the object more than once. If you unescape a string within - * the object more than once, you have unsafe code. Note that rewinding an object + * can iterate through the object more than once. If you unescape a string or a key + * within the object more than once, you have unsafe code. Note that rewinding an object * means that you may need to reparse it anew: it is not a free operation. * * @returns true if the object contains some elements (not empty) @@ -83571,6 +91269,8 @@ namespace simdjson { simdjson_inline simdjson_result operator[](std::string_view key) & noexcept; simdjson_inline simdjson_result operator[](std::string_view key) && noexcept; simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; + simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; + inline simdjson_result reset() noexcept; inline simdjson_result is_empty() noexcept; inline simdjson_result count_fields() & noexcept; @@ -83944,6 +91644,65 @@ namespace simdjson { return child; } + inline std::string json_path_to_pointer_conversion(std::string_view json_path) { + if (json_path.empty() || (json_path.front() != '.' && + json_path.front() != '[')) { + return "-1"; // This is just a sentinel value, the caller should check for this and return an error. + } + + std::string result; + // Reserve space to reduce allocations, adjusting for potential increases due + // to escaping. + result.reserve(json_path.size() * 2); + + size_t i = 0; + + while (i < json_path.length()) { + if (json_path[i] == '.') { + result += '/'; + } + else if (json_path[i] == '[') { + result += '/'; + ++i; // Move past the '[' + while (i < json_path.length() && json_path[i] != ']') { + if (json_path[i] == '~') { + result += "~0"; + } + else if (json_path[i] == '/') { + result += "~1"; + } + else { + result += json_path[i]; + } + ++i; + } + if (i == json_path.length() || json_path[i] != ']') { + return "-1"; // Using sentinel value that will be handled as an error by the caller. + } + } + else { + if (json_path[i] == '~') { + result += "~0"; + } + else if (json_path[i] == '/') { + result += "~1"; + } + else { + result += json_path[i]; + } + } + ++i; + } + + return result; + } + + inline simdjson_result array::at_path(std::string_view json_path) noexcept { + auto json_pointer = json_path_to_pointer_conversion(json_path); + if (json_pointer == "-1") { return INVALID_JSON_POINTER; } + return at_pointer(json_pointer); + } + simdjson_inline simdjson_result array::at(size_t index) noexcept { size_t i = 0; for (auto value : *this) { @@ -83998,6 +91757,10 @@ namespace simdjson { if (error()) { return error(); } return first.at_pointer(json_pointer); } + simdjson_inline simdjson_result simdjson_result::at_path(std::string_view json_path) noexcept { + if (error()) { return error(); } + return first.at_path(json_path); + } simdjson_inline simdjson_result simdjson_result::raw_json() noexcept { if (error()) { return error(); } return first.raw_json(); @@ -84094,14 +91857,14 @@ namespace simdjson { /* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ /* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_ONDEMAND_DOCUMENT_INL_H */ /* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/base.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/array-inl.h" */ /* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/array_iterator.h" */ /* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/document.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/json_iterator-inl.h" */ /* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/json_type.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/object-inl.h" */ /* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/raw_json_string.h" */ /* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/value.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/array-inl.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/json_iterator-inl.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/object-inl.h" */ /* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/value_iterator-inl.h" */ /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ @@ -84257,22 +92020,26 @@ namespace simdjson { template<> simdjson_inline simdjson_result document::get() & noexcept { return get_bool(); } template<> simdjson_inline simdjson_result document::get() & noexcept { return get_value(); } - template<> simdjson_inline simdjson_result document::get() && noexcept { return get_raw_json_string(); } - template<> simdjson_inline simdjson_result document::get() && noexcept { return get_string(false); } - template<> simdjson_inline simdjson_result document::get() && noexcept { return std::forward(*this).get_double(); } - template<> simdjson_inline simdjson_result document::get() && noexcept { return std::forward(*this).get_uint64(); } - template<> simdjson_inline simdjson_result document::get() && noexcept { return std::forward(*this).get_int64(); } - template<> simdjson_inline simdjson_result document::get() && noexcept { return std::forward(*this).get_bool(); } - template<> simdjson_inline simdjson_result document::get() && noexcept { return get_value(); } + template<> simdjson_deprecated simdjson_inline simdjson_result document::get() && noexcept { return get_raw_json_string(); } + template<> simdjson_deprecated simdjson_inline simdjson_result document::get() && noexcept { return get_string(false); } + template<> simdjson_deprecated simdjson_inline simdjson_result document::get() && noexcept { return std::forward(*this).get_double(); } + template<> simdjson_deprecated simdjson_inline simdjson_result document::get() && noexcept { return std::forward(*this).get_uint64(); } + template<> simdjson_deprecated simdjson_inline simdjson_result document::get() && noexcept { return std::forward(*this).get_int64(); } + template<> simdjson_deprecated simdjson_inline simdjson_result document::get() && noexcept { return std::forward(*this).get_bool(); } + template<> simdjson_deprecated simdjson_inline simdjson_result document::get() && noexcept { return get_value(); } template simdjson_inline error_code document::get(T& out) & noexcept { return get().get(out); } - template simdjson_inline error_code document::get(T& out) && noexcept { + template simdjson_deprecated simdjson_inline error_code document::get(T& out) && noexcept { return std::forward(*this).get().get(out); } #if SIMDJSON_EXCEPTIONS + template + simdjson_deprecated simdjson_inline document::operator T() && noexcept(false) { return get(); } + template + simdjson_inline document::operator T() & noexcept(false) { return get(); } simdjson_inline document::operator array() & noexcept(false) { return get_array(); } simdjson_inline document::operator object() & noexcept(false) { return get_object(); } simdjson_inline document::operator uint64_t() noexcept(false) { return get_uint64(); } @@ -84357,6 +92124,13 @@ namespace simdjson { return !((this_type == json_type::array) || (this_type == json_type::object)); } + simdjson_inline simdjson_result document::is_string() noexcept { + json_type this_type; + auto error = type().get(this_type); + if (error) { return error; } + return (this_type == json_type::string); + } + simdjson_inline bool document::is_negative() noexcept { return get_root_value_iterator().is_root_negative(); } @@ -84376,7 +92150,7 @@ namespace simdjson { simdjson_inline simdjson_result document::raw_json_token() noexcept { auto _iter = get_root_value_iterator(); - return std::string_view(reinterpret_cast(_iter.peek_start()), _iter.peek_start_length()); + return std::string_view(reinterpret_cast(_iter.peek_start()), _iter.peek_root_length()); } simdjson_inline simdjson_result document::at_pointer(std::string_view json_pointer) noexcept { @@ -84397,6 +92171,23 @@ namespace simdjson { } } + simdjson_inline simdjson_result document::at_path(std::string_view json_path) noexcept { + rewind(); // Rewind the document each time at_pointer is called + if (json_path.empty()) { + return this->get_value(); + } + json_type t; + SIMDJSON_TRY(type().get(t)); + switch (t) { + case json_type::array: + return (*this).get_array().at_path(json_path); + case json_type::object: + return (*this).get_object().at_path(json_path); + default: + return INVALID_JSON_POINTER; + } + } + } // namespace ondemand } // namespace westmere } // namespace simdjson @@ -84535,7 +92326,7 @@ namespace simdjson { return first.get(); } template - simdjson_inline simdjson_result simdjson_result::get() && noexcept { + simdjson_deprecated simdjson_inline simdjson_result simdjson_result::get() && noexcept { if (error()) { return error(); } return std::forward(first).get(); } @@ -84551,7 +92342,7 @@ namespace simdjson { } template<> simdjson_inline simdjson_result simdjson_result::get() & noexcept = delete; - template<> simdjson_inline simdjson_result simdjson_result::get() && noexcept { + template<> simdjson_deprecated simdjson_inline simdjson_result simdjson_result::get() && noexcept { if (error()) { return error(); } return std::forward(first); } @@ -84572,6 +92363,10 @@ namespace simdjson { return first.is_scalar(); } + simdjson_inline simdjson_result simdjson_result::is_string() noexcept { + if (error()) { return error(); } + return first.is_string(); + } simdjson_inline bool simdjson_result::is_negative() noexcept { if (error()) { return error(); } @@ -84595,6 +92390,11 @@ namespace simdjson { #if SIMDJSON_EXCEPTIONS + template ::value == false>::type> + simdjson_inline simdjson_result::operator T() noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return first; + } simdjson_inline simdjson_result::operator westmere::ondemand::array() & noexcept(false) { if (error()) { throw simdjson_error(error()); } return first; @@ -84660,6 +92460,10 @@ namespace simdjson { return first.at_pointer(json_pointer); } + simdjson_inline simdjson_result simdjson_result::at_path(std::string_view json_path) noexcept { + if (error()) { return error(); } + return first.at_path(json_path); + } } // namespace simdjson @@ -84701,15 +92505,25 @@ namespace simdjson { simdjson_inline simdjson_result document_reference::get_bool() noexcept { return doc->get_root_value_iterator().get_root_bool(false); } simdjson_inline simdjson_result document_reference::get_value() noexcept { return doc->get_value(); } simdjson_inline simdjson_result document_reference::is_null() noexcept { return doc->get_root_value_iterator().is_root_null(false); } - + template<> simdjson_inline simdjson_result document_reference::get() & noexcept { return get_array(); } + template<> simdjson_inline simdjson_result document_reference::get() & noexcept { return get_object(); } + template<> simdjson_inline simdjson_result document_reference::get() & noexcept { return get_raw_json_string(); } + template<> simdjson_inline simdjson_result document_reference::get() & noexcept { return get_string(false); } + template<> simdjson_inline simdjson_result document_reference::get() & noexcept { return get_double(); } + template<> simdjson_inline simdjson_result document_reference::get() & noexcept { return get_uint64(); } + template<> simdjson_inline simdjson_result document_reference::get() & noexcept { return get_int64(); } + template<> simdjson_inline simdjson_result document_reference::get() & noexcept { return get_bool(); } + template<> simdjson_inline simdjson_result document_reference::get() & noexcept { return get_value(); } #if SIMDJSON_EXCEPTIONS + template + simdjson_inline document_reference::operator T() noexcept(false) { return get(); } simdjson_inline document_reference::operator array() & noexcept(false) { return array(*doc); } simdjson_inline document_reference::operator object() & noexcept(false) { return object(*doc); } simdjson_inline document_reference::operator uint64_t() noexcept(false) { return get_uint64(); } simdjson_inline document_reference::operator int64_t() noexcept(false) { return get_int64(); } simdjson_inline document_reference::operator double() noexcept(false) { return get_double(); } simdjson_inline document_reference::operator std::string_view() noexcept(false) { return std::string_view(*doc); } - simdjson_inline document_reference::operator raw_json_string() noexcept(false) { return raw_json_string(*doc); } + simdjson_inline document_reference::operator raw_json_string() noexcept(false) { return get_raw_json_string(); } simdjson_inline document_reference::operator bool() noexcept(false) { return get_bool(); } simdjson_inline document_reference::operator value() noexcept(false) { return value(*doc); } #endif @@ -84726,6 +92540,7 @@ namespace simdjson { simdjson_inline simdjson_result document_reference::find_field_unordered(const char* key) & noexcept { return doc->find_field_unordered(key); } simdjson_inline simdjson_result document_reference::type() noexcept { return doc->type(); } simdjson_inline simdjson_result document_reference::is_scalar() noexcept { return doc->is_scalar(); } + simdjson_inline simdjson_result document_reference::is_string() noexcept { return doc->is_string(); } simdjson_inline simdjson_result document_reference::current_location() noexcept { return doc->current_location(); } simdjson_inline int32_t document_reference::current_depth() const noexcept { return doc->current_depth(); } simdjson_inline bool document_reference::is_negative() noexcept { return doc->is_negative(); } @@ -84734,6 +92549,7 @@ namespace simdjson { simdjson_inline simdjson_result document_reference::get_number() noexcept { return doc->get_root_value_iterator().get_root_number(false); } simdjson_inline simdjson_result document_reference::raw_json_token() noexcept { return doc->raw_json_token(); } simdjson_inline simdjson_result document_reference::at_pointer(std::string_view json_pointer) noexcept { return doc->at_pointer(json_pointer); } + simdjson_inline simdjson_result document_reference::at_path(std::string_view json_path) noexcept { return doc->at_path(json_path); } simdjson_inline simdjson_result document_reference::raw_json() noexcept { return doc->raw_json(); } simdjson_inline document_reference::operator document& () const noexcept { return *doc; } @@ -84865,6 +92681,10 @@ namespace simdjson { if (error()) { return error(); } return first.is_scalar(); } + simdjson_inline simdjson_result simdjson_result::is_string() noexcept { + if (error()) { return error(); } + return first.is_string(); + } simdjson_inline simdjson_result simdjson_result::is_negative() noexcept { if (error()) { return error(); } return first.is_negative(); @@ -84882,6 +92702,11 @@ namespace simdjson { return first.get_number(); } #if SIMDJSON_EXCEPTIONS + template ::value == false>::type> + simdjson_inline simdjson_result::operator T() noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return first; + } simdjson_inline simdjson_result::operator westmere::ondemand::array() & noexcept(false) { if (error()) { throw simdjson_error(error()); } return first; @@ -84935,6 +92760,12 @@ namespace simdjson { return first.at_pointer(json_pointer); } + simdjson_inline simdjson_result simdjson_result::at_path(std::string_view json_path) noexcept { + if (error()) { + return error(); + } + return first.at_path(json_path); + } } // namespace simdjson @@ -85288,9 +93119,18 @@ namespace simdjson { depth--; break; default: // Scalar value document - // TODO: Remove any trailing whitespaces + // TODO: We could remove trailing whitespaces // This returns a string spanning from start of value to the beginning of the next document (excluded) - return std::string_view(reinterpret_cast(stream->buf) + current_index(), stream->parser->implementation->structural_indexes[++cur_struct_index] - current_index() - 1); + { + auto next_index = stream->parser->implementation->structural_indexes[++cur_struct_index]; + // normally the length would be next_index - current_index() - 1, except for the last document + size_t svlen = next_index - current_index(); + const char* start = reinterpret_cast(stream->buf) + current_index(); + while (svlen > 1 && (std::isspace(start[svlen - 1]) || start[svlen - 1] == '\0')) { + svlen--; + } + return std::string_view(start, svlen); + } } cur_struct_index++; } @@ -85385,7 +93225,7 @@ namespace simdjson { namespace westmere { namespace ondemand { - // clang 6 doesn't think the default constructor can be noexcept, so we make it explicit + // clang 6 does not think the default constructor can be noexcept, so we make it explicit simdjson_inline field::field() noexcept : std::pair() {} simdjson_inline field::field(raw_json_string key, ondemand::value&& value) noexcept @@ -85411,11 +93251,32 @@ namespace simdjson { return answer; } + template + simdjson_inline simdjson_warn_unused error_code field::unescaped_key(string_type& receiver, bool allow_replacement) noexcept { + std::string_view key; + SIMDJSON_TRY(unescaped_key(allow_replacement).get(key)); + receiver = key; + return SUCCESS; + } + simdjson_inline raw_json_string field::key() const noexcept { SIMDJSON_ASSUME(first.buf != nullptr); // We would like to call .alive() by Visual Studio won't let us. return first; } + + simdjson_inline std::string_view field::key_raw_json_token() const noexcept { + SIMDJSON_ASSUME(first.buf != nullptr); // We would like to call .alive() by Visual Studio won't let us. + return std::string_view(reinterpret_cast(first.buf - 1), second.iter._json_iter->token.peek(-1) - first.buf + 1); + } + + simdjson_inline std::string_view field::escaped_key() const noexcept { + SIMDJSON_ASSUME(first.buf != nullptr); // We would like to call .alive() by Visual Studio won't let us. + auto end_quote = second.iter._json_iter->token.peek(-1); + while (*end_quote != '"') end_quote--; + return std::string_view(reinterpret_cast(first.buf), end_quote - first.buf); + } + simdjson_inline value& field::value() & noexcept { return second; } @@ -85449,10 +93310,28 @@ namespace simdjson { if (error()) { return error(); } return first.key(); } + + simdjson_inline simdjson_result simdjson_result::key_raw_json_token() noexcept { + if (error()) { return error(); } + return first.key_raw_json_token(); + } + + simdjson_inline simdjson_result simdjson_result::escaped_key() noexcept { + if (error()) { return error(); } + return first.escaped_key(); + } + simdjson_inline simdjson_result simdjson_result::unescaped_key(bool allow_replacement) noexcept { if (error()) { return error(); } return first.unescaped_key(allow_replacement); } + + template + simdjson_inline error_code simdjson_result::unescaped_key(string_type& receiver, bool allow_replacement) noexcept { + if (error()) { return error(); } + return first.unescaped_key(receiver, allow_replacement); + } + simdjson_inline simdjson_result simdjson_result::value() noexcept { if (error()) { return error(); } return std::move(first.value()); @@ -85520,6 +93399,23 @@ namespace simdjson { #endif } +#ifdef SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON + simdjson_inline json_iterator::json_iterator(const uint8_t* buf, ondemand::parser* _parser, bool streaming) noexcept + : token(buf, &_parser->implementation->structural_indexes[0]), + parser{ _parser }, + _string_buf_loc{ parser->string_buf.get() }, + _depth{ 1 }, + _root{ parser->implementation->structural_indexes.get() }, + _streaming{ streaming } + + { + logger::log_headers(); +#if SIMDJSON_CHECK_EOF + assert_more_tokens(); +#endif + } +#endif // SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON + inline void json_iterator::rewind() noexcept { token.set_position(root_position()); logger::log_headers(); // We start again @@ -85753,6 +93649,12 @@ namespace simdjson { #endif // SIMDJSON_CHECK_EOF return token.peek_length(position); } + simdjson_inline uint32_t json_iterator::peek_root_length(token_position position) const noexcept { +#if SIMDJSON_CHECK_EOF + assert_valid_position(position); +#endif // SIMDJSON_CHECK_EOF + return token.peek_root_length(position); + } simdjson_inline token_position json_iterator::last_position() const noexcept { // The following line fails under some compilers... @@ -85798,11 +93700,23 @@ namespace simdjson { } simdjson_inline simdjson_result json_iterator::unescape(raw_json_string in, bool allow_replacement) noexcept { +#if SIMDJSON_DEVELOPMENT_CHECKS + auto result = parser->unescape(in, _string_buf_loc, allow_replacement); + SIMDJSON_ASSUME(!parser->string_buffer_overflow(_string_buf_loc)); + return result; +#else return parser->unescape(in, _string_buf_loc, allow_replacement); +#endif } simdjson_inline simdjson_result json_iterator::unescape_wobbly(raw_json_string in) noexcept { +#if SIMDJSON_DEVELOPMENT_CHECKS + auto result = parser->unescape_wobbly(in, _string_buf_loc); + SIMDJSON_ASSUME(!parser->string_buffer_overflow(_string_buf_loc)); + return result; +#else return parser->unescape_wobbly(in, _string_buf_loc); +#endif } simdjson_inline void json_iterator::reenter_child(token_position position, depth_t child_depth) noexcept { @@ -85927,7 +93841,6 @@ namespace simdjson { return get_uint64(); } - simdjson_inline bool number::is_int64() const noexcept { return get_number_type() == number_type::signed_integer; } @@ -86395,6 +94308,14 @@ namespace simdjson { return child; } + inline simdjson_result object::at_path(std::string_view json_path) noexcept { + auto json_pointer = json_path_to_pointer_conversion(json_path); + if (json_pointer == "-1") { + return INVALID_JSON_POINTER; + } + return at_pointer(json_pointer); + } + simdjson_inline simdjson_result object::count_fields() & noexcept { size_t count{ 0 }; // Important: we do not consume any of the values. @@ -86467,6 +94388,14 @@ namespace simdjson { return first.at_pointer(json_pointer); } + simdjson_inline simdjson_result simdjson_result::at_path( + std::string_view json_path) noexcept { + if (error()) { + return error(); + } + return first.at_path(json_path); + } + inline simdjson_result simdjson_result::reset() noexcept { if (error()) { return error(); } return first.reset(); @@ -86678,6 +94607,11 @@ namespace simdjson { _max_depth = new_max_depth; return SUCCESS; } +#if SIMDJSON_DEVELOPMENT_CHECKS + simdjson_inline simdjson_warn_unused bool parser::string_buffer_overflow(const uint8_t* string_buf_loc) const noexcept { + return (string_buf_loc < string_buf.get()) || (size_t(string_buf_loc - string_buf.get()) >= capacity()); + } +#endif simdjson_warn_unused simdjson_inline simdjson_result parser::iterate(padded_string_view json) & noexcept { if (json.padding() < SIMDJSON_PADDING) { return INSUFFICIENT_PADDING; } @@ -86694,6 +94628,27 @@ namespace simdjson { return document::start({ reinterpret_cast(json.data()), this }); } +#ifdef SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON + simdjson_warn_unused simdjson_inline simdjson_result parser::iterate_allow_incomplete_json(padded_string_view json) & noexcept { + if (json.padding() < SIMDJSON_PADDING) { return INSUFFICIENT_PADDING; } + + json.remove_utf8_bom(); + + // Allocate if needed + if (capacity() < json.length() || !string_buf) { + SIMDJSON_TRY(allocate(json.length(), max_depth())); + } + + // Run stage 1. + const simdjson::error_code err = implementation->stage1(reinterpret_cast(json.data()), json.length(), stage1_mode::regular); + if (err) { + if (err != UNCLOSED_STRING) + return err; + } + return document::start({ reinterpret_cast(json.data()), this, true }); + } +#endif // SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON + simdjson_warn_unused simdjson_inline simdjson_result parser::iterate(const char* json, size_t len, size_t allocated) & noexcept { return iterate(padded_string_view(json, len, allocated)); } @@ -86765,13 +94720,13 @@ namespace simdjson { return iterate_many(s.data(), s.length(), batch_size, allow_comma_separated); } - simdjson_inline size_t parser::capacity() const noexcept { + simdjson_pure simdjson_inline size_t parser::capacity() const noexcept { return _capacity; } - simdjson_inline size_t parser::max_capacity() const noexcept { + simdjson_pure simdjson_inline size_t parser::max_capacity() const noexcept { return _max_capacity; } - simdjson_inline size_t parser::max_depth() const noexcept { + simdjson_pure simdjson_inline size_t parser::max_depth() const noexcept { return _max_depth; } @@ -87320,6 +95275,11 @@ namespace simdjson { return *(position + 1) - *position; } + simdjson_inline uint32_t token_iterator::peek_root_length(token_position position) const noexcept { + return *(position + 2) - *(position) > *(position + 1) - *(position) ? + *(position + 1) - *(position) + : *(position + 2) - *(position); + } simdjson_inline const uint8_t* token_iterator::peek(int32_t delta) const noexcept { return &buf[*(_position + delta)]; } @@ -87469,6 +95429,10 @@ namespace simdjson { } #if SIMDJSON_EXCEPTIONS + template + simdjson_inline value::operator T() noexcept(false) { + return get(); + } simdjson_inline value::operator array() noexcept(false) { return get_array(); } @@ -87555,6 +95519,14 @@ namespace simdjson { return !((this_type == json_type::array) || (this_type == json_type::object)); } + simdjson_inline simdjson_result value::is_string() noexcept { + json_type this_type; + auto error = type().get(this_type); + if (error) { return error; } + return (this_type == json_type::string); + } + + simdjson_inline bool value::is_negative() noexcept { return iter.is_negative(); } @@ -87601,6 +95573,26 @@ namespace simdjson { return iter.json_iter().depth(); } + inline bool is_pointer_well_formed(std::string_view json_pointer) noexcept { + if (simdjson_unlikely(json_pointer.empty())) { // can't be + return false; + } + if (simdjson_unlikely(json_pointer[0] != '/')) { + return false; + } + size_t escape = json_pointer.find('~'); + if (escape == std::string_view::npos) { + return true; + } + if (escape == json_pointer.size() - 1) { + return false; + } + if (json_pointer[escape + 1] != '0' && json_pointer[escape + 1] != '1') { + return false; + } + return true; + } + simdjson_inline simdjson_result value::at_pointer(std::string_view json_pointer) noexcept { json_type t; SIMDJSON_TRY(type().get(t)); @@ -87610,6 +95602,23 @@ namespace simdjson { return (*this).get_array().at_pointer(json_pointer); case json_type::object: return (*this).get_object().at_pointer(json_pointer); + default: + // a non-empty string can be invalid, or accessing a primitive (issue 2154) + if (is_pointer_well_formed(json_pointer)) { + return NO_SUCH_FIELD; + } + return INVALID_JSON_POINTER; + } + } + + simdjson_inline simdjson_result value::at_path(std::string_view json_path) noexcept { + json_type t; + SIMDJSON_TRY(type().get(t)); + switch (t) { + case json_type::array: + return (*this).get_array().at_path(json_path); + case json_type::object: + return (*this).get_object().at_path(json_path); default: return INVALID_JSON_POINTER; } @@ -87768,6 +95777,10 @@ namespace simdjson { if (error()) { return error(); } return first.is_scalar(); } + simdjson_inline simdjson_result simdjson_result::is_string() noexcept { + if (error()) { return error(); } + return first.is_string(); + } simdjson_inline simdjson_result simdjson_result::is_negative() noexcept { if (error()) { return error(); } return first.is_negative(); @@ -87785,6 +95798,11 @@ namespace simdjson { return first.get_number(); } #if SIMDJSON_EXCEPTIONS + template + simdjson_inline simdjson_result::operator T() noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return static_cast(first); + } simdjson_inline simdjson_result::operator westmere::ondemand::array() noexcept(false) { if (error()) { throw simdjson_error(error()); } return first; @@ -87839,11 +95857,22 @@ namespace simdjson { return first.current_depth(); } - simdjson_inline simdjson_result simdjson_result::at_pointer(std::string_view json_pointer) noexcept { - if (error()) { return error(); } + simdjson_inline simdjson_result simdjson_result::at_pointer( + std::string_view json_pointer) noexcept { + if (error()) { + return error(); + } return first.at_pointer(json_pointer); } + simdjson_inline simdjson_result simdjson_result::at_path( + std::string_view json_path) noexcept { + if (error()) { + return error(); + } + return first.at_path(json_path); + } + } // namespace simdjson #endif // SIMDJSON_GENERIC_ONDEMAND_VALUE_INL_H @@ -87858,9 +95887,9 @@ namespace simdjson { /* amalgamation skipped (editor-only): #include "simdjson/generic/atomparsing.h" */ /* amalgamation skipped (editor-only): #include "simdjson/generic/numberparsing.h" */ /* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/json_iterator.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/value_iterator.h" */ /* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/json_type-inl.h" */ /* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/raw_json_string-inl.h" */ -/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/value_iterator.h" */ /* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ namespace simdjson { @@ -87990,7 +96019,7 @@ namespace simdjson { else if (!is_open()) { #if SIMDJSON_DEVELOPMENT_CHECKS // If we're past the end of the object, we're being iterated out of order. - // Note: this isn't perfect detection. It's possible the user is inside some other object; if so, + // Note: this is not perfect detection. It's possible the user is inside some other object; if so, // this object iterator will blithely scan that object for fields. if (_json_iter->depth() < depth() - 1) { return OUT_OF_ORDER_ITERATION; } #endif @@ -88104,7 +96133,7 @@ namespace simdjson { #if SIMDJSON_DEVELOPMENT_CHECKS // If we're past the end of the object, we're being iterated out of order. - // Note: this isn't perfect detection. It's possible the user is inside some other object; if so, + // Note: this is not perfect detection. It's possible the user is inside some other object; if so, // this object iterator will blithely scan that object for fields. if (_json_iter->depth() < depth() - 1) { return OUT_OF_ORDER_ITERATION; } #endif @@ -88447,7 +96476,7 @@ namespace simdjson { } simdjson_inline simdjson_result value_iterator::is_root_integer(bool check_trailing) noexcept { - auto max_len = peek_start_length(); + auto max_len = peek_root_length(); auto json = peek_root_scalar("is_root_integer"); uint8_t tmpbuf[20 + 1 + 1]{}; // <20 digits> is the longest possible unsigned integer tmpbuf[20 + 1] = '\0'; // make sure that buffer is always null terminated. @@ -88463,7 +96492,7 @@ namespace simdjson { } simdjson_inline simdjson_result value_iterator::get_root_number_type(bool check_trailing) noexcept { - auto max_len = peek_start_length(); + auto max_len = peek_root_length(); auto json = peek_root_scalar("number"); // Per https://www.exploringbinary.com/maximum-number-of-decimal-digits-in-binary-floating-point-numbers/, // 1074 is the maximum number of significant fractional digits. Add 8 more digits for the biggest @@ -88471,7 +96500,12 @@ namespace simdjson { uint8_t tmpbuf[1074 + 8 + 1 + 1]; tmpbuf[1074 + 8 + 1] = '\0'; // make sure that buffer is always null terminated. if (!_json_iter->copy_to_buffer(json, max_len, tmpbuf, 1074 + 8 + 1)) { - logger::log_error(*_json_iter, start_position(), depth(), "Root number more than 1082 characters"); + if (numberparsing::check_if_integer(json, max_len)) { + if (check_trailing && !_json_iter->is_single_token()) { return TRAILING_CONTENT; } + logger::log_error(*_json_iter, start_position(), depth(), "Found big integer"); + return number_type::big_integer; + } + logger::log_error(*_json_iter, start_position(), depth(), "Root number more than 1082 characters and not a big integer"); return NUMBER_ERROR; } auto answer = numberparsing::get_number_type(tmpbuf); @@ -88479,15 +96513,21 @@ namespace simdjson { return answer; } simdjson_inline simdjson_result value_iterator::get_root_number(bool check_trailing) noexcept { - auto max_len = peek_start_length(); + auto max_len = peek_root_length(); auto json = peek_root_scalar("number"); // Per https://www.exploringbinary.com/maximum-number-of-decimal-digits-in-binary-floating-point-numbers/, // 1074 is the maximum number of significant fractional digits. Add 8 more digits for the biggest // number: -0.e-308. + // NOTE: the current approach doesn't work for very big integer numbers containing more than 1074 digits. uint8_t tmpbuf[1074 + 8 + 1 + 1]; tmpbuf[1074 + 8 + 1] = '\0'; // make sure that buffer is always null terminated. if (!_json_iter->copy_to_buffer(json, max_len, tmpbuf, 1074 + 8 + 1)) { - logger::log_error(*_json_iter, start_position(), depth(), "Root number more than 1082 characters"); + if (numberparsing::check_if_integer(json, max_len)) { + if (check_trailing && !_json_iter->is_single_token()) { return TRAILING_CONTENT; } + logger::log_error(*_json_iter, start_position(), depth(), "Found big integer"); + return BIGINT_ERROR; + } + logger::log_error(*_json_iter, start_position(), depth(), "Root number more than 1082 characters and not a big integer"); return NUMBER_ERROR; } number num; @@ -88519,7 +96559,7 @@ namespace simdjson { return raw_json_string(json + 1); } simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_root_uint64(bool check_trailing) noexcept { - auto max_len = peek_start_length(); + auto max_len = peek_root_length(); auto json = peek_root_scalar("uint64"); uint8_t tmpbuf[20 + 1 + 1]{}; // <20 digits> is the longest possible unsigned integer tmpbuf[20 + 1] = '\0'; // make sure that buffer is always null terminated. @@ -88535,7 +96575,7 @@ namespace simdjson { return result; } simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_root_uint64_in_string(bool check_trailing) noexcept { - auto max_len = peek_start_length(); + auto max_len = peek_root_length(); auto json = peek_root_scalar("uint64"); uint8_t tmpbuf[20 + 1 + 1]{}; // <20 digits> is the longest possible unsigned integer tmpbuf[20 + 1] = '\0'; // make sure that buffer is always null terminated. @@ -88551,7 +96591,7 @@ namespace simdjson { return result; } simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_root_int64(bool check_trailing) noexcept { - auto max_len = peek_start_length(); + auto max_len = peek_root_length(); auto json = peek_root_scalar("int64"); uint8_t tmpbuf[20 + 1 + 1]; // -<19 digits> is the longest possible integer tmpbuf[20 + 1] = '\0'; // make sure that buffer is always null terminated. @@ -88568,7 +96608,7 @@ namespace simdjson { return result; } simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_root_int64_in_string(bool check_trailing) noexcept { - auto max_len = peek_start_length(); + auto max_len = peek_root_length(); auto json = peek_root_scalar("int64"); uint8_t tmpbuf[20 + 1 + 1]; // -<19 digits> is the longest possible integer tmpbuf[20 + 1] = '\0'; // make sure that buffer is always null terminated. @@ -88585,7 +96625,7 @@ namespace simdjson { return result; } simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_root_double(bool check_trailing) noexcept { - auto max_len = peek_start_length(); + auto max_len = peek_root_length(); auto json = peek_root_scalar("double"); // Per https://www.exploringbinary.com/maximum-number-of-decimal-digits-in-binary-floating-point-numbers/, // 1074 is the maximum number of significant fractional digits. Add 8 more digits for the biggest @@ -88605,7 +96645,7 @@ namespace simdjson { } simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_root_double_in_string(bool check_trailing) noexcept { - auto max_len = peek_start_length(); + auto max_len = peek_root_length(); auto json = peek_root_scalar("double"); // Per https://www.exploringbinary.com/maximum-number-of-decimal-digits-in-binary-floating-point-numbers/, // 1074 is the maximum number of significant fractional digits. Add 8 more digits for the biggest @@ -88624,7 +96664,7 @@ namespace simdjson { return result; } simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_root_bool(bool check_trailing) noexcept { - auto max_len = peek_start_length(); + auto max_len = peek_root_length(); auto json = peek_root_scalar("bool"); uint8_t tmpbuf[5 + 1 + 1]; // +1 for null termination tmpbuf[5 + 1] = '\0'; // make sure that buffer is always null terminated. @@ -88637,7 +96677,7 @@ namespace simdjson { return result; } simdjson_inline simdjson_result value_iterator::is_root_null(bool check_trailing) noexcept { - auto max_len = peek_start_length(); + auto max_len = peek_root_length(); auto json = peek_root_scalar("null"); bool result = (max_len >= 4 && !atomparsing::str4ncmp(json, "null") && (max_len == 4 || jsoncharutils::is_structural_or_whitespace(json[4]))); @@ -88709,6 +96749,9 @@ namespace simdjson { simdjson_inline uint32_t value_iterator::peek_start_length() const noexcept { return _json_iter->peek_length(start_position()); } + simdjson_inline uint32_t value_iterator::peek_root_length() const noexcept { + return _json_iter->peek_root_length(start_position()); + } simdjson_inline const uint8_t* value_iterator::peek_scalar(const char* type) noexcept { logger::log_value(*_json_iter, start_position(), depth(), type); @@ -88950,6 +96993,21844 @@ SIMDJSON_UNTARGET_REGION #endif // SIMDJSON_WESTMERE_IMPLEMENTATION_H /* end file simdjson/westmere/ondemand.h */ +#elif SIMDJSON_BUILTIN_IMPLEMENTATION_IS(lsx) +/* including simdjson/lsx/ondemand.h: #include "simdjson/lsx/ondemand.h" */ +/* begin file simdjson/lsx/ondemand.h */ +#ifndef SIMDJSON_LSX_ONDEMAND_H +#define SIMDJSON_LSX_ONDEMAND_H + +/* including simdjson/lsx/begin.h: #include "simdjson/lsx/begin.h" */ +/* begin file simdjson/lsx/begin.h */ +/* defining SIMDJSON_IMPLEMENTATION to "lsx" */ +#define SIMDJSON_IMPLEMENTATION lsx +/* including simdjson/lsx/base.h: #include "simdjson/lsx/base.h" */ +/* begin file simdjson/lsx/base.h */ +#ifndef SIMDJSON_LSX_BASE_H +#define SIMDJSON_LSX_BASE_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/base.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + /** + * Implementation for LSX. + */ + namespace lsx { + + class implementation; + + namespace { + namespace simd { + template struct simd8; + template struct simd8x64; + } // namespace simd + } // unnamed namespace + + } // namespace lsx +} // namespace simdjson + +#endif // SIMDJSON_LSX_BASE_H +/* end file simdjson/lsx/base.h */ +/* including simdjson/lsx/intrinsics.h: #include "simdjson/lsx/intrinsics.h" */ +/* begin file simdjson/lsx/intrinsics.h */ +#ifndef SIMDJSON_LSX_INTRINSICS_H +#define SIMDJSON_LSX_INTRINSICS_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/lsx/base.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +// This should be the correct header whether +// you use visual studio or other compilers. +#include + +static_assert(sizeof(__m128i) <= simdjson::SIMDJSON_PADDING, "insufficient padding for LoongArch SX"); + +#endif // SIMDJSON_LSX_INTRINSICS_H +/* end file simdjson/lsx/intrinsics.h */ +/* including simdjson/lsx/bitmanipulation.h: #include "simdjson/lsx/bitmanipulation.h" */ +/* begin file simdjson/lsx/bitmanipulation.h */ +#ifndef SIMDJSON_LSX_BITMANIPULATION_H +#define SIMDJSON_LSX_BITMANIPULATION_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/lsx/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/lsx/intrinsics.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/lsx/bitmask.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lsx { + namespace { + + // We sometimes call trailing_zero on inputs that are zero, + // but the algorithms do not end up using the returned value. + // Sadly, sanitizers are not smart enough to figure it out. + SIMDJSON_NO_SANITIZE_UNDEFINED + // This function can be used safely even if not all bytes have been + // initialized. + // See issue https://github.com/simdjson/simdjson/issues/1965 + SIMDJSON_NO_SANITIZE_MEMORY + simdjson_inline int trailing_zeroes(uint64_t input_num) { + return __builtin_ctzll(input_num); + } + + /* result might be undefined when input_num is zero */ + simdjson_inline uint64_t clear_lowest_bit(uint64_t input_num) { + return input_num & (input_num - 1); + } + + /* result might be undefined when input_num is zero */ + simdjson_inline int leading_zeroes(uint64_t input_num) { + return __builtin_clzll(input_num); + } + + /* result might be undefined when input_num is zero */ + simdjson_inline int count_ones(uint64_t input_num) { + return __lsx_vpickve2gr_w(__lsx_vpcnt_d(__m128i(v2u64{ input_num, 0 })), 0); + } + + simdjson_inline bool add_overflow(uint64_t value1, uint64_t value2, uint64_t* result) { + return __builtin_uaddll_overflow(value1, value2, + reinterpret_cast(result)); + } + + } // unnamed namespace + } // namespace lsx +} // namespace simdjson + +#endif // SIMDJSON_LSX_BITMANIPULATION_H +/* end file simdjson/lsx/bitmanipulation.h */ +/* including simdjson/lsx/bitmask.h: #include "simdjson/lsx/bitmask.h" */ +/* begin file simdjson/lsx/bitmask.h */ +#ifndef SIMDJSON_LSX_BITMASK_H +#define SIMDJSON_LSX_BITMASK_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/lsx/base.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lsx { + namespace { + + // + // Perform a "cumulative bitwise xor," flipping bits each time a 1 is encountered. + // + // For example, prefix_xor(00100100) == 00011100 + // + simdjson_inline uint64_t prefix_xor(uint64_t bitmask) { + bitmask ^= bitmask << 1; + bitmask ^= bitmask << 2; + bitmask ^= bitmask << 4; + bitmask ^= bitmask << 8; + bitmask ^= bitmask << 16; + bitmask ^= bitmask << 32; + return bitmask; + } + + } // unnamed namespace + } // namespace lsx +} // namespace simdjson + +#endif +/* end file simdjson/lsx/bitmask.h */ +/* including simdjson/lsx/numberparsing_defs.h: #include "simdjson/lsx/numberparsing_defs.h" */ +/* begin file simdjson/lsx/numberparsing_defs.h */ +#ifndef SIMDJSON_LSX_NUMBERPARSING_DEFS_H +#define SIMDJSON_LSX_NUMBERPARSING_DEFS_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/lsx/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/lsx/intrinsics.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +#include + +namespace simdjson { + namespace lsx { + namespace numberparsing { + + // we don't have appropriate instructions, so let us use a scalar function + // credit: https://johnnylee-sde.github.io/Fast-numeric-string-to-int/ + /** @private */ + static simdjson_inline uint32_t parse_eight_digits_unrolled(const uint8_t* chars) { + uint64_t val; + std::memcpy(&val, chars, sizeof(uint64_t)); + val = (val & 0x0F0F0F0F0F0F0F0F) * 2561 >> 8; + val = (val & 0x00FF00FF00FF00FF) * 6553601 >> 16; + return uint32_t((val & 0x0000FFFF0000FFFF) * 42949672960001 >> 32); + } + + simdjson_inline internal::value128 full_multiplication(uint64_t value1, uint64_t value2) { + internal::value128 answer; + __uint128_t r = (static_cast<__uint128_t>(value1)) * value2; + answer.low = uint64_t(r); + answer.high = uint64_t(r >> 64); + return answer; + } + + } // namespace numberparsing + } // namespace lsx +} // namespace simdjson + +#define SIMDJSON_SWAR_NUMBER_PARSING 1 + +#endif // SIMDJSON_LSX_NUMBERPARSING_DEFS_H +/* end file simdjson/lsx/numberparsing_defs.h */ +/* including simdjson/lsx/simd.h: #include "simdjson/lsx/simd.h" */ +/* begin file simdjson/lsx/simd.h */ +#ifndef SIMDJSON_LSX_SIMD_H +#define SIMDJSON_LSX_SIMD_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/lsx/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/lsx/bitmanipulation.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/simdprune_tables.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lsx { + namespace { + namespace simd { + + // Forward-declared so they can be used by splat and friends. + template + struct base { + __m128i value; + + // Zero constructor + simdjson_inline base() : value{ __m128i() } {} + + // Conversion from SIMD register + simdjson_inline base(const __m128i _value) : value(_value) {} + + // Conversion to SIMD register + simdjson_inline operator const __m128i& () const { return this->value; } + simdjson_inline operator __m128i& () { return this->value; } + simdjson_inline operator const v16i8& () const { return (v16i8&)this->value; } + simdjson_inline operator v16i8& () { return (v16i8&)this->value; } + + // Bit operations + simdjson_inline Child operator|(const Child other) const { return __lsx_vor_v(*this, other); } + simdjson_inline Child operator&(const Child other) const { return __lsx_vand_v(*this, other); } + simdjson_inline Child operator^(const Child other) const { return __lsx_vxor_v(*this, other); } + simdjson_inline Child bit_andnot(const Child other) const { return __lsx_vandn_v(other, *this); } + simdjson_inline Child& operator|=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast | other; return *this_cast; } + simdjson_inline Child& operator&=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast & other; return *this_cast; } + simdjson_inline Child& operator^=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast ^ other; return *this_cast; } + }; + + // Forward-declared so they can be used by splat and friends. + template + struct simd8; + + template> + struct base8 : base> { + simdjson_inline base8() : base>() {} + simdjson_inline base8(const __m128i _value) : base>(_value) {} + + friend simdjson_really_inline Mask operator==(const simd8 lhs, const simd8 rhs) { return __lsx_vseq_b(lhs, rhs); } + + static const int SIZE = sizeof(base>::value); + + template + simdjson_inline simd8 prev(const simd8 prev_chunk) const { + return __lsx_vor_v(__lsx_vbsll_v(*this, N), __lsx_vbsrl_v(prev_chunk, 16 - N)); + } + }; + + // SIMD byte mask type (returned by things like eq and gt) + template<> + struct simd8 : base8 { + static simdjson_inline simd8 splat(bool _value) { + return __lsx_vreplgr2vr_b(uint8_t(-(!!_value))); + } + + simdjson_inline simd8() : base8() {} + simdjson_inline simd8(const __m128i _value) : base8(_value) {} + // Splat constructor + simdjson_inline simd8(bool _value) : base8(splat(_value)) {} + + simdjson_inline int to_bitmask() const { return __lsx_vpickve2gr_w(__lsx_vmskltz_b(*this), 0); } + simdjson_inline bool any() const { return 0 == __lsx_vpickve2gr_hu(__lsx_vmsknz_b(*this), 0); } + simdjson_inline simd8 operator~() const { return *this ^ true; } + }; + + template + struct base8_numeric : base8 { + static simdjson_inline simd8 splat(T _value) { return __lsx_vreplgr2vr_b(_value); } + static simdjson_inline simd8 zero() { return __lsx_vldi(0); } + static simdjson_inline simd8 load(const T values[16]) { + return __lsx_vld(reinterpret_cast(values), 0); + } + // Repeat 16 values as many times as necessary (usually for lookup tables) + static simdjson_inline simd8 repeat_16( + T v0, T v1, T v2, T v3, T v4, T v5, T v6, T v7, + T v8, T v9, T v10, T v11, T v12, T v13, T v14, T v15 + ) { + return simd8( + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15 + ); + } + + simdjson_inline base8_numeric() : base8() {} + simdjson_inline base8_numeric(const __m128i _value) : base8(_value) {} + + // Store to array + simdjson_inline void store(T dst[16]) const { + return __lsx_vst(*this, reinterpret_cast<__m128i*>(dst), 0); + } + + // Addition/subtraction are the same for signed and unsigned + simdjson_inline simd8 operator+(const simd8 other) const { return __lsx_vadd_b(*this, other); } + simdjson_inline simd8 operator-(const simd8 other) const { return __lsx_vsub_b(*this, other); } + simdjson_inline simd8& operator+=(const simd8 other) { *this = *this + other; return *static_cast*>(this); } + simdjson_inline simd8& operator-=(const simd8 other) { *this = *this - other; return *static_cast*>(this); } + + // Override to distinguish from bool version + simdjson_inline simd8 operator~() const { return *this ^ 0xFFu; } + + // Perform a lookup assuming the value is between 0 and 16 (undefined behavior for out of range values) + template + simdjson_inline simd8 lookup_16(simd8 lookup_table) const { + return __lsx_vshuf_b(lookup_table, lookup_table, *this); + } + + // Copies to 'output" all bytes corresponding to a 0 in the mask (interpreted as a bitset). + // Passing a 0 value for mask would be equivalent to writing out every byte to output. + // Only the first 16 - count_ones(mask) bytes of the result are significant but 16 bytes + // get written. + template + simdjson_inline void compress(uint16_t mask, L* output) const { + using internal::thintable_epi8; + using internal::BitsSetTable256mul2; + using internal::pshufb_combine_table; + // this particular implementation was inspired by haswell + // lsx do it in 2 steps, first 8 bytes and then second 8 bytes... + uint8_t mask1 = uint8_t(mask); // least significant 8 bits + uint8_t mask2 = uint8_t(mask >> 8); // second least significant 8 bits + // next line just loads the 64-bit values thintable_epi8[mask1] and + // thintable_epi8[mask2] into a 128-bit register. + __m128i shufmask = { int64_t(thintable_epi8[mask1]), int64_t(thintable_epi8[mask2]) + 0x0808080808080808 }; + // this is the version "nearly pruned" + __m128i pruned = __lsx_vshuf_b(*this, *this, shufmask); + // we still need to put the pieces back together. + // we compute the popcount of the first words: + int pop1 = BitsSetTable256mul2[mask1]; + // then load the corresponding mask + __m128i compactmask = __lsx_vldx(reinterpret_cast(reinterpret_cast(pshufb_combine_table)), pop1 * 8); + __m128i answer = __lsx_vshuf_b(pruned, pruned, compactmask); + __lsx_vst(answer, reinterpret_cast(output), 0); + } + + template + simdjson_inline simd8 lookup_16( + L replace0, L replace1, L replace2, L replace3, + L replace4, L replace5, L replace6, L replace7, + L replace8, L replace9, L replace10, L replace11, + L replace12, L replace13, L replace14, L replace15) const { + return lookup_16(simd8::repeat_16( + replace0, replace1, replace2, replace3, + replace4, replace5, replace6, replace7, + replace8, replace9, replace10, replace11, + replace12, replace13, replace14, replace15 + )); + } + }; + + // Signed bytes + template<> + struct simd8 : base8_numeric { + simdjson_inline simd8() : base8_numeric() {} + simdjson_inline simd8(const __m128i _value) : base8_numeric(_value) {} + // Splat constructor + simdjson_inline simd8(int8_t _value) : simd8(splat(_value)) {} + // Array constructor + simdjson_inline simd8(const int8_t values[16]) : simd8(load(values)) {} + // Member-by-member initialization + simdjson_inline simd8( + int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, + int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 + ) : simd8({ + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10,v11,v12,v13,v14,v15 + }) {} + // Repeat 16 values as many times as necessary (usually for lookup tables) + simdjson_inline static simd8 repeat_16( + int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, + int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 + ) { + return simd8( + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15 + ); + } + + // Order-sensitive comparisons + simdjson_inline simd8 max_val(const simd8 other) const { return __lsx_vmax_b(*this, other); } + simdjson_inline simd8 min_val(const simd8 other) const { return __lsx_vmin_b(*this, other); } + simdjson_inline simd8 operator>(const simd8 other) const { return __lsx_vslt_b(other, *this); } + simdjson_inline simd8 operator<(const simd8 other) const { return __lsx_vslt_b(*this, other); } + }; + + // Unsigned bytes + template<> + struct simd8 : base8_numeric { + simdjson_inline simd8() : base8_numeric() {} + simdjson_inline simd8(const __m128i _value) : base8_numeric(_value) {} + // Splat constructor + simdjson_inline simd8(uint8_t _value) : simd8(splat(_value)) {} + // Array constructor + simdjson_inline simd8(const uint8_t values[16]) : simd8(load(values)) {} + // Member-by-member initialization + simdjson_inline simd8( + uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, + uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 + ) : simd8(__m128i(v16u8{ + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10,v11,v12,v13,v14,v15 + })) {} + // Repeat 16 values as many times as necessary (usually for lookup tables) + simdjson_inline static simd8 repeat_16( + uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, + uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 + ) { + return simd8( + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15 + ); + } + + // Saturated math + simdjson_inline simd8 saturating_add(const simd8 other) const { return __lsx_vsadd_bu(*this, other); } + simdjson_inline simd8 saturating_sub(const simd8 other) const { return __lsx_vssub_bu(*this, other); } + + // Order-specific operations + simdjson_inline simd8 max_val(const simd8 other) const { return __lsx_vmax_bu(*this, other); } + simdjson_inline simd8 min_val(const simd8 other) const { return __lsx_vmin_bu(other, *this); } + // Same as >, but only guarantees true is nonzero (< guarantees true = -1) + simdjson_inline simd8 gt_bits(const simd8 other) const { return this->saturating_sub(other); } + // Same as <, but only guarantees true is nonzero (< guarantees true = -1) + simdjson_inline simd8 lt_bits(const simd8 other) const { return other.saturating_sub(*this); } + simdjson_inline simd8 operator<=(const simd8 other) const { return other.max_val(*this) == other; } + simdjson_inline simd8 operator>=(const simd8 other) const { return other.min_val(*this) == other; } + simdjson_inline simd8 operator>(const simd8 other) const { return this->gt_bits(other).any_bits_set(); } + simdjson_inline simd8 operator<(const simd8 other) const { return this->lt_bits(other).any_bits_set(); } + + // Bit-specific operations + simdjson_inline simd8 bits_not_set() const { return *this == uint8_t(0); } + simdjson_inline simd8 bits_not_set(simd8 bits) const { return (*this & bits).bits_not_set(); } + simdjson_inline simd8 any_bits_set() const { return ~this->bits_not_set(); } + simdjson_inline simd8 any_bits_set(simd8 bits) const { return ~this->bits_not_set(bits); } + simdjson_inline bool is_ascii() const { return 0 == __lsx_vpickve2gr_w(__lsx_vmskltz_b(*this), 0); } + simdjson_inline bool bits_not_set_anywhere() const { return 0 == __lsx_vpickve2gr_hu(__lsx_vmsknz_b(*this), 0); } + simdjson_inline bool any_bits_set_anywhere() const { return !bits_not_set_anywhere(); } + simdjson_inline bool bits_not_set_anywhere(simd8 bits) const { + return 0 == __lsx_vpickve2gr_hu(__lsx_vmsknz_b(__lsx_vand_v(*this, bits)), 0); + } + simdjson_inline bool any_bits_set_anywhere(simd8 bits) const { return !bits_not_set_anywhere(bits); } + template + simdjson_inline simd8 shr() const { return simd8(__lsx_vsrli_b(*this, N)); } + template + simdjson_inline simd8 shl() const { return simd8(__lsx_vslli_b(*this, N)); } + }; + + template + struct simd8x64 { + static constexpr int NUM_CHUNKS = 64 / sizeof(simd8); + static_assert(NUM_CHUNKS == 4, "LSX kernel should use four registers per 64-byte block."); + const simd8 chunks[NUM_CHUNKS]; + + simd8x64(const simd8x64& o) = delete; // no copy allowed + simd8x64& operator=(const simd8& other) = delete; // no assignment allowed + simd8x64() = delete; // no default constructor allowed + + simdjson_inline simd8x64(const simd8 chunk0, const simd8 chunk1, const simd8 chunk2, const simd8 chunk3) : chunks{ chunk0, chunk1, chunk2, chunk3 } {} + simdjson_inline simd8x64(const T ptr[64]) : chunks{ simd8::load(ptr), simd8::load(ptr + 16), simd8::load(ptr + 32), simd8::load(ptr + 48) } {} + + simdjson_inline uint64_t compress(uint64_t mask, T* output) const { + uint16_t mask1 = uint16_t(mask); + uint16_t mask2 = uint16_t(mask >> 16); + uint16_t mask3 = uint16_t(mask >> 32); + uint16_t mask4 = uint16_t(mask >> 48); + __m128i zcnt = __lsx_vpcnt_h(__m128i(v2u64{ ~mask, 0 })); + uint64_t zcnt1 = __lsx_vpickve2gr_hu(zcnt, 0); + uint64_t zcnt2 = __lsx_vpickve2gr_hu(zcnt, 1); + uint64_t zcnt3 = __lsx_vpickve2gr_hu(zcnt, 2); + uint64_t zcnt4 = __lsx_vpickve2gr_hu(zcnt, 3); + uint8_t* voutput = reinterpret_cast(output); + // There should be a critical value which processes in scaler is faster. + if (zcnt1) + this->chunks[0].compress(mask1, reinterpret_cast(voutput)); + voutput += zcnt1; + if (zcnt2) + this->chunks[1].compress(mask2, reinterpret_cast(voutput)); + voutput += zcnt2; + if (zcnt3) + this->chunks[2].compress(mask3, reinterpret_cast(voutput)); + voutput += zcnt3; + if (zcnt4) + this->chunks[3].compress(mask4, reinterpret_cast(voutput)); + voutput += zcnt4; + return reinterpret_cast(voutput) - reinterpret_cast(output); + } + + simdjson_inline void store(T ptr[64]) const { + this->chunks[0].store(ptr + sizeof(simd8) * 0); + this->chunks[1].store(ptr + sizeof(simd8) * 1); + this->chunks[2].store(ptr + sizeof(simd8) * 2); + this->chunks[3].store(ptr + sizeof(simd8) * 3); + } + + simdjson_inline uint64_t to_bitmask() const { + __m128i mask1 = __lsx_vmskltz_b(this->chunks[0]); + __m128i mask2 = __lsx_vmskltz_b(this->chunks[1]); + __m128i mask3 = __lsx_vmskltz_b(this->chunks[2]); + __m128i mask4 = __lsx_vmskltz_b(this->chunks[3]); + mask1 = __lsx_vilvl_h(mask2, mask1); + mask2 = __lsx_vilvl_h(mask4, mask3); + return __lsx_vpickve2gr_du(__lsx_vilvl_w(mask2, mask1), 0); + } + + simdjson_inline simd8 reduce_or() const { + return (this->chunks[0] | this->chunks[1]) | (this->chunks[2] | this->chunks[3]); + } + + simdjson_inline uint64_t eq(const T m) const { + const simd8 mask = simd8::splat(m); + return simd8x64( + this->chunks[0] == mask, + this->chunks[1] == mask, + this->chunks[2] == mask, + this->chunks[3] == mask + ).to_bitmask(); + } + + simdjson_inline uint64_t eq(const simd8x64& other) const { + return simd8x64( + this->chunks[0] == other.chunks[0], + this->chunks[1] == other.chunks[1], + this->chunks[2] == other.chunks[2], + this->chunks[3] == other.chunks[3] + ).to_bitmask(); + } + + simdjson_inline uint64_t lteq(const T m) const { + const simd8 mask = simd8::splat(m); + return simd8x64( + this->chunks[0] <= mask, + this->chunks[1] <= mask, + this->chunks[2] <= mask, + this->chunks[3] <= mask + ).to_bitmask(); + } + }; // struct simd8x64 + + } // namespace simd + } // unnamed namespace + } // namespace lsx +} // namespace simdjson + +#endif // SIMDJSON_LSX_SIMD_H +/* end file simdjson/lsx/simd.h */ +/* including simdjson/lsx/stringparsing_defs.h: #include "simdjson/lsx/stringparsing_defs.h" */ +/* begin file simdjson/lsx/stringparsing_defs.h */ +#ifndef SIMDJSON_LSX_STRINGPARSING_DEFS_H +#define SIMDJSON_LSX_STRINGPARSING_DEFS_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/lsx/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/lsx/simd.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/lsx/bitmanipulation.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lsx { + namespace { + + using namespace simd; + + // Holds backslashes and quotes locations. + struct backslash_and_quote { + public: + static constexpr uint32_t BYTES_PROCESSED = 32; + simdjson_inline static backslash_and_quote copy_and_find(const uint8_t* src, uint8_t* dst); + + simdjson_inline bool has_quote_first() { return ((bs_bits - 1) & quote_bits) != 0; } + simdjson_inline bool has_backslash() { return bs_bits != 0; } + simdjson_inline int quote_index() { return trailing_zeroes(quote_bits); } + simdjson_inline int backslash_index() { return trailing_zeroes(bs_bits); } + + uint32_t bs_bits; + uint32_t quote_bits; + }; // struct backslash_and_quote + + simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8_t* src, uint8_t* dst) { + // this can read up to 31 bytes beyond the buffer size, but we require + // SIMDJSON_PADDING of padding + static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "backslash and quote finder must process fewer than SIMDJSON_PADDING bytes"); + simd8 v0(src); + simd8 v1(src + sizeof(v0)); + v0.store(dst); + v1.store(dst + sizeof(v0)); + + // Getting a 64-bit bitmask is much cheaper than multiple 16-bit bitmasks on LSX; therefore, we + // smash them together into a 64-byte mask and get the bitmask from there. + uint64_t bs_and_quote = simd8x64(v0 == '\\', v1 == '\\', v0 == '"', v1 == '"').to_bitmask(); + return { + uint32_t(bs_and_quote), // bs_bits + uint32_t(bs_and_quote >> 32) // quote_bits + }; + } + + } // unnamed namespace + } // namespace lsx +} // namespace simdjson + +#endif // SIMDJSON_LSX_STRINGPARSING_DEFS_H +/* end file simdjson/lsx/stringparsing_defs.h */ + +#define SIMDJSON_SKIP_BACKSLASH_SHORT_CIRCUIT 1 +/* end file simdjson/lsx/begin.h */ +/* including simdjson/generic/ondemand/amalgamated.h for lsx: #include "simdjson/generic/ondemand/amalgamated.h" */ +/* begin file simdjson/generic/ondemand/amalgamated.h for lsx */ +#if defined(SIMDJSON_CONDITIONAL_INCLUDE) && !defined(SIMDJSON_GENERIC_ONDEMAND_DEPENDENCIES_H) +#error simdjson/generic/ondemand/dependencies.h must be included before simdjson/generic/ondemand/amalgamated.h! +#endif + +// Stuff other things depend on +/* including simdjson/generic/ondemand/base.h for lsx: #include "simdjson/generic/ondemand/base.h" */ +/* begin file simdjson/generic/ondemand/base.h for lsx */ +#ifndef SIMDJSON_GENERIC_ONDEMAND_BASE_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_ONDEMAND_BASE_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lsx { + /** + * A fast, simple, DOM-like interface that parses JSON as you use it. + * + * Designed for maximum speed and a lower memory profile. + */ + namespace ondemand { + + /** Represents the depth of a JSON value (number of nested arrays/objects). */ + using depth_t = int32_t; + + /** @copydoc simdjson::lsx::number_type */ + using number_type = simdjson::lsx::number_type; + + /** @private Position in the JSON buffer indexes */ + using token_position = const uint32_t*; + + class array; + class array_iterator; + class document; + class document_reference; + class document_stream; + class field; + class json_iterator; + enum class json_type; + struct number; + class object; + class object_iterator; + class parser; + class raw_json_string; + class token_iterator; + class value; + class value_iterator; + + } // namespace ondemand + } // namespace lsx +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_ONDEMAND_BASE_H +/* end file simdjson/generic/ondemand/base.h for lsx */ +/* including simdjson/generic/ondemand/value_iterator.h for lsx: #include "simdjson/generic/ondemand/value_iterator.h" */ +/* begin file simdjson/generic/ondemand/value_iterator.h for lsx */ +#ifndef SIMDJSON_GENERIC_ONDEMAND_VALUE_ITERATOR_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_ONDEMAND_VALUE_ITERATOR_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/implementation_simdjson_result_base.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lsx { + namespace ondemand { + + /** + * Iterates through a single JSON value at a particular depth. + * + * Does not keep track of the type of value: provides methods for objects, arrays and scalars and expects + * the caller to call the right ones. + * + * @private This is not intended for external use. + */ + class value_iterator { + protected: + /** The underlying JSON iterator */ + json_iterator* _json_iter{}; + /** The depth of this value */ + depth_t _depth{}; + /** + * The starting token index for this value + */ + token_position _start_position{}; + + public: + simdjson_inline value_iterator() noexcept = default; + + /** + * Denote that we're starting a document. + */ + simdjson_inline void start_document() noexcept; + + /** + * Skips a non-iterated or partially-iterated JSON value, whether it is a scalar, array or object. + * + * Optimized for scalars. + */ + simdjson_warn_unused simdjson_inline error_code skip_child() noexcept; + + /** + * Tell whether the iterator is at the EOF mark + */ + simdjson_inline bool at_end() const noexcept; + + /** + * Tell whether the iterator is at the start of the value + */ + simdjson_inline bool at_start() const noexcept; + + /** + * Tell whether the value is open--if the value has not been used, or the array/object is still open. + */ + simdjson_inline bool is_open() const noexcept; + + /** + * Tell whether the value is at an object's first field (just after the {). + */ + simdjson_inline bool at_first_field() const noexcept; + + /** + * Abandon all iteration. + */ + simdjson_inline void abandon() noexcept; + + /** + * Get the child value as a value_iterator. + */ + simdjson_inline value_iterator child_value() const noexcept; + + /** + * Get the depth of this value. + */ + simdjson_inline int32_t depth() const noexcept; + + /** + * Get the JSON type of this value. + * + * @error TAPE_ERROR when the JSON value is a bad token like "}" "," or "alse". + */ + simdjson_inline simdjson_result type() const noexcept; + + /** + * @addtogroup object Object iteration + * + * Methods to iterate and find object fields. These methods generally *assume* the value is + * actually an object; the caller is responsible for keeping track of that fact. + * + * @{ + */ + + /** + * Start an object iteration. + * + * @returns Whether the object had any fields (returns false for empty). + * @error INCORRECT_TYPE if there is no opening { + */ + simdjson_warn_unused simdjson_inline simdjson_result start_object() noexcept; + /** + * Start an object iteration from the root. + * + * @returns Whether the object had any fields (returns false for empty). + * @error INCORRECT_TYPE if there is no opening { + * @error TAPE_ERROR if there is no matching } at end of document + */ + simdjson_warn_unused simdjson_inline simdjson_result start_root_object() noexcept; + /** + * Checks whether an object could be started from the root. May be called by start_root_object. + * + * @returns SUCCESS if it is possible to safely start an object from the root (document level). + * @error INCORRECT_TYPE if there is no opening { + * @error TAPE_ERROR if there is no matching } at end of document + */ + simdjson_warn_unused simdjson_inline error_code check_root_object() noexcept; + /** + * Start an object iteration after the user has already checked and moved past the {. + * + * Does not move the iterator unless the object is empty ({}). + * + * @returns Whether the object had any fields (returns false for empty). + * @error INCOMPLETE_ARRAY_OR_OBJECT If there are no more tokens (implying the *parent* + * array or object is incomplete). + */ + simdjson_warn_unused simdjson_inline simdjson_result started_object() noexcept; + /** + * Start an object iteration from the root, after the user has already checked and moved past the {. + * + * Does not move the iterator unless the object is empty ({}). + * + * @returns Whether the object had any fields (returns false for empty). + * @error INCOMPLETE_ARRAY_OR_OBJECT If there are no more tokens (implying the *parent* + * array or object is incomplete). + */ + simdjson_warn_unused simdjson_inline simdjson_result started_root_object() noexcept; + + /** + * Moves to the next field in an object. + * + * Looks for , and }. If } is found, the object is finished and the iterator advances past it. + * Otherwise, it advances to the next value. + * + * @return whether there is another field in the object. + * @error TAPE_ERROR If there is a comma missing between fields. + * @error TAPE_ERROR If there is a comma, but not enough tokens remaining to have a key, :, and value. + */ + simdjson_warn_unused simdjson_inline simdjson_result has_next_field() noexcept; + + /** + * Get the current field's key. + */ + simdjson_warn_unused simdjson_inline simdjson_result field_key() noexcept; + + /** + * Pass the : in the field and move to its value. + */ + simdjson_warn_unused simdjson_inline error_code field_value() noexcept; + + /** + * Find the next field with the given key. + * + * Assumes you have called next_field() or otherwise matched the previous value. + * + * This means the iterator must be sitting at the next key: + * + * ``` + * { "a": 1, "b": 2 } + * ^ + * ``` + * + * Key is *raw JSON,* meaning it will be matched against the verbatim JSON without attempting to + * unescape it. This works well for typical ASCII and UTF-8 keys (almost all of them), but may + * fail to match some keys with escapes (\u, \n, etc.). + */ + simdjson_warn_unused simdjson_inline error_code find_field(const std::string_view key) noexcept; + + /** + * Find the next field with the given key, *without* unescaping. This assumes object order: it + * will not find the field if it was already passed when looking for some *other* field. + * + * Assumes you have called next_field() or otherwise matched the previous value. + * + * This means the iterator must be sitting at the next key: + * + * ``` + * { "a": 1, "b": 2 } + * ^ + * ``` + * + * Key is *raw JSON,* meaning it will be matched against the verbatim JSON without attempting to + * unescape it. This works well for typical ASCII and UTF-8 keys (almost all of them), but may + * fail to match some keys with escapes (\u, \n, etc.). + */ + simdjson_warn_unused simdjson_inline simdjson_result find_field_raw(const std::string_view key) noexcept; + + /** + * Find the field with the given key without regard to order, and *without* unescaping. + * + * This is an unordered object lookup: if the field is not found initially, it will cycle around and scan from the beginning. + * + * Assumes you have called next_field() or otherwise matched the previous value. + * + * This means the iterator must be sitting at the next key: + * + * ``` + * { "a": 1, "b": 2 } + * ^ + * ``` + * + * Key is *raw JSON,* meaning it will be matched against the verbatim JSON without attempting to + * unescape it. This works well for typical ASCII and UTF-8 keys (almost all of them), but may + * fail to match some keys with escapes (\u, \n, etc.). + */ + simdjson_warn_unused simdjson_inline simdjson_result find_field_unordered_raw(const std::string_view key) noexcept; + + /** @} */ + + /** + * @addtogroup array Array iteration + * Methods to iterate over array elements. These methods generally *assume* the value is actually + * an object; the caller is responsible for keeping track of that fact. + * @{ + */ + + /** + * Check for an opening [ and start an array iteration. + * + * @returns Whether the array had any elements (returns false for empty). + * @error INCORRECT_TYPE If there is no [. + */ + simdjson_warn_unused simdjson_inline simdjson_result start_array() noexcept; + /** + * Check for an opening [ and start an array iteration while at the root. + * + * @returns Whether the array had any elements (returns false for empty). + * @error INCORRECT_TYPE If there is no [. + * @error TAPE_ERROR if there is no matching ] at end of document + */ + simdjson_warn_unused simdjson_inline simdjson_result start_root_array() noexcept; + /** + * Checks whether an array could be started from the root. May be called by start_root_array. + * + * @returns SUCCESS if it is possible to safely start an array from the root (document level). + * @error INCORRECT_TYPE If there is no [. + * @error TAPE_ERROR if there is no matching ] at end of document + */ + simdjson_warn_unused simdjson_inline error_code check_root_array() noexcept; + /** + * Start an array iteration, after the user has already checked and moved past the [. + * + * Does not move the iterator unless the array is empty ([]). + * + * @returns Whether the array had any elements (returns false for empty). + * @error INCOMPLETE_ARRAY_OR_OBJECT If there are no more tokens (implying the *parent* + * array or object is incomplete). + */ + simdjson_warn_unused simdjson_inline simdjson_result started_array() noexcept; + /** + * Start an array iteration from the root, after the user has already checked and moved past the [. + * + * Does not move the iterator unless the array is empty ([]). + * + * @returns Whether the array had any elements (returns false for empty). + * @error INCOMPLETE_ARRAY_OR_OBJECT If there are no more tokens (implying the *parent* + * array or object is incomplete). + */ + simdjson_warn_unused simdjson_inline simdjson_result started_root_array() noexcept; + + /** + * Moves to the next element in an array. + * + * Looks for , and ]. If ] is found, the array is finished and the iterator advances past it. + * Otherwise, it advances to the next value. + * + * @return Whether there is another element in the array. + * @error TAPE_ERROR If there is a comma missing between elements. + */ + simdjson_warn_unused simdjson_inline simdjson_result has_next_element() noexcept; + + /** + * Get a child value iterator. + */ + simdjson_warn_unused simdjson_inline value_iterator child() const noexcept; + + /** @} */ + + /** + * @defgroup scalar Scalar values + * @addtogroup scalar + * @{ + */ + + simdjson_warn_unused simdjson_inline simdjson_result get_string(bool allow_replacement) noexcept; + template + simdjson_warn_unused simdjson_inline error_code get_string(string_type& receiver, bool allow_replacement) noexcept; + simdjson_warn_unused simdjson_inline simdjson_result get_wobbly_string() noexcept; + simdjson_warn_unused simdjson_inline simdjson_result get_raw_json_string() noexcept; + simdjson_warn_unused simdjson_inline simdjson_result get_uint64() noexcept; + simdjson_warn_unused simdjson_inline simdjson_result get_uint64_in_string() noexcept; + simdjson_warn_unused simdjson_inline simdjson_result get_int64() noexcept; + simdjson_warn_unused simdjson_inline simdjson_result get_int64_in_string() noexcept; + simdjson_warn_unused simdjson_inline simdjson_result get_double() noexcept; + simdjson_warn_unused simdjson_inline simdjson_result get_double_in_string() noexcept; + simdjson_warn_unused simdjson_inline simdjson_result get_bool() noexcept; + simdjson_warn_unused simdjson_inline simdjson_result is_null() noexcept; + simdjson_warn_unused simdjson_inline bool is_negative() noexcept; + simdjson_warn_unused simdjson_inline simdjson_result is_integer() noexcept; + simdjson_warn_unused simdjson_inline simdjson_result get_number_type() noexcept; + simdjson_warn_unused simdjson_inline simdjson_result get_number() noexcept; + + simdjson_warn_unused simdjson_inline simdjson_result get_root_string(bool check_trailing, bool allow_replacement) noexcept; + template + simdjson_warn_unused simdjson_inline error_code get_root_string(string_type& receiver, bool check_trailing, bool allow_replacement) noexcept; + simdjson_warn_unused simdjson_inline simdjson_result get_root_wobbly_string(bool check_trailing) noexcept; + simdjson_warn_unused simdjson_inline simdjson_result get_root_raw_json_string(bool check_trailing) noexcept; + simdjson_warn_unused simdjson_inline simdjson_result get_root_uint64(bool check_trailing) noexcept; + simdjson_warn_unused simdjson_inline simdjson_result get_root_uint64_in_string(bool check_trailing) noexcept; + simdjson_warn_unused simdjson_inline simdjson_result get_root_int64(bool check_trailing) noexcept; + simdjson_warn_unused simdjson_inline simdjson_result get_root_int64_in_string(bool check_trailing) noexcept; + simdjson_warn_unused simdjson_inline simdjson_result get_root_double(bool check_trailing) noexcept; + simdjson_warn_unused simdjson_inline simdjson_result get_root_double_in_string(bool check_trailing) noexcept; + simdjson_warn_unused simdjson_inline simdjson_result get_root_bool(bool check_trailing) noexcept; + simdjson_warn_unused simdjson_inline bool is_root_negative() noexcept; + simdjson_warn_unused simdjson_inline simdjson_result is_root_integer(bool check_trailing) noexcept; + simdjson_warn_unused simdjson_inline simdjson_result get_root_number_type(bool check_trailing) noexcept; + simdjson_warn_unused simdjson_inline simdjson_result get_root_number(bool check_trailing) noexcept; + simdjson_warn_unused simdjson_inline simdjson_result is_root_null(bool check_trailing) noexcept; + + simdjson_inline error_code error() const noexcept; + simdjson_inline uint8_t*& string_buf_loc() noexcept; + simdjson_inline const json_iterator& json_iter() const noexcept; + simdjson_inline json_iterator& json_iter() noexcept; + + simdjson_inline void assert_is_valid() const noexcept; + simdjson_inline bool is_valid() const noexcept; + + /** @} */ + protected: + /** + * Restarts an array iteration. + * @returns Whether the array has any elements (returns false for empty). + */ + simdjson_inline simdjson_result reset_array() noexcept; + /** + * Restarts an object iteration. + * @returns Whether the object has any fields (returns false for empty). + */ + simdjson_inline simdjson_result reset_object() noexcept; + /** + * move_at_start(): moves us so that we are pointing at the beginning of + * the container. It updates the index so that at_start() is true and it + * syncs the depth. The user can then create a new container instance. + * + * Usage: used with value::count_elements(). + **/ + simdjson_inline void move_at_start() noexcept; + + /** + * move_at_container_start(): moves us so that we are pointing at the beginning of + * the container so that assert_at_container_start() passes. + * + * Usage: used with reset_array() and reset_object(). + **/ + simdjson_inline void move_at_container_start() noexcept; + /* Useful for debugging and logging purposes. */ + inline std::string to_string() const noexcept; + simdjson_inline value_iterator(json_iterator* json_iter, depth_t depth, token_position start_index) noexcept; + + simdjson_inline simdjson_result parse_null(const uint8_t* json) const noexcept; + simdjson_inline simdjson_result parse_bool(const uint8_t* json) const noexcept; + simdjson_inline const uint8_t* peek_start() const noexcept; + simdjson_inline uint32_t peek_start_length() const noexcept; + simdjson_inline uint32_t peek_root_length() const noexcept; + + /** + * The general idea of the advance_... methods and the peek_* methods + * is that you first peek and check that you have desired type. If you do, + * and only if you do, then you advance. + * + * We used to unconditionally advance. But this made reasoning about our + * current state difficult. + * Suppose you always advance. Look at the 'value' matching the key + * "shadowable" in the following example... + * + * ({"globals":{"a":{"shadowable":[}}}}) + * + * If the user thinks it is a Boolean and asks for it, then we check the '[', + * decide it is not a Boolean, but still move into the next character ('}'). Now + * we are left pointing at '}' right after a '['. And we have not yet reported + * an error, only that we do not have a Boolean. + * + * If, instead, you just stand your ground until it is content that you know, then + * you will only even move beyond the '[' if the user tells you that you have an + * array. So you will be at the '}' character inside the array and, hopefully, you + * will then catch the error because an array cannot start with '}', but the code + * processing Boolean values does not know this. + * + * So the contract is: first call 'peek_...' and then call 'advance_...' only + * if you have determined that it is a type you can handle. + * + * Unfortunately, it makes the code more verbose, longer and maybe more error prone. + */ + + simdjson_inline void advance_scalar(const char* type) noexcept; + simdjson_inline void advance_root_scalar(const char* type) noexcept; + simdjson_inline void advance_non_root_scalar(const char* type) noexcept; + + simdjson_inline const uint8_t* peek_scalar(const char* type) noexcept; + simdjson_inline const uint8_t* peek_root_scalar(const char* type) noexcept; + simdjson_inline const uint8_t* peek_non_root_scalar(const char* type) noexcept; + + + simdjson_inline error_code start_container(uint8_t start_char, const char* incorrect_type_message, const char* type) noexcept; + simdjson_inline error_code end_container() noexcept; + + /** + * Advance to a place expecting a value (increasing depth). + * + * @return The current token (the one left behind). + * @error TAPE_ERROR If the document ended early. + */ + simdjson_inline simdjson_result advance_to_value() noexcept; + + simdjson_inline error_code incorrect_type_error(const char* message) const noexcept; + simdjson_inline error_code error_unless_more_tokens(uint32_t tokens = 1) const noexcept; + + simdjson_inline bool is_at_start() const noexcept; + /** + * is_at_iterator_start() returns true on an array or object after it has just been + * created, whether the instance is empty or not. + * + * Usage: used by array::begin() in debug mode (SIMDJSON_DEVELOPMENT_CHECKS) + */ + simdjson_inline bool is_at_iterator_start() const noexcept; + + /** + * Assuming that we are within an object, this returns true if we + * are pointing at a key. + * + * Usage: the skip_child() method should never be used while we are pointing + * at a key inside an object. + */ + simdjson_inline bool is_at_key() const noexcept; + + inline void assert_at_start() const noexcept; + inline void assert_at_container_start() const noexcept; + inline void assert_at_root() const noexcept; + inline void assert_at_child() const noexcept; + inline void assert_at_next() const noexcept; + inline void assert_at_non_root_start() const noexcept; + + /** Get the starting position of this value */ + simdjson_inline token_position start_position() const noexcept; + + /** @copydoc error_code json_iterator::position() const noexcept; */ + simdjson_inline token_position position() const noexcept; + /** @copydoc error_code json_iterator::end_position() const noexcept; */ + simdjson_inline token_position last_position() const noexcept; + /** @copydoc error_code json_iterator::end_position() const noexcept; */ + simdjson_inline token_position end_position() const noexcept; + /** @copydoc error_code json_iterator::report_error(error_code error, const char *message) noexcept; */ + simdjson_inline error_code report_error(error_code error, const char* message) noexcept; + + friend class document; + friend class object; + friend class array; + friend class value; + friend class field; + }; // value_iterator + + } // namespace ondemand + } // namespace lsx +} // namespace simdjson + +namespace simdjson { + + template<> + struct simdjson_result : public lsx::implementation_simdjson_result_base { + public: + simdjson_inline simdjson_result(lsx::ondemand::value_iterator&& value) noexcept; ///< @private + simdjson_inline simdjson_result(error_code error) noexcept; ///< @private + simdjson_inline simdjson_result() noexcept = default; + }; + +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_ONDEMAND_VALUE_ITERATOR_H +/* end file simdjson/generic/ondemand/value_iterator.h for lsx */ +/* including simdjson/generic/ondemand/value.h for lsx: #include "simdjson/generic/ondemand/value.h" */ +/* begin file simdjson/generic/ondemand/value.h for lsx */ +#ifndef SIMDJSON_GENERIC_ONDEMAND_VALUE_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_ONDEMAND_VALUE_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/implementation_simdjson_result_base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/value_iterator.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lsx { + namespace ondemand { + + /** + * An ephemeral JSON value returned during iteration. It is only valid for as long as you do + * not access more data in the JSON document. + */ + class value { + public: + /** + * Create a new invalid value. + * + * Exists so you can declare a variable and later assign to it before use. + */ + simdjson_inline value() noexcept = default; + + /** + * Get this value as the given type. + * + * Supported types: object, array, raw_json_string, string_view, uint64_t, int64_t, double, bool + * + * You may use get_double(), get_bool(), get_uint64(), get_int64(), + * get_object(), get_array(), get_raw_json_string(), or get_string() instead. + * + * @returns A value of the given type, parsed from the JSON. + * @returns INCORRECT_TYPE If the JSON value is not the given type. + */ + template simdjson_inline simdjson_result get() noexcept { + // Unless the simdjson library or the user provides an inline implementation, calling this method should + // immediately fail. + static_assert(!sizeof(T), "The get method with given type is not implemented by the simdjson library. " + "The supported types are ondemand::object, ondemand::array, raw_json_string, std::string_view, uint64_t, " + "int64_t, double, and bool. We recommend you use get_double(), get_bool(), get_uint64(), get_int64(), " + " get_object(), get_array(), get_raw_json_string(), or get_string() instead of the get template." + " You may also add support for custom types, see our documentation."); + } + + /** + * Get this value as the given type. + * + * Supported types: object, array, raw_json_string, string_view, uint64_t, int64_t, double, bool + * + * @param out This is set to a value of the given type, parsed from the JSON. If there is an error, this may not be initialized. + * @returns INCORRECT_TYPE If the JSON value is not an object. + * @returns SUCCESS If the parse succeeded and the out parameter was set to the value. + */ + template simdjson_inline error_code get(T& out) noexcept; + + /** + * Cast this JSON value to an array. + * + * @returns An object that can be used to iterate the array. + * @returns INCORRECT_TYPE If the JSON value is not an array. + */ + simdjson_inline simdjson_result get_array() noexcept; + + /** + * Cast this JSON value to an object. + * + * @returns An object that can be used to look up or iterate fields. + * @returns INCORRECT_TYPE If the JSON value is not an object. + */ + simdjson_inline simdjson_result get_object() noexcept; + + /** + * Cast this JSON value to an unsigned integer. + * + * @returns A unsigned 64-bit integer. + * @returns INCORRECT_TYPE If the JSON value is not a 64-bit unsigned integer. + */ + simdjson_inline simdjson_result get_uint64() noexcept; + + /** + * Cast this JSON value (inside string) to a unsigned integer. + * + * @returns A unsigned 64-bit integer. + * @returns INCORRECT_TYPE If the JSON value is not a 64-bit unsigned integer. + */ + simdjson_inline simdjson_result get_uint64_in_string() noexcept; + + /** + * Cast this JSON value to a signed integer. + * + * @returns A signed 64-bit integer. + * @returns INCORRECT_TYPE If the JSON value is not a 64-bit integer. + */ + simdjson_inline simdjson_result get_int64() noexcept; + + /** + * Cast this JSON value (inside string) to a signed integer. + * + * @returns A signed 64-bit integer. + * @returns INCORRECT_TYPE If the JSON value is not a 64-bit integer. + */ + simdjson_inline simdjson_result get_int64_in_string() noexcept; + + /** + * Cast this JSON value to a double. + * + * @returns A double. + * @returns INCORRECT_TYPE If the JSON value is not a valid floating-point number. + */ + simdjson_inline simdjson_result get_double() noexcept; + + /** + * Cast this JSON value (inside string) to a double + * + * @returns A double. + * @returns INCORRECT_TYPE If the JSON value is not a valid floating-point number. + */ + simdjson_inline simdjson_result get_double_in_string() noexcept; + + /** + * Cast this JSON value to a string. + * + * The string is guaranteed to be valid UTF-8. + * + * Equivalent to get(). + * + * Important: a value should be consumed once. Calling get_string() twice on the same value + * is an error. + * + * @returns An UTF-8 string. The string is stored in the parser and will be invalidated the next + * time it parses a document or when it is destroyed. + * @returns INCORRECT_TYPE if the JSON value is not a string. + */ + simdjson_inline simdjson_result get_string(bool allow_replacement = false) noexcept; + + /** + * Attempts to fill the provided std::string reference with the parsed value of the current string. + * + * The string is guaranteed to be valid UTF-8. + * + * Important: a value should be consumed once. Calling get_string() twice on the same value + * is an error. + * + * Performance: This method may be slower than get_string() or get_string(bool) because it may need to allocate memory. + * We recommend you avoid allocating an std::string unless you need to. + * + * @returns INCORRECT_TYPE if the JSON value is not a string. Otherwise, we return SUCCESS. + */ + template + simdjson_inline error_code get_string(string_type& receiver, bool allow_replacement = false) noexcept; + + /** + * Cast this JSON value to a "wobbly" string. + * + * The string is may not be a valid UTF-8 string. + * See https://simonsapin.github.io/wtf-8/ + * + * Important: a value should be consumed once. Calling get_wobbly_string() twice on the same value + * is an error. + * + * @returns An UTF-8 string. The string is stored in the parser and will be invalidated the next + * time it parses a document or when it is destroyed. + * @returns INCORRECT_TYPE if the JSON value is not a string. + */ + simdjson_inline simdjson_result get_wobbly_string() noexcept; + /** + * Cast this JSON value to a raw_json_string. + * + * The string is guaranteed to be valid UTF-8, and may have escapes in it (e.g. \\ or \n). + * + * @returns A pointer to the raw JSON for the given string. + * @returns INCORRECT_TYPE if the JSON value is not a string. + */ + simdjson_inline simdjson_result get_raw_json_string() noexcept; + + /** + * Cast this JSON value to a bool. + * + * @returns A bool value. + * @returns INCORRECT_TYPE if the JSON value is not true or false. + */ + simdjson_inline simdjson_result get_bool() noexcept; + + /** + * Checks if this JSON value is null. If and only if the value is + * null, then it is consumed (we advance). If we find a token that + * begins with 'n' but is not 'null', then an error is returned. + * + * @returns Whether the value is null. + * @returns INCORRECT_TYPE If the JSON value begins with 'n' and is not 'null'. + */ + simdjson_inline simdjson_result is_null() noexcept; + +#if SIMDJSON_EXCEPTIONS + /** + * Cast this JSON value to an instance of type T. The programmer is responsible for + * providing an implementation of get for the type T, if T is not one of the types + * supported by the library (object, array, raw_json_string, string_view, uint64_t, etc.). + * + * See https://github.com/simdjson/simdjson/blob/master/doc/basics.md#adding-support-for-custom-types + * + * @returns An instance of type T + */ + template + explicit simdjson_inline operator T() noexcept(false); + /** + * Cast this JSON value to an array. + * + * @returns An object that can be used to iterate the array. + * @exception simdjson_error(INCORRECT_TYPE) If the JSON value is not an array. + */ + simdjson_inline operator array() noexcept(false); + /** + * Cast this JSON value to an object. + * + * @returns An object that can be used to look up or iterate fields. + * @exception simdjson_error(INCORRECT_TYPE) If the JSON value is not an object. + */ + simdjson_inline operator object() noexcept(false); + /** + * Cast this JSON value to an unsigned integer. + * + * @returns A signed 64-bit integer. + * @exception simdjson_error(INCORRECT_TYPE) If the JSON value is not a 64-bit unsigned integer. + */ + simdjson_inline operator uint64_t() noexcept(false); + /** + * Cast this JSON value to a signed integer. + * + * @returns A signed 64-bit integer. + * @exception simdjson_error(INCORRECT_TYPE) If the JSON value is not a 64-bit integer. + */ + simdjson_inline operator int64_t() noexcept(false); + /** + * Cast this JSON value to a double. + * + * @returns A double. + * @exception simdjson_error(INCORRECT_TYPE) If the JSON value is not a valid floating-point number. + */ + simdjson_inline operator double() noexcept(false); + /** + * Cast this JSON value to a string. + * + * The string is guaranteed to be valid UTF-8. + * + * Equivalent to get(). + * + * @returns An UTF-8 string. The string is stored in the parser and will be invalidated the next + * time it parses a document or when it is destroyed. + * @exception simdjson_error(INCORRECT_TYPE) if the JSON value is not a string. + */ + simdjson_inline operator std::string_view() noexcept(false); + /** + * Cast this JSON value to a raw_json_string. + * + * The string is guaranteed to be valid UTF-8, and may have escapes in it (e.g. \\ or \n). + * + * @returns A pointer to the raw JSON for the given string. + * @exception simdjson_error(INCORRECT_TYPE) if the JSON value is not a string. + */ + simdjson_inline operator raw_json_string() noexcept(false); + /** + * Cast this JSON value to a bool. + * + * @returns A bool value. + * @exception simdjson_error(INCORRECT_TYPE) if the JSON value is not true or false. + */ + simdjson_inline operator bool() noexcept(false); +#endif + + /** + * Begin array iteration. + * + * Part of the std::iterable interface. + * + * @returns INCORRECT_TYPE If the JSON value is not an array. + */ + simdjson_inline simdjson_result begin() & noexcept; + /** + * Sentinel representing the end of the array. + * + * Part of the std::iterable interface. + */ + simdjson_inline simdjson_result end() & noexcept; + /** + * This method scans the array and counts the number of elements. + * The count_elements method should always be called before you have begun + * iterating through the array: it is expected that you are pointing at + * the beginning of the array. + * The runtime complexity is linear in the size of the array. After + * calling this function, if successful, the array is 'rewinded' at its + * beginning as if it had never been accessed. If the JSON is malformed (e.g., + * there is a missing comma), then an error is returned and it is no longer + * safe to continue. + * + * Performance hint: You should only call count_elements() as a last + * resort as it may require scanning the document twice or more. + */ + simdjson_inline simdjson_result count_elements() & noexcept; + /** + * This method scans the object and counts the number of key-value pairs. + * The count_fields method should always be called before you have begun + * iterating through the object: it is expected that you are pointing at + * the beginning of the object. + * The runtime complexity is linear in the size of the object. After + * calling this function, if successful, the object is 'rewinded' at its + * beginning as if it had never been accessed. If the JSON is malformed (e.g., + * there is a missing comma), then an error is returned and it is no longer + * safe to continue. + * + * To check that an object is empty, it is more performant to use + * the is_empty() method on the object instance. + * + * Performance hint: You should only call count_fields() as a last + * resort as it may require scanning the document twice or more. + */ + simdjson_inline simdjson_result count_fields() & noexcept; + /** + * Get the value at the given index in the array. This function has linear-time complexity. + * This function should only be called once on an array instance since the array iterator is not reset between each call. + * + * @return The value at the given index, or: + * - INDEX_OUT_OF_BOUNDS if the array index is larger than an array length + */ + simdjson_inline simdjson_result at(size_t index) noexcept; + /** + * Look up a field by name on an object (order-sensitive). + * + * The following code reads z, then y, then x, and thus will not retrieve x or y if fed the + * JSON `{ "x": 1, "y": 2, "z": 3 }`: + * + * ```c++ + * simdjson::ondemand::parser parser; + * auto obj = parser.parse(R"( { "x": 1, "y": 2, "z": 3 } )"_padded); + * double z = obj.find_field("z"); + * double y = obj.find_field("y"); + * double x = obj.find_field("x"); + * ``` + * If you have multiple fields with a matching key ({"x": 1, "x": 1}) be mindful + * that only one field is returned. + + * **Raw Keys:** The lookup will be done against the *raw* key, and will not unescape keys. + * e.g. `object["a"]` will match `{ "a": 1 }`, but will *not* match `{ "\u0061": 1 }`. + * + * @param key The key to look up. + * @returns The value of the field, or NO_SUCH_FIELD if the field is not in the object. + */ + simdjson_inline simdjson_result find_field(std::string_view key) noexcept; + /** @overload simdjson_inline simdjson_result find_field(std::string_view key) noexcept; */ + simdjson_inline simdjson_result find_field(const char* key) noexcept; + + /** + * Look up a field by name on an object, without regard to key order. + * + * **Performance Notes:** This is a bit less performant than find_field(), though its effect varies + * and often appears negligible. It starts out normally, starting out at the last field; but if + * the field is not found, it scans from the beginning of the object to see if it missed it. That + * missing case has a non-cache-friendly bump and lots of extra scanning, especially if the object + * in question is large. The fact that the extra code is there also bumps the executable size. + * + * It is the default, however, because it would be highly surprising (and hard to debug) if the + * default behavior failed to look up a field just because it was in the wrong order--and many + * APIs assume this. Therefore, you must be explicit if you want to treat objects as out of order. + * + * If you have multiple fields with a matching key ({"x": 1, "x": 1}) be mindful + * that only one field is returned. + * + * Use find_field() if you are sure fields will be in order (or are willing to treat it as if the + * field as not there when they are not in order). + * + * @param key The key to look up. + * @returns The value of the field, or NO_SUCH_FIELD if the field is not in the object. + */ + simdjson_inline simdjson_result find_field_unordered(std::string_view key) noexcept; + /** @overload simdjson_inline simdjson_result find_field_unordered(std::string_view key) noexcept; */ + simdjson_inline simdjson_result find_field_unordered(const char* key) noexcept; + /** @overload simdjson_inline simdjson_result find_field_unordered(std::string_view key) noexcept; */ + simdjson_inline simdjson_result operator[](std::string_view key) noexcept; + /** @overload simdjson_inline simdjson_result find_field_unordered(std::string_view key) noexcept; */ + simdjson_inline simdjson_result operator[](const char* key) noexcept; + + /** + * Get the type of this JSON value. It does not validate or consume the value. + * E.g., you must still call "is_null()" to check that a value is null even if + * "type()" returns json_type::null. + * + * NOTE: If you're only expecting a value to be one type (a typical case), it's generally + * better to just call .get_double, .get_string, etc. and check for INCORRECT_TYPE (or just + * let it throw an exception). + * + * @return The type of JSON value (json_type::array, json_type::object, json_type::string, + * json_type::number, json_type::boolean, or json_type::null). + * @error TAPE_ERROR when the JSON value is a bad token like "}" "," or "alse". + */ + simdjson_inline simdjson_result type() noexcept; + + /** + * Checks whether the value is a scalar (string, number, null, Boolean). + * Returns false when there it is an array or object. + * + * @returns true if the type is string, number, null, Boolean + * @error TAPE_ERROR when the JSON value is a bad token like "}" "," or "alse". + */ + simdjson_inline simdjson_result is_scalar() noexcept; + /** + * Checks whether the value is a string. + * + * @returns true if the type is string + * @error TAPE_ERROR when the JSON value is a bad token like "}" "," or "alse". + */ + simdjson_inline simdjson_result is_string() noexcept; + + /** + * Checks whether the value is a negative number. + * + * @returns true if the number if negative. + */ + simdjson_inline bool is_negative() noexcept; + /** + * Checks whether the value is an integer number. Note that + * this requires to partially parse the number string. If + * the value is determined to be an integer, it may still + * not parse properly as an integer in subsequent steps + * (e.g., it might overflow). + * + * Performance note: if you call this function systematically + * before parsing a number, you may have fallen for a performance + * anti-pattern. + * + * @returns true if the number if negative. + */ + simdjson_inline simdjson_result is_integer() noexcept; + /** + * Determine the number type (integer or floating-point number) as quickly + * as possible. This function does not fully validate the input. It is + * useful when you only need to classify the numbers, without parsing them. + * + * If you are planning to retrieve the value or you need full validation, + * consider using the get_number() method instead: it will fully parse + * and validate the input, and give you access to the type: + * get_number().get_number_type(). + * + * get_number_type() is number_type::unsigned_integer if we have + * an integer greater or equal to 9223372036854775808. + * get_number_type() is number_type::signed_integer if we have an + * integer that is less than 9223372036854775808. + * get_number_type() is number_type::big_integer for integers that do not fit in 64 bits, + * in which case the digit_count is set to the length of the big integer string. + * Otherwise, get_number_type() has value number_type::floating_point_number. + * + * This function requires processing the number string, but it is expected + * to be faster than get_number().get_number_type() because it is does not + * parse the number value. + * + * @returns the type of the number + */ + simdjson_inline simdjson_result get_number_type() noexcept; + + /** + * Attempt to parse an ondemand::number. An ondemand::number may + * contain an integer value or a floating-point value, the simdjson + * library will autodetect the type. Thus it is a dynamically typed + * number. Before accessing the value, you must determine the detected + * type. + * + * number.get_number_type() is number_type::signed_integer if we have + * an integer in [-9223372036854775808,9223372036854775808) + * You can recover the value by calling number.get_int64() and you + * have that number.is_int64() is true. + * + * number.get_number_type() is number_type::unsigned_integer if we have + * an integer in [9223372036854775808,18446744073709551616) + * You can recover the value by calling number.get_uint64() and you + * have that number.is_uint64() is true. + * + * For integers that do not fit in 64 bits, the function returns BIGINT_ERROR error code. + * + * Otherwise, number.get_number_type() has value number_type::floating_point_number + * and we have a binary64 number. + * You can recover the value by calling number.get_double() and you + * have that number.is_double() is true. + * + * You must check the type before accessing the value: it is an error + * to call "get_int64()" when number.get_number_type() is not + * number_type::signed_integer and when number.is_int64() is false. + * + * Performance note: this is designed with performance in mind. When + * calling 'get_number()', you scan the number string only once, determining + * efficiently the type and storing it in an efficient manner. + */ + simdjson_warn_unused simdjson_inline simdjson_result get_number() noexcept; + + /** + * Get the raw JSON for this token. + * + * The string_view will always point into the input buffer. + * + * The string_view will start at the beginning of the token, and include the entire token + * *as well as all spaces until the next token (or EOF).* This means, for example, that a + * string token always begins with a " and is always terminated by the final ", possibly + * followed by a number of spaces. + * + * The string_view is *not* null-terminated. However, if this is a scalar (string, number, + * boolean, or null), the character after the end of the string_view is guaranteed to be + * a non-space token. + * + * Tokens include: + * - { + * - [ + * - "a string (possibly with UTF-8 or backslashed characters like \\\")". + * - -1.2e-100 + * - true + * - false + * - null + * + * See also value::raw_json(). + */ + simdjson_inline std::string_view raw_json_token() noexcept; + + /** + * Get a string_view pointing at this value in the JSON document. + * If this element is an array or an object, it consumes the array or the object + * and returns a string_view instance corresponding to the + * array as represented in JSON. It points inside the original document. + * If this element is a scalar (string, number, Boolean, null), it returns what + * raw_json_token() would return. + */ + simdjson_inline simdjson_result raw_json() noexcept; + + /** + * Returns the current location in the document if in bounds. + */ + simdjson_inline simdjson_result current_location() noexcept; + + /** + * Returns the current depth in the document if in bounds. + * + * E.g., + * 0 = finished with document + * 1 = document root value (could be [ or {, not yet known) + * 2 = , or } inside root array/object + * 3 = key or value inside root array/object. + */ + simdjson_inline int32_t current_depth() const noexcept; + + /** + * Get the value associated with the given JSON pointer. We use the RFC 6901 + * https://tools.ietf.org/html/rfc6901 standard. + * + * ondemand::parser parser; + * auto json = R"({ "foo": { "a": [ 10, 20, 30 ] }})"_padded; + * auto doc = parser.iterate(json); + * doc.at_pointer("/foo/a/1") == 20 + * + * It is allowed for a key to be the empty string: + * + * ondemand::parser parser; + * auto json = R"({ "": { "a": [ 10, 20, 30 ] }})"_padded; + * auto doc = parser.iterate(json); + * doc.at_pointer("//a/1") == 20 + * + * Note that at_pointer() called on the document automatically calls the document's rewind + * method between each call. It invalidates all previously accessed arrays, objects and values + * that have not been consumed. + * + * Calling at_pointer() on non-document instances (e.g., arrays and objects) is not + * standardized (by RFC 6901). We provide some experimental support for JSON pointers + * on non-document instances. Yet it is not the case when calling at_pointer on an array + * or an object instance: there is no rewind and no invalidation. + * + * You may only call at_pointer on an array after it has been created, but before it has + * been first accessed. When calling at_pointer on an array, the pointer is advanced to + * the location indicated by the JSON pointer (in case of success). It is no longer possible + * to call at_pointer on the same array. + * + * You may call at_pointer more than once on an object, but each time the pointer is advanced + * to be within the value matched by the key indicated by the JSON pointer query. Thus any preceding + * key (as well as the current key) can no longer be used with following JSON pointer calls. + * + * Also note that at_pointer() relies on find_field() which implies that we do not unescape keys when matching + * + * @return The value associated with the given JSON pointer, or: + * - NO_SUCH_FIELD if a field does not exist in an object + * - INDEX_OUT_OF_BOUNDS if an array index is larger than an array length + * - INCORRECT_TYPE if a non-integer is used to access an array + * - INVALID_JSON_POINTER if the JSON pointer is invalid and cannot be parsed + */ + simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; + + /** + * Get the value associated with the given JSONPath expression. We only support + * JSONPath queries that trivially convertible to JSON Pointer queries: key + * names and array indices. + * + * @return The value associated with the given JSONPath expression, or: + * - INVALID_JSON_POINTER if the JSONPath to JSON Pointer conversion fails + * - NO_SUCH_FIELD if a field does not exist in an object + * - INDEX_OUT_OF_BOUNDS if an array index is larger than an array length + * - INCORRECT_TYPE if a non-integer is used to access an array + */ + simdjson_inline simdjson_result at_path(std::string_view at_path) noexcept; + + + protected: + /** + * Create a value. + */ + simdjson_inline value(const value_iterator& iter) noexcept; + + /** + * Skip this value, allowing iteration to continue. + */ + simdjson_inline void skip() noexcept; + + /** + * Start a value at the current position. + * + * (It should already be started; this is just a self-documentation method.) + */ + static simdjson_inline value start(const value_iterator& iter) noexcept; + + /** + * Resume a value. + */ + static simdjson_inline value resume(const value_iterator& iter) noexcept; + + /** + * Get the object, starting or resuming it as necessary + */ + simdjson_inline simdjson_result start_or_resume_object() noexcept; + + // simdjson_inline void log_value(const char *type) const noexcept; + // simdjson_inline void log_error(const char *message) const noexcept; + + value_iterator iter{}; + + friend class document; + friend class array_iterator; + friend class field; + friend class object; + friend struct simdjson_result; + friend struct simdjson_result; + friend class field; + }; + + } // namespace ondemand + } // namespace lsx +} // namespace simdjson + +namespace simdjson { + + template<> + struct simdjson_result : public lsx::implementation_simdjson_result_base { + public: + simdjson_inline simdjson_result(lsx::ondemand::value&& value) noexcept; ///< @private + simdjson_inline simdjson_result(error_code error) noexcept; ///< @private + simdjson_inline simdjson_result() noexcept = default; + + simdjson_inline simdjson_result get_array() noexcept; + simdjson_inline simdjson_result get_object() noexcept; + + simdjson_inline simdjson_result get_uint64() noexcept; + simdjson_inline simdjson_result get_uint64_in_string() noexcept; + simdjson_inline simdjson_result get_int64() noexcept; + simdjson_inline simdjson_result get_int64_in_string() noexcept; + simdjson_inline simdjson_result get_double() noexcept; + simdjson_inline simdjson_result get_double_in_string() noexcept; + simdjson_inline simdjson_result get_string(bool allow_replacement = false) noexcept; + template + simdjson_inline error_code get_string(string_type& receiver, bool allow_replacement = false) noexcept; + simdjson_inline simdjson_result get_wobbly_string() noexcept; + simdjson_inline simdjson_result get_raw_json_string() noexcept; + simdjson_inline simdjson_result get_bool() noexcept; + simdjson_inline simdjson_result is_null() noexcept; + + template simdjson_inline simdjson_result get() noexcept; + + template simdjson_inline error_code get(T& out) noexcept; + +#if SIMDJSON_EXCEPTIONS + template + explicit simdjson_inline operator T() noexcept(false); + simdjson_inline operator lsx::ondemand::array() noexcept(false); + simdjson_inline operator lsx::ondemand::object() noexcept(false); + simdjson_inline operator uint64_t() noexcept(false); + simdjson_inline operator int64_t() noexcept(false); + simdjson_inline operator double() noexcept(false); + simdjson_inline operator std::string_view() noexcept(false); + simdjson_inline operator lsx::ondemand::raw_json_string() noexcept(false); + simdjson_inline operator bool() noexcept(false); +#endif + simdjson_inline simdjson_result count_elements() & noexcept; + simdjson_inline simdjson_result count_fields() & noexcept; + simdjson_inline simdjson_result at(size_t index) noexcept; + simdjson_inline simdjson_result begin() & noexcept; + simdjson_inline simdjson_result end() & noexcept; + + /** + * Look up a field by name on an object (order-sensitive). + * + * The following code reads z, then y, then x, and thus will not retrieve x or y if fed the + * JSON `{ "x": 1, "y": 2, "z": 3 }`: + * + * ```c++ + * simdjson::ondemand::parser parser; + * auto obj = parser.parse(R"( { "x": 1, "y": 2, "z": 3 } )"_padded); + * double z = obj.find_field("z"); + * double y = obj.find_field("y"); + * double x = obj.find_field("x"); + * ``` + * + * **Raw Keys:** The lookup will be done against the *raw* key, and will not unescape keys. + * e.g. `object["a"]` will match `{ "a": 1 }`, but will *not* match `{ "\u0061": 1 }`. + * + * @param key The key to look up. + * @returns The value of the field, or NO_SUCH_FIELD if the field is not in the object. + */ + simdjson_inline simdjson_result find_field(std::string_view key) noexcept; + /** @overload simdjson_inline simdjson_result find_field(std::string_view key) noexcept; */ + simdjson_inline simdjson_result find_field(const char* key) noexcept; + + /** + * Look up a field by name on an object, without regard to key order. + * + * **Performance Notes:** This is a bit less performant than find_field(), though its effect varies + * and often appears negligible. It starts out normally, starting out at the last field; but if + * the field is not found, it scans from the beginning of the object to see if it missed it. That + * missing case has a non-cache-friendly bump and lots of extra scanning, especially if the object + * in question is large. The fact that the extra code is there also bumps the executable size. + * + * It is the default, however, because it would be highly surprising (and hard to debug) if the + * default behavior failed to look up a field just because it was in the wrong order--and many + * APIs assume this. Therefore, you must be explicit if you want to treat objects as out of order. + * + * Use find_field() if you are sure fields will be in order (or are willing to treat it as if the + * field as not there when they are not in order). + * + * @param key The key to look up. + * @returns The value of the field, or NO_SUCH_FIELD if the field is not in the object. + */ + simdjson_inline simdjson_result find_field_unordered(std::string_view key) noexcept; + /** @overload simdjson_inline simdjson_result find_field_unordered(std::string_view key) noexcept; */ + simdjson_inline simdjson_result find_field_unordered(const char* key) noexcept; + /** @overload simdjson_inline simdjson_result find_field_unordered(std::string_view key) noexcept; */ + simdjson_inline simdjson_result operator[](std::string_view key) noexcept; + /** @overload simdjson_inline simdjson_result find_field_unordered(std::string_view key) noexcept; */ + simdjson_inline simdjson_result operator[](const char* key) noexcept; + + /** + * Get the type of this JSON value. + * + * NOTE: If you're only expecting a value to be one type (a typical case), it's generally + * better to just call .get_double, .get_string, etc. and check for INCORRECT_TYPE (or just + * let it throw an exception). + */ + simdjson_inline simdjson_result type() noexcept; + simdjson_inline simdjson_result is_scalar() noexcept; + simdjson_inline simdjson_result is_string() noexcept; + simdjson_inline simdjson_result is_negative() noexcept; + simdjson_inline simdjson_result is_integer() noexcept; + simdjson_inline simdjson_result get_number_type() noexcept; + simdjson_inline simdjson_result get_number() noexcept; + + /** @copydoc simdjson_inline std::string_view value::raw_json_token() const noexcept */ + simdjson_inline simdjson_result raw_json_token() noexcept; + simdjson_inline simdjson_result raw_json() noexcept; + + /** @copydoc simdjson_inline simdjson_result current_location() noexcept */ + simdjson_inline simdjson_result current_location() noexcept; + /** @copydoc simdjson_inline int32_t current_depth() const noexcept */ + simdjson_inline simdjson_result current_depth() const noexcept; + simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; + simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; + }; + +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_ONDEMAND_VALUE_H +/* end file simdjson/generic/ondemand/value.h for lsx */ +/* including simdjson/generic/ondemand/logger.h for lsx: #include "simdjson/generic/ondemand/logger.h" */ +/* begin file simdjson/generic/ondemand/logger.h for lsx */ +#ifndef SIMDJSON_GENERIC_ONDEMAND_LOGGER_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_ONDEMAND_LOGGER_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/base.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lsx { + namespace ondemand { + + // Logging should be free unless SIMDJSON_VERBOSE_LOGGING is set. Importantly, it is critical + // that the call to the log functions be side-effect free. Thus, for example, you should not + // create temporary std::string instances. + namespace logger { + + enum class log_level : int32_t { + info = 0, + error = 1 + }; + +#if SIMDJSON_VERBOSE_LOGGING + static constexpr const bool LOG_ENABLED = true; +#else + static constexpr const bool LOG_ENABLED = false; +#endif + + // We do not want these functions to be 'really inlined' since real inlining is + // for performance purposes and if you are using the loggers, you do not care about + // performance (or should not). + static inline void log_headers() noexcept; + // If args are provided, title will be treated as format string + template + static inline void log_line(const json_iterator& iter, token_position index, depth_t depth, const char* title_prefix, const char* title, std::string_view detail, logger::log_level level, Args&&... args) noexcept; + template + static inline void log_line(const json_iterator& iter, const char* title_prefix, const char* title, std::string_view detail, int delta, int depth_delta, logger::log_level level, Args&&... args) noexcept; + static inline void log_event(const json_iterator& iter, const char* type, std::string_view detail = "", int delta = 0, int depth_delta = 0) noexcept; + static inline void log_value(const json_iterator& iter, token_position index, depth_t depth, const char* type, std::string_view detail = "") noexcept; + static inline void log_value(const json_iterator& iter, const char* type, std::string_view detail = "", int delta = -1, int depth_delta = 0) noexcept; + static inline void log_start_value(const json_iterator& iter, token_position index, depth_t depth, const char* type, std::string_view detail = "") noexcept; + static inline void log_start_value(const json_iterator& iter, const char* type, int delta = -1, int depth_delta = 0) noexcept; + static inline void log_end_value(const json_iterator& iter, const char* type, int delta = -1, int depth_delta = 0) noexcept; + + static inline void log_error(const json_iterator& iter, token_position index, depth_t depth, const char* error, const char* detail = "") noexcept; + static inline void log_error(const json_iterator& iter, const char* error, const char* detail = "", int delta = -1, int depth_delta = 0) noexcept; + + static inline void log_event(const value_iterator& iter, const char* type, std::string_view detail = "", int delta = 0, int depth_delta = 0) noexcept; + static inline void log_value(const value_iterator& iter, const char* type, std::string_view detail = "", int delta = -1, int depth_delta = 0) noexcept; + static inline void log_start_value(const value_iterator& iter, const char* type, int delta = -1, int depth_delta = 0) noexcept; + static inline void log_end_value(const value_iterator& iter, const char* type, int delta = -1, int depth_delta = 0) noexcept; + static inline void log_error(const value_iterator& iter, const char* error, const char* detail = "", int delta = -1, int depth_delta = 0) noexcept; + + } // namespace logger + } // namespace ondemand + } // namespace lsx +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_ONDEMAND_LOGGER_H +/* end file simdjson/generic/ondemand/logger.h for lsx */ +/* including simdjson/generic/ondemand/token_iterator.h for lsx: #include "simdjson/generic/ondemand/token_iterator.h" */ +/* begin file simdjson/generic/ondemand/token_iterator.h for lsx */ +#ifndef SIMDJSON_GENERIC_ONDEMAND_TOKEN_ITERATOR_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_ONDEMAND_TOKEN_ITERATOR_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/implementation_simdjson_result_base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/logger.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lsx { + namespace ondemand { + + /** + * Iterates through JSON tokens (`{` `}` `[` `]` `,` `:` `""` `123` `true` `false` `null`) + * detected by stage 1. + * + * @private This is not intended for external use. + */ + class token_iterator { + public: + /** + * Create a new invalid token_iterator. + * + * Exists so you can declare a variable and later assign to it before use. + */ + simdjson_inline token_iterator() noexcept = default; + simdjson_inline token_iterator(token_iterator&& other) noexcept = default; + simdjson_inline token_iterator& operator=(token_iterator&& other) noexcept = default; + simdjson_inline token_iterator(const token_iterator& other) noexcept = default; + simdjson_inline token_iterator& operator=(const token_iterator& other) noexcept = default; + + /** + * Advance to the next token (returning the current one). + */ + simdjson_inline const uint8_t* return_current_and_advance() noexcept; + /** + * Reports the current offset in bytes from the start of the underlying buffer. + */ + simdjson_inline uint32_t current_offset() const noexcept; + /** + * Get the JSON text for a given token (relative). + * + * This is not null-terminated; it is a view into the JSON. + * + * @param delta The relative position of the token to retrieve. e.g. 0 = current token, + * 1 = next token, -1 = prev token. + * + * TODO consider a string_view, assuming the length will get stripped out by the optimizer when + * it is not used... + */ + simdjson_inline const uint8_t* peek(int32_t delta = 0) const noexcept; + /** + * Get the maximum length of the JSON text for a given token. + * + * The length will include any whitespace at the end of the token. + * + * @param delta The relative position of the token to retrieve. e.g. 0 = current token, + * 1 = next token, -1 = prev token. + */ + simdjson_inline uint32_t peek_length(int32_t delta = 0) const noexcept; + + /** + * Get the JSON text for a given token. + * + * This is not null-terminated; it is a view into the JSON. + * + * @param position The position of the token. + * + */ + simdjson_inline const uint8_t* peek(token_position position) const noexcept; + /** + * Get the maximum length of the JSON text for a given token. + * + * The length will include any whitespace at the end of the token. + * + * @param position The position of the token. + */ + simdjson_inline uint32_t peek_length(token_position position) const noexcept; + /** + * Get the maximum length of the JSON text for a root token. + * + * The length will include any whitespace at the end of the token. + * + * @param position The position of the token (start of the document). + */ + simdjson_inline uint32_t peek_root_length(token_position position) const noexcept; + /** + * Return the current index. + */ + simdjson_inline token_position position() const noexcept; + /** + * Reset to a previously saved index. + */ + simdjson_inline void set_position(token_position target_position) noexcept; + + // NOTE: we don't support a full C++ iterator interface, because we expect people to make + // different calls to advance the iterator based on *their own* state. + + simdjson_inline bool operator==(const token_iterator& other) const noexcept; + simdjson_inline bool operator!=(const token_iterator& other) const noexcept; + simdjson_inline bool operator>(const token_iterator& other) const noexcept; + simdjson_inline bool operator>=(const token_iterator& other) const noexcept; + simdjson_inline bool operator<(const token_iterator& other) const noexcept; + simdjson_inline bool operator<=(const token_iterator& other) const noexcept; + + protected: + simdjson_inline token_iterator(const uint8_t* buf, token_position position) noexcept; + + /** + * Get the index of the JSON text for a given token (relative). + * + * This is not null-terminated; it is a view into the JSON. + * + * @param delta The relative position of the token to retrieve. e.g. 0 = current token, + * 1 = next token, -1 = prev token. + */ + simdjson_inline uint32_t peek_index(int32_t delta = 0) const noexcept; + /** + * Get the index of the JSON text for a given token. + * + * This is not null-terminated; it is a view into the JSON. + * + * @param position The position of the token. + * + */ + simdjson_inline uint32_t peek_index(token_position position) const noexcept; + + const uint8_t* buf{}; + token_position _position{}; + + friend class json_iterator; + friend class value_iterator; + friend class object; + template + friend simdjson_inline void logger::log_line(const json_iterator& iter, const char* title_prefix, const char* title, std::string_view detail, int delta, int depth_delta, logger::log_level level, Args&&... args) noexcept; + template + friend simdjson_inline void logger::log_line(const json_iterator& iter, token_position index, depth_t depth, const char* title_prefix, const char* title, std::string_view detail, logger::log_level level, Args&&... args) noexcept; + }; + + } // namespace ondemand + } // namespace lsx +} // namespace simdjson + +namespace simdjson { + + template<> + struct simdjson_result : public lsx::implementation_simdjson_result_base { + public: + simdjson_inline simdjson_result(lsx::ondemand::token_iterator&& value) noexcept; ///< @private + simdjson_inline simdjson_result(error_code error) noexcept; ///< @private + simdjson_inline simdjson_result() noexcept = default; + simdjson_inline ~simdjson_result() noexcept = default; ///< @private + }; + +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_ONDEMAND_TOKEN_ITERATOR_H +/* end file simdjson/generic/ondemand/token_iterator.h for lsx */ +/* including simdjson/generic/ondemand/json_iterator.h for lsx: #include "simdjson/generic/ondemand/json_iterator.h" */ +/* begin file simdjson/generic/ondemand/json_iterator.h for lsx */ +#ifndef SIMDJSON_GENERIC_ONDEMAND_JSON_ITERATOR_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_ONDEMAND_JSON_ITERATOR_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/implementation_simdjson_result_base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/token_iterator.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lsx { + namespace ondemand { + + /** + * Iterates through JSON tokens, keeping track of depth and string buffer. + * + * @private This is not intended for external use. + */ + class json_iterator { + protected: + token_iterator token{}; + ondemand::parser* parser{}; + /** + * Next free location in the string buffer. + * + * Used by raw_json_string::unescape() to have a place to unescape strings to. + */ + uint8_t* _string_buf_loc{}; + /** + * JSON error, if there is one. + * + * INCORRECT_TYPE and NO_SUCH_FIELD are *not* stored here, ever. + * + * PERF NOTE: we *hope* this will be elided into control flow, as it is only used (a) in the first + * iteration of the loop, or (b) for the final iteration after a missing comma is found in ++. If + * this is not elided, we should make sure it's at least not using up a register. Failing that, + * we should store it in document so there's only one of them. + */ + error_code error{ SUCCESS }; + /** + * Depth of the current token in the JSON. + * + * - 0 = finished with document + * - 1 = document root value (could be [ or {, not yet known) + * - 2 = , or } inside root array/object + * - 3 = key or value inside root array/object. + */ + depth_t _depth{}; + /** + * Beginning of the document indexes. + * Normally we have root == parser->implementation->structural_indexes.get() + * but this may differ, especially in streaming mode (where we have several + * documents); + */ + token_position _root{}; + /** + * Normally, a json_iterator operates over a single document, but in + * some cases, we may have a stream of documents. This attribute is meant + * as meta-data: the json_iterator works the same irrespective of the + * value of this attribute. + */ + bool _streaming{ false }; + + public: + simdjson_inline json_iterator() noexcept = default; + simdjson_inline json_iterator(json_iterator&& other) noexcept; + simdjson_inline json_iterator& operator=(json_iterator&& other) noexcept; + simdjson_inline explicit json_iterator(const json_iterator& other) noexcept = default; + simdjson_inline json_iterator& operator=(const json_iterator& other) noexcept = default; + /** + * Skips a JSON value, whether it is a scalar, array or object. + */ + simdjson_warn_unused simdjson_inline error_code skip_child(depth_t parent_depth) noexcept; + + /** + * Tell whether the iterator is still at the start + */ + simdjson_inline bool at_root() const noexcept; + + /** + * Tell whether we should be expected to run in streaming + * mode (iterating over many documents). It is pure metadata + * that does not affect how the iterator works. It is used by + * start_root_array() and start_root_object(). + */ + simdjson_inline bool streaming() const noexcept; + + /** + * Get the root value iterator + */ + simdjson_inline token_position root_position() const noexcept; + /** + * Assert that we are at the document depth (== 1) + */ + simdjson_inline void assert_at_document_depth() const noexcept; + /** + * Assert that we are at the root of the document + */ + simdjson_inline void assert_at_root() const noexcept; + + /** + * Tell whether the iterator is at the EOF mark + */ + simdjson_inline bool at_end() const noexcept; + + /** + * Tell whether the iterator is live (has not been moved). + */ + simdjson_inline bool is_alive() const noexcept; + + /** + * Abandon this iterator, setting depth to 0 (as if the document is finished). + */ + simdjson_inline void abandon() noexcept; + + /** + * Advance the current token without modifying depth. + */ + simdjson_inline const uint8_t* return_current_and_advance() noexcept; + + /** + * Returns true if there is a single token in the index (i.e., it is + * a JSON with a scalar value such as a single number). + * + * @return whether there is a single token + */ + simdjson_inline bool is_single_token() const noexcept; + + /** + * Assert that there are at least the given number of tokens left. + * + * Has no effect in release builds. + */ + simdjson_inline void assert_more_tokens(uint32_t required_tokens = 1) const noexcept; + /** + * Assert that the given position addresses an actual token (is within bounds). + * + * Has no effect in release builds. + */ + simdjson_inline void assert_valid_position(token_position position) const noexcept; + /** + * Get the JSON text for a given token (relative). + * + * This is not null-terminated; it is a view into the JSON. + * + * @param delta The relative position of the token to retrieve. e.g. 0 = next token, -1 = prev token. + * + * TODO consider a string_view, assuming the length will get stripped out by the optimizer when + * it is not used ... + */ + simdjson_inline const uint8_t* peek(int32_t delta = 0) const noexcept; + /** + * Get the maximum length of the JSON text for the current token (or relative). + * + * The length will include any whitespace at the end of the token. + * + * @param delta The relative position of the token to retrieve. e.g. 0 = next token, -1 = prev token. + */ + simdjson_inline uint32_t peek_length(int32_t delta = 0) const noexcept; + /** + * Get a pointer to the current location in the input buffer. + * + * This is not null-terminated; it is a view into the JSON. + * + * You may be pointing outside of the input buffer: it is not generally + * safe to dereference this pointer. + */ + simdjson_inline const uint8_t* unsafe_pointer() const noexcept; + /** + * Get the JSON text for a given token. + * + * This is not null-terminated; it is a view into the JSON. + * + * @param position The position of the token to retrieve. + * + * TODO consider a string_view, assuming the length will get stripped out by the optimizer when + * it is not used ... + */ + simdjson_inline const uint8_t* peek(token_position position) const noexcept; + /** + * Get the maximum length of the JSON text for the current token (or relative). + * + * The length will include any whitespace at the end of the token. + * + * @param position The position of the token to retrieve. + */ + simdjson_inline uint32_t peek_length(token_position position) const noexcept; + /** + * Get the maximum length of the JSON text for the current root token. + * + * The length will include any whitespace at the end of the token. + * + * @param position The position of the token to retrieve. + */ + simdjson_inline uint32_t peek_root_length(token_position position) const noexcept; + /** + * Get the JSON text for the last token in the document. + * + * This is not null-terminated; it is a view into the JSON. + * + * TODO consider a string_view, assuming the length will get stripped out by the optimizer when + * it is not used ... + */ + simdjson_inline const uint8_t* peek_last() const noexcept; + + /** + * Ascend one level. + * + * Validates that the depth - 1 == parent_depth. + * + * @param parent_depth the expected parent depth. + */ + simdjson_inline void ascend_to(depth_t parent_depth) noexcept; + + /** + * Descend one level. + * + * Validates that the new depth == child_depth. + * + * @param child_depth the expected child depth. + */ + simdjson_inline void descend_to(depth_t child_depth) noexcept; + simdjson_inline void descend_to(depth_t child_depth, int32_t delta) noexcept; + + /** + * Get current depth. + */ + simdjson_inline depth_t depth() const noexcept; + + /** + * Get current (writeable) location in the string buffer. + */ + simdjson_inline uint8_t*& string_buf_loc() noexcept; + + /** + * Report an unrecoverable error, preventing further iteration. + * + * @param error The error to report. Must not be SUCCESS, UNINITIALIZED, INCORRECT_TYPE, or NO_SUCH_FIELD. + * @param message An error message to report with the error. + */ + simdjson_inline error_code report_error(error_code error, const char* message) noexcept; + + /** + * Log error, but don't stop iteration. + * @param error The error to report. Must be INCORRECT_TYPE, or NO_SUCH_FIELD. + * @param message An error message to report with the error. + */ + simdjson_inline error_code optional_error(error_code error, const char* message) noexcept; + + /** + * Take an input in json containing max_len characters and attempt to copy it over to tmpbuf, a buffer with + * N bytes of capacity. It will return false if N is too small (smaller than max_len) of if it is zero. + * The buffer (tmpbuf) is padded with space characters. + */ + simdjson_warn_unused simdjson_inline bool copy_to_buffer(const uint8_t* json, uint32_t max_len, uint8_t* tmpbuf, size_t N) noexcept; + + simdjson_inline token_position position() const noexcept; + /** + * Write the raw_json_string to the string buffer and return a string_view. + * Each raw_json_string should be unescaped once, or else the string buffer might + * overflow. + */ + simdjson_inline simdjson_result unescape(raw_json_string in, bool allow_replacement) noexcept; + simdjson_inline simdjson_result unescape_wobbly(raw_json_string in) noexcept; + + simdjson_inline void reenter_child(token_position position, depth_t child_depth) noexcept; + + simdjson_inline error_code consume_character(char c) noexcept; +#if SIMDJSON_DEVELOPMENT_CHECKS + simdjson_inline token_position start_position(depth_t depth) const noexcept; + simdjson_inline void set_start_position(depth_t depth, token_position position) noexcept; +#endif + + /* Useful for debugging and logging purposes. */ + inline std::string to_string() const noexcept; + + /** + * Returns the current location in the document if in bounds. + */ + inline simdjson_result current_location() const noexcept; + + /** + * Updates this json iterator so that it is back at the beginning of the document, + * as if it had just been created. + */ + inline void rewind() noexcept; + /** + * This checks whether the {,},[,] are balanced so that the document + * ends with proper zero depth. This requires scanning the whole document + * and it may be expensive. It is expected that it will be rarely called. + * It does not attempt to match { with } and [ with ]. + */ + inline bool balanced() const noexcept; + protected: + simdjson_inline json_iterator(const uint8_t* buf, ondemand::parser* parser) noexcept; +#ifdef SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON + simdjson_inline json_iterator(const uint8_t* buf, ondemand::parser* parser, bool streaming) noexcept; +#endif // SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON + /// The last token before the end + simdjson_inline token_position last_position() const noexcept; + /// The token *at* the end. This points at gibberish and should only be used for comparison. + simdjson_inline token_position end_position() const noexcept; + /// The end of the buffer. + simdjson_inline token_position end() const noexcept; + + friend class document; + friend class document_stream; + friend class object; + friend class array; + friend class value; + friend class raw_json_string; + friend class parser; + friend class value_iterator; + friend class field; + template + friend simdjson_inline void logger::log_line(const json_iterator& iter, const char* title_prefix, const char* title, std::string_view detail, int delta, int depth_delta, logger::log_level level, Args&&... args) noexcept; + template + friend simdjson_inline void logger::log_line(const json_iterator& iter, token_position index, depth_t depth, const char* title_prefix, const char* title, std::string_view detail, logger::log_level level, Args&&... args) noexcept; + }; // json_iterator + + } // namespace ondemand + } // namespace lsx +} // namespace simdjson + +namespace simdjson { + + template<> + struct simdjson_result : public lsx::implementation_simdjson_result_base { + public: + simdjson_inline simdjson_result(lsx::ondemand::json_iterator&& value) noexcept; ///< @private + simdjson_inline simdjson_result(error_code error) noexcept; ///< @private + + simdjson_inline simdjson_result() noexcept = default; + }; + +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_ONDEMAND_JSON_ITERATOR_H +/* end file simdjson/generic/ondemand/json_iterator.h for lsx */ +/* including simdjson/generic/ondemand/json_type.h for lsx: #include "simdjson/generic/ondemand/json_type.h" */ +/* begin file simdjson/generic/ondemand/json_type.h for lsx */ +#ifndef SIMDJSON_GENERIC_ONDEMAND_JSON_TYPE_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_ONDEMAND_JSON_TYPE_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/implementation_simdjson_result_base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/numberparsing.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lsx { + namespace ondemand { + + /** + * The type of a JSON value. + */ + enum class json_type { + // Start at 1 to catch uninitialized / default values more easily + array = 1, ///< A JSON array ( [ 1, 2, 3 ... ] ) + object, ///< A JSON object ( { "a": 1, "b" 2, ... } ) + number, ///< A JSON number ( 1 or -2.3 or 4.5e6 ...) + string, ///< A JSON string ( "a" or "hello world\n" ...) + boolean, ///< A JSON boolean (true or false) + null ///< A JSON null (null) + }; + + /** + * A type representing a JSON number. + * The design of the struct is deliberately straight-forward. All + * functions return standard values with no error check. + */ + struct number { + + /** + * return the automatically determined type of + * the number: number_type::floating_point_number, + * number_type::signed_integer or number_type::unsigned_integer. + * + * enum class number_type { + * floating_point_number=1, /// a binary64 number + * signed_integer, /// a signed integer that fits in a 64-bit word using two's complement + * unsigned_integer /// a positive integer larger or equal to 1<<63 + * }; + */ + simdjson_inline ondemand::number_type get_number_type() const noexcept; + /** + * return true if the automatically determined type of + * the number is number_type::unsigned_integer. + */ + simdjson_inline bool is_uint64() const noexcept; + /** + * return the value as a uint64_t, only valid if is_uint64() is true. + */ + simdjson_inline uint64_t get_uint64() const noexcept; + simdjson_inline operator uint64_t() const noexcept; + + /** + * return true if the automatically determined type of + * the number is number_type::signed_integer. + */ + simdjson_inline bool is_int64() const noexcept; + /** + * return the value as a int64_t, only valid if is_int64() is true. + */ + simdjson_inline int64_t get_int64() const noexcept; + simdjson_inline operator int64_t() const noexcept; + + + /** + * return true if the automatically determined type of + * the number is number_type::floating_point_number. + */ + simdjson_inline bool is_double() const noexcept; + /** + * return the value as a double, only valid if is_double() is true. + */ + simdjson_inline double get_double() const noexcept; + simdjson_inline operator double() const noexcept; + + /** + * Convert the number to a double. Though it always succeed, the conversion + * may be lossy if the number cannot be represented exactly. + */ + simdjson_inline double as_double() const noexcept; + + + protected: + /** + * The next block of declaration is designed so that we can call the number parsing + * functions on a number type. They are protected and should never be used outside + * of the core simdjson library. + */ + friend class value_iterator; + template + friend error_code numberparsing::write_float(const uint8_t* const src, bool negative, uint64_t i, const uint8_t* start_digits, size_t digit_count, int64_t exponent, W& writer); + template + friend error_code numberparsing::parse_number(const uint8_t* const src, W& writer); + /** Store a signed 64-bit value to the number. */ + simdjson_inline void append_s64(int64_t value) noexcept; + /** Store an unsigned 64-bit value to the number. */ + simdjson_inline void append_u64(uint64_t value) noexcept; + /** Store a double value to the number. */ + simdjson_inline void append_double(double value) noexcept; + /** Specifies that the value is a double, but leave it undefined. */ + simdjson_inline void skip_double() noexcept; + /** + * End of friend declarations. + */ + + /** + * Our attributes are a union type (size = 64 bits) + * followed by a type indicator. + */ + union { + double floating_point_number; + int64_t signed_integer; + uint64_t unsigned_integer; + } payload{ 0 }; + number_type type{ number_type::signed_integer }; + }; + + /** + * Write the JSON type to the output stream + * + * @param out The output stream. + * @param type The json_type. + */ + inline std::ostream& operator<<(std::ostream& out, json_type type) noexcept; + +#if SIMDJSON_EXCEPTIONS + /** + * Send JSON type to an output stream. + * + * @param out The output stream. + * @param type The json_type. + * @throw simdjson_error if the result being printed has an error. If there is an error with the + * underlying output stream, that error will be propagated (simdjson_error will not be + * thrown). + */ + inline std::ostream& operator<<(std::ostream& out, simdjson_result& type) noexcept(false); +#endif + + } // namespace ondemand + } // namespace lsx +} // namespace simdjson + +namespace simdjson { + + template<> + struct simdjson_result : public lsx::implementation_simdjson_result_base { + public: + simdjson_inline simdjson_result(lsx::ondemand::json_type&& value) noexcept; ///< @private + simdjson_inline simdjson_result(error_code error) noexcept; ///< @private + simdjson_inline simdjson_result() noexcept = default; + simdjson_inline ~simdjson_result() noexcept = default; ///< @private + }; + +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_ONDEMAND_JSON_TYPE_H +/* end file simdjson/generic/ondemand/json_type.h for lsx */ +/* including simdjson/generic/ondemand/raw_json_string.h for lsx: #include "simdjson/generic/ondemand/raw_json_string.h" */ +/* begin file simdjson/generic/ondemand/raw_json_string.h for lsx */ +#ifndef SIMDJSON_GENERIC_ONDEMAND_RAW_JSON_STRING_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_ONDEMAND_RAW_JSON_STRING_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/implementation_simdjson_result_base.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lsx { + namespace ondemand { + + /** + * A string escaped per JSON rules, terminated with quote ("). They are used to represent + * unescaped keys inside JSON documents. + * + * (In other words, a pointer to the beginning of a string, just after the start quote, inside a + * JSON file.) + * + * This class is deliberately simplistic and has little functionality. You can + * compare a raw_json_string instance with an unescaped C string, but + * that is nearly all you can do. + * + * The raw_json_string is unescaped. If you wish to write an unescaped version of it to your own + * buffer, you may do so using the parser.unescape(string, buff) method, using an ondemand::parser + * instance. Doing so requires you to have a sufficiently large buffer. + * + * The raw_json_string instances originate typically from field instance which in turn represent + * key-value pairs from object instances. From a field instance, you get the raw_json_string + * instance by calling key(). You can, if you want a more usable string_view instance, call + * the unescaped_key() method on the field instance. You may also create a raw_json_string from + * any other string value, with the value.get_raw_json_string() method. Again, you can get + * a more usable string_view instance by calling get_string(). + * + */ + class raw_json_string { + public: + /** + * Create a new invalid raw_json_string. + * + * Exists so you can declare a variable and later assign to it before use. + */ + simdjson_inline raw_json_string() noexcept = default; + + /** + * Create a new invalid raw_json_string pointed at the given location in the JSON. + * + * The given location must be just *after* the beginning quote (") in the JSON file. + * + * It *must* be terminated by a ", and be a valid JSON string. + */ + simdjson_inline raw_json_string(const uint8_t* _buf) noexcept; + /** + * Get the raw pointer to the beginning of the string in the JSON (just after the "). + * + * It is possible for this function to return a null pointer if the instance + * has outlived its existence. + */ + simdjson_inline const char* raw() const noexcept; + + /** + * This compares the current instance to the std::string_view target: returns true if + * they are byte-by-byte equal (no escaping is done) on target.size() characters, + * and if the raw_json_string instance has a quote character at byte index target.size(). + * We never read more than length + 1 bytes in the raw_json_string instance. + * If length is smaller than target.size(), this will return false. + * + * The std::string_view instance may contain any characters. However, the caller + * is responsible for setting length so that length bytes may be read in the + * raw_json_string. + * + * Performance: the comparison may be done using memcmp which may be efficient + * for long strings. + */ + simdjson_inline bool unsafe_is_equal(size_t length, std::string_view target) const noexcept; + + /** + * This compares the current instance to the std::string_view target: returns true if + * they are byte-by-byte equal (no escaping is done). + * The std::string_view instance should not contain unescaped quote characters: + * the caller is responsible for this check. See is_free_from_unescaped_quote. + * + * Performance: the comparison is done byte-by-byte which might be inefficient for + * long strings. + * + * If target is a compile-time constant, and your compiler likes you, + * you should be able to do the following without performance penalty... + * + * static_assert(raw_json_string::is_free_from_unescaped_quote(target), ""); + * s.unsafe_is_equal(target); + */ + simdjson_inline bool unsafe_is_equal(std::string_view target) const noexcept; + + /** + * This compares the current instance to the C string target: returns true if + * they are byte-by-byte equal (no escaping is done). + * The provided C string should not contain an unescaped quote character: + * the caller is responsible for this check. See is_free_from_unescaped_quote. + * + * If target is a compile-time constant, and your compiler likes you, + * you should be able to do the following without performance penalty... + * + * static_assert(raw_json_string::is_free_from_unescaped_quote(target), ""); + * s.unsafe_is_equal(target); + */ + simdjson_inline bool unsafe_is_equal(const char* target) const noexcept; + + /** + * This compares the current instance to the std::string_view target: returns true if + * they are byte-by-byte equal (no escaping is done). + */ + simdjson_inline bool is_equal(std::string_view target) const noexcept; + + /** + * This compares the current instance to the C string target: returns true if + * they are byte-by-byte equal (no escaping is done). + */ + simdjson_inline bool is_equal(const char* target) const noexcept; + + /** + * Returns true if target is free from unescaped quote. If target is known at + * compile-time, we might expect the computation to happen at compile time with + * many compilers (not all!). + */ + static simdjson_inline bool is_free_from_unescaped_quote(std::string_view target) noexcept; + static simdjson_inline bool is_free_from_unescaped_quote(const char* target) noexcept; + + private: + + + /** + * This will set the inner pointer to zero, effectively making + * this instance unusable. + */ + simdjson_inline void consume() noexcept { buf = nullptr; } + + /** + * Checks whether the inner pointer is non-null and thus usable. + */ + simdjson_inline simdjson_warn_unused bool alive() const noexcept { return buf != nullptr; } + + /** + * Unescape this JSON string, replacing \\ with \, \n with newline, etc. + * The result will be a valid UTF-8. + * + * ## IMPORTANT: string_view lifetime + * + * The string_view is only valid until the next parse() call on the parser. + * + * @param iter A json_iterator, which contains a buffer where the string will be written. + * @param allow_replacement Whether we allow replacement of invalid surrogate pairs. + */ + simdjson_inline simdjson_warn_unused simdjson_result unescape(json_iterator& iter, bool allow_replacement) const noexcept; + + /** + * Unescape this JSON string, replacing \\ with \, \n with newline, etc. + * The result may not be a valid UTF-8. https://simonsapin.github.io/wtf-8/ + * + * ## IMPORTANT: string_view lifetime + * + * The string_view is only valid until the next parse() call on the parser. + * + * @param iter A json_iterator, which contains a buffer where the string will be written. + */ + simdjson_inline simdjson_warn_unused simdjson_result unescape_wobbly(json_iterator& iter) const noexcept; + const uint8_t* buf{}; + friend class object; + friend class field; + friend class parser; + friend struct simdjson_result; + }; + + simdjson_unused simdjson_inline std::ostream& operator<<(std::ostream&, const raw_json_string&) noexcept; + + /** + * Comparisons between raw_json_string and std::string_view instances are potentially unsafe: the user is responsible + * for providing a string with no unescaped quote. Note that unescaped quotes cannot be present in valid JSON strings. + */ + simdjson_unused simdjson_inline bool operator==(const raw_json_string& a, std::string_view c) noexcept; + simdjson_unused simdjson_inline bool operator==(std::string_view c, const raw_json_string& a) noexcept; + simdjson_unused simdjson_inline bool operator!=(const raw_json_string& a, std::string_view c) noexcept; + simdjson_unused simdjson_inline bool operator!=(std::string_view c, const raw_json_string& a) noexcept; + + + } // namespace ondemand + } // namespace lsx +} // namespace simdjson + +namespace simdjson { + + template<> + struct simdjson_result : public lsx::implementation_simdjson_result_base { + public: + simdjson_inline simdjson_result(lsx::ondemand::raw_json_string&& value) noexcept; ///< @private + simdjson_inline simdjson_result(error_code error) noexcept; ///< @private + simdjson_inline simdjson_result() noexcept = default; + simdjson_inline ~simdjson_result() noexcept = default; ///< @private + + simdjson_inline simdjson_result raw() const noexcept; + simdjson_inline simdjson_warn_unused simdjson_result unescape(lsx::ondemand::json_iterator& iter, bool allow_replacement) const noexcept; + simdjson_inline simdjson_warn_unused simdjson_result unescape_wobbly(lsx::ondemand::json_iterator& iter) const noexcept; + }; + +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_ONDEMAND_RAW_JSON_STRING_H +/* end file simdjson/generic/ondemand/raw_json_string.h for lsx */ +/* including simdjson/generic/ondemand/parser.h for lsx: #include "simdjson/generic/ondemand/parser.h" */ +/* begin file simdjson/generic/ondemand/parser.h for lsx */ +#ifndef SIMDJSON_GENERIC_ONDEMAND_PARSER_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_ONDEMAND_PARSER_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/implementation_simdjson_result_base.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +#include + +namespace simdjson { + namespace lsx { + namespace ondemand { + + /** + * The default batch size for document_stream instances for this On-Demand kernel. + * Note that different On-Demand kernel may use a different DEFAULT_BATCH_SIZE value + * in the future. + */ + static constexpr size_t DEFAULT_BATCH_SIZE = 1000000; + /** + * Some adversary might try to set the batch size to 0 or 1, which might cause problems. + * We set a minimum of 32B since anything else is highly likely to be an error. In practice, + * most users will want a much larger batch size. + * + * All non-negative MINIMAL_BATCH_SIZE values should be 'safe' except that, obviously, no JSON + * document can ever span 0 or 1 byte and that very large values would create memory allocation issues. + */ + static constexpr size_t MINIMAL_BATCH_SIZE = 32; + + /** + * A JSON fragment iterator. + * + * This holds the actual iterator as well as the buffer for writing strings. + */ + class parser { + public: + /** + * Create a JSON parser. + * + * The new parser will have zero capacity. + */ + inline explicit parser(size_t max_capacity = SIMDJSON_MAXSIZE_BYTES) noexcept; + + inline parser(parser&& other) noexcept = default; + simdjson_inline parser(const parser& other) = delete; + simdjson_inline parser& operator=(const parser& other) = delete; + simdjson_inline parser& operator=(parser&& other) noexcept = default; + + /** Deallocate the JSON parser. */ + inline ~parser() noexcept = default; + + /** + * Start iterating an on-demand JSON document. + * + * ondemand::parser parser; + * document doc = parser.iterate(json); + * + * It is expected that the content is a valid UTF-8 file, containing a valid JSON document. + * Otherwise the iterate method may return an error. In particular, the whole input should be + * valid: we do not attempt to tolerate incorrect content either before or after a JSON + * document. If there is a UTF-8 BOM, the parser skips it. + * + * ### IMPORTANT: Validate what you use + * + * Calling iterate on an invalid JSON document may not immediately trigger an error. The call to + * iterate does not parse and validate the whole document. + * + * ### IMPORTANT: Buffer Lifetime + * + * Because parsing is done while you iterate, you *must* keep the JSON buffer around at least as + * long as the document iteration. + * + * ### IMPORTANT: Document Lifetime + * + * Only one iteration at a time can happen per parser, and the parser *must* be kept alive during + * iteration to ensure intermediate buffers can be accessed. Any document must be destroyed before + * you call parse() again or destroy the parser. + * + * ### REQUIRED: Buffer Padding + * + * The buffer must have at least SIMDJSON_PADDING extra allocated bytes. It does not matter what + * those bytes are initialized to, as long as they are allocated. These bytes will be read: if you + * using a sanitizer that verifies that no uninitialized byte is read, then you should initialize the + * SIMDJSON_PADDING bytes to avoid runtime warnings. + * + * @param json The JSON to parse. + * @param len The length of the JSON. + * @param capacity The number of bytes allocated in the JSON (must be at least len+SIMDJSON_PADDING). + * + * @return The document, or an error: + * - INSUFFICIENT_PADDING if the input has less than SIMDJSON_PADDING extra bytes. + * - MEMALLOC if realloc_if_needed the parser does not have enough capacity, and memory + * allocation fails. + * - EMPTY if the document is all whitespace. + * - UTF8_ERROR if the document is not valid UTF-8. + * - UNESCAPED_CHARS if a string contains control characters that must be escaped + * - UNCLOSED_STRING if there is an unclosed string in the document. + */ + simdjson_warn_unused simdjson_result iterate(padded_string_view json) & noexcept; +#ifdef SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON + simdjson_warn_unused simdjson_result iterate_allow_incomplete_json(padded_string_view json) & noexcept; +#endif // SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON + /** @overload simdjson_result iterate(padded_string_view json) & noexcept */ + simdjson_warn_unused simdjson_result iterate(const char* json, size_t len, size_t capacity) & noexcept; + /** @overload simdjson_result iterate(padded_string_view json) & noexcept */ + simdjson_warn_unused simdjson_result iterate(const uint8_t* json, size_t len, size_t capacity) & noexcept; + /** @overload simdjson_result iterate(padded_string_view json) & noexcept */ + simdjson_warn_unused simdjson_result iterate(std::string_view json, size_t capacity) & noexcept; + /** @overload simdjson_result iterate(padded_string_view json) & noexcept */ + simdjson_warn_unused simdjson_result iterate(const std::string& json) & noexcept; + /** @overload simdjson_result iterate(padded_string_view json) & noexcept */ + simdjson_warn_unused simdjson_result iterate(std::string& json) & noexcept; + /** @overload simdjson_result iterate(padded_string_view json) & noexcept */ + simdjson_warn_unused simdjson_result iterate(const simdjson_result& json) & noexcept; + /** @overload simdjson_result iterate(padded_string_view json) & noexcept */ + simdjson_warn_unused simdjson_result iterate(const simdjson_result& json) & noexcept; + /** @overload simdjson_result iterate(padded_string_view json) & noexcept */ + simdjson_warn_unused simdjson_result iterate(padded_string&& json) & noexcept = delete; + + /** + * @private + * + * Start iterating an on-demand JSON document. + * + * ondemand::parser parser; + * json_iterator doc = parser.iterate(json); + * + * ### IMPORTANT: Buffer Lifetime + * + * Because parsing is done while you iterate, you *must* keep the JSON buffer around at least as + * long as the document iteration. + * + * ### IMPORTANT: Document Lifetime + * + * Only one iteration at a time can happen per parser, and the parser *must* be kept alive during + * iteration to ensure intermediate buffers can be accessed. Any document must be destroyed before + * you call parse() again or destroy the parser. + * + * The ondemand::document instance holds the iterator. The document must remain in scope + * while you are accessing instances of ondemand::value, ondemand::object, ondemand::array. + * + * ### REQUIRED: Buffer Padding + * + * The buffer must have at least SIMDJSON_PADDING extra allocated bytes. It does not matter what + * those bytes are initialized to, as long as they are allocated. These bytes will be read: if you + * using a sanitizer that verifies that no uninitialized byte is read, then you should initialize the + * SIMDJSON_PADDING bytes to avoid runtime warnings. + * + * @param json The JSON to parse. + * + * @return The iterator, or an error: + * - INSUFFICIENT_PADDING if the input has less than SIMDJSON_PADDING extra bytes. + * - MEMALLOC if realloc_if_needed the parser does not have enough capacity, and memory + * allocation fails. + * - EMPTY if the document is all whitespace. + * - UTF8_ERROR if the document is not valid UTF-8. + * - UNESCAPED_CHARS if a string contains control characters that must be escaped + * - UNCLOSED_STRING if there is an unclosed string in the document. + */ + simdjson_warn_unused simdjson_result iterate_raw(padded_string_view json) & noexcept; + + + /** + * Parse a buffer containing many JSON documents. + * + * auto json = R"({ "foo": 1 } { "foo": 2 } { "foo": 3 } )"_padded; + * ondemand::parser parser; + * ondemand::document_stream docs = parser.iterate_many(json); + * for (auto & doc : docs) { + * std::cout << doc["foo"] << std::endl; + * } + * // Prints 1 2 3 + * + * No copy of the input buffer is made. + * + * The function is lazy: it may be that no more than one JSON document at a time is parsed. + * + * The caller is responsabile to ensure that the input string data remains unchanged and is + * not deleted during the loop. + * + * ### Format + * + * The buffer must contain a series of one or more JSON documents, concatenated into a single + * buffer, separated by ASCII whitespace. It effectively parses until it has a fully valid document, + * then starts parsing the next document at that point. (It does this with more parallelism and + * lookahead than you might think, though.) + * + * documents that consist of an object or array may omit the whitespace between them, concatenating + * with no separator. Documents that consist of a single primitive (i.e. documents that are not + * arrays or objects) MUST be separated with ASCII whitespace. + * + * The characters inside a JSON document, and between JSON documents, must be valid Unicode (UTF-8). + * If there is a UTF-8 BOM, the parser skips it. + * + * The documents must not exceed batch_size bytes (by default 1MB) or they will fail to parse. + * Setting batch_size to excessively large or excessively small values may impact negatively the + * performance. + * + * ### REQUIRED: Buffer Padding + * + * The buffer must have at least SIMDJSON_PADDING extra allocated bytes. It does not matter what + * those bytes are initialized to, as long as they are allocated. These bytes will be read: if you + * using a sanitizer that verifies that no uninitialized byte is read, then you should initialize the + * SIMDJSON_PADDING bytes to avoid runtime warnings. + * + * ### Threads + * + * When compiled with SIMDJSON_THREADS_ENABLED, this method will use a single thread under the + * hood to do some lookahead. + * + * ### Parser Capacity + * + * If the parser's current capacity is less than batch_size, it will allocate enough capacity + * to handle it (up to max_capacity). + * + * @param buf The concatenated JSON to parse. + * @param len The length of the concatenated JSON. + * @param batch_size The batch size to use. MUST be larger than the largest document. The sweet + * spot is cache-related: small enough to fit in cache, yet big enough to + * parse as many documents as possible in one tight loop. + * Defaults to 10MB, which has been a reasonable sweet spot in our tests. + * @param allow_comma_separated (defaults on false) This allows a mode where the documents are + * separated by commas instead of whitespace. It comes with a performance + * penalty because the entire document is indexed at once (and the document must be + * less than 4 GB), and there is no multithreading. In this mode, the batch_size parameter + * is effectively ignored, as it is set to at least the document size. + * @return The stream, or an error. An empty input will yield 0 documents rather than an EMPTY error. Errors: + * - MEMALLOC if the parser does not have enough capacity and memory allocation fails + * - CAPACITY if the parser does not have enough capacity and batch_size > max_capacity. + * - other json errors if parsing fails. You should not rely on these errors to always the same for the + * same document: they may vary under runtime dispatch (so they may vary depending on your system and hardware). + */ + inline simdjson_result iterate_many(const uint8_t* buf, size_t len, size_t batch_size = DEFAULT_BATCH_SIZE, bool allow_comma_separated = false) noexcept; + /** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size) */ + inline simdjson_result iterate_many(const char* buf, size_t len, size_t batch_size = DEFAULT_BATCH_SIZE, bool allow_comma_separated = false) noexcept; + /** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size) */ + inline simdjson_result iterate_many(const std::string& s, size_t batch_size = DEFAULT_BATCH_SIZE, bool allow_comma_separated = false) noexcept; + inline simdjson_result iterate_many(const std::string&& s, size_t batch_size, bool allow_comma_separated = false) = delete;// unsafe + /** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size) */ + inline simdjson_result iterate_many(const padded_string& s, size_t batch_size = DEFAULT_BATCH_SIZE, bool allow_comma_separated = false) noexcept; + inline simdjson_result iterate_many(const padded_string&& s, size_t batch_size, bool allow_comma_separated = false) = delete;// unsafe + + /** @private We do not want to allow implicit conversion from C string to std::string. */ + simdjson_result iterate_many(const char* buf, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept = delete; + + /** The capacity of this parser (the largest document it can process). */ + simdjson_pure simdjson_inline size_t capacity() const noexcept; + /** The maximum capacity of this parser (the largest document it is allowed to process). */ + simdjson_pure simdjson_inline size_t max_capacity() const noexcept; + simdjson_inline void set_max_capacity(size_t max_capacity) noexcept; + /** + * The maximum depth of this parser (the most deeply nested objects and arrays it can process). + * This parameter is only relevant when the macro SIMDJSON_DEVELOPMENT_CHECKS is set to true. + * The document's instance current_depth() method should be used to monitor the parsing + * depth and limit it if desired. + */ + simdjson_pure simdjson_inline size_t max_depth() const noexcept; + + /** + * Ensure this parser has enough memory to process JSON documents up to `capacity` bytes in length + * and `max_depth` depth. + * + * The max_depth parameter is only relevant when the macro SIMDJSON_DEVELOPMENT_CHECKS is set to true. + * The document's instance current_depth() method should be used to monitor the parsing + * depth and limit it if desired. + * + * @param capacity The new capacity. + * @param max_depth The new max_depth. Defaults to DEFAULT_MAX_DEPTH. + * @return The error, if there is one. + */ + simdjson_warn_unused error_code allocate(size_t capacity, size_t max_depth = DEFAULT_MAX_DEPTH) noexcept; + +#ifdef SIMDJSON_THREADS_ENABLED + /** + * The parser instance can use threads when they are available to speed up some + * operations. It is enabled by default. Changing this attribute will change the + * behavior of the parser for future operations. + */ + bool threaded{ true }; +#endif + + /** + * Unescape this JSON string, replacing \\ with \, \n with newline, etc. to a user-provided buffer. + * The result must be valid UTF-8. + * The provided pointer is advanced to the end of the string by reference, and a string_view instance + * is returned. You can ensure that your buffer is large enough by allocating a block of memory at least + * as large as the input JSON plus SIMDJSON_PADDING and then unescape all strings to this one buffer. + * + * This unescape function is a low-level function. If you want a more user-friendly approach, you should + * avoid raw_json_string instances (e.g., by calling unescaped_key() instead of key() or get_string() + * instead of get_raw_json_string()). + * + * ## IMPORTANT: string_view lifetime + * + * The string_view is only valid as long as the bytes in dst. + * + * @param raw_json_string input + * @param dst A pointer to a buffer at least large enough to write this string as well as + * an additional SIMDJSON_PADDING bytes. + * @param allow_replacement Whether we allow a replacement if the input string contains unmatched surrogate pairs. + * @return A string_view pointing at the unescaped string in dst + * @error STRING_ERROR if escapes are incorrect. + */ + simdjson_inline simdjson_result unescape(raw_json_string in, uint8_t*& dst, bool allow_replacement = false) const noexcept; + + /** + * Unescape this JSON string, replacing \\ with \, \n with newline, etc. to a user-provided buffer. + * The result may not be valid UTF-8. See https://simonsapin.github.io/wtf-8/ + * The provided pointer is advanced to the end of the string by reference, and a string_view instance + * is returned. You can ensure that your buffer is large enough by allocating a block of memory at least + * as large as the input JSON plus SIMDJSON_PADDING and then unescape all strings to this one buffer. + * + * This unescape function is a low-level function. If you want a more user-friendly approach, you should + * avoid raw_json_string instances (e.g., by calling unescaped_key() instead of key() or get_string() + * instead of get_raw_json_string()). + * + * ## IMPORTANT: string_view lifetime + * + * The string_view is only valid as long as the bytes in dst. + * + * @param raw_json_string input + * @param dst A pointer to a buffer at least large enough to write this string as well as + * an additional SIMDJSON_PADDING bytes. + * @return A string_view pointing at the unescaped string in dst + * @error STRING_ERROR if escapes are incorrect. + */ + simdjson_inline simdjson_result unescape_wobbly(raw_json_string in, uint8_t*& dst) const noexcept; + +#if SIMDJSON_DEVELOPMENT_CHECKS + /** + * Returns true if string_buf_loc is outside of the allocated range for the + * the string buffer. When true, it indicates that the string buffer has overflowed. + * This is a development-time check that is not needed in production. It can be + * used to detect buffer overflows in the string buffer and usafe usage of the + * string buffer. + */ + bool string_buffer_overflow(const uint8_t* string_buf_loc) const noexcept; +#endif + + private: + /** @private [for benchmarking access] The implementation to use */ + std::unique_ptr implementation{}; + size_t _capacity{ 0 }; + size_t _max_capacity; + size_t _max_depth{ DEFAULT_MAX_DEPTH }; + std::unique_ptr string_buf{}; +#if SIMDJSON_DEVELOPMENT_CHECKS + std::unique_ptr start_positions{}; +#endif + + friend class json_iterator; + friend class document_stream; + }; + + } // namespace ondemand + } // namespace lsx +} // namespace simdjson + +namespace simdjson { + + template<> + struct simdjson_result : public lsx::implementation_simdjson_result_base { + public: + simdjson_inline simdjson_result(lsx::ondemand::parser&& value) noexcept; ///< @private + simdjson_inline simdjson_result(error_code error) noexcept; ///< @private + simdjson_inline simdjson_result() noexcept = default; + }; + +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_ONDEMAND_PARSER_H +/* end file simdjson/generic/ondemand/parser.h for lsx */ + +// All other declarations +/* including simdjson/generic/ondemand/array.h for lsx: #include "simdjson/generic/ondemand/array.h" */ +/* begin file simdjson/generic/ondemand/array.h for lsx */ +#ifndef SIMDJSON_GENERIC_ONDEMAND_ARRAY_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_ONDEMAND_ARRAY_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/implementation_simdjson_result_base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/value_iterator.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lsx { + namespace ondemand { + + /** + * A forward-only JSON array. + */ + class array { + public: + /** + * Create a new invalid array. + * + * Exists so you can declare a variable and later assign to it before use. + */ + simdjson_inline array() noexcept = default; + + /** + * Begin array iteration. + * + * Part of the std::iterable interface. + */ + simdjson_inline simdjson_result begin() noexcept; + /** + * Sentinel representing the end of the array. + * + * Part of the std::iterable interface. + */ + simdjson_inline simdjson_result end() noexcept; + /** + * This method scans the array and counts the number of elements. + * The count_elements method should always be called before you have begun + * iterating through the array: it is expected that you are pointing at + * the beginning of the array. + * The runtime complexity is linear in the size of the array. After + * calling this function, if successful, the array is 'rewinded' at its + * beginning as if it had never been accessed. If the JSON is malformed (e.g., + * there is a missing comma), then an error is returned and it is no longer + * safe to continue. + * + * To check that an array is empty, it is more performant to use + * the is_empty() method. + */ + simdjson_inline simdjson_result count_elements() & noexcept; + /** + * This method scans the beginning of the array and checks whether the + * array is empty. + * The runtime complexity is constant time. After + * calling this function, if successful, the array is 'rewinded' at its + * beginning as if it had never been accessed. If the JSON is malformed (e.g., + * there is a missing comma), then an error is returned and it is no longer + * safe to continue. + */ + simdjson_inline simdjson_result is_empty() & noexcept; + /** + * Reset the iterator so that we are pointing back at the + * beginning of the array. You should still consume values only once even if you + * can iterate through the array more than once. If you unescape a string + * within the array more than once, you have unsafe code. Note that rewinding + * an array means that you may need to reparse it anew: it is not a free + * operation. + * + * @returns true if the array contains some elements (not empty) + */ + inline simdjson_result reset() & noexcept; + /** + * Get the value associated with the given JSON pointer. We use the RFC 6901 + * https://tools.ietf.org/html/rfc6901 standard, interpreting the current node + * as the root of its own JSON document. + * + * ondemand::parser parser; + * auto json = R"([ { "foo": { "a": [ 10, 20, 30 ] }} ])"_padded; + * auto doc = parser.iterate(json); + * doc.at_pointer("/0/foo/a/1") == 20 + * + * Note that at_pointer() called on the document automatically calls the document's rewind + * method between each call. It invalidates all previously accessed arrays, objects and values + * that have not been consumed. Yet it is not the case when calling at_pointer on an array + * instance: there is no rewind and no invalidation. + * + * You may only call at_pointer on an array after it has been created, but before it has + * been first accessed. When calling at_pointer on an array, the pointer is advanced to + * the location indicated by the JSON pointer (in case of success). It is no longer possible + * to call at_pointer on the same array. + * + * Also note that at_pointer() relies on find_field() which implies that we do not unescape keys when matching. + * + * @return The value associated with the given JSON pointer, or: + * - NO_SUCH_FIELD if a field does not exist in an object + * - INDEX_OUT_OF_BOUNDS if an array index is larger than an array length + * - INCORRECT_TYPE if a non-integer is used to access an array + * - INVALID_JSON_POINTER if the JSON pointer is invalid and cannot be parsed + */ + inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; + + /** + * Get the value associated with the given JSONPath expression. We only support + * JSONPath queries that trivially convertible to JSON Pointer queries: key + * names and array indices. + * + * https://datatracker.ietf.org/doc/html/draft-normington-jsonpath-00 + * + * @return The value associated with the given JSONPath expression, or: + * - INVALID_JSON_POINTER if the JSONPath to JSON Pointer conversion fails + * - NO_SUCH_FIELD if a field does not exist in an object + * - INDEX_OUT_OF_BOUNDS if an array index is larger than an array length + * - INCORRECT_TYPE if a non-integer is used to access an array + */ + inline simdjson_result at_path(std::string_view json_path) noexcept; + + /** + * Consumes the array and returns a string_view instance corresponding to the + * array as represented in JSON. It points inside the original document. + */ + simdjson_inline simdjson_result raw_json() noexcept; + + /** + * Get the value at the given index. This function has linear-time complexity. + * This function should only be called once on an array instance since the array iterator is not reset between each call. + * + * @return The value at the given index, or: + * - INDEX_OUT_OF_BOUNDS if the array index is larger than an array length + */ + simdjson_inline simdjson_result at(size_t index) noexcept; + protected: + /** + * Go to the end of the array, no matter where you are right now. + */ + simdjson_inline error_code consume() noexcept; + + /** + * Begin array iteration. + * + * @param iter The iterator. Must be where the initial [ is expected. Will be *moved* into the + * resulting array. + * @error INCORRECT_TYPE if the iterator is not at [. + */ + static simdjson_inline simdjson_result start(value_iterator& iter) noexcept; + /** + * Begin array iteration from the root. + * + * @param iter The iterator. Must be where the initial [ is expected. Will be *moved* into the + * resulting array. + * @error INCORRECT_TYPE if the iterator is not at [. + * @error TAPE_ERROR if there is no closing ] at the end of the document. + */ + static simdjson_inline simdjson_result start_root(value_iterator& iter) noexcept; + /** + * Begin array iteration. + * + * This version of the method should be called after the initial [ has been verified, and is + * intended for use by switch statements that check the type of a value. + * + * @param iter The iterator. Must be after the initial [. Will be *moved* into the resulting array. + */ + static simdjson_inline simdjson_result started(value_iterator& iter) noexcept; + + /** + * Create an array at the given Internal array creation. Call array::start() or array::started() instead of this. + * + * @param iter The iterator. Must either be at the start of the first element with iter.is_alive() + * == true, or past the [] with is_alive() == false if the array is empty. Will be *moved* + * into the resulting array. + */ + simdjson_inline array(const value_iterator& iter) noexcept; + + /** + * Iterator marking current position. + * + * iter.is_alive() == false indicates iteration is complete. + */ + value_iterator iter{}; + + friend class value; + friend class document; + friend struct simdjson_result; + friend struct simdjson_result; + friend class array_iterator; + }; + + } // namespace ondemand + } // namespace lsx +} // namespace simdjson + +namespace simdjson { + + template<> + struct simdjson_result : public lsx::implementation_simdjson_result_base { + public: + simdjson_inline simdjson_result(lsx::ondemand::array&& value) noexcept; ///< @private + simdjson_inline simdjson_result(error_code error) noexcept; ///< @private + simdjson_inline simdjson_result() noexcept = default; + + simdjson_inline simdjson_result begin() noexcept; + simdjson_inline simdjson_result end() noexcept; + inline simdjson_result count_elements() & noexcept; + inline simdjson_result is_empty() & noexcept; + inline simdjson_result reset() & noexcept; + simdjson_inline simdjson_result at(size_t index) noexcept; + simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; + simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; + simdjson_inline simdjson_result raw_json() noexcept; + + }; + +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_ONDEMAND_ARRAY_H +/* end file simdjson/generic/ondemand/array.h for lsx */ +/* including simdjson/generic/ondemand/array_iterator.h for lsx: #include "simdjson/generic/ondemand/array_iterator.h" */ +/* begin file simdjson/generic/ondemand/array_iterator.h for lsx */ +#ifndef SIMDJSON_GENERIC_ONDEMAND_ARRAY_ITERATOR_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_ONDEMAND_ARRAY_ITERATOR_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/implementation_simdjson_result_base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/value_iterator.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + + +namespace simdjson { + namespace lsx { + namespace ondemand { + + /** + * A forward-only JSON array. + * + * This is an input_iterator, meaning: + * - It is forward-only + * - * must be called exactly once per element. + * - ++ must be called exactly once in between each * (*, ++, *, ++, * ...) + */ + class array_iterator { + public: + /** Create a new, invalid array iterator. */ + simdjson_inline array_iterator() noexcept = default; + + // + // Iterator interface + // + + /** + * Get the current element. + * + * Part of the std::iterator interface. + */ + simdjson_inline simdjson_result operator*() noexcept; // MUST ONLY BE CALLED ONCE PER ITERATION. + /** + * Check if we are at the end of the JSON. + * + * Part of the std::iterator interface. + * + * @return true if there are no more elements in the JSON array. + */ + simdjson_inline bool operator==(const array_iterator&) const noexcept; + /** + * Check if there are more elements in the JSON array. + * + * Part of the std::iterator interface. + * + * @return true if there are more elements in the JSON array. + */ + simdjson_inline bool operator!=(const array_iterator&) const noexcept; + /** + * Move to the next element. + * + * Part of the std::iterator interface. + */ + simdjson_inline array_iterator& operator++() noexcept; + + private: + value_iterator iter{}; + + simdjson_inline array_iterator(const value_iterator& iter) noexcept; + + friend class array; + friend class value; + friend struct simdjson_result; + }; + + } // namespace ondemand + } // namespace lsx +} // namespace simdjson + +namespace simdjson { + + template<> + struct simdjson_result : public lsx::implementation_simdjson_result_base { + public: + simdjson_inline simdjson_result(lsx::ondemand::array_iterator&& value) noexcept; ///< @private + simdjson_inline simdjson_result(error_code error) noexcept; ///< @private + simdjson_inline simdjson_result() noexcept = default; + + // + // Iterator interface + // + + simdjson_inline simdjson_result operator*() noexcept; // MUST ONLY BE CALLED ONCE PER ITERATION. + simdjson_inline bool operator==(const simdjson_result&) const noexcept; + simdjson_inline bool operator!=(const simdjson_result&) const noexcept; + simdjson_inline simdjson_result& operator++() noexcept; + }; + +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_ONDEMAND_ARRAY_ITERATOR_H +/* end file simdjson/generic/ondemand/array_iterator.h for lsx */ +/* including simdjson/generic/ondemand/document.h for lsx: #include "simdjson/generic/ondemand/document.h" */ +/* begin file simdjson/generic/ondemand/document.h for lsx */ +#ifndef SIMDJSON_GENERIC_ONDEMAND_DOCUMENT_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_ONDEMAND_DOCUMENT_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/json_iterator.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lsx { + namespace ondemand { + + /** + * A JSON document. It holds a json_iterator instance. + * + * Used by tokens to get text, and string buffer location. + * + * You must keep the document around during iteration. + */ + class document { + public: + /** + * Create a new invalid document. + * + * Exists so you can declare a variable and later assign to it before use. + */ + simdjson_inline document() noexcept = default; + simdjson_inline document(const document& other) noexcept = delete; // pass your documents by reference, not by copy + simdjson_inline document(document&& other) noexcept = default; + simdjson_inline document& operator=(const document& other) noexcept = delete; + simdjson_inline document& operator=(document&& other) noexcept = default; + + /** + * Cast this JSON value to an array. + * + * @returns An object that can be used to iterate the array. + * @returns INCORRECT_TYPE If the JSON value is not an array. + */ + simdjson_inline simdjson_result get_array() & noexcept; + /** + * Cast this JSON value to an object. + * + * @returns An object that can be used to look up or iterate fields. + * @returns INCORRECT_TYPE If the JSON value is not an object. + */ + simdjson_inline simdjson_result get_object() & noexcept; + /** + * Cast this JSON value to an unsigned integer. + * + * @returns A signed 64-bit integer. + * @returns INCORRECT_TYPE If the JSON value is not a 64-bit unsigned integer. + */ + simdjson_inline simdjson_result get_uint64() noexcept; + /** + * Cast this JSON value (inside string) to an unsigned integer. + * + * @returns A signed 64-bit integer. + * @returns INCORRECT_TYPE If the JSON value is not a 64-bit unsigned integer. + */ + simdjson_inline simdjson_result get_uint64_in_string() noexcept; + /** + * Cast this JSON value to a signed integer. + * + * @returns A signed 64-bit integer. + * @returns INCORRECT_TYPE If the JSON value is not a 64-bit integer. + */ + simdjson_inline simdjson_result get_int64() noexcept; + /** + * Cast this JSON value (inside string) to a signed integer. + * + * @returns A signed 64-bit integer. + * @returns INCORRECT_TYPE If the JSON value is not a 64-bit integer. + */ + simdjson_inline simdjson_result get_int64_in_string() noexcept; + /** + * Cast this JSON value to a double. + * + * @returns A double. + * @returns INCORRECT_TYPE If the JSON value is not a valid floating-point number. + */ + simdjson_inline simdjson_result get_double() noexcept; + + /** + * Cast this JSON value (inside string) to a double. + * + * @returns A double. + * @returns INCORRECT_TYPE If the JSON value is not a valid floating-point number. + */ + simdjson_inline simdjson_result get_double_in_string() noexcept; + /** + * Cast this JSON value to a string. + * + * The string is guaranteed to be valid UTF-8. + * + * Important: Calling get_string() twice on the same document is an error. + * + * @param Whether to allow a replacement character for unmatched surrogate pairs. + * @returns An UTF-8 string. The string is stored in the parser and will be invalidated the next + * time it parses a document or when it is destroyed. + * @returns INCORRECT_TYPE if the JSON value is not a string. + */ + simdjson_inline simdjson_result get_string(bool allow_replacement = false) noexcept; + /** + * Attempts to fill the provided std::string reference with the parsed value of the current string. + * + * The string is guaranteed to be valid UTF-8. + * + * Important: a value should be consumed once. Calling get_string() twice on the same value + * is an error. + * + * Performance: This method may be slower than get_string() or get_string(bool) because it may need to allocate memory. + * We recommend you avoid allocating an std::string unless you need to. + * + * @returns INCORRECT_TYPE if the JSON value is not a string. Otherwise, we return SUCCESS. + */ + template + simdjson_inline error_code get_string(string_type& receiver, bool allow_replacement = false) noexcept; + /** + * Cast this JSON value to a string. + * + * The string is not guaranteed to be valid UTF-8. See https://simonsapin.github.io/wtf-8/ + * + * Important: Calling get_wobbly_string() twice on the same document is an error. + * + * @returns An UTF-8 string. The string is stored in the parser and will be invalidated the next + * time it parses a document or when it is destroyed. + * @returns INCORRECT_TYPE if the JSON value is not a string. + */ + simdjson_inline simdjson_result get_wobbly_string() noexcept; + /** + * Cast this JSON value to a raw_json_string. + * + * The string is guaranteed to be valid UTF-8, and may have escapes in it (e.g. \\ or \n). + * + * @returns A pointer to the raw JSON for the given string. + * @returns INCORRECT_TYPE if the JSON value is not a string. + */ + simdjson_inline simdjson_result get_raw_json_string() noexcept; + /** + * Cast this JSON value to a bool. + * + * @returns A bool value. + * @returns INCORRECT_TYPE if the JSON value is not true or false. + */ + simdjson_inline simdjson_result get_bool() noexcept; + /** + * Cast this JSON value to a value when the document is an object or an array. + * + * You must not have begun iterating through the object or array. When + * SIMDJSON_DEVELOPMENT_CHECKS is set to 1 (which is the case when building in Debug mode + * by default), and you have already begun iterating, + * you will get an OUT_OF_ORDER_ITERATION error. If you have begun iterating, you can use + * rewind() to reset the document to its initial state before calling this method. + * + * @returns A value if a JSON array or object cannot be found. + * @returns SCALAR_DOCUMENT_AS_VALUE error is the document is a scalar (see is_scalar() function). + */ + simdjson_inline simdjson_result get_value() noexcept; + + /** + * Checks if this JSON value is null. If and only if the value is + * null, then it is consumed (we advance). If we find a token that + * begins with 'n' but is not 'null', then an error is returned. + * + * @returns Whether the value is null. + * @returns INCORRECT_TYPE If the JSON value begins with 'n' and is not 'null'. + */ + simdjson_inline simdjson_result is_null() noexcept; + + /** + * Get this value as the given type. + * + * Supported types: object, array, raw_json_string, string_view, uint64_t, int64_t, double, bool + * + * You may use get_double(), get_bool(), get_uint64(), get_int64(), + * get_object(), get_array(), get_raw_json_string(), or get_string() instead. + * + * @returns A value of the given type, parsed from the JSON. + * @returns INCORRECT_TYPE If the JSON value is not the given type. + */ + template simdjson_inline simdjson_result get() & noexcept { + // Unless the simdjson library or the user provides an inline implementation, calling this method should + // immediately fail. + static_assert(!sizeof(T), "The get method with given type is not implemented by the simdjson library. " + "The supported types are ondemand::object, ondemand::array, raw_json_string, std::string_view, uint64_t, " + "int64_t, double, and bool. We recommend you use get_double(), get_bool(), get_uint64(), get_int64(), " + " get_object(), get_array(), get_raw_json_string(), or get_string() instead of the get template." + " You may also add support for custom types, see our documentation."); + } + /** @overload template simdjson_result get() & noexcept */ + template simdjson_deprecated simdjson_inline simdjson_result get() && noexcept { + // Unless the simdjson library or the user provides an inline implementation, calling this method should + // immediately fail. + static_assert(!sizeof(T), "The get method with given type is not implemented by the simdjson library. " + "The supported types are ondemand::object, ondemand::array, raw_json_string, std::string_view, uint64_t, " + "int64_t, double, and bool. We recommend you use get_double(), get_bool(), get_uint64(), get_int64(), " + " get_object(), get_array(), get_raw_json_string(), or get_string() instead of the get template." + " You may also add support for custom types, see our documentation."); + } + + /** + * Get this value as the given type. + * + * Supported types: object, array, raw_json_string, string_view, uint64_t, int64_t, double, bool, value + * + * Be mindful that the document instance must remain in scope while you are accessing object, array and value instances. + * + * @param out This is set to a value of the given type, parsed from the JSON. If there is an error, this may not be initialized. + * @returns INCORRECT_TYPE If the JSON value is not an object. + * @returns SUCCESS If the parse succeeded and the out parameter was set to the value. + */ + template simdjson_inline error_code get(T& out) & noexcept; + /** @overload template error_code get(T &out) & noexcept */ + template simdjson_deprecated simdjson_inline error_code get(T& out) && noexcept; + +#if SIMDJSON_EXCEPTIONS + /** + * Cast this JSON value to an instance of type T. The programmer is responsible for + * providing an implementation of get for the type T, if T is not one of the types + * supported by the library (object, array, raw_json_string, string_view, uint64_t, etc.) + * + * See https://github.com/simdjson/simdjson/blob/master/doc/basics.md#adding-support-for-custom-types + * + * @returns An instance of type T + */ + template + explicit simdjson_inline operator T() & noexcept(false); + template + explicit simdjson_deprecated simdjson_inline operator T() && noexcept(false); + + /** + * Cast this JSON value to an array. + * + * @returns An object that can be used to iterate the array. + * @exception simdjson_error(INCORRECT_TYPE) If the JSON value is not an array. + */ + simdjson_inline operator array() & noexcept(false); + /** + * Cast this JSON value to an object. + * + * @returns An object that can be used to look up or iterate fields. + * @exception simdjson_error(INCORRECT_TYPE) If the JSON value is not an object. + */ + simdjson_inline operator object() & noexcept(false); + /** + * Cast this JSON value to an unsigned integer. + * + * @returns A signed 64-bit integer. + * @exception simdjson_error(INCORRECT_TYPE) If the JSON value is not a 64-bit unsigned integer. + */ + simdjson_inline operator uint64_t() noexcept(false); + /** + * Cast this JSON value to a signed integer. + * + * @returns A signed 64-bit integer. + * @exception simdjson_error(INCORRECT_TYPE) If the JSON value is not a 64-bit integer. + */ + simdjson_inline operator int64_t() noexcept(false); + /** + * Cast this JSON value to a double. + * + * @returns A double. + * @exception simdjson_error(INCORRECT_TYPE) If the JSON value is not a valid floating-point number. + */ + simdjson_inline operator double() noexcept(false); + /** + * Cast this JSON value to a string. + * + * The string is guaranteed to be valid UTF-8. + * + * @returns An UTF-8 string. The string is stored in the parser and will be invalidated the next + * time it parses a document or when it is destroyed. + * @exception simdjson_error(INCORRECT_TYPE) if the JSON value is not a string. + */ + simdjson_inline operator std::string_view() noexcept(false); + /** + * Cast this JSON value to a raw_json_string. + * + * The string is guaranteed to be valid UTF-8, and may have escapes in it (e.g. \\ or \n). + * + * @returns A pointer to the raw JSON for the given string. + * @exception simdjson_error(INCORRECT_TYPE) if the JSON value is not a string. + */ + simdjson_inline operator raw_json_string() noexcept(false); + /** + * Cast this JSON value to a bool. + * + * @returns A bool value. + * @exception simdjson_error(INCORRECT_TYPE) if the JSON value is not true or false. + */ + simdjson_inline operator bool() noexcept(false); + /** + * Cast this JSON value to a value when the document is an object or an array. + * + * You must not have begun iterating through the object or array. When + * SIMDJSON_DEVELOPMENT_CHECKS is defined, and you have already begun iterating, + * you will get an OUT_OF_ORDER_ITERATION error. If you have begun iterating, you can use + * rewind() to reset the document to its initial state before calling this method. + * + * @returns A value value if a JSON array or object cannot be found. + * @exception SCALAR_DOCUMENT_AS_VALUE error is the document is a scalar (see is_scalar() function). + */ + simdjson_inline operator value() noexcept(false); +#endif + /** + * This method scans the array and counts the number of elements. + * The count_elements method should always be called before you have begun + * iterating through the array: it is expected that you are pointing at + * the beginning of the array. + * The runtime complexity is linear in the size of the array. After + * calling this function, if successful, the array is 'rewinded' at its + * beginning as if it had never been accessed. If the JSON is malformed (e.g., + * there is a missing comma), then an error is returned and it is no longer + * safe to continue. + */ + simdjson_inline simdjson_result count_elements() & noexcept; + /** + * This method scans the object and counts the number of key-value pairs. + * The count_fields method should always be called before you have begun + * iterating through the object: it is expected that you are pointing at + * the beginning of the object. + * The runtime complexity is linear in the size of the object. After + * calling this function, if successful, the object is 'rewinded' at its + * beginning as if it had never been accessed. If the JSON is malformed (e.g., + * there is a missing comma), then an error is returned and it is no longer + * safe to continue. + * + * To check that an object is empty, it is more performant to use + * the is_empty() method. + */ + simdjson_inline simdjson_result count_fields() & noexcept; + /** + * Get the value at the given index in the array. This function has linear-time complexity. + * This function should only be called once on an array instance since the array iterator is not reset between each call. + * + * @return The value at the given index, or: + * - INDEX_OUT_OF_BOUNDS if the array index is larger than an array length + */ + simdjson_inline simdjson_result at(size_t index) & noexcept; + /** + * Begin array iteration. + * + * Part of the std::iterable interface. + */ + simdjson_inline simdjson_result begin() & noexcept; + /** + * Sentinel representing the end of the array. + * + * Part of the std::iterable interface. + */ + simdjson_inline simdjson_result end() & noexcept; + + /** + * Look up a field by name on an object (order-sensitive). + * + * The following code reads z, then y, then x, and thus will not retrieve x or y if fed the + * JSON `{ "x": 1, "y": 2, "z": 3 }`: + * + * ```c++ + * simdjson::ondemand::parser parser; + * auto obj = parser.parse(R"( { "x": 1, "y": 2, "z": 3 } )"_padded); + * double z = obj.find_field("z"); + * double y = obj.find_field("y"); + * double x = obj.find_field("x"); + * ``` + * + * **Raw Keys:** The lookup will be done against the *raw* key, and will not unescape keys. + * e.g. `object["a"]` will match `{ "a": 1 }`, but will *not* match `{ "\u0061": 1 }`. + * + * + * You must consume the fields on an object one at a time. A request for a new key + * invalidates previous field values: it makes them unsafe. E.g., the array + * given by content["bids"].get_array() should not be accessed after you have called + * content["asks"].get_array(). You can detect such mistakes by first compiling and running + * the code in Debug mode (or with the macro `SIMDJSON_DEVELOPMENT_CHECKS` set to 1): an + * OUT_OF_ORDER_ITERATION error is generated. + * + * You are expected to access keys only once. You should access the value corresponding to + * a key a single time. Doing object["mykey"].to_string()and then again object["mykey"].to_string() + * is an error. + * + * @param key The key to look up. + * @returns The value of the field, or NO_SUCH_FIELD if the field is not in the object. + */ + simdjson_inline simdjson_result find_field(std::string_view key) & noexcept; + /** @overload simdjson_inline simdjson_result find_field(std::string_view key) & noexcept; */ + simdjson_inline simdjson_result find_field(const char* key) & noexcept; + + /** + * Look up a field by name on an object, without regard to key order. + * + * **Performance Notes:** This is a bit less performant than find_field(), though its effect varies + * and often appears negligible. It starts out normally, starting out at the last field; but if + * the field is not found, it scans from the beginning of the object to see if it missed it. That + * missing case has a non-cache-friendly bump and lots of extra scanning, especially if the object + * in question is large. The fact that the extra code is there also bumps the executable size. + * + * It is the default, however, because it would be highly surprising (and hard to debug) if the + * default behavior failed to look up a field just because it was in the wrong order--and many + * APIs assume this. Therefore, you must be explicit if you want to treat objects as out of order. + * + * Use find_field() if you are sure fields will be in order (or are willing to treat it as if the + * field was not there when they are not in order). + * + * You must consume the fields on an object one at a time. A request for a new key + * invalidates previous field values: it makes them unsafe. E.g., the array + * given by content["bids"].get_array() should not be accessed after you have called + * content["asks"].get_array(). You can detect such mistakes by first compiling and running + * the code in Debug mode (or with the macro `SIMDJSON_DEVELOPMENT_CHECKS` set to 1): an + * OUT_OF_ORDER_ITERATION error is generated. + * + * You are expected to access keys only once. You should access the value corresponding to a key + * a single time. Doing object["mykey"].to_string() and then again object["mykey"].to_string() + * is an error. + * + * @param key The key to look up. + * @returns The value of the field, or NO_SUCH_FIELD if the field is not in the object. + */ + simdjson_inline simdjson_result find_field_unordered(std::string_view key) & noexcept; + /** @overload simdjson_inline simdjson_result find_field_unordered(std::string_view key) & noexcept; */ + simdjson_inline simdjson_result find_field_unordered(const char* key) & noexcept; + /** @overload simdjson_inline simdjson_result find_field_unordered(std::string_view key) & noexcept; */ + simdjson_inline simdjson_result operator[](std::string_view key) & noexcept; + /** @overload simdjson_inline simdjson_result find_field_unordered(std::string_view key) & noexcept; */ + simdjson_inline simdjson_result operator[](const char* key) & noexcept; + + /** + * Get the type of this JSON value. It does not validate or consume the value. + * E.g., you must still call "is_null()" to check that a value is null even if + * "type()" returns json_type::null. + * + * NOTE: If you're only expecting a value to be one type (a typical case), it's generally + * better to just call .get_double, .get_string, etc. and check for INCORRECT_TYPE (or just + * let it throw an exception). + * + * @error TAPE_ERROR when the JSON value is a bad token like "}" "," or "alse". + */ + simdjson_inline simdjson_result type() noexcept; + + /** + * Checks whether the document is a scalar (string, number, null, Boolean). + * Returns false when there it is an array or object. + * + * @returns true if the type is string, number, null, Boolean + * @error TAPE_ERROR when the JSON value is a bad token like "}" "," or "alse". + */ + simdjson_inline simdjson_result is_scalar() noexcept; + + /** + * Checks whether the document is a string. + * + * @returns true if the type is string + * @error TAPE_ERROR when the JSON value is a bad token like "}" "," or "alse". + */ + simdjson_inline simdjson_result is_string() noexcept; + + /** + * Checks whether the document is a negative number. + * + * @returns true if the number if negative. + */ + simdjson_inline bool is_negative() noexcept; + /** + * Checks whether the document is an integer number. Note that + * this requires to partially parse the number string. If + * the value is determined to be an integer, it may still + * not parse properly as an integer in subsequent steps + * (e.g., it might overflow). + * + * @returns true if the number if negative. + */ + simdjson_inline simdjson_result is_integer() noexcept; + /** + * Determine the number type (integer or floating-point number) as quickly + * as possible. This function does not fully validate the input. It is + * useful when you only need to classify the numbers, without parsing them. + * + * If you are planning to retrieve the value or you need full validation, + * consider using the get_number() method instead: it will fully parse + * and validate the input, and give you access to the type: + * get_number().get_number_type(). + * + * get_number_type() is number_type::unsigned_integer if we have + * an integer greater or equal to 9223372036854775808 and no larger than 18446744073709551615. + * get_number_type() is number_type::signed_integer if we have an + * integer that is less than 9223372036854775808 and greater or equal to -9223372036854775808. + * get_number_type() is number_type::big_integer if we have an integer outside + * of those ranges (either larger than 18446744073709551615 or smaller than -9223372036854775808). + * Otherwise, get_number_type() has value number_type::floating_point_number + * + * This function requires processing the number string, but it is expected + * to be faster than get_number().get_number_type() because it is does not + * parse the number value. + * + * @returns the type of the number + */ + simdjson_inline simdjson_result get_number_type() noexcept; + + /** + * Attempt to parse an ondemand::number. An ondemand::number may + * contain an integer value or a floating-point value, the simdjson + * library will autodetect the type. Thus it is a dynamically typed + * number. Before accessing the value, you must determine the detected + * type. + * + * number.get_number_type() is number_type::signed_integer if we have + * an integer in [-9223372036854775808,9223372036854775808) + * You can recover the value by calling number.get_int64() and you + * have that number.is_int64() is true. + * + * number.get_number_type() is number_type::unsigned_integer if we have + * an integer in [9223372036854775808,18446744073709551616) + * You can recover the value by calling number.get_uint64() and you + * have that number.is_uint64() is true. + * + * Otherwise, number.get_number_type() has value number_type::floating_point_number + * and we have a binary64 number. + * You can recover the value by calling number.get_double() and you + * have that number.is_double() is true. + * + * You must check the type before accessing the value: it is an error + * to call "get_int64()" when number.get_number_type() is not + * number_type::signed_integer and when number.is_int64() is false. + */ + simdjson_warn_unused simdjson_inline simdjson_result get_number() noexcept; + + /** + * Get the raw JSON for this token. + * + * The string_view will always point into the input buffer. + * + * The string_view will start at the beginning of the token, and include the entire token + * *as well as all spaces until the next token (or EOF).* This means, for example, that a + * string token always begins with a " and is always terminated by the final ", possibly + * followed by a number of spaces. + * + * The string_view is *not* null-terminated. If this is a scalar (string, number, + * boolean, or null), the character after the end of the string_view may be the padded buffer. + * + * Tokens include: + * - { + * - [ + * - "a string (possibly with UTF-8 or backslashed characters like \\\")". + * - -1.2e-100 + * - true + * - false + * - null + */ + simdjson_inline simdjson_result raw_json_token() noexcept; + + /** + * Reset the iterator inside the document instance so we are pointing back at the + * beginning of the document, as if it had just been created. It invalidates all + * values, objects and arrays that you have created so far (including unescaped strings). + */ + inline void rewind() noexcept; + /** + * Returns debugging information. + */ + inline std::string to_debug_string() noexcept; + /** + * Some unrecoverable error conditions may render the document instance unusable. + * The is_alive() method returns true when the document is still suitable. + */ + inline bool is_alive() noexcept; + + /** + * Returns the current location in the document if in bounds. + */ + inline simdjson_result current_location() const noexcept; + + /** + * Returns true if this document has been fully parsed. + * If you have consumed the whole document and at_end() returns + * false, then there may be trailing content. + */ + inline bool at_end() const noexcept; + + /** + * Returns the current depth in the document if in bounds. + * + * E.g., + * 0 = finished with document + * 1 = document root value (could be [ or {, not yet known) + * 2 = , or } inside root array/object + * 3 = key or value inside root array/object. + */ + simdjson_inline int32_t current_depth() const noexcept; + + /** + * Get the value associated with the given JSON pointer. We use the RFC 6901 + * https://tools.ietf.org/html/rfc6901 standard. + * + * ondemand::parser parser; + * auto json = R"({ "foo": { "a": [ 10, 20, 30 ] }})"_padded; + * auto doc = parser.iterate(json); + * doc.at_pointer("/foo/a/1") == 20 + * + * It is allowed for a key to be the empty string: + * + * ondemand::parser parser; + * auto json = R"({ "": { "a": [ 10, 20, 30 ] }})"_padded; + * auto doc = parser.iterate(json); + * doc.at_pointer("//a/1") == 20 + * + * Key values are matched exactly, without unescaping or Unicode normalization. + * We do a byte-by-byte comparison. E.g. + * + * const padded_string json = "{\"\\u00E9\":123}"_padded; + * auto doc = parser.iterate(json); + * doc.at_pointer("/\\u00E9") == 123 + * doc.at_pointer((const char*)u8"/\u00E9") returns an error (NO_SUCH_FIELD) + * + * Note that at_pointer() automatically calls rewind between each call. Thus + * all values, objects and arrays that you have created so far (including unescaped strings) + * are invalidated. After calling at_pointer, you need to consume the result: string values + * should be stored in your own variables, arrays should be decoded and stored in your own array-like + * structures and so forth. + * + * Also note that at_pointer() relies on find_field() which implies that we do not unescape keys when matching + * + * @return The value associated with the given JSON pointer, or: + * - NO_SUCH_FIELD if a field does not exist in an object + * - INDEX_OUT_OF_BOUNDS if an array index is larger than an array length + * - INCORRECT_TYPE if a non-integer is used to access an array + * - INVALID_JSON_POINTER if the JSON pointer is invalid and cannot be parsed + * - SCALAR_DOCUMENT_AS_VALUE if the json_pointer is empty and the document is not a scalar (see is_scalar() function). + */ + simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; + + /** + * Get the value associated with the given JSONPath expression. We only support + * JSONPath queries that trivially convertible to JSON Pointer queries: key + * names and array indices. + * + * https://datatracker.ietf.org/doc/html/draft-normington-jsonpath-00 + * + * Key values are matched exactly, without unescaping or Unicode normalization. + * We do a byte-by-byte comparison. E.g. + * + * const padded_string json = "{\"\\u00E9\":123}"_padded; + * auto doc = parser.iterate(json); + * doc.at_path(".\\u00E9") == 123 + * doc.at_path((const char*)u8".\u00E9") returns an error (NO_SUCH_FIELD) + * + * @return The value associated with the given JSONPath expression, or: + * - INVALID_JSON_POINTER if the JSONPath to JSON Pointer conversion fails + * - NO_SUCH_FIELD if a field does not exist in an object + * - INDEX_OUT_OF_BOUNDS if an array index is larger than an array length + * - INCORRECT_TYPE if a non-integer is used to access an array + */ + simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; + + /** + * Consumes the document and returns a string_view instance corresponding to the + * document as represented in JSON. It points inside the original byte array containing + * the JSON document. + */ + simdjson_inline simdjson_result raw_json() noexcept; + protected: + /** + * Consumes the document. + */ + simdjson_inline error_code consume() noexcept; + + simdjson_inline document(ondemand::json_iterator&& iter) noexcept; + simdjson_inline const uint8_t* text(uint32_t idx) const noexcept; + + simdjson_inline value_iterator resume_value_iterator() noexcept; + simdjson_inline value_iterator get_root_value_iterator() noexcept; + simdjson_inline simdjson_result start_or_resume_object() noexcept; + static simdjson_inline document start(ondemand::json_iterator&& iter) noexcept; + + // + // Fields + // + json_iterator iter{}; ///< Current position in the document + static constexpr depth_t DOCUMENT_DEPTH = 0; ///< document depth is always 0 + + friend class array_iterator; + friend class value; + friend class ondemand::parser; + friend class object; + friend class array; + friend class field; + friend class token; + friend class document_stream; + friend class document_reference; + }; + + + /** + * A document_reference is a thin wrapper around a document reference instance. + */ + class document_reference { + public: + simdjson_inline document_reference() noexcept; + simdjson_inline document_reference(document& d) noexcept; + simdjson_inline document_reference(const document_reference& other) noexcept = default; + simdjson_inline document_reference& operator=(const document_reference& other) noexcept = default; + simdjson_inline void rewind() noexcept; + simdjson_inline simdjson_result get_array() & noexcept; + simdjson_inline simdjson_result get_object() & noexcept; + simdjson_inline simdjson_result get_uint64() noexcept; + simdjson_inline simdjson_result get_uint64_in_string() noexcept; + simdjson_inline simdjson_result get_int64() noexcept; + simdjson_inline simdjson_result get_int64_in_string() noexcept; + simdjson_inline simdjson_result get_double() noexcept; + simdjson_inline simdjson_result get_double_in_string() noexcept; + simdjson_inline simdjson_result get_string(bool allow_replacement = false) noexcept; + template + simdjson_inline error_code get_string(string_type& receiver, bool allow_replacement = false) noexcept; + simdjson_inline simdjson_result get_wobbly_string() noexcept; + simdjson_inline simdjson_result get_raw_json_string() noexcept; + simdjson_inline simdjson_result get_bool() noexcept; + simdjson_inline simdjson_result get_value() noexcept; + + simdjson_inline simdjson_result is_null() noexcept; + template simdjson_inline simdjson_result get() & noexcept; + simdjson_inline simdjson_result raw_json() noexcept; + simdjson_inline operator document& () const noexcept; +#if SIMDJSON_EXCEPTIONS + template + explicit simdjson_inline operator T() noexcept(false); + simdjson_inline operator array() & noexcept(false); + simdjson_inline operator object() & noexcept(false); + simdjson_inline operator uint64_t() noexcept(false); + simdjson_inline operator int64_t() noexcept(false); + simdjson_inline operator double() noexcept(false); + simdjson_inline operator std::string_view() noexcept(false); + simdjson_inline operator raw_json_string() noexcept(false); + simdjson_inline operator bool() noexcept(false); + simdjson_inline operator value() noexcept(false); +#endif + simdjson_inline simdjson_result count_elements() & noexcept; + simdjson_inline simdjson_result count_fields() & noexcept; + simdjson_inline simdjson_result at(size_t index) & noexcept; + simdjson_inline simdjson_result begin() & noexcept; + simdjson_inline simdjson_result end() & noexcept; + simdjson_inline simdjson_result find_field(std::string_view key) & noexcept; + simdjson_inline simdjson_result find_field(const char* key) & noexcept; + simdjson_inline simdjson_result operator[](std::string_view key) & noexcept; + simdjson_inline simdjson_result operator[](const char* key) & noexcept; + simdjson_inline simdjson_result find_field_unordered(std::string_view key) & noexcept; + simdjson_inline simdjson_result find_field_unordered(const char* key) & noexcept; + + simdjson_inline simdjson_result type() noexcept; + simdjson_inline simdjson_result is_scalar() noexcept; + simdjson_inline simdjson_result is_string() noexcept; + + simdjson_inline simdjson_result current_location() noexcept; + simdjson_inline int32_t current_depth() const noexcept; + simdjson_inline bool is_negative() noexcept; + simdjson_inline simdjson_result is_integer() noexcept; + simdjson_inline simdjson_result get_number_type() noexcept; + simdjson_inline simdjson_result get_number() noexcept; + simdjson_inline simdjson_result raw_json_token() noexcept; + simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; + simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; + + private: + document* doc{ nullptr }; + }; + } // namespace ondemand + } // namespace lsx +} // namespace simdjson + +namespace simdjson { + + template<> + struct simdjson_result : public lsx::implementation_simdjson_result_base { + public: + simdjson_inline simdjson_result(lsx::ondemand::document&& value) noexcept; ///< @private + simdjson_inline simdjson_result(error_code error) noexcept; ///< @private + simdjson_inline simdjson_result() noexcept = default; + simdjson_inline error_code rewind() noexcept; + + simdjson_inline simdjson_result get_array() & noexcept; + simdjson_inline simdjson_result get_object() & noexcept; + simdjson_inline simdjson_result get_uint64() noexcept; + simdjson_inline simdjson_result get_uint64_in_string() noexcept; + simdjson_inline simdjson_result get_int64() noexcept; + simdjson_inline simdjson_result get_int64_in_string() noexcept; + simdjson_inline simdjson_result get_double() noexcept; + simdjson_inline simdjson_result get_double_in_string() noexcept; + simdjson_inline simdjson_result get_string(bool allow_replacement = false) noexcept; + template + simdjson_inline error_code get_string(string_type& receiver, bool allow_replacement = false) noexcept; + simdjson_inline simdjson_result get_wobbly_string() noexcept; + simdjson_inline simdjson_result get_raw_json_string() noexcept; + simdjson_inline simdjson_result get_bool() noexcept; + simdjson_inline simdjson_result get_value() noexcept; + simdjson_inline simdjson_result is_null() noexcept; + + template simdjson_inline simdjson_result get() & noexcept; + template simdjson_deprecated simdjson_inline simdjson_result get() && noexcept; + + template simdjson_inline error_code get(T& out) & noexcept; + template simdjson_inline error_code get(T& out) && noexcept; +#if SIMDJSON_EXCEPTIONS + template ::value == false>::type> + explicit simdjson_inline operator T() noexcept(false); + simdjson_inline operator lsx::ondemand::array() & noexcept(false); + simdjson_inline operator lsx::ondemand::object() & noexcept(false); + simdjson_inline operator uint64_t() noexcept(false); + simdjson_inline operator int64_t() noexcept(false); + simdjson_inline operator double() noexcept(false); + simdjson_inline operator std::string_view() noexcept(false); + simdjson_inline operator lsx::ondemand::raw_json_string() noexcept(false); + simdjson_inline operator bool() noexcept(false); + simdjson_inline operator lsx::ondemand::value() noexcept(false); +#endif + simdjson_inline simdjson_result count_elements() & noexcept; + simdjson_inline simdjson_result count_fields() & noexcept; + simdjson_inline simdjson_result at(size_t index) & noexcept; + simdjson_inline simdjson_result begin() & noexcept; + simdjson_inline simdjson_result end() & noexcept; + simdjson_inline simdjson_result find_field(std::string_view key) & noexcept; + simdjson_inline simdjson_result find_field(const char* key) & noexcept; + simdjson_inline simdjson_result operator[](std::string_view key) & noexcept; + simdjson_inline simdjson_result operator[](const char* key) & noexcept; + simdjson_inline simdjson_result find_field_unordered(std::string_view key) & noexcept; + simdjson_inline simdjson_result find_field_unordered(const char* key) & noexcept; + simdjson_inline simdjson_result type() noexcept; + simdjson_inline simdjson_result is_scalar() noexcept; + simdjson_inline simdjson_result is_string() noexcept; + simdjson_inline simdjson_result current_location() noexcept; + simdjson_inline int32_t current_depth() const noexcept; + simdjson_inline bool at_end() const noexcept; + simdjson_inline bool is_negative() noexcept; + simdjson_inline simdjson_result is_integer() noexcept; + simdjson_inline simdjson_result get_number_type() noexcept; + simdjson_inline simdjson_result get_number() noexcept; + /** @copydoc simdjson_inline std::string_view document::raw_json_token() const noexcept */ + simdjson_inline simdjson_result raw_json_token() noexcept; + + simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; + simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; + }; + + +} // namespace simdjson + + + +namespace simdjson { + + template<> + struct simdjson_result : public lsx::implementation_simdjson_result_base { + public: + simdjson_inline simdjson_result(lsx::ondemand::document_reference value, error_code error) noexcept; + simdjson_inline simdjson_result() noexcept = default; + simdjson_inline error_code rewind() noexcept; + + simdjson_inline simdjson_result get_array() & noexcept; + simdjson_inline simdjson_result get_object() & noexcept; + simdjson_inline simdjson_result get_uint64() noexcept; + simdjson_inline simdjson_result get_uint64_in_string() noexcept; + simdjson_inline simdjson_result get_int64() noexcept; + simdjson_inline simdjson_result get_int64_in_string() noexcept; + simdjson_inline simdjson_result get_double() noexcept; + simdjson_inline simdjson_result get_double_in_string() noexcept; + simdjson_inline simdjson_result get_string(bool allow_replacement = false) noexcept; + template + simdjson_inline error_code get_string(string_type& receiver, bool allow_replacement = false) noexcept; + simdjson_inline simdjson_result get_wobbly_string() noexcept; + simdjson_inline simdjson_result get_raw_json_string() noexcept; + simdjson_inline simdjson_result get_bool() noexcept; + simdjson_inline simdjson_result get_value() noexcept; + simdjson_inline simdjson_result is_null() noexcept; +#if SIMDJSON_EXCEPTIONS + template ::value == false>::type> + explicit simdjson_inline operator T() noexcept(false); + simdjson_inline operator lsx::ondemand::array() & noexcept(false); + simdjson_inline operator lsx::ondemand::object() & noexcept(false); + simdjson_inline operator uint64_t() noexcept(false); + simdjson_inline operator int64_t() noexcept(false); + simdjson_inline operator double() noexcept(false); + simdjson_inline operator std::string_view() noexcept(false); + simdjson_inline operator lsx::ondemand::raw_json_string() noexcept(false); + simdjson_inline operator bool() noexcept(false); + simdjson_inline operator lsx::ondemand::value() noexcept(false); +#endif + simdjson_inline simdjson_result count_elements() & noexcept; + simdjson_inline simdjson_result count_fields() & noexcept; + simdjson_inline simdjson_result at(size_t index) & noexcept; + simdjson_inline simdjson_result begin() & noexcept; + simdjson_inline simdjson_result end() & noexcept; + simdjson_inline simdjson_result find_field(std::string_view key) & noexcept; + simdjson_inline simdjson_result find_field(const char* key) & noexcept; + simdjson_inline simdjson_result operator[](std::string_view key) & noexcept; + simdjson_inline simdjson_result operator[](const char* key) & noexcept; + simdjson_inline simdjson_result find_field_unordered(std::string_view key) & noexcept; + simdjson_inline simdjson_result find_field_unordered(const char* key) & noexcept; + simdjson_inline simdjson_result type() noexcept; + simdjson_inline simdjson_result is_scalar() noexcept; + simdjson_inline simdjson_result is_string() noexcept; + simdjson_inline simdjson_result current_location() noexcept; + simdjson_inline simdjson_result current_depth() const noexcept; + simdjson_inline simdjson_result is_negative() noexcept; + simdjson_inline simdjson_result is_integer() noexcept; + simdjson_inline simdjson_result get_number_type() noexcept; + simdjson_inline simdjson_result get_number() noexcept; + /** @copydoc simdjson_inline std::string_view document_reference::raw_json_token() const noexcept */ + simdjson_inline simdjson_result raw_json_token() noexcept; + + simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; + simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; + }; + + +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_ONDEMAND_DOCUMENT_H +/* end file simdjson/generic/ondemand/document.h for lsx */ +/* including simdjson/generic/ondemand/document_stream.h for lsx: #include "simdjson/generic/ondemand/document_stream.h" */ +/* begin file simdjson/generic/ondemand/document_stream.h for lsx */ +#ifndef SIMDJSON_GENERIC_ONDEMAND_DOCUMENT_STREAM_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_ONDEMAND_DOCUMENT_STREAM_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/implementation_simdjson_result_base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/document.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/parser.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +#ifdef SIMDJSON_THREADS_ENABLED +#include +#include +#include +#endif + +namespace simdjson { + namespace lsx { + namespace ondemand { + +#ifdef SIMDJSON_THREADS_ENABLED + /** @private Custom worker class **/ + struct stage1_worker { + stage1_worker() noexcept = default; + stage1_worker(const stage1_worker&) = delete; + stage1_worker(stage1_worker&&) = delete; + stage1_worker operator=(const stage1_worker&) = delete; + ~stage1_worker(); + /** + * We only start the thread when it is needed, not at object construction, this may throw. + * You should only call this once. + **/ + void start_thread(); + /** + * Start a stage 1 job. You should first call 'run', then 'finish'. + * You must call start_thread once before. + */ + void run(document_stream* ds, parser* stage1, size_t next_batch_start); + /** Wait for the run to finish (blocking). You should first call 'run', then 'finish'. **/ + void finish(); + + private: + + /** + * Normally, we would never stop the thread. But we do in the destructor. + * This function is only safe assuming that you are not waiting for results. You + * should have called run, then finish, and be done. + **/ + void stop_thread(); + + std::thread thread{}; + /** These three variables define the work done by the thread. **/ + ondemand::parser* stage1_thread_parser{}; + size_t _next_batch_start{}; + document_stream* owner{}; + /** + * We have two state variables. This could be streamlined to one variable in the future but + * we use two for clarity. + */ + bool has_work{ false }; + bool can_work{ true }; + + /** + * We lock using a mutex. + */ + std::mutex locking_mutex{}; + std::condition_variable cond_var{}; + + friend class document_stream; + }; +#endif // SIMDJSON_THREADS_ENABLED + + /** + * A forward-only stream of documents. + * + * Produced by parser::iterate_many. + * + */ + class document_stream { + public: + /** + * Construct an uninitialized document_stream. + * + * ```c++ + * document_stream docs; + * auto error = parser.iterate_many(json).get(docs); + * ``` + */ + simdjson_inline document_stream() noexcept; + /** Move one document_stream to another. */ + simdjson_inline document_stream(document_stream&& other) noexcept = default; + /** Move one document_stream to another. */ + simdjson_inline document_stream& operator=(document_stream&& other) noexcept = default; + + simdjson_inline ~document_stream() noexcept; + + /** + * Returns the input size in bytes. + */ + inline size_t size_in_bytes() const noexcept; + + /** + * After iterating through the stream, this method + * returns the number of bytes that were not parsed at the end + * of the stream. If truncated_bytes() differs from zero, + * then the input was truncated maybe because incomplete JSON + * documents were found at the end of the stream. You + * may need to process the bytes in the interval [size_in_bytes()-truncated_bytes(), size_in_bytes()). + * + * You should only call truncated_bytes() after streaming through all + * documents, like so: + * + * document_stream stream = parser.iterate_many(json,window); + * for(auto & doc : stream) { + * // do something with doc + * } + * size_t truncated = stream.truncated_bytes(); + * + */ + inline size_t truncated_bytes() const noexcept; + + class iterator { + public: + using value_type = simdjson_result; + using reference = simdjson_result; + using pointer = void; + using difference_type = std::ptrdiff_t; + using iterator_category = std::input_iterator_tag; + + /** + * Default constructor. + */ + simdjson_inline iterator() noexcept; + /** + * Get the current document (or error). + */ + simdjson_inline reference operator*() noexcept; + /** + * Advance to the next document (prefix). + */ + inline iterator& operator++() noexcept; + /** + * Check if we're at the end yet. + * @param other the end iterator to compare to. + */ + simdjson_inline bool operator!=(const iterator& other) const noexcept; + /** + * @private + * + * Gives the current index in the input document in bytes. + * + * document_stream stream = parser.parse_many(json,window); + * for(auto i = stream.begin(); i != stream.end(); ++i) { + * auto doc = *i; + * size_t index = i.current_index(); + * } + * + * This function (current_index()) is experimental and the usage + * may change in future versions of simdjson: we find the API somewhat + * awkward and we would like to offer something friendlier. + */ + simdjson_inline size_t current_index() const noexcept; + + /** + * @private + * + * Gives a view of the current document at the current position. + * + * document_stream stream = parser.iterate_many(json,window); + * for(auto i = stream.begin(); i != stream.end(); ++i) { + * std::string_view v = i.source(); + * } + * + * The returned string_view instance is simply a map to the (unparsed) + * source string: it may thus include white-space characters and all manner + * of padding. + * + * This function (source()) is experimental and the usage + * may change in future versions of simdjson: we find the API somewhat + * awkward and we would like to offer something friendlier. + * + */ + simdjson_inline std::string_view source() const noexcept; + + /** + * Returns error of the stream (if any). + */ + inline error_code error() const noexcept; + + private: + simdjson_inline iterator(document_stream* s, bool finished) noexcept; + /** The document_stream we're iterating through. */ + document_stream* stream; + /** Whether we're finished or not. */ + bool finished; + + friend class document; + friend class document_stream; + friend class json_iterator; + }; + + /** + * Start iterating the documents in the stream. + */ + simdjson_inline iterator begin() noexcept; + /** + * The end of the stream, for iterator comparison purposes. + */ + simdjson_inline iterator end() noexcept; + + private: + + document_stream& operator=(const document_stream&) = delete; // Disallow copying + document_stream(const document_stream& other) = delete; // Disallow copying + + /** + * Construct a document_stream. Does not allocate or parse anything until the iterator is + * used. + * + * @param parser is a reference to the parser instance used to generate this document_stream + * @param buf is the raw byte buffer we need to process + * @param len is the length of the raw byte buffer in bytes + * @param batch_size is the size of the windows (must be strictly greater or equal to the largest JSON document) + */ + simdjson_inline document_stream( + ondemand::parser& parser, + const uint8_t* buf, + size_t len, + size_t batch_size, + bool allow_comma_separated + ) noexcept; + + /** + * Parse the first document in the buffer. Used by begin(), to handle allocation and + * initialization. + */ + inline void start() noexcept; + + /** + * Parse the next document found in the buffer previously given to document_stream. + * + * The content should be a valid JSON document encoded as UTF-8. If there is a + * UTF-8 BOM, the parser skips it. + * + * You do NOT need to pre-allocate a parser. This function takes care of + * pre-allocating a capacity defined by the batch_size defined when creating the + * document_stream object. + * + * The function returns simdjson::EMPTY if there is no more data to be parsed. + * + * The function returns simdjson::SUCCESS (as integer = 0) in case of success + * and indicates that the buffer has successfully been parsed to the end. + * Every document it contained has been parsed without error. + * + * The function returns an error code from simdjson/simdjson.h in case of failure + * such as simdjson::CAPACITY, simdjson::MEMALLOC, simdjson::DEPTH_ERROR and so forth; + * the simdjson::error_message function converts these error codes into a string). + * + * You can also check validity by calling parser.is_valid(). The same parser can + * and should be reused for the other documents in the buffer. + */ + inline void next() noexcept; + + /** Move the json_iterator of the document to the location of the next document in the stream. */ + inline void next_document() noexcept; + + /** Get the next document index. */ + inline size_t next_batch_start() const noexcept; + + /** Pass the next batch through stage 1 with the given parser. */ + inline error_code run_stage1(ondemand::parser& p, size_t batch_start) noexcept; + + // Fields + ondemand::parser* parser; + const uint8_t* buf; + size_t len; + size_t batch_size; + bool allow_comma_separated; + /** + * We are going to use just one document instance. The document owns + * the json_iterator. It implies that we only ever pass a reference + * to the document to the users. + */ + document doc{}; + /** The error (or lack thereof) from the current document. */ + error_code error; + size_t batch_start{ 0 }; + size_t doc_index{}; + +#ifdef SIMDJSON_THREADS_ENABLED + /** Indicates whether we use threads. Note that this needs to be a constant during the execution of the parsing. */ + bool use_thread; + + inline void load_from_stage1_thread() noexcept; + + /** Start a thread to run stage 1 on the next batch. */ + inline void start_stage1_thread() noexcept; + + /** Wait for the stage 1 thread to finish and capture the results. */ + inline void finish_stage1_thread() noexcept; + + /** The error returned from the stage 1 thread. */ + error_code stage1_thread_error{ UNINITIALIZED }; + /** The thread used to run stage 1 against the next batch in the background. */ + std::unique_ptr worker{ new(std::nothrow) stage1_worker() }; + /** + * The parser used to run stage 1 in the background. Will be swapped + * with the regular parser when finished. + */ + ondemand::parser stage1_thread_parser{}; + + friend struct stage1_worker; +#endif // SIMDJSON_THREADS_ENABLED + + friend class parser; + friend class document; + friend class json_iterator; + friend struct simdjson_result; + friend struct simdjson::internal::simdjson_result_base; + }; // document_stream + + } // namespace ondemand + } // namespace lsx +} // namespace simdjson + +namespace simdjson { + template<> + struct simdjson_result : public lsx::implementation_simdjson_result_base { + public: + simdjson_inline simdjson_result(lsx::ondemand::document_stream&& value) noexcept; ///< @private + simdjson_inline simdjson_result(error_code error) noexcept; ///< @private + simdjson_inline simdjson_result() noexcept = default; + }; + +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_ONDEMAND_DOCUMENT_STREAM_H +/* end file simdjson/generic/ondemand/document_stream.h for lsx */ +/* including simdjson/generic/ondemand/field.h for lsx: #include "simdjson/generic/ondemand/field.h" */ +/* begin file simdjson/generic/ondemand/field.h for lsx */ +#ifndef SIMDJSON_GENERIC_ONDEMAND_FIELD_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_ONDEMAND_FIELD_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/implementation_simdjson_result_base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/raw_json_string.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/value.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lsx { + namespace ondemand { + + /** + * A JSON field (key/value pair) in an object. + * + * Returned from object iteration. + * + * Extends from std::pair so you can use C++ algorithms that rely on pairs. + */ + class field : public std::pair { + public: + /** + * Create a new invalid field. + * + * Exists so you can declare a variable and later assign to it before use. + */ + simdjson_inline field() noexcept; + + /** + * Get the key as a string_view (for higher speed, consider raw_key). + * We deliberately use a more cumbersome name (unescaped_key) to force users + * to think twice about using it. + * + * This consumes the key: once you have called unescaped_key(), you cannot + * call it again nor can you call key(). + */ + simdjson_inline simdjson_warn_unused simdjson_result unescaped_key(bool allow_replacement = false) noexcept; + /** + * Get the key as a string_view (for higher speed, consider raw_key). + * We deliberately use a more cumbersome name (unescaped_key) to force users + * to think twice about using it. The content is stored in the receiver. + * + * This consumes the key: once you have called unescaped_key(), you cannot + * call it again nor can you call key(). + */ + template + simdjson_inline simdjson_warn_unused error_code unescaped_key(string_type& receiver, bool allow_replacement = false) noexcept; + /** + * Get the key as a raw_json_string. Can be used for direct comparison with + * an unescaped C string: e.g., key() == "test". This does not count as + * consumption of the content: you can safely call it repeatedly. + * See escaped_key() for a similar function which returns + * a more convenient std::string_view result. + */ + simdjson_inline raw_json_string key() const noexcept; + /** + * Get the unprocessed key as a string_view. This includes the quotes and may include + * some spaces after the last quote. This does not count as + * consumption of the content: you can safely call it repeatedly. + * See escaped_key(). + */ + simdjson_inline std::string_view key_raw_json_token() const noexcept; + /** + * Get the key as a string_view. This does not include the quotes and + * the string is unprocessed key so it may contain escape characters + * (e.g., \uXXXX or \n). It does not count as a consumption of the content: + * you can safely call it repeatedly. Use unescaped_key() to get the unescaped key. + */ + simdjson_inline std::string_view escaped_key() const noexcept; + /** + * Get the field value. + */ + simdjson_inline ondemand::value& value() & noexcept; + /** + * @overload ondemand::value &ondemand::value() & noexcept + */ + simdjson_inline ondemand::value value() && noexcept; + + protected: + simdjson_inline field(raw_json_string key, ondemand::value&& value) noexcept; + static simdjson_inline simdjson_result start(value_iterator& parent_iter) noexcept; + static simdjson_inline simdjson_result start(const value_iterator& parent_iter, raw_json_string key) noexcept; + friend struct simdjson_result; + friend class object_iterator; + }; + + } // namespace ondemand + } // namespace lsx +} // namespace simdjson + +namespace simdjson { + + template<> + struct simdjson_result : public lsx::implementation_simdjson_result_base { + public: + simdjson_inline simdjson_result(lsx::ondemand::field&& value) noexcept; ///< @private + simdjson_inline simdjson_result(error_code error) noexcept; ///< @private + simdjson_inline simdjson_result() noexcept = default; + + simdjson_inline simdjson_result unescaped_key(bool allow_replacement = false) noexcept; + template + simdjson_inline error_code unescaped_key(string_type& receiver, bool allow_replacement = false) noexcept; + simdjson_inline simdjson_result key() noexcept; + simdjson_inline simdjson_result key_raw_json_token() noexcept; + simdjson_inline simdjson_result escaped_key() noexcept; + simdjson_inline simdjson_result value() noexcept; + }; + +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_ONDEMAND_FIELD_H +/* end file simdjson/generic/ondemand/field.h for lsx */ +/* including simdjson/generic/ondemand/object.h for lsx: #include "simdjson/generic/ondemand/object.h" */ +/* begin file simdjson/generic/ondemand/object.h for lsx */ +#ifndef SIMDJSON_GENERIC_ONDEMAND_OBJECT_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_ONDEMAND_OBJECT_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/implementation_simdjson_result_base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/value_iterator.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lsx { + namespace ondemand { + + /** + * A forward-only JSON object field iterator. + */ + class object { + public: + /** + * Create a new invalid object. + * + * Exists so you can declare a variable and later assign to it before use. + */ + simdjson_inline object() noexcept = default; + + simdjson_inline simdjson_result begin() noexcept; + simdjson_inline simdjson_result end() noexcept; + /** + * Look up a field by name on an object (order-sensitive). + * + * The following code reads z, then y, then x, and thus will not retrieve x or y if fed the + * JSON `{ "x": 1, "y": 2, "z": 3 }`: + * + * ```c++ + * simdjson::ondemand::parser parser; + * auto obj = parser.parse(R"( { "x": 1, "y": 2, "z": 3 } )"_padded); + * double z = obj.find_field("z"); + * double y = obj.find_field("y"); + * double x = obj.find_field("x"); + * ``` + * If you have multiple fields with a matching key ({"x": 1, "x": 1}) be mindful + * that only one field is returned. + * + * **Raw Keys:** The lookup will be done against the *raw* key, and will not unescape keys. + * e.g. `object["a"]` will match `{ "a": 1 }`, but will *not* match `{ "\u0061": 1 }`. + * + * You must consume the fields on an object one at a time. A request for a new key + * invalidates previous field values: it makes them unsafe. The value instance you get + * from `content["bids"]` becomes invalid when you call `content["asks"]`. The array + * given by content["bids"].get_array() should not be accessed after you have called + * content["asks"].get_array(). You can detect such mistakes by first compiling and running + * the code in Debug mode (or with the macro `SIMDJSON_DEVELOPMENT_CHECKS` set to 1): an + * OUT_OF_ORDER_ITERATION error is generated. + * + * You are expected to access keys only once. You should access the value corresponding to a + * key a single time. Doing object["mykey"].to_string() and then again object["mykey"].to_string() + * is an error. + * + * If you expect to have keys with escape characters, please review our documentation. + * + * @param key The key to look up. + * @returns The value of the field, or NO_SUCH_FIELD if the field is not in the object. + */ + simdjson_inline simdjson_result find_field(std::string_view key) & noexcept; + /** @overload simdjson_inline simdjson_result find_field(std::string_view key) & noexcept; */ + simdjson_inline simdjson_result find_field(std::string_view key) && noexcept; + + /** + * Look up a field by name on an object, without regard to key order. + * + * **Performance Notes:** This is a bit less performant than find_field(), though its effect varies + * and often appears negligible. It starts out normally, starting out at the last field; but if + * the field is not found, it scans from the beginning of the object to see if it missed it. That + * missing case has a non-cache-friendly bump and lots of extra scanning, especially if the object + * in question is large. The fact that the extra code is there also bumps the executable size. + * + * It is the default, however, because it would be highly surprising (and hard to debug) if the + * default behavior failed to look up a field just because it was in the wrong order--and many + * APIs assume this. Therefore, you must be explicit if you want to treat objects as out of order. + * + * Use find_field() if you are sure fields will be in order (or are willing to treat it as if the + * field was not there when they are not in order). + * + * If you have multiple fields with a matching key ({"x": 1, "x": 1}) be mindful + * that only one field is returned. + * + * You must consume the fields on an object one at a time. A request for a new key + * invalidates previous field values: it makes them unsafe. The value instance you get + * from `content["bids"]` becomes invalid when you call `content["asks"]`. The array + * given by content["bids"].get_array() should not be accessed after you have called + * content["asks"].get_array(). You can detect such mistakes by first compiling and running + * the code in Debug mode (or with the macro `SIMDJSON_DEVELOPMENT_CHECKS` set to 1): an + * OUT_OF_ORDER_ITERATION error is generated. + * + * You are expected to access keys only once. You should access the value corresponding to a key + * a single time. Doing object["mykey"].to_string() and then again object["mykey"].to_string() is an error. + * + * If you expect to have keys with escape characters, please review our documentation. + * + * @param key The key to look up. + * @returns The value of the field, or NO_SUCH_FIELD if the field is not in the object. + */ + simdjson_inline simdjson_result find_field_unordered(std::string_view key) & noexcept; + /** @overload simdjson_inline simdjson_result find_field_unordered(std::string_view key) & noexcept; */ + simdjson_inline simdjson_result find_field_unordered(std::string_view key) && noexcept; + /** @overload simdjson_inline simdjson_result find_field_unordered(std::string_view key) & noexcept; */ + simdjson_inline simdjson_result operator[](std::string_view key) & noexcept; + /** @overload simdjson_inline simdjson_result find_field_unordered(std::string_view key) & noexcept; */ + simdjson_inline simdjson_result operator[](std::string_view key) && noexcept; + + /** + * Get the value associated with the given JSON pointer. We use the RFC 6901 + * https://tools.ietf.org/html/rfc6901 standard, interpreting the current node + * as the root of its own JSON document. + * + * ondemand::parser parser; + * auto json = R"({ "foo": { "a": [ 10, 20, 30 ] }})"_padded; + * auto doc = parser.iterate(json); + * doc.at_pointer("/foo/a/1") == 20 + * + * It is allowed for a key to be the empty string: + * + * ondemand::parser parser; + * auto json = R"({ "": { "a": [ 10, 20, 30 ] }})"_padded; + * auto doc = parser.iterate(json); + * doc.at_pointer("//a/1") == 20 + * + * Note that at_pointer() called on the document automatically calls the document's rewind + * method between each call. It invalidates all previously accessed arrays, objects and values + * that have not been consumed. Yet it is not the case when calling at_pointer on an object + * instance: there is no rewind and no invalidation. + * + * You may call at_pointer more than once on an object, but each time the pointer is advanced + * to be within the value matched by the key indicated by the JSON pointer query. Thus any preceding + * key (as well as the current key) can no longer be used with following JSON pointer calls. + * + * Also note that at_pointer() relies on find_field() which implies that we do not unescape keys when matching. + * + * @return The value associated with the given JSON pointer, or: + * - NO_SUCH_FIELD if a field does not exist in an object + * - INDEX_OUT_OF_BOUNDS if an array index is larger than an array length + * - INCORRECT_TYPE if a non-integer is used to access an array + * - INVALID_JSON_POINTER if the JSON pointer is invalid and cannot be parsed + */ + inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; + + /** + * Get the value associated with the given JSONPath expression. We only support + * JSONPath queries that trivially convertible to JSON Pointer queries: key + * names and array indices. + * + * @return The value associated with the given JSONPath expression, or: + * - INVALID_JSON_POINTER if the JSONPath to JSON Pointer conversion fails + * - NO_SUCH_FIELD if a field does not exist in an object + * - INDEX_OUT_OF_BOUNDS if an array index is larger than an array length + * - INCORRECT_TYPE if a non-integer is used to access an array + */ + inline simdjson_result at_path(std::string_view json_path) noexcept; + + /** + * Reset the iterator so that we are pointing back at the + * beginning of the object. You should still consume values only once even if you + * can iterate through the object more than once. If you unescape a string or a key + * within the object more than once, you have unsafe code. Note that rewinding an object + * means that you may need to reparse it anew: it is not a free operation. + * + * @returns true if the object contains some elements (not empty) + */ + inline simdjson_result reset() & noexcept; + /** + * This method scans the beginning of the object and checks whether the + * object is empty. + * The runtime complexity is constant time. After + * calling this function, if successful, the object is 'rewinded' at its + * beginning as if it had never been accessed. If the JSON is malformed (e.g., + * there is a missing comma), then an error is returned and it is no longer + * safe to continue. + */ + inline simdjson_result is_empty() & noexcept; + /** + * This method scans the object and counts the number of key-value pairs. + * The count_fields method should always be called before you have begun + * iterating through the object: it is expected that you are pointing at + * the beginning of the object. + * The runtime complexity is linear in the size of the object. After + * calling this function, if successful, the object is 'rewinded' at its + * beginning as if it had never been accessed. If the JSON is malformed (e.g., + * there is a missing comma), then an error is returned and it is no longer + * safe to continue. + * + * To check that an object is empty, it is more performant to use + * the is_empty() method. + * + * Performance hint: You should only call count_fields() as a last + * resort as it may require scanning the document twice or more. + */ + simdjson_inline simdjson_result count_fields() & noexcept; + /** + * Consumes the object and returns a string_view instance corresponding to the + * object as represented in JSON. It points inside the original byte array containing + * the JSON document. + */ + simdjson_inline simdjson_result raw_json() noexcept; + + protected: + /** + * Go to the end of the object, no matter where you are right now. + */ + simdjson_inline error_code consume() noexcept; + static simdjson_inline simdjson_result start(value_iterator& iter) noexcept; + static simdjson_inline simdjson_result start_root(value_iterator& iter) noexcept; + static simdjson_inline simdjson_result started(value_iterator& iter) noexcept; + static simdjson_inline object resume(const value_iterator& iter) noexcept; + simdjson_inline object(const value_iterator& iter) noexcept; + + simdjson_warn_unused simdjson_inline error_code find_field_raw(const std::string_view key) noexcept; + + value_iterator iter{}; + + friend class value; + friend class document; + friend struct simdjson_result; + }; + + } // namespace ondemand + } // namespace lsx +} // namespace simdjson + +namespace simdjson { + + template<> + struct simdjson_result : public lsx::implementation_simdjson_result_base { + public: + simdjson_inline simdjson_result(lsx::ondemand::object&& value) noexcept; ///< @private + simdjson_inline simdjson_result(error_code error) noexcept; ///< @private + simdjson_inline simdjson_result() noexcept = default; + + simdjson_inline simdjson_result begin() noexcept; + simdjson_inline simdjson_result end() noexcept; + simdjson_inline simdjson_result find_field(std::string_view key) & noexcept; + simdjson_inline simdjson_result find_field(std::string_view key) && noexcept; + simdjson_inline simdjson_result find_field_unordered(std::string_view key) & noexcept; + simdjson_inline simdjson_result find_field_unordered(std::string_view key) && noexcept; + simdjson_inline simdjson_result operator[](std::string_view key) & noexcept; + simdjson_inline simdjson_result operator[](std::string_view key) && noexcept; + simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; + simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; + + inline simdjson_result reset() noexcept; + inline simdjson_result is_empty() noexcept; + inline simdjson_result count_fields() & noexcept; + inline simdjson_result raw_json() noexcept; + + }; + +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_ONDEMAND_OBJECT_H +/* end file simdjson/generic/ondemand/object.h for lsx */ +/* including simdjson/generic/ondemand/object_iterator.h for lsx: #include "simdjson/generic/ondemand/object_iterator.h" */ +/* begin file simdjson/generic/ondemand/object_iterator.h for lsx */ +#ifndef SIMDJSON_GENERIC_ONDEMAND_OBJECT_ITERATOR_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_ONDEMAND_OBJECT_ITERATOR_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/implementation_simdjson_result_base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/value_iterator.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lsx { + namespace ondemand { + + class object_iterator { + public: + /** + * Create a new invalid object_iterator. + * + * Exists so you can declare a variable and later assign to it before use. + */ + simdjson_inline object_iterator() noexcept = default; + + // + // Iterator interface + // + + // Reads key and value, yielding them to the user. + // MUST ONLY BE CALLED ONCE PER ITERATION. + simdjson_inline simdjson_result operator*() noexcept; + // Assumes it's being compared with the end. true if depth < iter->depth. + simdjson_inline bool operator==(const object_iterator&) const noexcept; + // Assumes it's being compared with the end. true if depth >= iter->depth. + simdjson_inline bool operator!=(const object_iterator&) const noexcept; + // Checks for ']' and ',' + simdjson_inline object_iterator& operator++() noexcept; + + private: + /** + * The underlying JSON iterator. + * + * PERF NOTE: expected to be elided in favor of the parent document: this is set when the object + * is first used, and never changes afterwards. + */ + value_iterator iter{}; + + simdjson_inline object_iterator(const value_iterator& iter) noexcept; + friend struct simdjson_result; + friend class object; + }; + + } // namespace ondemand + } // namespace lsx +} // namespace simdjson + +namespace simdjson { + + template<> + struct simdjson_result : public lsx::implementation_simdjson_result_base { + public: + simdjson_inline simdjson_result(lsx::ondemand::object_iterator&& value) noexcept; ///< @private + simdjson_inline simdjson_result(error_code error) noexcept; ///< @private + simdjson_inline simdjson_result() noexcept = default; + + // + // Iterator interface + // + + // Reads key and value, yielding them to the user. + simdjson_inline simdjson_result operator*() noexcept; // MUST ONLY BE CALLED ONCE PER ITERATION. + // Assumes it's being compared with the end. true if depth < iter->depth. + simdjson_inline bool operator==(const simdjson_result&) const noexcept; + // Assumes it's being compared with the end. true if depth >= iter->depth. + simdjson_inline bool operator!=(const simdjson_result&) const noexcept; + // Checks for ']' and ',' + simdjson_inline simdjson_result& operator++() noexcept; + }; + +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_ONDEMAND_OBJECT_ITERATOR_H +/* end file simdjson/generic/ondemand/object_iterator.h for lsx */ +/* including simdjson/generic/ondemand/serialization.h for lsx: #include "simdjson/generic/ondemand/serialization.h" */ +/* begin file simdjson/generic/ondemand/serialization.h for lsx */ +#ifndef SIMDJSON_GENERIC_ONDEMAND_SERIALIZATION_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_ONDEMAND_SERIALIZATION_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/base.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + /** + * Create a string-view instance out of a document instance. The string-view instance + * contains JSON text that is suitable to be parsed as JSON again. It does not + * validate the content. + */ + inline simdjson_result to_json_string(lsx::ondemand::document& x) noexcept; + /** + * Create a string-view instance out of a value instance. The string-view instance + * contains JSON text that is suitable to be parsed as JSON again. The value must + * not have been accessed previously. It does not + * validate the content. + */ + inline simdjson_result to_json_string(lsx::ondemand::value& x) noexcept; + /** + * Create a string-view instance out of an object instance. The string-view instance + * contains JSON text that is suitable to be parsed as JSON again. It does not + * validate the content. + */ + inline simdjson_result to_json_string(lsx::ondemand::object& x) noexcept; + /** + * Create a string-view instance out of an array instance. The string-view instance + * contains JSON text that is suitable to be parsed as JSON again. It does not + * validate the content. + */ + inline simdjson_result to_json_string(lsx::ondemand::array& x) noexcept; + inline simdjson_result to_json_string(simdjson_result x); + inline simdjson_result to_json_string(simdjson_result x); + inline simdjson_result to_json_string(simdjson_result x); + inline simdjson_result to_json_string(simdjson_result x); +} // namespace simdjson + +/** + * We want to support argument-dependent lookup (ADL). + * Hence we should define operator<< in the namespace + * where the argument (here value, object, etc.) resides. + * Credit: @madhur4127 + * See https://github.com/simdjson/simdjson/issues/1768 + */ +namespace simdjson { + namespace lsx { + namespace ondemand { + + /** + * Print JSON to an output stream. It does not + * validate the content. + * + * @param out The output stream. + * @param value The element. + * @throw if there is an error with the underlying output stream. simdjson itself will not throw. + */ + inline std::ostream& operator<<(std::ostream& out, simdjson::lsx::ondemand::value x); +#if SIMDJSON_EXCEPTIONS + inline std::ostream& operator<<(std::ostream& out, simdjson::simdjson_result x); +#endif + /** + * Print JSON to an output stream. It does not + * validate the content. + * + * @param out The output stream. + * @param value The array. + * @throw if there is an error with the underlying output stream. simdjson itself will not throw. + */ + inline std::ostream& operator<<(std::ostream& out, simdjson::lsx::ondemand::array value); +#if SIMDJSON_EXCEPTIONS + inline std::ostream& operator<<(std::ostream& out, simdjson::simdjson_result x); +#endif + /** + * Print JSON to an output stream. It does not + * validate the content. + * + * @param out The output stream. + * @param value The array. + * @throw if there is an error with the underlying output stream. simdjson itself will not throw. + */ + inline std::ostream& operator<<(std::ostream& out, simdjson::lsx::ondemand::document& value); +#if SIMDJSON_EXCEPTIONS + inline std::ostream& operator<<(std::ostream& out, simdjson::simdjson_result&& x); +#endif + inline std::ostream& operator<<(std::ostream& out, simdjson::lsx::ondemand::document_reference& value); +#if SIMDJSON_EXCEPTIONS + inline std::ostream& operator<<(std::ostream& out, simdjson::simdjson_result&& x); +#endif + /** + * Print JSON to an output stream. It does not + * validate the content. + * + * @param out The output stream. + * @param value The object. + * @throw if there is an error with the underlying output stream. simdjson itself will not throw. + */ + inline std::ostream& operator<<(std::ostream& out, simdjson::lsx::ondemand::object value); +#if SIMDJSON_EXCEPTIONS + inline std::ostream& operator<<(std::ostream& out, simdjson::simdjson_result x); +#endif + } + } +} // namespace simdjson::lsx::ondemand + +#endif // SIMDJSON_GENERIC_ONDEMAND_SERIALIZATION_H +/* end file simdjson/generic/ondemand/serialization.h for lsx */ + +// Inline definitions +/* including simdjson/generic/ondemand/array-inl.h for lsx: #include "simdjson/generic/ondemand/array-inl.h" */ +/* begin file simdjson/generic/ondemand/array-inl.h for lsx */ +#ifndef SIMDJSON_GENERIC_ONDEMAND_ARRAY_INL_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_ONDEMAND_ARRAY_INL_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/array.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/array_iterator-inl.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/json_iterator.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/value.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/value_iterator-inl.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lsx { + namespace ondemand { + + // + // ### Live States + // + // While iterating or looking up values, depth >= iter->depth. at_start may vary. Error is + // always SUCCESS: + // + // - Start: This is the state when the array is first found and the iterator is just past the `{`. + // In this state, at_start == true. + // - Next: After we hand a scalar value to the user, or an array/object which they then fully + // iterate over, the iterator is at the `,` before the next value (or `]`). In this state, + // depth == iter->depth, at_start == false, and error == SUCCESS. + // - Unfinished Business: When we hand an array/object to the user which they do not fully + // iterate over, we need to finish that iteration by skipping child values until we reach the + // Next state. In this state, depth > iter->depth, at_start == false, and error == SUCCESS. + // + // ## Error States + // + // In error states, we will yield exactly one more value before stopping. iter->depth == depth + // and at_start is always false. We decrement after yielding the error, moving to the Finished + // state. + // + // - Chained Error: When the array iterator is part of an error chain--for example, in + // `for (auto tweet : doc["tweets"])`, where the tweet element may be missing or not be an + // array--we yield that error in the loop, exactly once. In this state, error != SUCCESS and + // iter->depth == depth, and at_start == false. We decrement depth when we yield the error. + // - Missing Comma Error: When the iterator ++ method discovers there is no comma between elements, + // we flag that as an error and treat it exactly the same as a Chained Error. In this state, + // error == TAPE_ERROR, iter->depth == depth, and at_start == false. + // + // ## Terminal State + // + // The terminal state has iter->depth < depth. at_start is always false. + // + // - Finished: When we have reached a `]` or have reported an error, we are finished. We signal this + // by decrementing depth. In this state, iter->depth < depth, at_start == false, and + // error == SUCCESS. + // + + simdjson_inline array::array(const value_iterator& _iter) noexcept + : iter{ _iter } + { + } + + simdjson_inline simdjson_result array::start(value_iterator& iter) noexcept { + // We don't need to know if the array is empty to start iteration, but we do want to know if there + // is an error--thus `simdjson_unused`. + simdjson_unused bool has_value; + SIMDJSON_TRY(iter.start_array().get(has_value)); + return array(iter); + } + simdjson_inline simdjson_result array::start_root(value_iterator& iter) noexcept { + simdjson_unused bool has_value; + SIMDJSON_TRY(iter.start_root_array().get(has_value)); + return array(iter); + } + simdjson_inline simdjson_result array::started(value_iterator& iter) noexcept { + bool has_value; + SIMDJSON_TRY(iter.started_array().get(has_value)); + return array(iter); + } + + simdjson_inline simdjson_result array::begin() noexcept { +#if SIMDJSON_DEVELOPMENT_CHECKS + if (!iter.is_at_iterator_start()) { return OUT_OF_ORDER_ITERATION; } +#endif + return array_iterator(iter); + } + simdjson_inline simdjson_result array::end() noexcept { + return array_iterator(iter); + } + simdjson_inline error_code array::consume() noexcept { + auto error = iter.json_iter().skip_child(iter.depth() - 1); + if (error) { iter.abandon(); } + return error; + } + + simdjson_inline simdjson_result array::raw_json() noexcept { + const uint8_t* starting_point{ iter.peek_start() }; + auto error = consume(); + if (error) { return error; } + // After 'consume()', we could be left pointing just beyond the document, but that + // is ok because we are not going to dereference the final pointer position, we just + // use it to compute the length in bytes. + const uint8_t* final_point{ iter._json_iter->unsafe_pointer() }; + return std::string_view(reinterpret_cast(starting_point), size_t(final_point - starting_point)); + } + + SIMDJSON_PUSH_DISABLE_WARNINGS + SIMDJSON_DISABLE_STRICT_OVERFLOW_WARNING + simdjson_inline simdjson_result array::count_elements() & noexcept { + size_t count{ 0 }; + // Important: we do not consume any of the values. + for (simdjson_unused auto v : *this) { count++; } + // The above loop will always succeed, but we want to report errors. + if (iter.error()) { return iter.error(); } + // We need to move back at the start because we expect users to iterate through + // the array after counting the number of elements. + iter.reset_array(); + return count; + } + SIMDJSON_POP_DISABLE_WARNINGS + + simdjson_inline simdjson_result array::is_empty() & noexcept { + bool is_not_empty; + auto error = iter.reset_array().get(is_not_empty); + if (error) { return error; } + return !is_not_empty; + } + + inline simdjson_result array::reset() & noexcept { + return iter.reset_array(); + } + + inline simdjson_result array::at_pointer(std::string_view json_pointer) noexcept { + if (json_pointer[0] != '/') { return INVALID_JSON_POINTER; } + json_pointer = json_pointer.substr(1); + // - means "the append position" or "the element after the end of the array" + // We don't support this, because we're returning a real element, not a position. + if (json_pointer == "-") { return INDEX_OUT_OF_BOUNDS; } + + // Read the array index + size_t array_index = 0; + size_t i; + for (i = 0; i < json_pointer.length() && json_pointer[i] != '/'; i++) { + uint8_t digit = uint8_t(json_pointer[i] - '0'); + // Check for non-digit in array index. If it's there, we're trying to get a field in an object + if (digit > 9) { return INCORRECT_TYPE; } + array_index = array_index * 10 + digit; + } + + // 0 followed by other digits is invalid + if (i > 1 && json_pointer[0] == '0') { return INVALID_JSON_POINTER; } // "JSON pointer array index has other characters after 0" + + // Empty string is invalid; so is a "/" with no digits before it + if (i == 0) { return INVALID_JSON_POINTER; } // "Empty string in JSON pointer array index" + // Get the child + auto child = at(array_index); + // If there is an error, it ends here + if (child.error()) { + return child; + } + + // If there is a /, we're not done yet, call recursively. + if (i < json_pointer.length()) { + child = child.at_pointer(json_pointer.substr(i)); + } + return child; + } + + inline std::string json_path_to_pointer_conversion(std::string_view json_path) { + if (json_path.empty() || (json_path.front() != '.' && + json_path.front() != '[')) { + return "-1"; // This is just a sentinel value, the caller should check for this and return an error. + } + + std::string result; + // Reserve space to reduce allocations, adjusting for potential increases due + // to escaping. + result.reserve(json_path.size() * 2); + + size_t i = 0; + + while (i < json_path.length()) { + if (json_path[i] == '.') { + result += '/'; + } + else if (json_path[i] == '[') { + result += '/'; + ++i; // Move past the '[' + while (i < json_path.length() && json_path[i] != ']') { + if (json_path[i] == '~') { + result += "~0"; + } + else if (json_path[i] == '/') { + result += "~1"; + } + else { + result += json_path[i]; + } + ++i; + } + if (i == json_path.length() || json_path[i] != ']') { + return "-1"; // Using sentinel value that will be handled as an error by the caller. + } + } + else { + if (json_path[i] == '~') { + result += "~0"; + } + else if (json_path[i] == '/') { + result += "~1"; + } + else { + result += json_path[i]; + } + } + ++i; + } + + return result; + } + + inline simdjson_result array::at_path(std::string_view json_path) noexcept { + auto json_pointer = json_path_to_pointer_conversion(json_path); + if (json_pointer == "-1") { return INVALID_JSON_POINTER; } + return at_pointer(json_pointer); + } + + simdjson_inline simdjson_result array::at(size_t index) noexcept { + size_t i = 0; + for (auto value : *this) { + if (i == index) { return value; } + i++; + } + return INDEX_OUT_OF_BOUNDS; + } + + } // namespace ondemand + } // namespace lsx +} // namespace simdjson + +namespace simdjson { + + simdjson_inline simdjson_result::simdjson_result( + lsx::ondemand::array&& value + ) noexcept + : implementation_simdjson_result_base( + std::forward(value) + ) + { + } + simdjson_inline simdjson_result::simdjson_result( + error_code error + ) noexcept + : implementation_simdjson_result_base(error) + { + } + + simdjson_inline simdjson_result simdjson_result::begin() noexcept { + if (error()) { return error(); } + return first.begin(); + } + simdjson_inline simdjson_result simdjson_result::end() noexcept { + if (error()) { return error(); } + return first.end(); + } + simdjson_inline simdjson_result simdjson_result::count_elements() & noexcept { + if (error()) { return error(); } + return first.count_elements(); + } + simdjson_inline simdjson_result simdjson_result::is_empty() & noexcept { + if (error()) { return error(); } + return first.is_empty(); + } + simdjson_inline simdjson_result simdjson_result::at(size_t index) noexcept { + if (error()) { return error(); } + return first.at(index); + } + simdjson_inline simdjson_result simdjson_result::at_pointer(std::string_view json_pointer) noexcept { + if (error()) { return error(); } + return first.at_pointer(json_pointer); + } + simdjson_inline simdjson_result simdjson_result::at_path(std::string_view json_path) noexcept { + if (error()) { return error(); } + return first.at_path(json_path); + } + simdjson_inline simdjson_result simdjson_result::raw_json() noexcept { + if (error()) { return error(); } + return first.raw_json(); + } +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_ONDEMAND_ARRAY_INL_H +/* end file simdjson/generic/ondemand/array-inl.h for lsx */ +/* including simdjson/generic/ondemand/array_iterator-inl.h for lsx: #include "simdjson/generic/ondemand/array_iterator-inl.h" */ +/* begin file simdjson/generic/ondemand/array_iterator-inl.h for lsx */ +#ifndef SIMDJSON_GENERIC_ONDEMAND_ARRAY_ITERATOR_INL_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_ONDEMAND_ARRAY_ITERATOR_INL_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/array_iterator.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/value-inl.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/value_iterator-inl.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lsx { + namespace ondemand { + + simdjson_inline array_iterator::array_iterator(const value_iterator& _iter) noexcept + : iter{ _iter } + {} + + simdjson_inline simdjson_result array_iterator::operator*() noexcept { + if (iter.error()) { iter.abandon(); return iter.error(); } + return value(iter.child()); + } + simdjson_inline bool array_iterator::operator==(const array_iterator& other) const noexcept { + return !(*this != other); + } + simdjson_inline bool array_iterator::operator!=(const array_iterator&) const noexcept { + return iter.is_open(); + } + simdjson_inline array_iterator& array_iterator::operator++() noexcept { + error_code error; + // PERF NOTE this is a safety rail ... users should exit loops as soon as they receive an error, so we'll never get here. + // However, it does not seem to make a perf difference, so we add it out of an abundance of caution. + if ((error = iter.error())) { return *this; } + if ((error = iter.skip_child())) { return *this; } + if ((error = iter.has_next_element().error())) { return *this; } + return *this; + } + + } // namespace ondemand + } // namespace lsx +} // namespace simdjson + +namespace simdjson { + + simdjson_inline simdjson_result::simdjson_result( + lsx::ondemand::array_iterator&& value + ) noexcept + : lsx::implementation_simdjson_result_base(std::forward(value)) + { + first.iter.assert_is_valid(); + } + simdjson_inline simdjson_result::simdjson_result(error_code error) noexcept + : lsx::implementation_simdjson_result_base({}, error) + { + } + + simdjson_inline simdjson_result simdjson_result::operator*() noexcept { + if (error()) { return error(); } + return *first; + } + simdjson_inline bool simdjson_result::operator==(const simdjson_result& other) const noexcept { + if (!first.iter.is_valid()) { return !error(); } + return first == other.first; + } + simdjson_inline bool simdjson_result::operator!=(const simdjson_result& other) const noexcept { + if (!first.iter.is_valid()) { return error(); } + return first != other.first; + } + simdjson_inline simdjson_result& simdjson_result::operator++() noexcept { + // Clear the error if there is one, so we don't yield it twice + if (error()) { second = SUCCESS; return *this; } + ++(first); + return *this; + } + +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_ONDEMAND_ARRAY_ITERATOR_INL_H +/* end file simdjson/generic/ondemand/array_iterator-inl.h for lsx */ +/* including simdjson/generic/ondemand/document-inl.h for lsx: #include "simdjson/generic/ondemand/document-inl.h" */ +/* begin file simdjson/generic/ondemand/document-inl.h for lsx */ +#ifndef SIMDJSON_GENERIC_ONDEMAND_DOCUMENT_INL_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_ONDEMAND_DOCUMENT_INL_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/array_iterator.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/document.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/json_type.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/raw_json_string.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/value.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/array-inl.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/json_iterator-inl.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/object-inl.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/value_iterator-inl.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lsx { + namespace ondemand { + + simdjson_inline document::document(ondemand::json_iterator&& _iter) noexcept + : iter{ std::forward(_iter) } + { + logger::log_start_value(iter, "document"); + } + + simdjson_inline document document::start(json_iterator&& iter) noexcept { + return document(std::forward(iter)); + } + + inline void document::rewind() noexcept { + iter.rewind(); + } + + inline std::string document::to_debug_string() noexcept { + return iter.to_string(); + } + + inline simdjson_result document::current_location() const noexcept { + return iter.current_location(); + } + + inline int32_t document::current_depth() const noexcept { + return iter.depth(); + } + + inline bool document::at_end() const noexcept { + return iter.at_end(); + } + + + inline bool document::is_alive() noexcept { + return iter.is_alive(); + } + simdjson_inline value_iterator document::resume_value_iterator() noexcept { + return value_iterator(&iter, 1, iter.root_position()); + } + simdjson_inline value_iterator document::get_root_value_iterator() noexcept { + return resume_value_iterator(); + } + simdjson_inline simdjson_result document::start_or_resume_object() noexcept { + if (iter.at_root()) { + return get_object(); + } + else { + return object::resume(resume_value_iterator()); + } + } + simdjson_inline simdjson_result document::get_value() noexcept { + // Make sure we start any arrays or objects before returning, so that start_root_() + // gets called. + + // It is the convention throughout the code that the macro `SIMDJSON_DEVELOPMENT_CHECKS` determines whether + // we check for OUT_OF_ORDER_ITERATION. Proper on::demand code should never trigger this error. +#if SIMDJSON_DEVELOPMENT_CHECKS + if (!iter.at_root()) { return OUT_OF_ORDER_ITERATION; } +#endif + // assert_at_root() serves two purposes: in Debug mode, whether or not + // SIMDJSON_DEVELOPMENT_CHECKS is set or not, it checks that we are at the root of + // the document (this will typically be redundant). In release mode, it generates + // SIMDJSON_ASSUME statements to allow the compiler to make assumptions. + iter.assert_at_root(); + switch (*iter.peek()) { + case '[': { + // The following lines check that the document ends with ]. + auto value_iterator = get_root_value_iterator(); + auto error = value_iterator.check_root_array(); + if (error) { return error; } + return value(get_root_value_iterator()); + } + case '{': { + // The following lines would check that the document ends with }. + auto value_iterator = get_root_value_iterator(); + auto error = value_iterator.check_root_object(); + if (error) { return error; } + return value(get_root_value_iterator()); + } + default: + // Unfortunately, scalar documents are a special case in simdjson and they cannot + // be safely converted to value instances. + return SCALAR_DOCUMENT_AS_VALUE; + } + } + simdjson_inline simdjson_result document::get_array() & noexcept { + auto value = get_root_value_iterator(); + return array::start_root(value); + } + simdjson_inline simdjson_result document::get_object() & noexcept { + auto value = get_root_value_iterator(); + return object::start_root(value); + } + + /** + * We decided that calling 'get_double()' on the JSON document '1.233 blabla' should + * give an error, so we check for trailing content. We want to disallow trailing + * content. + * Thus, in several implementations below, we pass a 'true' parameter value to + * a get_root_value_iterator() method: this indicates that we disallow trailing content. + */ + + simdjson_inline simdjson_result document::get_uint64() noexcept { + return get_root_value_iterator().get_root_uint64(true); + } + simdjson_inline simdjson_result document::get_uint64_in_string() noexcept { + return get_root_value_iterator().get_root_uint64_in_string(true); + } + simdjson_inline simdjson_result document::get_int64() noexcept { + return get_root_value_iterator().get_root_int64(true); + } + simdjson_inline simdjson_result document::get_int64_in_string() noexcept { + return get_root_value_iterator().get_root_int64_in_string(true); + } + simdjson_inline simdjson_result document::get_double() noexcept { + return get_root_value_iterator().get_root_double(true); + } + simdjson_inline simdjson_result document::get_double_in_string() noexcept { + return get_root_value_iterator().get_root_double_in_string(true); + } + simdjson_inline simdjson_result document::get_string(bool allow_replacement) noexcept { + return get_root_value_iterator().get_root_string(true, allow_replacement); + } + template + simdjson_inline error_code document::get_string(string_type& receiver, bool allow_replacement) noexcept { + return get_root_value_iterator().get_root_string(receiver, true, allow_replacement); + } + simdjson_inline simdjson_result document::get_wobbly_string() noexcept { + return get_root_value_iterator().get_root_wobbly_string(true); + } + simdjson_inline simdjson_result document::get_raw_json_string() noexcept { + return get_root_value_iterator().get_root_raw_json_string(true); + } + simdjson_inline simdjson_result document::get_bool() noexcept { + return get_root_value_iterator().get_root_bool(true); + } + simdjson_inline simdjson_result document::is_null() noexcept { + return get_root_value_iterator().is_root_null(true); + } + + template<> simdjson_inline simdjson_result document::get() & noexcept { return get_array(); } + template<> simdjson_inline simdjson_result document::get() & noexcept { return get_object(); } + template<> simdjson_inline simdjson_result document::get() & noexcept { return get_raw_json_string(); } + template<> simdjson_inline simdjson_result document::get() & noexcept { return get_string(false); } + template<> simdjson_inline simdjson_result document::get() & noexcept { return get_double(); } + template<> simdjson_inline simdjson_result document::get() & noexcept { return get_uint64(); } + template<> simdjson_inline simdjson_result document::get() & noexcept { return get_int64(); } + template<> simdjson_inline simdjson_result document::get() & noexcept { return get_bool(); } + template<> simdjson_inline simdjson_result document::get() & noexcept { return get_value(); } + + template<> simdjson_deprecated simdjson_inline simdjson_result document::get() && noexcept { return get_raw_json_string(); } + template<> simdjson_deprecated simdjson_inline simdjson_result document::get() && noexcept { return get_string(false); } + template<> simdjson_deprecated simdjson_inline simdjson_result document::get() && noexcept { return std::forward(*this).get_double(); } + template<> simdjson_deprecated simdjson_inline simdjson_result document::get() && noexcept { return std::forward(*this).get_uint64(); } + template<> simdjson_deprecated simdjson_inline simdjson_result document::get() && noexcept { return std::forward(*this).get_int64(); } + template<> simdjson_deprecated simdjson_inline simdjson_result document::get() && noexcept { return std::forward(*this).get_bool(); } + template<> simdjson_deprecated simdjson_inline simdjson_result document::get() && noexcept { return get_value(); } + + template simdjson_inline error_code document::get(T& out) & noexcept { + return get().get(out); + } + template simdjson_deprecated simdjson_inline error_code document::get(T& out) && noexcept { + return std::forward(*this).get().get(out); + } + +#if SIMDJSON_EXCEPTIONS + template + simdjson_deprecated simdjson_inline document::operator T() && noexcept(false) { return get(); } + template + simdjson_inline document::operator T() & noexcept(false) { return get(); } + simdjson_inline document::operator array() & noexcept(false) { return get_array(); } + simdjson_inline document::operator object() & noexcept(false) { return get_object(); } + simdjson_inline document::operator uint64_t() noexcept(false) { return get_uint64(); } + simdjson_inline document::operator int64_t() noexcept(false) { return get_int64(); } + simdjson_inline document::operator double() noexcept(false) { return get_double(); } + simdjson_inline document::operator std::string_view() noexcept(false) { return get_string(false); } + simdjson_inline document::operator raw_json_string() noexcept(false) { return get_raw_json_string(); } + simdjson_inline document::operator bool() noexcept(false) { return get_bool(); } + simdjson_inline document::operator value() noexcept(false) { return get_value(); } + +#endif + simdjson_inline simdjson_result document::count_elements() & noexcept { + auto a = get_array(); + simdjson_result answer = a.count_elements(); + /* If there was an array, we are now left pointing at its first element. */ + if (answer.error() == SUCCESS) { rewind(); } + return answer; + } + simdjson_inline simdjson_result document::count_fields() & noexcept { + auto a = get_object(); + simdjson_result answer = a.count_fields(); + /* If there was an object, we are now left pointing at its first element. */ + if (answer.error() == SUCCESS) { rewind(); } + return answer; + } + simdjson_inline simdjson_result document::at(size_t index) & noexcept { + auto a = get_array(); + return a.at(index); + } + simdjson_inline simdjson_result document::begin() & noexcept { + return get_array().begin(); + } + simdjson_inline simdjson_result document::end() & noexcept { + return {}; + } + + simdjson_inline simdjson_result document::find_field(std::string_view key) & noexcept { + return start_or_resume_object().find_field(key); + } + simdjson_inline simdjson_result document::find_field(const char* key) & noexcept { + return start_or_resume_object().find_field(key); + } + simdjson_inline simdjson_result document::find_field_unordered(std::string_view key) & noexcept { + return start_or_resume_object().find_field_unordered(key); + } + simdjson_inline simdjson_result document::find_field_unordered(const char* key) & noexcept { + return start_or_resume_object().find_field_unordered(key); + } + simdjson_inline simdjson_result document::operator[](std::string_view key) & noexcept { + return start_or_resume_object()[key]; + } + simdjson_inline simdjson_result document::operator[](const char* key) & noexcept { + return start_or_resume_object()[key]; + } + + simdjson_inline error_code document::consume() noexcept { + auto error = iter.skip_child(0); + if (error) { iter.abandon(); } + return error; + } + + simdjson_inline simdjson_result document::raw_json() noexcept { + auto _iter = get_root_value_iterator(); + const uint8_t* starting_point{ _iter.peek_start() }; + auto error = consume(); + if (error) { return error; } + // After 'consume()', we could be left pointing just beyond the document, but that + // is ok because we are not going to dereference the final pointer position, we just + // use it to compute the length in bytes. + const uint8_t* final_point{ iter.unsafe_pointer() }; + return std::string_view(reinterpret_cast(starting_point), size_t(final_point - starting_point)); + } + + simdjson_inline simdjson_result document::type() noexcept { + return get_root_value_iterator().type(); + } + + simdjson_inline simdjson_result document::is_scalar() noexcept { + json_type this_type; + auto error = type().get(this_type); + if (error) { return error; } + return !((this_type == json_type::array) || (this_type == json_type::object)); + } + + simdjson_inline simdjson_result document::is_string() noexcept { + json_type this_type; + auto error = type().get(this_type); + if (error) { return error; } + return (this_type == json_type::string); + } + + simdjson_inline bool document::is_negative() noexcept { + return get_root_value_iterator().is_root_negative(); + } + + simdjson_inline simdjson_result document::is_integer() noexcept { + return get_root_value_iterator().is_root_integer(true); + } + + simdjson_inline simdjson_result document::get_number_type() noexcept { + return get_root_value_iterator().get_root_number_type(true); + } + + simdjson_inline simdjson_result document::get_number() noexcept { + return get_root_value_iterator().get_root_number(true); + } + + + simdjson_inline simdjson_result document::raw_json_token() noexcept { + auto _iter = get_root_value_iterator(); + return std::string_view(reinterpret_cast(_iter.peek_start()), _iter.peek_root_length()); + } + + simdjson_inline simdjson_result document::at_pointer(std::string_view json_pointer) noexcept { + rewind(); // Rewind the document each time at_pointer is called + if (json_pointer.empty()) { + return this->get_value(); + } + json_type t; + SIMDJSON_TRY(type().get(t)); + switch (t) + { + case json_type::array: + return (*this).get_array().at_pointer(json_pointer); + case json_type::object: + return (*this).get_object().at_pointer(json_pointer); + default: + return INVALID_JSON_POINTER; + } + } + + simdjson_inline simdjson_result document::at_path(std::string_view json_path) noexcept { + rewind(); // Rewind the document each time at_pointer is called + if (json_path.empty()) { + return this->get_value(); + } + json_type t; + SIMDJSON_TRY(type().get(t)); + switch (t) { + case json_type::array: + return (*this).get_array().at_path(json_path); + case json_type::object: + return (*this).get_object().at_path(json_path); + default: + return INVALID_JSON_POINTER; + } + } + + } // namespace ondemand + } // namespace lsx +} // namespace simdjson + +namespace simdjson { + + simdjson_inline simdjson_result::simdjson_result( + lsx::ondemand::document&& value + ) noexcept : + implementation_simdjson_result_base( + std::forward(value) + ) + { + } + simdjson_inline simdjson_result::simdjson_result( + error_code error + ) noexcept : + implementation_simdjson_result_base( + error + ) + { + } + simdjson_inline simdjson_result simdjson_result::count_elements() & noexcept { + if (error()) { return error(); } + return first.count_elements(); + } + simdjson_inline simdjson_result simdjson_result::count_fields() & noexcept { + if (error()) { return error(); } + return first.count_fields(); + } + simdjson_inline simdjson_result simdjson_result::at(size_t index) & noexcept { + if (error()) { return error(); } + return first.at(index); + } + simdjson_inline error_code simdjson_result::rewind() noexcept { + if (error()) { return error(); } + first.rewind(); + return SUCCESS; + } + simdjson_inline simdjson_result simdjson_result::begin() & noexcept { + if (error()) { return error(); } + return first.begin(); + } + simdjson_inline simdjson_result simdjson_result::end() & noexcept { + return {}; + } + simdjson_inline simdjson_result simdjson_result::find_field_unordered(std::string_view key) & noexcept { + if (error()) { return error(); } + return first.find_field_unordered(key); + } + simdjson_inline simdjson_result simdjson_result::find_field_unordered(const char* key) & noexcept { + if (error()) { return error(); } + return first.find_field_unordered(key); + } + simdjson_inline simdjson_result simdjson_result::operator[](std::string_view key) & noexcept { + if (error()) { return error(); } + return first[key]; + } + simdjson_inline simdjson_result simdjson_result::operator[](const char* key) & noexcept { + if (error()) { return error(); } + return first[key]; + } + simdjson_inline simdjson_result simdjson_result::find_field(std::string_view key) & noexcept { + if (error()) { return error(); } + return first.find_field(key); + } + simdjson_inline simdjson_result simdjson_result::find_field(const char* key) & noexcept { + if (error()) { return error(); } + return first.find_field(key); + } + simdjson_inline simdjson_result simdjson_result::get_array() & noexcept { + if (error()) { return error(); } + return first.get_array(); + } + simdjson_inline simdjson_result simdjson_result::get_object() & noexcept { + if (error()) { return error(); } + return first.get_object(); + } + simdjson_inline simdjson_result simdjson_result::get_uint64() noexcept { + if (error()) { return error(); } + return first.get_uint64(); + } + simdjson_inline simdjson_result simdjson_result::get_uint64_in_string() noexcept { + if (error()) { return error(); } + return first.get_uint64_in_string(); + } + simdjson_inline simdjson_result simdjson_result::get_int64() noexcept { + if (error()) { return error(); } + return first.get_int64(); + } + simdjson_inline simdjson_result simdjson_result::get_int64_in_string() noexcept { + if (error()) { return error(); } + return first.get_int64_in_string(); + } + simdjson_inline simdjson_result simdjson_result::get_double() noexcept { + if (error()) { return error(); } + return first.get_double(); + } + simdjson_inline simdjson_result simdjson_result::get_double_in_string() noexcept { + if (error()) { return error(); } + return first.get_double_in_string(); + } + simdjson_inline simdjson_result simdjson_result::get_string(bool allow_replacement) noexcept { + if (error()) { return error(); } + return first.get_string(allow_replacement); + } + template + simdjson_inline error_code simdjson_result::get_string(string_type& receiver, bool allow_replacement) noexcept { + if (error()) { return error(); } + return first.get_string(receiver, allow_replacement); + } + simdjson_inline simdjson_result simdjson_result::get_wobbly_string() noexcept { + if (error()) { return error(); } + return first.get_wobbly_string(); + } + simdjson_inline simdjson_result simdjson_result::get_raw_json_string() noexcept { + if (error()) { return error(); } + return first.get_raw_json_string(); + } + simdjson_inline simdjson_result simdjson_result::get_bool() noexcept { + if (error()) { return error(); } + return first.get_bool(); + } + simdjson_inline simdjson_result simdjson_result::get_value() noexcept { + if (error()) { return error(); } + return first.get_value(); + } + simdjson_inline simdjson_result simdjson_result::is_null() noexcept { + if (error()) { return error(); } + return first.is_null(); + } + + template + simdjson_inline simdjson_result simdjson_result::get() & noexcept { + if (error()) { return error(); } + return first.get(); + } + template + simdjson_deprecated simdjson_inline simdjson_result simdjson_result::get() && noexcept { + if (error()) { return error(); } + return std::forward(first).get(); + } + template + simdjson_inline error_code simdjson_result::get(T& out) & noexcept { + if (error()) { return error(); } + return first.get(out); + } + template + simdjson_inline error_code simdjson_result::get(T& out) && noexcept { + if (error()) { return error(); } + return std::forward(first).get(out); + } + + template<> simdjson_inline simdjson_result simdjson_result::get() & noexcept = delete; + template<> simdjson_deprecated simdjson_inline simdjson_result simdjson_result::get() && noexcept { + if (error()) { return error(); } + return std::forward(first); + } + template<> simdjson_inline error_code simdjson_result::get(lsx::ondemand::document& out) & noexcept = delete; + template<> simdjson_inline error_code simdjson_result::get(lsx::ondemand::document& out) && noexcept { + if (error()) { return error(); } + out = std::forward(first); + return SUCCESS; + } + + simdjson_inline simdjson_result simdjson_result::type() noexcept { + if (error()) { return error(); } + return first.type(); + } + + simdjson_inline simdjson_result simdjson_result::is_scalar() noexcept { + if (error()) { return error(); } + return first.is_scalar(); + } + + simdjson_inline simdjson_result simdjson_result::is_string() noexcept { + if (error()) { return error(); } + return first.is_string(); + } + + simdjson_inline bool simdjson_result::is_negative() noexcept { + if (error()) { return error(); } + return first.is_negative(); + } + + simdjson_inline simdjson_result simdjson_result::is_integer() noexcept { + if (error()) { return error(); } + return first.is_integer(); + } + + simdjson_inline simdjson_result simdjson_result::get_number_type() noexcept { + if (error()) { return error(); } + return first.get_number_type(); + } + + simdjson_inline simdjson_result simdjson_result::get_number() noexcept { + if (error()) { return error(); } + return first.get_number(); + } + + +#if SIMDJSON_EXCEPTIONS + template ::value == false>::type> + simdjson_inline simdjson_result::operator T() noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return first; + } + simdjson_inline simdjson_result::operator lsx::ondemand::array() & noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return first; + } + simdjson_inline simdjson_result::operator lsx::ondemand::object() & noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return first; + } + simdjson_inline simdjson_result::operator uint64_t() noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return first; + } + simdjson_inline simdjson_result::operator int64_t() noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return first; + } + simdjson_inline simdjson_result::operator double() noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return first; + } + simdjson_inline simdjson_result::operator std::string_view() noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return first; + } + simdjson_inline simdjson_result::operator lsx::ondemand::raw_json_string() noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return first; + } + simdjson_inline simdjson_result::operator bool() noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return first; + } + simdjson_inline simdjson_result::operator lsx::ondemand::value() noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return first; + } +#endif + + + simdjson_inline simdjson_result simdjson_result::current_location() noexcept { + if (error()) { return error(); } + return first.current_location(); + } + + simdjson_inline bool simdjson_result::at_end() const noexcept { + if (error()) { return error(); } + return first.at_end(); + } + + + simdjson_inline int32_t simdjson_result::current_depth() const noexcept { + if (error()) { return error(); } + return first.current_depth(); + } + + simdjson_inline simdjson_result simdjson_result::raw_json_token() noexcept { + if (error()) { return error(); } + return first.raw_json_token(); + } + + simdjson_inline simdjson_result simdjson_result::at_pointer(std::string_view json_pointer) noexcept { + if (error()) { return error(); } + return first.at_pointer(json_pointer); + } + + simdjson_inline simdjson_result simdjson_result::at_path(std::string_view json_path) noexcept { + if (error()) { return error(); } + return first.at_path(json_path); + } + +} // namespace simdjson + + +namespace simdjson { + namespace lsx { + namespace ondemand { + + simdjson_inline document_reference::document_reference() noexcept : doc{ nullptr } {} + simdjson_inline document_reference::document_reference(document& d) noexcept : doc(&d) {} + simdjson_inline void document_reference::rewind() noexcept { doc->rewind(); } + simdjson_inline simdjson_result document_reference::get_array() & noexcept { return doc->get_array(); } + simdjson_inline simdjson_result document_reference::get_object() & noexcept { return doc->get_object(); } + /** + * The document_reference instances are used primarily/solely for streams of JSON + * documents. + * We decided that calling 'get_double()' on the JSON document '1.233 blabla' should + * give an error, so we check for trailing content. + * + * However, for streams of JSON documents, we want to be able to start from + * "321" "321" "321" + * and parse it successfully as a stream of JSON documents, calling get_uint64_in_string() + * successfully each time. + * + * To achieve this result, we pass a 'false' to a get_root_value_iterator() method: + * this indicates that we allow trailing content. + */ + simdjson_inline simdjson_result document_reference::get_uint64() noexcept { return doc->get_root_value_iterator().get_root_uint64(false); } + simdjson_inline simdjson_result document_reference::get_uint64_in_string() noexcept { return doc->get_root_value_iterator().get_root_uint64_in_string(false); } + simdjson_inline simdjson_result document_reference::get_int64() noexcept { return doc->get_root_value_iterator().get_root_int64(false); } + simdjson_inline simdjson_result document_reference::get_int64_in_string() noexcept { return doc->get_root_value_iterator().get_root_int64_in_string(false); } + simdjson_inline simdjson_result document_reference::get_double() noexcept { return doc->get_root_value_iterator().get_root_double(false); } + simdjson_inline simdjson_result document_reference::get_double_in_string() noexcept { return doc->get_root_value_iterator().get_root_double(false); } + simdjson_inline simdjson_result document_reference::get_string(bool allow_replacement) noexcept { return doc->get_root_value_iterator().get_root_string(false, allow_replacement); } + template + simdjson_inline error_code document_reference::get_string(string_type& receiver, bool allow_replacement) noexcept { return doc->get_root_value_iterator().get_root_string(receiver, false, allow_replacement); } + simdjson_inline simdjson_result document_reference::get_wobbly_string() noexcept { return doc->get_root_value_iterator().get_root_wobbly_string(false); } + simdjson_inline simdjson_result document_reference::get_raw_json_string() noexcept { return doc->get_root_value_iterator().get_root_raw_json_string(false); } + simdjson_inline simdjson_result document_reference::get_bool() noexcept { return doc->get_root_value_iterator().get_root_bool(false); } + simdjson_inline simdjson_result document_reference::get_value() noexcept { return doc->get_value(); } + simdjson_inline simdjson_result document_reference::is_null() noexcept { return doc->get_root_value_iterator().is_root_null(false); } + template<> simdjson_inline simdjson_result document_reference::get() & noexcept { return get_array(); } + template<> simdjson_inline simdjson_result document_reference::get() & noexcept { return get_object(); } + template<> simdjson_inline simdjson_result document_reference::get() & noexcept { return get_raw_json_string(); } + template<> simdjson_inline simdjson_result document_reference::get() & noexcept { return get_string(false); } + template<> simdjson_inline simdjson_result document_reference::get() & noexcept { return get_double(); } + template<> simdjson_inline simdjson_result document_reference::get() & noexcept { return get_uint64(); } + template<> simdjson_inline simdjson_result document_reference::get() & noexcept { return get_int64(); } + template<> simdjson_inline simdjson_result document_reference::get() & noexcept { return get_bool(); } + template<> simdjson_inline simdjson_result document_reference::get() & noexcept { return get_value(); } +#if SIMDJSON_EXCEPTIONS + template + simdjson_inline document_reference::operator T() noexcept(false) { return get(); } + simdjson_inline document_reference::operator array() & noexcept(false) { return array(*doc); } + simdjson_inline document_reference::operator object() & noexcept(false) { return object(*doc); } + simdjson_inline document_reference::operator uint64_t() noexcept(false) { return get_uint64(); } + simdjson_inline document_reference::operator int64_t() noexcept(false) { return get_int64(); } + simdjson_inline document_reference::operator double() noexcept(false) { return get_double(); } + simdjson_inline document_reference::operator std::string_view() noexcept(false) { return std::string_view(*doc); } + simdjson_inline document_reference::operator raw_json_string() noexcept(false) { return get_raw_json_string(); } + simdjson_inline document_reference::operator bool() noexcept(false) { return get_bool(); } + simdjson_inline document_reference::operator value() noexcept(false) { return value(*doc); } +#endif + simdjson_inline simdjson_result document_reference::count_elements() & noexcept { return doc->count_elements(); } + simdjson_inline simdjson_result document_reference::count_fields() & noexcept { return doc->count_fields(); } + simdjson_inline simdjson_result document_reference::at(size_t index) & noexcept { return doc->at(index); } + simdjson_inline simdjson_result document_reference::begin() & noexcept { return doc->begin(); } + simdjson_inline simdjson_result document_reference::end() & noexcept { return doc->end(); } + simdjson_inline simdjson_result document_reference::find_field(std::string_view key) & noexcept { return doc->find_field(key); } + simdjson_inline simdjson_result document_reference::find_field(const char* key) & noexcept { return doc->find_field(key); } + simdjson_inline simdjson_result document_reference::operator[](std::string_view key) & noexcept { return (*doc)[key]; } + simdjson_inline simdjson_result document_reference::operator[](const char* key) & noexcept { return (*doc)[key]; } + simdjson_inline simdjson_result document_reference::find_field_unordered(std::string_view key) & noexcept { return doc->find_field_unordered(key); } + simdjson_inline simdjson_result document_reference::find_field_unordered(const char* key) & noexcept { return doc->find_field_unordered(key); } + simdjson_inline simdjson_result document_reference::type() noexcept { return doc->type(); } + simdjson_inline simdjson_result document_reference::is_scalar() noexcept { return doc->is_scalar(); } + simdjson_inline simdjson_result document_reference::is_string() noexcept { return doc->is_string(); } + simdjson_inline simdjson_result document_reference::current_location() noexcept { return doc->current_location(); } + simdjson_inline int32_t document_reference::current_depth() const noexcept { return doc->current_depth(); } + simdjson_inline bool document_reference::is_negative() noexcept { return doc->is_negative(); } + simdjson_inline simdjson_result document_reference::is_integer() noexcept { return doc->get_root_value_iterator().is_root_integer(false); } + simdjson_inline simdjson_result document_reference::get_number_type() noexcept { return doc->get_root_value_iterator().get_root_number_type(false); } + simdjson_inline simdjson_result document_reference::get_number() noexcept { return doc->get_root_value_iterator().get_root_number(false); } + simdjson_inline simdjson_result document_reference::raw_json_token() noexcept { return doc->raw_json_token(); } + simdjson_inline simdjson_result document_reference::at_pointer(std::string_view json_pointer) noexcept { return doc->at_pointer(json_pointer); } + simdjson_inline simdjson_result document_reference::at_path(std::string_view json_path) noexcept { return doc->at_path(json_path); } + simdjson_inline simdjson_result document_reference::raw_json() noexcept { return doc->raw_json(); } + simdjson_inline document_reference::operator document& () const noexcept { return *doc; } + + } // namespace ondemand + } // namespace lsx +} // namespace simdjson + + + +namespace simdjson { + simdjson_inline simdjson_result::simdjson_result(lsx::ondemand::document_reference value, error_code error) + noexcept : implementation_simdjson_result_base(std::forward(value), error) {} + + + simdjson_inline simdjson_result simdjson_result::count_elements() & noexcept { + if (error()) { return error(); } + return first.count_elements(); + } + simdjson_inline simdjson_result simdjson_result::count_fields() & noexcept { + if (error()) { return error(); } + return first.count_fields(); + } + simdjson_inline simdjson_result simdjson_result::at(size_t index) & noexcept { + if (error()) { return error(); } + return first.at(index); + } + simdjson_inline error_code simdjson_result::rewind() noexcept { + if (error()) { return error(); } + first.rewind(); + return SUCCESS; + } + simdjson_inline simdjson_result simdjson_result::begin() & noexcept { + if (error()) { return error(); } + return first.begin(); + } + simdjson_inline simdjson_result simdjson_result::end() & noexcept { + return {}; + } + simdjson_inline simdjson_result simdjson_result::find_field_unordered(std::string_view key) & noexcept { + if (error()) { return error(); } + return first.find_field_unordered(key); + } + simdjson_inline simdjson_result simdjson_result::find_field_unordered(const char* key) & noexcept { + if (error()) { return error(); } + return first.find_field_unordered(key); + } + simdjson_inline simdjson_result simdjson_result::operator[](std::string_view key) & noexcept { + if (error()) { return error(); } + return first[key]; + } + simdjson_inline simdjson_result simdjson_result::operator[](const char* key) & noexcept { + if (error()) { return error(); } + return first[key]; + } + simdjson_inline simdjson_result simdjson_result::find_field(std::string_view key) & noexcept { + if (error()) { return error(); } + return first.find_field(key); + } + simdjson_inline simdjson_result simdjson_result::find_field(const char* key) & noexcept { + if (error()) { return error(); } + return first.find_field(key); + } + simdjson_inline simdjson_result simdjson_result::get_array() & noexcept { + if (error()) { return error(); } + return first.get_array(); + } + simdjson_inline simdjson_result simdjson_result::get_object() & noexcept { + if (error()) { return error(); } + return first.get_object(); + } + simdjson_inline simdjson_result simdjson_result::get_uint64() noexcept { + if (error()) { return error(); } + return first.get_uint64(); + } + simdjson_inline simdjson_result simdjson_result::get_uint64_in_string() noexcept { + if (error()) { return error(); } + return first.get_uint64_in_string(); + } + simdjson_inline simdjson_result simdjson_result::get_int64() noexcept { + if (error()) { return error(); } + return first.get_int64(); + } + simdjson_inline simdjson_result simdjson_result::get_int64_in_string() noexcept { + if (error()) { return error(); } + return first.get_int64_in_string(); + } + simdjson_inline simdjson_result simdjson_result::get_double() noexcept { + if (error()) { return error(); } + return first.get_double(); + } + simdjson_inline simdjson_result simdjson_result::get_double_in_string() noexcept { + if (error()) { return error(); } + return first.get_double_in_string(); + } + simdjson_inline simdjson_result simdjson_result::get_string(bool allow_replacement) noexcept { + if (error()) { return error(); } + return first.get_string(allow_replacement); + } + template + simdjson_inline error_code simdjson_result::get_string(string_type& receiver, bool allow_replacement) noexcept { + if (error()) { return error(); } + return first.get_string(receiver, allow_replacement); + } + simdjson_inline simdjson_result simdjson_result::get_wobbly_string() noexcept { + if (error()) { return error(); } + return first.get_wobbly_string(); + } + simdjson_inline simdjson_result simdjson_result::get_raw_json_string() noexcept { + if (error()) { return error(); } + return first.get_raw_json_string(); + } + simdjson_inline simdjson_result simdjson_result::get_bool() noexcept { + if (error()) { return error(); } + return first.get_bool(); + } + simdjson_inline simdjson_result simdjson_result::get_value() noexcept { + if (error()) { return error(); } + return first.get_value(); + } + simdjson_inline simdjson_result simdjson_result::is_null() noexcept { + if (error()) { return error(); } + return first.is_null(); + } + simdjson_inline simdjson_result simdjson_result::type() noexcept { + if (error()) { return error(); } + return first.type(); + } + simdjson_inline simdjson_result simdjson_result::is_scalar() noexcept { + if (error()) { return error(); } + return first.is_scalar(); + } + simdjson_inline simdjson_result simdjson_result::is_string() noexcept { + if (error()) { return error(); } + return first.is_string(); + } + simdjson_inline simdjson_result simdjson_result::is_negative() noexcept { + if (error()) { return error(); } + return first.is_negative(); + } + simdjson_inline simdjson_result simdjson_result::is_integer() noexcept { + if (error()) { return error(); } + return first.is_integer(); + } + simdjson_inline simdjson_result simdjson_result::get_number_type() noexcept { + if (error()) { return error(); } + return first.get_number_type(); + } + simdjson_inline simdjson_result simdjson_result::get_number() noexcept { + if (error()) { return error(); } + return first.get_number(); + } +#if SIMDJSON_EXCEPTIONS + template ::value == false>::type> + simdjson_inline simdjson_result::operator T() noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return first; + } + simdjson_inline simdjson_result::operator lsx::ondemand::array() & noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return first; + } + simdjson_inline simdjson_result::operator lsx::ondemand::object() & noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return first; + } + simdjson_inline simdjson_result::operator uint64_t() noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return first; + } + simdjson_inline simdjson_result::operator int64_t() noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return first; + } + simdjson_inline simdjson_result::operator double() noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return first; + } + simdjson_inline simdjson_result::operator std::string_view() noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return first; + } + simdjson_inline simdjson_result::operator lsx::ondemand::raw_json_string() noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return first; + } + simdjson_inline simdjson_result::operator bool() noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return first; + } + simdjson_inline simdjson_result::operator lsx::ondemand::value() noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return first; + } +#endif + + simdjson_inline simdjson_result simdjson_result::current_location() noexcept { + if (error()) { return error(); } + return first.current_location(); + } + + simdjson_inline simdjson_result simdjson_result::raw_json_token() noexcept { + if (error()) { return error(); } + return first.raw_json_token(); + } + + simdjson_inline simdjson_result simdjson_result::at_pointer(std::string_view json_pointer) noexcept { + if (error()) { return error(); } + return first.at_pointer(json_pointer); + } + + simdjson_inline simdjson_result simdjson_result::at_path(std::string_view json_path) noexcept { + if (error()) { + return error(); + } + return first.at_path(json_path); + } + +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_ONDEMAND_DOCUMENT_INL_H +/* end file simdjson/generic/ondemand/document-inl.h for lsx */ +/* including simdjson/generic/ondemand/document_stream-inl.h for lsx: #include "simdjson/generic/ondemand/document_stream-inl.h" */ +/* begin file simdjson/generic/ondemand/document_stream-inl.h for lsx */ +#ifndef SIMDJSON_GENERIC_ONDEMAND_DOCUMENT_STREAM_INL_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_ONDEMAND_DOCUMENT_STREAM_INL_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/document_stream.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/document-inl.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/implementation_simdjson_result_base-inl.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +#include +#include + +namespace simdjson { + namespace lsx { + namespace ondemand { + +#ifdef SIMDJSON_THREADS_ENABLED + + inline void stage1_worker::finish() { + // After calling "run" someone would call finish() to wait + // for the end of the processing. + // This function will wait until either the thread has done + // the processing or, else, the destructor has been called. + std::unique_lock lock(locking_mutex); + cond_var.wait(lock, [this] {return has_work == false; }); + } + + inline stage1_worker::~stage1_worker() { + // The thread may never outlive the stage1_worker instance + // and will always be stopped/joined before the stage1_worker + // instance is gone. + stop_thread(); + } + + inline void stage1_worker::start_thread() { + std::unique_lock lock(locking_mutex); + if (thread.joinable()) { + return; // This should never happen but we never want to create more than one thread. + } + thread = std::thread([this] { + while (true) { + std::unique_lock thread_lock(locking_mutex); + // We wait for either "run" or "stop_thread" to be called. + cond_var.wait(thread_lock, [this] {return has_work || !can_work; }); + // If, for some reason, the stop_thread() method was called (i.e., the + // destructor of stage1_worker is called, then we want to immediately destroy + // the thread (and not do any more processing). + if (!can_work) { + break; + } + this->owner->stage1_thread_error = this->owner->run_stage1(*this->stage1_thread_parser, + this->_next_batch_start); + this->has_work = false; + // The condition variable call should be moved after thread_lock.unlock() for performance + // reasons but thread sanitizers may report it as a data race if we do. + // See https://stackoverflow.com/questions/35775501/c-should-condition-variable-be-notified-under-lock + cond_var.notify_one(); // will notify "finish" + thread_lock.unlock(); + } + } + ); + } + + + inline void stage1_worker::stop_thread() { + std::unique_lock lock(locking_mutex); + // We have to make sure that all locks can be released. + can_work = false; + has_work = false; + cond_var.notify_all(); + lock.unlock(); + if (thread.joinable()) { + thread.join(); + } + } + + inline void stage1_worker::run(document_stream* ds, parser* stage1, size_t next_batch_start) { + std::unique_lock lock(locking_mutex); + owner = ds; + _next_batch_start = next_batch_start; + stage1_thread_parser = stage1; + has_work = true; + // The condition variable call should be moved after thread_lock.unlock() for performance + // reasons but thread sanitizers may report it as a data race if we do. + // See https://stackoverflow.com/questions/35775501/c-should-condition-variable-be-notified-under-lock + cond_var.notify_one(); // will notify the thread lock that we have work + lock.unlock(); + } + +#endif // SIMDJSON_THREADS_ENABLED + + simdjson_inline document_stream::document_stream( + ondemand::parser& _parser, + const uint8_t* _buf, + size_t _len, + size_t _batch_size, + bool _allow_comma_separated + ) noexcept + : parser{ &_parser }, + buf{ _buf }, + len{ _len }, + batch_size{ _batch_size <= MINIMAL_BATCH_SIZE ? MINIMAL_BATCH_SIZE : _batch_size }, + allow_comma_separated{ _allow_comma_separated }, + error{ SUCCESS } +#ifdef SIMDJSON_THREADS_ENABLED + , use_thread(_parser.threaded) // we need to make a copy because _parser.threaded can change +#endif + { +#ifdef SIMDJSON_THREADS_ENABLED + if (worker.get() == nullptr) { + error = MEMALLOC; + } +#endif + } + + simdjson_inline document_stream::document_stream() noexcept + : parser{ nullptr }, + buf{ nullptr }, + len{ 0 }, + batch_size{ 0 }, + allow_comma_separated{ false }, + error{ UNINITIALIZED } +#ifdef SIMDJSON_THREADS_ENABLED + , use_thread(false) +#endif + { + } + + simdjson_inline document_stream::~document_stream() noexcept + { +#ifdef SIMDJSON_THREADS_ENABLED + worker.reset(); +#endif + } + + inline size_t document_stream::size_in_bytes() const noexcept { + return len; + } + + inline size_t document_stream::truncated_bytes() const noexcept { + if (error == CAPACITY) { return len - batch_start; } + return parser->implementation->structural_indexes[parser->implementation->n_structural_indexes] - parser->implementation->structural_indexes[parser->implementation->n_structural_indexes + 1]; + } + + simdjson_inline document_stream::iterator::iterator() noexcept + : stream{ nullptr }, finished{ true } { + } + + simdjson_inline document_stream::iterator::iterator(document_stream* _stream, bool is_end) noexcept + : stream{ _stream }, finished{ is_end } { + } + + simdjson_inline simdjson_result document_stream::iterator::operator*() noexcept { + //if(stream->error) { return stream->error; } + return simdjson_result(stream->doc, stream->error); + } + + simdjson_inline document_stream::iterator& document_stream::iterator::operator++() noexcept { + // If there is an error, then we want the iterator + // to be finished, no matter what. (E.g., we do not + // keep generating documents with errors, or go beyond + // a document with errors.) + // + // Users do not have to call "operator*()" when they use operator++, + // so we need to end the stream in the operator++ function. + // + // Note that setting finished = true is essential otherwise + // we would enter an infinite loop. + if (stream->error) { finished = true; } + // Note that stream->error() is guarded against error conditions + // (it will immediately return if stream->error casts to false). + // In effect, this next function does nothing when (stream->error) + // is true (hence the risk of an infinite loop). + stream->next(); + // If that was the last document, we're finished. + // It is the only type of error we do not want to appear + // in operator*. + if (stream->error == EMPTY) { finished = true; } + // If we had any other kind of error (not EMPTY) then we want + // to pass it along to the operator* and we cannot mark the result + // as "finished" just yet. + return *this; + } + + simdjson_inline bool document_stream::iterator::operator!=(const document_stream::iterator& other) const noexcept { + return finished != other.finished; + } + + simdjson_inline document_stream::iterator document_stream::begin() noexcept { + start(); + // If there are no documents, we're finished. + return iterator(this, error == EMPTY); + } + + simdjson_inline document_stream::iterator document_stream::end() noexcept { + return iterator(this, true); + } + + inline void document_stream::start() noexcept { + if (error) { return; } + error = parser->allocate(batch_size); + if (error) { return; } + // Always run the first stage 1 parse immediately + batch_start = 0; + error = run_stage1(*parser, batch_start); + while (error == EMPTY) { + // In exceptional cases, we may start with an empty block + batch_start = next_batch_start(); + if (batch_start >= len) { return; } + error = run_stage1(*parser, batch_start); + } + if (error) { return; } + doc_index = batch_start; + doc = document(json_iterator(&buf[batch_start], parser)); + doc.iter._streaming = true; + +#ifdef SIMDJSON_THREADS_ENABLED + if (use_thread && next_batch_start() < len) { + // Kick off the first thread on next batch if needed + error = stage1_thread_parser.allocate(batch_size); + if (error) { return; } + worker->start_thread(); + start_stage1_thread(); + if (error) { return; } + } +#endif // SIMDJSON_THREADS_ENABLED + } + + inline void document_stream::next() noexcept { + // We always enter at once once in an error condition. + if (error) { return; } + next_document(); + if (error) { return; } + auto cur_struct_index = doc.iter._root - parser->implementation->structural_indexes.get(); + doc_index = batch_start + parser->implementation->structural_indexes[cur_struct_index]; + + // Check if at end of structural indexes (i.e. at end of batch) + if (cur_struct_index >= static_cast(parser->implementation->n_structural_indexes)) { + error = EMPTY; + // Load another batch (if available) + while (error == EMPTY) { + batch_start = next_batch_start(); + if (batch_start >= len) { break; } +#ifdef SIMDJSON_THREADS_ENABLED + if (use_thread) { + load_from_stage1_thread(); + } + else { + error = run_stage1(*parser, batch_start); + } +#else + error = run_stage1(*parser, batch_start); +#endif + /** + * Whenever we move to another window, we need to update all pointers to make + * it appear as if the input buffer started at the beginning of the window. + * + * Take this input: + * + * {"z":5} {"1":1,"2":2,"4":4} [7, 10, 9] [15, 11, 12, 13] [154, 110, 112, 1311] + * + * Say you process the following window... + * + * '{"z":5} {"1":1,"2":2,"4":4} [7, 10, 9]' + * + * When you do so, the json_iterator has a pointer at the beginning of the memory region + * (pointing at the beginning of '{"z"...'. + * + * When you move to the window that starts at... + * + * '[7, 10, 9] [15, 11, 12, 13] ... + * + * then it is not sufficient to just run stage 1. You also need to re-anchor the + * json_iterator so that it believes we are starting at '[7, 10, 9]...'. + * + * Under the DOM front-end, this gets done automatically because the parser owns + * the pointer the data, and when you call stage1 and then stage2 on the same + * parser, then stage2 will run on the pointer acquired by stage1. + * + * That is, stage1 calls "this->buf = _buf" so the parser remembers the buffer that + * we used. But json_iterator has no callback when stage1 is called on the parser. + * In fact, I think that the parser is unaware of json_iterator. + * + * + * So we need to re-anchor the json_iterator after each call to stage 1 so that + * all of the pointers are in sync. + */ + doc.iter = json_iterator(&buf[batch_start], parser); + doc.iter._streaming = true; + /** + * End of resync. + */ + + if (error) { continue; } // If the error was EMPTY, we may want to load another batch. + doc_index = batch_start; + } + } + } + + inline void document_stream::next_document() noexcept { + // Go to next place where depth=0 (document depth) + error = doc.iter.skip_child(0); + if (error) { return; } + // Always set depth=1 at the start of document + doc.iter._depth = 1; + // consume comma if comma separated is allowed + if (allow_comma_separated) { doc.iter.consume_character(','); } + // Resets the string buffer at the beginning, thus invalidating the strings. + doc.iter._string_buf_loc = parser->string_buf.get(); + doc.iter._root = doc.iter.position(); + } + + inline size_t document_stream::next_batch_start() const noexcept { + return batch_start + parser->implementation->structural_indexes[parser->implementation->n_structural_indexes]; + } + + inline error_code document_stream::run_stage1(ondemand::parser& p, size_t _batch_start) noexcept { + // This code only updates the structural index in the parser, it does not update any json_iterator + // instance. + size_t remaining = len - _batch_start; + if (remaining <= batch_size) { + return p.implementation->stage1(&buf[_batch_start], remaining, stage1_mode::streaming_final); + } + else { + return p.implementation->stage1(&buf[_batch_start], batch_size, stage1_mode::streaming_partial); + } + } + + simdjson_inline size_t document_stream::iterator::current_index() const noexcept { + return stream->doc_index; + } + + simdjson_inline std::string_view document_stream::iterator::source() const noexcept { + auto depth = stream->doc.iter.depth(); + auto cur_struct_index = stream->doc.iter._root - stream->parser->implementation->structural_indexes.get(); + + // If at root, process the first token to determine if scalar value + if (stream->doc.iter.at_root()) { + switch (stream->buf[stream->batch_start + stream->parser->implementation->structural_indexes[cur_struct_index]]) { + case '{': case '[': // Depth=1 already at start of document + break; + case '}': case ']': + depth--; + break; + default: // Scalar value document + // TODO: We could remove trailing whitespaces + // This returns a string spanning from start of value to the beginning of the next document (excluded) + { + auto next_index = stream->parser->implementation->structural_indexes[++cur_struct_index]; + // normally the length would be next_index - current_index() - 1, except for the last document + size_t svlen = next_index - current_index(); + const char* start = reinterpret_cast(stream->buf) + current_index(); + while (svlen > 1 && (std::isspace(start[svlen - 1]) || start[svlen - 1] == '\0')) { + svlen--; + } + return std::string_view(start, svlen); + } + } + cur_struct_index++; + } + + while (cur_struct_index <= static_cast(stream->parser->implementation->n_structural_indexes)) { + switch (stream->buf[stream->batch_start + stream->parser->implementation->structural_indexes[cur_struct_index]]) { + case '{': case '[': + depth++; + break; + case '}': case ']': + depth--; + break; + } + if (depth == 0) { break; } + cur_struct_index++; + } + + return std::string_view(reinterpret_cast(stream->buf) + current_index(), stream->parser->implementation->structural_indexes[cur_struct_index] - current_index() + stream->batch_start + 1);; + } + + inline error_code document_stream::iterator::error() const noexcept { + return stream->error; + } + +#ifdef SIMDJSON_THREADS_ENABLED + + inline void document_stream::load_from_stage1_thread() noexcept { + worker->finish(); + // Swap to the parser that was loaded up in the thread. Make sure the parser has + // enough memory to swap to, as well. + std::swap(stage1_thread_parser, *parser); + error = stage1_thread_error; + if (error) { return; } + + // If there's anything left, start the stage 1 thread! + if (next_batch_start() < len) { + start_stage1_thread(); + } + } + + inline void document_stream::start_stage1_thread() noexcept { + // we call the thread on a lambda that will update + // this->stage1_thread_error + // there is only one thread that may write to this value + // TODO this is NOT exception-safe. + this->stage1_thread_error = UNINITIALIZED; // In case something goes wrong, make sure it's an error + size_t _next_batch_start = this->next_batch_start(); + + worker->run(this, &this->stage1_thread_parser, _next_batch_start); + } + +#endif // SIMDJSON_THREADS_ENABLED + + } // namespace ondemand + } // namespace lsx +} // namespace simdjson + +namespace simdjson { + + simdjson_inline simdjson_result::simdjson_result( + error_code error + ) noexcept : + implementation_simdjson_result_base(error) + { + } + simdjson_inline simdjson_result::simdjson_result( + lsx::ondemand::document_stream&& value + ) noexcept : + implementation_simdjson_result_base( + std::forward(value) + ) + { + } + +} + +#endif // SIMDJSON_GENERIC_ONDEMAND_DOCUMENT_STREAM_INL_H +/* end file simdjson/generic/ondemand/document_stream-inl.h for lsx */ +/* including simdjson/generic/ondemand/field-inl.h for lsx: #include "simdjson/generic/ondemand/field-inl.h" */ +/* begin file simdjson/generic/ondemand/field-inl.h for lsx */ +#ifndef SIMDJSON_GENERIC_ONDEMAND_FIELD_INL_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_ONDEMAND_FIELD_INL_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/field.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/value-inl.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/value_iterator-inl.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lsx { + namespace ondemand { + + // clang 6 does not think the default constructor can be noexcept, so we make it explicit + simdjson_inline field::field() noexcept : std::pair() {} + + simdjson_inline field::field(raw_json_string key, ondemand::value&& value) noexcept + : std::pair(key, std::forward(value)) + { + } + + simdjson_inline simdjson_result field::start(value_iterator& parent_iter) noexcept { + raw_json_string key; + SIMDJSON_TRY(parent_iter.field_key().get(key)); + SIMDJSON_TRY(parent_iter.field_value()); + return field::start(parent_iter, key); + } + + simdjson_inline simdjson_result field::start(const value_iterator& parent_iter, raw_json_string key) noexcept { + return field(key, parent_iter.child()); + } + + simdjson_inline simdjson_warn_unused simdjson_result field::unescaped_key(bool allow_replacement) noexcept { + SIMDJSON_ASSUME(first.buf != nullptr); // We would like to call .alive() but Visual Studio won't let us. + simdjson_result answer = first.unescape(second.iter.json_iter(), allow_replacement); + first.consume(); + return answer; + } + + template + simdjson_inline simdjson_warn_unused error_code field::unescaped_key(string_type& receiver, bool allow_replacement) noexcept { + std::string_view key; + SIMDJSON_TRY(unescaped_key(allow_replacement).get(key)); + receiver = key; + return SUCCESS; + } + + simdjson_inline raw_json_string field::key() const noexcept { + SIMDJSON_ASSUME(first.buf != nullptr); // We would like to call .alive() by Visual Studio won't let us. + return first; + } + + + simdjson_inline std::string_view field::key_raw_json_token() const noexcept { + SIMDJSON_ASSUME(first.buf != nullptr); // We would like to call .alive() by Visual Studio won't let us. + return std::string_view(reinterpret_cast(first.buf - 1), second.iter._json_iter->token.peek(-1) - first.buf + 1); + } + + simdjson_inline std::string_view field::escaped_key() const noexcept { + SIMDJSON_ASSUME(first.buf != nullptr); // We would like to call .alive() by Visual Studio won't let us. + auto end_quote = second.iter._json_iter->token.peek(-1); + while (*end_quote != '"') end_quote--; + return std::string_view(reinterpret_cast(first.buf), end_quote - first.buf); + } + + simdjson_inline value& field::value() & noexcept { + return second; + } + + simdjson_inline value field::value() && noexcept { + return std::forward(*this).second; + } + + } // namespace ondemand + } // namespace lsx +} // namespace simdjson + +namespace simdjson { + + simdjson_inline simdjson_result::simdjson_result( + lsx::ondemand::field&& value + ) noexcept : + implementation_simdjson_result_base( + std::forward(value) + ) + { + } + simdjson_inline simdjson_result::simdjson_result( + error_code error + ) noexcept : + implementation_simdjson_result_base(error) + { + } + + simdjson_inline simdjson_result simdjson_result::key() noexcept { + if (error()) { return error(); } + return first.key(); + } + + simdjson_inline simdjson_result simdjson_result::key_raw_json_token() noexcept { + if (error()) { return error(); } + return first.key_raw_json_token(); + } + + simdjson_inline simdjson_result simdjson_result::escaped_key() noexcept { + if (error()) { return error(); } + return first.escaped_key(); + } + + simdjson_inline simdjson_result simdjson_result::unescaped_key(bool allow_replacement) noexcept { + if (error()) { return error(); } + return first.unescaped_key(allow_replacement); + } + + template + simdjson_inline error_code simdjson_result::unescaped_key(string_type& receiver, bool allow_replacement) noexcept { + if (error()) { return error(); } + return first.unescaped_key(receiver, allow_replacement); + } + + simdjson_inline simdjson_result simdjson_result::value() noexcept { + if (error()) { return error(); } + return std::move(first.value()); + } + +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_ONDEMAND_FIELD_INL_H +/* end file simdjson/generic/ondemand/field-inl.h for lsx */ +/* including simdjson/generic/ondemand/json_iterator-inl.h for lsx: #include "simdjson/generic/ondemand/json_iterator-inl.h" */ +/* begin file simdjson/generic/ondemand/json_iterator-inl.h for lsx */ +#ifndef SIMDJSON_GENERIC_ONDEMAND_JSON_ITERATOR_INL_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_ONDEMAND_JSON_ITERATOR_INL_H */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/dom_parser_implementation.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/json_iterator.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/parser.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/raw_json_string.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/logger-inl.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/parser-inl.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/token_iterator-inl.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lsx { + namespace ondemand { + + simdjson_inline json_iterator::json_iterator(json_iterator&& other) noexcept + : token(std::forward(other.token)), + parser{ other.parser }, + _string_buf_loc{ other._string_buf_loc }, + error{ other.error }, + _depth{ other._depth }, + _root{ other._root }, + _streaming{ other._streaming } + { + other.parser = nullptr; + } + simdjson_inline json_iterator& json_iterator::operator=(json_iterator&& other) noexcept { + token = other.token; + parser = other.parser; + _string_buf_loc = other._string_buf_loc; + error = other.error; + _depth = other._depth; + _root = other._root; + _streaming = other._streaming; + other.parser = nullptr; + return *this; + } + + simdjson_inline json_iterator::json_iterator(const uint8_t* buf, ondemand::parser* _parser) noexcept + : token(buf, &_parser->implementation->structural_indexes[0]), + parser{ _parser }, + _string_buf_loc{ parser->string_buf.get() }, + _depth{ 1 }, + _root{ parser->implementation->structural_indexes.get() }, + _streaming{ false } + + { + logger::log_headers(); +#if SIMDJSON_CHECK_EOF + assert_more_tokens(); +#endif + } + +#ifdef SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON + simdjson_inline json_iterator::json_iterator(const uint8_t* buf, ondemand::parser* _parser, bool streaming) noexcept + : token(buf, &_parser->implementation->structural_indexes[0]), + parser{ _parser }, + _string_buf_loc{ parser->string_buf.get() }, + _depth{ 1 }, + _root{ parser->implementation->structural_indexes.get() }, + _streaming{ streaming } + + { + logger::log_headers(); +#if SIMDJSON_CHECK_EOF + assert_more_tokens(); +#endif + } +#endif // SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON + + inline void json_iterator::rewind() noexcept { + token.set_position(root_position()); + logger::log_headers(); // We start again + _string_buf_loc = parser->string_buf.get(); + _depth = 1; + } + + inline bool json_iterator::balanced() const noexcept { + token_iterator ti(token); + int32_t count{ 0 }; + ti.set_position(root_position()); + while (ti.peek() <= peek_last()) { + switch (*ti.return_current_and_advance()) + { + case '[': case '{': + count++; + break; + case ']': case '}': + count--; + break; + default: + break; + } + } + return count == 0; + } + + + // GCC 7 warns when the first line of this function is inlined away into oblivion due to the caller + // relating depth and parent_depth, which is a desired effect. The warning does not show up if the + // skip_child() function is not marked inline). + SIMDJSON_PUSH_DISABLE_WARNINGS + SIMDJSON_DISABLE_STRICT_OVERFLOW_WARNING + simdjson_warn_unused simdjson_inline error_code json_iterator::skip_child(depth_t parent_depth) noexcept { + if (depth() <= parent_depth) { return SUCCESS; } + switch (*return_current_and_advance()) { + // TODO consider whether matching braces is a requirement: if non-matching braces indicates + // *missing* braces, then future lookups are not in the object/arrays they think they are, + // violating the rule "validate enough structure that the user can be confident they are + // looking at the right values." + // PERF TODO we can eliminate the switch here with a lookup of how much to add to depth + + // For the first open array/object in a value, we've already incremented depth, so keep it the same + // We never stop at colon, but if we did, it wouldn't affect depth + case '[': case '{': case ':': + logger::log_start_value(*this, "skip"); + break; + // If there is a comma, we have just finished a value in an array/object, and need to get back in + case ',': + logger::log_value(*this, "skip"); + break; + // ] or } means we just finished a value and need to jump out of the array/object + case ']': case '}': + logger::log_end_value(*this, "skip"); + _depth--; + if (depth() <= parent_depth) { return SUCCESS; } +#if SIMDJSON_CHECK_EOF + // If there are no more tokens, the parent is incomplete. + if (at_end()) { return report_error(INCOMPLETE_ARRAY_OR_OBJECT, "Missing [ or { at start"); } +#endif // SIMDJSON_CHECK_EOF + break; + case '"': + if (*peek() == ':') { + // We are at a key!!! + // This might happen if you just started an object and you skip it immediately. + // Performance note: it would be nice to get rid of this check as it is somewhat + // expensive. + // https://github.com/simdjson/simdjson/issues/1742 + logger::log_value(*this, "key"); + return_current_and_advance(); // eat up the ':' + break; // important!!! + } + simdjson_fallthrough; + // Anything else must be a scalar value + default: + // For the first scalar, we will have incremented depth already, so we decrement it here. + logger::log_value(*this, "skip"); + _depth--; + if (depth() <= parent_depth) { return SUCCESS; } + break; + } + + // Now that we've considered the first value, we only increment/decrement for arrays/objects + while (position() < end_position()) { + switch (*return_current_and_advance()) { + case '[': case '{': + logger::log_start_value(*this, "skip"); + _depth++; + break; + // TODO consider whether matching braces is a requirement: if non-matching braces indicates + // *missing* braces, then future lookups are not in the object/arrays they think they are, + // violating the rule "validate enough structure that the user can be confident they are + // looking at the right values." + // PERF TODO we can eliminate the switch here with a lookup of how much to add to depth + case ']': case '}': + logger::log_end_value(*this, "skip"); + _depth--; + if (depth() <= parent_depth) { return SUCCESS; } + break; + default: + logger::log_value(*this, "skip", ""); + break; + } + } + + return report_error(TAPE_ERROR, "not enough close braces"); + } + + SIMDJSON_POP_DISABLE_WARNINGS + + simdjson_inline bool json_iterator::at_root() const noexcept { + return position() == root_position(); + } + + simdjson_inline bool json_iterator::is_single_token() const noexcept { + return parser->implementation->n_structural_indexes == 1; + } + + simdjson_inline bool json_iterator::streaming() const noexcept { + return _streaming; + } + + simdjson_inline token_position json_iterator::root_position() const noexcept { + return _root; + } + + simdjson_inline void json_iterator::assert_at_document_depth() const noexcept { + SIMDJSON_ASSUME(_depth == 1); + } + + simdjson_inline void json_iterator::assert_at_root() const noexcept { + SIMDJSON_ASSUME(_depth == 1); +#ifndef SIMDJSON_CLANG_VISUAL_STUDIO + // Under Visual Studio, the next SIMDJSON_ASSUME fails with: the argument + // has side effects that will be discarded. + SIMDJSON_ASSUME(token.position() == _root); +#endif + } + + simdjson_inline void json_iterator::assert_more_tokens(uint32_t required_tokens) const noexcept { + assert_valid_position(token._position + required_tokens - 1); + } + + simdjson_inline void json_iterator::assert_valid_position(token_position position) const noexcept { +#ifndef SIMDJSON_CLANG_VISUAL_STUDIO + SIMDJSON_ASSUME(position >= &parser->implementation->structural_indexes[0]); + SIMDJSON_ASSUME(position < &parser->implementation->structural_indexes[parser->implementation->n_structural_indexes]); +#endif + } + + simdjson_inline bool json_iterator::at_end() const noexcept { + return position() == end_position(); + } + simdjson_inline token_position json_iterator::end_position() const noexcept { + uint32_t n_structural_indexes{ parser->implementation->n_structural_indexes }; + return &parser->implementation->structural_indexes[n_structural_indexes]; + } + + inline std::string json_iterator::to_string() const noexcept { + if (!is_alive()) { return "dead json_iterator instance"; } + const char* current_structural = reinterpret_cast(token.peek()); + return std::string("json_iterator [ depth : ") + std::to_string(_depth) + + std::string(", structural : '") + std::string(current_structural, 1) + + std::string("', offset : ") + std::to_string(token.current_offset()) + + std::string("', error : ") + error_message(error) + + std::string(" ]"); + } + + inline simdjson_result json_iterator::current_location() const noexcept { + if (!is_alive()) { // Unrecoverable error + if (!at_root()) { + return reinterpret_cast(token.peek(-1)); + } + else { + return reinterpret_cast(token.peek()); + } + } + if (at_end()) { + return OUT_OF_BOUNDS; + } + return reinterpret_cast(token.peek()); + } + + simdjson_inline bool json_iterator::is_alive() const noexcept { + return parser; + } + + simdjson_inline void json_iterator::abandon() noexcept { + parser = nullptr; + _depth = 0; + } + + simdjson_inline const uint8_t* json_iterator::return_current_and_advance() noexcept { +#if SIMDJSON_CHECK_EOF + assert_more_tokens(); +#endif // SIMDJSON_CHECK_EOF + return token.return_current_and_advance(); + } + + simdjson_inline const uint8_t* json_iterator::unsafe_pointer() const noexcept { + // deliberately done without safety guard: + return token.peek(); + } + + simdjson_inline const uint8_t* json_iterator::peek(int32_t delta) const noexcept { +#if SIMDJSON_CHECK_EOF + assert_more_tokens(delta + 1); +#endif // SIMDJSON_CHECK_EOF + return token.peek(delta); + } + + simdjson_inline uint32_t json_iterator::peek_length(int32_t delta) const noexcept { +#if SIMDJSON_CHECK_EOF + assert_more_tokens(delta + 1); +#endif // #if SIMDJSON_CHECK_EOF + return token.peek_length(delta); + } + + simdjson_inline const uint8_t* json_iterator::peek(token_position position) const noexcept { + // todo: currently we require end-of-string buffering, but the following + // assert_valid_position should be turned on if/when we lift that condition. + // assert_valid_position(position); + // This is almost surely related to SIMDJSON_CHECK_EOF but given that SIMDJSON_CHECK_EOF + // is ON by default, we have no choice but to disable it for real with a comment. + return token.peek(position); + } + + simdjson_inline uint32_t json_iterator::peek_length(token_position position) const noexcept { +#if SIMDJSON_CHECK_EOF + assert_valid_position(position); +#endif // SIMDJSON_CHECK_EOF + return token.peek_length(position); + } + simdjson_inline uint32_t json_iterator::peek_root_length(token_position position) const noexcept { +#if SIMDJSON_CHECK_EOF + assert_valid_position(position); +#endif // SIMDJSON_CHECK_EOF + return token.peek_root_length(position); + } + + simdjson_inline token_position json_iterator::last_position() const noexcept { + // The following line fails under some compilers... + // SIMDJSON_ASSUME(parser->implementation->n_structural_indexes > 0); + // since it has side-effects. + uint32_t n_structural_indexes{ parser->implementation->n_structural_indexes }; + SIMDJSON_ASSUME(n_structural_indexes > 0); + return &parser->implementation->structural_indexes[n_structural_indexes - 1]; + } + simdjson_inline const uint8_t* json_iterator::peek_last() const noexcept { + return token.peek(last_position()); + } + + simdjson_inline void json_iterator::ascend_to(depth_t parent_depth) noexcept { + SIMDJSON_ASSUME(parent_depth >= 0 && parent_depth < INT32_MAX - 1); + SIMDJSON_ASSUME(_depth == parent_depth + 1); + _depth = parent_depth; + } + + simdjson_inline void json_iterator::descend_to(depth_t child_depth) noexcept { + SIMDJSON_ASSUME(child_depth >= 1 && child_depth < INT32_MAX); + SIMDJSON_ASSUME(_depth == child_depth - 1); + _depth = child_depth; + } + + simdjson_inline depth_t json_iterator::depth() const noexcept { + return _depth; + } + + simdjson_inline uint8_t*& json_iterator::string_buf_loc() noexcept { + return _string_buf_loc; + } + + simdjson_inline error_code json_iterator::report_error(error_code _error, const char* message) noexcept { + SIMDJSON_ASSUME(_error != SUCCESS && _error != UNINITIALIZED && _error != INCORRECT_TYPE && _error != NO_SUCH_FIELD); + logger::log_error(*this, message); + error = _error; + return error; + } + + simdjson_inline token_position json_iterator::position() const noexcept { + return token.position(); + } + + simdjson_inline simdjson_result json_iterator::unescape(raw_json_string in, bool allow_replacement) noexcept { +#if SIMDJSON_DEVELOPMENT_CHECKS + auto result = parser->unescape(in, _string_buf_loc, allow_replacement); + SIMDJSON_ASSUME(!parser->string_buffer_overflow(_string_buf_loc)); + return result; +#else + return parser->unescape(in, _string_buf_loc, allow_replacement); +#endif + } + + simdjson_inline simdjson_result json_iterator::unescape_wobbly(raw_json_string in) noexcept { +#if SIMDJSON_DEVELOPMENT_CHECKS + auto result = parser->unescape_wobbly(in, _string_buf_loc); + SIMDJSON_ASSUME(!parser->string_buffer_overflow(_string_buf_loc)); + return result; +#else + return parser->unescape_wobbly(in, _string_buf_loc); +#endif + } + + simdjson_inline void json_iterator::reenter_child(token_position position, depth_t child_depth) noexcept { + SIMDJSON_ASSUME(child_depth >= 1 && child_depth < INT32_MAX); + SIMDJSON_ASSUME(_depth == child_depth - 1); +#if SIMDJSON_DEVELOPMENT_CHECKS +#ifndef SIMDJSON_CLANG_VISUAL_STUDIO + SIMDJSON_ASSUME(size_t(child_depth) < parser->max_depth()); + SIMDJSON_ASSUME(position >= parser->start_positions[child_depth]); +#endif +#endif + token.set_position(position); + _depth = child_depth; + } + + simdjson_inline error_code json_iterator::consume_character(char c) noexcept { + if (*peek() == c) { + return_current_and_advance(); + return SUCCESS; + } + return TAPE_ERROR; + } + +#if SIMDJSON_DEVELOPMENT_CHECKS + + simdjson_inline token_position json_iterator::start_position(depth_t depth) const noexcept { + SIMDJSON_ASSUME(size_t(depth) < parser->max_depth()); + return size_t(depth) < parser->max_depth() ? parser->start_positions[depth] : 0; + } + + simdjson_inline void json_iterator::set_start_position(depth_t depth, token_position position) noexcept { + SIMDJSON_ASSUME(size_t(depth) < parser->max_depth()); + if (size_t(depth) < parser->max_depth()) { parser->start_positions[depth] = position; } + } + +#endif + + + simdjson_inline error_code json_iterator::optional_error(error_code _error, const char* message) noexcept { + SIMDJSON_ASSUME(_error == INCORRECT_TYPE || _error == NO_SUCH_FIELD); + logger::log_error(*this, message); + return _error; + } + + + simdjson_warn_unused simdjson_inline bool json_iterator::copy_to_buffer(const uint8_t* json, uint32_t max_len, uint8_t* tmpbuf, size_t N) noexcept { + // This function is not expected to be called in performance-sensitive settings. + // Let us guard against silly cases: + if ((N < max_len) || (N == 0)) { return false; } + // Copy to the buffer. + std::memcpy(tmpbuf, json, max_len); + if (N > max_len) { // We pad whatever remains with ' '. + std::memset(tmpbuf + max_len, ' ', N - max_len); + } + return true; + } + + } // namespace ondemand + } // namespace lsx +} // namespace simdjson + +namespace simdjson { + + simdjson_inline simdjson_result::simdjson_result(lsx::ondemand::json_iterator&& value) noexcept + : implementation_simdjson_result_base(std::forward(value)) {} + simdjson_inline simdjson_result::simdjson_result(error_code error) noexcept + : implementation_simdjson_result_base(error) {} + +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_ONDEMAND_JSON_ITERATOR_INL_H +/* end file simdjson/generic/ondemand/json_iterator-inl.h for lsx */ +/* including simdjson/generic/ondemand/json_type-inl.h for lsx: #include "simdjson/generic/ondemand/json_type-inl.h" */ +/* begin file simdjson/generic/ondemand/json_type-inl.h for lsx */ +#ifndef SIMDJSON_GENERIC_ONDEMAND_JSON_TYPE_INL_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_ONDEMAND_JSON_TYPE_INL_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/json_type.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/implementation_simdjson_result_base-inl.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lsx { + namespace ondemand { + + inline std::ostream& operator<<(std::ostream& out, json_type type) noexcept { + switch (type) { + case json_type::array: out << "array"; break; + case json_type::object: out << "object"; break; + case json_type::number: out << "number"; break; + case json_type::string: out << "string"; break; + case json_type::boolean: out << "boolean"; break; + case json_type::null: out << "null"; break; + default: SIMDJSON_UNREACHABLE(); + } + return out; + } + +#if SIMDJSON_EXCEPTIONS + inline std::ostream& operator<<(std::ostream& out, simdjson_result& type) noexcept(false) { + return out << type.value(); + } +#endif + + + + simdjson_inline number_type number::get_number_type() const noexcept { + return type; + } + + simdjson_inline bool number::is_uint64() const noexcept { + return get_number_type() == number_type::unsigned_integer; + } + + simdjson_inline uint64_t number::get_uint64() const noexcept { + return payload.unsigned_integer; + } + + simdjson_inline number::operator uint64_t() const noexcept { + return get_uint64(); + } + + simdjson_inline bool number::is_int64() const noexcept { + return get_number_type() == number_type::signed_integer; + } + + simdjson_inline int64_t number::get_int64() const noexcept { + return payload.signed_integer; + } + + simdjson_inline number::operator int64_t() const noexcept { + return get_int64(); + } + + simdjson_inline bool number::is_double() const noexcept { + return get_number_type() == number_type::floating_point_number; + } + + simdjson_inline double number::get_double() const noexcept { + return payload.floating_point_number; + } + + simdjson_inline number::operator double() const noexcept { + return get_double(); + } + + simdjson_inline double number::as_double() const noexcept { + if (is_double()) { + return payload.floating_point_number; + } + if (is_int64()) { + return double(payload.signed_integer); + } + return double(payload.unsigned_integer); + } + + simdjson_inline void number::append_s64(int64_t value) noexcept { + payload.signed_integer = value; + type = number_type::signed_integer; + } + + simdjson_inline void number::append_u64(uint64_t value) noexcept { + payload.unsigned_integer = value; + type = number_type::unsigned_integer; + } + + simdjson_inline void number::append_double(double value) noexcept { + payload.floating_point_number = value; + type = number_type::floating_point_number; + } + + simdjson_inline void number::skip_double() noexcept { + type = number_type::floating_point_number; + } + + } // namespace ondemand + } // namespace lsx +} // namespace simdjson + +namespace simdjson { + + simdjson_inline simdjson_result::simdjson_result(lsx::ondemand::json_type&& value) noexcept + : implementation_simdjson_result_base(std::forward(value)) {} + simdjson_inline simdjson_result::simdjson_result(error_code error) noexcept + : implementation_simdjson_result_base(error) {} + +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_ONDEMAND_JSON_TYPE_INL_H +/* end file simdjson/generic/ondemand/json_type-inl.h for lsx */ +/* including simdjson/generic/ondemand/logger-inl.h for lsx: #include "simdjson/generic/ondemand/logger-inl.h" */ +/* begin file simdjson/generic/ondemand/logger-inl.h for lsx */ +#ifndef SIMDJSON_GENERIC_ONDEMAND_LOGGER_INL_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_ONDEMAND_LOGGER_INL_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/logger.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/json_iterator.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/value_iterator.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +#include +#include + +namespace simdjson { + namespace lsx { + namespace ondemand { + namespace logger { + + static constexpr const char* DASHES = "----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------"; + static constexpr const int LOG_EVENT_LEN = 20; + static constexpr const int LOG_BUFFER_LEN = 30; + static constexpr const int LOG_SMALL_BUFFER_LEN = 10; + static int log_depth = 0; // Not threadsafe. Log only. + + // Helper to turn unprintable or newline characters into spaces + static inline char printable_char(char c) { + if (c >= 0x20) { + return c; + } + else { + return ' '; + } + } + + template + static inline std::string string_format(const std::string& format, const Args&... args) + { + SIMDJSON_PUSH_DISABLE_ALL_WARNINGS + int size_s = std::snprintf(nullptr, 0, format.c_str(), args...) + 1; + auto size = static_cast(size_s); + if (size <= 0) return std::string(); + std::unique_ptr buf(new char[size]); + std::snprintf(buf.get(), size, format.c_str(), args...); + SIMDJSON_POP_DISABLE_WARNINGS + return std::string(buf.get(), buf.get() + size - 1); + } + + static inline log_level get_log_level_from_env() + { + SIMDJSON_PUSH_DISABLE_WARNINGS + SIMDJSON_DISABLE_DEPRECATED_WARNING // Disable CRT_SECURE warning on MSVC: manually verified this is safe + char* lvl = getenv("SIMDJSON_LOG_LEVEL"); + SIMDJSON_POP_DISABLE_WARNINGS + if (lvl && simdjson_strcasecmp(lvl, "ERROR") == 0) { return log_level::error; } + return log_level::info; + } + + static inline log_level log_threshold() + { + static log_level threshold = get_log_level_from_env(); + return threshold; + } + + static inline bool should_log(log_level level) + { + return level >= log_threshold(); + } + + inline void log_event(const json_iterator& iter, const char* type, std::string_view detail, int delta, int depth_delta) noexcept { + log_line(iter, "", type, detail, delta, depth_delta, log_level::info); + } + + inline void log_value(const json_iterator& iter, token_position index, depth_t depth, const char* type, std::string_view detail) noexcept { + log_line(iter, index, depth, "", type, detail, log_level::info); + } + inline void log_value(const json_iterator& iter, const char* type, std::string_view detail, int delta, int depth_delta) noexcept { + log_line(iter, "", type, detail, delta, depth_delta, log_level::info); + } + + inline void log_start_value(const json_iterator& iter, token_position index, depth_t depth, const char* type, std::string_view detail) noexcept { + log_line(iter, index, depth, "+", type, detail, log_level::info); + if (LOG_ENABLED) { log_depth++; } + } + inline void log_start_value(const json_iterator& iter, const char* type, int delta, int depth_delta) noexcept { + log_line(iter, "+", type, "", delta, depth_delta, log_level::info); + if (LOG_ENABLED) { log_depth++; } + } + + inline void log_end_value(const json_iterator& iter, const char* type, int delta, int depth_delta) noexcept { + if (LOG_ENABLED) { log_depth--; } + log_line(iter, "-", type, "", delta, depth_delta, log_level::info); + } + + inline void log_error(const json_iterator& iter, const char* error, const char* detail, int delta, int depth_delta) noexcept { + log_line(iter, "ERROR: ", error, detail, delta, depth_delta, log_level::error); + } + inline void log_error(const json_iterator& iter, token_position index, depth_t depth, const char* error, const char* detail) noexcept { + log_line(iter, index, depth, "ERROR: ", error, detail, log_level::error); + } + + inline void log_event(const value_iterator& iter, const char* type, std::string_view detail, int delta, int depth_delta) noexcept { + log_event(iter.json_iter(), type, detail, delta, depth_delta); + } + + inline void log_value(const value_iterator& iter, const char* type, std::string_view detail, int delta, int depth_delta) noexcept { + log_value(iter.json_iter(), type, detail, delta, depth_delta); + } + + inline void log_start_value(const value_iterator& iter, const char* type, int delta, int depth_delta) noexcept { + log_start_value(iter.json_iter(), type, delta, depth_delta); + } + + inline void log_end_value(const value_iterator& iter, const char* type, int delta, int depth_delta) noexcept { + log_end_value(iter.json_iter(), type, delta, depth_delta); + } + + inline void log_error(const value_iterator& iter, const char* error, const char* detail, int delta, int depth_delta) noexcept { + log_error(iter.json_iter(), error, detail, delta, depth_delta); + } + + inline void log_headers() noexcept { + if (LOG_ENABLED) { + if (simdjson_unlikely(should_log(log_level::info))) { + // Technically a static variable is not thread-safe, but if you are using threads and logging... well... + static bool displayed_hint{ false }; + log_depth = 0; + printf("\n"); + if (!displayed_hint) { + // We only print this helpful header once. + printf("# Logging provides the depth and position of the iterator user-visible steps:\n"); + printf("# +array says 'this is where we were when we discovered the start array'\n"); + printf( + "# -array says 'this is where we were when we ended the array'\n"); + printf("# skip says 'this is a structural or value I am skipping'\n"); + printf("# +/-skip says 'this is a start/end array or object I am skipping'\n"); + printf("#\n"); + printf("# The indentation of the terms (array, string,...) indicates the depth,\n"); + printf("# in addition to the depth being displayed.\n"); + printf("#\n"); + printf("# Every token in the document has a single depth determined by the tokens before it,\n"); + printf("# and is not affected by what the token actually is.\n"); + printf("#\n"); + printf("# Not all structural elements are presented as tokens in the logs.\n"); + printf("#\n"); + printf("# We never give control to the user within an empty array or an empty object.\n"); + printf("#\n"); + printf("# Inside an array, having a depth greater than the array's depth means that\n"); + printf("# we are pointing inside a value.\n"); + printf("# Having a depth equal to the array means that we are pointing right before a value.\n"); + printf("# Having a depth smaller than the array means that we have moved beyond the array.\n"); + displayed_hint = true; + } + printf("\n"); + printf("| %-*s ", LOG_EVENT_LEN, "Event"); + printf("| %-*s ", LOG_BUFFER_LEN, "Buffer"); + printf("| %-*s ", LOG_SMALL_BUFFER_LEN, "Next"); + // printf("| %-*s ", 5, "Next#"); + printf("| %-*s ", 5, "Depth"); + printf("| Detail "); + printf("|\n"); + + printf("|%.*s", LOG_EVENT_LEN + 2, DASHES); + printf("|%.*s", LOG_BUFFER_LEN + 2, DASHES); + printf("|%.*s", LOG_SMALL_BUFFER_LEN + 2, DASHES); + // printf("|%.*s", 5+2, DASHES); + printf("|%.*s", 5 + 2, DASHES); + printf("|--------"); + printf("|\n"); + fflush(stdout); + } + } + } + + template + inline void log_line(const json_iterator& iter, const char* title_prefix, const char* title, std::string_view detail, int delta, int depth_delta, log_level level, Args&&... args) noexcept { + log_line(iter, iter.position() + delta, depth_t(iter.depth() + depth_delta), title_prefix, title, detail, level, std::forward(args)...); + } + + template + inline void log_line(const json_iterator& iter, token_position index, depth_t depth, const char* title_prefix, const char* title, std::string_view detail, log_level level, Args&&... args) noexcept { + if (LOG_ENABLED) { + if (simdjson_unlikely(should_log(level))) { + const int indent = depth * 2; + const auto buf = iter.token.buf; + auto msg = string_format(title, std::forward(args)...); + printf("| %*s%s%-*s ", indent, "", title_prefix, + LOG_EVENT_LEN - indent - int(strlen(title_prefix)), msg.c_str()); + { + // Print the current structural. + printf("| "); + // Before we begin, the index might point right before the document. + // This could be unsafe, see https://github.com/simdjson/simdjson/discussions/1938 + if (index < iter._root) { + printf("%*s", LOG_BUFFER_LEN, ""); + } + else { + auto current_structural = &buf[*index]; + for (int i = 0; i < LOG_BUFFER_LEN; i++) { + printf("%c", printable_char(current_structural[i])); + } + } + printf(" "); + } + { + // Print the next structural. + printf("| "); + auto next_structural = &buf[*(index + 1)]; + for (int i = 0; i < LOG_SMALL_BUFFER_LEN; i++) { + printf("%c", printable_char(next_structural[i])); + } + printf(" "); + } + // printf("| %5u ", *(index+1)); + printf("| %5i ", depth); + printf("| %6.*s ", int(detail.size()), detail.data()); + printf("|\n"); + fflush(stdout); + } + } + } + + } // namespace logger + } // namespace ondemand + } // namespace lsx +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_ONDEMAND_LOGGER_INL_H +/* end file simdjson/generic/ondemand/logger-inl.h for lsx */ +/* including simdjson/generic/ondemand/object-inl.h for lsx: #include "simdjson/generic/ondemand/object-inl.h" */ +/* begin file simdjson/generic/ondemand/object-inl.h for lsx */ +#ifndef SIMDJSON_GENERIC_ONDEMAND_OBJECT_INL_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_ONDEMAND_OBJECT_INL_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/field.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/object.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/object_iterator.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/raw_json_string.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/json_iterator.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/value-inl.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lsx { + namespace ondemand { + + simdjson_inline simdjson_result object::find_field_unordered(const std::string_view key) & noexcept { + bool has_value; + SIMDJSON_TRY(iter.find_field_unordered_raw(key).get(has_value)); + if (!has_value) { + logger::log_line(iter.json_iter(), "ERROR: ", "Cannot find key %.*s", "", -1, 0, logger::log_level::error, static_cast(key.size()), key.data()); + return NO_SUCH_FIELD; + } + return value(iter.child()); + } + simdjson_inline simdjson_result object::find_field_unordered(const std::string_view key) && noexcept { + bool has_value; + SIMDJSON_TRY(iter.find_field_unordered_raw(key).get(has_value)); + if (!has_value) { + logger::log_line(iter.json_iter(), "ERROR: ", "Cannot find key %.*s", "", -1, 0, logger::log_level::error, static_cast(key.size()), key.data()); + return NO_SUCH_FIELD; + } + return value(iter.child()); + } + simdjson_inline simdjson_result object::operator[](const std::string_view key) & noexcept { + return find_field_unordered(key); + } + simdjson_inline simdjson_result object::operator[](const std::string_view key) && noexcept { + return std::forward(*this).find_field_unordered(key); + } + simdjson_inline simdjson_result object::find_field(const std::string_view key) & noexcept { + bool has_value; + SIMDJSON_TRY(iter.find_field_raw(key).get(has_value)); + if (!has_value) { + logger::log_line(iter.json_iter(), "ERROR: ", "Cannot find key %.*s", "", -1, 0, logger::log_level::error, static_cast(key.size()), key.data()); + return NO_SUCH_FIELD; + } + return value(iter.child()); + } + simdjson_inline simdjson_result object::find_field(const std::string_view key) && noexcept { + bool has_value; + SIMDJSON_TRY(iter.find_field_raw(key).get(has_value)); + if (!has_value) { + logger::log_line(iter.json_iter(), "ERROR: ", "Cannot find key %.*s", "", -1, 0, logger::log_level::error, static_cast(key.size()), key.data()); + return NO_SUCH_FIELD; + } + return value(iter.child()); + } + + simdjson_inline simdjson_result object::start(value_iterator& iter) noexcept { + SIMDJSON_TRY(iter.start_object().error()); + return object(iter); + } + simdjson_inline simdjson_result object::start_root(value_iterator& iter) noexcept { + SIMDJSON_TRY(iter.start_root_object().error()); + return object(iter); + } + simdjson_inline error_code object::consume() noexcept { + if (iter.is_at_key()) { + /** + * whenever you are pointing at a key, calling skip_child() is + * unsafe because you will hit a string and you will assume that + * it is string value, and this mistake will lead you to make bad + * depth computation. + */ + /** + * We want to 'consume' the key. We could really + * just do _json_iter->return_current_and_advance(); at this + * point, but, for clarity, we will use the high-level API to + * eat the key. We assume that the compiler optimizes away + * most of the work. + */ + simdjson_unused raw_json_string actual_key; + auto error = iter.field_key().get(actual_key); + if (error) { iter.abandon(); return error; }; + // Let us move to the value while we are at it. + if ((error = iter.field_value())) { iter.abandon(); return error; } + } + auto error_skip = iter.json_iter().skip_child(iter.depth() - 1); + if (error_skip) { iter.abandon(); } + return error_skip; + } + + simdjson_inline simdjson_result object::raw_json() noexcept { + const uint8_t* starting_point{ iter.peek_start() }; + auto error = consume(); + if (error) { return error; } + const uint8_t* final_point{ iter._json_iter->peek() }; + return std::string_view(reinterpret_cast(starting_point), size_t(final_point - starting_point)); + } + + simdjson_inline simdjson_result object::started(value_iterator& iter) noexcept { + SIMDJSON_TRY(iter.started_object().error()); + return object(iter); + } + + simdjson_inline object object::resume(const value_iterator& iter) noexcept { + return iter; + } + + simdjson_inline object::object(const value_iterator& _iter) noexcept + : iter{ _iter } + { + } + + simdjson_inline simdjson_result object::begin() noexcept { +#if SIMDJSON_DEVELOPMENT_CHECKS + if (!iter.is_at_iterator_start()) { return OUT_OF_ORDER_ITERATION; } +#endif + return object_iterator(iter); + } + simdjson_inline simdjson_result object::end() noexcept { + return object_iterator(iter); + } + + inline simdjson_result object::at_pointer(std::string_view json_pointer) noexcept { + if (json_pointer[0] != '/') { return INVALID_JSON_POINTER; } + json_pointer = json_pointer.substr(1); + size_t slash = json_pointer.find('/'); + std::string_view key = json_pointer.substr(0, slash); + // Grab the child with the given key + simdjson_result child; + + // If there is an escape character in the key, unescape it and then get the child. + size_t escape = key.find('~'); + if (escape != std::string_view::npos) { + // Unescape the key + std::string unescaped(key); + do { + switch (unescaped[escape + 1]) { + case '0': + unescaped.replace(escape, 2, "~"); + break; + case '1': + unescaped.replace(escape, 2, "/"); + break; + default: + return INVALID_JSON_POINTER; // "Unexpected ~ escape character in JSON pointer"); + } + escape = unescaped.find('~', escape + 1); + } while (escape != std::string::npos); + child = find_field(unescaped); // Take note find_field does not unescape keys when matching + } + else { + child = find_field(key); + } + if (child.error()) { + return child; // we do not continue if there was an error + } + // If there is a /, we have to recurse and look up more of the path + if (slash != std::string_view::npos) { + child = child.at_pointer(json_pointer.substr(slash)); + } + return child; + } + + inline simdjson_result object::at_path(std::string_view json_path) noexcept { + auto json_pointer = json_path_to_pointer_conversion(json_path); + if (json_pointer == "-1") { + return INVALID_JSON_POINTER; + } + return at_pointer(json_pointer); + } + + simdjson_inline simdjson_result object::count_fields() & noexcept { + size_t count{ 0 }; + // Important: we do not consume any of the values. + for (simdjson_unused auto v : *this) { count++; } + // The above loop will always succeed, but we want to report errors. + if (iter.error()) { return iter.error(); } + // We need to move back at the start because we expect users to iterate through + // the object after counting the number of elements. + iter.reset_object(); + return count; + } + + simdjson_inline simdjson_result object::is_empty() & noexcept { + bool is_not_empty; + auto error = iter.reset_object().get(is_not_empty); + if (error) { return error; } + return !is_not_empty; + } + + simdjson_inline simdjson_result object::reset() & noexcept { + return iter.reset_object(); + } + + } // namespace ondemand + } // namespace lsx +} // namespace simdjson + +namespace simdjson { + + simdjson_inline simdjson_result::simdjson_result(lsx::ondemand::object&& value) noexcept + : implementation_simdjson_result_base(std::forward(value)) {} + simdjson_inline simdjson_result::simdjson_result(error_code error) noexcept + : implementation_simdjson_result_base(error) {} + + simdjson_inline simdjson_result simdjson_result::begin() noexcept { + if (error()) { return error(); } + return first.begin(); + } + simdjson_inline simdjson_result simdjson_result::end() noexcept { + if (error()) { return error(); } + return first.end(); + } + simdjson_inline simdjson_result simdjson_result::find_field_unordered(std::string_view key) & noexcept { + if (error()) { return error(); } + return first.find_field_unordered(key); + } + simdjson_inline simdjson_result simdjson_result::find_field_unordered(std::string_view key) && noexcept { + if (error()) { return error(); } + return std::forward(first).find_field_unordered(key); + } + simdjson_inline simdjson_result simdjson_result::operator[](std::string_view key) & noexcept { + if (error()) { return error(); } + return first[key]; + } + simdjson_inline simdjson_result simdjson_result::operator[](std::string_view key) && noexcept { + if (error()) { return error(); } + return std::forward(first)[key]; + } + simdjson_inline simdjson_result simdjson_result::find_field(std::string_view key) & noexcept { + if (error()) { return error(); } + return first.find_field(key); + } + simdjson_inline simdjson_result simdjson_result::find_field(std::string_view key) && noexcept { + if (error()) { return error(); } + return std::forward(first).find_field(key); + } + + simdjson_inline simdjson_result simdjson_result::at_pointer(std::string_view json_pointer) noexcept { + if (error()) { return error(); } + return first.at_pointer(json_pointer); + } + + simdjson_inline simdjson_result simdjson_result::at_path( + std::string_view json_path) noexcept { + if (error()) { + return error(); + } + return first.at_path(json_path); + } + + inline simdjson_result simdjson_result::reset() noexcept { + if (error()) { return error(); } + return first.reset(); + } + + inline simdjson_result simdjson_result::is_empty() noexcept { + if (error()) { return error(); } + return first.is_empty(); + } + + simdjson_inline simdjson_result simdjson_result::count_fields() & noexcept { + if (error()) { return error(); } + return first.count_fields(); + } + + simdjson_inline simdjson_result simdjson_result::raw_json() noexcept { + if (error()) { return error(); } + return first.raw_json(); + } +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_ONDEMAND_OBJECT_INL_H +/* end file simdjson/generic/ondemand/object-inl.h for lsx */ +/* including simdjson/generic/ondemand/object_iterator-inl.h for lsx: #include "simdjson/generic/ondemand/object_iterator-inl.h" */ +/* begin file simdjson/generic/ondemand/object_iterator-inl.h for lsx */ +#ifndef SIMDJSON_GENERIC_ONDEMAND_OBJECT_ITERATOR_INL_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_ONDEMAND_OBJECT_ITERATOR_INL_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/object_iterator.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/field-inl.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/value_iterator-inl.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lsx { + namespace ondemand { + + // + // object_iterator + // + + simdjson_inline object_iterator::object_iterator(const value_iterator& _iter) noexcept + : iter{ _iter } + {} + + simdjson_inline simdjson_result object_iterator::operator*() noexcept { + error_code error = iter.error(); + if (error) { iter.abandon(); return error; } + auto result = field::start(iter); + // TODO this is a safety rail ... users should exit loops as soon as they receive an error. + // Nonetheless, let's see if performance is OK with this if statement--the compiler may give it to us for free. + if (result.error()) { iter.abandon(); } + return result; + } + simdjson_inline bool object_iterator::operator==(const object_iterator& other) const noexcept { + return !(*this != other); + } + simdjson_inline bool object_iterator::operator!=(const object_iterator&) const noexcept { + return iter.is_open(); + } + + SIMDJSON_PUSH_DISABLE_WARNINGS + SIMDJSON_DISABLE_STRICT_OVERFLOW_WARNING + simdjson_inline object_iterator& object_iterator::operator++() noexcept { + // TODO this is a safety rail ... users should exit loops as soon as they receive an error. + // Nonetheless, let's see if performance is OK with this if statement--the compiler may give it to us for free. + if (!iter.is_open()) { return *this; } // Iterator will be released if there is an error + + simdjson_unused error_code error; + if ((error = iter.skip_child())) { return *this; } + + simdjson_unused bool has_value; + if ((error = iter.has_next_field().get(has_value))) { return *this; }; + return *this; + } + SIMDJSON_POP_DISABLE_WARNINGS + + // + // ### Live States + // + // While iterating or looking up values, depth >= iter.depth. at_start may vary. Error is + // always SUCCESS: + // + // - Start: This is the state when the object is first found and the iterator is just past the {. + // In this state, at_start == true. + // - Next: After we hand a scalar value to the user, or an array/object which they then fully + // iterate over, the iterator is at the , or } before the next value. In this state, + // depth == iter.depth, at_start == false, and error == SUCCESS. + // - Unfinished Business: When we hand an array/object to the user which they do not fully + // iterate over, we need to finish that iteration by skipping child values until we reach the + // Next state. In this state, depth > iter.depth, at_start == false, and error == SUCCESS. + // + // ## Error States + // + // In error states, we will yield exactly one more value before stopping. iter.depth == depth + // and at_start is always false. We decrement after yielding the error, moving to the Finished + // state. + // + // - Chained Error: When the object iterator is part of an error chain--for example, in + // `for (auto tweet : doc["tweets"])`, where the tweet field may be missing or not be an + // object--we yield that error in the loop, exactly once. In this state, error != SUCCESS and + // iter.depth == depth, and at_start == false. We decrement depth when we yield the error. + // - Missing Comma Error: When the iterator ++ method discovers there is no comma between fields, + // we flag that as an error and treat it exactly the same as a Chained Error. In this state, + // error == TAPE_ERROR, iter.depth == depth, and at_start == false. + // + // Errors that occur while reading a field to give to the user (such as when the key is not a + // string or the field is missing a colon) are yielded immediately. Depth is then decremented, + // moving to the Finished state without transitioning through an Error state at all. + // + // ## Terminal State + // + // The terminal state has iter.depth < depth. at_start is always false. + // + // - Finished: When we have reached a }, we are finished. We signal this by decrementing depth. + // In this state, iter.depth < depth, at_start == false, and error == SUCCESS. + // + + } // namespace ondemand + } // namespace lsx +} // namespace simdjson + +namespace simdjson { + + simdjson_inline simdjson_result::simdjson_result( + lsx::ondemand::object_iterator&& value + ) noexcept + : implementation_simdjson_result_base(std::forward(value)) + { + first.iter.assert_is_valid(); + } + simdjson_inline simdjson_result::simdjson_result(error_code error) noexcept + : implementation_simdjson_result_base({}, error) + { + } + + simdjson_inline simdjson_result simdjson_result::operator*() noexcept { + if (error()) { return error(); } + return *first; + } + // If we're iterating and there is an error, return the error once. + simdjson_inline bool simdjson_result::operator==(const simdjson_result& other) const noexcept { + if (!first.iter.is_valid()) { return !error(); } + return first == other.first; + } + // If we're iterating and there is an error, return the error once. + simdjson_inline bool simdjson_result::operator!=(const simdjson_result& other) const noexcept { + if (!first.iter.is_valid()) { return error(); } + return first != other.first; + } + // Checks for ']' and ',' + simdjson_inline simdjson_result& simdjson_result::operator++() noexcept { + // Clear the error if there is one, so we don't yield it twice + if (error()) { second = SUCCESS; return *this; } + ++first; + return *this; + } + +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_ONDEMAND_OBJECT_ITERATOR_INL_H +/* end file simdjson/generic/ondemand/object_iterator-inl.h for lsx */ +/* including simdjson/generic/ondemand/parser-inl.h for lsx: #include "simdjson/generic/ondemand/parser-inl.h" */ +/* begin file simdjson/generic/ondemand/parser-inl.h for lsx */ +#ifndef SIMDJSON_GENERIC_ONDEMAND_PARSER_INL_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_ONDEMAND_PARSER_INL_H */ +/* amalgamation skipped (editor-only): #include "simdjson/padded_string.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/padded_string_view.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/implementation.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/dom_parser_implementation.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/dom/base.h" // for MINIMAL_DOCUMENT_CAPACITY */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/document_stream.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/parser.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/raw_json_string.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lsx { + namespace ondemand { + + simdjson_inline parser::parser(size_t max_capacity) noexcept + : _max_capacity{ max_capacity } { + } + + simdjson_warn_unused simdjson_inline error_code parser::allocate(size_t new_capacity, size_t new_max_depth) noexcept { + if (new_capacity > max_capacity()) { return CAPACITY; } + if (string_buf && new_capacity == capacity() && new_max_depth == max_depth()) { return SUCCESS; } + + // string_capacity copied from document::allocate + _capacity = 0; + size_t string_capacity = SIMDJSON_ROUNDUP_N(5 * new_capacity / 3 + SIMDJSON_PADDING, 64); + string_buf.reset(new (std::nothrow) uint8_t[string_capacity]); +#if SIMDJSON_DEVELOPMENT_CHECKS + start_positions.reset(new (std::nothrow) token_position[new_max_depth]); +#endif + if (implementation) { + SIMDJSON_TRY(implementation->set_capacity(new_capacity)); + SIMDJSON_TRY(implementation->set_max_depth(new_max_depth)); + } + else { + SIMDJSON_TRY(simdjson::get_active_implementation()->create_dom_parser_implementation(new_capacity, new_max_depth, implementation)); + } + _capacity = new_capacity; + _max_depth = new_max_depth; + return SUCCESS; + } +#if SIMDJSON_DEVELOPMENT_CHECKS + simdjson_inline simdjson_warn_unused bool parser::string_buffer_overflow(const uint8_t* string_buf_loc) const noexcept { + return (string_buf_loc < string_buf.get()) || (size_t(string_buf_loc - string_buf.get()) >= capacity()); + } +#endif + + simdjson_warn_unused simdjson_inline simdjson_result parser::iterate(padded_string_view json) & noexcept { + if (json.padding() < SIMDJSON_PADDING) { return INSUFFICIENT_PADDING; } + + json.remove_utf8_bom(); + + // Allocate if needed + if (capacity() < json.length() || !string_buf) { + SIMDJSON_TRY(allocate(json.length(), max_depth())); + } + + // Run stage 1. + SIMDJSON_TRY(implementation->stage1(reinterpret_cast(json.data()), json.length(), stage1_mode::regular)); + return document::start({ reinterpret_cast(json.data()), this }); + } + +#ifdef SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON + simdjson_warn_unused simdjson_inline simdjson_result parser::iterate_allow_incomplete_json(padded_string_view json) & noexcept { + if (json.padding() < SIMDJSON_PADDING) { return INSUFFICIENT_PADDING; } + + json.remove_utf8_bom(); + + // Allocate if needed + if (capacity() < json.length() || !string_buf) { + SIMDJSON_TRY(allocate(json.length(), max_depth())); + } + + // Run stage 1. + const simdjson::error_code err = implementation->stage1(reinterpret_cast(json.data()), json.length(), stage1_mode::regular); + if (err) { + if (err != UNCLOSED_STRING) + return err; + } + return document::start({ reinterpret_cast(json.data()), this, true }); + } +#endif // SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON + + simdjson_warn_unused simdjson_inline simdjson_result parser::iterate(const char* json, size_t len, size_t allocated) & noexcept { + return iterate(padded_string_view(json, len, allocated)); + } + + simdjson_warn_unused simdjson_inline simdjson_result parser::iterate(const uint8_t* json, size_t len, size_t allocated) & noexcept { + return iterate(padded_string_view(json, len, allocated)); + } + + simdjson_warn_unused simdjson_inline simdjson_result parser::iterate(std::string_view json, size_t allocated) & noexcept { + return iterate(padded_string_view(json, allocated)); + } + + simdjson_warn_unused simdjson_inline simdjson_result parser::iterate(std::string& json) & noexcept { + if (json.capacity() - json.size() < SIMDJSON_PADDING) { + json.reserve(json.size() + SIMDJSON_PADDING); + } + return iterate(padded_string_view(json)); + } + + simdjson_warn_unused simdjson_inline simdjson_result parser::iterate(const std::string& json) & noexcept { + return iterate(padded_string_view(json)); + } + + simdjson_warn_unused simdjson_inline simdjson_result parser::iterate(const simdjson_result& result) & noexcept { + // We don't presently have a way to temporarily get a const T& from a simdjson_result without throwing an exception + SIMDJSON_TRY(result.error()); + padded_string_view json = result.value_unsafe(); + return iterate(json); + } + + simdjson_warn_unused simdjson_inline simdjson_result parser::iterate(const simdjson_result& result) & noexcept { + // We don't presently have a way to temporarily get a const T& from a simdjson_result without throwing an exception + SIMDJSON_TRY(result.error()); + const padded_string& json = result.value_unsafe(); + return iterate(json); + } + + simdjson_warn_unused simdjson_inline simdjson_result parser::iterate_raw(padded_string_view json) & noexcept { + if (json.padding() < SIMDJSON_PADDING) { return INSUFFICIENT_PADDING; } + + json.remove_utf8_bom(); + + // Allocate if needed + if (capacity() < json.length()) { + SIMDJSON_TRY(allocate(json.length(), max_depth())); + } + + // Run stage 1. + SIMDJSON_TRY(implementation->stage1(reinterpret_cast(json.data()), json.length(), stage1_mode::regular)); + return json_iterator(reinterpret_cast(json.data()), this); + } + + inline simdjson_result parser::iterate_many(const uint8_t* buf, size_t len, size_t batch_size, bool allow_comma_separated) noexcept { + if (batch_size < MINIMAL_BATCH_SIZE) { batch_size = MINIMAL_BATCH_SIZE; } + if ((len >= 3) && (std::memcmp(buf, "\xEF\xBB\xBF", 3) == 0)) { + buf += 3; + len -= 3; + } + if (allow_comma_separated && batch_size < len) { batch_size = len; } + return document_stream(*this, buf, len, batch_size, allow_comma_separated); + } + inline simdjson_result parser::iterate_many(const char* buf, size_t len, size_t batch_size, bool allow_comma_separated) noexcept { + return iterate_many(reinterpret_cast(buf), len, batch_size, allow_comma_separated); + } + inline simdjson_result parser::iterate_many(const std::string& s, size_t batch_size, bool allow_comma_separated) noexcept { + return iterate_many(s.data(), s.length(), batch_size, allow_comma_separated); + } + inline simdjson_result parser::iterate_many(const padded_string& s, size_t batch_size, bool allow_comma_separated) noexcept { + return iterate_many(s.data(), s.length(), batch_size, allow_comma_separated); + } + + simdjson_pure simdjson_inline size_t parser::capacity() const noexcept { + return _capacity; + } + simdjson_pure simdjson_inline size_t parser::max_capacity() const noexcept { + return _max_capacity; + } + simdjson_pure simdjson_inline size_t parser::max_depth() const noexcept { + return _max_depth; + } + + simdjson_inline void parser::set_max_capacity(size_t max_capacity) noexcept { + if (max_capacity < dom::MINIMAL_DOCUMENT_CAPACITY) { + _max_capacity = max_capacity; + } + else { + _max_capacity = dom::MINIMAL_DOCUMENT_CAPACITY; + } + } + + simdjson_inline simdjson_warn_unused simdjson_result parser::unescape(raw_json_string in, uint8_t*& dst, bool allow_replacement) const noexcept { + uint8_t* end = implementation->parse_string(in.buf, dst, allow_replacement); + if (!end) { return STRING_ERROR; } + std::string_view result(reinterpret_cast(dst), end - dst); + dst = end; + return result; + } + + simdjson_inline simdjson_warn_unused simdjson_result parser::unescape_wobbly(raw_json_string in, uint8_t*& dst) const noexcept { + uint8_t* end = implementation->parse_wobbly_string(in.buf, dst); + if (!end) { return STRING_ERROR; } + std::string_view result(reinterpret_cast(dst), end - dst); + dst = end; + return result; + } + + } // namespace ondemand + } // namespace lsx +} // namespace simdjson + +namespace simdjson { + + simdjson_inline simdjson_result::simdjson_result(lsx::ondemand::parser&& value) noexcept + : implementation_simdjson_result_base(std::forward(value)) {} + simdjson_inline simdjson_result::simdjson_result(error_code error) noexcept + : implementation_simdjson_result_base(error) {} + +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_ONDEMAND_PARSER_INL_H +/* end file simdjson/generic/ondemand/parser-inl.h for lsx */ +/* including simdjson/generic/ondemand/raw_json_string-inl.h for lsx: #include "simdjson/generic/ondemand/raw_json_string-inl.h" */ +/* begin file simdjson/generic/ondemand/raw_json_string-inl.h for lsx */ +#ifndef SIMDJSON_GENERIC_ONDEMAND_RAW_JSON_STRING_INL_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_ONDEMAND_RAW_JSON_STRING_INL_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/raw_json_string.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/json_iterator-inl.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/implementation_simdjson_result_base-inl.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + + namespace lsx { + namespace ondemand { + + simdjson_inline raw_json_string::raw_json_string(const uint8_t* _buf) noexcept : buf{ _buf } {} + + simdjson_inline const char* raw_json_string::raw() const noexcept { return reinterpret_cast(buf); } + + + simdjson_inline bool raw_json_string::is_free_from_unescaped_quote(std::string_view target) noexcept { + size_t pos{ 0 }; + // if the content has no escape character, just scan through it quickly! + for (; pos < target.size() && target[pos] != '\\'; pos++) {} + // slow path may begin. + bool escaping{ false }; + for (; pos < target.size(); pos++) { + if ((target[pos] == '"') && !escaping) { + return false; + } + else if (target[pos] == '\\') { + escaping = !escaping; + } + else { + escaping = false; + } + } + return true; + } + + simdjson_inline bool raw_json_string::is_free_from_unescaped_quote(const char* target) noexcept { + size_t pos{ 0 }; + // if the content has no escape character, just scan through it quickly! + for (; target[pos] && target[pos] != '\\'; pos++) {} + // slow path may begin. + bool escaping{ false }; + for (; target[pos]; pos++) { + if ((target[pos] == '"') && !escaping) { + return false; + } + else if (target[pos] == '\\') { + escaping = !escaping; + } + else { + escaping = false; + } + } + return true; + } + + + simdjson_inline bool raw_json_string::unsafe_is_equal(size_t length, std::string_view target) const noexcept { + // If we are going to call memcmp, then we must know something about the length of the raw_json_string. + return (length >= target.size()) && (raw()[target.size()] == '"') && !memcmp(raw(), target.data(), target.size()); + } + + simdjson_inline bool raw_json_string::unsafe_is_equal(std::string_view target) const noexcept { + // Assumptions: does not contain unescaped quote characters, and + // the raw content is quote terminated within a valid JSON string. + if (target.size() <= SIMDJSON_PADDING) { + return (raw()[target.size()] == '"') && !memcmp(raw(), target.data(), target.size()); + } + const char* r{ raw() }; + size_t pos{ 0 }; + for (; pos < target.size(); pos++) { + if (r[pos] != target[pos]) { return false; } + } + if (r[pos] != '"') { return false; } + return true; + } + + simdjson_inline bool raw_json_string::is_equal(std::string_view target) const noexcept { + const char* r{ raw() }; + size_t pos{ 0 }; + bool escaping{ false }; + for (; pos < target.size(); pos++) { + if (r[pos] != target[pos]) { return false; } + // if target is a compile-time constant and it is free from + // quotes, then the next part could get optimized away through + // inlining. + if ((target[pos] == '"') && !escaping) { + // We have reached the end of the raw_json_string but + // the target is not done. + return false; + } + else if (target[pos] == '\\') { + escaping = !escaping; + } + else { + escaping = false; + } + } + if (r[pos] != '"') { return false; } + return true; + } + + + simdjson_inline bool raw_json_string::unsafe_is_equal(const char* target) const noexcept { + // Assumptions: 'target' does not contain unescaped quote characters, is null terminated and + // the raw content is quote terminated within a valid JSON string. + const char* r{ raw() }; + size_t pos{ 0 }; + for (; target[pos]; pos++) { + if (r[pos] != target[pos]) { return false; } + } + if (r[pos] != '"') { return false; } + return true; + } + + simdjson_inline bool raw_json_string::is_equal(const char* target) const noexcept { + // Assumptions: does not contain unescaped quote characters, and + // the raw content is quote terminated within a valid JSON string. + const char* r{ raw() }; + size_t pos{ 0 }; + bool escaping{ false }; + for (; target[pos]; pos++) { + if (r[pos] != target[pos]) { return false; } + // if target is a compile-time constant and it is free from + // quotes, then the next part could get optimized away through + // inlining. + if ((target[pos] == '"') && !escaping) { + // We have reached the end of the raw_json_string but + // the target is not done. + return false; + } + else if (target[pos] == '\\') { + escaping = !escaping; + } + else { + escaping = false; + } + } + if (r[pos] != '"') { return false; } + return true; + } + + simdjson_unused simdjson_inline bool operator==(const raw_json_string& a, std::string_view c) noexcept { + return a.unsafe_is_equal(c); + } + + simdjson_unused simdjson_inline bool operator==(std::string_view c, const raw_json_string& a) noexcept { + return a == c; + } + + simdjson_unused simdjson_inline bool operator!=(const raw_json_string& a, std::string_view c) noexcept { + return !(a == c); + } + + simdjson_unused simdjson_inline bool operator!=(std::string_view c, const raw_json_string& a) noexcept { + return !(a == c); + } + + + simdjson_inline simdjson_warn_unused simdjson_result raw_json_string::unescape(json_iterator& iter, bool allow_replacement) const noexcept { + return iter.unescape(*this, allow_replacement); + } + + simdjson_inline simdjson_warn_unused simdjson_result raw_json_string::unescape_wobbly(json_iterator& iter) const noexcept { + return iter.unescape_wobbly(*this); + } + + simdjson_unused simdjson_inline std::ostream& operator<<(std::ostream& out, const raw_json_string& str) noexcept { + bool in_escape = false; + const char* s = str.raw(); + while (true) { + switch (*s) { + case '\\': in_escape = !in_escape; break; + case '"': if (in_escape) { in_escape = false; } + else { return out; } break; + default: if (in_escape) { in_escape = false; } + } + out << *s; + s++; + } + } + + } // namespace ondemand + } // namespace lsx +} // namespace simdjson + +namespace simdjson { + + simdjson_inline simdjson_result::simdjson_result(lsx::ondemand::raw_json_string&& value) noexcept + : implementation_simdjson_result_base(std::forward(value)) {} + simdjson_inline simdjson_result::simdjson_result(error_code error) noexcept + : implementation_simdjson_result_base(error) {} + + simdjson_inline simdjson_result simdjson_result::raw() const noexcept { + if (error()) { return error(); } + return first.raw(); + } + simdjson_inline simdjson_warn_unused simdjson_result simdjson_result::unescape(lsx::ondemand::json_iterator& iter, bool allow_replacement) const noexcept { + if (error()) { return error(); } + return first.unescape(iter, allow_replacement); + } + simdjson_inline simdjson_warn_unused simdjson_result simdjson_result::unescape_wobbly(lsx::ondemand::json_iterator& iter) const noexcept { + if (error()) { return error(); } + return first.unescape_wobbly(iter); + } +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_ONDEMAND_RAW_JSON_STRING_INL_H +/* end file simdjson/generic/ondemand/raw_json_string-inl.h for lsx */ +/* including simdjson/generic/ondemand/serialization-inl.h for lsx: #include "simdjson/generic/ondemand/serialization-inl.h" */ +/* begin file simdjson/generic/ondemand/serialization-inl.h for lsx */ +#ifndef SIMDJSON_GENERIC_ONDEMAND_SERIALIZATION_INL_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_ONDEMAND_SERIALIZATION_INL_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/array.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/document-inl.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/json_type.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/object.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/serialization.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/value.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + + inline std::string_view trim(const std::string_view str) noexcept { + // We can almost surely do better by rolling our own find_first_not_of function. + size_t first = str.find_first_not_of(" \t\n\r"); + // If we have the empty string (just white space), then no trimming is possible, and + // we return the empty string_view. + if (std::string_view::npos == first) { return std::string_view(); } + size_t last = str.find_last_not_of(" \t\n\r"); + return str.substr(first, (last - first + 1)); + } + + + inline simdjson_result to_json_string(lsx::ondemand::document& x) noexcept { + std::string_view v; + auto error = x.raw_json().get(v); + if (error) { return error; } + return trim(v); + } + + inline simdjson_result to_json_string(lsx::ondemand::document_reference& x) noexcept { + std::string_view v; + auto error = x.raw_json().get(v); + if (error) { return error; } + return trim(v); + } + + inline simdjson_result to_json_string(lsx::ondemand::value& x) noexcept { + /** + * If we somehow receive a value that has already been consumed, + * then the following code could be in trouble. E.g., we create + * an array as needed, but if an array was already created, then + * it could be bad. + */ + using namespace lsx::ondemand; + lsx::ondemand::json_type t; + auto error = x.type().get(t); + if (error != SUCCESS) { return error; } + switch (t) + { + case json_type::array: + { + lsx::ondemand::array array; + error = x.get_array().get(array); + if (error) { return error; } + return to_json_string(array); + } + case json_type::object: + { + lsx::ondemand::object object; + error = x.get_object().get(object); + if (error) { return error; } + return to_json_string(object); + } + default: + return trim(x.raw_json_token()); + } + } + + inline simdjson_result to_json_string(lsx::ondemand::object& x) noexcept { + std::string_view v; + auto error = x.raw_json().get(v); + if (error) { return error; } + return trim(v); + } + + inline simdjson_result to_json_string(lsx::ondemand::array& x) noexcept { + std::string_view v; + auto error = x.raw_json().get(v); + if (error) { return error; } + return trim(v); + } + + inline simdjson_result to_json_string(simdjson_result x) { + if (x.error()) { return x.error(); } + return to_json_string(x.value_unsafe()); + } + + inline simdjson_result to_json_string(simdjson_result x) { + if (x.error()) { return x.error(); } + return to_json_string(x.value_unsafe()); + } + + inline simdjson_result to_json_string(simdjson_result x) { + if (x.error()) { return x.error(); } + return to_json_string(x.value_unsafe()); + } + + inline simdjson_result to_json_string(simdjson_result x) { + if (x.error()) { return x.error(); } + return to_json_string(x.value_unsafe()); + } + + inline simdjson_result to_json_string(simdjson_result x) { + if (x.error()) { return x.error(); } + return to_json_string(x.value_unsafe()); + } +} // namespace simdjson + +namespace simdjson { + namespace lsx { + namespace ondemand { + +#if SIMDJSON_EXCEPTIONS + inline std::ostream& operator<<(std::ostream& out, simdjson::lsx::ondemand::value x) { + std::string_view v; + auto error = simdjson::to_json_string(x).get(v); + if (error == simdjson::SUCCESS) { + return (out << v); + } + else { + throw simdjson::simdjson_error(error); + } + } + inline std::ostream& operator<<(std::ostream& out, simdjson::simdjson_result x) { + if (x.error()) { throw simdjson::simdjson_error(x.error()); } + return (out << x.value()); + } +#else + inline std::ostream& operator<<(std::ostream& out, simdjson::lsx::ondemand::value x) { + std::string_view v; + auto error = simdjson::to_json_string(x).get(v); + if (error == simdjson::SUCCESS) { + return (out << v); + } + else { + return (out << error); + } + } +#endif + +#if SIMDJSON_EXCEPTIONS + inline std::ostream& operator<<(std::ostream& out, simdjson::lsx::ondemand::array value) { + std::string_view v; + auto error = simdjson::to_json_string(value).get(v); + if (error == simdjson::SUCCESS) { + return (out << v); + } + else { + throw simdjson::simdjson_error(error); + } + } + inline std::ostream& operator<<(std::ostream& out, simdjson::simdjson_result x) { + if (x.error()) { throw simdjson::simdjson_error(x.error()); } + return (out << x.value()); + } +#else + inline std::ostream& operator<<(std::ostream& out, simdjson::lsx::ondemand::array value) { + std::string_view v; + auto error = simdjson::to_json_string(value).get(v); + if (error == simdjson::SUCCESS) { + return (out << v); + } + else { + return (out << error); + } + } +#endif + +#if SIMDJSON_EXCEPTIONS + inline std::ostream& operator<<(std::ostream& out, simdjson::lsx::ondemand::document& value) { + std::string_view v; + auto error = simdjson::to_json_string(value).get(v); + if (error == simdjson::SUCCESS) { + return (out << v); + } + else { + throw simdjson::simdjson_error(error); + } + } + inline std::ostream& operator<<(std::ostream& out, simdjson::lsx::ondemand::document_reference& value) { + std::string_view v; + auto error = simdjson::to_json_string(value).get(v); + if (error == simdjson::SUCCESS) { + return (out << v); + } + else { + throw simdjson::simdjson_error(error); + } + } + inline std::ostream& operator<<(std::ostream& out, simdjson::simdjson_result&& x) { + if (x.error()) { throw simdjson::simdjson_error(x.error()); } + return (out << x.value()); + } + inline std::ostream& operator<<(std::ostream& out, simdjson::simdjson_result&& x) { + if (x.error()) { throw simdjson::simdjson_error(x.error()); } + return (out << x.value()); + } +#else + inline std::ostream& operator<<(std::ostream& out, simdjson::lsx::ondemand::document& value) { + std::string_view v; + auto error = simdjson::to_json_string(value).get(v); + if (error == simdjson::SUCCESS) { + return (out << v); + } + else { + return (out << error); + } + } +#endif + +#if SIMDJSON_EXCEPTIONS + inline std::ostream& operator<<(std::ostream& out, simdjson::lsx::ondemand::object value) { + std::string_view v; + auto error = simdjson::to_json_string(value).get(v); + if (error == simdjson::SUCCESS) { + return (out << v); + } + else { + throw simdjson::simdjson_error(error); + } + } + inline std::ostream& operator<<(std::ostream& out, simdjson::simdjson_result x) { + if (x.error()) { throw simdjson::simdjson_error(x.error()); } + return (out << x.value()); + } +#else + inline std::ostream& operator<<(std::ostream& out, simdjson::lsx::ondemand::object value) { + std::string_view v; + auto error = simdjson::to_json_string(value).get(v); + if (error == simdjson::SUCCESS) { + return (out << v); + } + else { + return (out << error); + } + } +#endif + } + } +} // namespace simdjson::lsx::ondemand + +#endif // SIMDJSON_GENERIC_ONDEMAND_SERIALIZATION_INL_H +/* end file simdjson/generic/ondemand/serialization-inl.h for lsx */ +/* including simdjson/generic/ondemand/token_iterator-inl.h for lsx: #include "simdjson/generic/ondemand/token_iterator-inl.h" */ +/* begin file simdjson/generic/ondemand/token_iterator-inl.h for lsx */ +#ifndef SIMDJSON_GENERIC_ONDEMAND_TOKEN_ITERATOR_INL_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_ONDEMAND_TOKEN_ITERATOR_INL_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/token_iterator.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/implementation_simdjson_result_base-inl.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lsx { + namespace ondemand { + + simdjson_inline token_iterator::token_iterator( + const uint8_t* _buf, + token_position position + ) noexcept : buf{ _buf }, _position{ position } + { + } + + simdjson_inline uint32_t token_iterator::current_offset() const noexcept { + return *(_position); + } + + + simdjson_inline const uint8_t* token_iterator::return_current_and_advance() noexcept { + return &buf[*(_position++)]; + } + + simdjson_inline const uint8_t* token_iterator::peek(token_position position) const noexcept { + return &buf[*position]; + } + simdjson_inline uint32_t token_iterator::peek_index(token_position position) const noexcept { + return *position; + } + simdjson_inline uint32_t token_iterator::peek_length(token_position position) const noexcept { + return *(position + 1) - *position; + } + + simdjson_inline uint32_t token_iterator::peek_root_length(token_position position) const noexcept { + return *(position + 2) - *(position) > *(position + 1) - *(position) ? + *(position + 1) - *(position) + : *(position + 2) - *(position); + } + simdjson_inline const uint8_t* token_iterator::peek(int32_t delta) const noexcept { + return &buf[*(_position + delta)]; + } + simdjson_inline uint32_t token_iterator::peek_index(int32_t delta) const noexcept { + return *(_position + delta); + } + simdjson_inline uint32_t token_iterator::peek_length(int32_t delta) const noexcept { + return *(_position + delta + 1) - *(_position + delta); + } + + simdjson_inline token_position token_iterator::position() const noexcept { + return _position; + } + simdjson_inline void token_iterator::set_position(token_position target_position) noexcept { + _position = target_position; + } + + simdjson_inline bool token_iterator::operator==(const token_iterator& other) const noexcept { + return _position == other._position; + } + simdjson_inline bool token_iterator::operator!=(const token_iterator& other) const noexcept { + return _position != other._position; + } + simdjson_inline bool token_iterator::operator>(const token_iterator& other) const noexcept { + return _position > other._position; + } + simdjson_inline bool token_iterator::operator>=(const token_iterator& other) const noexcept { + return _position >= other._position; + } + simdjson_inline bool token_iterator::operator<(const token_iterator& other) const noexcept { + return _position < other._position; + } + simdjson_inline bool token_iterator::operator<=(const token_iterator& other) const noexcept { + return _position <= other._position; + } + + } // namespace ondemand + } // namespace lsx +} // namespace simdjson + +namespace simdjson { + + simdjson_inline simdjson_result::simdjson_result(lsx::ondemand::token_iterator&& value) noexcept + : implementation_simdjson_result_base(std::forward(value)) {} + simdjson_inline simdjson_result::simdjson_result(error_code error) noexcept + : implementation_simdjson_result_base(error) {} + +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_ONDEMAND_TOKEN_ITERATOR_INL_H +/* end file simdjson/generic/ondemand/token_iterator-inl.h for lsx */ +/* including simdjson/generic/ondemand/value-inl.h for lsx: #include "simdjson/generic/ondemand/value-inl.h" */ +/* begin file simdjson/generic/ondemand/value-inl.h for lsx */ +#ifndef SIMDJSON_GENERIC_ONDEMAND_VALUE_INL_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_ONDEMAND_VALUE_INL_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/array.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/array_iterator.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/json_iterator.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/json_type.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/object.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/raw_json_string.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/value.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lsx { + namespace ondemand { + + simdjson_inline value::value(const value_iterator& _iter) noexcept + : iter{ _iter } + { + } + simdjson_inline value value::start(const value_iterator& iter) noexcept { + return iter; + } + simdjson_inline value value::resume(const value_iterator& iter) noexcept { + return iter; + } + + simdjson_inline simdjson_result value::get_array() noexcept { + return array::start(iter); + } + simdjson_inline simdjson_result value::get_object() noexcept { + return object::start(iter); + } + simdjson_inline simdjson_result value::start_or_resume_object() noexcept { + if (iter.at_start()) { + return get_object(); + } + else { + return object::resume(iter); + } + } + + simdjson_inline simdjson_result value::get_raw_json_string() noexcept { + return iter.get_raw_json_string(); + } + simdjson_inline simdjson_result value::get_string(bool allow_replacement) noexcept { + return iter.get_string(allow_replacement); + } + template + simdjson_inline error_code value::get_string(string_type& receiver, bool allow_replacement) noexcept { + return iter.get_string(receiver, allow_replacement); + } + simdjson_inline simdjson_result value::get_wobbly_string() noexcept { + return iter.get_wobbly_string(); + } + simdjson_inline simdjson_result value::get_double() noexcept { + return iter.get_double(); + } + simdjson_inline simdjson_result value::get_double_in_string() noexcept { + return iter.get_double_in_string(); + } + simdjson_inline simdjson_result value::get_uint64() noexcept { + return iter.get_uint64(); + } + simdjson_inline simdjson_result value::get_uint64_in_string() noexcept { + return iter.get_uint64_in_string(); + } + simdjson_inline simdjson_result value::get_int64() noexcept { + return iter.get_int64(); + } + simdjson_inline simdjson_result value::get_int64_in_string() noexcept { + return iter.get_int64_in_string(); + } + simdjson_inline simdjson_result value::get_bool() noexcept { + return iter.get_bool(); + } + simdjson_inline simdjson_result value::is_null() noexcept { + return iter.is_null(); + } + template<> simdjson_inline simdjson_result value::get() noexcept { return get_array(); } + template<> simdjson_inline simdjson_result value::get() noexcept { return get_object(); } + template<> simdjson_inline simdjson_result value::get() noexcept { return get_raw_json_string(); } + template<> simdjson_inline simdjson_result value::get() noexcept { return get_string(false); } + template<> simdjson_inline simdjson_result value::get() noexcept { return get_number(); } + template<> simdjson_inline simdjson_result value::get() noexcept { return get_double(); } + template<> simdjson_inline simdjson_result value::get() noexcept { return get_uint64(); } + template<> simdjson_inline simdjson_result value::get() noexcept { return get_int64(); } + template<> simdjson_inline simdjson_result value::get() noexcept { return get_bool(); } + + template simdjson_inline error_code value::get(T& out) noexcept { + return get().get(out); + } + +#if SIMDJSON_EXCEPTIONS + template + simdjson_inline value::operator T() noexcept(false) { + return get(); + } + simdjson_inline value::operator array() noexcept(false) { + return get_array(); + } + simdjson_inline value::operator object() noexcept(false) { + return get_object(); + } + simdjson_inline value::operator uint64_t() noexcept(false) { + return get_uint64(); + } + simdjson_inline value::operator int64_t() noexcept(false) { + return get_int64(); + } + simdjson_inline value::operator double() noexcept(false) { + return get_double(); + } + simdjson_inline value::operator std::string_view() noexcept(false) { + return get_string(false); + } + simdjson_inline value::operator raw_json_string() noexcept(false) { + return get_raw_json_string(); + } + simdjson_inline value::operator bool() noexcept(false) { + return get_bool(); + } +#endif + + simdjson_inline simdjson_result value::begin() & noexcept { + return get_array().begin(); + } + simdjson_inline simdjson_result value::end() & noexcept { + return {}; + } + simdjson_inline simdjson_result value::count_elements() & noexcept { + simdjson_result answer; + auto a = get_array(); + answer = a.count_elements(); + // count_elements leaves you pointing inside the array, at the first element. + // We need to move back so that the user can create a new array (which requires that + // we point at '['). + iter.move_at_start(); + return answer; + } + simdjson_inline simdjson_result value::count_fields() & noexcept { + simdjson_result answer; + auto a = get_object(); + answer = a.count_fields(); + iter.move_at_start(); + return answer; + } + simdjson_inline simdjson_result value::at(size_t index) noexcept { + auto a = get_array(); + return a.at(index); + } + + simdjson_inline simdjson_result value::find_field(std::string_view key) noexcept { + return start_or_resume_object().find_field(key); + } + simdjson_inline simdjson_result value::find_field(const char* key) noexcept { + return start_or_resume_object().find_field(key); + } + + simdjson_inline simdjson_result value::find_field_unordered(std::string_view key) noexcept { + return start_or_resume_object().find_field_unordered(key); + } + simdjson_inline simdjson_result value::find_field_unordered(const char* key) noexcept { + return start_or_resume_object().find_field_unordered(key); + } + + simdjson_inline simdjson_result value::operator[](std::string_view key) noexcept { + return start_or_resume_object()[key]; + } + simdjson_inline simdjson_result value::operator[](const char* key) noexcept { + return start_or_resume_object()[key]; + } + + simdjson_inline simdjson_result value::type() noexcept { + return iter.type(); + } + + simdjson_inline simdjson_result value::is_scalar() noexcept { + json_type this_type; + auto error = type().get(this_type); + if (error) { return error; } + return !((this_type == json_type::array) || (this_type == json_type::object)); + } + + simdjson_inline simdjson_result value::is_string() noexcept { + json_type this_type; + auto error = type().get(this_type); + if (error) { return error; } + return (this_type == json_type::string); + } + + + simdjson_inline bool value::is_negative() noexcept { + return iter.is_negative(); + } + + simdjson_inline simdjson_result value::is_integer() noexcept { + return iter.is_integer(); + } + simdjson_warn_unused simdjson_inline simdjson_result value::get_number_type() noexcept { + return iter.get_number_type(); + } + simdjson_warn_unused simdjson_inline simdjson_result value::get_number() noexcept { + return iter.get_number(); + } + + simdjson_inline std::string_view value::raw_json_token() noexcept { + return std::string_view(reinterpret_cast(iter.peek_start()), iter.peek_start_length()); + } + + simdjson_inline simdjson_result value::raw_json() noexcept { + json_type t; + SIMDJSON_TRY(type().get(t)); + switch (t) + { + case json_type::array: { + ondemand::array array; + SIMDJSON_TRY(get_array().get(array)); + return array.raw_json(); + } + case json_type::object: { + ondemand::object object; + SIMDJSON_TRY(get_object().get(object)); + return object.raw_json(); + } + default: + return raw_json_token(); + } + } + + simdjson_inline simdjson_result value::current_location() noexcept { + return iter.json_iter().current_location(); + } + + simdjson_inline int32_t value::current_depth() const noexcept { + return iter.json_iter().depth(); + } + + inline bool is_pointer_well_formed(std::string_view json_pointer) noexcept { + if (simdjson_unlikely(json_pointer.empty())) { // can't be + return false; + } + if (simdjson_unlikely(json_pointer[0] != '/')) { + return false; + } + size_t escape = json_pointer.find('~'); + if (escape == std::string_view::npos) { + return true; + } + if (escape == json_pointer.size() - 1) { + return false; + } + if (json_pointer[escape + 1] != '0' && json_pointer[escape + 1] != '1') { + return false; + } + return true; + } + + simdjson_inline simdjson_result value::at_pointer(std::string_view json_pointer) noexcept { + json_type t; + SIMDJSON_TRY(type().get(t)); + switch (t) + { + case json_type::array: + return (*this).get_array().at_pointer(json_pointer); + case json_type::object: + return (*this).get_object().at_pointer(json_pointer); + default: + // a non-empty string can be invalid, or accessing a primitive (issue 2154) + if (is_pointer_well_formed(json_pointer)) { + return NO_SUCH_FIELD; + } + return INVALID_JSON_POINTER; + } + } + + simdjson_inline simdjson_result value::at_path(std::string_view json_path) noexcept { + json_type t; + SIMDJSON_TRY(type().get(t)); + switch (t) { + case json_type::array: + return (*this).get_array().at_path(json_path); + case json_type::object: + return (*this).get_object().at_path(json_path); + default: + return INVALID_JSON_POINTER; + } + } + + } // namespace ondemand + } // namespace lsx +} // namespace simdjson + +namespace simdjson { + + simdjson_inline simdjson_result::simdjson_result( + lsx::ondemand::value&& value + ) noexcept : + implementation_simdjson_result_base( + std::forward(value) + ) + { + } + simdjson_inline simdjson_result::simdjson_result( + error_code error + ) noexcept : + implementation_simdjson_result_base(error) + { + } + simdjson_inline simdjson_result simdjson_result::count_elements() & noexcept { + if (error()) { return error(); } + return first.count_elements(); + } + simdjson_inline simdjson_result simdjson_result::count_fields() & noexcept { + if (error()) { return error(); } + return first.count_fields(); + } + simdjson_inline simdjson_result simdjson_result::at(size_t index) noexcept { + if (error()) { return error(); } + return first.at(index); + } + simdjson_inline simdjson_result simdjson_result::begin() & noexcept { + if (error()) { return error(); } + return first.begin(); + } + simdjson_inline simdjson_result simdjson_result::end() & noexcept { + if (error()) { return error(); } + return {}; + } + + simdjson_inline simdjson_result simdjson_result::find_field(std::string_view key) noexcept { + if (error()) { return error(); } + return first.find_field(key); + } + simdjson_inline simdjson_result simdjson_result::find_field(const char* key) noexcept { + if (error()) { return error(); } + return first.find_field(key); + } + + simdjson_inline simdjson_result simdjson_result::find_field_unordered(std::string_view key) noexcept { + if (error()) { return error(); } + return first.find_field_unordered(key); + } + simdjson_inline simdjson_result simdjson_result::find_field_unordered(const char* key) noexcept { + if (error()) { return error(); } + return first.find_field_unordered(key); + } + + simdjson_inline simdjson_result simdjson_result::operator[](std::string_view key) noexcept { + if (error()) { return error(); } + return first[key]; + } + simdjson_inline simdjson_result simdjson_result::operator[](const char* key) noexcept { + if (error()) { return error(); } + return first[key]; + } + + simdjson_inline simdjson_result simdjson_result::get_array() noexcept { + if (error()) { return error(); } + return first.get_array(); + } + simdjson_inline simdjson_result simdjson_result::get_object() noexcept { + if (error()) { return error(); } + return first.get_object(); + } + simdjson_inline simdjson_result simdjson_result::get_uint64() noexcept { + if (error()) { return error(); } + return first.get_uint64(); + } + simdjson_inline simdjson_result simdjson_result::get_uint64_in_string() noexcept { + if (error()) { return error(); } + return first.get_uint64_in_string(); + } + simdjson_inline simdjson_result simdjson_result::get_int64() noexcept { + if (error()) { return error(); } + return first.get_int64(); + } + simdjson_inline simdjson_result simdjson_result::get_int64_in_string() noexcept { + if (error()) { return error(); } + return first.get_int64_in_string(); + } + simdjson_inline simdjson_result simdjson_result::get_double() noexcept { + if (error()) { return error(); } + return first.get_double(); + } + simdjson_inline simdjson_result simdjson_result::get_double_in_string() noexcept { + if (error()) { return error(); } + return first.get_double_in_string(); + } + simdjson_inline simdjson_result simdjson_result::get_string(bool allow_replacement) noexcept { + if (error()) { return error(); } + return first.get_string(allow_replacement); + } + template + simdjson_inline error_code simdjson_result::get_string(string_type& receiver, bool allow_replacement) noexcept { + if (error()) { return error(); } + return first.get_string(receiver, allow_replacement); + } + simdjson_inline simdjson_result simdjson_result::get_wobbly_string() noexcept { + if (error()) { return error(); } + return first.get_wobbly_string(); + } + simdjson_inline simdjson_result simdjson_result::get_raw_json_string() noexcept { + if (error()) { return error(); } + return first.get_raw_json_string(); + } + simdjson_inline simdjson_result simdjson_result::get_bool() noexcept { + if (error()) { return error(); } + return first.get_bool(); + } + simdjson_inline simdjson_result simdjson_result::is_null() noexcept { + if (error()) { return error(); } + return first.is_null(); + } + + template simdjson_inline simdjson_result simdjson_result::get() noexcept { + if (error()) { return error(); } + return first.get(); + } + template simdjson_inline error_code simdjson_result::get(T& out) noexcept { + if (error()) { return error(); } + return first.get(out); + } + + template<> simdjson_inline simdjson_result simdjson_result::get() noexcept { + if (error()) { return error(); } + return std::move(first); + } + template<> simdjson_inline error_code simdjson_result::get(lsx::ondemand::value& out) noexcept { + if (error()) { return error(); } + out = first; + return SUCCESS; + } + + simdjson_inline simdjson_result simdjson_result::type() noexcept { + if (error()) { return error(); } + return first.type(); + } + simdjson_inline simdjson_result simdjson_result::is_scalar() noexcept { + if (error()) { return error(); } + return first.is_scalar(); + } + simdjson_inline simdjson_result simdjson_result::is_string() noexcept { + if (error()) { return error(); } + return first.is_string(); + } + simdjson_inline simdjson_result simdjson_result::is_negative() noexcept { + if (error()) { return error(); } + return first.is_negative(); + } + simdjson_inline simdjson_result simdjson_result::is_integer() noexcept { + if (error()) { return error(); } + return first.is_integer(); + } + simdjson_inline simdjson_result simdjson_result::get_number_type() noexcept { + if (error()) { return error(); } + return first.get_number_type(); + } + simdjson_inline simdjson_result simdjson_result::get_number() noexcept { + if (error()) { return error(); } + return first.get_number(); + } +#if SIMDJSON_EXCEPTIONS + template + simdjson_inline simdjson_result::operator T() noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return static_cast(first); + } + simdjson_inline simdjson_result::operator lsx::ondemand::array() noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return first; + } + simdjson_inline simdjson_result::operator lsx::ondemand::object() noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return first; + } + simdjson_inline simdjson_result::operator uint64_t() noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return first; + } + simdjson_inline simdjson_result::operator int64_t() noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return first; + } + simdjson_inline simdjson_result::operator double() noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return first; + } + simdjson_inline simdjson_result::operator std::string_view() noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return first; + } + simdjson_inline simdjson_result::operator lsx::ondemand::raw_json_string() noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return first; + } + simdjson_inline simdjson_result::operator bool() noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return first; + } +#endif + + simdjson_inline simdjson_result simdjson_result::raw_json_token() noexcept { + if (error()) { return error(); } + return first.raw_json_token(); + } + + simdjson_inline simdjson_result simdjson_result::raw_json() noexcept { + if (error()) { return error(); } + return first.raw_json(); + } + + simdjson_inline simdjson_result simdjson_result::current_location() noexcept { + if (error()) { return error(); } + return first.current_location(); + } + + simdjson_inline simdjson_result simdjson_result::current_depth() const noexcept { + if (error()) { return error(); } + return first.current_depth(); + } + + simdjson_inline simdjson_result simdjson_result::at_pointer( + std::string_view json_pointer) noexcept { + if (error()) { + return error(); + } + return first.at_pointer(json_pointer); + } + + simdjson_inline simdjson_result simdjson_result::at_path( + std::string_view json_path) noexcept { + if (error()) { + return error(); + } + return first.at_path(json_path); + } + +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_ONDEMAND_VALUE_INL_H +/* end file simdjson/generic/ondemand/value-inl.h for lsx */ +/* including simdjson/generic/ondemand/value_iterator-inl.h for lsx: #include "simdjson/generic/ondemand/value_iterator-inl.h" */ +/* begin file simdjson/generic/ondemand/value_iterator-inl.h for lsx */ +#ifndef SIMDJSON_GENERIC_ONDEMAND_VALUE_ITERATOR_INL_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_ONDEMAND_VALUE_ITERATOR_INL_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/atomparsing.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/numberparsing.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/json_iterator.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/value_iterator.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/json_type-inl.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/raw_json_string-inl.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lsx { + namespace ondemand { + + simdjson_inline value_iterator::value_iterator( + json_iterator* json_iter, + depth_t depth, + token_position start_position + ) noexcept : _json_iter{ json_iter }, _depth{ depth }, _start_position{ start_position } + { + } + + simdjson_warn_unused simdjson_inline simdjson_result value_iterator::start_object() noexcept { + SIMDJSON_TRY(start_container('{', "Not an object", "object")); + return started_object(); + } + + simdjson_warn_unused simdjson_inline simdjson_result value_iterator::start_root_object() noexcept { + SIMDJSON_TRY(start_container('{', "Not an object", "object")); + return started_root_object(); + } + + simdjson_warn_unused simdjson_inline simdjson_result value_iterator::started_object() noexcept { + assert_at_container_start(); +#if SIMDJSON_DEVELOPMENT_CHECKS + _json_iter->set_start_position(_depth, start_position()); +#endif + if (*_json_iter->peek() == '}') { + logger::log_value(*_json_iter, "empty object"); + _json_iter->return_current_and_advance(); + end_container(); + return false; + } + return true; + } + + simdjson_warn_unused simdjson_inline error_code value_iterator::check_root_object() noexcept { + // When in streaming mode, we cannot expect peek_last() to be the last structural element of the + // current document. It only works in the normal mode where we have indexed a single document. + // Note that adding a check for 'streaming' is not expensive since we only have at most + // one root element. + if (!_json_iter->streaming()) { + // The following lines do not fully protect against garbage content within the + // object: e.g., `{"a":2} foo }`. Users concerned with garbage content should + // call `at_end()` on the document instance at the end of the processing to + // ensure that the processing has finished at the end. + // + if (*_json_iter->peek_last() != '}') { + _json_iter->abandon(); + return report_error(INCOMPLETE_ARRAY_OR_OBJECT, "missing } at end"); + } + // If the last character is } *and* the first gibberish character is also '}' + // then on-demand could accidentally go over. So we need additional checks. + // https://github.com/simdjson/simdjson/issues/1834 + // Checking that the document is balanced requires a full scan which is potentially + // expensive, but it only happens in edge cases where the first padding character is + // a closing bracket. + if ((*_json_iter->peek(_json_iter->end_position()) == '}') && (!_json_iter->balanced())) { + _json_iter->abandon(); + // The exact error would require more work. It will typically be an unclosed object. + return report_error(INCOMPLETE_ARRAY_OR_OBJECT, "the document is unbalanced"); + } + } + return SUCCESS; + } + + simdjson_warn_unused simdjson_inline simdjson_result value_iterator::started_root_object() noexcept { + auto error = check_root_object(); + if (error) { return error; } + return started_object(); + } + + simdjson_warn_unused simdjson_inline error_code value_iterator::end_container() noexcept { +#if SIMDJSON_CHECK_EOF + if (depth() > 1 && at_end()) { return report_error(INCOMPLETE_ARRAY_OR_OBJECT, "missing parent ] or }"); } + // if (depth() <= 1 && !at_end()) { return report_error(INCOMPLETE_ARRAY_OR_OBJECT, "missing [ or { at start"); } +#endif // SIMDJSON_CHECK_EOF + _json_iter->ascend_to(depth() - 1); + return SUCCESS; + } + + simdjson_warn_unused simdjson_inline simdjson_result value_iterator::has_next_field() noexcept { + assert_at_next(); + + // It's illegal to call this unless there are more tokens: anything that ends in } or ] is + // obligated to verify there are more tokens if they are not the top level. + switch (*_json_iter->return_current_and_advance()) { + case '}': + logger::log_end_value(*_json_iter, "object"); + SIMDJSON_TRY(end_container()); + return false; + case ',': + return true; + default: + return report_error(TAPE_ERROR, "Missing comma between object fields"); + } + } + + simdjson_warn_unused simdjson_inline simdjson_result value_iterator::find_field_raw(const std::string_view key) noexcept { + error_code error; + bool has_value; + // + // Initially, the object can be in one of a few different places: + // + // 1. The start of the object, at the first field: + // + // ``` + // { "a": [ 1, 2 ], "b": [ 3, 4 ] } + // ^ (depth 2, index 1) + // ``` + if (at_first_field()) { + has_value = true; + + // + // 2. When a previous search did not yield a value or the object is empty: + // + // ``` + // { "a": [ 1, 2 ], "b": [ 3, 4 ] } + // ^ (depth 0) + // { } + // ^ (depth 0, index 2) + // ``` + // + } + else if (!is_open()) { +#if SIMDJSON_DEVELOPMENT_CHECKS + // If we're past the end of the object, we're being iterated out of order. + // Note: this is not perfect detection. It's possible the user is inside some other object; if so, + // this object iterator will blithely scan that object for fields. + if (_json_iter->depth() < depth() - 1) { return OUT_OF_ORDER_ITERATION; } +#endif + return false; + + // 3. When a previous search found a field or an iterator yielded a value: + // + // ``` + // // When a field was not fully consumed (or not even touched at all) + // { "a": [ 1, 2 ], "b": [ 3, 4 ] } + // ^ (depth 2) + // // When a field was fully consumed + // { "a": [ 1, 2 ], "b": [ 3, 4 ] } + // ^ (depth 1) + // // When the last field was fully consumed + // { "a": [ 1, 2 ], "b": [ 3, 4 ] } + // ^ (depth 1) + // ``` + // + } + else { + if ((error = skip_child())) { abandon(); return error; } + if ((error = has_next_field().get(has_value))) { abandon(); return error; } +#if SIMDJSON_DEVELOPMENT_CHECKS + if (_json_iter->start_position(_depth) != start_position()) { return OUT_OF_ORDER_ITERATION; } +#endif + } + while (has_value) { + // Get the key and colon, stopping at the value. + raw_json_string actual_key; + // size_t max_key_length = _json_iter->peek_length() - 2; // -2 for the two quotes + // Note: _json_iter->peek_length() - 2 might overflow if _json_iter->peek_length() < 2. + // field_key() advances the pointer and checks that '"' is found (corresponding to a key). + // The depth is left unchanged by field_key(). + if ((error = field_key().get(actual_key))) { abandon(); return error; }; + // field_value() will advance and check that we find a ':' separating the + // key and the value. It will also increment the depth by one. + if ((error = field_value())) { abandon(); return error; } + // If it matches, stop and return + // We could do it this way if we wanted to allow arbitrary + // key content (including escaped quotes). + //if (actual_key.unsafe_is_equal(max_key_length, key)) { + // Instead we do the following which may trigger buffer overruns if the + // user provides an adversarial key (containing a well placed unescaped quote + // character and being longer than the number of bytes remaining in the JSON + // input). + if (actual_key.unsafe_is_equal(key)) { + logger::log_event(*this, "match", key, -2); + // If we return here, then we return while pointing at the ':' that we just checked. + return true; + } + + // No match: skip the value and see if , or } is next + logger::log_event(*this, "no match", key, -2); + // The call to skip_child is meant to skip over the value corresponding to the key. + // After skip_child(), we are right before the next comma (',') or the final brace ('}'). + SIMDJSON_TRY(skip_child()); // Skip the value entirely + // The has_next_field() advances the pointer and check that either ',' or '}' is found. + // It returns true if ',' is found, false otherwise. If anything other than ',' or '}' is found, + // then we are in error and we abort. + if ((error = has_next_field().get(has_value))) { abandon(); return error; } + } + + // If the loop ended, we're out of fields to look at. + return false; + } + + SIMDJSON_PUSH_DISABLE_WARNINGS + SIMDJSON_DISABLE_STRICT_OVERFLOW_WARNING + simdjson_warn_unused simdjson_inline simdjson_result value_iterator::find_field_unordered_raw(const std::string_view key) noexcept { + /** + * When find_field_unordered_raw is called, we can either be pointing at the + * first key, pointing outside (at the closing brace) or if a key was matched + * we can be either pointing right afterthe ':' right before the value (that we need skip), + * or we may have consumed the value and we might be at a comma or at the + * final brace (ready for a call to has_next_field()). + */ + error_code error; + bool has_value; + + // First, we scan from that point to the end. + // If we don't find a match, we may loop back around, and scan from the beginning to that point. + token_position search_start = _json_iter->position(); + + // We want to know whether we need to go back to the beginning. + bool at_first = at_first_field(); + /////////////// + // Initially, the object can be in one of a few different places: + // + // 1. At the first key: + // + // ``` + // { "a": [ 1, 2 ], "b": [ 3, 4 ] } + // ^ (depth 2, index 1) + // ``` + // + if (at_first) { + has_value = true; + + // 2. When a previous search did not yield a value or the object is empty: + // + // ``` + // { "a": [ 1, 2 ], "b": [ 3, 4 ] } + // ^ (depth 0) + // { } + // ^ (depth 0, index 2) + // ``` + // + } + else if (!is_open()) { + +#if SIMDJSON_DEVELOPMENT_CHECKS + // If we're past the end of the object, we're being iterated out of order. + // Note: this is not perfect detection. It's possible the user is inside some other object; if so, + // this object iterator will blithely scan that object for fields. + if (_json_iter->depth() < depth() - 1) { return OUT_OF_ORDER_ITERATION; } +#endif + SIMDJSON_TRY(reset_object().get(has_value)); + at_first = true; + // 3. When a previous search found a field or an iterator yielded a value: + // + // ``` + // // When a field was not fully consumed (or not even touched at all) + // { "a": [ 1, 2 ], "b": [ 3, 4 ] } + // ^ (depth 2) + // // When a field was fully consumed + // { "a": [ 1, 2 ], "b": [ 3, 4 ] } + // ^ (depth 1) + // // When the last field was fully consumed + // { "a": [ 1, 2 ], "b": [ 3, 4 ] } + // ^ (depth 1) + // ``` + // + } + else { + // If someone queried a key but they not did access the value, then we are left pointing + // at the ':' and we need to move forward through the value... If the value was + // processed then skip_child() does not move the iterator (but may adjust the depth). + if ((error = skip_child())) { abandon(); return error; } + search_start = _json_iter->position(); + if ((error = has_next_field().get(has_value))) { abandon(); return error; } +#if SIMDJSON_DEVELOPMENT_CHECKS + if (_json_iter->start_position(_depth) != start_position()) { return OUT_OF_ORDER_ITERATION; } +#endif + } + + // After initial processing, we will be in one of two states: + // + // ``` + // // At the beginning of a field + // { "a": [ 1, 2 ], "b": [ 3, 4 ] } + // ^ (depth 1) + // { "a": [ 1, 2 ], "b": [ 3, 4 ] } + // ^ (depth 1) + // // At the end of the object + // { "a": [ 1, 2 ], "b": [ 3, 4 ] } + // ^ (depth 0) + // ``` + // + // Next, we find a match starting from the current position. + while (has_value) { + SIMDJSON_ASSUME(_json_iter->_depth == _depth); // We must be at the start of a field + + // Get the key and colon, stopping at the value. + raw_json_string actual_key; + // size_t max_key_length = _json_iter->peek_length() - 2; // -2 for the two quotes + // Note: _json_iter->peek_length() - 2 might overflow if _json_iter->peek_length() < 2. + // field_key() advances the pointer and checks that '"' is found (corresponding to a key). + // The depth is left unchanged by field_key(). + if ((error = field_key().get(actual_key))) { abandon(); return error; }; + // field_value() will advance and check that we find a ':' separating the + // key and the value. It will also increment the depth by one. + if ((error = field_value())) { abandon(); return error; } + + // If it matches, stop and return + // We could do it this way if we wanted to allow arbitrary + // key content (including escaped quotes). + // if (actual_key.unsafe_is_equal(max_key_length, key)) { + // Instead we do the following which may trigger buffer overruns if the + // user provides an adversarial key (containing a well placed unescaped quote + // character and being longer than the number of bytes remaining in the JSON + // input). + if (actual_key.unsafe_is_equal(key)) { + logger::log_event(*this, "match", key, -2); + // If we return here, then we return while pointing at the ':' that we just checked. + return true; + } + + // No match: skip the value and see if , or } is next + logger::log_event(*this, "no match", key, -2); + // The call to skip_child is meant to skip over the value corresponding to the key. + // After skip_child(), we are right before the next comma (',') or the final brace ('}'). + SIMDJSON_TRY(skip_child()); + // The has_next_field() advances the pointer and check that either ',' or '}' is found. + // It returns true if ',' is found, false otherwise. If anything other than ',' or '}' is found, + // then we are in error and we abort. + if ((error = has_next_field().get(has_value))) { abandon(); return error; } + } + // Performance note: it maybe wasteful to rewind to the beginning when there might be + // no other query following. Indeed, it would require reskipping the whole object. + // Instead, you can just stay where you are. If there is a new query, there is always time + // to rewind. + if (at_first) { return false; } + + // If we reach the end without finding a match, search the rest of the fields starting at the + // beginning of the object. + // (We have already run through the object before, so we've already validated its structure. We + // don't check errors in this bit.) + SIMDJSON_TRY(reset_object().get(has_value)); + while (true) { + SIMDJSON_ASSUME(has_value); // we should reach search_start before ever reaching the end of the object + SIMDJSON_ASSUME(_json_iter->_depth == _depth); // We must be at the start of a field + + // Get the key and colon, stopping at the value. + raw_json_string actual_key; + // size_t max_key_length = _json_iter->peek_length() - 2; // -2 for the two quotes + // Note: _json_iter->peek_length() - 2 might overflow if _json_iter->peek_length() < 2. + // field_key() advances the pointer and checks that '"' is found (corresponding to a key). + // The depth is left unchanged by field_key(). + error = field_key().get(actual_key); SIMDJSON_ASSUME(!error); + // field_value() will advance and check that we find a ':' separating the + // key and the value. It will also increment the depth by one. + error = field_value(); SIMDJSON_ASSUME(!error); + + // If it matches, stop and return + // We could do it this way if we wanted to allow arbitrary + // key content (including escaped quotes). + // if (actual_key.unsafe_is_equal(max_key_length, key)) { + // Instead we do the following which may trigger buffer overruns if the + // user provides an adversarial key (containing a well placed unescaped quote + // character and being longer than the number of bytes remaining in the JSON + // input). + if (actual_key.unsafe_is_equal(key)) { + logger::log_event(*this, "match", key, -2); + // If we return here, then we return while pointing at the ':' that we just checked. + return true; + } + + // No match: skip the value and see if , or } is next + logger::log_event(*this, "no match", key, -2); + // The call to skip_child is meant to skip over the value corresponding to the key. + // After skip_child(), we are right before the next comma (',') or the final brace ('}'). + SIMDJSON_TRY(skip_child()); + // If we reached the end of the key-value pair we started from, then we know + // that the key is not there so we return false. We are either right before + // the next comma or the final brace. + if (_json_iter->position() == search_start) { return false; } + // The has_next_field() advances the pointer and check that either ',' or '}' is found. + // It returns true if ',' is found, false otherwise. If anything other than ',' or '}' is found, + // then we are in error and we abort. + error = has_next_field().get(has_value); SIMDJSON_ASSUME(!error); + // If we make the mistake of exiting here, then we could be left pointing at a key + // in the middle of an object. That's not an allowable state. + } + // If the loop ended, we're out of fields to look at. The program should + // never reach this point. + return false; + } + SIMDJSON_POP_DISABLE_WARNINGS + + simdjson_warn_unused simdjson_inline simdjson_result value_iterator::field_key() noexcept { + assert_at_next(); + + const uint8_t* key = _json_iter->return_current_and_advance(); + if (*(key++) != '"') { return report_error(TAPE_ERROR, "Object key is not a string"); } + return raw_json_string(key); + } + + simdjson_warn_unused simdjson_inline error_code value_iterator::field_value() noexcept { + assert_at_next(); + + if (*_json_iter->return_current_and_advance() != ':') { return report_error(TAPE_ERROR, "Missing colon in object field"); } + _json_iter->descend_to(depth() + 1); + return SUCCESS; + } + + simdjson_warn_unused simdjson_inline simdjson_result value_iterator::start_array() noexcept { + SIMDJSON_TRY(start_container('[', "Not an array", "array")); + return started_array(); + } + + simdjson_warn_unused simdjson_inline simdjson_result value_iterator::start_root_array() noexcept { + SIMDJSON_TRY(start_container('[', "Not an array", "array")); + return started_root_array(); + } + + inline std::string value_iterator::to_string() const noexcept { + auto answer = std::string("value_iterator [ depth : ") + std::to_string(_depth) + std::string(", "); + if (_json_iter != nullptr) { answer += _json_iter->to_string(); } + answer += std::string(" ]"); + return answer; + } + + simdjson_warn_unused simdjson_inline simdjson_result value_iterator::started_array() noexcept { + assert_at_container_start(); + if (*_json_iter->peek() == ']') { + logger::log_value(*_json_iter, "empty array"); + _json_iter->return_current_and_advance(); + SIMDJSON_TRY(end_container()); + return false; + } + _json_iter->descend_to(depth() + 1); +#if SIMDJSON_DEVELOPMENT_CHECKS + _json_iter->set_start_position(_depth, start_position()); +#endif + return true; + } + + simdjson_warn_unused simdjson_inline error_code value_iterator::check_root_array() noexcept { + // When in streaming mode, we cannot expect peek_last() to be the last structural element of the + // current document. It only works in the normal mode where we have indexed a single document. + // Note that adding a check for 'streaming' is not expensive since we only have at most + // one root element. + if (!_json_iter->streaming()) { + // The following lines do not fully protect against garbage content within the + // array: e.g., `[1, 2] foo]`. Users concerned with garbage content should + // also call `at_end()` on the document instance at the end of the processing to + // ensure that the processing has finished at the end. + // + if (*_json_iter->peek_last() != ']') { + _json_iter->abandon(); + return report_error(INCOMPLETE_ARRAY_OR_OBJECT, "missing ] at end"); + } + // If the last character is ] *and* the first gibberish character is also ']' + // then on-demand could accidentally go over. So we need additional checks. + // https://github.com/simdjson/simdjson/issues/1834 + // Checking that the document is balanced requires a full scan which is potentially + // expensive, but it only happens in edge cases where the first padding character is + // a closing bracket. + if ((*_json_iter->peek(_json_iter->end_position()) == ']') && (!_json_iter->balanced())) { + _json_iter->abandon(); + // The exact error would require more work. It will typically be an unclosed array. + return report_error(INCOMPLETE_ARRAY_OR_OBJECT, "the document is unbalanced"); + } + } + return SUCCESS; + } + + simdjson_warn_unused simdjson_inline simdjson_result value_iterator::started_root_array() noexcept { + auto error = check_root_array(); + if (error) { return error; } + return started_array(); + } + + simdjson_warn_unused simdjson_inline simdjson_result value_iterator::has_next_element() noexcept { + assert_at_next(); + + logger::log_event(*this, "has_next_element"); + switch (*_json_iter->return_current_and_advance()) { + case ']': + logger::log_end_value(*_json_iter, "array"); + SIMDJSON_TRY(end_container()); + return false; + case ',': + _json_iter->descend_to(depth() + 1); + return true; + default: + return report_error(TAPE_ERROR, "Missing comma between array elements"); + } + } + + simdjson_warn_unused simdjson_inline simdjson_result value_iterator::parse_bool(const uint8_t* json) const noexcept { + auto not_true = atomparsing::str4ncmp(json, "true"); + auto not_false = atomparsing::str4ncmp(json, "fals") | (json[4] ^ 'e'); + bool error = (not_true && not_false) || jsoncharutils::is_not_structural_or_whitespace(json[not_true ? 5 : 4]); + if (error) { return incorrect_type_error("Not a boolean"); } + return simdjson_result(!not_true); + } + simdjson_warn_unused simdjson_inline simdjson_result value_iterator::parse_null(const uint8_t* json) const noexcept { + bool is_null_string = !atomparsing::str4ncmp(json, "null") && jsoncharutils::is_structural_or_whitespace(json[4]); + // if we start with 'n', we must be a null + if (!is_null_string && json[0] == 'n') { return incorrect_type_error("Not a null but starts with n"); } + return is_null_string; + } + + simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_string(bool allow_replacement) noexcept { + return get_raw_json_string().unescape(json_iter(), allow_replacement); + } + template + simdjson_warn_unused simdjson_inline error_code value_iterator::get_string(string_type& receiver, bool allow_replacement) noexcept { + std::string_view content; + auto err = get_string(allow_replacement).get(content); + if (err) { return err; } + receiver = content; + return SUCCESS; + } + simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_wobbly_string() noexcept { + return get_raw_json_string().unescape_wobbly(json_iter()); + } + simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_raw_json_string() noexcept { + auto json = peek_scalar("string"); + if (*json != '"') { return incorrect_type_error("Not a string"); } + advance_scalar("string"); + return raw_json_string(json + 1); + } + simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_uint64() noexcept { + auto result = numberparsing::parse_unsigned(peek_non_root_scalar("uint64")); + if (result.error() == SUCCESS) { advance_non_root_scalar("uint64"); } + return result; + } + simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_uint64_in_string() noexcept { + auto result = numberparsing::parse_unsigned_in_string(peek_non_root_scalar("uint64")); + if (result.error() == SUCCESS) { advance_non_root_scalar("uint64"); } + return result; + } + simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_int64() noexcept { + auto result = numberparsing::parse_integer(peek_non_root_scalar("int64")); + if (result.error() == SUCCESS) { advance_non_root_scalar("int64"); } + return result; + } + simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_int64_in_string() noexcept { + auto result = numberparsing::parse_integer_in_string(peek_non_root_scalar("int64")); + if (result.error() == SUCCESS) { advance_non_root_scalar("int64"); } + return result; + } + simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_double() noexcept { + auto result = numberparsing::parse_double(peek_non_root_scalar("double")); + if (result.error() == SUCCESS) { advance_non_root_scalar("double"); } + return result; + } + simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_double_in_string() noexcept { + auto result = numberparsing::parse_double_in_string(peek_non_root_scalar("double")); + if (result.error() == SUCCESS) { advance_non_root_scalar("double"); } + return result; + } + simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_bool() noexcept { + auto result = parse_bool(peek_non_root_scalar("bool")); + if (result.error() == SUCCESS) { advance_non_root_scalar("bool"); } + return result; + } + simdjson_inline simdjson_result value_iterator::is_null() noexcept { + bool is_null_value; + SIMDJSON_TRY(parse_null(peek_non_root_scalar("null")).get(is_null_value)); + if (is_null_value) { advance_non_root_scalar("null"); } + return is_null_value; + } + simdjson_inline bool value_iterator::is_negative() noexcept { + return numberparsing::is_negative(peek_non_root_scalar("numbersign")); + } + simdjson_inline bool value_iterator::is_root_negative() noexcept { + return numberparsing::is_negative(peek_root_scalar("numbersign")); + } + simdjson_inline simdjson_result value_iterator::is_integer() noexcept { + return numberparsing::is_integer(peek_non_root_scalar("integer")); + } + simdjson_inline simdjson_result value_iterator::get_number_type() noexcept { + return numberparsing::get_number_type(peek_non_root_scalar("integer")); + } + simdjson_inline simdjson_result value_iterator::get_number() noexcept { + number num; + error_code error = numberparsing::parse_number(peek_non_root_scalar("number"), num); + if (error) { return error; } + return num; + } + + simdjson_inline simdjson_result value_iterator::is_root_integer(bool check_trailing) noexcept { + auto max_len = peek_root_length(); + auto json = peek_root_scalar("is_root_integer"); + uint8_t tmpbuf[20 + 1 + 1]{}; // <20 digits> is the longest possible unsigned integer + tmpbuf[20 + 1] = '\0'; // make sure that buffer is always null terminated. + if (!_json_iter->copy_to_buffer(json, max_len, tmpbuf, 20 + 1)) { + return false; // if there are more than 20 characters, it cannot be represented as an integer. + } + auto answer = numberparsing::is_integer(tmpbuf); + // If the parsing was a success, we must still check that it is + // a single scalar. Note that we parse first because of cases like '[]' where + // getting TRAILING_CONTENT is wrong. + if (check_trailing && (answer.error() == SUCCESS) && (!_json_iter->is_single_token())) { return TRAILING_CONTENT; } + return answer; + } + + simdjson_inline simdjson_result value_iterator::get_root_number_type(bool check_trailing) noexcept { + auto max_len = peek_root_length(); + auto json = peek_root_scalar("number"); + // Per https://www.exploringbinary.com/maximum-number-of-decimal-digits-in-binary-floating-point-numbers/, + // 1074 is the maximum number of significant fractional digits. Add 8 more digits for the biggest + // number: -0.e-308. + uint8_t tmpbuf[1074 + 8 + 1 + 1]; + tmpbuf[1074 + 8 + 1] = '\0'; // make sure that buffer is always null terminated. + if (!_json_iter->copy_to_buffer(json, max_len, tmpbuf, 1074 + 8 + 1)) { + if (numberparsing::check_if_integer(json, max_len)) { + if (check_trailing && !_json_iter->is_single_token()) { return TRAILING_CONTENT; } + logger::log_error(*_json_iter, start_position(), depth(), "Found big integer"); + return number_type::big_integer; + } + logger::log_error(*_json_iter, start_position(), depth(), "Root number more than 1082 characters and not a big integer"); + return NUMBER_ERROR; + } + auto answer = numberparsing::get_number_type(tmpbuf); + if (check_trailing && (answer.error() == SUCCESS) && !_json_iter->is_single_token()) { return TRAILING_CONTENT; } + return answer; + } + simdjson_inline simdjson_result value_iterator::get_root_number(bool check_trailing) noexcept { + auto max_len = peek_root_length(); + auto json = peek_root_scalar("number"); + // Per https://www.exploringbinary.com/maximum-number-of-decimal-digits-in-binary-floating-point-numbers/, + // 1074 is the maximum number of significant fractional digits. Add 8 more digits for the biggest + // number: -0.e-308. + // NOTE: the current approach doesn't work for very big integer numbers containing more than 1074 digits. + uint8_t tmpbuf[1074 + 8 + 1 + 1]; + tmpbuf[1074 + 8 + 1] = '\0'; // make sure that buffer is always null terminated. + if (!_json_iter->copy_to_buffer(json, max_len, tmpbuf, 1074 + 8 + 1)) { + if (numberparsing::check_if_integer(json, max_len)) { + if (check_trailing && !_json_iter->is_single_token()) { return TRAILING_CONTENT; } + logger::log_error(*_json_iter, start_position(), depth(), "Found big integer"); + return BIGINT_ERROR; + } + logger::log_error(*_json_iter, start_position(), depth(), "Root number more than 1082 characters and not a big integer"); + return NUMBER_ERROR; + } + number num; + error_code error = numberparsing::parse_number(tmpbuf, num); + if (error) { return error; } + if (check_trailing && !_json_iter->is_single_token()) { return TRAILING_CONTENT; } + advance_root_scalar("number"); + return num; + } + simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_root_string(bool check_trailing, bool allow_replacement) noexcept { + return get_root_raw_json_string(check_trailing).unescape(json_iter(), allow_replacement); + } + template + simdjson_warn_unused simdjson_inline error_code value_iterator::get_root_string(string_type& receiver, bool check_trailing, bool allow_replacement) noexcept { + std::string_view content; + auto err = get_root_string(check_trailing, allow_replacement).get(content); + if (err) { return err; } + receiver = content; + return SUCCESS; + } + simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_root_wobbly_string(bool check_trailing) noexcept { + return get_root_raw_json_string(check_trailing).unescape_wobbly(json_iter()); + } + simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_root_raw_json_string(bool check_trailing) noexcept { + auto json = peek_scalar("string"); + if (*json != '"') { return incorrect_type_error("Not a string"); } + if (check_trailing && !_json_iter->is_single_token()) { return TRAILING_CONTENT; } + advance_scalar("string"); + return raw_json_string(json + 1); + } + simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_root_uint64(bool check_trailing) noexcept { + auto max_len = peek_root_length(); + auto json = peek_root_scalar("uint64"); + uint8_t tmpbuf[20 + 1 + 1]{}; // <20 digits> is the longest possible unsigned integer + tmpbuf[20 + 1] = '\0'; // make sure that buffer is always null terminated. + if (!_json_iter->copy_to_buffer(json, max_len, tmpbuf, 20 + 1)) { + logger::log_error(*_json_iter, start_position(), depth(), "Root number more than 20 characters"); + return NUMBER_ERROR; + } + auto result = numberparsing::parse_unsigned(tmpbuf); + if (result.error() == SUCCESS) { + if (check_trailing && !_json_iter->is_single_token()) { return TRAILING_CONTENT; } + advance_root_scalar("uint64"); + } + return result; + } + simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_root_uint64_in_string(bool check_trailing) noexcept { + auto max_len = peek_root_length(); + auto json = peek_root_scalar("uint64"); + uint8_t tmpbuf[20 + 1 + 1]{}; // <20 digits> is the longest possible unsigned integer + tmpbuf[20 + 1] = '\0'; // make sure that buffer is always null terminated. + if (!_json_iter->copy_to_buffer(json, max_len, tmpbuf, 20 + 1)) { + logger::log_error(*_json_iter, start_position(), depth(), "Root number more than 20 characters"); + return NUMBER_ERROR; + } + auto result = numberparsing::parse_unsigned_in_string(tmpbuf); + if (result.error() == SUCCESS) { + if (check_trailing && !_json_iter->is_single_token()) { return TRAILING_CONTENT; } + advance_root_scalar("uint64"); + } + return result; + } + simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_root_int64(bool check_trailing) noexcept { + auto max_len = peek_root_length(); + auto json = peek_root_scalar("int64"); + uint8_t tmpbuf[20 + 1 + 1]; // -<19 digits> is the longest possible integer + tmpbuf[20 + 1] = '\0'; // make sure that buffer is always null terminated. + if (!_json_iter->copy_to_buffer(json, max_len, tmpbuf, 20 + 1)) { + logger::log_error(*_json_iter, start_position(), depth(), "Root number more than 20 characters"); + return NUMBER_ERROR; + } + + auto result = numberparsing::parse_integer(tmpbuf); + if (result.error() == SUCCESS) { + if (check_trailing && !_json_iter->is_single_token()) { return TRAILING_CONTENT; } + advance_root_scalar("int64"); + } + return result; + } + simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_root_int64_in_string(bool check_trailing) noexcept { + auto max_len = peek_root_length(); + auto json = peek_root_scalar("int64"); + uint8_t tmpbuf[20 + 1 + 1]; // -<19 digits> is the longest possible integer + tmpbuf[20 + 1] = '\0'; // make sure that buffer is always null terminated. + if (!_json_iter->copy_to_buffer(json, max_len, tmpbuf, 20 + 1)) { + logger::log_error(*_json_iter, start_position(), depth(), "Root number more than 20 characters"); + return NUMBER_ERROR; + } + + auto result = numberparsing::parse_integer_in_string(tmpbuf); + if (result.error() == SUCCESS) { + if (check_trailing && !_json_iter->is_single_token()) { return TRAILING_CONTENT; } + advance_root_scalar("int64"); + } + return result; + } + simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_root_double(bool check_trailing) noexcept { + auto max_len = peek_root_length(); + auto json = peek_root_scalar("double"); + // Per https://www.exploringbinary.com/maximum-number-of-decimal-digits-in-binary-floating-point-numbers/, + // 1074 is the maximum number of significant fractional digits. Add 8 more digits for the biggest + // number: -0.e-308. + uint8_t tmpbuf[1074 + 8 + 1 + 1]; // +1 for null termination. + tmpbuf[1074 + 8 + 1] = '\0'; // make sure that buffer is always null terminated. + if (!_json_iter->copy_to_buffer(json, max_len, tmpbuf, 1074 + 8 + 1)) { + logger::log_error(*_json_iter, start_position(), depth(), "Root number more than 1082 characters"); + return NUMBER_ERROR; + } + auto result = numberparsing::parse_double(tmpbuf); + if (result.error() == SUCCESS) { + if (check_trailing && !_json_iter->is_single_token()) { return TRAILING_CONTENT; } + advance_root_scalar("double"); + } + return result; + } + + simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_root_double_in_string(bool check_trailing) noexcept { + auto max_len = peek_root_length(); + auto json = peek_root_scalar("double"); + // Per https://www.exploringbinary.com/maximum-number-of-decimal-digits-in-binary-floating-point-numbers/, + // 1074 is the maximum number of significant fractional digits. Add 8 more digits for the biggest + // number: -0.e-308. + uint8_t tmpbuf[1074 + 8 + 1 + 1]; // +1 for null termination. + tmpbuf[1074 + 8 + 1] = '\0'; // make sure that buffer is always null terminated. + if (!_json_iter->copy_to_buffer(json, max_len, tmpbuf, 1074 + 8 + 1)) { + logger::log_error(*_json_iter, start_position(), depth(), "Root number more than 1082 characters"); + return NUMBER_ERROR; + } + auto result = numberparsing::parse_double_in_string(tmpbuf); + if (result.error() == SUCCESS) { + if (check_trailing && !_json_iter->is_single_token()) { return TRAILING_CONTENT; } + advance_root_scalar("double"); + } + return result; + } + simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_root_bool(bool check_trailing) noexcept { + auto max_len = peek_root_length(); + auto json = peek_root_scalar("bool"); + uint8_t tmpbuf[5 + 1 + 1]; // +1 for null termination + tmpbuf[5 + 1] = '\0'; // make sure that buffer is always null terminated. + if (!_json_iter->copy_to_buffer(json, max_len, tmpbuf, 5 + 1)) { return incorrect_type_error("Not a boolean"); } + auto result = parse_bool(tmpbuf); + if (result.error() == SUCCESS) { + if (check_trailing && !_json_iter->is_single_token()) { return TRAILING_CONTENT; } + advance_root_scalar("bool"); + } + return result; + } + simdjson_inline simdjson_result value_iterator::is_root_null(bool check_trailing) noexcept { + auto max_len = peek_root_length(); + auto json = peek_root_scalar("null"); + bool result = (max_len >= 4 && !atomparsing::str4ncmp(json, "null") && + (max_len == 4 || jsoncharutils::is_structural_or_whitespace(json[4]))); + if (result) { // we have something that looks like a null. + if (check_trailing && !_json_iter->is_single_token()) { return TRAILING_CONTENT; } + advance_root_scalar("null"); + } + return result; + } + + simdjson_warn_unused simdjson_inline error_code value_iterator::skip_child() noexcept { + SIMDJSON_ASSUME(_json_iter->token._position > _start_position); + SIMDJSON_ASSUME(_json_iter->_depth >= _depth); + + return _json_iter->skip_child(depth()); + } + + simdjson_inline value_iterator value_iterator::child() const noexcept { + assert_at_child(); + return { _json_iter, depth() + 1, _json_iter->token.position() }; + } + + // GCC 7 warns when the first line of this function is inlined away into oblivion due to the caller + // relating depth and iterator depth, which is a desired effect. It does not happen if is_open is + // marked non-inline. + SIMDJSON_PUSH_DISABLE_WARNINGS + SIMDJSON_DISABLE_STRICT_OVERFLOW_WARNING + simdjson_inline bool value_iterator::is_open() const noexcept { + return _json_iter->depth() >= depth(); + } + SIMDJSON_POP_DISABLE_WARNINGS + + simdjson_inline bool value_iterator::at_end() const noexcept { + return _json_iter->at_end(); + } + + simdjson_inline bool value_iterator::at_start() const noexcept { + return _json_iter->token.position() == start_position(); + } + + simdjson_inline bool value_iterator::at_first_field() const noexcept { + SIMDJSON_ASSUME(_json_iter->token._position > _start_position); + return _json_iter->token.position() == start_position() + 1; + } + + simdjson_inline void value_iterator::abandon() noexcept { + _json_iter->abandon(); + } + + simdjson_warn_unused simdjson_inline depth_t value_iterator::depth() const noexcept { + return _depth; + } + simdjson_warn_unused simdjson_inline error_code value_iterator::error() const noexcept { + return _json_iter->error; + } + simdjson_warn_unused simdjson_inline uint8_t*& value_iterator::string_buf_loc() noexcept { + return _json_iter->string_buf_loc(); + } + simdjson_warn_unused simdjson_inline const json_iterator& value_iterator::json_iter() const noexcept { + return *_json_iter; + } + simdjson_warn_unused simdjson_inline json_iterator& value_iterator::json_iter() noexcept { + return *_json_iter; + } + + simdjson_inline const uint8_t* value_iterator::peek_start() const noexcept { + return _json_iter->peek(start_position()); + } + simdjson_inline uint32_t value_iterator::peek_start_length() const noexcept { + return _json_iter->peek_length(start_position()); + } + simdjson_inline uint32_t value_iterator::peek_root_length() const noexcept { + return _json_iter->peek_root_length(start_position()); + } + + simdjson_inline const uint8_t* value_iterator::peek_scalar(const char* type) noexcept { + logger::log_value(*_json_iter, start_position(), depth(), type); + // If we're not at the position anymore, we don't want to advance the cursor. + if (!is_at_start()) { return peek_start(); } + + // Get the JSON and advance the cursor, decreasing depth to signify that we have retrieved the value. + assert_at_start(); + return _json_iter->peek(); + } + + simdjson_inline void value_iterator::advance_scalar(const char* type) noexcept { + logger::log_value(*_json_iter, start_position(), depth(), type); + // If we're not at the position anymore, we don't want to advance the cursor. + if (!is_at_start()) { return; } + + // Get the JSON and advance the cursor, decreasing depth to signify that we have retrieved the value. + assert_at_start(); + _json_iter->return_current_and_advance(); + _json_iter->ascend_to(depth() - 1); + } + + simdjson_inline error_code value_iterator::start_container(uint8_t start_char, const char* incorrect_type_message, const char* type) noexcept { + logger::log_start_value(*_json_iter, start_position(), depth(), type); + // If we're not at the position anymore, we don't want to advance the cursor. + const uint8_t* json; + if (!is_at_start()) { +#if SIMDJSON_DEVELOPMENT_CHECKS + if (!is_at_iterator_start()) { return OUT_OF_ORDER_ITERATION; } +#endif + json = peek_start(); + if (*json != start_char) { return incorrect_type_error(incorrect_type_message); } + } + else { + assert_at_start(); + /** + * We should be prudent. Let us peek. If it is not the right type, we + * return an error. Only once we have determined that we have the right + * type are we allowed to advance! + */ + json = _json_iter->peek(); + if (*json != start_char) { return incorrect_type_error(incorrect_type_message); } + _json_iter->return_current_and_advance(); + } + + + return SUCCESS; + } + + + simdjson_inline const uint8_t* value_iterator::peek_root_scalar(const char* type) noexcept { + logger::log_value(*_json_iter, start_position(), depth(), type); + if (!is_at_start()) { return peek_start(); } + + assert_at_root(); + return _json_iter->peek(); + } + simdjson_inline const uint8_t* value_iterator::peek_non_root_scalar(const char* type) noexcept { + logger::log_value(*_json_iter, start_position(), depth(), type); + if (!is_at_start()) { return peek_start(); } + + assert_at_non_root_start(); + return _json_iter->peek(); + } + + simdjson_inline void value_iterator::advance_root_scalar(const char* type) noexcept { + logger::log_value(*_json_iter, start_position(), depth(), type); + if (!is_at_start()) { return; } + + assert_at_root(); + _json_iter->return_current_and_advance(); + _json_iter->ascend_to(depth() - 1); + } + simdjson_inline void value_iterator::advance_non_root_scalar(const char* type) noexcept { + logger::log_value(*_json_iter, start_position(), depth(), type); + if (!is_at_start()) { return; } + + assert_at_non_root_start(); + _json_iter->return_current_and_advance(); + _json_iter->ascend_to(depth() - 1); + } + + simdjson_inline error_code value_iterator::incorrect_type_error(const char* message) const noexcept { + logger::log_error(*_json_iter, start_position(), depth(), message); + return INCORRECT_TYPE; + } + + simdjson_inline bool value_iterator::is_at_start() const noexcept { + return position() == start_position(); + } + + simdjson_inline bool value_iterator::is_at_key() const noexcept { + // Keys are at the same depth as the object. + // Note here that we could be safer and check that we are within an object, + // but we do not. + return _depth == _json_iter->_depth && *_json_iter->peek() == '"'; + } + + simdjson_inline bool value_iterator::is_at_iterator_start() const noexcept { + // We can legitimately be either at the first value ([1]), or after the array if it's empty ([]). + auto delta = position() - start_position(); + return delta == 1 || delta == 2; + } + + inline void value_iterator::assert_at_start() const noexcept { + SIMDJSON_ASSUME(_json_iter->token._position == _start_position); + SIMDJSON_ASSUME(_json_iter->_depth == _depth); + SIMDJSON_ASSUME(_depth > 0); + } + + inline void value_iterator::assert_at_container_start() const noexcept { + SIMDJSON_ASSUME(_json_iter->token._position == _start_position + 1); + SIMDJSON_ASSUME(_json_iter->_depth == _depth); + SIMDJSON_ASSUME(_depth > 0); + } + + inline void value_iterator::assert_at_next() const noexcept { + SIMDJSON_ASSUME(_json_iter->token._position > _start_position); + SIMDJSON_ASSUME(_json_iter->_depth == _depth); + SIMDJSON_ASSUME(_depth > 0); + } + + simdjson_inline void value_iterator::move_at_start() noexcept { + _json_iter->_depth = _depth; + _json_iter->token.set_position(_start_position); + } + + simdjson_inline void value_iterator::move_at_container_start() noexcept { + _json_iter->_depth = _depth; + _json_iter->token.set_position(_start_position + 1); + } + + simdjson_inline simdjson_result value_iterator::reset_array() noexcept { + if (error()) { return error(); } + move_at_container_start(); + return started_array(); + } + + simdjson_inline simdjson_result value_iterator::reset_object() noexcept { + if (error()) { return error(); } + move_at_container_start(); + return started_object(); + } + + inline void value_iterator::assert_at_child() const noexcept { + SIMDJSON_ASSUME(_json_iter->token._position > _start_position); + SIMDJSON_ASSUME(_json_iter->_depth == _depth + 1); + SIMDJSON_ASSUME(_depth > 0); + } + + inline void value_iterator::assert_at_root() const noexcept { + assert_at_start(); + SIMDJSON_ASSUME(_depth == 1); + } + + inline void value_iterator::assert_at_non_root_start() const noexcept { + assert_at_start(); + SIMDJSON_ASSUME(_depth > 1); + } + + inline void value_iterator::assert_is_valid() const noexcept { + SIMDJSON_ASSUME(_json_iter != nullptr); + } + + simdjson_inline bool value_iterator::is_valid() const noexcept { + return _json_iter != nullptr; + } + + simdjson_inline simdjson_result value_iterator::type() const noexcept { + switch (*peek_start()) { + case '{': + return json_type::object; + case '[': + return json_type::array; + case '"': + return json_type::string; + case 'n': + return json_type::null; + case 't': case 'f': + return json_type::boolean; + case '-': + case '0': case '1': case '2': case '3': case '4': + case '5': case '6': case '7': case '8': case '9': + return json_type::number; + default: + return TAPE_ERROR; + } + } + + simdjson_inline token_position value_iterator::start_position() const noexcept { + return _start_position; + } + + simdjson_inline token_position value_iterator::position() const noexcept { + return _json_iter->position(); + } + + simdjson_inline token_position value_iterator::end_position() const noexcept { + return _json_iter->end_position(); + } + + simdjson_inline token_position value_iterator::last_position() const noexcept { + return _json_iter->last_position(); + } + + simdjson_inline error_code value_iterator::report_error(error_code error, const char* message) noexcept { + return _json_iter->report_error(error, message); + } + + } // namespace ondemand + } // namespace lsx +} // namespace simdjson + +namespace simdjson { + + simdjson_inline simdjson_result::simdjson_result(lsx::ondemand::value_iterator&& value) noexcept + : implementation_simdjson_result_base(std::forward(value)) {} + simdjson_inline simdjson_result::simdjson_result(error_code error) noexcept + : implementation_simdjson_result_base(error) {} + +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_ONDEMAND_VALUE_ITERATOR_INL_H +/* end file simdjson/generic/ondemand/value_iterator-inl.h for lsx */ +/* end file simdjson/generic/ondemand/amalgamated.h for lsx */ +/* including simdjson/lsx/end.h: #include "simdjson/lsx/end.h" */ +/* begin file simdjson/lsx/end.h */ +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/lsx/base.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +#undef SIMDJSON_SKIP_BACKSLASH_SHORT_CIRCUIT +/* undefining SIMDJSON_IMPLEMENTATION from "lsx" */ +#undef SIMDJSON_IMPLEMENTATION +/* end file simdjson/lsx/end.h */ + +#endif // SIMDJSON_LSX_ONDEMAND_H +/* end file simdjson/lsx/ondemand.h */ +#elif SIMDJSON_BUILTIN_IMPLEMENTATION_IS(lasx) +/* including simdjson/lasx/ondemand.h: #include "simdjson/lasx/ondemand.h" */ +/* begin file simdjson/lasx/ondemand.h */ +#ifndef SIMDJSON_LASX_ONDEMAND_H +#define SIMDJSON_LASX_ONDEMAND_H + +/* including simdjson/lasx/begin.h: #include "simdjson/lasx/begin.h" */ +/* begin file simdjson/lasx/begin.h */ +/* defining SIMDJSON_IMPLEMENTATION to "lasx" */ +#define SIMDJSON_IMPLEMENTATION lasx +/* including simdjson/lasx/base.h: #include "simdjson/lasx/base.h" */ +/* begin file simdjson/lasx/base.h */ +#ifndef SIMDJSON_LASX_BASE_H +#define SIMDJSON_LASX_BASE_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/base.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + /** + * Implementation for LASX. + */ + namespace lasx { + + class implementation; + + namespace { + namespace simd { + template struct simd8; + template struct simd8x64; + } // namespace simd + } // unnamed namespace + + } // namespace lasx +} // namespace simdjson + +#endif // SIMDJSON_LASX_BASE_H +/* end file simdjson/lasx/base.h */ +/* including simdjson/lasx/intrinsics.h: #include "simdjson/lasx/intrinsics.h" */ +/* begin file simdjson/lasx/intrinsics.h */ +#ifndef SIMDJSON_LASX_INTRINSICS_H +#define SIMDJSON_LASX_INTRINSICS_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/lasx/base.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +// This should be the correct header whether +// you use visual studio or other compilers. +#include + +static_assert(sizeof(__m256i) <= simdjson::SIMDJSON_PADDING, "insufficient padding for LoongArch ASX"); + +#endif // SIMDJSON_LASX_INTRINSICS_H +/* end file simdjson/lasx/intrinsics.h */ +/* including simdjson/lasx/bitmanipulation.h: #include "simdjson/lasx/bitmanipulation.h" */ +/* begin file simdjson/lasx/bitmanipulation.h */ +#ifndef SIMDJSON_LASX_BITMANIPULATION_H +#define SIMDJSON_LASX_BITMANIPULATION_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/lasx/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/lasx/intrinsics.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/lasx/bitmask.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lasx { + namespace { + + // We sometimes call trailing_zero on inputs that are zero, + // but the algorithms do not end up using the returned value. + // Sadly, sanitizers are not smart enough to figure it out. + SIMDJSON_NO_SANITIZE_UNDEFINED + // This function can be used safely even if not all bytes have been + // initialized. + // See issue https://github.com/simdjson/simdjson/issues/1965 + SIMDJSON_NO_SANITIZE_MEMORY + simdjson_inline int trailing_zeroes(uint64_t input_num) { + return __builtin_ctzll(input_num); + } + + /* result might be undefined when input_num is zero */ + simdjson_inline uint64_t clear_lowest_bit(uint64_t input_num) { + return input_num & (input_num - 1); + } + + /* result might be undefined when input_num is zero */ + simdjson_inline int leading_zeroes(uint64_t input_num) { + return __builtin_clzll(input_num); + } + + /* result might be undefined when input_num is zero */ + simdjson_inline int count_ones(uint64_t input_num) { + return __lasx_xvpickve2gr_w(__lasx_xvpcnt_d(__m256i(v4u64{ input_num, 0, 0, 0 })), 0); + } + + simdjson_inline bool add_overflow(uint64_t value1, uint64_t value2, uint64_t* result) { + return __builtin_uaddll_overflow(value1, value2, + reinterpret_cast(result)); + } + + } // unnamed namespace + } // namespace lasx +} // namespace simdjson + +#endif // SIMDJSON_LASX_BITMANIPULATION_H +/* end file simdjson/lasx/bitmanipulation.h */ +/* including simdjson/lasx/bitmask.h: #include "simdjson/lasx/bitmask.h" */ +/* begin file simdjson/lasx/bitmask.h */ +#ifndef SIMDJSON_LASX_BITMASK_H +#define SIMDJSON_LASX_BITMASK_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/lasx/base.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lasx { + namespace { + + // + // Perform a "cumulative bitwise xor," flipping bits each time a 1 is encountered. + // + // For example, prefix_xor(00100100) == 00011100 + // + simdjson_inline uint64_t prefix_xor(uint64_t bitmask) { + bitmask ^= bitmask << 1; + bitmask ^= bitmask << 2; + bitmask ^= bitmask << 4; + bitmask ^= bitmask << 8; + bitmask ^= bitmask << 16; + bitmask ^= bitmask << 32; + return bitmask; + } + + } // unnamed namespace + } // namespace lasx +} // namespace simdjson + +#endif +/* end file simdjson/lasx/bitmask.h */ +/* including simdjson/lasx/numberparsing_defs.h: #include "simdjson/lasx/numberparsing_defs.h" */ +/* begin file simdjson/lasx/numberparsing_defs.h */ +#ifndef SIMDJSON_LASX_NUMBERPARSING_DEFS_H +#define SIMDJSON_LASX_NUMBERPARSING_DEFS_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/lasx/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/lasx/intrinsics.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/numberparsing_tables.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +#include + +namespace simdjson { + namespace lasx { + namespace numberparsing { + + // we don't have appropriate instructions, so let us use a scalar function + // credit: https://johnnylee-sde.github.io/Fast-numeric-string-to-int/ + /** @private */ + static simdjson_inline uint32_t parse_eight_digits_unrolled(const uint8_t* chars) { + uint64_t val; + std::memcpy(&val, chars, sizeof(uint64_t)); + val = (val & 0x0F0F0F0F0F0F0F0F) * 2561 >> 8; + val = (val & 0x00FF00FF00FF00FF) * 6553601 >> 16; + return uint32_t((val & 0x0000FFFF0000FFFF) * 42949672960001 >> 32); + } + + simdjson_inline internal::value128 full_multiplication(uint64_t value1, uint64_t value2) { + internal::value128 answer; + __uint128_t r = (static_cast<__uint128_t>(value1)) * value2; + answer.low = uint64_t(r); + answer.high = uint64_t(r >> 64); + return answer; + } + + } // namespace numberparsing + } // namespace lasx +} // namespace simdjson + +#define SIMDJSON_SWAR_NUMBER_PARSING 1 + +#endif // SIMDJSON_LASX_NUMBERPARSING_DEFS_H +/* end file simdjson/lasx/numberparsing_defs.h */ +/* including simdjson/lasx/simd.h: #include "simdjson/lasx/simd.h" */ +/* begin file simdjson/lasx/simd.h */ +#ifndef SIMDJSON_LASX_SIMD_H +#define SIMDJSON_LASX_SIMD_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/lasx/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/lasx/bitmanipulation.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/simdprune_tables.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lasx { + namespace { + namespace simd { + + // Forward-declared so they can be used by splat and friends. + template + struct base { + __m256i value; + + // Zero constructor + simdjson_inline base() : value{ __m256i() } {} + + // Conversion from SIMD register + simdjson_inline base(const __m256i _value) : value(_value) {} + + // Conversion to SIMD register + simdjson_inline operator const __m256i& () const { return this->value; } + simdjson_inline operator __m256i& () { return this->value; } + simdjson_inline operator const v32i8& () const { return (v32i8&)this->value; } + simdjson_inline operator v32i8& () { return (v32i8&)this->value; } + + // Bit operations + simdjson_inline Child operator|(const Child other) const { return __lasx_xvor_v(*this, other); } + simdjson_inline Child operator&(const Child other) const { return __lasx_xvand_v(*this, other); } + simdjson_inline Child operator^(const Child other) const { return __lasx_xvxor_v(*this, other); } + simdjson_inline Child bit_andnot(const Child other) const { return __lasx_xvandn_v(other, *this); } + simdjson_inline Child& operator|=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast | other; return *this_cast; } + simdjson_inline Child& operator&=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast & other; return *this_cast; } + simdjson_inline Child& operator^=(const Child other) { auto this_cast = static_cast(this); *this_cast = *this_cast ^ other; return *this_cast; } + }; + + // Forward-declared so they can be used by splat and friends. + template + struct simd8; + + template> + struct base8 : base> { + simdjson_inline base8() : base>() {} + simdjson_inline base8(const __m256i _value) : base>(_value) {} + + friend simdjson_really_inline Mask operator==(const simd8 lhs, const simd8 rhs) { return __lasx_xvseq_b(lhs, rhs); } + + static const int SIZE = sizeof(base>::value); + + template + simdjson_inline simd8 prev(const simd8 prev_chunk) const { + __m256i hi = __lasx_xvbsll_v(*this, N); + __m256i lo = __lasx_xvbsrl_v(*this, 16 - N); + __m256i tmp = __lasx_xvbsrl_v(prev_chunk, 16 - N); + lo = __lasx_xvpermi_q(lo, tmp, 0x21); + return __lasx_xvor_v(hi, lo); + } + }; + + // SIMD byte mask type (returned by things like eq and gt) + template<> + struct simd8 : base8 { + static simdjson_inline simd8 splat(bool _value) { return __lasx_xvreplgr2vr_b(uint8_t(-(!!_value))); } + + simdjson_inline simd8() : base8() {} + simdjson_inline simd8(const __m256i _value) : base8(_value) {} + // Splat constructor + simdjson_inline simd8(bool _value) : base8(splat(_value)) {} + + simdjson_inline int to_bitmask() const { + __m256i mask = __lasx_xvmskltz_b(*this); + return (__lasx_xvpickve2gr_w(mask, 4) << 16) | (__lasx_xvpickve2gr_w(mask, 0)); + } + simdjson_inline bool any() const { + __m256i v = __lasx_xvmsknz_b(*this); + return (0 == __lasx_xvpickve2gr_w(v, 0)) && (0 == __lasx_xvpickve2gr_w(v, 4)); + } + simdjson_inline simd8 operator~() const { return *this ^ true; } + }; + + template + struct base8_numeric : base8 { + static simdjson_inline simd8 splat(T _value) { + return __lasx_xvreplgr2vr_b(_value); + } + static simdjson_inline simd8 zero() { return __lasx_xvldi(0); } + static simdjson_inline simd8 load(const T values[32]) { + return __lasx_xvld(reinterpret_cast(values), 0); + } + // Repeat 16 values as many times as necessary (usually for lookup tables) + static simdjson_inline simd8 repeat_16( + T v0, T v1, T v2, T v3, T v4, T v5, T v6, T v7, + T v8, T v9, T v10, T v11, T v12, T v13, T v14, T v15 + ) { + return simd8( + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15, + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15 + ); + } + + simdjson_inline base8_numeric() : base8() {} + simdjson_inline base8_numeric(const __m256i _value) : base8(_value) {} + + // Store to array + simdjson_inline void store(T dst[32]) const { + return __lasx_xvst(*this, reinterpret_cast<__m256i*>(dst), 0); + } + + // Addition/subtraction are the same for signed and unsigned + simdjson_inline simd8 operator+(const simd8 other) const { return __lasx_xvadd_b(*this, other); } + simdjson_inline simd8 operator-(const simd8 other) const { return __lasx_xvsub_b(*this, other); } + simdjson_inline simd8& operator+=(const simd8 other) { *this = *this + other; return *static_cast*>(this); } + simdjson_inline simd8& operator-=(const simd8 other) { *this = *this - other; return *static_cast*>(this); } + + // Override to distinguish from bool version + simdjson_inline simd8 operator~() const { return *this ^ 0xFFu; } + + // Perform a lookup assuming the value is between 0 and 16 (undefined behavior for out of range values) + template + simdjson_inline simd8 lookup_16(simd8 lookup_table) const { + return __lasx_xvshuf_b(lookup_table, lookup_table, *this); + } + + // Copies to 'output" all bytes corresponding to a 0 in the mask (interpreted as a bitset). + // Passing a 0 value for mask would be equivalent to writing out every byte to output. + // Only the first 16 - count_ones(mask) bytes of the result are significant but 16 bytes + // get written. + template + simdjson_inline void compress(uint32_t mask, L* output) const { + using internal::thintable_epi8; + using internal::BitsSetTable256mul2; + using internal::pshufb_combine_table; + // this particular implementation was inspired by haswell + // lasx do it in 4 steps, first 8 bytes and then second 8 bytes... + uint8_t mask1 = uint8_t(mask); // least significant 8 bits + uint8_t mask2 = uint8_t(mask >> 8); // second significant 8 bits + uint8_t mask3 = uint8_t(mask >> 16); // ... + uint8_t mask4 = uint8_t(mask >> 24); // ... + // next line just loads the 64-bit values thintable_epi8[mask{1,2,3,4}] + // into a 256-bit register. + __m256i shufmask = { int64_t(thintable_epi8[mask1]), int64_t(thintable_epi8[mask2]) + 0x0808080808080808, int64_t(thintable_epi8[mask3]), int64_t(thintable_epi8[mask4]) + 0x0808080808080808 }; + // this is the version "nearly pruned" + __m256i pruned = __lasx_xvshuf_b(*this, *this, shufmask); + // we still need to put the pieces back together. + // we compute the popcount of the first words: + int pop1 = BitsSetTable256mul2[mask1]; + int pop2 = BitsSetTable256mul2[mask2]; + int pop3 = BitsSetTable256mul2[mask3]; + + // then load the corresponding mask + __m256i masklo = __lasx_xvldx(reinterpret_cast(reinterpret_cast(pshufb_combine_table)), pop1 * 8); + __m256i maskhi = __lasx_xvldx(reinterpret_cast(reinterpret_cast(pshufb_combine_table)), pop3 * 8); + __m256i compactmask = __lasx_xvpermi_q(maskhi, masklo, 0x20); + __m256i answer = __lasx_xvshuf_b(pruned, pruned, compactmask); + __lasx_xvst(answer, reinterpret_cast(output), 0); + uint64_t value3 = __lasx_xvpickve2gr_du(answer, 2); + uint64_t value4 = __lasx_xvpickve2gr_du(answer, 3); + uint64_t* pos = reinterpret_cast(reinterpret_cast(output) + 16 - (pop1 + pop2) / 2); + pos[0] = value3; + pos[1] = value4; + } + + template + simdjson_inline simd8 lookup_16( + L replace0, L replace1, L replace2, L replace3, + L replace4, L replace5, L replace6, L replace7, + L replace8, L replace9, L replace10, L replace11, + L replace12, L replace13, L replace14, L replace15) const { + return lookup_16(simd8::repeat_16( + replace0, replace1, replace2, replace3, + replace4, replace5, replace6, replace7, + replace8, replace9, replace10, replace11, + replace12, replace13, replace14, replace15 + )); + } + }; + + // Signed bytes + template<> + struct simd8 : base8_numeric { + simdjson_inline simd8() : base8_numeric() {} + simdjson_inline simd8(const __m256i _value) : base8_numeric(_value) {} + // Splat constructor + simdjson_inline simd8(int8_t _value) : simd8(splat(_value)) {} + // Array constructor + simdjson_inline simd8(const int8_t values[32]) : simd8(load(values)) {} + // Member-by-member initialization + simdjson_inline simd8( + int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, + int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15, + int8_t v16, int8_t v17, int8_t v18, int8_t v19, int8_t v20, int8_t v21, int8_t v22, int8_t v23, + int8_t v24, int8_t v25, int8_t v26, int8_t v27, int8_t v28, int8_t v29, int8_t v30, int8_t v31 + ) : simd8({ + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10,v11,v12,v13,v14,v15, + v16,v17,v18,v19,v20,v21,v22,v23, + v24,v25,v26,v27,v28,v29,v30,v31 + }) {} + // Repeat 16 values as many times as necessary (usually for lookup tables) + simdjson_inline static simd8 repeat_16( + int8_t v0, int8_t v1, int8_t v2, int8_t v3, int8_t v4, int8_t v5, int8_t v6, int8_t v7, + int8_t v8, int8_t v9, int8_t v10, int8_t v11, int8_t v12, int8_t v13, int8_t v14, int8_t v15 + ) { + return simd8( + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15, + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15 + ); + } + + // Order-sensitive comparisons + simdjson_inline simd8 max_val(const simd8 other) const { return __lasx_xvmax_b(*this, other); } + simdjson_inline simd8 min_val(const simd8 other) const { return __lasx_xvmin_b(*this, other); } + simdjson_inline simd8 operator>(const simd8 other) const { return __lasx_xvslt_b(other, *this); } + simdjson_inline simd8 operator<(const simd8 other) const { return __lasx_xvslt_b(*this, other); } + }; + + // Unsigned bytes + template<> + struct simd8 : base8_numeric { + simdjson_inline simd8() : base8_numeric() {} + simdjson_inline simd8(const __m256i _value) : base8_numeric(_value) {} + // Splat constructor + simdjson_inline simd8(uint8_t _value) : simd8(splat(_value)) {} + // Array constructor + simdjson_inline simd8(const uint8_t values[32]) : simd8(load(values)) {} + // Member-by-member initialization + simdjson_inline simd8( + uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, + uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15, + uint8_t v16, uint8_t v17, uint8_t v18, uint8_t v19, uint8_t v20, uint8_t v21, uint8_t v22, uint8_t v23, + uint8_t v24, uint8_t v25, uint8_t v26, uint8_t v27, uint8_t v28, uint8_t v29, uint8_t v30, uint8_t v31 + ) : simd8(__m256i(v32u8{ + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10,v11,v12,v13,v14,v15, + v16,v17,v18,v19,v20,v21,v22,v23, + v24,v25,v26,v27,v28,v29,v30,v31 + })) {} + // Repeat 16 values as many times as necessary (usually for lookup tables) + simdjson_inline static simd8 repeat_16( + uint8_t v0, uint8_t v1, uint8_t v2, uint8_t v3, uint8_t v4, uint8_t v5, uint8_t v6, uint8_t v7, + uint8_t v8, uint8_t v9, uint8_t v10, uint8_t v11, uint8_t v12, uint8_t v13, uint8_t v14, uint8_t v15 + ) { + return simd8( + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15, + v0, v1, v2, v3, v4, v5, v6, v7, + v8, v9, v10, v11, v12, v13, v14, v15 + ); + } + + // Saturated math + simdjson_inline simd8 saturating_add(const simd8 other) const { return __lasx_xvsadd_bu(*this, other); } + simdjson_inline simd8 saturating_sub(const simd8 other) const { return __lasx_xvssub_bu(*this, other); } + + // Order-specific operations + simdjson_inline simd8 max_val(const simd8 other) const { return __lasx_xvmax_bu(*this, other); } + simdjson_inline simd8 min_val(const simd8 other) const { return __lasx_xvmin_bu(other, *this); } + // Same as >, but only guarantees true is nonzero (< guarantees true = -1) + simdjson_inline simd8 gt_bits(const simd8 other) const { return this->saturating_sub(other); } + // Same as <, but only guarantees true is nonzero (< guarantees true = -1) + simdjson_inline simd8 lt_bits(const simd8 other) const { return other.saturating_sub(*this); } + simdjson_inline simd8 operator<=(const simd8 other) const { return other.max_val(*this) == other; } + simdjson_inline simd8 operator>=(const simd8 other) const { return other.min_val(*this) == other; } + simdjson_inline simd8 operator>(const simd8 other) const { return this->gt_bits(other).any_bits_set(); } + simdjson_inline simd8 operator<(const simd8 other) const { return this->lt_bits(other).any_bits_set(); } + + // Bit-specific operations + simdjson_inline simd8 bits_not_set() const { return *this == uint8_t(0); } + simdjson_inline simd8 bits_not_set(simd8 bits) const { return (*this & bits).bits_not_set(); } + simdjson_inline simd8 any_bits_set() const { return ~this->bits_not_set(); } + simdjson_inline simd8 any_bits_set(simd8 bits) const { return ~this->bits_not_set(bits); } + simdjson_inline bool is_ascii() const { + __m256i mask = __lasx_xvmskltz_b(*this); + return (0 == __lasx_xvpickve2gr_w(mask, 0)) && (0 == __lasx_xvpickve2gr_w(mask, 4)); + } + simdjson_inline bool bits_not_set_anywhere() const { + __m256i v = __lasx_xvmsknz_b(*this); + return (0 == __lasx_xvpickve2gr_w(v, 0)) && (0 == __lasx_xvpickve2gr_w(v, 4)); + } + simdjson_inline bool any_bits_set_anywhere() const { return !bits_not_set_anywhere(); } + simdjson_inline bool bits_not_set_anywhere(simd8 bits) const { + __m256i v = __lasx_xvmsknz_b(__lasx_xvand_v(*this, bits)); + return (0 == __lasx_xvpickve2gr_w(v, 0)) && (0 == __lasx_xvpickve2gr_w(v, 4)); + } + simdjson_inline bool any_bits_set_anywhere(simd8 bits) const { return !bits_not_set_anywhere(bits); } + template + simdjson_inline simd8 shr() const { return simd8(__lasx_xvsrli_b(*this, N)); } + template + simdjson_inline simd8 shl() const { return simd8(__lasx_xvslli_b(*this, N)); } + }; + + template + struct simd8x64 { + static constexpr int NUM_CHUNKS = 64 / sizeof(simd8); + static_assert(NUM_CHUNKS == 2, "LASX kernel should use two registers per 64-byte block."); + const simd8 chunks[NUM_CHUNKS]; + + simd8x64(const simd8x64& o) = delete; // no copy allowed + simd8x64& operator=(const simd8& other) = delete; // no assignment allowed + simd8x64() = delete; // no default constructor allowed + + simdjson_inline simd8x64(const simd8 chunk0, const simd8 chunk1) : chunks{ chunk0, chunk1 } {} + simdjson_inline simd8x64(const T ptr[64]) : chunks{ simd8::load(ptr), simd8::load(ptr + 32) } {} + + simdjson_inline uint64_t compress(uint64_t mask, T* output) const { + uint32_t mask1 = uint32_t(mask); + uint32_t mask2 = uint32_t(mask >> 32); + __m256i zcnt = __lasx_xvpcnt_w(__m256i(v4u64{ ~mask, 0, 0, 0 })); + uint64_t zcnt1 = __lasx_xvpickve2gr_wu(zcnt, 0); + uint64_t zcnt2 = __lasx_xvpickve2gr_wu(zcnt, 1); + // There should be a critical value which processes in scaler is faster. + if (zcnt1) + this->chunks[0].compress(mask1, output); + if (zcnt2) + this->chunks[1].compress(mask2, output + zcnt1); + return zcnt1 + zcnt2; + } + + simdjson_inline void store(T ptr[64]) const { + this->chunks[0].store(ptr + sizeof(simd8) * 0); + this->chunks[1].store(ptr + sizeof(simd8) * 1); + } + + simdjson_inline uint64_t to_bitmask() const { + __m256i mask0 = __lasx_xvmskltz_b(this->chunks[0]); + __m256i mask1 = __lasx_xvmskltz_b(this->chunks[1]); + __m256i mask_tmp = __lasx_xvpickve_w(mask0, 4); + __m256i tmp = __lasx_xvpickve_w(mask1, 4); + mask0 = __lasx_xvinsve0_w(mask0, mask1, 1); + mask_tmp = __lasx_xvinsve0_w(mask_tmp, tmp, 1); + return __lasx_xvpickve2gr_du(__lasx_xvpackev_h(mask_tmp, mask0), 0); + } + + simdjson_inline simd8 reduce_or() const { + return this->chunks[0] | this->chunks[1]; + } + + simdjson_inline uint64_t eq(const T m) const { + const simd8 mask = simd8::splat(m); + return simd8x64( + this->chunks[0] == mask, + this->chunks[1] == mask + ).to_bitmask(); + } + + simdjson_inline uint64_t eq(const simd8x64& other) const { + return simd8x64( + this->chunks[0] == other.chunks[0], + this->chunks[1] == other.chunks[1] + ).to_bitmask(); + } + + simdjson_inline uint64_t lteq(const T m) const { + const simd8 mask = simd8::splat(m); + return simd8x64( + this->chunks[0] <= mask, + this->chunks[1] <= mask + ).to_bitmask(); + } + }; // struct simd8x64 + + } // namespace simd + } // unnamed namespace + } // namespace lasx +} // namespace simdjson + +#endif // SIMDJSON_LASX_SIMD_H +/* end file simdjson/lasx/simd.h */ +/* including simdjson/lasx/stringparsing_defs.h: #include "simdjson/lasx/stringparsing_defs.h" */ +/* begin file simdjson/lasx/stringparsing_defs.h */ +#ifndef SIMDJSON_LASX_STRINGPARSING_DEFS_H +#define SIMDJSON_LASX_STRINGPARSING_DEFS_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/lasx/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/lasx/simd.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/lasx/bitmanipulation.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lasx { + namespace { + + using namespace simd; + + // Holds backslashes and quotes locations. + struct backslash_and_quote { + public: + static constexpr uint32_t BYTES_PROCESSED = 32; + simdjson_inline static backslash_and_quote copy_and_find(const uint8_t* src, uint8_t* dst); + + simdjson_inline bool has_quote_first() { return ((bs_bits - 1) & quote_bits) != 0; } + simdjson_inline bool has_backslash() { return bs_bits != 0; } + simdjson_inline int quote_index() { return trailing_zeroes(quote_bits); } + simdjson_inline int backslash_index() { return trailing_zeroes(bs_bits); } + + uint32_t bs_bits; + uint32_t quote_bits; + }; // struct backslash_and_quote + + simdjson_inline backslash_and_quote backslash_and_quote::copy_and_find(const uint8_t* src, uint8_t* dst) { + // this can read up to 31 bytes beyond the buffer size, but we require + // SIMDJSON_PADDING of padding + static_assert(SIMDJSON_PADDING >= (BYTES_PROCESSED - 1), "backslash and quote finder must process fewer than SIMDJSON_PADDING bytes"); + simd8 v(src); + v.store(dst); + return { + static_cast((v == '\\').to_bitmask()), // bs_bits + static_cast((v == '"').to_bitmask()), // quote_bits + }; + } + + } // unnamed namespace + } // namespace lasx +} // namespace simdjson + +#endif // SIMDJSON_LASX_STRINGPARSING_DEFS_H +/* end file simdjson/lasx/stringparsing_defs.h */ + +#define SIMDJSON_SKIP_BACKSLASH_SHORT_CIRCUIT 1 +/* end file simdjson/lasx/begin.h */ +/* including simdjson/generic/ondemand/amalgamated.h for lasx: #include "simdjson/generic/ondemand/amalgamated.h" */ +/* begin file simdjson/generic/ondemand/amalgamated.h for lasx */ +#if defined(SIMDJSON_CONDITIONAL_INCLUDE) && !defined(SIMDJSON_GENERIC_ONDEMAND_DEPENDENCIES_H) +#error simdjson/generic/ondemand/dependencies.h must be included before simdjson/generic/ondemand/amalgamated.h! +#endif + +// Stuff other things depend on +/* including simdjson/generic/ondemand/base.h for lasx: #include "simdjson/generic/ondemand/base.h" */ +/* begin file simdjson/generic/ondemand/base.h for lasx */ +#ifndef SIMDJSON_GENERIC_ONDEMAND_BASE_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_ONDEMAND_BASE_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/base.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lasx { + /** + * A fast, simple, DOM-like interface that parses JSON as you use it. + * + * Designed for maximum speed and a lower memory profile. + */ + namespace ondemand { + + /** Represents the depth of a JSON value (number of nested arrays/objects). */ + using depth_t = int32_t; + + /** @copydoc simdjson::lasx::number_type */ + using number_type = simdjson::lasx::number_type; + + /** @private Position in the JSON buffer indexes */ + using token_position = const uint32_t*; + + class array; + class array_iterator; + class document; + class document_reference; + class document_stream; + class field; + class json_iterator; + enum class json_type; + struct number; + class object; + class object_iterator; + class parser; + class raw_json_string; + class token_iterator; + class value; + class value_iterator; + + } // namespace ondemand + } // namespace lasx +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_ONDEMAND_BASE_H +/* end file simdjson/generic/ondemand/base.h for lasx */ +/* including simdjson/generic/ondemand/value_iterator.h for lasx: #include "simdjson/generic/ondemand/value_iterator.h" */ +/* begin file simdjson/generic/ondemand/value_iterator.h for lasx */ +#ifndef SIMDJSON_GENERIC_ONDEMAND_VALUE_ITERATOR_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_ONDEMAND_VALUE_ITERATOR_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/implementation_simdjson_result_base.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lasx { + namespace ondemand { + + /** + * Iterates through a single JSON value at a particular depth. + * + * Does not keep track of the type of value: provides methods for objects, arrays and scalars and expects + * the caller to call the right ones. + * + * @private This is not intended for external use. + */ + class value_iterator { + protected: + /** The underlying JSON iterator */ + json_iterator* _json_iter{}; + /** The depth of this value */ + depth_t _depth{}; + /** + * The starting token index for this value + */ + token_position _start_position{}; + + public: + simdjson_inline value_iterator() noexcept = default; + + /** + * Denote that we're starting a document. + */ + simdjson_inline void start_document() noexcept; + + /** + * Skips a non-iterated or partially-iterated JSON value, whether it is a scalar, array or object. + * + * Optimized for scalars. + */ + simdjson_warn_unused simdjson_inline error_code skip_child() noexcept; + + /** + * Tell whether the iterator is at the EOF mark + */ + simdjson_inline bool at_end() const noexcept; + + /** + * Tell whether the iterator is at the start of the value + */ + simdjson_inline bool at_start() const noexcept; + + /** + * Tell whether the value is open--if the value has not been used, or the array/object is still open. + */ + simdjson_inline bool is_open() const noexcept; + + /** + * Tell whether the value is at an object's first field (just after the {). + */ + simdjson_inline bool at_first_field() const noexcept; + + /** + * Abandon all iteration. + */ + simdjson_inline void abandon() noexcept; + + /** + * Get the child value as a value_iterator. + */ + simdjson_inline value_iterator child_value() const noexcept; + + /** + * Get the depth of this value. + */ + simdjson_inline int32_t depth() const noexcept; + + /** + * Get the JSON type of this value. + * + * @error TAPE_ERROR when the JSON value is a bad token like "}" "," or "alse". + */ + simdjson_inline simdjson_result type() const noexcept; + + /** + * @addtogroup object Object iteration + * + * Methods to iterate and find object fields. These methods generally *assume* the value is + * actually an object; the caller is responsible for keeping track of that fact. + * + * @{ + */ + + /** + * Start an object iteration. + * + * @returns Whether the object had any fields (returns false for empty). + * @error INCORRECT_TYPE if there is no opening { + */ + simdjson_warn_unused simdjson_inline simdjson_result start_object() noexcept; + /** + * Start an object iteration from the root. + * + * @returns Whether the object had any fields (returns false for empty). + * @error INCORRECT_TYPE if there is no opening { + * @error TAPE_ERROR if there is no matching } at end of document + */ + simdjson_warn_unused simdjson_inline simdjson_result start_root_object() noexcept; + /** + * Checks whether an object could be started from the root. May be called by start_root_object. + * + * @returns SUCCESS if it is possible to safely start an object from the root (document level). + * @error INCORRECT_TYPE if there is no opening { + * @error TAPE_ERROR if there is no matching } at end of document + */ + simdjson_warn_unused simdjson_inline error_code check_root_object() noexcept; + /** + * Start an object iteration after the user has already checked and moved past the {. + * + * Does not move the iterator unless the object is empty ({}). + * + * @returns Whether the object had any fields (returns false for empty). + * @error INCOMPLETE_ARRAY_OR_OBJECT If there are no more tokens (implying the *parent* + * array or object is incomplete). + */ + simdjson_warn_unused simdjson_inline simdjson_result started_object() noexcept; + /** + * Start an object iteration from the root, after the user has already checked and moved past the {. + * + * Does not move the iterator unless the object is empty ({}). + * + * @returns Whether the object had any fields (returns false for empty). + * @error INCOMPLETE_ARRAY_OR_OBJECT If there are no more tokens (implying the *parent* + * array or object is incomplete). + */ + simdjson_warn_unused simdjson_inline simdjson_result started_root_object() noexcept; + + /** + * Moves to the next field in an object. + * + * Looks for , and }. If } is found, the object is finished and the iterator advances past it. + * Otherwise, it advances to the next value. + * + * @return whether there is another field in the object. + * @error TAPE_ERROR If there is a comma missing between fields. + * @error TAPE_ERROR If there is a comma, but not enough tokens remaining to have a key, :, and value. + */ + simdjson_warn_unused simdjson_inline simdjson_result has_next_field() noexcept; + + /** + * Get the current field's key. + */ + simdjson_warn_unused simdjson_inline simdjson_result field_key() noexcept; + + /** + * Pass the : in the field and move to its value. + */ + simdjson_warn_unused simdjson_inline error_code field_value() noexcept; + + /** + * Find the next field with the given key. + * + * Assumes you have called next_field() or otherwise matched the previous value. + * + * This means the iterator must be sitting at the next key: + * + * ``` + * { "a": 1, "b": 2 } + * ^ + * ``` + * + * Key is *raw JSON,* meaning it will be matched against the verbatim JSON without attempting to + * unescape it. This works well for typical ASCII and UTF-8 keys (almost all of them), but may + * fail to match some keys with escapes (\u, \n, etc.). + */ + simdjson_warn_unused simdjson_inline error_code find_field(const std::string_view key) noexcept; + + /** + * Find the next field with the given key, *without* unescaping. This assumes object order: it + * will not find the field if it was already passed when looking for some *other* field. + * + * Assumes you have called next_field() or otherwise matched the previous value. + * + * This means the iterator must be sitting at the next key: + * + * ``` + * { "a": 1, "b": 2 } + * ^ + * ``` + * + * Key is *raw JSON,* meaning it will be matched against the verbatim JSON without attempting to + * unescape it. This works well for typical ASCII and UTF-8 keys (almost all of them), but may + * fail to match some keys with escapes (\u, \n, etc.). + */ + simdjson_warn_unused simdjson_inline simdjson_result find_field_raw(const std::string_view key) noexcept; + + /** + * Find the field with the given key without regard to order, and *without* unescaping. + * + * This is an unordered object lookup: if the field is not found initially, it will cycle around and scan from the beginning. + * + * Assumes you have called next_field() or otherwise matched the previous value. + * + * This means the iterator must be sitting at the next key: + * + * ``` + * { "a": 1, "b": 2 } + * ^ + * ``` + * + * Key is *raw JSON,* meaning it will be matched against the verbatim JSON without attempting to + * unescape it. This works well for typical ASCII and UTF-8 keys (almost all of them), but may + * fail to match some keys with escapes (\u, \n, etc.). + */ + simdjson_warn_unused simdjson_inline simdjson_result find_field_unordered_raw(const std::string_view key) noexcept; + + /** @} */ + + /** + * @addtogroup array Array iteration + * Methods to iterate over array elements. These methods generally *assume* the value is actually + * an object; the caller is responsible for keeping track of that fact. + * @{ + */ + + /** + * Check for an opening [ and start an array iteration. + * + * @returns Whether the array had any elements (returns false for empty). + * @error INCORRECT_TYPE If there is no [. + */ + simdjson_warn_unused simdjson_inline simdjson_result start_array() noexcept; + /** + * Check for an opening [ and start an array iteration while at the root. + * + * @returns Whether the array had any elements (returns false for empty). + * @error INCORRECT_TYPE If there is no [. + * @error TAPE_ERROR if there is no matching ] at end of document + */ + simdjson_warn_unused simdjson_inline simdjson_result start_root_array() noexcept; + /** + * Checks whether an array could be started from the root. May be called by start_root_array. + * + * @returns SUCCESS if it is possible to safely start an array from the root (document level). + * @error INCORRECT_TYPE If there is no [. + * @error TAPE_ERROR if there is no matching ] at end of document + */ + simdjson_warn_unused simdjson_inline error_code check_root_array() noexcept; + /** + * Start an array iteration, after the user has already checked and moved past the [. + * + * Does not move the iterator unless the array is empty ([]). + * + * @returns Whether the array had any elements (returns false for empty). + * @error INCOMPLETE_ARRAY_OR_OBJECT If there are no more tokens (implying the *parent* + * array or object is incomplete). + */ + simdjson_warn_unused simdjson_inline simdjson_result started_array() noexcept; + /** + * Start an array iteration from the root, after the user has already checked and moved past the [. + * + * Does not move the iterator unless the array is empty ([]). + * + * @returns Whether the array had any elements (returns false for empty). + * @error INCOMPLETE_ARRAY_OR_OBJECT If there are no more tokens (implying the *parent* + * array or object is incomplete). + */ + simdjson_warn_unused simdjson_inline simdjson_result started_root_array() noexcept; + + /** + * Moves to the next element in an array. + * + * Looks for , and ]. If ] is found, the array is finished and the iterator advances past it. + * Otherwise, it advances to the next value. + * + * @return Whether there is another element in the array. + * @error TAPE_ERROR If there is a comma missing between elements. + */ + simdjson_warn_unused simdjson_inline simdjson_result has_next_element() noexcept; + + /** + * Get a child value iterator. + */ + simdjson_warn_unused simdjson_inline value_iterator child() const noexcept; + + /** @} */ + + /** + * @defgroup scalar Scalar values + * @addtogroup scalar + * @{ + */ + + simdjson_warn_unused simdjson_inline simdjson_result get_string(bool allow_replacement) noexcept; + template + simdjson_warn_unused simdjson_inline error_code get_string(string_type& receiver, bool allow_replacement) noexcept; + simdjson_warn_unused simdjson_inline simdjson_result get_wobbly_string() noexcept; + simdjson_warn_unused simdjson_inline simdjson_result get_raw_json_string() noexcept; + simdjson_warn_unused simdjson_inline simdjson_result get_uint64() noexcept; + simdjson_warn_unused simdjson_inline simdjson_result get_uint64_in_string() noexcept; + simdjson_warn_unused simdjson_inline simdjson_result get_int64() noexcept; + simdjson_warn_unused simdjson_inline simdjson_result get_int64_in_string() noexcept; + simdjson_warn_unused simdjson_inline simdjson_result get_double() noexcept; + simdjson_warn_unused simdjson_inline simdjson_result get_double_in_string() noexcept; + simdjson_warn_unused simdjson_inline simdjson_result get_bool() noexcept; + simdjson_warn_unused simdjson_inline simdjson_result is_null() noexcept; + simdjson_warn_unused simdjson_inline bool is_negative() noexcept; + simdjson_warn_unused simdjson_inline simdjson_result is_integer() noexcept; + simdjson_warn_unused simdjson_inline simdjson_result get_number_type() noexcept; + simdjson_warn_unused simdjson_inline simdjson_result get_number() noexcept; + + simdjson_warn_unused simdjson_inline simdjson_result get_root_string(bool check_trailing, bool allow_replacement) noexcept; + template + simdjson_warn_unused simdjson_inline error_code get_root_string(string_type& receiver, bool check_trailing, bool allow_replacement) noexcept; + simdjson_warn_unused simdjson_inline simdjson_result get_root_wobbly_string(bool check_trailing) noexcept; + simdjson_warn_unused simdjson_inline simdjson_result get_root_raw_json_string(bool check_trailing) noexcept; + simdjson_warn_unused simdjson_inline simdjson_result get_root_uint64(bool check_trailing) noexcept; + simdjson_warn_unused simdjson_inline simdjson_result get_root_uint64_in_string(bool check_trailing) noexcept; + simdjson_warn_unused simdjson_inline simdjson_result get_root_int64(bool check_trailing) noexcept; + simdjson_warn_unused simdjson_inline simdjson_result get_root_int64_in_string(bool check_trailing) noexcept; + simdjson_warn_unused simdjson_inline simdjson_result get_root_double(bool check_trailing) noexcept; + simdjson_warn_unused simdjson_inline simdjson_result get_root_double_in_string(bool check_trailing) noexcept; + simdjson_warn_unused simdjson_inline simdjson_result get_root_bool(bool check_trailing) noexcept; + simdjson_warn_unused simdjson_inline bool is_root_negative() noexcept; + simdjson_warn_unused simdjson_inline simdjson_result is_root_integer(bool check_trailing) noexcept; + simdjson_warn_unused simdjson_inline simdjson_result get_root_number_type(bool check_trailing) noexcept; + simdjson_warn_unused simdjson_inline simdjson_result get_root_number(bool check_trailing) noexcept; + simdjson_warn_unused simdjson_inline simdjson_result is_root_null(bool check_trailing) noexcept; + + simdjson_inline error_code error() const noexcept; + simdjson_inline uint8_t*& string_buf_loc() noexcept; + simdjson_inline const json_iterator& json_iter() const noexcept; + simdjson_inline json_iterator& json_iter() noexcept; + + simdjson_inline void assert_is_valid() const noexcept; + simdjson_inline bool is_valid() const noexcept; + + /** @} */ + protected: + /** + * Restarts an array iteration. + * @returns Whether the array has any elements (returns false for empty). + */ + simdjson_inline simdjson_result reset_array() noexcept; + /** + * Restarts an object iteration. + * @returns Whether the object has any fields (returns false for empty). + */ + simdjson_inline simdjson_result reset_object() noexcept; + /** + * move_at_start(): moves us so that we are pointing at the beginning of + * the container. It updates the index so that at_start() is true and it + * syncs the depth. The user can then create a new container instance. + * + * Usage: used with value::count_elements(). + **/ + simdjson_inline void move_at_start() noexcept; + + /** + * move_at_container_start(): moves us so that we are pointing at the beginning of + * the container so that assert_at_container_start() passes. + * + * Usage: used with reset_array() and reset_object(). + **/ + simdjson_inline void move_at_container_start() noexcept; + /* Useful for debugging and logging purposes. */ + inline std::string to_string() const noexcept; + simdjson_inline value_iterator(json_iterator* json_iter, depth_t depth, token_position start_index) noexcept; + + simdjson_inline simdjson_result parse_null(const uint8_t* json) const noexcept; + simdjson_inline simdjson_result parse_bool(const uint8_t* json) const noexcept; + simdjson_inline const uint8_t* peek_start() const noexcept; + simdjson_inline uint32_t peek_start_length() const noexcept; + simdjson_inline uint32_t peek_root_length() const noexcept; + + /** + * The general idea of the advance_... methods and the peek_* methods + * is that you first peek and check that you have desired type. If you do, + * and only if you do, then you advance. + * + * We used to unconditionally advance. But this made reasoning about our + * current state difficult. + * Suppose you always advance. Look at the 'value' matching the key + * "shadowable" in the following example... + * + * ({"globals":{"a":{"shadowable":[}}}}) + * + * If the user thinks it is a Boolean and asks for it, then we check the '[', + * decide it is not a Boolean, but still move into the next character ('}'). Now + * we are left pointing at '}' right after a '['. And we have not yet reported + * an error, only that we do not have a Boolean. + * + * If, instead, you just stand your ground until it is content that you know, then + * you will only even move beyond the '[' if the user tells you that you have an + * array. So you will be at the '}' character inside the array and, hopefully, you + * will then catch the error because an array cannot start with '}', but the code + * processing Boolean values does not know this. + * + * So the contract is: first call 'peek_...' and then call 'advance_...' only + * if you have determined that it is a type you can handle. + * + * Unfortunately, it makes the code more verbose, longer and maybe more error prone. + */ + + simdjson_inline void advance_scalar(const char* type) noexcept; + simdjson_inline void advance_root_scalar(const char* type) noexcept; + simdjson_inline void advance_non_root_scalar(const char* type) noexcept; + + simdjson_inline const uint8_t* peek_scalar(const char* type) noexcept; + simdjson_inline const uint8_t* peek_root_scalar(const char* type) noexcept; + simdjson_inline const uint8_t* peek_non_root_scalar(const char* type) noexcept; + + + simdjson_inline error_code start_container(uint8_t start_char, const char* incorrect_type_message, const char* type) noexcept; + simdjson_inline error_code end_container() noexcept; + + /** + * Advance to a place expecting a value (increasing depth). + * + * @return The current token (the one left behind). + * @error TAPE_ERROR If the document ended early. + */ + simdjson_inline simdjson_result advance_to_value() noexcept; + + simdjson_inline error_code incorrect_type_error(const char* message) const noexcept; + simdjson_inline error_code error_unless_more_tokens(uint32_t tokens = 1) const noexcept; + + simdjson_inline bool is_at_start() const noexcept; + /** + * is_at_iterator_start() returns true on an array or object after it has just been + * created, whether the instance is empty or not. + * + * Usage: used by array::begin() in debug mode (SIMDJSON_DEVELOPMENT_CHECKS) + */ + simdjson_inline bool is_at_iterator_start() const noexcept; + + /** + * Assuming that we are within an object, this returns true if we + * are pointing at a key. + * + * Usage: the skip_child() method should never be used while we are pointing + * at a key inside an object. + */ + simdjson_inline bool is_at_key() const noexcept; + + inline void assert_at_start() const noexcept; + inline void assert_at_container_start() const noexcept; + inline void assert_at_root() const noexcept; + inline void assert_at_child() const noexcept; + inline void assert_at_next() const noexcept; + inline void assert_at_non_root_start() const noexcept; + + /** Get the starting position of this value */ + simdjson_inline token_position start_position() const noexcept; + + /** @copydoc error_code json_iterator::position() const noexcept; */ + simdjson_inline token_position position() const noexcept; + /** @copydoc error_code json_iterator::end_position() const noexcept; */ + simdjson_inline token_position last_position() const noexcept; + /** @copydoc error_code json_iterator::end_position() const noexcept; */ + simdjson_inline token_position end_position() const noexcept; + /** @copydoc error_code json_iterator::report_error(error_code error, const char *message) noexcept; */ + simdjson_inline error_code report_error(error_code error, const char* message) noexcept; + + friend class document; + friend class object; + friend class array; + friend class value; + friend class field; + }; // value_iterator + + } // namespace ondemand + } // namespace lasx +} // namespace simdjson + +namespace simdjson { + + template<> + struct simdjson_result : public lasx::implementation_simdjson_result_base { + public: + simdjson_inline simdjson_result(lasx::ondemand::value_iterator&& value) noexcept; ///< @private + simdjson_inline simdjson_result(error_code error) noexcept; ///< @private + simdjson_inline simdjson_result() noexcept = default; + }; + +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_ONDEMAND_VALUE_ITERATOR_H +/* end file simdjson/generic/ondemand/value_iterator.h for lasx */ +/* including simdjson/generic/ondemand/value.h for lasx: #include "simdjson/generic/ondemand/value.h" */ +/* begin file simdjson/generic/ondemand/value.h for lasx */ +#ifndef SIMDJSON_GENERIC_ONDEMAND_VALUE_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_ONDEMAND_VALUE_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/implementation_simdjson_result_base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/value_iterator.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lasx { + namespace ondemand { + + /** + * An ephemeral JSON value returned during iteration. It is only valid for as long as you do + * not access more data in the JSON document. + */ + class value { + public: + /** + * Create a new invalid value. + * + * Exists so you can declare a variable and later assign to it before use. + */ + simdjson_inline value() noexcept = default; + + /** + * Get this value as the given type. + * + * Supported types: object, array, raw_json_string, string_view, uint64_t, int64_t, double, bool + * + * You may use get_double(), get_bool(), get_uint64(), get_int64(), + * get_object(), get_array(), get_raw_json_string(), or get_string() instead. + * + * @returns A value of the given type, parsed from the JSON. + * @returns INCORRECT_TYPE If the JSON value is not the given type. + */ + template simdjson_inline simdjson_result get() noexcept { + // Unless the simdjson library or the user provides an inline implementation, calling this method should + // immediately fail. + static_assert(!sizeof(T), "The get method with given type is not implemented by the simdjson library. " + "The supported types are ondemand::object, ondemand::array, raw_json_string, std::string_view, uint64_t, " + "int64_t, double, and bool. We recommend you use get_double(), get_bool(), get_uint64(), get_int64(), " + " get_object(), get_array(), get_raw_json_string(), or get_string() instead of the get template." + " You may also add support for custom types, see our documentation."); + } + + /** + * Get this value as the given type. + * + * Supported types: object, array, raw_json_string, string_view, uint64_t, int64_t, double, bool + * + * @param out This is set to a value of the given type, parsed from the JSON. If there is an error, this may not be initialized. + * @returns INCORRECT_TYPE If the JSON value is not an object. + * @returns SUCCESS If the parse succeeded and the out parameter was set to the value. + */ + template simdjson_inline error_code get(T& out) noexcept; + + /** + * Cast this JSON value to an array. + * + * @returns An object that can be used to iterate the array. + * @returns INCORRECT_TYPE If the JSON value is not an array. + */ + simdjson_inline simdjson_result get_array() noexcept; + + /** + * Cast this JSON value to an object. + * + * @returns An object that can be used to look up or iterate fields. + * @returns INCORRECT_TYPE If the JSON value is not an object. + */ + simdjson_inline simdjson_result get_object() noexcept; + + /** + * Cast this JSON value to an unsigned integer. + * + * @returns A unsigned 64-bit integer. + * @returns INCORRECT_TYPE If the JSON value is not a 64-bit unsigned integer. + */ + simdjson_inline simdjson_result get_uint64() noexcept; + + /** + * Cast this JSON value (inside string) to a unsigned integer. + * + * @returns A unsigned 64-bit integer. + * @returns INCORRECT_TYPE If the JSON value is not a 64-bit unsigned integer. + */ + simdjson_inline simdjson_result get_uint64_in_string() noexcept; + + /** + * Cast this JSON value to a signed integer. + * + * @returns A signed 64-bit integer. + * @returns INCORRECT_TYPE If the JSON value is not a 64-bit integer. + */ + simdjson_inline simdjson_result get_int64() noexcept; + + /** + * Cast this JSON value (inside string) to a signed integer. + * + * @returns A signed 64-bit integer. + * @returns INCORRECT_TYPE If the JSON value is not a 64-bit integer. + */ + simdjson_inline simdjson_result get_int64_in_string() noexcept; + + /** + * Cast this JSON value to a double. + * + * @returns A double. + * @returns INCORRECT_TYPE If the JSON value is not a valid floating-point number. + */ + simdjson_inline simdjson_result get_double() noexcept; + + /** + * Cast this JSON value (inside string) to a double + * + * @returns A double. + * @returns INCORRECT_TYPE If the JSON value is not a valid floating-point number. + */ + simdjson_inline simdjson_result get_double_in_string() noexcept; + + /** + * Cast this JSON value to a string. + * + * The string is guaranteed to be valid UTF-8. + * + * Equivalent to get(). + * + * Important: a value should be consumed once. Calling get_string() twice on the same value + * is an error. + * + * @returns An UTF-8 string. The string is stored in the parser and will be invalidated the next + * time it parses a document or when it is destroyed. + * @returns INCORRECT_TYPE if the JSON value is not a string. + */ + simdjson_inline simdjson_result get_string(bool allow_replacement = false) noexcept; + + /** + * Attempts to fill the provided std::string reference with the parsed value of the current string. + * + * The string is guaranteed to be valid UTF-8. + * + * Important: a value should be consumed once. Calling get_string() twice on the same value + * is an error. + * + * Performance: This method may be slower than get_string() or get_string(bool) because it may need to allocate memory. + * We recommend you avoid allocating an std::string unless you need to. + * + * @returns INCORRECT_TYPE if the JSON value is not a string. Otherwise, we return SUCCESS. + */ + template + simdjson_inline error_code get_string(string_type& receiver, bool allow_replacement = false) noexcept; + + /** + * Cast this JSON value to a "wobbly" string. + * + * The string is may not be a valid UTF-8 string. + * See https://simonsapin.github.io/wtf-8/ + * + * Important: a value should be consumed once. Calling get_wobbly_string() twice on the same value + * is an error. + * + * @returns An UTF-8 string. The string is stored in the parser and will be invalidated the next + * time it parses a document or when it is destroyed. + * @returns INCORRECT_TYPE if the JSON value is not a string. + */ + simdjson_inline simdjson_result get_wobbly_string() noexcept; + /** + * Cast this JSON value to a raw_json_string. + * + * The string is guaranteed to be valid UTF-8, and may have escapes in it (e.g. \\ or \n). + * + * @returns A pointer to the raw JSON for the given string. + * @returns INCORRECT_TYPE if the JSON value is not a string. + */ + simdjson_inline simdjson_result get_raw_json_string() noexcept; + + /** + * Cast this JSON value to a bool. + * + * @returns A bool value. + * @returns INCORRECT_TYPE if the JSON value is not true or false. + */ + simdjson_inline simdjson_result get_bool() noexcept; + + /** + * Checks if this JSON value is null. If and only if the value is + * null, then it is consumed (we advance). If we find a token that + * begins with 'n' but is not 'null', then an error is returned. + * + * @returns Whether the value is null. + * @returns INCORRECT_TYPE If the JSON value begins with 'n' and is not 'null'. + */ + simdjson_inline simdjson_result is_null() noexcept; + +#if SIMDJSON_EXCEPTIONS + /** + * Cast this JSON value to an instance of type T. The programmer is responsible for + * providing an implementation of get for the type T, if T is not one of the types + * supported by the library (object, array, raw_json_string, string_view, uint64_t, etc.). + * + * See https://github.com/simdjson/simdjson/blob/master/doc/basics.md#adding-support-for-custom-types + * + * @returns An instance of type T + */ + template + explicit simdjson_inline operator T() noexcept(false); + /** + * Cast this JSON value to an array. + * + * @returns An object that can be used to iterate the array. + * @exception simdjson_error(INCORRECT_TYPE) If the JSON value is not an array. + */ + simdjson_inline operator array() noexcept(false); + /** + * Cast this JSON value to an object. + * + * @returns An object that can be used to look up or iterate fields. + * @exception simdjson_error(INCORRECT_TYPE) If the JSON value is not an object. + */ + simdjson_inline operator object() noexcept(false); + /** + * Cast this JSON value to an unsigned integer. + * + * @returns A signed 64-bit integer. + * @exception simdjson_error(INCORRECT_TYPE) If the JSON value is not a 64-bit unsigned integer. + */ + simdjson_inline operator uint64_t() noexcept(false); + /** + * Cast this JSON value to a signed integer. + * + * @returns A signed 64-bit integer. + * @exception simdjson_error(INCORRECT_TYPE) If the JSON value is not a 64-bit integer. + */ + simdjson_inline operator int64_t() noexcept(false); + /** + * Cast this JSON value to a double. + * + * @returns A double. + * @exception simdjson_error(INCORRECT_TYPE) If the JSON value is not a valid floating-point number. + */ + simdjson_inline operator double() noexcept(false); + /** + * Cast this JSON value to a string. + * + * The string is guaranteed to be valid UTF-8. + * + * Equivalent to get(). + * + * @returns An UTF-8 string. The string is stored in the parser and will be invalidated the next + * time it parses a document or when it is destroyed. + * @exception simdjson_error(INCORRECT_TYPE) if the JSON value is not a string. + */ + simdjson_inline operator std::string_view() noexcept(false); + /** + * Cast this JSON value to a raw_json_string. + * + * The string is guaranteed to be valid UTF-8, and may have escapes in it (e.g. \\ or \n). + * + * @returns A pointer to the raw JSON for the given string. + * @exception simdjson_error(INCORRECT_TYPE) if the JSON value is not a string. + */ + simdjson_inline operator raw_json_string() noexcept(false); + /** + * Cast this JSON value to a bool. + * + * @returns A bool value. + * @exception simdjson_error(INCORRECT_TYPE) if the JSON value is not true or false. + */ + simdjson_inline operator bool() noexcept(false); +#endif + + /** + * Begin array iteration. + * + * Part of the std::iterable interface. + * + * @returns INCORRECT_TYPE If the JSON value is not an array. + */ + simdjson_inline simdjson_result begin() & noexcept; + /** + * Sentinel representing the end of the array. + * + * Part of the std::iterable interface. + */ + simdjson_inline simdjson_result end() & noexcept; + /** + * This method scans the array and counts the number of elements. + * The count_elements method should always be called before you have begun + * iterating through the array: it is expected that you are pointing at + * the beginning of the array. + * The runtime complexity is linear in the size of the array. After + * calling this function, if successful, the array is 'rewinded' at its + * beginning as if it had never been accessed. If the JSON is malformed (e.g., + * there is a missing comma), then an error is returned and it is no longer + * safe to continue. + * + * Performance hint: You should only call count_elements() as a last + * resort as it may require scanning the document twice or more. + */ + simdjson_inline simdjson_result count_elements() & noexcept; + /** + * This method scans the object and counts the number of key-value pairs. + * The count_fields method should always be called before you have begun + * iterating through the object: it is expected that you are pointing at + * the beginning of the object. + * The runtime complexity is linear in the size of the object. After + * calling this function, if successful, the object is 'rewinded' at its + * beginning as if it had never been accessed. If the JSON is malformed (e.g., + * there is a missing comma), then an error is returned and it is no longer + * safe to continue. + * + * To check that an object is empty, it is more performant to use + * the is_empty() method on the object instance. + * + * Performance hint: You should only call count_fields() as a last + * resort as it may require scanning the document twice or more. + */ + simdjson_inline simdjson_result count_fields() & noexcept; + /** + * Get the value at the given index in the array. This function has linear-time complexity. + * This function should only be called once on an array instance since the array iterator is not reset between each call. + * + * @return The value at the given index, or: + * - INDEX_OUT_OF_BOUNDS if the array index is larger than an array length + */ + simdjson_inline simdjson_result at(size_t index) noexcept; + /** + * Look up a field by name on an object (order-sensitive). + * + * The following code reads z, then y, then x, and thus will not retrieve x or y if fed the + * JSON `{ "x": 1, "y": 2, "z": 3 }`: + * + * ```c++ + * simdjson::ondemand::parser parser; + * auto obj = parser.parse(R"( { "x": 1, "y": 2, "z": 3 } )"_padded); + * double z = obj.find_field("z"); + * double y = obj.find_field("y"); + * double x = obj.find_field("x"); + * ``` + * If you have multiple fields with a matching key ({"x": 1, "x": 1}) be mindful + * that only one field is returned. + + * **Raw Keys:** The lookup will be done against the *raw* key, and will not unescape keys. + * e.g. `object["a"]` will match `{ "a": 1 }`, but will *not* match `{ "\u0061": 1 }`. + * + * @param key The key to look up. + * @returns The value of the field, or NO_SUCH_FIELD if the field is not in the object. + */ + simdjson_inline simdjson_result find_field(std::string_view key) noexcept; + /** @overload simdjson_inline simdjson_result find_field(std::string_view key) noexcept; */ + simdjson_inline simdjson_result find_field(const char* key) noexcept; + + /** + * Look up a field by name on an object, without regard to key order. + * + * **Performance Notes:** This is a bit less performant than find_field(), though its effect varies + * and often appears negligible. It starts out normally, starting out at the last field; but if + * the field is not found, it scans from the beginning of the object to see if it missed it. That + * missing case has a non-cache-friendly bump and lots of extra scanning, especially if the object + * in question is large. The fact that the extra code is there also bumps the executable size. + * + * It is the default, however, because it would be highly surprising (and hard to debug) if the + * default behavior failed to look up a field just because it was in the wrong order--and many + * APIs assume this. Therefore, you must be explicit if you want to treat objects as out of order. + * + * If you have multiple fields with a matching key ({"x": 1, "x": 1}) be mindful + * that only one field is returned. + * + * Use find_field() if you are sure fields will be in order (or are willing to treat it as if the + * field as not there when they are not in order). + * + * @param key The key to look up. + * @returns The value of the field, or NO_SUCH_FIELD if the field is not in the object. + */ + simdjson_inline simdjson_result find_field_unordered(std::string_view key) noexcept; + /** @overload simdjson_inline simdjson_result find_field_unordered(std::string_view key) noexcept; */ + simdjson_inline simdjson_result find_field_unordered(const char* key) noexcept; + /** @overload simdjson_inline simdjson_result find_field_unordered(std::string_view key) noexcept; */ + simdjson_inline simdjson_result operator[](std::string_view key) noexcept; + /** @overload simdjson_inline simdjson_result find_field_unordered(std::string_view key) noexcept; */ + simdjson_inline simdjson_result operator[](const char* key) noexcept; + + /** + * Get the type of this JSON value. It does not validate or consume the value. + * E.g., you must still call "is_null()" to check that a value is null even if + * "type()" returns json_type::null. + * + * NOTE: If you're only expecting a value to be one type (a typical case), it's generally + * better to just call .get_double, .get_string, etc. and check for INCORRECT_TYPE (or just + * let it throw an exception). + * + * @return The type of JSON value (json_type::array, json_type::object, json_type::string, + * json_type::number, json_type::boolean, or json_type::null). + * @error TAPE_ERROR when the JSON value is a bad token like "}" "," or "alse". + */ + simdjson_inline simdjson_result type() noexcept; + + /** + * Checks whether the value is a scalar (string, number, null, Boolean). + * Returns false when there it is an array or object. + * + * @returns true if the type is string, number, null, Boolean + * @error TAPE_ERROR when the JSON value is a bad token like "}" "," or "alse". + */ + simdjson_inline simdjson_result is_scalar() noexcept; + /** + * Checks whether the value is a string. + * + * @returns true if the type is string + * @error TAPE_ERROR when the JSON value is a bad token like "}" "," or "alse". + */ + simdjson_inline simdjson_result is_string() noexcept; + + /** + * Checks whether the value is a negative number. + * + * @returns true if the number if negative. + */ + simdjson_inline bool is_negative() noexcept; + /** + * Checks whether the value is an integer number. Note that + * this requires to partially parse the number string. If + * the value is determined to be an integer, it may still + * not parse properly as an integer in subsequent steps + * (e.g., it might overflow). + * + * Performance note: if you call this function systematically + * before parsing a number, you may have fallen for a performance + * anti-pattern. + * + * @returns true if the number if negative. + */ + simdjson_inline simdjson_result is_integer() noexcept; + /** + * Determine the number type (integer or floating-point number) as quickly + * as possible. This function does not fully validate the input. It is + * useful when you only need to classify the numbers, without parsing them. + * + * If you are planning to retrieve the value or you need full validation, + * consider using the get_number() method instead: it will fully parse + * and validate the input, and give you access to the type: + * get_number().get_number_type(). + * + * get_number_type() is number_type::unsigned_integer if we have + * an integer greater or equal to 9223372036854775808. + * get_number_type() is number_type::signed_integer if we have an + * integer that is less than 9223372036854775808. + * get_number_type() is number_type::big_integer for integers that do not fit in 64 bits, + * in which case the digit_count is set to the length of the big integer string. + * Otherwise, get_number_type() has value number_type::floating_point_number. + * + * This function requires processing the number string, but it is expected + * to be faster than get_number().get_number_type() because it is does not + * parse the number value. + * + * @returns the type of the number + */ + simdjson_inline simdjson_result get_number_type() noexcept; + + /** + * Attempt to parse an ondemand::number. An ondemand::number may + * contain an integer value or a floating-point value, the simdjson + * library will autodetect the type. Thus it is a dynamically typed + * number. Before accessing the value, you must determine the detected + * type. + * + * number.get_number_type() is number_type::signed_integer if we have + * an integer in [-9223372036854775808,9223372036854775808) + * You can recover the value by calling number.get_int64() and you + * have that number.is_int64() is true. + * + * number.get_number_type() is number_type::unsigned_integer if we have + * an integer in [9223372036854775808,18446744073709551616) + * You can recover the value by calling number.get_uint64() and you + * have that number.is_uint64() is true. + * + * For integers that do not fit in 64 bits, the function returns BIGINT_ERROR error code. + * + * Otherwise, number.get_number_type() has value number_type::floating_point_number + * and we have a binary64 number. + * You can recover the value by calling number.get_double() and you + * have that number.is_double() is true. + * + * You must check the type before accessing the value: it is an error + * to call "get_int64()" when number.get_number_type() is not + * number_type::signed_integer and when number.is_int64() is false. + * + * Performance note: this is designed with performance in mind. When + * calling 'get_number()', you scan the number string only once, determining + * efficiently the type and storing it in an efficient manner. + */ + simdjson_warn_unused simdjson_inline simdjson_result get_number() noexcept; + + /** + * Get the raw JSON for this token. + * + * The string_view will always point into the input buffer. + * + * The string_view will start at the beginning of the token, and include the entire token + * *as well as all spaces until the next token (or EOF).* This means, for example, that a + * string token always begins with a " and is always terminated by the final ", possibly + * followed by a number of spaces. + * + * The string_view is *not* null-terminated. However, if this is a scalar (string, number, + * boolean, or null), the character after the end of the string_view is guaranteed to be + * a non-space token. + * + * Tokens include: + * - { + * - [ + * - "a string (possibly with UTF-8 or backslashed characters like \\\")". + * - -1.2e-100 + * - true + * - false + * - null + * + * See also value::raw_json(). + */ + simdjson_inline std::string_view raw_json_token() noexcept; + + /** + * Get a string_view pointing at this value in the JSON document. + * If this element is an array or an object, it consumes the array or the object + * and returns a string_view instance corresponding to the + * array as represented in JSON. It points inside the original document. + * If this element is a scalar (string, number, Boolean, null), it returns what + * raw_json_token() would return. + */ + simdjson_inline simdjson_result raw_json() noexcept; + + /** + * Returns the current location in the document if in bounds. + */ + simdjson_inline simdjson_result current_location() noexcept; + + /** + * Returns the current depth in the document if in bounds. + * + * E.g., + * 0 = finished with document + * 1 = document root value (could be [ or {, not yet known) + * 2 = , or } inside root array/object + * 3 = key or value inside root array/object. + */ + simdjson_inline int32_t current_depth() const noexcept; + + /** + * Get the value associated with the given JSON pointer. We use the RFC 6901 + * https://tools.ietf.org/html/rfc6901 standard. + * + * ondemand::parser parser; + * auto json = R"({ "foo": { "a": [ 10, 20, 30 ] }})"_padded; + * auto doc = parser.iterate(json); + * doc.at_pointer("/foo/a/1") == 20 + * + * It is allowed for a key to be the empty string: + * + * ondemand::parser parser; + * auto json = R"({ "": { "a": [ 10, 20, 30 ] }})"_padded; + * auto doc = parser.iterate(json); + * doc.at_pointer("//a/1") == 20 + * + * Note that at_pointer() called on the document automatically calls the document's rewind + * method between each call. It invalidates all previously accessed arrays, objects and values + * that have not been consumed. + * + * Calling at_pointer() on non-document instances (e.g., arrays and objects) is not + * standardized (by RFC 6901). We provide some experimental support for JSON pointers + * on non-document instances. Yet it is not the case when calling at_pointer on an array + * or an object instance: there is no rewind and no invalidation. + * + * You may only call at_pointer on an array after it has been created, but before it has + * been first accessed. When calling at_pointer on an array, the pointer is advanced to + * the location indicated by the JSON pointer (in case of success). It is no longer possible + * to call at_pointer on the same array. + * + * You may call at_pointer more than once on an object, but each time the pointer is advanced + * to be within the value matched by the key indicated by the JSON pointer query. Thus any preceding + * key (as well as the current key) can no longer be used with following JSON pointer calls. + * + * Also note that at_pointer() relies on find_field() which implies that we do not unescape keys when matching + * + * @return The value associated with the given JSON pointer, or: + * - NO_SUCH_FIELD if a field does not exist in an object + * - INDEX_OUT_OF_BOUNDS if an array index is larger than an array length + * - INCORRECT_TYPE if a non-integer is used to access an array + * - INVALID_JSON_POINTER if the JSON pointer is invalid and cannot be parsed + */ + simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; + + /** + * Get the value associated with the given JSONPath expression. We only support + * JSONPath queries that trivially convertible to JSON Pointer queries: key + * names and array indices. + * + * @return The value associated with the given JSONPath expression, or: + * - INVALID_JSON_POINTER if the JSONPath to JSON Pointer conversion fails + * - NO_SUCH_FIELD if a field does not exist in an object + * - INDEX_OUT_OF_BOUNDS if an array index is larger than an array length + * - INCORRECT_TYPE if a non-integer is used to access an array + */ + simdjson_inline simdjson_result at_path(std::string_view at_path) noexcept; + + + protected: + /** + * Create a value. + */ + simdjson_inline value(const value_iterator& iter) noexcept; + + /** + * Skip this value, allowing iteration to continue. + */ + simdjson_inline void skip() noexcept; + + /** + * Start a value at the current position. + * + * (It should already be started; this is just a self-documentation method.) + */ + static simdjson_inline value start(const value_iterator& iter) noexcept; + + /** + * Resume a value. + */ + static simdjson_inline value resume(const value_iterator& iter) noexcept; + + /** + * Get the object, starting or resuming it as necessary + */ + simdjson_inline simdjson_result start_or_resume_object() noexcept; + + // simdjson_inline void log_value(const char *type) const noexcept; + // simdjson_inline void log_error(const char *message) const noexcept; + + value_iterator iter{}; + + friend class document; + friend class array_iterator; + friend class field; + friend class object; + friend struct simdjson_result; + friend struct simdjson_result; + friend class field; + }; + + } // namespace ondemand + } // namespace lasx +} // namespace simdjson + +namespace simdjson { + + template<> + struct simdjson_result : public lasx::implementation_simdjson_result_base { + public: + simdjson_inline simdjson_result(lasx::ondemand::value&& value) noexcept; ///< @private + simdjson_inline simdjson_result(error_code error) noexcept; ///< @private + simdjson_inline simdjson_result() noexcept = default; + + simdjson_inline simdjson_result get_array() noexcept; + simdjson_inline simdjson_result get_object() noexcept; + + simdjson_inline simdjson_result get_uint64() noexcept; + simdjson_inline simdjson_result get_uint64_in_string() noexcept; + simdjson_inline simdjson_result get_int64() noexcept; + simdjson_inline simdjson_result get_int64_in_string() noexcept; + simdjson_inline simdjson_result get_double() noexcept; + simdjson_inline simdjson_result get_double_in_string() noexcept; + simdjson_inline simdjson_result get_string(bool allow_replacement = false) noexcept; + template + simdjson_inline error_code get_string(string_type& receiver, bool allow_replacement = false) noexcept; + simdjson_inline simdjson_result get_wobbly_string() noexcept; + simdjson_inline simdjson_result get_raw_json_string() noexcept; + simdjson_inline simdjson_result get_bool() noexcept; + simdjson_inline simdjson_result is_null() noexcept; + + template simdjson_inline simdjson_result get() noexcept; + + template simdjson_inline error_code get(T& out) noexcept; + +#if SIMDJSON_EXCEPTIONS + template + explicit simdjson_inline operator T() noexcept(false); + simdjson_inline operator lasx::ondemand::array() noexcept(false); + simdjson_inline operator lasx::ondemand::object() noexcept(false); + simdjson_inline operator uint64_t() noexcept(false); + simdjson_inline operator int64_t() noexcept(false); + simdjson_inline operator double() noexcept(false); + simdjson_inline operator std::string_view() noexcept(false); + simdjson_inline operator lasx::ondemand::raw_json_string() noexcept(false); + simdjson_inline operator bool() noexcept(false); +#endif + simdjson_inline simdjson_result count_elements() & noexcept; + simdjson_inline simdjson_result count_fields() & noexcept; + simdjson_inline simdjson_result at(size_t index) noexcept; + simdjson_inline simdjson_result begin() & noexcept; + simdjson_inline simdjson_result end() & noexcept; + + /** + * Look up a field by name on an object (order-sensitive). + * + * The following code reads z, then y, then x, and thus will not retrieve x or y if fed the + * JSON `{ "x": 1, "y": 2, "z": 3 }`: + * + * ```c++ + * simdjson::ondemand::parser parser; + * auto obj = parser.parse(R"( { "x": 1, "y": 2, "z": 3 } )"_padded); + * double z = obj.find_field("z"); + * double y = obj.find_field("y"); + * double x = obj.find_field("x"); + * ``` + * + * **Raw Keys:** The lookup will be done against the *raw* key, and will not unescape keys. + * e.g. `object["a"]` will match `{ "a": 1 }`, but will *not* match `{ "\u0061": 1 }`. + * + * @param key The key to look up. + * @returns The value of the field, or NO_SUCH_FIELD if the field is not in the object. + */ + simdjson_inline simdjson_result find_field(std::string_view key) noexcept; + /** @overload simdjson_inline simdjson_result find_field(std::string_view key) noexcept; */ + simdjson_inline simdjson_result find_field(const char* key) noexcept; + + /** + * Look up a field by name on an object, without regard to key order. + * + * **Performance Notes:** This is a bit less performant than find_field(), though its effect varies + * and often appears negligible. It starts out normally, starting out at the last field; but if + * the field is not found, it scans from the beginning of the object to see if it missed it. That + * missing case has a non-cache-friendly bump and lots of extra scanning, especially if the object + * in question is large. The fact that the extra code is there also bumps the executable size. + * + * It is the default, however, because it would be highly surprising (and hard to debug) if the + * default behavior failed to look up a field just because it was in the wrong order--and many + * APIs assume this. Therefore, you must be explicit if you want to treat objects as out of order. + * + * Use find_field() if you are sure fields will be in order (or are willing to treat it as if the + * field as not there when they are not in order). + * + * @param key The key to look up. + * @returns The value of the field, or NO_SUCH_FIELD if the field is not in the object. + */ + simdjson_inline simdjson_result find_field_unordered(std::string_view key) noexcept; + /** @overload simdjson_inline simdjson_result find_field_unordered(std::string_view key) noexcept; */ + simdjson_inline simdjson_result find_field_unordered(const char* key) noexcept; + /** @overload simdjson_inline simdjson_result find_field_unordered(std::string_view key) noexcept; */ + simdjson_inline simdjson_result operator[](std::string_view key) noexcept; + /** @overload simdjson_inline simdjson_result find_field_unordered(std::string_view key) noexcept; */ + simdjson_inline simdjson_result operator[](const char* key) noexcept; + + /** + * Get the type of this JSON value. + * + * NOTE: If you're only expecting a value to be one type (a typical case), it's generally + * better to just call .get_double, .get_string, etc. and check for INCORRECT_TYPE (or just + * let it throw an exception). + */ + simdjson_inline simdjson_result type() noexcept; + simdjson_inline simdjson_result is_scalar() noexcept; + simdjson_inline simdjson_result is_string() noexcept; + simdjson_inline simdjson_result is_negative() noexcept; + simdjson_inline simdjson_result is_integer() noexcept; + simdjson_inline simdjson_result get_number_type() noexcept; + simdjson_inline simdjson_result get_number() noexcept; + + /** @copydoc simdjson_inline std::string_view value::raw_json_token() const noexcept */ + simdjson_inline simdjson_result raw_json_token() noexcept; + simdjson_inline simdjson_result raw_json() noexcept; + + /** @copydoc simdjson_inline simdjson_result current_location() noexcept */ + simdjson_inline simdjson_result current_location() noexcept; + /** @copydoc simdjson_inline int32_t current_depth() const noexcept */ + simdjson_inline simdjson_result current_depth() const noexcept; + simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; + simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; + }; + +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_ONDEMAND_VALUE_H +/* end file simdjson/generic/ondemand/value.h for lasx */ +/* including simdjson/generic/ondemand/logger.h for lasx: #include "simdjson/generic/ondemand/logger.h" */ +/* begin file simdjson/generic/ondemand/logger.h for lasx */ +#ifndef SIMDJSON_GENERIC_ONDEMAND_LOGGER_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_ONDEMAND_LOGGER_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/base.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lasx { + namespace ondemand { + + // Logging should be free unless SIMDJSON_VERBOSE_LOGGING is set. Importantly, it is critical + // that the call to the log functions be side-effect free. Thus, for example, you should not + // create temporary std::string instances. + namespace logger { + + enum class log_level : int32_t { + info = 0, + error = 1 + }; + +#if SIMDJSON_VERBOSE_LOGGING + static constexpr const bool LOG_ENABLED = true; +#else + static constexpr const bool LOG_ENABLED = false; +#endif + + // We do not want these functions to be 'really inlined' since real inlining is + // for performance purposes and if you are using the loggers, you do not care about + // performance (or should not). + static inline void log_headers() noexcept; + // If args are provided, title will be treated as format string + template + static inline void log_line(const json_iterator& iter, token_position index, depth_t depth, const char* title_prefix, const char* title, std::string_view detail, logger::log_level level, Args&&... args) noexcept; + template + static inline void log_line(const json_iterator& iter, const char* title_prefix, const char* title, std::string_view detail, int delta, int depth_delta, logger::log_level level, Args&&... args) noexcept; + static inline void log_event(const json_iterator& iter, const char* type, std::string_view detail = "", int delta = 0, int depth_delta = 0) noexcept; + static inline void log_value(const json_iterator& iter, token_position index, depth_t depth, const char* type, std::string_view detail = "") noexcept; + static inline void log_value(const json_iterator& iter, const char* type, std::string_view detail = "", int delta = -1, int depth_delta = 0) noexcept; + static inline void log_start_value(const json_iterator& iter, token_position index, depth_t depth, const char* type, std::string_view detail = "") noexcept; + static inline void log_start_value(const json_iterator& iter, const char* type, int delta = -1, int depth_delta = 0) noexcept; + static inline void log_end_value(const json_iterator& iter, const char* type, int delta = -1, int depth_delta = 0) noexcept; + + static inline void log_error(const json_iterator& iter, token_position index, depth_t depth, const char* error, const char* detail = "") noexcept; + static inline void log_error(const json_iterator& iter, const char* error, const char* detail = "", int delta = -1, int depth_delta = 0) noexcept; + + static inline void log_event(const value_iterator& iter, const char* type, std::string_view detail = "", int delta = 0, int depth_delta = 0) noexcept; + static inline void log_value(const value_iterator& iter, const char* type, std::string_view detail = "", int delta = -1, int depth_delta = 0) noexcept; + static inline void log_start_value(const value_iterator& iter, const char* type, int delta = -1, int depth_delta = 0) noexcept; + static inline void log_end_value(const value_iterator& iter, const char* type, int delta = -1, int depth_delta = 0) noexcept; + static inline void log_error(const value_iterator& iter, const char* error, const char* detail = "", int delta = -1, int depth_delta = 0) noexcept; + + } // namespace logger + } // namespace ondemand + } // namespace lasx +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_ONDEMAND_LOGGER_H +/* end file simdjson/generic/ondemand/logger.h for lasx */ +/* including simdjson/generic/ondemand/token_iterator.h for lasx: #include "simdjson/generic/ondemand/token_iterator.h" */ +/* begin file simdjson/generic/ondemand/token_iterator.h for lasx */ +#ifndef SIMDJSON_GENERIC_ONDEMAND_TOKEN_ITERATOR_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_ONDEMAND_TOKEN_ITERATOR_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/implementation_simdjson_result_base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/logger.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lasx { + namespace ondemand { + + /** + * Iterates through JSON tokens (`{` `}` `[` `]` `,` `:` `""` `123` `true` `false` `null`) + * detected by stage 1. + * + * @private This is not intended for external use. + */ + class token_iterator { + public: + /** + * Create a new invalid token_iterator. + * + * Exists so you can declare a variable and later assign to it before use. + */ + simdjson_inline token_iterator() noexcept = default; + simdjson_inline token_iterator(token_iterator&& other) noexcept = default; + simdjson_inline token_iterator& operator=(token_iterator&& other) noexcept = default; + simdjson_inline token_iterator(const token_iterator& other) noexcept = default; + simdjson_inline token_iterator& operator=(const token_iterator& other) noexcept = default; + + /** + * Advance to the next token (returning the current one). + */ + simdjson_inline const uint8_t* return_current_and_advance() noexcept; + /** + * Reports the current offset in bytes from the start of the underlying buffer. + */ + simdjson_inline uint32_t current_offset() const noexcept; + /** + * Get the JSON text for a given token (relative). + * + * This is not null-terminated; it is a view into the JSON. + * + * @param delta The relative position of the token to retrieve. e.g. 0 = current token, + * 1 = next token, -1 = prev token. + * + * TODO consider a string_view, assuming the length will get stripped out by the optimizer when + * it is not used... + */ + simdjson_inline const uint8_t* peek(int32_t delta = 0) const noexcept; + /** + * Get the maximum length of the JSON text for a given token. + * + * The length will include any whitespace at the end of the token. + * + * @param delta The relative position of the token to retrieve. e.g. 0 = current token, + * 1 = next token, -1 = prev token. + */ + simdjson_inline uint32_t peek_length(int32_t delta = 0) const noexcept; + + /** + * Get the JSON text for a given token. + * + * This is not null-terminated; it is a view into the JSON. + * + * @param position The position of the token. + * + */ + simdjson_inline const uint8_t* peek(token_position position) const noexcept; + /** + * Get the maximum length of the JSON text for a given token. + * + * The length will include any whitespace at the end of the token. + * + * @param position The position of the token. + */ + simdjson_inline uint32_t peek_length(token_position position) const noexcept; + /** + * Get the maximum length of the JSON text for a root token. + * + * The length will include any whitespace at the end of the token. + * + * @param position The position of the token (start of the document). + */ + simdjson_inline uint32_t peek_root_length(token_position position) const noexcept; + /** + * Return the current index. + */ + simdjson_inline token_position position() const noexcept; + /** + * Reset to a previously saved index. + */ + simdjson_inline void set_position(token_position target_position) noexcept; + + // NOTE: we don't support a full C++ iterator interface, because we expect people to make + // different calls to advance the iterator based on *their own* state. + + simdjson_inline bool operator==(const token_iterator& other) const noexcept; + simdjson_inline bool operator!=(const token_iterator& other) const noexcept; + simdjson_inline bool operator>(const token_iterator& other) const noexcept; + simdjson_inline bool operator>=(const token_iterator& other) const noexcept; + simdjson_inline bool operator<(const token_iterator& other) const noexcept; + simdjson_inline bool operator<=(const token_iterator& other) const noexcept; + + protected: + simdjson_inline token_iterator(const uint8_t* buf, token_position position) noexcept; + + /** + * Get the index of the JSON text for a given token (relative). + * + * This is not null-terminated; it is a view into the JSON. + * + * @param delta The relative position of the token to retrieve. e.g. 0 = current token, + * 1 = next token, -1 = prev token. + */ + simdjson_inline uint32_t peek_index(int32_t delta = 0) const noexcept; + /** + * Get the index of the JSON text for a given token. + * + * This is not null-terminated; it is a view into the JSON. + * + * @param position The position of the token. + * + */ + simdjson_inline uint32_t peek_index(token_position position) const noexcept; + + const uint8_t* buf{}; + token_position _position{}; + + friend class json_iterator; + friend class value_iterator; + friend class object; + template + friend simdjson_inline void logger::log_line(const json_iterator& iter, const char* title_prefix, const char* title, std::string_view detail, int delta, int depth_delta, logger::log_level level, Args&&... args) noexcept; + template + friend simdjson_inline void logger::log_line(const json_iterator& iter, token_position index, depth_t depth, const char* title_prefix, const char* title, std::string_view detail, logger::log_level level, Args&&... args) noexcept; + }; + + } // namespace ondemand + } // namespace lasx +} // namespace simdjson + +namespace simdjson { + + template<> + struct simdjson_result : public lasx::implementation_simdjson_result_base { + public: + simdjson_inline simdjson_result(lasx::ondemand::token_iterator&& value) noexcept; ///< @private + simdjson_inline simdjson_result(error_code error) noexcept; ///< @private + simdjson_inline simdjson_result() noexcept = default; + simdjson_inline ~simdjson_result() noexcept = default; ///< @private + }; + +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_ONDEMAND_TOKEN_ITERATOR_H +/* end file simdjson/generic/ondemand/token_iterator.h for lasx */ +/* including simdjson/generic/ondemand/json_iterator.h for lasx: #include "simdjson/generic/ondemand/json_iterator.h" */ +/* begin file simdjson/generic/ondemand/json_iterator.h for lasx */ +#ifndef SIMDJSON_GENERIC_ONDEMAND_JSON_ITERATOR_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_ONDEMAND_JSON_ITERATOR_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/implementation_simdjson_result_base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/token_iterator.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lasx { + namespace ondemand { + + /** + * Iterates through JSON tokens, keeping track of depth and string buffer. + * + * @private This is not intended for external use. + */ + class json_iterator { + protected: + token_iterator token{}; + ondemand::parser* parser{}; + /** + * Next free location in the string buffer. + * + * Used by raw_json_string::unescape() to have a place to unescape strings to. + */ + uint8_t* _string_buf_loc{}; + /** + * JSON error, if there is one. + * + * INCORRECT_TYPE and NO_SUCH_FIELD are *not* stored here, ever. + * + * PERF NOTE: we *hope* this will be elided into control flow, as it is only used (a) in the first + * iteration of the loop, or (b) for the final iteration after a missing comma is found in ++. If + * this is not elided, we should make sure it's at least not using up a register. Failing that, + * we should store it in document so there's only one of them. + */ + error_code error{ SUCCESS }; + /** + * Depth of the current token in the JSON. + * + * - 0 = finished with document + * - 1 = document root value (could be [ or {, not yet known) + * - 2 = , or } inside root array/object + * - 3 = key or value inside root array/object. + */ + depth_t _depth{}; + /** + * Beginning of the document indexes. + * Normally we have root == parser->implementation->structural_indexes.get() + * but this may differ, especially in streaming mode (where we have several + * documents); + */ + token_position _root{}; + /** + * Normally, a json_iterator operates over a single document, but in + * some cases, we may have a stream of documents. This attribute is meant + * as meta-data: the json_iterator works the same irrespective of the + * value of this attribute. + */ + bool _streaming{ false }; + + public: + simdjson_inline json_iterator() noexcept = default; + simdjson_inline json_iterator(json_iterator&& other) noexcept; + simdjson_inline json_iterator& operator=(json_iterator&& other) noexcept; + simdjson_inline explicit json_iterator(const json_iterator& other) noexcept = default; + simdjson_inline json_iterator& operator=(const json_iterator& other) noexcept = default; + /** + * Skips a JSON value, whether it is a scalar, array or object. + */ + simdjson_warn_unused simdjson_inline error_code skip_child(depth_t parent_depth) noexcept; + + /** + * Tell whether the iterator is still at the start + */ + simdjson_inline bool at_root() const noexcept; + + /** + * Tell whether we should be expected to run in streaming + * mode (iterating over many documents). It is pure metadata + * that does not affect how the iterator works. It is used by + * start_root_array() and start_root_object(). + */ + simdjson_inline bool streaming() const noexcept; + + /** + * Get the root value iterator + */ + simdjson_inline token_position root_position() const noexcept; + /** + * Assert that we are at the document depth (== 1) + */ + simdjson_inline void assert_at_document_depth() const noexcept; + /** + * Assert that we are at the root of the document + */ + simdjson_inline void assert_at_root() const noexcept; + + /** + * Tell whether the iterator is at the EOF mark + */ + simdjson_inline bool at_end() const noexcept; + + /** + * Tell whether the iterator is live (has not been moved). + */ + simdjson_inline bool is_alive() const noexcept; + + /** + * Abandon this iterator, setting depth to 0 (as if the document is finished). + */ + simdjson_inline void abandon() noexcept; + + /** + * Advance the current token without modifying depth. + */ + simdjson_inline const uint8_t* return_current_and_advance() noexcept; + + /** + * Returns true if there is a single token in the index (i.e., it is + * a JSON with a scalar value such as a single number). + * + * @return whether there is a single token + */ + simdjson_inline bool is_single_token() const noexcept; + + /** + * Assert that there are at least the given number of tokens left. + * + * Has no effect in release builds. + */ + simdjson_inline void assert_more_tokens(uint32_t required_tokens = 1) const noexcept; + /** + * Assert that the given position addresses an actual token (is within bounds). + * + * Has no effect in release builds. + */ + simdjson_inline void assert_valid_position(token_position position) const noexcept; + /** + * Get the JSON text for a given token (relative). + * + * This is not null-terminated; it is a view into the JSON. + * + * @param delta The relative position of the token to retrieve. e.g. 0 = next token, -1 = prev token. + * + * TODO consider a string_view, assuming the length will get stripped out by the optimizer when + * it is not used ... + */ + simdjson_inline const uint8_t* peek(int32_t delta = 0) const noexcept; + /** + * Get the maximum length of the JSON text for the current token (or relative). + * + * The length will include any whitespace at the end of the token. + * + * @param delta The relative position of the token to retrieve. e.g. 0 = next token, -1 = prev token. + */ + simdjson_inline uint32_t peek_length(int32_t delta = 0) const noexcept; + /** + * Get a pointer to the current location in the input buffer. + * + * This is not null-terminated; it is a view into the JSON. + * + * You may be pointing outside of the input buffer: it is not generally + * safe to dereference this pointer. + */ + simdjson_inline const uint8_t* unsafe_pointer() const noexcept; + /** + * Get the JSON text for a given token. + * + * This is not null-terminated; it is a view into the JSON. + * + * @param position The position of the token to retrieve. + * + * TODO consider a string_view, assuming the length will get stripped out by the optimizer when + * it is not used ... + */ + simdjson_inline const uint8_t* peek(token_position position) const noexcept; + /** + * Get the maximum length of the JSON text for the current token (or relative). + * + * The length will include any whitespace at the end of the token. + * + * @param position The position of the token to retrieve. + */ + simdjson_inline uint32_t peek_length(token_position position) const noexcept; + /** + * Get the maximum length of the JSON text for the current root token. + * + * The length will include any whitespace at the end of the token. + * + * @param position The position of the token to retrieve. + */ + simdjson_inline uint32_t peek_root_length(token_position position) const noexcept; + /** + * Get the JSON text for the last token in the document. + * + * This is not null-terminated; it is a view into the JSON. + * + * TODO consider a string_view, assuming the length will get stripped out by the optimizer when + * it is not used ... + */ + simdjson_inline const uint8_t* peek_last() const noexcept; + + /** + * Ascend one level. + * + * Validates that the depth - 1 == parent_depth. + * + * @param parent_depth the expected parent depth. + */ + simdjson_inline void ascend_to(depth_t parent_depth) noexcept; + + /** + * Descend one level. + * + * Validates that the new depth == child_depth. + * + * @param child_depth the expected child depth. + */ + simdjson_inline void descend_to(depth_t child_depth) noexcept; + simdjson_inline void descend_to(depth_t child_depth, int32_t delta) noexcept; + + /** + * Get current depth. + */ + simdjson_inline depth_t depth() const noexcept; + + /** + * Get current (writeable) location in the string buffer. + */ + simdjson_inline uint8_t*& string_buf_loc() noexcept; + + /** + * Report an unrecoverable error, preventing further iteration. + * + * @param error The error to report. Must not be SUCCESS, UNINITIALIZED, INCORRECT_TYPE, or NO_SUCH_FIELD. + * @param message An error message to report with the error. + */ + simdjson_inline error_code report_error(error_code error, const char* message) noexcept; + + /** + * Log error, but don't stop iteration. + * @param error The error to report. Must be INCORRECT_TYPE, or NO_SUCH_FIELD. + * @param message An error message to report with the error. + */ + simdjson_inline error_code optional_error(error_code error, const char* message) noexcept; + + /** + * Take an input in json containing max_len characters and attempt to copy it over to tmpbuf, a buffer with + * N bytes of capacity. It will return false if N is too small (smaller than max_len) of if it is zero. + * The buffer (tmpbuf) is padded with space characters. + */ + simdjson_warn_unused simdjson_inline bool copy_to_buffer(const uint8_t* json, uint32_t max_len, uint8_t* tmpbuf, size_t N) noexcept; + + simdjson_inline token_position position() const noexcept; + /** + * Write the raw_json_string to the string buffer and return a string_view. + * Each raw_json_string should be unescaped once, or else the string buffer might + * overflow. + */ + simdjson_inline simdjson_result unescape(raw_json_string in, bool allow_replacement) noexcept; + simdjson_inline simdjson_result unescape_wobbly(raw_json_string in) noexcept; + + simdjson_inline void reenter_child(token_position position, depth_t child_depth) noexcept; + + simdjson_inline error_code consume_character(char c) noexcept; +#if SIMDJSON_DEVELOPMENT_CHECKS + simdjson_inline token_position start_position(depth_t depth) const noexcept; + simdjson_inline void set_start_position(depth_t depth, token_position position) noexcept; +#endif + + /* Useful for debugging and logging purposes. */ + inline std::string to_string() const noexcept; + + /** + * Returns the current location in the document if in bounds. + */ + inline simdjson_result current_location() const noexcept; + + /** + * Updates this json iterator so that it is back at the beginning of the document, + * as if it had just been created. + */ + inline void rewind() noexcept; + /** + * This checks whether the {,},[,] are balanced so that the document + * ends with proper zero depth. This requires scanning the whole document + * and it may be expensive. It is expected that it will be rarely called. + * It does not attempt to match { with } and [ with ]. + */ + inline bool balanced() const noexcept; + protected: + simdjson_inline json_iterator(const uint8_t* buf, ondemand::parser* parser) noexcept; +#ifdef SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON + simdjson_inline json_iterator(const uint8_t* buf, ondemand::parser* parser, bool streaming) noexcept; +#endif // SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON + /// The last token before the end + simdjson_inline token_position last_position() const noexcept; + /// The token *at* the end. This points at gibberish and should only be used for comparison. + simdjson_inline token_position end_position() const noexcept; + /// The end of the buffer. + simdjson_inline token_position end() const noexcept; + + friend class document; + friend class document_stream; + friend class object; + friend class array; + friend class value; + friend class raw_json_string; + friend class parser; + friend class value_iterator; + friend class field; + template + friend simdjson_inline void logger::log_line(const json_iterator& iter, const char* title_prefix, const char* title, std::string_view detail, int delta, int depth_delta, logger::log_level level, Args&&... args) noexcept; + template + friend simdjson_inline void logger::log_line(const json_iterator& iter, token_position index, depth_t depth, const char* title_prefix, const char* title, std::string_view detail, logger::log_level level, Args&&... args) noexcept; + }; // json_iterator + + } // namespace ondemand + } // namespace lasx +} // namespace simdjson + +namespace simdjson { + + template<> + struct simdjson_result : public lasx::implementation_simdjson_result_base { + public: + simdjson_inline simdjson_result(lasx::ondemand::json_iterator&& value) noexcept; ///< @private + simdjson_inline simdjson_result(error_code error) noexcept; ///< @private + + simdjson_inline simdjson_result() noexcept = default; + }; + +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_ONDEMAND_JSON_ITERATOR_H +/* end file simdjson/generic/ondemand/json_iterator.h for lasx */ +/* including simdjson/generic/ondemand/json_type.h for lasx: #include "simdjson/generic/ondemand/json_type.h" */ +/* begin file simdjson/generic/ondemand/json_type.h for lasx */ +#ifndef SIMDJSON_GENERIC_ONDEMAND_JSON_TYPE_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_ONDEMAND_JSON_TYPE_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/implementation_simdjson_result_base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/numberparsing.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lasx { + namespace ondemand { + + /** + * The type of a JSON value. + */ + enum class json_type { + // Start at 1 to catch uninitialized / default values more easily + array = 1, ///< A JSON array ( [ 1, 2, 3 ... ] ) + object, ///< A JSON object ( { "a": 1, "b" 2, ... } ) + number, ///< A JSON number ( 1 or -2.3 or 4.5e6 ...) + string, ///< A JSON string ( "a" or "hello world\n" ...) + boolean, ///< A JSON boolean (true or false) + null ///< A JSON null (null) + }; + + /** + * A type representing a JSON number. + * The design of the struct is deliberately straight-forward. All + * functions return standard values with no error check. + */ + struct number { + + /** + * return the automatically determined type of + * the number: number_type::floating_point_number, + * number_type::signed_integer or number_type::unsigned_integer. + * + * enum class number_type { + * floating_point_number=1, /// a binary64 number + * signed_integer, /// a signed integer that fits in a 64-bit word using two's complement + * unsigned_integer /// a positive integer larger or equal to 1<<63 + * }; + */ + simdjson_inline ondemand::number_type get_number_type() const noexcept; + /** + * return true if the automatically determined type of + * the number is number_type::unsigned_integer. + */ + simdjson_inline bool is_uint64() const noexcept; + /** + * return the value as a uint64_t, only valid if is_uint64() is true. + */ + simdjson_inline uint64_t get_uint64() const noexcept; + simdjson_inline operator uint64_t() const noexcept; + + /** + * return true if the automatically determined type of + * the number is number_type::signed_integer. + */ + simdjson_inline bool is_int64() const noexcept; + /** + * return the value as a int64_t, only valid if is_int64() is true. + */ + simdjson_inline int64_t get_int64() const noexcept; + simdjson_inline operator int64_t() const noexcept; + + + /** + * return true if the automatically determined type of + * the number is number_type::floating_point_number. + */ + simdjson_inline bool is_double() const noexcept; + /** + * return the value as a double, only valid if is_double() is true. + */ + simdjson_inline double get_double() const noexcept; + simdjson_inline operator double() const noexcept; + + /** + * Convert the number to a double. Though it always succeed, the conversion + * may be lossy if the number cannot be represented exactly. + */ + simdjson_inline double as_double() const noexcept; + + + protected: + /** + * The next block of declaration is designed so that we can call the number parsing + * functions on a number type. They are protected and should never be used outside + * of the core simdjson library. + */ + friend class value_iterator; + template + friend error_code numberparsing::write_float(const uint8_t* const src, bool negative, uint64_t i, const uint8_t* start_digits, size_t digit_count, int64_t exponent, W& writer); + template + friend error_code numberparsing::parse_number(const uint8_t* const src, W& writer); + /** Store a signed 64-bit value to the number. */ + simdjson_inline void append_s64(int64_t value) noexcept; + /** Store an unsigned 64-bit value to the number. */ + simdjson_inline void append_u64(uint64_t value) noexcept; + /** Store a double value to the number. */ + simdjson_inline void append_double(double value) noexcept; + /** Specifies that the value is a double, but leave it undefined. */ + simdjson_inline void skip_double() noexcept; + /** + * End of friend declarations. + */ + + /** + * Our attributes are a union type (size = 64 bits) + * followed by a type indicator. + */ + union { + double floating_point_number; + int64_t signed_integer; + uint64_t unsigned_integer; + } payload{ 0 }; + number_type type{ number_type::signed_integer }; + }; + + /** + * Write the JSON type to the output stream + * + * @param out The output stream. + * @param type The json_type. + */ + inline std::ostream& operator<<(std::ostream& out, json_type type) noexcept; + +#if SIMDJSON_EXCEPTIONS + /** + * Send JSON type to an output stream. + * + * @param out The output stream. + * @param type The json_type. + * @throw simdjson_error if the result being printed has an error. If there is an error with the + * underlying output stream, that error will be propagated (simdjson_error will not be + * thrown). + */ + inline std::ostream& operator<<(std::ostream& out, simdjson_result& type) noexcept(false); +#endif + + } // namespace ondemand + } // namespace lasx +} // namespace simdjson + +namespace simdjson { + + template<> + struct simdjson_result : public lasx::implementation_simdjson_result_base { + public: + simdjson_inline simdjson_result(lasx::ondemand::json_type&& value) noexcept; ///< @private + simdjson_inline simdjson_result(error_code error) noexcept; ///< @private + simdjson_inline simdjson_result() noexcept = default; + simdjson_inline ~simdjson_result() noexcept = default; ///< @private + }; + +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_ONDEMAND_JSON_TYPE_H +/* end file simdjson/generic/ondemand/json_type.h for lasx */ +/* including simdjson/generic/ondemand/raw_json_string.h for lasx: #include "simdjson/generic/ondemand/raw_json_string.h" */ +/* begin file simdjson/generic/ondemand/raw_json_string.h for lasx */ +#ifndef SIMDJSON_GENERIC_ONDEMAND_RAW_JSON_STRING_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_ONDEMAND_RAW_JSON_STRING_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/implementation_simdjson_result_base.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lasx { + namespace ondemand { + + /** + * A string escaped per JSON rules, terminated with quote ("). They are used to represent + * unescaped keys inside JSON documents. + * + * (In other words, a pointer to the beginning of a string, just after the start quote, inside a + * JSON file.) + * + * This class is deliberately simplistic and has little functionality. You can + * compare a raw_json_string instance with an unescaped C string, but + * that is nearly all you can do. + * + * The raw_json_string is unescaped. If you wish to write an unescaped version of it to your own + * buffer, you may do so using the parser.unescape(string, buff) method, using an ondemand::parser + * instance. Doing so requires you to have a sufficiently large buffer. + * + * The raw_json_string instances originate typically from field instance which in turn represent + * key-value pairs from object instances. From a field instance, you get the raw_json_string + * instance by calling key(). You can, if you want a more usable string_view instance, call + * the unescaped_key() method on the field instance. You may also create a raw_json_string from + * any other string value, with the value.get_raw_json_string() method. Again, you can get + * a more usable string_view instance by calling get_string(). + * + */ + class raw_json_string { + public: + /** + * Create a new invalid raw_json_string. + * + * Exists so you can declare a variable and later assign to it before use. + */ + simdjson_inline raw_json_string() noexcept = default; + + /** + * Create a new invalid raw_json_string pointed at the given location in the JSON. + * + * The given location must be just *after* the beginning quote (") in the JSON file. + * + * It *must* be terminated by a ", and be a valid JSON string. + */ + simdjson_inline raw_json_string(const uint8_t* _buf) noexcept; + /** + * Get the raw pointer to the beginning of the string in the JSON (just after the "). + * + * It is possible for this function to return a null pointer if the instance + * has outlived its existence. + */ + simdjson_inline const char* raw() const noexcept; + + /** + * This compares the current instance to the std::string_view target: returns true if + * they are byte-by-byte equal (no escaping is done) on target.size() characters, + * and if the raw_json_string instance has a quote character at byte index target.size(). + * We never read more than length + 1 bytes in the raw_json_string instance. + * If length is smaller than target.size(), this will return false. + * + * The std::string_view instance may contain any characters. However, the caller + * is responsible for setting length so that length bytes may be read in the + * raw_json_string. + * + * Performance: the comparison may be done using memcmp which may be efficient + * for long strings. + */ + simdjson_inline bool unsafe_is_equal(size_t length, std::string_view target) const noexcept; + + /** + * This compares the current instance to the std::string_view target: returns true if + * they are byte-by-byte equal (no escaping is done). + * The std::string_view instance should not contain unescaped quote characters: + * the caller is responsible for this check. See is_free_from_unescaped_quote. + * + * Performance: the comparison is done byte-by-byte which might be inefficient for + * long strings. + * + * If target is a compile-time constant, and your compiler likes you, + * you should be able to do the following without performance penalty... + * + * static_assert(raw_json_string::is_free_from_unescaped_quote(target), ""); + * s.unsafe_is_equal(target); + */ + simdjson_inline bool unsafe_is_equal(std::string_view target) const noexcept; + + /** + * This compares the current instance to the C string target: returns true if + * they are byte-by-byte equal (no escaping is done). + * The provided C string should not contain an unescaped quote character: + * the caller is responsible for this check. See is_free_from_unescaped_quote. + * + * If target is a compile-time constant, and your compiler likes you, + * you should be able to do the following without performance penalty... + * + * static_assert(raw_json_string::is_free_from_unescaped_quote(target), ""); + * s.unsafe_is_equal(target); + */ + simdjson_inline bool unsafe_is_equal(const char* target) const noexcept; + + /** + * This compares the current instance to the std::string_view target: returns true if + * they are byte-by-byte equal (no escaping is done). + */ + simdjson_inline bool is_equal(std::string_view target) const noexcept; + + /** + * This compares the current instance to the C string target: returns true if + * they are byte-by-byte equal (no escaping is done). + */ + simdjson_inline bool is_equal(const char* target) const noexcept; + + /** + * Returns true if target is free from unescaped quote. If target is known at + * compile-time, we might expect the computation to happen at compile time with + * many compilers (not all!). + */ + static simdjson_inline bool is_free_from_unescaped_quote(std::string_view target) noexcept; + static simdjson_inline bool is_free_from_unescaped_quote(const char* target) noexcept; + + private: + + + /** + * This will set the inner pointer to zero, effectively making + * this instance unusable. + */ + simdjson_inline void consume() noexcept { buf = nullptr; } + + /** + * Checks whether the inner pointer is non-null and thus usable. + */ + simdjson_inline simdjson_warn_unused bool alive() const noexcept { return buf != nullptr; } + + /** + * Unescape this JSON string, replacing \\ with \, \n with newline, etc. + * The result will be a valid UTF-8. + * + * ## IMPORTANT: string_view lifetime + * + * The string_view is only valid until the next parse() call on the parser. + * + * @param iter A json_iterator, which contains a buffer where the string will be written. + * @param allow_replacement Whether we allow replacement of invalid surrogate pairs. + */ + simdjson_inline simdjson_warn_unused simdjson_result unescape(json_iterator& iter, bool allow_replacement) const noexcept; + + /** + * Unescape this JSON string, replacing \\ with \, \n with newline, etc. + * The result may not be a valid UTF-8. https://simonsapin.github.io/wtf-8/ + * + * ## IMPORTANT: string_view lifetime + * + * The string_view is only valid until the next parse() call on the parser. + * + * @param iter A json_iterator, which contains a buffer where the string will be written. + */ + simdjson_inline simdjson_warn_unused simdjson_result unescape_wobbly(json_iterator& iter) const noexcept; + const uint8_t* buf{}; + friend class object; + friend class field; + friend class parser; + friend struct simdjson_result; + }; + + simdjson_unused simdjson_inline std::ostream& operator<<(std::ostream&, const raw_json_string&) noexcept; + + /** + * Comparisons between raw_json_string and std::string_view instances are potentially unsafe: the user is responsible + * for providing a string with no unescaped quote. Note that unescaped quotes cannot be present in valid JSON strings. + */ + simdjson_unused simdjson_inline bool operator==(const raw_json_string& a, std::string_view c) noexcept; + simdjson_unused simdjson_inline bool operator==(std::string_view c, const raw_json_string& a) noexcept; + simdjson_unused simdjson_inline bool operator!=(const raw_json_string& a, std::string_view c) noexcept; + simdjson_unused simdjson_inline bool operator!=(std::string_view c, const raw_json_string& a) noexcept; + + + } // namespace ondemand + } // namespace lasx +} // namespace simdjson + +namespace simdjson { + + template<> + struct simdjson_result : public lasx::implementation_simdjson_result_base { + public: + simdjson_inline simdjson_result(lasx::ondemand::raw_json_string&& value) noexcept; ///< @private + simdjson_inline simdjson_result(error_code error) noexcept; ///< @private + simdjson_inline simdjson_result() noexcept = default; + simdjson_inline ~simdjson_result() noexcept = default; ///< @private + + simdjson_inline simdjson_result raw() const noexcept; + simdjson_inline simdjson_warn_unused simdjson_result unescape(lasx::ondemand::json_iterator& iter, bool allow_replacement) const noexcept; + simdjson_inline simdjson_warn_unused simdjson_result unescape_wobbly(lasx::ondemand::json_iterator& iter) const noexcept; + }; + +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_ONDEMAND_RAW_JSON_STRING_H +/* end file simdjson/generic/ondemand/raw_json_string.h for lasx */ +/* including simdjson/generic/ondemand/parser.h for lasx: #include "simdjson/generic/ondemand/parser.h" */ +/* begin file simdjson/generic/ondemand/parser.h for lasx */ +#ifndef SIMDJSON_GENERIC_ONDEMAND_PARSER_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_ONDEMAND_PARSER_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/implementation_simdjson_result_base.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +#include + +namespace simdjson { + namespace lasx { + namespace ondemand { + + /** + * The default batch size for document_stream instances for this On-Demand kernel. + * Note that different On-Demand kernel may use a different DEFAULT_BATCH_SIZE value + * in the future. + */ + static constexpr size_t DEFAULT_BATCH_SIZE = 1000000; + /** + * Some adversary might try to set the batch size to 0 or 1, which might cause problems. + * We set a minimum of 32B since anything else is highly likely to be an error. In practice, + * most users will want a much larger batch size. + * + * All non-negative MINIMAL_BATCH_SIZE values should be 'safe' except that, obviously, no JSON + * document can ever span 0 or 1 byte and that very large values would create memory allocation issues. + */ + static constexpr size_t MINIMAL_BATCH_SIZE = 32; + + /** + * A JSON fragment iterator. + * + * This holds the actual iterator as well as the buffer for writing strings. + */ + class parser { + public: + /** + * Create a JSON parser. + * + * The new parser will have zero capacity. + */ + inline explicit parser(size_t max_capacity = SIMDJSON_MAXSIZE_BYTES) noexcept; + + inline parser(parser&& other) noexcept = default; + simdjson_inline parser(const parser& other) = delete; + simdjson_inline parser& operator=(const parser& other) = delete; + simdjson_inline parser& operator=(parser&& other) noexcept = default; + + /** Deallocate the JSON parser. */ + inline ~parser() noexcept = default; + + /** + * Start iterating an on-demand JSON document. + * + * ondemand::parser parser; + * document doc = parser.iterate(json); + * + * It is expected that the content is a valid UTF-8 file, containing a valid JSON document. + * Otherwise the iterate method may return an error. In particular, the whole input should be + * valid: we do not attempt to tolerate incorrect content either before or after a JSON + * document. If there is a UTF-8 BOM, the parser skips it. + * + * ### IMPORTANT: Validate what you use + * + * Calling iterate on an invalid JSON document may not immediately trigger an error. The call to + * iterate does not parse and validate the whole document. + * + * ### IMPORTANT: Buffer Lifetime + * + * Because parsing is done while you iterate, you *must* keep the JSON buffer around at least as + * long as the document iteration. + * + * ### IMPORTANT: Document Lifetime + * + * Only one iteration at a time can happen per parser, and the parser *must* be kept alive during + * iteration to ensure intermediate buffers can be accessed. Any document must be destroyed before + * you call parse() again or destroy the parser. + * + * ### REQUIRED: Buffer Padding + * + * The buffer must have at least SIMDJSON_PADDING extra allocated bytes. It does not matter what + * those bytes are initialized to, as long as they are allocated. These bytes will be read: if you + * using a sanitizer that verifies that no uninitialized byte is read, then you should initialize the + * SIMDJSON_PADDING bytes to avoid runtime warnings. + * + * @param json The JSON to parse. + * @param len The length of the JSON. + * @param capacity The number of bytes allocated in the JSON (must be at least len+SIMDJSON_PADDING). + * + * @return The document, or an error: + * - INSUFFICIENT_PADDING if the input has less than SIMDJSON_PADDING extra bytes. + * - MEMALLOC if realloc_if_needed the parser does not have enough capacity, and memory + * allocation fails. + * - EMPTY if the document is all whitespace. + * - UTF8_ERROR if the document is not valid UTF-8. + * - UNESCAPED_CHARS if a string contains control characters that must be escaped + * - UNCLOSED_STRING if there is an unclosed string in the document. + */ + simdjson_warn_unused simdjson_result iterate(padded_string_view json) & noexcept; +#ifdef SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON + simdjson_warn_unused simdjson_result iterate_allow_incomplete_json(padded_string_view json) & noexcept; +#endif // SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON + /** @overload simdjson_result iterate(padded_string_view json) & noexcept */ + simdjson_warn_unused simdjson_result iterate(const char* json, size_t len, size_t capacity) & noexcept; + /** @overload simdjson_result iterate(padded_string_view json) & noexcept */ + simdjson_warn_unused simdjson_result iterate(const uint8_t* json, size_t len, size_t capacity) & noexcept; + /** @overload simdjson_result iterate(padded_string_view json) & noexcept */ + simdjson_warn_unused simdjson_result iterate(std::string_view json, size_t capacity) & noexcept; + /** @overload simdjson_result iterate(padded_string_view json) & noexcept */ + simdjson_warn_unused simdjson_result iterate(const std::string& json) & noexcept; + /** @overload simdjson_result iterate(padded_string_view json) & noexcept */ + simdjson_warn_unused simdjson_result iterate(std::string& json) & noexcept; + /** @overload simdjson_result iterate(padded_string_view json) & noexcept */ + simdjson_warn_unused simdjson_result iterate(const simdjson_result& json) & noexcept; + /** @overload simdjson_result iterate(padded_string_view json) & noexcept */ + simdjson_warn_unused simdjson_result iterate(const simdjson_result& json) & noexcept; + /** @overload simdjson_result iterate(padded_string_view json) & noexcept */ + simdjson_warn_unused simdjson_result iterate(padded_string&& json) & noexcept = delete; + + /** + * @private + * + * Start iterating an on-demand JSON document. + * + * ondemand::parser parser; + * json_iterator doc = parser.iterate(json); + * + * ### IMPORTANT: Buffer Lifetime + * + * Because parsing is done while you iterate, you *must* keep the JSON buffer around at least as + * long as the document iteration. + * + * ### IMPORTANT: Document Lifetime + * + * Only one iteration at a time can happen per parser, and the parser *must* be kept alive during + * iteration to ensure intermediate buffers can be accessed. Any document must be destroyed before + * you call parse() again or destroy the parser. + * + * The ondemand::document instance holds the iterator. The document must remain in scope + * while you are accessing instances of ondemand::value, ondemand::object, ondemand::array. + * + * ### REQUIRED: Buffer Padding + * + * The buffer must have at least SIMDJSON_PADDING extra allocated bytes. It does not matter what + * those bytes are initialized to, as long as they are allocated. These bytes will be read: if you + * using a sanitizer that verifies that no uninitialized byte is read, then you should initialize the + * SIMDJSON_PADDING bytes to avoid runtime warnings. + * + * @param json The JSON to parse. + * + * @return The iterator, or an error: + * - INSUFFICIENT_PADDING if the input has less than SIMDJSON_PADDING extra bytes. + * - MEMALLOC if realloc_if_needed the parser does not have enough capacity, and memory + * allocation fails. + * - EMPTY if the document is all whitespace. + * - UTF8_ERROR if the document is not valid UTF-8. + * - UNESCAPED_CHARS if a string contains control characters that must be escaped + * - UNCLOSED_STRING if there is an unclosed string in the document. + */ + simdjson_warn_unused simdjson_result iterate_raw(padded_string_view json) & noexcept; + + + /** + * Parse a buffer containing many JSON documents. + * + * auto json = R"({ "foo": 1 } { "foo": 2 } { "foo": 3 } )"_padded; + * ondemand::parser parser; + * ondemand::document_stream docs = parser.iterate_many(json); + * for (auto & doc : docs) { + * std::cout << doc["foo"] << std::endl; + * } + * // Prints 1 2 3 + * + * No copy of the input buffer is made. + * + * The function is lazy: it may be that no more than one JSON document at a time is parsed. + * + * The caller is responsabile to ensure that the input string data remains unchanged and is + * not deleted during the loop. + * + * ### Format + * + * The buffer must contain a series of one or more JSON documents, concatenated into a single + * buffer, separated by ASCII whitespace. It effectively parses until it has a fully valid document, + * then starts parsing the next document at that point. (It does this with more parallelism and + * lookahead than you might think, though.) + * + * documents that consist of an object or array may omit the whitespace between them, concatenating + * with no separator. Documents that consist of a single primitive (i.e. documents that are not + * arrays or objects) MUST be separated with ASCII whitespace. + * + * The characters inside a JSON document, and between JSON documents, must be valid Unicode (UTF-8). + * If there is a UTF-8 BOM, the parser skips it. + * + * The documents must not exceed batch_size bytes (by default 1MB) or they will fail to parse. + * Setting batch_size to excessively large or excessively small values may impact negatively the + * performance. + * + * ### REQUIRED: Buffer Padding + * + * The buffer must have at least SIMDJSON_PADDING extra allocated bytes. It does not matter what + * those bytes are initialized to, as long as they are allocated. These bytes will be read: if you + * using a sanitizer that verifies that no uninitialized byte is read, then you should initialize the + * SIMDJSON_PADDING bytes to avoid runtime warnings. + * + * ### Threads + * + * When compiled with SIMDJSON_THREADS_ENABLED, this method will use a single thread under the + * hood to do some lookahead. + * + * ### Parser Capacity + * + * If the parser's current capacity is less than batch_size, it will allocate enough capacity + * to handle it (up to max_capacity). + * + * @param buf The concatenated JSON to parse. + * @param len The length of the concatenated JSON. + * @param batch_size The batch size to use. MUST be larger than the largest document. The sweet + * spot is cache-related: small enough to fit in cache, yet big enough to + * parse as many documents as possible in one tight loop. + * Defaults to 10MB, which has been a reasonable sweet spot in our tests. + * @param allow_comma_separated (defaults on false) This allows a mode where the documents are + * separated by commas instead of whitespace. It comes with a performance + * penalty because the entire document is indexed at once (and the document must be + * less than 4 GB), and there is no multithreading. In this mode, the batch_size parameter + * is effectively ignored, as it is set to at least the document size. + * @return The stream, or an error. An empty input will yield 0 documents rather than an EMPTY error. Errors: + * - MEMALLOC if the parser does not have enough capacity and memory allocation fails + * - CAPACITY if the parser does not have enough capacity and batch_size > max_capacity. + * - other json errors if parsing fails. You should not rely on these errors to always the same for the + * same document: they may vary under runtime dispatch (so they may vary depending on your system and hardware). + */ + inline simdjson_result iterate_many(const uint8_t* buf, size_t len, size_t batch_size = DEFAULT_BATCH_SIZE, bool allow_comma_separated = false) noexcept; + /** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size) */ + inline simdjson_result iterate_many(const char* buf, size_t len, size_t batch_size = DEFAULT_BATCH_SIZE, bool allow_comma_separated = false) noexcept; + /** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size) */ + inline simdjson_result iterate_many(const std::string& s, size_t batch_size = DEFAULT_BATCH_SIZE, bool allow_comma_separated = false) noexcept; + inline simdjson_result iterate_many(const std::string&& s, size_t batch_size, bool allow_comma_separated = false) = delete;// unsafe + /** @overload parse_many(const uint8_t *buf, size_t len, size_t batch_size) */ + inline simdjson_result iterate_many(const padded_string& s, size_t batch_size = DEFAULT_BATCH_SIZE, bool allow_comma_separated = false) noexcept; + inline simdjson_result iterate_many(const padded_string&& s, size_t batch_size, bool allow_comma_separated = false) = delete;// unsafe + + /** @private We do not want to allow implicit conversion from C string to std::string. */ + simdjson_result iterate_many(const char* buf, size_t batch_size = DEFAULT_BATCH_SIZE) noexcept = delete; + + /** The capacity of this parser (the largest document it can process). */ + simdjson_pure simdjson_inline size_t capacity() const noexcept; + /** The maximum capacity of this parser (the largest document it is allowed to process). */ + simdjson_pure simdjson_inline size_t max_capacity() const noexcept; + simdjson_inline void set_max_capacity(size_t max_capacity) noexcept; + /** + * The maximum depth of this parser (the most deeply nested objects and arrays it can process). + * This parameter is only relevant when the macro SIMDJSON_DEVELOPMENT_CHECKS is set to true. + * The document's instance current_depth() method should be used to monitor the parsing + * depth and limit it if desired. + */ + simdjson_pure simdjson_inline size_t max_depth() const noexcept; + + /** + * Ensure this parser has enough memory to process JSON documents up to `capacity` bytes in length + * and `max_depth` depth. + * + * The max_depth parameter is only relevant when the macro SIMDJSON_DEVELOPMENT_CHECKS is set to true. + * The document's instance current_depth() method should be used to monitor the parsing + * depth and limit it if desired. + * + * @param capacity The new capacity. + * @param max_depth The new max_depth. Defaults to DEFAULT_MAX_DEPTH. + * @return The error, if there is one. + */ + simdjson_warn_unused error_code allocate(size_t capacity, size_t max_depth = DEFAULT_MAX_DEPTH) noexcept; + +#ifdef SIMDJSON_THREADS_ENABLED + /** + * The parser instance can use threads when they are available to speed up some + * operations. It is enabled by default. Changing this attribute will change the + * behavior of the parser for future operations. + */ + bool threaded{ true }; +#endif + + /** + * Unescape this JSON string, replacing \\ with \, \n with newline, etc. to a user-provided buffer. + * The result must be valid UTF-8. + * The provided pointer is advanced to the end of the string by reference, and a string_view instance + * is returned. You can ensure that your buffer is large enough by allocating a block of memory at least + * as large as the input JSON plus SIMDJSON_PADDING and then unescape all strings to this one buffer. + * + * This unescape function is a low-level function. If you want a more user-friendly approach, you should + * avoid raw_json_string instances (e.g., by calling unescaped_key() instead of key() or get_string() + * instead of get_raw_json_string()). + * + * ## IMPORTANT: string_view lifetime + * + * The string_view is only valid as long as the bytes in dst. + * + * @param raw_json_string input + * @param dst A pointer to a buffer at least large enough to write this string as well as + * an additional SIMDJSON_PADDING bytes. + * @param allow_replacement Whether we allow a replacement if the input string contains unmatched surrogate pairs. + * @return A string_view pointing at the unescaped string in dst + * @error STRING_ERROR if escapes are incorrect. + */ + simdjson_inline simdjson_result unescape(raw_json_string in, uint8_t*& dst, bool allow_replacement = false) const noexcept; + + /** + * Unescape this JSON string, replacing \\ with \, \n with newline, etc. to a user-provided buffer. + * The result may not be valid UTF-8. See https://simonsapin.github.io/wtf-8/ + * The provided pointer is advanced to the end of the string by reference, and a string_view instance + * is returned. You can ensure that your buffer is large enough by allocating a block of memory at least + * as large as the input JSON plus SIMDJSON_PADDING and then unescape all strings to this one buffer. + * + * This unescape function is a low-level function. If you want a more user-friendly approach, you should + * avoid raw_json_string instances (e.g., by calling unescaped_key() instead of key() or get_string() + * instead of get_raw_json_string()). + * + * ## IMPORTANT: string_view lifetime + * + * The string_view is only valid as long as the bytes in dst. + * + * @param raw_json_string input + * @param dst A pointer to a buffer at least large enough to write this string as well as + * an additional SIMDJSON_PADDING bytes. + * @return A string_view pointing at the unescaped string in dst + * @error STRING_ERROR if escapes are incorrect. + */ + simdjson_inline simdjson_result unescape_wobbly(raw_json_string in, uint8_t*& dst) const noexcept; + +#if SIMDJSON_DEVELOPMENT_CHECKS + /** + * Returns true if string_buf_loc is outside of the allocated range for the + * the string buffer. When true, it indicates that the string buffer has overflowed. + * This is a development-time check that is not needed in production. It can be + * used to detect buffer overflows in the string buffer and usafe usage of the + * string buffer. + */ + bool string_buffer_overflow(const uint8_t* string_buf_loc) const noexcept; +#endif + + private: + /** @private [for benchmarking access] The implementation to use */ + std::unique_ptr implementation{}; + size_t _capacity{ 0 }; + size_t _max_capacity; + size_t _max_depth{ DEFAULT_MAX_DEPTH }; + std::unique_ptr string_buf{}; +#if SIMDJSON_DEVELOPMENT_CHECKS + std::unique_ptr start_positions{}; +#endif + + friend class json_iterator; + friend class document_stream; + }; + + } // namespace ondemand + } // namespace lasx +} // namespace simdjson + +namespace simdjson { + + template<> + struct simdjson_result : public lasx::implementation_simdjson_result_base { + public: + simdjson_inline simdjson_result(lasx::ondemand::parser&& value) noexcept; ///< @private + simdjson_inline simdjson_result(error_code error) noexcept; ///< @private + simdjson_inline simdjson_result() noexcept = default; + }; + +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_ONDEMAND_PARSER_H +/* end file simdjson/generic/ondemand/parser.h for lasx */ + +// All other declarations +/* including simdjson/generic/ondemand/array.h for lasx: #include "simdjson/generic/ondemand/array.h" */ +/* begin file simdjson/generic/ondemand/array.h for lasx */ +#ifndef SIMDJSON_GENERIC_ONDEMAND_ARRAY_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_ONDEMAND_ARRAY_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/implementation_simdjson_result_base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/value_iterator.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lasx { + namespace ondemand { + + /** + * A forward-only JSON array. + */ + class array { + public: + /** + * Create a new invalid array. + * + * Exists so you can declare a variable and later assign to it before use. + */ + simdjson_inline array() noexcept = default; + + /** + * Begin array iteration. + * + * Part of the std::iterable interface. + */ + simdjson_inline simdjson_result begin() noexcept; + /** + * Sentinel representing the end of the array. + * + * Part of the std::iterable interface. + */ + simdjson_inline simdjson_result end() noexcept; + /** + * This method scans the array and counts the number of elements. + * The count_elements method should always be called before you have begun + * iterating through the array: it is expected that you are pointing at + * the beginning of the array. + * The runtime complexity is linear in the size of the array. After + * calling this function, if successful, the array is 'rewinded' at its + * beginning as if it had never been accessed. If the JSON is malformed (e.g., + * there is a missing comma), then an error is returned and it is no longer + * safe to continue. + * + * To check that an array is empty, it is more performant to use + * the is_empty() method. + */ + simdjson_inline simdjson_result count_elements() & noexcept; + /** + * This method scans the beginning of the array and checks whether the + * array is empty. + * The runtime complexity is constant time. After + * calling this function, if successful, the array is 'rewinded' at its + * beginning as if it had never been accessed. If the JSON is malformed (e.g., + * there is a missing comma), then an error is returned and it is no longer + * safe to continue. + */ + simdjson_inline simdjson_result is_empty() & noexcept; + /** + * Reset the iterator so that we are pointing back at the + * beginning of the array. You should still consume values only once even if you + * can iterate through the array more than once. If you unescape a string + * within the array more than once, you have unsafe code. Note that rewinding + * an array means that you may need to reparse it anew: it is not a free + * operation. + * + * @returns true if the array contains some elements (not empty) + */ + inline simdjson_result reset() & noexcept; + /** + * Get the value associated with the given JSON pointer. We use the RFC 6901 + * https://tools.ietf.org/html/rfc6901 standard, interpreting the current node + * as the root of its own JSON document. + * + * ondemand::parser parser; + * auto json = R"([ { "foo": { "a": [ 10, 20, 30 ] }} ])"_padded; + * auto doc = parser.iterate(json); + * doc.at_pointer("/0/foo/a/1") == 20 + * + * Note that at_pointer() called on the document automatically calls the document's rewind + * method between each call. It invalidates all previously accessed arrays, objects and values + * that have not been consumed. Yet it is not the case when calling at_pointer on an array + * instance: there is no rewind and no invalidation. + * + * You may only call at_pointer on an array after it has been created, but before it has + * been first accessed. When calling at_pointer on an array, the pointer is advanced to + * the location indicated by the JSON pointer (in case of success). It is no longer possible + * to call at_pointer on the same array. + * + * Also note that at_pointer() relies on find_field() which implies that we do not unescape keys when matching. + * + * @return The value associated with the given JSON pointer, or: + * - NO_SUCH_FIELD if a field does not exist in an object + * - INDEX_OUT_OF_BOUNDS if an array index is larger than an array length + * - INCORRECT_TYPE if a non-integer is used to access an array + * - INVALID_JSON_POINTER if the JSON pointer is invalid and cannot be parsed + */ + inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; + + /** + * Get the value associated with the given JSONPath expression. We only support + * JSONPath queries that trivially convertible to JSON Pointer queries: key + * names and array indices. + * + * https://datatracker.ietf.org/doc/html/draft-normington-jsonpath-00 + * + * @return The value associated with the given JSONPath expression, or: + * - INVALID_JSON_POINTER if the JSONPath to JSON Pointer conversion fails + * - NO_SUCH_FIELD if a field does not exist in an object + * - INDEX_OUT_OF_BOUNDS if an array index is larger than an array length + * - INCORRECT_TYPE if a non-integer is used to access an array + */ + inline simdjson_result at_path(std::string_view json_path) noexcept; + + /** + * Consumes the array and returns a string_view instance corresponding to the + * array as represented in JSON. It points inside the original document. + */ + simdjson_inline simdjson_result raw_json() noexcept; + + /** + * Get the value at the given index. This function has linear-time complexity. + * This function should only be called once on an array instance since the array iterator is not reset between each call. + * + * @return The value at the given index, or: + * - INDEX_OUT_OF_BOUNDS if the array index is larger than an array length + */ + simdjson_inline simdjson_result at(size_t index) noexcept; + protected: + /** + * Go to the end of the array, no matter where you are right now. + */ + simdjson_inline error_code consume() noexcept; + + /** + * Begin array iteration. + * + * @param iter The iterator. Must be where the initial [ is expected. Will be *moved* into the + * resulting array. + * @error INCORRECT_TYPE if the iterator is not at [. + */ + static simdjson_inline simdjson_result start(value_iterator& iter) noexcept; + /** + * Begin array iteration from the root. + * + * @param iter The iterator. Must be where the initial [ is expected. Will be *moved* into the + * resulting array. + * @error INCORRECT_TYPE if the iterator is not at [. + * @error TAPE_ERROR if there is no closing ] at the end of the document. + */ + static simdjson_inline simdjson_result start_root(value_iterator& iter) noexcept; + /** + * Begin array iteration. + * + * This version of the method should be called after the initial [ has been verified, and is + * intended for use by switch statements that check the type of a value. + * + * @param iter The iterator. Must be after the initial [. Will be *moved* into the resulting array. + */ + static simdjson_inline simdjson_result started(value_iterator& iter) noexcept; + + /** + * Create an array at the given Internal array creation. Call array::start() or array::started() instead of this. + * + * @param iter The iterator. Must either be at the start of the first element with iter.is_alive() + * == true, or past the [] with is_alive() == false if the array is empty. Will be *moved* + * into the resulting array. + */ + simdjson_inline array(const value_iterator& iter) noexcept; + + /** + * Iterator marking current position. + * + * iter.is_alive() == false indicates iteration is complete. + */ + value_iterator iter{}; + + friend class value; + friend class document; + friend struct simdjson_result; + friend struct simdjson_result; + friend class array_iterator; + }; + + } // namespace ondemand + } // namespace lasx +} // namespace simdjson + +namespace simdjson { + + template<> + struct simdjson_result : public lasx::implementation_simdjson_result_base { + public: + simdjson_inline simdjson_result(lasx::ondemand::array&& value) noexcept; ///< @private + simdjson_inline simdjson_result(error_code error) noexcept; ///< @private + simdjson_inline simdjson_result() noexcept = default; + + simdjson_inline simdjson_result begin() noexcept; + simdjson_inline simdjson_result end() noexcept; + inline simdjson_result count_elements() & noexcept; + inline simdjson_result is_empty() & noexcept; + inline simdjson_result reset() & noexcept; + simdjson_inline simdjson_result at(size_t index) noexcept; + simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; + simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; + simdjson_inline simdjson_result raw_json() noexcept; + + }; + +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_ONDEMAND_ARRAY_H +/* end file simdjson/generic/ondemand/array.h for lasx */ +/* including simdjson/generic/ondemand/array_iterator.h for lasx: #include "simdjson/generic/ondemand/array_iterator.h" */ +/* begin file simdjson/generic/ondemand/array_iterator.h for lasx */ +#ifndef SIMDJSON_GENERIC_ONDEMAND_ARRAY_ITERATOR_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_ONDEMAND_ARRAY_ITERATOR_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/implementation_simdjson_result_base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/value_iterator.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + + +namespace simdjson { + namespace lasx { + namespace ondemand { + + /** + * A forward-only JSON array. + * + * This is an input_iterator, meaning: + * - It is forward-only + * - * must be called exactly once per element. + * - ++ must be called exactly once in between each * (*, ++, *, ++, * ...) + */ + class array_iterator { + public: + /** Create a new, invalid array iterator. */ + simdjson_inline array_iterator() noexcept = default; + + // + // Iterator interface + // + + /** + * Get the current element. + * + * Part of the std::iterator interface. + */ + simdjson_inline simdjson_result operator*() noexcept; // MUST ONLY BE CALLED ONCE PER ITERATION. + /** + * Check if we are at the end of the JSON. + * + * Part of the std::iterator interface. + * + * @return true if there are no more elements in the JSON array. + */ + simdjson_inline bool operator==(const array_iterator&) const noexcept; + /** + * Check if there are more elements in the JSON array. + * + * Part of the std::iterator interface. + * + * @return true if there are more elements in the JSON array. + */ + simdjson_inline bool operator!=(const array_iterator&) const noexcept; + /** + * Move to the next element. + * + * Part of the std::iterator interface. + */ + simdjson_inline array_iterator& operator++() noexcept; + + private: + value_iterator iter{}; + + simdjson_inline array_iterator(const value_iterator& iter) noexcept; + + friend class array; + friend class value; + friend struct simdjson_result; + }; + + } // namespace ondemand + } // namespace lasx +} // namespace simdjson + +namespace simdjson { + + template<> + struct simdjson_result : public lasx::implementation_simdjson_result_base { + public: + simdjson_inline simdjson_result(lasx::ondemand::array_iterator&& value) noexcept; ///< @private + simdjson_inline simdjson_result(error_code error) noexcept; ///< @private + simdjson_inline simdjson_result() noexcept = default; + + // + // Iterator interface + // + + simdjson_inline simdjson_result operator*() noexcept; // MUST ONLY BE CALLED ONCE PER ITERATION. + simdjson_inline bool operator==(const simdjson_result&) const noexcept; + simdjson_inline bool operator!=(const simdjson_result&) const noexcept; + simdjson_inline simdjson_result& operator++() noexcept; + }; + +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_ONDEMAND_ARRAY_ITERATOR_H +/* end file simdjson/generic/ondemand/array_iterator.h for lasx */ +/* including simdjson/generic/ondemand/document.h for lasx: #include "simdjson/generic/ondemand/document.h" */ +/* begin file simdjson/generic/ondemand/document.h for lasx */ +#ifndef SIMDJSON_GENERIC_ONDEMAND_DOCUMENT_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_ONDEMAND_DOCUMENT_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/json_iterator.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lasx { + namespace ondemand { + + /** + * A JSON document. It holds a json_iterator instance. + * + * Used by tokens to get text, and string buffer location. + * + * You must keep the document around during iteration. + */ + class document { + public: + /** + * Create a new invalid document. + * + * Exists so you can declare a variable and later assign to it before use. + */ + simdjson_inline document() noexcept = default; + simdjson_inline document(const document& other) noexcept = delete; // pass your documents by reference, not by copy + simdjson_inline document(document&& other) noexcept = default; + simdjson_inline document& operator=(const document& other) noexcept = delete; + simdjson_inline document& operator=(document&& other) noexcept = default; + + /** + * Cast this JSON value to an array. + * + * @returns An object that can be used to iterate the array. + * @returns INCORRECT_TYPE If the JSON value is not an array. + */ + simdjson_inline simdjson_result get_array() & noexcept; + /** + * Cast this JSON value to an object. + * + * @returns An object that can be used to look up or iterate fields. + * @returns INCORRECT_TYPE If the JSON value is not an object. + */ + simdjson_inline simdjson_result get_object() & noexcept; + /** + * Cast this JSON value to an unsigned integer. + * + * @returns A signed 64-bit integer. + * @returns INCORRECT_TYPE If the JSON value is not a 64-bit unsigned integer. + */ + simdjson_inline simdjson_result get_uint64() noexcept; + /** + * Cast this JSON value (inside string) to an unsigned integer. + * + * @returns A signed 64-bit integer. + * @returns INCORRECT_TYPE If the JSON value is not a 64-bit unsigned integer. + */ + simdjson_inline simdjson_result get_uint64_in_string() noexcept; + /** + * Cast this JSON value to a signed integer. + * + * @returns A signed 64-bit integer. + * @returns INCORRECT_TYPE If the JSON value is not a 64-bit integer. + */ + simdjson_inline simdjson_result get_int64() noexcept; + /** + * Cast this JSON value (inside string) to a signed integer. + * + * @returns A signed 64-bit integer. + * @returns INCORRECT_TYPE If the JSON value is not a 64-bit integer. + */ + simdjson_inline simdjson_result get_int64_in_string() noexcept; + /** + * Cast this JSON value to a double. + * + * @returns A double. + * @returns INCORRECT_TYPE If the JSON value is not a valid floating-point number. + */ + simdjson_inline simdjson_result get_double() noexcept; + + /** + * Cast this JSON value (inside string) to a double. + * + * @returns A double. + * @returns INCORRECT_TYPE If the JSON value is not a valid floating-point number. + */ + simdjson_inline simdjson_result get_double_in_string() noexcept; + /** + * Cast this JSON value to a string. + * + * The string is guaranteed to be valid UTF-8. + * + * Important: Calling get_string() twice on the same document is an error. + * + * @param Whether to allow a replacement character for unmatched surrogate pairs. + * @returns An UTF-8 string. The string is stored in the parser and will be invalidated the next + * time it parses a document or when it is destroyed. + * @returns INCORRECT_TYPE if the JSON value is not a string. + */ + simdjson_inline simdjson_result get_string(bool allow_replacement = false) noexcept; + /** + * Attempts to fill the provided std::string reference with the parsed value of the current string. + * + * The string is guaranteed to be valid UTF-8. + * + * Important: a value should be consumed once. Calling get_string() twice on the same value + * is an error. + * + * Performance: This method may be slower than get_string() or get_string(bool) because it may need to allocate memory. + * We recommend you avoid allocating an std::string unless you need to. + * + * @returns INCORRECT_TYPE if the JSON value is not a string. Otherwise, we return SUCCESS. + */ + template + simdjson_inline error_code get_string(string_type& receiver, bool allow_replacement = false) noexcept; + /** + * Cast this JSON value to a string. + * + * The string is not guaranteed to be valid UTF-8. See https://simonsapin.github.io/wtf-8/ + * + * Important: Calling get_wobbly_string() twice on the same document is an error. + * + * @returns An UTF-8 string. The string is stored in the parser and will be invalidated the next + * time it parses a document or when it is destroyed. + * @returns INCORRECT_TYPE if the JSON value is not a string. + */ + simdjson_inline simdjson_result get_wobbly_string() noexcept; + /** + * Cast this JSON value to a raw_json_string. + * + * The string is guaranteed to be valid UTF-8, and may have escapes in it (e.g. \\ or \n). + * + * @returns A pointer to the raw JSON for the given string. + * @returns INCORRECT_TYPE if the JSON value is not a string. + */ + simdjson_inline simdjson_result get_raw_json_string() noexcept; + /** + * Cast this JSON value to a bool. + * + * @returns A bool value. + * @returns INCORRECT_TYPE if the JSON value is not true or false. + */ + simdjson_inline simdjson_result get_bool() noexcept; + /** + * Cast this JSON value to a value when the document is an object or an array. + * + * You must not have begun iterating through the object or array. When + * SIMDJSON_DEVELOPMENT_CHECKS is set to 1 (which is the case when building in Debug mode + * by default), and you have already begun iterating, + * you will get an OUT_OF_ORDER_ITERATION error. If you have begun iterating, you can use + * rewind() to reset the document to its initial state before calling this method. + * + * @returns A value if a JSON array or object cannot be found. + * @returns SCALAR_DOCUMENT_AS_VALUE error is the document is a scalar (see is_scalar() function). + */ + simdjson_inline simdjson_result get_value() noexcept; + + /** + * Checks if this JSON value is null. If and only if the value is + * null, then it is consumed (we advance). If we find a token that + * begins with 'n' but is not 'null', then an error is returned. + * + * @returns Whether the value is null. + * @returns INCORRECT_TYPE If the JSON value begins with 'n' and is not 'null'. + */ + simdjson_inline simdjson_result is_null() noexcept; + + /** + * Get this value as the given type. + * + * Supported types: object, array, raw_json_string, string_view, uint64_t, int64_t, double, bool + * + * You may use get_double(), get_bool(), get_uint64(), get_int64(), + * get_object(), get_array(), get_raw_json_string(), or get_string() instead. + * + * @returns A value of the given type, parsed from the JSON. + * @returns INCORRECT_TYPE If the JSON value is not the given type. + */ + template simdjson_inline simdjson_result get() & noexcept { + // Unless the simdjson library or the user provides an inline implementation, calling this method should + // immediately fail. + static_assert(!sizeof(T), "The get method with given type is not implemented by the simdjson library. " + "The supported types are ondemand::object, ondemand::array, raw_json_string, std::string_view, uint64_t, " + "int64_t, double, and bool. We recommend you use get_double(), get_bool(), get_uint64(), get_int64(), " + " get_object(), get_array(), get_raw_json_string(), or get_string() instead of the get template." + " You may also add support for custom types, see our documentation."); + } + /** @overload template simdjson_result get() & noexcept */ + template simdjson_deprecated simdjson_inline simdjson_result get() && noexcept { + // Unless the simdjson library or the user provides an inline implementation, calling this method should + // immediately fail. + static_assert(!sizeof(T), "The get method with given type is not implemented by the simdjson library. " + "The supported types are ondemand::object, ondemand::array, raw_json_string, std::string_view, uint64_t, " + "int64_t, double, and bool. We recommend you use get_double(), get_bool(), get_uint64(), get_int64(), " + " get_object(), get_array(), get_raw_json_string(), or get_string() instead of the get template." + " You may also add support for custom types, see our documentation."); + } + + /** + * Get this value as the given type. + * + * Supported types: object, array, raw_json_string, string_view, uint64_t, int64_t, double, bool, value + * + * Be mindful that the document instance must remain in scope while you are accessing object, array and value instances. + * + * @param out This is set to a value of the given type, parsed from the JSON. If there is an error, this may not be initialized. + * @returns INCORRECT_TYPE If the JSON value is not an object. + * @returns SUCCESS If the parse succeeded and the out parameter was set to the value. + */ + template simdjson_inline error_code get(T& out) & noexcept; + /** @overload template error_code get(T &out) & noexcept */ + template simdjson_deprecated simdjson_inline error_code get(T& out) && noexcept; + +#if SIMDJSON_EXCEPTIONS + /** + * Cast this JSON value to an instance of type T. The programmer is responsible for + * providing an implementation of get for the type T, if T is not one of the types + * supported by the library (object, array, raw_json_string, string_view, uint64_t, etc.) + * + * See https://github.com/simdjson/simdjson/blob/master/doc/basics.md#adding-support-for-custom-types + * + * @returns An instance of type T + */ + template + explicit simdjson_inline operator T() & noexcept(false); + template + explicit simdjson_deprecated simdjson_inline operator T() && noexcept(false); + + /** + * Cast this JSON value to an array. + * + * @returns An object that can be used to iterate the array. + * @exception simdjson_error(INCORRECT_TYPE) If the JSON value is not an array. + */ + simdjson_inline operator array() & noexcept(false); + /** + * Cast this JSON value to an object. + * + * @returns An object that can be used to look up or iterate fields. + * @exception simdjson_error(INCORRECT_TYPE) If the JSON value is not an object. + */ + simdjson_inline operator object() & noexcept(false); + /** + * Cast this JSON value to an unsigned integer. + * + * @returns A signed 64-bit integer. + * @exception simdjson_error(INCORRECT_TYPE) If the JSON value is not a 64-bit unsigned integer. + */ + simdjson_inline operator uint64_t() noexcept(false); + /** + * Cast this JSON value to a signed integer. + * + * @returns A signed 64-bit integer. + * @exception simdjson_error(INCORRECT_TYPE) If the JSON value is not a 64-bit integer. + */ + simdjson_inline operator int64_t() noexcept(false); + /** + * Cast this JSON value to a double. + * + * @returns A double. + * @exception simdjson_error(INCORRECT_TYPE) If the JSON value is not a valid floating-point number. + */ + simdjson_inline operator double() noexcept(false); + /** + * Cast this JSON value to a string. + * + * The string is guaranteed to be valid UTF-8. + * + * @returns An UTF-8 string. The string is stored in the parser and will be invalidated the next + * time it parses a document or when it is destroyed. + * @exception simdjson_error(INCORRECT_TYPE) if the JSON value is not a string. + */ + simdjson_inline operator std::string_view() noexcept(false); + /** + * Cast this JSON value to a raw_json_string. + * + * The string is guaranteed to be valid UTF-8, and may have escapes in it (e.g. \\ or \n). + * + * @returns A pointer to the raw JSON for the given string. + * @exception simdjson_error(INCORRECT_TYPE) if the JSON value is not a string. + */ + simdjson_inline operator raw_json_string() noexcept(false); + /** + * Cast this JSON value to a bool. + * + * @returns A bool value. + * @exception simdjson_error(INCORRECT_TYPE) if the JSON value is not true or false. + */ + simdjson_inline operator bool() noexcept(false); + /** + * Cast this JSON value to a value when the document is an object or an array. + * + * You must not have begun iterating through the object or array. When + * SIMDJSON_DEVELOPMENT_CHECKS is defined, and you have already begun iterating, + * you will get an OUT_OF_ORDER_ITERATION error. If you have begun iterating, you can use + * rewind() to reset the document to its initial state before calling this method. + * + * @returns A value value if a JSON array or object cannot be found. + * @exception SCALAR_DOCUMENT_AS_VALUE error is the document is a scalar (see is_scalar() function). + */ + simdjson_inline operator value() noexcept(false); +#endif + /** + * This method scans the array and counts the number of elements. + * The count_elements method should always be called before you have begun + * iterating through the array: it is expected that you are pointing at + * the beginning of the array. + * The runtime complexity is linear in the size of the array. After + * calling this function, if successful, the array is 'rewinded' at its + * beginning as if it had never been accessed. If the JSON is malformed (e.g., + * there is a missing comma), then an error is returned and it is no longer + * safe to continue. + */ + simdjson_inline simdjson_result count_elements() & noexcept; + /** + * This method scans the object and counts the number of key-value pairs. + * The count_fields method should always be called before you have begun + * iterating through the object: it is expected that you are pointing at + * the beginning of the object. + * The runtime complexity is linear in the size of the object. After + * calling this function, if successful, the object is 'rewinded' at its + * beginning as if it had never been accessed. If the JSON is malformed (e.g., + * there is a missing comma), then an error is returned and it is no longer + * safe to continue. + * + * To check that an object is empty, it is more performant to use + * the is_empty() method. + */ + simdjson_inline simdjson_result count_fields() & noexcept; + /** + * Get the value at the given index in the array. This function has linear-time complexity. + * This function should only be called once on an array instance since the array iterator is not reset between each call. + * + * @return The value at the given index, or: + * - INDEX_OUT_OF_BOUNDS if the array index is larger than an array length + */ + simdjson_inline simdjson_result at(size_t index) & noexcept; + /** + * Begin array iteration. + * + * Part of the std::iterable interface. + */ + simdjson_inline simdjson_result begin() & noexcept; + /** + * Sentinel representing the end of the array. + * + * Part of the std::iterable interface. + */ + simdjson_inline simdjson_result end() & noexcept; + + /** + * Look up a field by name on an object (order-sensitive). + * + * The following code reads z, then y, then x, and thus will not retrieve x or y if fed the + * JSON `{ "x": 1, "y": 2, "z": 3 }`: + * + * ```c++ + * simdjson::ondemand::parser parser; + * auto obj = parser.parse(R"( { "x": 1, "y": 2, "z": 3 } )"_padded); + * double z = obj.find_field("z"); + * double y = obj.find_field("y"); + * double x = obj.find_field("x"); + * ``` + * + * **Raw Keys:** The lookup will be done against the *raw* key, and will not unescape keys. + * e.g. `object["a"]` will match `{ "a": 1 }`, but will *not* match `{ "\u0061": 1 }`. + * + * + * You must consume the fields on an object one at a time. A request for a new key + * invalidates previous field values: it makes them unsafe. E.g., the array + * given by content["bids"].get_array() should not be accessed after you have called + * content["asks"].get_array(). You can detect such mistakes by first compiling and running + * the code in Debug mode (or with the macro `SIMDJSON_DEVELOPMENT_CHECKS` set to 1): an + * OUT_OF_ORDER_ITERATION error is generated. + * + * You are expected to access keys only once. You should access the value corresponding to + * a key a single time. Doing object["mykey"].to_string()and then again object["mykey"].to_string() + * is an error. + * + * @param key The key to look up. + * @returns The value of the field, or NO_SUCH_FIELD if the field is not in the object. + */ + simdjson_inline simdjson_result find_field(std::string_view key) & noexcept; + /** @overload simdjson_inline simdjson_result find_field(std::string_view key) & noexcept; */ + simdjson_inline simdjson_result find_field(const char* key) & noexcept; + + /** + * Look up a field by name on an object, without regard to key order. + * + * **Performance Notes:** This is a bit less performant than find_field(), though its effect varies + * and often appears negligible. It starts out normally, starting out at the last field; but if + * the field is not found, it scans from the beginning of the object to see if it missed it. That + * missing case has a non-cache-friendly bump and lots of extra scanning, especially if the object + * in question is large. The fact that the extra code is there also bumps the executable size. + * + * It is the default, however, because it would be highly surprising (and hard to debug) if the + * default behavior failed to look up a field just because it was in the wrong order--and many + * APIs assume this. Therefore, you must be explicit if you want to treat objects as out of order. + * + * Use find_field() if you are sure fields will be in order (or are willing to treat it as if the + * field was not there when they are not in order). + * + * You must consume the fields on an object one at a time. A request for a new key + * invalidates previous field values: it makes them unsafe. E.g., the array + * given by content["bids"].get_array() should not be accessed after you have called + * content["asks"].get_array(). You can detect such mistakes by first compiling and running + * the code in Debug mode (or with the macro `SIMDJSON_DEVELOPMENT_CHECKS` set to 1): an + * OUT_OF_ORDER_ITERATION error is generated. + * + * You are expected to access keys only once. You should access the value corresponding to a key + * a single time. Doing object["mykey"].to_string() and then again object["mykey"].to_string() + * is an error. + * + * @param key The key to look up. + * @returns The value of the field, or NO_SUCH_FIELD if the field is not in the object. + */ + simdjson_inline simdjson_result find_field_unordered(std::string_view key) & noexcept; + /** @overload simdjson_inline simdjson_result find_field_unordered(std::string_view key) & noexcept; */ + simdjson_inline simdjson_result find_field_unordered(const char* key) & noexcept; + /** @overload simdjson_inline simdjson_result find_field_unordered(std::string_view key) & noexcept; */ + simdjson_inline simdjson_result operator[](std::string_view key) & noexcept; + /** @overload simdjson_inline simdjson_result find_field_unordered(std::string_view key) & noexcept; */ + simdjson_inline simdjson_result operator[](const char* key) & noexcept; + + /** + * Get the type of this JSON value. It does not validate or consume the value. + * E.g., you must still call "is_null()" to check that a value is null even if + * "type()" returns json_type::null. + * + * NOTE: If you're only expecting a value to be one type (a typical case), it's generally + * better to just call .get_double, .get_string, etc. and check for INCORRECT_TYPE (or just + * let it throw an exception). + * + * @error TAPE_ERROR when the JSON value is a bad token like "}" "," or "alse". + */ + simdjson_inline simdjson_result type() noexcept; + + /** + * Checks whether the document is a scalar (string, number, null, Boolean). + * Returns false when there it is an array or object. + * + * @returns true if the type is string, number, null, Boolean + * @error TAPE_ERROR when the JSON value is a bad token like "}" "," or "alse". + */ + simdjson_inline simdjson_result is_scalar() noexcept; + + /** + * Checks whether the document is a string. + * + * @returns true if the type is string + * @error TAPE_ERROR when the JSON value is a bad token like "}" "," or "alse". + */ + simdjson_inline simdjson_result is_string() noexcept; + + /** + * Checks whether the document is a negative number. + * + * @returns true if the number if negative. + */ + simdjson_inline bool is_negative() noexcept; + /** + * Checks whether the document is an integer number. Note that + * this requires to partially parse the number string. If + * the value is determined to be an integer, it may still + * not parse properly as an integer in subsequent steps + * (e.g., it might overflow). + * + * @returns true if the number if negative. + */ + simdjson_inline simdjson_result is_integer() noexcept; + /** + * Determine the number type (integer or floating-point number) as quickly + * as possible. This function does not fully validate the input. It is + * useful when you only need to classify the numbers, without parsing them. + * + * If you are planning to retrieve the value or you need full validation, + * consider using the get_number() method instead: it will fully parse + * and validate the input, and give you access to the type: + * get_number().get_number_type(). + * + * get_number_type() is number_type::unsigned_integer if we have + * an integer greater or equal to 9223372036854775808 and no larger than 18446744073709551615. + * get_number_type() is number_type::signed_integer if we have an + * integer that is less than 9223372036854775808 and greater or equal to -9223372036854775808. + * get_number_type() is number_type::big_integer if we have an integer outside + * of those ranges (either larger than 18446744073709551615 or smaller than -9223372036854775808). + * Otherwise, get_number_type() has value number_type::floating_point_number + * + * This function requires processing the number string, but it is expected + * to be faster than get_number().get_number_type() because it is does not + * parse the number value. + * + * @returns the type of the number + */ + simdjson_inline simdjson_result get_number_type() noexcept; + + /** + * Attempt to parse an ondemand::number. An ondemand::number may + * contain an integer value or a floating-point value, the simdjson + * library will autodetect the type. Thus it is a dynamically typed + * number. Before accessing the value, you must determine the detected + * type. + * + * number.get_number_type() is number_type::signed_integer if we have + * an integer in [-9223372036854775808,9223372036854775808) + * You can recover the value by calling number.get_int64() and you + * have that number.is_int64() is true. + * + * number.get_number_type() is number_type::unsigned_integer if we have + * an integer in [9223372036854775808,18446744073709551616) + * You can recover the value by calling number.get_uint64() and you + * have that number.is_uint64() is true. + * + * Otherwise, number.get_number_type() has value number_type::floating_point_number + * and we have a binary64 number. + * You can recover the value by calling number.get_double() and you + * have that number.is_double() is true. + * + * You must check the type before accessing the value: it is an error + * to call "get_int64()" when number.get_number_type() is not + * number_type::signed_integer and when number.is_int64() is false. + */ + simdjson_warn_unused simdjson_inline simdjson_result get_number() noexcept; + + /** + * Get the raw JSON for this token. + * + * The string_view will always point into the input buffer. + * + * The string_view will start at the beginning of the token, and include the entire token + * *as well as all spaces until the next token (or EOF).* This means, for example, that a + * string token always begins with a " and is always terminated by the final ", possibly + * followed by a number of spaces. + * + * The string_view is *not* null-terminated. If this is a scalar (string, number, + * boolean, or null), the character after the end of the string_view may be the padded buffer. + * + * Tokens include: + * - { + * - [ + * - "a string (possibly with UTF-8 or backslashed characters like \\\")". + * - -1.2e-100 + * - true + * - false + * - null + */ + simdjson_inline simdjson_result raw_json_token() noexcept; + + /** + * Reset the iterator inside the document instance so we are pointing back at the + * beginning of the document, as if it had just been created. It invalidates all + * values, objects and arrays that you have created so far (including unescaped strings). + */ + inline void rewind() noexcept; + /** + * Returns debugging information. + */ + inline std::string to_debug_string() noexcept; + /** + * Some unrecoverable error conditions may render the document instance unusable. + * The is_alive() method returns true when the document is still suitable. + */ + inline bool is_alive() noexcept; + + /** + * Returns the current location in the document if in bounds. + */ + inline simdjson_result current_location() const noexcept; + + /** + * Returns true if this document has been fully parsed. + * If you have consumed the whole document and at_end() returns + * false, then there may be trailing content. + */ + inline bool at_end() const noexcept; + + /** + * Returns the current depth in the document if in bounds. + * + * E.g., + * 0 = finished with document + * 1 = document root value (could be [ or {, not yet known) + * 2 = , or } inside root array/object + * 3 = key or value inside root array/object. + */ + simdjson_inline int32_t current_depth() const noexcept; + + /** + * Get the value associated with the given JSON pointer. We use the RFC 6901 + * https://tools.ietf.org/html/rfc6901 standard. + * + * ondemand::parser parser; + * auto json = R"({ "foo": { "a": [ 10, 20, 30 ] }})"_padded; + * auto doc = parser.iterate(json); + * doc.at_pointer("/foo/a/1") == 20 + * + * It is allowed for a key to be the empty string: + * + * ondemand::parser parser; + * auto json = R"({ "": { "a": [ 10, 20, 30 ] }})"_padded; + * auto doc = parser.iterate(json); + * doc.at_pointer("//a/1") == 20 + * + * Key values are matched exactly, without unescaping or Unicode normalization. + * We do a byte-by-byte comparison. E.g. + * + * const padded_string json = "{\"\\u00E9\":123}"_padded; + * auto doc = parser.iterate(json); + * doc.at_pointer("/\\u00E9") == 123 + * doc.at_pointer((const char*)u8"/\u00E9") returns an error (NO_SUCH_FIELD) + * + * Note that at_pointer() automatically calls rewind between each call. Thus + * all values, objects and arrays that you have created so far (including unescaped strings) + * are invalidated. After calling at_pointer, you need to consume the result: string values + * should be stored in your own variables, arrays should be decoded and stored in your own array-like + * structures and so forth. + * + * Also note that at_pointer() relies on find_field() which implies that we do not unescape keys when matching + * + * @return The value associated with the given JSON pointer, or: + * - NO_SUCH_FIELD if a field does not exist in an object + * - INDEX_OUT_OF_BOUNDS if an array index is larger than an array length + * - INCORRECT_TYPE if a non-integer is used to access an array + * - INVALID_JSON_POINTER if the JSON pointer is invalid and cannot be parsed + * - SCALAR_DOCUMENT_AS_VALUE if the json_pointer is empty and the document is not a scalar (see is_scalar() function). + */ + simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; + + /** + * Get the value associated with the given JSONPath expression. We only support + * JSONPath queries that trivially convertible to JSON Pointer queries: key + * names and array indices. + * + * https://datatracker.ietf.org/doc/html/draft-normington-jsonpath-00 + * + * Key values are matched exactly, without unescaping or Unicode normalization. + * We do a byte-by-byte comparison. E.g. + * + * const padded_string json = "{\"\\u00E9\":123}"_padded; + * auto doc = parser.iterate(json); + * doc.at_path(".\\u00E9") == 123 + * doc.at_path((const char*)u8".\u00E9") returns an error (NO_SUCH_FIELD) + * + * @return The value associated with the given JSONPath expression, or: + * - INVALID_JSON_POINTER if the JSONPath to JSON Pointer conversion fails + * - NO_SUCH_FIELD if a field does not exist in an object + * - INDEX_OUT_OF_BOUNDS if an array index is larger than an array length + * - INCORRECT_TYPE if a non-integer is used to access an array + */ + simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; + + /** + * Consumes the document and returns a string_view instance corresponding to the + * document as represented in JSON. It points inside the original byte array containing + * the JSON document. + */ + simdjson_inline simdjson_result raw_json() noexcept; + protected: + /** + * Consumes the document. + */ + simdjson_inline error_code consume() noexcept; + + simdjson_inline document(ondemand::json_iterator&& iter) noexcept; + simdjson_inline const uint8_t* text(uint32_t idx) const noexcept; + + simdjson_inline value_iterator resume_value_iterator() noexcept; + simdjson_inline value_iterator get_root_value_iterator() noexcept; + simdjson_inline simdjson_result start_or_resume_object() noexcept; + static simdjson_inline document start(ondemand::json_iterator&& iter) noexcept; + + // + // Fields + // + json_iterator iter{}; ///< Current position in the document + static constexpr depth_t DOCUMENT_DEPTH = 0; ///< document depth is always 0 + + friend class array_iterator; + friend class value; + friend class ondemand::parser; + friend class object; + friend class array; + friend class field; + friend class token; + friend class document_stream; + friend class document_reference; + }; + + + /** + * A document_reference is a thin wrapper around a document reference instance. + */ + class document_reference { + public: + simdjson_inline document_reference() noexcept; + simdjson_inline document_reference(document& d) noexcept; + simdjson_inline document_reference(const document_reference& other) noexcept = default; + simdjson_inline document_reference& operator=(const document_reference& other) noexcept = default; + simdjson_inline void rewind() noexcept; + simdjson_inline simdjson_result get_array() & noexcept; + simdjson_inline simdjson_result get_object() & noexcept; + simdjson_inline simdjson_result get_uint64() noexcept; + simdjson_inline simdjson_result get_uint64_in_string() noexcept; + simdjson_inline simdjson_result get_int64() noexcept; + simdjson_inline simdjson_result get_int64_in_string() noexcept; + simdjson_inline simdjson_result get_double() noexcept; + simdjson_inline simdjson_result get_double_in_string() noexcept; + simdjson_inline simdjson_result get_string(bool allow_replacement = false) noexcept; + template + simdjson_inline error_code get_string(string_type& receiver, bool allow_replacement = false) noexcept; + simdjson_inline simdjson_result get_wobbly_string() noexcept; + simdjson_inline simdjson_result get_raw_json_string() noexcept; + simdjson_inline simdjson_result get_bool() noexcept; + simdjson_inline simdjson_result get_value() noexcept; + + simdjson_inline simdjson_result is_null() noexcept; + template simdjson_inline simdjson_result get() & noexcept; + simdjson_inline simdjson_result raw_json() noexcept; + simdjson_inline operator document& () const noexcept; +#if SIMDJSON_EXCEPTIONS + template + explicit simdjson_inline operator T() noexcept(false); + simdjson_inline operator array() & noexcept(false); + simdjson_inline operator object() & noexcept(false); + simdjson_inline operator uint64_t() noexcept(false); + simdjson_inline operator int64_t() noexcept(false); + simdjson_inline operator double() noexcept(false); + simdjson_inline operator std::string_view() noexcept(false); + simdjson_inline operator raw_json_string() noexcept(false); + simdjson_inline operator bool() noexcept(false); + simdjson_inline operator value() noexcept(false); +#endif + simdjson_inline simdjson_result count_elements() & noexcept; + simdjson_inline simdjson_result count_fields() & noexcept; + simdjson_inline simdjson_result at(size_t index) & noexcept; + simdjson_inline simdjson_result begin() & noexcept; + simdjson_inline simdjson_result end() & noexcept; + simdjson_inline simdjson_result find_field(std::string_view key) & noexcept; + simdjson_inline simdjson_result find_field(const char* key) & noexcept; + simdjson_inline simdjson_result operator[](std::string_view key) & noexcept; + simdjson_inline simdjson_result operator[](const char* key) & noexcept; + simdjson_inline simdjson_result find_field_unordered(std::string_view key) & noexcept; + simdjson_inline simdjson_result find_field_unordered(const char* key) & noexcept; + + simdjson_inline simdjson_result type() noexcept; + simdjson_inline simdjson_result is_scalar() noexcept; + simdjson_inline simdjson_result is_string() noexcept; + + simdjson_inline simdjson_result current_location() noexcept; + simdjson_inline int32_t current_depth() const noexcept; + simdjson_inline bool is_negative() noexcept; + simdjson_inline simdjson_result is_integer() noexcept; + simdjson_inline simdjson_result get_number_type() noexcept; + simdjson_inline simdjson_result get_number() noexcept; + simdjson_inline simdjson_result raw_json_token() noexcept; + simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; + simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; + + private: + document* doc{ nullptr }; + }; + } // namespace ondemand + } // namespace lasx +} // namespace simdjson + +namespace simdjson { + + template<> + struct simdjson_result : public lasx::implementation_simdjson_result_base { + public: + simdjson_inline simdjson_result(lasx::ondemand::document&& value) noexcept; ///< @private + simdjson_inline simdjson_result(error_code error) noexcept; ///< @private + simdjson_inline simdjson_result() noexcept = default; + simdjson_inline error_code rewind() noexcept; + + simdjson_inline simdjson_result get_array() & noexcept; + simdjson_inline simdjson_result get_object() & noexcept; + simdjson_inline simdjson_result get_uint64() noexcept; + simdjson_inline simdjson_result get_uint64_in_string() noexcept; + simdjson_inline simdjson_result get_int64() noexcept; + simdjson_inline simdjson_result get_int64_in_string() noexcept; + simdjson_inline simdjson_result get_double() noexcept; + simdjson_inline simdjson_result get_double_in_string() noexcept; + simdjson_inline simdjson_result get_string(bool allow_replacement = false) noexcept; + template + simdjson_inline error_code get_string(string_type& receiver, bool allow_replacement = false) noexcept; + simdjson_inline simdjson_result get_wobbly_string() noexcept; + simdjson_inline simdjson_result get_raw_json_string() noexcept; + simdjson_inline simdjson_result get_bool() noexcept; + simdjson_inline simdjson_result get_value() noexcept; + simdjson_inline simdjson_result is_null() noexcept; + + template simdjson_inline simdjson_result get() & noexcept; + template simdjson_deprecated simdjson_inline simdjson_result get() && noexcept; + + template simdjson_inline error_code get(T& out) & noexcept; + template simdjson_inline error_code get(T& out) && noexcept; +#if SIMDJSON_EXCEPTIONS + template ::value == false>::type> + explicit simdjson_inline operator T() noexcept(false); + simdjson_inline operator lasx::ondemand::array() & noexcept(false); + simdjson_inline operator lasx::ondemand::object() & noexcept(false); + simdjson_inline operator uint64_t() noexcept(false); + simdjson_inline operator int64_t() noexcept(false); + simdjson_inline operator double() noexcept(false); + simdjson_inline operator std::string_view() noexcept(false); + simdjson_inline operator lasx::ondemand::raw_json_string() noexcept(false); + simdjson_inline operator bool() noexcept(false); + simdjson_inline operator lasx::ondemand::value() noexcept(false); +#endif + simdjson_inline simdjson_result count_elements() & noexcept; + simdjson_inline simdjson_result count_fields() & noexcept; + simdjson_inline simdjson_result at(size_t index) & noexcept; + simdjson_inline simdjson_result begin() & noexcept; + simdjson_inline simdjson_result end() & noexcept; + simdjson_inline simdjson_result find_field(std::string_view key) & noexcept; + simdjson_inline simdjson_result find_field(const char* key) & noexcept; + simdjson_inline simdjson_result operator[](std::string_view key) & noexcept; + simdjson_inline simdjson_result operator[](const char* key) & noexcept; + simdjson_inline simdjson_result find_field_unordered(std::string_view key) & noexcept; + simdjson_inline simdjson_result find_field_unordered(const char* key) & noexcept; + simdjson_inline simdjson_result type() noexcept; + simdjson_inline simdjson_result is_scalar() noexcept; + simdjson_inline simdjson_result is_string() noexcept; + simdjson_inline simdjson_result current_location() noexcept; + simdjson_inline int32_t current_depth() const noexcept; + simdjson_inline bool at_end() const noexcept; + simdjson_inline bool is_negative() noexcept; + simdjson_inline simdjson_result is_integer() noexcept; + simdjson_inline simdjson_result get_number_type() noexcept; + simdjson_inline simdjson_result get_number() noexcept; + /** @copydoc simdjson_inline std::string_view document::raw_json_token() const noexcept */ + simdjson_inline simdjson_result raw_json_token() noexcept; + + simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; + simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; + }; + + +} // namespace simdjson + + + +namespace simdjson { + + template<> + struct simdjson_result : public lasx::implementation_simdjson_result_base { + public: + simdjson_inline simdjson_result(lasx::ondemand::document_reference value, error_code error) noexcept; + simdjson_inline simdjson_result() noexcept = default; + simdjson_inline error_code rewind() noexcept; + + simdjson_inline simdjson_result get_array() & noexcept; + simdjson_inline simdjson_result get_object() & noexcept; + simdjson_inline simdjson_result get_uint64() noexcept; + simdjson_inline simdjson_result get_uint64_in_string() noexcept; + simdjson_inline simdjson_result get_int64() noexcept; + simdjson_inline simdjson_result get_int64_in_string() noexcept; + simdjson_inline simdjson_result get_double() noexcept; + simdjson_inline simdjson_result get_double_in_string() noexcept; + simdjson_inline simdjson_result get_string(bool allow_replacement = false) noexcept; + template + simdjson_inline error_code get_string(string_type& receiver, bool allow_replacement = false) noexcept; + simdjson_inline simdjson_result get_wobbly_string() noexcept; + simdjson_inline simdjson_result get_raw_json_string() noexcept; + simdjson_inline simdjson_result get_bool() noexcept; + simdjson_inline simdjson_result get_value() noexcept; + simdjson_inline simdjson_result is_null() noexcept; +#if SIMDJSON_EXCEPTIONS + template ::value == false>::type> + explicit simdjson_inline operator T() noexcept(false); + simdjson_inline operator lasx::ondemand::array() & noexcept(false); + simdjson_inline operator lasx::ondemand::object() & noexcept(false); + simdjson_inline operator uint64_t() noexcept(false); + simdjson_inline operator int64_t() noexcept(false); + simdjson_inline operator double() noexcept(false); + simdjson_inline operator std::string_view() noexcept(false); + simdjson_inline operator lasx::ondemand::raw_json_string() noexcept(false); + simdjson_inline operator bool() noexcept(false); + simdjson_inline operator lasx::ondemand::value() noexcept(false); +#endif + simdjson_inline simdjson_result count_elements() & noexcept; + simdjson_inline simdjson_result count_fields() & noexcept; + simdjson_inline simdjson_result at(size_t index) & noexcept; + simdjson_inline simdjson_result begin() & noexcept; + simdjson_inline simdjson_result end() & noexcept; + simdjson_inline simdjson_result find_field(std::string_view key) & noexcept; + simdjson_inline simdjson_result find_field(const char* key) & noexcept; + simdjson_inline simdjson_result operator[](std::string_view key) & noexcept; + simdjson_inline simdjson_result operator[](const char* key) & noexcept; + simdjson_inline simdjson_result find_field_unordered(std::string_view key) & noexcept; + simdjson_inline simdjson_result find_field_unordered(const char* key) & noexcept; + simdjson_inline simdjson_result type() noexcept; + simdjson_inline simdjson_result is_scalar() noexcept; + simdjson_inline simdjson_result is_string() noexcept; + simdjson_inline simdjson_result current_location() noexcept; + simdjson_inline simdjson_result current_depth() const noexcept; + simdjson_inline simdjson_result is_negative() noexcept; + simdjson_inline simdjson_result is_integer() noexcept; + simdjson_inline simdjson_result get_number_type() noexcept; + simdjson_inline simdjson_result get_number() noexcept; + /** @copydoc simdjson_inline std::string_view document_reference::raw_json_token() const noexcept */ + simdjson_inline simdjson_result raw_json_token() noexcept; + + simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; + simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; + }; + + +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_ONDEMAND_DOCUMENT_H +/* end file simdjson/generic/ondemand/document.h for lasx */ +/* including simdjson/generic/ondemand/document_stream.h for lasx: #include "simdjson/generic/ondemand/document_stream.h" */ +/* begin file simdjson/generic/ondemand/document_stream.h for lasx */ +#ifndef SIMDJSON_GENERIC_ONDEMAND_DOCUMENT_STREAM_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_ONDEMAND_DOCUMENT_STREAM_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/implementation_simdjson_result_base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/document.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/parser.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +#ifdef SIMDJSON_THREADS_ENABLED +#include +#include +#include +#endif + +namespace simdjson { + namespace lasx { + namespace ondemand { + +#ifdef SIMDJSON_THREADS_ENABLED + /** @private Custom worker class **/ + struct stage1_worker { + stage1_worker() noexcept = default; + stage1_worker(const stage1_worker&) = delete; + stage1_worker(stage1_worker&&) = delete; + stage1_worker operator=(const stage1_worker&) = delete; + ~stage1_worker(); + /** + * We only start the thread when it is needed, not at object construction, this may throw. + * You should only call this once. + **/ + void start_thread(); + /** + * Start a stage 1 job. You should first call 'run', then 'finish'. + * You must call start_thread once before. + */ + void run(document_stream* ds, parser* stage1, size_t next_batch_start); + /** Wait for the run to finish (blocking). You should first call 'run', then 'finish'. **/ + void finish(); + + private: + + /** + * Normally, we would never stop the thread. But we do in the destructor. + * This function is only safe assuming that you are not waiting for results. You + * should have called run, then finish, and be done. + **/ + void stop_thread(); + + std::thread thread{}; + /** These three variables define the work done by the thread. **/ + ondemand::parser* stage1_thread_parser{}; + size_t _next_batch_start{}; + document_stream* owner{}; + /** + * We have two state variables. This could be streamlined to one variable in the future but + * we use two for clarity. + */ + bool has_work{ false }; + bool can_work{ true }; + + /** + * We lock using a mutex. + */ + std::mutex locking_mutex{}; + std::condition_variable cond_var{}; + + friend class document_stream; + }; +#endif // SIMDJSON_THREADS_ENABLED + + /** + * A forward-only stream of documents. + * + * Produced by parser::iterate_many. + * + */ + class document_stream { + public: + /** + * Construct an uninitialized document_stream. + * + * ```c++ + * document_stream docs; + * auto error = parser.iterate_many(json).get(docs); + * ``` + */ + simdjson_inline document_stream() noexcept; + /** Move one document_stream to another. */ + simdjson_inline document_stream(document_stream&& other) noexcept = default; + /** Move one document_stream to another. */ + simdjson_inline document_stream& operator=(document_stream&& other) noexcept = default; + + simdjson_inline ~document_stream() noexcept; + + /** + * Returns the input size in bytes. + */ + inline size_t size_in_bytes() const noexcept; + + /** + * After iterating through the stream, this method + * returns the number of bytes that were not parsed at the end + * of the stream. If truncated_bytes() differs from zero, + * then the input was truncated maybe because incomplete JSON + * documents were found at the end of the stream. You + * may need to process the bytes in the interval [size_in_bytes()-truncated_bytes(), size_in_bytes()). + * + * You should only call truncated_bytes() after streaming through all + * documents, like so: + * + * document_stream stream = parser.iterate_many(json,window); + * for(auto & doc : stream) { + * // do something with doc + * } + * size_t truncated = stream.truncated_bytes(); + * + */ + inline size_t truncated_bytes() const noexcept; + + class iterator { + public: + using value_type = simdjson_result; + using reference = simdjson_result; + using pointer = void; + using difference_type = std::ptrdiff_t; + using iterator_category = std::input_iterator_tag; + + /** + * Default constructor. + */ + simdjson_inline iterator() noexcept; + /** + * Get the current document (or error). + */ + simdjson_inline reference operator*() noexcept; + /** + * Advance to the next document (prefix). + */ + inline iterator& operator++() noexcept; + /** + * Check if we're at the end yet. + * @param other the end iterator to compare to. + */ + simdjson_inline bool operator!=(const iterator& other) const noexcept; + /** + * @private + * + * Gives the current index in the input document in bytes. + * + * document_stream stream = parser.parse_many(json,window); + * for(auto i = stream.begin(); i != stream.end(); ++i) { + * auto doc = *i; + * size_t index = i.current_index(); + * } + * + * This function (current_index()) is experimental and the usage + * may change in future versions of simdjson: we find the API somewhat + * awkward and we would like to offer something friendlier. + */ + simdjson_inline size_t current_index() const noexcept; + + /** + * @private + * + * Gives a view of the current document at the current position. + * + * document_stream stream = parser.iterate_many(json,window); + * for(auto i = stream.begin(); i != stream.end(); ++i) { + * std::string_view v = i.source(); + * } + * + * The returned string_view instance is simply a map to the (unparsed) + * source string: it may thus include white-space characters and all manner + * of padding. + * + * This function (source()) is experimental and the usage + * may change in future versions of simdjson: we find the API somewhat + * awkward and we would like to offer something friendlier. + * + */ + simdjson_inline std::string_view source() const noexcept; + + /** + * Returns error of the stream (if any). + */ + inline error_code error() const noexcept; + + private: + simdjson_inline iterator(document_stream* s, bool finished) noexcept; + /** The document_stream we're iterating through. */ + document_stream* stream; + /** Whether we're finished or not. */ + bool finished; + + friend class document; + friend class document_stream; + friend class json_iterator; + }; + + /** + * Start iterating the documents in the stream. + */ + simdjson_inline iterator begin() noexcept; + /** + * The end of the stream, for iterator comparison purposes. + */ + simdjson_inline iterator end() noexcept; + + private: + + document_stream& operator=(const document_stream&) = delete; // Disallow copying + document_stream(const document_stream& other) = delete; // Disallow copying + + /** + * Construct a document_stream. Does not allocate or parse anything until the iterator is + * used. + * + * @param parser is a reference to the parser instance used to generate this document_stream + * @param buf is the raw byte buffer we need to process + * @param len is the length of the raw byte buffer in bytes + * @param batch_size is the size of the windows (must be strictly greater or equal to the largest JSON document) + */ + simdjson_inline document_stream( + ondemand::parser& parser, + const uint8_t* buf, + size_t len, + size_t batch_size, + bool allow_comma_separated + ) noexcept; + + /** + * Parse the first document in the buffer. Used by begin(), to handle allocation and + * initialization. + */ + inline void start() noexcept; + + /** + * Parse the next document found in the buffer previously given to document_stream. + * + * The content should be a valid JSON document encoded as UTF-8. If there is a + * UTF-8 BOM, the parser skips it. + * + * You do NOT need to pre-allocate a parser. This function takes care of + * pre-allocating a capacity defined by the batch_size defined when creating the + * document_stream object. + * + * The function returns simdjson::EMPTY if there is no more data to be parsed. + * + * The function returns simdjson::SUCCESS (as integer = 0) in case of success + * and indicates that the buffer has successfully been parsed to the end. + * Every document it contained has been parsed without error. + * + * The function returns an error code from simdjson/simdjson.h in case of failure + * such as simdjson::CAPACITY, simdjson::MEMALLOC, simdjson::DEPTH_ERROR and so forth; + * the simdjson::error_message function converts these error codes into a string). + * + * You can also check validity by calling parser.is_valid(). The same parser can + * and should be reused for the other documents in the buffer. + */ + inline void next() noexcept; + + /** Move the json_iterator of the document to the location of the next document in the stream. */ + inline void next_document() noexcept; + + /** Get the next document index. */ + inline size_t next_batch_start() const noexcept; + + /** Pass the next batch through stage 1 with the given parser. */ + inline error_code run_stage1(ondemand::parser& p, size_t batch_start) noexcept; + + // Fields + ondemand::parser* parser; + const uint8_t* buf; + size_t len; + size_t batch_size; + bool allow_comma_separated; + /** + * We are going to use just one document instance. The document owns + * the json_iterator. It implies that we only ever pass a reference + * to the document to the users. + */ + document doc{}; + /** The error (or lack thereof) from the current document. */ + error_code error; + size_t batch_start{ 0 }; + size_t doc_index{}; + +#ifdef SIMDJSON_THREADS_ENABLED + /** Indicates whether we use threads. Note that this needs to be a constant during the execution of the parsing. */ + bool use_thread; + + inline void load_from_stage1_thread() noexcept; + + /** Start a thread to run stage 1 on the next batch. */ + inline void start_stage1_thread() noexcept; + + /** Wait for the stage 1 thread to finish and capture the results. */ + inline void finish_stage1_thread() noexcept; + + /** The error returned from the stage 1 thread. */ + error_code stage1_thread_error{ UNINITIALIZED }; + /** The thread used to run stage 1 against the next batch in the background. */ + std::unique_ptr worker{ new(std::nothrow) stage1_worker() }; + /** + * The parser used to run stage 1 in the background. Will be swapped + * with the regular parser when finished. + */ + ondemand::parser stage1_thread_parser{}; + + friend struct stage1_worker; +#endif // SIMDJSON_THREADS_ENABLED + + friend class parser; + friend class document; + friend class json_iterator; + friend struct simdjson_result; + friend struct simdjson::internal::simdjson_result_base; + }; // document_stream + + } // namespace ondemand + } // namespace lasx +} // namespace simdjson + +namespace simdjson { + template<> + struct simdjson_result : public lasx::implementation_simdjson_result_base { + public: + simdjson_inline simdjson_result(lasx::ondemand::document_stream&& value) noexcept; ///< @private + simdjson_inline simdjson_result(error_code error) noexcept; ///< @private + simdjson_inline simdjson_result() noexcept = default; + }; + +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_ONDEMAND_DOCUMENT_STREAM_H +/* end file simdjson/generic/ondemand/document_stream.h for lasx */ +/* including simdjson/generic/ondemand/field.h for lasx: #include "simdjson/generic/ondemand/field.h" */ +/* begin file simdjson/generic/ondemand/field.h for lasx */ +#ifndef SIMDJSON_GENERIC_ONDEMAND_FIELD_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_ONDEMAND_FIELD_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/implementation_simdjson_result_base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/raw_json_string.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/value.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lasx { + namespace ondemand { + + /** + * A JSON field (key/value pair) in an object. + * + * Returned from object iteration. + * + * Extends from std::pair so you can use C++ algorithms that rely on pairs. + */ + class field : public std::pair { + public: + /** + * Create a new invalid field. + * + * Exists so you can declare a variable and later assign to it before use. + */ + simdjson_inline field() noexcept; + + /** + * Get the key as a string_view (for higher speed, consider raw_key). + * We deliberately use a more cumbersome name (unescaped_key) to force users + * to think twice about using it. + * + * This consumes the key: once you have called unescaped_key(), you cannot + * call it again nor can you call key(). + */ + simdjson_inline simdjson_warn_unused simdjson_result unescaped_key(bool allow_replacement = false) noexcept; + /** + * Get the key as a string_view (for higher speed, consider raw_key). + * We deliberately use a more cumbersome name (unescaped_key) to force users + * to think twice about using it. The content is stored in the receiver. + * + * This consumes the key: once you have called unescaped_key(), you cannot + * call it again nor can you call key(). + */ + template + simdjson_inline simdjson_warn_unused error_code unescaped_key(string_type& receiver, bool allow_replacement = false) noexcept; + /** + * Get the key as a raw_json_string. Can be used for direct comparison with + * an unescaped C string: e.g., key() == "test". This does not count as + * consumption of the content: you can safely call it repeatedly. + * See escaped_key() for a similar function which returns + * a more convenient std::string_view result. + */ + simdjson_inline raw_json_string key() const noexcept; + /** + * Get the unprocessed key as a string_view. This includes the quotes and may include + * some spaces after the last quote. This does not count as + * consumption of the content: you can safely call it repeatedly. + * See escaped_key(). + */ + simdjson_inline std::string_view key_raw_json_token() const noexcept; + /** + * Get the key as a string_view. This does not include the quotes and + * the string is unprocessed key so it may contain escape characters + * (e.g., \uXXXX or \n). It does not count as a consumption of the content: + * you can safely call it repeatedly. Use unescaped_key() to get the unescaped key. + */ + simdjson_inline std::string_view escaped_key() const noexcept; + /** + * Get the field value. + */ + simdjson_inline ondemand::value& value() & noexcept; + /** + * @overload ondemand::value &ondemand::value() & noexcept + */ + simdjson_inline ondemand::value value() && noexcept; + + protected: + simdjson_inline field(raw_json_string key, ondemand::value&& value) noexcept; + static simdjson_inline simdjson_result start(value_iterator& parent_iter) noexcept; + static simdjson_inline simdjson_result start(const value_iterator& parent_iter, raw_json_string key) noexcept; + friend struct simdjson_result; + friend class object_iterator; + }; + + } // namespace ondemand + } // namespace lasx +} // namespace simdjson + +namespace simdjson { + + template<> + struct simdjson_result : public lasx::implementation_simdjson_result_base { + public: + simdjson_inline simdjson_result(lasx::ondemand::field&& value) noexcept; ///< @private + simdjson_inline simdjson_result(error_code error) noexcept; ///< @private + simdjson_inline simdjson_result() noexcept = default; + + simdjson_inline simdjson_result unescaped_key(bool allow_replacement = false) noexcept; + template + simdjson_inline error_code unescaped_key(string_type& receiver, bool allow_replacement = false) noexcept; + simdjson_inline simdjson_result key() noexcept; + simdjson_inline simdjson_result key_raw_json_token() noexcept; + simdjson_inline simdjson_result escaped_key() noexcept; + simdjson_inline simdjson_result value() noexcept; + }; + +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_ONDEMAND_FIELD_H +/* end file simdjson/generic/ondemand/field.h for lasx */ +/* including simdjson/generic/ondemand/object.h for lasx: #include "simdjson/generic/ondemand/object.h" */ +/* begin file simdjson/generic/ondemand/object.h for lasx */ +#ifndef SIMDJSON_GENERIC_ONDEMAND_OBJECT_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_ONDEMAND_OBJECT_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/implementation_simdjson_result_base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/value_iterator.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lasx { + namespace ondemand { + + /** + * A forward-only JSON object field iterator. + */ + class object { + public: + /** + * Create a new invalid object. + * + * Exists so you can declare a variable and later assign to it before use. + */ + simdjson_inline object() noexcept = default; + + simdjson_inline simdjson_result begin() noexcept; + simdjson_inline simdjson_result end() noexcept; + /** + * Look up a field by name on an object (order-sensitive). + * + * The following code reads z, then y, then x, and thus will not retrieve x or y if fed the + * JSON `{ "x": 1, "y": 2, "z": 3 }`: + * + * ```c++ + * simdjson::ondemand::parser parser; + * auto obj = parser.parse(R"( { "x": 1, "y": 2, "z": 3 } )"_padded); + * double z = obj.find_field("z"); + * double y = obj.find_field("y"); + * double x = obj.find_field("x"); + * ``` + * If you have multiple fields with a matching key ({"x": 1, "x": 1}) be mindful + * that only one field is returned. + * + * **Raw Keys:** The lookup will be done against the *raw* key, and will not unescape keys. + * e.g. `object["a"]` will match `{ "a": 1 }`, but will *not* match `{ "\u0061": 1 }`. + * + * You must consume the fields on an object one at a time. A request for a new key + * invalidates previous field values: it makes them unsafe. The value instance you get + * from `content["bids"]` becomes invalid when you call `content["asks"]`. The array + * given by content["bids"].get_array() should not be accessed after you have called + * content["asks"].get_array(). You can detect such mistakes by first compiling and running + * the code in Debug mode (or with the macro `SIMDJSON_DEVELOPMENT_CHECKS` set to 1): an + * OUT_OF_ORDER_ITERATION error is generated. + * + * You are expected to access keys only once. You should access the value corresponding to a + * key a single time. Doing object["mykey"].to_string() and then again object["mykey"].to_string() + * is an error. + * + * If you expect to have keys with escape characters, please review our documentation. + * + * @param key The key to look up. + * @returns The value of the field, or NO_SUCH_FIELD if the field is not in the object. + */ + simdjson_inline simdjson_result find_field(std::string_view key) & noexcept; + /** @overload simdjson_inline simdjson_result find_field(std::string_view key) & noexcept; */ + simdjson_inline simdjson_result find_field(std::string_view key) && noexcept; + + /** + * Look up a field by name on an object, without regard to key order. + * + * **Performance Notes:** This is a bit less performant than find_field(), though its effect varies + * and often appears negligible. It starts out normally, starting out at the last field; but if + * the field is not found, it scans from the beginning of the object to see if it missed it. That + * missing case has a non-cache-friendly bump and lots of extra scanning, especially if the object + * in question is large. The fact that the extra code is there also bumps the executable size. + * + * It is the default, however, because it would be highly surprising (and hard to debug) if the + * default behavior failed to look up a field just because it was in the wrong order--and many + * APIs assume this. Therefore, you must be explicit if you want to treat objects as out of order. + * + * Use find_field() if you are sure fields will be in order (or are willing to treat it as if the + * field was not there when they are not in order). + * + * If you have multiple fields with a matching key ({"x": 1, "x": 1}) be mindful + * that only one field is returned. + * + * You must consume the fields on an object one at a time. A request for a new key + * invalidates previous field values: it makes them unsafe. The value instance you get + * from `content["bids"]` becomes invalid when you call `content["asks"]`. The array + * given by content["bids"].get_array() should not be accessed after you have called + * content["asks"].get_array(). You can detect such mistakes by first compiling and running + * the code in Debug mode (or with the macro `SIMDJSON_DEVELOPMENT_CHECKS` set to 1): an + * OUT_OF_ORDER_ITERATION error is generated. + * + * You are expected to access keys only once. You should access the value corresponding to a key + * a single time. Doing object["mykey"].to_string() and then again object["mykey"].to_string() is an error. + * + * If you expect to have keys with escape characters, please review our documentation. + * + * @param key The key to look up. + * @returns The value of the field, or NO_SUCH_FIELD if the field is not in the object. + */ + simdjson_inline simdjson_result find_field_unordered(std::string_view key) & noexcept; + /** @overload simdjson_inline simdjson_result find_field_unordered(std::string_view key) & noexcept; */ + simdjson_inline simdjson_result find_field_unordered(std::string_view key) && noexcept; + /** @overload simdjson_inline simdjson_result find_field_unordered(std::string_view key) & noexcept; */ + simdjson_inline simdjson_result operator[](std::string_view key) & noexcept; + /** @overload simdjson_inline simdjson_result find_field_unordered(std::string_view key) & noexcept; */ + simdjson_inline simdjson_result operator[](std::string_view key) && noexcept; + + /** + * Get the value associated with the given JSON pointer. We use the RFC 6901 + * https://tools.ietf.org/html/rfc6901 standard, interpreting the current node + * as the root of its own JSON document. + * + * ondemand::parser parser; + * auto json = R"({ "foo": { "a": [ 10, 20, 30 ] }})"_padded; + * auto doc = parser.iterate(json); + * doc.at_pointer("/foo/a/1") == 20 + * + * It is allowed for a key to be the empty string: + * + * ondemand::parser parser; + * auto json = R"({ "": { "a": [ 10, 20, 30 ] }})"_padded; + * auto doc = parser.iterate(json); + * doc.at_pointer("//a/1") == 20 + * + * Note that at_pointer() called on the document automatically calls the document's rewind + * method between each call. It invalidates all previously accessed arrays, objects and values + * that have not been consumed. Yet it is not the case when calling at_pointer on an object + * instance: there is no rewind and no invalidation. + * + * You may call at_pointer more than once on an object, but each time the pointer is advanced + * to be within the value matched by the key indicated by the JSON pointer query. Thus any preceding + * key (as well as the current key) can no longer be used with following JSON pointer calls. + * + * Also note that at_pointer() relies on find_field() which implies that we do not unescape keys when matching. + * + * @return The value associated with the given JSON pointer, or: + * - NO_SUCH_FIELD if a field does not exist in an object + * - INDEX_OUT_OF_BOUNDS if an array index is larger than an array length + * - INCORRECT_TYPE if a non-integer is used to access an array + * - INVALID_JSON_POINTER if the JSON pointer is invalid and cannot be parsed + */ + inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; + + /** + * Get the value associated with the given JSONPath expression. We only support + * JSONPath queries that trivially convertible to JSON Pointer queries: key + * names and array indices. + * + * @return The value associated with the given JSONPath expression, or: + * - INVALID_JSON_POINTER if the JSONPath to JSON Pointer conversion fails + * - NO_SUCH_FIELD if a field does not exist in an object + * - INDEX_OUT_OF_BOUNDS if an array index is larger than an array length + * - INCORRECT_TYPE if a non-integer is used to access an array + */ + inline simdjson_result at_path(std::string_view json_path) noexcept; + + /** + * Reset the iterator so that we are pointing back at the + * beginning of the object. You should still consume values only once even if you + * can iterate through the object more than once. If you unescape a string or a key + * within the object more than once, you have unsafe code. Note that rewinding an object + * means that you may need to reparse it anew: it is not a free operation. + * + * @returns true if the object contains some elements (not empty) + */ + inline simdjson_result reset() & noexcept; + /** + * This method scans the beginning of the object and checks whether the + * object is empty. + * The runtime complexity is constant time. After + * calling this function, if successful, the object is 'rewinded' at its + * beginning as if it had never been accessed. If the JSON is malformed (e.g., + * there is a missing comma), then an error is returned and it is no longer + * safe to continue. + */ + inline simdjson_result is_empty() & noexcept; + /** + * This method scans the object and counts the number of key-value pairs. + * The count_fields method should always be called before you have begun + * iterating through the object: it is expected that you are pointing at + * the beginning of the object. + * The runtime complexity is linear in the size of the object. After + * calling this function, if successful, the object is 'rewinded' at its + * beginning as if it had never been accessed. If the JSON is malformed (e.g., + * there is a missing comma), then an error is returned and it is no longer + * safe to continue. + * + * To check that an object is empty, it is more performant to use + * the is_empty() method. + * + * Performance hint: You should only call count_fields() as a last + * resort as it may require scanning the document twice or more. + */ + simdjson_inline simdjson_result count_fields() & noexcept; + /** + * Consumes the object and returns a string_view instance corresponding to the + * object as represented in JSON. It points inside the original byte array containing + * the JSON document. + */ + simdjson_inline simdjson_result raw_json() noexcept; + + protected: + /** + * Go to the end of the object, no matter where you are right now. + */ + simdjson_inline error_code consume() noexcept; + static simdjson_inline simdjson_result start(value_iterator& iter) noexcept; + static simdjson_inline simdjson_result start_root(value_iterator& iter) noexcept; + static simdjson_inline simdjson_result started(value_iterator& iter) noexcept; + static simdjson_inline object resume(const value_iterator& iter) noexcept; + simdjson_inline object(const value_iterator& iter) noexcept; + + simdjson_warn_unused simdjson_inline error_code find_field_raw(const std::string_view key) noexcept; + + value_iterator iter{}; + + friend class value; + friend class document; + friend struct simdjson_result; + }; + + } // namespace ondemand + } // namespace lasx +} // namespace simdjson + +namespace simdjson { + + template<> + struct simdjson_result : public lasx::implementation_simdjson_result_base { + public: + simdjson_inline simdjson_result(lasx::ondemand::object&& value) noexcept; ///< @private + simdjson_inline simdjson_result(error_code error) noexcept; ///< @private + simdjson_inline simdjson_result() noexcept = default; + + simdjson_inline simdjson_result begin() noexcept; + simdjson_inline simdjson_result end() noexcept; + simdjson_inline simdjson_result find_field(std::string_view key) & noexcept; + simdjson_inline simdjson_result find_field(std::string_view key) && noexcept; + simdjson_inline simdjson_result find_field_unordered(std::string_view key) & noexcept; + simdjson_inline simdjson_result find_field_unordered(std::string_view key) && noexcept; + simdjson_inline simdjson_result operator[](std::string_view key) & noexcept; + simdjson_inline simdjson_result operator[](std::string_view key) && noexcept; + simdjson_inline simdjson_result at_pointer(std::string_view json_pointer) noexcept; + simdjson_inline simdjson_result at_path(std::string_view json_path) noexcept; + + inline simdjson_result reset() noexcept; + inline simdjson_result is_empty() noexcept; + inline simdjson_result count_fields() & noexcept; + inline simdjson_result raw_json() noexcept; + + }; + +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_ONDEMAND_OBJECT_H +/* end file simdjson/generic/ondemand/object.h for lasx */ +/* including simdjson/generic/ondemand/object_iterator.h for lasx: #include "simdjson/generic/ondemand/object_iterator.h" */ +/* begin file simdjson/generic/ondemand/object_iterator.h for lasx */ +#ifndef SIMDJSON_GENERIC_ONDEMAND_OBJECT_ITERATOR_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_ONDEMAND_OBJECT_ITERATOR_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/implementation_simdjson_result_base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/value_iterator.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lasx { + namespace ondemand { + + class object_iterator { + public: + /** + * Create a new invalid object_iterator. + * + * Exists so you can declare a variable and later assign to it before use. + */ + simdjson_inline object_iterator() noexcept = default; + + // + // Iterator interface + // + + // Reads key and value, yielding them to the user. + // MUST ONLY BE CALLED ONCE PER ITERATION. + simdjson_inline simdjson_result operator*() noexcept; + // Assumes it's being compared with the end. true if depth < iter->depth. + simdjson_inline bool operator==(const object_iterator&) const noexcept; + // Assumes it's being compared with the end. true if depth >= iter->depth. + simdjson_inline bool operator!=(const object_iterator&) const noexcept; + // Checks for ']' and ',' + simdjson_inline object_iterator& operator++() noexcept; + + private: + /** + * The underlying JSON iterator. + * + * PERF NOTE: expected to be elided in favor of the parent document: this is set when the object + * is first used, and never changes afterwards. + */ + value_iterator iter{}; + + simdjson_inline object_iterator(const value_iterator& iter) noexcept; + friend struct simdjson_result; + friend class object; + }; + + } // namespace ondemand + } // namespace lasx +} // namespace simdjson + +namespace simdjson { + + template<> + struct simdjson_result : public lasx::implementation_simdjson_result_base { + public: + simdjson_inline simdjson_result(lasx::ondemand::object_iterator&& value) noexcept; ///< @private + simdjson_inline simdjson_result(error_code error) noexcept; ///< @private + simdjson_inline simdjson_result() noexcept = default; + + // + // Iterator interface + // + + // Reads key and value, yielding them to the user. + simdjson_inline simdjson_result operator*() noexcept; // MUST ONLY BE CALLED ONCE PER ITERATION. + // Assumes it's being compared with the end. true if depth < iter->depth. + simdjson_inline bool operator==(const simdjson_result&) const noexcept; + // Assumes it's being compared with the end. true if depth >= iter->depth. + simdjson_inline bool operator!=(const simdjson_result&) const noexcept; + // Checks for ']' and ',' + simdjson_inline simdjson_result& operator++() noexcept; + }; + +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_ONDEMAND_OBJECT_ITERATOR_H +/* end file simdjson/generic/ondemand/object_iterator.h for lasx */ +/* including simdjson/generic/ondemand/serialization.h for lasx: #include "simdjson/generic/ondemand/serialization.h" */ +/* begin file simdjson/generic/ondemand/serialization.h for lasx */ +#ifndef SIMDJSON_GENERIC_ONDEMAND_SERIALIZATION_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_ONDEMAND_SERIALIZATION_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/base.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + /** + * Create a string-view instance out of a document instance. The string-view instance + * contains JSON text that is suitable to be parsed as JSON again. It does not + * validate the content. + */ + inline simdjson_result to_json_string(lasx::ondemand::document& x) noexcept; + /** + * Create a string-view instance out of a value instance. The string-view instance + * contains JSON text that is suitable to be parsed as JSON again. The value must + * not have been accessed previously. It does not + * validate the content. + */ + inline simdjson_result to_json_string(lasx::ondemand::value& x) noexcept; + /** + * Create a string-view instance out of an object instance. The string-view instance + * contains JSON text that is suitable to be parsed as JSON again. It does not + * validate the content. + */ + inline simdjson_result to_json_string(lasx::ondemand::object& x) noexcept; + /** + * Create a string-view instance out of an array instance. The string-view instance + * contains JSON text that is suitable to be parsed as JSON again. It does not + * validate the content. + */ + inline simdjson_result to_json_string(lasx::ondemand::array& x) noexcept; + inline simdjson_result to_json_string(simdjson_result x); + inline simdjson_result to_json_string(simdjson_result x); + inline simdjson_result to_json_string(simdjson_result x); + inline simdjson_result to_json_string(simdjson_result x); +} // namespace simdjson + +/** + * We want to support argument-dependent lookup (ADL). + * Hence we should define operator<< in the namespace + * where the argument (here value, object, etc.) resides. + * Credit: @madhur4127 + * See https://github.com/simdjson/simdjson/issues/1768 + */ +namespace simdjson { + namespace lasx { + namespace ondemand { + + /** + * Print JSON to an output stream. It does not + * validate the content. + * + * @param out The output stream. + * @param value The element. + * @throw if there is an error with the underlying output stream. simdjson itself will not throw. + */ + inline std::ostream& operator<<(std::ostream& out, simdjson::lasx::ondemand::value x); +#if SIMDJSON_EXCEPTIONS + inline std::ostream& operator<<(std::ostream& out, simdjson::simdjson_result x); +#endif + /** + * Print JSON to an output stream. It does not + * validate the content. + * + * @param out The output stream. + * @param value The array. + * @throw if there is an error with the underlying output stream. simdjson itself will not throw. + */ + inline std::ostream& operator<<(std::ostream& out, simdjson::lasx::ondemand::array value); +#if SIMDJSON_EXCEPTIONS + inline std::ostream& operator<<(std::ostream& out, simdjson::simdjson_result x); +#endif + /** + * Print JSON to an output stream. It does not + * validate the content. + * + * @param out The output stream. + * @param value The array. + * @throw if there is an error with the underlying output stream. simdjson itself will not throw. + */ + inline std::ostream& operator<<(std::ostream& out, simdjson::lasx::ondemand::document& value); +#if SIMDJSON_EXCEPTIONS + inline std::ostream& operator<<(std::ostream& out, simdjson::simdjson_result&& x); +#endif + inline std::ostream& operator<<(std::ostream& out, simdjson::lasx::ondemand::document_reference& value); +#if SIMDJSON_EXCEPTIONS + inline std::ostream& operator<<(std::ostream& out, simdjson::simdjson_result&& x); +#endif + /** + * Print JSON to an output stream. It does not + * validate the content. + * + * @param out The output stream. + * @param value The object. + * @throw if there is an error with the underlying output stream. simdjson itself will not throw. + */ + inline std::ostream& operator<<(std::ostream& out, simdjson::lasx::ondemand::object value); +#if SIMDJSON_EXCEPTIONS + inline std::ostream& operator<<(std::ostream& out, simdjson::simdjson_result x); +#endif + } + } +} // namespace simdjson::lasx::ondemand + +#endif // SIMDJSON_GENERIC_ONDEMAND_SERIALIZATION_H +/* end file simdjson/generic/ondemand/serialization.h for lasx */ + +// Inline definitions +/* including simdjson/generic/ondemand/array-inl.h for lasx: #include "simdjson/generic/ondemand/array-inl.h" */ +/* begin file simdjson/generic/ondemand/array-inl.h for lasx */ +#ifndef SIMDJSON_GENERIC_ONDEMAND_ARRAY_INL_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_ONDEMAND_ARRAY_INL_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/array.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/array_iterator-inl.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/json_iterator.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/value.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/value_iterator-inl.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lasx { + namespace ondemand { + + // + // ### Live States + // + // While iterating or looking up values, depth >= iter->depth. at_start may vary. Error is + // always SUCCESS: + // + // - Start: This is the state when the array is first found and the iterator is just past the `{`. + // In this state, at_start == true. + // - Next: After we hand a scalar value to the user, or an array/object which they then fully + // iterate over, the iterator is at the `,` before the next value (or `]`). In this state, + // depth == iter->depth, at_start == false, and error == SUCCESS. + // - Unfinished Business: When we hand an array/object to the user which they do not fully + // iterate over, we need to finish that iteration by skipping child values until we reach the + // Next state. In this state, depth > iter->depth, at_start == false, and error == SUCCESS. + // + // ## Error States + // + // In error states, we will yield exactly one more value before stopping. iter->depth == depth + // and at_start is always false. We decrement after yielding the error, moving to the Finished + // state. + // + // - Chained Error: When the array iterator is part of an error chain--for example, in + // `for (auto tweet : doc["tweets"])`, where the tweet element may be missing or not be an + // array--we yield that error in the loop, exactly once. In this state, error != SUCCESS and + // iter->depth == depth, and at_start == false. We decrement depth when we yield the error. + // - Missing Comma Error: When the iterator ++ method discovers there is no comma between elements, + // we flag that as an error and treat it exactly the same as a Chained Error. In this state, + // error == TAPE_ERROR, iter->depth == depth, and at_start == false. + // + // ## Terminal State + // + // The terminal state has iter->depth < depth. at_start is always false. + // + // - Finished: When we have reached a `]` or have reported an error, we are finished. We signal this + // by decrementing depth. In this state, iter->depth < depth, at_start == false, and + // error == SUCCESS. + // + + simdjson_inline array::array(const value_iterator& _iter) noexcept + : iter{ _iter } + { + } + + simdjson_inline simdjson_result array::start(value_iterator& iter) noexcept { + // We don't need to know if the array is empty to start iteration, but we do want to know if there + // is an error--thus `simdjson_unused`. + simdjson_unused bool has_value; + SIMDJSON_TRY(iter.start_array().get(has_value)); + return array(iter); + } + simdjson_inline simdjson_result array::start_root(value_iterator& iter) noexcept { + simdjson_unused bool has_value; + SIMDJSON_TRY(iter.start_root_array().get(has_value)); + return array(iter); + } + simdjson_inline simdjson_result array::started(value_iterator& iter) noexcept { + bool has_value; + SIMDJSON_TRY(iter.started_array().get(has_value)); + return array(iter); + } + + simdjson_inline simdjson_result array::begin() noexcept { +#if SIMDJSON_DEVELOPMENT_CHECKS + if (!iter.is_at_iterator_start()) { return OUT_OF_ORDER_ITERATION; } +#endif + return array_iterator(iter); + } + simdjson_inline simdjson_result array::end() noexcept { + return array_iterator(iter); + } + simdjson_inline error_code array::consume() noexcept { + auto error = iter.json_iter().skip_child(iter.depth() - 1); + if (error) { iter.abandon(); } + return error; + } + + simdjson_inline simdjson_result array::raw_json() noexcept { + const uint8_t* starting_point{ iter.peek_start() }; + auto error = consume(); + if (error) { return error; } + // After 'consume()', we could be left pointing just beyond the document, but that + // is ok because we are not going to dereference the final pointer position, we just + // use it to compute the length in bytes. + const uint8_t* final_point{ iter._json_iter->unsafe_pointer() }; + return std::string_view(reinterpret_cast(starting_point), size_t(final_point - starting_point)); + } + + SIMDJSON_PUSH_DISABLE_WARNINGS + SIMDJSON_DISABLE_STRICT_OVERFLOW_WARNING + simdjson_inline simdjson_result array::count_elements() & noexcept { + size_t count{ 0 }; + // Important: we do not consume any of the values. + for (simdjson_unused auto v : *this) { count++; } + // The above loop will always succeed, but we want to report errors. + if (iter.error()) { return iter.error(); } + // We need to move back at the start because we expect users to iterate through + // the array after counting the number of elements. + iter.reset_array(); + return count; + } + SIMDJSON_POP_DISABLE_WARNINGS + + simdjson_inline simdjson_result array::is_empty() & noexcept { + bool is_not_empty; + auto error = iter.reset_array().get(is_not_empty); + if (error) { return error; } + return !is_not_empty; + } + + inline simdjson_result array::reset() & noexcept { + return iter.reset_array(); + } + + inline simdjson_result array::at_pointer(std::string_view json_pointer) noexcept { + if (json_pointer[0] != '/') { return INVALID_JSON_POINTER; } + json_pointer = json_pointer.substr(1); + // - means "the append position" or "the element after the end of the array" + // We don't support this, because we're returning a real element, not a position. + if (json_pointer == "-") { return INDEX_OUT_OF_BOUNDS; } + + // Read the array index + size_t array_index = 0; + size_t i; + for (i = 0; i < json_pointer.length() && json_pointer[i] != '/'; i++) { + uint8_t digit = uint8_t(json_pointer[i] - '0'); + // Check for non-digit in array index. If it's there, we're trying to get a field in an object + if (digit > 9) { return INCORRECT_TYPE; } + array_index = array_index * 10 + digit; + } + + // 0 followed by other digits is invalid + if (i > 1 && json_pointer[0] == '0') { return INVALID_JSON_POINTER; } // "JSON pointer array index has other characters after 0" + + // Empty string is invalid; so is a "/" with no digits before it + if (i == 0) { return INVALID_JSON_POINTER; } // "Empty string in JSON pointer array index" + // Get the child + auto child = at(array_index); + // If there is an error, it ends here + if (child.error()) { + return child; + } + + // If there is a /, we're not done yet, call recursively. + if (i < json_pointer.length()) { + child = child.at_pointer(json_pointer.substr(i)); + } + return child; + } + + inline std::string json_path_to_pointer_conversion(std::string_view json_path) { + if (json_path.empty() || (json_path.front() != '.' && + json_path.front() != '[')) { + return "-1"; // This is just a sentinel value, the caller should check for this and return an error. + } + + std::string result; + // Reserve space to reduce allocations, adjusting for potential increases due + // to escaping. + result.reserve(json_path.size() * 2); + + size_t i = 0; + + while (i < json_path.length()) { + if (json_path[i] == '.') { + result += '/'; + } + else if (json_path[i] == '[') { + result += '/'; + ++i; // Move past the '[' + while (i < json_path.length() && json_path[i] != ']') { + if (json_path[i] == '~') { + result += "~0"; + } + else if (json_path[i] == '/') { + result += "~1"; + } + else { + result += json_path[i]; + } + ++i; + } + if (i == json_path.length() || json_path[i] != ']') { + return "-1"; // Using sentinel value that will be handled as an error by the caller. + } + } + else { + if (json_path[i] == '~') { + result += "~0"; + } + else if (json_path[i] == '/') { + result += "~1"; + } + else { + result += json_path[i]; + } + } + ++i; + } + + return result; + } + + inline simdjson_result array::at_path(std::string_view json_path) noexcept { + auto json_pointer = json_path_to_pointer_conversion(json_path); + if (json_pointer == "-1") { return INVALID_JSON_POINTER; } + return at_pointer(json_pointer); + } + + simdjson_inline simdjson_result array::at(size_t index) noexcept { + size_t i = 0; + for (auto value : *this) { + if (i == index) { return value; } + i++; + } + return INDEX_OUT_OF_BOUNDS; + } + + } // namespace ondemand + } // namespace lasx +} // namespace simdjson + +namespace simdjson { + + simdjson_inline simdjson_result::simdjson_result( + lasx::ondemand::array&& value + ) noexcept + : implementation_simdjson_result_base( + std::forward(value) + ) + { + } + simdjson_inline simdjson_result::simdjson_result( + error_code error + ) noexcept + : implementation_simdjson_result_base(error) + { + } + + simdjson_inline simdjson_result simdjson_result::begin() noexcept { + if (error()) { return error(); } + return first.begin(); + } + simdjson_inline simdjson_result simdjson_result::end() noexcept { + if (error()) { return error(); } + return first.end(); + } + simdjson_inline simdjson_result simdjson_result::count_elements() & noexcept { + if (error()) { return error(); } + return first.count_elements(); + } + simdjson_inline simdjson_result simdjson_result::is_empty() & noexcept { + if (error()) { return error(); } + return first.is_empty(); + } + simdjson_inline simdjson_result simdjson_result::at(size_t index) noexcept { + if (error()) { return error(); } + return first.at(index); + } + simdjson_inline simdjson_result simdjson_result::at_pointer(std::string_view json_pointer) noexcept { + if (error()) { return error(); } + return first.at_pointer(json_pointer); + } + simdjson_inline simdjson_result simdjson_result::at_path(std::string_view json_path) noexcept { + if (error()) { return error(); } + return first.at_path(json_path); + } + simdjson_inline simdjson_result simdjson_result::raw_json() noexcept { + if (error()) { return error(); } + return first.raw_json(); + } +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_ONDEMAND_ARRAY_INL_H +/* end file simdjson/generic/ondemand/array-inl.h for lasx */ +/* including simdjson/generic/ondemand/array_iterator-inl.h for lasx: #include "simdjson/generic/ondemand/array_iterator-inl.h" */ +/* begin file simdjson/generic/ondemand/array_iterator-inl.h for lasx */ +#ifndef SIMDJSON_GENERIC_ONDEMAND_ARRAY_ITERATOR_INL_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_ONDEMAND_ARRAY_ITERATOR_INL_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/array_iterator.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/value-inl.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/value_iterator-inl.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lasx { + namespace ondemand { + + simdjson_inline array_iterator::array_iterator(const value_iterator& _iter) noexcept + : iter{ _iter } + {} + + simdjson_inline simdjson_result array_iterator::operator*() noexcept { + if (iter.error()) { iter.abandon(); return iter.error(); } + return value(iter.child()); + } + simdjson_inline bool array_iterator::operator==(const array_iterator& other) const noexcept { + return !(*this != other); + } + simdjson_inline bool array_iterator::operator!=(const array_iterator&) const noexcept { + return iter.is_open(); + } + simdjson_inline array_iterator& array_iterator::operator++() noexcept { + error_code error; + // PERF NOTE this is a safety rail ... users should exit loops as soon as they receive an error, so we'll never get here. + // However, it does not seem to make a perf difference, so we add it out of an abundance of caution. + if ((error = iter.error())) { return *this; } + if ((error = iter.skip_child())) { return *this; } + if ((error = iter.has_next_element().error())) { return *this; } + return *this; + } + + } // namespace ondemand + } // namespace lasx +} // namespace simdjson + +namespace simdjson { + + simdjson_inline simdjson_result::simdjson_result( + lasx::ondemand::array_iterator&& value + ) noexcept + : lasx::implementation_simdjson_result_base(std::forward(value)) + { + first.iter.assert_is_valid(); + } + simdjson_inline simdjson_result::simdjson_result(error_code error) noexcept + : lasx::implementation_simdjson_result_base({}, error) + { + } + + simdjson_inline simdjson_result simdjson_result::operator*() noexcept { + if (error()) { return error(); } + return *first; + } + simdjson_inline bool simdjson_result::operator==(const simdjson_result& other) const noexcept { + if (!first.iter.is_valid()) { return !error(); } + return first == other.first; + } + simdjson_inline bool simdjson_result::operator!=(const simdjson_result& other) const noexcept { + if (!first.iter.is_valid()) { return error(); } + return first != other.first; + } + simdjson_inline simdjson_result& simdjson_result::operator++() noexcept { + // Clear the error if there is one, so we don't yield it twice + if (error()) { second = SUCCESS; return *this; } + ++(first); + return *this; + } + +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_ONDEMAND_ARRAY_ITERATOR_INL_H +/* end file simdjson/generic/ondemand/array_iterator-inl.h for lasx */ +/* including simdjson/generic/ondemand/document-inl.h for lasx: #include "simdjson/generic/ondemand/document-inl.h" */ +/* begin file simdjson/generic/ondemand/document-inl.h for lasx */ +#ifndef SIMDJSON_GENERIC_ONDEMAND_DOCUMENT_INL_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_ONDEMAND_DOCUMENT_INL_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/array_iterator.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/document.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/json_type.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/raw_json_string.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/value.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/array-inl.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/json_iterator-inl.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/object-inl.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/value_iterator-inl.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lasx { + namespace ondemand { + + simdjson_inline document::document(ondemand::json_iterator&& _iter) noexcept + : iter{ std::forward(_iter) } + { + logger::log_start_value(iter, "document"); + } + + simdjson_inline document document::start(json_iterator&& iter) noexcept { + return document(std::forward(iter)); + } + + inline void document::rewind() noexcept { + iter.rewind(); + } + + inline std::string document::to_debug_string() noexcept { + return iter.to_string(); + } + + inline simdjson_result document::current_location() const noexcept { + return iter.current_location(); + } + + inline int32_t document::current_depth() const noexcept { + return iter.depth(); + } + + inline bool document::at_end() const noexcept { + return iter.at_end(); + } + + + inline bool document::is_alive() noexcept { + return iter.is_alive(); + } + simdjson_inline value_iterator document::resume_value_iterator() noexcept { + return value_iterator(&iter, 1, iter.root_position()); + } + simdjson_inline value_iterator document::get_root_value_iterator() noexcept { + return resume_value_iterator(); + } + simdjson_inline simdjson_result document::start_or_resume_object() noexcept { + if (iter.at_root()) { + return get_object(); + } + else { + return object::resume(resume_value_iterator()); + } + } + simdjson_inline simdjson_result document::get_value() noexcept { + // Make sure we start any arrays or objects before returning, so that start_root_() + // gets called. + + // It is the convention throughout the code that the macro `SIMDJSON_DEVELOPMENT_CHECKS` determines whether + // we check for OUT_OF_ORDER_ITERATION. Proper on::demand code should never trigger this error. +#if SIMDJSON_DEVELOPMENT_CHECKS + if (!iter.at_root()) { return OUT_OF_ORDER_ITERATION; } +#endif + // assert_at_root() serves two purposes: in Debug mode, whether or not + // SIMDJSON_DEVELOPMENT_CHECKS is set or not, it checks that we are at the root of + // the document (this will typically be redundant). In release mode, it generates + // SIMDJSON_ASSUME statements to allow the compiler to make assumptions. + iter.assert_at_root(); + switch (*iter.peek()) { + case '[': { + // The following lines check that the document ends with ]. + auto value_iterator = get_root_value_iterator(); + auto error = value_iterator.check_root_array(); + if (error) { return error; } + return value(get_root_value_iterator()); + } + case '{': { + // The following lines would check that the document ends with }. + auto value_iterator = get_root_value_iterator(); + auto error = value_iterator.check_root_object(); + if (error) { return error; } + return value(get_root_value_iterator()); + } + default: + // Unfortunately, scalar documents are a special case in simdjson and they cannot + // be safely converted to value instances. + return SCALAR_DOCUMENT_AS_VALUE; + } + } + simdjson_inline simdjson_result document::get_array() & noexcept { + auto value = get_root_value_iterator(); + return array::start_root(value); + } + simdjson_inline simdjson_result document::get_object() & noexcept { + auto value = get_root_value_iterator(); + return object::start_root(value); + } + + /** + * We decided that calling 'get_double()' on the JSON document '1.233 blabla' should + * give an error, so we check for trailing content. We want to disallow trailing + * content. + * Thus, in several implementations below, we pass a 'true' parameter value to + * a get_root_value_iterator() method: this indicates that we disallow trailing content. + */ + + simdjson_inline simdjson_result document::get_uint64() noexcept { + return get_root_value_iterator().get_root_uint64(true); + } + simdjson_inline simdjson_result document::get_uint64_in_string() noexcept { + return get_root_value_iterator().get_root_uint64_in_string(true); + } + simdjson_inline simdjson_result document::get_int64() noexcept { + return get_root_value_iterator().get_root_int64(true); + } + simdjson_inline simdjson_result document::get_int64_in_string() noexcept { + return get_root_value_iterator().get_root_int64_in_string(true); + } + simdjson_inline simdjson_result document::get_double() noexcept { + return get_root_value_iterator().get_root_double(true); + } + simdjson_inline simdjson_result document::get_double_in_string() noexcept { + return get_root_value_iterator().get_root_double_in_string(true); + } + simdjson_inline simdjson_result document::get_string(bool allow_replacement) noexcept { + return get_root_value_iterator().get_root_string(true, allow_replacement); + } + template + simdjson_inline error_code document::get_string(string_type& receiver, bool allow_replacement) noexcept { + return get_root_value_iterator().get_root_string(receiver, true, allow_replacement); + } + simdjson_inline simdjson_result document::get_wobbly_string() noexcept { + return get_root_value_iterator().get_root_wobbly_string(true); + } + simdjson_inline simdjson_result document::get_raw_json_string() noexcept { + return get_root_value_iterator().get_root_raw_json_string(true); + } + simdjson_inline simdjson_result document::get_bool() noexcept { + return get_root_value_iterator().get_root_bool(true); + } + simdjson_inline simdjson_result document::is_null() noexcept { + return get_root_value_iterator().is_root_null(true); + } + + template<> simdjson_inline simdjson_result document::get() & noexcept { return get_array(); } + template<> simdjson_inline simdjson_result document::get() & noexcept { return get_object(); } + template<> simdjson_inline simdjson_result document::get() & noexcept { return get_raw_json_string(); } + template<> simdjson_inline simdjson_result document::get() & noexcept { return get_string(false); } + template<> simdjson_inline simdjson_result document::get() & noexcept { return get_double(); } + template<> simdjson_inline simdjson_result document::get() & noexcept { return get_uint64(); } + template<> simdjson_inline simdjson_result document::get() & noexcept { return get_int64(); } + template<> simdjson_inline simdjson_result document::get() & noexcept { return get_bool(); } + template<> simdjson_inline simdjson_result document::get() & noexcept { return get_value(); } + + template<> simdjson_deprecated simdjson_inline simdjson_result document::get() && noexcept { return get_raw_json_string(); } + template<> simdjson_deprecated simdjson_inline simdjson_result document::get() && noexcept { return get_string(false); } + template<> simdjson_deprecated simdjson_inline simdjson_result document::get() && noexcept { return std::forward(*this).get_double(); } + template<> simdjson_deprecated simdjson_inline simdjson_result document::get() && noexcept { return std::forward(*this).get_uint64(); } + template<> simdjson_deprecated simdjson_inline simdjson_result document::get() && noexcept { return std::forward(*this).get_int64(); } + template<> simdjson_deprecated simdjson_inline simdjson_result document::get() && noexcept { return std::forward(*this).get_bool(); } + template<> simdjson_deprecated simdjson_inline simdjson_result document::get() && noexcept { return get_value(); } + + template simdjson_inline error_code document::get(T& out) & noexcept { + return get().get(out); + } + template simdjson_deprecated simdjson_inline error_code document::get(T& out) && noexcept { + return std::forward(*this).get().get(out); + } + +#if SIMDJSON_EXCEPTIONS + template + simdjson_deprecated simdjson_inline document::operator T() && noexcept(false) { return get(); } + template + simdjson_inline document::operator T() & noexcept(false) { return get(); } + simdjson_inline document::operator array() & noexcept(false) { return get_array(); } + simdjson_inline document::operator object() & noexcept(false) { return get_object(); } + simdjson_inline document::operator uint64_t() noexcept(false) { return get_uint64(); } + simdjson_inline document::operator int64_t() noexcept(false) { return get_int64(); } + simdjson_inline document::operator double() noexcept(false) { return get_double(); } + simdjson_inline document::operator std::string_view() noexcept(false) { return get_string(false); } + simdjson_inline document::operator raw_json_string() noexcept(false) { return get_raw_json_string(); } + simdjson_inline document::operator bool() noexcept(false) { return get_bool(); } + simdjson_inline document::operator value() noexcept(false) { return get_value(); } + +#endif + simdjson_inline simdjson_result document::count_elements() & noexcept { + auto a = get_array(); + simdjson_result answer = a.count_elements(); + /* If there was an array, we are now left pointing at its first element. */ + if (answer.error() == SUCCESS) { rewind(); } + return answer; + } + simdjson_inline simdjson_result document::count_fields() & noexcept { + auto a = get_object(); + simdjson_result answer = a.count_fields(); + /* If there was an object, we are now left pointing at its first element. */ + if (answer.error() == SUCCESS) { rewind(); } + return answer; + } + simdjson_inline simdjson_result document::at(size_t index) & noexcept { + auto a = get_array(); + return a.at(index); + } + simdjson_inline simdjson_result document::begin() & noexcept { + return get_array().begin(); + } + simdjson_inline simdjson_result document::end() & noexcept { + return {}; + } + + simdjson_inline simdjson_result document::find_field(std::string_view key) & noexcept { + return start_or_resume_object().find_field(key); + } + simdjson_inline simdjson_result document::find_field(const char* key) & noexcept { + return start_or_resume_object().find_field(key); + } + simdjson_inline simdjson_result document::find_field_unordered(std::string_view key) & noexcept { + return start_or_resume_object().find_field_unordered(key); + } + simdjson_inline simdjson_result document::find_field_unordered(const char* key) & noexcept { + return start_or_resume_object().find_field_unordered(key); + } + simdjson_inline simdjson_result document::operator[](std::string_view key) & noexcept { + return start_or_resume_object()[key]; + } + simdjson_inline simdjson_result document::operator[](const char* key) & noexcept { + return start_or_resume_object()[key]; + } + + simdjson_inline error_code document::consume() noexcept { + auto error = iter.skip_child(0); + if (error) { iter.abandon(); } + return error; + } + + simdjson_inline simdjson_result document::raw_json() noexcept { + auto _iter = get_root_value_iterator(); + const uint8_t* starting_point{ _iter.peek_start() }; + auto error = consume(); + if (error) { return error; } + // After 'consume()', we could be left pointing just beyond the document, but that + // is ok because we are not going to dereference the final pointer position, we just + // use it to compute the length in bytes. + const uint8_t* final_point{ iter.unsafe_pointer() }; + return std::string_view(reinterpret_cast(starting_point), size_t(final_point - starting_point)); + } + + simdjson_inline simdjson_result document::type() noexcept { + return get_root_value_iterator().type(); + } + + simdjson_inline simdjson_result document::is_scalar() noexcept { + json_type this_type; + auto error = type().get(this_type); + if (error) { return error; } + return !((this_type == json_type::array) || (this_type == json_type::object)); + } + + simdjson_inline simdjson_result document::is_string() noexcept { + json_type this_type; + auto error = type().get(this_type); + if (error) { return error; } + return (this_type == json_type::string); + } + + simdjson_inline bool document::is_negative() noexcept { + return get_root_value_iterator().is_root_negative(); + } + + simdjson_inline simdjson_result document::is_integer() noexcept { + return get_root_value_iterator().is_root_integer(true); + } + + simdjson_inline simdjson_result document::get_number_type() noexcept { + return get_root_value_iterator().get_root_number_type(true); + } + + simdjson_inline simdjson_result document::get_number() noexcept { + return get_root_value_iterator().get_root_number(true); + } + + + simdjson_inline simdjson_result document::raw_json_token() noexcept { + auto _iter = get_root_value_iterator(); + return std::string_view(reinterpret_cast(_iter.peek_start()), _iter.peek_root_length()); + } + + simdjson_inline simdjson_result document::at_pointer(std::string_view json_pointer) noexcept { + rewind(); // Rewind the document each time at_pointer is called + if (json_pointer.empty()) { + return this->get_value(); + } + json_type t; + SIMDJSON_TRY(type().get(t)); + switch (t) + { + case json_type::array: + return (*this).get_array().at_pointer(json_pointer); + case json_type::object: + return (*this).get_object().at_pointer(json_pointer); + default: + return INVALID_JSON_POINTER; + } + } + + simdjson_inline simdjson_result document::at_path(std::string_view json_path) noexcept { + rewind(); // Rewind the document each time at_pointer is called + if (json_path.empty()) { + return this->get_value(); + } + json_type t; + SIMDJSON_TRY(type().get(t)); + switch (t) { + case json_type::array: + return (*this).get_array().at_path(json_path); + case json_type::object: + return (*this).get_object().at_path(json_path); + default: + return INVALID_JSON_POINTER; + } + } + + } // namespace ondemand + } // namespace lasx +} // namespace simdjson + +namespace simdjson { + + simdjson_inline simdjson_result::simdjson_result( + lasx::ondemand::document&& value + ) noexcept : + implementation_simdjson_result_base( + std::forward(value) + ) + { + } + simdjson_inline simdjson_result::simdjson_result( + error_code error + ) noexcept : + implementation_simdjson_result_base( + error + ) + { + } + simdjson_inline simdjson_result simdjson_result::count_elements() & noexcept { + if (error()) { return error(); } + return first.count_elements(); + } + simdjson_inline simdjson_result simdjson_result::count_fields() & noexcept { + if (error()) { return error(); } + return first.count_fields(); + } + simdjson_inline simdjson_result simdjson_result::at(size_t index) & noexcept { + if (error()) { return error(); } + return first.at(index); + } + simdjson_inline error_code simdjson_result::rewind() noexcept { + if (error()) { return error(); } + first.rewind(); + return SUCCESS; + } + simdjson_inline simdjson_result simdjson_result::begin() & noexcept { + if (error()) { return error(); } + return first.begin(); + } + simdjson_inline simdjson_result simdjson_result::end() & noexcept { + return {}; + } + simdjson_inline simdjson_result simdjson_result::find_field_unordered(std::string_view key) & noexcept { + if (error()) { return error(); } + return first.find_field_unordered(key); + } + simdjson_inline simdjson_result simdjson_result::find_field_unordered(const char* key) & noexcept { + if (error()) { return error(); } + return first.find_field_unordered(key); + } + simdjson_inline simdjson_result simdjson_result::operator[](std::string_view key) & noexcept { + if (error()) { return error(); } + return first[key]; + } + simdjson_inline simdjson_result simdjson_result::operator[](const char* key) & noexcept { + if (error()) { return error(); } + return first[key]; + } + simdjson_inline simdjson_result simdjson_result::find_field(std::string_view key) & noexcept { + if (error()) { return error(); } + return first.find_field(key); + } + simdjson_inline simdjson_result simdjson_result::find_field(const char* key) & noexcept { + if (error()) { return error(); } + return first.find_field(key); + } + simdjson_inline simdjson_result simdjson_result::get_array() & noexcept { + if (error()) { return error(); } + return first.get_array(); + } + simdjson_inline simdjson_result simdjson_result::get_object() & noexcept { + if (error()) { return error(); } + return first.get_object(); + } + simdjson_inline simdjson_result simdjson_result::get_uint64() noexcept { + if (error()) { return error(); } + return first.get_uint64(); + } + simdjson_inline simdjson_result simdjson_result::get_uint64_in_string() noexcept { + if (error()) { return error(); } + return first.get_uint64_in_string(); + } + simdjson_inline simdjson_result simdjson_result::get_int64() noexcept { + if (error()) { return error(); } + return first.get_int64(); + } + simdjson_inline simdjson_result simdjson_result::get_int64_in_string() noexcept { + if (error()) { return error(); } + return first.get_int64_in_string(); + } + simdjson_inline simdjson_result simdjson_result::get_double() noexcept { + if (error()) { return error(); } + return first.get_double(); + } + simdjson_inline simdjson_result simdjson_result::get_double_in_string() noexcept { + if (error()) { return error(); } + return first.get_double_in_string(); + } + simdjson_inline simdjson_result simdjson_result::get_string(bool allow_replacement) noexcept { + if (error()) { return error(); } + return first.get_string(allow_replacement); + } + template + simdjson_inline error_code simdjson_result::get_string(string_type& receiver, bool allow_replacement) noexcept { + if (error()) { return error(); } + return first.get_string(receiver, allow_replacement); + } + simdjson_inline simdjson_result simdjson_result::get_wobbly_string() noexcept { + if (error()) { return error(); } + return first.get_wobbly_string(); + } + simdjson_inline simdjson_result simdjson_result::get_raw_json_string() noexcept { + if (error()) { return error(); } + return first.get_raw_json_string(); + } + simdjson_inline simdjson_result simdjson_result::get_bool() noexcept { + if (error()) { return error(); } + return first.get_bool(); + } + simdjson_inline simdjson_result simdjson_result::get_value() noexcept { + if (error()) { return error(); } + return first.get_value(); + } + simdjson_inline simdjson_result simdjson_result::is_null() noexcept { + if (error()) { return error(); } + return first.is_null(); + } + + template + simdjson_inline simdjson_result simdjson_result::get() & noexcept { + if (error()) { return error(); } + return first.get(); + } + template + simdjson_deprecated simdjson_inline simdjson_result simdjson_result::get() && noexcept { + if (error()) { return error(); } + return std::forward(first).get(); + } + template + simdjson_inline error_code simdjson_result::get(T& out) & noexcept { + if (error()) { return error(); } + return first.get(out); + } + template + simdjson_inline error_code simdjson_result::get(T& out) && noexcept { + if (error()) { return error(); } + return std::forward(first).get(out); + } + + template<> simdjson_inline simdjson_result simdjson_result::get() & noexcept = delete; + template<> simdjson_deprecated simdjson_inline simdjson_result simdjson_result::get() && noexcept { + if (error()) { return error(); } + return std::forward(first); + } + template<> simdjson_inline error_code simdjson_result::get(lasx::ondemand::document& out) & noexcept = delete; + template<> simdjson_inline error_code simdjson_result::get(lasx::ondemand::document& out) && noexcept { + if (error()) { return error(); } + out = std::forward(first); + return SUCCESS; + } + + simdjson_inline simdjson_result simdjson_result::type() noexcept { + if (error()) { return error(); } + return first.type(); + } + + simdjson_inline simdjson_result simdjson_result::is_scalar() noexcept { + if (error()) { return error(); } + return first.is_scalar(); + } + + simdjson_inline simdjson_result simdjson_result::is_string() noexcept { + if (error()) { return error(); } + return first.is_string(); + } + + simdjson_inline bool simdjson_result::is_negative() noexcept { + if (error()) { return error(); } + return first.is_negative(); + } + + simdjson_inline simdjson_result simdjson_result::is_integer() noexcept { + if (error()) { return error(); } + return first.is_integer(); + } + + simdjson_inline simdjson_result simdjson_result::get_number_type() noexcept { + if (error()) { return error(); } + return first.get_number_type(); + } + + simdjson_inline simdjson_result simdjson_result::get_number() noexcept { + if (error()) { return error(); } + return first.get_number(); + } + + +#if SIMDJSON_EXCEPTIONS + template ::value == false>::type> + simdjson_inline simdjson_result::operator T() noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return first; + } + simdjson_inline simdjson_result::operator lasx::ondemand::array() & noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return first; + } + simdjson_inline simdjson_result::operator lasx::ondemand::object() & noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return first; + } + simdjson_inline simdjson_result::operator uint64_t() noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return first; + } + simdjson_inline simdjson_result::operator int64_t() noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return first; + } + simdjson_inline simdjson_result::operator double() noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return first; + } + simdjson_inline simdjson_result::operator std::string_view() noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return first; + } + simdjson_inline simdjson_result::operator lasx::ondemand::raw_json_string() noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return first; + } + simdjson_inline simdjson_result::operator bool() noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return first; + } + simdjson_inline simdjson_result::operator lasx::ondemand::value() noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return first; + } +#endif + + + simdjson_inline simdjson_result simdjson_result::current_location() noexcept { + if (error()) { return error(); } + return first.current_location(); + } + + simdjson_inline bool simdjson_result::at_end() const noexcept { + if (error()) { return error(); } + return first.at_end(); + } + + + simdjson_inline int32_t simdjson_result::current_depth() const noexcept { + if (error()) { return error(); } + return first.current_depth(); + } + + simdjson_inline simdjson_result simdjson_result::raw_json_token() noexcept { + if (error()) { return error(); } + return first.raw_json_token(); + } + + simdjson_inline simdjson_result simdjson_result::at_pointer(std::string_view json_pointer) noexcept { + if (error()) { return error(); } + return first.at_pointer(json_pointer); + } + + simdjson_inline simdjson_result simdjson_result::at_path(std::string_view json_path) noexcept { + if (error()) { return error(); } + return first.at_path(json_path); + } + +} // namespace simdjson + + +namespace simdjson { + namespace lasx { + namespace ondemand { + + simdjson_inline document_reference::document_reference() noexcept : doc{ nullptr } {} + simdjson_inline document_reference::document_reference(document& d) noexcept : doc(&d) {} + simdjson_inline void document_reference::rewind() noexcept { doc->rewind(); } + simdjson_inline simdjson_result document_reference::get_array() & noexcept { return doc->get_array(); } + simdjson_inline simdjson_result document_reference::get_object() & noexcept { return doc->get_object(); } + /** + * The document_reference instances are used primarily/solely for streams of JSON + * documents. + * We decided that calling 'get_double()' on the JSON document '1.233 blabla' should + * give an error, so we check for trailing content. + * + * However, for streams of JSON documents, we want to be able to start from + * "321" "321" "321" + * and parse it successfully as a stream of JSON documents, calling get_uint64_in_string() + * successfully each time. + * + * To achieve this result, we pass a 'false' to a get_root_value_iterator() method: + * this indicates that we allow trailing content. + */ + simdjson_inline simdjson_result document_reference::get_uint64() noexcept { return doc->get_root_value_iterator().get_root_uint64(false); } + simdjson_inline simdjson_result document_reference::get_uint64_in_string() noexcept { return doc->get_root_value_iterator().get_root_uint64_in_string(false); } + simdjson_inline simdjson_result document_reference::get_int64() noexcept { return doc->get_root_value_iterator().get_root_int64(false); } + simdjson_inline simdjson_result document_reference::get_int64_in_string() noexcept { return doc->get_root_value_iterator().get_root_int64_in_string(false); } + simdjson_inline simdjson_result document_reference::get_double() noexcept { return doc->get_root_value_iterator().get_root_double(false); } + simdjson_inline simdjson_result document_reference::get_double_in_string() noexcept { return doc->get_root_value_iterator().get_root_double(false); } + simdjson_inline simdjson_result document_reference::get_string(bool allow_replacement) noexcept { return doc->get_root_value_iterator().get_root_string(false, allow_replacement); } + template + simdjson_inline error_code document_reference::get_string(string_type& receiver, bool allow_replacement) noexcept { return doc->get_root_value_iterator().get_root_string(receiver, false, allow_replacement); } + simdjson_inline simdjson_result document_reference::get_wobbly_string() noexcept { return doc->get_root_value_iterator().get_root_wobbly_string(false); } + simdjson_inline simdjson_result document_reference::get_raw_json_string() noexcept { return doc->get_root_value_iterator().get_root_raw_json_string(false); } + simdjson_inline simdjson_result document_reference::get_bool() noexcept { return doc->get_root_value_iterator().get_root_bool(false); } + simdjson_inline simdjson_result document_reference::get_value() noexcept { return doc->get_value(); } + simdjson_inline simdjson_result document_reference::is_null() noexcept { return doc->get_root_value_iterator().is_root_null(false); } + template<> simdjson_inline simdjson_result document_reference::get() & noexcept { return get_array(); } + template<> simdjson_inline simdjson_result document_reference::get() & noexcept { return get_object(); } + template<> simdjson_inline simdjson_result document_reference::get() & noexcept { return get_raw_json_string(); } + template<> simdjson_inline simdjson_result document_reference::get() & noexcept { return get_string(false); } + template<> simdjson_inline simdjson_result document_reference::get() & noexcept { return get_double(); } + template<> simdjson_inline simdjson_result document_reference::get() & noexcept { return get_uint64(); } + template<> simdjson_inline simdjson_result document_reference::get() & noexcept { return get_int64(); } + template<> simdjson_inline simdjson_result document_reference::get() & noexcept { return get_bool(); } + template<> simdjson_inline simdjson_result document_reference::get() & noexcept { return get_value(); } +#if SIMDJSON_EXCEPTIONS + template + simdjson_inline document_reference::operator T() noexcept(false) { return get(); } + simdjson_inline document_reference::operator array() & noexcept(false) { return array(*doc); } + simdjson_inline document_reference::operator object() & noexcept(false) { return object(*doc); } + simdjson_inline document_reference::operator uint64_t() noexcept(false) { return get_uint64(); } + simdjson_inline document_reference::operator int64_t() noexcept(false) { return get_int64(); } + simdjson_inline document_reference::operator double() noexcept(false) { return get_double(); } + simdjson_inline document_reference::operator std::string_view() noexcept(false) { return std::string_view(*doc); } + simdjson_inline document_reference::operator raw_json_string() noexcept(false) { return get_raw_json_string(); } + simdjson_inline document_reference::operator bool() noexcept(false) { return get_bool(); } + simdjson_inline document_reference::operator value() noexcept(false) { return value(*doc); } +#endif + simdjson_inline simdjson_result document_reference::count_elements() & noexcept { return doc->count_elements(); } + simdjson_inline simdjson_result document_reference::count_fields() & noexcept { return doc->count_fields(); } + simdjson_inline simdjson_result document_reference::at(size_t index) & noexcept { return doc->at(index); } + simdjson_inline simdjson_result document_reference::begin() & noexcept { return doc->begin(); } + simdjson_inline simdjson_result document_reference::end() & noexcept { return doc->end(); } + simdjson_inline simdjson_result document_reference::find_field(std::string_view key) & noexcept { return doc->find_field(key); } + simdjson_inline simdjson_result document_reference::find_field(const char* key) & noexcept { return doc->find_field(key); } + simdjson_inline simdjson_result document_reference::operator[](std::string_view key) & noexcept { return (*doc)[key]; } + simdjson_inline simdjson_result document_reference::operator[](const char* key) & noexcept { return (*doc)[key]; } + simdjson_inline simdjson_result document_reference::find_field_unordered(std::string_view key) & noexcept { return doc->find_field_unordered(key); } + simdjson_inline simdjson_result document_reference::find_field_unordered(const char* key) & noexcept { return doc->find_field_unordered(key); } + simdjson_inline simdjson_result document_reference::type() noexcept { return doc->type(); } + simdjson_inline simdjson_result document_reference::is_scalar() noexcept { return doc->is_scalar(); } + simdjson_inline simdjson_result document_reference::is_string() noexcept { return doc->is_string(); } + simdjson_inline simdjson_result document_reference::current_location() noexcept { return doc->current_location(); } + simdjson_inline int32_t document_reference::current_depth() const noexcept { return doc->current_depth(); } + simdjson_inline bool document_reference::is_negative() noexcept { return doc->is_negative(); } + simdjson_inline simdjson_result document_reference::is_integer() noexcept { return doc->get_root_value_iterator().is_root_integer(false); } + simdjson_inline simdjson_result document_reference::get_number_type() noexcept { return doc->get_root_value_iterator().get_root_number_type(false); } + simdjson_inline simdjson_result document_reference::get_number() noexcept { return doc->get_root_value_iterator().get_root_number(false); } + simdjson_inline simdjson_result document_reference::raw_json_token() noexcept { return doc->raw_json_token(); } + simdjson_inline simdjson_result document_reference::at_pointer(std::string_view json_pointer) noexcept { return doc->at_pointer(json_pointer); } + simdjson_inline simdjson_result document_reference::at_path(std::string_view json_path) noexcept { return doc->at_path(json_path); } + simdjson_inline simdjson_result document_reference::raw_json() noexcept { return doc->raw_json(); } + simdjson_inline document_reference::operator document& () const noexcept { return *doc; } + + } // namespace ondemand + } // namespace lasx +} // namespace simdjson + + + +namespace simdjson { + simdjson_inline simdjson_result::simdjson_result(lasx::ondemand::document_reference value, error_code error) + noexcept : implementation_simdjson_result_base(std::forward(value), error) {} + + + simdjson_inline simdjson_result simdjson_result::count_elements() & noexcept { + if (error()) { return error(); } + return first.count_elements(); + } + simdjson_inline simdjson_result simdjson_result::count_fields() & noexcept { + if (error()) { return error(); } + return first.count_fields(); + } + simdjson_inline simdjson_result simdjson_result::at(size_t index) & noexcept { + if (error()) { return error(); } + return first.at(index); + } + simdjson_inline error_code simdjson_result::rewind() noexcept { + if (error()) { return error(); } + first.rewind(); + return SUCCESS; + } + simdjson_inline simdjson_result simdjson_result::begin() & noexcept { + if (error()) { return error(); } + return first.begin(); + } + simdjson_inline simdjson_result simdjson_result::end() & noexcept { + return {}; + } + simdjson_inline simdjson_result simdjson_result::find_field_unordered(std::string_view key) & noexcept { + if (error()) { return error(); } + return first.find_field_unordered(key); + } + simdjson_inline simdjson_result simdjson_result::find_field_unordered(const char* key) & noexcept { + if (error()) { return error(); } + return first.find_field_unordered(key); + } + simdjson_inline simdjson_result simdjson_result::operator[](std::string_view key) & noexcept { + if (error()) { return error(); } + return first[key]; + } + simdjson_inline simdjson_result simdjson_result::operator[](const char* key) & noexcept { + if (error()) { return error(); } + return first[key]; + } + simdjson_inline simdjson_result simdjson_result::find_field(std::string_view key) & noexcept { + if (error()) { return error(); } + return first.find_field(key); + } + simdjson_inline simdjson_result simdjson_result::find_field(const char* key) & noexcept { + if (error()) { return error(); } + return first.find_field(key); + } + simdjson_inline simdjson_result simdjson_result::get_array() & noexcept { + if (error()) { return error(); } + return first.get_array(); + } + simdjson_inline simdjson_result simdjson_result::get_object() & noexcept { + if (error()) { return error(); } + return first.get_object(); + } + simdjson_inline simdjson_result simdjson_result::get_uint64() noexcept { + if (error()) { return error(); } + return first.get_uint64(); + } + simdjson_inline simdjson_result simdjson_result::get_uint64_in_string() noexcept { + if (error()) { return error(); } + return first.get_uint64_in_string(); + } + simdjson_inline simdjson_result simdjson_result::get_int64() noexcept { + if (error()) { return error(); } + return first.get_int64(); + } + simdjson_inline simdjson_result simdjson_result::get_int64_in_string() noexcept { + if (error()) { return error(); } + return first.get_int64_in_string(); + } + simdjson_inline simdjson_result simdjson_result::get_double() noexcept { + if (error()) { return error(); } + return first.get_double(); + } + simdjson_inline simdjson_result simdjson_result::get_double_in_string() noexcept { + if (error()) { return error(); } + return first.get_double_in_string(); + } + simdjson_inline simdjson_result simdjson_result::get_string(bool allow_replacement) noexcept { + if (error()) { return error(); } + return first.get_string(allow_replacement); + } + template + simdjson_inline error_code simdjson_result::get_string(string_type& receiver, bool allow_replacement) noexcept { + if (error()) { return error(); } + return first.get_string(receiver, allow_replacement); + } + simdjson_inline simdjson_result simdjson_result::get_wobbly_string() noexcept { + if (error()) { return error(); } + return first.get_wobbly_string(); + } + simdjson_inline simdjson_result simdjson_result::get_raw_json_string() noexcept { + if (error()) { return error(); } + return first.get_raw_json_string(); + } + simdjson_inline simdjson_result simdjson_result::get_bool() noexcept { + if (error()) { return error(); } + return first.get_bool(); + } + simdjson_inline simdjson_result simdjson_result::get_value() noexcept { + if (error()) { return error(); } + return first.get_value(); + } + simdjson_inline simdjson_result simdjson_result::is_null() noexcept { + if (error()) { return error(); } + return first.is_null(); + } + simdjson_inline simdjson_result simdjson_result::type() noexcept { + if (error()) { return error(); } + return first.type(); + } + simdjson_inline simdjson_result simdjson_result::is_scalar() noexcept { + if (error()) { return error(); } + return first.is_scalar(); + } + simdjson_inline simdjson_result simdjson_result::is_string() noexcept { + if (error()) { return error(); } + return first.is_string(); + } + simdjson_inline simdjson_result simdjson_result::is_negative() noexcept { + if (error()) { return error(); } + return first.is_negative(); + } + simdjson_inline simdjson_result simdjson_result::is_integer() noexcept { + if (error()) { return error(); } + return first.is_integer(); + } + simdjson_inline simdjson_result simdjson_result::get_number_type() noexcept { + if (error()) { return error(); } + return first.get_number_type(); + } + simdjson_inline simdjson_result simdjson_result::get_number() noexcept { + if (error()) { return error(); } + return first.get_number(); + } +#if SIMDJSON_EXCEPTIONS + template ::value == false>::type> + simdjson_inline simdjson_result::operator T() noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return first; + } + simdjson_inline simdjson_result::operator lasx::ondemand::array() & noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return first; + } + simdjson_inline simdjson_result::operator lasx::ondemand::object() & noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return first; + } + simdjson_inline simdjson_result::operator uint64_t() noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return first; + } + simdjson_inline simdjson_result::operator int64_t() noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return first; + } + simdjson_inline simdjson_result::operator double() noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return first; + } + simdjson_inline simdjson_result::operator std::string_view() noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return first; + } + simdjson_inline simdjson_result::operator lasx::ondemand::raw_json_string() noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return first; + } + simdjson_inline simdjson_result::operator bool() noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return first; + } + simdjson_inline simdjson_result::operator lasx::ondemand::value() noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return first; + } +#endif + + simdjson_inline simdjson_result simdjson_result::current_location() noexcept { + if (error()) { return error(); } + return first.current_location(); + } + + simdjson_inline simdjson_result simdjson_result::raw_json_token() noexcept { + if (error()) { return error(); } + return first.raw_json_token(); + } + + simdjson_inline simdjson_result simdjson_result::at_pointer(std::string_view json_pointer) noexcept { + if (error()) { return error(); } + return first.at_pointer(json_pointer); + } + + simdjson_inline simdjson_result simdjson_result::at_path(std::string_view json_path) noexcept { + if (error()) { + return error(); + } + return first.at_path(json_path); + } + +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_ONDEMAND_DOCUMENT_INL_H +/* end file simdjson/generic/ondemand/document-inl.h for lasx */ +/* including simdjson/generic/ondemand/document_stream-inl.h for lasx: #include "simdjson/generic/ondemand/document_stream-inl.h" */ +/* begin file simdjson/generic/ondemand/document_stream-inl.h for lasx */ +#ifndef SIMDJSON_GENERIC_ONDEMAND_DOCUMENT_STREAM_INL_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_ONDEMAND_DOCUMENT_STREAM_INL_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/document_stream.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/document-inl.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/implementation_simdjson_result_base-inl.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +#include +#include + +namespace simdjson { + namespace lasx { + namespace ondemand { + +#ifdef SIMDJSON_THREADS_ENABLED + + inline void stage1_worker::finish() { + // After calling "run" someone would call finish() to wait + // for the end of the processing. + // This function will wait until either the thread has done + // the processing or, else, the destructor has been called. + std::unique_lock lock(locking_mutex); + cond_var.wait(lock, [this] {return has_work == false; }); + } + + inline stage1_worker::~stage1_worker() { + // The thread may never outlive the stage1_worker instance + // and will always be stopped/joined before the stage1_worker + // instance is gone. + stop_thread(); + } + + inline void stage1_worker::start_thread() { + std::unique_lock lock(locking_mutex); + if (thread.joinable()) { + return; // This should never happen but we never want to create more than one thread. + } + thread = std::thread([this] { + while (true) { + std::unique_lock thread_lock(locking_mutex); + // We wait for either "run" or "stop_thread" to be called. + cond_var.wait(thread_lock, [this] {return has_work || !can_work; }); + // If, for some reason, the stop_thread() method was called (i.e., the + // destructor of stage1_worker is called, then we want to immediately destroy + // the thread (and not do any more processing). + if (!can_work) { + break; + } + this->owner->stage1_thread_error = this->owner->run_stage1(*this->stage1_thread_parser, + this->_next_batch_start); + this->has_work = false; + // The condition variable call should be moved after thread_lock.unlock() for performance + // reasons but thread sanitizers may report it as a data race if we do. + // See https://stackoverflow.com/questions/35775501/c-should-condition-variable-be-notified-under-lock + cond_var.notify_one(); // will notify "finish" + thread_lock.unlock(); + } + } + ); + } + + + inline void stage1_worker::stop_thread() { + std::unique_lock lock(locking_mutex); + // We have to make sure that all locks can be released. + can_work = false; + has_work = false; + cond_var.notify_all(); + lock.unlock(); + if (thread.joinable()) { + thread.join(); + } + } + + inline void stage1_worker::run(document_stream* ds, parser* stage1, size_t next_batch_start) { + std::unique_lock lock(locking_mutex); + owner = ds; + _next_batch_start = next_batch_start; + stage1_thread_parser = stage1; + has_work = true; + // The condition variable call should be moved after thread_lock.unlock() for performance + // reasons but thread sanitizers may report it as a data race if we do. + // See https://stackoverflow.com/questions/35775501/c-should-condition-variable-be-notified-under-lock + cond_var.notify_one(); // will notify the thread lock that we have work + lock.unlock(); + } + +#endif // SIMDJSON_THREADS_ENABLED + + simdjson_inline document_stream::document_stream( + ondemand::parser& _parser, + const uint8_t* _buf, + size_t _len, + size_t _batch_size, + bool _allow_comma_separated + ) noexcept + : parser{ &_parser }, + buf{ _buf }, + len{ _len }, + batch_size{ _batch_size <= MINIMAL_BATCH_SIZE ? MINIMAL_BATCH_SIZE : _batch_size }, + allow_comma_separated{ _allow_comma_separated }, + error{ SUCCESS } +#ifdef SIMDJSON_THREADS_ENABLED + , use_thread(_parser.threaded) // we need to make a copy because _parser.threaded can change +#endif + { +#ifdef SIMDJSON_THREADS_ENABLED + if (worker.get() == nullptr) { + error = MEMALLOC; + } +#endif + } + + simdjson_inline document_stream::document_stream() noexcept + : parser{ nullptr }, + buf{ nullptr }, + len{ 0 }, + batch_size{ 0 }, + allow_comma_separated{ false }, + error{ UNINITIALIZED } +#ifdef SIMDJSON_THREADS_ENABLED + , use_thread(false) +#endif + { + } + + simdjson_inline document_stream::~document_stream() noexcept + { +#ifdef SIMDJSON_THREADS_ENABLED + worker.reset(); +#endif + } + + inline size_t document_stream::size_in_bytes() const noexcept { + return len; + } + + inline size_t document_stream::truncated_bytes() const noexcept { + if (error == CAPACITY) { return len - batch_start; } + return parser->implementation->structural_indexes[parser->implementation->n_structural_indexes] - parser->implementation->structural_indexes[parser->implementation->n_structural_indexes + 1]; + } + + simdjson_inline document_stream::iterator::iterator() noexcept + : stream{ nullptr }, finished{ true } { + } + + simdjson_inline document_stream::iterator::iterator(document_stream* _stream, bool is_end) noexcept + : stream{ _stream }, finished{ is_end } { + } + + simdjson_inline simdjson_result document_stream::iterator::operator*() noexcept { + //if(stream->error) { return stream->error; } + return simdjson_result(stream->doc, stream->error); + } + + simdjson_inline document_stream::iterator& document_stream::iterator::operator++() noexcept { + // If there is an error, then we want the iterator + // to be finished, no matter what. (E.g., we do not + // keep generating documents with errors, or go beyond + // a document with errors.) + // + // Users do not have to call "operator*()" when they use operator++, + // so we need to end the stream in the operator++ function. + // + // Note that setting finished = true is essential otherwise + // we would enter an infinite loop. + if (stream->error) { finished = true; } + // Note that stream->error() is guarded against error conditions + // (it will immediately return if stream->error casts to false). + // In effect, this next function does nothing when (stream->error) + // is true (hence the risk of an infinite loop). + stream->next(); + // If that was the last document, we're finished. + // It is the only type of error we do not want to appear + // in operator*. + if (stream->error == EMPTY) { finished = true; } + // If we had any other kind of error (not EMPTY) then we want + // to pass it along to the operator* and we cannot mark the result + // as "finished" just yet. + return *this; + } + + simdjson_inline bool document_stream::iterator::operator!=(const document_stream::iterator& other) const noexcept { + return finished != other.finished; + } + + simdjson_inline document_stream::iterator document_stream::begin() noexcept { + start(); + // If there are no documents, we're finished. + return iterator(this, error == EMPTY); + } + + simdjson_inline document_stream::iterator document_stream::end() noexcept { + return iterator(this, true); + } + + inline void document_stream::start() noexcept { + if (error) { return; } + error = parser->allocate(batch_size); + if (error) { return; } + // Always run the first stage 1 parse immediately + batch_start = 0; + error = run_stage1(*parser, batch_start); + while (error == EMPTY) { + // In exceptional cases, we may start with an empty block + batch_start = next_batch_start(); + if (batch_start >= len) { return; } + error = run_stage1(*parser, batch_start); + } + if (error) { return; } + doc_index = batch_start; + doc = document(json_iterator(&buf[batch_start], parser)); + doc.iter._streaming = true; + +#ifdef SIMDJSON_THREADS_ENABLED + if (use_thread && next_batch_start() < len) { + // Kick off the first thread on next batch if needed + error = stage1_thread_parser.allocate(batch_size); + if (error) { return; } + worker->start_thread(); + start_stage1_thread(); + if (error) { return; } + } +#endif // SIMDJSON_THREADS_ENABLED + } + + inline void document_stream::next() noexcept { + // We always enter at once once in an error condition. + if (error) { return; } + next_document(); + if (error) { return; } + auto cur_struct_index = doc.iter._root - parser->implementation->structural_indexes.get(); + doc_index = batch_start + parser->implementation->structural_indexes[cur_struct_index]; + + // Check if at end of structural indexes (i.e. at end of batch) + if (cur_struct_index >= static_cast(parser->implementation->n_structural_indexes)) { + error = EMPTY; + // Load another batch (if available) + while (error == EMPTY) { + batch_start = next_batch_start(); + if (batch_start >= len) { break; } +#ifdef SIMDJSON_THREADS_ENABLED + if (use_thread) { + load_from_stage1_thread(); + } + else { + error = run_stage1(*parser, batch_start); + } +#else + error = run_stage1(*parser, batch_start); +#endif + /** + * Whenever we move to another window, we need to update all pointers to make + * it appear as if the input buffer started at the beginning of the window. + * + * Take this input: + * + * {"z":5} {"1":1,"2":2,"4":4} [7, 10, 9] [15, 11, 12, 13] [154, 110, 112, 1311] + * + * Say you process the following window... + * + * '{"z":5} {"1":1,"2":2,"4":4} [7, 10, 9]' + * + * When you do so, the json_iterator has a pointer at the beginning of the memory region + * (pointing at the beginning of '{"z"...'. + * + * When you move to the window that starts at... + * + * '[7, 10, 9] [15, 11, 12, 13] ... + * + * then it is not sufficient to just run stage 1. You also need to re-anchor the + * json_iterator so that it believes we are starting at '[7, 10, 9]...'. + * + * Under the DOM front-end, this gets done automatically because the parser owns + * the pointer the data, and when you call stage1 and then stage2 on the same + * parser, then stage2 will run on the pointer acquired by stage1. + * + * That is, stage1 calls "this->buf = _buf" so the parser remembers the buffer that + * we used. But json_iterator has no callback when stage1 is called on the parser. + * In fact, I think that the parser is unaware of json_iterator. + * + * + * So we need to re-anchor the json_iterator after each call to stage 1 so that + * all of the pointers are in sync. + */ + doc.iter = json_iterator(&buf[batch_start], parser); + doc.iter._streaming = true; + /** + * End of resync. + */ + + if (error) { continue; } // If the error was EMPTY, we may want to load another batch. + doc_index = batch_start; + } + } + } + + inline void document_stream::next_document() noexcept { + // Go to next place where depth=0 (document depth) + error = doc.iter.skip_child(0); + if (error) { return; } + // Always set depth=1 at the start of document + doc.iter._depth = 1; + // consume comma if comma separated is allowed + if (allow_comma_separated) { doc.iter.consume_character(','); } + // Resets the string buffer at the beginning, thus invalidating the strings. + doc.iter._string_buf_loc = parser->string_buf.get(); + doc.iter._root = doc.iter.position(); + } + + inline size_t document_stream::next_batch_start() const noexcept { + return batch_start + parser->implementation->structural_indexes[parser->implementation->n_structural_indexes]; + } + + inline error_code document_stream::run_stage1(ondemand::parser& p, size_t _batch_start) noexcept { + // This code only updates the structural index in the parser, it does not update any json_iterator + // instance. + size_t remaining = len - _batch_start; + if (remaining <= batch_size) { + return p.implementation->stage1(&buf[_batch_start], remaining, stage1_mode::streaming_final); + } + else { + return p.implementation->stage1(&buf[_batch_start], batch_size, stage1_mode::streaming_partial); + } + } + + simdjson_inline size_t document_stream::iterator::current_index() const noexcept { + return stream->doc_index; + } + + simdjson_inline std::string_view document_stream::iterator::source() const noexcept { + auto depth = stream->doc.iter.depth(); + auto cur_struct_index = stream->doc.iter._root - stream->parser->implementation->structural_indexes.get(); + + // If at root, process the first token to determine if scalar value + if (stream->doc.iter.at_root()) { + switch (stream->buf[stream->batch_start + stream->parser->implementation->structural_indexes[cur_struct_index]]) { + case '{': case '[': // Depth=1 already at start of document + break; + case '}': case ']': + depth--; + break; + default: // Scalar value document + // TODO: We could remove trailing whitespaces + // This returns a string spanning from start of value to the beginning of the next document (excluded) + { + auto next_index = stream->parser->implementation->structural_indexes[++cur_struct_index]; + // normally the length would be next_index - current_index() - 1, except for the last document + size_t svlen = next_index - current_index(); + const char* start = reinterpret_cast(stream->buf) + current_index(); + while (svlen > 1 && (std::isspace(start[svlen - 1]) || start[svlen - 1] == '\0')) { + svlen--; + } + return std::string_view(start, svlen); + } + } + cur_struct_index++; + } + + while (cur_struct_index <= static_cast(stream->parser->implementation->n_structural_indexes)) { + switch (stream->buf[stream->batch_start + stream->parser->implementation->structural_indexes[cur_struct_index]]) { + case '{': case '[': + depth++; + break; + case '}': case ']': + depth--; + break; + } + if (depth == 0) { break; } + cur_struct_index++; + } + + return std::string_view(reinterpret_cast(stream->buf) + current_index(), stream->parser->implementation->structural_indexes[cur_struct_index] - current_index() + stream->batch_start + 1);; + } + + inline error_code document_stream::iterator::error() const noexcept { + return stream->error; + } + +#ifdef SIMDJSON_THREADS_ENABLED + + inline void document_stream::load_from_stage1_thread() noexcept { + worker->finish(); + // Swap to the parser that was loaded up in the thread. Make sure the parser has + // enough memory to swap to, as well. + std::swap(stage1_thread_parser, *parser); + error = stage1_thread_error; + if (error) { return; } + + // If there's anything left, start the stage 1 thread! + if (next_batch_start() < len) { + start_stage1_thread(); + } + } + + inline void document_stream::start_stage1_thread() noexcept { + // we call the thread on a lambda that will update + // this->stage1_thread_error + // there is only one thread that may write to this value + // TODO this is NOT exception-safe. + this->stage1_thread_error = UNINITIALIZED; // In case something goes wrong, make sure it's an error + size_t _next_batch_start = this->next_batch_start(); + + worker->run(this, &this->stage1_thread_parser, _next_batch_start); + } + +#endif // SIMDJSON_THREADS_ENABLED + + } // namespace ondemand + } // namespace lasx +} // namespace simdjson + +namespace simdjson { + + simdjson_inline simdjson_result::simdjson_result( + error_code error + ) noexcept : + implementation_simdjson_result_base(error) + { + } + simdjson_inline simdjson_result::simdjson_result( + lasx::ondemand::document_stream&& value + ) noexcept : + implementation_simdjson_result_base( + std::forward(value) + ) + { + } + +} + +#endif // SIMDJSON_GENERIC_ONDEMAND_DOCUMENT_STREAM_INL_H +/* end file simdjson/generic/ondemand/document_stream-inl.h for lasx */ +/* including simdjson/generic/ondemand/field-inl.h for lasx: #include "simdjson/generic/ondemand/field-inl.h" */ +/* begin file simdjson/generic/ondemand/field-inl.h for lasx */ +#ifndef SIMDJSON_GENERIC_ONDEMAND_FIELD_INL_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_ONDEMAND_FIELD_INL_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/field.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/value-inl.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/value_iterator-inl.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lasx { + namespace ondemand { + + // clang 6 does not think the default constructor can be noexcept, so we make it explicit + simdjson_inline field::field() noexcept : std::pair() {} + + simdjson_inline field::field(raw_json_string key, ondemand::value&& value) noexcept + : std::pair(key, std::forward(value)) + { + } + + simdjson_inline simdjson_result field::start(value_iterator& parent_iter) noexcept { + raw_json_string key; + SIMDJSON_TRY(parent_iter.field_key().get(key)); + SIMDJSON_TRY(parent_iter.field_value()); + return field::start(parent_iter, key); + } + + simdjson_inline simdjson_result field::start(const value_iterator& parent_iter, raw_json_string key) noexcept { + return field(key, parent_iter.child()); + } + + simdjson_inline simdjson_warn_unused simdjson_result field::unescaped_key(bool allow_replacement) noexcept { + SIMDJSON_ASSUME(first.buf != nullptr); // We would like to call .alive() but Visual Studio won't let us. + simdjson_result answer = first.unescape(second.iter.json_iter(), allow_replacement); + first.consume(); + return answer; + } + + template + simdjson_inline simdjson_warn_unused error_code field::unescaped_key(string_type& receiver, bool allow_replacement) noexcept { + std::string_view key; + SIMDJSON_TRY(unescaped_key(allow_replacement).get(key)); + receiver = key; + return SUCCESS; + } + + simdjson_inline raw_json_string field::key() const noexcept { + SIMDJSON_ASSUME(first.buf != nullptr); // We would like to call .alive() by Visual Studio won't let us. + return first; + } + + + simdjson_inline std::string_view field::key_raw_json_token() const noexcept { + SIMDJSON_ASSUME(first.buf != nullptr); // We would like to call .alive() by Visual Studio won't let us. + return std::string_view(reinterpret_cast(first.buf - 1), second.iter._json_iter->token.peek(-1) - first.buf + 1); + } + + simdjson_inline std::string_view field::escaped_key() const noexcept { + SIMDJSON_ASSUME(first.buf != nullptr); // We would like to call .alive() by Visual Studio won't let us. + auto end_quote = second.iter._json_iter->token.peek(-1); + while (*end_quote != '"') end_quote--; + return std::string_view(reinterpret_cast(first.buf), end_quote - first.buf); + } + + simdjson_inline value& field::value() & noexcept { + return second; + } + + simdjson_inline value field::value() && noexcept { + return std::forward(*this).second; + } + + } // namespace ondemand + } // namespace lasx +} // namespace simdjson + +namespace simdjson { + + simdjson_inline simdjson_result::simdjson_result( + lasx::ondemand::field&& value + ) noexcept : + implementation_simdjson_result_base( + std::forward(value) + ) + { + } + simdjson_inline simdjson_result::simdjson_result( + error_code error + ) noexcept : + implementation_simdjson_result_base(error) + { + } + + simdjson_inline simdjson_result simdjson_result::key() noexcept { + if (error()) { return error(); } + return first.key(); + } + + simdjson_inline simdjson_result simdjson_result::key_raw_json_token() noexcept { + if (error()) { return error(); } + return first.key_raw_json_token(); + } + + simdjson_inline simdjson_result simdjson_result::escaped_key() noexcept { + if (error()) { return error(); } + return first.escaped_key(); + } + + simdjson_inline simdjson_result simdjson_result::unescaped_key(bool allow_replacement) noexcept { + if (error()) { return error(); } + return first.unescaped_key(allow_replacement); + } + + template + simdjson_inline error_code simdjson_result::unescaped_key(string_type& receiver, bool allow_replacement) noexcept { + if (error()) { return error(); } + return first.unescaped_key(receiver, allow_replacement); + } + + simdjson_inline simdjson_result simdjson_result::value() noexcept { + if (error()) { return error(); } + return std::move(first.value()); + } + +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_ONDEMAND_FIELD_INL_H +/* end file simdjson/generic/ondemand/field-inl.h for lasx */ +/* including simdjson/generic/ondemand/json_iterator-inl.h for lasx: #include "simdjson/generic/ondemand/json_iterator-inl.h" */ +/* begin file simdjson/generic/ondemand/json_iterator-inl.h for lasx */ +#ifndef SIMDJSON_GENERIC_ONDEMAND_JSON_ITERATOR_INL_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_ONDEMAND_JSON_ITERATOR_INL_H */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/dom_parser_implementation.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/json_iterator.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/parser.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/raw_json_string.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/logger-inl.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/parser-inl.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/token_iterator-inl.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lasx { + namespace ondemand { + + simdjson_inline json_iterator::json_iterator(json_iterator&& other) noexcept + : token(std::forward(other.token)), + parser{ other.parser }, + _string_buf_loc{ other._string_buf_loc }, + error{ other.error }, + _depth{ other._depth }, + _root{ other._root }, + _streaming{ other._streaming } + { + other.parser = nullptr; + } + simdjson_inline json_iterator& json_iterator::operator=(json_iterator&& other) noexcept { + token = other.token; + parser = other.parser; + _string_buf_loc = other._string_buf_loc; + error = other.error; + _depth = other._depth; + _root = other._root; + _streaming = other._streaming; + other.parser = nullptr; + return *this; + } + + simdjson_inline json_iterator::json_iterator(const uint8_t* buf, ondemand::parser* _parser) noexcept + : token(buf, &_parser->implementation->structural_indexes[0]), + parser{ _parser }, + _string_buf_loc{ parser->string_buf.get() }, + _depth{ 1 }, + _root{ parser->implementation->structural_indexes.get() }, + _streaming{ false } + + { + logger::log_headers(); +#if SIMDJSON_CHECK_EOF + assert_more_tokens(); +#endif + } + +#ifdef SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON + simdjson_inline json_iterator::json_iterator(const uint8_t* buf, ondemand::parser* _parser, bool streaming) noexcept + : token(buf, &_parser->implementation->structural_indexes[0]), + parser{ _parser }, + _string_buf_loc{ parser->string_buf.get() }, + _depth{ 1 }, + _root{ parser->implementation->structural_indexes.get() }, + _streaming{ streaming } + + { + logger::log_headers(); +#if SIMDJSON_CHECK_EOF + assert_more_tokens(); +#endif + } +#endif // SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON + + inline void json_iterator::rewind() noexcept { + token.set_position(root_position()); + logger::log_headers(); // We start again + _string_buf_loc = parser->string_buf.get(); + _depth = 1; + } + + inline bool json_iterator::balanced() const noexcept { + token_iterator ti(token); + int32_t count{ 0 }; + ti.set_position(root_position()); + while (ti.peek() <= peek_last()) { + switch (*ti.return_current_and_advance()) + { + case '[': case '{': + count++; + break; + case ']': case '}': + count--; + break; + default: + break; + } + } + return count == 0; + } + + + // GCC 7 warns when the first line of this function is inlined away into oblivion due to the caller + // relating depth and parent_depth, which is a desired effect. The warning does not show up if the + // skip_child() function is not marked inline). + SIMDJSON_PUSH_DISABLE_WARNINGS + SIMDJSON_DISABLE_STRICT_OVERFLOW_WARNING + simdjson_warn_unused simdjson_inline error_code json_iterator::skip_child(depth_t parent_depth) noexcept { + if (depth() <= parent_depth) { return SUCCESS; } + switch (*return_current_and_advance()) { + // TODO consider whether matching braces is a requirement: if non-matching braces indicates + // *missing* braces, then future lookups are not in the object/arrays they think they are, + // violating the rule "validate enough structure that the user can be confident they are + // looking at the right values." + // PERF TODO we can eliminate the switch here with a lookup of how much to add to depth + + // For the first open array/object in a value, we've already incremented depth, so keep it the same + // We never stop at colon, but if we did, it wouldn't affect depth + case '[': case '{': case ':': + logger::log_start_value(*this, "skip"); + break; + // If there is a comma, we have just finished a value in an array/object, and need to get back in + case ',': + logger::log_value(*this, "skip"); + break; + // ] or } means we just finished a value and need to jump out of the array/object + case ']': case '}': + logger::log_end_value(*this, "skip"); + _depth--; + if (depth() <= parent_depth) { return SUCCESS; } +#if SIMDJSON_CHECK_EOF + // If there are no more tokens, the parent is incomplete. + if (at_end()) { return report_error(INCOMPLETE_ARRAY_OR_OBJECT, "Missing [ or { at start"); } +#endif // SIMDJSON_CHECK_EOF + break; + case '"': + if (*peek() == ':') { + // We are at a key!!! + // This might happen if you just started an object and you skip it immediately. + // Performance note: it would be nice to get rid of this check as it is somewhat + // expensive. + // https://github.com/simdjson/simdjson/issues/1742 + logger::log_value(*this, "key"); + return_current_and_advance(); // eat up the ':' + break; // important!!! + } + simdjson_fallthrough; + // Anything else must be a scalar value + default: + // For the first scalar, we will have incremented depth already, so we decrement it here. + logger::log_value(*this, "skip"); + _depth--; + if (depth() <= parent_depth) { return SUCCESS; } + break; + } + + // Now that we've considered the first value, we only increment/decrement for arrays/objects + while (position() < end_position()) { + switch (*return_current_and_advance()) { + case '[': case '{': + logger::log_start_value(*this, "skip"); + _depth++; + break; + // TODO consider whether matching braces is a requirement: if non-matching braces indicates + // *missing* braces, then future lookups are not in the object/arrays they think they are, + // violating the rule "validate enough structure that the user can be confident they are + // looking at the right values." + // PERF TODO we can eliminate the switch here with a lookup of how much to add to depth + case ']': case '}': + logger::log_end_value(*this, "skip"); + _depth--; + if (depth() <= parent_depth) { return SUCCESS; } + break; + default: + logger::log_value(*this, "skip", ""); + break; + } + } + + return report_error(TAPE_ERROR, "not enough close braces"); + } + + SIMDJSON_POP_DISABLE_WARNINGS + + simdjson_inline bool json_iterator::at_root() const noexcept { + return position() == root_position(); + } + + simdjson_inline bool json_iterator::is_single_token() const noexcept { + return parser->implementation->n_structural_indexes == 1; + } + + simdjson_inline bool json_iterator::streaming() const noexcept { + return _streaming; + } + + simdjson_inline token_position json_iterator::root_position() const noexcept { + return _root; + } + + simdjson_inline void json_iterator::assert_at_document_depth() const noexcept { + SIMDJSON_ASSUME(_depth == 1); + } + + simdjson_inline void json_iterator::assert_at_root() const noexcept { + SIMDJSON_ASSUME(_depth == 1); +#ifndef SIMDJSON_CLANG_VISUAL_STUDIO + // Under Visual Studio, the next SIMDJSON_ASSUME fails with: the argument + // has side effects that will be discarded. + SIMDJSON_ASSUME(token.position() == _root); +#endif + } + + simdjson_inline void json_iterator::assert_more_tokens(uint32_t required_tokens) const noexcept { + assert_valid_position(token._position + required_tokens - 1); + } + + simdjson_inline void json_iterator::assert_valid_position(token_position position) const noexcept { +#ifndef SIMDJSON_CLANG_VISUAL_STUDIO + SIMDJSON_ASSUME(position >= &parser->implementation->structural_indexes[0]); + SIMDJSON_ASSUME(position < &parser->implementation->structural_indexes[parser->implementation->n_structural_indexes]); +#endif + } + + simdjson_inline bool json_iterator::at_end() const noexcept { + return position() == end_position(); + } + simdjson_inline token_position json_iterator::end_position() const noexcept { + uint32_t n_structural_indexes{ parser->implementation->n_structural_indexes }; + return &parser->implementation->structural_indexes[n_structural_indexes]; + } + + inline std::string json_iterator::to_string() const noexcept { + if (!is_alive()) { return "dead json_iterator instance"; } + const char* current_structural = reinterpret_cast(token.peek()); + return std::string("json_iterator [ depth : ") + std::to_string(_depth) + + std::string(", structural : '") + std::string(current_structural, 1) + + std::string("', offset : ") + std::to_string(token.current_offset()) + + std::string("', error : ") + error_message(error) + + std::string(" ]"); + } + + inline simdjson_result json_iterator::current_location() const noexcept { + if (!is_alive()) { // Unrecoverable error + if (!at_root()) { + return reinterpret_cast(token.peek(-1)); + } + else { + return reinterpret_cast(token.peek()); + } + } + if (at_end()) { + return OUT_OF_BOUNDS; + } + return reinterpret_cast(token.peek()); + } + + simdjson_inline bool json_iterator::is_alive() const noexcept { + return parser; + } + + simdjson_inline void json_iterator::abandon() noexcept { + parser = nullptr; + _depth = 0; + } + + simdjson_inline const uint8_t* json_iterator::return_current_and_advance() noexcept { +#if SIMDJSON_CHECK_EOF + assert_more_tokens(); +#endif // SIMDJSON_CHECK_EOF + return token.return_current_and_advance(); + } + + simdjson_inline const uint8_t* json_iterator::unsafe_pointer() const noexcept { + // deliberately done without safety guard: + return token.peek(); + } + + simdjson_inline const uint8_t* json_iterator::peek(int32_t delta) const noexcept { +#if SIMDJSON_CHECK_EOF + assert_more_tokens(delta + 1); +#endif // SIMDJSON_CHECK_EOF + return token.peek(delta); + } + + simdjson_inline uint32_t json_iterator::peek_length(int32_t delta) const noexcept { +#if SIMDJSON_CHECK_EOF + assert_more_tokens(delta + 1); +#endif // #if SIMDJSON_CHECK_EOF + return token.peek_length(delta); + } + + simdjson_inline const uint8_t* json_iterator::peek(token_position position) const noexcept { + // todo: currently we require end-of-string buffering, but the following + // assert_valid_position should be turned on if/when we lift that condition. + // assert_valid_position(position); + // This is almost surely related to SIMDJSON_CHECK_EOF but given that SIMDJSON_CHECK_EOF + // is ON by default, we have no choice but to disable it for real with a comment. + return token.peek(position); + } + + simdjson_inline uint32_t json_iterator::peek_length(token_position position) const noexcept { +#if SIMDJSON_CHECK_EOF + assert_valid_position(position); +#endif // SIMDJSON_CHECK_EOF + return token.peek_length(position); + } + simdjson_inline uint32_t json_iterator::peek_root_length(token_position position) const noexcept { +#if SIMDJSON_CHECK_EOF + assert_valid_position(position); +#endif // SIMDJSON_CHECK_EOF + return token.peek_root_length(position); + } + + simdjson_inline token_position json_iterator::last_position() const noexcept { + // The following line fails under some compilers... + // SIMDJSON_ASSUME(parser->implementation->n_structural_indexes > 0); + // since it has side-effects. + uint32_t n_structural_indexes{ parser->implementation->n_structural_indexes }; + SIMDJSON_ASSUME(n_structural_indexes > 0); + return &parser->implementation->structural_indexes[n_structural_indexes - 1]; + } + simdjson_inline const uint8_t* json_iterator::peek_last() const noexcept { + return token.peek(last_position()); + } + + simdjson_inline void json_iterator::ascend_to(depth_t parent_depth) noexcept { + SIMDJSON_ASSUME(parent_depth >= 0 && parent_depth < INT32_MAX - 1); + SIMDJSON_ASSUME(_depth == parent_depth + 1); + _depth = parent_depth; + } + + simdjson_inline void json_iterator::descend_to(depth_t child_depth) noexcept { + SIMDJSON_ASSUME(child_depth >= 1 && child_depth < INT32_MAX); + SIMDJSON_ASSUME(_depth == child_depth - 1); + _depth = child_depth; + } + + simdjson_inline depth_t json_iterator::depth() const noexcept { + return _depth; + } + + simdjson_inline uint8_t*& json_iterator::string_buf_loc() noexcept { + return _string_buf_loc; + } + + simdjson_inline error_code json_iterator::report_error(error_code _error, const char* message) noexcept { + SIMDJSON_ASSUME(_error != SUCCESS && _error != UNINITIALIZED && _error != INCORRECT_TYPE && _error != NO_SUCH_FIELD); + logger::log_error(*this, message); + error = _error; + return error; + } + + simdjson_inline token_position json_iterator::position() const noexcept { + return token.position(); + } + + simdjson_inline simdjson_result json_iterator::unescape(raw_json_string in, bool allow_replacement) noexcept { +#if SIMDJSON_DEVELOPMENT_CHECKS + auto result = parser->unescape(in, _string_buf_loc, allow_replacement); + SIMDJSON_ASSUME(!parser->string_buffer_overflow(_string_buf_loc)); + return result; +#else + return parser->unescape(in, _string_buf_loc, allow_replacement); +#endif + } + + simdjson_inline simdjson_result json_iterator::unescape_wobbly(raw_json_string in) noexcept { +#if SIMDJSON_DEVELOPMENT_CHECKS + auto result = parser->unescape_wobbly(in, _string_buf_loc); + SIMDJSON_ASSUME(!parser->string_buffer_overflow(_string_buf_loc)); + return result; +#else + return parser->unescape_wobbly(in, _string_buf_loc); +#endif + } + + simdjson_inline void json_iterator::reenter_child(token_position position, depth_t child_depth) noexcept { + SIMDJSON_ASSUME(child_depth >= 1 && child_depth < INT32_MAX); + SIMDJSON_ASSUME(_depth == child_depth - 1); +#if SIMDJSON_DEVELOPMENT_CHECKS +#ifndef SIMDJSON_CLANG_VISUAL_STUDIO + SIMDJSON_ASSUME(size_t(child_depth) < parser->max_depth()); + SIMDJSON_ASSUME(position >= parser->start_positions[child_depth]); +#endif +#endif + token.set_position(position); + _depth = child_depth; + } + + simdjson_inline error_code json_iterator::consume_character(char c) noexcept { + if (*peek() == c) { + return_current_and_advance(); + return SUCCESS; + } + return TAPE_ERROR; + } + +#if SIMDJSON_DEVELOPMENT_CHECKS + + simdjson_inline token_position json_iterator::start_position(depth_t depth) const noexcept { + SIMDJSON_ASSUME(size_t(depth) < parser->max_depth()); + return size_t(depth) < parser->max_depth() ? parser->start_positions[depth] : 0; + } + + simdjson_inline void json_iterator::set_start_position(depth_t depth, token_position position) noexcept { + SIMDJSON_ASSUME(size_t(depth) < parser->max_depth()); + if (size_t(depth) < parser->max_depth()) { parser->start_positions[depth] = position; } + } + +#endif + + + simdjson_inline error_code json_iterator::optional_error(error_code _error, const char* message) noexcept { + SIMDJSON_ASSUME(_error == INCORRECT_TYPE || _error == NO_SUCH_FIELD); + logger::log_error(*this, message); + return _error; + } + + + simdjson_warn_unused simdjson_inline bool json_iterator::copy_to_buffer(const uint8_t* json, uint32_t max_len, uint8_t* tmpbuf, size_t N) noexcept { + // This function is not expected to be called in performance-sensitive settings. + // Let us guard against silly cases: + if ((N < max_len) || (N == 0)) { return false; } + // Copy to the buffer. + std::memcpy(tmpbuf, json, max_len); + if (N > max_len) { // We pad whatever remains with ' '. + std::memset(tmpbuf + max_len, ' ', N - max_len); + } + return true; + } + + } // namespace ondemand + } // namespace lasx +} // namespace simdjson + +namespace simdjson { + + simdjson_inline simdjson_result::simdjson_result(lasx::ondemand::json_iterator&& value) noexcept + : implementation_simdjson_result_base(std::forward(value)) {} + simdjson_inline simdjson_result::simdjson_result(error_code error) noexcept + : implementation_simdjson_result_base(error) {} + +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_ONDEMAND_JSON_ITERATOR_INL_H +/* end file simdjson/generic/ondemand/json_iterator-inl.h for lasx */ +/* including simdjson/generic/ondemand/json_type-inl.h for lasx: #include "simdjson/generic/ondemand/json_type-inl.h" */ +/* begin file simdjson/generic/ondemand/json_type-inl.h for lasx */ +#ifndef SIMDJSON_GENERIC_ONDEMAND_JSON_TYPE_INL_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_ONDEMAND_JSON_TYPE_INL_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/json_type.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/implementation_simdjson_result_base-inl.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lasx { + namespace ondemand { + + inline std::ostream& operator<<(std::ostream& out, json_type type) noexcept { + switch (type) { + case json_type::array: out << "array"; break; + case json_type::object: out << "object"; break; + case json_type::number: out << "number"; break; + case json_type::string: out << "string"; break; + case json_type::boolean: out << "boolean"; break; + case json_type::null: out << "null"; break; + default: SIMDJSON_UNREACHABLE(); + } + return out; + } + +#if SIMDJSON_EXCEPTIONS + inline std::ostream& operator<<(std::ostream& out, simdjson_result& type) noexcept(false) { + return out << type.value(); + } +#endif + + + + simdjson_inline number_type number::get_number_type() const noexcept { + return type; + } + + simdjson_inline bool number::is_uint64() const noexcept { + return get_number_type() == number_type::unsigned_integer; + } + + simdjson_inline uint64_t number::get_uint64() const noexcept { + return payload.unsigned_integer; + } + + simdjson_inline number::operator uint64_t() const noexcept { + return get_uint64(); + } + + simdjson_inline bool number::is_int64() const noexcept { + return get_number_type() == number_type::signed_integer; + } + + simdjson_inline int64_t number::get_int64() const noexcept { + return payload.signed_integer; + } + + simdjson_inline number::operator int64_t() const noexcept { + return get_int64(); + } + + simdjson_inline bool number::is_double() const noexcept { + return get_number_type() == number_type::floating_point_number; + } + + simdjson_inline double number::get_double() const noexcept { + return payload.floating_point_number; + } + + simdjson_inline number::operator double() const noexcept { + return get_double(); + } + + simdjson_inline double number::as_double() const noexcept { + if (is_double()) { + return payload.floating_point_number; + } + if (is_int64()) { + return double(payload.signed_integer); + } + return double(payload.unsigned_integer); + } + + simdjson_inline void number::append_s64(int64_t value) noexcept { + payload.signed_integer = value; + type = number_type::signed_integer; + } + + simdjson_inline void number::append_u64(uint64_t value) noexcept { + payload.unsigned_integer = value; + type = number_type::unsigned_integer; + } + + simdjson_inline void number::append_double(double value) noexcept { + payload.floating_point_number = value; + type = number_type::floating_point_number; + } + + simdjson_inline void number::skip_double() noexcept { + type = number_type::floating_point_number; + } + + } // namespace ondemand + } // namespace lasx +} // namespace simdjson + +namespace simdjson { + + simdjson_inline simdjson_result::simdjson_result(lasx::ondemand::json_type&& value) noexcept + : implementation_simdjson_result_base(std::forward(value)) {} + simdjson_inline simdjson_result::simdjson_result(error_code error) noexcept + : implementation_simdjson_result_base(error) {} + +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_ONDEMAND_JSON_TYPE_INL_H +/* end file simdjson/generic/ondemand/json_type-inl.h for lasx */ +/* including simdjson/generic/ondemand/logger-inl.h for lasx: #include "simdjson/generic/ondemand/logger-inl.h" */ +/* begin file simdjson/generic/ondemand/logger-inl.h for lasx */ +#ifndef SIMDJSON_GENERIC_ONDEMAND_LOGGER_INL_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_ONDEMAND_LOGGER_INL_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/logger.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/json_iterator.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/value_iterator.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +#include +#include + +namespace simdjson { + namespace lasx { + namespace ondemand { + namespace logger { + + static constexpr const char* DASHES = "----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------"; + static constexpr const int LOG_EVENT_LEN = 20; + static constexpr const int LOG_BUFFER_LEN = 30; + static constexpr const int LOG_SMALL_BUFFER_LEN = 10; + static int log_depth = 0; // Not threadsafe. Log only. + + // Helper to turn unprintable or newline characters into spaces + static inline char printable_char(char c) { + if (c >= 0x20) { + return c; + } + else { + return ' '; + } + } + + template + static inline std::string string_format(const std::string& format, const Args&... args) + { + SIMDJSON_PUSH_DISABLE_ALL_WARNINGS + int size_s = std::snprintf(nullptr, 0, format.c_str(), args...) + 1; + auto size = static_cast(size_s); + if (size <= 0) return std::string(); + std::unique_ptr buf(new char[size]); + std::snprintf(buf.get(), size, format.c_str(), args...); + SIMDJSON_POP_DISABLE_WARNINGS + return std::string(buf.get(), buf.get() + size - 1); + } + + static inline log_level get_log_level_from_env() + { + SIMDJSON_PUSH_DISABLE_WARNINGS + SIMDJSON_DISABLE_DEPRECATED_WARNING // Disable CRT_SECURE warning on MSVC: manually verified this is safe + char* lvl = getenv("SIMDJSON_LOG_LEVEL"); + SIMDJSON_POP_DISABLE_WARNINGS + if (lvl && simdjson_strcasecmp(lvl, "ERROR") == 0) { return log_level::error; } + return log_level::info; + } + + static inline log_level log_threshold() + { + static log_level threshold = get_log_level_from_env(); + return threshold; + } + + static inline bool should_log(log_level level) + { + return level >= log_threshold(); + } + + inline void log_event(const json_iterator& iter, const char* type, std::string_view detail, int delta, int depth_delta) noexcept { + log_line(iter, "", type, detail, delta, depth_delta, log_level::info); + } + + inline void log_value(const json_iterator& iter, token_position index, depth_t depth, const char* type, std::string_view detail) noexcept { + log_line(iter, index, depth, "", type, detail, log_level::info); + } + inline void log_value(const json_iterator& iter, const char* type, std::string_view detail, int delta, int depth_delta) noexcept { + log_line(iter, "", type, detail, delta, depth_delta, log_level::info); + } + + inline void log_start_value(const json_iterator& iter, token_position index, depth_t depth, const char* type, std::string_view detail) noexcept { + log_line(iter, index, depth, "+", type, detail, log_level::info); + if (LOG_ENABLED) { log_depth++; } + } + inline void log_start_value(const json_iterator& iter, const char* type, int delta, int depth_delta) noexcept { + log_line(iter, "+", type, "", delta, depth_delta, log_level::info); + if (LOG_ENABLED) { log_depth++; } + } + + inline void log_end_value(const json_iterator& iter, const char* type, int delta, int depth_delta) noexcept { + if (LOG_ENABLED) { log_depth--; } + log_line(iter, "-", type, "", delta, depth_delta, log_level::info); + } + + inline void log_error(const json_iterator& iter, const char* error, const char* detail, int delta, int depth_delta) noexcept { + log_line(iter, "ERROR: ", error, detail, delta, depth_delta, log_level::error); + } + inline void log_error(const json_iterator& iter, token_position index, depth_t depth, const char* error, const char* detail) noexcept { + log_line(iter, index, depth, "ERROR: ", error, detail, log_level::error); + } + + inline void log_event(const value_iterator& iter, const char* type, std::string_view detail, int delta, int depth_delta) noexcept { + log_event(iter.json_iter(), type, detail, delta, depth_delta); + } + + inline void log_value(const value_iterator& iter, const char* type, std::string_view detail, int delta, int depth_delta) noexcept { + log_value(iter.json_iter(), type, detail, delta, depth_delta); + } + + inline void log_start_value(const value_iterator& iter, const char* type, int delta, int depth_delta) noexcept { + log_start_value(iter.json_iter(), type, delta, depth_delta); + } + + inline void log_end_value(const value_iterator& iter, const char* type, int delta, int depth_delta) noexcept { + log_end_value(iter.json_iter(), type, delta, depth_delta); + } + + inline void log_error(const value_iterator& iter, const char* error, const char* detail, int delta, int depth_delta) noexcept { + log_error(iter.json_iter(), error, detail, delta, depth_delta); + } + + inline void log_headers() noexcept { + if (LOG_ENABLED) { + if (simdjson_unlikely(should_log(log_level::info))) { + // Technically a static variable is not thread-safe, but if you are using threads and logging... well... + static bool displayed_hint{ false }; + log_depth = 0; + printf("\n"); + if (!displayed_hint) { + // We only print this helpful header once. + printf("# Logging provides the depth and position of the iterator user-visible steps:\n"); + printf("# +array says 'this is where we were when we discovered the start array'\n"); + printf( + "# -array says 'this is where we were when we ended the array'\n"); + printf("# skip says 'this is a structural or value I am skipping'\n"); + printf("# +/-skip says 'this is a start/end array or object I am skipping'\n"); + printf("#\n"); + printf("# The indentation of the terms (array, string,...) indicates the depth,\n"); + printf("# in addition to the depth being displayed.\n"); + printf("#\n"); + printf("# Every token in the document has a single depth determined by the tokens before it,\n"); + printf("# and is not affected by what the token actually is.\n"); + printf("#\n"); + printf("# Not all structural elements are presented as tokens in the logs.\n"); + printf("#\n"); + printf("# We never give control to the user within an empty array or an empty object.\n"); + printf("#\n"); + printf("# Inside an array, having a depth greater than the array's depth means that\n"); + printf("# we are pointing inside a value.\n"); + printf("# Having a depth equal to the array means that we are pointing right before a value.\n"); + printf("# Having a depth smaller than the array means that we have moved beyond the array.\n"); + displayed_hint = true; + } + printf("\n"); + printf("| %-*s ", LOG_EVENT_LEN, "Event"); + printf("| %-*s ", LOG_BUFFER_LEN, "Buffer"); + printf("| %-*s ", LOG_SMALL_BUFFER_LEN, "Next"); + // printf("| %-*s ", 5, "Next#"); + printf("| %-*s ", 5, "Depth"); + printf("| Detail "); + printf("|\n"); + + printf("|%.*s", LOG_EVENT_LEN + 2, DASHES); + printf("|%.*s", LOG_BUFFER_LEN + 2, DASHES); + printf("|%.*s", LOG_SMALL_BUFFER_LEN + 2, DASHES); + // printf("|%.*s", 5+2, DASHES); + printf("|%.*s", 5 + 2, DASHES); + printf("|--------"); + printf("|\n"); + fflush(stdout); + } + } + } + + template + inline void log_line(const json_iterator& iter, const char* title_prefix, const char* title, std::string_view detail, int delta, int depth_delta, log_level level, Args&&... args) noexcept { + log_line(iter, iter.position() + delta, depth_t(iter.depth() + depth_delta), title_prefix, title, detail, level, std::forward(args)...); + } + + template + inline void log_line(const json_iterator& iter, token_position index, depth_t depth, const char* title_prefix, const char* title, std::string_view detail, log_level level, Args&&... args) noexcept { + if (LOG_ENABLED) { + if (simdjson_unlikely(should_log(level))) { + const int indent = depth * 2; + const auto buf = iter.token.buf; + auto msg = string_format(title, std::forward(args)...); + printf("| %*s%s%-*s ", indent, "", title_prefix, + LOG_EVENT_LEN - indent - int(strlen(title_prefix)), msg.c_str()); + { + // Print the current structural. + printf("| "); + // Before we begin, the index might point right before the document. + // This could be unsafe, see https://github.com/simdjson/simdjson/discussions/1938 + if (index < iter._root) { + printf("%*s", LOG_BUFFER_LEN, ""); + } + else { + auto current_structural = &buf[*index]; + for (int i = 0; i < LOG_BUFFER_LEN; i++) { + printf("%c", printable_char(current_structural[i])); + } + } + printf(" "); + } + { + // Print the next structural. + printf("| "); + auto next_structural = &buf[*(index + 1)]; + for (int i = 0; i < LOG_SMALL_BUFFER_LEN; i++) { + printf("%c", printable_char(next_structural[i])); + } + printf(" "); + } + // printf("| %5u ", *(index+1)); + printf("| %5i ", depth); + printf("| %6.*s ", int(detail.size()), detail.data()); + printf("|\n"); + fflush(stdout); + } + } + } + + } // namespace logger + } // namespace ondemand + } // namespace lasx +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_ONDEMAND_LOGGER_INL_H +/* end file simdjson/generic/ondemand/logger-inl.h for lasx */ +/* including simdjson/generic/ondemand/object-inl.h for lasx: #include "simdjson/generic/ondemand/object-inl.h" */ +/* begin file simdjson/generic/ondemand/object-inl.h for lasx */ +#ifndef SIMDJSON_GENERIC_ONDEMAND_OBJECT_INL_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_ONDEMAND_OBJECT_INL_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/field.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/object.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/object_iterator.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/raw_json_string.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/json_iterator.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/value-inl.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lasx { + namespace ondemand { + + simdjson_inline simdjson_result object::find_field_unordered(const std::string_view key) & noexcept { + bool has_value; + SIMDJSON_TRY(iter.find_field_unordered_raw(key).get(has_value)); + if (!has_value) { + logger::log_line(iter.json_iter(), "ERROR: ", "Cannot find key %.*s", "", -1, 0, logger::log_level::error, static_cast(key.size()), key.data()); + return NO_SUCH_FIELD; + } + return value(iter.child()); + } + simdjson_inline simdjson_result object::find_field_unordered(const std::string_view key) && noexcept { + bool has_value; + SIMDJSON_TRY(iter.find_field_unordered_raw(key).get(has_value)); + if (!has_value) { + logger::log_line(iter.json_iter(), "ERROR: ", "Cannot find key %.*s", "", -1, 0, logger::log_level::error, static_cast(key.size()), key.data()); + return NO_SUCH_FIELD; + } + return value(iter.child()); + } + simdjson_inline simdjson_result object::operator[](const std::string_view key) & noexcept { + return find_field_unordered(key); + } + simdjson_inline simdjson_result object::operator[](const std::string_view key) && noexcept { + return std::forward(*this).find_field_unordered(key); + } + simdjson_inline simdjson_result object::find_field(const std::string_view key) & noexcept { + bool has_value; + SIMDJSON_TRY(iter.find_field_raw(key).get(has_value)); + if (!has_value) { + logger::log_line(iter.json_iter(), "ERROR: ", "Cannot find key %.*s", "", -1, 0, logger::log_level::error, static_cast(key.size()), key.data()); + return NO_SUCH_FIELD; + } + return value(iter.child()); + } + simdjson_inline simdjson_result object::find_field(const std::string_view key) && noexcept { + bool has_value; + SIMDJSON_TRY(iter.find_field_raw(key).get(has_value)); + if (!has_value) { + logger::log_line(iter.json_iter(), "ERROR: ", "Cannot find key %.*s", "", -1, 0, logger::log_level::error, static_cast(key.size()), key.data()); + return NO_SUCH_FIELD; + } + return value(iter.child()); + } + + simdjson_inline simdjson_result object::start(value_iterator& iter) noexcept { + SIMDJSON_TRY(iter.start_object().error()); + return object(iter); + } + simdjson_inline simdjson_result object::start_root(value_iterator& iter) noexcept { + SIMDJSON_TRY(iter.start_root_object().error()); + return object(iter); + } + simdjson_inline error_code object::consume() noexcept { + if (iter.is_at_key()) { + /** + * whenever you are pointing at a key, calling skip_child() is + * unsafe because you will hit a string and you will assume that + * it is string value, and this mistake will lead you to make bad + * depth computation. + */ + /** + * We want to 'consume' the key. We could really + * just do _json_iter->return_current_and_advance(); at this + * point, but, for clarity, we will use the high-level API to + * eat the key. We assume that the compiler optimizes away + * most of the work. + */ + simdjson_unused raw_json_string actual_key; + auto error = iter.field_key().get(actual_key); + if (error) { iter.abandon(); return error; }; + // Let us move to the value while we are at it. + if ((error = iter.field_value())) { iter.abandon(); return error; } + } + auto error_skip = iter.json_iter().skip_child(iter.depth() - 1); + if (error_skip) { iter.abandon(); } + return error_skip; + } + + simdjson_inline simdjson_result object::raw_json() noexcept { + const uint8_t* starting_point{ iter.peek_start() }; + auto error = consume(); + if (error) { return error; } + const uint8_t* final_point{ iter._json_iter->peek() }; + return std::string_view(reinterpret_cast(starting_point), size_t(final_point - starting_point)); + } + + simdjson_inline simdjson_result object::started(value_iterator& iter) noexcept { + SIMDJSON_TRY(iter.started_object().error()); + return object(iter); + } + + simdjson_inline object object::resume(const value_iterator& iter) noexcept { + return iter; + } + + simdjson_inline object::object(const value_iterator& _iter) noexcept + : iter{ _iter } + { + } + + simdjson_inline simdjson_result object::begin() noexcept { +#if SIMDJSON_DEVELOPMENT_CHECKS + if (!iter.is_at_iterator_start()) { return OUT_OF_ORDER_ITERATION; } +#endif + return object_iterator(iter); + } + simdjson_inline simdjson_result object::end() noexcept { + return object_iterator(iter); + } + + inline simdjson_result object::at_pointer(std::string_view json_pointer) noexcept { + if (json_pointer[0] != '/') { return INVALID_JSON_POINTER; } + json_pointer = json_pointer.substr(1); + size_t slash = json_pointer.find('/'); + std::string_view key = json_pointer.substr(0, slash); + // Grab the child with the given key + simdjson_result child; + + // If there is an escape character in the key, unescape it and then get the child. + size_t escape = key.find('~'); + if (escape != std::string_view::npos) { + // Unescape the key + std::string unescaped(key); + do { + switch (unescaped[escape + 1]) { + case '0': + unescaped.replace(escape, 2, "~"); + break; + case '1': + unescaped.replace(escape, 2, "/"); + break; + default: + return INVALID_JSON_POINTER; // "Unexpected ~ escape character in JSON pointer"); + } + escape = unescaped.find('~', escape + 1); + } while (escape != std::string::npos); + child = find_field(unescaped); // Take note find_field does not unescape keys when matching + } + else { + child = find_field(key); + } + if (child.error()) { + return child; // we do not continue if there was an error + } + // If there is a /, we have to recurse and look up more of the path + if (slash != std::string_view::npos) { + child = child.at_pointer(json_pointer.substr(slash)); + } + return child; + } + + inline simdjson_result object::at_path(std::string_view json_path) noexcept { + auto json_pointer = json_path_to_pointer_conversion(json_path); + if (json_pointer == "-1") { + return INVALID_JSON_POINTER; + } + return at_pointer(json_pointer); + } + + simdjson_inline simdjson_result object::count_fields() & noexcept { + size_t count{ 0 }; + // Important: we do not consume any of the values. + for (simdjson_unused auto v : *this) { count++; } + // The above loop will always succeed, but we want to report errors. + if (iter.error()) { return iter.error(); } + // We need to move back at the start because we expect users to iterate through + // the object after counting the number of elements. + iter.reset_object(); + return count; + } + + simdjson_inline simdjson_result object::is_empty() & noexcept { + bool is_not_empty; + auto error = iter.reset_object().get(is_not_empty); + if (error) { return error; } + return !is_not_empty; + } + + simdjson_inline simdjson_result object::reset() & noexcept { + return iter.reset_object(); + } + + } // namespace ondemand + } // namespace lasx +} // namespace simdjson + +namespace simdjson { + + simdjson_inline simdjson_result::simdjson_result(lasx::ondemand::object&& value) noexcept + : implementation_simdjson_result_base(std::forward(value)) {} + simdjson_inline simdjson_result::simdjson_result(error_code error) noexcept + : implementation_simdjson_result_base(error) {} + + simdjson_inline simdjson_result simdjson_result::begin() noexcept { + if (error()) { return error(); } + return first.begin(); + } + simdjson_inline simdjson_result simdjson_result::end() noexcept { + if (error()) { return error(); } + return first.end(); + } + simdjson_inline simdjson_result simdjson_result::find_field_unordered(std::string_view key) & noexcept { + if (error()) { return error(); } + return first.find_field_unordered(key); + } + simdjson_inline simdjson_result simdjson_result::find_field_unordered(std::string_view key) && noexcept { + if (error()) { return error(); } + return std::forward(first).find_field_unordered(key); + } + simdjson_inline simdjson_result simdjson_result::operator[](std::string_view key) & noexcept { + if (error()) { return error(); } + return first[key]; + } + simdjson_inline simdjson_result simdjson_result::operator[](std::string_view key) && noexcept { + if (error()) { return error(); } + return std::forward(first)[key]; + } + simdjson_inline simdjson_result simdjson_result::find_field(std::string_view key) & noexcept { + if (error()) { return error(); } + return first.find_field(key); + } + simdjson_inline simdjson_result simdjson_result::find_field(std::string_view key) && noexcept { + if (error()) { return error(); } + return std::forward(first).find_field(key); + } + + simdjson_inline simdjson_result simdjson_result::at_pointer(std::string_view json_pointer) noexcept { + if (error()) { return error(); } + return first.at_pointer(json_pointer); + } + + simdjson_inline simdjson_result simdjson_result::at_path( + std::string_view json_path) noexcept { + if (error()) { + return error(); + } + return first.at_path(json_path); + } + + inline simdjson_result simdjson_result::reset() noexcept { + if (error()) { return error(); } + return first.reset(); + } + + inline simdjson_result simdjson_result::is_empty() noexcept { + if (error()) { return error(); } + return first.is_empty(); + } + + simdjson_inline simdjson_result simdjson_result::count_fields() & noexcept { + if (error()) { return error(); } + return first.count_fields(); + } + + simdjson_inline simdjson_result simdjson_result::raw_json() noexcept { + if (error()) { return error(); } + return first.raw_json(); + } +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_ONDEMAND_OBJECT_INL_H +/* end file simdjson/generic/ondemand/object-inl.h for lasx */ +/* including simdjson/generic/ondemand/object_iterator-inl.h for lasx: #include "simdjson/generic/ondemand/object_iterator-inl.h" */ +/* begin file simdjson/generic/ondemand/object_iterator-inl.h for lasx */ +#ifndef SIMDJSON_GENERIC_ONDEMAND_OBJECT_ITERATOR_INL_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_ONDEMAND_OBJECT_ITERATOR_INL_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/object_iterator.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/field-inl.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/value_iterator-inl.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lasx { + namespace ondemand { + + // + // object_iterator + // + + simdjson_inline object_iterator::object_iterator(const value_iterator& _iter) noexcept + : iter{ _iter } + {} + + simdjson_inline simdjson_result object_iterator::operator*() noexcept { + error_code error = iter.error(); + if (error) { iter.abandon(); return error; } + auto result = field::start(iter); + // TODO this is a safety rail ... users should exit loops as soon as they receive an error. + // Nonetheless, let's see if performance is OK with this if statement--the compiler may give it to us for free. + if (result.error()) { iter.abandon(); } + return result; + } + simdjson_inline bool object_iterator::operator==(const object_iterator& other) const noexcept { + return !(*this != other); + } + simdjson_inline bool object_iterator::operator!=(const object_iterator&) const noexcept { + return iter.is_open(); + } + + SIMDJSON_PUSH_DISABLE_WARNINGS + SIMDJSON_DISABLE_STRICT_OVERFLOW_WARNING + simdjson_inline object_iterator& object_iterator::operator++() noexcept { + // TODO this is a safety rail ... users should exit loops as soon as they receive an error. + // Nonetheless, let's see if performance is OK with this if statement--the compiler may give it to us for free. + if (!iter.is_open()) { return *this; } // Iterator will be released if there is an error + + simdjson_unused error_code error; + if ((error = iter.skip_child())) { return *this; } + + simdjson_unused bool has_value; + if ((error = iter.has_next_field().get(has_value))) { return *this; }; + return *this; + } + SIMDJSON_POP_DISABLE_WARNINGS + + // + // ### Live States + // + // While iterating or looking up values, depth >= iter.depth. at_start may vary. Error is + // always SUCCESS: + // + // - Start: This is the state when the object is first found and the iterator is just past the {. + // In this state, at_start == true. + // - Next: After we hand a scalar value to the user, or an array/object which they then fully + // iterate over, the iterator is at the , or } before the next value. In this state, + // depth == iter.depth, at_start == false, and error == SUCCESS. + // - Unfinished Business: When we hand an array/object to the user which they do not fully + // iterate over, we need to finish that iteration by skipping child values until we reach the + // Next state. In this state, depth > iter.depth, at_start == false, and error == SUCCESS. + // + // ## Error States + // + // In error states, we will yield exactly one more value before stopping. iter.depth == depth + // and at_start is always false. We decrement after yielding the error, moving to the Finished + // state. + // + // - Chained Error: When the object iterator is part of an error chain--for example, in + // `for (auto tweet : doc["tweets"])`, where the tweet field may be missing or not be an + // object--we yield that error in the loop, exactly once. In this state, error != SUCCESS and + // iter.depth == depth, and at_start == false. We decrement depth when we yield the error. + // - Missing Comma Error: When the iterator ++ method discovers there is no comma between fields, + // we flag that as an error and treat it exactly the same as a Chained Error. In this state, + // error == TAPE_ERROR, iter.depth == depth, and at_start == false. + // + // Errors that occur while reading a field to give to the user (such as when the key is not a + // string or the field is missing a colon) are yielded immediately. Depth is then decremented, + // moving to the Finished state without transitioning through an Error state at all. + // + // ## Terminal State + // + // The terminal state has iter.depth < depth. at_start is always false. + // + // - Finished: When we have reached a }, we are finished. We signal this by decrementing depth. + // In this state, iter.depth < depth, at_start == false, and error == SUCCESS. + // + + } // namespace ondemand + } // namespace lasx +} // namespace simdjson + +namespace simdjson { + + simdjson_inline simdjson_result::simdjson_result( + lasx::ondemand::object_iterator&& value + ) noexcept + : implementation_simdjson_result_base(std::forward(value)) + { + first.iter.assert_is_valid(); + } + simdjson_inline simdjson_result::simdjson_result(error_code error) noexcept + : implementation_simdjson_result_base({}, error) + { + } + + simdjson_inline simdjson_result simdjson_result::operator*() noexcept { + if (error()) { return error(); } + return *first; + } + // If we're iterating and there is an error, return the error once. + simdjson_inline bool simdjson_result::operator==(const simdjson_result& other) const noexcept { + if (!first.iter.is_valid()) { return !error(); } + return first == other.first; + } + // If we're iterating and there is an error, return the error once. + simdjson_inline bool simdjson_result::operator!=(const simdjson_result& other) const noexcept { + if (!first.iter.is_valid()) { return error(); } + return first != other.first; + } + // Checks for ']' and ',' + simdjson_inline simdjson_result& simdjson_result::operator++() noexcept { + // Clear the error if there is one, so we don't yield it twice + if (error()) { second = SUCCESS; return *this; } + ++first; + return *this; + } + +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_ONDEMAND_OBJECT_ITERATOR_INL_H +/* end file simdjson/generic/ondemand/object_iterator-inl.h for lasx */ +/* including simdjson/generic/ondemand/parser-inl.h for lasx: #include "simdjson/generic/ondemand/parser-inl.h" */ +/* begin file simdjson/generic/ondemand/parser-inl.h for lasx */ +#ifndef SIMDJSON_GENERIC_ONDEMAND_PARSER_INL_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_ONDEMAND_PARSER_INL_H */ +/* amalgamation skipped (editor-only): #include "simdjson/padded_string.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/padded_string_view.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/implementation.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/internal/dom_parser_implementation.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/dom/base.h" // for MINIMAL_DOCUMENT_CAPACITY */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/document_stream.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/parser.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/raw_json_string.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lasx { + namespace ondemand { + + simdjson_inline parser::parser(size_t max_capacity) noexcept + : _max_capacity{ max_capacity } { + } + + simdjson_warn_unused simdjson_inline error_code parser::allocate(size_t new_capacity, size_t new_max_depth) noexcept { + if (new_capacity > max_capacity()) { return CAPACITY; } + if (string_buf && new_capacity == capacity() && new_max_depth == max_depth()) { return SUCCESS; } + + // string_capacity copied from document::allocate + _capacity = 0; + size_t string_capacity = SIMDJSON_ROUNDUP_N(5 * new_capacity / 3 + SIMDJSON_PADDING, 64); + string_buf.reset(new (std::nothrow) uint8_t[string_capacity]); +#if SIMDJSON_DEVELOPMENT_CHECKS + start_positions.reset(new (std::nothrow) token_position[new_max_depth]); +#endif + if (implementation) { + SIMDJSON_TRY(implementation->set_capacity(new_capacity)); + SIMDJSON_TRY(implementation->set_max_depth(new_max_depth)); + } + else { + SIMDJSON_TRY(simdjson::get_active_implementation()->create_dom_parser_implementation(new_capacity, new_max_depth, implementation)); + } + _capacity = new_capacity; + _max_depth = new_max_depth; + return SUCCESS; + } +#if SIMDJSON_DEVELOPMENT_CHECKS + simdjson_inline simdjson_warn_unused bool parser::string_buffer_overflow(const uint8_t* string_buf_loc) const noexcept { + return (string_buf_loc < string_buf.get()) || (size_t(string_buf_loc - string_buf.get()) >= capacity()); + } +#endif + + simdjson_warn_unused simdjson_inline simdjson_result parser::iterate(padded_string_view json) & noexcept { + if (json.padding() < SIMDJSON_PADDING) { return INSUFFICIENT_PADDING; } + + json.remove_utf8_bom(); + + // Allocate if needed + if (capacity() < json.length() || !string_buf) { + SIMDJSON_TRY(allocate(json.length(), max_depth())); + } + + // Run stage 1. + SIMDJSON_TRY(implementation->stage1(reinterpret_cast(json.data()), json.length(), stage1_mode::regular)); + return document::start({ reinterpret_cast(json.data()), this }); + } + +#ifdef SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON + simdjson_warn_unused simdjson_inline simdjson_result parser::iterate_allow_incomplete_json(padded_string_view json) & noexcept { + if (json.padding() < SIMDJSON_PADDING) { return INSUFFICIENT_PADDING; } + + json.remove_utf8_bom(); + + // Allocate if needed + if (capacity() < json.length() || !string_buf) { + SIMDJSON_TRY(allocate(json.length(), max_depth())); + } + + // Run stage 1. + const simdjson::error_code err = implementation->stage1(reinterpret_cast(json.data()), json.length(), stage1_mode::regular); + if (err) { + if (err != UNCLOSED_STRING) + return err; + } + return document::start({ reinterpret_cast(json.data()), this, true }); + } +#endif // SIMDJSON_EXPERIMENTAL_ALLOW_INCOMPLETE_JSON + + simdjson_warn_unused simdjson_inline simdjson_result parser::iterate(const char* json, size_t len, size_t allocated) & noexcept { + return iterate(padded_string_view(json, len, allocated)); + } + + simdjson_warn_unused simdjson_inline simdjson_result parser::iterate(const uint8_t* json, size_t len, size_t allocated) & noexcept { + return iterate(padded_string_view(json, len, allocated)); + } + + simdjson_warn_unused simdjson_inline simdjson_result parser::iterate(std::string_view json, size_t allocated) & noexcept { + return iterate(padded_string_view(json, allocated)); + } + + simdjson_warn_unused simdjson_inline simdjson_result parser::iterate(std::string& json) & noexcept { + if (json.capacity() - json.size() < SIMDJSON_PADDING) { + json.reserve(json.size() + SIMDJSON_PADDING); + } + return iterate(padded_string_view(json)); + } + + simdjson_warn_unused simdjson_inline simdjson_result parser::iterate(const std::string& json) & noexcept { + return iterate(padded_string_view(json)); + } + + simdjson_warn_unused simdjson_inline simdjson_result parser::iterate(const simdjson_result& result) & noexcept { + // We don't presently have a way to temporarily get a const T& from a simdjson_result without throwing an exception + SIMDJSON_TRY(result.error()); + padded_string_view json = result.value_unsafe(); + return iterate(json); + } + + simdjson_warn_unused simdjson_inline simdjson_result parser::iterate(const simdjson_result& result) & noexcept { + // We don't presently have a way to temporarily get a const T& from a simdjson_result without throwing an exception + SIMDJSON_TRY(result.error()); + const padded_string& json = result.value_unsafe(); + return iterate(json); + } + + simdjson_warn_unused simdjson_inline simdjson_result parser::iterate_raw(padded_string_view json) & noexcept { + if (json.padding() < SIMDJSON_PADDING) { return INSUFFICIENT_PADDING; } + + json.remove_utf8_bom(); + + // Allocate if needed + if (capacity() < json.length()) { + SIMDJSON_TRY(allocate(json.length(), max_depth())); + } + + // Run stage 1. + SIMDJSON_TRY(implementation->stage1(reinterpret_cast(json.data()), json.length(), stage1_mode::regular)); + return json_iterator(reinterpret_cast(json.data()), this); + } + + inline simdjson_result parser::iterate_many(const uint8_t* buf, size_t len, size_t batch_size, bool allow_comma_separated) noexcept { + if (batch_size < MINIMAL_BATCH_SIZE) { batch_size = MINIMAL_BATCH_SIZE; } + if ((len >= 3) && (std::memcmp(buf, "\xEF\xBB\xBF", 3) == 0)) { + buf += 3; + len -= 3; + } + if (allow_comma_separated && batch_size < len) { batch_size = len; } + return document_stream(*this, buf, len, batch_size, allow_comma_separated); + } + inline simdjson_result parser::iterate_many(const char* buf, size_t len, size_t batch_size, bool allow_comma_separated) noexcept { + return iterate_many(reinterpret_cast(buf), len, batch_size, allow_comma_separated); + } + inline simdjson_result parser::iterate_many(const std::string& s, size_t batch_size, bool allow_comma_separated) noexcept { + return iterate_many(s.data(), s.length(), batch_size, allow_comma_separated); + } + inline simdjson_result parser::iterate_many(const padded_string& s, size_t batch_size, bool allow_comma_separated) noexcept { + return iterate_many(s.data(), s.length(), batch_size, allow_comma_separated); + } + + simdjson_pure simdjson_inline size_t parser::capacity() const noexcept { + return _capacity; + } + simdjson_pure simdjson_inline size_t parser::max_capacity() const noexcept { + return _max_capacity; + } + simdjson_pure simdjson_inline size_t parser::max_depth() const noexcept { + return _max_depth; + } + + simdjson_inline void parser::set_max_capacity(size_t max_capacity) noexcept { + if (max_capacity < dom::MINIMAL_DOCUMENT_CAPACITY) { + _max_capacity = max_capacity; + } + else { + _max_capacity = dom::MINIMAL_DOCUMENT_CAPACITY; + } + } + + simdjson_inline simdjson_warn_unused simdjson_result parser::unescape(raw_json_string in, uint8_t*& dst, bool allow_replacement) const noexcept { + uint8_t* end = implementation->parse_string(in.buf, dst, allow_replacement); + if (!end) { return STRING_ERROR; } + std::string_view result(reinterpret_cast(dst), end - dst); + dst = end; + return result; + } + + simdjson_inline simdjson_warn_unused simdjson_result parser::unescape_wobbly(raw_json_string in, uint8_t*& dst) const noexcept { + uint8_t* end = implementation->parse_wobbly_string(in.buf, dst); + if (!end) { return STRING_ERROR; } + std::string_view result(reinterpret_cast(dst), end - dst); + dst = end; + return result; + } + + } // namespace ondemand + } // namespace lasx +} // namespace simdjson + +namespace simdjson { + + simdjson_inline simdjson_result::simdjson_result(lasx::ondemand::parser&& value) noexcept + : implementation_simdjson_result_base(std::forward(value)) {} + simdjson_inline simdjson_result::simdjson_result(error_code error) noexcept + : implementation_simdjson_result_base(error) {} + +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_ONDEMAND_PARSER_INL_H +/* end file simdjson/generic/ondemand/parser-inl.h for lasx */ +/* including simdjson/generic/ondemand/raw_json_string-inl.h for lasx: #include "simdjson/generic/ondemand/raw_json_string-inl.h" */ +/* begin file simdjson/generic/ondemand/raw_json_string-inl.h for lasx */ +#ifndef SIMDJSON_GENERIC_ONDEMAND_RAW_JSON_STRING_INL_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_ONDEMAND_RAW_JSON_STRING_INL_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/raw_json_string.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/json_iterator-inl.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/implementation_simdjson_result_base-inl.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + + namespace lasx { + namespace ondemand { + + simdjson_inline raw_json_string::raw_json_string(const uint8_t* _buf) noexcept : buf{ _buf } {} + + simdjson_inline const char* raw_json_string::raw() const noexcept { return reinterpret_cast(buf); } + + + simdjson_inline bool raw_json_string::is_free_from_unescaped_quote(std::string_view target) noexcept { + size_t pos{ 0 }; + // if the content has no escape character, just scan through it quickly! + for (; pos < target.size() && target[pos] != '\\'; pos++) {} + // slow path may begin. + bool escaping{ false }; + for (; pos < target.size(); pos++) { + if ((target[pos] == '"') && !escaping) { + return false; + } + else if (target[pos] == '\\') { + escaping = !escaping; + } + else { + escaping = false; + } + } + return true; + } + + simdjson_inline bool raw_json_string::is_free_from_unescaped_quote(const char* target) noexcept { + size_t pos{ 0 }; + // if the content has no escape character, just scan through it quickly! + for (; target[pos] && target[pos] != '\\'; pos++) {} + // slow path may begin. + bool escaping{ false }; + for (; target[pos]; pos++) { + if ((target[pos] == '"') && !escaping) { + return false; + } + else if (target[pos] == '\\') { + escaping = !escaping; + } + else { + escaping = false; + } + } + return true; + } + + + simdjson_inline bool raw_json_string::unsafe_is_equal(size_t length, std::string_view target) const noexcept { + // If we are going to call memcmp, then we must know something about the length of the raw_json_string. + return (length >= target.size()) && (raw()[target.size()] == '"') && !memcmp(raw(), target.data(), target.size()); + } + + simdjson_inline bool raw_json_string::unsafe_is_equal(std::string_view target) const noexcept { + // Assumptions: does not contain unescaped quote characters, and + // the raw content is quote terminated within a valid JSON string. + if (target.size() <= SIMDJSON_PADDING) { + return (raw()[target.size()] == '"') && !memcmp(raw(), target.data(), target.size()); + } + const char* r{ raw() }; + size_t pos{ 0 }; + for (; pos < target.size(); pos++) { + if (r[pos] != target[pos]) { return false; } + } + if (r[pos] != '"') { return false; } + return true; + } + + simdjson_inline bool raw_json_string::is_equal(std::string_view target) const noexcept { + const char* r{ raw() }; + size_t pos{ 0 }; + bool escaping{ false }; + for (; pos < target.size(); pos++) { + if (r[pos] != target[pos]) { return false; } + // if target is a compile-time constant and it is free from + // quotes, then the next part could get optimized away through + // inlining. + if ((target[pos] == '"') && !escaping) { + // We have reached the end of the raw_json_string but + // the target is not done. + return false; + } + else if (target[pos] == '\\') { + escaping = !escaping; + } + else { + escaping = false; + } + } + if (r[pos] != '"') { return false; } + return true; + } + + + simdjson_inline bool raw_json_string::unsafe_is_equal(const char* target) const noexcept { + // Assumptions: 'target' does not contain unescaped quote characters, is null terminated and + // the raw content is quote terminated within a valid JSON string. + const char* r{ raw() }; + size_t pos{ 0 }; + for (; target[pos]; pos++) { + if (r[pos] != target[pos]) { return false; } + } + if (r[pos] != '"') { return false; } + return true; + } + + simdjson_inline bool raw_json_string::is_equal(const char* target) const noexcept { + // Assumptions: does not contain unescaped quote characters, and + // the raw content is quote terminated within a valid JSON string. + const char* r{ raw() }; + size_t pos{ 0 }; + bool escaping{ false }; + for (; target[pos]; pos++) { + if (r[pos] != target[pos]) { return false; } + // if target is a compile-time constant and it is free from + // quotes, then the next part could get optimized away through + // inlining. + if ((target[pos] == '"') && !escaping) { + // We have reached the end of the raw_json_string but + // the target is not done. + return false; + } + else if (target[pos] == '\\') { + escaping = !escaping; + } + else { + escaping = false; + } + } + if (r[pos] != '"') { return false; } + return true; + } + + simdjson_unused simdjson_inline bool operator==(const raw_json_string& a, std::string_view c) noexcept { + return a.unsafe_is_equal(c); + } + + simdjson_unused simdjson_inline bool operator==(std::string_view c, const raw_json_string& a) noexcept { + return a == c; + } + + simdjson_unused simdjson_inline bool operator!=(const raw_json_string& a, std::string_view c) noexcept { + return !(a == c); + } + + simdjson_unused simdjson_inline bool operator!=(std::string_view c, const raw_json_string& a) noexcept { + return !(a == c); + } + + + simdjson_inline simdjson_warn_unused simdjson_result raw_json_string::unescape(json_iterator& iter, bool allow_replacement) const noexcept { + return iter.unescape(*this, allow_replacement); + } + + simdjson_inline simdjson_warn_unused simdjson_result raw_json_string::unescape_wobbly(json_iterator& iter) const noexcept { + return iter.unescape_wobbly(*this); + } + + simdjson_unused simdjson_inline std::ostream& operator<<(std::ostream& out, const raw_json_string& str) noexcept { + bool in_escape = false; + const char* s = str.raw(); + while (true) { + switch (*s) { + case '\\': in_escape = !in_escape; break; + case '"': if (in_escape) { in_escape = false; } + else { return out; } break; + default: if (in_escape) { in_escape = false; } + } + out << *s; + s++; + } + } + + } // namespace ondemand + } // namespace lasx +} // namespace simdjson + +namespace simdjson { + + simdjson_inline simdjson_result::simdjson_result(lasx::ondemand::raw_json_string&& value) noexcept + : implementation_simdjson_result_base(std::forward(value)) {} + simdjson_inline simdjson_result::simdjson_result(error_code error) noexcept + : implementation_simdjson_result_base(error) {} + + simdjson_inline simdjson_result simdjson_result::raw() const noexcept { + if (error()) { return error(); } + return first.raw(); + } + simdjson_inline simdjson_warn_unused simdjson_result simdjson_result::unescape(lasx::ondemand::json_iterator& iter, bool allow_replacement) const noexcept { + if (error()) { return error(); } + return first.unescape(iter, allow_replacement); + } + simdjson_inline simdjson_warn_unused simdjson_result simdjson_result::unescape_wobbly(lasx::ondemand::json_iterator& iter) const noexcept { + if (error()) { return error(); } + return first.unescape_wobbly(iter); + } +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_ONDEMAND_RAW_JSON_STRING_INL_H +/* end file simdjson/generic/ondemand/raw_json_string-inl.h for lasx */ +/* including simdjson/generic/ondemand/serialization-inl.h for lasx: #include "simdjson/generic/ondemand/serialization-inl.h" */ +/* begin file simdjson/generic/ondemand/serialization-inl.h for lasx */ +#ifndef SIMDJSON_GENERIC_ONDEMAND_SERIALIZATION_INL_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_ONDEMAND_SERIALIZATION_INL_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/array.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/document-inl.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/json_type.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/object.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/serialization.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/value.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + + inline std::string_view trim(const std::string_view str) noexcept { + // We can almost surely do better by rolling our own find_first_not_of function. + size_t first = str.find_first_not_of(" \t\n\r"); + // If we have the empty string (just white space), then no trimming is possible, and + // we return the empty string_view. + if (std::string_view::npos == first) { return std::string_view(); } + size_t last = str.find_last_not_of(" \t\n\r"); + return str.substr(first, (last - first + 1)); + } + + + inline simdjson_result to_json_string(lasx::ondemand::document& x) noexcept { + std::string_view v; + auto error = x.raw_json().get(v); + if (error) { return error; } + return trim(v); + } + + inline simdjson_result to_json_string(lasx::ondemand::document_reference& x) noexcept { + std::string_view v; + auto error = x.raw_json().get(v); + if (error) { return error; } + return trim(v); + } + + inline simdjson_result to_json_string(lasx::ondemand::value& x) noexcept { + /** + * If we somehow receive a value that has already been consumed, + * then the following code could be in trouble. E.g., we create + * an array as needed, but if an array was already created, then + * it could be bad. + */ + using namespace lasx::ondemand; + lasx::ondemand::json_type t; + auto error = x.type().get(t); + if (error != SUCCESS) { return error; } + switch (t) + { + case json_type::array: + { + lasx::ondemand::array array; + error = x.get_array().get(array); + if (error) { return error; } + return to_json_string(array); + } + case json_type::object: + { + lasx::ondemand::object object; + error = x.get_object().get(object); + if (error) { return error; } + return to_json_string(object); + } + default: + return trim(x.raw_json_token()); + } + } + + inline simdjson_result to_json_string(lasx::ondemand::object& x) noexcept { + std::string_view v; + auto error = x.raw_json().get(v); + if (error) { return error; } + return trim(v); + } + + inline simdjson_result to_json_string(lasx::ondemand::array& x) noexcept { + std::string_view v; + auto error = x.raw_json().get(v); + if (error) { return error; } + return trim(v); + } + + inline simdjson_result to_json_string(simdjson_result x) { + if (x.error()) { return x.error(); } + return to_json_string(x.value_unsafe()); + } + + inline simdjson_result to_json_string(simdjson_result x) { + if (x.error()) { return x.error(); } + return to_json_string(x.value_unsafe()); + } + + inline simdjson_result to_json_string(simdjson_result x) { + if (x.error()) { return x.error(); } + return to_json_string(x.value_unsafe()); + } + + inline simdjson_result to_json_string(simdjson_result x) { + if (x.error()) { return x.error(); } + return to_json_string(x.value_unsafe()); + } + + inline simdjson_result to_json_string(simdjson_result x) { + if (x.error()) { return x.error(); } + return to_json_string(x.value_unsafe()); + } +} // namespace simdjson + +namespace simdjson { + namespace lasx { + namespace ondemand { + +#if SIMDJSON_EXCEPTIONS + inline std::ostream& operator<<(std::ostream& out, simdjson::lasx::ondemand::value x) { + std::string_view v; + auto error = simdjson::to_json_string(x).get(v); + if (error == simdjson::SUCCESS) { + return (out << v); + } + else { + throw simdjson::simdjson_error(error); + } + } + inline std::ostream& operator<<(std::ostream& out, simdjson::simdjson_result x) { + if (x.error()) { throw simdjson::simdjson_error(x.error()); } + return (out << x.value()); + } +#else + inline std::ostream& operator<<(std::ostream& out, simdjson::lasx::ondemand::value x) { + std::string_view v; + auto error = simdjson::to_json_string(x).get(v); + if (error == simdjson::SUCCESS) { + return (out << v); + } + else { + return (out << error); + } + } +#endif + +#if SIMDJSON_EXCEPTIONS + inline std::ostream& operator<<(std::ostream& out, simdjson::lasx::ondemand::array value) { + std::string_view v; + auto error = simdjson::to_json_string(value).get(v); + if (error == simdjson::SUCCESS) { + return (out << v); + } + else { + throw simdjson::simdjson_error(error); + } + } + inline std::ostream& operator<<(std::ostream& out, simdjson::simdjson_result x) { + if (x.error()) { throw simdjson::simdjson_error(x.error()); } + return (out << x.value()); + } +#else + inline std::ostream& operator<<(std::ostream& out, simdjson::lasx::ondemand::array value) { + std::string_view v; + auto error = simdjson::to_json_string(value).get(v); + if (error == simdjson::SUCCESS) { + return (out << v); + } + else { + return (out << error); + } + } +#endif + +#if SIMDJSON_EXCEPTIONS + inline std::ostream& operator<<(std::ostream& out, simdjson::lasx::ondemand::document& value) { + std::string_view v; + auto error = simdjson::to_json_string(value).get(v); + if (error == simdjson::SUCCESS) { + return (out << v); + } + else { + throw simdjson::simdjson_error(error); + } + } + inline std::ostream& operator<<(std::ostream& out, simdjson::lasx::ondemand::document_reference& value) { + std::string_view v; + auto error = simdjson::to_json_string(value).get(v); + if (error == simdjson::SUCCESS) { + return (out << v); + } + else { + throw simdjson::simdjson_error(error); + } + } + inline std::ostream& operator<<(std::ostream& out, simdjson::simdjson_result&& x) { + if (x.error()) { throw simdjson::simdjson_error(x.error()); } + return (out << x.value()); + } + inline std::ostream& operator<<(std::ostream& out, simdjson::simdjson_result&& x) { + if (x.error()) { throw simdjson::simdjson_error(x.error()); } + return (out << x.value()); + } +#else + inline std::ostream& operator<<(std::ostream& out, simdjson::lasx::ondemand::document& value) { + std::string_view v; + auto error = simdjson::to_json_string(value).get(v); + if (error == simdjson::SUCCESS) { + return (out << v); + } + else { + return (out << error); + } + } +#endif + +#if SIMDJSON_EXCEPTIONS + inline std::ostream& operator<<(std::ostream& out, simdjson::lasx::ondemand::object value) { + std::string_view v; + auto error = simdjson::to_json_string(value).get(v); + if (error == simdjson::SUCCESS) { + return (out << v); + } + else { + throw simdjson::simdjson_error(error); + } + } + inline std::ostream& operator<<(std::ostream& out, simdjson::simdjson_result x) { + if (x.error()) { throw simdjson::simdjson_error(x.error()); } + return (out << x.value()); + } +#else + inline std::ostream& operator<<(std::ostream& out, simdjson::lasx::ondemand::object value) { + std::string_view v; + auto error = simdjson::to_json_string(value).get(v); + if (error == simdjson::SUCCESS) { + return (out << v); + } + else { + return (out << error); + } + } +#endif + } + } +} // namespace simdjson::lasx::ondemand + +#endif // SIMDJSON_GENERIC_ONDEMAND_SERIALIZATION_INL_H +/* end file simdjson/generic/ondemand/serialization-inl.h for lasx */ +/* including simdjson/generic/ondemand/token_iterator-inl.h for lasx: #include "simdjson/generic/ondemand/token_iterator-inl.h" */ +/* begin file simdjson/generic/ondemand/token_iterator-inl.h for lasx */ +#ifndef SIMDJSON_GENERIC_ONDEMAND_TOKEN_ITERATOR_INL_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_ONDEMAND_TOKEN_ITERATOR_INL_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/token_iterator.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/implementation_simdjson_result_base-inl.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lasx { + namespace ondemand { + + simdjson_inline token_iterator::token_iterator( + const uint8_t* _buf, + token_position position + ) noexcept : buf{ _buf }, _position{ position } + { + } + + simdjson_inline uint32_t token_iterator::current_offset() const noexcept { + return *(_position); + } + + + simdjson_inline const uint8_t* token_iterator::return_current_and_advance() noexcept { + return &buf[*(_position++)]; + } + + simdjson_inline const uint8_t* token_iterator::peek(token_position position) const noexcept { + return &buf[*position]; + } + simdjson_inline uint32_t token_iterator::peek_index(token_position position) const noexcept { + return *position; + } + simdjson_inline uint32_t token_iterator::peek_length(token_position position) const noexcept { + return *(position + 1) - *position; + } + + simdjson_inline uint32_t token_iterator::peek_root_length(token_position position) const noexcept { + return *(position + 2) - *(position) > *(position + 1) - *(position) ? + *(position + 1) - *(position) + : *(position + 2) - *(position); + } + simdjson_inline const uint8_t* token_iterator::peek(int32_t delta) const noexcept { + return &buf[*(_position + delta)]; + } + simdjson_inline uint32_t token_iterator::peek_index(int32_t delta) const noexcept { + return *(_position + delta); + } + simdjson_inline uint32_t token_iterator::peek_length(int32_t delta) const noexcept { + return *(_position + delta + 1) - *(_position + delta); + } + + simdjson_inline token_position token_iterator::position() const noexcept { + return _position; + } + simdjson_inline void token_iterator::set_position(token_position target_position) noexcept { + _position = target_position; + } + + simdjson_inline bool token_iterator::operator==(const token_iterator& other) const noexcept { + return _position == other._position; + } + simdjson_inline bool token_iterator::operator!=(const token_iterator& other) const noexcept { + return _position != other._position; + } + simdjson_inline bool token_iterator::operator>(const token_iterator& other) const noexcept { + return _position > other._position; + } + simdjson_inline bool token_iterator::operator>=(const token_iterator& other) const noexcept { + return _position >= other._position; + } + simdjson_inline bool token_iterator::operator<(const token_iterator& other) const noexcept { + return _position < other._position; + } + simdjson_inline bool token_iterator::operator<=(const token_iterator& other) const noexcept { + return _position <= other._position; + } + + } // namespace ondemand + } // namespace lasx +} // namespace simdjson + +namespace simdjson { + + simdjson_inline simdjson_result::simdjson_result(lasx::ondemand::token_iterator&& value) noexcept + : implementation_simdjson_result_base(std::forward(value)) {} + simdjson_inline simdjson_result::simdjson_result(error_code error) noexcept + : implementation_simdjson_result_base(error) {} + +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_ONDEMAND_TOKEN_ITERATOR_INL_H +/* end file simdjson/generic/ondemand/token_iterator-inl.h for lasx */ +/* including simdjson/generic/ondemand/value-inl.h for lasx: #include "simdjson/generic/ondemand/value-inl.h" */ +/* begin file simdjson/generic/ondemand/value-inl.h for lasx */ +#ifndef SIMDJSON_GENERIC_ONDEMAND_VALUE_INL_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_ONDEMAND_VALUE_INL_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/array.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/array_iterator.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/json_iterator.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/json_type.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/object.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/raw_json_string.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/value.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lasx { + namespace ondemand { + + simdjson_inline value::value(const value_iterator& _iter) noexcept + : iter{ _iter } + { + } + simdjson_inline value value::start(const value_iterator& iter) noexcept { + return iter; + } + simdjson_inline value value::resume(const value_iterator& iter) noexcept { + return iter; + } + + simdjson_inline simdjson_result value::get_array() noexcept { + return array::start(iter); + } + simdjson_inline simdjson_result value::get_object() noexcept { + return object::start(iter); + } + simdjson_inline simdjson_result value::start_or_resume_object() noexcept { + if (iter.at_start()) { + return get_object(); + } + else { + return object::resume(iter); + } + } + + simdjson_inline simdjson_result value::get_raw_json_string() noexcept { + return iter.get_raw_json_string(); + } + simdjson_inline simdjson_result value::get_string(bool allow_replacement) noexcept { + return iter.get_string(allow_replacement); + } + template + simdjson_inline error_code value::get_string(string_type& receiver, bool allow_replacement) noexcept { + return iter.get_string(receiver, allow_replacement); + } + simdjson_inline simdjson_result value::get_wobbly_string() noexcept { + return iter.get_wobbly_string(); + } + simdjson_inline simdjson_result value::get_double() noexcept { + return iter.get_double(); + } + simdjson_inline simdjson_result value::get_double_in_string() noexcept { + return iter.get_double_in_string(); + } + simdjson_inline simdjson_result value::get_uint64() noexcept { + return iter.get_uint64(); + } + simdjson_inline simdjson_result value::get_uint64_in_string() noexcept { + return iter.get_uint64_in_string(); + } + simdjson_inline simdjson_result value::get_int64() noexcept { + return iter.get_int64(); + } + simdjson_inline simdjson_result value::get_int64_in_string() noexcept { + return iter.get_int64_in_string(); + } + simdjson_inline simdjson_result value::get_bool() noexcept { + return iter.get_bool(); + } + simdjson_inline simdjson_result value::is_null() noexcept { + return iter.is_null(); + } + template<> simdjson_inline simdjson_result value::get() noexcept { return get_array(); } + template<> simdjson_inline simdjson_result value::get() noexcept { return get_object(); } + template<> simdjson_inline simdjson_result value::get() noexcept { return get_raw_json_string(); } + template<> simdjson_inline simdjson_result value::get() noexcept { return get_string(false); } + template<> simdjson_inline simdjson_result value::get() noexcept { return get_number(); } + template<> simdjson_inline simdjson_result value::get() noexcept { return get_double(); } + template<> simdjson_inline simdjson_result value::get() noexcept { return get_uint64(); } + template<> simdjson_inline simdjson_result value::get() noexcept { return get_int64(); } + template<> simdjson_inline simdjson_result value::get() noexcept { return get_bool(); } + + template simdjson_inline error_code value::get(T& out) noexcept { + return get().get(out); + } + +#if SIMDJSON_EXCEPTIONS + template + simdjson_inline value::operator T() noexcept(false) { + return get(); + } + simdjson_inline value::operator array() noexcept(false) { + return get_array(); + } + simdjson_inline value::operator object() noexcept(false) { + return get_object(); + } + simdjson_inline value::operator uint64_t() noexcept(false) { + return get_uint64(); + } + simdjson_inline value::operator int64_t() noexcept(false) { + return get_int64(); + } + simdjson_inline value::operator double() noexcept(false) { + return get_double(); + } + simdjson_inline value::operator std::string_view() noexcept(false) { + return get_string(false); + } + simdjson_inline value::operator raw_json_string() noexcept(false) { + return get_raw_json_string(); + } + simdjson_inline value::operator bool() noexcept(false) { + return get_bool(); + } +#endif + + simdjson_inline simdjson_result value::begin() & noexcept { + return get_array().begin(); + } + simdjson_inline simdjson_result value::end() & noexcept { + return {}; + } + simdjson_inline simdjson_result value::count_elements() & noexcept { + simdjson_result answer; + auto a = get_array(); + answer = a.count_elements(); + // count_elements leaves you pointing inside the array, at the first element. + // We need to move back so that the user can create a new array (which requires that + // we point at '['). + iter.move_at_start(); + return answer; + } + simdjson_inline simdjson_result value::count_fields() & noexcept { + simdjson_result answer; + auto a = get_object(); + answer = a.count_fields(); + iter.move_at_start(); + return answer; + } + simdjson_inline simdjson_result value::at(size_t index) noexcept { + auto a = get_array(); + return a.at(index); + } + + simdjson_inline simdjson_result value::find_field(std::string_view key) noexcept { + return start_or_resume_object().find_field(key); + } + simdjson_inline simdjson_result value::find_field(const char* key) noexcept { + return start_or_resume_object().find_field(key); + } + + simdjson_inline simdjson_result value::find_field_unordered(std::string_view key) noexcept { + return start_or_resume_object().find_field_unordered(key); + } + simdjson_inline simdjson_result value::find_field_unordered(const char* key) noexcept { + return start_or_resume_object().find_field_unordered(key); + } + + simdjson_inline simdjson_result value::operator[](std::string_view key) noexcept { + return start_or_resume_object()[key]; + } + simdjson_inline simdjson_result value::operator[](const char* key) noexcept { + return start_or_resume_object()[key]; + } + + simdjson_inline simdjson_result value::type() noexcept { + return iter.type(); + } + + simdjson_inline simdjson_result value::is_scalar() noexcept { + json_type this_type; + auto error = type().get(this_type); + if (error) { return error; } + return !((this_type == json_type::array) || (this_type == json_type::object)); + } + + simdjson_inline simdjson_result value::is_string() noexcept { + json_type this_type; + auto error = type().get(this_type); + if (error) { return error; } + return (this_type == json_type::string); + } + + + simdjson_inline bool value::is_negative() noexcept { + return iter.is_negative(); + } + + simdjson_inline simdjson_result value::is_integer() noexcept { + return iter.is_integer(); + } + simdjson_warn_unused simdjson_inline simdjson_result value::get_number_type() noexcept { + return iter.get_number_type(); + } + simdjson_warn_unused simdjson_inline simdjson_result value::get_number() noexcept { + return iter.get_number(); + } + + simdjson_inline std::string_view value::raw_json_token() noexcept { + return std::string_view(reinterpret_cast(iter.peek_start()), iter.peek_start_length()); + } + + simdjson_inline simdjson_result value::raw_json() noexcept { + json_type t; + SIMDJSON_TRY(type().get(t)); + switch (t) + { + case json_type::array: { + ondemand::array array; + SIMDJSON_TRY(get_array().get(array)); + return array.raw_json(); + } + case json_type::object: { + ondemand::object object; + SIMDJSON_TRY(get_object().get(object)); + return object.raw_json(); + } + default: + return raw_json_token(); + } + } + + simdjson_inline simdjson_result value::current_location() noexcept { + return iter.json_iter().current_location(); + } + + simdjson_inline int32_t value::current_depth() const noexcept { + return iter.json_iter().depth(); + } + + inline bool is_pointer_well_formed(std::string_view json_pointer) noexcept { + if (simdjson_unlikely(json_pointer.empty())) { // can't be + return false; + } + if (simdjson_unlikely(json_pointer[0] != '/')) { + return false; + } + size_t escape = json_pointer.find('~'); + if (escape == std::string_view::npos) { + return true; + } + if (escape == json_pointer.size() - 1) { + return false; + } + if (json_pointer[escape + 1] != '0' && json_pointer[escape + 1] != '1') { + return false; + } + return true; + } + + simdjson_inline simdjson_result value::at_pointer(std::string_view json_pointer) noexcept { + json_type t; + SIMDJSON_TRY(type().get(t)); + switch (t) + { + case json_type::array: + return (*this).get_array().at_pointer(json_pointer); + case json_type::object: + return (*this).get_object().at_pointer(json_pointer); + default: + // a non-empty string can be invalid, or accessing a primitive (issue 2154) + if (is_pointer_well_formed(json_pointer)) { + return NO_SUCH_FIELD; + } + return INVALID_JSON_POINTER; + } + } + + simdjson_inline simdjson_result value::at_path(std::string_view json_path) noexcept { + json_type t; + SIMDJSON_TRY(type().get(t)); + switch (t) { + case json_type::array: + return (*this).get_array().at_path(json_path); + case json_type::object: + return (*this).get_object().at_path(json_path); + default: + return INVALID_JSON_POINTER; + } + } + + } // namespace ondemand + } // namespace lasx +} // namespace simdjson + +namespace simdjson { + + simdjson_inline simdjson_result::simdjson_result( + lasx::ondemand::value&& value + ) noexcept : + implementation_simdjson_result_base( + std::forward(value) + ) + { + } + simdjson_inline simdjson_result::simdjson_result( + error_code error + ) noexcept : + implementation_simdjson_result_base(error) + { + } + simdjson_inline simdjson_result simdjson_result::count_elements() & noexcept { + if (error()) { return error(); } + return first.count_elements(); + } + simdjson_inline simdjson_result simdjson_result::count_fields() & noexcept { + if (error()) { return error(); } + return first.count_fields(); + } + simdjson_inline simdjson_result simdjson_result::at(size_t index) noexcept { + if (error()) { return error(); } + return first.at(index); + } + simdjson_inline simdjson_result simdjson_result::begin() & noexcept { + if (error()) { return error(); } + return first.begin(); + } + simdjson_inline simdjson_result simdjson_result::end() & noexcept { + if (error()) { return error(); } + return {}; + } + + simdjson_inline simdjson_result simdjson_result::find_field(std::string_view key) noexcept { + if (error()) { return error(); } + return first.find_field(key); + } + simdjson_inline simdjson_result simdjson_result::find_field(const char* key) noexcept { + if (error()) { return error(); } + return first.find_field(key); + } + + simdjson_inline simdjson_result simdjson_result::find_field_unordered(std::string_view key) noexcept { + if (error()) { return error(); } + return first.find_field_unordered(key); + } + simdjson_inline simdjson_result simdjson_result::find_field_unordered(const char* key) noexcept { + if (error()) { return error(); } + return first.find_field_unordered(key); + } + + simdjson_inline simdjson_result simdjson_result::operator[](std::string_view key) noexcept { + if (error()) { return error(); } + return first[key]; + } + simdjson_inline simdjson_result simdjson_result::operator[](const char* key) noexcept { + if (error()) { return error(); } + return first[key]; + } + + simdjson_inline simdjson_result simdjson_result::get_array() noexcept { + if (error()) { return error(); } + return first.get_array(); + } + simdjson_inline simdjson_result simdjson_result::get_object() noexcept { + if (error()) { return error(); } + return first.get_object(); + } + simdjson_inline simdjson_result simdjson_result::get_uint64() noexcept { + if (error()) { return error(); } + return first.get_uint64(); + } + simdjson_inline simdjson_result simdjson_result::get_uint64_in_string() noexcept { + if (error()) { return error(); } + return first.get_uint64_in_string(); + } + simdjson_inline simdjson_result simdjson_result::get_int64() noexcept { + if (error()) { return error(); } + return first.get_int64(); + } + simdjson_inline simdjson_result simdjson_result::get_int64_in_string() noexcept { + if (error()) { return error(); } + return first.get_int64_in_string(); + } + simdjson_inline simdjson_result simdjson_result::get_double() noexcept { + if (error()) { return error(); } + return first.get_double(); + } + simdjson_inline simdjson_result simdjson_result::get_double_in_string() noexcept { + if (error()) { return error(); } + return first.get_double_in_string(); + } + simdjson_inline simdjson_result simdjson_result::get_string(bool allow_replacement) noexcept { + if (error()) { return error(); } + return first.get_string(allow_replacement); + } + template + simdjson_inline error_code simdjson_result::get_string(string_type& receiver, bool allow_replacement) noexcept { + if (error()) { return error(); } + return first.get_string(receiver, allow_replacement); + } + simdjson_inline simdjson_result simdjson_result::get_wobbly_string() noexcept { + if (error()) { return error(); } + return first.get_wobbly_string(); + } + simdjson_inline simdjson_result simdjson_result::get_raw_json_string() noexcept { + if (error()) { return error(); } + return first.get_raw_json_string(); + } + simdjson_inline simdjson_result simdjson_result::get_bool() noexcept { + if (error()) { return error(); } + return first.get_bool(); + } + simdjson_inline simdjson_result simdjson_result::is_null() noexcept { + if (error()) { return error(); } + return first.is_null(); + } + + template simdjson_inline simdjson_result simdjson_result::get() noexcept { + if (error()) { return error(); } + return first.get(); + } + template simdjson_inline error_code simdjson_result::get(T& out) noexcept { + if (error()) { return error(); } + return first.get(out); + } + + template<> simdjson_inline simdjson_result simdjson_result::get() noexcept { + if (error()) { return error(); } + return std::move(first); + } + template<> simdjson_inline error_code simdjson_result::get(lasx::ondemand::value& out) noexcept { + if (error()) { return error(); } + out = first; + return SUCCESS; + } + + simdjson_inline simdjson_result simdjson_result::type() noexcept { + if (error()) { return error(); } + return first.type(); + } + simdjson_inline simdjson_result simdjson_result::is_scalar() noexcept { + if (error()) { return error(); } + return first.is_scalar(); + } + simdjson_inline simdjson_result simdjson_result::is_string() noexcept { + if (error()) { return error(); } + return first.is_string(); + } + simdjson_inline simdjson_result simdjson_result::is_negative() noexcept { + if (error()) { return error(); } + return first.is_negative(); + } + simdjson_inline simdjson_result simdjson_result::is_integer() noexcept { + if (error()) { return error(); } + return first.is_integer(); + } + simdjson_inline simdjson_result simdjson_result::get_number_type() noexcept { + if (error()) { return error(); } + return first.get_number_type(); + } + simdjson_inline simdjson_result simdjson_result::get_number() noexcept { + if (error()) { return error(); } + return first.get_number(); + } +#if SIMDJSON_EXCEPTIONS + template + simdjson_inline simdjson_result::operator T() noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return static_cast(first); + } + simdjson_inline simdjson_result::operator lasx::ondemand::array() noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return first; + } + simdjson_inline simdjson_result::operator lasx::ondemand::object() noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return first; + } + simdjson_inline simdjson_result::operator uint64_t() noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return first; + } + simdjson_inline simdjson_result::operator int64_t() noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return first; + } + simdjson_inline simdjson_result::operator double() noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return first; + } + simdjson_inline simdjson_result::operator std::string_view() noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return first; + } + simdjson_inline simdjson_result::operator lasx::ondemand::raw_json_string() noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return first; + } + simdjson_inline simdjson_result::operator bool() noexcept(false) { + if (error()) { throw simdjson_error(error()); } + return first; + } +#endif + + simdjson_inline simdjson_result simdjson_result::raw_json_token() noexcept { + if (error()) { return error(); } + return first.raw_json_token(); + } + + simdjson_inline simdjson_result simdjson_result::raw_json() noexcept { + if (error()) { return error(); } + return first.raw_json(); + } + + simdjson_inline simdjson_result simdjson_result::current_location() noexcept { + if (error()) { return error(); } + return first.current_location(); + } + + simdjson_inline simdjson_result simdjson_result::current_depth() const noexcept { + if (error()) { return error(); } + return first.current_depth(); + } + + simdjson_inline simdjson_result simdjson_result::at_pointer( + std::string_view json_pointer) noexcept { + if (error()) { + return error(); + } + return first.at_pointer(json_pointer); + } + + simdjson_inline simdjson_result simdjson_result::at_path( + std::string_view json_path) noexcept { + if (error()) { + return error(); + } + return first.at_path(json_path); + } + +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_ONDEMAND_VALUE_INL_H +/* end file simdjson/generic/ondemand/value-inl.h for lasx */ +/* including simdjson/generic/ondemand/value_iterator-inl.h for lasx: #include "simdjson/generic/ondemand/value_iterator-inl.h" */ +/* begin file simdjson/generic/ondemand/value_iterator-inl.h for lasx */ +#ifndef SIMDJSON_GENERIC_ONDEMAND_VALUE_ITERATOR_INL_H + +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #define SIMDJSON_GENERIC_ONDEMAND_VALUE_ITERATOR_INL_H */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/base.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/atomparsing.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/numberparsing.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/json_iterator.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/value_iterator.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/json_type-inl.h" */ +/* amalgamation skipped (editor-only): #include "simdjson/generic/ondemand/raw_json_string-inl.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +namespace simdjson { + namespace lasx { + namespace ondemand { + + simdjson_inline value_iterator::value_iterator( + json_iterator* json_iter, + depth_t depth, + token_position start_position + ) noexcept : _json_iter{ json_iter }, _depth{ depth }, _start_position{ start_position } + { + } + + simdjson_warn_unused simdjson_inline simdjson_result value_iterator::start_object() noexcept { + SIMDJSON_TRY(start_container('{', "Not an object", "object")); + return started_object(); + } + + simdjson_warn_unused simdjson_inline simdjson_result value_iterator::start_root_object() noexcept { + SIMDJSON_TRY(start_container('{', "Not an object", "object")); + return started_root_object(); + } + + simdjson_warn_unused simdjson_inline simdjson_result value_iterator::started_object() noexcept { + assert_at_container_start(); +#if SIMDJSON_DEVELOPMENT_CHECKS + _json_iter->set_start_position(_depth, start_position()); +#endif + if (*_json_iter->peek() == '}') { + logger::log_value(*_json_iter, "empty object"); + _json_iter->return_current_and_advance(); + end_container(); + return false; + } + return true; + } + + simdjson_warn_unused simdjson_inline error_code value_iterator::check_root_object() noexcept { + // When in streaming mode, we cannot expect peek_last() to be the last structural element of the + // current document. It only works in the normal mode where we have indexed a single document. + // Note that adding a check for 'streaming' is not expensive since we only have at most + // one root element. + if (!_json_iter->streaming()) { + // The following lines do not fully protect against garbage content within the + // object: e.g., `{"a":2} foo }`. Users concerned with garbage content should + // call `at_end()` on the document instance at the end of the processing to + // ensure that the processing has finished at the end. + // + if (*_json_iter->peek_last() != '}') { + _json_iter->abandon(); + return report_error(INCOMPLETE_ARRAY_OR_OBJECT, "missing } at end"); + } + // If the last character is } *and* the first gibberish character is also '}' + // then on-demand could accidentally go over. So we need additional checks. + // https://github.com/simdjson/simdjson/issues/1834 + // Checking that the document is balanced requires a full scan which is potentially + // expensive, but it only happens in edge cases where the first padding character is + // a closing bracket. + if ((*_json_iter->peek(_json_iter->end_position()) == '}') && (!_json_iter->balanced())) { + _json_iter->abandon(); + // The exact error would require more work. It will typically be an unclosed object. + return report_error(INCOMPLETE_ARRAY_OR_OBJECT, "the document is unbalanced"); + } + } + return SUCCESS; + } + + simdjson_warn_unused simdjson_inline simdjson_result value_iterator::started_root_object() noexcept { + auto error = check_root_object(); + if (error) { return error; } + return started_object(); + } + + simdjson_warn_unused simdjson_inline error_code value_iterator::end_container() noexcept { +#if SIMDJSON_CHECK_EOF + if (depth() > 1 && at_end()) { return report_error(INCOMPLETE_ARRAY_OR_OBJECT, "missing parent ] or }"); } + // if (depth() <= 1 && !at_end()) { return report_error(INCOMPLETE_ARRAY_OR_OBJECT, "missing [ or { at start"); } +#endif // SIMDJSON_CHECK_EOF + _json_iter->ascend_to(depth() - 1); + return SUCCESS; + } + + simdjson_warn_unused simdjson_inline simdjson_result value_iterator::has_next_field() noexcept { + assert_at_next(); + + // It's illegal to call this unless there are more tokens: anything that ends in } or ] is + // obligated to verify there are more tokens if they are not the top level. + switch (*_json_iter->return_current_and_advance()) { + case '}': + logger::log_end_value(*_json_iter, "object"); + SIMDJSON_TRY(end_container()); + return false; + case ',': + return true; + default: + return report_error(TAPE_ERROR, "Missing comma between object fields"); + } + } + + simdjson_warn_unused simdjson_inline simdjson_result value_iterator::find_field_raw(const std::string_view key) noexcept { + error_code error; + bool has_value; + // + // Initially, the object can be in one of a few different places: + // + // 1. The start of the object, at the first field: + // + // ``` + // { "a": [ 1, 2 ], "b": [ 3, 4 ] } + // ^ (depth 2, index 1) + // ``` + if (at_first_field()) { + has_value = true; + + // + // 2. When a previous search did not yield a value or the object is empty: + // + // ``` + // { "a": [ 1, 2 ], "b": [ 3, 4 ] } + // ^ (depth 0) + // { } + // ^ (depth 0, index 2) + // ``` + // + } + else if (!is_open()) { +#if SIMDJSON_DEVELOPMENT_CHECKS + // If we're past the end of the object, we're being iterated out of order. + // Note: this is not perfect detection. It's possible the user is inside some other object; if so, + // this object iterator will blithely scan that object for fields. + if (_json_iter->depth() < depth() - 1) { return OUT_OF_ORDER_ITERATION; } +#endif + return false; + + // 3. When a previous search found a field or an iterator yielded a value: + // + // ``` + // // When a field was not fully consumed (or not even touched at all) + // { "a": [ 1, 2 ], "b": [ 3, 4 ] } + // ^ (depth 2) + // // When a field was fully consumed + // { "a": [ 1, 2 ], "b": [ 3, 4 ] } + // ^ (depth 1) + // // When the last field was fully consumed + // { "a": [ 1, 2 ], "b": [ 3, 4 ] } + // ^ (depth 1) + // ``` + // + } + else { + if ((error = skip_child())) { abandon(); return error; } + if ((error = has_next_field().get(has_value))) { abandon(); return error; } +#if SIMDJSON_DEVELOPMENT_CHECKS + if (_json_iter->start_position(_depth) != start_position()) { return OUT_OF_ORDER_ITERATION; } +#endif + } + while (has_value) { + // Get the key and colon, stopping at the value. + raw_json_string actual_key; + // size_t max_key_length = _json_iter->peek_length() - 2; // -2 for the two quotes + // Note: _json_iter->peek_length() - 2 might overflow if _json_iter->peek_length() < 2. + // field_key() advances the pointer and checks that '"' is found (corresponding to a key). + // The depth is left unchanged by field_key(). + if ((error = field_key().get(actual_key))) { abandon(); return error; }; + // field_value() will advance and check that we find a ':' separating the + // key and the value. It will also increment the depth by one. + if ((error = field_value())) { abandon(); return error; } + // If it matches, stop and return + // We could do it this way if we wanted to allow arbitrary + // key content (including escaped quotes). + //if (actual_key.unsafe_is_equal(max_key_length, key)) { + // Instead we do the following which may trigger buffer overruns if the + // user provides an adversarial key (containing a well placed unescaped quote + // character and being longer than the number of bytes remaining in the JSON + // input). + if (actual_key.unsafe_is_equal(key)) { + logger::log_event(*this, "match", key, -2); + // If we return here, then we return while pointing at the ':' that we just checked. + return true; + } + + // No match: skip the value and see if , or } is next + logger::log_event(*this, "no match", key, -2); + // The call to skip_child is meant to skip over the value corresponding to the key. + // After skip_child(), we are right before the next comma (',') or the final brace ('}'). + SIMDJSON_TRY(skip_child()); // Skip the value entirely + // The has_next_field() advances the pointer and check that either ',' or '}' is found. + // It returns true if ',' is found, false otherwise. If anything other than ',' or '}' is found, + // then we are in error and we abort. + if ((error = has_next_field().get(has_value))) { abandon(); return error; } + } + + // If the loop ended, we're out of fields to look at. + return false; + } + + SIMDJSON_PUSH_DISABLE_WARNINGS + SIMDJSON_DISABLE_STRICT_OVERFLOW_WARNING + simdjson_warn_unused simdjson_inline simdjson_result value_iterator::find_field_unordered_raw(const std::string_view key) noexcept { + /** + * When find_field_unordered_raw is called, we can either be pointing at the + * first key, pointing outside (at the closing brace) or if a key was matched + * we can be either pointing right afterthe ':' right before the value (that we need skip), + * or we may have consumed the value and we might be at a comma or at the + * final brace (ready for a call to has_next_field()). + */ + error_code error; + bool has_value; + + // First, we scan from that point to the end. + // If we don't find a match, we may loop back around, and scan from the beginning to that point. + token_position search_start = _json_iter->position(); + + // We want to know whether we need to go back to the beginning. + bool at_first = at_first_field(); + /////////////// + // Initially, the object can be in one of a few different places: + // + // 1. At the first key: + // + // ``` + // { "a": [ 1, 2 ], "b": [ 3, 4 ] } + // ^ (depth 2, index 1) + // ``` + // + if (at_first) { + has_value = true; + + // 2. When a previous search did not yield a value or the object is empty: + // + // ``` + // { "a": [ 1, 2 ], "b": [ 3, 4 ] } + // ^ (depth 0) + // { } + // ^ (depth 0, index 2) + // ``` + // + } + else if (!is_open()) { + +#if SIMDJSON_DEVELOPMENT_CHECKS + // If we're past the end of the object, we're being iterated out of order. + // Note: this is not perfect detection. It's possible the user is inside some other object; if so, + // this object iterator will blithely scan that object for fields. + if (_json_iter->depth() < depth() - 1) { return OUT_OF_ORDER_ITERATION; } +#endif + SIMDJSON_TRY(reset_object().get(has_value)); + at_first = true; + // 3. When a previous search found a field or an iterator yielded a value: + // + // ``` + // // When a field was not fully consumed (or not even touched at all) + // { "a": [ 1, 2 ], "b": [ 3, 4 ] } + // ^ (depth 2) + // // When a field was fully consumed + // { "a": [ 1, 2 ], "b": [ 3, 4 ] } + // ^ (depth 1) + // // When the last field was fully consumed + // { "a": [ 1, 2 ], "b": [ 3, 4 ] } + // ^ (depth 1) + // ``` + // + } + else { + // If someone queried a key but they not did access the value, then we are left pointing + // at the ':' and we need to move forward through the value... If the value was + // processed then skip_child() does not move the iterator (but may adjust the depth). + if ((error = skip_child())) { abandon(); return error; } + search_start = _json_iter->position(); + if ((error = has_next_field().get(has_value))) { abandon(); return error; } +#if SIMDJSON_DEVELOPMENT_CHECKS + if (_json_iter->start_position(_depth) != start_position()) { return OUT_OF_ORDER_ITERATION; } +#endif + } + + // After initial processing, we will be in one of two states: + // + // ``` + // // At the beginning of a field + // { "a": [ 1, 2 ], "b": [ 3, 4 ] } + // ^ (depth 1) + // { "a": [ 1, 2 ], "b": [ 3, 4 ] } + // ^ (depth 1) + // // At the end of the object + // { "a": [ 1, 2 ], "b": [ 3, 4 ] } + // ^ (depth 0) + // ``` + // + // Next, we find a match starting from the current position. + while (has_value) { + SIMDJSON_ASSUME(_json_iter->_depth == _depth); // We must be at the start of a field + + // Get the key and colon, stopping at the value. + raw_json_string actual_key; + // size_t max_key_length = _json_iter->peek_length() - 2; // -2 for the two quotes + // Note: _json_iter->peek_length() - 2 might overflow if _json_iter->peek_length() < 2. + // field_key() advances the pointer and checks that '"' is found (corresponding to a key). + // The depth is left unchanged by field_key(). + if ((error = field_key().get(actual_key))) { abandon(); return error; }; + // field_value() will advance and check that we find a ':' separating the + // key and the value. It will also increment the depth by one. + if ((error = field_value())) { abandon(); return error; } + + // If it matches, stop and return + // We could do it this way if we wanted to allow arbitrary + // key content (including escaped quotes). + // if (actual_key.unsafe_is_equal(max_key_length, key)) { + // Instead we do the following which may trigger buffer overruns if the + // user provides an adversarial key (containing a well placed unescaped quote + // character and being longer than the number of bytes remaining in the JSON + // input). + if (actual_key.unsafe_is_equal(key)) { + logger::log_event(*this, "match", key, -2); + // If we return here, then we return while pointing at the ':' that we just checked. + return true; + } + + // No match: skip the value and see if , or } is next + logger::log_event(*this, "no match", key, -2); + // The call to skip_child is meant to skip over the value corresponding to the key. + // After skip_child(), we are right before the next comma (',') or the final brace ('}'). + SIMDJSON_TRY(skip_child()); + // The has_next_field() advances the pointer and check that either ',' or '}' is found. + // It returns true if ',' is found, false otherwise. If anything other than ',' or '}' is found, + // then we are in error and we abort. + if ((error = has_next_field().get(has_value))) { abandon(); return error; } + } + // Performance note: it maybe wasteful to rewind to the beginning when there might be + // no other query following. Indeed, it would require reskipping the whole object. + // Instead, you can just stay where you are. If there is a new query, there is always time + // to rewind. + if (at_first) { return false; } + + // If we reach the end without finding a match, search the rest of the fields starting at the + // beginning of the object. + // (We have already run through the object before, so we've already validated its structure. We + // don't check errors in this bit.) + SIMDJSON_TRY(reset_object().get(has_value)); + while (true) { + SIMDJSON_ASSUME(has_value); // we should reach search_start before ever reaching the end of the object + SIMDJSON_ASSUME(_json_iter->_depth == _depth); // We must be at the start of a field + + // Get the key and colon, stopping at the value. + raw_json_string actual_key; + // size_t max_key_length = _json_iter->peek_length() - 2; // -2 for the two quotes + // Note: _json_iter->peek_length() - 2 might overflow if _json_iter->peek_length() < 2. + // field_key() advances the pointer and checks that '"' is found (corresponding to a key). + // The depth is left unchanged by field_key(). + error = field_key().get(actual_key); SIMDJSON_ASSUME(!error); + // field_value() will advance and check that we find a ':' separating the + // key and the value. It will also increment the depth by one. + error = field_value(); SIMDJSON_ASSUME(!error); + + // If it matches, stop and return + // We could do it this way if we wanted to allow arbitrary + // key content (including escaped quotes). + // if (actual_key.unsafe_is_equal(max_key_length, key)) { + // Instead we do the following which may trigger buffer overruns if the + // user provides an adversarial key (containing a well placed unescaped quote + // character and being longer than the number of bytes remaining in the JSON + // input). + if (actual_key.unsafe_is_equal(key)) { + logger::log_event(*this, "match", key, -2); + // If we return here, then we return while pointing at the ':' that we just checked. + return true; + } + + // No match: skip the value and see if , or } is next + logger::log_event(*this, "no match", key, -2); + // The call to skip_child is meant to skip over the value corresponding to the key. + // After skip_child(), we are right before the next comma (',') or the final brace ('}'). + SIMDJSON_TRY(skip_child()); + // If we reached the end of the key-value pair we started from, then we know + // that the key is not there so we return false. We are either right before + // the next comma or the final brace. + if (_json_iter->position() == search_start) { return false; } + // The has_next_field() advances the pointer and check that either ',' or '}' is found. + // It returns true if ',' is found, false otherwise. If anything other than ',' or '}' is found, + // then we are in error and we abort. + error = has_next_field().get(has_value); SIMDJSON_ASSUME(!error); + // If we make the mistake of exiting here, then we could be left pointing at a key + // in the middle of an object. That's not an allowable state. + } + // If the loop ended, we're out of fields to look at. The program should + // never reach this point. + return false; + } + SIMDJSON_POP_DISABLE_WARNINGS + + simdjson_warn_unused simdjson_inline simdjson_result value_iterator::field_key() noexcept { + assert_at_next(); + + const uint8_t* key = _json_iter->return_current_and_advance(); + if (*(key++) != '"') { return report_error(TAPE_ERROR, "Object key is not a string"); } + return raw_json_string(key); + } + + simdjson_warn_unused simdjson_inline error_code value_iterator::field_value() noexcept { + assert_at_next(); + + if (*_json_iter->return_current_and_advance() != ':') { return report_error(TAPE_ERROR, "Missing colon in object field"); } + _json_iter->descend_to(depth() + 1); + return SUCCESS; + } + + simdjson_warn_unused simdjson_inline simdjson_result value_iterator::start_array() noexcept { + SIMDJSON_TRY(start_container('[', "Not an array", "array")); + return started_array(); + } + + simdjson_warn_unused simdjson_inline simdjson_result value_iterator::start_root_array() noexcept { + SIMDJSON_TRY(start_container('[', "Not an array", "array")); + return started_root_array(); + } + + inline std::string value_iterator::to_string() const noexcept { + auto answer = std::string("value_iterator [ depth : ") + std::to_string(_depth) + std::string(", "); + if (_json_iter != nullptr) { answer += _json_iter->to_string(); } + answer += std::string(" ]"); + return answer; + } + + simdjson_warn_unused simdjson_inline simdjson_result value_iterator::started_array() noexcept { + assert_at_container_start(); + if (*_json_iter->peek() == ']') { + logger::log_value(*_json_iter, "empty array"); + _json_iter->return_current_and_advance(); + SIMDJSON_TRY(end_container()); + return false; + } + _json_iter->descend_to(depth() + 1); +#if SIMDJSON_DEVELOPMENT_CHECKS + _json_iter->set_start_position(_depth, start_position()); +#endif + return true; + } + + simdjson_warn_unused simdjson_inline error_code value_iterator::check_root_array() noexcept { + // When in streaming mode, we cannot expect peek_last() to be the last structural element of the + // current document. It only works in the normal mode where we have indexed a single document. + // Note that adding a check for 'streaming' is not expensive since we only have at most + // one root element. + if (!_json_iter->streaming()) { + // The following lines do not fully protect against garbage content within the + // array: e.g., `[1, 2] foo]`. Users concerned with garbage content should + // also call `at_end()` on the document instance at the end of the processing to + // ensure that the processing has finished at the end. + // + if (*_json_iter->peek_last() != ']') { + _json_iter->abandon(); + return report_error(INCOMPLETE_ARRAY_OR_OBJECT, "missing ] at end"); + } + // If the last character is ] *and* the first gibberish character is also ']' + // then on-demand could accidentally go over. So we need additional checks. + // https://github.com/simdjson/simdjson/issues/1834 + // Checking that the document is balanced requires a full scan which is potentially + // expensive, but it only happens in edge cases where the first padding character is + // a closing bracket. + if ((*_json_iter->peek(_json_iter->end_position()) == ']') && (!_json_iter->balanced())) { + _json_iter->abandon(); + // The exact error would require more work. It will typically be an unclosed array. + return report_error(INCOMPLETE_ARRAY_OR_OBJECT, "the document is unbalanced"); + } + } + return SUCCESS; + } + + simdjson_warn_unused simdjson_inline simdjson_result value_iterator::started_root_array() noexcept { + auto error = check_root_array(); + if (error) { return error; } + return started_array(); + } + + simdjson_warn_unused simdjson_inline simdjson_result value_iterator::has_next_element() noexcept { + assert_at_next(); + + logger::log_event(*this, "has_next_element"); + switch (*_json_iter->return_current_and_advance()) { + case ']': + logger::log_end_value(*_json_iter, "array"); + SIMDJSON_TRY(end_container()); + return false; + case ',': + _json_iter->descend_to(depth() + 1); + return true; + default: + return report_error(TAPE_ERROR, "Missing comma between array elements"); + } + } + + simdjson_warn_unused simdjson_inline simdjson_result value_iterator::parse_bool(const uint8_t* json) const noexcept { + auto not_true = atomparsing::str4ncmp(json, "true"); + auto not_false = atomparsing::str4ncmp(json, "fals") | (json[4] ^ 'e'); + bool error = (not_true && not_false) || jsoncharutils::is_not_structural_or_whitespace(json[not_true ? 5 : 4]); + if (error) { return incorrect_type_error("Not a boolean"); } + return simdjson_result(!not_true); + } + simdjson_warn_unused simdjson_inline simdjson_result value_iterator::parse_null(const uint8_t* json) const noexcept { + bool is_null_string = !atomparsing::str4ncmp(json, "null") && jsoncharutils::is_structural_or_whitespace(json[4]); + // if we start with 'n', we must be a null + if (!is_null_string && json[0] == 'n') { return incorrect_type_error("Not a null but starts with n"); } + return is_null_string; + } + + simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_string(bool allow_replacement) noexcept { + return get_raw_json_string().unescape(json_iter(), allow_replacement); + } + template + simdjson_warn_unused simdjson_inline error_code value_iterator::get_string(string_type& receiver, bool allow_replacement) noexcept { + std::string_view content; + auto err = get_string(allow_replacement).get(content); + if (err) { return err; } + receiver = content; + return SUCCESS; + } + simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_wobbly_string() noexcept { + return get_raw_json_string().unescape_wobbly(json_iter()); + } + simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_raw_json_string() noexcept { + auto json = peek_scalar("string"); + if (*json != '"') { return incorrect_type_error("Not a string"); } + advance_scalar("string"); + return raw_json_string(json + 1); + } + simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_uint64() noexcept { + auto result = numberparsing::parse_unsigned(peek_non_root_scalar("uint64")); + if (result.error() == SUCCESS) { advance_non_root_scalar("uint64"); } + return result; + } + simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_uint64_in_string() noexcept { + auto result = numberparsing::parse_unsigned_in_string(peek_non_root_scalar("uint64")); + if (result.error() == SUCCESS) { advance_non_root_scalar("uint64"); } + return result; + } + simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_int64() noexcept { + auto result = numberparsing::parse_integer(peek_non_root_scalar("int64")); + if (result.error() == SUCCESS) { advance_non_root_scalar("int64"); } + return result; + } + simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_int64_in_string() noexcept { + auto result = numberparsing::parse_integer_in_string(peek_non_root_scalar("int64")); + if (result.error() == SUCCESS) { advance_non_root_scalar("int64"); } + return result; + } + simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_double() noexcept { + auto result = numberparsing::parse_double(peek_non_root_scalar("double")); + if (result.error() == SUCCESS) { advance_non_root_scalar("double"); } + return result; + } + simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_double_in_string() noexcept { + auto result = numberparsing::parse_double_in_string(peek_non_root_scalar("double")); + if (result.error() == SUCCESS) { advance_non_root_scalar("double"); } + return result; + } + simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_bool() noexcept { + auto result = parse_bool(peek_non_root_scalar("bool")); + if (result.error() == SUCCESS) { advance_non_root_scalar("bool"); } + return result; + } + simdjson_inline simdjson_result value_iterator::is_null() noexcept { + bool is_null_value; + SIMDJSON_TRY(parse_null(peek_non_root_scalar("null")).get(is_null_value)); + if (is_null_value) { advance_non_root_scalar("null"); } + return is_null_value; + } + simdjson_inline bool value_iterator::is_negative() noexcept { + return numberparsing::is_negative(peek_non_root_scalar("numbersign")); + } + simdjson_inline bool value_iterator::is_root_negative() noexcept { + return numberparsing::is_negative(peek_root_scalar("numbersign")); + } + simdjson_inline simdjson_result value_iterator::is_integer() noexcept { + return numberparsing::is_integer(peek_non_root_scalar("integer")); + } + simdjson_inline simdjson_result value_iterator::get_number_type() noexcept { + return numberparsing::get_number_type(peek_non_root_scalar("integer")); + } + simdjson_inline simdjson_result value_iterator::get_number() noexcept { + number num; + error_code error = numberparsing::parse_number(peek_non_root_scalar("number"), num); + if (error) { return error; } + return num; + } + + simdjson_inline simdjson_result value_iterator::is_root_integer(bool check_trailing) noexcept { + auto max_len = peek_root_length(); + auto json = peek_root_scalar("is_root_integer"); + uint8_t tmpbuf[20 + 1 + 1]{}; // <20 digits> is the longest possible unsigned integer + tmpbuf[20 + 1] = '\0'; // make sure that buffer is always null terminated. + if (!_json_iter->copy_to_buffer(json, max_len, tmpbuf, 20 + 1)) { + return false; // if there are more than 20 characters, it cannot be represented as an integer. + } + auto answer = numberparsing::is_integer(tmpbuf); + // If the parsing was a success, we must still check that it is + // a single scalar. Note that we parse first because of cases like '[]' where + // getting TRAILING_CONTENT is wrong. + if (check_trailing && (answer.error() == SUCCESS) && (!_json_iter->is_single_token())) { return TRAILING_CONTENT; } + return answer; + } + + simdjson_inline simdjson_result value_iterator::get_root_number_type(bool check_trailing) noexcept { + auto max_len = peek_root_length(); + auto json = peek_root_scalar("number"); + // Per https://www.exploringbinary.com/maximum-number-of-decimal-digits-in-binary-floating-point-numbers/, + // 1074 is the maximum number of significant fractional digits. Add 8 more digits for the biggest + // number: -0.e-308. + uint8_t tmpbuf[1074 + 8 + 1 + 1]; + tmpbuf[1074 + 8 + 1] = '\0'; // make sure that buffer is always null terminated. + if (!_json_iter->copy_to_buffer(json, max_len, tmpbuf, 1074 + 8 + 1)) { + if (numberparsing::check_if_integer(json, max_len)) { + if (check_trailing && !_json_iter->is_single_token()) { return TRAILING_CONTENT; } + logger::log_error(*_json_iter, start_position(), depth(), "Found big integer"); + return number_type::big_integer; + } + logger::log_error(*_json_iter, start_position(), depth(), "Root number more than 1082 characters and not a big integer"); + return NUMBER_ERROR; + } + auto answer = numberparsing::get_number_type(tmpbuf); + if (check_trailing && (answer.error() == SUCCESS) && !_json_iter->is_single_token()) { return TRAILING_CONTENT; } + return answer; + } + simdjson_inline simdjson_result value_iterator::get_root_number(bool check_trailing) noexcept { + auto max_len = peek_root_length(); + auto json = peek_root_scalar("number"); + // Per https://www.exploringbinary.com/maximum-number-of-decimal-digits-in-binary-floating-point-numbers/, + // 1074 is the maximum number of significant fractional digits. Add 8 more digits for the biggest + // number: -0.e-308. + // NOTE: the current approach doesn't work for very big integer numbers containing more than 1074 digits. + uint8_t tmpbuf[1074 + 8 + 1 + 1]; + tmpbuf[1074 + 8 + 1] = '\0'; // make sure that buffer is always null terminated. + if (!_json_iter->copy_to_buffer(json, max_len, tmpbuf, 1074 + 8 + 1)) { + if (numberparsing::check_if_integer(json, max_len)) { + if (check_trailing && !_json_iter->is_single_token()) { return TRAILING_CONTENT; } + logger::log_error(*_json_iter, start_position(), depth(), "Found big integer"); + return BIGINT_ERROR; + } + logger::log_error(*_json_iter, start_position(), depth(), "Root number more than 1082 characters and not a big integer"); + return NUMBER_ERROR; + } + number num; + error_code error = numberparsing::parse_number(tmpbuf, num); + if (error) { return error; } + if (check_trailing && !_json_iter->is_single_token()) { return TRAILING_CONTENT; } + advance_root_scalar("number"); + return num; + } + simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_root_string(bool check_trailing, bool allow_replacement) noexcept { + return get_root_raw_json_string(check_trailing).unescape(json_iter(), allow_replacement); + } + template + simdjson_warn_unused simdjson_inline error_code value_iterator::get_root_string(string_type& receiver, bool check_trailing, bool allow_replacement) noexcept { + std::string_view content; + auto err = get_root_string(check_trailing, allow_replacement).get(content); + if (err) { return err; } + receiver = content; + return SUCCESS; + } + simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_root_wobbly_string(bool check_trailing) noexcept { + return get_root_raw_json_string(check_trailing).unescape_wobbly(json_iter()); + } + simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_root_raw_json_string(bool check_trailing) noexcept { + auto json = peek_scalar("string"); + if (*json != '"') { return incorrect_type_error("Not a string"); } + if (check_trailing && !_json_iter->is_single_token()) { return TRAILING_CONTENT; } + advance_scalar("string"); + return raw_json_string(json + 1); + } + simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_root_uint64(bool check_trailing) noexcept { + auto max_len = peek_root_length(); + auto json = peek_root_scalar("uint64"); + uint8_t tmpbuf[20 + 1 + 1]{}; // <20 digits> is the longest possible unsigned integer + tmpbuf[20 + 1] = '\0'; // make sure that buffer is always null terminated. + if (!_json_iter->copy_to_buffer(json, max_len, tmpbuf, 20 + 1)) { + logger::log_error(*_json_iter, start_position(), depth(), "Root number more than 20 characters"); + return NUMBER_ERROR; + } + auto result = numberparsing::parse_unsigned(tmpbuf); + if (result.error() == SUCCESS) { + if (check_trailing && !_json_iter->is_single_token()) { return TRAILING_CONTENT; } + advance_root_scalar("uint64"); + } + return result; + } + simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_root_uint64_in_string(bool check_trailing) noexcept { + auto max_len = peek_root_length(); + auto json = peek_root_scalar("uint64"); + uint8_t tmpbuf[20 + 1 + 1]{}; // <20 digits> is the longest possible unsigned integer + tmpbuf[20 + 1] = '\0'; // make sure that buffer is always null terminated. + if (!_json_iter->copy_to_buffer(json, max_len, tmpbuf, 20 + 1)) { + logger::log_error(*_json_iter, start_position(), depth(), "Root number more than 20 characters"); + return NUMBER_ERROR; + } + auto result = numberparsing::parse_unsigned_in_string(tmpbuf); + if (result.error() == SUCCESS) { + if (check_trailing && !_json_iter->is_single_token()) { return TRAILING_CONTENT; } + advance_root_scalar("uint64"); + } + return result; + } + simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_root_int64(bool check_trailing) noexcept { + auto max_len = peek_root_length(); + auto json = peek_root_scalar("int64"); + uint8_t tmpbuf[20 + 1 + 1]; // -<19 digits> is the longest possible integer + tmpbuf[20 + 1] = '\0'; // make sure that buffer is always null terminated. + if (!_json_iter->copy_to_buffer(json, max_len, tmpbuf, 20 + 1)) { + logger::log_error(*_json_iter, start_position(), depth(), "Root number more than 20 characters"); + return NUMBER_ERROR; + } + + auto result = numberparsing::parse_integer(tmpbuf); + if (result.error() == SUCCESS) { + if (check_trailing && !_json_iter->is_single_token()) { return TRAILING_CONTENT; } + advance_root_scalar("int64"); + } + return result; + } + simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_root_int64_in_string(bool check_trailing) noexcept { + auto max_len = peek_root_length(); + auto json = peek_root_scalar("int64"); + uint8_t tmpbuf[20 + 1 + 1]; // -<19 digits> is the longest possible integer + tmpbuf[20 + 1] = '\0'; // make sure that buffer is always null terminated. + if (!_json_iter->copy_to_buffer(json, max_len, tmpbuf, 20 + 1)) { + logger::log_error(*_json_iter, start_position(), depth(), "Root number more than 20 characters"); + return NUMBER_ERROR; + } + + auto result = numberparsing::parse_integer_in_string(tmpbuf); + if (result.error() == SUCCESS) { + if (check_trailing && !_json_iter->is_single_token()) { return TRAILING_CONTENT; } + advance_root_scalar("int64"); + } + return result; + } + simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_root_double(bool check_trailing) noexcept { + auto max_len = peek_root_length(); + auto json = peek_root_scalar("double"); + // Per https://www.exploringbinary.com/maximum-number-of-decimal-digits-in-binary-floating-point-numbers/, + // 1074 is the maximum number of significant fractional digits. Add 8 more digits for the biggest + // number: -0.e-308. + uint8_t tmpbuf[1074 + 8 + 1 + 1]; // +1 for null termination. + tmpbuf[1074 + 8 + 1] = '\0'; // make sure that buffer is always null terminated. + if (!_json_iter->copy_to_buffer(json, max_len, tmpbuf, 1074 + 8 + 1)) { + logger::log_error(*_json_iter, start_position(), depth(), "Root number more than 1082 characters"); + return NUMBER_ERROR; + } + auto result = numberparsing::parse_double(tmpbuf); + if (result.error() == SUCCESS) { + if (check_trailing && !_json_iter->is_single_token()) { return TRAILING_CONTENT; } + advance_root_scalar("double"); + } + return result; + } + + simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_root_double_in_string(bool check_trailing) noexcept { + auto max_len = peek_root_length(); + auto json = peek_root_scalar("double"); + // Per https://www.exploringbinary.com/maximum-number-of-decimal-digits-in-binary-floating-point-numbers/, + // 1074 is the maximum number of significant fractional digits. Add 8 more digits for the biggest + // number: -0.e-308. + uint8_t tmpbuf[1074 + 8 + 1 + 1]; // +1 for null termination. + tmpbuf[1074 + 8 + 1] = '\0'; // make sure that buffer is always null terminated. + if (!_json_iter->copy_to_buffer(json, max_len, tmpbuf, 1074 + 8 + 1)) { + logger::log_error(*_json_iter, start_position(), depth(), "Root number more than 1082 characters"); + return NUMBER_ERROR; + } + auto result = numberparsing::parse_double_in_string(tmpbuf); + if (result.error() == SUCCESS) { + if (check_trailing && !_json_iter->is_single_token()) { return TRAILING_CONTENT; } + advance_root_scalar("double"); + } + return result; + } + simdjson_warn_unused simdjson_inline simdjson_result value_iterator::get_root_bool(bool check_trailing) noexcept { + auto max_len = peek_root_length(); + auto json = peek_root_scalar("bool"); + uint8_t tmpbuf[5 + 1 + 1]; // +1 for null termination + tmpbuf[5 + 1] = '\0'; // make sure that buffer is always null terminated. + if (!_json_iter->copy_to_buffer(json, max_len, tmpbuf, 5 + 1)) { return incorrect_type_error("Not a boolean"); } + auto result = parse_bool(tmpbuf); + if (result.error() == SUCCESS) { + if (check_trailing && !_json_iter->is_single_token()) { return TRAILING_CONTENT; } + advance_root_scalar("bool"); + } + return result; + } + simdjson_inline simdjson_result value_iterator::is_root_null(bool check_trailing) noexcept { + auto max_len = peek_root_length(); + auto json = peek_root_scalar("null"); + bool result = (max_len >= 4 && !atomparsing::str4ncmp(json, "null") && + (max_len == 4 || jsoncharutils::is_structural_or_whitespace(json[4]))); + if (result) { // we have something that looks like a null. + if (check_trailing && !_json_iter->is_single_token()) { return TRAILING_CONTENT; } + advance_root_scalar("null"); + } + return result; + } + + simdjson_warn_unused simdjson_inline error_code value_iterator::skip_child() noexcept { + SIMDJSON_ASSUME(_json_iter->token._position > _start_position); + SIMDJSON_ASSUME(_json_iter->_depth >= _depth); + + return _json_iter->skip_child(depth()); + } + + simdjson_inline value_iterator value_iterator::child() const noexcept { + assert_at_child(); + return { _json_iter, depth() + 1, _json_iter->token.position() }; + } + + // GCC 7 warns when the first line of this function is inlined away into oblivion due to the caller + // relating depth and iterator depth, which is a desired effect. It does not happen if is_open is + // marked non-inline. + SIMDJSON_PUSH_DISABLE_WARNINGS + SIMDJSON_DISABLE_STRICT_OVERFLOW_WARNING + simdjson_inline bool value_iterator::is_open() const noexcept { + return _json_iter->depth() >= depth(); + } + SIMDJSON_POP_DISABLE_WARNINGS + + simdjson_inline bool value_iterator::at_end() const noexcept { + return _json_iter->at_end(); + } + + simdjson_inline bool value_iterator::at_start() const noexcept { + return _json_iter->token.position() == start_position(); + } + + simdjson_inline bool value_iterator::at_first_field() const noexcept { + SIMDJSON_ASSUME(_json_iter->token._position > _start_position); + return _json_iter->token.position() == start_position() + 1; + } + + simdjson_inline void value_iterator::abandon() noexcept { + _json_iter->abandon(); + } + + simdjson_warn_unused simdjson_inline depth_t value_iterator::depth() const noexcept { + return _depth; + } + simdjson_warn_unused simdjson_inline error_code value_iterator::error() const noexcept { + return _json_iter->error; + } + simdjson_warn_unused simdjson_inline uint8_t*& value_iterator::string_buf_loc() noexcept { + return _json_iter->string_buf_loc(); + } + simdjson_warn_unused simdjson_inline const json_iterator& value_iterator::json_iter() const noexcept { + return *_json_iter; + } + simdjson_warn_unused simdjson_inline json_iterator& value_iterator::json_iter() noexcept { + return *_json_iter; + } + + simdjson_inline const uint8_t* value_iterator::peek_start() const noexcept { + return _json_iter->peek(start_position()); + } + simdjson_inline uint32_t value_iterator::peek_start_length() const noexcept { + return _json_iter->peek_length(start_position()); + } + simdjson_inline uint32_t value_iterator::peek_root_length() const noexcept { + return _json_iter->peek_root_length(start_position()); + } + + simdjson_inline const uint8_t* value_iterator::peek_scalar(const char* type) noexcept { + logger::log_value(*_json_iter, start_position(), depth(), type); + // If we're not at the position anymore, we don't want to advance the cursor. + if (!is_at_start()) { return peek_start(); } + + // Get the JSON and advance the cursor, decreasing depth to signify that we have retrieved the value. + assert_at_start(); + return _json_iter->peek(); + } + + simdjson_inline void value_iterator::advance_scalar(const char* type) noexcept { + logger::log_value(*_json_iter, start_position(), depth(), type); + // If we're not at the position anymore, we don't want to advance the cursor. + if (!is_at_start()) { return; } + + // Get the JSON and advance the cursor, decreasing depth to signify that we have retrieved the value. + assert_at_start(); + _json_iter->return_current_and_advance(); + _json_iter->ascend_to(depth() - 1); + } + + simdjson_inline error_code value_iterator::start_container(uint8_t start_char, const char* incorrect_type_message, const char* type) noexcept { + logger::log_start_value(*_json_iter, start_position(), depth(), type); + // If we're not at the position anymore, we don't want to advance the cursor. + const uint8_t* json; + if (!is_at_start()) { +#if SIMDJSON_DEVELOPMENT_CHECKS + if (!is_at_iterator_start()) { return OUT_OF_ORDER_ITERATION; } +#endif + json = peek_start(); + if (*json != start_char) { return incorrect_type_error(incorrect_type_message); } + } + else { + assert_at_start(); + /** + * We should be prudent. Let us peek. If it is not the right type, we + * return an error. Only once we have determined that we have the right + * type are we allowed to advance! + */ + json = _json_iter->peek(); + if (*json != start_char) { return incorrect_type_error(incorrect_type_message); } + _json_iter->return_current_and_advance(); + } + + + return SUCCESS; + } + + + simdjson_inline const uint8_t* value_iterator::peek_root_scalar(const char* type) noexcept { + logger::log_value(*_json_iter, start_position(), depth(), type); + if (!is_at_start()) { return peek_start(); } + + assert_at_root(); + return _json_iter->peek(); + } + simdjson_inline const uint8_t* value_iterator::peek_non_root_scalar(const char* type) noexcept { + logger::log_value(*_json_iter, start_position(), depth(), type); + if (!is_at_start()) { return peek_start(); } + + assert_at_non_root_start(); + return _json_iter->peek(); + } + + simdjson_inline void value_iterator::advance_root_scalar(const char* type) noexcept { + logger::log_value(*_json_iter, start_position(), depth(), type); + if (!is_at_start()) { return; } + + assert_at_root(); + _json_iter->return_current_and_advance(); + _json_iter->ascend_to(depth() - 1); + } + simdjson_inline void value_iterator::advance_non_root_scalar(const char* type) noexcept { + logger::log_value(*_json_iter, start_position(), depth(), type); + if (!is_at_start()) { return; } + + assert_at_non_root_start(); + _json_iter->return_current_and_advance(); + _json_iter->ascend_to(depth() - 1); + } + + simdjson_inline error_code value_iterator::incorrect_type_error(const char* message) const noexcept { + logger::log_error(*_json_iter, start_position(), depth(), message); + return INCORRECT_TYPE; + } + + simdjson_inline bool value_iterator::is_at_start() const noexcept { + return position() == start_position(); + } + + simdjson_inline bool value_iterator::is_at_key() const noexcept { + // Keys are at the same depth as the object. + // Note here that we could be safer and check that we are within an object, + // but we do not. + return _depth == _json_iter->_depth && *_json_iter->peek() == '"'; + } + + simdjson_inline bool value_iterator::is_at_iterator_start() const noexcept { + // We can legitimately be either at the first value ([1]), or after the array if it's empty ([]). + auto delta = position() - start_position(); + return delta == 1 || delta == 2; + } + + inline void value_iterator::assert_at_start() const noexcept { + SIMDJSON_ASSUME(_json_iter->token._position == _start_position); + SIMDJSON_ASSUME(_json_iter->_depth == _depth); + SIMDJSON_ASSUME(_depth > 0); + } + + inline void value_iterator::assert_at_container_start() const noexcept { + SIMDJSON_ASSUME(_json_iter->token._position == _start_position + 1); + SIMDJSON_ASSUME(_json_iter->_depth == _depth); + SIMDJSON_ASSUME(_depth > 0); + } + + inline void value_iterator::assert_at_next() const noexcept { + SIMDJSON_ASSUME(_json_iter->token._position > _start_position); + SIMDJSON_ASSUME(_json_iter->_depth == _depth); + SIMDJSON_ASSUME(_depth > 0); + } + + simdjson_inline void value_iterator::move_at_start() noexcept { + _json_iter->_depth = _depth; + _json_iter->token.set_position(_start_position); + } + + simdjson_inline void value_iterator::move_at_container_start() noexcept { + _json_iter->_depth = _depth; + _json_iter->token.set_position(_start_position + 1); + } + + simdjson_inline simdjson_result value_iterator::reset_array() noexcept { + if (error()) { return error(); } + move_at_container_start(); + return started_array(); + } + + simdjson_inline simdjson_result value_iterator::reset_object() noexcept { + if (error()) { return error(); } + move_at_container_start(); + return started_object(); + } + + inline void value_iterator::assert_at_child() const noexcept { + SIMDJSON_ASSUME(_json_iter->token._position > _start_position); + SIMDJSON_ASSUME(_json_iter->_depth == _depth + 1); + SIMDJSON_ASSUME(_depth > 0); + } + + inline void value_iterator::assert_at_root() const noexcept { + assert_at_start(); + SIMDJSON_ASSUME(_depth == 1); + } + + inline void value_iterator::assert_at_non_root_start() const noexcept { + assert_at_start(); + SIMDJSON_ASSUME(_depth > 1); + } + + inline void value_iterator::assert_is_valid() const noexcept { + SIMDJSON_ASSUME(_json_iter != nullptr); + } + + simdjson_inline bool value_iterator::is_valid() const noexcept { + return _json_iter != nullptr; + } + + simdjson_inline simdjson_result value_iterator::type() const noexcept { + switch (*peek_start()) { + case '{': + return json_type::object; + case '[': + return json_type::array; + case '"': + return json_type::string; + case 'n': + return json_type::null; + case 't': case 'f': + return json_type::boolean; + case '-': + case '0': case '1': case '2': case '3': case '4': + case '5': case '6': case '7': case '8': case '9': + return json_type::number; + default: + return TAPE_ERROR; + } + } + + simdjson_inline token_position value_iterator::start_position() const noexcept { + return _start_position; + } + + simdjson_inline token_position value_iterator::position() const noexcept { + return _json_iter->position(); + } + + simdjson_inline token_position value_iterator::end_position() const noexcept { + return _json_iter->end_position(); + } + + simdjson_inline token_position value_iterator::last_position() const noexcept { + return _json_iter->last_position(); + } + + simdjson_inline error_code value_iterator::report_error(error_code error, const char* message) noexcept { + return _json_iter->report_error(error, message); + } + + } // namespace ondemand + } // namespace lasx +} // namespace simdjson + +namespace simdjson { + + simdjson_inline simdjson_result::simdjson_result(lasx::ondemand::value_iterator&& value) noexcept + : implementation_simdjson_result_base(std::forward(value)) {} + simdjson_inline simdjson_result::simdjson_result(error_code error) noexcept + : implementation_simdjson_result_base(error) {} + +} // namespace simdjson + +#endif // SIMDJSON_GENERIC_ONDEMAND_VALUE_ITERATOR_INL_H +/* end file simdjson/generic/ondemand/value_iterator-inl.h for lasx */ +/* end file simdjson/generic/ondemand/amalgamated.h for lasx */ +/* including simdjson/lasx/end.h: #include "simdjson/lasx/end.h" */ +/* begin file simdjson/lasx/end.h */ +/* amalgamation skipped (editor-only): #ifndef SIMDJSON_CONDITIONAL_INCLUDE */ +/* amalgamation skipped (editor-only): #include "simdjson/lasx/base.h" */ +/* amalgamation skipped (editor-only): #endif // SIMDJSON_CONDITIONAL_INCLUDE */ + +#undef SIMDJSON_SKIP_BACKSLASH_SHORT_CIRCUIT +/* undefining SIMDJSON_IMPLEMENTATION from "lasx" */ +#undef SIMDJSON_IMPLEMENTATION +/* end file simdjson/lasx/end.h */ + +#endif // SIMDJSON_LASX_ONDEMAND_H +/* end file simdjson/lasx/ondemand.h */ #else #error Unknown SIMDJSON_BUILTIN_IMPLEMENTATION #endif