From d36e592afb3ca25125bc099ef3d46c3e9cb632b0 Mon Sep 17 00:00:00 2001 From: Thomas Harte Date: Fri, 22 May 2020 00:31:40 -0400 Subject: [PATCH 01/10] Starts towards BSON serialisation for all deflectable structs. Still to be tackled: arrays, enumerated types should probably be encoded as strings, deserialisation, probably distinguish get and fuzzy_get... --- Reflection/Struct.cpp | 165 ++++++++++++++++++++++++++++++++++++++++++ Reflection/Struct.hpp | 24 +++--- 2 files changed, 179 insertions(+), 10 deletions(-) diff --git a/Reflection/Struct.cpp b/Reflection/Struct.cpp index c3b676805..55e6b90bd 100644 --- a/Reflection/Struct.cpp +++ b/Reflection/Struct.cpp @@ -10,9 +10,51 @@ #include #include +#include #include #include +#define ForAllInts(x) \ + x(uint8_t); \ + x(int8_t); \ + x(uint16_t); \ + x(int16_t); \ + x(uint32_t); \ + x(int32_t); \ + x(uint64_t); \ + x(int64_t); + +#define ForAllFloats(x) \ + x(float); \ + x(double); + +namespace TypeInfo { + +static bool is_integral(const std::type_info *type) { + return + *type == typeid(uint8_t) || *type == typeid(int8_t) || + *type == typeid(uint16_t) || *type == typeid(int16_t) || + *type == typeid(uint32_t) || *type == typeid(int32_t) || + *type == typeid(uint64_t) || *type == typeid(int64_t); +} + +static bool is_floating_point(const std::type_info *type) { + return *type == typeid(float) || *type == typeid(double); +} + +static size_t size(const std::type_info *type) { +#define TestType(x) if(*type == typeid(x)) return sizeof(x); + ForAllInts(TestType); + ForAllFloats(TestType); + TestType(char *); +#undef TestType + + // This is some sort of struct or object type. + return 0; +} + +} + // MARK: - Setters template <> bool Reflection::set(Struct &target, const std::string &name, int value) { @@ -128,6 +170,34 @@ template bool Reflection::get(const Struct &target, const std::s } } + // If the type is an int that is larger than the stored type, cast upward. + if constexpr (std::is_integral::value) { + constexpr size_t size = sizeof(Type); + const bool target_is_integral = TypeInfo::is_integral(target_type); + const size_t target_size = TypeInfo::size(target_type); + + if(size > target_size && target_is_integral) { +#define Map(x) if(*target_type == typeid(x)) { value = static_cast(*reinterpret_cast(target.get(name))); } + ForAllInts(Map); +#undef Map + return true; + } + } + + // If the type is a double and stored type is a float, cast upward. + if constexpr (std::is_floating_point::value) { + constexpr size_t size = sizeof(Type); + const bool target_is_floating_point = TypeInfo::is_floating_point(target_type); + const size_t target_size = TypeInfo::size(target_type); + + if(size > target_size && target_is_floating_point) { +#define Map(x) if(*target_type == typeid(x)) { value = static_cast(*reinterpret_cast(target.get(name))); } + ForAllFloats(Map); +#undef Map + return true; + } + } + return false; } @@ -215,3 +285,98 @@ std::string Reflection::Struct::description() const { return stream.str(); } + +std::vector Reflection::Struct::serialise() const { + std::vector result; + + /* Contractually, this serialises as BSON. */ + + for(const auto &key: all_keys()) { + if(!should_serialise(key)) continue; + + /* Here: e_list ::= element e_list | "" */ + const auto count = count_of(key); + const auto type = type_of(key); + + if(count > 1) { + // TODO: Arrays. + } else { + auto push_name = [&result, &key] () { + std::copy(key.begin(), key.end(), std::back_inserter(result)); + result.push_back(0); + }; + + auto push_int = [push_name, &result] (uint8_t type, auto x) { + result.push_back(type); + push_name(); + for(size_t c = 0; c < sizeof(x); ++c) + result.push_back(uint8_t((x) >> (8 * c))); + }; + + // Test for an exact match on Booleans. + if(*type == typeid(bool)) { + result.push_back(0x08); + push_name(); + result.push_back(uint8_t(Reflection::get(*this, key))); + continue; + } + + // Test for ints that will safely convert to an int32. + int32_t int32; + if(Reflection::get(*this, key, int32)) { + push_int(0x10, int32); + continue; + } + + // Test for ints that can be converted to a uint64. + uint32_t uint64; + if(Reflection::get(*this, key, uint64)) { + push_int(0x11, uint64); + continue; + } + + // Test for ints that can be converted to an int64. + int32_t int64; + if(Reflection::get(*this, key, int64)) { + push_int(0x12, int64); + continue; + } + + + /* All ints should now be dealt with. */ + + // There's only one potential float type: a double. + double float64; + if(Reflection::get(*this, key, float64)) { + // TODO: place as little-endian IEEE 754-2008. + continue; + } + + // Okay, check for a potential recursion. + if(*type == typeid(Reflection::Struct)) { + result.push_back(0x03); + push_name(); + + const Reflection::Struct *const child = reinterpret_cast(get(key)); + const auto sub_document = child->serialise(); + std::copy(sub_document.begin(), sub_document.end(), std::back_inserter(result)); + continue; + } + + // Should never reach here; that means a type was discovered in a struct which is intended for + // serialisation but which could not be parsed. + assert(false); + } + } + + /* + document ::= int32 e_list "\x00" + The int32 is the total number of bytes comprising the document. + */ + result.push_back(0); + const uint32_t size_with_prefix = uint32_t(result.size()) + 2; + result.insert(result.begin(), uint8_t(size_with_prefix >> 8)); + result.insert(result.begin(), uint8_t(size_with_prefix & 0xff)); + + return result; +} diff --git a/Reflection/Struct.hpp b/Reflection/Struct.hpp index f0d22e5e9..f777f1a16 100644 --- a/Reflection/Struct.hpp +++ b/Reflection/Struct.hpp @@ -38,6 +38,20 @@ struct Struct { */ std::string description() const; + /*! + @returns The BSON serialisation of this struct. + */ + std::vector serialise() const; + + /*! + Applies as many fields as possible from the incoming BSON. + */ + bool deserialise(const std::vector &bson); + + /*! + */ + virtual bool should_serialise(const std::string &key) const { return true; } + private: void append(std::ostringstream &stream, const std::string &key, const std::type_info *type, size_t offset) const; }; @@ -105,16 +119,6 @@ template bool get(const Struct &target, const std::string &name, */ template Type get(const Struct &target, const std::string &name, size_t offset = 0); - -// TODO: move this elsewhere. It's just a sketch anyway. -struct Serialisable { - /// Serialises this object, appending it to @c target. - virtual void serialise(std::vector &target) = 0; - /// Deserialises this object from @c source. - /// @returns @c true if the deserialisation was successful; @c false otherwise. - virtual bool deserialise(const std::vector &source) = 0; -}; - template class StructImpl: public Struct { public: /*! From 39ffe45f3c2a6eb2ca638ce80fd0d452e388a387 Mon Sep 17 00:00:00 2001 From: Thomas Harte Date: Fri, 22 May 2020 21:55:12 -0400 Subject: [PATCH 02/10] Attempts to add support for arrays. --- Reflection/Struct.cpp | 184 ++++++++++++++++++++++++------------------ 1 file changed, 105 insertions(+), 79 deletions(-) diff --git a/Reflection/Struct.cpp b/Reflection/Struct.cpp index 55e6b90bd..1bcaf547d 100644 --- a/Reflection/Struct.cpp +++ b/Reflection/Struct.cpp @@ -9,6 +9,7 @@ #include "Struct.hpp" #include +#include #include #include #include @@ -42,6 +43,16 @@ static bool is_floating_point(const std::type_info *type) { return *type == typeid(float) || *type == typeid(double); } +static bool is_signed(const std::type_info *type) { + return + *type == typeid(int8_t) || + *type == typeid(int16_t) || + *type == typeid(int32_t) || + *type == typeid(int64_t) || + *type == typeid(double) || + *type == typeid(float); +} + static size_t size(const std::type_info *type) { #define TestType(x) if(*type == typeid(x)) return sizeof(x); ForAllInts(TestType); @@ -170,13 +181,14 @@ template bool Reflection::get(const Struct &target, const std::s } } - // If the type is an int that is larger than the stored type, cast upward. + // If the type is an int that is larger than the stored type and matches the signedness, cast upward. if constexpr (std::is_integral::value) { constexpr size_t size = sizeof(Type); const bool target_is_integral = TypeInfo::is_integral(target_type); + const bool signs_match = std::is_signed::value == TypeInfo::is_signed(target_type); const size_t target_size = TypeInfo::size(target_type); - if(size > target_size && target_is_integral) { + if(signs_match && size > target_size && target_is_integral) { #define Map(x) if(*target_type == typeid(x)) { value = static_cast(*reinterpret_cast(target.get(name))); } ForAllInts(Map); #undef Map @@ -286,10 +298,87 @@ std::string Reflection::Struct::description() const { return stream.str(); } +/* Contractually, this serialises as BSON. */ std::vector Reflection::Struct::serialise() const { - std::vector result; + auto append = [this] (std::vector &result, const std::string &key, const std::string &output_name, const std::type_info *type, size_t offset) { + auto push_name = [&result, &output_name] () { + std::copy(output_name.begin(), output_name.end(), std::back_inserter(result)); + result.push_back(0); + }; - /* Contractually, this serialises as BSON. */ + auto push_int = [push_name, &result] (uint8_t type, auto x) { + result.push_back(type); + push_name(); + for(size_t c = 0; c < sizeof(x); ++c) + result.push_back(uint8_t((x) >> (8 * c))); + }; + + // Test for an exact match on Booleans. + if(*type == typeid(bool)) { + result.push_back(0x08); + push_name(); + result.push_back(uint8_t(Reflection::get(*this, key))); + return; + } + + // Test for ints that will safely convert to an int32. + int32_t int32; + if(Reflection::get(*this, key, int32)) { + push_int(0x10, int32); + return; + } + + // Test for ints that can be converted to a uint64. + uint32_t uint64; + if(Reflection::get(*this, key, uint64)) { + push_int(0x11, uint64); + return; + } + + // Test for ints that can be converted to an int64. + int32_t int64; + if(Reflection::get(*this, key, int64)) { + push_int(0x12, int64); + return; + } + + /* All ints should now be dealt with. */ + + // There's only one potential float type: a double. + double float64; + if(Reflection::get(*this, key, float64)) { + // TODO: place as little-endian IEEE 754-2008. + return; + } + + // Okay, check for a potential recursion. + if(*type == typeid(Reflection::Struct)) { + result.push_back(0x03); + push_name(); + + const Reflection::Struct *const child = reinterpret_cast(get(key)); + const auto sub_document = child->serialise(); + std::copy(sub_document.begin(), sub_document.end(), std::back_inserter(result)); + return; + } + + // Should never reach here; that means a type was discovered in a struct which is intended for + // serialisation but which could not be parsed. + assert(false); + }; + + auto wrap_object = [] (std::vector &data) { + /* + document ::= int32 e_list "\x00" + The int32 is the total number of bytes comprising the document. + */ + data.push_back(0); + const uint32_t size_with_prefix = uint32_t(data.size()) + 2; + data.insert(data.begin(), uint8_t(size_with_prefix >> 8)); + data.insert(data.begin(), uint8_t(size_with_prefix & 0xff)); + }; + + std::vector result; for(const auto &key: all_keys()) { if(!should_serialise(key)) continue; @@ -299,84 +388,21 @@ std::vector Reflection::Struct::serialise() const { const auto type = type_of(key); if(count > 1) { - // TODO: Arrays. + // In BSON, an array is a sub-document with ASCII keys '0', '1', etc. + result.push_back(0x04); + + std::vector array; + for(size_t c = 0; c < count; ++c) { + append(array, key, std::to_string(c), type, c); + } + wrap_object(array); + + std::copy(array.begin(), array.end(), std::back_inserter(result)); } else { - auto push_name = [&result, &key] () { - std::copy(key.begin(), key.end(), std::back_inserter(result)); - result.push_back(0); - }; - - auto push_int = [push_name, &result] (uint8_t type, auto x) { - result.push_back(type); - push_name(); - for(size_t c = 0; c < sizeof(x); ++c) - result.push_back(uint8_t((x) >> (8 * c))); - }; - - // Test for an exact match on Booleans. - if(*type == typeid(bool)) { - result.push_back(0x08); - push_name(); - result.push_back(uint8_t(Reflection::get(*this, key))); - continue; - } - - // Test for ints that will safely convert to an int32. - int32_t int32; - if(Reflection::get(*this, key, int32)) { - push_int(0x10, int32); - continue; - } - - // Test for ints that can be converted to a uint64. - uint32_t uint64; - if(Reflection::get(*this, key, uint64)) { - push_int(0x11, uint64); - continue; - } - - // Test for ints that can be converted to an int64. - int32_t int64; - if(Reflection::get(*this, key, int64)) { - push_int(0x12, int64); - continue; - } - - - /* All ints should now be dealt with. */ - - // There's only one potential float type: a double. - double float64; - if(Reflection::get(*this, key, float64)) { - // TODO: place as little-endian IEEE 754-2008. - continue; - } - - // Okay, check for a potential recursion. - if(*type == typeid(Reflection::Struct)) { - result.push_back(0x03); - push_name(); - - const Reflection::Struct *const child = reinterpret_cast(get(key)); - const auto sub_document = child->serialise(); - std::copy(sub_document.begin(), sub_document.end(), std::back_inserter(result)); - continue; - } - - // Should never reach here; that means a type was discovered in a struct which is intended for - // serialisation but which could not be parsed. - assert(false); + append(result, key, key, type, 0); } } - /* - document ::= int32 e_list "\x00" - The int32 is the total number of bytes comprising the document. - */ - result.push_back(0); - const uint32_t size_with_prefix = uint32_t(result.size()) + 2; - result.insert(result.begin(), uint8_t(size_with_prefix >> 8)); - result.insert(result.begin(), uint8_t(size_with_prefix & 0xff)); - + wrap_object(result); return result; } From b3587d4cde67969aae7cd93848f51b2386dfee2c Mon Sep 17 00:00:00 2001 From: Thomas Harte Date: Fri, 22 May 2020 23:38:07 -0400 Subject: [PATCH 03/10] Corrects: logic for int promotion, object sizes, int64_t gets, sizes prefixed to objects. --- Reflection/Struct.cpp | 48 +++++++++++++++++++++++-------------------- 1 file changed, 26 insertions(+), 22 deletions(-) diff --git a/Reflection/Struct.cpp b/Reflection/Struct.cpp index 1bcaf547d..7bb62d5a0 100644 --- a/Reflection/Struct.cpp +++ b/Reflection/Struct.cpp @@ -183,16 +183,20 @@ template bool Reflection::get(const Struct &target, const std::s // If the type is an int that is larger than the stored type and matches the signedness, cast upward. if constexpr (std::is_integral::value) { - constexpr size_t size = sizeof(Type); - const bool target_is_integral = TypeInfo::is_integral(target_type); - const bool signs_match = std::is_signed::value == TypeInfo::is_signed(target_type); - const size_t target_size = TypeInfo::size(target_type); + if(TypeInfo::is_integral(target_type)) { + const bool target_is_signed = TypeInfo::is_signed(target_type); + const size_t target_size = TypeInfo::size(target_type); - if(signs_match && size > target_size && target_is_integral) { -#define Map(x) if(*target_type == typeid(x)) { value = static_cast(*reinterpret_cast(target.get(name))); } - ForAllInts(Map); + // An unsigned type can map to any larger type, signed or unsigned; + // a signed type can map to a larger type only if it also is signed. + if(sizeof(Type) > target_size && (!target_is_signed || std::is_signed::value)) { + const auto address = reinterpret_cast(target.get(name)) + offset * target_size; + +#define Map(x) if(*target_type == typeid(x)) { value = static_cast(*reinterpret_cast(address)); } + ForAllInts(Map); #undef Map - return true; + return true; + } } } @@ -203,7 +207,9 @@ template bool Reflection::get(const Struct &target, const std::s const size_t target_size = TypeInfo::size(target_type); if(size > target_size && target_is_floating_point) { -#define Map(x) if(*target_type == typeid(x)) { value = static_cast(*reinterpret_cast(target.get(name))); } + const auto address = reinterpret_cast(target.get(name)) + offset * target_size; + +#define Map(x) if(*target_type == typeid(x)) { value = static_cast(*reinterpret_cast(address)); } ForAllFloats(Map); #undef Map return true; @@ -317,27 +323,20 @@ std::vector Reflection::Struct::serialise() const { if(*type == typeid(bool)) { result.push_back(0x08); push_name(); - result.push_back(uint8_t(Reflection::get(*this, key))); + result.push_back(uint8_t(Reflection::get(*this, key, offset))); return; } // Test for ints that will safely convert to an int32. int32_t int32; - if(Reflection::get(*this, key, int32)) { + if(Reflection::get(*this, key, int32, offset)) { push_int(0x10, int32); return; } - // Test for ints that can be converted to a uint64. - uint32_t uint64; - if(Reflection::get(*this, key, uint64)) { - push_int(0x11, uint64); - return; - } - // Test for ints that can be converted to an int64. - int32_t int64; - if(Reflection::get(*this, key, int64)) { + int64_t int64; + if(Reflection::get(*this, key, int64, offset)) { push_int(0x12, int64); return; } @@ -346,12 +345,13 @@ std::vector Reflection::Struct::serialise() const { // There's only one potential float type: a double. double float64; - if(Reflection::get(*this, key, float64)) { + if(Reflection::get(*this, key, float64, offset)) { // TODO: place as little-endian IEEE 754-2008. return; } // Okay, check for a potential recursion. + // Not currently supported: arrays of structs. if(*type == typeid(Reflection::Struct)) { result.push_back(0x03); push_name(); @@ -373,7 +373,9 @@ std::vector Reflection::Struct::serialise() const { The int32 is the total number of bytes comprising the document. */ data.push_back(0); - const uint32_t size_with_prefix = uint32_t(data.size()) + 2; + const uint32_t size_with_prefix = uint32_t(data.size()) + 4; + data.insert(data.begin(), uint8_t(size_with_prefix >> 24)); + data.insert(data.begin(), uint8_t(size_with_prefix >> 16)); data.insert(data.begin(), uint8_t(size_with_prefix >> 8)); data.insert(data.begin(), uint8_t(size_with_prefix & 0xff)); }; @@ -390,6 +392,8 @@ std::vector Reflection::Struct::serialise() const { if(count > 1) { // In BSON, an array is a sub-document with ASCII keys '0', '1', etc. result.push_back(0x04); + std::copy(key.begin(), key.end(), std::back_inserter(result)); + result.push_back(0); std::vector array; for(size_t c = 0; c < count; ++c) { From bb2f21a22e6a9dcceec2d0442f018de525c6d0f5 Mon Sep 17 00:00:00 2001 From: Thomas Harte Date: Sat, 23 May 2020 22:54:43 -0400 Subject: [PATCH 04/10] Encodes enumerated values as strings. --- Reflection/Struct.cpp | 45 +++++++++++++++++++++++++++++++------------ 1 file changed, 33 insertions(+), 12 deletions(-) diff --git a/Reflection/Struct.cpp b/Reflection/Struct.cpp index 7bb62d5a0..34d0b2c7e 100644 --- a/Reflection/Struct.cpp +++ b/Reflection/Struct.cpp @@ -306,27 +306,49 @@ std::string Reflection::Struct::description() const { /* Contractually, this serialises as BSON. */ std::vector Reflection::Struct::serialise() const { - auto append = [this] (std::vector &result, const std::string &key, const std::string &output_name, const std::type_info *type, size_t offset) { - auto push_name = [&result, &output_name] () { - std::copy(output_name.begin(), output_name.end(), std::back_inserter(result)); - result.push_back(0); - }; + auto push_name = [] (std::vector &result, const std::string &name) { + std::copy(name.begin(), name.end(), std::back_inserter(result)); + result.push_back(0); + }; - auto push_int = [push_name, &result] (uint8_t type, auto x) { + auto append = [push_name, this] (std::vector &result, const std::string &key, const std::string &output_name, const std::type_info *type, size_t offset) { + auto push_int = [push_name, &result, &output_name] (uint8_t type, auto x) { result.push_back(type); - push_name(); + push_name(result, output_name); for(size_t c = 0; c < sizeof(x); ++c) result.push_back(uint8_t((x) >> (8 * c))); }; + auto push_string = [push_name, &result, &output_name] (const std::string &text) { + result.push_back(0x02); + push_name(result, output_name); + + const uint32_t string_length = uint32_t(text.size() + 1); + result.push_back(uint8_t(string_length)); + result.push_back(uint8_t(string_length >> 8)); + result.push_back(uint8_t(string_length >> 16)); + result.push_back(uint8_t(string_length >> 24)); + std::copy(text.begin(), text.end(), std::back_inserter(result)); + result.push_back(0); + }; + // Test for an exact match on Booleans. if(*type == typeid(bool)) { result.push_back(0x08); - push_name(); + push_name(result, output_name); result.push_back(uint8_t(Reflection::get(*this, key, offset))); return; } + // Record the string value for enums. + if(!Reflection::Enum::name(*type).empty()) { + int value; + Reflection::get(*this, key, value, offset); + const auto text = Reflection::Enum::to_string(*type, 0); + push_string(text); + return; + } + // Test for ints that will safely convert to an int32. int32_t int32; if(Reflection::get(*this, key, int32, offset)) { @@ -354,7 +376,7 @@ std::vector Reflection::Struct::serialise() const { // Not currently supported: arrays of structs. if(*type == typeid(Reflection::Struct)) { result.push_back(0x03); - push_name(); + push_name(result, output_name); const Reflection::Struct *const child = reinterpret_cast(get(key)); const auto sub_document = child->serialise(); @@ -391,9 +413,8 @@ std::vector Reflection::Struct::serialise() const { if(count > 1) { // In BSON, an array is a sub-document with ASCII keys '0', '1', etc. - result.push_back(0x04); - std::copy(key.begin(), key.end(), std::back_inserter(result)); - result.push_back(0); + result.push_back(0x03); // TODO: 0x04. + push_name(result, key); std::vector array; for(size_t c = 0; c < count; ++c) { From b8b880a91d379a7b9b618ad82bf68f46dbf022d7 Mon Sep 17 00:00:00 2001 From: Thomas Harte Date: Sun, 24 May 2020 01:20:48 -0400 Subject: [PATCH 05/10] Extends encoding to handle vector, floats and doubles. --- Reflection/Struct.cpp | 56 ++++++++++++++++++++++++++++++++----------- Reflection/Struct.hpp | 5 ++-- 2 files changed, 45 insertions(+), 16 deletions(-) diff --git a/Reflection/Struct.cpp b/Reflection/Struct.cpp index 34d0b2c7e..330a4f545 100644 --- a/Reflection/Struct.cpp +++ b/Reflection/Struct.cpp @@ -312,22 +312,23 @@ std::vector Reflection::Struct::serialise() const { }; auto append = [push_name, this] (std::vector &result, const std::string &key, const std::string &output_name, const std::type_info *type, size_t offset) { - auto push_int = [push_name, &result, &output_name] (uint8_t type, auto x) { - result.push_back(type); - push_name(result, output_name); + auto push_int = [push_name, &result, &output_name] (auto x) { for(size_t c = 0; c < sizeof(x); ++c) result.push_back(uint8_t((x) >> (8 * c))); }; - auto push_string = [push_name, &result, &output_name] (const std::string &text) { + auto push_named_int = [push_int, push_name, &result, &output_name] (uint8_t type, auto x) { + result.push_back(type); + push_name(result, output_name); + push_int(x); + }; + + auto push_string = [push_int, push_name, &result, &output_name] (const std::string &text) { result.push_back(0x02); push_name(result, output_name); const uint32_t string_length = uint32_t(text.size() + 1); - result.push_back(uint8_t(string_length)); - result.push_back(uint8_t(string_length >> 8)); - result.push_back(uint8_t(string_length >> 16)); - result.push_back(uint8_t(string_length >> 24)); + push_int(string_length); std::copy(text.begin(), text.end(), std::back_inserter(result)); result.push_back(0); }; @@ -352,23 +353,50 @@ std::vector Reflection::Struct::serialise() const { // Test for ints that will safely convert to an int32. int32_t int32; if(Reflection::get(*this, key, int32, offset)) { - push_int(0x10, int32); + push_named_int(0x10, int32); return; } // Test for ints that can be converted to an int64. int64_t int64; if(Reflection::get(*this, key, int64, offset)) { - push_int(0x12, int64); + push_named_int(0x12, int64); return; } - /* All ints should now be dealt with. */ - - // There's only one potential float type: a double. + // There's only one BSON float type: a double. double float64; if(Reflection::get(*this, key, float64, offset)) { - // TODO: place as little-endian IEEE 754-2008. + result.push_back(0x01); + push_name(result, output_name); + + // The following declines to assume an internal representation + // for doubles, constructing IEEE 708 from first principles. + // Which is probably absurd given how often I've assumed + // e.g. two's complement. + int exponent; + const double mantissa = frexp(fabs(float64), &exponent); + exponent += 1022; + const uint64_t integer_mantissa = + static_cast(mantissa * 9007199254740992.0); + const uint64_t binary64 = + ((float64 < 0) ? 0x8000'0000'0000'0000 : 0) | + (integer_mantissa & 0x000f'ffff'ffff'ffff) | + (static_cast(exponent) << 52); + push_int(binary64); + + return; + } + + // Store std::vectors as binary data. + if(*type == typeid(std::vector)) { + result.push_back(0x05); + push_name(result, output_name); + + auto source = reinterpret_cast *>(get(key)); + push_int(uint32_t(source->size())); + result.push_back(0x00); + std::copy(source->begin(), source->end(), std::back_inserter(result)); return; } diff --git a/Reflection/Struct.hpp b/Reflection/Struct.hpp index f777f1a16..703e39ad1 100644 --- a/Reflection/Struct.hpp +++ b/Reflection/Struct.hpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include @@ -295,8 +296,8 @@ template class StructImpl: public Struct { private: template bool declare_reflectable(Type *t, const std::string &name) { - Reflection::Struct *const str = static_cast(t); - if(str) { + if constexpr (std::is_base_of::value) { + Reflection::Struct *const str = static_cast(t); declare_emplace(str, name); return true; } From c83c82748408356fd88f15a84bbc2ef9a024b84b Mon Sep 17 00:00:00 2001 From: Thomas Harte Date: Sun, 24 May 2020 12:19:20 -0400 Subject: [PATCH 06/10] Adds necessary header for math. --- Reflection/Struct.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/Reflection/Struct.cpp b/Reflection/Struct.cpp index 330a4f545..e07f9cc79 100644 --- a/Reflection/Struct.cpp +++ b/Reflection/Struct.cpp @@ -10,6 +10,7 @@ #include #include +#include #include #include #include From 4d34d9ae2bc754fde179f551875b068d0b13a3c8 Mon Sep 17 00:00:00 2001 From: Thomas Harte Date: Mon, 25 May 2020 23:39:00 -0400 Subject: [PATCH 07/10] Implements BSON deserialisation, other than arrays. --- .../Implementation/MultiConfigurable.cpp | 8 + Reflection/Struct.cpp | 175 +++++++++++++++++- Reflection/Struct.hpp | 33 +++- 3 files changed, 207 insertions(+), 9 deletions(-) diff --git a/Analyser/Dynamic/MultiMachine/Implementation/MultiConfigurable.cpp b/Analyser/Dynamic/MultiMachine/Implementation/MultiConfigurable.cpp index 5d8fb89cd..df7333924 100644 --- a/Analyser/Dynamic/MultiMachine/Implementation/MultiConfigurable.cpp +++ b/Analyser/Dynamic/MultiMachine/Implementation/MultiConfigurable.cpp @@ -72,6 +72,14 @@ class MultiStruct: public Reflection::Struct { return nullptr; } + void *get(const std::string &name) final { + for(auto &options: options_) { + auto value = options->get(name); + if(value) return value; + } + return nullptr; + } + void set(const std::string &name, const void *value) final { const auto safe_type = type_of(name); if(!safe_type) return; diff --git a/Reflection/Struct.cpp b/Reflection/Struct.cpp index e07f9cc79..8dc88f166 100644 --- a/Reflection/Struct.cpp +++ b/Reflection/Struct.cpp @@ -69,22 +69,58 @@ static size_t size(const std::type_info *type) { // MARK: - Setters +template <> bool Reflection::set(Struct &target, const std::string &name, float value) { + const auto target_type = target.type_of(name); + if(!target_type) return false; + + if(*target_type == typeid(float)) { + target.set(name, &value); + return true; + } + + return set(target, name, value); +} + +template <> bool Reflection::set(Struct &target, const std::string &name, double value) { + const auto target_type = target.type_of(name); + if(!target_type) return false; + + if(*target_type == typeid(double)) { + target.set(name, &value); + return true; + } + + if(*target_type == typeid(float)) { + const float float_value = float(value); + target.set(name, &float_value); + return true; + } + + return false; +} + template <> bool Reflection::set(Struct &target, const std::string &name, int value) { + return set(target, name, value); +} + +template <> bool Reflection::set(Struct &target, const std::string &name, int64_t value) { const auto target_type = target.type_of(name); if(!target_type) return false; // No need to convert an int or a registered enum. if(*target_type == typeid(int) || !Reflection::Enum::name(*target_type).empty()) { + const int value32 = int(value); + target.set(name, &value32); + return true; + } + + // Set an int64_t directly. + if(*target_type == typeid(int64_t)) { target.set(name, &value); return true; } - // Promote to an int64_t. - if(*target_type == typeid(int64_t)) { - const auto int64 = int64_t(value); - target.set(name, &int64); - return true; - } + // TODO: other int sizes. return false; } @@ -93,6 +129,14 @@ template <> bool Reflection::set(Struct &target, const std::string &name, const const auto target_type = target.type_of(name); if(!target_type) return false; + // If the target is a string, assign. + if(*target_type == typeid(std::string)) { + auto child = reinterpret_cast(target.get(name)); + *child = value; + return true; + } + + // From here on, make an attempt to convert to a named enum. if(Reflection::Enum::name(*target_type).empty()) { return false; } @@ -389,6 +433,14 @@ std::vector Reflection::Struct::serialise() const { return; } + // Strings are written naturally. + if(*type == typeid(std::string)) { + const uint8_t *address = reinterpret_cast(get(key)); + const std::string *const text = reinterpret_cast(address + offset*sizeof(std::string)); + push_string(*text); + return; + } + // Store std::vectors as binary data. if(*type == typeid(std::vector)) { result.push_back(0x05); @@ -442,7 +494,7 @@ std::vector Reflection::Struct::serialise() const { if(count > 1) { // In BSON, an array is a sub-document with ASCII keys '0', '1', etc. - result.push_back(0x03); // TODO: 0x04. + result.push_back(0x04); push_name(result, key); std::vector array; @@ -460,3 +512,112 @@ std::vector Reflection::Struct::serialise() const { wrap_object(result); return result; } + +bool Reflection::Struct::deserialise(const std::vector &bson) { + return deserialise(bson.data(), bson.size()); +} + +bool Reflection::Struct::deserialise(const uint8_t *bson, size_t size) { + // Validate the object's declared size. + const auto end = bson + size; + auto read_int = [&bson] (auto &target) { + constexpr auto shift = 8 * (sizeof(target) - 1); + target = 0; + for(size_t c = 0; c < sizeof(target); ++c) { + target >>= 8; + target |= decltype(target)(*bson) << shift; + ++bson; + } + }; + + uint32_t object_size; + read_int(object_size); + if(object_size > size) return false; + + while(true) { + const uint8_t next_type = *bson; + ++bson; + if(!next_type) break; + + std::string key; + while(*bson) { + key.push_back(char(*bson)); + ++bson; + } + ++bson; + + switch(next_type) { + default: + return false; + + // TODO: arrays. + + // 0x03: A subdocument; try to install the inner BSON. + // 0x05: Binary data. Seek to populate a std::vector. + case 0x03: + case 0x05: { + const auto type = type_of(key); + + uint32_t subobject_size; + read_int(subobject_size); + + if(next_type == 0x03 && *type == typeid(Reflection::Struct)) { + auto child = reinterpret_cast(get(key)); + child->deserialise(bson - 4, size_t(end - bson + 4)); + bson += subobject_size - 4; + } + + if(next_type == 0x05 && *type == typeid(std::vector)) { + auto child = reinterpret_cast *>(get(key)); + *child = std::vector(bson, bson + subobject_size); + bson += subobject_size; + } + } break; + + // String. + case 0x02: { + uint32_t length; + read_int(length); + + const std::string value(bson, bson + length - 1); + ::Reflection::set(*this, key, value); + + bson += length; + } break; + + // Boolean. + case 0x08: { + const bool value = *bson; + ++bson; + ::Reflection::set(*this, key, value); + } break; + + // 32-bit int. + case 0x10: { + int32_t value; + read_int(value); + ::Reflection::set(*this, key, value); + } break; + + // 64-bit int. + case 0x12: { + int64_t value; + read_int(value); + ::Reflection::set(*this, key, value); + } break; + + // 64-bit double. + case 0x01: { + uint64_t value; + read_int(value); + + // TODO: real conversion. + double double_value = double(value); + + ::Reflection::set(*this, key, double_value); + } break; + } + } + + return true; +} diff --git a/Reflection/Struct.hpp b/Reflection/Struct.hpp index 703e39ad1..3cf0a4ab0 100644 --- a/Reflection/Struct.hpp +++ b/Reflection/Struct.hpp @@ -29,6 +29,7 @@ struct Struct { virtual const std::type_info *type_of(const std::string &name) const = 0; virtual size_t count_of(const std::string &name) const = 0; virtual void set(const std::string &name, const void *value) = 0; + virtual void *get(const std::string &name) = 0; virtual const void *get(const std::string &name) const = 0; virtual std::vector values_for(const std::string &name) const = 0; virtual ~Struct() {} @@ -40,21 +41,38 @@ struct Struct { std::string description() const; /*! - @returns The BSON serialisation of this struct. + Serialises this struct in BSON format. + + Supported field types: + + * [u]int[8/16/32/64]_t; + * float and double; + * bool; + * std::string; + * plain C-style arrays of any of the above; + * other reflective structs; + * std::vector as raw binary data. + + TODO: vector of string, possibly? Or more general vector support? + + @returns The BSON serialisation. */ std::vector serialise() const; /*! - Applies as many fields as possible from the incoming BSON. + Applies as many fields as possible from the incoming BSON. Supports the same types + as @c serialise. */ bool deserialise(const std::vector &bson); /*! + Called to determine whether @c key should be included in the serialisation of this struct. */ virtual bool should_serialise(const std::string &key) const { return true; } private: void append(std::ostringstream &stream, const std::string &key, const std::type_info *type, size_t offset) const; + bool deserialise(const uint8_t *bson, size_t size); }; /*! @@ -68,9 +86,11 @@ template bool set(Struct &target, const std::string &name, Type Setting an int: * to an int copies the int; + * to a smaller type, truncates the int; * to an int64_t promotes the int; and * to a registered enum, copies the int. */ +template <> bool set(Struct &target, const std::string &name, int64_t value); template <> bool set(Struct &target, const std::string &name, int value); /*! @@ -89,6 +109,9 @@ template <> bool set(Struct &target, const std::string &name, const char *value) template <> bool set(Struct &target, const std::string &name, bool value); +template <> bool set(Struct &target, const std::string &name, float value); +template <> bool set(Struct &target, const std::string &name, double value); + /*! Fuzzy-set attempts to set any property based on a string value. This is intended to allow input provided by the user. @@ -126,6 +149,12 @@ template class StructImpl: public Struct { @returns the value of type @c Type that is loaded from the offset registered for the field @c name. It is the caller's responsibility to provide an appropriate type of data. */ + void *get(const std::string &name) final { + const auto iterator = contents_.find(name); + if(iterator == contents_.end()) return nullptr; + return reinterpret_cast(this) + iterator->second.offset; + } + const void *get(const std::string &name) const final { const auto iterator = contents_.find(name); if(iterator == contents_.end()) return nullptr; From 023d76a3e74dea0cc58882eec5135849429a0bc4 Mon Sep 17 00:00:00 2001 From: Thomas Harte Date: Tue, 26 May 2020 22:20:15 -0400 Subject: [PATCH 08/10] Permits int truncation, adds double decoder. Arrays still TODO. --- Reflection/Struct.cpp | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/Reflection/Struct.cpp b/Reflection/Struct.cpp index 8dc88f166..8c66593ab 100644 --- a/Reflection/Struct.cpp +++ b/Reflection/Struct.cpp @@ -120,7 +120,9 @@ template <> bool Reflection::set(Struct &target, const std::string &name, int64_ return true; } - // TODO: other int sizes. +#define SetInt(x) if(*target_type == typeid(x)) { x truncated_value = x(value); target.set(name, &truncated_value); } + ForAllInts(SetInt); +#undef SetInt return false; } @@ -537,7 +539,8 @@ bool Reflection::Struct::deserialise(const uint8_t *bson, size_t size) { while(true) { const uint8_t next_type = *bson; ++bson; - if(!next_type) break; + if(!next_type) + break; std::string key; while(*bson) { @@ -550,8 +553,6 @@ bool Reflection::Struct::deserialise(const uint8_t *bson, size_t size) { default: return false; - // TODO: arrays. - // 0x03: A subdocument; try to install the inner BSON. // 0x05: Binary data. Seek to populate a std::vector. case 0x03: @@ -574,6 +575,19 @@ bool Reflection::Struct::deserialise(const uint8_t *bson, size_t size) { } } break; + // Array. BSON's encoding of these is a minor pain, but could be worse; + // they're presented as a subobject with objects serialised in array order + // but given the string keys "0", "1", etc. So: validate the keys, decode + // the objects. + case 0x04: { + uint32_t subobject_size; + read_int(subobject_size); + + // TODO: parse objects. + + bson += subobject_size - 4; + } break; + // String. case 0x02: { uint32_t length; @@ -611,10 +625,12 @@ bool Reflection::Struct::deserialise(const uint8_t *bson, size_t size) { uint64_t value; read_int(value); - // TODO: real conversion. - double double_value = double(value); + const double mantissa = 0.5 + double(value & 0x000f'ffff'ffff'ffff) / 9007199254740992.0; + const int exponent = ((value >> 52) & 2047) - 1022; + const double double_value = ldexp(mantissa, exponent); + const double sign = (value & 0x8000'0000'0000'0000) ? -1 : 1; - ::Reflection::set(*this, key, double_value); + ::Reflection::set(*this, key, double_value * sign); } break; } } From 51d684820f3ba5d582e1b5830ed26fdd93d149a6 Mon Sep 17 00:00:00 2001 From: Thomas Harte Date: Tue, 26 May 2020 22:55:55 -0400 Subject: [PATCH 09/10] Attempts to add array support to ::set and BSON deserialisation. --- .../Implementation/MultiConfigurable.cpp | 4 +- Reflection/Struct.cpp | 73 +++++++++++++++---- Reflection/Struct.hpp | 33 ++++----- 3 files changed, 74 insertions(+), 36 deletions(-) diff --git a/Analyser/Dynamic/MultiMachine/Implementation/MultiConfigurable.cpp b/Analyser/Dynamic/MultiMachine/Implementation/MultiConfigurable.cpp index df7333924..28dbd9175 100644 --- a/Analyser/Dynamic/MultiMachine/Implementation/MultiConfigurable.cpp +++ b/Analyser/Dynamic/MultiMachine/Implementation/MultiConfigurable.cpp @@ -80,7 +80,7 @@ class MultiStruct: public Reflection::Struct { return nullptr; } - void set(const std::string &name, const void *value) final { + void set(const std::string &name, const void *value, size_t offset) final { const auto safe_type = type_of(name); if(!safe_type) return; @@ -91,7 +91,7 @@ class MultiStruct: public Reflection::Struct { if(!type) continue; if(*type == *safe_type) { - options->set(name, value); + options->set(name, value, offset); } } } diff --git a/Reflection/Struct.cpp b/Reflection/Struct.cpp index 8c66593ab..d6ef45f60 100644 --- a/Reflection/Struct.cpp +++ b/Reflection/Struct.cpp @@ -69,65 +69,65 @@ static size_t size(const std::type_info *type) { // MARK: - Setters -template <> bool Reflection::set(Struct &target, const std::string &name, float value) { +template <> bool Reflection::set(Struct &target, const std::string &name, float value, size_t offset) { const auto target_type = target.type_of(name); if(!target_type) return false; if(*target_type == typeid(float)) { - target.set(name, &value); + target.set(name, &value, offset); return true; } return set(target, name, value); } -template <> bool Reflection::set(Struct &target, const std::string &name, double value) { +template <> bool Reflection::set(Struct &target, const std::string &name, double value, size_t offset) { const auto target_type = target.type_of(name); if(!target_type) return false; if(*target_type == typeid(double)) { - target.set(name, &value); + target.set(name, &value, offset); return true; } if(*target_type == typeid(float)) { const float float_value = float(value); - target.set(name, &float_value); + target.set(name, &float_value, offset); return true; } return false; } -template <> bool Reflection::set(Struct &target, const std::string &name, int value) { +template <> bool Reflection::set(Struct &target, const std::string &name, int value, size_t offset) { return set(target, name, value); } -template <> bool Reflection::set(Struct &target, const std::string &name, int64_t value) { +template <> bool Reflection::set(Struct &target, const std::string &name, int64_t value, size_t offset) { const auto target_type = target.type_of(name); if(!target_type) return false; // No need to convert an int or a registered enum. if(*target_type == typeid(int) || !Reflection::Enum::name(*target_type).empty()) { const int value32 = int(value); - target.set(name, &value32); + target.set(name, &value32, offset); return true; } // Set an int64_t directly. if(*target_type == typeid(int64_t)) { - target.set(name, &value); + target.set(name, &value, offset); return true; } -#define SetInt(x) if(*target_type == typeid(x)) { x truncated_value = x(value); target.set(name, &truncated_value); } +#define SetInt(x) if(*target_type == typeid(x)) { x truncated_value = x(value); target.set(name, &truncated_value, offset); } ForAllInts(SetInt); #undef SetInt return false; } -template <> bool Reflection::set(Struct &target, const std::string &name, const std::string &value) { +template <> bool Reflection::set(Struct &target, const std::string &name, const std::string &value, size_t offset) { const auto target_type = target.type_of(name); if(!target_type) return false; @@ -147,22 +147,22 @@ template <> bool Reflection::set(Struct &target, const std::string &name, const if(enum_value < 0) { return false; } - target.set(name, &enum_value); + target.set(name, &enum_value, offset); return true; } -template <> bool Reflection::set(Struct &target, const std::string &name, const char *value) { +template <> bool Reflection::set(Struct &target, const std::string &name, const char *value, size_t offset) { const std::string string(value); return set(target, name, string); } -template <> bool Reflection::set(Struct &target, const std::string &name, bool value) { +template <> bool Reflection::set(Struct &target, const std::string &name, bool value, size_t offset) { const auto target_type = target.type_of(name); if(!target_type) return false; if(*target_type == typeid(bool)) { - target.set(name, &value);; + target.set(name, &value, offset);; } return false; @@ -519,6 +519,45 @@ bool Reflection::Struct::deserialise(const std::vector &bson) { return deserialise(bson.data(), bson.size()); } +namespace { + +/*! + Provides a proxy struct that redirects calls to set to another object and property, picking + an offset based on the propety name specified here. +*/ +struct ArrayReceiver: public Reflection::Struct { + ArrayReceiver(Reflection::Struct *target, const std::type_info *type, const std::string &key, size_t count) : + target_(target), type_(type), key_(key), count_(count) {} + + std::vector all_keys() const final { return {}; } + const std::type_info *type_of(const std::string &name) const final { return type_; } + size_t count_of(const std::string &name) const final { return 0; } + + void set(const std::string &name, const void *value, size_t offset) final { + const auto index = size_t(std::stoi(name)); + if(index >= count_) { + return; + } + target_->set(key_, value, index); + } + + virtual std::vector values_for(const std::string &name) const final { + return {}; + } + + void *get(const std::string &name) final { + return nullptr; + } + + private: + Reflection::Struct *target_; + const std::type_info *type_; + std::string key_; + size_t count_; +}; + +} + bool Reflection::Struct::deserialise(const uint8_t *bson, size_t size) { // Validate the object's declared size. const auto end = bson + size; @@ -580,10 +619,12 @@ bool Reflection::Struct::deserialise(const uint8_t *bson, size_t size) { // but given the string keys "0", "1", etc. So: validate the keys, decode // the objects. case 0x04: { + ArrayReceiver receiver(this, type_of(key), key, count_of(key)); + uint32_t subobject_size; read_int(subobject_size); - // TODO: parse objects. + receiver.deserialise(bson - 4, size_t(end - bson + 4)); bson += subobject_size - 4; } break; diff --git a/Reflection/Struct.hpp b/Reflection/Struct.hpp index 3cf0a4ab0..f2d744283 100644 --- a/Reflection/Struct.hpp +++ b/Reflection/Struct.hpp @@ -28,9 +28,11 @@ struct Struct { virtual std::vector all_keys() const = 0; virtual const std::type_info *type_of(const std::string &name) const = 0; virtual size_t count_of(const std::string &name) const = 0; - virtual void set(const std::string &name, const void *value) = 0; + virtual void set(const std::string &name, const void *value, size_t offset = 0) = 0; virtual void *get(const std::string &name) = 0; - virtual const void *get(const std::string &name) const = 0; + virtual const void *get(const std::string &name) const { + return const_cast(this)->get(name); + } virtual std::vector values_for(const std::string &name) const = 0; virtual ~Struct() {} @@ -80,7 +82,7 @@ struct Struct { @returns @c true if the property was successfully set; @c false otherwise. */ -template bool set(Struct &target, const std::string &name, Type value); +template bool set(Struct &target, const std::string &name, Type value, size_t offset = 0); /*! Setting an int: @@ -90,27 +92,27 @@ template bool set(Struct &target, const std::string &name, Type * to an int64_t promotes the int; and * to a registered enum, copies the int. */ -template <> bool set(Struct &target, const std::string &name, int64_t value); -template <> bool set(Struct &target, const std::string &name, int value); +template <> bool set(Struct &target, const std::string &name, int64_t value, size_t offset); +template <> bool set(Struct &target, const std::string &name, int value, size_t offset); /*! Setting a string: * to an enum, if the string names a member of the enum, sets the value. */ -template <> bool set(Struct &target, const std::string &name, const std::string &value); -template <> bool set(Struct &target, const std::string &name, const char *value); +template <> bool set(Struct &target, const std::string &name, const std::string &value, size_t offset); +template <> bool set(Struct &target, const std::string &name, const char *value, size_t offset); /*! Setting a bool: * to a bool, copies the value. */ -template <> bool set(Struct &target, const std::string &name, bool value); +template <> bool set(Struct &target, const std::string &name, bool value, size_t offset); -template <> bool set(Struct &target, const std::string &name, float value); -template <> bool set(Struct &target, const std::string &name, double value); +template <> bool set(Struct &target, const std::string &name, float value, size_t offset); +template <> bool set(Struct &target, const std::string &name, double value, size_t offset); /*! Fuzzy-set attempts to set any property based on a string value. This is intended to allow input provided by the user. @@ -155,21 +157,16 @@ template class StructImpl: public Struct { return reinterpret_cast(this) + iterator->second.offset; } - const void *get(const std::string &name) const final { - const auto iterator = contents_.find(name); - if(iterator == contents_.end()) return nullptr; - return reinterpret_cast(this) + iterator->second.offset; - } - /*! Stores the @c value of type @c Type to the offset registered for the field @c name. It is the caller's responsibility to provide an appropriate type of data. */ - void set(const std::string &name, const void *value) final { + void set(const std::string &name, const void *value, size_t offset) final { const auto iterator = contents_.find(name); if(iterator == contents_.end()) return; - memcpy(reinterpret_cast(this) + iterator->second.offset, value, iterator->second.size); + assert(offset < iterator->second.count); + memcpy(reinterpret_cast(this) + iterator->second.offset + offset * iterator->second.size, value, iterator->second.size); } /*! From 1308f119a6d66c2f42f9b6e806b528931e36e696 Mon Sep 17 00:00:00 2001 From: Thomas Harte Date: Tue, 26 May 2020 23:07:26 -0400 Subject: [PATCH 10/10] Relocates cassert. --- Reflection/Struct.cpp | 1 - Reflection/Struct.hpp | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/Reflection/Struct.cpp b/Reflection/Struct.cpp index d6ef45f60..e4d772adc 100644 --- a/Reflection/Struct.cpp +++ b/Reflection/Struct.cpp @@ -9,7 +9,6 @@ #include "Struct.hpp" #include -#include #include #include #include diff --git a/Reflection/Struct.hpp b/Reflection/Struct.hpp index f2d744283..68278c7ab 100644 --- a/Reflection/Struct.hpp +++ b/Reflection/Struct.hpp @@ -9,6 +9,7 @@ #ifndef Struct_hpp #define Struct_hpp +#include #include #include #include