Map/Array support as "tokens"

This essentially completes the token_reader. In the future, we will
probably want something that lets us allocate maps/arrays more
seamlessly, but we can start working on client-server architecture with
this reader.

source/include/parselink/msgpack/core/format.h

@@ -145,6 +145,16 @@ namespace format {
         constexpr static type value = type::invalid;
     };
 
+    template <>
+    struct resolve_token_type<array_desc> {
+        constexpr static type value = type::array;
+    };
+
+    template <>
+    struct resolve_token_type<map_desc> {
+        constexpr static type value = type::map;
+    };
+
     template <typename T>
     constexpr static auto resolve_type_v = resolve_token_type<T>::value;
 

source/include/parselink/msgpack/token/reader.h

@@ -92,6 +92,16 @@ constexpr inline format::traits const& traits_lookup(std::byte id) {
         case format::bin32::marker:
             return format::get_traits<format::bin32>();
 
+        case format::array16::marker:
+            return format::get_traits<format::array16>();
+        case format::array32::marker:
+            return format::get_traits<format::array32>();
+
+        case format::map16::marker:
+            return format::get_traits<format::map16>();
+        case format::map32::marker:
+            return format::get_traits<format::map32>();
+
         case format::nil::marker:
             return format::get_traits<format::nil>();
         case format::invalid::marker:
@@ -104,15 +114,21 @@ constexpr inline format::traits const& traits_lookup(std::byte id) {
     }
 
     // TODO(ksassenrath): Handle fixtype formats.
-    if ((id & ~format::fixstr::mask) == format::fixstr::marker) {
-        return format::get_traits<format::fixstr>();
-    }
     if ((id & ~format::negative_fixint::mask) == format::negative_fixint::marker) {
         return format::get_traits<format::negative_fixint>();
     }
     if ((id & ~format::positive_fixint::mask) == format::positive_fixint::marker) {
         return format::get_traits<format::positive_fixint>();
     }
+    if ((id & ~format::fixstr::mask) == format::fixstr::marker) {
+        return format::get_traits<format::fixstr>();
+    }
+    if ((id & ~format::fixmap::mask) == format::fixmap::marker) {
+        return format::get_traits<format::fixmap>();
+    }
+    if ((id & ~format::fixarray::mask) == format::fixarray::marker) {
+        return format::get_traits<format::fixarray>();
+    }
 
     return format::no_traits;
 }
@@ -200,20 +216,24 @@ public:
             }
             case format::type::string: {
                 using type = token_traits<format::type::string>::storage_type;
-                {
+                auto ptr = reinterpret_cast<type>(&*curr);
-                    auto ptr = reinterpret_cast<type>(&*curr);
+                tok = token{std::string_view{ptr, var_size}};
-                    tok = token{std::string_view{ptr, var_size}};
+                curr += var_size;
-                    curr += var_size;
-                }
                 break;
             }
             case format::type::binary: {
                 using type = token_traits<format::type::binary>::storage_type;
-                {
+                auto ptr = reinterpret_cast<type>(&*curr);
-                    auto ptr = reinterpret_cast<type>(&*curr);
+                tok = token{std::span<std::byte const>{ptr, var_size}};
-                    tok = token{std::span<std::byte const>{ptr, var_size}};
+                curr += var_size;
-                    curr += var_size;
+                break;
-                }
+            }
+            case format::type::array: {
+                tok = token{array_desc(var_size)};
+                break;
+            }
+            case format::type::map: {
+                tok = token{map_desc(var_size)};
                 break;
             }
             default:

source/include/parselink/msgpack/token/type.h

@@ -132,6 +132,16 @@ struct token_traits<format::type::binary> {
     using storage_type = std::byte const*;
 };
 
+template <>
+struct token_traits<format::type::array> {
+    using storage_type = std::uint64_t;
+};
+
+template <>
+struct token_traits<format::type::map> {
+    using storage_type = std::uint64_t;
+};
+
 template <>
 struct token_traits<format::type::boolean> {
     using storage_type = bool;
@@ -180,6 +190,16 @@ public:
         value_.bp = bv.data();
     }
 
+    explicit token_base(array_desc const& value) noexcept {
+        size_and_type_.set_enum(format::type::array);
+        size_and_type_.set_size(value.count);
+    }
+
+    explicit token_base(map_desc const& value) noexcept {
+        size_and_type_.set_enum(format::type::map);
+        size_and_type_.set_size(value.count);
+    }
+
     explicit token_base(nil) noexcept {
         size_and_type_.set_enum(format::type::nil);
     }
@@ -227,7 +247,8 @@ private:
         typename token_traits<format::type::string>::storage_type str;
         typename token_traits<format::type::binary>::storage_type bp;
         typename token_traits<format::type::boolean>::storage_type b;
-        token_base* obj;
+        // TODO: What to store here for array/map types? Possibly a pointer
+        // to the end of the array/map, if/when known?
     } value_;
     size_and_enum<std::uint32_t, format::type> size_and_type_{};
 };
@@ -296,6 +317,26 @@ token_base<8>::get() const noexcept
     return invalid{};
 }
 
+template<>
+constexpr tl::expected<array_desc, error>
+token_base<8>::get() const noexcept
+{
+    if (type() != format::type::array) {
+        return tl::make_unexpected(error::wrong_type);
+    }
+    return array_desc{size_and_type_.get_size()};
+}
+
+template<>
+constexpr tl::expected<map_desc, error>
+token_base<8>::get() const noexcept
+{
+    if (type() != format::type::map) {
+        return tl::make_unexpected(error::wrong_type);
+    }
+    return map_desc{size_and_type_.get_size()};
+}
+
 using token = token_base<sizeof(void*)>;
 
 } // namespace msgpack

tests/msgpack/test_token_reader.cpp

@@ -1,7 +1,7 @@
 #include <msgpack/token.h>
 
-#include <fmt/format.h>
 #include <boost/ut.hpp>
+#include <fmt/format.h>
 #include <magic_enum.hpp>
 
 using namespace boost::ut;
@@ -486,6 +486,156 @@ suite reader_tests = [] {
         expect(test_end_of_message(reader));
     };
     //--------------------------------------------------------------------------
+    // Structural format tests (map/array)
+    //--------------------------------------------------------------------------
+    "read format::fixarray"_test = [] {
+        // A MessagePack array of 5 8-bit unsigned integers.
+        constexpr auto payload = make_bytes(0x95, 0xcc, 0x85, 0xcc, 0x84, 0xcc,
+                                           0x83, 0xcc, 0x82, 0xcc, 0x81);
+
+        msgpack::token_reader reader(payload);
+        auto token = reader.read_one();
+        expect(token && token->type() == msgpack::format::type::array);
+        auto array = token->get<msgpack::array_desc>();
+        expect(array && array->count == 5);
+
+        for (std::size_t i = 0; i < array->count; ++i) {
+          auto v = reader.read_one();
+          expect(v && v->get<std::uint8_t>() == 0x85 - i);
+        }
+
+        expect(test_end_of_message(reader));
+    };
+    "read format::array16"_test = [] {
+        // A MessagePack array of 5 8-bit unsigned integers.
+        constexpr auto payload = make_bytes(0xdc, 0x00, 0x05,
+                0xcc, 0x85, 0xcc, 0x84, 0xcc, 0x83, 0xcc, 0x82, 0xcc, 0x81);
+
+        msgpack::token_reader reader(payload);
+        auto token = reader.read_one();
+        expect(token && token->type() == msgpack::format::type::array);
+        auto array = token->get<msgpack::array_desc>();
+        expect(array && array->count == 5);
+
+        for (std::size_t i = 0; i < array->count; ++i) {
+          auto v = reader.read_one();
+          expect(v && v->get<std::uint8_t>() == 0x85 - i);
+        }
+
+        expect(test_end_of_message(reader));
+    };
+    "read format::array32"_test = [] {
+        // A MessagePack array of 5 8-bit unsigned integers.
+        constexpr auto payload = make_bytes(0xdd, 0x00, 0x00, 0x00, 0x05,
+                0xcc, 0x85, 0xcc, 0x84, 0xcc, 0x83, 0xcc, 0x82, 0xcc, 0x81);
+
+        msgpack::token_reader reader(payload);
+        auto token = reader.read_one();
+        expect(token && token->type() == msgpack::format::type::array);
+        auto array = token->get<msgpack::array_desc>();
+        expect(array && array->count == 5);
+
+        for (std::size_t i = 0; i < array->count; ++i) {
+          auto v = reader.read_one();
+          expect(v && v->get<std::uint8_t>() == 0x85 - i);
+        }
+
+        expect(test_end_of_message(reader));
+    };
+    "read format::fixmap"_test = [] {
+        // A MessagePack map of 3 strings to 8-bit unsigned integers.
+        static constexpr std::array<std::string_view, 3> strings = {
+            "one", "two", "three"
+        };
+
+        constexpr auto payload = cat(
+                make_bytes(0x83, 0xa3),
+                generate_bytes([] { return from_string_view(strings[0]); }),
+                make_bytes(0x01, 0xa3),
+                generate_bytes([] { return from_string_view(strings[1]); }),
+                make_bytes(0x02, 0xa5),
+                generate_bytes([] { return from_string_view(strings[2]); }),
+                make_bytes(0x03)
+        );
+
+        msgpack::token_reader reader(payload);
+        auto token = reader.read_one();
+        expect(token && token->type() == msgpack::format::type::map);
+        auto map = token->get<msgpack::map_desc>();
+        expect(map && map->count == 3);
+
+        for (std::size_t i = 0; i < map->count; ++i) {
+            auto str = reader.read_one();
+            expect(str && str->get<std::string_view>() == strings[i]);
+            auto value = reader.read_one();
+            expect(value && value->get<std::uint8_t>() == i + 1);
+        }
+
+        expect(test_end_of_message(reader));
+    };
+    "read format::map16"_test = [] {
+        // A MessagePack map of 3 strings to 8-bit unsigned integers.
+        static constexpr std::array<std::string_view, 3> strings = {
+            "one", "two", "three"
+        };
+
+        constexpr auto payload = cat(
+                make_bytes(0xde, 0x00, 0x03, 0xa3),
+                generate_bytes([] { return from_string_view(strings[0]); }),
+                make_bytes(0x01, 0xa3),
+                generate_bytes([] { return from_string_view(strings[1]); }),
+                make_bytes(0x02, 0xa5),
+                generate_bytes([] { return from_string_view(strings[2]); }),
+                make_bytes(0x03)
+        );
+
+        msgpack::token_reader reader(payload);
+        auto token = reader.read_one();
+        expect(token && token->type() == msgpack::format::type::map);
+        auto map = token->get<msgpack::map_desc>();
+        expect(map && map->count == 3);
+
+        for (std::size_t i = 0; i < map->count; ++i) {
+            auto str = reader.read_one();
+            expect(str && str->get<std::string_view>() == strings[i]);
+            auto value = reader.read_one();
+            expect(value && value->get<std::uint8_t>() == i + 1);
+        }
+
+        expect(test_end_of_message(reader));
+    };
+    "read format::map32"_test = [] {
+        // A MessagePack map of 3 strings to 8-bit unsigned integers.
+        static constexpr std::array<std::string_view, 3> strings = {
+            "one", "two", "three"
+        };
+
+        constexpr auto payload = cat(
+                make_bytes(0xdf, 0x00, 0x00, 0x00, 0x03, 0xa3),
+                generate_bytes([] { return from_string_view(strings[0]); }),
+                make_bytes(0x01, 0xa3),
+                generate_bytes([] { return from_string_view(strings[1]); }),
+                make_bytes(0x02, 0xa5),
+                generate_bytes([] { return from_string_view(strings[2]); }),
+                make_bytes(0x03)
+        );
+
+        msgpack::token_reader reader(payload);
+        auto token = reader.read_one();
+        expect(token && token->type() == msgpack::format::type::map);
+        auto map = token->get<msgpack::map_desc>();
+        expect(map && map->count == 3);
+
+        for (std::size_t i = 0; i < map->count; ++i) {
+            auto str = reader.read_one();
+            expect(str && str->get<std::string_view>() == strings[i]);
+            auto value = reader.read_one();
+            expect(value && value->get<std::uint8_t>() == i + 1);
+        }
+
+        expect(test_end_of_message(reader));
+    };
+    //--------------------------------------------------------------------------
     // Assorted format tests
     //--------------------------------------------------------------------------
     "read format::boolean"_test = [] {