Many more supported types in token reader.

source/include/parselink/msgpack/core/format.h

@@ -66,7 +66,7 @@ namespace format {
         unsigned_int,
         signed_int,
         string,
-        bytes,
+        binary,
         nil,
         boolean,
         array,
@@ -74,8 +74,9 @@ namespace format {
     };
 
     // Flags that may control the behavior of readers/writers.
-    enum flag {
+    enum flag : std::uint8_t {
-        apply_mask = 1
+        apply_mask = 1,
+        fixed_size = 2,
     };
 
     // MessagePack formats break down into one of the following schemes:
@@ -84,7 +85,7 @@ namespace format {
     //
     // In addition, there are "fix" types that embed the literal value or the
     // payload length within the marker byte.
-    enum payload {
+    enum payload : std::uint8_t {
         fixed,
         variable
     };
@@ -94,20 +95,72 @@ namespace format {
         constexpr static flag flags = {};
     };
 
+    // Traits describing a particular format.
+    struct traits {
+        std::uint8_t flags{};
+        std::uint8_t size{};
+        std::byte mask{};
+        type token_type{};
+        constexpr auto operator<=>(traits const&) const noexcept = default;
+    };
+
+    // "Sentinel" traits instance indicating no trait found
+    constexpr static traits no_traits {};
+
     struct fixtype_format : base_format {};
 
+    template <class>
+    struct resolve_token_type {
+        constexpr static type value = type::invalid;
+    };
+
+    template <std::integral T>
+    struct resolve_token_type<T> {
+        constexpr static type value = std::is_signed_v<T> ?
+                type::signed_int : type::unsigned_int;
+    };
+
+    template <>
+    struct resolve_token_type<std::string_view> {
+        constexpr static type value = type::string;
+    };
+
+    template <>
+    struct resolve_token_type<std::span<std::byte const>> {
+        constexpr static type value = type::binary;
+    };
+
+    template <>
+    struct resolve_token_type<bool> {
+        constexpr static type value = type::boolean;
+    };
+
+    template <>
+    struct resolve_token_type<nil> {
+        constexpr static type value = type::nil;
+    };
+
+    template <>
+    struct resolve_token_type<invalid> {
+        constexpr static type value = type::invalid;
+    };
+
+    template <typename T>
+    constexpr static auto resolve_type_v = resolve_token_type<T>::value;
+
     // The library's representation of certain data types does not always
     // match the underlying data serialized in the message. For example,
     // arrays and maps are encoded as a marker byte + fixed length value, but
     // msgpack will present them wrapped in a structure for stronger type
     // semantics.
 
-    template <std::uint8_t Marker, typename First>
+    template <std::uint8_t Marker, typename First, typename Value = First>
     struct format : base_format {
         using first_type = First;
-        using value_type = first_type; // Can be overridden
+        using value_type = Value; // Can be overridden
         constexpr static std::byte marker{Marker};
         constexpr static payload payload_type{payload::fixed};
+        constexpr static std::uint8_t flags{};
     };
 
     template <std::uint8_t Marker, std::uint8_t Mask, typename First = std::uint8_t>
@@ -127,10 +180,10 @@ namespace format {
     // Positive/negative fixint represent the literal value specified, do not
     // need to mask it off.
     struct positive_fixint : fixtype<0x00, 0x7f> {
-        constexpr static flag flags = {};
+        constexpr static flag flags{};
     };
     struct negative_fixint : fixtype<0xe0, 0x1f, std::int8_t> {
-        constexpr static flag flags = {};
+        constexpr static flag flags{};
     };
 
     struct uint8 : format<0xcc, std::uint8_t> {};
@@ -146,9 +199,20 @@ namespace format {
     /*
      * Other primitive types
      */
-    struct nil : fixtype<0xc0, 0x00> { using value_type = msgpack::nil; };
+    struct nil : fixtype<0xc0, 0x00> {
-    struct invalid : fixtype<0xc1, 0x00> { using value_type = msgpack::invalid; };
+        using value_type = msgpack::nil;
-    struct boolean : fixtype<0xc2, 0x01> { using value_type = bool; };
+        constexpr static flag flags{flag::fixed_size | flag::apply_mask};
+    };
+
+    struct invalid : fixtype<0xc1, 0x00> {
+        using value_type = msgpack::invalid;
+        constexpr static flag flags{flag::fixed_size | flag::apply_mask};
+    };
+
+    struct boolean : fixtype<0xc2, 0x01> {
+        using value_type = bool;
+        constexpr static flag flags{flag::fixed_size | flag::apply_mask};
+    };
 
     /*
      * Maps
@@ -183,6 +247,7 @@ namespace format {
         constexpr static auto payload_type = payload::variable;
     };
 
+
     struct fixstr : string_format<fixtype<0xa0, 0x1f>> {};
     struct str8 : string_format<format<0xd9, std::uint8_t>> {};
     struct str16 : string_format<format<0xda, std::uint16_t>> {};
@@ -280,6 +345,38 @@ concept format_type = std::is_base_of_v<format::base_format, T>;
 template <typename T>
 concept is_fixtype = std::is_base_of_v<format::fixtype_format, T>;
 
+namespace format {
+    // This template instantiates a format::traits object for a given format.
+    // This should only occur once per format type, and should happen at
+    // compile time.
+    //
+    // This isn't the prettiest way to go about it.
+    template <format_type Format>
+    inline traits const& get_traits() noexcept {
+        constexpr static auto inst = []{
+            // Fixtypes define the size within the identifier byte, so the
+            // trait size must be zero.
+            auto size = is_fixtype<Format> ?
+                    0 : sizeof(typename Format::first_type);
+
+            traits inst{
+                .flags = Format::flags,
+                .size = std::uint8_t(size),
+                .token_type = resolve_type_v<typename Format::value_type>
+            };
+            // Add in the fixed_size flag for integral types.
+            if constexpr (std::is_integral_v<typename Format::value_type>) {
+                inst.flags |= flag::fixed_size;
+            }
+            if constexpr (is_fixtype<Format>) {
+                inst.mask = Format::mask;
+            }
+            return inst;
+        }();
+        return inst;
+    };
+} // namespace format
+
 template <format_type... Formats>
 using format_list = format::detail::type_list<Formats...>;
 

source/include/parselink/msgpack/token/reader.h

@@ -32,18 +32,89 @@ namespace msgpack {
 
 namespace detail {
 
-constexpr inline decltype(auto) read(std::size_t size, auto& inp)
+constexpr std::int64_t sign_extend(std::size_t size, auto bytes) noexcept {
+    switch (size) {
+        case 0:
+        case 1: return std::int8_t(std::bit_cast<std::int64_t>(bytes));
+        case 2: return std::int16_t(std::bit_cast<std::int64_t>(bytes));
+        case 4: return std::int32_t(std::bit_cast<std::int64_t>(bytes));
+        case 8: return std::bit_cast<std::int64_t>(bytes);
+        default: break; // Not reachable.
+    }
+    return {};
+}
+
+constexpr decltype(auto) read(std::size_t size, auto& inp)
         noexcept {
-    std::array<std::byte, token::word_size> value;
+    std::array<std::byte, token::word_size> value{};
     be_to_host(inp, inp + size, std::begin(value));
     inp += size;
     return value;
 }
 
 template <format::type FormatType>
-constexpr inline decltype(auto) read_value(std::size_t size, auto& inp) {
+constexpr inline decltype(auto) make_token(auto bytes) {
     using value_type = typename token_traits<FormatType>::storage_type;
-    return token{std::bit_cast<value_type>(read(size, inp))};
+    return token{std::bit_cast<value_type>(bytes)};
+}
+
+// Define in token/type.h instead? Avoid instantiations in the core API.
+constexpr inline format::traits const& traits_lookup(std::byte id) {
+    switch (id) {
+        case format::uint8::marker:
+            return format::get_traits<format::uint8>();
+        case format::uint16::marker:
+            return format::get_traits<format::uint16>();
+        case format::uint32::marker:
+            return format::get_traits<format::uint32>();
+        case format::uint64::marker:
+            return format::get_traits<format::uint64>();
+
+        case format::int8::marker:
+            return format::get_traits<format::int8>();
+        case format::int16::marker:
+            return format::get_traits<format::int16>();
+        case format::int32::marker:
+            return format::get_traits<format::int32>();
+        case format::int64::marker:
+            return format::get_traits<format::int64>();
+
+        case format::str8::marker:
+            return format::get_traits<format::str8>();
+        case format::str16::marker:
+            return format::get_traits<format::str16>();
+        case format::str32::marker:
+            return format::get_traits<format::str32>();
+        case format::bin8::marker:
+            return format::get_traits<format::bin8>();
+        case format::bin16::marker:
+            return format::get_traits<format::bin16>();
+        case format::bin32::marker:
+            return format::get_traits<format::bin32>();
+
+        case format::nil::marker:
+            return format::get_traits<format::nil>();
+        case format::invalid::marker:
+            return format::get_traits<format::invalid>();
+        case format::boolean::marker:
+        case format::boolean::marker | std::byte{1}:
+            return format::get_traits<format::boolean>();
+        default:
+            break;
+    }
+
+    // TODO(ksassenrath): Handle fixtype formats.
+    if ((id & ~format::fixstr::mask) == format::fixstr::marker) {
+        return format::get_traits<format::fixstr>();
+    }
+    if ((id & ~format::negative_fixint::mask) == format::negative_fixint::marker) {
+        return format::get_traits<format::negative_fixint>();
+    }
+    if ((id & ~format::positive_fixint::mask) == format::positive_fixint::marker) {
+        return format::get_traits<format::positive_fixint>();
+    }
+
+    return format::no_traits;
 }
 
 } // namespace detail
@@ -73,100 +144,80 @@ public:
             return tl::make_unexpected(error::end_of_message);
         }
 
+        // curr_ will be updated after everything succeeds.
         auto curr = curr_;
-
-        // Enumerate the current byte first by switch statement, then by
-        // fix types.
-        long int size = 0;
         std::size_t var_size = 0;
-        auto id = *curr;
+        auto id = *curr++;
-        traits&
+        auto const& traits = detail::traits_lookup(id);
-        format::type token_type;
+
-        ++curr;
+        if (traits == format::no_traits) {
-        switch (id) {
-            case format::uint8::marker:
-                size = sizeof(format::uint8::value_type);
-                token_type = format::type::unsigned_int;
-                break;
-            case format::uint16::marker:
-                size = sizeof(format::uint16::value_type);
-                token_type = format::type::unsigned_int;
-                break;
-            case format::uint32::marker:
-                size = sizeof(format::uint32::value_type);
-                token_type = format::type::unsigned_int;
-                break;
-            case format::uint64::marker:
-                size = sizeof(format::uint64::value_type);
-                token_type = format::type::unsigned_int;
-                break;
-            case format::int8::marker:
-                size = sizeof(format::int8::value_type);
-                token_type = format::type::signed_int;
-                break;
-            case format::int16::marker:
-                size = sizeof(format::int16::value_type);
-                token_type = format::type::signed_int;
-                break;
-            case format::int32::marker:
-                size = sizeof(format::int32::value_type);
-                token_type = format::type::signed_int;
-                break;
-            case format::int64::marker:
-                size = sizeof(format::int64::value_type);
-                token_type = format::type::signed_int;
-                break;
-            case format::str8::marker:
-                size = sizeof(format::str8::first_type);
-                token_type = format::type::string;
-                break;
-            case format::str16::marker:
-                size = sizeof(format::str16::first_type);
-                token_type = format::type::string;
-                break;
-            case format::str32::marker:
-                size = sizeof(format::str32::first_type);
-                token_type = format::type::string;
-                break;
-            case format::bin8::marker:
-                size = sizeof(format::bin8::first_type);
-                token_type = format::type::bytes;
-                break;
-            case format::bin16::marker:
-                size = sizeof(format::bin16::first_type);
-                token_type = format::type::bytes;
-                break;
-            case format::bin32::marker:
-                size = sizeof(format::bin32::first_type);
-                token_type = format::type::bytes;
-                break;
-            default:
             return tl::make_unexpected(error::not_implemented);
         }
 
-        if (remaining(curr) < size) {
+        if (remaining(curr) < traits.size) {
             return tl::make_unexpected(error::incomplete_message);
         }
 
-        switch (token_type) {
+        // This is either the value of the format, or the size of the format.
-            case format::type::unsigned_int:
+        auto first_bytes = [&]{
-                tok = detail::read_value<format::type::unsigned_int>(size, curr);
+            if (traits.size) {
-                break;
+                return detail::read(traits.size, curr);
-            case format::type::signed_int:
+            } else {
-                tok = detail::read_value<format::type::signed_int>(size, curr);
+                auto lsb = id;
-                break;
+                if (traits.flags & format::flag::apply_mask) {
-            case format::type::string:
+                    lsb &= std::byte(traits.mask);
-                var_size = std::bit_cast<std::size_t>(detail::read(size, curr));
+                }
+                return std::array<std::byte, token::word_size>{lsb};
+            }
+        }();
+
+        // This indicates first_bytes is the size of the format payload.
+        if (!(traits.flags & format::flag::fixed_size)) {
+            var_size = std::bit_cast<std::size_t>(first_bytes);
             if (std::size_t(remaining(curr)) < var_size) {
                 return tl::make_unexpected(error::incomplete_message);
             }
+            // For variable-sized types we don't actually read anything yet.
+        }
+
+        switch (traits.token_type) {
+            case format::type::boolean:
+                tok = token{bool(first_bytes[0] & traits.mask)};
+                break;
+            case format::type::invalid:
+                tok = token{invalid{}};
+                break;
+            case format::type::nil:
+                tok = token{nil{}};
+                break;
+            case format::type::unsigned_int:
+                tok = detail::make_token<format::type::unsigned_int>(first_bytes);
+                break;
+            case format::type::signed_int: {
+                auto value = detail::sign_extend(traits.size, first_bytes);
+                tok = token{value};
+                break;
+            }
+            case format::type::string: {
                 using type = token_traits<format::type::string>::storage_type;
                 {
                     auto ptr = reinterpret_cast<type>(&*curr);
                     tok = token{std::string_view{ptr, var_size}};
                     curr += var_size;
                 }
-            default: break;
+                break;
+            }
+            case format::type::binary: {
+                using type = token_traits<format::type::binary>::storage_type;
+                {
+                    auto ptr = reinterpret_cast<type>(&*curr);
+                    tok = token{std::span<std::byte const>{ptr, var_size}};
+                    curr += var_size;
+                }
+                break;
+            }
+            default:
+                return tl::make_unexpected(error::not_implemented);
         }
 
         curr_ = curr;

source/include/parselink/msgpack/token/type.h

@@ -128,7 +128,7 @@ struct token_traits<format::type::string> {
 };
 
 template <>
-struct token_traits<format::type::bytes> {
+struct token_traits<format::type::binary> {
     using storage_type = std::byte const*;
 };
 
@@ -175,11 +175,19 @@ public:
     template <std::convertible_to<std::span<std::byte const>> T>
     explicit token_base(T const& value) noexcept {
         std::span<std::byte const> bv(value);
-        size_and_type_.set_enum(format::type::bytes);
+        size_and_type_.set_enum(format::type::binary);
         size_and_type_.set_size(bv.size());
         value_.bp = bv.data();
     }
 
+    explicit token_base(nil) noexcept {
+        size_and_type_.set_enum(format::type::nil);
+    }
+
+    explicit token_base(invalid) noexcept {
+        size_and_type_.set_enum(format::type::invalid);
+    }
+
     constexpr format::type type() const noexcept {
         return size_and_type_.get_enum();
     }
@@ -217,7 +225,7 @@ private:
         typename token_traits<format::type::unsigned_int>::storage_type u;
         typename token_traits<format::type::signed_int>::storage_type i;
         typename token_traits<format::type::string>::storage_type str;
-        typename token_traits<format::type::bytes>::storage_type bp;
+        typename token_traits<format::type::binary>::storage_type bp;
         typename token_traits<format::type::boolean>::storage_type b;
         token_base* obj;
     } value_;
@@ -249,7 +257,7 @@ inline tl::expected<std::vector<std::byte>, error> token_base<8>::get()
         const noexcept
 {
     tl::expected<std::vector<std::byte>, error> result;
-    if (type() != format::type::bytes) {
+    if (type() != format::type::binary) {
         result = tl::make_unexpected(error::wrong_type);
     } else {
         result = std::vector<std::byte>(value_.bp,
@@ -262,12 +270,32 @@ template<>
 constexpr tl::expected<std::span<std::byte const>, error>
 token_base<8>::get() const noexcept
 {
-    if (type() != format::type::bytes) {
+    if (type() != format::type::binary) {
         return tl::make_unexpected(error::wrong_type);
     }
     return std::span<std::byte const>(value_.bp, size_and_type_.get_size());
 }
 
+template<>
+constexpr tl::expected<nil, error>
+token_base<8>::get() const noexcept
+{
+    if (type() != format::type::nil) {
+        return tl::make_unexpected(error::wrong_type);
+    }
+    return nil{};
+}
+
+template<>
+constexpr tl::expected<invalid, error>
+token_base<8>::get() const noexcept
+{
+    if (type() != format::type::invalid) {
+        return tl::make_unexpected(error::wrong_type);
+    }
+    return invalid{};
+}
+
 using token = token_base<sizeof(void*)>;
 
 } // namespace msgpack

source/include/parselink/msgpack/util/endianness.h

@@ -121,9 +121,10 @@ constexpr auto value_cast(Array&& data) noexcept {
  */
 template <endianness From, endianness To, typename T>
 requires std::is_trivial_v<std::remove_reference_t<T>>
-constexpr T swap(T val) noexcept {
+constexpr T byte_swap(T val) noexcept {
-    return value_cast<T>(
+    using array_type = std::array<std::byte, sizeof(T)>;
-            maybe_swap<From, To>(raw_cast(val)));
+    return std::bit_cast<T>(
+            maybe_swap<From, To>(std::bit_cast<array_type>(val)));
 }
 
 } // namespace detail;
@@ -144,7 +145,7 @@ struct endian {
 
 template <typename T>
 constexpr T be_to_host(T val) noexcept {
-    return detail::swap<endian::big, endian::host>(val);
+    return detail::byte_swap<endian::big, endian::host>(val);
 }
 
 template <typename Iter, typename OutIter>
@@ -154,17 +155,17 @@ constexpr void be_to_host(Iter begin, Iter end, OutIter dest) noexcept {
 
 template <typename T>
 constexpr T host_to_be(T val) noexcept {
-    return detail::swap<endian::host, endian::big>(val);
+    return detail::byte_swap<endian::host, endian::big>(val);
 }
 
 template <typename T>
 constexpr T le_to_host(T val) noexcept {
-    return detail::swap<endian::little, endian::host>(val);
+    return detail::byte_swap<endian::little, endian::host>(val);
 }
 
 template <typename T>
 constexpr T host_to_le(T val) noexcept {
-    return detail::swap<endian::host, endian::little>(val);
+    return detail::byte_swap<endian::host, endian::little>(val);
 }
 
 #endif // msgpack_endianness_d8ae54f45851ed13

tests/msgpack/test_token.cpp

@@ -118,7 +118,7 @@ suite assignment_and_access = [] {
         std::vector<std::byte> extracted_val;
         {
             auto val = make_bytes(0x32, 0xff, 0xaa, 0xce);
-            msgpack::token obj(val); expect(obj.type() == msgpack::format::type::bytes);
+            msgpack::token obj(val); expect(obj.type() == msgpack::format::type::binary);
             auto retrieved = obj.get<std::span<std::byte const>>();
             expect(bool(retrieved));
             expect(std::equal(retrieved->begin(), retrieved->end(),

tests/msgpack/test_token_reader.cpp

@@ -1,48 +1,83 @@
 #include <msgpack/token.h>
 
+#include <fmt/format.h>
 #include <boost/ut.hpp>
+#include <magic_enum.hpp>
 
 using namespace boost::ut;
 
 namespace {
-    template <typename... Bytes>
+template <typename T>
-    constexpr std::array<std::byte, sizeof...(Bytes)> make_bytes(Bytes &&...bytes) {
+constexpr bool operator==(std::span<T const> a, std::span<T const> b) noexcept {
-      return {std::byte(std::forward<Bytes>(bytes))...};
+    return std::equal(a.begin(), a.end(), b.begin(), b.end());
-    }
+}
 
-template <typename T, std::size_t C = 1024> struct oversized_array {
+template <typename T>
+constexpr std::array<std::byte, sizeof(T)> as_bytes(T&& t) {
+    return std::bit_cast<std::array<std::byte, sizeof(T)>>(std::forward<T>(t));
+}
+
+template <typename... Bytes>
+constexpr std::array<std::byte, sizeof...(Bytes)> make_bytes(Bytes &&...bytes) {
+    return {std::byte(std::forward<Bytes>(bytes))...};
+}
+
+template <typename T, std::size_t C = 1024 * 1024> struct oversized_array {
     std::array<T, C> data;
     std::size_t size;
 };
 
+
 constexpr auto to_bytes_array_oversized(auto const &container) {
-  oversized_array<std::byte> arr;
+    using value_type = std::decay_t<decltype(container[0])>;
+    oversized_array<value_type> arr;
     std::copy(std::begin(container), std::end(container), std::begin(arr.data));
     arr.size = std::distance(std::begin(container), std::end(container));
     return arr;
 }
 
-    consteval auto generate_bytes(auto callable) {
+consteval auto generate_bytes(auto callable) {
     constexpr auto oversized = to_bytes_array_oversized(callable());
-      std::array<std::byte, oversized.size> out;
+    using value_type = std::decay_t<decltype(oversized.data[0])>;
+    std::array<value_type, oversized.size> out;
     std::copy(std::begin(oversized.data),
             std::next(std::begin(oversized.data), oversized.size),
             std::begin(out));
     return out;
-    }
+}
 
-template <std::size_t A, std::size_t B>
+consteval auto build_string(auto callable) {
-consteval auto cat(std::array<std::byte, A> const &a,
+    constexpr auto string_array = generate_bytes(callable);
-                   std::array<std::byte, B> const &b) {
+    return string_array;
-  std::array<std::byte, A + B> out;
+}
-  std::copy(std::begin(a), std::next(std::begin(a), std::size(a)),
+
-            std::begin(out));
+template <std::size_t ... Sizes>
-  std::copy(std::begin(b), std::next(std::begin(b), std::size(b)),
+constexpr auto cat(std::array<std::byte, Sizes>const&... a) noexcept {
-            std::next(std::begin(out), std::size(a)));
+    std::array<std::byte, (Sizes + ...)> out;
+    std::size_t index{};
+    ((std::copy_n(a.begin(), Sizes, out.begin() + index), index += Sizes), ...);
     return out;
 }
 
-    constexpr auto from_string_view(std::string_view sv) {
+constexpr auto repeat(std::span<std::byte const> sv, std::size_t count) {
+    std::vector<std::byte> range;
+    range.reserve(sv.size() * count);
+    for (decltype(count) i = 0; i < count; ++i) {
+        std::copy_n(sv.begin(), sv.size(), std::back_inserter(range));
+    }
+    return range;
+}
+
+constexpr auto repeat(std::string_view sv, std::size_t count) {
+    std::vector<char> range;
+    range.reserve(sv.size() * count);
+    for (decltype(count) i = 0; i < count; ++i) {
+        std::copy_n(sv.begin(), sv.size(), std::back_inserter(range));
+    }
+    return range;
+}
+
+constexpr auto from_string_view(std::string_view sv) {
     std::vector<std::byte> range;
     range.resize(sv.size());
     auto itr = range.begin();
@@ -51,20 +86,9 @@ consteval auto cat(std::array<std::byte, A> const &a,
         ++itr;
     }
     return range;
-    }
+}
 
-    template <typename First, typename... Others>
+template <typename T, typename E>
-    constexpr bool wrong_types(auto const& obj) {
-        auto err = tl::make_unexpected(msgpack::error::wrong_type);
-        if (obj.template get<First>() != err) return false;
-        if constexpr (sizeof...(Others)) {
-            return wrong_types<Others...>(obj);
-        } else {
-            return true;
-        }
-    }
-
-    template <typename T, typename E>
     std::ostream &operator<<(std::ostream &os, tl::expected<T, E> const &exp) {
         if (exp.has_value()) {
             os << "Value: '" << *exp << "'";
@@ -73,32 +97,431 @@ consteval auto cat(std::array<std::byte, A> const &a,
         }
         return os;
     }
+
+bool test_incomplete_message(auto const& payload) {
+    // Test incomplete message.
+    for (decltype(payload.size()) i = 1; i < payload.size() - 1; ++i) {
+        // Test incomplete message.
+        msgpack::token_reader reader(std::span(payload.data(), i));
+        auto token = reader.read_one();
+        if (token != tl::make_unexpected(msgpack::error::incomplete_message)) {
+            fmt::print("Got the wrong response reading subview[0,{}] of payload: {}\n",
+                    i,
+                    token->get<std::string_view>().value());
+            return false;
+        }
+    }
+    return true;
 }
 
-suite reader = [] {
+bool test_end_of_message(auto& reader) {
-    "read uint32"_test = [] {
+    return reader.read_one() ==
-        constexpr auto payload = make_bytes(0xce, 0x01, 0x02, 0x03, 0x09);
+        tl::make_unexpected(msgpack::error::end_of_message);
+}
+
+}
+
+suite reader_tests = [] {
+    //--------------------------------------------------------------------------
+    // Unsigned integer format tests
+    //--------------------------------------------------------------------------
+    "read format::positive_fixint"_test = [] {
+        static constexpr auto payload = make_bytes(0x35);
+
+        // Test incomplete message.
+        expect(test_incomplete_message(payload));
+
+        // Test complete message.
         msgpack::token_reader reader(payload);
         auto token = reader.read_one();
         expect(token && token->type() == msgpack::format::type::unsigned_int);
-        expect(token->get<std::uint8_t>() == tl::make_unexpected(msgpack::error::will_truncate));
+        expect(token->get<std::uint8_t>() == 0x35);
-        expect(token->get<std::uint16_t>() == tl::make_unexpected(msgpack::error::will_truncate));
+        expect(token->get<std::uint16_t>() == 0x35);
-        expect(token->get<std::uint32_t>() == 0x01020309);
+        expect(token->get<std::uint32_t>() == 0x35);
-        expect(token->get<std::uint64_t>() == 0x01020309);
+        expect(token->get<std::uint64_t>() == 0x35);
-        token = reader.read_one();
+
-        expect(token == tl::make_unexpected(msgpack::error::end_of_message));
+        // EOM
+        expect(test_end_of_message(reader));
     };
-    "read str8"_test = [] {
+    "read format::uint8"_test = [] {
-        constexpr std::string_view sv = "hello d";
+        static constexpr auto payload = make_bytes(0xcc, 0x02);
+
+        // Test incomplete message.
+        expect(test_incomplete_message(payload));
+
+        // Test complete message.
+        msgpack::token_reader reader(payload);
+        auto token = reader.read_one();
+        expect(token && token->type() == msgpack::format::type::unsigned_int);
+        expect(token->get<std::uint8_t>() == 0x02);
+        expect(token->get<std::uint16_t>() == 0x02);
+        expect(token->get<std::uint32_t>() == 0x02);
+        expect(token->get<std::uint64_t>() == 0x02);
+
+        // EOM
+        expect(test_end_of_message(reader));
+    };
+    "read format::uint16"_test = [] {
+        constexpr auto payload = make_bytes(0xcd, 0x01, 0x02);
+
+        // Test incomplete message.
+        expect(test_incomplete_message(payload));
+
+        // Test complete message.
+        msgpack::token_reader reader(payload);
+        auto token = reader.read_one();
+        expect(token && token->type() == msgpack::format::type::unsigned_int);
+        expect(token->get<std::uint8_t>() ==
+                tl::make_unexpected(msgpack::error::will_truncate));
+        expect(token->get<std::uint16_t>() == 0x0102);
+        expect(token->get<std::uint32_t>() == 0x0102);
+        expect(token->get<std::uint64_t>() == 0x0102);
+
+        // EOM
+        expect(test_end_of_message(reader));
+    };
+    "read format::uint32"_test = [] {
+        constexpr auto payload = make_bytes(0xce, 0x01, 0x02, 0x03, 0x04);
+
+        // Test incomplete message.
+        expect(test_incomplete_message(payload));
+
+        // Test complete message.
+        msgpack::token_reader reader(payload);
+        auto token = reader.read_one();
+        expect(token && token->type() == msgpack::format::type::unsigned_int);
+        expect(token->get<std::uint8_t>() ==
+                tl::make_unexpected(msgpack::error::will_truncate));
+        expect(token->get<std::uint16_t>() ==
+                tl::make_unexpected(msgpack::error::will_truncate));
+        expect(token->get<std::uint32_t>() == 0x01020304);
+        expect(token->get<std::uint64_t>() == 0x01020304);
+        // EOM
+        expect(test_end_of_message(reader));
+    };
+    "read format::uint64"_test = [] {
+        constexpr auto payload = make_bytes(0xcf,
+                0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08);
+
+        // Test incomplete message.
+        expect(test_incomplete_message(payload));
+
+        // Test complete message.
+        msgpack::token_reader reader(payload);
+        auto token = reader.read_one();
+        expect(token && token->type() == msgpack::format::type::unsigned_int);
+        expect(token->get<std::uint8_t>() ==
+                tl::make_unexpected(msgpack::error::will_truncate));
+        expect(token->get<std::uint16_t>() ==
+                tl::make_unexpected(msgpack::error::will_truncate));
+        expect(token->get<std::uint32_t>() ==
+                tl::make_unexpected(msgpack::error::will_truncate));
+        expect(token->get<std::uint64_t>() == 0x0102030405060708);
+        // EOM
+        expect(test_end_of_message(reader));
+    };
+
+    //--------------------------------------------------------------------------
+    // Signed integer format tests
+    //--------------------------------------------------------------------------
+    "read format::negative_fixint"_test = [] {
+        static constexpr auto payload = make_bytes(0xe5);
+
+        // Test incomplete message.
+        expect(test_incomplete_message(payload));
+
+        // Test complete message.
+        msgpack::token_reader reader(payload);
+        auto token = reader.read_one();
+        expect(token && token->type() == msgpack::format::type::signed_int);
+        expect(token->get<std::int8_t>() == -27);
+        expect(token->get<std::int16_t>() == -27);
+        expect(token->get<std::int32_t>() == -27);
+        expect(token->get<std::int64_t>() == -27);
+
+        // EOM
+        expect(test_end_of_message(reader));
+    };
+    "read format::int8"_test = [] {
+        static constexpr auto payload = make_bytes(0xd0, 0xff);
+
+        // Test incomplete message.
+        expect(test_incomplete_message(payload));
+
+        // Test complete message.
+        msgpack::token_reader reader(payload);
+        auto token = reader.read_one();
+        expect(token && token->type() == msgpack::format::type::signed_int);
+        expect(token->get<std::int8_t>() == -1);
+        expect(token->get<std::int16_t>() == -1);
+        expect(token->get<std::int32_t>() == -1);
+        expect(token->get<std::int64_t>() == -1);
+
+        // EOM
+        expect(test_end_of_message(reader));
+    };
+    "read format::int16"_test = [] {
+        constexpr auto payload = make_bytes(0xd1, 0xff, 0x00);
+
+        // Test incomplete message.
+        expect(test_incomplete_message(payload));
+
+        // Test complete message.
+        msgpack::token_reader reader(payload);
+        auto token = reader.read_one();
+        expect(token && token->type() == msgpack::format::type::signed_int);
+        expect(token->get<std::int8_t>() ==
+                tl::make_unexpected(msgpack::error::will_truncate));
+        expect(token->get<std::int16_t>() == -256);
+        expect(token->get<std::int32_t>() == -256);
+        expect(token->get<std::int64_t>() == -256);
+
+        // EOM
+        expect(test_end_of_message(reader));
+    };
+    "read format::int32"_test = [] {
+        constexpr auto payload = make_bytes(0xd2, 0xff, 0xff, 0x00, 0x00);
+
+        // Test incomplete message.
+        expect(test_incomplete_message(payload));
+
+        // Test complete message.
+        msgpack::token_reader reader(payload);
+        auto token = reader.read_one();
+        expect(token && token->type() == msgpack::format::type::signed_int);
+        expect(token->get<std::int8_t>() ==
+                tl::make_unexpected(msgpack::error::will_truncate));
+        expect(token->get<std::int16_t>() ==
+                tl::make_unexpected(msgpack::error::will_truncate));
+        expect(token->get<std::int32_t>() == -65536);
+        expect(token->get<std::int64_t>() == -65536);
+        // EOM
+        expect(test_end_of_message(reader));
+    };
+    "read format::int64"_test = [] {
+        constexpr auto payload = make_bytes(0xd3,
+                0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00);
+
+        // Test incomplete message.
+        expect(test_incomplete_message(payload));
+
+        // Test complete message.
+        msgpack::token_reader reader(payload);
+        auto token = reader.read_one();
+        expect(token && token->type() == msgpack::format::type::signed_int);
+        expect(token->get<std::int8_t>() ==
+                tl::make_unexpected(msgpack::error::will_truncate));
+        expect(token->get<std::int16_t>() ==
+                tl::make_unexpected(msgpack::error::will_truncate));
+        expect(token->get<std::int32_t>() ==
+                tl::make_unexpected(msgpack::error::will_truncate));
+        expect(token->get<std::int64_t>() == -4294967296);
+        // EOM
+        expect(test_end_of_message(reader));
+    };
+
+    //--------------------------------------------------------------------------
+    // String format tests
+    //--------------------------------------------------------------------------
+    "read format::fixstr"_test = [] {
+        constexpr std::string_view sv = "hello";
+        constexpr auto payload =
+            cat(make_bytes(0xa5),
+                generate_bytes([sv] { return from_string_view(sv); }));
+
+        // Test incomplete message.
+        expect(test_incomplete_message(payload));
+
+        // Test complete message.
+        msgpack::token_reader reader(payload);
+        auto token = reader.read_one();
+
+        expect(token && token->type() == msgpack::format::type::string);
+        expect(token->get<std::string_view>() == sv);
+
+        // EOM
+        expect(test_end_of_message(reader));
+    };
+    "read format::str8"_test = [] {
+        constexpr std::string_view sv = "hello world";
         constexpr auto payload =
             cat(make_bytes(0xd9, sv.size()),
                 generate_bytes([sv] { return from_string_view(sv); }));
 
+        // Test incomplete message.
+        expect(test_incomplete_message(payload));
+
+        // Test complete message.
         msgpack::token_reader reader(payload);
         auto token = reader.read_one();
+
         expect(token && token->type() == msgpack::format::type::string);
         expect(token->get<std::string_view>() == sv);
+
+        // EOM
+        expect(test_end_of_message(reader));
+    };
+    "read format::str16"_test = [] {
+        static constexpr auto sa = build_string([]{
+                return repeat("0123456789abcdef", 17);
+        });
+        constexpr auto sv = std::string_view(sa.data(), sa.size());
+        constexpr auto sv_size = host_to_be(std::uint16_t(sv.size()));
+
+        auto payload =
+            cat(make_bytes(0xda), as_bytes(sv_size),
+                generate_bytes([] { return from_string_view(sv); }));
+
+        // Test incomplete message.
+        expect(test_incomplete_message(payload));
+
+        // Test complete message.
+        msgpack::token_reader reader(payload);
+        auto token = reader.read_one();
+
+        expect(token && token->type() == msgpack::format::type::string);
+        expect(token->get<std::string_view>() == sv);
+
+        // EOM
+        expect(test_end_of_message(reader));
+    };
+    "read format::str32"_test = [] {
+        static constexpr auto sa = build_string([]{
+                return repeat("0123456789abcdef", 4097);
+        });
+        constexpr auto sv = std::string_view(sa.data(), sa.size());
+        constexpr auto sv_size = host_to_be(std::uint32_t(sv.size()));
+
+        constexpr auto payload =
+            cat(make_bytes(0xdb), as_bytes(sv_size),
+                generate_bytes([sv] { return from_string_view(sv); }));
+
+        // Test incomplete message.
+        expect(test_incomplete_message(payload));
+
+        // Test complete message.
+        msgpack::token_reader reader(payload);
+        auto token = reader.read_one();
+
+        expect(token && token->type() == msgpack::format::type::string);
+        expect(token->get<std::string_view>() == sv);
+
+        // EOM
+        expect(test_end_of_message(reader));
+    };
+
+    //--------------------------------------------------------------------------
+    // Binary format tests
+    //--------------------------------------------------------------------------
+    "read format::bin8"_test = [] {
+        constexpr auto bytes = make_bytes(
+                0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,
+                0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10
+        );
+        constexpr auto bytes_size = host_to_be(std::uint8_t(bytes.size()));
+        constexpr auto payload =
+            cat(make_bytes(0xc4), as_bytes(bytes_size), bytes);
+
+        // Test incomplete message.
+        expect(test_incomplete_message(payload));
+
+        // Test complete message.
+        msgpack::token_reader reader(payload);
+        auto token = reader.read_one();
+
+        expect(token && token->type() == msgpack::format::type::binary);
+        auto value = token->get<std::span<std::byte const>>().value();
+        expect(value == std::span(bytes.data(), bytes.size()));
+
+        // EOM
+        expect(test_end_of_message(reader));
+    };
+    "read format::bin16"_test = [] {
+        static constexpr auto bytes = generate_bytes([]{
+            return repeat(
+                    make_bytes(0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08),
+                    16);
+        });
+        constexpr auto bytes_size = host_to_be(std::uint16_t(bytes.size()));
+
+        auto payload = cat(make_bytes(0xc5), as_bytes(bytes_size), bytes);
+
+        // Test incomplete message.
+        expect(test_incomplete_message(payload));
+
+        // Test complete message.
+        msgpack::token_reader reader(payload);
+        auto token = reader.read_one();
+
+        expect(token && token->type() == msgpack::format::type::binary);
+        auto value = token->get<std::span<std::byte const>>().value();
+        expect(value == std::span(bytes.data(), bytes.size()));
+
+        // EOM
+        expect(test_end_of_message(reader));
+    };
+    "read format::bin32"_test = [] {
+        static constexpr auto bytes = generate_bytes([]{
+            return repeat(
+                    make_bytes(0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08),
+                    8193);
+        });
+        constexpr auto bytes_size = host_to_be(std::uint32_t(bytes.size()));
+
+        constexpr auto payload =
+            cat(make_bytes(0xc6), as_bytes(bytes_size), bytes);
+
+        // Test incomplete message.
+        expect(test_incomplete_message(payload));
+
+        // Test complete message.
+        msgpack::token_reader reader(payload);
+        auto token = reader.read_one();
+
+        expect(token && token->type() == msgpack::format::type::binary);
+        auto value = token->get<std::span<std::byte const>>().value();
+        expect(value == std::span(bytes.data(), bytes.size()));
+
+        // EOM
+        expect(test_end_of_message(reader));
+    };
+    //--------------------------------------------------------------------------
+    // Assorted format tests
+    //--------------------------------------------------------------------------
+    "read format::boolean"_test = [] {
+        constexpr auto payload = make_bytes(0xc2, 0xc3);
+
+        // Test complete message.
+        msgpack::token_reader reader(payload);
+        auto token = reader.read_one();
+        expect(token && token->type() == msgpack::format::type::boolean);
+        expect(token->get<bool>() == false);
         token = reader.read_one();
-        expect(token == tl::make_unexpected(msgpack::error::end_of_message));
+        expect(token && token->type() == msgpack::format::type::boolean);
+        expect(token->get<bool>() == true);
+        // EOM
+        expect(test_end_of_message(reader));
+    };
+    "read format::nil"_test = [] {
+        constexpr auto payload = make_bytes(0xc0);
+
+        // Test complete message.
+        msgpack::token_reader reader(payload);
+        auto token = reader.read_one();
+        expect(token && token->type() == msgpack::format::type::nil);
+        expect(token->get<msgpack::nil>().has_value());
+        // EOM
+        expect(test_end_of_message(reader));
+    };
+    "read format::invalid"_test = [] {
+        constexpr auto payload = make_bytes(0xc1);
+
+        // Test complete message.
+        msgpack::token_reader reader(payload);
+        auto token = reader.read_one();
+        expect(token && token->type() == msgpack::format::type::invalid);
+        expect(token->get<msgpack::invalid>().has_value());
+        // EOM
+        expect(test_end_of_message(reader));
     };
 };