Packer WIP

include/parselink/msgpack/core/packer.h

@@ -69,8 +69,16 @@ struct pack_helper {};
 template <>
 struct pack_helper<format::type::unsigned_int> {
     static constexpr std::size_t num_bytes(std::uint64_t value) noexcept {
+#ifdef PARSELINK_CFG_MSGPACK_PACKER_USE_BRANCHES
+        if (count <= 0x7f) return 0;
+        if (count < 0x100) return 1;
+        if (count < 0x10000) return 2;
+        if (count < 0x100000000) return 4;
+        return 8;
+#else
         if (value <= std::uint64_t(format::positive_fixint::mask)) return 0;
         return std::bit_ceil(std::uint64_t((std::bit_width(value) + 7) >> 3));
+#endif
     }
 
     static constexpr std::byte marker(std::uint64_t value) noexcept {
@@ -113,15 +121,73 @@ struct pack_helper<format::type::signed_int> {
     }
 };
 
+template <>
+struct pack_helper<format::type::binary> {
+    static constexpr uint32_t max_supported_payload = 4;
+
+    static constexpr std::size_t num_bytes(
+            std::span<std::byte const> value) noexcept {
+        return std::min(max_supported_payload,
+                std::bit_ceil(std::uint32_t(
+                        ((std::bit_width(value.size()) + 7) >> 3))));
+    }
+
+    static constexpr auto marker(std::span<std::byte const> value) noexcept {
+        auto const len = num_bytes(value);
+        switch (len) {
+            case 1: return format::bin8::marker;
+            case 2: return format::bin16::marker;
+            case 4:
+            default: return format::bin32::marker;
+        }
+    }
+};
+
+template <>
+struct pack_helper<format::type::array> {
+    static constexpr uint32_t max_supported_payload = 4;
+
+    static constexpr std::size_t num_bytes(std::size_t count) noexcept {
+        if (count < 16) return 0;
+        return std::min(max_supported_payload,
+                std::bit_ceil(std::uint32_t(
+                        ((std::bit_width(count) + 15) >> 4) << 1)));
+    }
+
+    static constexpr auto marker(std::size_t value) noexcept {
+        auto const len = num_bytes(value);
+        switch (len) {
+            case 0:
+            case 2: return format::array16::marker;
+            case 4:
+            default: return format::bin32::marker;
+        }
+    }
+};
+
+constexpr auto x = pack_helper<format::type::array>::num_bytes(15);
+constexpr auto y = pack_helper<format::type::array>::num_bytes(16);
+constexpr auto z = pack_helper<format::type::array>::num_bytes(256);
+constexpr auto a = pack_helper<format::type::array>::num_bytes(65536);
+
 template <>
 struct pack_helper<format::type::string> {
-    static constexpr std::size_t num_bytes(std::uint64_t value) noexcept {
-        if (value <= std::uint32_t(format::fixstr::mask)) return 0;
-        return std::bit_ceil(std::uint32_t((std::bit_width(value) + 7) >> 3));
+    static constexpr uint32_t max_supported_payload = 4;
+
+    static constexpr std::size_t num_bytes(std::string_view value) noexcept {
+        auto const len = value.size();
+        if (len <= std::uint32_t(format::fixstr::mask)) return 0;
+        return std::min(max_supported_payload,
+                std::bit_ceil(std::uint32_t((std::bit_width(len) + 7) >> 3)));
+    }
+
+    static constexpr std::size_t payload_size(std::string_view value) noexcept {
+        return value.size();
     }
 
     static constexpr auto marker(std::string_view value) noexcept {
-        switch (num_bytes(value.size())) {
+        auto const len = num_bytes(value);
+        switch (len) {
             case 0: return format::fixstr::marker | std::byte(value.size());
             case 1: return format::str8::marker;
             case 2: return format::str16::marker;
@@ -138,21 +204,38 @@ template <typename T>
 struct pack_adapter {};
 
 template <typename T>
-concept builtin_packable_type = requires(T const& t, std::byte* b) {
-    { pack_adapter<T>::size(t) } -> std::same_as<std::size_t>;
+concept basic_packable_type = requires(T const& t, std::byte* b) {
+    { pack_adapter<T>::total_size(t) } -> std::same_as<std::size_t>;
     { pack_adapter<T>::write(t, b) } -> std::same_as<decltype(b)>;
     { pack_adapter<T>::marker(t) } -> std::same_as<std::byte>;
 };
 
 template <typename T>
-concept packable_type = builtin_packable_type<T>;
+concept packable_type = basic_packable_type<T>;
 
 template <format::type F>
 struct builtin_pack_adapter {
     static constexpr auto format_type = F;
+    using pack_helper = detail::pack_helper<F>;
 
-    static constexpr auto size(auto value) noexcept {
-        return detail::pack_helper<F>::num_bytes(value);
+    // Retrieves the number of bytes to store the format, aside from the marker
+    // byte.
+    static constexpr auto format_size(auto value) noexcept {
+        return pack_helper::num_bytes(value);
+    }
+
+    // Retrieves the number of bytes to store the payload, if it is
+    // variable-length.
+    static constexpr std::size_t payload_size(auto value) noexcept {
+        if constexpr (requires { pack_helper::payload_size(value); }) {
+            return pack_helper::payload_size(value);
+        } else {
+            return 0;
+        }
+    }
+
+    static constexpr auto total_size(auto value) noexcept {
+        return 1 + format_size(value) + payload_size(value);
     }
 
     static constexpr auto marker(auto value) noexcept {
@@ -164,7 +247,7 @@ template <std::signed_integral T>
 struct pack_adapter<T> : builtin_pack_adapter<format::type::signed_int> {
     template <typename Itr>
     static constexpr Itr write(T value, Itr out) noexcept {
-        return detail::write_integral(value, out, size(value));
+        return detail::write_integral(value, out, format_size(value));
     }
 };
 
@@ -172,10 +255,73 @@ template <std::unsigned_integral T>
 struct pack_adapter<T> : builtin_pack_adapter<format::type::unsigned_int> {
     template <typename Itr>
     static constexpr Itr write(T value, Itr out) noexcept {
-        return detail::write_integral(value, out, size(value));
+        return detail::write_integral(value, out, format_size(value));
     }
 };
 
+template <>
+struct pack_adapter<std::string_view>
+        : builtin_pack_adapter<format::type::string> {
+    template <typename Itr>
+    static constexpr Itr write(std::string_view value, Itr out) noexcept {
+        out = detail::write_integral(value.size(), out, format_size(value));
+        auto const* beg = reinterpret_cast<std::byte const*>(&*value.begin());
+        std::copy(beg, beg + value.size(), out);
+        return out + value.size();
+    }
+};
+
+template <std::convertible_to<std::string_view> T>
+struct pack_adapter<T> : pack_adapter<std::string_view> {};
+
+template <>
+struct pack_adapter<std::span<std::byte const>>
+        : builtin_pack_adapter<format::type::binary> {
+    template <typename Itr>
+    static constexpr Itr write(
+            std::span<std::byte const> value, Itr out) noexcept {
+        out = detail::write_integral(value.size(), out, format_size(value));
+        std::copy(value.begin(), value.end(), out);
+        return out + value.size();
+    }
+};
+
+class packer {
+    using BufferType = std::span<std::byte>;
+
+public:
+    constexpr packer(BufferType buff)
+            : buff_(buff)
+            , curr_(std::begin(buff_))
+            , end_(std::end(buff_)) {}
+
+    template <packable_type T>
+    constexpr tl::expected<tl::monostate, error> pack(T&& v) noexcept {
+        using diff_type =
+                std::iterator_traits<decltype(curr_)>::difference_type;
+        diff_type const space_needed = pack_adapter<T>::total_size(v);
+        if (space_needed > std::distance(curr_, end_)) {
+            return tl::make_unexpected(error::out_of_space);
+        }
+        *curr_++ = pack_adapter<T>::marker(v);
+        if (space_needed > 1) {
+            curr_ = pack_adapter<T>::write(v, curr_);
+        }
+        return tl::monostate{};
+    }
+
+    constexpr auto tell() const noexcept {
+        return std::distance(std::begin(buff_), curr_);
+    }
+
+    constexpr auto subspan() const noexcept { return buff_.subspan(0, tell()); }
+
+private:
+    BufferType buff_;
+    decltype(std::begin(buff_)) curr_;
+    decltype(std::end(buff_)) end_;
+};
+
 } // namespace msgpack
 
 #endif // msgpack_core_packer_1d5939e9c1498568

tests/msgpack/BUILD

@@ -16,6 +16,7 @@ cc_library(
 
 cc_test(
     name = "reader",
+    size = "small",
     srcs = [
         "test_reader_relaxed.cpp",
         "test_reader_strict.cpp",
@@ -23,8 +24,18 @@ cc_test(
     deps = ["test_deps"],
 )
 
+cc_test(
+    name = "packer",
+    size = "small",
+    srcs = [
+        "test_packer.cpp",
+    ],
+    deps = ["test_deps"],
+)
+
 cc_test(
     name = "writer",
+    size = "small",
     srcs = [
         "test_writer.cpp",
     ],
@@ -33,6 +44,7 @@ cc_test(
 
 cc_test(
     name = "token",
+    size = "small",
     srcs = [
         "test_token.cpp",
     ],
@@ -41,6 +53,7 @@ cc_test(
 
 cc_test(
     name = "token_reader",
+    size = "small",
     srcs = [
         "test_token_reader.cpp",
     ],
@@ -49,6 +62,7 @@ cc_test(
 
 cc_test(
     name = "token_views",
+    size = "small",
     srcs = [
         "test_token_views.cpp",
     ],

tests/msgpack/test_packer.cpp

@@ -0,0 +1,287 @@
+//-----------------------------------------------------------------------------
+//     ___                      __ _       _
+//    / _ \__ _ _ __ ___  ___  / /(_)_ __ | | __
+//   / /_)/ _` | '__/ __|/ _ \/ / | | '_ \| |/ /
+//  / ___/ (_| | |  \__ \  __/ /__| | | | |   <
+//  \/    \__,_|_|  |___/\___\____/_|_| |_|_|\_\ .
+//
+//-----------------------------------------------------------------------------
+// Author: Kurt Sassenrath
+// Module: msgpack
+//
+// Default packer tests.
+//
+// Copyright (c) 2023 Kurt Sassenrath.
+//
+// License TBD.
+//-----------------------------------------------------------------------------
+
+#include "parselink/msgpack/core/packer.h"
+
+#include <fmt/format.h>
+#include <fmt/ranges.h>
+
+#include "rng.h"
+#include <boost/ut.hpp>
+#include <chrono>
+
+using namespace boost::ut;
+
+namespace {
+
+template <typename... Bytes>
+constexpr std::array<std::byte, sizeof...(Bytes)> make_bytes(Bytes&&... bytes) {
+    return {static_cast<std::byte>(std::forward<Bytes>(bytes))...};
+}
+
+constexpr auto equal(auto a, auto b) {
+    return std::equal(std::begin(a), std::end(a), std::begin(b), std::end(b));
+}
+
+template <typename T>
+struct test_data {};
+
+template <>
+struct test_data<std::uint64_t> {
+    static constexpr auto values = std::to_array<std::uint64_t>({
+            0x00,              // positive fixint
+            0x79,              // positive fixint
+            0x80,              // uint8
+            0xff,              // uint8
+            0x100,             // uint16
+            0xffff,            // uint16
+            0x10000,           // uint32
+            0xffffffff,        // uint32
+            0x100000000,       // uint64
+            0xffffffffffffffff // uint64
+    });
+
+    static constexpr auto payload = make_bytes(0x00, // positive fixint
+            0x79,                                    // positive fixint
+            0xcc, 0x80,                              // uint8
+            0xcc, 0xff,                              // uint8
+            0xcd, 0x01, 0x00,                        // uint16
+            0xcd, 0xff, 0xff,                        // uint16
+            0xce, 0x00, 0x01, 0x00, 0x00,            // uint32
+            0xce, 0xff, 0xff, 0xff, 0xff,            // uint32
+            0xcf, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, // uint64
+            0xcf, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff  // uint64
+    );
+
+    static constexpr auto valid(auto value) noexcept {
+        return value <= std::numeric_limits<std::uint64_t>::max();
+    }
+};
+
+template <std::unsigned_integral T>
+struct test_data<T> : test_data<std::uint64_t> {
+    static constexpr auto valid(auto value) noexcept {
+        return value <= std::numeric_limits<T>::max();
+    }
+};
+
+template <>
+struct test_data<std::int64_t> {
+    static constexpr auto values = std::to_array<std::int64_t>({
+            -1,                                          // negative fixint
+            -32,                                         // negative fixint
+            -33,                                         // int8
+            -128,                                        // int8
+            0,                                           // int8
+            127,                                         // int8
+            128,                                         // int16
+            -129,                                        // int16
+            -32768,                                      // int16
+            32767,                                       // int16
+            -32769,                                      // int32
+            32768,                                       // int32
+            -2147483648,                                 // int32
+            2147483647,                                  // int32
+            -2147483649,                                 // int64
+            2147483648,                                  // int64
+            std::numeric_limits<std::int64_t>::lowest(), // int64
+            std::numeric_limits<std::int64_t>::max(),    // int64
+    });
+
+    static constexpr auto payload = make_bytes(0xff, // negative fixint
+            0xe0,                                    // negative fixint
+            0xd0, 0xdf,                              // int8
+            0xd0, 0x80,                              // int8
+            0xd0, 0x0,                               // int8
+            0xd0, 0x7f,                              // int8
+            0xd1, 0x00, 0x80,                        // int16
+            0xd1, 0xff, 0x7f,                        // int16
+            0xd1, 0x80, 0x00,                        // int16
+            0xd1, 0x7f, 0xff,                        // int16
+            0xd2, 0xff, 0xff, 0x7f, 0xff,            // int32
+            0xd2, 0x00, 0x00, 0x80, 0x00,            // int32
+            0xd2, 0x80, 0x00, 0x00, 0x00,            // int32
+            0xd2, 0x7f, 0xff, 0xff, 0xff,            // int32
+            0xd3, 0xff, 0xff, 0xff, 0xff, 0x7f, 0xff, 0xff, 0xff, // int64
+            0xd3, 0x00, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, // int64
+            0xd3, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // int64
+            0xd3, 0x7f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff  // int64
+    );
+
+    static constexpr auto valid(auto value) noexcept {
+        return value <= std::numeric_limits<std::int64_t>::max()
+            && value >= std::numeric_limits<std::int64_t>::lowest();
+    }
+};
+
+template <std::convertible_to<std::string_view> T>
+struct test_data<T> {
+    static constexpr auto values = std::to_array<T>({
+            "",                                 // fixstr
+            "0",                                // fixstr
+            "0123456789abcdef0123456789abcde",  // fixstr
+            "0123456789abcdef0123456789abcdef", // str8
+    });
+
+    static constexpr auto payload = make_bytes(0xa0, // fixstr
+            0xa1, 0x30,                              // fixstr
+
+            // fixstr
+            0xbf, 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39,
+            0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x30, 0x31, 0x32, 0x33, 0x34,
+            0x35, 0x36, 0x37, 0x38, 0x39, 0x61, 0x62, 0x63, 0x64, 0x65,
+
+            // str8
+            0xd9, 0x20, 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38,
+            0x39, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x30, 0x31, 0x32, 0x33,
+            0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x61, 0x62, 0x63, 0x64, 0x65,
+            0x66);
+
+    static constexpr auto valid(T value) noexcept {
+        return std::string_view(value).size()
+            <= std::numeric_limits<std::uint32_t>::max();
+    }
+};
+
+template <>
+struct test_data<std::span<std::byte const>> {
+    static constexpr auto test1 = make_bytes(0x02, 0x03);
+    static constexpr auto values =
+            std::to_array<std::span<std::byte const>>({test1});
+    static constexpr auto payload = make_bytes(0xc4, 0x02, 0x02, 0x03);
+
+    static constexpr auto valid(auto value) noexcept {
+        return value.size() <= std::numeric_limits<std::uint32_t>::max();
+    }
+};
+
+template <std::signed_integral T>
+struct test_data<T> : test_data<std::int64_t> {
+    static constexpr auto valid(auto value) noexcept {
+        return value <= std::numeric_limits<T>::max()
+            && value >= std::numeric_limits<T>::lowest();
+    }
+};
+
+template <typename T>
+bool test_deduced() noexcept {
+    constexpr auto const& expected_payload = test_data<T>::payload;
+    std::array<std::byte, expected_payload.size()> payload;
+
+    msgpack::packer packer(payload);
+    for (auto const& value : test_data<T>::values) {
+        if (!test_data<T>::valid(value)) break;
+        expect(!!packer.pack(T(value)));
+        auto expect = std::span(expected_payload.begin(), packer.tell());
+        auto correct = equal(packer.subspan(), expect);
+        if (!correct) {
+            fmt::print("Deduction failed for '{}'\n", T(value));
+            fmt::print("\tActual: {::#04x}\n", packer.subspan());
+            fmt::print("\tExpect: {::#04x}\n", expect);
+            return false;
+        }
+    }
+    return true;
+}
+
+} // anonymous namespace
+
+suite packer = [] {
+    "packer::pack<std::uint8_t>"_test = [] {
+        expect(test_deduced<std::uint8_t>());
+    };
+    "packer::pack<std::uint16_t>"_test = [] {
+        expect(test_deduced<std::uint16_t>());
+    };
+    "packer::pack<std::uint32_t>"_test = [] {
+        expect(test_deduced<std::uint32_t>());
+    };
+    "packer::pack<std::uint64_t>"_test = [] {
+        expect(test_deduced<std::uint64_t>());
+    };
+
+    "packer::pack<std::int8_t>"_test = [] {
+        expect(test_deduced<std::int8_t>());
+    };
+    "packer::pack<std::int16_t>"_test = [] {
+        expect(test_deduced<std::int16_t>());
+    };
+    "packer::pack<std::int32_t>"_test = [] {
+        expect(test_deduced<std::int32_t>());
+    };
+    "packer::pack<std::int64_t>"_test = [] {
+        expect(test_deduced<std::int64_t>());
+    };
+
+    "packer::pack<std::string_view>"_test = [] {
+        expect(test_deduced<std::string_view>());
+    };
+    "packer::pack<char const*>"_test = [] {
+        expect(test_deduced<char const*>());
+    };
+    "packer::pack<std::array<std::byte>>"_test = [] {
+        expect(test_deduced<std::span<std::byte const>>());
+    };
+};
+
+static constexpr auto num_bytes(std::uint64_t count) {
+    if (count <= 0x7f) return 0;
+    if (count < 0x100) return 1;
+    if (count < 0x10000) return 2;
+    if (count < 0x100000000) return 4;
+    return 8;
+}
+
+static constexpr std::size_t bit_count_bytes(std::uint64_t count) {
+    if (count <= 0x7f) return 0;
+    return std::bit_ceil(std::uint64_t((std::bit_width(count) + 7) >> 3));
+}
+
+suite perf = [] {
+    "test"_test = [] {
+        for (auto j = 0; j < 10; ++j) {
+            using namespace std::chrono_literals;
+            std::vector<std::uint64_t> a;
+            std::vector<std::uint64_t> r1;
+            std::vector<std::uint64_t> r2;
+            a.reserve(10000000);
+            r1.reserve(10000000);
+            r2.reserve(10000000);
+            rng x;
+
+            for (auto i = 0; i < 10000000; ++i) {
+                a.push_back(x.get<std::size_t>());
+            }
+
+            auto start2 = std::chrono::steady_clock::now();
+            for (auto v : a) {
+                r2.push_back(num_bytes(v));
+            }
+            auto end2 = std::chrono::steady_clock::now();
+
+            auto start = std::chrono::steady_clock::now();
+            for (auto v : a) {
+                r1.push_back(bit_count_bytes(v));
+            }
+            auto end = std::chrono::steady_clock::now();
+
+            fmt::print("Time taken: {}us vs {}us\n", (end - start) / 1us,
+                    (end2 - start2) / 1us);
+        }
+    };
+};