token_reader::read_some, operator==, "speed test"

- Added token_reader::read_some(), which takes a view of a token buffer
  reads/unpacks up to that many tokens from a buffer. If an error occurs,
  or if there are fewer msgpack formats within the buffer, then the
  returned span will be a subspan of the original buffer. If no tokens
  were read before encountering an error, token_reader will return the
  error instead.
- token::operator== overload for types that might be stored within the
  token. Handy for cleaning up tests, even though both methods of
  `token::get<T>() == value` and `token == value` are represented for
  now.
- Added a simple tool to open a file containing msgpack content and read
  tokens into a buffer. This is for testing speed, but needs to be made
  into a proper benchmark once more work has been done. Pretty fast
  right now, only taking 2us to parse ~200 tokens (that takes the
  pythong msgpack implementation ~1s to handle).

include/parselink/msgpack/token/reader.h

@@ -244,15 +244,37 @@ public:
         return {tok};
     }
 
-    // Read multiple tokens from the byte buffer. The number of tokens parsed
+    // Read multiple tokens from the byte buffer, returning a subspan with the
-    // can be surmised from the returned span of tokens. If the reader
+    // number of tokens parsed, or a relevant error. If the reader currently
-    // currently points to the end of the byte buffer, then
+    // points to the end of the byte buffer, then error::end_of_message is
-    // error::end_of_message is returned, and if there is not enough data to
+    // returned, and if there is not enough data to fully parse a token, then
-    // fully parse a token, then incomplete_message is returned.
+    // incomplete_message is returned.
     constexpr tl::expected<std::span<token>, error> read_some(
-            std::span<token> token_buffer) noexcept;
+            std::span<token> token_buffer) noexcept {
+        auto tok = token_buffer.begin();
+        error err = error::end_of_message;
+
+        while (tok != token_buffer.end()) {
+            auto result = read_one().map([&tok](auto&& t){
+                    *tok = t;
+                    ++tok;
+            });
+            if (!result) {
+                err = result.error();
+                break;
+            }
+        }
+
+        auto parsed = std::distance(token_buffer.begin(), tok);
+        if (parsed == 0) {
+            return tl::make_unexpected(err);
+        }
+
+        return token_buffer.subspan(0, parsed);
+    }
 
 private:
+
     std::span<std::byte const> data_;
     decltype(data_)::iterator curr_;
     decltype(data_)::iterator end_;

include/parselink/msgpack/token/type.h

@@ -240,6 +240,19 @@ public:
         }
     }
 
+    template <typename T>
+    constexpr bool operator==(T const& t) const noexcept {
+        if constexpr (std::equality_comparable<T>) {
+            auto result = get<T>().map([&t](auto v) { return v == t; });
+            return result && *result;
+        } else {
+            auto result = get<T>().map([&t](auto v) {
+                return std::equal(v.begin(), v.end(), t.begin(), t.end());
+            });
+            return result && *result;
+        }
+    }
+
 private:
     union {
         typename token_traits<format::type::unsigned_int>::storage_type u;

tests/msgpack/BUILD

@@ -46,3 +46,11 @@ cc_test(
     ],
     deps = ["test_deps"],
 )
+
+cc_binary(
+    name = "speed",
+    srcs = [
+        "test_speed.cpp",
+    ],
+    deps = ["test_deps"],
+)

tests/msgpack/test_speed.cpp

@@ -0,0 +1,48 @@
+#include "parselink/msgpack/token.h"
+
+#include <fmt/format.h>
+
+#include <chrono>
+#include <cstdlib>
+#include <fcntl.h>
+#include <vector>
+
+using namespace std::chrono_literals;
+
+auto read_file(char const* filename) {
+    std::vector<std::byte> data;
+    auto fd = open(filename, O_RDONLY);
+    if (!fd) {
+        fmt::print("Could not open {}: {}\n", filename, strerror(errno));
+        return data;
+    }
+
+    auto size = lseek(fd, 0, SEEK_END);
+    data.resize(size);
+    lseek(fd, 0, SEEK_SET);
+
+    fmt::print("Reading {} bytes from {}\n", size, filename);
+    read(fd, data.data(), data.size());
+    close(fd);
+
+    return data;
+}
+
+int main(int argc, char** argv) {
+
+    std::array<msgpack::token, 4096> buf;
+
+    auto data = read_file(argc < 2 ? "output.bin" : argv[1]);
+
+    msgpack::token_reader reader(data);
+
+    auto before = std::chrono::steady_clock::now();
+    auto test = reader.read_some(buf);
+    auto after = std::chrono::steady_clock::now();
+    test.map([&](auto sp){
+        fmt::print("Read {} tokens\n", sp.size());
+    }).map_error([&](auto err){
+        fmt::print("Failed to read tokens: {}\n", (int)err);
+    });
+    fmt::print("Took {} us\n", (after - before) / 1us);
+}

tests/msgpack/test_token_reader.cpp

@@ -139,6 +139,7 @@ suite reader_tests = [] {
         expect(token->get<std::uint16_t>() == 0x35);
         expect(token->get<std::uint32_t>() == 0x35);
         expect(token->get<std::uint64_t>() == 0x35);
+        expect(token == 0x35u);
 
         // EOM
         expect(test_end_of_message(reader));
@@ -157,6 +158,7 @@ suite reader_tests = [] {
         expect(token->get<std::uint16_t>() == 0x02);
         expect(token->get<std::uint32_t>() == 0x02);
         expect(token->get<std::uint64_t>() == 0x02);
+        expect(token == 0x02u);
 
         // EOM
         expect(test_end_of_message(reader));
@@ -176,6 +178,7 @@ suite reader_tests = [] {
         expect(token->get<std::uint16_t>() == 0x0102);
         expect(token->get<std::uint32_t>() == 0x0102);
         expect(token->get<std::uint64_t>() == 0x0102);
+        expect(token == 0x0102u);
 
         // EOM
         expect(test_end_of_message(reader));
@@ -196,6 +199,7 @@ suite reader_tests = [] {
                 tl::make_unexpected(msgpack::error::will_truncate));
         expect(token->get<std::uint32_t>() == 0x01020304);
         expect(token->get<std::uint64_t>() == 0x01020304);
+        expect(token == 0x01020304u);
         // EOM
         expect(test_end_of_message(reader));
     };
@@ -217,6 +221,7 @@ suite reader_tests = [] {
         expect(token->get<std::uint32_t>() ==
                 tl::make_unexpected(msgpack::error::will_truncate));
         expect(token->get<std::uint64_t>() == 0x0102030405060708);
+        expect(token == 0x0102030405060708u);
         // EOM
         expect(test_end_of_message(reader));
     };
@@ -238,6 +243,7 @@ suite reader_tests = [] {
         expect(token->get<std::int16_t>() == -27);
         expect(token->get<std::int32_t>() == -27);
         expect(token->get<std::int64_t>() == -27);
+        expect(token == -27);
 
         // EOM
         expect(test_end_of_message(reader));
@@ -256,6 +262,7 @@ suite reader_tests = [] {
         expect(token->get<std::int16_t>() == -1);
         expect(token->get<std::int32_t>() == -1);
         expect(token->get<std::int64_t>() == -1);
+        expect(token == -1);
 
         // EOM
         expect(test_end_of_message(reader));
@@ -275,6 +282,7 @@ suite reader_tests = [] {
         expect(token->get<std::int16_t>() == -256);
         expect(token->get<std::int32_t>() == -256);
         expect(token->get<std::int64_t>() == -256);
+        expect(token == -256);
 
         // EOM
         expect(test_end_of_message(reader));
@@ -295,6 +303,7 @@ suite reader_tests = [] {
                 tl::make_unexpected(msgpack::error::will_truncate));
         expect(token->get<std::int32_t>() == -65536);
         expect(token->get<std::int64_t>() == -65536);
+        expect(token == -65536);
         // EOM
         expect(test_end_of_message(reader));
     };
@@ -316,6 +325,7 @@ suite reader_tests = [] {
         expect(token->get<std::int32_t>() ==
                 tl::make_unexpected(msgpack::error::will_truncate));
         expect(token->get<std::int64_t>() == -4294967296);
+        expect(token == -4294967296);
         // EOM
         expect(test_end_of_message(reader));
     };
@@ -338,6 +348,7 @@ suite reader_tests = [] {
 
         expect(token && token->type() == msgpack::format::type::string);
         expect(token->get<std::string_view>() == sv);
+        expect(token == sv);
 
         // EOM
         expect(test_end_of_message(reader));
@@ -357,6 +368,7 @@ suite reader_tests = [] {
 
         expect(token && token->type() == msgpack::format::type::string);
         expect(token->get<std::string_view>() == sv);
+        expect(token == sv);
 
         // EOM
         expect(test_end_of_message(reader));
@@ -381,6 +393,7 @@ suite reader_tests = [] {
 
         expect(token && token->type() == msgpack::format::type::string);
         expect(token->get<std::string_view>() == sv);
+        expect(token == sv);
 
         // EOM
         expect(test_end_of_message(reader));
@@ -405,6 +418,7 @@ suite reader_tests = [] {
 
         expect(token && token->type() == msgpack::format::type::string);
         expect(token->get<std::string_view>() == sv);
+        expect(token == sv);
 
         // EOM
         expect(test_end_of_message(reader));
@@ -431,7 +445,8 @@ suite reader_tests = [] {
 
         expect(token && token->type() == msgpack::format::type::binary);
         auto value = token->get<std::span<std::byte const>>().value();
-        expect(value == std::span(bytes.data(), bytes.size()));
+        expect(value == std::span<std::byte const>(bytes));
+        expect(token == std::span<std::byte const>(bytes));
 
         // EOM
         expect(test_end_of_message(reader));
@@ -455,7 +470,8 @@ suite reader_tests = [] {
 
         expect(token && token->type() == msgpack::format::type::binary);
         auto value = token->get<std::span<std::byte const>>().value();
-        expect(value == std::span(bytes.data(), bytes.size()));
+        expect(value == std::span<std::byte const>(bytes));
+        expect(token == std::span<std::byte const>(bytes));
 
         // EOM
         expect(test_end_of_message(reader));
@@ -480,7 +496,8 @@ suite reader_tests = [] {
 
         expect(token && token->type() == msgpack::format::type::binary);
         auto value = token->get<std::span<std::byte const>>().value();
-        expect(value == std::span(bytes.data(), bytes.size()));
+        expect(value == std::span<std::byte const>(bytes));
+        expect(token == std::span<std::byte const>(bytes));
 
         // EOM
         expect(test_end_of_message(reader));
@@ -519,7 +536,7 @@ suite reader_tests = [] {
 
         for (std::size_t i = 0; i < array->count; ++i) {
           auto v = reader.read_one();
-          expect(v && v->get<std::uint8_t>() == 0x85 - i);
+          expect(v && v == 0x85u - i);
         }
 
         expect(test_end_of_message(reader));
@@ -598,8 +615,10 @@ suite reader_tests = [] {
         for (std::size_t i = 0; i < map->count; ++i) {
             auto str = reader.read_one();
             expect(str && str->get<std::string_view>() == strings[i]);
+            expect(str == strings[i]);
             auto value = reader.read_one();
             expect(value && value->get<std::uint8_t>() == i + 1);
+            expect(value == i + 1);
         }
 
         expect(test_end_of_message(reader));
@@ -631,6 +650,7 @@ suite reader_tests = [] {
             expect(str && str->get<std::string_view>() == strings[i]);
             auto value = reader.read_one();
             expect(value && value->get<std::uint8_t>() == i + 1);
+            expect(value == i + 1);
         }
 
         expect(test_end_of_message(reader));
@@ -646,9 +666,11 @@ suite reader_tests = [] {
         auto token = reader.read_one();
         expect(token && token->type() == msgpack::format::type::boolean);
         expect(token->get<bool>() == false);
+        expect(token == false);
         token = reader.read_one();
         expect(token && token->type() == msgpack::format::type::boolean);
         expect(token->get<bool>() == true);
+        expect(token == true);
         // EOM
         expect(test_end_of_message(reader));
     };
@@ -674,4 +696,19 @@ suite reader_tests = [] {
         // EOM
         expect(test_end_of_message(reader));
     };
+
+    //--------------------------------------------------------------------------
+    // Assorted format tests
+    //--------------------------------------------------------------------------
+    "read multiple tokens"_test = [] {
+        std::array<msgpack::token, 5> arr;
+        constexpr auto payload = make_bytes(0xc2, 0xc3);
+        msgpack::token_reader reader(payload);
+        auto output = reader.read_some(arr);
+        expect(output && output->size() == 2);
+        expect((*output)[0] == false);
+        expect((*output)[1] == true);
+        output = reader.read_some(arr);
+        expect(output == tl::make_unexpected(msgpack::error::end_of_message));
+    };
 };