ventrue/parselink-old
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

token_reader::read_some, operator==, "speed test"

Browse Source 
- Added token_reader::read_some(), which takes a view of a token buffer
  reads/unpacks up to that many tokens from a buffer. If an error occurs,
  or if there are fewer msgpack formats within the buffer, then the
  returned span will be a subspan of the original buffer. If no tokens
  were read before encountering an error, token_reader will return the
  error instead.
- token::operator== overload for types that might be stored within the
  token. Handy for cleaning up tests, even though both methods of
  `token::get<T>() == value` and `token == value` are represented for
  now.
- Added a simple tool to open a file containing msgpack content and read
  tokens into a buffer. This is for testing speed, but needs to be made
  into a proper benchmark once more work has been done. Pretty fast
  right now, only taking 2us to parse ~200 tokens (that takes the
  pythong msgpack implementation ~1s to handle).
This commit is contained in:
Kurt Sassenrath 2023-10-06 16:16:56 -07:00
parent 2a4c819f4f
commit a80b9d0fc6
5 changed files with 138 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
include/parselink/msgpack/token/reader.h
View File

@ -244,15 +244,37 @@ public:
return {tok};
}
// Read multiple tokens from the byte buffer. The number of tokens parsed
// can be surmised from the returned span of tokens. If the reader
// currently points to the end of the byte buffer, then
// error::end_of_message is returned, and if there is not enough data to
// fully parse a token, then incomplete_message is returned.
// Read multiple tokens from the byte buffer, returning a subspan with the
// number of tokens parsed, or a relevant error. If the reader currently
// points to the end of the byte buffer, then error::end_of_message is
// returned, and if there is not enough data to fully parse a token, then
// incomplete_message is returned.
constexpr tl::expected<std::span<token>, error> read_some(
std::span<token> token_buffer) noexcept;
std::span<token> token_buffer) noexcept {
auto tok = token_buffer.begin();
error err = error::end_of_message;
while (tok != token_buffer.end()) {
auto result = read_one().map([&tok](auto&& t){
*tok = t;
++tok;
});
if (!result) {
err = result.error();
break;
}
}
auto parsed = std::distance(token_buffer.begin(), tok);
if (parsed == 0) {
return tl::make_unexpected(err);
}
return token_buffer.subspan(0, parsed);
}
private:
std::span<std::byte const> data_;
decltype(data_)::iterator curr_;
decltype(data_)::iterator end_;

13
include/parselink/msgpack/token/type.h
View File

@ -240,6 +240,19 @@ public:
}
}
template <typename T>
constexpr bool operator==(T const& t) const noexcept {
if constexpr (std::equality_comparable<T>) {
auto result = get<T>().map([&t](auto v) { return v == t; });
return result && *result;
} else {
auto result = get<T>().map([&t](auto v) {
return std::equal(v.begin(), v.end(), t.begin(), t.end());
});
return result && *result;
}
}
private:
union {
typename token_traits<format::type::unsigned_int>::storage_type u;

8
tests/msgpack/BUILD
View File

@ -46,3 +46,11 @@ cc_test(
],
deps = ["test_deps"],
)
cc_binary(
name = "speed",
srcs = [
"test_speed.cpp",
],
deps = ["test_deps"],
)

48
tests/msgpack/test_speed.cpp Normal file
View File

@ -0,0 +1,48 @@
#include "parselink/msgpack/token.h"
#include <fmt/format.h>
#include <chrono>
#include <cstdlib>
#include <fcntl.h>
#include <vector>
using namespace std::chrono_literals;
auto read_file(char const* filename) {
std::vector<std::byte> data;
auto fd = open(filename, O_RDONLY);
if (!fd) {
fmt::print("Could not open {}: {}\n", filename, strerror(errno));
return data;
}
auto size = lseek(fd, 0, SEEK_END);
data.resize(size);
lseek(fd, 0, SEEK_SET);
fmt::print("Reading {} bytes from {}\n", size, filename);
read(fd, data.data(), data.size());
close(fd);
return data;
}
int main(int argc, char** argv) {
std::array<msgpack::token, 4096> buf;
auto data = read_file(argc < 2 ? "output.bin" : argv[1]);
msgpack::token_reader reader(data);
auto before = std::chrono::steady_clock::now();
auto test = reader.read_some(buf);
auto after = std::chrono::steady_clock::now();
test.map([&](auto sp){
fmt::print("Read {} tokens\n", sp.size());
}).map_error([&](auto err){
fmt::print("Failed to read tokens: {}\n", (int)err);
});
fmt::print("Took {} us\n", (after - before) / 1us);
}

45
tests/msgpack/test_token_reader.cpp
View File

@ -139,6 +139,7 @@ suite reader_tests = [] {
expect(token->get<std::uint16_t>() == 0x35);
expect(token->get<std::uint32_t>() == 0x35);
expect(token->get<std::uint64_t>() == 0x35);
expect(token == 0x35u);
// EOM
expect(test_end_of_message(reader));
@ -157,6 +158,7 @@ suite reader_tests = [] {
expect(token->get<std::uint16_t>() == 0x02);
expect(token->get<std::uint32_t>() == 0x02);
expect(token->get<std::uint64_t>() == 0x02);
expect(token == 0x02u);
// EOM
expect(test_end_of_message(reader));
@ -176,6 +178,7 @@ suite reader_tests = [] {
expect(token->get<std::uint16_t>() == 0x0102);
expect(token->get<std::uint32_t>() == 0x0102);
expect(token->get<std::uint64_t>() == 0x0102);
expect(token == 0x0102u);
// EOM
expect(test_end_of_message(reader));
@ -196,6 +199,7 @@ suite reader_tests = [] {
tl::make_unexpected(msgpack::error::will_truncate));
expect(token->get<std::uint32_t>() == 0x01020304);
expect(token->get<std::uint64_t>() == 0x01020304);
expect(token == 0x01020304u);
// EOM
expect(test_end_of_message(reader));
};
@ -217,6 +221,7 @@ suite reader_tests = [] {
expect(token->get<std::uint32_t>() ==
tl::make_unexpected(msgpack::error::will_truncate));
expect(token->get<std::uint64_t>() == 0x0102030405060708);
expect(token == 0x0102030405060708u);
// EOM
expect(test_end_of_message(reader));
};
@ -238,6 +243,7 @@ suite reader_tests = [] {
expect(token->get<std::int16_t>() == -27);
expect(token->get<std::int32_t>() == -27);
expect(token->get<std::int64_t>() == -27);
expect(token == -27);
// EOM
expect(test_end_of_message(reader));
@ -256,6 +262,7 @@ suite reader_tests = [] {
expect(token->get<std::int16_t>() == -1);
expect(token->get<std::int32_t>() == -1);
expect(token->get<std::int64_t>() == -1);
expect(token == -1);
// EOM
expect(test_end_of_message(reader));
@ -275,6 +282,7 @@ suite reader_tests = [] {
expect(token->get<std::int16_t>() == -256);
expect(token->get<std::int32_t>() == -256);
expect(token->get<std::int64_t>() == -256);
expect(token == -256);
// EOM
expect(test_end_of_message(reader));
@ -295,6 +303,7 @@ suite reader_tests = [] {
tl::make_unexpected(msgpack::error::will_truncate));
expect(token->get<std::int32_t>() == -65536);
expect(token->get<std::int64_t>() == -65536);
expect(token == -65536);
// EOM
expect(test_end_of_message(reader));
};
@ -316,6 +325,7 @@ suite reader_tests = [] {
expect(token->get<std::int32_t>() ==
tl::make_unexpected(msgpack::error::will_truncate));
expect(token->get<std::int64_t>() == -4294967296);
expect(token == -4294967296);
// EOM
expect(test_end_of_message(reader));
};
@ -338,6 +348,7 @@ suite reader_tests = [] {
expect(token && token->type() == msgpack::format::type::string);
expect(token->get<std::string_view>() == sv);
expect(token == sv);
// EOM
expect(test_end_of_message(reader));
@ -357,6 +368,7 @@ suite reader_tests = [] {
expect(token && token->type() == msgpack::format::type::string);
expect(token->get<std::string_view>() == sv);
expect(token == sv);
// EOM
expect(test_end_of_message(reader));
@ -381,6 +393,7 @@ suite reader_tests = [] {
expect(token && token->type() == msgpack::format::type::string);
expect(token->get<std::string_view>() == sv);
expect(token == sv);
// EOM
expect(test_end_of_message(reader));
@ -405,6 +418,7 @@ suite reader_tests = [] {
expect(token && token->type() == msgpack::format::type::string);
expect(token->get<std::string_view>() == sv);
expect(token == sv);
// EOM
expect(test_end_of_message(reader));
@ -431,7 +445,8 @@ suite reader_tests = [] {
expect(token && token->type() == msgpack::format::type::binary);
auto value = token->get<std::span<std::byte const>>().value();
expect(value == std::span(bytes.data(), bytes.size()));
expect(value == std::span<std::byte const>(bytes));
expect(token == std::span<std::byte const>(bytes));
// EOM
expect(test_end_of_message(reader));
@ -455,7 +470,8 @@ suite reader_tests = [] {
expect(token && token->type() == msgpack::format::type::binary);
auto value = token->get<std::span<std::byte const>>().value();
expect(value == std::span(bytes.data(), bytes.size()));
expect(value == std::span<std::byte const>(bytes));
expect(token == std::span<std::byte const>(bytes));
// EOM
expect(test_end_of_message(reader));
@ -480,7 +496,8 @@ suite reader_tests = [] {
expect(token && token->type() == msgpack::format::type::binary);
auto value = token->get<std::span<std::byte const>>().value();
expect(value == std::span(bytes.data(), bytes.size()));
expect(value == std::span<std::byte const>(bytes));
expect(token == std::span<std::byte const>(bytes));
// EOM
expect(test_end_of_message(reader));
@ -519,7 +536,7 @@ suite reader_tests = [] {
for (std::size_t i = 0; i < array->count; ++i) {
auto v = reader.read_one();
expect(v && v->get<std::uint8_t>() == 0x85 - i);
expect(v && v == 0x85u - i);
}
expect(test_end_of_message(reader));
@ -598,8 +615,10 @@ suite reader_tests = [] {
for (std::size_t i = 0; i < map->count; ++i) {
auto str = reader.read_one();
expect(str && str->get<std::string_view>() == strings[i]);
expect(str == strings[i]);
auto value = reader.read_one();
expect(value && value->get<std::uint8_t>() == i + 1);
expect(value == i + 1);
}
expect(test_end_of_message(reader));
@ -631,6 +650,7 @@ suite reader_tests = [] {
expect(str && str->get<std::string_view>() == strings[i]);
auto value = reader.read_one();
expect(value && value->get<std::uint8_t>() == i + 1);
expect(value == i + 1);
}
expect(test_end_of_message(reader));
@ -646,9 +666,11 @@ suite reader_tests = [] {
auto token = reader.read_one();
expect(token && token->type() == msgpack::format::type::boolean);
expect(token->get<bool>() == false);
expect(token == false);
token = reader.read_one();
expect(token && token->type() == msgpack::format::type::boolean);
expect(token->get<bool>() == true);
expect(token == true);
// EOM
expect(test_end_of_message(reader));
};
@ -674,4 +696,19 @@ suite reader_tests = [] {
// EOM
expect(test_end_of_message(reader));
};
//--------------------------------------------------------------------------
// Assorted format tests
//--------------------------------------------------------------------------
"read multiple tokens"_test = [] {
std::array<msgpack::token, 5> arr;
constexpr auto payload = make_bytes(0xc2, 0xc3);
msgpack::token_reader reader(payload);
auto output = reader.read_some(arr);
expect(output && output->size() == 2);
expect((*output)[0] == false);
expect((*output)[1] == true);
output = reader.read_some(arr);
expect(output == tl::make_unexpected(msgpack::error::end_of_message));
};
};