ventrue/parselink-old
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Map/Array support as "tokens"

Browse Source 
This essentially completes the token_reader. In the future, we will
probably want something that lets us allocate maps/arrays more
seamlessly, but we can start working on client-server architecture with
this reader.
This commit is contained in:
Kurt Sassenrath 2023-09-18 07:46:23 -07:00
parent c5c03a2a40
commit f894b8eaf6
4 changed files with 236 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
source/include/parselink/msgpack/core/format.h
View File

@ -145,6 +145,16 @@ namespace format {
constexpr static type value = type::invalid;
};
template <>
struct resolve_token_type<array_desc> {
constexpr static type value = type::array;
};
template <>
struct resolve_token_type<map_desc> {
constexpr static type value = type::map;
};
template <typename T>
constexpr static auto resolve_type_v = resolve_token_type<T>::value;

46
source/include/parselink/msgpack/token/reader.h
View File

@ -92,6 +92,16 @@ constexpr inline format::traits const& traits_lookup(std::byte id) {
case format::bin32::marker:
return format::get_traits<format::bin32>();
case format::array16::marker:
return format::get_traits<format::array16>();
case format::array32::marker:
return format::get_traits<format::array32>();
case format::map16::marker:
return format::get_traits<format::map16>();
case format::map32::marker:
return format::get_traits<format::map32>();
case format::nil::marker:
return format::get_traits<format::nil>();
case format::invalid::marker:
@ -104,15 +114,21 @@ constexpr inline format::traits const& traits_lookup(std::byte id) {
}
// TODO(ksassenrath): Handle fixtype formats.
if ((id & ~format::fixstr::mask) == format::fixstr::marker) {
return format::get_traits<format::fixstr>();
}
if ((id & ~format::negative_fixint::mask) == format::negative_fixint::marker) {
return format::get_traits<format::negative_fixint>();
}
if ((id & ~format::positive_fixint::mask) == format::positive_fixint::marker) {
return format::get_traits<format::positive_fixint>();
}
if ((id & ~format::fixstr::mask) == format::fixstr::marker) {
return format::get_traits<format::fixstr>();
}
if ((id & ~format::fixmap::mask) == format::fixmap::marker) {
return format::get_traits<format::fixmap>();
}
if ((id & ~format::fixarray::mask) == format::fixarray::marker) {
return format::get_traits<format::fixarray>();
}
return format::no_traits;
}
@ -200,20 +216,24 @@ public:
}
case format::type::string: {
using type = token_traits<format::type::string>::storage_type;
{
auto ptr = reinterpret_cast<type>(&*curr);
tok = token{std::string_view{ptr, var_size}};
curr += var_size;
}
auto ptr = reinterpret_cast<type>(&*curr);
tok = token{std::string_view{ptr, var_size}};
curr += var_size;
break;
}
case format::type::binary: {
using type = token_traits<format::type::binary>::storage_type;
{
auto ptr = reinterpret_cast<type>(&*curr);
tok = token{std::span<std::byte const>{ptr, var_size}};
curr += var_size;
}
auto ptr = reinterpret_cast<type>(&*curr);
tok = token{std::span<std::byte const>{ptr, var_size}};
curr += var_size;
break;
}
case format::type::array: {
tok = token{array_desc(var_size)};
break;
}
case format::type::map: {
tok = token{map_desc(var_size)};
break;
}
default:

43
source/include/parselink/msgpack/token/type.h
View File

@ -132,6 +132,16 @@ struct token_traits<format::type::binary> {
using storage_type = std::byte const*;
};
template <>
struct token_traits<format::type::array> {
using storage_type = std::uint64_t;
};
template <>
struct token_traits<format::type::map> {
using storage_type = std::uint64_t;
};
template <>
struct token_traits<format::type::boolean> {
using storage_type = bool;
@ -180,6 +190,16 @@ public:
value_.bp = bv.data();
}
explicit token_base(array_desc const& value) noexcept {
size_and_type_.set_enum(format::type::array);
size_and_type_.set_size(value.count);
}
explicit token_base(map_desc const& value) noexcept {
size_and_type_.set_enum(format::type::map);
size_and_type_.set_size(value.count);
}
explicit token_base(nil) noexcept {
size_and_type_.set_enum(format::type::nil);
}
@ -227,7 +247,8 @@ private:
typename token_traits<format::type::string>::storage_type str;
typename token_traits<format::type::binary>::storage_type bp;
typename token_traits<format::type::boolean>::storage_type b;
token_base* obj;
// TODO: What to store here for array/map types? Possibly a pointer
// to the end of the array/map, if/when known?
} value_;
size_and_enum<std::uint32_t, format::type> size_and_type_{};
};
@ -296,6 +317,26 @@ token_base<8>::get() const noexcept
return invalid{};
}
template<>
constexpr tl::expected<array_desc, error>
token_base<8>::get() const noexcept
{
if (type() != format::type::array) {
return tl::make_unexpected(error::wrong_type);
}
return array_desc{size_and_type_.get_size()};
}
template<>
constexpr tl::expected<map_desc, error>
token_base<8>::get() const noexcept
{
if (type() != format::type::map) {
return tl::make_unexpected(error::wrong_type);
}
return map_desc{size_and_type_.get_size()};
}
using token = token_base<sizeof(void*)>;
} // namespace msgpack

152
tests/msgpack/test_token_reader.cpp
View File

@ -1,7 +1,7 @@
#include <msgpack/token.h>
#include <fmt/format.h>
#include <boost/ut.hpp>
#include <fmt/format.h>
#include <magic_enum.hpp>
using namespace boost::ut;
@ -486,6 +486,156 @@ suite reader_tests = [] {
expect(test_end_of_message(reader));
};
//--------------------------------------------------------------------------
// Structural format tests (map/array)
//--------------------------------------------------------------------------
"read format::fixarray"_test = [] {
// A MessagePack array of 5 8-bit unsigned integers.
constexpr auto payload = make_bytes(0x95, 0xcc, 0x85, 0xcc, 0x84, 0xcc,
0x83, 0xcc, 0x82, 0xcc, 0x81);
msgpack::token_reader reader(payload);
auto token = reader.read_one();
expect(token && token->type() == msgpack::format::type::array);
auto array = token->get<msgpack::array_desc>();
expect(array && array->count == 5);
for (std::size_t i = 0; i < array->count; ++i) {
auto v = reader.read_one();
expect(v && v->get<std::uint8_t>() == 0x85 - i);
}
expect(test_end_of_message(reader));
};
"read format::array16"_test = [] {
// A MessagePack array of 5 8-bit unsigned integers.
constexpr auto payload = make_bytes(0xdc, 0x00, 0x05,
0xcc, 0x85, 0xcc, 0x84, 0xcc, 0x83, 0xcc, 0x82, 0xcc, 0x81);
msgpack::token_reader reader(payload);
auto token = reader.read_one();
expect(token && token->type() == msgpack::format::type::array);
auto array = token->get<msgpack::array_desc>();
expect(array && array->count == 5);
for (std::size_t i = 0; i < array->count; ++i) {
auto v = reader.read_one();
expect(v && v->get<std::uint8_t>() == 0x85 - i);
}
expect(test_end_of_message(reader));
};
"read format::array32"_test = [] {
// A MessagePack array of 5 8-bit unsigned integers.
constexpr auto payload = make_bytes(0xdd, 0x00, 0x00, 0x00, 0x05,
0xcc, 0x85, 0xcc, 0x84, 0xcc, 0x83, 0xcc, 0x82, 0xcc, 0x81);
msgpack::token_reader reader(payload);
auto token = reader.read_one();
expect(token && token->type() == msgpack::format::type::array);
auto array = token->get<msgpack::array_desc>();
expect(array && array->count == 5);
for (std::size_t i = 0; i < array->count; ++i) {
auto v = reader.read_one();
expect(v && v->get<std::uint8_t>() == 0x85 - i);
}
expect(test_end_of_message(reader));
};
"read format::fixmap"_test = [] {
// A MessagePack map of 3 strings to 8-bit unsigned integers.
static constexpr std::array<std::string_view, 3> strings = {
"one", "two", "three"
};
constexpr auto payload = cat(
make_bytes(0x83, 0xa3),
generate_bytes([] { return from_string_view(strings[0]); }),
make_bytes(0x01, 0xa3),
generate_bytes([] { return from_string_view(strings[1]); }),
make_bytes(0x02, 0xa5),
generate_bytes([] { return from_string_view(strings[2]); }),
make_bytes(0x03)
);
msgpack::token_reader reader(payload);
auto token = reader.read_one();
expect(token && token->type() == msgpack::format::type::map);
auto map = token->get<msgpack::map_desc>();
expect(map && map->count == 3);
for (std::size_t i = 0; i < map->count; ++i) {
auto str = reader.read_one();
expect(str && str->get<std::string_view>() == strings[i]);
auto value = reader.read_one();
expect(value && value->get<std::uint8_t>() == i + 1);
}
expect(test_end_of_message(reader));
};
"read format::map16"_test = [] {
// A MessagePack map of 3 strings to 8-bit unsigned integers.
static constexpr std::array<std::string_view, 3> strings = {
"one", "two", "three"
};
constexpr auto payload = cat(
make_bytes(0xde, 0x00, 0x03, 0xa3),
generate_bytes([] { return from_string_view(strings[0]); }),
make_bytes(0x01, 0xa3),
generate_bytes([] { return from_string_view(strings[1]); }),
make_bytes(0x02, 0xa5),
generate_bytes([] { return from_string_view(strings[2]); }),
make_bytes(0x03)
);
msgpack::token_reader reader(payload);
auto token = reader.read_one();
expect(token && token->type() == msgpack::format::type::map);
auto map = token->get<msgpack::map_desc>();
expect(map && map->count == 3);
for (std::size_t i = 0; i < map->count; ++i) {
auto str = reader.read_one();
expect(str && str->get<std::string_view>() == strings[i]);
auto value = reader.read_one();
expect(value && value->get<std::uint8_t>() == i + 1);
}
expect(test_end_of_message(reader));
};
"read format::map32"_test = [] {
// A MessagePack map of 3 strings to 8-bit unsigned integers.
static constexpr std::array<std::string_view, 3> strings = {
"one", "two", "three"
};
constexpr auto payload = cat(
make_bytes(0xdf, 0x00, 0x00, 0x00, 0x03, 0xa3),
generate_bytes([] { return from_string_view(strings[0]); }),
make_bytes(0x01, 0xa3),
generate_bytes([] { return from_string_view(strings[1]); }),
make_bytes(0x02, 0xa5),
generate_bytes([] { return from_string_view(strings[2]); }),
make_bytes(0x03)
);
msgpack::token_reader reader(payload);
auto token = reader.read_one();
expect(token && token->type() == msgpack::format::type::map);
auto map = token->get<msgpack::map_desc>();
expect(map && map->count == 3);
for (std::size_t i = 0; i < map->count; ++i) {
auto str = reader.read_one();
expect(str && str->get<std::string_view>() == strings[i]);
auto value = reader.read_one();
expect(value && value->get<std::uint8_t>() == i + 1);
}
expect(test_end_of_message(reader));
};
//--------------------------------------------------------------------------
// Assorted format tests
//--------------------------------------------------------------------------
"read format::boolean"_test = [] {