ventrue/parselink-old
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Implement token map_view, additional formatters.

Browse Source 
msgpack::map_view can be used to iterate, pair-wise, over a range of
msgpack::token. It will immediately return if the first token is not a
map, and will skip over nested map/arrays.

Note for the future: It will be handy to be able to get the subspan
corresponding to the nested map/array. Will think about how to solve
that later.

Begin incorporating map_view into the server.

Add formatters for std::byte, dynamic theme for bool, and spans thereof.
Maybe switch to range?
This commit is contained in:
Kurt Sassenrath 2023-10-12 14:52:06 -07:00
parent 22f78cc7d7
commit 1c7047e314
10 changed files with 644 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
include/parselink/logging.h
View File

@ -36,7 +36,7 @@ namespace logging {
// enabled in the library, but the compiler should be able to optimize away
// some/most of the calls.
constexpr inline auto static_threshold = level::trace;
constexpr inline auto default_threshold = level::info;
constexpr inline auto default_threshold = level::debug;
// Structure for holding a message. Note: message is a view over some buffer,
// not it's own string. It will either be a static buffer supplied by the

14
include/parselink/logging/formatters.h
View File

@ -96,6 +96,20 @@ struct fmt::formatter<std::errc> : fmt::formatter<std::error_code> {
}
};
// Support printing bytes as their hex representation.
template <>
struct fmt::formatter<std::byte> {
template <typename ParseContext>
constexpr auto parse(ParseContext& ctx) -> decltype(ctx.begin()) {
return ctx.begin();
}
template<typename FormatContext>
auto format(std::byte const v, FormatContext& ctx) const {
return fmt::format_to(ctx.out(), "0x{:0x}", std::uint8_t(v));
}
};
// Support printing raw/smart pointers without needing to wrap them in fmt::ptr
template <parselink::logging::detail::printable_pointer T>
struct fmt::formatter<T> : fmt::formatter<void const*> {

16
include/parselink/logging/theme.h
View File

@ -72,8 +72,12 @@ struct static_theme {
};
template <std::integral T>
requires (!std::same_as<T, bool>)
struct theme<T> : static_theme<fmt::color::light_sky_blue> {};
template <>
struct theme<std::byte> : static_theme<fmt::color::light_sky_blue> {};
template <std::convertible_to<std::string_view> T>
struct theme<T> : static_theme<fmt::color::pale_green> {};
@ -84,7 +88,6 @@ template <typename E>
requires std::is_enum_v<E>
struct theme<E> : static_theme<fmt::color::gray> {};
// Errors
template <>
struct theme<std::error_code> : static_theme<fmt::color::fire_brick> {};
@ -96,6 +99,13 @@ template <>
struct theme<error_str> : static_theme<fmt::color::fire_brick> {};
template <>
struct theme<bool> {
static constexpr auto style(bool l) noexcept {
return fmt::fg(l ? fmt::color::spring_green : fmt::color::indian_red);
}
};
template <>
struct theme<enum_name_only<logging::level>> {
constexpr static fmt::color colors[] = {
@ -127,7 +137,9 @@ concept has_dynamic_theme = requires (T const& t) {
template <typename T>
concept has_theme = has_static_theme<T> || has_dynamic_theme<T>;
static_assert(has_static_theme<std::errc>);
template <typename T>
requires (has_theme<T>)
struct theme<std::span<T>> : theme<T> {};
template <typename T>
constexpr auto get_theme(T const&) {

4
include/parselink/msgpack/core/format.h
View File

@ -70,7 +70,9 @@ namespace format {
nil,
boolean,
array,
map
map,
array_view,
map_view,
};
// Flags that may control the behavior of readers/writers.

1
include/parselink/msgpack/token.h
View File

@ -3,5 +3,6 @@
#include "token/type.h"
#include "token/reader.h"
#include "token/views.h"
#endif // msgpack_object_f1f3a9e5c8be6a11

138
include/parselink/msgpack/token/views.h Normal file
View File

@ -0,0 +1,138 @@
//-----------------------------------------------------------------------------
// ___ __ _ _
// / _ \__ _ _ __ ___ ___ / /(_)_ __ | | __
// / /_)/ _` | '__/ __|/ _ \/ / | | '_ \| |/ /
// / ___/ (_| | | \__ \ __/ /__| | | | | <
// \/ \__,_|_| |___/\___\____/_|_| |_|_|\_\ .
//
//-----------------------------------------------------------------------------
// Author: Kurt Sassenrath
// Module: msgpack
//
// Token view utilities.
//
// MessagePack maps and arrays are nested, and the token reader only parses
// out the type. This file provides utilities for iterating over these
// "container" formats without incurring additional overhead on the parser when
// it is not needed.
//
// Copyright (c) 2023 Kurt Sassenrath.
//
// License TBD.
//-----------------------------------------------------------------------------
#ifndef msgpack_token_views_f19c250e782ed51c
#define msgpack_token_views_f19c250e782ed51c
#include "type.h"
#include <fmt/format.h>
#include <ranges>
namespace msgpack {
template <std::ranges::view V>
requires std::ranges::input_range<V> && std::same_as<token,
std::ranges::range_value_t<V>>
struct map_view : public std::ranges::view_interface<map_view<V>> {
public:
class iterator {
friend class sentinel;
using base_iterator = std::ranges::iterator_t<V>;
using base_sentinel = std::ranges::sentinel_t<V>;
using base_value_type = std::ranges::range_value_t<V>;
using base_reference = std::ranges::range_reference_t<V>;
base_iterator next(base_iterator current, std::size_t n = 1) {
while (n && current != std::ranges::end(*base_)) {
if (auto m = current->template get<map_desc>(); m) {
n += m->count * 2;
} else if (auto m = current->template get<array_desc>(); m) {
n += m->count;
}
++current;
--n;
}
return current;
}
public:
using value_type = std::pair<V, V>;
using reference = std::pair<base_reference, base_reference>;
using difference_type = std::ptrdiff_t;
using iterator_category = std::forward_iterator_tag;
iterator() = default;
iterator(V const& base)
: base_{&base}
, k_{std::ranges::begin(base)} {
// Ensure that k_ points to a map_desc. If not, then we
// effectively treat this as the end.
if (k_->type() == msgpack::format::type::map) {
remaining_ = k_->template get<msgpack::map_desc>()->count + 1;
// Advance to the first entry in the map.
++k_;
v_ = next(k_);
}
}
[[nodiscard]] reference operator*() const {
return { *k_, *v_ };
}
iterator& operator++() {
k_ = next(v_);
v_ = next(k_);
--remaining_;
return *this;
}
[[nodiscard]] iterator operator++(int) {
auto tmp = *this;
++(*this);
return tmp;
}
[[nodiscard]] bool operator==(iterator const& rhs) const {
return k_ == rhs.remaining_ &&
base_ == rhs.base_;
}
V const* base_{};
base_iterator k_{};
base_iterator v_{};
std::size_t remaining_{};
};
class sentinel {
public:
[[nodiscard]] bool operator==(sentinel const&) const {
return true;
}
[[nodiscard]] bool operator==(iterator const& rhs) const {
return rhs.remaining_ == 0
|| rhs.k_ == std::ranges::end(*rhs.base_);
}
};
constexpr map_view() noexcept = default;
constexpr map_view(V base) : base_{std::move(base)} {}
[[nodiscard]] constexpr iterator begin() const {
return { base_ };
}
[[nodiscard]] constexpr sentinel end() const {
return {};
}
private:
V base_;
};
template <class Range>
map_view(Range&&) -> map_view<std::views::all_t<Range>>;
} // namespace msgpack
#endif // msgpack_token_views_f19c250e782ed51c

167
source/server.cpp
View File

@ -22,6 +22,11 @@
#include "parselink/server.h"
#include "parselink/msgpack/token/reader.h"
#include "parselink/msgpack/token/views.h"
#include "parselink/proto/message.h"
#include <fmt/ranges.h>
#include <boost/asio/io_context.hpp>
#include <boost/asio/signal_set.hpp>
@ -63,6 +68,54 @@ struct fmt::formatter<boost::system::error_code>
}
};
template <>
struct fmt::formatter<msgpack::token> {
template <typename ParseContext>
constexpr auto parse(ParseContext& ctx) -> decltype(ctx.begin()) {
return ctx.begin();
}
template<typename FormatContext>
auto format(msgpack::token const& v, FormatContext& ctx) const {
using parselink::logging::themed_arg;
auto out = fmt::format_to(ctx.out(), "<msgpack {} = ", themed_arg(v.type()));
switch (v.type()) {
case msgpack::format::type::unsigned_int:
fmt::format_to(out, "{}", themed_arg(*(v.get<std::uint64_t>())));
break;
case msgpack::format::type::signed_int:
out = fmt::format_to(out, "{}", themed_arg(*(v.get<std::uint64_t>())));
break;
case msgpack::format::type::boolean:
out = fmt::format_to(out, "{}", themed_arg(*(v.get<bool>())));
break;
case msgpack::format::type::string:
out = fmt::format_to(out, "{}", themed_arg(*(v.get<std::string_view>())));
break;
case msgpack::format::type::binary:
out = fmt::format_to(out, "{}",
themed_arg(*(v.get<std::span<std::byte const>>())));
break;
case msgpack::format::type::map:
out = fmt::format_to(out, "(arity: {})",
themed_arg(v.get<msgpack::map_desc>()->count));
break;
case msgpack::format::type::array:
out = fmt::format_to(out, "(arity: {})",
themed_arg(v.get<msgpack::array_desc>()->count));
break;
case msgpack::format::type::nil:
out = fmt::format_to(out, "(nil)");
break;
case msgpack::format::type::invalid:
out = fmt::format_to(out, "(invalid)");
break;
default:
break;
}
return fmt::format_to(out, ">");
}
};
template <typename T>
concept endpoint = requires(T const& t) {
{t.address()};
@ -102,6 +155,9 @@ public:
user_session(net::ip::tcp::socket sock) : socket_(std::move(sock)) {}
~user_session() {
logger.debug("Closing connection to {}", socket_.remote_endpoint());
boost::system::error_code ec;
socket_.shutdown(net::ip::tcp::socket::shutdown_both, ec);
socket_.close();
}
void start() {
@ -111,40 +167,111 @@ public:
}, detached);
}
awaitable<void> await_auth() {
std::array<std::byte, 16> buffer;
auto [ec, n] = co_await socket_.async_read_some(
net::buffer(buffer), no_ex_coro);
if (ec) {
logger.error("Reading from user socket failed: {}", ec);
co_return;
}
logger.debug("Read {} bytes from client: {}", n,
std::string_view(reinterpret_cast<char*>(buffer.data()), n));
// TODO(ksassenrath): Clean this part up. This could be handled in its
// own read_message_header() method.
auto reader = msgpack::token_reader(std::span(buffer.data(), n));
tl::expected<std::vector<std::byte>, msgpack::error> parse_header(
std::span<std::byte> data) noexcept {
auto reader = msgpack::token_reader(data);
auto magic = reader.read_one().map(
[](auto t){ return t == std::string_view{"prs"}; });
if (magic && *magic) {
logger.debug("Got magic from client");
} else {
logger.debug("Got error from client: {}", magic);
co_return;
logger.error("Failed to get magic from client: {}", magic);
return tl::unexpected(magic.error());
}
auto sz = reader.read_one().and_then(
[](auto t){ return t.template get<std::size_t>(); });
if (sz && *sz) {
logger.debug("Got packet size from client: {}", *sz);
} else {
logger.debug("Got error from client: {}", sz);
co_return;
logger.debug("Failed to get packet size from client: {}", sz);
return tl::unexpected(magic.error());
}
// Copy the rest of the message to the buffer for parsing.
std::vector<std::byte> msg(*sz);
// TODO(ksassenrath): Replace vector with custom buffer.
std::vector<std::byte> msg;
msg.reserve(*sz);
msg.resize(reader.remaining());
std::copy(reader.current(), reader.end(), msg.begin());
//auto [ec, n] = co_await socket_.async_read_some(net::buffer());
return msg;
}
awaitable<tl::expected<std::monostate, boost::system::error_code>>
buffer_message(std::vector<std::byte>& buffer) noexcept {
auto amt = buffer.size();
auto total = buffer.capacity();
buffer.resize(total);
while (amt < total) {
auto subf = std::span(buffer.begin() + amt, buffer.end());
auto [ec, n] = co_await socket_.async_read_some(
net::buffer(subf), no_ex_coro);
logger.debug("Read {} bytes, total is now {}", n, amt + n);
if (ec || n == 0) {
logger.error("Reading from user socket failed: {}", ec);
co_return tl::make_unexpected(ec);
}
amt += n;
}
co_return std::monostate{};
}
tl::expected<bool, msgpack::error> handle_auth(std::span<msgpack::token> tokens) {
auto message_type = tokens.begin()->get<std::string_view>();
if (message_type) {
logger.debug("Received '{}' packet. Parsing body", *message_type);
proto::connect_message message;
for (auto const& [k, v] : msgpack::map_view(tokens.subspan(1))) {
logger.debug("Parsing {} -> {}", k, v);
}
} else {
logger.error("Did not get message type: {}", message_type.error());
}
// The first token should include
return true;
}
awaitable<void> await_auth() noexcept {
std::array<std::byte, 16> buffer;
auto [ec, n] = co_await socket_.async_read_some(
net::buffer(buffer), no_ex_coro);
if (ec) {
logger.error("Reading from user socket failed: {}", ec);
co_return;
}
logger.debug("Read {} bytes from client: {}", n,
std::span(buffer.data(), n));
auto hdr_result = parse_header(std::span(buffer.data(), n));
if (!hdr_result) {
co_return;
}
auto msg = std::move(*hdr_result);
auto maybe_error = co_await buffer_message(msg);
if (!maybe_error) {
logger.error("Unable to buffer message: {}",
maybe_error.error());
co_return;
}
logger.trace("Message: {}", msg);
auto reader = msgpack::token_reader(msg);
std::array<msgpack::token, 32> tokens;
auto parsed = reader.read_some(tokens).and_then(
[this](auto c) { for (auto t : c) logger.trace("{}", t); return handle_auth(c); })
.or_else([](auto const& error) {
logger.error("Unable to parse msgpack tokens: {}", error);
});
if (!parsed) {
co_return;
}
// Authenticate against database.
}
enum class state {

10
tests/msgpack/BUILD
View File

@ -3,7 +3,7 @@ cc_library(
name = "test_deps",
srcs = [
"test_main.cpp",
"rng.h"
"rng.h",
],
deps = [
"//include/parselink:msgpack",
@ -47,6 +47,14 @@ cc_test(
deps = ["test_deps"],
)
cc_test(
name = "token_views",
srcs = [
"test_token_views.cpp",
],
deps = ["test_deps"],
)
cc_binary(
name = "speed",
srcs = [

219
tests/msgpack/test_token_views.cpp Normal file
View File

@ -0,0 +1,219 @@
#include <msgpack/token.h>
#include <boost/ut.hpp>
#include <fmt/format.h>
#include <fmt/ranges.h>
#include <magic_enum.hpp>
using namespace boost::ut;
namespace {
template <typename T>
constexpr bool operator==(std::span<T const> a, std::span<T const> b) noexcept {
return std::equal(a.begin(), a.end(), b.begin(), b.end());
}
template <typename T>
constexpr std::array<std::byte, sizeof(T)> as_bytes(T&& t) {
return std::bit_cast<std::array<std::byte, sizeof(T)>>(std::forward<T>(t));
}
template <typename... Bytes>
constexpr std::array<std::byte, sizeof...(Bytes)> make_bytes(Bytes &&...bytes) {
return {std::byte(std::forward<Bytes>(bytes))...};
}
template <typename T, std::size_t C = 1024 * 1024> struct oversized_array {
std::array<T, C> data;
std::size_t size;
};
constexpr auto to_bytes_array_oversized(auto const &container) {
using value_type = std::decay_t<decltype(container[0])>;
oversized_array<value_type> arr;
std::copy(std::begin(container), std::end(container), std::begin(arr.data));
arr.size = std::distance(std::begin(container), std::end(container));
return arr;
}
consteval auto generate_bytes(auto callable) {
constexpr auto oversized = to_bytes_array_oversized(callable());
using value_type = std::decay_t<decltype(oversized.data[0])>;
std::array<value_type, oversized.size> out;
std::copy(std::begin(oversized.data),
std::next(std::begin(oversized.data), oversized.size),
std::begin(out));
return out;
}
consteval auto build_string(auto callable) {
constexpr auto string_array = generate_bytes(callable);
return string_array;
}
template <std::size_t ... Sizes>
constexpr auto cat(std::array<std::byte, Sizes>const&... a) noexcept {
std::array<std::byte, (Sizes + ...)> out;
std::size_t index{};
((std::copy_n(a.begin(), Sizes, out.begin() + index), index += Sizes), ...);
return out;
}
constexpr auto repeat(std::span<std::byte const> sv, std::size_t count) {
std::vector<std::byte> range;
range.reserve(sv.size() * count);
for (decltype(count) i = 0; i < count; ++i) {
std::copy_n(sv.begin(), sv.size(), std::back_inserter(range));
}
return range;
}
constexpr auto repeat(std::string_view sv, std::size_t count) {
std::vector<char> range;
range.reserve(sv.size() * count);
for (decltype(count) i = 0; i < count; ++i) {
std::copy_n(sv.begin(), sv.size(), std::back_inserter(range));
}
return range;
}
constexpr auto from_string_view(std::string_view sv) {
std::vector<std::byte> range;
range.resize(sv.size());
auto itr = range.begin();
for (auto c : sv) {
*itr = std::byte(c);
++itr;
}
return range;
}
template <typename T, typename E>
std::ostream &operator<<(std::ostream &os, tl::expected<T, E> const &exp) {
if (exp.has_value()) {
os << "Value: '" << *exp << "'";
} else {
os << "Error";
}
return os;
}
bool test_incomplete_message(auto const& payload) {
// Test incomplete message.
for (decltype(payload.size()) i = 1; i < payload.size() - 1; ++i) {
// Test incomplete message.
msgpack::token_reader reader(std::span(payload.data(), i));
auto token = reader.read_one();
if (token != tl::make_unexpected(msgpack::error::incomplete_message)) {
fmt::print("Got the wrong response reading subview[0,{}] of payload: {}\n",
i,
token->get<std::string_view>().value());
return false;
}
}
return true;
}
bool test_end_of_message(auto& reader) {
return reader.read_one() ==
tl::make_unexpected(msgpack::error::end_of_message);
}
}
template <>
struct fmt::formatter<msgpack::token> {
template <typename ParseContext>
constexpr auto parse(ParseContext& ctx) -> decltype(ctx.begin()) {
return ctx.begin();
}
template<typename FormatContext>
auto format(msgpack::token const& v, FormatContext& ctx) const {
auto out = fmt::format_to(ctx.out(), "<msgpack {} = ", magic_enum::enum_name(v.type()));
switch (v.type()) {
case msgpack::format::type::unsigned_int:
fmt::format_to(out, "{}", (*(v.get<std::uint64_t>())));
break;
case msgpack::format::type::signed_int:
out = fmt::format_to(out, "{}", (*(v.get<std::uint64_t>())));
break;
case msgpack::format::type::boolean:
out = fmt::format_to(out, "{}", (*(v.get<bool>())));
break;
case msgpack::format::type::string:
out = fmt::format_to(out, "{}", (*(v.get<std::string_view>())));
break;
case msgpack::format::type::binary:
out = fmt::format_to(out, "{}",
(*(v.get<std::span<std::byte const>>())));
break;
case msgpack::format::type::map:
out = fmt::format_to(out, "(arity: {})",
(v.get<msgpack::map_desc>()->count));
break;
case msgpack::format::type::array:
out = fmt::format_to(out, "(arity: {})",
(v.get<msgpack::array_desc>()->count));
break;
case msgpack::format::type::nil:
out = fmt::format_to(out, "(nil)");
break;
case msgpack::format::type::invalid:
out = fmt::format_to(out, "(invalid)");
break;
default:
break;
}
return fmt::format_to(out, ">");
}
};
suite views_tests = [] {
"read format::fixmap"_test = [] {
// A MessagePack map of 3 strings to 8-bit unsigned integers.
static constexpr auto strings = std::to_array<std::string_view>({
"one", "two", "three", "array", "four", "map", "five",
});
constexpr auto payload = cat(
make_bytes(0x87, 0xa3),
generate_bytes([] { return from_string_view(strings[0]); }),
make_bytes(0x01, 0xa3),
generate_bytes([] { return from_string_view(strings[1]); }),
make_bytes(0x02, 0xa5),
generate_bytes([] { return from_string_view(strings[2]); }),
make_bytes(0x03, 0xa5),
generate_bytes([] { return from_string_view(strings[3]); }),
// Array of size 2, two fixints:
make_bytes(0x92, 0x32, 0x33),
make_bytes(0xa4),
generate_bytes([] { return from_string_view(strings[4]); }),
make_bytes(0x04, 0xa3),
generate_bytes([] { return from_string_view(strings[5]); }),
// Map of size 3, 6 total fixints.
make_bytes(0x83, 0x01, 0x02, 0x03, 0x04, 0x33, 0x11),
make_bytes(0xa4),
generate_bytes([] { return from_string_view(strings[6]); })
//make_bytes(0x05)
);
std::array<msgpack::token, 32> tokens;
msgpack::token_reader reader(payload);
auto result = reader.read_some(tokens).map([](auto read_tokens) {
for (auto const& [k, v] : msgpack::map_view(read_tokens)) {
fmt::print("map[{}] = {}\n", k, v);
}
});
expect(result.has_value());
};
};

98
tests/msgpack/test_utils.h Normal file
View File

@ -0,0 +1,98 @@
#ifndef msgpack_test_utils_4573e6627d8efe78
#define msgpack_test_utils_4573e6627d8efe78
#include <boost/ut.hpp>
#include <fmt/format.h>
#include <magic_enum.hpp>
template <typename T>
constexpr bool operator==(std::span<T const> a, std::span<T const> b) noexcept {
return std::equal(a.begin(), a.end(), b.begin(), b.end());
}
template <typename T>
constexpr std::array<std::byte, sizeof(T)> as_bytes(T&& t) {
return std::bit_cast<std::array<std::byte, sizeof(T)>>(std::forward<T>(t));
}
template <typename... Bytes>
constexpr std::array<std::byte, sizeof...(Bytes)> make_bytes(Bytes &&...bytes) {
return {std::byte(std::forward<Bytes>(bytes))...};
}
template <typename T, std::size_t C = 1024 * 1024> struct oversized_array {
std::array<T, C> data;
std::size_t size;
};
constexpr auto to_bytes_array_oversized(auto const &container) {
using value_type = std::decay_t<decltype(container[0])>;
oversized_array<value_type> arr;
std::copy(std::begin(container), std::end(container), std::begin(arr.data));
arr.size = std::distance(std::begin(container), std::end(container));
return arr;
}
consteval auto generate_bytes(auto callable) {
constexpr auto oversized = to_bytes_array_oversized(callable());
using value_type = std::decay_t<decltype(oversized.data[0])>;
std::array<value_type, oversized.size> out;
std::copy(std::begin(oversized.data),
std::next(std::begin(oversized.data), oversized.size),
std::begin(out));
return out;
}
consteval auto build_string(auto callable) {
constexpr auto string_array = generate_bytes(callable);
return string_array;
}
template <std::size_t ... Sizes>
constexpr auto cat(std::array<std::byte, Sizes>const&... a) noexcept {
std::array<std::byte, (Sizes + ...)> out;
std::size_t index{};
((std::copy_n(a.begin(), Sizes, out.begin() + index), index += Sizes), ...);
return out;
}
constexpr auto repeat(std::span<std::byte const> sv, std::size_t count) {
std::vector<std::byte> range;
range.reserve(sv.size() * count);
for (decltype(count) i = 0; i < count; ++i) {
std::copy_n(sv.begin(), sv.size(), std::back_inserter(range));
}
return range;
}
constexpr auto repeat(std::string_view sv, std::size_t count) {
std::vector<char> range;
range.reserve(sv.size() * count);
for (decltype(count) i = 0; i < count; ++i) {
std::copy_n(sv.begin(), sv.size(), std::back_inserter(range));
}
return range;
}
constexpr auto from_string_view(std::string_view sv) {
std::vector<std::byte> range;
range.resize(sv.size());
auto itr = range.begin();
for (auto c : sv) {
*itr = std::byte(c);
++itr;
}
return range;
}
template <typename T, typename E>
std::ostream &operator<<(std::ostream &os, tl::expected<T, E> const &exp) {
if (exp.has_value()) {
os << "Value: '" << *exp << "'";
} else {
os << "Error";
}
return os;
}
#endif // msgpack_test_utils_4573e6627d8efe78