Update ::wire:: to be closer to Monero variant (#70)

src/admin_main.cpp

@@ -31,6 +31,7 @@
 #include <boost/program_options/parsers.hpp>
 #include <boost/program_options/variables_map.hpp>
 #include <boost/range/adaptor/filtered.hpp>
+#include <boost/range/adaptor/transformed.hpp>
 #include <cassert>
 #include <cstring>
 #include <iostream>
@@ -54,6 +55,8 @@
 #include "wire/crypto.h"
 #include "wire/filters.h"
 #include "wire/json/write.h"
+#include "wire/wrapper/array.h"
+#include "wire/wrappers_impl.h"
 
 namespace
 {
@@ -79,7 +82,7 @@ namespace
     const auto transform = [] (lws::db::account src)
     { return admin_display<lws::db::account>{std::move(src)}; };
 
-    wire::array(dest, (source.value | boost::adaptors::filtered(filter)), transform);
+    wire_write::bytes(dest, wire::array(source.value | boost::adaptors::filtered(filter) | boost::adaptors::transformed(transform)));
   }
 
   template<typename F, typename... T>

src/db/storage.cpp

@@ -59,6 +59,8 @@
 #include "wire/filters.h"
 #include "wire/json.h"
 #include "wire/vector.h"
+#include "wire/wrapper/array.h"
+#include "wire/wrappers_impl.h"
 
 namespace lws
 {
@@ -746,7 +748,7 @@ namespace db
   {
     wire::object(dest,
       wire::field("scan_height", self.first),
-      wire::field("accounts", wire::as_array(std::move(self.second)))
+      wire::field("accounts", wire::array(std::move(self.second)))
     );
   }
 
@@ -839,10 +841,10 @@ namespace db
     const wire::as_array_filter<toggle_key_output> toggle_keys_filter{{show_keys}};
     wire::json_stream_writer json_stream{out};
     wire::object(json_stream,
-      wire::field(blocks.name, wire::as_array(reverse(*blocks_partial))),
+      wire::field(blocks.name, wire::array(reverse(*blocks_partial))),
       wire::field(accounts.name, wire::as_object(accounts_stream->make_range(), wire::enum_as_string, toggle_keys_filter)),
       wire::field(accounts_by_address.name, wire::as_object(transform(accounts_ba_stream->make_range(), address_as_key))),
-      wire::field(accounts_by_height.name, wire::as_array(accounts_bh_stream->make_range())),
+      wire::field(accounts_by_height.name, wire::array(accounts_bh_stream->make_range())),
       wire::field(outputs.name, wire::as_object(outputs_stream->make_range(), wire::as_integer, wire::as_array)),
       wire::field(spends.name, wire::as_object(spends_stream->make_range(), wire::as_integer, wire::as_array)),
       wire::field(images.name, wire::as_object(images_stream->make_range(), output_id_key{}, wire::as_array)),

src/lmdb/msgpack_table.h

@@ -37,7 +37,10 @@ namespace lmdb
     {
       epee::byte_stream initial;
       initial.write({reinterpret_cast<const char*>(std::addressof(val1)), sizeof(val1)});
-      return wire_write::to_bytes(wire::msgpack_slice_writer{std::move(initial), true}, val2);
+
+      wire::msgpack_slice_writer dest{std::move(initial), true};
+      wire_write::bytes(dest, val2);
+      return epee::byte_slice{dest.take_sink()};
     }
 
     /*!
@@ -76,10 +79,12 @@ namespace lmdb
 
       auto msgpack_bytes = lmdb::to_byte_span(value);
       msgpack_bytes.remove_prefix(sizeof(out.first));
-      auto msgpack = wire::msgpack::from_bytes<msgpack_value_type>(epee::byte_slice{{msgpack_bytes}});
+
-      if (!msgpack)
+      msgpack_value_type second{};
-        return msgpack.error();
+      const std::error_code error = wire::msgpack::from_bytes(epee::byte_slice{{msgpack_bytes}}, second);
-      out.second = std::move(*msgpack);
+      if (error)
+        return error;
+      out.second = std::move(second);
 
       return out;
     }

src/rest_server.cpp

@@ -661,14 +661,19 @@ namespace lws
       using request = typename E::request;
       using response = typename E::response;
 
-      expect<request> req = wire::json::from_bytes<request>(std::move(root));
+      request req{};
-      if (!req)
+      std::error_code error = wire::json::from_bytes(std::move(root), req);
-        return req.error();
+      if (error)
+        return error;
 
-      expect<response> resp = E::handle(std::move(*req), std::move(disk), gclient);
+      expect<response> resp = E::handle(std::move(req), std::move(disk), gclient);
       if (!resp)
         return resp.error();
-      return wire::json::to_bytes<response>(*resp);
+
+      epee::byte_slice out{};
+      if ((error = wire::json::to_bytes(out, *resp)))
+        return error;
+      return {std::move(out)};
     }
 
     template<typename T>
@@ -693,17 +698,21 @@ namespace lws
     expect<epee::byte_slice> call_admin(std::string&& root, db::storage disk, const rpc::client&, const bool disable_auth)
     {
       using request = typename E::request;
-      expect<admin<request>> req = wire::json::from_bytes<admin<request>>(std::move(root));
+      
-      if (!req)
+      admin<request> req{};
-        return req.error();
+      {
+        const std::error_code error = wire::json::from_bytes(std::move(root), req);
+        if (error)
+          return error;
+      }
 
       if (!disable_auth)
       {
-        if (!req->auth)
+        if (!req.auth)
           return {error::account_not_found};
 
         db::account_address address{};
-        if (!crypto::secret_key_to_public_key(*(req->auth), address.view_public))
+        if (!crypto::secret_key_to_public_key(*(req.auth), address.view_public))
           return {error::crypto_failure};
 
         auto reader = disk.start_read();
@@ -719,8 +728,8 @@ namespace lws
       }
 
       wire::json_slice_writer dest{};
-      MONERO_CHECK(E{}(dest, std::move(disk), std::move(req->params)));
+      MONERO_CHECK(E{}(dest, std::move(disk), std::move(req.params)));
-      return dest.take_bytes();
+      return epee::byte_slice{dest.take_sink()};
     }
 
     struct endpoint

src/rpc/CMakeLists.txt

@@ -30,4 +30,4 @@ set(monero-lws-rpc_sources admin.cpp client.cpp daemon_pub.cpp daemon_zmq.cpp li
 set(monero-lws-rpc_headers admin.h client.h daemon_pub.h daemon_zmq.h fwd.h json.h light_wallet.h rates.h)
 
 add_library(monero-lws-rpc ${monero-lws-rpc_sources} ${monero-lws-rpc_headers})
-target_link_libraries(monero-lws-rpc monero::libraries monero-lws-wire-json)
+target_link_libraries(monero-lws-rpc monero::libraries monero-lws-wire-json monero-lws-wire-wrapper)

src/rpc/admin.cpp

@@ -27,6 +27,7 @@
 
 #include "admin.h"
 
+#include <boost/range/adaptor/transformed.hpp>
 #include <boost/range/iterator_range.hpp>
 #include <boost/uuid/random_generator.hpp>
 #include <functional>
@@ -41,6 +42,8 @@
 #include "wire/traits.h"
 #include "wire/uuid.h"
 #include "wire/vector.h"
+#include "wire/wrapper/array.h"
+#include "wire/wrappers_impl.h"
 
 namespace wire
 {
@@ -94,7 +97,7 @@ namespace
   void write_bytes(wire::json_writer& dest, const truncated<boost::iterator_range<lmdb::value_iterator<V>>> self)
   {
     const auto truncate = [] (V src) { return truncated<V>{std::move(src)}; };
-    wire::array(dest, std::move(self.value), truncate);
+    wire_write::bytes(dest, wire::array(boost::adaptors::transform(std::move(self.value), truncate)));
   }
 
   template<typename K, typename V, typename C>
@@ -127,7 +130,10 @@ namespace
   void write_addresses(wire::writer& dest, epee::span<const lws::db::account_address> self)
   {
     // writes an array of monero base58 address strings
-    wire::object(dest, wire::field("updated", wire::as_array(self, lws::db::address_string)));
+
+    wire::object(dest,
+      wire::field("updated", wire::array(boost::adaptors::transform(self, lws::db::address_string)))
+    );
   }
 
   expect<void> write_addresses(wire::writer& dest, const expect<std::vector<lws::db::account_address>>& self)

src/rpc/daemon_pub.cpp

@@ -77,7 +77,11 @@ namespace rpc
 
   expect<minimal_chain_pub> minimal_chain_pub::from_json(std::string&& source)
   {
-    return wire::json::from_bytes<minimal_chain_pub>(std::move(source));
+    minimal_chain_pub out{};
+    std::error_code err = wire::json::from_bytes(std::move(source), out);
+    if (err)
+      return err;
+    return {std::move(out)};
   }
 }
 }

src/rpc/daemon_zmq.cpp

@@ -32,6 +32,7 @@
 #include "rpc/message_data_structs.h" // monero/src
 #include "wire/crypto.h"
 #include "wire/json.h"
+#include "wire/wrapper/variant.h"
 #include "wire/vector.h"
 
 namespace
@@ -103,14 +104,13 @@ namespace cryptonote
   }
   static void read_bytes(wire::json_reader& source, tx_out& self)
   {
+    auto variant = wire::variant(std::ref(self.target));
     wire::object(source,
       WIRE_FIELD(amount),
-      wire::variant_field("transaction output variant", std::ref(self.target),
+      WIRE_OPTION("to_key", txout_to_key, variant),
-        wire::option<txout_to_key>{"to_key"},
+      WIRE_OPTION("to_tagged_key", txout_to_tagged_key, variant),
-        wire::option<txout_to_tagged_key>{"to_tagged_key"},
+      WIRE_OPTION("to_script", txout_to_script, variant),
-        wire::option<txout_to_script>{"to_script"},
+      WIRE_OPTION("to_scripthash", txout_to_scripthash, variant)
-        wire::option<txout_to_scripthash>{"to_scripthash"}
-      )
     );
   }
 
@@ -132,13 +132,12 @@ namespace cryptonote
   }
   static void read_bytes(wire::json_reader& source, txin_v& self)
   {
+    auto variant = wire::variant(std::ref(self));
     wire::object(source,
-      wire::variant_field("transaction input variant", std::ref(self),
+      WIRE_OPTION("to_key", txin_to_key, variant),
-        wire::option<txin_to_key>{"to_key"},
+      WIRE_OPTION("gen", txin_gen, variant),
-        wire::option<txin_gen>{"gen"},
+      WIRE_OPTION("to_script", txin_to_script, variant),
-        wire::option<txin_to_script>{"to_script"},
+      WIRE_OPTION("to_scripthash", txin_to_scripthash, variant)
-        wire::option<txin_to_scripthash>{"to_scripthash"}
-      )
     );
   }
 

src/rpc/light_wallet.cpp

@@ -28,6 +28,7 @@
 #include "light_wallet.h"
 
 #include <boost/range/adaptor/indexed.hpp>
+#include <boost/range/adaptor/transformed.hpp>
 #include <ctime>
 #include <limits>
 #include <stdexcept>
@@ -44,6 +45,8 @@
 #include "wire/json.h"
 #include "wire/traits.h"
 #include "wire/vector.h"
+#include "wire/wrapper/array.h"
+#include "wire/wrappers_impl.h"
 
 namespace
 {
@@ -261,7 +264,7 @@ namespace lws
       WIRE_FIELD_COPY(start_height),
       WIRE_FIELD_COPY(transaction_height),
       WIRE_FIELD_COPY(blockchain_height),
-      wire::field("transactions", wire::as_array(boost::adaptors::index(self.transactions)))
+      wire::field("transactions", wire::array(boost::adaptors::index(self.transactions)))
     );
   }
 
@@ -297,7 +300,7 @@ namespace lws
       WIRE_FIELD_COPY(per_byte_fee),
       WIRE_FIELD_COPY(fee_mask),
       WIRE_FIELD_COPY(amount),
-      wire::field("outputs", wire::as_array(std::cref(self.outputs), expand))
+      wire::field("outputs", wire::array(boost::adaptors::transform(self.outputs, expand)))
     );
   }
 

src/rpc/rates.cpp

@@ -72,7 +72,11 @@ namespace lws
 
     expect<lws::rates> crypto_compare_::operator()(std::string&& body) const
     {
-      return wire::json::from_bytes<lws::rates>(std::move(body));
+      lws::rates out{};
+      const std::error_code error = wire::json::from_bytes(std::move(body), out);
+      if (error)
+        return error;
+      return {std::move(out)};
     }
   } // rpc
 } // lws

src/scanner.cpp

@@ -158,9 +158,15 @@ namespace lws
       if (uri.empty())
         uri = "/";
 
+      epee::byte_slice bytes{};
       const std::string& url = event.value.second.url;
-      const epee::byte_slice bytes = wire::json::to_bytes(event);
+      const std::error_code json_error = wire::json::to_bytes(bytes, event);
       const net::http::http_response_info* info = nullptr;
+      if (json_error)
+      {
+        MERROR("Failed to generate webhook JSON: " << json_error.message());
+        return;
+      }
 
       MINFO("Sending webhook to " << url);
       if (!client.invoke(uri, "POST", std::string{bytes.begin(), bytes.end()}, timeout, std::addressof(info), params))
@@ -460,7 +466,12 @@ namespace lws
             MONERO_THROW(resp.error(), "Failed to retrieve blocks from daemon");
           }
 
-          auto fetched = MONERO_UNWRAP(wire::json::from_bytes<rpc::json<rpc::get_blocks_fast>::response>(std::move(*resp)));
+          rpc::json<rpc::get_blocks_fast>::response fetched{};
+          {
+            const std::error_code error = wire::json::from_bytes(std::move(*resp), fetched);
+            if (error)
+              throw std::system_error{error};
+          }
           if (fetched.result.blocks.empty())
             throw std::runtime_error{"Daemon unexpectedly returned zero blocks"};
 

src/wire/CMakeLists.txt

@@ -1,4 +1,4 @@
-# Copyright (c) 2020, The Monero Project
+# Copyright (c) 2020-2023, The Monero Project
 #
 # All rights reserved.
 #
@@ -35,3 +35,4 @@ target_link_libraries(monero-lws-wire PRIVATE monero::libraries)
 
 add_subdirectory(json)
 add_subdirectory(msgpack)
+add_subdirectory(wrapper)

src/wire/crypto.h

@@ -45,48 +45,13 @@ namespace crypto
 
 namespace wire
 {
-  template<>
+  WIRE_DECLARE_BLOB(crypto::ec_scalar);
-  struct is_blob<crypto::ec_scalar>
+  WIRE_DECLARE_BLOB(crypto::hash);
-    : std::true_type
+  WIRE_DECLARE_BLOB(crypto::hash8);
-  {};
+  WIRE_DECLARE_BLOB(crypto::key_derivation);
-
+  WIRE_DECLARE_BLOB(crypto::key_image);
-  template<>
+  WIRE_DECLARE_BLOB(crypto::public_key);
-  struct is_blob<crypto::hash8>
+  WIRE_DECLARE_BLOB(crypto::signature);
-    : std::true_type
+  WIRE_DECLARE_BLOB(crypto::view_tag);
-  {};
+  WIRE_DECLARE_BLOB(rct::key);
-
-  template<>
-  struct is_blob<crypto::hash>
-    : std::true_type
-  {};
-
-  template<>
-  struct is_blob<crypto::key_derivation>
-    : std::true_type
-  {};
-
-  template<>
-  struct is_blob<crypto::key_image>
-    : std::true_type
-  {};
-
-  template<>
-  struct is_blob<crypto::public_key>
-    : std::true_type
-  {};
-
-  template<>
-  struct is_blob<crypto::signature>
-    : std::true_type
-  {};
-
-  template<>
-  struct is_blob<rct::key>
-    : std::true_type
-  {};
-
-  template<>
-  struct is_blob<crypto::view_tag>
-    : std::true_type
-  {};
 }

src/wire/field.h

@@ -55,46 +55,70 @@
 
 namespace wire
 {
-  template<typename T>
+  /*! Links `name` to a `value` and index `I` for object serialization.
-  struct unwrap_reference
+
+  `value_type` is `T` with optional `std::reference_wrapper` removed.
+  `value_type` needs a `read_bytes` function when parsing with a
+  `wire::reader` - see `read.h` for more info. `value_type` needs a
+  `write_bytes` function when parsing with a `wire::writer` - see `write.h`
+  for more info.
+
+  Any `value_type` where `is_optional_on_empty<value_type> == true`, will
+  automatically be converted to an optional field iff `value_type` has an
+  `empty()` method that returns `true`. The old output engine omitted fields
+  when an array was empty, and the standard input macro would ignore the
+  `false` return for the missing field. For compability reasons, the
+  input/output engine here matches that behavior. See `wrapper/array.h` to
+  enforce a required field even when the array is empty or specialize the
+  `is_optional_on_empty` trait. Only new fields should use this behavior.
+
+  Additional concept requirements for `value_type` when `Required == false`:
+    * must have an `operator*()` function.
+    * must have a conversion to bool function that returns true when
+      `operator*()` is safe to call (and implicitly when the associated field
+      should be written as opposed to skipped/omitted).
+  Additional concept requirements for `value_type` when `Required == false`
+  when reading:
+    * must have an `emplace()` method that ensures `operator*()` is safe to call.
+    * must have a `reset()` method to indicate a field was skipped/omitted.
+
+  If a standard type needs custom serialization, one "trick":
+  ```
+  struct custom_tag{};
+  void read_bytes(wire::reader&, boost::fusion::pair<custom_tag, std::string&>)
+  { ... }
+  void write_bytes(wire::writer&, boost::fusion::pair<custom_tag, const std::string&>)
+  { ... }
+
+  template<typename F, typename T>
+  void object_map(F& format, T& self)
   {
-    using type = T;
+    wire::object(format,
-  };
+      wire::field("foo", boost::fusion::make_pair<custom_tag>(std::ref(self.foo)))
+    );
+  }
+  ```
 
-  template<typename T>
+  Basically each input/output format needs a unique type so that the compiler
-  struct unwrap_reference<std::reference_wrapper<T>>
+  knows how to "dispatch" the read/write calls. */
-  {
-    using type = T;
-  };
-
-
-  //! Links `name` to a `value` for object serialization.
   template<typename T, bool Required, unsigned I = 0>
   struct field_
   {
-    using value_type = typename unwrap_reference<T>::type;
+    using value_type = unwrap_reference_t<T>;
-    static constexpr bool is_required() noexcept { return Required; }
-    static constexpr std::size_t count() noexcept { return 1; }
-    static constexpr unsigned id() noexcept { return I; }
 
     //! \return True if field is forced optional when `get_value().empty()`.
     static constexpr bool optional_on_empty() noexcept
     { return is_optional_on_empty<value_type>::value; }
 
+    static constexpr bool is_required() noexcept { return Required; }
+    static constexpr std::size_t count() noexcept { return 1; }
+    static constexpr unsigned id() noexcept { return I; }
+
     const char* name;
     T value;
 
-    //! \return `value` with `std::reference_wrapper` removed.
+    constexpr const value_type& get_value() const noexcept { return value; }
-    constexpr const value_type& get_value() const noexcept
+    value_type& get_value() noexcept { return value; }
-    {
-      return value;
-    }
-
-    //! \return `value` with `std::reference_wrapper` removed.
-    value_type& get_value() noexcept
-    {
-      return value;
-    }
   };
 
   //! Links `name` to `value`. Use `std::ref` if de-serializing.
@@ -112,76 +136,6 @@ namespace wire
   }
 
 
-  //! Links `name` to a type `T` for variant serialization.
-  template<typename T>
-  struct option
-  {
-    static constexpr unsigned id() noexcept { return 0; }
-    const char* name;
-  };
-
-  //! \return Name associated with type `T` for variant `field`.
-  template<typename T, typename U>
-  constexpr const char* get_option_name(const U& field) noexcept
-  {
-    return static_cast< const option<T>& >(field).name;
-  }
-
-  //! Links each type in a variant to a string key.
-  template<typename T, bool Required, typename... U>
-  struct variant_field_ : option<U>...
-  {
-    using value_type = typename unwrap_reference<T>::type;
-    static constexpr bool is_required() noexcept { return Required; }
-    static constexpr std::size_t count() noexcept { return sizeof...(U); }
-
-    constexpr variant_field_(const char* name, T value, option<U>... opts)
-      : option<U>(std::move(opts))..., name(name), value(std::move(value))
-    {}
-
-    const char* name;
-    T value;
-
-    constexpr const value_type& get_value() const noexcept
-    {
-      return value;
-    }
-
-    value_type& get_value() noexcept
-    {
-      return value;
-    }
-
-    template<typename V>
-    struct wrap
-    {
-      using result_type = void;
-
-      variant_field_ self;
-      V visitor;
-
-      template<typename X>
-      void operator()(const X& value) const
-      {
-        visitor(get_option_name<X>(self), value);
-      }
-    };
-
-    template<typename V>
-    void visit(V visitor) const
-    {
-      apply_visitor(wrap<V>{*this, std::move(visitor)}, get_value());
-    }
-  };
-
-  //! Links variant `value` to a unique name per type in `opts`. Use `std::ref` for `value` if de-serializing.
-  template<typename T, typename... U>
-  constexpr inline variant_field_<T, true, U...> variant_field(const char* name, T value, option<U>... opts)
-  {
-    return {name, std::move(value), std::move(opts)...};
-  }
-
-
   //! Indicates a field value should be written as an array
   template<typename T, typename F>
   struct as_array_
@@ -260,6 +214,8 @@ namespace wire
   template<typename T, unsigned I>
   inline constexpr bool available(const field_<T, true, I>& elem) noexcept
   {
+    /* The old output engine always skipped fields when it was an empty array,
+       this follows that behavior. See comments for `field_`. */
     return elem.is_required() || (elem.optional_on_empty() && !wire::empty(elem.get_value()));
   }
   template<typename T, unsigned I>
@@ -267,15 +223,5 @@ namespace wire
   {
     return bool(elem.get_value());
   }
-  template<typename T, typename... U>
-  inline constexpr bool available(const variant_field_<T, true, U...>&) noexcept
-  {
-    return true;
-  }
-  template<typename T, typename... U>
-  inline constexpr bool available(const variant_field_<T, false, U...>& elem)
-  {
-    return elem != nullptr;
-  }
 }
 

src/wire/json/base.h

@@ -1,4 +1,4 @@
-// Copyright (c) 2020, The Monero Project
+// Copyright (c) 2022-2023, The Monero Project
 // All rights reserved.
 //
 // Redistribution and use in source and binary forms, with or without modification, are
@@ -28,23 +28,31 @@
 #pragma once
 
 #include <string>
+#include <system_error>
 
-#include "byte_slice.h"
+#include "byte_stream.h"
-#include "common/expect.h"
 #include "wire/json/fwd.h"
+#include "wire/read.h"
+#include "wire/write.h"
 
 namespace wire
 {
   struct json
   {
     using input_type = json_reader;
-    using output_type = json_writer;
+    using output_type = json_slice_writer;
 
     template<typename T>
-    static expect<T> from_bytes(std::string&& source);
+    static std::error_code from_bytes(std::string&& source, T& dest)
+    {
+      return wire_read::from_bytes<input_type>(std::move(source), dest);
+    }
 
-    template<typename T>
+    template<typename T, typename U>
-    static epee::byte_slice to_bytes(const T& source);
+    static std::error_code to_bytes(T& dest, const U& source)
+    {
+      return wire_write::to_bytes<output_type>(dest, source);
+    }
   };
 }
 

src/wire/json/fwd.h

@@ -40,6 +40,7 @@ namespace wire
 {
   struct json;
   class json_reader;
+  class json_slice_writer;
   class json_writer;
 }
 

src/wire/json/read.h

@@ -107,28 +107,4 @@ namespace wire
         \return True if another value to read. */
     bool key(epee::span<const key_map> map, std::size_t&, std::size_t& index) override final;
   };
-
-
-  // Don't call `read` directly in this namespace, do it from `wire_read`.
-
-  template<typename T>
-  expect<T> json::from_bytes(std::string&& bytes)
-  {
-    json_reader source{std::move(bytes)};
-    return wire_read::to<T>(source);
-  }
-
-  // specialization prevents type "downgrading" to base type in cpp files
-
-  template<typename T>
-  inline void array(json_reader& source, T& dest)
-  {
-    wire_read::array(source, dest);
-  }
-
-  template<typename... T>
-  inline void object(json_reader& source, T... fields)
-  {
-    wire_read::object(source, wire_read::tracker<T>{std::move(fields)}...);
-  }
 } // wire

src/wire/json/write.cpp

@@ -54,10 +54,11 @@ namespace wire
     if (!formatter_.IsComplete())
       throw std::logic_error{"json_writer::take_json() failed with incomplete JSON tree"};
   }
-  epee::byte_slice json_writer::take_json()
+  epee::byte_stream json_writer::take_json()
   {
     check_complete();
-    epee::byte_slice out{std::move(bytes_)};
+    epee::byte_stream out{std::move(bytes_)};
+    bytes_.clear();
     formatter_.Reset(bytes_);
     return out;
   }

src/wire/json/write.h

@@ -60,15 +60,15 @@ namespace wire
     void check_flush();
 
   protected:
-    json_writer(bool needs_flush)
+    json_writer(epee::byte_stream&& out, bool needs_flush)
-      : writer(), bytes_(), formatter_(bytes_), needs_flush_(needs_flush)
+      : writer(), bytes_(std::move(out)), formatter_(bytes_), needs_flush_(needs_flush)
     {}
 
     //! \throw std::logic_error if incomplete JSON tree
     void check_complete();
 
     //! \throw std::logic_error if incomplete JSON tree. \return JSON bytes
-    epee::byte_slice take_json();
+    epee::byte_stream take_json();
 
     //! Flush bytes in local buffer to `do_flush(...)`
     void flush()
@@ -78,6 +78,9 @@ namespace wire
     }
 
   public:
+    //! JSON does not need array sizes.
+    static constexpr std::false_type need_array_size() noexcept { return{}; }
+
     json_writer(const json_writer&) = delete;
     virtual ~json_writer() noexcept;
     json_writer& operator=(const json_writer&) = delete;
@@ -113,12 +116,18 @@ namespace wire
   //! Buffers entire JSON message in memory
   struct json_slice_writer final : json_writer
   {
+    using sink = epee::byte_stream;
+
+    explicit json_slice_writer(sink&& out)
+      : json_writer(std::move(out), false)
+    {}
+
     explicit json_slice_writer()
-      : json_writer(false)
+      : json_writer(epee::byte_stream{}, false)
     {}
 
     //! \throw std::logic_error if incomplete JSON tree \return JSON bytes
-    epee::byte_slice take_bytes()
+    epee::byte_stream take_sink()
     {
       return json_writer::take_json();
     }
@@ -132,7 +141,7 @@ namespace wire
     virtual void do_flush(epee::span<const std::uint8_t>) override final;
   public:
     explicit json_stream_writer(std::ostream& dest)
-      : json_writer(true), dest(dest)
+      : json_writer(epee::byte_stream{}, true), dest(dest)
     {}
 
     //! Flush remaining bytes to stream \throw std::logic_error if incomplete JSON tree
@@ -142,45 +151,4 @@ namespace wire
       flush();
     }
   };
-
-  template<typename T>
-  epee::byte_slice json::to_bytes(const T& source)
-  {
-    return wire_write::to_bytes<json_slice_writer>(source);
-  }
-
-  template<typename T, typename F = identity_>
-  inline void array(json_writer& dest, const T& source, F filter = F{})
-  {
-    // works with "lazily" computed ranges
-    wire_write::array(dest, source, 0, std::move(filter));
-  }
-  template<typename T, typename F>
-  inline void write_bytes(json_writer& dest, as_array_<T, F> source)
-  {
-    wire::array(dest, source.get_value(), std::move(source.filter));
-  }
-  template<typename T>
-  inline enable_if<is_array<T>::value> write_bytes(json_writer& dest, const T& source)
-  {
-    wire::array(dest, source);
-  }
-
-  template<typename T, typename F = identity_, typename G = identity_>
-  inline void dynamic_object(json_writer& dest, const T& source, F key_filter = F{}, G value_filter = G{})
-  {
-    // works with "lazily" computed ranges
-    wire_write::dynamic_object(dest, source, 0, std::move(key_filter), std::move(value_filter));
-  }
-  template<typename T, typename F, typename G>
-  inline void write_bytes(json_writer& dest, as_object_<T, F, G> source)
-  {
-    wire::dynamic_object(dest, source.get_map(), std::move(source.key_filter), std::move(source.value_filter));
-  }
-
-  template<typename... T>
-  inline void object(json_writer& dest, T... fields)
-  {
-    wire_write::object(dest, std::move(fields)...);
-  }
 }

src/wire/msgpack/base.h

@@ -36,13 +36,15 @@
 #include "byte_slice.h"
 #include "common/expect.h"
 #include "wire/msgpack/fwd.h"
+#include "wire/read.h"
+#include "wire/write.h"
 
 namespace wire
 {
   struct msgpack
   {
     using input_type = msgpack_reader;
-    using output_type = msgpack_writer;
+    using output_type = msgpack_slice_writer;
 
     //! Tags that do not require bitmask to identify
     enum class tag : std::uint8_t
@@ -162,12 +164,18 @@ namespace wire
     using object16 = type<std::uint16_t, tag::object16>;
     using object32 = type<std::uint32_t, tag::object32>;
     using object_types = std::tuple<object16, object32>;
-
+    
     template<typename T>
-    static expect<T> from_bytes(epee::byte_slice&& source);
+    static std::error_code from_bytes(epee::byte_slice&& source, T& dest)
+    {
+      return wire_read::from_bytes<input_type>(std::move(source), dest);
+    }
 
-    template<typename T>
+    template<typename T, typename U>
-    static epee::byte_slice to_bytes(const T& source);
+    static std::error_code to_bytes(T& dest, const U& source)
+    {
+      return wire_write::to_bytes<output_type>(dest, source);
+    }
   };
 }
 

src/wire/msgpack/fwd.h

@@ -40,6 +40,7 @@ namespace wire
 {
   struct msgpack;
   class msgpack_reader;
+  class msgpack_slice_writer;
   class msgpack_writer;
 }
 

src/wire/msgpack/read.h

@@ -137,28 +137,4 @@ namespace wire
         \return True if another value to read. */
     bool key(epee::span<const key_map> map, std::size_t&, std::size_t& index) override final;
   };
-
-
-  // Don't call `read` directly in this namespace, do it from `wire_read`.
-
-  template<typename T>
-  expect<T> msgpack::from_bytes(epee::byte_slice&& bytes)
-  {
-    msgpack_reader source{std::move(bytes)};
-    return wire_read::to<T>(source);
-  }
-
-  // specialization prevents type "downgrading" to base type in cpp files
-
-  template<typename T>
-  inline void array(msgpack_reader& source, T& dest)
-  {
-    wire_read::array(source, dest);
-  }
-
-  template<typename... T>
-  inline void object(msgpack_reader& source, T... fields)
-  {
-    wire_read::object(source, wire_read::tracker<T>{std::move(fields)}...);
-  }
 } // wire

src/wire/msgpack/write.cpp

@@ -147,10 +147,10 @@ namespace wire
     if (expected_)
       throw std::logic_error{"msgpack_writer::take_msgpack() failed with incomplete tree"};
   }
-  epee::byte_slice msgpack_writer::take_msgpack()
+  epee::byte_stream msgpack_writer::take_msgpack()
   {
     check_complete();
-    epee::byte_slice out{std::move(bytes_)};
+    epee::byte_stream out{std::move(bytes_)};
     bytes_.clear();
     return out;
   }

src/wire/msgpack/write.h

@@ -86,19 +86,15 @@ namespace wire
     }
 
   protected:
-    msgpack_writer(epee::byte_stream&& initial, bool integer_keys, bool needs_flush)
+    msgpack_writer(epee::byte_stream&& sink, bool integer_keys, bool needs_flush)
-      : writer(), bytes_(std::move(initial)), expected_(1), integer_keys_(integer_keys), needs_flush_(needs_flush)
+      : writer(), bytes_(std::move(sink)), expected_(1), integer_keys_(integer_keys), needs_flush_(needs_flush)
-    {}
-
-    msgpack_writer(bool integer_keys, bool needs_flush)
-      : msgpack_writer(epee::byte_stream{}, integer_keys, needs_flush)
     {}
 
     //! \throw std::logic_error if tree was not completed
     void check_complete();
 
     //! \throw std::logic_error if incomplete msgpack tree. \return msgpack bytes
-    epee::byte_slice take_msgpack();
+    epee::byte_stream take_msgpack();
 
     //! Flush bytes in local buffer to `do_flush(...)`
     void flush()
@@ -157,22 +153,22 @@ namespace wire
   //! Buffers entire JSON message in memory
   struct msgpack_slice_writer final : msgpack_writer
   {
-    msgpack_slice_writer(epee::byte_stream&& initial, bool integer_keys = false)
+    explicit msgpack_slice_writer(epee::byte_stream&& sink, bool integer_keys = false)
-      : msgpack_writer(std::move(initial), integer_keys, false)
+      : msgpack_writer(std::move(sink), integer_keys, false)
     {}
 
     explicit msgpack_slice_writer(bool integer_keys = false)
-      : msgpack_writer(integer_keys, false)
+      : msgpack_slice_writer(epee::byte_stream{}, integer_keys)
     {}
 
     //! \throw std::logic_error if incomplete JSON tree \return JSON bytes
-    epee::byte_slice take_bytes()
+    epee::byte_stream take_sink()
     {
       return msgpack_writer::take_msgpack();
     }
   };
 
-  //! Periodically flushes JSON data to `std::ostream`
+  //! Periodically flushes MsgPack data to `std::ostream`
   class msgpack_stream_writer final : public msgpack_writer
   {
     std::ostream& dest;
@@ -180,7 +176,7 @@ namespace wire
     virtual void do_flush(epee::span<const std::uint8_t>) override final;
   public:
     explicit msgpack_stream_writer(std::ostream& dest, bool integer_keys = false)
-      : msgpack_writer(integer_keys, true), dest(dest)
+      : msgpack_writer(epee::byte_stream{}, integer_keys, true), dest(dest)
     {}
 
     //! Flush remaining bytes to stream \throw std::logic_error if incomplete JSON tree
@@ -190,44 +186,4 @@ namespace wire
       flush();
     }
   };
-
-  template<typename T>
-  epee::byte_slice msgpack::to_bytes(const T& source)
-  {
-    return wire_write::to_bytes<msgpack_slice_writer>(source);
-  }
-
-  template<typename T, typename F = identity_>
-  inline void array(msgpack_writer& dest, const T& source, F filter = F{})
-  {
-    wire_write::array(dest, source, source.size(), std::move(filter));
-  }
-  template<typename T, typename F>
-  inline void write_bytes(msgpack_writer& dest, as_array_<T, F> source)
-  {
-    wire::array(dest, source.get_value(), std::move(source.filter));
-  }
-  template<typename T>
-  inline enable_if<is_array<T>::value> write_bytes(msgpack_writer& dest, const T& source)
-  {
-    wire::array(dest, source);
-  }
-
-  template<typename T, typename F = identity_, typename G = identity_>
-  inline void dynamic_object(msgpack_writer& dest, const T& source, F key_filter = F{}, G value_filter = G{})
-  {
-    // works with "lazily" computed ranges
-    wire_write::dynamic_object(dest, source, 0, std::move(key_filter), std::move(value_filter));
-  }
-  template<typename T, typename F, typename G>
-  inline void write_bytes(msgpack_writer& dest, as_object_<T, F, G> source)
-  {
-    wire::dynamic_object(dest, source.get_map(), std::move(source.key_filter), std::move(source.value_filter));
-  }
-
-  template<typename... T>
-  inline void object(msgpack_writer& dest, T... fields)
-  {
-    wire_write::object(dest, std::move(fields)...);
-  }
 }

src/wire/read.h

@@ -67,6 +67,9 @@ namespace wire
     //! \return Maximum read depth for both objects and arrays before erroring
     static constexpr std::size_t max_read_depth() noexcept { return 100; }
 
+    //! \return Assume delimited arrays in generic interface (some optimizations disabled)
+    static constexpr std::true_type delimited_arrays() noexcept { return {}; }
+
     reader() noexcept
       : depth_(0)
     {}
@@ -137,32 +140,25 @@ namespace wire
     void end_object() noexcept { decrement_depth(); }
   };
 
-  inline void read_bytes(reader& source, bool& dest)
+  template<typename R>
-  {
+  inline void read_bytes(R& source, bool& dest)
-    dest = source.boolean();
+  { dest = source.boolean(); }
-  }
 
-  inline void read_bytes(reader& source, double& dest)
+  template<typename R>
-  {
+  inline void read_bytes(R& source, double& dest)
-    dest = source.real();
+  { dest = source.real(); }
-  }
 
-  inline void read_bytes(reader& source, std::string& dest)
+  template<typename R>
-  {
+  inline void read_bytes(R& source, std::string& dest)
-    dest = source.string();
+  { dest = source.string(); }
-  }
 
   template<typename R>
   inline void read_bytes(R& source, std::vector<std::uint8_t>& dest)
-  {
+  { dest = source.binary(); }
-    dest = source.binary();
-  }
 
-  template<typename T>
+  template<typename R, typename T>
-  inline enable_if<is_blob<T>::value> read_bytes(reader& source, T& dest)
+  inline std::enable_if_t<is_blob<T>::value> read_bytes(R& source, T& dest)
-  {
+  { source.binary(epee::as_mut_byte_span(dest)); }
-    source.binary(epee::as_mut_byte_span(dest));
-  }
 
   namespace integer
   {
@@ -205,17 +201,17 @@ namespace wire
   }
 
   //! read all current and future signed integer types
-  template<typename T>
+  template<typename R, typename T>
-  inline enable_if<std::is_signed<T>::value && std::is_integral<T>::value>
+  inline std::enable_if_t<std::is_signed<T>::value && std::is_integral<T>::value>
-  read_bytes(reader& source, T& dest)
+  read_bytes(R& source, T& dest)
   {
     dest = integer::cast_signed<T>(source.integer());
   }
 
   //! read all current and future unsigned integer types
-  template<typename T>
+  template<typename R, typename T>
-  inline enable_if<std::is_unsigned<T>::value && std::is_integral<T>::value>
+  inline std::enable_if_t<std::is_unsigned<T>::value && std::is_integral<T>::value>
-  read_bytes(reader& source, T& dest)
+  read_bytes(R& source, T& dest)
   {
     dest = integer::cast_unsigned<T>(source.unsigned_integer());
   }
@@ -231,21 +227,27 @@ namespace wire_read
 
   [[noreturn]] void throw_exception(wire::error::schema code, const char* display, epee::span<char const* const> name_list);
 
+  template<typename R, typename T>
+  inline void bytes(R& source, T&& dest)
+  {
+    read_bytes(source, dest); // ADL (searches every associated namespace)
+  }
+
   //! \return `T` converted from `source` or error.
-  template<typename T, typename R>
+  template<typename R, typename T, typename U>
-  inline expect<T> to(R& source)
+  inline std::error_code from_bytes(T&& source, U& dest)
   {
     try
     {
-      T dest{};
+      R in{std::forward<T>(source)};
-      read_bytes(source, dest);
+      bytes(in, dest);
-      source.check_complete();
+      in.check_complete();
-      return dest;
     }
     catch (const wire::exception& e)
     {
       return e.code();
     }
+    return {};
   }
 
   template<typename R, typename T>
@@ -258,7 +260,7 @@ namespace wire_read
     std::size_t count = source.start_array();
 
     dest.clear();
-    dest.reserve(count);
+    wire::reserve(dest, count);
 
     bool more = count;
     while (more || !source.is_array_end(count))
@@ -272,37 +274,6 @@ namespace wire_read
     return source.end_array();
   }
 
-  // `unpack_variant_field` identifies which of the variant types was selected. starts with index-0
-
-  template<typename R, typename T>
-  inline void unpack_variant_field(std::size_t, R&, const T&)
-  {}
-
-  template<typename R, typename T, typename U, typename... X>
-  inline void unpack_variant_field(const std::size_t index, R& source, T& variant, const wire::option<U>& head, const wire::option<X>&... tail)
-  {
-    if (index)
-      unpack_variant_field(index - 1, source, variant, tail...);
-    else
-    {
-      U dest{};
-      read_bytes(source, dest);
-      variant = std::move(dest);
-    }
-  }
-
-  // `unpack_field` expands `variant_field_`s or reads `field_`s directly
-
-  template<typename R, typename T, bool Required, typename... U>
-  inline void unpack_field(const std::size_t index, R& source, wire::variant_field_<T, Required, U...>& dest)
-  {
-    unpack_variant_field(index, source, dest.get_value(), static_cast< const wire::option<U>& >(dest)...);
-  }
-
-  template<typename T, bool Required, typename... U>
-  inline void reset_field(wire::variant_field_<T, Required, U...>& dest)
-  {}
-
   template<typename T, unsigned I>
   inline void reset_field(wire::field_<T, true, I>& dest)
   {
@@ -320,34 +291,15 @@ namespace wire_read
   template<typename R, typename T, unsigned I>
   inline void unpack_field(std::size_t, R& source, wire::field_<T, true, I>& dest)
   {
-    read_bytes(source, dest.get_value());
+    bytes(source, dest.get_value());
   }
 
   template<typename R, typename T, unsigned I>
   inline void unpack_field(std::size_t, R& source, wire::field_<T, false, I>& dest)
   {
-    dest.get_value().emplace();
+    if (!bool(dest.get_value()))
-    read_bytes(source, *dest.get_value());
+      dest.get_value().emplace();
-  }
+    bytes(source, *dest.get_value());
-
-  // `expand_field_map` writes a single `field_` name or all option names in a `variant_field_` to a table
-
-  template<std::size_t N>
-  inline void expand_field_map(std::size_t, wire::reader::key_map (&)[N])
-  {}
-
-  template<std::size_t N, typename T, typename... U>
-  inline void expand_field_map(std::size_t index, wire::reader::key_map (&map)[N], const T& head, const U&... tail)
-  {
-    map[index].name = head.name;
-    map[index].id = head.id();
-    expand_field_map(index + 1, map, tail...);
-  }
-
-  template<std::size_t N, typename T, bool Required, typename... U>
-  inline void expand_field_map(std::size_t index, wire::reader::key_map (&map)[N], const wire::variant_field_<T, Required, U...>& field)
-  {
-    expand_field_map(index, map, static_cast< const wire::option<U> & >(field)...);
   }
 
   //! Tracks read status of every object field instance.
@@ -378,7 +330,8 @@ namespace wire_read
     std::size_t set_mapping(std::size_t index, wire::reader::key_map (&map)[N])
     {
       our_index_ = index;
-      expand_field_map(index, map, field_); // expands possible inner options
+      map[index].id = field_.id();
+      map[index].name = field_.name;
       return index + count();
     }
 
@@ -460,19 +413,14 @@ namespace wire_read
 
 namespace wire
 {
-  template<typename T>
-  inline void array(reader& source, T& dest)
-  {
-    wire_read::array(source, dest);
-  }
   template<typename R, typename T>
-  inline enable_if<is_array<T>::value> read_bytes(R& source, T& dest)
+  inline std::enable_if_t<is_array<T>::value> read_bytes(R& source, T& dest)
   {
     wire_read::array(source, dest);
   }
 
-  template<typename... T>
+  template<typename R, typename... T>
-  inline void object(reader& source, T... fields)
+  inline std::enable_if_t<std::is_base_of<reader, R>::value> object(R& source, T... fields)
   {
     wire_read::object(source, wire_read::tracker<T>{std::move(fields)}...);
   }

src/wire/traits.h

@@ -30,20 +30,50 @@
 #include <type_traits>
 #include <utility>
 
+#define WIRE_DECLARE_BLOB(type)                 \
+  template<>                                    \
+  struct is_blob<type>                          \
+    : std::true_type                            \
+  {}
+
+#define WIRE_DECLARE_BLOB_NS(type)              \
+  namespace wire { WIRE_DECLARE_BLOB(type); }
+
 namespace wire
 {
-  template<bool C>
+  template<typename T>
-  using enable_if = typename std::enable_if<C>::type;
+  struct unwrap_reference
+  {
+    using type = std::remove_cv_t<std::remove_reference_t<T>>;
+  };
 
+  template<typename T>
+  struct unwrap_reference<std::reference_wrapper<T>>
+    : std::remove_cv<T>
+  {};
+
+  template<typename T>
+  using unwrap_reference_t = typename unwrap_reference<T>::type;
+
+  /*! Mark `T` as an array for writing, and reading when
+   `default_min_element_size<T::value_type>::value != 0`. See `array_` in
+   `wrapper/array.h`. */
   template<typename T>
   struct is_array : std::false_type
   {};
 
+  /*! Mark `T` as fixed binary data for reading+writing. Concept requirements
+    for reading:
+      * `T` must be compatible with `epee::as_mut_byte_span` (`std::is_pod<T>`
+        and no padding).
+    Concept requirements for writing:
+      * `T` must be compatible with `epee::as_byte_span` (std::is_pod<T>` and
+        no padding). */
   template<typename T>
   struct is_blob : std::false_type
   {};
 
-/*! Forces field to be optional when empty. Concept requirements for `T` when
+  /*! Forces field to be optional when empty. Concept requirements for `T` when
     `is_optional_on_empty<T>::value == true`:
       * must have an `empty()` method that toggles whether the associated
         `wire::field_<...>` is omitted by the `wire::writer`.
@@ -66,6 +96,15 @@ namespace wire
     return head + sum(tail...);
   }
 
+  //! If container has no `reserve(0)` function, this function is used
+  template<typename... T>
+  inline void reserve(const T&...) noexcept
+  {}
+
+  //! Container has `reserve(std::size_t)` function, use it
+  template<typename T>
+  inline auto reserve(T& container, const std::size_t count) -> decltype(container.reserve(count))
+  { return container.reserve(count); }
 
   //! If `T` has no `empty()` function, this function is used
   template<typename... T>

src/wire/wrapper/CMakeLists.txt

@@ -0,0 +1,35 @@
+# Copyright (c) 2020, The Monero Project
+#
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are
+# permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice, this list of
+#    conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice, this list
+#    of conditions and the following disclaimer in the documentation and/or other
+#    materials provided with the distribution.
+#
+# 3. Neither the name of the copyright holder nor the names of its contributors may be
+#    used to endorse or promote products derived from this software without specific
+#    prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+# THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+set(monero-lws-wire_sources variant.cpp)
+set(monero-lws-wire_headers variant.h)
+
+add_library(monero-lws-wire-wrapper ${monero-lws-wire_sources} ${monero-lws-wire_headers})
+target_include_directories(monero-lws-wire-wrapper PUBLIC "${LMDB_INCLUDE}")
+target_link_libraries(monero-lws-wire-wrapper PRIVATE monero::libraries)
+

src/wire/wrapper/array.h

@@ -0,0 +1,158 @@
+// Copyright (c) 2021, The Monero Project
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#pragma once
+
+#include <cstdint>
+#include <utility>
+
+//#include "wire/field.h"
+#include "wire/traits.h"
+
+/*! An array field with read constraint. See `array_` for more info. All (empty)
+  arrays were "optional" (omitted) historically in epee, so this matches prior
+  behavior. */
+#define WIRE_FIELD_ARRAY(name, read_constraint)         \
+  ::wire::optional_field( #name , ::wire::array< read_constraint >(std::ref( self . name )))
+
+namespace wire
+{
+  /*! A wrapper that ensures `T` is written as an array, with `C` constraints
+    when reading (`max_element_count` or `min_element_size`). `C` can be `void`
+    if write-only.
+    This wrapper meets the requirements for an optional field; `wire::field`
+    and `wire::optional_field` determine whether an empty array must be
+    encoded on the wire. Historically, empty arrays were always omitted on
+    the wire (a defacto optional field).
+    The `is_array` trait can also be used, but is default treated as an optional
+    field. The trait `is_optional_on_empty` traits can be specialized to disable
+    the optional on empty behavior. See `wire/traits.h` for more ifnormation
+    on the `is_optional_on_empty` trait.
+    `container_type` is `T` with optional `std::reference_wrapper` removed.
+    `container_type` concept requirements:
+      * `typedef` `value_type` that specifies inner type.
+      * must have `size()` method that returns number of elements.
+    Additional concept requirements for `container_type` when reading:
+      * must have `clear()` method that removes all elements (`size() == 0`).
+      * must have either: (1) `end()` and `emplace_hint(iterator, value_type&&)`
+        or (2) `emplace_back()` and `back()`:
+        * `end()` method that returns one-past the last element.
+        * `emplace_hint(iterator, value_type&&)` method that move constructs a new
+           element.
+        * `emplace_back()` method that default initializes new element
+        * `back()` method that retrieves last element by reference.
+    Additional concept requirements for `container_type` when writing:
+      * must work with foreach loop (`std::begin` and `std::end`).
+      * must work with `boost::size` (from the `boost::range` library). */
+  template<typename T, typename C = void>
+  struct array_
+  {
+    using constraint = C;
+    using container_type = unwrap_reference_t<T>;
+    using value_type = typename container_type::value_type;
+
+    // See nested `array_` overload below
+    using inner_array = std::reference_wrapper<value_type>;
+    using inner_array_const = std::reference_wrapper<const value_type>;
+
+    T container;
+
+    constexpr const container_type& get_container() const noexcept { return container; }
+    container_type& get_container() noexcept { return container; }
+
+    //! Read directly into the non-nested array
+    container_type& get_read_object() noexcept { return get_container(); }
+
+
+    // concept requirements for optional fields
+
+    explicit operator bool() const noexcept { return !get_container().empty(); }
+    array_& emplace() noexcept { return *this; }
+
+    array_& operator*() noexcept { return *this; }
+    const array_& operator*() const noexcept { return *this; }
+
+    void reset() { get_container().clear(); }
+  };
+
+  //! Nested array case
+  template<typename T, typename C, typename D>
+  struct array_<array_<T, C>, D>
+  {
+    // compute `container_type` and `value_type` recursively
+    using constraint = D;
+    using container_type = typename array_<T, C>::container_type;
+    using value_type = typename container_type::value_type;
+
+    // Re-compute `array_` for inner values
+    using inner_array = array_<typename array_<T, C>::inner_array, C>;
+    using inner_array_const = array_<typename array_<T, C>::inner_array_const, C>;
+
+    array_<T, C> nested;
+
+    const container_type& get_container() const noexcept { return nested.get_container(); }
+    container_type& get_container() noexcept { return nested.get_container(); }
+
+    //! Read through this proxy to track nested array
+    array_& get_read_object() noexcept { return *this; }
+
+
+    // concept requirements for optional fields
+
+    explicit operator bool() const noexcept { return !empty(); }
+    array_& emplace() noexcept { return *this; }
+
+    array_& operator*() noexcept { return *this; }
+    const array_& operator*() const noexcept { return *this; }
+
+    void reset() { clear(); }
+
+
+    /* For reading nested arrays. writing nested arrays is handled in
+       `wrappers_impl.h` with range transform. */
+
+    void clear() { get_container().clear(); }
+    bool empty() const noexcept { return get_container().empty(); }
+    std::size_t size() const noexcept { return get_container().size(); }
+
+    void emplace_back() { get_container().emplace_back(); }
+
+    //! \return A proxy object for tracking inner-array constraints
+    inner_array back() noexcept { return {std::ref(get_container().back())}; }
+  };
+
+  //! Treat `value` as an array when reading/writing, and constrain reading with `C`.
+  template<typename C = void, typename T = void>
+  inline constexpr array_<T, C> array(T value)
+  {
+    return {std::move(value)};
+  }
+
+  /* Do not register with `is_optional_on_empty` trait, this allows selection
+     on whether an array is mandatory on wire. */
+
+} // wire

src/wire/wrapper/variant.cpp

@@ -0,0 +1,40 @@
+// Copyright (c) 2022, The Monero Project
+//
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "wire/wrapper/variant.h"
+
+#include <boost/core/demangle.hpp>
+#include "wire/error.h"
+
+namespace wire
+{
+  [[noreturn]] void throw_variant_exception(wire::error::schema type, const char* variant_name)
+  {
+    WIRE_DLOG_THROW(type, "error with variant type: " << boost::core::demangle(variant_name));
+  }
+}

src/wire/wrapper/variant.h

@@ -0,0 +1,207 @@
+// Copyright (c) 2022, The Monero Project
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#pragma once
+
+#include <boost/variant/get.hpp>
+#include <boost/variant/variant_fwd.hpp>
+#include <functional>
+#include <utility>
+
+#include "wire/error.h"
+#include "wire/fwd.h"
+#include "wire/field.h"
+#include "wire/read.h"
+#include "wire/write.h"
+
+#define WIRE_OPTION(name, type, cpp_name)                               \
+  wire::optional_field(name, wire::option<type>(std::ref(cpp_name)))
+
+namespace wire
+{
+  [[noreturn]] void throw_variant_exception(error::schema type, const char* variant_name);
+
+  /*! Wrapper for any C++ variant type that tracks if a `read_bytes` call has
+    completed on wrapped `value`. This wrapper is not needed if the variant is
+    being used for writes only - see `wire::option_` below for more information.
+
+    `variant_type` is `T` with optional `std::reference_wrapper` removed. See
+    `option_` for concept requirements of `variant_type`.
+
+    Example usage:
+    ```
+    template<typename F, typename T>
+    void type_map(F& format, T& self)
+    {
+      auto variant = wire::variant(std::ref(self.field3));
+      wire::object(format,
+        ...
+        WIRE_OPTION("type1", type1, variant),
+        WIRE_OPTION("type2", type2, variant)
+      );
+    }
+    ``` */
+  template<typename T>
+  struct variant_
+  {
+    using variant_type = unwrap_reference_t<T>;
+
+    //! \throw wire::exception with `type` and mangled C++ name of `variant_type`.
+    [[noreturn]] static void throw_exception(const error::schema type)
+    { throw_variant_exception(type, typeid(variant_type).name()); }
+
+    constexpr variant_(T&& value)
+      : value(std::move(value)), read(false)
+    {}
+
+    T value;
+    bool read;
+
+    constexpr const variant_type& get_variant() const noexcept { return value; }
+    variant_type& get_variant() noexcept { return value; }
+
+    //! Makes `variant_` compatible with `emplace()` in `option_`.
+    template<typename U>
+    variant_& operator=(U&& rhs)
+    {
+      get_variant() = std::forward<U>(rhs);
+      return *this;
+    }
+  };
+
+  template<typename T>
+  inline constexpr variant_<T> variant(T value)
+  { return {std::move(value)}; }
+
+  namespace adapt
+  {
+    template<typename T>
+    inline void throw_if_not_read(const T&)
+    { throw_variant_exception(error::schema::missing_key, typeid(T).name()); }
+
+    template<typename T>
+    inline void throw_if_not_read(const variant_<T>& value)
+    {
+      if (!value.read)
+        value.throw_exception(error::schema::missing_key);
+    }
+
+
+    // other variant overloads can be added here as needed
+
+    template<typename U, typename... T>
+    inline const U* get_if(const boost::variant<T...>& value)
+    { return boost::get<U>(std::addressof(value)); }
+
+    template<typename U, typename T>
+    inline const U* get_if(const variant_<T>& value)
+    { return adapt::get_if<U>(value.get_variant()); }
+  }
+
+  /*! Wrapper that makes a variant compatible with `wire::optional_field`.
+    Currently `wire::variant_` and `boost::variant` are valid variant types
+    for writing, and only `wire::variant_` is valid for reading.
+
+   `variant_type` is `T` with optional `std::reference_wrapper` removed.
+   `variant_type` concept requirements:
+     * must have two overloads for `get<U>` function in `adapt` namespace - one
+       `const` and one non-`const` that returns `const U&` and `U&` respectively
+       iff `variant_type` is storing type `U`. Otherwise, the function should
+       throw an exception.
+     * must have overload for `get_if<U>` function in `adapt` namespace that
+       returns `const U*` when `variant_type` is storing type `U`. Otherwise, the
+       function should return `nullptr`.
+     * must have a member function `operator=(U&&)` that changes the stored type
+       to `U` (`get<U>` and `get_if<U>` will return `U` after `operator=`
+       completion).
+
+   The `wire::variant(std::ref(self.field3))` step in the example above can be
+   omitted if only writing is needed. The `boost::variant` value should be
+   given directly to `wire::option<U>(...)` or `WIRE_OPTION` macro - only one
+   type is active so `wire::optional_field` will omit all other types/fields. */
+  template<typename T, typename U>
+  struct option_
+  {
+    using variant_type = unwrap_reference_t<T>;
+    using option_type = U;
+
+    T value;
+
+    constexpr const variant_type& get_variant() const noexcept { return value; }
+    variant_type& get_variant() noexcept { return value; }
+
+    //! \return `true` iff `U` is active type in variant.
+    bool is_active() const { return adapt::get_if<U>(get_variant()) != nullptr; }
+
+
+    // concept requirements for optional fields
+
+    explicit operator bool() const { return is_active(); }
+    void emplace() { get_variant() = U{}; }
+
+    const option_& operator*() const { return *this; }
+    option_& operator*() { return *this; }
+
+    //! \throw wire::exception iff no variant type was read.
+    void reset() { adapt::throw_if_not_read(get_variant()); }
+  };
+
+  template<typename U, typename T>
+  inline constexpr option_<T, U> option(T value)
+  { return {std::move(value)}; }
+
+  namespace adapt
+  {
+    // other variant overloads can be added here as needed
+
+    template<typename U, typename... T>
+    inline U& get(boost::variant<T...>& value)
+    { return boost::get<U>(value); }
+
+    template<typename U, typename... T>
+    inline const U& get(const boost::variant<T...>& value)
+    { return boost::get<U>(value); }
+
+    template<typename U, typename T>
+    inline const U& get(const variant_<T>& value)
+    { return adapt::get<U>(value.get_variant()); }
+  }
+
+  //! \throw wire::exception if `dest.get_variant()` has already been used in `read_bytes`.
+  template<typename R, typename T, typename U>
+  inline void read_bytes(R& source, option_<std::reference_wrapper<variant_<T>>, U> dest)
+  {
+    if (dest.get_variant().read)
+      dest.get_variant().throw_exception(error::schema::invalid_key);
+    wire_read::bytes(source, adapt::get<U>(dest.get_variant().get_variant()));
+    dest.get_variant().read = true;
+  }
+
+  template<typename W, typename T, typename U>
+  inline void write_bytes(W& dest, const option_<T, U>& source)
+  { wire_write::bytes(dest, adapt::get<U>(source.get_variant())); }
+}

src/wire/wrappers_impl.h

@@ -0,0 +1,62 @@
+// Copyright (c) 2022, The Monero Project
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#pragma once
+
+#include <boost/range/adaptor/transformed.hpp>
+#include "wire/error.h"
+#include "wire/read.h"
+#include "wire/write.h"
+#include "wire/wrapper/array.h"
+
+namespace wire
+{
+  //
+  // free functions for `array_` wrapper
+  //
+
+  template<typename R, typename T, typename C>
+  inline void read_bytes(R& source, array_<T, C> wrapper)
+  {
+    // see constraints directly above `array_` definition
+    static_assert(std::is_same<R, void>::value, "array_ must have a read constraint for memory purposes");
+    wire_read::array(source, wrapper.get_read_object());
+  }
+
+  template<typename W, typename T, typename C>
+  inline void write_bytes(W& dest, const array_<T, C>& wrapper)
+  {
+    wire_write::array(dest, wrapper.get_container());
+  }
+  template<typename W, typename T, typename C, typename D>
+  inline void write_bytes(W& dest, const array_<array_<T, C>, D>& wrapper)
+  {
+    using inner_type = typename array_<array_<T, C>, D>::inner_array_const;
+    const auto wrap = [](const auto& val) -> inner_type { return {std::ref(val)}; };
+    wire_write::array(dest, boost::adaptors::transform(wrapper.get_container(), wrap));
+  }
+} // wire

src/wire/write.h

@@ -29,11 +29,16 @@
 
 #include <array>
 #include <boost/utility/string_ref.hpp>
+#include <boost/range/size.hpp>
+#include <boost/range/adaptor/transformed.hpp>
 #include <cstdint>
 #include <type_traits>
+#include <system_error>
 
 #include "byte_slice.h" // monero/contrib/epee/include
+#include "byte_stream.h"// monero/contrib/epee/include
 #include "span.h"       // monero/contrib/epee/include
+#include "wire/error.h"
 #include "wire/field.h"
 #include "wire/filters.h"
 #include "wire/traits.h"
@@ -47,6 +52,9 @@ namespace wire
 
     virtual ~writer() noexcept;
 
+    //! By default, insist on retrieving array size before writing array
+    static constexpr std::true_type need_array_size() noexcept { return{}; }
+
     virtual void boolean(bool) = 0;
 
     virtual void integer(int) = 0;
@@ -78,59 +86,62 @@ namespace wire
     writer& operator=(writer&&) = default;
   };
 
-  // leave in header, compiler can de-virtualize when final type is given
+  template<typename W>
+  inline void write_bytes(W& dest, const bool source)
+  { dest.boolean(source); }
 
-  inline void write_bytes(writer& dest, const bool source)
+  template<typename W>
-  {
+  inline void write_arithmetic(W& dest, const int source)
-    dest.boolean(source);
+  { dest.integer(source); }
-  }
 
-  inline void write_bytes(writer& dest, const int source)
+  template<typename W>
-  {
+  inline void write_arithmetic(W& dest, const long source)
-    dest.integer(source);
+  { dest.integer(std::intmax_t(source)); }
-  }
-  inline void write_bytes(writer& dest, const long source)
-  {
-    dest.integer(std::intmax_t(source));
-  }
-  inline void write_bytes(writer& dest, const long long source)
-  {
-    dest.integer(std::intmax_t(source));
-  }
 
-  inline void write_bytes(writer& dest, const unsigned source)
+  template<typename W>
-  {
+  inline void write_arithmetic(W& dest, const long long source)
-    dest.unsigned_integer(source);
+  { dest.integer(std::intmax_t(source)); }
-  }
-  inline void write_bytes(writer& dest, const unsigned long source)
-  {
-    dest.unsigned_integer(std::uintmax_t(source));
-  }
-  inline void write_bytes(writer& dest, const unsigned long long source)
-  {
-    dest.unsigned_integer(std::uintmax_t(source));
-  }
 
-  inline void write_bytes(writer& dest, const double source)
+  template<typename W>
-  {
+  inline void write_arithmetic(W& dest, const unsigned source)
-    dest.real(source);
+  { dest.unsigned_integer(source); }
-  }
 
-  inline void write_bytes(writer& dest, const boost::string_ref source)
+  template<typename W>
-  {
+  inline void write_arithmetic(W& dest, const unsigned long source)
-    dest.string(source);
+  { dest.unsigned_integer(std::uintmax_t(source)); }
-  }
 
-  template<typename T>
+  template<typename W>
-  inline enable_if<is_blob<T>::value> write_bytes(writer& dest, const T& source)
+  inline void write_arithmetic(W& dest, const unsigned long long source)
-  {
+  { dest.unsigned_integer(std::uintmax_t(source)); }
-    dest.binary(epee::as_byte_span(source));
-  }
 
-  inline void write_bytes(writer& dest, const epee::span<const std::uint8_t> source)
+  template<typename W, typename T>
-  {
+  inline std::enable_if_t<std::is_arithmetic<T>::value> write_bytes(W& dest, const T source)
-    dest.binary(source);
+  { write_arithmetic(dest, source); }
-  }
+
+  template<typename W>
+  inline void write_bytes(W& dest, const double source)
+  { dest.real(source); }
+
+  template<typename W>
+  inline void write_bytes(W& dest, const boost::string_ref source)
+  { dest.string(source); }
+
+  template<typename W, typename T>
+  inline std::enable_if_t<is_blob<T>::value> write_bytes(W& dest, const T& source)
+  { dest.binary(epee::as_byte_span(source)); }
+
+  template<typename W>
+  inline void write_bytes(W& dest, const epee::span<const std::uint8_t> source)
+  { dest.binary(source); }
+
+  template<typename W>
+  inline void write_bytes(W& dest, const epee::byte_slice& source)
+  { write_bytes(dest, epee::to_span(source)); }
+
+  //! Use `write_bytes(...)` method if available for `T`.
+  template<typename W, typename T>
+  inline auto write_bytes(W& dest, const T& source) -> decltype(source.write_bytes(dest))
+  { return source.write_bytes(dest); }
 }
 
 namespace wire_write
@@ -142,39 +153,80 @@ namespace wire_write
       declared after these functions. */
 
   template<typename W, typename T>
-  inline epee::byte_slice to_bytes(W&& dest, const T& value)
+  inline void bytes(W& dest, const T& source)
   {
-    write_bytes(dest, value);
+    write_bytes(dest, source); // ADL (searches every associated namespace)
-    return dest.take_bytes();
+  }
+
+  template<typename W, typename T, typename U>
+  inline std::error_code to_bytes(T& dest, const U& source)
+  {
+    try
+    {
+      W out{std::move(dest)};
+      bytes(out, source);
+      dest = out.take_sink();
+    }
+    catch (const wire::exception& e)
+    {
+      dest.clear();
+      return e.code();
+    }
+    catch (...)
+    {
+      dest.clear();
+      throw;
+    }
+    return {};
   }
 
   template<typename W, typename T>
-  inline epee::byte_slice to_bytes(const T& value)
+  inline std::error_code to_bytes(epee::byte_slice& dest, const T& source)
   {
-    return wire_write::to_bytes(W{}, value);
+    epee::byte_stream sink{};
+    const std::error_code error = wire_write::to_bytes<W>(sink, source);
+    if (error)
+    {
+      dest = nullptr;
+      return error;
+    }
+    dest = epee::byte_slice{std::move(sink)};
+    return {};
   }
 
-  template<typename W, typename T, typename F = wire::identity_>
+  template<typename T>
-  inline void array(W& dest, const T& source, const std::size_t count, F filter = F{})
+  inline std::size_t array_size_(std::true_type, const T& source)
+  { return boost::size(source); }
+
+  template<typename T>
+  inline constexpr std::size_t array_size_(std::false_type, const T&) noexcept
+  { return 0; }
+
+  template<typename W, typename T>
+  inline constexpr std::size_t array_size(const W& dest, const T& source) noexcept
+  { return array_size_(dest.need_array_size(), source); }
+
+  template<typename W, typename T>
+  inline void array(W& dest, const T& source)
   {
     using value_type = typename T::value_type;
     static_assert(!std::is_same<value_type, char>::value, "write array of chars as binary");
     static_assert(!std::is_same<value_type, std::uint8_t>::value, "write array of unsigned chars as binary");
 
-    dest.start_array(count);
+    dest.start_array(array_size(dest, source));
     for (const auto& elem : source)
-      write_bytes(dest, filter(elem));
+      bytes(dest, elem);
     dest.end_array();
   }
 
   template<typename W, typename T, unsigned I>
   inline bool field(W& dest, const wire::field_<T, true, I> elem)
   {
-    // Arrays always optional, see `wire/field.h`
+    // Arrays always optional, see `wire::field.h`
     if (wire::available(elem))
     {
       dest.key(I, elem.name);
-      write_bytes(dest, elem.get_value());
+      bytes(dest, elem.get_value());
     }
     return true;
   }
@@ -185,7 +237,7 @@ namespace wire_write
     if (wire::available(elem))
     {
       dest.key(I, elem.name);
-      write_bytes(dest, *elem.get_value());
+      bytes(dest, *elem.get_value());
     }
     return true;
   }
@@ -199,13 +251,13 @@ namespace wire_write
   }
 
   template<typename W, typename T, typename F, typename G>
-  inline void dynamic_object(W& dest, const T& values, const std::size_t count, F key_filter, G value_filter)
+  inline void dynamic_object(W& dest, const T& values, F key_filter, G value_filter)
   {
-    dest.start_object(count);
+    dest.start_object(array_size(dest, values));
     for (const auto& elem : values)
     {
       dest.key(key_filter(elem.first));
-      write_bytes(dest, value_filter(elem.second));
+      bytes(dest, value_filter(elem.second));
     }
     dest.end_object();
   }
@@ -213,35 +265,31 @@ namespace wire_write
 
 namespace wire
 {
-  template<typename T, typename F = identity_>
+  template<typename W, typename T, typename F>
-  inline void array(writer& dest, const T& source, F filter = F{})
+  inline void write_bytes(W& dest, const as_array_<T, F> source)
   {
-    wire_write::array(dest, source, source.size(), std::move(filter));
+    wire_write::array(dest, boost::adaptors::transform(source.get_value(), source.filter));
   }
-  template<typename T, typename F>
+  template<typename W, typename T>
-  inline void write_bytes(writer& dest, as_array_<T, F> source)
+  inline std::enable_if_t<is_array<T>::value> write_bytes(W& dest, const T& source)
   {
-    wire::array(dest, source.get_value(), std::move(source.filter));
+    wire_write::array(dest, source);
-  }
-  template<typename T>
-  inline enable_if<is_array<T>::value> write_bytes(writer& dest, const T& source)
-  {
-    wire::array(dest, source);
   }
 
-  template<typename T, typename F = identity_, typename G = identity_>
+  template<typename W, typename T, typename F = identity_, typename G = identity_>
-  inline void dynamic_object(writer& dest, const T& source, F key_filter = F{}, G value_filter = G{})
+  inline std::enable_if_t<std::is_base_of<writer, W>::value>
+  dynamic_object(W& dest, const T& source, F key_filter = F{}, G value_filter = G{})
   {
-    wire_write::dynamic_object(dest, source, source.size(), std::move(key_filter), std::move(value_filter));
+    wire_write::dynamic_object(dest, source, std::move(key_filter), std::move(value_filter));
   }
-  template<typename T, typename F, typename G>
+  template<typename W, typename T, typename F, typename G>
-  inline void write_bytes(writer& dest, as_object_<T, F, G> source)
+  inline void write_bytes(W& dest, as_object_<T, F, G> source)
   {
     wire::dynamic_object(dest, source.get_map(), std::move(source.key_filter), std::move(source.value_filter));
   }
 
-  template<typename... T>
+  template<typename W, typename... T>
-  inline void object(writer& dest, T... fields)
+  inline std::enable_if_t<std::is_base_of<writer, W>::value> object(W& dest, T... fields)
   {
     wire_write::object(dest, std::move(fields)...);
   }

tests/unit/rpc/admin.test.cpp

@@ -46,12 +46,13 @@ namespace
   expect<epee::byte_slice> call_endpoint(lws::db::storage disk, std::string json)
   {
     using request_type = typename T::request;
-    expect<request_type> req = wire::json::from_bytes<request_type>(std::move(json));
+    request_type req{};
-    if (!req)
+    const std::error_code error = wire::json::from_bytes(std::move(json), req);
-      return req.error();
+    if (error)
+      return error;
     wire::json_slice_writer out{};
-    MONERO_CHECK(T{}(out, std::move(disk), std::move(*req)));
+    MONERO_CHECK(T{}(out, std::move(disk), std::move(req)));
-    return out.take_bytes();
+    return epee::byte_slice{out.take_sink()};
   }
 }
 
@@ -138,7 +139,7 @@ LWS_CASE("rpc::admin")
     {
       wire::json_slice_writer out{};
       EXPECT(lws::rpc::webhook_list(out, db.clone()));
-      expect<epee::byte_slice> bytes = out.take_bytes();
+      expect<epee::byte_slice> bytes = epee::byte_slice{out.take_sink()};
       EXPECT(!bytes.has_error());
 
       {

tests/unit/wire/json/read.write.test.cpp

@@ -47,16 +47,15 @@ namespace
   {
     SETUP("Basic values with " + boost::core::demangle(typeid(T).name()) + " integers")
     {
-      const auto result =
+      basic_object<T> result{};
-        wire::json::from_bytes<basic_object<T>>(std::string{basic_json});
+      EXPECT(!wire::json::from_bytes(std::string{basic_json}, result));
-      EXPECT(result);
+      EXPECT(result.utf8 == basic_string);
-      EXPECT(result->utf8 == basic_string);
       {
         const std::vector<T> expected{0, 127};
-        EXPECT(result->vec == expected);
+        EXPECT(result.vec == expected);
       }
-      EXPECT(result->data == lws_test::blob_test1);
+      EXPECT(result.data == lws_test::blob_test1);
-      EXPECT(result->choice);
+      EXPECT(result.choice);
     }
   }
 
@@ -66,7 +65,8 @@ namespace
     SETUP("Basic values with " + boost::core::demangle(typeid(T).name()) + " integers")
     {
       const basic_object<T> val{basic_string, std::vector<T>{0, 127}, lws_test::blob_test1, true};
-      const auto result = wire::json::to_bytes(val);
+      epee::byte_slice result{};
+      EXPECT(!wire::json::to_bytes(result, val));
       EXPECT(boost::range::equal(result, std::string{basic_json}));
     }
   }
@@ -91,9 +91,11 @@ LWS_CASE("wire::json_reader")
     i64_limit::max() <= std::numeric_limits<std::uintmax_t>::max(),
     "expected int64_t::max <= uintmax_t::max"
   );
+  std::uint64_t one = 0;
+  std::int64_t two = 0;
   std::string big_number = std::to_string(std::uintmax_t(i64_limit::max()) + 1);
-  EXPECT(wire::json::from_bytes<std::uint64_t>(negative_number) == wire::error::schema::larger_integer);
+  EXPECT(wire::json::from_bytes(negative_number, one) == wire::error::schema::larger_integer);
-  EXPECT(wire::json::from_bytes<std::int64_t>(std::move(big_number)) == wire::error::schema::smaller_integer);
+  EXPECT(wire::json::from_bytes(std::move(big_number), two) == wire::error::schema::smaller_integer);
 }
 
 LWS_CASE("wire::json_writer")

tests/unit/wire/msgpack/read.write.test.cpp

@@ -86,16 +86,17 @@ namespace
   {
     SETUP("Basic (string keys) with " + boost::core::demangle(typeid(T).name()) + " integers")
     {
-      const auto result =
+      basic_object<T> result{};
-        wire::msgpack::from_bytes<basic_object<T>>(epee::byte_slice{{basic_msgpack}});
+      const std::error_code error =
-      EXPECT(result);
+        wire::msgpack::from_bytes(epee::byte_slice{{basic_msgpack}}, result);
-      EXPECT(result->utf8 == basic_string);
+      EXPECT(!error);
+      EXPECT(result.utf8 == basic_string);
       {
         const std::vector<T> expected{0, 127};
-        EXPECT(result->vec == expected);
+        EXPECT(result.vec == expected);
       }
-      EXPECT(result->data == lws_test::blob_test1);
+      EXPECT(result.data == lws_test::blob_test1);
-      EXPECT(result->choice);
+      EXPECT(result.choice);
     }
   }
 
@@ -104,16 +105,17 @@ namespace
   {
     SETUP("Advanced (integer keys) with " + boost::core::demangle(typeid(T).name()) + " integers")
     {
-      const auto result =
+      basic_object<T> result{};
-        wire::msgpack::from_bytes<basic_object<T>>(epee::byte_slice{{advanced_msgpack}});
+      const std::error_code error =
-      EXPECT(result);
+        wire::msgpack::from_bytes(epee::byte_slice{{advanced_msgpack}}, result);
-      EXPECT(result->utf8 == basic_string);
+      EXPECT(!error);
+      EXPECT(result.utf8 == basic_string);
       {
         const std::vector<T> expected{0, 127};
-        EXPECT(result->vec == expected);
+        EXPECT(result.vec == expected);
       }
-      EXPECT(result->data == lws_test::blob_test1);
+      EXPECT(result.data == lws_test::blob_test1);
-      EXPECT(result->choice);
+      EXPECT(result.choice);
     }
   }
 
@@ -123,7 +125,9 @@ namespace
     SETUP("Basic (string keys) with " + boost::core::demangle(typeid(T).name()) + " integers")
     {
       const basic_object<T> val{basic_string, std::vector<T>{0, 127}, lws_test::blob_test1, true};
-      const auto result = wire::msgpack::to_bytes(val);
+      epee::byte_slice result{};
+      const std::error_code error = wire::msgpack::to_bytes(result, val);
+      EXPECT(!error);
       EXPECT(boost::range::equal(result, epee::byte_slice{{basic_msgpack}}));
     }
   }
@@ -134,7 +138,12 @@ namespace
     SETUP("Advanced (integer keys) with " + boost::core::demangle(typeid(T).name()) + " integers")
     {
       const basic_object<T> val{basic_string, std::vector<T>{0, 127}, lws_test::blob_test1, true};
-      const auto result = wire_write::to_bytes(wire::msgpack_slice_writer{true}, val);
+      epee::byte_slice result{};
+      {
+        wire::msgpack_slice_writer out{true};
+        wire_write::bytes(out, val);
+        result = epee::byte_slice{out.take_sink()};
+      }
       EXPECT(boost::range::equal(result, epee::byte_slice{{advanced_msgpack}}));
     }
   }

tests/unit/wire/read.write.test.cpp

@@ -146,23 +146,23 @@ namespace
       verify_initial(lest_env, base);
 
       {
-        const expect<epee::byte_slice> bytes = T::to_bytes(base);
+        epee::byte_slice bytes{};
-        EXPECT(bytes);
+        EXPECT(!T::to_bytes(bytes, base));
 
-        const expect<complex> derived = T::template from_bytes<complex>(U{std::string{bytes->begin(), bytes->end()}});
+        complex derived{};
-        EXPECT(derived);
+        EXPECT(!T::template from_bytes<complex>(U{std::string{bytes.begin(), bytes.end()}}, derived));
-        verify_initial(lest_env, *derived);
+        verify_initial(lest_env, derived);
       }
 
       fill(base);
 
       {
-        const expect<epee::byte_slice> bytes = T::to_bytes(base);
+        epee::byte_slice bytes{};
-        EXPECT(bytes);
+        EXPECT(!T::to_bytes(bytes, base));
 
-        const expect<complex> derived = T::template from_bytes<complex>(U{std::string{bytes->begin(), bytes->end()}});
+        complex derived{};
-        EXPECT(derived);
+        EXPECT(!T::template from_bytes<complex>(U{std::string{bytes.begin(), bytes.end()}}, derived));
-        verify_filled(lest_env, *derived);
+        verify_filled(lest_env, derived);
       }
     }
   }
@@ -184,12 +184,15 @@ namespace
   template<typename T, typename U>
   expect<small> round_trip(lest::env& lest_env, std::int64_t value)
   {
-    expect<small> out = small{0};
+    small out{0};
     SETUP("Testing round-trip with " + std::to_string(value))
     {
-      const expect<epee::byte_slice> bytes = T::template to_bytes(big{value});
+      epee::byte_slice bytes{};
-      EXPECT(bytes);
+      EXPECT(!T::template to_bytes(bytes, big{value}));
-      out = T::template from_bytes<small>(U{std::string{bytes->begin(), bytes->end()}});
+      const std::error_code error =
+        T::template from_bytes(U{std::string{bytes.begin(), bytes.end()}}, out);
+      if (error)
+        return error;
     }
     return out;
   }
@@ -236,12 +239,12 @@ namespace
     verify_initial(lest_env, base);
     fill(base);
 
-    const expect<epee::byte_slice> bytes = T::to_bytes(base);
+    epee::byte_slice bytes{};
-    EXPECT(bytes);
+    EXPECT(!T::to_bytes(bytes, base));
 
-    const expect<simple> derived = T::template from_bytes<simple>(U{std::string{bytes->begin(), bytes->end()}});
+    simple derived{};
-    EXPECT(derived);
+    EXPECT(!T::template from_bytes<simple>(U{std::string{bytes.begin(), bytes.end()}}, derived));
-    EXPECT(derived->choice);
+    EXPECT(derived.choice);
   }
 }