Initial working base separated from top-level monero project

CMakeLists.txt

@@ -0,0 +1,172 @@
+# Copyright (c) 2020, The Monero Project
+#
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are
+# permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice, this list of
+#    conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice, this list
+#    of conditions and the following disclaimer in the documentation and/or other
+#    materials provided with the distribution.
+#
+# 3. Neither the name of the copyright holder nor the names of its contributors may be
+#    used to endorse or promote products derived from this software without specific
+#    prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+# THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+cmake_minimum_required(VERSION 3.1.0)
+project(monero-lws)
+
+set(MONERO_LIBRARIES
+  daemon_messages
+  serialization
+  lmdb_lib
+  net
+  cryptonote_core
+  cryptonote_basic
+  ringct
+  ringct_basic
+  multisig
+  hardforks
+  checkpoints
+  blockchain_db
+  common
+  lmdb
+  device
+  cncrypto
+  randomx
+  epee
+  easylogging
+  version
+  wallet-crypto
+)
+
+set(MONERO_OPTIONAL wallet-crypto)
+
+set(MONERO_SEARCH_PATHS
+  "/contrib/epee/src"
+  "/external/db_drivers/liblmdb"
+  "/external/easylogging++"
+  "/src"
+  "/src/crypto"
+  "/src/crypto/wallet"
+  "/src/lmdb"
+  "/src/ringct"
+  "/src/rpc"
+)
+
+
+#
+# Pull some information from monero build
+#
+
+# Needed due to "bug" in monero CMake - the `project` function is used twice!
+if (NOT MONERO_SOURCE_DIR)
+  message(FATAL_ERROR "The argument -DMONERO_SOURCE_DIR must specify a location of a monero source tree")
+endif()
+
+if (NOT MONERO_BUILD_DIR)
+  message(FATAL_ERROR "The argument -DMONERO_BUILD_DIR must specify a location of an existing monero build")
+endif()
+
+load_cache(${MONERO_BUILD_DIR} READ_WITH_PREFIX monero_
+  Boost_THREAD_LIBRARY_RELEASE
+  CMAKE_CXX_COMPILER
+  EXTRA_LIBRARIES
+  HIDAPI_LIBRARY
+  LMDB_INCLUDE
+  monero_SOURCE_DIR
+  OPENSSL_CRYPTO_LIBRARY
+  OPENSSL_SSL_LIBRARY
+  SODIUM_LIBRARY
+  UNBOUND_LIBRARIES
+  ZMQ_INCLUDE_PATH
+  ZMQ_LIB
+)
+
+if (NOT (monero_monero_SOURCE_DIR MATCHES "${MONERO_SOURCE_DIR}(/src/cryptonote_protocol)"))
+  message(FATAL_ERROR "Invalid Monero source dir - does not appear to match source used for build directory")
+endif()
+
+if (NOT (CMAKE_CXX_COMPILER STREQUAL monero_CMAKE_CXX_COMPILER))
+  message(FATAL_ERROR "Compiler for monero build differs from this project")
+endif()
+
+#
+# Dependencies specific to monero-lws
+#
+
+set(THREADS_PREFER_PTHREAD_FLAG ON)
+find_package(Threads REQUIRED)
+
+find_package(Boost 1.58 QUIET REQUIRED COMPONENTS chrono filesystem program_options regex serialization thread)
+
+if (NOT (Boost_THREAD_LIBRARY STREQUAL monero_Boost_THREAD_LIBRARY_RELEASE))
+  message(FATAL_ERROR "Boost libraries for monero build differs from this project")
+endif()
+
+foreach (LIB ${MONERO_LIBRARIES})
+  find_library(LIB_PATH NAMES "${LIB}" PATHS ${MONERO_BUILD_DIR} PATH_SUFFIXES "/src/${LIB}" "external/${LIB}" ${MONERO_SEARCH_PATHS} REQUIRED NO_DEFAULT_PATH)
+
+  list(FIND MONERO_OPTIONAL "${LIB}" LIB_OPTIONAL)
+  if (NOT LIB_PATH AND NOT LIB_OPTIONAL)
+    message(FATAL_ERROR "Unable to find required Monero library ${LIB}")
+  endif()
+
+  set(LIB_NAME "monero::${LIB}")
+  add_library(${LIB_NAME} STATIC IMPORTED)
+  set_target_properties(${LIB_NAME} PROPERTIES IMPORTED_LOCATION ${LIB_PATH})
+  
+  list(APPEND IMPORTED_MONERO_LIBRARIES "${LIB_NAME}")
+  unset(LIB_PATH CACHE)
+endforeach()
+
+add_library(monero::libraries INTERFACE IMPORTED)
+target_include_directories(monero::libraries SYSTEM
+  INTERFACE
+    ${Boost_INCLUDE_DIR}
+    "${MONERO_BUILD_DIR}/generated_include"
+    "${MONERO_SOURCE_DIR}/contrib/epee/include"
+    "${MONERO_SOURCE_DIR}/external/easylogging++"
+    "${MONERO_SOURCE_DIR}/external/rapidjson/include"
+    "${MONERO_SOURCE_DIR}/src"
+)
+target_link_libraries(monero::libraries
+  INTERFACE
+    ${CMAKE_DL_LIBS}
+    ${Boost_CHRONO_LIBRARY}
+    ${Boost_FILESYSTEM_LIBRARY}
+    ${Boost_REGEX_LIBRARY}
+    ${Boost_SERIALIZATION_LIBRARY}
+    ${Boost_THREAD_LIBRARY}
+    ${monero_HIDAPI_LIBRARY}
+    ${monero_OPENSSL_CRYPTO_LIBRARY}
+    ${monero_OPENSSL_SSL_LIBRARY}
+    ${monero_SODIUM_LIBRARY}
+    ${monero_UNBOUND_LIBRARIES}
+    ${IMPORTED_MONERO_LIBRARIES}
+)
+
+set(LMDB_INCLUDE "${monero_LMDB_INCLUDE}")
+set(LMDB_LIB_PATH "monero::lmdb")
+set(ZMQ_LIB "${monero_ZMQ_LIB}")
+set(ZMQ_INCLUDE_PATH "${monero_ZMQ_INCLUDE_PATH}")
+
+
+#
+# Build monero-lws code
+#
+
+add_subdirectory(src)

src/CMakeLists.txt

@@ -0,0 +1,77 @@
+# Copyright (c) 2018-2020, The Monero Project
+#
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are
+# permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice, this list of
+#    conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice, this list
+#    of conditions and the following disclaimer in the documentation and/or other
+#    materials provided with the distribution.
+#
+# 3. Neither the name of the copyright holder nor the names of its contributors may be
+#    used to endorse or promote products derived from this software without specific
+#    prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+# THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+include_directories(.)
+
+add_subdirectory(db)
+add_subdirectory(rpc)
+add_subdirectory(util)
+add_subdirectory(wire)
+
+# For both the server and admin utility.
+set(monero-lws-common_sources config.cpp error.cpp)
+set(monero-lws-common_headers config.h error.h fwd.h)
+
+add_library(monero-lws-common ${monero-lws-common_sources} ${monero-lws-common_headers})
+target_link_libraries(monero-lws-common monero::libraries)
+
+
+add_executable(monero-lws-daemon server_main.cpp rest_server.cpp scanner.cpp)
+target_include_directories(monero-lws-daemon PUBLIC ${ZMQ_INCLUDE_PATH})
+target_link_libraries(monero-lws-daemon
+  PRIVATE
+    monero::libraries
+    ${MONERO_lmdb}
+    monero-lws-common
+    monero-lws-db
+    monero-lws-rpc
+    monero-lws-util
+    monero-lws-wire-json
+    ${Boost_CHRONO_LIBRARY}
+    ${Boost_FILESYSTEM_LIBRARY}
+    ${Boost_PROGRAM_OPTIONS_LIBRARY}
+    ${Boost_THREAD_LIBRARY}
+    ${CMAKE_THREAD_LIBS_INIT}
+    ${EXTRA_LIBRARIES}
+    ${ZMQ_LIB}
+    Threads::Threads
+)
+
+add_executable(monero-lws-admin admin_main.cpp)
+target_link_libraries(monero-lws-admin
+  PRIVATE
+    monero::libraries
+    monero-lws-common
+    monero-lws-db
+    monero-lws-wire-json
+    ${Boost_PROGRAM_OPTIONS_LIBRARY}
+    Threads::Threads
+)
+
+install(TARGETS monero-lws-daemon DESTINATION bin)
+install(TARGETS monero-lws-admin DESTINATION bin)

src/admin_main.cpp

@@ -0,0 +1,423 @@
+// Copyright (c) 2018-2020, The Monero Project
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include <algorithm>
+#include <boost/optional/optional.hpp>
+#include <boost/program_options/options_description.hpp>
+#include <boost/program_options/parsers.hpp>
+#include <boost/program_options/variables_map.hpp>
+#include <cassert>
+#include <cstring>
+#include <iostream>
+#include <iterator>
+#include <stdexcept>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "common/command_line.h" // monero/src
+#include "common/expect.h"       // monero/src
+#include "config.h"
+#include "error.h"
+#include "db/storage.h"
+#include "db/string.h"
+#include "options.h"
+#include "misc_log_ex.h"  // monero/contrib/epee/include
+#include "span.h"         // monero/contrib/epee/include
+#include "string_tools.h" // monero/contrib/epee/include
+#include "wire/filters.h"
+#include "wire/json/write.h"
+
+namespace
+{
+  // Do not output "full" debug data provided by `db::data.h` header; truncate output
+  template<typename T>
+  struct truncated
+  {
+    T value;
+  };
+
+  void write_bytes(wire::json_writer& dest, const truncated<lws::db::account>& self)
+  {
+    wire::object(dest,
+      wire::field("address", lws::db::address_string(self.value.address)),
+      wire::field("scan_height", self.value.scan_height),
+      wire::field("access_time", self.value.access)
+    );
+  };
+
+  void write_bytes(wire::json_writer& dest, const truncated<lws::db::request_info>& self)
+  {
+    wire::object(dest,
+      wire::field("address", lws::db::address_string(self.value.address)),
+      wire::field("start_height", self.value.start_height)
+    );
+  }
+
+  template<typename V>
+  void write_bytes(wire::json_writer& dest, const truncated<boost::iterator_range<lmdb::value_iterator<V>>> self)
+  {
+    const auto truncate = [] (V src) { return truncated<V>{std::move(src)}; };
+    wire::array(dest, std::move(self.value), truncate);
+  }
+
+  template<typename K, typename V>
+  void stream_json_object(std::ostream& dest, boost::iterator_range<lmdb::key_iterator<K, V>> self)
+  {
+    using value_range = boost::iterator_range<lmdb::value_iterator<V>>;
+    const auto truncate = [] (value_range src) -> truncated<value_range>
+    {
+     return {std::move(src)};
+    };
+
+    wire::json_stream_writer json{dest};
+    wire::dynamic_object(json, std::move(self), wire::enum_as_string, truncate);
+    json.finish();
+  }
+
+  void write_json_addresses(std::ostream& dest, epee::span<const lws::db::account_address> self)
+  {
+    // writes an array of monero base58 address strings
+    wire::json_stream_writer stream{dest};
+    wire::object(stream, wire::field("updated", wire::as_array(self, lws::db::address_string)));
+    stream.finish();
+  }
+
+  struct options : lws::options
+  {
+    const command_line::arg_descriptor<bool> show_sensitive;
+    const command_line::arg_descriptor<std::string> command;
+    const command_line::arg_descriptor<std::vector<std::string>> arguments;
+
+    options()
+      : lws::options()
+      , show_sensitive{"show-sensitive", "Show view keys", false}
+      , command{"command", "Admin command to execute", ""}
+      , arguments{"arguments", "Arguments to command"}
+    {}
+
+    void prepare(boost::program_options::options_description& description) const
+    {
+      lws::options::prepare(description);
+      command_line::add_arg(description, show_sensitive);
+      command_line::add_arg(description, command);
+      command_line::add_arg(description, arguments);
+    }
+  };
+
+  struct program
+  {
+    lws::db::storage disk;
+    std::vector<std::string> arguments;
+    bool show_sensitive;
+  };
+
+  crypto::secret_key get_key(std::string const& hex)
+  {
+    crypto::secret_key out{};
+    if (!epee::string_tools::hex_to_pod(hex, out))
+      MONERO_THROW(lws::error::bad_view_key, "View key has invalid hex");
+    return out;
+  }
+
+  std::vector<lws::db::account_address> get_addresses(epee::span<const std::string> arguments)
+  {
+    // first entry is currently always some other option
+    assert(!arguments.empty());
+    arguments.remove_prefix(1);
+
+    std::vector<lws::db::account_address> addresses{};
+    for (std::string const& address : arguments)
+      addresses.push_back(lws::db::address_string(address).value());
+    return addresses;
+  }
+
+  void accept_requests(program prog, std::ostream& out)
+  {
+    if (prog.arguments.size() < 2)
+      throw std::runtime_error{"accept_requests requires 2 or more arguments"};
+
+    const lws::db::request req =
+      MONERO_UNWRAP(lws::db::request_from_string(prog.arguments[0]));
+    std::vector<lws::db::account_address> addresses =
+      get_addresses(epee::to_span(prog.arguments));
+
+    const std::vector<lws::db::account_address> updated =
+      prog.disk.accept_requests(req, epee::to_span(addresses)).value();
+
+    write_json_addresses(out, epee::to_span(updated));
+  }
+
+  void add_account(program prog, std::ostream& out)
+  {
+    if (prog.arguments.size() != 2)
+      throw std::runtime_error{"add_account needs exactly two arguments"};
+
+    const lws::db::account_address address[1] = {
+      lws::db::address_string(prog.arguments[0]).value()
+    };
+    const crypto::secret_key key{get_key(prog.arguments[1])};
+
+    MONERO_UNWRAP(prog.disk.add_account(address[0], key));
+    write_json_addresses(out, address);
+  }
+
+  void debug_database(program prog, std::ostream& out)
+  {
+    if (!prog.arguments.empty())
+      throw std::runtime_error{"debug_database takes zero arguments"};
+
+    auto reader = prog.disk.start_read().value();
+    reader.json_debug(out, prog.show_sensitive);
+  }
+
+  void list_accounts(program prog, std::ostream& out)
+  {
+    if (!prog.arguments.empty())
+      throw std::runtime_error{"list_accounts takes zero arguments"};
+
+    auto reader = prog.disk.start_read().value();
+    auto stream = reader.get_accounts().value();
+    stream_json_object(out, stream.make_range());
+  }
+
+  void list_requests(program prog, std::ostream& out)
+  {
+    if (!prog.arguments.empty())
+      throw std::runtime_error{"list_requests takes zero arguments"};
+
+    auto reader = prog.disk.start_read().value();
+    auto stream = reader.get_requests().value();
+    stream_json_object(out, stream.make_range());
+  }
+
+  void modify_account(program prog, std::ostream& out)
+  {
+    if (prog.arguments.size() < 2)
+      throw std::runtime_error{"modify_account_status requires 2 or more arguments"};
+
+    const lws::db::account_status status =
+      lws::db::account_status_from_string(prog.arguments[0]).value();
+    std::vector<lws::db::account_address> addresses =
+      get_addresses(epee::to_span(prog.arguments));
+
+    const std::vector<lws::db::account_address> updated =
+      prog.disk.change_status(status, epee::to_span(addresses)).value();
+
+    write_json_addresses(out, epee::to_span(updated));
+  }
+
+  void reject_requests(program prog, std::ostream& out)
+  {
+    if (prog.arguments.size() < 2)
+      MONERO_THROW(common_error::kInvalidArgument, "reject_requests requires 2 or more arguments");
+
+    const lws::db::request req =
+      lws::db::request_from_string(prog.arguments[0]).value();
+    std::vector<lws::db::account_address> addresses =
+      get_addresses(epee::to_span(prog.arguments));
+
+    MONERO_UNWRAP(prog.disk.reject_requests(req, epee::to_span(addresses)));
+  }
+
+  void rescan(program prog, std::ostream& out)
+  {
+    if (prog.arguments.size() < 2)
+      throw std::runtime_error{"rescan requires 2 or more arguments"};
+
+    const auto height = lws::db::block_id(std::stoull(prog.arguments[0]));
+    const std::vector<lws::db::account_address> addresses =
+      get_addresses(epee::to_span(prog.arguments));
+
+    const std::vector<lws::db::account_address> updated =
+      prog.disk.rescan(height, epee::to_span(addresses)).value();
+
+    write_json_addresses(out, epee::to_span(updated));
+  }
+
+  void rollback(program prog, std::ostream& out)
+  {
+    if (prog.arguments.size() != 1)
+      throw std::runtime_error{"rollback requires 1 argument"};
+
+    const auto height = lws::db::block_id(std::stoull(prog.arguments[0]));
+    MONERO_UNWRAP(prog.disk.rollback(height));
+
+    wire::json_stream_writer json{out};
+    wire::object(json, wire::field("new_height", height));
+    json.finish();
+  }
+
+  struct command
+  {
+    char const* const name;
+    void (*const handler)(program, std::ostream&);
+    char const* const parameters;
+    };
+
+  static constexpr const command commands[] =
+  {
+    {"accept_requests",       &accept_requests, "<\"create\"|\"import\"> <base58 address> [base 58 address]..."},
+    {"add_account",           &add_account,     "<base58 address> <view key hex>"},
+    {"debug_database",        &debug_database,  ""},
+    {"list_accounts",         &list_accounts,   ""},
+    {"list_requests",         &list_requests,   ""},
+    {"modify_account_status", &modify_account,  "<\"active\"|\"inactive\"|\"hidden\"> <base58 address> [base 58 address]..."},
+    {"reject_requests",       &reject_requests, "<\"create\"|\"import\"> <base58 address> [base 58 address]..."},
+    {"rescan",                &rescan,          "<height> <base58 address> [base 58 address]..."},
+    {"rollback",              &rollback,        "<height>"}
+  };
+
+  void print_help(std::ostream& out)
+  {
+    boost::program_options::options_description description{"Options"};
+    options{}.prepare(description);
+
+    out << "Usage: [options] [command] [arguments]" << std::endl;
+    out << description << std::endl;
+    out << "Commands:" << std::endl;
+    for (command cmd : commands)
+    {
+      out << "  " << cmd.name << "\t\t" << cmd.parameters << std::endl;
+    }
+  }
+
+  boost::optional<std::pair<std::string, program>> get_program(int argc, char** argv)
+  {
+    namespace po = boost::program_options;
+
+    const options opts{};
+    po::variables_map args{};
+    {
+      po::options_description description{"Options"};
+      opts.prepare(description);
+
+      po::positional_options_description positional{};
+      positional.add(opts.command.name, 1);
+      positional.add(opts.arguments.name, -1);
+
+      po::store(
+        po::command_line_parser(argc, argv)
+        .options(description).positional(positional).run()
+        , args
+      );
+      po::notify(args);
+    }
+
+    if (command_line::get_arg(args, command_line::arg_help))
+    {
+      print_help(std::cout);
+      return boost::none;
+    }
+
+    opts.set_network(args); // do this first, sets global variable :/
+
+    program prog{
+      lws::db::storage::open(command_line::get_arg(args, opts.db_path).c_str(), 0)
+    };
+
+    prog.show_sensitive = command_line::get_arg(args, opts.show_sensitive);
+    auto cmd = args[opts.command.name];
+    if (cmd.empty())
+      throw std::runtime_error{"No command given"};
+
+    prog.arguments = command_line::get_arg(args, opts.arguments);
+    return {{cmd.as<std::string>(), std::move(prog)}};
+  }
+
+  void run(boost::string_ref name, program prog, std::ostream& out)
+  {
+    struct by_name
+    {
+      bool operator()(command const& left, command const& right) const noexcept
+      {
+        assert(left.name && right.name);
+        return std::strcmp(left.name, right.name) < 0;
+      }
+      bool operator()(boost::string_ref left, command const& right) const noexcept
+      {
+        assert(right.name);
+        return left < right.name;
+      }
+      bool operator()(command const& left, boost::string_ref right) const noexcept
+      {
+        assert(left.name);
+        return left.name < right;
+      }
+    };
+
+    assert(std::is_sorted(std::begin(commands), std::end(commands), by_name{}));
+    const auto found = std::lower_bound(
+      std::begin(commands), std::end(commands), name, by_name{}
+    );
+    if (found == std::end(commands) || found->name != name)
+      throw std::runtime_error{"No such command"};
+
+    assert(found->handler != nullptr);
+    found->handler(std::move(prog), out);
+
+    if (out.bad())
+      MONERO_THROW(std::io_errc::stream, "Writing to stdout failed");
+
+    out << std::endl;
+  }
+} // anonymous
+
+int main (int argc, char** argv)
+{
+  try
+  {
+    mlog_configure("", false, 0, 0); // disable logging
+
+    boost::optional<std::pair<std::string, program>> prog;
+
+    try
+    {
+      prog = get_program(argc, argv);
+    }
+    catch (std::exception const& e)
+    {
+      std::cerr << e.what() << std::endl << std::endl;
+      print_help(std::cerr);
+      return EXIT_FAILURE;
+    }
+
+    if (prog)
+      run(prog->first, std::move(prog->second), std::cout);
+  }
+  catch (std::exception const& e)
+  {
+    std::cerr << e.what() << std::endl;
+    return EXIT_FAILURE;
+  }
+  catch (...)
+  {
+    std::cerr << "Unknown exception" << std::endl;
+    return EXIT_FAILURE;
+  }
+  return EXIT_SUCCESS;
+}

src/config.cpp

@@ -0,0 +1,9 @@
+#include "config.h"
+
+namespace lws
+{
+namespace config
+{
+  cryptonote::network_type network = cryptonote::MAINNET;
+}
+}

src/config.h

@@ -0,0 +1,11 @@
+#pragma once
+
+#include "cryptonote_config.h"
+
+namespace lws
+{
+namespace config
+{
+  extern cryptonote::network_type network;
+}
+}

src/db/CMakeLists.txt

@@ -0,0 +1,34 @@
+# Copyright (c) 2018-2020, The Monero Project
+#
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are
+# permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice, this list of
+#    conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice, this list
+#    of conditions and the following disclaimer in the documentation and/or other
+#    materials provided with the distribution.
+#
+# 3. Neither the name of the copyright holder nor the names of its contributors may be
+#    used to endorse or promote products derived from this software without specific
+#    prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+# THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+set(monero-lws-db_sources account.cpp data.cpp storage.cpp string.cpp)
+set(monero-lws-db_headers account.h data.h fwd.h storage.h string.h)
+
+add_library(monero-lws-db ${monero-lws-db_sources} ${monero-lws-db_headers})
+target_include_directories(monero-lws-db PUBLIC "${LMDB_INCLUDE}")
+target_link_libraries(monero-lws-db monero::libraries ${LMDB_LIB_PATH})

src/db/account.cpp

@@ -0,0 +1,179 @@
+// Copyright (c) 2018, The Monero Project
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#include "account.h"
+
+#include <algorithm>
+#include <cstring>
+
+#include "common/error.h"
+#include "common/expect.h"
+#include "db/data.h"
+#include "db/string.h"
+
+namespace lws
+{
+  namespace
+  {
+    // update if `crypto::public_key` gets `operator<`
+    struct sort_pubs
+    {
+      bool operator()(crypto::public_key const& lhs, crypto::public_key const& rhs) const noexcept
+      {
+        return std::memcmp(std::addressof(lhs), std::addressof(rhs), sizeof(lhs)) < 0;
+      }
+    };
+  }
+
+  struct account::internal
+  {
+    explicit internal(db::account const& source)
+      : address(db::address_string(source.address)), id(source.id), pubs(source.address), view_key()
+    {
+      using inner_type =
+        std::remove_reference<decltype(tools::unwrap(view_key))>::type;
+
+      static_assert(std::is_standard_layout<db::view_key>(), "need standard layout source");
+      static_assert(std::is_pod<inner_type>(), "need pod target");
+      static_assert(sizeof(view_key) == sizeof(source.key), "different size keys");
+      std::memcpy(
+        std::addressof(tools::unwrap(view_key)),
+        std::addressof(source.key),
+        sizeof(source.key)
+      );
+    }
+
+    std::string address;
+    db::account_id id;
+    db::account_address pubs;
+    crypto::secret_key view_key;
+  };
+
+  account::account(std::shared_ptr<const internal> immutable, db::block_id height, std::vector<db::output_id> spendable, std::vector<crypto::public_key> pubs) noexcept
+    : immutable_(std::move(immutable))
+    , spendable_(std::move(spendable))
+    , pubs_(std::move(pubs))
+      , spends_()
+    , outputs_()
+    , height_(height)
+  {}
+
+  void account::null_check() const
+  {
+    if (!immutable_)
+      MONERO_THROW(::common_error::kInvalidArgument, "using moved from account");
+  }
+
+  account::account(db::account const& source, std::vector<db::output_id> spendable, std::vector<crypto::public_key> pubs)
+    : account(std::make_shared<internal>(source), source.scan_height, std::move(spendable), std::move(pubs))
+  {
+    std::sort(spendable_.begin(), spendable_.end());
+    std::sort(pubs_.begin(), pubs_.end(), sort_pubs{});
+  }
+
+  account::~account() noexcept
+  {}
+
+  account account::clone() const
+  {
+    account result{immutable_, height_, spendable_, pubs_};
+    result.outputs_ = outputs_;
+    result.spends_ = spends_;
+    return result;
+  }
+
+  void account::updated(db::block_id new_height) noexcept
+  {
+    height_ = new_height;
+    spends_.clear();
+    spends_.shrink_to_fit();
+    outputs_.clear();
+    outputs_.shrink_to_fit();
+  }
+
+  db::account_id account::id() const noexcept
+  {
+    if (immutable_)
+      return immutable_->id;
+    return db::account_id::invalid;
+  }
+
+  std::string const& account::address() const
+  {
+    null_check();
+    return immutable_->address;
+  }
+
+  db::account_address const& account::db_address() const
+  {
+    null_check();
+    return immutable_->pubs;
+  }
+
+  crypto::public_key const& account::view_public() const
+  {
+    null_check();
+    return immutable_->pubs.view_public;
+  }
+
+  crypto::public_key const& account::spend_public() const
+  {
+    null_check();
+    return immutable_->pubs.spend_public;
+  }
+
+  crypto::secret_key const& account::view_key() const
+  {
+    null_check();
+    return immutable_->view_key;
+  }
+
+  bool account::has_spendable(db::output_id const& id) const noexcept
+  {
+    return std::binary_search(spendable_.begin(), spendable_.end(), id);
+  }
+
+  bool account::add_out(db::output const& out)
+  {
+    auto existing_pub = std::lower_bound(pubs_.begin(), pubs_.end(), out.pub, sort_pubs{});
+    if (existing_pub != pubs_.end() && *existing_pub == out.pub)
+      return false;
+
+    pubs_.insert(existing_pub, out.pub);
+    spendable_.insert(
+      std::lower_bound(spendable_.begin(), spendable_.end(), out.spend_meta.id),
+      out.spend_meta.id
+    );
+    outputs_.push_back(out);
+    return true;
+  }
+
+  void account::add_spend(db::spend const& spend)
+  {
+    spends_.push_back(spend);
+  }
+} // lws
+

src/db/account.h

@@ -0,0 +1,114 @@
+// Copyright (c) 2018, The Monero Project
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#pragma once
+
+#include <cstdint>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "crypto/crypto.h"
+#include "fwd.h"
+#include "db/fwd.h"
+
+namespace lws
+{
+  //! Tracks a subset of DB account info for scanning/updating.
+  class account
+  {
+    struct internal;
+
+    std::shared_ptr<const internal> immutable_;
+    std::vector<db::output_id> spendable_;
+    std::vector<crypto::public_key> pubs_;
+    std::vector<db::spend> spends_;
+    std::vector<db::output> outputs_;
+    db::block_id height_;
+
+    explicit account(std::shared_ptr<const internal> immutable, db::block_id height, std::vector<db::output_id> spendable, std::vector<crypto::public_key> pubs) noexcept;
+    void null_check() const;
+
+  public:
+
+    //! Construct an account from `source` and current `spendable` outputs.
+    explicit account(db::account const& source, std::vector<db::output_id> spendable, std::vector<crypto::public_key> pubs);
+
+    /*!
+      \return False if this is a "moved-from" account (i.e. the internal memory
+        has been moved to another object).
+    */
+    explicit operator bool() const noexcept { return immutable_ != nullptr; }
+
+    account(const account&) = delete;
+    account(account&&) = default;
+    ~account() noexcept;
+    account& operator=(const account&) = delete;
+    account& operator=(account&&) = default;
+
+    //! \return A copy of `this`.
+    account clone() const;
+
+    //! \return A copy of `this` with a new height and `outputs().empty()`.
+    void updated(db::block_id new_height) noexcept;
+
+    //! \return Unique ID from the account database, possibly `db::account_id::kInvalid`.
+    db::account_id id() const noexcept;
+
+    //! \return Monero base58 string for account.
+    std::string const& address() const;
+
+    //! \return Object used for lookup in LMDB.
+    db::account_address const& db_address() const;
+
+    //! \return Extracted view public key from `address()`
+    crypto::public_key const& view_public() const;
+
+    //! \return Extracted spend public key from `address()`.
+    crypto::public_key const& spend_public() const;
+
+    //! \return Secret view key for the account.
+    crypto::secret_key const& view_key() const;
+
+    //! \return Current scan height of `this`.
+    db::block_id scan_height() const noexcept { return height_; }
+
+    //! \return True iff `id` is spendable by `this`.
+    bool has_spendable(db::output_id const& id) const noexcept;
+
+    //! \return Outputs matched during the latest scan.
+    std::vector<db::output> const& outputs() const noexcept { return outputs_; }
+
+    //! \return Spends matched during the latest scan.
+    std::vector<db::spend> const& spends() const noexcept { return spends_; }
+
+    //! Track a newly received `out`, \return `false` if `out.pub` is duplicated.
+    bool add_out(db::output const& out);
+
+    //! Track a possible `spend`.
+    void add_spend(db::spend const& spend);
+  };
+} // lws

src/db/data.cpp

@@ -0,0 +1,225 @@
+// Copyright (c) 2018, The Monero Project
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#include "data.h"
+
+#include <cstring>
+#include <memory>
+
+#include "wire/crypto.h"
+#include "wire.h"
+
+namespace lws
+{
+namespace db
+{
+  namespace
+  {
+    template<typename F, typename T>
+    void map_output_id(F& format, T& self)
+    {
+      wire::object(format, WIRE_FIELD(high), WIRE_FIELD(low));
+    }
+  }
+  WIRE_DEFINE_OBJECT(output_id, map_output_id);
+
+  namespace
+  {
+    constexpr const char* map_account_status[] = {"active", "inactive", "hidden"};
+    constexpr const char* map_request[] = {"create", "import"};
+  }
+  WIRE_DEFINE_ENUM(account_status, map_account_status);
+  WIRE_DEFINE_ENUM(request, map_request);
+
+  namespace
+  {
+    template<typename F, typename T>
+    void map_account_address(F& format, T& self)
+    {
+      wire::object(format, WIRE_FIELD(spend_public), WIRE_FIELD(view_public));
+    }
+  }
+  WIRE_DEFINE_OBJECT(account_address, map_account_address);
+
+  void write_bytes(wire::writer& dest, const account& self, const bool show_key)
+  {
+    view_key const* const key =
+      show_key ? std::addressof(self.key) : nullptr;
+    const bool admin = (self.flags & admin_account);
+    const bool generated_locally = (self.flags & account_generated_locally);
+    
+    wire::object(dest,
+      WIRE_FIELD(id),
+      wire::field("access_time", self.access),
+      WIRE_FIELD(address),
+      wire::optional_field("view_key", key),
+      WIRE_FIELD(scan_height),
+      WIRE_FIELD(start_height),
+      wire::field("creation_time", self.creation),
+      wire::field("admin", admin),
+      wire::field("generated_locally", generated_locally)
+    );
+  }
+
+  namespace
+  {
+    template<typename F, typename T>
+    void map_block_info(F& format, T& self)
+    {
+      wire::object(format, WIRE_FIELD(id), WIRE_FIELD(hash));
+    }
+  }
+  WIRE_DEFINE_OBJECT(block_info, map_block_info);
+
+  namespace
+  {
+    template<typename F, typename T>
+    void map_transaction_link(F& format, T& self)
+    {
+      wire::object(format, WIRE_FIELD(height), WIRE_FIELD(tx_hash));
+    }
+  }
+  WIRE_DEFINE_OBJECT(transaction_link, map_transaction_link);
+  
+  void write_bytes(wire::writer& dest, const output& self)
+  {
+    const std::pair<db::extra, std::uint8_t> unpacked =
+      db::unpack(self.extra);
+    
+    const bool coinbase = (unpacked.first & lws::db::coinbase_output);
+    const bool rct = (unpacked.first & lws::db::ringct_output);
+    
+    const auto rct_mask = rct ? std::addressof(self.ringct_mask) : nullptr;
+    
+    epee::span<const std::uint8_t> payment_bytes{};
+    if (unpacked.second == 32)
+      payment_bytes = epee::as_byte_span(self.payment_id.long_);
+    else if (unpacked.second == 8)
+      payment_bytes = epee::as_byte_span(self.payment_id.short_);
+
+    const auto payment_id = payment_bytes.empty() ?
+      nullptr : std::addressof(payment_bytes);
+
+    wire::object(dest,
+      wire::field("id", std::cref(self.spend_meta.id)),
+      wire::field("block", self.link.height),
+      wire::field("index", self.spend_meta.index),
+      wire::field("amount", self.spend_meta.amount),
+      wire::field("timestamp", self.timestamp),
+      wire::field("tx_hash", std::cref(self.link.tx_hash)),
+      wire::field("tx_prefix_hash", std::cref(self.tx_prefix_hash)),
+      wire::field("tx_public", std::cref(self.spend_meta.tx_public)),
+      wire::optional_field("rct_mask", rct_mask),
+      wire::optional_field("payment_id", payment_id),
+      wire::field("unlock_time", self.unlock_time),
+      wire::field("mixin_count", self.spend_meta.mixin_count),
+      wire::field("coinbase", coinbase)
+    );
+  }
+
+  namespace
+  {
+    template<typename F, typename T1, typename T2>
+    void map_spend(F& format, T1& self, T2& payment_id)
+    {
+      wire::object(format,
+        wire::field("height", self.link.height),
+        wire::field("tx_hash", std::ref(self.link.tx_hash)),
+	WIRE_FIELD(image),
+	WIRE_FIELD(source),
+	WIRE_FIELD(timestamp),
+	WIRE_FIELD(unlock_time),
+	WIRE_FIELD(mixin_count),
+	wire::optional_field("payment_id", payment_id)
+      );
+    }
+  }
+  void read_bytes(wire::reader& source, spend& dest)
+  {
+    boost::optional<crypto::hash> payment_id;
+    map_spend(source, dest, payment_id);
+    
+    if (payment_id)
+    {
+      dest.length = sizeof(dest.payment_id);
+      dest.payment_id = std::move(*payment_id);
+    }
+    else
+      dest.length = 0;
+  }
+  void write_bytes(wire::writer& dest, const spend& source)
+  {
+    crypto::hash const* const payment_id =
+      (source.length == sizeof(source.payment_id) ?
+        std::addressof(source.payment_id) : nullptr);
+    return map_spend(dest, source, payment_id);
+  }
+
+  namespace
+  {
+    template<typename F, typename T>
+    void map_key_image(F& format, T& self)
+    {
+      wire::object(format,
+        wire::field("key_image", std::ref(self.value)),
+	wire::field("tx_hash", std::ref(self.link.tx_hash)),
+	wire::field("height", self.link.height)
+      );
+    }
+  }
+  WIRE_DEFINE_OBJECT(key_image, map_key_image);
+
+  void write_bytes(wire::writer& dest, const request_info& self, const bool show_key)
+  {
+    db::view_key const* const key =
+      show_key ? std::addressof(self.key) : nullptr;
+    const bool generated = (self.creation_flags & lws::db::account_generated_locally);
+      
+    wire::object(dest,
+      WIRE_FIELD(address),
+      wire::optional_field("view_key", key),
+      WIRE_FIELD(start_height),
+      wire::field("generated_locally", generated)
+    );
+  }
+
+  /*! TODO consider making an `operator<` for `crypto::tx_hash`. Not known to be
+    needed elsewhere yet. */
+
+  bool operator<(transaction_link const& left, transaction_link const& right) noexcept
+  {
+    return left.height == right.height ?
+      std::memcmp(std::addressof(left.tx_hash), std::addressof(right.tx_hash), sizeof(left.tx_hash)) < 0 :
+      left.height < right.height;
+  }
+  bool operator<=(transaction_link const& left, transaction_link const& right) noexcept
+  {
+    return right.height == left.height ?
+      std::memcmp(std::addressof(left.tx_hash), std::addressof(right.tx_hash), sizeof(left.tx_hash)) <= 0 :
+      left.height < right.height;
+  }
+} // db
+} // lws

src/db/data.h

@@ -0,0 +1,275 @@
+// Copyright (c) 2018-2020, The Monero Project
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#pragma once
+
+#include <cassert>
+#include <cstdint>
+#include <iosfwd>
+#include <utility>
+
+#include "crypto/crypto.h"
+#include "lmdb/util.h"
+#include "ringct/rctTypes.h" //! \TODO brings in lots of includes, try to remove
+#include "wire/fwd.h"
+#include "wire/traits.h"
+
+namespace lws
+{
+namespace db
+{
+  /*
+    Enum classes are used because they generate identical code to native integer
+    types, but are not implicitly convertible to each other or any integer types.
+    They also have comparison but not arithmetic operators defined.
+  */
+
+  //! References an account stored in the database, faster than by address
+  enum class account_id : std::uint32_t
+  {
+    invalid = std::uint32_t(-1) //!< Always represents _not an_ account id.
+  };
+  WIRE_AS_INTEGER(account_id);
+
+  //! Number of seconds since UNIX epoch.
+  enum class account_time : std::uint32_t {};
+  WIRE_AS_INTEGER(account_time);
+
+  //! References a block height
+  enum class block_id : std::uint64_t {};
+  WIRE_AS_INTEGER(block_id);
+
+  //! References a global output number, as determined by the public chain
+  struct output_id
+  {
+    std::uint64_t high; //!< Amount on public chain; rct outputs are `0`
+    std::uint64_t low;  //!< Offset within `amount` on the public chain
+  };
+  WIRE_DECLARE_OBJECT(output_id);
+
+  enum class account_status : std::uint8_t
+  {
+    active = 0, //!< Actively being scanned and reported by API
+    inactive,   //!< Not being scanned, but still reported by API
+    hidden      //!< Not being scanned or reported by API
+  };
+  WIRE_DECLARE_ENUM(account_status);
+
+  enum account_flags : std::uint8_t
+  {
+    default_account = 0,
+    admin_account   = 1,          //!< Not currently used, for future extensions
+    account_generated_locally = 2 //!< Flag sent by client on initial login request
+  };
+
+  enum class request : std::uint8_t
+  {
+    create = 0, //!< Add a new account
+    import_scan //!< Set account start and scan height to zero.
+  };
+  WIRE_DECLARE_ENUM(request);
+
+  /*!
+    DB does not use `crypto::secret_key` because it is not POD (UB to copy over
+    entire struct). LMDB is keeping a copy in process memory anyway (row
+    encryption not currently used). The roadmap recommends process isolation
+    per-connection by default as a defense against viewkey leaks due to bug. */
+
+  struct view_key : crypto::ec_scalar {};
+  // wire::is_blob trait below
+
+  //! The public keys of a monero address
+  struct account_address
+  {
+    crypto::public_key spend_public; //!< Must be first for LMDB optimizations.
+    crypto::public_key view_public;
+  };
+  static_assert(sizeof(account_address) == 64, "padding in account_address");
+  WIRE_DECLARE_OBJECT(account_address);
+
+  struct account
+  {
+    account_id id;          //!< Must be first for LMDB optimizations
+    account_time access;    //!< Last time `get_address_info` was called.
+    account_address address;
+    view_key key;           //!< Doubles as authorization handle for REST API.
+    block_id scan_height;   //!< Last block scanned; check-ins are always by block
+    block_id start_height;  //!< Account started scanning at this block height
+    account_time creation;  //!< Time account first appeared in database.
+    account_flags flags;    //!< Additional account info bitmask.
+    char reserved[3];
+  };
+  static_assert(sizeof(account) == (4 * 2) + 64 + 32 + (8 * 2) + (4 * 2), "padding in account");
+  void write_bytes(wire::writer&, const account&, bool show_key = false);
+
+  struct block_info
+  {
+    block_id id;      //!< Must be first for LMDB optimizations
+    crypto::hash hash;
+  };
+  static_assert(sizeof(block_info) == 8 + 32, "padding in block_info");
+  WIRE_DECLARE_OBJECT(block_info);
+
+  //! `output`s and `spend`s are sorted by these fields to make merging easier.
+  struct transaction_link
+  {
+    block_id height;      //!< Block height containing transaction
+    crypto::hash tx_hash; //!< Hash of the transaction
+  };
+
+  //! Additional flags stored in `output`s.
+  enum extra : std::uint8_t
+  {
+    coinbase_output = 1,
+    ringct_output   = 2
+  };
+
+  //! Packed information stored in `output`s.
+  enum class extra_and_length : std::uint8_t {};
+
+  //! \return `val` and `length` packed into a single byte.
+  inline extra_and_length pack(extra val, std::uint8_t length) noexcept
+  {
+    assert(length <= 32);
+    return extra_and_length((std::uint8_t(val) << 6) | (length & 0x3f));
+  }
+
+  //! \return `extra` and length unpacked from a single byte.
+  inline std::pair<extra, std::uint8_t> unpack(extra_and_length val) noexcept
+  {
+    const std::uint8_t real_val = std::uint8_t(val);
+    return {extra(real_val >> 6), std::uint8_t(real_val & 0x3f)};
+  }
+
+  //! Information for an output that has been received by an `account`.
+  struct output
+  {
+    transaction_link link;        //! Orders and links `output` to `spend`s.
+
+    //! Data that a linked `spend` needs in some REST endpoints.
+    struct spend_meta_
+    {
+      output_id id;             //!< Unique id for output within monero
+      // `link` and `id` must be in this order for LMDB optimizations
+      std::uint64_t amount;
+      std::uint32_t mixin_count;//!< Ring-size of TX
+      std::uint32_t index;      //!< Offset within a tx
+      crypto::public_key tx_public;
+    } spend_meta;
+
+    std::uint64_t timestamp;
+    std::uint64_t unlock_time; //!< Not always a timestamp; mirrors chain value.
+    crypto::hash tx_prefix_hash;
+    crypto::public_key pub;    //!< One-time spendable public key.
+    rct::key ringct_mask;      //!< Unencrypted CT mask
+    char reserved[7];
+    extra_and_length extra;    //!< Extra info + length of payment id
+    union payment_id_
+    {
+      crypto::hash8 short_;  //!< Decrypted short payment id
+      crypto::hash long_;    //!< Long version of payment id (always decrypted)
+    } payment_id;
+  };
+  static_assert(
+    sizeof(output) == 8 + 32 + (8 * 3) + (4 * 2) + 32 + (8 * 2) + (32 * 3) + 7 + 1 + 32,
+    "padding in output"
+  );
+  void write_bytes(wire::writer&, const output&);
+
+  //! Information about a possible spend of a received `output`.
+  struct spend
+  {
+    transaction_link link;    //!< Orders and links `spend` to `output`.
+    crypto::key_image image;  //!< Unique ID for the spend
+    // `link` and `image` must in this order for LMDB optimizations
+    output_id source;         //!< The output being spent
+    std::uint64_t timestamp;  //!< Timestamp of spend
+    std::uint64_t unlock_time;//!< Unlock time of spend
+    std::uint32_t mixin_count;//!< Ring-size of TX output
+    char reserved[3];
+    std::uint8_t length;      //!< Length of `payment_id` field (0..32).
+    crypto::hash payment_id;  //!< Unencrypted only, can't decrypt spend
+  };
+  static_assert(sizeof(spend) == 8 + 32 * 2 + 8 * 4 + 4 + 3 + 1 + 32, "padding in spend");
+  WIRE_DECLARE_OBJECT(spend);
+
+  //! Key image and info needed to retrieve primary `spend` data.
+  struct key_image
+  {
+    crypto::key_image value; //!< Actual key image value
+    // The above field needs to be first for LMDB optimizations
+    transaction_link link;   //!< Link to `spend` and `output`.
+  };
+  WIRE_DECLARE_OBJECT(key_image);
+
+  struct request_info
+  {
+    account_address address;//!< Must be first for LMDB optimizations
+    view_key key;
+    block_id start_height;
+    account_time creation;        //!< Time the request was created.
+    account_flags creation_flags; //!< Generated locally?
+    char reserved[3];
+  };
+  static_assert(sizeof(request_info) == 64 + 32 + 8 + (4 * 2), "padding in request_info");
+  void write_bytes(wire::writer& dest, const request_info& self, bool show_key = false);
+
+  inline constexpr bool operator==(output_id left, output_id right) noexcept
+  {
+    return left.high == right.high && left.low == right.low;
+  }
+  inline constexpr bool operator!=(output_id left, output_id right) noexcept
+  {
+    return left.high != right.high || left.low != right.low;
+  }
+  inline constexpr bool operator<(output_id left, output_id right) noexcept
+  {
+    return left.high == right.high ?
+      left.low < right.low : left.high < right.high;
+  }
+  inline constexpr bool operator<=(output_id left, output_id right) noexcept
+  {
+    return left.high == right.high ?
+      left.low <= right.low : left.high < right.high;
+  }
+
+  bool operator<(transaction_link const& left, transaction_link const& right) noexcept;
+  bool operator<=(transaction_link const& left, transaction_link const& right) noexcept;
+
+  /*!
+    Write `address` to `out` in base58 format using `lws::config::network` to
+    determine tag. */
+  std::ostream& operator<<(std::ostream& out, account_address const& address);
+} // db
+} // lws
+
+namespace wire
+{
+  template<>
+   struct is_blob<lws::db::view_key>
+    : std::true_type
+  {};
+}

src/db/fwd.h

@@ -0,0 +1,56 @@
+// Copyright (c) 2018-2020, The Monero Project
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#pragma once
+
+#include <cstdint>
+
+namespace lws
+{
+namespace db
+{
+  enum account_flags : std::uint8_t;
+  enum class account_id : std::uint32_t;
+  enum class account_status : std::uint8_t;
+  enum class block_id : std::uint64_t;
+  enum extra : std::uint8_t;
+  enum class extra_and_length : std::uint8_t;
+  enum class request : std::uint8_t;
+
+  struct account;
+  struct account_address;
+  struct block_info;
+  struct key_image;
+  struct output;
+  struct output_id;
+  struct request_info;
+  struct spend;
+  class storage;
+  struct transaction_link;
+  struct view_key;
+} // db
+} // lws

src/db/storage.h

@@ -0,0 +1,239 @@
+// Copyright (c) 2018, The Monero Project
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#pragma once
+
+#include <iosfwd>
+#include <list>
+#include <memory>
+#include <utility>
+#include <vector>
+
+#include "common/expect.h"
+#include "crypto/crypto.h"
+#include "db/account.h"
+#include "db/data.h"
+#include "fwd.h"
+#include "lmdb/transaction.h"
+#include "lmdb/key_stream.h"
+#include "lmdb/value_stream.h"
+
+namespace lws
+{
+namespace db
+{
+  namespace cursor
+  {
+    MONERO_CURSOR(accounts);
+    MONERO_CURSOR(outputs);
+    MONERO_CURSOR(spends);
+    MONERO_CURSOR(images);
+    MONERO_CURSOR(requests);
+
+    MONERO_CURSOR(blocks);
+    MONERO_CURSOR(accounts_by_address);
+    MONERO_CURSOR(accounts_by_height);
+  }
+
+  struct storage_internal;
+  struct reader_internal
+  {
+    cursor::blocks blocks_cur;
+    cursor::accounts_by_address accounts_ba_cur;
+    cursor::accounts_by_height accounts_bh_cur;
+  };
+
+  //! Wrapper for LMDB read access to on-disk storage of light-weight server data.
+  class storage_reader
+  {
+    std::shared_ptr<storage_internal> db;
+    lmdb::read_txn txn;
+    reader_internal curs;
+
+  public:
+    storage_reader(std::shared_ptr<storage_internal> db, lmdb::read_txn txn) noexcept
+      : db(std::move(db)), txn(std::move(txn)), curs{}
+    {}
+
+    storage_reader(storage_reader&&) = default;
+    storage_reader(storage_reader const&) = delete;
+
+    ~storage_reader() noexcept;
+
+    storage_reader& operator=(storage_reader&&) = default;
+    storage_reader& operator=(storage_reader const&) = delete;
+
+    //! \return Last known block.
+    expect<block_info> get_last_block() noexcept;
+
+    //! \return List for `GetHashesFast` to sync blockchain with daemon.
+    expect<std::list<crypto::hash>> get_chain_sync();
+
+    //! \return All registered `account`s.
+    expect<lmdb::key_stream<account_status, account, cursor::close_accounts>>
+      get_accounts(cursor::accounts cur = nullptr) noexcept;
+
+    //! \return All `account`s currently in `status` or `lmdb::error(MDB_NOT_FOUND)`.
+    expect<lmdb::value_stream<account, cursor::close_accounts>>
+      get_accounts(account_status status, cursor::accounts cur = nullptr) noexcept;
+
+    //! \return Info related to `address` or `lmdb::error(MDB_NOT_FOUND)`.
+    expect<std::pair<account_status, account>>
+      get_account(account_address const& address, cursor::accounts& cur) noexcept;
+
+    expect<std::pair<account_status, account>>
+      get_account(account_address const& address) noexcept
+    {
+      cursor::accounts cur;
+      return get_account(address, cur);
+    }
+
+    //! \return All outputs received by `id`.
+    expect<lmdb::value_stream<output, cursor::close_outputs>>
+      get_outputs(account_id id, cursor::outputs cur = nullptr) noexcept;
+
+    //! \return All potential spends by `id`.
+    expect<lmdb::value_stream<spend, cursor::close_spends>>
+      get_spends(account_id id, cursor::spends cur = nullptr) noexcept;
+
+    //! \return All key images associated with `id`.
+    expect<lmdb::value_stream<db::key_image, cursor::close_images>>
+      get_images(output_id id, cursor::images cur = nullptr) noexcept;
+
+    //! \return All `request_info`s.
+    expect<lmdb::key_stream<request, request_info, cursor::close_requests>>
+      get_requests(cursor::requests cur = nullptr) noexcept;
+
+    //! \return A specific request from `address` of `type`.
+    expect<request_info>
+      get_request(request type, account_address const& address, cursor::requests cur = nullptr) noexcept;
+
+    //! Dump the contents of the database in JSON format to `out`.
+    expect<void> json_debug(std::ostream& out, bool show_keys);
+
+    //! \return Read txn that can be re-used via `storage::start_read`.
+    lmdb::suspended_txn finish_read() noexcept;
+  };
+
+  //! Wrapper for LMDB on-disk storage of light-weight server data.
+  class storage
+  {
+    std::shared_ptr<storage_internal> db;
+
+    storage(std::shared_ptr<storage_internal> db) noexcept
+      : db(std::move(db))
+    {}
+
+  public:
+    /*!
+      Open a light_wallet_server LDMB database.
+
+      \param path Directory for LMDB storage
+      \param create_queue_max Maximum number of create account requests allowed.
+
+      \throw std::system_error on any LMDB error (all treated as fatal).
+      \throw std::bad_alloc If `std::shared_ptr` fails to allocate.
+
+      \return A ready light-wallet server database.
+    */
+    static storage open(const char* path, unsigned create_queue_max);
+
+    storage(storage&&) = default;
+    storage(storage const&) = delete;
+
+    ~storage() noexcept;
+
+    storage& operator=(storage&&) = default;
+    storage& operator=(storage const&) = delete;
+
+    //! \return A copy of the LMDB environment, but not reusable txn/cursors.
+    storage clone() const noexcept;
+
+    //! Rollback chain and accounts to `height`.
+    expect<void> rollback(block_id height);
+
+    /*!
+      Sync the local blockchain with a remote version. Pops user txes if reorg
+      detected.
+
+      \param height The height of the element in `hashes`
+      \param hashes List of blockchain hashes starting at `height`.
+
+      \return True if the local blockchain is correctly synced.
+    */
+    expect<void> sync_chain(block_id height, epee::span<const crypto::hash> hashes);
+
+    //! Bump the last access time of `address` to the current time.
+    expect<void> update_access_time(account_address const& address) noexcept;
+
+    //! Change state of `address` to `status`. \return Updated `addresses`.
+    expect<std::vector<account_address>>
+      change_status(account_status status, epee::span<const account_address> addresses);
+
+
+    //! Add an account, for immediate inclusion in the active list.
+    expect<void> add_account(account_address const& address, crypto::secret_key const& key) noexcept;
+
+    //! Reset `addresses` to `height` for scanning.
+    expect<std::vector<account_address>>
+      rescan(block_id height, epee::span<const account_address> addresses);
+
+    //! Add an account for later approval. For use with the login endpoint.
+    expect<void> creation_request(account_address const& address, crypto::secret_key const& key, account_flags flags) noexcept;
+
+    /*!
+      Request lock height of an existing account. No effect if the `start_height`
+      is already older.
+    */
+    expect<void> import_request(account_address const& address, block_id height) noexcept;
+
+    //! Accept requests by `addresses` of type `req`. \return Accepted addresses.
+    expect<std::vector<account_address>>
+      accept_requests(request req, epee::span<const account_address> addresses);
+
+    //! Reject requests by `addresses` of type `req`. \return Rejected addresses.
+    expect<std::vector<account_address>>
+      reject_requests(request req, epee::span<const account_address> addresses);
+
+    /*!
+      Updates the status of user accounts, even if inactive or hidden. Duplicate
+      receives or spends provided in `accts` are silently ignored. If a gap in
+      `height` vs the stored account record is detected, the entire update will
+      fail.
+
+      \param height The first hash in `chain` is at this height.
+      \param chain List of block hashes that `accts` were scanned against.
+      \param accts Updated to `height + chain.size()` scan height.
+
+      \return True iff LMDB successfully committed the update.
+    */
+    expect<std::size_t> update(block_id height, epee::span<const crypto::hash> chain, epee::span<const lws::account> accts);
+
+    //! `txn` must have come from a previous call on the same thread.
+    expect<storage_reader> start_read(lmdb::suspended_txn txn = nullptr) const;
+  };
+} // db
+} // lws

src/db/string.cpp

@@ -0,0 +1,62 @@
+// Copyright (c) 2018, The Monero Project
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#include "string.h"
+
+#include "config.h"
+#include "cryptonote_basic/cryptonote_basic_impl.h"
+#include "db/data.h"
+#include "error.h"
+
+namespace lws
+{
+namespace  db
+{
+  std::string address_string_::operator()(account_address const& address) const
+  {
+    const cryptonote::account_public_address address_{
+      address.spend_public, address.view_public
+    };
+    return cryptonote::get_account_address_as_str(
+      lws::config::network, false, address_
+    );
+  }
+  expect<account_address>
+  address_string_::operator()(boost::string_ref address) const noexcept
+  {
+    cryptonote::address_parse_info info{};
+
+    if (!cryptonote::get_account_address_from_str(info, lws::config::network, std::string{address}))
+      return {lws::error::bad_address};
+    if (info.is_subaddress || info.has_payment_id)
+      return {lws::error::bad_address};
+
+    return account_address{
+      info.address.m_spend_public_key, info.address.m_view_public_key
+    };
+  }
+} // db
+} // lws

src/db/string.h

@@ -0,0 +1,55 @@
+// Copyright (c) 2018, The Monero Project
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#pragma once
+
+#include <boost/utility/string_ref.hpp>
+#include <string>
+
+#include "common/expect.h"
+#include "db/fwd.h"
+
+namespace lws
+{
+namespace  db
+{
+  //! Callable for converting `account_address` to/from monero base58 public address.
+  struct address_string_
+  {
+    /*!
+      \return `address` as a monero base58 public address, using
+        `lws::config::network` for the tag.
+    */
+    std::string operator()(account_address const& address) const;
+    /*!
+      \return `address`, as base58 public address, using `lws::config::network`
+        for the tag.
+    */
+    expect<account_address> operator()(boost::string_ref address) const noexcept;
+  };
+  constexpr const address_string_ address_string{};
+} // db
+} // lws

src/error.cpp

@@ -0,0 +1,131 @@
+// Copyright (c) 2018-2020, The Monero Project
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#include "error.h"
+
+#include <string>
+
+namespace lws
+{
+  struct category final : std::error_category
+  {
+    virtual const char* name() const noexcept override final
+    {
+      return "lws::error_category()";
+    }
+
+    virtual std::string message(int value) const override final
+    {
+      switch (lws::error(value))
+      {
+      case error::account_exists:
+        return "Account with specified address already exists";
+      case error::account_max:
+        return "Account limit has been reached";
+      case error::account_not_found:
+        return "No account with the specified address exists";
+      case error::bad_address:
+        return "Invalid base58 public address - wrong --network ?";
+      case error::bad_view_key:
+        return "Address/viewkey mismatch";
+      case error::bad_blockchain:
+        return "Unable to sync blockchain - wrong --network ?";
+      case error::bad_client_tx:
+        return "Received invalid transaction from REST client";
+      case error::bad_daemon_response:
+        return "Response from monerod daemon was bad/unexpected";
+      case error::blockchain_reorg:
+        return "A blockchain reorg has been detected";
+      case error::configuration:
+        return "Invalid process configuration";
+      case error::create_queue_max:
+        return "Exceeded maxmimum number of pending account requests";
+      case error::crypto_failure:
+        return "A cryptographic function failed";
+      case error::daemon_timeout:
+        return "Connection failed with daemon";
+      case error::duplicate_request:
+        return "A request of this type for this address has already been made";
+      case error::exceeded_blockchain_buffer:
+        return "Exceeded internal buffer for blockchain hashes";
+      case error::exceeded_rest_request_limit:
+        return "Request from client via REST exceeded enforced limits";
+      case error::exchange_rates_disabled:
+        return "Exchange rates feature is disabled";
+      case error::exchange_rates_fetch:
+        return "Unspecified error when retrieving exchange rates";
+      case error::http_server:
+        return "HTTP server failed";
+      case error::exchange_rates_old:
+        return "Exchange rates are older than cache interval";
+      case error::not_enough_mixin:
+        return "Not enough outputs to meet requested mixin count";
+      case error::signal_abort_process:
+        return "An in-process message was received to abort the process";
+      case error::signal_abort_scan:
+        return "An in-process message was received to abort account scanning";
+      case error::signal_unknown:
+        return "An unknown in-process message was received";
+      case error::system_clock_invalid_range:
+        return "System clock is out of range for account storage format";
+      case error::tx_relay_failed:
+        return "The daemon failed to relay transaction from REST client";
+      default:
+        break;
+      }
+      return "Unknown lws::error_category() value";
+    }
+
+    virtual std::error_condition default_error_condition(int value) const noexcept override final
+    {
+      switch (lws::error(value))
+      {
+      case error::bad_address:
+      case error::bad_view_key:
+        return std::errc::bad_address;
+      case error::daemon_timeout:
+        return std::errc::timed_out;
+      case error::exceeded_blockchain_buffer:
+        return std::errc::no_buffer_space;
+      case error::signal_abort_process:
+      case error::signal_abort_scan:
+      case error::signal_unknown:
+        return std::errc::interrupted;
+      case error::system_clock_invalid_range:
+        return std::errc::result_out_of_range;
+      default:
+        break; // map to unmatchable category
+      }
+      return std::error_condition{value, *this};
+    }
+  };
+
+  std::error_category const& error_category() noexcept
+  {
+    static const category instance{};
+    return instance;
+  }
+} // lws

src/error.h

@@ -0,0 +1,79 @@
+// Copyright (c) 2018-2020, The Monero Project
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#pragma once
+
+#include <system_error>
+#include <type_traits>
+
+namespace lws
+{
+  enum class error : int
+  {
+    // 0 is reserved for no error, as per expect<T>
+    account_exists = 1,         //!< Tried to create an account that already exists
+    account_max,                //!< Maximum number of accounts have been created
+    account_not_found,          //!< Account address is not in database.
+    bad_address,                //!< Invalid base58 public address
+    bad_view_key,               //!< Account has address/viewkey mismatch
+    bad_blockchain,             //!< Blockchain is invalid or wrong network type
+    bad_client_tx,              //!< REST client submitted invalid transaction
+    bad_daemon_response,        //!< RPC Response from daemon was invalid
+    blockchain_reorg,           //!< Blockchain reorg after fetching/scanning block(s)
+    configuration,              //!< Process configuration invalid
+    crypto_failure,             //!< Cryptographic function failed
+    create_queue_max,           //!< Reached maximum pending account requests
+    daemon_timeout,             //!< ZMQ send/receive timeout
+    duplicate_request,          //!< Account already has a request of  this type pending
+    exceeded_blockchain_buffer, //!< Out buffer for blockchain is too small
+    exceeded_rest_request_limit,//!< Exceeded enforced size limits for request
+    exchange_rates_disabled,    //!< Exchange rates fetching is disabled
+    exchange_rates_fetch,       //!< Exchange rates fetching failed
+    exchange_rates_old,         //!< Exchange rates are older than cache interval
+    http_server,                //!< HTTP server failure (init or run)
+    not_enough_mixin,           //!< Not enough outputs to meet mixin count
+    signal_abort_process,       //!< In process ZMQ PUB to abort the process was received
+    signal_abort_scan,          //!< In process ZMQ PUB to abort the scan was received
+    signal_unknown,             //!< An unknown in process ZMQ PUB was received
+    system_clock_invalid_range, //!< System clock is out of range for storage format
+    tx_relay_failed             //!< Daemon failed to relayed tx from REST client
+  };
+
+  std::error_category const& error_category() noexcept;
+
+  inline std::error_code make_error_code(lws::error value) noexcept
+  {
+    return std::error_code{int(value), error_category()};
+  }
+}
+
+namespace std
+{
+  template<>
+  struct is_error_code_enum<::lws::error>
+    : true_type
+  {};
+}

src/fwd.h

@@ -0,0 +1,35 @@
+// Copyright (c) 2020, The Monero Project
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#pragma once
+
+namespace lws
+{
+  class account;
+  class rest_server;
+  class scanner;
+}

src/options.h

@@ -0,0 +1,72 @@
+// Copyright (c) 2018-2020, The Monero Project
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#pragma once
+
+#include <boost/program_options/options_description.hpp>
+#include <boost/program_options/variables_map.hpp>
+#include <stdexcept>
+#include <string>
+
+#include "common/command_line.h" // monero/src
+#include "common/util.h"         // monero/src
+
+namespace lws
+{
+  constexpr const char default_db_subdir[] = "/light_wallet_server";
+
+  struct options
+  {
+    const command_line::arg_descriptor<std::string> db_path;
+    const command_line::arg_descriptor<std::string> network;
+
+    options()
+      : db_path{"db-path", "Folder for LMDB files", tools::get_default_data_dir() + default_db_subdir}
+      , network{"network", "<\"main\"|\"stage\"|\"test\"> - Blockchain net type", "main"}
+    {}
+
+    void prepare(boost::program_options::options_description& description) const
+    {
+      command_line::add_arg(description, db_path);
+      command_line::add_arg(description, network);
+      command_line::add_arg(description, command_line::arg_help);
+    }
+
+    void set_network(boost::program_options::variables_map const& args) const
+    {
+      const std::string net = command_line::get_arg(args, network);
+      if (net == "main")
+        lws::config::network = cryptonote::MAINNET;
+      else if (net == "stage")
+        lws::config::network = cryptonote::STAGENET;
+      else if (net == "test")
+        lws::config::network = cryptonote::TESTNET;
+      else
+        throw std::runtime_error{"Bad --network value"};
+    }
+  };
+}

src/rest_server.cpp

@@ -0,0 +1,863 @@
+// Copyright (c) 2018-2020, The Monero Project
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#include "rest_server.h"
+
+#include <algorithm>
+#include <boost/utility/string_ref.hpp>
+#include <cstring>
+#include <limits>
+#include <string>
+#include <utility>
+
+#include "common/error.h"                             // monero/src
+#include "common/expect.h"                            // monero/src
+#include "crypto/crypto.h"                            // monero/src
+#include "cryptonote_config.h"      // monero/src
+#include "db/data.h"
+#include "db/storage.h"
+#include "error.h"
+#include "lmdb/util.h"                 // monero/src
+#include "net/http_base.h"             // monero/contrib/epee/include
+#include "net/net_parse_helpers.h"     // monero/contrib/epee/include
+#include "rpc/daemon_messages.h"       // monero/src
+#include "rpc/light_wallet.h"
+#include "rpc/rates.h"
+#include "util/http_server.h"
+#include "util/gamma_picker.h"
+#include "util/random_outputs.h"
+#include "wire/json.h"
+
+namespace lws
+{
+  namespace
+  {
+    namespace http = epee::net_utils::http;
+
+    struct context : epee::net_utils::connection_context_base
+    {
+      context()
+        : epee::net_utils::connection_context_base()
+      {}
+    };
+
+    bool is_locked(std::uint64_t unlock_time, db::block_id last) noexcept
+    {
+      if (unlock_time > CRYPTONOTE_MAX_BLOCK_NUMBER)
+        return std::chrono::seconds{unlock_time} > std::chrono::system_clock::now().time_since_epoch();
+      return db::block_id(unlock_time) > last;
+    }
+
+    std::vector<db::output::spend_meta_>::const_iterator
+    find_metadata(std::vector<db::output::spend_meta_> const& metas, db::output_id id)
+    {
+      struct by_output_id
+      {
+        bool operator()(db::output::spend_meta_ const& left, db::output_id right) const noexcept
+        {
+          return left.id < right;
+        }
+        bool operator()(db::output_id left, db::output::spend_meta_ const& right) const noexcept
+        {
+          return left < right.id;
+        }
+      };
+      return std::lower_bound(metas.begin(), metas.end(), id, by_output_id{});
+    }
+
+    bool is_hidden(db::account_status status) noexcept
+    {
+      switch (status)
+      {
+      case db::account_status::active:
+      case db::account_status::inactive:
+        return false;
+      default:
+      case db::account_status::hidden:
+        break;
+      }
+      return true;
+    }
+
+    bool key_check(const rpc::account_credentials& creds)
+    {
+      crypto::public_key verify{};
+      if (!crypto::secret_key_to_public_key(creds.key, verify))
+        return false;
+      if (verify != creds.address.view_public)
+        return false;
+      return true;
+    }
+
+    //! \return Account info from the DB, iff key matches address AND address is NOT hidden.
+    expect<std::pair<db::account, db::storage_reader>> open_account(const rpc::account_credentials& creds, db::storage disk)
+    {
+      if (!key_check(creds))
+        return {lws::error::bad_view_key};
+
+      auto reader = disk.start_read();
+      if (!reader)
+        return reader.error();
+
+      const auto user = reader->get_account(creds.address);
+      if (!user)
+        return user.error();
+      if (is_hidden(user->first))
+        return {lws::error::account_not_found};
+      return {std::make_pair(user->second, std::move(*reader))};
+    }
+
+    struct get_address_info
+    {
+      using request = rpc::account_credentials;
+      using response = rpc::get_address_info_response;
+
+      static expect<response> handle(const request& req, db::storage disk, rpc::client const& client)
+      {
+        auto user = open_account(req, std::move(disk));
+        if (!user)
+          return user.error();
+
+        response resp{.rates = {common_error::kInvalidArgument}};
+
+        auto outputs = user->second.get_outputs(user->first.id);
+        if (!outputs)
+          return outputs.error();
+
+        auto spends = user->second.get_spends(user->first.id);
+        if (!spends)
+          return spends.error();
+
+        const expect<db::block_info> last = user->second.get_last_block();
+        if (!last)
+          return last.error();
+
+        resp.blockchain_height = std::uint64_t(last->id);
+        resp.transaction_height = resp.blockchain_height;
+        resp.scanned_height = std::uint64_t(user->first.scan_height);
+        resp.scanned_block_height = resp.scanned_height;
+        resp.start_height = std::uint64_t(user->first.start_height);
+
+        std::vector<db::output::spend_meta_> metas{};
+        metas.reserve(outputs->count());
+
+        for (auto output = outputs->make_iterator(); !output.is_end(); ++output)
+        {
+          const db::output::spend_meta_ meta =
+            output.get_value<MONERO_FIELD(db::output, spend_meta)>();
+
+          // these outputs will usually be in correct order post ringct
+          if (metas.empty() || metas.back().id < meta.id)
+            metas.push_back(meta);
+          else
+            metas.insert(find_metadata(metas, meta.id), meta);
+
+          resp.total_received = rpc::safe_uint64(std::uint64_t(resp.total_received) + meta.amount);
+          if (is_locked(output.get_value<MONERO_FIELD(db::output, unlock_time)>(), last->id))
+            resp.locked_funds = rpc::safe_uint64(std::uint64_t(resp.locked_funds) + meta.amount);
+        }
+
+        resp.spent_outputs.reserve(spends->count());
+        for (auto const& spend : spends->make_range())
+        {
+          const auto meta = find_metadata(metas, spend.source);
+          if (meta == metas.end() || meta->id != spend.source)
+          {
+            throw std::logic_error{
+              "Serious database error, no receive for spend"
+            };
+          }
+
+          resp.spent_outputs.push_back({*meta, spend});
+          resp.total_sent = rpc::safe_uint64(std::uint64_t(resp.total_sent) + meta->amount);
+        }
+
+        resp.rates = client.get_rates();
+        if (!resp.rates)
+          MWARNING("Unable to retrieve exchange rates: " << resp.rates.error().message());
+
+        return resp;
+      }
+    };
+
+    struct get_address_txs
+    {
+      using request = rpc::account_credentials;
+      using response = rpc::get_address_txs_response;
+
+      static expect<response> handle(const request& req, db::storage disk, rpc::client const&)
+      {
+        auto user = open_account(req, std::move(disk));
+        if (!user)
+          return user.error();
+
+        auto outputs = user->second.get_outputs(user->first.id);
+        if (!outputs)
+          return outputs.error();
+
+        auto spends = user->second.get_spends(user->first.id);
+        if (!spends)
+          return spends.error();
+
+        const expect<db::block_info> last = user->second.get_last_block();
+        if (!last)
+          return last.error();
+
+        response resp{};
+        resp.scanned_height = std::uint64_t(user->first.scan_height);
+        resp.scanned_block_height = resp.scanned_height;
+        resp.start_height = std::uint64_t(user->first.start_height);
+        resp.blockchain_height = std::uint64_t(last->id);
+        resp.transaction_height = resp.blockchain_height;
+
+        // merge input and output info into a single set of txes.
+
+        auto output = outputs->make_iterator();
+        auto spend = spends->make_iterator();
+
+        std::vector<db::output::spend_meta_> metas{};
+
+        resp.transactions.reserve(outputs->count());
+        metas.reserve(resp.transactions.capacity());
+
+        db::transaction_link next_output{};
+        db::transaction_link next_spend{};
+
+        if (!output.is_end())
+          next_output = output.get_value<MONERO_FIELD(db::output, link)>();
+        if (!spend.is_end())
+          next_spend = spend.get_value<MONERO_FIELD(db::spend, link)>();
+
+        while (!output.is_end() || !spend.is_end())
+        {
+          if (!resp.transactions.empty())
+          {
+            db::transaction_link const& last = resp.transactions.back().info.link;
+
+            if ((!output.is_end() && next_output < last) || (!spend.is_end() && next_spend < last))
+            {
+              throw std::logic_error{"DB has unexpected sort order"};
+            }
+          }
+
+          if (spend.is_end() || (!output.is_end() && next_output <= next_spend))
+          {
+            std::uint64_t amount = 0;
+            if (resp.transactions.empty() || resp.transactions.back().info.link.tx_hash != next_output.tx_hash)
+            {
+              resp.transactions.push_back({*output});
+              amount = resp.transactions.back().info.spend_meta.amount;
+            }
+            else
+            {
+              amount = output.get_value<MONERO_FIELD(db::output, spend_meta.amount)>();
+              resp.transactions.back().info.spend_meta.amount += amount;
+            }
+
+            const db::output_id this_id = resp.transactions.back().info.spend_meta.id;
+            if (metas.empty() || metas.back().id < this_id)
+              metas.push_back(resp.transactions.back().info.spend_meta);
+            else
+              metas.insert(find_metadata(metas, this_id), resp.transactions.back().info.spend_meta);
+
+            resp.total_received = rpc::safe_uint64(std::uint64_t(resp.total_received) + amount);
+
+            ++output;
+            if (!output.is_end())
+              next_output = output.get_value<MONERO_FIELD(db::output, link)>();
+          }
+          else if (output.is_end() || (next_spend < next_output))
+          {
+            const db::output_id source_id = spend.get_value<MONERO_FIELD(db::spend, source)>();
+            const auto meta = find_metadata(metas, source_id);
+            if (meta == metas.end() || meta->id != source_id)
+            {
+              throw std::logic_error{
+                "Serious database error, no receive for spend"
+              };
+            }
+
+            if (resp.transactions.empty() || resp.transactions.back().info.link.tx_hash != next_spend.tx_hash)
+            {
+              resp.transactions.push_back({});
+              resp.transactions.back().spends.push_back({*meta, *spend});
+              resp.transactions.back().info.link.height = resp.transactions.back().spends.back().possible_spend.link.height;
+              resp.transactions.back().info.link.tx_hash = resp.transactions.back().spends.back().possible_spend.link.tx_hash;
+              resp.transactions.back().info.spend_meta.mixin_count =
+                resp.transactions.back().spends.back().possible_spend.mixin_count;
+              resp.transactions.back().info.timestamp = resp.transactions.back().spends.back().possible_spend.timestamp;
+              resp.transactions.back().info.unlock_time = resp.transactions.back().spends.back().possible_spend.unlock_time;
+            }
+            else
+              resp.transactions.back().spends.push_back({*meta, *spend});
+
+            resp.transactions.back().spent += meta->amount;
+
+            ++spend;
+            if (!spend.is_end())
+              next_spend = spend.get_value<MONERO_FIELD(db::spend, link)>();
+          }
+        }
+
+        return resp;
+      }
+    };
+
+    struct get_random_outs
+    {
+      using request = rpc::get_random_outs_request;
+      using response = rpc::get_random_outs_response;
+
+      static expect<response> handle(request req, const db::storage&, rpc::client const& gclient)
+      {
+        using distribution_rpc = cryptonote::rpc::GetOutputDistribution;
+        using histogram_rpc = cryptonote::rpc::GetOutputHistogram;
+        using distribution_rpc = cryptonote::rpc::GetOutputDistribution;
+
+        std::vector<std::uint64_t> amounts = std::move(req.amounts.values);
+
+        if (50 < req.count || 20 < amounts.size())
+          return {lws::error::exceeded_rest_request_limit};
+
+        expect<rpc::client> client = gclient.clone();
+        if (!client)
+          return client.error();
+
+        const std::greater<std::uint64_t> rsort{};
+        std::sort(amounts.begin(), amounts.end(), rsort);
+        const std::size_t ringct_count =
+          amounts.end() - std::lower_bound(amounts.begin(), amounts.end(), 0, rsort);
+
+        std::vector<lws::histogram> histograms{};
+        if (ringct_count < amounts.size())
+        {
+          // reuse allocated vector memory
+          amounts.resize(amounts.size() - ringct_count);
+
+          histogram_rpc::Request histogram_req{};
+          histogram_req.amounts = std::move(amounts);
+          histogram_req.min_count = 0;
+          histogram_req.max_count = 0;
+          histogram_req.unlocked = true;
+          histogram_req.recent_cutoff = 0;
+
+          epee::byte_slice msg = rpc::client::make_message("get_output_histogram", histogram_req);
+          MONERO_CHECK(client->send(std::move(msg), std::chrono::seconds{10}));
+
+          auto histogram_resp = client->receive<histogram_rpc::Response>(std::chrono::minutes{3});
+          if (!histogram_resp)
+            return histogram_resp.error();
+          if (histogram_resp->histogram.size() != histogram_req.amounts.size())
+            return {lws::error::bad_daemon_response};
+
+          histograms = std::move(histogram_resp->histogram);
+
+          amounts = std::move(histogram_req.amounts);
+          amounts.insert(amounts.end(), ringct_count, 0);
+        }
+
+        std::vector<std::uint64_t> distributions{};
+        if (ringct_count)
+        {
+          distribution_rpc::Request distribution_req{};
+          if (ringct_count == amounts.size())
+            distribution_req.amounts = std::move(amounts);
+
+          distribution_req.amounts.resize(1);
+          distribution_req.from_height = 0;
+          distribution_req.to_height = 0;
+          distribution_req.cumulative = true;
+
+          epee::byte_slice msg =
+            rpc::client::make_message("get_output_distribution", distribution_req);
+          MONERO_CHECK(client->send(std::move(msg), std::chrono::seconds{10}));
+
+          auto distribution_resp =
+            client->receive<distribution_rpc::Response>(std::chrono::minutes{3});
+          if (!distribution_resp)
+            return distribution_resp.error();
+
+          if (distribution_resp->distributions.size() != 1)
+            return {lws::error::bad_daemon_response};
+          if (distribution_resp->distributions[0].amount != 0)
+            return {lws::error::bad_daemon_response};
+
+          distributions = std::move(distribution_resp->distributions[0].data.distribution);
+
+          if (amounts.empty())
+          {
+            amounts = std::move(distribution_req.amounts);
+            amounts.insert(amounts.end(), ringct_count - 1, 0);
+          }
+        }
+
+        class zmq_fetch_keys
+        {
+          /* `std::function` needs a copyable functor. The functor was made
+             const and copied in the function instead of using a reference to
+             make the callback in `std::function` thread-safe. This shouldn't
+             be a problem now, but this is just-in-case of a future refactor. */
+          rpc::client gclient;
+        public:
+          zmq_fetch_keys(rpc::client src) noexcept
+            : gclient(std::move(src))
+          {}
+
+          zmq_fetch_keys(zmq_fetch_keys&&) = default;
+          zmq_fetch_keys(zmq_fetch_keys const& rhs)
+            : gclient(MONERO_UNWRAP(rhs.gclient.clone()))
+          {}
+
+          expect<std::vector<output_keys>> operator()(std::vector<lws::output_ref> ids) const
+          {
+            using get_keys_rpc = cryptonote::rpc::GetOutputKeys;
+
+            get_keys_rpc::Request keys_req{};
+            keys_req.outputs = std::move(ids);
+
+            expect<rpc::client> client = gclient.clone();
+            if (!client)
+              return client.error();
+
+            epee::byte_slice msg = rpc::client::make_message("get_output_keys", keys_req);
+            MONERO_CHECK(client->send(std::move(msg), std::chrono::seconds{10}));
+
+            auto keys_resp = client->receive<get_keys_rpc::Response>(std::chrono::seconds{10});
+            if (!keys_resp)
+              return keys_resp.error();
+
+            return {std::move(keys_resp->keys)};
+          }
+        };
+
+        lws::gamma_picker pick_rct{std::move(distributions)};
+        auto rings = pick_random_outputs(
+          req.count,
+          epee::to_span(amounts),
+          pick_rct,
+          epee::to_mut_span(histograms),
+          zmq_fetch_keys{std::move(*client)}
+        );
+        if (!rings)
+          return rings.error();
+
+        return response{std::move(*rings)};
+      }
+    };
+
+    struct get_unspent_outs
+    {
+      using request = rpc::get_unspent_outs_request;
+      using response = rpc::get_unspent_outs_response;
+
+      static expect<response> handle(request req, db::storage disk, rpc::client const& gclient)
+      {
+        using rpc_command = cryptonote::rpc::GetFeeEstimate;
+
+        auto user = open_account(req.creds, std::move(disk));
+        if (!user)
+          return user.error();
+
+        expect<rpc::client> client = gclient.clone();
+        if (!client)
+          return client.error();
+
+        {
+          rpc_command::Request req{};
+          req.num_grace_blocks = 10;
+          epee::byte_slice msg = rpc::client::make_message("get_dynamic_fee_estimate", req);
+          MONERO_CHECK(client->send(std::move(msg), std::chrono::seconds{10}));
+        }
+
+        if ((req.use_dust && req.use_dust) || !req.dust_threshold)
+          req.dust_threshold = rpc::safe_uint64(0);
+
+        if (!req.mixin)
+          req.mixin = 0;
+
+        auto outputs = user->second.get_outputs(user->first.id);
+        if (!outputs)
+          return outputs.error();
+
+        std::uint64_t received = 0;
+        std::vector<std::pair<db::output, std::vector<crypto::key_image>>> unspent;
+
+        unspent.reserve(outputs->count());
+        for (db::output const& out : outputs->make_range())
+        {
+          if (out.spend_meta.amount < std::uint64_t(*req.dust_threshold) || out.spend_meta.mixin_count < *req.mixin)
+            continue;
+
+          received += out.spend_meta.amount;
+          unspent.push_back({out, {}});
+
+          auto images = user->second.get_images(out.spend_meta.id);
+          if (!images)
+            return images.error();
+
+          unspent.back().second.reserve(images->count());
+          auto range = images->make_range<MONERO_FIELD(db::key_image, value)>();
+          std::copy(range.begin(), range.end(), std::back_inserter(unspent.back().second));
+        }
+
+        if (received < std::uint64_t(req.amount))
+          return {lws::error::account_not_found};
+
+        const auto resp = client->receive<rpc_command::Response>(std::chrono::seconds{20});
+        if (!resp)
+          return resp.error();
+
+        if (resp->size_scale == 0 || 1024 < resp->size_scale || resp->fee_mask == 0)
+          return {lws::error::bad_daemon_response};
+
+        const std::uint64_t per_kb_fee =
+          resp->estimated_base_fee * (1024 / resp->size_scale);
+        const std::uint64_t per_kb_fee_masked =
+          ((per_kb_fee + (resp->fee_mask - 1)) / resp->fee_mask) * resp->fee_mask;
+
+        return response{per_kb_fee_masked, resp->fee_mask, rpc::safe_uint64(received), std::move(unspent), std::move(req.creds.key)};
+      }
+    };
+
+    struct import_request
+    {
+      using request = rpc::account_credentials;
+      using response = rpc::import_response;
+
+      static expect<response> handle(request req, db::storage disk, rpc::client const&)
+      {
+        bool new_request = false;
+        bool fulfilled = false;
+        {
+          auto user = open_account(req, disk.clone());
+          if (!user)
+            return user.error();
+
+          if (user->first.start_height == db::block_id(0))
+            fulfilled = true;
+          else
+          {
+            const expect<db::request_info> info =
+              user->second.get_request(db::request::import_scan, req.address);
+
+            if (!info)
+            {
+              if (info != lmdb::error(MDB_NOTFOUND))
+                return info.error();
+
+              new_request = true;
+            }
+          }
+        } // close reader
+
+        if (new_request)
+          MONERO_CHECK(disk.import_request(req.address, db::block_id(0)));
+
+        const char* status = new_request ?
+          "Accepted, waiting for approval" : (fulfilled ? "Approved" : "Waiting for Approval");
+        return response{rpc::safe_uint64(0), status, new_request, fulfilled};
+      }
+    };
+
+    struct login
+    {
+      using request = rpc::login_request;
+      using response = rpc::login_response;
+
+      static expect<response> handle(request req, db::storage disk, rpc::client const&)
+      {
+        if (!key_check(req.creds))
+          return {lws::error::bad_view_key};
+
+        {
+          auto reader = disk.start_read();
+          if (!reader)
+            return reader.error();
+
+          const auto account = reader->get_account(req.creds.address);
+          reader->finish_read();
+
+          if (account)
+          {
+            if (is_hidden(account->first))
+              return {lws::error::account_not_found};
+
+            // Do not count a request for account creation as login
+            return response{false, bool(account->second.flags & db::account_generated_locally)};
+          }
+          else if (!req.create_account || account != lws::error::account_not_found)
+            return account.error();
+        }
+
+        const auto flags = req.generated_locally ? db::account_generated_locally : db::default_account;
+        MONERO_CHECK(disk.creation_request(req.creds.address, req.creds.key, flags));
+        return response{true, req.generated_locally};
+      }
+    };
+
+    struct submit_raw_tx
+    {
+      using request = rpc::submit_raw_tx_request;
+      using response = rpc::submit_raw_tx_response;
+
+      static expect<response> handle(request req, const db::storage& disk, const rpc::client& gclient)
+      {
+        using transaction_rpc = cryptonote::rpc::SendRawTxHex;
+
+        expect<rpc::client> client = gclient.clone();
+        if (!client)
+          return client.error();
+
+        transaction_rpc::Request daemon_req{};
+        daemon_req.relay = true;
+        daemon_req.tx_as_hex = std::move(req.tx);
+
+        epee::byte_slice message = rpc::client::make_message("send_raw_tx_hex", daemon_req);
+        MONERO_CHECK(client->send(std::move(message), std::chrono::seconds{10}));
+
+        const auto daemon_resp = client->receive<transaction_rpc::Response>(std::chrono::seconds{20});
+        if (!daemon_resp)
+          return daemon_resp.error();
+        if (!daemon_resp->relayed)
+          return {lws::error::tx_relay_failed};
+
+        return response{"OK"};
+      }
+    };
+
+    template<typename E>
+    expect<epee::byte_slice> call(std::string&& root, db::storage disk, const rpc::client& gclient)
+    {
+      using request = typename E::request;
+      using response = typename E::response;
+
+      expect<request> req = wire::json::from_bytes<request>(std::move(root));
+      if (!req)
+        return req.error();
+
+      expect<response> resp = E::handle(std::move(*req), std::move(disk), gclient);
+      if (!resp)
+        return resp.error();
+      return wire::json::to_bytes<response>(*resp);
+    }
+
+    struct endpoint
+    {
+      char const* const name;
+      expect<epee::byte_slice> (*const run)(std::string&&, db::storage, rpc::client const&);
+      const unsigned max_size;
+    };
+
+    constexpr const endpoint endpoints[] =
+    {
+      {"/get_address_info",      call<get_address_info>, 2 * 1024},
+      {"/get_address_txs",       call<get_address_txs>,  2 * 1024},
+      {"/get_random_outs",       call<get_random_outs>,  2 * 1024},
+      {"/get_txt_records",       nullptr,                0       },
+      {"/get_unspent_outs",      call<get_unspent_outs>, 2 * 1024},
+      {"/import_wallet_request", call<import_request>,   2 * 1024},
+      {"/login",                 call<login>,            2 * 1024},
+      {"/submit_raw_tx",         call<submit_raw_tx>,   50 * 1024}
+    };
+
+    struct by_name_
+    {
+      bool operator()(endpoint const& left, endpoint const& right) const noexcept
+      {
+        if (left.name && right.name)
+          return std::strcmp(left.name, right.name) < 0;
+        return false;
+      }
+      bool operator()(const boost::string_ref left, endpoint const& right) const noexcept
+      {
+        if (right.name)
+          return left < right.name;
+        return false;
+      }
+      bool operator()(endpoint const& left, const boost::string_ref right) const noexcept
+      {
+        if (left.name)
+          return left.name < right;
+        return false;
+      }
+    };
+    constexpr const by_name_ by_name{};
+  } // anonymous
+
+  struct rest_server::internal final : public lws::http_server_impl_base<rest_server::internal, context>
+  {
+    db::storage disk;
+    rpc::client client;
+
+    explicit internal(boost::asio::io_service& io_service, lws::db::storage disk, rpc::client client)
+      : lws::http_server_impl_base<rest_server::internal, context>(io_service)
+      , disk(std::move(disk))
+      , client(std::move(client))
+    {
+      assert(std::is_sorted(std::begin(endpoints), std::end(endpoints), by_name));
+    }
+
+    virtual bool
+    handle_http_request(const http::http_request_info& query, http::http_response_info& response, context& ctx)
+    override final
+    {
+      const auto handler = std::lower_bound(
+        std::begin(endpoints), std::end(endpoints), query.m_URI, by_name
+      );
+      if (handler == std::end(endpoints) || handler->name != query.m_URI)
+      {
+        response.m_response_code = 404;
+        response.m_response_comment = "Not Found";
+        return true;
+      }
+
+      if (handler->run == nullptr)
+      {
+        response.m_response_code = 501;
+        response.m_response_comment = "Not Implemented";
+        return true;
+      }
+
+      if (handler->max_size < query.m_body.size())
+      {
+        MINFO("Client exceeded maximum body size (" << handler->max_size << " bytes)");
+        response.m_response_code = 400;
+        response.m_response_comment = "Bad Request";
+        return true;
+      }
+
+      if (query.m_http_method != http::http_method_post)
+      {
+        response.m_response_code = 405;
+        response.m_response_comment = "Method Not Allowed";
+        return true;
+      }
+
+      // \TODO remove copy of json string here :/
+      auto body = handler->run(std::string{query.m_body}, disk.clone(), client);
+      if (!body)
+      {
+        MINFO(body.error().message() << " from " << ctx.m_remote_address.str() << " on " << handler->name);
+
+        if (body.error().category() == wire::error::rapidjson_category())
+        {
+          response.m_response_code = 400;
+          response.m_response_comment = "Bad Request";
+        }
+        else if (body == lws::error::account_not_found)
+        {
+          response.m_response_code = 403;
+          response.m_response_comment = "Forbidden";
+        }
+        else if (body.matches(std::errc::timed_out) || body.matches(std::errc::no_lock_available))
+        {
+          response.m_response_code = 503;
+          response.m_response_comment = "Service Unavailable";
+        }
+        else
+        {
+          response.m_response_code = 500;
+          response.m_response_comment = "Internal Server Error";
+        }
+        return true;
+      }
+
+      response.m_response_code = 200;
+      response.m_response_comment = "OK";
+      response.m_mime_tipe = "application/json";
+      response.m_header_info.m_content_type = "application/json";
+      response.m_body.assign(reinterpret_cast<const char*>(body->data()), body->size()); // \TODO Remove copy here too!
+      response.m_additional_fields.push_back({"Access-Control-Allow-Credentials", "true"});
+      return true;
+    }
+  };
+
+  rest_server::rest_server(epee::span<const std::string> addresses, db::storage disk, rpc::client client, configuration config)
+    : io_service_(), ports_()
+  {
+    ports_.emplace_back(io_service_, std::move(disk), std::move(client));
+
+    if (addresses.empty())
+      MONERO_THROW(common_error::kInvalidArgument, "REST server requires 1 or more addresses");
+
+    const auto init_port = [] (internal& port, const std::string& address, configuration config) -> bool
+    {
+      epee::net_utils::http::url_content url{};
+      if (!epee::net_utils::parse_url(address, url))
+        MONERO_THROW(lws::error::configuration, "REST Server URL/address is invalid");
+
+      const bool https = url.schema == "https";
+      if (!https && url.schema != "http")
+        MONERO_THROW(lws::error::configuration, "Unsupported scheme, only http or https supported");
+
+      if (std::numeric_limits<std::uint16_t>::max() < url.port)
+        MONERO_THROW(lws::error::configuration, "Specified port for rest server is out of range");
+
+      if (!https)
+      {
+        boost::system::error_code error{};
+        const boost::asio::ip::address ip_host =
+          ip_host.from_string(url.host, error);
+        if (error)
+          MONERO_THROW(lws::error::configuration, "Invalid IP address for REST server");
+        if (!ip_host.is_loopback() && !config.allow_external)
+          MONERO_THROW(lws::error::configuration, "Binding to external interface with http - consider using https or secure tunnel (ssh, etc). Use --confirm-external-bind to override");
+      }
+
+      if (url.port == 0)
+        url.port = https ? 8443 : 8080;
+
+      epee::net_utils::ssl_options_t ssl_options = https ?
+        epee::net_utils::ssl_support_t::e_ssl_support_enabled :
+        epee::net_utils::ssl_support_t::e_ssl_support_disabled;
+      ssl_options.verification = epee::net_utils::ssl_verification_t::none; // clients verified with view key
+      ssl_options.auth = std::move(config.auth);
+
+      if (!port.init(std::to_string(url.port), std::move(url.host), std::move(config.access_controls), std::move(ssl_options)))
+        MONERO_THROW(lws::error::http_server, "REST server failed to initialize");
+      return https;
+    };
+
+    bool any_ssl = false;
+    for (std::size_t index = 1; index < addresses.size(); ++index)
+    {
+      ports_.emplace_back(io_service_, ports_.front().disk.clone(), MONERO_UNWRAP(ports_.front().client.clone()));
+      any_ssl |= init_port(ports_.back(), addresses[index], config);
+    }
+
+    const bool expect_ssl = !config.auth.private_key_path.empty();
+    const std::size_t threads = config.threads;
+    any_ssl |= init_port(ports_.front(), addresses[0], std::move(config));
+    if (!any_ssl && expect_ssl)
+      MONERO_THROW(lws::error::configuration, "Specified SSL key/cert without specifying https capable REST server");
+
+    if (!ports_.front().run(threads, false))
+      MONERO_THROW(lws::error::http_server, "REST server failed to run");
+  }
+
+  rest_server::~rest_server() noexcept
+  {}
+} // lws

src/rest_server.h

@@ -0,0 +1,69 @@
+// Copyright (c) 2018-2019, The Monero Project
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#pragma once
+
+#include <boost/asio/io_service.hpp>
+#include <cstddef>
+#include <list>
+#include <string>
+#include <vector>
+
+#include "db/storage.h"
+#include "net/net_ssl.h"
+#include "rpc/client.h"
+#include "span.h"
+
+namespace lws
+{
+  class rest_server
+  {
+    struct internal;
+    
+    boost::asio::io_service io_service_;
+    std::list<internal> ports_;
+    
+  public:
+    struct configuration
+    {
+      epee::net_utils::ssl_authentication_t auth;
+      std::vector<std::string> access_controls;
+      std::size_t threads;
+      bool allow_external;
+    };
+    
+    explicit rest_server(epee::span<const std::string> addresses, db::storage disk, rpc::client client, configuration config);
+    
+    rest_server(rest_server&&) = delete;
+    rest_server(rest_server const&) = delete;
+    
+    ~rest_server() noexcept;
+    
+    rest_server& operator=(rest_server&&) = delete;
+    rest_server& operator=(rest_server const&) = delete;
+  };
+}

src/rpc/CMakeLists.txt

@@ -0,0 +1,33 @@
+# Copyright (c) 2020, The Monero Project
+#
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are
+# permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice, this list of
+#    conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice, this list
+#    of conditions and the following disclaimer in the documentation and/or other
+#    materials provided with the distribution.
+#
+# 3. Neither the name of the copyright holder nor the names of its contributors may be
+#    used to endorse or promote products derived from this software without specific
+#    prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+# THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+set(monero-lws-rpc_sources client.cpp daemon_zmq.cpp light_wallet.cpp rates.cpp)
+set(monero-lws-rpc_headers client.h daemon_zmq.h fwd.h json.h light_wallet.h rates.h)
+
+add_library(monero-lws-rpc ${monero-lws-rpc_sources} ${monero-lws-rpc_headers})
+target_link_libraries(monero-lws-rpc monero::libraries monero-lws-wire-json)

src/rpc/client.cpp

@@ -0,0 +1,325 @@
+// Copyright (c) 2018-2020, The Monero Project
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "client.h"
+
+#include <boost/thread/mutex.hpp>
+#include <cassert>
+#include <system_error>
+
+#include "common/error.h"    // monero/contrib/epee/include
+#include "error.h"
+#include "net/http_client.h" // monero/contrib/epee/include/net
+#include "net/zmq.h"         // monero/src
+
+namespace lws
+{
+namespace rpc
+{
+  namespace http = epee::net_utils::http;
+
+  namespace
+  {
+    constexpr const char signal_endpoint[] = "inproc://signal";
+    constexpr const char abort_scan_signal[] = "SCAN";
+    constexpr const char abort_process_signal[] = "PROCESS";
+    constexpr const int daemon_zmq_linger = 0;
+    
+    struct terminate
+    {
+      void operator()(void* ptr) const noexcept
+      {
+        if (ptr)
+        {
+          while (zmq_term(ptr))
+          {
+            if (zmq_errno() != EINTR)
+              break;
+          }
+        }
+      }
+    };
+    using zcontext = std::unique_ptr<void, terminate>;
+
+    expect<void> do_wait(void* daemon, void* signal_sub, short events, std::chrono::milliseconds timeout) noexcept
+    {
+      if (timeout <= std::chrono::seconds{0})
+        return {lws::error::daemon_timeout};
+
+      zmq_pollitem_t items[2] {
+        {daemon, 0, short(events | ZMQ_POLLERR), 0},
+        {signal_sub, 0, short(ZMQ_POLLIN | ZMQ_POLLERR), 0}
+      };
+
+      for (;;)
+      {
+        const auto start = std::chrono::steady_clock::now();
+        const int ready = zmq_poll(items, 2, timeout.count());
+        const auto end = std::chrono::steady_clock::now();
+        const auto spent = std::chrono::duration_cast<std::chrono::milliseconds>(start - end);
+        timeout -= std::min(spent, timeout);
+
+        if (ready == 0)
+          return {lws::error::daemon_timeout};
+        if (0 < ready)
+          break;
+        const int err = zmq_errno();
+        if (err != EINTR)
+          return net::zmq::make_error_code(err);
+      }
+      if (items[0].revents)
+        return success();
+
+      char buf[1];
+      MONERO_ZMQ_CHECK(zmq_recv(signal_sub, buf, 1, 0));
+
+      switch (buf[0])
+      {
+      case 'P':
+        return {lws::error::signal_abort_process};
+      case 'S':
+        return {lws::error::signal_abort_scan};
+      default:
+        break;
+      }
+      return {lws::error::signal_unknown};
+    }
+
+    template<std::size_t N>
+    expect<void> do_signal(void* signal_pub, const char (&signal)[N]) noexcept
+    {
+      MONERO_ZMQ_CHECK(zmq_send(signal_pub, signal, sizeof(signal), 0));
+      return success();
+    }
+
+    template<std::size_t N>
+    expect<void> do_subscribe(void* signal_sub, const char (&signal)[N]) noexcept
+    {
+      MONERO_ZMQ_CHECK(zmq_setsockopt(signal_sub, ZMQ_SUBSCRIBE, signal, sizeof(signal)));
+      return success();
+    }
+  } // anonymous
+
+  namespace detail
+  {
+    struct context
+    {
+      explicit context(zcontext comm, socket signal_pub, std::string daemon_addr, std::chrono::minutes interval)
+        : comm(std::move(comm))
+        , signal_pub(std::move(signal_pub))
+        , daemon_addr(std::move(daemon_addr))
+        , rates_conn()
+        , cache_time()
+        , cache_interval(interval)
+        , cached{}
+        , sync_rates()
+      {
+        if (std::chrono::minutes{0} < cache_interval)
+          rates_conn.set_server(crypto_compare.host, boost::none, epee::net_utils::ssl_support_t::e_ssl_support_enabled);
+      }
+
+      zcontext comm;
+      socket signal_pub;
+      std::string daemon_addr;
+      http::http_simple_client rates_conn;
+      std::chrono::steady_clock::time_point cache_time;
+      const std::chrono::minutes cache_interval;
+      rates cached;
+      boost::mutex sync_rates;
+    };
+  } // detail
+
+  expect<std::string> client::get_message(std::chrono::seconds timeout)
+  {
+    MONERO_PRECOND(ctx != nullptr);
+    assert(daemon != nullptr);
+    assert(signal_sub != nullptr);
+
+    expect<std::string> msg{common_error::kInvalidArgument};
+    while (!(msg = net::zmq::receive(daemon.get(), ZMQ_DONTWAIT)))
+    {
+      if (msg != net::zmq::make_error_code(EAGAIN))
+        break;
+
+      MONERO_CHECK(do_wait(daemon.get(), signal_sub.get(), ZMQ_POLLIN, timeout));
+      timeout = std::chrono::seconds{0};
+    }
+    // std::string move constructor is noexcept
+    return msg;
+  }
+
+  expect<client> client::make(std::shared_ptr<detail::context> ctx) noexcept
+  {
+    MONERO_PRECOND(ctx != nullptr);
+
+    const int linger = daemon_zmq_linger;
+    client out{std::move(ctx)};
+
+    out.daemon.reset(zmq_socket(out.ctx->comm.get(), ZMQ_REQ));
+    if (out.daemon.get() == nullptr)
+      return net::zmq::get_error_code();
+    MONERO_ZMQ_CHECK(zmq_connect(out.daemon.get(), out.ctx->daemon_addr.c_str()));
+    MONERO_ZMQ_CHECK(zmq_setsockopt(out.daemon.get(), ZMQ_LINGER, &linger, sizeof(linger)));
+
+    out.signal_sub.reset(zmq_socket(out.ctx->comm.get(), ZMQ_SUB));
+    if (out.signal_sub.get() == nullptr)
+      return net::zmq::get_error_code();
+    MONERO_ZMQ_CHECK(zmq_connect(out.signal_sub.get(), signal_endpoint));
+
+    MONERO_CHECK(do_subscribe(out.signal_sub.get(), abort_process_signal));
+    return {std::move(out)};
+  }
+
+  client::~client() noexcept
+  {}
+
+  expect<void> client::watch_scan_signals() noexcept
+  {
+    MONERO_PRECOND(ctx != nullptr);
+    assert(signal_sub != nullptr);
+    return do_subscribe(signal_sub.get(), abort_scan_signal);
+  }
+
+  expect<void> client::wait(std::chrono::seconds timeout) noexcept
+  {
+    MONERO_PRECOND(ctx != nullptr);
+    assert(daemon != nullptr);
+    assert(signal_sub != nullptr);
+    return do_wait(daemon.get(), signal_sub.get(), 0, timeout);
+  }
+
+  expect<void> client::send(epee::byte_slice message, std::chrono::seconds timeout) noexcept
+  {
+    MONERO_PRECOND(ctx != nullptr);
+    assert(daemon != nullptr);
+    assert(signal_sub != nullptr);
+
+    expect<void> sent;
+    while (!(sent = net::zmq::send(message.clone(), daemon.get(), ZMQ_DONTWAIT)))
+    {
+      if (sent != net::zmq::make_error_code(EAGAIN))
+        return sent.error();
+
+      MONERO_CHECK(do_wait(daemon.get(), signal_sub.get(), ZMQ_POLLOUT, timeout));
+      timeout = std::chrono::seconds{0};
+    }
+    return success();
+  }
+
+  expect<rates> client::get_rates() const
+  {
+    MONERO_PRECOND(ctx != nullptr);
+    if (ctx->cache_interval <= std::chrono::minutes{0})
+      return {lws::error::exchange_rates_disabled};
+
+    const auto now  = std::chrono::steady_clock::now();
+    const boost::unique_lock<boost::mutex> lock{ctx->sync_rates};
+    if (now - ctx->cache_time >= ctx->cache_interval + std::chrono::seconds{30})
+      return {lws::error::exchange_rates_old};
+    return ctx->cached;
+  }
+
+  context context::make(std::string daemon_addr, std::chrono::minutes rates_interval)
+  {
+    zcontext comm{zmq_init(1)};
+    if (comm == nullptr)
+      MONERO_THROW(net::zmq::get_error_code(), "zmq_init");
+
+    detail::socket pub{zmq_socket(comm.get(), ZMQ_PUB)};
+    if (pub == nullptr)
+      MONERO_THROW(net::zmq::get_error_code(), "zmq_socket");
+    if (zmq_bind(pub.get(), signal_endpoint) < 0)
+      MONERO_THROW(net::zmq::get_error_code(), "zmq_bind");
+
+    return context{
+      std::make_shared<detail::context>(
+        std::move(comm), std::move(pub), std::move(daemon_addr), rates_interval
+      )
+    };
+  }
+
+  context::~context() noexcept
+  {
+    if (ctx)
+      raise_abort_process();
+  }
+
+  std::string const& context::daemon_address() const
+  {
+    if (ctx == nullptr)
+      MONERO_THROW(common_error::kInvalidArgument, "Invalid lws::rpc::context");
+    return ctx->daemon_addr;
+  }
+
+  expect<void> context::raise_abort_scan() noexcept
+  {
+    MONERO_PRECOND(ctx != nullptr);
+    assert(ctx->signal_pub != nullptr);
+    return do_signal(ctx->signal_pub.get(), abort_scan_signal);
+  }
+
+  expect<void> context::raise_abort_process() noexcept
+  {
+    MONERO_PRECOND(ctx != nullptr);
+    assert(ctx->signal_pub != nullptr);
+    return do_signal(ctx->signal_pub.get(), abort_process_signal);
+  }
+
+  expect<boost::optional<lws::rates>> context::retrieve_rates()
+  {
+    MONERO_PRECOND(ctx != nullptr);
+
+    if (ctx->cache_interval <= std::chrono::minutes{0})
+      return boost::make_optional(false, ctx->cached);
+
+    const auto now = std::chrono::steady_clock::now();
+    if (now - ctx->cache_time < ctx->cache_interval)
+      return boost::make_optional(false, ctx->cached);
+
+    expect<rates> fresh{lws::error::exchange_rates_fetch};
+
+    const http::http_response_info* info = nullptr;
+    const bool retrieved =
+      ctx->rates_conn.invoke_get(crypto_compare.path, std::chrono::seconds{20}, std::string{}, std::addressof(info)) &&
+      info != nullptr &&
+      info->m_response_code == 200;
+
+    // \TODO Remove copy below
+    if (retrieved)
+      fresh = crypto_compare(std::string{info->m_body});
+
+    const boost::unique_lock<boost::mutex> lock{ctx->sync_rates};
+    ctx->cache_time = now;
+    if (fresh)
+    {
+      ctx->cached = *fresh;
+      return boost::make_optional(*fresh);
+    }
+    return fresh.error();
+  }
+} // rpc
+} // lws

src/rpc/client.h

@@ -0,0 +1,219 @@
+// Copyright (c) 2018-2020, The Monero Project
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#pragma once
+
+#include <boost/optional/optional.hpp>
+#include <chrono>
+#include <memory>
+#include <string>
+#include <utility>
+#include <zmq.h>
+
+#include "byte_slice.h"    // monero/contrib/epee/include
+#include "common/expect.h" // monero/src
+#include "rates.h"
+#include "rpc/message.h"   // monero/src
+
+namespace lws
+{
+namespace rpc
+{
+  namespace detail
+  {
+    struct close
+    {
+      void operator()(void* ptr) const noexcept
+      {
+        if (ptr)
+          zmq_close(ptr);
+      }
+    };
+    using socket = std::unique_ptr<void, close>;
+
+    struct context;
+  }
+
+  //! Abstraction for ZMQ RPC client. Only `get_rates()` thread-safe; use `clone()`.
+  class client
+  {
+    std::shared_ptr<detail::context> ctx;
+    detail::socket daemon;
+    detail::socket signal_sub;
+
+    explicit client(std::shared_ptr<detail::context> ctx)
+      : ctx(std::move(ctx)), daemon(), signal_sub()
+    {}
+
+  public:
+    //! A client with no connection (all send/receive functions fail).
+    explicit client() noexcept
+      : ctx(), daemon(), signal_sub()
+    {}
+
+    static expect<client> make(std::shared_ptr<detail::context> ctx) noexcept;
+
+    client(client&&) = default;
+    client(client const&) = delete;
+
+    ~client() noexcept;
+
+    client& operator=(client&&) = default;
+    client& operator=(client const&) = delete;
+
+    /*!
+      \note `watch_scan_signals()` status is not cloned.
+      \note The copy is not cheap - it creates a new ZMQ socket.
+      \return A client connected to same daemon as `this`.
+    */
+    expect<client> clone() const noexcept
+    {
+      return make(ctx);
+    }
+
+    //! \return True if `this` is valid (i.e. not default or moved from).
+    explicit operator bool() const noexcept
+    {
+      return ctx != nullptr;
+    }
+
+    //! `wait`, `send`, and `receive` will watch for `raise_abort_scan()`.
+    expect<void> watch_scan_signals() noexcept;
+
+    //! Block until `timeout` or until `context::stop()` is invoked.
+    expect<void> wait(std::chrono::seconds timeout) noexcept;
+
+    //! \return A JSON message for RPC request `M`.
+    template<typename M>
+    static epee::byte_slice make_message(char const* const name, const M& message)
+    {
+      return cryptonote::rpc::FullMessage::getRequest(name, message, 0);
+    }
+
+    /*!
+      Queue `message` for sending to daemon. If the queue is full, wait a
+      maximum of `timeout` seconds or until `context::raise_abort_scan` or
+      `context::raise_abort_process()` is called.
+    */
+    expect<void> send(epee::byte_slice message, std::chrono::seconds timeout) noexcept;
+
+    //! \return Next available RPC message response from server
+    expect<std::string> get_message(std::chrono::seconds timeout);
+
+    //! \return RPC response `M`, waiting a max of `timeout` seconds.
+    template<typename M>
+    expect<M> receive(std::chrono::seconds timeout)
+    {
+      expect<std::string> message = get_message(timeout);
+      if (!message)
+        return message.error();
+
+      cryptonote::rpc::FullMessage fm{std::move(*message)};
+      M out{};
+      out.fromJson(fm.getMessage());
+      return out;
+    }
+
+    /*!
+      \note This is the one function that IS thread-safe. Multiple threads can
+        call this function with the same `this` argument.
+
+        \return Recent exchange rates.
+    */
+    expect<rates> get_rates() const;
+  };
+
+  //! Owns ZMQ context, and ZMQ PUB socket for signalling child `client`s.
+  class context
+  {
+    std::shared_ptr<detail::context> ctx;
+
+    explicit context(std::shared_ptr<detail::context> ctx)
+      : ctx(std::move(ctx))
+    {}
+
+  public:
+    /*! Use `daemon_addr` for call child client objects.
+
+      \throw std::bad_alloc if internal `shared_ptr` allocation failed.
+      \throw std::system_error if any ZMQ errors occur.
+
+      \note All errors are exceptions; no recovery can occur.
+
+      \param daemon_addr Location of ZMQ enabled `monerod` RPC.
+      \param rates_interval Frequency to retrieve exchange rates. Set value to
+        `<= 0` to disable exchange rate retrieval.
+    */
+    static context make(std::string daemon_addr, std::chrono::minutes rates_interval);
+
+    context(context&&) = default;
+    context(context const&) = delete;
+
+    //! Calls `raise_abort_process()`. Clients can safely destruct later.
+    ~context() noexcept;
+
+    context& operator=(context&&) = default;
+    context& operator=(context const&) = delete;
+
+    // Do not create clone method, only one of these should exist right now.
+
+    //! \return The full address of the monerod ZMQ daemon.
+    std::string const& daemon_address() const;
+
+    //! \return Client connection. Thread-safe.
+    expect<client> connect() const noexcept
+    {
+      return client::make(ctx);
+    }
+
+    /*!
+      All block `client::send`, `client::receive`, and `client::wait` calls
+      originating from `this` object AND whose `watch_scan_signal` method was
+      invoked, will immediately return with `lws::error::kSignlAbortScan`. This
+      is NOT signal-safe NOR signal-safe NOR thread-safe.
+    */
+    expect<void> raise_abort_scan() noexcept;
+
+    /*!
+      All blocked `client::send`, `client::receive`, and `client::wait` calls
+      originating from `this` object will immediately return with
+      `lws::error::kSignalAbortProcess`. This call is NOT signal-safe NOR
+      thread-safe.
+    */
+    expect<void> raise_abort_process() noexcept;
+
+    /*!
+      Retrieve exchange rates, if enabled and past cache interval. Not
+      thread-safe (this can be invoked from one thread only, but this is
+      thread-safe with `client::get_rates()`). All clients will see new rates
+      immediately.
+
+      \return Rates iff they were updated.
+    */
+    expect<boost::optional<lws::rates>> retrieve_rates();
+  };
+} // rpc
+} // lws

src/rpc/daemon_zmq.cpp

@@ -0,0 +1,169 @@
+// Copyright (c) 2020, The Monero Project
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "daemon_zmq.h"
+
+#include "crypto/crypto.h"            // monero/src
+#include "rpc/message_data_structs.h" // monero/src
+#include "wire/crypto.h"
+#include "wire/json.h"
+#include "wire/vector.h"
+
+namespace
+{
+  constexpr const std::size_t default_blocks_fetched = 1000;
+  constexpr const std::size_t default_transaction_count = 100;
+  constexpr const std::size_t default_inputs = 2;
+  constexpr const std::size_t default_outputs = 4;
+  constexpr const std::size_t default_txextra_size = 2048;
+}
+
+namespace rct
+{
+  static void read_bytes(wire::json_reader& source, ctkey& self)
+  {
+    self.dest = {};
+    read_bytes(source, self.mask);
+  }
+
+  static void read_bytes(wire::json_reader& source, ecdhTuple& self)
+  {
+    wire::object(source, WIRE_FIELD(mask), WIRE_FIELD(amount));
+  }
+
+  static void read_bytes(wire::json_reader& source, rctSig& self)
+  {
+    self.outPk.reserve(default_inputs);
+    wire::object(source,
+      WIRE_FIELD(type),
+      wire::field("encrypted", std::ref(self.ecdhInfo)),
+      wire::field("commitments", std::ref(self.outPk)),
+      wire::field("fee", std::ref(self.txnFee))
+    );
+  }
+} // rct
+
+namespace cryptonote
+{
+  static void read_bytes(wire::json_reader& source, txout_to_script& self)
+  {
+    wire::object(source, WIRE_FIELD(keys), WIRE_FIELD(script));
+  }
+  static void read_bytes(wire::json_reader& source, txout_to_scripthash& self)
+  {
+    wire::object(source, WIRE_FIELD(hash));
+  }
+  static void read_bytes(wire::json_reader& source, txout_to_key& self)
+  {
+    wire::object(source, WIRE_FIELD(key));
+  }
+  static void read_bytes(wire::json_reader& source, tx_out& self)
+  {
+    wire::object(source,
+      WIRE_FIELD(amount),
+        wire::variant_field("transaction output variant", std::ref(self.target),
+        wire::option<txout_to_key>{"to_key"},
+        wire::option<txout_to_script>{"to_script"},
+        wire::option<txout_to_scripthash>{"to_scripthash"}
+      )
+    );
+  }
+
+  static void read_bytes(wire::json_reader& source, txin_gen& self)
+  {
+    wire::object(source, WIRE_FIELD(height));
+  }
+  static void read_bytes(wire::json_reader& source, txin_to_script& self)
+  {
+    wire::object(source, WIRE_FIELD(prev), WIRE_FIELD(prevout), WIRE_FIELD(sigset));
+  }
+  static void read_bytes(wire::json_reader& source, txin_to_scripthash& self)
+  {
+    wire::object(source, WIRE_FIELD(prev), WIRE_FIELD(prevout), WIRE_FIELD(script), WIRE_FIELD(sigset));
+  }
+  static void read_bytes(wire::json_reader& source, txin_to_key& self)
+  {
+    wire::object(source, WIRE_FIELD(amount), WIRE_FIELD(key_offsets), wire::field("key_image", std::ref(self.k_image)));
+  }
+  static void read_bytes(wire::json_reader& source, txin_v& self)
+  {
+    wire::object(source,
+        wire::variant_field("transaction input variant", std::ref(self),
+        wire::option<txin_to_key>{"to_key"},
+        wire::option<txin_gen>{"gen"},
+        wire::option<txin_to_script>{"to_script"},
+        wire::option<txin_to_scripthash>{"to_scripthash"}
+      )
+    );
+  }
+
+  static void read_bytes(wire::json_reader& source, transaction& self)
+  {
+    self.vin.reserve(default_inputs);
+    self.vout.reserve(default_outputs);
+    self.extra.reserve(default_txextra_size);
+    wire::object(source,
+      WIRE_FIELD(version),
+      WIRE_FIELD(unlock_time),
+      wire::field("inputs", std::ref(self.vin)),
+      wire::field("outputs", std::ref(self.vout)),
+      WIRE_FIELD(extra),
+      wire::field("ringct", std::ref(self.rct_signatures))
+    );
+  }
+
+  static void read_bytes(wire::json_reader& source, block& self)
+  {
+    self.tx_hashes.reserve(default_transaction_count);
+    wire::object(source,
+      WIRE_FIELD(major_version),
+      WIRE_FIELD(minor_version),
+      WIRE_FIELD(timestamp),
+      WIRE_FIELD(miner_tx),
+      WIRE_FIELD(tx_hashes),
+      WIRE_FIELD(prev_id),
+      WIRE_FIELD(nonce)
+    );
+  }
+
+  namespace rpc
+  {
+    static void read_bytes(wire::json_reader& source, block_with_transactions& self)
+    {
+      self.transactions.reserve(default_transaction_count);
+      wire::object(source, WIRE_FIELD(block), WIRE_FIELD(transactions));
+    }
+  } // rpc
+} // cryptonote
+
+void lws::rpc::read_bytes(wire::json_reader& source, get_blocks_fast_response& self)
+{
+  self.blocks.reserve(default_blocks_fetched);
+  self.output_indices.reserve(default_blocks_fetched);
+  wire::object(source, WIRE_FIELD(blocks), WIRE_FIELD(output_indices), WIRE_FIELD(start_height), WIRE_FIELD(current_height));
+}
+

src/rpc/daemon_zmq.h

@@ -0,0 +1,75 @@
+// Copyright (c) 2020, The Monero Project
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#pragma once
+
+#include <cstdint>
+#include <vector>
+
+#include "common/pod-class.h" // monero/src
+#include "wire/json/fwd.h"
+
+namespace crypto
+{
+  POD_CLASS hash;
+}
+
+namespace cryptonote
+{
+  namespace rpc
+  {
+    struct block_with_transactions;
+  }
+}
+
+namespace lws
+{
+namespace rpc
+{
+  struct get_blocks_fast_request
+  {
+    get_blocks_fast_request() = delete;
+    std::vector<crypto::hash> block_ids;
+    std::uint64_t start_height;
+    bool prune;
+  };
+  struct get_blocks_fast_response
+  {
+    get_blocks_fast_response() = delete;
+    std::vector<cryptonote::rpc::block_with_transactions> blocks;
+    std::vector<std::vector<std::vector<std::uint64_t>>> output_indices;
+    std::uint64_t start_height;
+    std::uint64_t current_height;
+  };
+  struct get_blocks_fast
+  {
+    using request = get_blocks_fast_request;
+    using response = get_blocks_fast_response;
+  };
+  void read_bytes(wire::json_reader&, get_blocks_fast_response&);
+} // rpc
+} // lws

src/rpc/fwd.h

@@ -0,0 +1,33 @@
+// Copyright (c) 2020, The Monero Project
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#pragma once
+
+namespace lws
+{
+  struct rates;
+}

src/rpc/json.h

@@ -0,0 +1,96 @@
+// Copyright (c) 2020, The Monero Project
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "wire/json.h"
+
+namespace lws
+{
+namespace rpc
+{
+  struct json_request_base
+  {
+    static constexpr const char jsonrpc[] = "2.0";
+
+    //! `method` must be in static memory.
+    explicit json_request_base(const char* method)
+      : id(0), method(method)
+    {}
+    
+    unsigned id;
+    const char* method; //!< Must be in static memory
+  };
+  const char json_request_base::jsonrpc[];
+
+  //! \tparam W implements the WRITE concept \tparam M implements the METHOD concept
+  template<typename W, typename M>
+  struct json_request : json_request_base
+  {
+    template<typename... U>
+    explicit json_request(U&&... args)
+      : json_request_base(M::name()),
+        params{std::forward<U>(args)...}
+    {}
+
+    W params;
+  };
+
+  template<typename W, typename M>
+  inline void write_bytes(wire::json_writer& dest, const json_request<W, M>& self)
+  {
+    // pull fields from base class into the same object
+    wire::object(dest, WIRE_FIELD_COPY(id), WIRE_FIELD_COPY(jsonrpc), WIRE_FIELD_COPY(method), WIRE_FIELD(params));
+  }
+
+
+  //! \tparam R implements the READ concept
+  template<typename R>
+  struct json_response
+  {
+    json_response() = delete;
+
+    unsigned id;
+    R result;
+  };
+
+  template<typename R>
+  inline void read_bytes(wire::json_reader& source, json_response<R>& self)
+  {
+    wire::object(source, WIRE_FIELD(id), WIRE_FIELD(result));
+  }
+
+
+  /*! Implements the RPC concept (JSON-RPC 2.0).
+    \tparam M must implement the METHOD concept. */
+  template<typename M>
+  struct json
+  {
+    using wire_type = wire::json;
+    using request = json_request<typename M::request, M>;
+    using response = json_response<typename M::response>;
+  };
+} // rpc
+} // lws

src/rpc/light_wallet.cpp

@@ -0,0 +1,338 @@
+// Copyright (c) 2018-2020, The Monero Project
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "light_wallet.h"
+
+#include <boost/range/adaptor/indexed.hpp>
+#include <ctime>
+#include <limits>
+#include <stdexcept>
+#include <type_traits>
+
+#include "db/string.h"
+#include "error.h"
+#include "misc_os_dependent.h" // monero/contrib/epee/include
+#include "ringct/rctOps.h"     // monero/src
+#include "span.h"              // monero/contrib/epee/include
+#include "util/random_outputs.h"
+#include "wire/crypto.h"
+#include "wire/error.h"
+#include "wire/json.h"
+#include "wire/traits.h"
+#include "wire/vector.h"
+
+namespace
+{
+  enum class iso_timestamp : std::uint64_t {};
+
+  struct rct_bytes
+  {
+    rct::key commitment;
+    rct::key mask;
+    rct::key amount;
+  };
+  static_assert(sizeof(rct_bytes) == 32 * 3, "padding in rct struct");
+
+  struct expand_outputs
+  {
+    const std::pair<lws::db::output, std::vector<crypto::key_image>>& data;
+    const crypto::secret_key& user_key;
+  };
+} // anonymous
+
+namespace wire
+{
+  template<>
+  struct is_blob<rct_bytes>
+    : std::true_type
+  {};
+}
+
+namespace
+{
+  void write_bytes(wire::json_writer& dest, const iso_timestamp self)
+  {
+    static_assert(std::is_integral<std::time_t>::value, "unexpected  time_t type");
+    if (std::numeric_limits<std::time_t>::max() < std::uint64_t(self))
+      throw std::runtime_error{"Exceeded max time_t value"};
+
+    std::tm value;
+    if (!epee::misc_utils::get_gmt_time(std::time_t(self), value))
+      throw std::runtime_error{"Failed to convert std::time_t to std::tm"};
+
+    char buf[28] = {0};
+    if (sizeof(buf) - 1 != std::strftime(buf, sizeof(buf), "%Y-%m-%dT%H:%M:%S.0-00:00", std::addressof(value)))
+      throw std::runtime_error{"strftime failed"};
+
+    dest.string({buf, sizeof(buf) - 1});
+  }
+
+  void write_bytes(wire::json_writer& dest, const expand_outputs self)
+  {
+    /*! \TODO Sending the public key for the output isn't necessary, as it can be
+      re-computed from the other parts. Same with the rct commitment and rct
+      amount. Consider dropping these from the API after client upgrades. Not
+      storing them in the DB saves 96-bytes per received out. */
+
+    rct_bytes rct{};
+    rct_bytes const* optional_rct = nullptr;
+    if (unpack(self.data.first.extra).first & lws::db::ringct_output)
+    {
+      crypto::key_derivation derived;
+      if (!crypto::generate_key_derivation(self.data.first.spend_meta.tx_public, self.user_key, derived))
+        MONERO_THROW(lws::error::crypto_failure, "generate_key_derivation failed");
+
+      crypto::secret_key scalar;
+      rct::ecdhTuple encrypted{self.data.first.ringct_mask, rct::d2h(self.data.first.spend_meta.amount)};
+
+      crypto::derivation_to_scalar(derived, self.data.first.spend_meta.index, scalar);
+      rct::ecdhEncode(encrypted, rct::sk2rct(scalar), false);
+
+      rct.commitment = rct::commit(self.data.first.spend_meta.amount, self.data.first.ringct_mask);
+      rct.mask = encrypted.mask;
+      rct.amount = encrypted.amount;
+
+      optional_rct = std::addressof(rct);
+    }
+
+    wire::object(dest,
+      wire::field("amount", lws::rpc::safe_uint64(self.data.first.spend_meta.amount)),
+      wire::field("public_key", self.data.first.pub),
+      wire::field("index", self.data.first.spend_meta.index),
+      wire::field("global_index", self.data.first.spend_meta.id.low),
+      wire::field("tx_id", self.data.first.spend_meta.id.low),
+      wire::field("tx_hash", std::cref(self.data.first.link.tx_hash)),
+      wire::field("tx_prefix_hash", std::cref(self.data.first.tx_prefix_hash)),
+      wire::field("tx_pub_key", self.data.first.spend_meta.tx_public),
+      wire::field("timestamp", iso_timestamp(self.data.first.timestamp)),
+      wire::field("height", self.data.first.link.height),
+      wire::field("spend_key_images", std::cref(self.data.second)),
+      wire::optional_field("rct", optional_rct)
+    );
+  }
+
+  void convert_address(const boost::string_ref source, lws::db::account_address& dest)
+  {
+    expect<lws::db::account_address> bytes = lws::db::address_string(source);
+    if (!bytes)
+      WIRE_DLOG_THROW(wire::error::schema::fixed_binary, "invalid Monero address format - " << bytes.error());
+    dest = std::move(*bytes);
+  }
+} // anonymous
+
+namespace lws
+{
+  static void write_bytes(wire::json_writer& dest, random_output const& self)
+  {
+    const rct_bytes rct{self.keys.mask, rct::zero(), rct::zero()};
+    wire::object(dest,
+      wire::field("global_index", rpc::safe_uint64(self.index)),
+      wire::field("public_key", std::cref(self.keys.key)),
+      wire::field("rct", std::cref(rct))
+    );
+  }
+  static void write_bytes(wire::json_writer& dest, random_ring const& self)
+  {
+    wire::object(dest,
+      wire::field("amount", rpc::safe_uint64(self.amount)),
+      wire::field("outputs", std::cref(self.ring))
+    );
+  };
+
+  void rpc::read_bytes(wire::json_reader& source, safe_uint64& self)
+  {
+    self = safe_uint64(wire::integer::convert_to<std::uint64_t>(source.safe_unsigned_integer()));
+  }
+  void rpc::write_bytes(wire::json_writer& dest, const safe_uint64 self)
+  {
+    auto buf = wire::json_writer::to_string(std::uint64_t(self));
+    dest.string(buf.data());
+  }
+  void rpc::read_bytes(wire::json_reader& source, safe_uint64_array& self)
+  {
+    for (std::size_t count = source.start_array(); !source.is_array_end(count); --count)
+      self.values.emplace_back(wire::integer::convert_to<std::uint64_t>(source.safe_unsigned_integer()));
+    source.end_array();
+  }
+
+  void rpc::read_bytes(wire::json_reader& source, account_credentials& self)
+  {
+    std::string address;
+    wire::object(source,
+      wire::field("address", std::ref(address)),
+      wire::field("view_key", std::ref(unwrap(unwrap(self.key))))
+    );
+    convert_address(address, self.address);
+  }
+
+  void rpc::write_bytes(wire::json_writer& dest, const transaction_spend& self)
+  {
+    wire::object(dest,
+      wire::field("amount", safe_uint64(self.meta.amount)),
+      wire::field("key_image", std::cref(self.possible_spend.image)),
+      wire::field("tx_pub_key", std::cref(self.meta.tx_public)),
+      wire::field("out_index", self.meta.index),
+      wire::field("mixin", self.possible_spend.mixin_count)
+    );
+  }
+
+  void rpc::write_bytes(wire::json_writer& dest, const get_address_info_response& self)
+  {
+    wire::object(dest,
+      WIRE_FIELD_COPY(locked_funds),
+      WIRE_FIELD_COPY(total_received),
+      WIRE_FIELD_COPY(total_sent),
+      WIRE_FIELD_COPY(scanned_height),
+      WIRE_FIELD_COPY(scanned_block_height),
+      WIRE_FIELD_COPY(start_height),
+      WIRE_FIELD_COPY(transaction_height),
+      WIRE_FIELD_COPY(blockchain_height),
+      WIRE_FIELD(spent_outputs),
+      WIRE_OPTIONAL_FIELD(rates)
+    );
+  }
+
+  namespace rpc
+  {
+    static void write_bytes(wire::json_writer& dest, boost::range::index_value<const get_address_txs_response::transaction&> self)
+    {
+      epee::span<const std::uint8_t> const* payment_id = nullptr;
+      epee::span<const std::uint8_t> payment_id_bytes;
+
+      const auto extra = db::unpack(self.value().info.extra);
+      if (extra.second)
+      {
+        payment_id = std::addressof(payment_id_bytes);
+
+        if (extra.second == sizeof(self.value().info.payment_id.short_))
+          payment_id_bytes = epee::as_byte_span(self.value().info.payment_id.short_);
+        else
+          payment_id_bytes = epee::as_byte_span(self.value().info.payment_id.long_);
+      }
+
+      const bool is_coinbase = (extra.first & db::coinbase_output);
+
+      wire::object(dest,
+        wire::field("id", std::uint64_t(self.index())),
+        wire::field("hash", std::cref(self.value().info.link.tx_hash)),
+        wire::field("timestamp", iso_timestamp(self.value().info.timestamp)),
+        wire::field("total_received", safe_uint64(self.value().info.spend_meta.amount)),
+        wire::field("total_sent", safe_uint64(self.value().spent)),
+        wire::field("unlock_time", self.value().info.unlock_time),
+        wire::field("height", self.value().info.link.height),
+        wire::optional_field("payment_id", payment_id),
+        wire::field("coinbase", is_coinbase),
+        wire::field("mempool", false),
+        wire::field("mixin", self.value().info.spend_meta.mixin_count),
+        wire::field("spent_outputs", std::cref(self.value().spends))
+      );
+    }
+  } // rpc
+  void rpc::write_bytes(wire::json_writer& dest, const get_address_txs_response& self)
+  {
+    wire::object(dest,
+      wire::field("total_received", safe_uint64(self.total_received)),
+      WIRE_FIELD_COPY(scanned_height),
+      WIRE_FIELD_COPY(scanned_block_height),
+      WIRE_FIELD_COPY(start_height),
+      WIRE_FIELD_COPY(transaction_height),
+      WIRE_FIELD_COPY(blockchain_height),
+      wire::field("transactions", wire::as_array(boost::adaptors::index(self.transactions)))
+    );
+  }
+
+  void rpc::read_bytes(wire::json_reader& source, get_random_outs_request& self)
+  {
+    wire::object(source, WIRE_FIELD(count), WIRE_FIELD(amounts));
+  }
+  void rpc::write_bytes(wire::json_writer& dest, const get_random_outs_response& self)
+  {
+    wire::object(dest, WIRE_FIELD(amount_outs));
+  }
+
+  void rpc::read_bytes(wire::json_reader& source, get_unspent_outs_request& self)
+  {
+    std::string address;
+    wire::object(source,
+      wire::field("address", std::ref(address)),
+      wire::field("view_key", std::ref(unwrap(unwrap(self.creds.key)))),
+      WIRE_FIELD(amount),
+      WIRE_OPTIONAL_FIELD(mixin),
+      WIRE_OPTIONAL_FIELD(use_dust),
+      WIRE_OPTIONAL_FIELD(dust_threshold)
+    );
+    convert_address(address, self.creds.address);
+  }
+  void rpc::write_bytes(wire::json_writer& dest, const get_unspent_outs_response& self)
+  {
+    const auto expand = [&self] (const std::pair<db::output, std::vector<crypto::key_image>>& src)
+    {
+      return expand_outputs{src, self.user_key};
+    };
+    wire::object(dest,
+      WIRE_FIELD_COPY(per_kb_fee),
+      WIRE_FIELD_COPY(fee_mask),
+      WIRE_FIELD_COPY(amount),
+      wire::field("outputs", wire::as_array(std::cref(self.outputs), expand))
+    );
+  }
+
+  void rpc::write_bytes(wire::json_writer& dest, const import_response& self)
+  {
+    wire::object(dest,
+      WIRE_FIELD_COPY(import_fee),
+      WIRE_FIELD_COPY(status),
+      WIRE_FIELD_COPY(new_request),
+      WIRE_FIELD_COPY(request_fulfilled)
+    );
+  }
+
+  void rpc::read_bytes(wire::json_reader& source, login_request& self)
+  {
+    std::string address;
+    wire::object(source,
+      wire::field("address", std::ref(address)),
+      wire::field("view_key", std::ref(unwrap(unwrap(self.creds.key)))),
+      WIRE_FIELD(create_account),
+      WIRE_FIELD(generated_locally)
+    );
+    convert_address(address, self.creds.address);
+  }
+  void rpc::write_bytes(wire::json_writer& dest, const login_response self)
+  {
+    wire::object(dest, WIRE_FIELD_COPY(new_address), WIRE_FIELD_COPY(generated_locally));
+  }
+
+  void rpc::read_bytes(wire::json_reader& source, submit_raw_tx_request& self)
+  {
+    wire::object(source, WIRE_FIELD(tx));
+  }
+  void rpc::write_bytes(wire::json_writer& dest, const submit_raw_tx_response self)
+  {
+    wire::object(dest, WIRE_FIELD_COPY(status));
+  }
+} // lws

src/rpc/light_wallet.h

@@ -0,0 +1,198 @@
+// Copyright (c) 2018-2020, The Monero Project
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#pragma once
+
+#include <boost/optional/optional.hpp>
+#include <cstdint>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "common/expect.h" // monero/src
+#include "crypto/crypto.h" // monero/src
+#include "db/data.h"
+#include "rpc/rates.h"
+#include "util/fwd.h"
+#include "wire/json/fwd.h"
+
+namespace lws
+{
+namespace rpc
+{
+  //! Read/write uint64 value as JSON string.
+  enum class safe_uint64 : std::uint64_t {};
+  void read_bytes(wire::json_reader&, safe_uint64&);
+  void write_bytes(wire::json_writer&, safe_uint64);
+
+  //! Read an array of uint64 values as JSON strings.
+  struct safe_uint64_array
+  {
+    std::vector<std::uint64_t> values; // so this can be passed to another function without copy
+  };
+  void read_bytes(wire::json_reader&, safe_uint64_array&);
+
+
+  struct account_credentials
+  {
+    lws::db::account_address address;
+    crypto::secret_key key;
+  };
+  void read_bytes(wire::json_reader&, account_credentials&);
+
+
+  struct transaction_spend
+  {
+    transaction_spend() = delete;
+    lws::db::output::spend_meta_ meta;
+    lws::db::spend possible_spend;
+  };
+  void write_bytes(wire::json_writer&, const transaction_spend&);
+
+
+  struct get_address_info_response
+  {
+    get_address_info_response() = delete;
+    safe_uint64 locked_funds;
+    safe_uint64 total_received;
+    safe_uint64 total_sent;
+    std::uint64_t scanned_height;
+    std::uint64_t scanned_block_height;
+    std::uint64_t start_height;
+    std::uint64_t transaction_height;
+    std::uint64_t blockchain_height;
+    std::vector<transaction_spend> spent_outputs;
+    expect<lws::rates> rates;
+  };
+  void write_bytes(wire::json_writer&, const get_address_info_response&);
+
+
+  struct get_address_txs_response
+  {
+    get_address_txs_response() = delete;
+    struct transaction
+    {
+      transaction() = delete;
+      db::output info;
+      std::vector<transaction_spend> spends;
+      std::uint64_t spent;
+    };
+
+    safe_uint64 total_received;
+    std::uint64_t scanned_height;
+    std::uint64_t scanned_block_height;
+    std::uint64_t start_height;
+    std::uint64_t transaction_height;
+    std::uint64_t blockchain_height;
+    std::vector<transaction> transactions;
+  };
+  void write_bytes(wire::json_writer&, const get_address_txs_response&);
+
+
+  struct get_random_outs_request
+  {
+    get_random_outs_request() = delete;
+    std::uint64_t count;
+    safe_uint64_array amounts;
+  };
+  void read_bytes(wire::json_reader&, get_random_outs_request&);
+
+  struct get_random_outs_response
+  {
+    get_random_outs_response() = delete;
+    std::vector<random_ring> amount_outs;
+  };
+  void write_bytes(wire::json_writer&, const get_random_outs_response&);
+
+
+  struct get_unspent_outs_request
+  {
+    get_unspent_outs_request() = delete;
+    safe_uint64 amount;
+    boost::optional<safe_uint64> dust_threshold;
+    boost::optional<std::uint32_t> mixin;
+    boost::optional<bool> use_dust;
+    account_credentials creds;
+  };
+  void read_bytes(wire::json_reader&, get_unspent_outs_request&);
+
+  struct get_unspent_outs_response
+  {
+    get_unspent_outs_response() = delete;
+    std::uint64_t per_kb_fee;
+    std::uint64_t fee_mask;
+    safe_uint64 amount;
+    std::vector<std::pair<db::output, std::vector<crypto::key_image>>> outputs;
+    crypto::secret_key user_key;
+  };
+  void write_bytes(wire::json_writer&, const get_unspent_outs_response&);
+
+
+  struct import_response
+  {
+    import_response() = delete;
+    safe_uint64 import_fee;
+    const char* status;
+    bool new_request;
+    bool request_fulfilled;
+  };
+  void write_bytes(wire::json_writer&, const import_response&);
+
+
+  struct login_request
+  {
+    login_request() = delete;
+    account_credentials creds;
+    bool create_account;
+    bool generated_locally;
+  };
+  void read_bytes(wire::json_reader&, login_request&);
+
+  struct login_response
+  {
+    login_response() = delete;
+    bool new_address;
+    bool generated_locally;
+  };
+  void write_bytes(wire::json_writer&, login_response);
+
+
+  struct submit_raw_tx_request
+  {
+    submit_raw_tx_request() = delete;
+    std::string tx;
+  };
+  void read_bytes(wire::json_reader&, submit_raw_tx_request&);
+
+  struct submit_raw_tx_response
+  {
+    submit_raw_tx_response() = delete;
+    const char* status;
+  };
+  void write_bytes(wire::json_writer&, submit_raw_tx_response);
+} // rpc
+} // lws

src/rpc/rates.cpp

@@ -0,0 +1,78 @@
+// Copyright (c) 2018-2020, The Monero Project
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#include "rates.h"
+
+#include "wire/json.h"
+
+namespace
+{
+  template<typename F, typename T>
+  void map_rates(F& format, T& self)
+  {
+    wire::object(format,
+      WIRE_FIELD(AUD),
+      WIRE_FIELD(BRL),
+      WIRE_FIELD(BTC),
+      WIRE_FIELD(CAD),
+      WIRE_FIELD(CHF),
+      WIRE_FIELD(CNY),
+      WIRE_FIELD(EUR),
+      WIRE_FIELD(GBP),
+      WIRE_FIELD(HKD),
+      WIRE_FIELD(INR),
+      WIRE_FIELD(JPY),
+      WIRE_FIELD(KRW),
+      WIRE_FIELD(MXN),
+      WIRE_FIELD(NOK),
+      WIRE_FIELD(NZD),
+      WIRE_FIELD(SEK),
+      WIRE_FIELD(SGD),
+      WIRE_FIELD(TRY),
+      WIRE_FIELD(USD),
+      WIRE_FIELD(RUB),
+      WIRE_FIELD(ZAR)
+    );
+  }
+}
+
+namespace lws
+{
+  WIRE_JSON_DEFINE_OBJECT(rates, map_rates);
+
+  namespace rpc
+  {
+    const char crypto_compare_::host[] = "https://min-api.cryptocompare.com:443";
+    const char crypto_compare_::path[] =
+      "/data/price?fsym=XMR&tsyms=AUD,BRL,BTC,CAD,CHF,CNY,EUR,GBP,"
+      "HKD,INR,JPY,KRW,MXN,NOK,NZD,SEK,SGD,TRY,USD,RUB,ZAR";
+
+    expect<lws::rates> crypto_compare_::operator()(std::string&& body) const
+    {
+      return wire::json::from_bytes<lws::rates>(std::move(body));
+    }
+  } // rpc
+} // lws

src/rpc/rates.h

@@ -0,0 +1,74 @@
+// Copyright (c) 2018-2020, The Monero Project
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#pragma once
+
+#include <string>
+
+#include "byte_slice.h"
+#include "common/expect.h"
+#include "wire/json/fwd.h"
+
+namespace lws
+{
+  struct rates
+  {
+    double AUD;
+    double BRL;
+    double BTC;
+    double CAD;
+    double CHF;
+    double CNY;
+    double EUR;
+    double GBP;
+    double HKD;
+    double INR;
+    double JPY;
+    double KRW;
+    double MXN;
+    double NOK;
+    double NZD;
+    double SEK;
+    double SGD;
+    double TRY;
+    double USD;
+    double RUB;
+    double ZAR;
+  };
+  WIRE_JSON_DECLARE_OBJECT(rates);
+
+  namespace rpc
+  {
+    struct crypto_compare_
+    {
+      static const char host[];
+      static const char path[];
+
+      expect<lws::rates> operator()(std::string&& body) const;
+    };
+    constexpr const crypto_compare_ crypto_compare{};
+  } // rpc
+} // lws

src/scanner.cpp

@@ -0,0 +1,780 @@
+// Copyright (c) 2018-2020, The Monero Project
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#include "scanner.h"
+
+#include <algorithm>
+#include <boost/numeric/conversion/cast.hpp>
+#include <boost/range/combine.hpp>
+#include <boost/thread/condition_variable.hpp>
+#include <boost/thread/mutex.hpp>
+#include <boost/thread/thread.hpp>
+#include <cassert>
+#include <chrono>
+#include <cstring>
+#include <type_traits>
+#include <utility>
+
+#include "common/error.h"                             // monero/src
+#include "crypto/crypto.h"                            // monero/src
+#include "crypto/wallet/crypto.h"                     // monero/src
+#include "cryptonote_basic/cryptonote_basic.h"        // monero/src
+#include "cryptonote_basic/cryptonote_format_utils.h" // monero/src
+#include "db/account.h"
+#include "db/data.h"
+#include "error.h"
+#include "misc_log_ex.h"             // monero/contrib/epee/include
+#include "rpc/daemon_messages.h"     // monero/src
+#include "rpc/daemon_zmq.h"
+#include "rpc/json.h"
+#include "util/transactions.h"
+#include "wire/json.h"
+
+#include "serialization/json_object.h"
+
+#undef MONERO_DEFAULT_LOG_CATEGORY
+#define MONERO_DEFAULT_LOG_CATEGORY "lws"
+
+namespace lws
+{
+  std::atomic<bool> scanner::running{true};
+
+  // Not in `rates.h` - defaulting to JSON output seems odd
+  std::ostream& operator<<(std::ostream& out, lws::rates const& src)
+  {
+    wire::json_stream_writer dest{out};
+    lws::write_bytes(dest, src);
+    dest.finish();
+    return out;
+  }
+
+  namespace
+  {
+    constexpr const std::chrono::seconds account_poll_interval{10};
+    constexpr const std::chrono::seconds block_poll_interval{20};
+    constexpr const std::chrono::minutes block_rpc_timeout{2};
+    constexpr const std::chrono::seconds send_timeout{30};
+    constexpr const std::chrono::seconds sync_rpc_timeout{30};
+
+    struct thread_sync
+    {
+      boost::mutex sync;
+      boost::condition_variable user_poll;
+      std::atomic<bool> update;
+    };
+
+    struct thread_data
+    {
+      explicit thread_data(rpc::client client, db::storage disk, std::vector<lws::account> users)
+        : client(std::move(client)), disk(std::move(disk)), users(std::move(users))
+      {}
+
+      rpc::client client;
+      db::storage disk;
+      std::vector<lws::account> users;
+    };
+
+    // until we have a signal-handler safe notification system
+    void checked_wait(const std::chrono::nanoseconds wait)
+    {
+      static constexpr const std::chrono::milliseconds interval{500};
+
+      const auto start = std::chrono::steady_clock::now();
+      while (scanner::is_running())
+      {
+        const auto current = std::chrono::steady_clock::now() - start;
+        if (wait <= current)
+          break;
+        const auto sleep_time = std::min(wait - current, std::chrono::nanoseconds{interval});
+        boost::this_thread::sleep_for(boost::chrono::nanoseconds{sleep_time.count()});
+      }
+    }
+
+    bool send(rpc::client& client, epee::byte_slice message)
+    {
+      const expect<void> sent = client.send(std::move(message), send_timeout);
+      if (!sent)
+      {
+        if (sent.matches(std::errc::interrupted))
+          return false;
+        MONERO_THROW(sent.error(), "Failed to send ZMQ RPC message");
+      }
+      return true;
+    }
+
+    struct by_height
+    {
+      bool operator()(account const& left, account const& right) const noexcept
+      {
+        return left.scan_height() < right.scan_height();
+      }
+    };
+
+    void scan_transaction(
+      epee::span<lws::account> users,
+      const db::block_id height,
+      const std::uint64_t timestamp,
+      crypto::hash const& tx_hash,
+      cryptonote::transaction const& tx,
+      std::vector<std::uint64_t> const& out_ids)
+    {
+      if (2 < tx.version)
+        throw std::runtime_error{"Unsupported tx version"};
+
+      cryptonote::tx_extra_pub_key key;
+      boost::optional<crypto::hash> prefix_hash;
+      boost::optional<cryptonote::tx_extra_nonce> extra_nonce;
+      std::pair<std::uint8_t, db::output::payment_id_> payment_id;
+
+      {
+        std::vector<cryptonote::tx_extra_field> extra;
+        cryptonote::parse_tx_extra(tx.extra, extra);
+        // allow partial parsing of tx extra (similar to wallet2.cpp)
+
+        if (!cryptonote::find_tx_extra_field_by_type(extra, key))
+          return;
+
+        extra_nonce.emplace();
+        if (cryptonote::find_tx_extra_field_by_type(extra, *extra_nonce))
+        {
+          if (cryptonote::get_payment_id_from_tx_extra_nonce(extra_nonce->nonce, payment_id.second.long_))
+            payment_id.first = sizeof(crypto::hash);
+        }
+        else
+          extra_nonce = boost::none;
+      } // destruct `extra` vector
+
+      for (account& user : users)
+      {
+        if (height <= user.scan_height())
+          continue; // to next user
+
+        crypto::key_derivation derived;
+        if (!crypto::wallet::generate_key_derivation(key.pub_key, user.view_key(), derived))
+          throw std::runtime_error{"Key derivation failed"};
+
+        db::extra ext{};
+        std::uint32_t mixin = 0;
+        for (auto const& in : tx.vin)
+        {
+          cryptonote::txin_to_key const* const in_data =
+            boost::get<cryptonote::txin_to_key>(std::addressof(in));
+          if (in_data)
+          {
+            mixin = boost::numeric_cast<std::uint32_t>(
+              std::max(std::size_t(1), in_data->key_offsets.size()) - 1
+            );
+
+            std::uint64_t goffset = 0;
+            for (std::uint64_t offset : in_data->key_offsets)
+            {
+              goffset += offset;
+              if (user.has_spendable(db::output_id{in_data->amount, goffset}))
+              {
+                user.add_spend(
+                  db::spend{
+                    db::transaction_link{height, tx_hash},
+                    in_data->k_image,
+                    db::output_id{in_data->amount, goffset},
+                    timestamp,
+                    tx.unlock_time,
+                    mixin,
+                    {0, 0, 0}, // reserved
+                    payment_id.first,
+                    payment_id.second.long_
+                  }
+                );
+              }
+            }
+          }
+          else if (boost::get<cryptonote::txin_gen>(std::addressof(in)))
+            ext = db::extra(ext | db::coinbase_output);
+        }
+
+        std::size_t index = -1;
+        for (auto const& out : tx.vout)
+        {
+          ++index;
+
+          cryptonote::txout_to_key const* const out_data =
+            boost::get<cryptonote::txout_to_key>(std::addressof(out.target));
+          if (!out_data)
+            continue; // to next output
+
+          crypto::public_key derived_pub;
+          const bool received =
+            crypto::wallet::derive_subaddress_public_key(out_data->key, derived, index, derived_pub) &&
+            derived_pub == user.spend_public();
+
+          if (!received)
+            continue; // to next output
+
+          if (!prefix_hash)
+          {
+            prefix_hash.emplace();
+            cryptonote::get_transaction_prefix_hash(tx, *prefix_hash);
+          }
+
+          std::uint64_t amount = out.amount;
+          rct::key mask = rct::identity();
+          if (!amount && !(ext & db::coinbase_output) && 1 < tx.version)
+          {
+            const bool bulletproof2 = (tx.rct_signatures.type == rct::RCTTypeBulletproof2);
+            const auto decrypted = lws::decode_amount(
+              tx.rct_signatures.outPk.at(index).mask, tx.rct_signatures.ecdhInfo.at(index), derived, index, bulletproof2
+            );
+            if (!decrypted)
+            {
+              MWARNING(user.address() << " failed to decrypt amount for tx " << tx_hash << ", skipping output");
+              continue; // to next output
+            }
+            amount = decrypted->first;
+            mask = decrypted->second;
+            ext = db::extra(ext | db::ringct_output);
+          }
+
+          if (extra_nonce)
+          {
+            if (!payment_id.first && cryptonote::get_encrypted_payment_id_from_tx_extra_nonce(extra_nonce->nonce, payment_id.second.short_))
+            {
+              payment_id.first = sizeof(crypto::hash8);
+              lws::decrypt_payment_id(payment_id.second.short_, derived);
+            }
+          }
+
+          const bool added = user.add_out(
+            db::output{
+              db::transaction_link{height, tx_hash},
+              db::output::spend_meta_{
+                db::output_id{out.amount, out_ids.at(index)},
+                amount,
+                mixin,
+                boost::numeric_cast<std::uint32_t>(index),
+                key.pub_key
+              },
+              timestamp,
+              tx.unlock_time,
+              *prefix_hash,
+              out_data->key,
+              mask,
+              {0, 0, 0, 0, 0, 0, 0}, // reserved bytes
+              db::pack(ext, payment_id.first),
+              payment_id.second
+            }
+          );
+
+          if (!added)
+            MWARNING("Output not added, duplicate public key encountered");
+        } // for all tx outs
+      } // for all users
+    }
+
+    void update_rates(rpc::context& ctx)
+    {
+      const expect<boost::optional<lws::rates>> new_rates = ctx.retrieve_rates();
+      if (!new_rates)
+        MERROR("Failed to retrieve exchange rates: " << new_rates.error().message());
+      else if (*new_rates)
+        MINFO("Updated exchange rates: " << *(*new_rates));
+    }
+
+    void scan_loop(thread_sync& self, std::shared_ptr<thread_data> data) noexcept
+    {
+      try
+      {
+        // boost::thread doesn't support move-only types + attributes
+        rpc::client client{std::move(data->client)};
+        db::storage disk{std::move(data->disk)};
+        std::vector<lws::account> users{std::move(data->users)};
+
+        assert(!users.empty());
+        assert(std::is_sorted(users.begin(), users.end(), by_height{}));
+
+        data.reset();
+
+        struct stop_
+        {
+          thread_sync& self;
+          ~stop_() noexcept
+          {
+            self.update = true;
+            self.user_poll.notify_one();
+          }
+        } stop{self};
+
+        // RPC server assumes that `start_height == 0` means use
+        // block ids. This technically skips genesis block.
+        cryptonote::rpc::GetBlocksFast::Request req{};
+        req.start_height = std::uint64_t(users.begin()->scan_height());
+        req.start_height = std::max(std::uint64_t(1), req.start_height);
+        req.prune = true;
+
+        epee::byte_slice block_request = rpc::client::make_message("get_blocks_fast", req);
+        if (!send(client, block_request.clone()))
+          return;
+
+        std::vector<crypto::hash> blockchain{};
+
+        while (!self.update && scanner::is_running())
+        {
+          blockchain.clear();
+
+          auto resp = client.get_message(block_rpc_timeout);
+          if (!resp)
+          {
+            if (resp.matches(std::errc::interrupted))
+              return; // a signal was sent over ZMQ
+            if (resp.matches(std::errc::timed_out))
+            {
+              MWARNING("Block retrieval timeout, retrying");
+              if (!send(client, block_request.clone()))
+                return;
+              continue;
+            }
+            MONERO_THROW(resp.error(), "Failed to retrieve blocks from daemon");
+          }
+
+          auto fetched = MONERO_UNWRAP(wire::json::from_bytes<rpc::json<rpc::get_blocks_fast>::response>(std::move(*resp)));
+          if (fetched.result.blocks.empty())
+            throw std::runtime_error{"Daemon unexpectedly returned zero blocks"};
+
+          if (fetched.result.start_height != req.start_height)
+          {
+            MWARNING("Daemon sent wrong blocks, resetting state");
+            return;
+          }
+
+          // retrieve next blocks in background
+          req.start_height = fetched.result.start_height + fetched.result.blocks.size() - 1;
+          block_request = rpc::client::make_message("get_blocks_fast", req);
+          if (!send(client, block_request.clone()))
+            return;
+
+          if (fetched.result.blocks.size() <= 1)
+          {
+            // ... how about some ZMQ push stuff? we can only dream ...
+            if (client.wait(block_poll_interval).matches(std::errc::interrupted))
+              return;
+            continue;
+          }
+
+          if (fetched.result.blocks.size() != fetched.result.output_indices.size())
+            throw std::runtime_error{"Bad daemon response - need same number of blocks and indices"};
+
+          blockchain.push_back(cryptonote::get_block_hash(fetched.result.blocks.front().block));
+
+          auto blocks = epee::to_span(fetched.result.blocks);
+          auto indices = epee::to_span(fetched.result.output_indices);
+
+          if (fetched.result.start_height != 1)
+          {
+            // skip overlap block
+            blocks.remove_prefix(1);
+            indices.remove_prefix(1);
+          }
+          else
+            fetched.result.start_height = 0;
+
+          for (auto block_data : boost::combine(blocks, indices))
+          {
+            ++(fetched.result.start_height);
+
+            cryptonote::block const& block = boost::get<0>(block_data).block;
+            auto const& txes = boost::get<0>(block_data).transactions;
+
+            if (block.tx_hashes.size() != txes.size())
+              throw std::runtime_error{"Bad daemon response - need same number of txes and tx hashes"};
+
+            auto indices = epee::to_span(boost::get<1>(block_data));
+            if (indices.empty())
+              throw std::runtime_error{"Bad daemon response - missing /coinbase tx indices"};
+
+            crypto::hash miner_tx_hash;
+            if (!cryptonote::get_transaction_hash(block.miner_tx, miner_tx_hash))
+              throw std::runtime_error{"Failed to calculate miner tx hash"};
+
+            scan_transaction(
+              epee::to_mut_span(users),
+              db::block_id(fetched.result.start_height),
+              block.timestamp,
+              miner_tx_hash,
+              block.miner_tx,
+              *(indices.begin())
+            );
+
+            indices.remove_prefix(1);
+            if (txes.size() != indices.size())
+              throw std::runtime_error{"Bad daemon respnse - need same number of txes and indices"};
+
+            for (auto tx_data : boost::combine(block.tx_hashes, txes, indices))
+            {
+              scan_transaction(
+                epee::to_mut_span(users),
+                db::block_id(fetched.result.start_height),
+                block.timestamp,
+                boost::get<0>(tx_data),
+                boost::get<1>(tx_data),
+                boost::get<2>(tx_data)
+              );
+            }
+
+            blockchain.push_back(cryptonote::get_block_hash(block));
+          } // for each block
+
+          expect<std::size_t> updated = disk.update(
+            users.front().scan_height(), epee::to_span(blockchain), epee::to_span(users)
+          );
+          if (!updated)
+          {
+            if (updated == lws::error::blockchain_reorg)
+            {
+              epee::byte_stream dest{};
+              {
+                rapidjson::Writer<epee::byte_stream> out{dest};
+                cryptonote::json::toJsonValue(out, blocks[998]);
+              }
+              MINFO("Blockchain reorg detected, resetting state");
+              return;
+            }
+            MONERO_THROW(updated.error(), "Failed to update accounts on disk");
+          }
+
+          MINFO("Processed " << blocks.size() << " block(s) against " << users.size() << " account(s)");
+          if (*updated != users.size())
+          {
+            MWARNING("Only updated " << *updated << " account(s) out of " << users.size() << ", resetting");
+            return;
+          }
+
+          for (account& user : users)
+            user.updated(db::block_id(fetched.result.start_height));
+        }
+      }
+      catch (std::exception const& e)
+      {
+        scanner::stop();
+        MERROR(e.what());
+      }
+      catch (...)
+      {
+        scanner::stop();
+        MERROR("Unknown exception");
+      }
+    }
+
+    /*!
+      Launches `thread_count` threads to run `scan_loop`, and then polls for
+      active account changes in background
+    */
+    void check_loop(db::storage disk, rpc::context& ctx, std::size_t thread_count, std::vector<lws::account> users, std::vector<db::account_id> active)
+    {
+      assert(0 < thread_count);
+      assert(0 < users.size());
+
+      thread_sync self{};
+      std::vector<boost::thread> threads{};
+
+      struct join_
+      {
+        thread_sync& self;
+        std::vector<boost::thread>& threads;
+        rpc::context& ctx;
+
+        ~join_() noexcept
+        {
+          self.update = true;
+          ctx.raise_abort_scan();
+          for (auto& thread : threads)
+            thread.join();
+        }
+      } join{self, threads, ctx};
+
+      /*
+        The algorithm here is extremely basic. Users are divided evenly amongst
+        the configurable thread count, and grouped by scan height. If an old
+        account appears, some accounts (grouped on that thread) will be delayed
+        in processing waiting for that account to catch up. Its not the greatest,
+        but this "will have to do" for the first cut.
+        Its not expected that many people will be running
+        "enterprise level" of nodes where accounts are constantly added.
+
+        Another "issue" is that each thread works independently instead of more
+        cooperatively for scanning. This requires a bit more synchronization, so
+        was left for later. Its likely worth doing to reduce the number of
+        transfers from the daemon, and the bottleneck on the writes into LMDB.
+
+        If the active user list changes, all threads are stopped/joined, and
+        everything is re-started.
+      */
+
+      boost::thread::attributes attrs;
+      attrs.set_stack_size(THREAD_STACK_SIZE);
+
+      threads.reserve(thread_count);
+      std::sort(users.begin(), users.end(), by_height{});
+
+      MINFO("Starting scan loops on " << std::min(thread_count, users.size()) << " thread(s) with " << users.size() << " account(s)");
+
+      while (!users.empty() && --thread_count)
+      {
+        const std::size_t per_thread = std::max(std::size_t(1), users.size() / (thread_count + 1));
+        const std::size_t count = std::min(per_thread, users.size());
+        std::vector<lws::account> thread_users{
+          std::make_move_iterator(users.end() - count), std::make_move_iterator(users.end())
+        };
+        users.erase(users.end() - count, users.end());
+
+        rpc::client client = MONERO_UNWRAP(ctx.connect());
+        client.watch_scan_signals();
+
+        auto data = std::make_shared<thread_data>(
+          std::move(client), disk.clone(), std::move(thread_users)
+        );
+        threads.emplace_back(attrs, std::bind(&scan_loop, std::ref(self), std::move(data)));
+      }
+
+      if (!users.empty())
+      {
+        rpc::client client = MONERO_UNWRAP(ctx.connect());
+        client.watch_scan_signals();
+
+        auto data = std::make_shared<thread_data>(
+          std::move(client), disk.clone(), std::move(users)
+        );
+        threads.emplace_back(attrs, std::bind(&scan_loop, std::ref(self), std::move(data)));
+      }
+
+      auto last_check = std::chrono::steady_clock::now();
+
+      lmdb::suspended_txn read_txn{};
+      db::cursor::accounts accounts_cur{};
+      boost::unique_lock<boost::mutex> lock{self.sync};
+
+      while (scanner::is_running())
+      {
+        update_rates(ctx);
+
+        for (;;)
+        {
+          //! \TODO use signalfd + ZMQ? Windows is the difficult case...
+          self.user_poll.wait_for(lock, boost::chrono::seconds{1});
+          if (self.update || !scanner::is_running())
+            return;
+          auto this_check = std::chrono::steady_clock::now();
+          if (account_poll_interval <= (this_check - last_check))
+          {
+            last_check = this_check;
+            break;
+          }
+        }
+
+        auto reader = disk.start_read(std::move(read_txn));
+        if (!reader)
+        {
+          if (reader.matches(std::errc::no_lock_available))
+          {
+            MWARNING("Failed to open DB read handle, retrying later");
+            continue;
+          }
+          MONERO_THROW(reader.error(), "Failed to open DB read handle");
+        }
+
+        auto current_users = MONERO_UNWRAP(
+          reader->get_accounts(db::account_status::active, std::move(accounts_cur))
+        );
+        if (current_users.count() != active.size())
+        {
+          MINFO("Change in active user accounts detected, stopping scan threads...");
+          return;
+        }
+
+        for (auto user = current_users.make_iterator(); !user.is_end(); ++user)
+        {
+          const db::account_id user_id = user.get_value<MONERO_FIELD(db::account, id)>();
+          if (!std::binary_search(active.begin(), active.end(), user_id))
+          {
+            MINFO("Change in active user accounts detected, stopping scan threads...");
+            return;
+          }
+        }
+
+        read_txn = reader->finish_read();
+        accounts_cur = current_users.give_cursor();
+      } // while scanning
+    }
+  } // anonymous
+
+  expect<rpc::client> scanner::sync(db::storage disk, rpc::client client)
+  {
+    using get_hashes = cryptonote::rpc::GetHashesFast;
+
+    MINFO("Starting blockchain sync with daemon");
+
+    get_hashes::Request req{};
+    req.start_height = 0;
+    {
+      auto reader = disk.start_read();
+      if (!reader)
+        return reader.error();
+
+      auto chain = reader->get_chain_sync();
+      if (!chain)
+        return chain.error();
+
+      req.known_hashes = std::move(*chain);
+    }
+
+    for (;;)
+    {
+      if (req.known_hashes.empty())
+        return {lws::error::bad_blockchain};
+
+      expect<void> sent{lws::error::daemon_timeout};
+
+      epee::byte_slice msg = rpc::client::make_message("get_hashes_fast", req);
+      auto start = std::chrono::steady_clock::now();
+
+      while (!(sent = client.send(std::move(msg), std::chrono::seconds{1})))
+      {
+        if (!scanner::is_running())
+          return {lws::error::signal_abort_process};
+
+        if (sync_rpc_timeout <= (std::chrono::steady_clock::now() - start))
+          return {lws::error::daemon_timeout};
+
+        if (!sent.matches(std::errc::timed_out))
+          return sent.error();
+      }
+
+      expect<get_hashes::Response> resp{lws::error::daemon_timeout};
+      start = std::chrono::steady_clock::now();
+
+      while (!(resp = client.receive<get_hashes::Response>(std::chrono::seconds{1})))
+      {
+        if (!scanner::is_running())
+          return {lws::error::signal_abort_process};
+
+        if (sync_rpc_timeout <= (std::chrono::steady_clock::now() - start))
+          return {lws::error::daemon_timeout};
+
+        if (!resp.matches(std::errc::timed_out))
+          return resp.error();
+      }
+
+      //
+      // Exit loop if it appears we have synced to top of chain
+      //
+      if (resp->hashes.size() <= 1 || resp->hashes.back() == req.known_hashes.front())
+        return {std::move(client)};
+
+      MONERO_CHECK(disk.sync_chain(db::block_id(resp->start_height), epee::to_span(resp->hashes)));
+
+      req.known_hashes.erase(req.known_hashes.begin(), --(req.known_hashes.end()));
+      for (std::size_t num = 0; num < 10; ++num)
+      {
+        if (resp->hashes.empty())
+          break;
+
+        req.known_hashes.insert(--(req.known_hashes.end()), resp->hashes.back());
+      }
+    }
+
+    return {std::move(client)};
+  }
+
+  void scanner::run(db::storage disk, rpc::context ctx, std::size_t thread_count)
+  {
+    thread_count = std::max(std::size_t(1), thread_count);
+
+    rpc::client client{};
+    for (;;)
+    {
+      const auto last = std::chrono::steady_clock::now();
+      update_rates(ctx);
+
+      std::vector<db::account_id> active;
+      std::vector<lws::account> users;
+
+      {
+        MINFO("Retrieving current active account list");
+
+        auto reader = MONERO_UNWRAP(disk.start_read());
+        auto accounts = MONERO_UNWRAP(
+          reader.get_accounts(db::account_status::active)
+        );
+
+        for (db::account user : accounts.make_range())
+        {
+          std::vector<db::output_id> receives{};
+          std::vector<crypto::public_key> pubs{};
+          auto receive_list = MONERO_UNWRAP(reader.get_outputs(user.id));
+
+          const std::size_t elems = receive_list.count();
+          receives.reserve(elems);
+          pubs.reserve(elems);
+
+          for (auto output = receive_list.make_iterator(); !output.is_end(); ++output)
+          {
+            receives.emplace_back(output.get_value<MONERO_FIELD(db::output, spend_meta.id)>());
+            pubs.emplace_back(output.get_value<MONERO_FIELD(db::output, pub)>());
+          }
+
+          users.emplace_back(user, std::move(receives), std::move(pubs));
+          active.insert(
+            std::lower_bound(active.begin(), active.end(), user.id), user.id
+          );
+        }
+
+        reader.finish_read();
+      } // cleanup DB reader
+
+      if (users.empty())
+      {
+        MINFO("No active accounts");
+        checked_wait(account_poll_interval - (std::chrono::steady_clock::now() - last));
+      }
+      else
+        check_loop(disk.clone(), ctx, thread_count, std::move(users), std::move(active));
+
+      if (!scanner::is_running())
+        return;
+
+      if (!client)
+        client = MONERO_UNWRAP(ctx.connect());
+
+      expect<rpc::client> synced = sync(disk.clone(), std::move(client));
+      if (!synced)
+      {
+        if (!synced.matches(std::errc::timed_out))
+          MONERO_THROW(synced.error(), "Unable to sync blockchain");
+
+        MWARNING("Failed to connect to daemon at " << ctx.daemon_address());
+      }
+      else
+        client = std::move(*synced);
+    } // while scanning
+  }
+} // lws

src/scanner.h

@@ -0,0 +1,59 @@
+// Copyright (c) 2018-2020, The Monero Project
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#pragma once
+
+#include <atomic>
+#include <boost/optional/optional.hpp>
+#include <cstdint>
+#include <string>
+
+#include "db/storage.h"
+#include "rpc/client.h"
+
+namespace lws
+{
+  //! Scans all active `db::account`s. Detects if another process changes active list.
+  class scanner
+  {
+    static std::atomic<bool> running;
+
+    scanner() = delete;
+
+  public:
+    //! Use `client` to sync blockchain data, and \return client if successful.
+    static expect<rpc::client> sync(db::storage disk, rpc::client client);
+
+    //! Poll daemon until `stop()` is called, using `thread_count` threads.
+    static void run(db::storage disk, rpc::context ctx, std::size_t thread_count);
+
+    //! \return True if `stop()` has never been called.
+    static bool is_running() noexcept { return running; }
+
+    //! Stops all scanner instances globally.
+    static void stop() noexcept { running = false; }
+  };
+} // lws

src/server_main.cpp

@@ -0,0 +1,241 @@
+// Copyright (c) 2018-2020, The Monero Project
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include <boost/filesystem/operations.hpp>
+#include <boost/optional/optional.hpp>
+#include <boost/program_options/options_description.hpp>
+#include <boost/program_options/parsers.hpp>
+#include <boost/program_options/variables_map.hpp>
+#include <boost/thread/thread.hpp>
+#include <chrono>
+#include <iostream>
+#include <stdexcept>
+#include <string>
+#include <vector>
+
+#include "common/command_line.h" // monero/src/
+#include "common/expect.h"       // monero/src/
+#include "common/util.h"         // monero/src/
+#include "config.h"
+#include "cryptonote_config.h"   // monero/src/
+#include "db/storage.h"
+#include "options.h"
+#include "rest_server.h"
+#include "scanner.h"
+
+#undef MONERO_DEFAULT_LOG_CATEGORY
+#define MONERO_DEFAULT_LOG_CATEGORY "lws"
+
+namespace
+{
+  struct options : lws::options
+  {
+    const command_line::arg_descriptor<std::string> daemon_rpc;
+    const command_line::arg_descriptor<std::vector<std::string>> rest_servers;
+    const command_line::arg_descriptor<std::string> rest_ssl_key;
+    const command_line::arg_descriptor<std::string> rest_ssl_cert;
+    const command_line::arg_descriptor<std::size_t> rest_threads;
+    const command_line::arg_descriptor<std::size_t> scan_threads;
+    const command_line::arg_descriptor<std::vector<std::string>> access_controls;
+    const command_line::arg_descriptor<bool> external_bind;
+    const command_line::arg_descriptor<unsigned> create_queue_max;
+    const command_line::arg_descriptor<std::chrono::minutes::rep> rates_interval;
+    const command_line::arg_descriptor<unsigned short> log_level;
+
+    static std::string get_default_zmq()
+    {
+      static constexpr const char base[] = "tcp://127.0.0.1:";
+      switch (lws::config::network)
+      {
+      case cryptonote::TESTNET:
+        return base + std::to_string(config::testnet::ZMQ_RPC_DEFAULT_PORT);
+      case cryptonote::STAGENET:
+        return base + std::to_string(config::stagenet::ZMQ_RPC_DEFAULT_PORT);
+      case cryptonote::MAINNET:
+      default:
+        break;
+      }
+      return base + std::to_string(config::ZMQ_RPC_DEFAULT_PORT);
+    }
+
+    options()
+      : lws::options()
+      , daemon_rpc{"daemon", "<protocol>://<address>:<port> of a monerod ZMQ RPC", get_default_zmq()}
+      , rest_servers{"rest-server", "[(https|http)://<address>:]<port> for incoming connections, multiple declarations allowed"}
+      , rest_ssl_key{"rest-ssl-key", "<path> to PEM formatted SSL key for https REST server", ""}
+      , rest_ssl_cert{"rest-ssl-certificate", "<path> to PEM formatted SSL certificate (chains supported) for https REST server", ""}
+      , rest_threads{"rest-threads", "Number of threads to process REST connections", 1}
+      , scan_threads{"scan-threads", "Maximum number of threads for account scanning", boost::thread::hardware_concurrency()}
+      , access_controls{"access-control-origin", "Specify a whitelisted HTTP control origin domain"}
+      , external_bind{"confirm-external-bind", "Allow listening for external connections", false}
+      , create_queue_max{"create-queue-max", "Set pending create account requests maximum", 10000}
+      , rates_interval{"exchange-rate-interval", "Retrieve exchange rates in minute intervals from cryptocompare.com if greater than 0", 0}
+      , log_level{"log-level", "Log level [0-4]", 1}
+    {}
+
+    void prepare(boost::program_options::options_description& description) const
+    {
+      static constexpr const char rest_default[] = "https://0.0.0.0:8443";
+
+      lws::options::prepare(description);
+      command_line::add_arg(description, daemon_rpc);
+      description.add_options()(rest_servers.name, boost::program_options::value<std::vector<std::string>>()->default_value({rest_default}, rest_default), rest_servers.description);
+      command_line::add_arg(description, rest_ssl_key);
+      command_line::add_arg(description, rest_ssl_cert);
+      command_line::add_arg(description, rest_threads);
+      command_line::add_arg(description, scan_threads);
+      command_line::add_arg(description, access_controls);
+      command_line::add_arg(description, external_bind);
+      command_line::add_arg(description, create_queue_max);
+      command_line::add_arg(description, rates_interval);
+      command_line::add_arg(description, log_level);
+    }
+  };
+
+  struct program
+  {
+    std::string db_path;
+    std::vector<std::string> rest_servers;
+    lws::rest_server::configuration rest_config;
+    std::string daemon_rpc;
+    std::chrono::minutes rates_interval;
+    std::size_t scan_threads;
+    unsigned create_queue_max;
+  };
+
+  void print_help(std::ostream& out)
+  {
+    boost::program_options::options_description description{"Options"};
+    options{}.prepare(description);
+
+    out << "Usage: [options]" << std::endl;
+    out << description;
+  }
+
+  boost::optional<program> get_program(int argc, char** argv)
+  {
+    namespace po = boost::program_options;
+
+    const options opts{};
+    po::variables_map args{};
+    {
+      po::options_description description{"Options"};
+      opts.prepare(description);
+
+      po::store(
+        po::command_line_parser(argc, argv).options(description).run(), args
+      );
+      po::notify(args);
+    }
+
+    if (command_line::get_arg(args, command_line::arg_help))
+    {
+      print_help(std::cout);
+      return boost::none;
+    }
+
+    opts.set_network(args); // do this first, sets global variable :/
+    mlog_set_log_level(command_line::get_arg(args, opts.log_level));
+
+    program prog{
+      command_line::get_arg(args, opts.db_path),
+      command_line::get_arg(args, opts.rest_servers),
+      lws::rest_server::configuration{
+        {command_line::get_arg(args, opts.rest_ssl_key), command_line::get_arg(args, opts.rest_ssl_cert)},
+        command_line::get_arg(args, opts.access_controls),
+        command_line::get_arg(args, opts.rest_threads),
+        command_line::get_arg(args, opts.external_bind)
+      },
+      command_line::get_arg(args, opts.daemon_rpc),
+      std::chrono::minutes{command_line::get_arg(args, opts.rates_interval)},
+      command_line::get_arg(args, opts.scan_threads),
+      command_line::get_arg(args, opts.create_queue_max),
+    };
+
+    prog.rest_config.threads = std::max(std::size_t(1), prog.rest_config.threads);
+    prog.scan_threads = std::max(std::size_t(1), prog.scan_threads);
+
+    if (command_line::is_arg_defaulted(args, opts.daemon_rpc))
+      prog.daemon_rpc = options::get_default_zmq();
+
+    return prog;
+  }
+
+  void run(program prog)
+  {
+    std::signal(SIGINT, [] (int) { lws::scanner::stop(); });
+
+    boost::filesystem::create_directories(prog.db_path);
+    auto disk = lws::db::storage::open(prog.db_path.c_str(), prog.create_queue_max);
+    auto ctx = lws::rpc::context::make(std::move(prog.daemon_rpc), prog.rates_interval);
+
+    MINFO("Using monerod ZMQ RPC at " << ctx.daemon_address());
+    auto client = lws::scanner::sync(disk.clone(), ctx.connect().value()).value();
+
+    lws::rest_server server{epee::to_span(prog.rest_servers), disk.clone(), std::move(client), std::move(prog.rest_config)};
+    for (const std::string& address : prog.rest_servers)
+      MINFO("Listening for REST clients at " << address);
+
+    // blocks until SIGINT
+    lws::scanner::run(std::move(disk), std::move(ctx), prog.scan_threads);
+  }
+} // anonymous
+
+int main(int argc, char** argv)
+{
+  tools::on_startup(); // if it throws, don't use MERROR just print default msg
+
+  try
+  {
+    boost::optional<program> prog;
+
+    try
+    {
+      prog = get_program(argc, argv);
+    }
+    catch (std::exception const& e)
+    {
+      std::cerr << e.what() << std::endl << std::endl;
+      print_help(std::cerr);
+      return EXIT_FAILURE;
+    }
+
+    if (prog)
+      run(std::move(*prog));
+  }
+  catch (std::exception const& e)
+  {
+    MERROR(e.what());
+    return EXIT_FAILURE;
+  }
+  catch (...)
+  {
+    MERROR("Unknown exception");
+    return EXIT_FAILURE;
+  }
+  return EXIT_SUCCESS;
+}

src/util/CMakeLists.txt

@@ -0,0 +1,33 @@
+# Copyright (c) 2020, The Monero Project
+#
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are
+# permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice, this list of
+#    conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice, this list
+#    of conditions and the following disclaimer in the documentation and/or other
+#    materials provided with the distribution.
+#
+# 3. Neither the name of the copyright holder nor the names of its contributors may be
+#    used to endorse or promote products derived from this software without specific
+#    prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+# THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+set(monero-lws-util_sources gamma_picker.cpp random_outputs.cpp transactions.cpp)
+set(monero-lws-util_headers fwd.h gamma_picker.h http_server.h random_outputs.h transactions.h)
+
+add_library(monero-lws-util ${monero-lws-util_sources} ${monero-lws-util_headers})
+target_link_libraries(monero-lws-util monero::libraries)

src/util/fwd.h

@@ -0,0 +1,35 @@
+// Copyright (c) 2020, The Monero Project
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#pragma once
+
+namespace lws
+{
+  class gamma_picker;
+  struct random_output;
+  struct random_ring;
+}

src/util/gamma_picker.cpp

@@ -0,0 +1,112 @@
+// Copyright (c) 2019-2020, The Monero Project
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "gamma_picker.h"
+
+#include <algorithm>
+#include <stdexcept>
+
+#include "crypto/crypto.h"
+#include "cryptonote_config.h"
+
+namespace lws
+{
+  namespace
+  {
+    constexpr const double gamma_shape = 19.28;
+    constexpr const double gamma_scale = 1 / double(1.61);
+    constexpr const std::size_t blocks_in_a_year = (86400 * 365) / DIFFICULTY_TARGET_V2;
+  }
+
+  gamma_picker::gamma_picker(std::vector<uint64_t> rct_offsets)
+    : gamma_picker(std::move(rct_offsets), gamma_shape, gamma_scale)
+  {}
+
+  gamma_picker::gamma_picker(std::vector<std::uint64_t> offsets_in, double shape, double scale)
+    : rct_offsets(std::move(offsets_in)),
+      gamma(shape, scale),
+      outputs_per_second(0)
+  {
+    if (!rct_offsets.empty())
+    {
+      const std::size_t blocks_to_consider = std::min(rct_offsets.size(), blocks_in_a_year);
+      const std::uint64_t initial = blocks_to_consider < rct_offsets.size() ?
+        rct_offsets[rct_offsets.size() - blocks_to_consider - 1] : 0;
+      const std::size_t outputs_to_consider = rct_offsets.back() - initial;
+
+      static_assert(0 < DIFFICULTY_TARGET_V2, "block target time cannot be zero");
+      // this assumes constant target over the whole rct range
+      outputs_per_second = outputs_to_consider / double(DIFFICULTY_TARGET_V2 * blocks_to_consider);
+    }
+  }
+
+  bool gamma_picker::is_valid() const noexcept
+  {
+    return CRYPTONOTE_DEFAULT_TX_SPENDABLE_AGE < rct_offsets.size();
+  }
+
+  std::uint64_t gamma_picker::spendable_upper_bound() const noexcept
+  {
+    if (!is_valid())
+      return 0;
+    return *(rct_offsets.end() - CRYPTONOTE_DEFAULT_TX_SPENDABLE_AGE - 1);
+  }
+
+  std::uint64_t gamma_picker::operator()()
+  {
+    if (!is_valid())
+      throw std::logic_error{"Cannot select random output - blockchain height too small"};
+
+    static_assert(std::is_empty<crypto::random_device>(), "random_device is no longer cheap to construct");
+    static constexpr const crypto::random_device engine{};
+    const auto end = offsets().end() - CRYPTONOTE_DEFAULT_TX_SPENDABLE_AGE;
+
+    for (unsigned tries = 0; tries < 100; ++tries)
+    {
+      std::uint64_t output_index = std::exp(gamma(engine)) * outputs_per_second;
+      if (offsets().back() <= output_index)
+        continue; // gamma selected older than blockchain height (rare)
+
+      output_index = offsets().back() - 1 - output_index;
+      const auto selection = std::lower_bound(offsets().begin(), end, output_index);
+      if (selection == end)
+        continue; // gamma selected within locked/non-spendable range (rare)
+
+      const std::uint64_t first_rct = offsets().begin() == selection ? 0 : *(selection - 1);
+      const std::uint64_t n_rct = *selection - first_rct;
+      if (n_rct != 0)
+        return first_rct + crypto::rand_idx(n_rct);
+      // block had zero outputs (miner didn't collect XMR?)
+    }
+    throw std::runtime_error{"Unable to select random output in spendable range using gamma distribution after 1,024 attempts"};
+  }
+
+  std::vector<std::uint64_t> gamma_picker::take_offsets()
+  {
+    return std::vector<std::uint64_t>{std::move(rct_offsets)};
+  }
+} // lws

src/util/gamma_picker.h

@@ -0,0 +1,88 @@
+// Copyright (c) 2019-2020, The Monero Project
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include <cstdint>
+#include <random>
+#include <vector>
+
+namespace lws
+{
+  //! Select outputs using a gamma distribution with Sarang's output-lineup method
+  class gamma_picker
+  {
+    std::vector<uint64_t> rct_offsets;
+    std::gamma_distribution<double> gamma;
+    double outputs_per_second;
+
+    gamma_picker(const gamma_picker&) = default; // force explicit usage of `clone()` to copy.
+  public:
+    //! \post `!is_valid()` since the chain of offsets is empty.
+    gamma_picker()
+      : gamma_picker(std::vector<std::uint64_t>{})
+    {}
+
+    //! Use default (recommended) gamma parameters with `rct_offsets`.
+    explicit gamma_picker(std::vector<std::uint64_t> rct_offsets);
+    explicit gamma_picker(std::vector<std::uint64_t> rct_offsets, double shape, double scale);
+
+    //! \post Source of move `!is_valid()`.
+    gamma_picker(gamma_picker&&) = default;
+
+    //! \post Source of move `!is_valid()`.
+    gamma_picker& operator=(gamma_picker&&) = default;
+
+    //! \return A copy of `this`.
+    gamma_picker clone() const { return gamma_picker{*this}; }
+
+    //! \return `is_valid()`.
+    explicit operator bool() const noexcept { return is_valid(); }
+
+    //! \return True if `operator()()` can pick an output using `offsets()`.
+    bool is_valid() const noexcept;
+
+    //! \return An upper-bound on the number of unlocked/spendable outputs based on block age.
+    std::uint64_t spendable_upper_bound() const noexcept;
+
+    /*!
+      Select a random output index for use in a ring. Outputs in the unspendable
+      range (too new) and older than the chain (too old) are filtered out by
+      retrying the gamma distribution.
+
+      \throw std::logic_error if `!is_valid()` - considered unrecoverable.
+      \throw std::runtiime_error if no output within spendable range was selected
+        after 100 attempts.
+      \return Selected output using gamma distribution.
+    */
+    std::uint64_t operator()();
+
+    //! \return Current ringct distribution used for `operator()()` output selection.
+    const std::vector<std::uint64_t>& offsets() const noexcept { return rct_offsets; }
+
+    //! \return Ownership of `offsets()` by move. \post `!is_valid()`
+    std::vector<std::uint64_t> take_offsets();
+  };
+} // lws

src/util/http_server.h

@@ -0,0 +1,124 @@
+// Copyright (c) 2020, The Monero Project
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include <boost/bind/bind.hpp>
+#include <boost/thread.hpp>
+#include <boost/optional/optional.hpp>
+
+#include "misc_log_ex.h"
+#include "net/abstract_tcp_server2.h"      // monero/contrib/epee/include
+#include "net/http_protocol_handler.h"     // monero/contrib/epee/include
+#include "net/http_server_handlers_map2.h" // monero/contrib/epee/include
+
+#undef MONERO_DEFAULT_LOG_CATEGORY
+#define MONERO_DEFAULT_LOG_CATEGORY "net.http"
+
+
+namespace lws
+{
+  template<class t_child_class, class t_connection_context = epee::net_utils::connection_context_base>
+  class http_server_impl_base: public epee::net_utils::http::i_http_server_handler<t_connection_context>
+  {
+
+  public:
+    http_server_impl_base()
+        : m_net_server(epee::net_utils::e_connection_type_RPC)
+    {}
+
+    explicit http_server_impl_base(boost::asio::io_service& external_io_service)
+      : m_net_server(external_io_service, epee::net_utils::e_connection_type_RPC)
+    {}
+
+    bool init(const std::string& bind_port, const std::string& bind_ip,
+      std::vector<std::string> access_control_origins, epee::net_utils::ssl_options_t ssl_options)
+    {
+
+      //set self as callback handler
+      m_net_server.get_config_object().m_phandler = static_cast<t_child_class*>(this);
+  
+      //here set folder for hosting reqests
+      m_net_server.get_config_object().m_folder = "";
+
+      //set access control allow origins if configured
+      std::sort(access_control_origins.begin(), access_control_origins.end());
+      m_net_server.get_config_object().m_access_control_origins = std::move(access_control_origins);
+
+  
+      MGINFO("Binding on " << bind_ip << " (IPv4):" << bind_port);
+      bool res = m_net_server.init_server(bind_port, bind_ip, bind_port, std::string{}, false, true, std::move(ssl_options));
+      if(!res)
+      {
+        LOG_ERROR("Failed to bind server");
+        return false;
+      }
+      return true;
+    }
+
+    bool run(size_t threads_count, bool wait = true)
+    {
+      //go to loop
+      MINFO("Run net_service loop( " << threads_count << " threads)...");
+      if(!m_net_server.run_server(threads_count, wait))
+      {
+        LOG_ERROR("Failed to run net tcp server!");
+      }
+
+      if(wait)
+        MINFO("net_service loop stopped.");
+      return true;
+    }
+
+    bool deinit()
+    {
+      return m_net_server.deinit_server();
+    }
+
+    bool timed_wait_server_stop(uint64_t ms)
+    {
+      return m_net_server.timed_wait_server_stop(ms);
+    }
+
+    bool send_stop_signal()
+    {
+      m_net_server.send_stop_signal();
+      return true;
+    }
+
+    int get_binded_port()
+    {
+      return m_net_server.get_binded_port();
+    }
+
+    long get_connections_count() const
+    {
+      return m_net_server.get_connections_count();
+    }
+
+  protected: 
+    epee::net_utils::boosted_tcp_server<epee::net_utils::http::http_custom_handler<t_connection_context> > m_net_server;
+  };
+} // lws

src/util/random_outputs.cpp

@@ -0,0 +1,309 @@
+// Copyright (c) 2018-2020, The Monero Project
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "random_outputs.h"
+
+#include <algorithm>
+#include <boost/range/combine.hpp>
+#include <cmath>
+#include <limits>
+#include <random>
+#include <utility>
+
+#include "cryptonote_config.h"       // monero/src
+#include "error.h"
+#include "util/gamma_picker.h"
+
+namespace lws
+{
+    namespace
+    {
+        struct by_amount
+        {
+            template<typename T>
+            bool operator()(T const& left, T const& right) const noexcept
+            {
+                return left.amount < right.amount;
+            }
+
+            template<typename T>
+            bool operator()(std::uint64_t left, T const& right) const noexcept
+            {
+                return left < right.amount;
+            }
+
+            template<typename T>
+            bool operator()(T const& left, std::uint64_t right) const noexcept
+            {
+                return left.amount < right;
+            }
+        };
+
+        struct by_index
+        {
+            template<typename T, typename U>
+            bool operator()(T const& left, U const& right) const noexcept
+            {
+                return left.index < right.index;
+            }
+        };
+
+        struct same_index
+        {
+            bool operator()(lws::output_ref const& left, lws::output_ref const& right) const noexcept
+            {
+                return left.index == right.index;
+            }
+        };
+
+        expect<void> pick_all(epee::span<lws::output_ref> out, const std::uint64_t amount) noexcept
+        {
+            static_assert(
+                std::numeric_limits<std::size_t>::max() <= std::numeric_limits<std::uint64_t>::max(),
+                "size_t is really large"
+            );
+
+            std::size_t index = 0;
+            for (auto& entry : out)
+            {
+                entry.amount = amount;
+                entry.index = index++;
+            }
+            return success();
+        }
+
+        expect<void> triangular_pick(epee::span<lws::output_ref> out, lws::histogram const& hist)
+        {
+            MONERO_PRECOND(hist.unlocked_count <= hist.total_count);
+
+            if (hist.unlocked_count < out.size())
+                return {lws::error::not_enough_mixin};
+
+            if (hist.unlocked_count == out.size())
+                return pick_all(out, hist.amount);
+
+            /* This does not match the wallet2 selection code - recents are not
+            considered. There should be no new recents for this selection
+            algorithm because it is only used for non-ringct outputs. */
+
+            static constexpr const std::uint64_t max = std::uint64_t(1) << 53;
+            for (auto& entry : out)
+            {
+                entry.amount = hist.amount;
+
+               /* \TODO Update REST API to send real outputs so selection
+               algorithm can use fork information (like wallet2). */
+
+                do
+                {
+                    const std::uint64_t r = crypto::rand<std::uint64_t>() % max;
+                    const double frac = std::sqrt(double(r) / double(max));
+                    entry.index = std::uint64_t(frac * double(hist.total_count));
+                } while (hist.unlocked_count < entry.index);
+            }
+
+            return success();
+        }
+
+        expect<void> gamma_pick(epee::span<lws::output_ref> out, gamma_picker& pick_rct)
+        {
+            if (!pick_rct)
+                return {lws::error::not_enough_mixin};
+
+            const std::uint64_t spendable = pick_rct.spendable_upper_bound();
+            if (spendable < out.size())
+                return {lws::error::not_enough_mixin};
+            if (spendable == out.size())
+                return pick_all(out, 0);
+
+            for (auto& entry : out)
+            {
+                entry.amount = 0;
+                entry.index = pick_rct();
+
+                /* \TODO Update REST API to send real outputs so selection
+                algorithm can use fork information (like wallet2). */
+            }
+            return success();
+        }
+    }
+
+    expect<std::vector<random_ring>> pick_random_outputs(
+        const std::uint32_t mixin,
+        const epee::span<const std::uint64_t> amounts, 
+        gamma_picker& pick_rct,
+        epee::span<histogram> histograms,
+        const std::function<key_fetcher> fetch
+    ) {
+        if (mixin == 0 || amounts.empty())
+            return std::vector<random_ring>{amounts.size()};
+
+        const std::size_t sizet_max = std::numeric_limits<std::size_t>::max();
+        MONERO_PRECOND(bool(fetch));
+        MONERO_PRECOND(mixin <= (sizet_max / amounts.size()));
+
+        std::vector<output_ref> proposed{};
+        std::vector<random_ring> rings{};
+        rings.resize(amounts.size());
+
+        for (auto ring : boost::combine(amounts, rings))
+            boost::get<1>(ring).amount = boost::get<0>(ring);
+
+        std::sort(histograms.begin(), histograms.end(), by_amount{});
+        for (unsigned tries = 0; tries < 64; ++tries)
+        {
+            proposed.clear();
+            proposed.reserve(rings.size() * mixin);
+
+            // select indexes foreach ring below mixin count
+            for (auto ring = rings.begin(); ring != rings.end(); /* handled below */)
+            {
+                const std::size_t count = proposed.size();
+                if (ring->ring.size() < mixin)
+                {
+                    const std::size_t diff = mixin - ring->ring.size();
+                    proposed.resize(proposed.size() + diff);
+                    {
+                        const epee::span<output_ref> latest{proposed.data() + count, diff};
+
+                        expect<void> picked{};
+                        const std::uint64_t amount = ring->amount;
+                        if (amount == 0)
+                            picked = gamma_pick(latest, pick_rct);
+                        else
+                        {
+                            const auto match =
+                                std::lower_bound(histograms.begin(), histograms.end(), amount, by_amount{});
+                            MONERO_PRECOND(match != histograms.end() && match->amount == amount);
+                            picked = triangular_pick(latest, *match);
+                        }
+
+                        if (!picked)
+                        {
+                            if (picked == lws::error::not_enough_mixin)
+                            {
+                                proposed.resize(proposed.size() - diff);
+                                ring = rings.erase(ring);
+                                continue;
+                            }
+                            return picked.error();
+                        }
+
+                        // drop dupes in latest selection
+                        std::sort(latest.begin(), latest.end(), by_index{});
+                        const auto last = std::unique(latest.begin(), latest.end(), same_index{});
+                        proposed.resize(last - proposed.data());
+                    }
+
+                    ring->ring.reserve(mixin);
+                    epee::span<random_output> current = epee::to_mut_span(ring->ring);
+                    std::sort(current.begin(), current.end(), by_index{});
+
+                    // See if new list has duplicates with existing ring
+                    for (auto ref = proposed.begin() + count; ref < proposed.end(); /* see branches */ )
+                    {
+                        // must update after push_back call
+                        current = {ring->ring.data(), current.size()};
+
+                        const auto match =
+                            std::lower_bound(current.begin(), current.end(), *ref, by_index{});
+                        if (match == current.end() || match->index != ref->index)
+                        {
+                            ring->ring.push_back(random_output{{}, ref->index});
+                            ring->ring.back().keys.unlocked = false; // for tracking below
+                            ++ref;
+                        }
+                        else // dupe
+                            ref = proposed.erase(ref);
+                    }
+                }
+
+                ++ring;
+            }
+
+            // all amounts lack enough mixin
+            if (rings.empty())
+                return rings;
+
+            /* \TODO For maximum privacy, the real outputs need to be fetched
+            below. This requires an update of the REST API. */
+
+            // fetch all new keys in one shot
+            const std::size_t expected = proposed.size();
+            auto result = fetch(std::move(proposed));
+            if (!result)
+                return result.error();
+
+            if (expected != result->size())
+                return {lws::error::bad_daemon_response};
+
+            bool done = true;
+            std::size_t offset = 0;
+            for (auto& ring : rings)
+            {
+                // this should never fail, else the logic in here is bad
+                assert(ring.ring.size() <= ring.ring.size());
+
+                // if we dropped a selection due to dupe, must try again
+                done = (done && mixin <= ring.ring.size());
+
+                for (auto entry = ring.ring.begin(); entry < ring.ring.end(); /* see branches */)
+                {
+                    // check only new keys
+                    if (entry->keys.unlocked)
+                        ++entry;
+                    else
+                    {
+                        if (result->size() <= offset)
+                            return {lws::error::bad_daemon_response};
+
+                        output_keys const& keys = result->at(offset);
+                        ++offset;
+
+                        if (keys.unlocked)
+                        {
+                            entry->keys = keys;
+                            ++entry;
+                        }
+                        else
+                        {
+                            done = false;
+                            entry = ring.ring.erase(entry);
+                        }
+                    }
+                }
+            }
+            assert(offset == result->size());
+
+            if (done)
+                return {std::move(rings)};
+        }
+
+        return {lws::error::not_enough_mixin};
+    }
+}
+

src/util/random_outputs.h

@@ -0,0 +1,92 @@
+// Copyright (c) 2018-2019, The Monero Project
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#pragma once
+
+#include <cstdint>
+#include <functional>
+#include <vector>
+
+#include "common/expect.h"            // monero/src
+#include "rpc/message_data_structs.h" // monero/src
+#include "span.h"                     // monero/src
+#include "util/fwd.h"
+
+namespace lws
+{
+    using histogram = cryptonote::rpc::output_amount_count;
+    using output_ref = cryptonote::rpc::output_amount_and_index;
+    using output_keys = cryptonote::rpc::output_key_mask_unlocked;
+
+    struct random_output
+    {
+        output_keys keys;
+        std::uint64_t index;
+    };
+
+    struct random_ring
+    {
+        std::vector<random_output> ring;
+        std::uint64_t amount;
+    };
+
+    using key_fetcher = expect<std::vector<output_keys>>(std::vector<output_ref>);
+
+    /*!
+        Selects random outputs for use in a ring signature. `amounts` of `0`
+        use a gamma distribution algorithm and all other amounts use a
+        triangular distribution.
+
+        \param mixin The number of dummy outputs per ring.
+        \param amounts The amounts that need dummy outputs to be selected.
+        \param pick_rct Ring-ct distribution from the daemon
+        \param histograms A histogram from the daemon foreach non-zero value
+            in `amounts`.
+        \param fetch A function that can retrieve the keys for the randomly
+            selected outputs.
+
+        \note `histograms` is modified - the list is sorted by amount.
+
+        \note This currenty leaks the real outputs to `fetch`, because the
+            real output is not provided alongside the dummy outputs. This is a
+            limitation of the current openmonero/mymonero API. When this is
+            resolved, this function can possibly be moved outside of the `lws`
+            namespace for use by simple wallet.
+
+        \return Randomly selected outputs in rings of size `mixin`, one for
+            each element in `amounts`. Amounts with less than `mixin` available
+            are not returned. All outputs are unlocked.
+    */
+    expect<std::vector<random_ring>> pick_random_outputs(
+        std::uint32_t mixin,
+        epee::span<const std::uint64_t> amounts,
+        gamma_picker& pick_rct,
+        epee::span<histogram> histograms,
+        std::function<key_fetcher> fetch
+    );
+}
+

src/util/transactions.cpp

@@ -0,0 +1,61 @@
+// Copyright (c) 2020, The Monero Project
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "transactions.h"
+
+#include "cryptonote_config.h"
+#include "crypto/crypto.h"
+#include "crypto/hash.h"
+#include "ringct/rctOps.h"
+
+void lws::decrypt_payment_id(crypto::hash8& out, const crypto::key_derivation& key)
+{
+  crypto::hash hash;
+  char data[33]; /* A hash, and an extra byte */
+
+  memcpy(data, &key, 32);
+  data[32] = config::HASH_KEY_ENCRYPTED_PAYMENT_ID;
+  cn_fast_hash(data, 33, hash);
+
+  for (size_t b = 0; b < 8; ++b)
+    out.data[b] ^= hash.data[b];
+}
+
+boost::optional<std::pair<std::uint64_t, rct::key>> lws::decode_amount(const rct::key& commitment, const rct::ecdhTuple& info, const crypto::key_derivation& sk, std::size_t index, const bool bulletproof2)
+{
+  crypto::secret_key scalar{};
+  crypto::derivation_to_scalar(sk, index, scalar);
+
+  rct::ecdhTuple copy{info};
+  rct::ecdhDecode(copy, rct::sk2rct(scalar), bulletproof2);
+
+  rct::key Ctmp;
+  rct::addKeys2(Ctmp, copy.mask, copy.amount, rct::H);
+  if (rct::equalKeys(commitment, Ctmp))
+    return {{rct::h2d(copy.amount), copy.mask}};
+  return boost::none;
+}

src/util/transactions.h

@@ -0,0 +1,45 @@
+// Copyright (c) 2020, The Monero Project
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include <boost/optional/optional.hpp>
+#include <cstdint>
+#include <utility>
+
+#include "common/pod-class.h"
+#include "ringct/rctTypes.h"
+
+namespace crypto
+{
+  POD_CLASS hash8;
+  POD_CLASS key_derivation;
+}
+
+namespace lws
+{
+  void decrypt_payment_id(crypto::hash8& out, const crypto::key_derivation& key);
+  boost::optional<std::pair<std::uint64_t, rct::key>> decode_amount(const rct::key& commitment, const rct::ecdhTuple& info, const crypto::key_derivation& sk, std::size_t index, const bool bulletproof2);
+}

src/wire.h

@@ -0,0 +1,77 @@
+// Copyright (c) 2020, The Monero Project
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#pragma once
+
+#include <algorithm>
+#include <boost/utility/string_ref.hpp>
+#include <iterator>
+#include <type_traits>
+
+#include "common/expect.h" // monero/src
+#include "wire/error.h"
+#include "wire/read.h"
+#include "wire/write.h"
+
+#define WIRE_DEFINE_ENUM(type_, map)                                    \
+  static_assert(std::is_enum<type_>::value, "get_string will fail");    \
+  static_assert(!std::is_signed<type_>::value, "write_bytes will fail"); \
+  const char* get_string(const type_ source) noexcept                   \
+  {                                                                     \
+    using native_type = std::underlying_type<type_>::type;              \
+    const native_type value = native_type(source);                      \
+    if (value < std::end(map) - std::begin(map))                        \
+      return map[value];                                                \
+    return "invalid enum value";                                        \
+  }                                                                     \
+  expect<type_> type_ ## _from_string(const boost::string_ref source) noexcept \
+  {                                                                     \
+    if (const auto elem = std::find(std::begin(map), std::end(map), source)) \
+    {                                                                   \
+      if (elem != std::end(map))                                        \
+        return type_(elem - std::begin(map));                           \
+    }                                                                   \
+    return {::wire::error::schema::enumeration};                        \
+  }                                                                     \
+  void read_bytes(::wire::reader& source, type_& dest)                  \
+  {                                                                     \
+    dest = type_(source.enumeration(map));                              \
+  }                                                                     \
+  void write_bytes(::wire::writer& dest, const type_ source)            \
+  {                                                                     \
+    dest.enumeration(std::size_t(source), map);                         \
+  }
+
+#define WIRE_DEFINE_OBJECT(type, map)                          \
+  void read_bytes(::wire::reader& source, type& dest)          \
+  {                                                            \
+    map(source, dest);                                         \
+  }                                                            \
+  void write_bytes(::wire::writer& dest, const type& source)   \
+  {                                                            \
+    map(dest, source);                                         \
+  }

src/wire/CMakeLists.txt

@@ -0,0 +1,36 @@
+# Copyright (c) 2020, The Monero Project
+#
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are
+# permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice, this list of
+#    conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice, this list
+#    of conditions and the following disclaimer in the documentation and/or other
+#    materials provided with the distribution.
+#
+# 3. Neither the name of the copyright holder nor the names of its contributors may be
+#    used to endorse or promote products derived from this software without specific
+#    prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+# THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+set(monero-lws-wire_sources error.cpp read.cpp write.cpp)
+set(monero-lws-wire_headers crypto.h error.h field.h filters.h fwd.h json.h read.h traits.h vector.h write.h)
+
+add_library(monero-lws-wire ${monero-lws-wire_sources} ${monero-lws-wire_headers})
+target_include_directories(monero-lws-wire PUBLIC "${LMDB_INCLUDE}")
+target_link_libraries(monero-lws-wire PRIVATE monero::libraries)
+
+add_subdirectory(json)

src/wire/crypto.h

@@ -0,0 +1,72 @@
+// Copyright (c) 2020, The Monero Project
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#pragma once
+
+#include <type_traits>
+
+#include "crypto/crypto.h"   // monero/src
+#include "ringct/rctTypes.h" // monero/src
+#include "wire/traits.h"
+
+namespace wire
+{
+  template<>
+  struct is_blob<crypto::ec_scalar>
+    : std::true_type
+  {};
+
+  template<>
+  struct is_blob<crypto::hash>
+    : std::true_type
+  {};
+
+  template<>
+  struct is_blob<crypto::key_derivation>
+    : std::true_type
+  {};
+
+  template<>
+  struct is_blob<crypto::key_image>
+    : std::true_type
+  {};
+
+  template<>
+  struct is_blob<crypto::public_key>
+    : std::true_type
+  {};
+
+  template<>
+  struct is_blob<crypto::signature>
+    : std::true_type
+  {};
+
+  template<>
+  struct is_blob<rct::key>
+    : std::true_type
+  {};
+}

src/wire/error.cpp

@@ -0,0 +1,93 @@
+// Copyright (c) 2020, The Monero Project
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "wire/error.h"
+
+namespace wire
+{
+  namespace error
+  {
+    const char* get_string(const schema value) noexcept
+    {
+      switch (value)
+      {
+      default:
+        break;
+
+      case schema::none:
+        return "No schema errors";
+      case schema::array:
+        return "Schema expected array";
+      case schema::binary:
+        return "Schema expected binary value of variable size";
+      case schema::boolean:
+        return "Schema expected boolean value";
+      case schema::enumeration:
+        return "Schema expected a specific of enumeration value(s)";
+      case schema::fixed_binary:
+        return "Schema expected binary of fixed size";
+      case schema::integer:
+        return "Schema expected integer value";
+      case schema::invalid_key:
+        return "Schema does not allow object field key";
+      case schema::larger_integer:
+        return "Schema expected a larger integer value";
+      case schema::maximum_depth:
+        return "Schema hit maximum array+object depth tracking";
+      case schema::missing_key:
+        return "Schema missing required field key";
+      case schema::number:
+        return "Schema expected number (integer or float) value";
+      case schema::object:
+        return "Schema expected object";
+      case schema::smaller_integer:
+        return "Schema expected a smaller integer value";
+      case schema::string:
+        return "Schema expected string";
+      }
+      return "Unknown schema error";
+    }
+
+    const std::error_category& schema_category() noexcept
+    {
+      struct category final : std::error_category
+      {
+        virtual const char* name() const noexcept override final
+          {
+            return "wire::error::schema_category()";
+          }
+
+          virtual std::string message(int value) const override final
+          {
+            return get_string(schema(value));
+          }
+      };
+      static const category instance{};
+      return instance;
+    }
+  }
+}

src/wire/error.h

@@ -0,0 +1,135 @@
+// Copyright (c) 2020, The Monero Project
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#pragma once
+
+#include <exception>
+#include <system_error>
+#include <type_traits>
+
+#include "misc_log_ex.h" // monero/contrib/epee/include
+
+//! Print default `code` message followed by optional message to debug log then throw `code`.
+#define WIRE_DLOG_THROW_(code, ...)					\
+  do									\
+  {									\
+    MDEBUG( get_string(code) __VA_ARGS__ );				\
+    throw ::wire::exception_t<decltype(code)>{code};			\
+  }									\
+  while (0)
+
+//! Print default `code` message followed by `msg` to debug log then throw `code`.
+#define WIRE_DLOG_THROW(code, msg)			\
+  WIRE_DLOG_THROW_(code, << ": " << msg)
+
+namespace wire
+{
+  namespace error
+  {
+    enum class schema : int
+    {
+      none = 0,        //!< Must be zero for `expect<..>`
+      array,           //!< Expected an array value
+      binary,          //!< Expected a binary value of variable length
+      boolean,         //!< Expected a boolean value
+      enumeration,     //!< Expected a value from a specific set
+      fixed_binary,    //!< Expected a binary value of fixed length
+      integer,         //!< Expected an integer value
+      invalid_key,     //!< Key for object is invalid
+      larger_integer,  //!< Expected a larger integer value
+      maximum_depth,   //!< Hit maximum number of object+array tracking
+      missing_key,     //!< Missing required key for object
+      number,          //!< Expected a number (integer or float) value
+      object,          //!< Expected object value
+      smaller_integer, //!< Expected a smaller integer value
+      string,          //!< Expected string value
+    };
+
+    //! \return Error message string.
+    const char* get_string(schema value) noexcept;
+
+    //! \return Category for `schema_error`.
+    const std::error_category& schema_category() noexcept;
+
+    //! \return Error code with `value` and `schema_category()`.
+    inline std::error_code make_error_code(const schema value) noexcept
+    {
+      return std::error_code{int(value), schema_category()};
+    }
+  } // error
+
+  //! `std::exception` doesn't require dynamic memory like `std::runtime_error`
+  struct exception : std::exception
+  {
+    exception() noexcept
+      : std::exception()
+    {}
+
+    exception(const exception&) = default;
+    exception& operator=(const exception&) = default;
+    virtual ~exception() noexcept
+    {}
+
+    virtual std::error_code code() const noexcept = 0;
+  };
+
+  template<typename T>
+  class exception_t final : public wire::exception
+  {
+    static_assert(std::is_enum<T>(), "only enumerated types allowed");
+    T value;
+
+  public:
+    exception_t(T value) noexcept
+      : value(value)
+    {}
+
+    exception_t(const exception_t&) = default;
+    ~exception_t() = default;
+    exception_t& operator=(const exception_t&) = default;
+
+    const char* what() const noexcept override final
+    {
+      static_assert(noexcept(noexcept(get_string(value))), "get_string function must be noexcept");
+      return get_string(value);
+    }
+
+    std::error_code code() const noexcept override final
+    {
+      static_assert(noexcept(noexcept(make_error_code(value))), "make_error_code funcion must be noexcept");
+      return make_error_code(value);
+    }
+  };
+}
+
+namespace std
+{
+  template<>
+  struct is_error_code_enum<wire::error::schema>
+    : true_type
+  {};
+}

src/wire/field.h

@@ -0,0 +1,280 @@
+// Copyright (c) 2020, The Monero Project
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#pragma once
+
+#include <functional>
+#include <utility>
+
+#include "wire/filters.h"
+#include "wire/traits.h"
+
+//! A required field has the same key name and C/C++ name
+#define WIRE_FIELD(name)                                \
+  ::wire::field( #name , std::ref( self . name ))
+
+//! A required field has the same key name and C/C++ name AND is cheap to copy (faster output).
+#define WIRE_FIELD_COPY(name)                   \
+  ::wire::field( #name , self . name )
+
+//! The optional field has the same key name and C/C++ name
+#define WIRE_OPTIONAL_FIELD(name)                               \
+  ::wire::optional_field( #name , std::ref( self . name ))
+
+namespace wire
+{
+  template<typename T>
+  struct unwrap_reference
+  {
+    using type = T;
+  };
+
+  template<typename T>
+  struct unwrap_reference<std::reference_wrapper<T>>
+  {
+    using type = T;
+  };
+
+
+  //! Links `name` to a `value` for object serialization.
+  template<typename T, bool Required>
+  struct field_
+  {
+    using value_type = typename unwrap_reference<T>::type;
+    static constexpr bool is_required() noexcept { return Required; }
+    static constexpr std::size_t count() noexcept { return 1; }
+
+    const char* name;
+    T value;
+
+    //! \return `value` with `std::reference_wrapper` removed.
+    constexpr const value_type& get_value() const noexcept
+    {
+      return value;
+    }
+
+    //! \return `value` with `std::reference_wrapper` removed.
+    value_type& get_value() noexcept
+    {
+      return value;
+    }
+  };
+
+  //! Links `name` to `value`. Use `std::ref` if de-serializing.
+  template<typename T>
+  constexpr inline field_<T, true> field(const char* name, T value)
+  {
+    return {name, std::move(value)};
+  }
+
+  //! Links `name` to `value`. Use `std::ref` if de-serializing.
+  template<typename T>
+  constexpr inline field_<T, false> optional_field(const char* name, T value)
+  {
+    return {name, std::move(value)};
+  }
+
+
+  //! Links `name` to a type `T` for variant serialization.
+  template<typename T>
+  struct option
+  {
+    const char* name;
+  };
+
+  //! \return Name associated with type `T` for variant `field`.
+  template<typename T, typename U>
+  constexpr const char* get_option_name(const U& field) noexcept
+  {
+    return static_cast< const option<T>& >(field).name;
+  }
+
+  //! Links each type in a variant to a string key.
+  template<typename T, bool Required, typename... U>
+  struct variant_field_ : option<U>...
+  {
+    using value_type = typename unwrap_reference<T>::type;
+    static constexpr bool is_required() noexcept { return Required; }
+    static constexpr std::size_t count() noexcept { return sizeof...(U); }
+
+    constexpr variant_field_(const char* name, T value, option<U>... opts)
+      : option<U>(std::move(opts))..., name(name), value(std::move(value))
+    {}
+
+    const char* name;
+    T value;
+
+    constexpr const value_type& get_value() const noexcept
+    {
+      return value;
+    }
+
+    value_type& get_value() noexcept
+    {
+      return value;
+    }
+
+    template<typename V>
+    struct wrap
+    {
+      using result_type = void;
+
+      variant_field_ self;
+      V visitor;
+
+      template<typename X>
+      void operator()(const X& value) const
+      {
+        visitor(get_option_name<X>(self), value);
+      }
+    };
+
+    template<typename V>
+    void visit(V visitor) const
+    {
+      apply_visitor(wrap<V>{*this, std::move(visitor)}, get_value());
+    }
+  };
+
+  //! Links variant `value` to a unique name per type in `opts`. Use `std::ref` for `value` if de-serializing.
+  template<typename T, typename... U>
+  constexpr inline variant_field_<T, true, U...> variant_field(const char* name, T value, option<U>... opts)
+  {
+    return {name, std::move(value), std::move(opts)...};
+  }
+
+
+  //! Indicates a field value should be written as an array
+  template<typename T, typename F>
+  struct as_array_
+  {
+    using value_type = typename unwrap_reference<T>::type;
+
+    T value;
+    F filter; //!< Each element in `value` given to this callable before `write_bytes`.
+
+    //! \return `value` with `std::reference_wrapper` removed.
+    constexpr const value_type& get_value() const noexcept
+    {
+      return value;
+    }
+
+    //! \return `value` with `std::reference_wrapper` removed.
+    value_type& get_value() noexcept
+    {
+      return value;
+    }
+  };
+
+  //! Callable that can filter `as_object` values or be used immediately.
+  template<typename Default>
+  struct as_array_filter
+  {
+    Default default_filter;
+
+    template<typename T>
+    constexpr as_array_<T, Default> operator()(T value) const
+    {
+      return {std::move(value), default_filter};
+    }
+
+    template<typename T, typename F>
+    constexpr as_array_<T, F> operator()(T value, F filter) const
+    {
+      return {std::move(value), std::move(filter)};
+    }
+  };
+  //! Usage: `wire::field("foo", wire::as_array(self.foo, to_string{})`. Consider `std::ref`.
+  constexpr as_array_filter<identity_> as_array{};
+
+
+  //! Indicates a field value should be written as an object
+  template<typename T, typename F, typename G>
+  struct as_object_
+  {
+    using map_type = typename unwrap_reference<T>::type;
+
+    T map;
+    F key_filter;   //!< Each key (`.first`) in `map` given to this callable before writing field key.
+    G value_filter; //!< Each value (`.second`) in `map` given to this callable before `write_bytes`.
+
+    //! \return `map` with `std::reference_wrapper` removed.
+    constexpr const map_type& get_map() const noexcept
+    {
+      return map;
+    }
+
+    //! \return `map` with `std::reference_wrapper` removed.
+    map_type& get_map() noexcept
+    {
+      return map;
+    }
+  };
+
+  //! Usage: `wire::field("foo", wire::as_object(self.foo, to_string{}, wire::as_array))`. Consider `std::ref`.
+  template<typename T, typename F = identity_, typename G = identity_>
+  inline constexpr as_object_<T, F, G> as_object(T map, F key_filter = F{}, G value_filter = G{})
+  {
+    return {std::move(map), std::move(key_filter), std::move(value_filter)};
+  }
+
+
+  template<typename T>
+  inline constexpr bool available(const field_<T, true>&) noexcept
+  {
+    return true;
+  }
+  template<typename T>
+  inline bool available(const field_<T, false>& elem)
+  {
+    return bool(elem.get_value());
+  }
+  template<typename T, typename... U>
+  inline constexpr bool available(const variant_field_<T, true, U...>&) noexcept
+  {
+    return true;
+  }
+  template<typename T, typename... U>
+  inline constexpr bool available(const variant_field_<T, false, U...>& elem)
+  {
+    return elem != nullptr;
+  }
+
+
+  // example usage : `wire::sum(std::size_t(wire::available(fields))...)`
+
+  inline constexpr int sum() noexcept
+  {
+    return 0;
+  }
+  template<typename T, typename... U>
+  inline constexpr T sum(const T head, const U... tail) noexcept
+  {
+    return head + sum(tail...);
+  }
+}
+

src/wire/filters.h

@@ -0,0 +1,73 @@
+// Copyright (c) 2020, The Monero Project
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#pragma once
+
+#include <cassert>
+#include <type_traits>
+
+#include "lmdb/util.h"
+
+// These functions are to be used with `wire::as_object(...)` key filtering
+
+namespace wire
+{
+  //! Callable that returns the value unchanged; default filter for `as_array` and `as_object`.
+  struct identity_
+  {
+    template<typename T>
+    const T& operator()(const T& value) const noexcept
+    {
+      return value;
+    }
+  };
+  constexpr const identity_ identity{};
+
+  //! Callable that forwards enum to get_string.
+  struct enum_as_string_
+  {
+    template<typename T>
+    auto operator()(const T value) const noexcept(noexcept(get_string(value))) -> decltype(get_string(value))
+    {
+      return get_string(value);
+    }
+  };
+  constexpr const enum_as_string_ enum_as_string{};
+
+  //! Callable that converts C++11 enum class or integer to integer value.
+  struct as_integer_
+  {
+    template<typename T>
+    lmdb::native_type<T> operator()(const T value) const noexcept
+    {
+      using native = lmdb::native_type<T>;
+      static_assert(!std::is_signed<native>::value, "integer cannot be signed");
+      return native(value);
+    }
+  };
+  constexpr const as_integer_ as_integer{};
+}

src/wire/fwd.h

@@ -0,0 +1,68 @@
+// Copyright (c) 2020, The Monero Project
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#pragma once
+
+#include <boost/utility/string_ref.hpp>
+#include <type_traits>
+
+#include "common/expect.h" // monero/src
+
+//! Declare an enum to be serialized as an integer
+#define WIRE_AS_INTEGER(type_)						\
+  static_assert(std::is_enum<type_>(), "AS_INTEGER only enum types");	\
+  template<typename R>							\
+  inline void read_bytes(R& source, type_& dest)                        \
+  {									\
+    std::underlying_type<type_>::type temp{};                           \
+    read_bytes(source, temp);                                           \
+    dest = type_(temp);                                                 \
+  }									\
+  template<typename W>							\
+  inline void write_bytes(W& dest, const type_ source)			\
+  {                                                                     \
+    write_bytes(dest, std::underlying_type<type_>::type(source));       \
+  }
+
+//! Declare an enum to be serialized as a string (json) or integer (msgpack)
+#define WIRE_DECLARE_ENUM(type)                                         \
+  const char* get_string(type) noexcept;                                \
+  expect<type> type ## _from_string(const boost::string_ref) noexcept; \
+  void read_bytes(::wire::reader&, type&);                              \
+  void write_bytes(::wire::writer&, type)
+
+//! Declare a class/struct serialization for all available formats
+#define WIRE_DECLARE_OBJECT(type)                     \
+  void read_bytes(::wire::reader&, type&);            \
+  void write_bytes(::wire::writer&, const type&)
+
+namespace wire
+{
+  class reader;
+  struct writer;
+}
+

src/wire/json.h

@@ -0,0 +1,54 @@
+// Copyright (c) 2020, The Monero Project
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#pragma once
+
+#include "wire/json/base.h"
+#include "wire/json/error.h"
+#include "wire/json/read.h"
+#include "wire/json/write.h"
+
+#define WIRE_JSON_DEFINE_ENUM(type, map)				\
+  void read_bytes(::wire::json_reader& source, type& dest)		\
+  {									\
+    dest = type(source.enumeration(map));				\
+  }									\
+  void write_bytes(::wire::json_writer& dest, const type source)	\
+  {									\
+    dest.enumeration(std::size_t(source), map);				\
+  }
+
+#define WIRE_JSON_DEFINE_OBJECT(type, map)                              \
+  void read_bytes(::wire::json_reader& source, type& dest)              \
+  {                                                                     \
+    map(source, dest);                                                  \
+  }                                                                     \
+  void write_bytes(::wire::json_writer& dest, const type& source)       \
+  {                                                                     \
+    map(dest, source);                                                  \
+  }
+

src/wire/json/CMakeLists.txt

@@ -0,0 +1,33 @@
+# Copyright (c) 2020, The Monero Project
+#
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification, are
+# permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice, this list of
+#    conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice, this list
+#    of conditions and the following disclaimer in the documentation and/or other
+#    materials provided with the distribution.
+#
+# 3. Neither the name of the copyright holder nor the names of its contributors may be
+#    used to endorse or promote products derived from this software without specific
+#    prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+# THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+set(monero-lws_wire-json_sources error.cpp read.cpp write.cpp)
+set(monero-lws_wire-json_headers base.h error.h fwd.h read.h write.h)
+
+add_library(monero-lws-wire-json ${monero-lws_wire-json_sources} ${monero-lws-wire-json_headers})
+target_link_libraries(monero-lws-wire-json  monero::libraries monero-lws-wire)

src/wire/json/base.h

@@ -0,0 +1,50 @@
+// Copyright (c) 2020, The Monero Project
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#pragma once
+
+#include <string>
+
+#include "byte_slice.h"
+#include "common/expect.h"
+#include "wire/json/fwd.h"
+
+namespace wire
+{
+  struct json
+  {
+    using input_type = json_reader;
+    using output_type = json_writer;
+
+    template<typename T>
+    static expect<T> from_bytes(std::string&& source);
+
+    template<typename T>
+    static epee::byte_slice to_bytes(const T& source);
+  };
+}
+

src/wire/json/error.cpp

@@ -0,0 +1,108 @@
+// Copyright (c) 2020, The Monero Project
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "error.h"
+
+namespace wire
+{
+namespace error
+{
+  const char* get_string(const rapidjson_e value) noexcept
+  {
+    switch (rapidjson::ParseErrorCode(value))
+    {
+    default:
+      break;
+
+    case rapidjson::kParseErrorNone:
+      return "No JSON parsing errors";
+
+      // from rapidjson
+    case rapidjson::kParseErrorDocumentEmpty:
+      return "JSON parser expected non-empty document";
+    case rapidjson::kParseErrorDocumentRootNotSingular:
+      return "JSON parser expected one value at root level";
+
+    case rapidjson::kParseErrorValueInvalid:
+      return "JSON parser found invalid value";
+
+    case rapidjson::kParseErrorObjectMissName:
+      return "JSON parser expected name for object field";
+    case rapidjson::kParseErrorObjectMissColon:
+      return "JSON parser expected ':' between name and value";
+    case rapidjson::kParseErrorObjectMissCommaOrCurlyBracket:
+      return "JSON parser expected ',' or '}'";
+
+    case rapidjson::kParseErrorArrayMissCommaOrSquareBracket:
+      return "JSON parser expected ',' or ']'";
+
+    case rapidjson::kParseErrorStringUnicodeEscapeInvalidHex:
+      return "JSON parser found invalid unicode escape";
+    case rapidjson::kParseErrorStringUnicodeSurrogateInvalid:
+      return "JSON parser found invalid unicode surrogate value";
+    case rapidjson::kParseErrorStringEscapeInvalid:
+      return "JSON parser found invalid escape sequence in string value";
+    case rapidjson::kParseErrorStringMissQuotationMark:
+      return "JSON parser expected '\"'";
+    case rapidjson::kParseErrorStringInvalidEncoding:
+      return "JSON parser found invalid encoding";
+
+    case rapidjson::kParseErrorNumberTooBig:
+      return "JSON parser found number value larger than double float precision";
+    case rapidjson::kParseErrorNumberMissFraction:
+      return "JSON parser found number missing fractional component";
+    case rapidjson::kParseErrorNumberMissExponent:
+      return "JSON parser found number missing exponent";
+
+    case rapidjson::kParseErrorTermination:
+      return "JSON parser was stopped";
+    case rapidjson::kParseErrorUnspecificSyntaxError:
+      return "JSON parser found syntax error";
+    }
+
+    return "Unknown JSON parser error";
+  }
+
+  const std::error_category& rapidjson_category() noexcept
+  {
+    struct category final : std::error_category
+    {
+      virtual const char* name() const noexcept override final
+      {
+        return "wire::error::rapidjson_category()";
+      }
+
+      virtual std::string message(int value) const override final
+      {
+        return get_string(rapidjson_e(value));
+      }
+    };
+    static const category instance{};
+    return instance;
+  }
+} // error
+} // wire

src/wire/json/error.h

@@ -0,0 +1,53 @@
+// Copyright (c) 2020, The Monero Project
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#pragma once
+
+#include <rapidjson/error/error.h>
+#include <system_error>
+
+namespace wire
+{
+namespace error
+{
+  //! Type wrapper to "grab" rapidjson errors
+  enum class rapidjson_e : int {};
+
+  //! \return Static string describing error `value`.
+  const char* get_string(rapidjson_e value) noexcept;
+
+  //! \return Category for rapidjson generated errors.
+  const std::error_category& rapidjson_category() noexcept;
+
+  //! \return Error code with `value` and `rapidjson_category()`.
+  inline std::error_code make_error_code(rapidjson_e value) noexcept
+  {
+    return std::error_code{int(value), rapidjson_category()};
+  }
+}
+}
+

src/wire/json/fwd.h

@@ -0,0 +1,45 @@
+// Copyright (c) 2020, The Monero Project
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#pragma once
+
+#define WIRE_JSON_DECLARE_ENUM(type)		\
+  const char* get_string(type) noexcept;        \
+  void read_bytes(::wire::json_reader&, type&);	\
+  void write_bytes(:wire::json_writer&, type)
+
+#define WIRE_JSON_DECLARE_OBJECT(type)			\
+  void read_bytes(::wire::json_reader&, type&);         \
+  void write_bytes(::wire::json_writer&, const type&)
+
+namespace wire
+{
+  struct json;
+  class json_reader;
+  class json_writer;
+}
+

src/wire/json/read.cpp

@@ -0,0 +1,413 @@
+// Copyright (c) 2020, The Monero Project
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "read.h"
+
+#include <algorithm>
+#include <limits>
+#include <rapidjson/memorystream.h>
+#include <stdexcept>
+
+#include "common/expect.h" // monero/src
+#include "hex.h"           // monero/contrib/epee/include
+#include "wire/error.h"
+#include "wire/json/error.h"
+
+namespace
+{
+  //! Maximum number of bytes to display "near" JSON error.
+  constexpr const std::size_t snippet_size = 30;
+
+  struct json_default_reject : rapidjson::BaseReaderHandler<rapidjson::UTF8<>, json_default_reject>
+  {
+    bool Default() const noexcept { return false; }
+  };
+
+  //! \throw std::system_error by converting `code` into a std::error_code
+  [[noreturn]] void throw_json_error(const epee::span<char> source, const rapidjson::Reader& reader, const wire::error::schema expected)
+  {
+    const std::size_t offset = std::min(source.size(), reader.GetErrorOffset());
+    const std::size_t start = offset;//std::max(snippet_size / 2, offset) - (snippet_size / 2);
+    const std::size_t end = start + std::min(snippet_size, source.size() - start);
+
+    const boost::string_ref text{source.data() + start, end - start};
+    const rapidjson::ParseErrorCode parse_error = reader.GetParseErrorCode();
+    switch (parse_error)
+    {
+    default:
+      WIRE_DLOG_THROW(wire::error::rapidjson_e(parse_error), "near \"" << text << '"');
+    case rapidjson::kParseErrorNone:
+    case rapidjson::kParseErrorTermination: // the handler returned false
+      break;
+    }
+    WIRE_DLOG_THROW(expected, "near '" << text << '\'');
+  }
+}
+
+namespace wire
+{
+  struct json_reader::rapidjson_sax
+  {
+    struct string_contents
+    {
+       const char* ptr;
+       std::size_t length;
+    };
+
+    union
+    {
+      bool boolean;
+      std::intmax_t integer;
+      std::uintmax_t unsigned_integer;
+      double number;
+      string_contents string;
+    } value;
+
+    error::schema expected_;
+    bool negative;
+
+    explicit rapidjson_sax(error::schema expected) noexcept
+      : expected_(expected), negative(false)
+    {}
+
+    bool Null() const noexcept
+    {
+      return expected_ == error::schema::none;
+    }
+
+    bool Bool(bool i) noexcept
+    {
+      value.boolean = i;
+      return expected_ == error::schema::boolean || expected_ == error::schema::none;
+    }
+
+    bool Int(int i) noexcept
+    {
+      return Int64(i);
+    }
+    bool Uint(unsigned i) noexcept
+    {
+      return Uint64(i);
+    }
+    bool Int64(std::int64_t i) noexcept
+    {
+      negative = true;
+      switch(expected_)
+      {
+      default:
+        return false;
+      case error::schema::integer:
+        value.integer = i;
+        break;
+      case error::schema::number:
+        value.number = i;
+        break;
+      case error::schema::none:
+        break;
+      }
+      return true;
+    }
+    bool Uint64(std::uint64_t i) noexcept
+    {
+      switch (expected_)
+      {
+      default:
+        return false;
+      case error::schema::integer:
+        value.unsigned_integer = i;
+        break;
+      case error::schema::number:
+        value.number = i;
+        break;
+      case error::schema::none:
+        break;
+      }
+      return true;
+    }
+
+    bool Double(double i) noexcept
+    {
+      value.number = i;
+      return expected_ == error::schema::number || expected_ == error::schema::none;
+    }
+
+    bool RawNumber(const char*, std::size_t, bool) const noexcept
+    {
+      return false;
+    }
+
+    bool String(const char* str, std::size_t length, bool) noexcept
+    {
+      value.string = {str, length};
+      return expected_ == error::schema::string || expected_ == error::schema::none;
+    }
+    bool Key(const char* str, std::size_t length, bool)
+    {
+      return String(str, length, true);
+    }
+
+    bool StartArray() const noexcept { return expected_ == error::schema::none; }
+    bool EndArray(std::size_t) const noexcept { return expected_ == error::schema::none; }
+    bool StartObject() const noexcept { return expected_ == error::schema::none; }
+    bool EndObject(std::size_t) const noexcept { return expected_ == error::schema::none; }
+  };
+
+  void json_reader::read_next_value(rapidjson_sax& handler)
+  {
+    rapidjson::InsituStringStream stream{current_.data()};
+    if (!reader_.Parse<rapidjson::kParseStopWhenDoneFlag>(stream, handler))
+      throw_json_error(current_, reader_, handler.expected_);
+    current_.remove_prefix(stream.Tell());
+  }
+
+  char json_reader::get_next_token()
+  {
+    rapidjson::InsituStringStream stream{current_.data()};
+    rapidjson::SkipWhitespace(stream);
+    current_.remove_prefix(stream.Tell());
+    return stream.Peek();
+  }
+
+  boost::string_ref json_reader::get_next_string()
+  {
+    if (get_next_token() != '"')
+      WIRE_DLOG_THROW_(error::schema::string);
+    current_.remove_prefix(1);
+
+    void const* const end = std::memchr(current_.data(), '"', current_.size());
+    if (!end)
+      WIRE_DLOG_THROW_(error::rapidjson_e(rapidjson::kParseErrorStringMissQuotationMark));
+
+    char const* const begin = current_.data();
+    const std::size_t length = current_.remove_prefix(static_cast<const char*>(end) - current_.data() + 1);
+    return {begin, length - 1};
+  }
+
+  void json_reader::skip_value()
+  {
+    rapidjson_sax accept_all{error::schema::none};
+    read_next_value(accept_all);
+  }
+
+  json_reader::json_reader(std::string&& source)
+    : reader(),
+      source_(std::move(source)),
+      current_(std::addressof(source_[0]), source_.size()),
+      reader_()
+  {}
+
+  void json_reader::check_complete() const
+  {
+    if (depth())
+      WIRE_DLOG_THROW(error::rapidjson_e(rapidjson::kParseErrorUnspecificSyntaxError), "Unexpected end");
+  }
+
+  bool json_reader::boolean()
+  {
+    rapidjson_sax json_bool{error::schema::boolean};
+    read_next_value(json_bool);
+    return json_bool.value.boolean;
+  }
+
+  std::intmax_t json_reader::integer()
+  {
+    rapidjson_sax json_int{error::schema::integer};
+    read_next_value(json_int);
+    if (json_int.negative)
+      return json_int.value.integer;
+    return integer::convert_to<std::intmax_t>(json_int.value.unsigned_integer);
+  }
+
+  std::uintmax_t json_reader::unsigned_integer()
+  {
+    rapidjson_sax json_uint{error::schema::integer};
+    read_next_value(json_uint);
+    if (!json_uint.negative)
+      return json_uint.value.unsigned_integer;
+    return integer::convert_to<std::uintmax_t>(json_uint.value.integer);
+  }
+    /*
+  const std::vector<std::uintmax_t>& json_reader::unsigned_integer_array()
+  {
+      read_next_unsigned_array(
+      }*/
+
+  std::uintmax_t json_reader::safe_unsigned_integer()
+  {
+    if (get_next_token() != '"')
+      WIRE_DLOG_THROW_(error::schema::string);
+    current_.remove_prefix(1);
+
+    const std::uintmax_t out = unsigned_integer();
+
+    if (get_next_token() != '"')
+      WIRE_DLOG_THROW_(error::rapidjson_e(rapidjson::kParseErrorStringMissQuotationMark));
+    current_.remove_prefix(1);
+
+    return out;
+  }
+
+  double json_reader::real()
+  {
+    rapidjson_sax json_number{error::schema::number};
+    read_next_value(json_number);
+    return json_number.value.number;
+  }
+
+  std::string json_reader::string()
+  {
+    rapidjson_sax json_string{error::schema::string};
+    read_next_value(json_string);
+    return std::string{json_string.value.string.ptr, json_string.value.string.length};
+  }
+
+  std::vector<std::uint8_t> json_reader::binary()
+  {
+    const boost::string_ref value = get_next_string();
+
+    std::vector<std::uint8_t> out;
+    out.resize(value.size() / 2);
+
+    if (!epee::from_hex::to_buffer(epee::to_mut_span(out), value))
+      WIRE_DLOG_THROW_(error::schema::binary);
+
+    return out;
+  }
+
+  void json_reader::binary(epee::span<std::uint8_t> dest)
+  {
+    const boost::string_ref value = get_next_string();
+    if (!epee::from_hex::to_buffer(dest, value))
+      WIRE_DLOG_THROW(error::schema::fixed_binary, "of size" << dest.size() * 2 << " but got " << value.size());
+  }
+
+  std::size_t json_reader::enumeration(epee::span<char const* const> enums)
+  {
+    rapidjson_sax json_enum{error::schema::string};
+    read_next_value(json_enum);
+
+    const boost::string_ref value{json_enum.value.string.ptr, json_enum.value.string.length};
+    for (std::size_t i = 0; i < enums.size(); ++i)
+    {
+      if (value == enums[i])
+        return i;
+    }
+
+    WIRE_DLOG_THROW(error::schema::enumeration, value << " is not a valid enum");
+    return enums.size();
+  }
+
+  std::size_t json_reader::start_array()
+  {
+    if (get_next_token() != '[')
+      WIRE_DLOG_THROW_(error::schema::array);
+    current_.remove_prefix(1);
+    increment_depth();
+    return 0;
+  }
+
+  bool json_reader::is_array_end(const std::size_t count)
+  {
+    const char next = get_next_token();
+    if (next == 0)
+      WIRE_DLOG_THROW_(error::rapidjson_e(rapidjson::kParseErrorArrayMissCommaOrSquareBracket));
+    if (next == ']')
+    {
+      current_.remove_prefix(1);
+      return true;
+    }
+
+    if (count)
+    {
+      if (next != ',')
+        WIRE_DLOG_THROW_(error::rapidjson_e(rapidjson::kParseErrorArrayMissCommaOrSquareBracket));
+      current_.remove_prefix(1);
+    }
+    return false;
+  }
+
+  std::size_t json_reader::start_object()
+  {
+    if (get_next_token() != '{')
+      WIRE_DLOG_THROW_(error::schema::object);
+    current_.remove_prefix(1);
+    increment_depth();
+    return 0;
+  }
+
+  bool json_reader::key(const epee::span<const key_map> map, std::size_t& state, std::size_t& index)
+  {
+    rapidjson_sax json_key{error::schema::string};
+    const auto process_key = [map] (const rapidjson_sax::string_contents value)
+    {
+      const boost::string_ref key{value.ptr, value.length};
+      for (std::size_t i = 0; i < map.size(); ++i)
+      {
+        if (map[i].name == key)
+          return i;
+      }
+      return map.size();
+    };
+
+    index = map.size();
+    for (;;)
+    {
+      // check for object or text end
+      const char next = get_next_token();
+      if (next == 0)
+        WIRE_DLOG_THROW_(error::rapidjson_e(rapidjson::kParseErrorObjectMissCommaOrCurlyBracket));
+      if (next == '}')
+      {
+        current_.remove_prefix(1);
+        return false;
+      }
+
+      // parse next field token
+      if (state)
+      {
+        if (next != ',')
+          WIRE_DLOG_THROW_(error::rapidjson_e(rapidjson::kParseErrorObjectMissCommaOrCurlyBracket));
+        current_.remove_prefix(1);
+      }
+      ++state;
+
+      // parse key
+      read_next_value(json_key);
+      index = process_key(json_key.value.string);
+      if (get_next_token() != ':')
+        WIRE_DLOG_THROW_(error::rapidjson_e(rapidjson::kParseErrorObjectMissColon));
+      current_.remove_prefix(1);
+
+      // parse value
+      if (index != map.size())
+        break;
+      skip_value();
+    }
+    return true;
+  }
+}
+

src/wire/json/read.h

@@ -0,0 +1,134 @@
+// Copyright (c) 2020, The Monero Project
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#pragma once
+
+#include <boost/utility/string_ref.hpp>
+#include <cstddef>
+#include <rapidjson/reader.h>
+#include <string>
+#include <type_traits>
+#include <utility>
+#include <vector>
+
+#include "wire/field.h"
+#include "wire/json/base.h"
+#include "wire/read.h"
+#include "wire/traits.h"
+
+namespace wire
+{
+  //! Reads JSON tokens one-at-a-time for DOMless parsing
+  class json_reader : public reader
+  {
+    struct rapidjson_sax;
+
+    std::string source_;
+    epee::span<char> current_;
+    rapidjson::Reader reader_;
+
+    void read_next_value(rapidjson_sax& handler);
+    char get_next_token();
+    boost::string_ref get_next_string();
+
+    //! Skips next value. \throw wire::exception if invalid JSON syntax.
+    void skip_value();
+
+  public:
+    explicit json_reader(std::string&& source);
+
+    //! \throw wire::exception if JSON parsing is incomplete.
+    void check_complete() const override final;
+
+    //! \throw wire::exception if next token not a boolean.
+    bool boolean() override final;
+
+    //! \throw wire::expception if next token not an integer.
+    std::intmax_t integer() override final;
+
+    //! \throw wire::exception if next token not an unsigned integer.
+    std::uintmax_t unsigned_integer() override final;
+
+    //! \throw wire::exception if next token is not an integer encoded as string
+    std::uintmax_t safe_unsigned_integer();
+
+    //! \throw wire::exception if next token not a valid real number
+    double real() override final;
+
+    //! \throw wire::exception if next token not a string
+    std::string string() override final;
+
+    //! \throw wire::exception if next token cannot be read as hex
+    std::vector<std::uint8_t> binary() override final;
+
+    //! \throw wire::exception if next token cannot be read as hex into `dest`.
+    void binary(epee::span<std::uint8_t> dest) override final;
+
+    //! \throw wire::exception if invalid next token invalid enum. \return Index in `enums`.
+    std::size_t enumeration(epee::span<char const* const> enums) override final;
+
+
+    //! \throw wire::exception if next token not `[`.
+    std::size_t start_array() override final;
+
+    //! Skips whitespace to next token. \return True if next token is eof or ']'.
+    bool is_array_end(std::size_t count) override final;
+
+
+    //! \throw wire::exception if next token not `{`.
+    std::size_t start_object() override final;
+
+    /*! \throw wire::exception if next token not key or `}`.
+        \param[out] index of key match within `map`.
+        \return True if another value to read. */
+    bool key(epee::span<const key_map> map, std::size_t&, std::size_t& index) override final;
+  };
+
+
+  // Don't call `read` directly in this namespace, do it from `wire_read`.
+
+  template<typename T>
+  expect<T> json::from_bytes(std::string&& bytes)
+  {
+    json_reader source{std::move(bytes)};
+    return wire_read::to<T>(source);
+  }
+
+  // specialization prevents type "downgrading" to base type in cpp files
+
+  template<typename T>
+  inline void array(json_reader& source, T& dest)
+  {
+    wire_read::array(source, dest);
+  }
+
+  template<typename... T>
+  inline void object(json_reader& source, T... fields)
+  {
+    wire_read::object(source, wire_read::tracker<T>{std::move(fields)}...);
+  }
+} // wire

src/wire/json/write.cpp

@@ -0,0 +1,167 @@
+// Copyright (c) 2020, The Monero Project
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "write.h"
+
+#include <stdexcept>
+
+#include "hex.h" // monero/contrib/epee/include
+
+namespace
+{
+  constexpr const unsigned flush_threshold = 100;
+}
+
+namespace wire
+{
+  void json_writer::do_flush(epee::span<const std::uint8_t>)
+  {}
+
+  void json_writer::check_flush()
+  {
+    if (needs_flush_ && (bytes_.increase_size() < flush_threshold || bytes_.increase_size() - flush_threshold < bytes_.size()))
+      flush();
+  }
+
+  void json_writer::check_complete()
+  {
+    if (!formatter_.IsComplete())
+      throw std::logic_error{"json_writer::take_json() failed with incomplete JSON tree"};
+  }
+  epee::byte_slice json_writer::take_json()
+  {
+    check_complete();
+    epee::byte_slice out{std::move(bytes_)};
+    formatter_.Reset(bytes_);
+    return out;
+  }
+
+  json_writer::~json_writer() noexcept
+  {}
+
+  std::array<char, uint_to_string_size> json_writer::to_string(const std::uintmax_t value) noexcept
+  {
+    static_assert(std::numeric_limits<std::uintmax_t>::max() <= std::numeric_limits<std::uint64_t>::max(), "bad uint conversion");
+    std::array<char, uint_to_string_size> buf{{}};
+    rapidjson::internal::u64toa(std::uint64_t(value), buf.data());
+    return buf;
+  }
+
+  void json_writer::integer(const int source)
+  {
+    formatter_.Int(source);
+    check_flush();
+  }
+  void json_writer::integer(const std::intmax_t source)
+  {
+    static_assert(std::numeric_limits<std::int64_t>::min() <= std::numeric_limits<std::intmax_t>::min(), "too small");
+    static_assert(std::numeric_limits<std::intmax_t>::max() <= std::numeric_limits<std::int64_t>::max(), "too large");
+    formatter_.Int64(source);
+    check_flush();
+  }
+  void json_writer::unsigned_integer(const unsigned source)
+  {
+    formatter_.Uint(source);
+    check_flush();
+  }
+    void json_writer::unsigned_integer(const std::uintmax_t source)
+  {
+    static_assert(std::numeric_limits<std::uintmax_t>::max() <= std::numeric_limits<std::uint64_t>::max(), "too large");
+    formatter_.Uint64(source);
+    check_flush();
+  }
+  void json_writer::real(const double source)
+  {
+    formatter_.Double(source);
+    check_flush();
+  }
+
+  void json_writer::string(const boost::string_ref source)
+  {
+    formatter_.String(source.data(), source.size());
+    check_flush();
+  }
+  void json_writer::binary(epee::span<const std::uint8_t> source)
+  {/* TODO update monero project
+    std::array<char, 256> buffer;
+    if (source.size() <= buffer.size() / 2)
+    {
+      if (!epee::to_hex::buffer({buffer.data(), source.size() * 2}, source))
+        throw std::logic_error{"Invalid buffer size for binary->hex conversion"};
+      string({buffer.data(), source.size() * 2});
+    }
+    else
+    {*/
+      const auto hex = epee::to_hex::string(source);
+      string(hex);
+      //}
+  }
+
+  void json_writer::enumeration(const std::size_t index, const epee::span<char const* const> enums)
+  {
+    if (enums.size() < index)
+      throw std::logic_error{"Invalid enum/string value"};
+    string({enums[index], std::strlen(enums[index])});
+  }
+
+  void json_writer::start_array(std::size_t)
+  {
+    formatter_.StartArray();
+  }
+  void json_writer::end_array()
+  {
+    formatter_.EndArray();
+  }
+
+  void json_writer::start_object(std::size_t)
+  {
+    formatter_.StartObject();
+  }
+  void json_writer::key(const boost::string_ref str)
+  {
+    formatter_.Key(str.data(), str.size());
+    check_flush();
+  }
+  void json_writer::key(const std::uintmax_t id)
+  {
+    auto str = json_writer::to_string(id);
+    key(str.data());
+  }
+  void json_writer::key(unsigned, const boost::string_ref str)
+  {
+    key(str);
+  }
+  void json_writer::end_object()
+  {
+    formatter_.EndObject();
+  }
+
+  void json_stream_writer::do_flush(epee::span<const std::uint8_t> bytes)
+  {
+    dest.write(reinterpret_cast<const char*>(bytes.data()), bytes.size());
+  }
+}

src/wire/json/write.h

@@ -0,0 +1,184 @@
+// Copyright (c) 2020, The Monero Project
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#pragma once
+
+#include <array>
+#include <boost/utility/string_ref.hpp>
+#include <cstdint>
+#include <limits>
+#include <rapidjson/writer.h>
+
+#include "byte_stream.h" // monero/contrib/epee/include
+#include "span.h"        // monero/contrib/epee/include
+#include "wire/field.h"
+#include "wire/filters.h"
+#include "wire/json/base.h"
+#include "wire/traits.h"
+#include "wire/write.h"
+
+namespace wire
+{
+  constexpr const std::size_t uint_to_string_size =
+    std::numeric_limits<std::uintmax_t>::digits10 + 2;
+
+  //! Writes JSON tokens one-at-a-time for DOMless output.
+  class json_writer : public writer
+  {
+    epee::byte_stream bytes_;
+    rapidjson::Writer<epee::byte_stream> formatter_;
+    bool needs_flush_;
+
+    //! \return True if buffer needs to be cleared
+    virtual void do_flush(epee::span<const uint8_t>);
+
+    //! Flush written bytes to `do_flush(...)` if configured
+    void check_flush();
+
+  protected:
+    json_writer(bool needs_flush)
+      : writer(), bytes_(), formatter_(bytes_), needs_flush_(needs_flush)
+    {}
+
+    //! \throw std::logic_error if incomplete JSON tree
+    void check_complete();
+
+    //! \throw std::logic_error if incomplete JSON tree. \return JSON bytes
+    epee::byte_slice take_json();
+
+    //! Flush bytes in local buffer to `do_flush(...)`
+    void flush()
+    {
+      do_flush({bytes_.data(), bytes_.size()});
+      bytes_ = epee::byte_stream{}; // TODO create .clear() method in monero project
+    }
+
+  public:
+    json_writer(const json_writer&) = delete;
+    virtual ~json_writer() noexcept;
+    json_writer& operator=(const json_writer&) = delete;
+
+    //! \return Null-terminated buffer containing uint as decimal ascii
+    static std::array<char, uint_to_string_size> to_string(std::uintmax_t) noexcept;
+
+    void integer(int) override final;
+    void integer(std::intmax_t) override final;
+
+    void unsigned_integer(unsigned) override final;
+    void unsigned_integer(std::uintmax_t) override final;
+
+    void real(double) override final;
+
+    void string(boost::string_ref) override final;
+    void binary(epee::span<const std::uint8_t> source) override final;
+
+    void enumeration(std::size_t index, epee::span<char const* const> enums) override final;
+
+    void start_array(std::size_t) override final;
+    void end_array() override final;
+
+    void start_object(std::size_t) override final;
+    void key(std::uintmax_t) override final;
+    void key(boost::string_ref) override final;
+    void key(unsigned, boost::string_ref) override final;
+    void end_object() override final;
+  };
+
+  //! Buffers entire JSON message in memory
+  struct json_slice_writer final : json_writer
+  {
+    explicit json_slice_writer()
+      : json_writer(false)
+    {}
+
+    //! \throw std::logic_error if incomplete JSON tree \return JSON bytes
+    epee::byte_slice take_bytes()
+    {
+      return json_writer::take_json();
+    }
+  };
+
+  //! Periodically flushes JSON data to `std::ostream`
+  class json_stream_writer final : public json_writer
+  {
+    std::ostream& dest;
+
+    virtual void do_flush(epee::span<const std::uint8_t>) override final;
+  public:
+    explicit json_stream_writer(std::ostream& dest)
+      : json_writer(true), dest(dest)
+    {}
+
+    //! Flush remaining bytes to stream \throw std::logic_error if incomplete JSON tree
+    void finish()
+    {
+      check_complete();
+      flush();
+    }
+  };
+
+  template<typename T>
+  epee::byte_slice json::to_bytes(const T& source)
+  {
+    return wire_write::to_bytes<json_slice_writer>(source);
+  }
+
+  template<typename T, typename F = identity_>
+  inline void array(json_writer& dest, const T& source, F filter = F{})
+  {
+    // works with "lazily" computed ranges
+    wire_write::array(dest, source, 0, std::move(filter));
+  }
+  template<typename T, typename F>
+  inline void write_bytes(json_writer& dest, as_array_<T, F> source)
+  {
+    wire::array(dest, source.get_value(), std::move(source.filter));
+  }
+  template<typename T>
+  inline enable_if<is_array<T>::value> write_bytes(json_writer& dest, const T& source)
+  {
+    wire::array(dest, source);
+  }
+
+  template<typename T, typename F = identity_, typename G = identity_>
+  inline void dynamic_object(json_writer& dest, const T& source, F key_filter = F{}, G value_filter = G{})
+  {
+    // works with "lazily" computed ranges
+    wire_write::dynamic_object(dest, source, 0, std::move(key_filter), std::move(value_filter));
+  }
+  template<typename T, typename F, typename G>
+  inline void write_bytes(json_writer& dest, as_object_<T, F, G> source)
+  {
+    wire::dynamic_object(dest, source.get_map(), std::move(source.key_filter), std::move(source.value_filter));
+  }
+
+  template<typename... T>
+  inline void object(json_writer& dest, T... fields)
+  {
+    wire_write::object(dest, std::move(fields)...);
+  }
+}

src/wire/read.cpp

@@ -0,0 +1,61 @@
+// Copyright (c) 2020, The Monero Project
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "wire/read.h"
+
+#include <stdexcept>
+
+void wire::reader::increment_depth()
+{
+  if (++depth_ == max_read_depth())
+    WIRE_DLOG_THROW_(error::schema::maximum_depth);
+}
+
+[[noreturn]] void wire::integer::throw_exception(std::intmax_t source, std::intmax_t min)
+{
+  WIRE_DLOG_THROW(error::schema::larger_integer, source << " given when " << min << " is minimum permitted");
+}
+[[noreturn]] void wire::integer::throw_exception(std::uintmax_t source, std::uintmax_t max)
+{
+  WIRE_DLOG_THROW(error::schema::smaller_integer, source << " given when " << max << "is maximum permitted");
+}
+
+[[noreturn]] void wire_read::throw_exception(const wire::error::schema code, const char* display, epee::span<char const* const> names)
+{
+  const char* name = nullptr;
+  for (const char* elem : names)
+  {
+    if (elem != nullptr)
+    {
+      name = elem;
+      break;
+    }
+  }
+  WIRE_DLOG_THROW(code, display << (name ? name : ""));
+}
+
+

src/wire/read.h

@@ -0,0 +1,471 @@
+// Copyright (c) 2020, The Monero Project
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#pragma once
+
+#include <cstdint>
+#include <limits>
+#include <string>
+#include <tuple>
+#include <type_traits>
+#include <vector>
+
+#include "common/expect.h" // monero/src
+#include "span.h"          // monero/contrib/epee/include
+#include "wire/error.h"
+#include "wire/field.h"
+#include "wire/traits.h"
+
+namespace wire
+{
+  //! Interface for converting "wire" (byte) formats to C/C++ objects without a DOM.
+  class reader
+  {
+    std::size_t depth_; //!< Tracks number of recursive objects and arrays
+
+  protected:
+    //! \throw wire::exception if max depth is reached
+    void increment_depth();
+    void decrement_depth() noexcept { --depth_; }
+
+    reader(const reader&) = default;
+    reader(reader&&) = default;
+    reader& operator=(const reader&) = default;
+    reader& operator=(reader&&) = default;
+
+  public:
+    struct key_map
+    {
+        const char* name;
+        unsigned id; //<! For integer key formats;
+    };
+
+    //! \return Maximum read depth for both objects and arrays before erroring
+    static constexpr std::size_t max_read_depth() noexcept { return 100; }
+
+    reader() noexcept
+      : depth_(0)
+    {}
+
+    virtual ~reader() noexcept
+    {}
+
+    //! \return Number of recursive objects and arrays
+    std::size_t depth() const noexcept { return depth_; }
+
+    //! \throw wire::exception if parsing is incomplete.
+    virtual void check_complete() const = 0;
+
+    //! \throw wire::exception if next value not a boolean.
+    virtual bool boolean() = 0;
+
+    //! \throw wire::expception if next value not an integer.
+    virtual std::intmax_t integer() = 0;
+
+    //! \throw wire::exception if next value not an unsigned integer.
+    virtual std::uintmax_t unsigned_integer() = 0;
+
+    //! \throw wire::exception if next value not number
+    virtual double real() = 0;
+
+    //! throw wire::exception if next value not string
+    virtual std::string string() = 0;
+
+    // ! \throw wire::exception if next value cannot be read as binary
+    virtual std::vector<std::uint8_t> binary() = 0;
+
+    //! \throw wire::exception if next value cannot be read as binary into `dest`.
+    virtual void binary(epee::span<std::uint8_t> dest) = 0;
+
+    //! \throw wire::exception if next value invalid enum. \return Index in `enums`.
+    virtual std::size_t enumeration(epee::span<char const* const> enums) = 0;
+
+    /*! \throw wire::exception if next value not array
+        \return Number of values to read before calling `is_array_end()`. */
+    virtual std::size_t start_array() = 0;
+
+    //! \return True if there is another element to read.
+    virtual bool is_array_end(std::size_t count) = 0;
+
+    //! \throw wire::exception if array end delimiter not present.
+    void end_array() noexcept { decrement_depth(); }
+
+
+    //! \throw wire::exception if not object begin. \return State to be given to `key(...)` function.
+    virtual std::size_t start_object() = 0;
+
+    /*! Read a key of an object field and match against a known list of keys.
+       Skips or throws exceptions on unknown fields depending on implementation
+       settings.
+
+      \param map of known keys (strings and integer) that are valid.
+      \param[in,out] state returned by `start_object()` or `key(...)` whichever
+        was last.
+      \param[out] index of match found in `map`.
+
+      \throw wire::exception if next value not a key.
+      \throw wire::exception if next key not found in `map` and skipping
+        fields disabled.
+
+      \return True if this function found a field in `map` to process.
+     */
+    virtual bool key(epee::span<const key_map> map, std::size_t& state, std::size_t& index) = 0;
+
+    void end_object() noexcept { decrement_depth(); }
+  };
+
+  inline void read_bytes(reader& source, bool& dest)
+  {
+    dest = source.boolean();
+  }
+
+  inline void read_bytes(reader& source, double& dest)
+  {
+    dest = source.real();
+  }
+
+  inline void read_bytes(reader& source, std::string& dest)
+  {
+    dest = source.string();
+  }
+
+  template<typename R>
+  inline void read_bytes(R& source, std::vector<std::uint8_t>& dest)
+  {
+    dest = source.binary();
+  }
+
+  template<typename T>
+  inline enable_if<is_blob<T>::value> read_bytes(reader& source, T& dest)
+  {
+    source.binary(epee::as_mut_byte_span(dest));
+  }
+
+  namespace integer
+  {
+    [[noreturn]] void throw_exception(std::intmax_t source, std::intmax_t min);
+    [[noreturn]] void throw_exception(std::uintmax_t source, std::uintmax_t max);
+
+    template<typename Target, typename U>
+    inline Target convert_to(const U source)
+    {
+      using common = typename std::common_type<Target, U>::type;
+      static constexpr const Target target_min = std::numeric_limits<Target>::min();
+      static constexpr const Target target_max = std::numeric_limits<Target>::max();
+
+      /* After optimizations, this is:
+           * 1 check for unsigned -> unsigned (uint, uint)
+           * 2 checks for signed -> signed (int, int)
+           * 2 checks for signed -> unsigned-- (
+           * 1 check for unsigned -> signed (uint, uint)
+
+         Put `WIRE_DLOG_THROW` in cpp to reduce code/ASM duplication. Do not
+         remove first check, signed values can be implicitly converted to
+         unsigned in some checks. */
+      if (!std::numeric_limits<Target>::is_signed && source < 0)
+        throw_exception(std::intmax_t(source), std::intmax_t(0));
+      else if (common(source) < common(target_min))
+        throw_exception(std::intmax_t(source), std::intmax_t(target_min));
+      else if (common(target_max) < common(source))
+        throw_exception(std::uintmax_t(source), std::uintmax_t(target_max));
+
+      return Target(source);
+    }
+  }
+
+  inline void read_bytes(reader& source, char& dest)
+  {
+    dest = integer::convert_to<char>(source.integer());
+  }
+  inline void read_bytes(reader& source, short& dest)
+  {
+    dest = integer::convert_to<short>(source.integer());
+  }
+  inline void read_bytes(reader& source, int& dest)
+  {
+    dest = integer::convert_to<int>(source.integer());
+  }
+  inline void read_bytes(reader& source, long& dest)
+  {
+    dest = integer::convert_to<long>(source.integer());
+  }
+  inline void read_bytes(reader& source, long long& dest)
+  {
+    dest = integer::convert_to<long long>(source.integer());
+  }
+
+  inline void read_bytes(reader& source, unsigned char& dest)
+  {
+    dest = integer::convert_to<unsigned char>(source.unsigned_integer());
+  }
+  inline void read_bytes(reader& source, unsigned short& dest)
+  {
+    dest = integer::convert_to<unsigned short>(source.unsigned_integer());
+  }
+  inline void read_bytes(reader& source, unsigned& dest)
+  {
+    dest = integer::convert_to<unsigned>(source.unsigned_integer());
+  }
+  inline void read_bytes(reader& source, unsigned long& dest)
+  {
+    dest = integer::convert_to<unsigned long>(source.unsigned_integer());
+  }
+  inline void read_bytes(reader& source, unsigned long long& dest)
+  {
+    dest = integer::convert_to<unsigned long long>(source.unsigned_integer());
+  }
+} // wire
+
+namespace wire_read
+{
+    /*! Don't add a function called `read_bytes` to this namespace, it will prevent
+      ADL lookup. ADL lookup delays the function searching until the template
+      is used instead of when its defined. This allows the unqualified calls to
+      `read_bytes` in this namespace to "find" user functions that are declared
+      after these functions (the technique behind `boost::serialization`). */
+
+  [[noreturn]] void throw_exception(wire::error::schema code, const char* display, epee::span<char const* const> name_list);
+
+  //! \return `T` converted from `source` or error.
+  template<typename T, typename R>
+  inline expect<T> to(R& source)
+  {
+    try
+    {
+      T dest{};
+      read_bytes(source, dest);
+      source.check_complete();
+      return dest;
+    }
+    catch (const wire::exception& e)
+    {
+      return e.code();
+    }
+  }
+
+  template<typename R, typename T>
+  inline void array(R& source, T& dest)
+  {
+    using value_type = typename T::value_type;
+    static_assert(!std::is_same<value_type, char>::value, "read array of chars as binary");
+    static_assert(!std::is_same<value_type, std::uint8_t>::value, "read array of unsigned chars as binary");
+    
+    std::size_t count = source.start_array();
+
+    dest.clear();
+    dest.reserve(count);
+
+    bool more = count;
+    while (more || !source.is_array_end(count))
+    {
+      dest.emplace_back();
+      read_bytes(source, dest.back());
+      --count;
+      more &= bool(count);
+    }
+
+    return source.end_array();
+  }
+
+  // `unpack_variant_field` identifies which of the variant types was selected. starts with index-0
+
+  template<typename R, typename T>
+  inline void unpack_variant_field(std::size_t, R&, const T&)
+  {}
+
+  template<typename R, typename T, typename U, typename... X>
+  inline void unpack_variant_field(const std::size_t index, R& source, T& variant, const wire::option<U>& head, const wire::option<X>&... tail)
+  {
+    if (index)
+      unpack_variant_field(index - 1, source, variant, tail...);
+    else
+    {
+      U dest{};
+      read_bytes(source, dest);
+      variant = std::move(dest);
+    }
+  }
+
+  // `unpack_field` expands `variant_field_`s or reads `field_`s directly
+
+  template<typename R, typename T, bool Required, typename... U>
+  inline void unpack_field(const std::size_t index, R& source, wire::variant_field_<T, Required, U...>& dest)
+  {
+    unpack_variant_field(index, source, dest.get_value(), static_cast< const wire::option<U>& >(dest)...);
+  }
+
+  template<typename R, typename T>
+  inline void unpack_field(std::size_t, R& source, wire::field_<T, true>& dest)
+  {
+    read_bytes(source, dest.get_value());
+  }
+
+  template<typename R, typename T>
+  inline void unpack_field(std::size_t, R& source, wire::field_<T, false>& dest)
+  {
+    dest.get_value().emplace();
+    read_bytes(source, *dest.get_value());
+  }
+
+  // `expand_field_map` writes a single `field_` name or all option names in a `variant_field_` to a table
+
+  template<std::size_t N>
+  inline void expand_field_map(std::size_t, wire::reader::key_map (&)[N])
+  {}
+
+  template<std::size_t N, typename T, typename... U>
+  inline void expand_field_map(std::size_t index, wire::reader::key_map (&map)[N], const T& head, const U&... tail)
+  {
+    map[index].name = head.name;
+    map[index].id = 0;
+    expand_field_map(index + 1, map, tail...);
+  }
+
+  template<std::size_t N, typename T, bool Required, typename... U>
+  inline void expand_field_map(std::size_t index, wire::reader::key_map (&map)[N], const wire::variant_field_<T, Required, U...>& field)
+  {
+    expand_field_map(index, map, static_cast< const wire::option<U> & >(field)...);
+  }
+
+  //! Tracks read status of every object field instance.
+  template<typename T>
+  class tracker
+  {
+    T field_;
+    std::size_t our_index_;
+    bool read_;
+
+  public:
+    static constexpr bool is_required() noexcept { return T::is_required(); }
+    static constexpr std::size_t count() noexcept { return T::count(); }
+
+    explicit tracker(T field)
+      : field_(std::move(field)), our_index_(0), read_(false)
+    {}
+
+    //! \return Field name if required and not read, otherwise `nullptr`.
+    const char* name_if_missing() const noexcept
+    {
+      return (is_required() && !read_) ? field_.name : nullptr;
+    }
+
+
+    //! Set all entries in `map` related to this field (expand variant types!).
+    template<std::size_t N>
+    std::size_t set_mapping(std::size_t index, wire::reader::key_map (&map)[N])
+    {
+      our_index_ = index;
+      expand_field_map(index, map, field_); // expands possible inner options
+      return index + count();
+    }
+
+    //! Try to read next value if `index` matches `this`. \return 0 if no match, 1 if optional field read, and 2 if required field read
+    template<typename R>
+    std::size_t try_read(R& source, const std::size_t index)
+    {
+      if (index < our_index_ || our_index_ + count() <= index)
+        return 0;
+      if (read_)
+        throw_exception(wire::error::schema::invalid_key, "duplicate", {std::addressof(field_.name), 1});
+
+      unpack_field(index - our_index_, source, field_);
+      read_ = true;
+      return 1 + is_required();
+    }
+  };
+
+  // `expand_tracker_map` writes all `tracker` types to a table
+
+  template<std::size_t N>
+  inline constexpr std::size_t expand_tracker_map(std::size_t index, const wire::reader::key_map (&)[N])
+  {
+    return index;
+  }
+
+  template<std::size_t N, typename T, typename... U>
+  inline void expand_tracker_map(std::size_t index, wire::reader::key_map (&map)[N], tracker<T>& head, tracker<U>&... tail)
+  {
+    expand_tracker_map(head.set_mapping(index, map), map, tail...);
+  }
+
+  template<typename R, typename... T>
+  inline void object(R& source, tracker<T>... fields)
+  {
+    static constexpr const std::size_t total_subfields = wire::sum(fields.count()...);
+    static_assert(total_subfields < 100, "algorithm uses too much stack space and linear searching");
+
+    std::size_t state = source.start_object();
+    std::size_t required = wire::sum(std::size_t(fields.is_required())...);
+
+    wire::reader::key_map map[total_subfields] = {};
+    expand_tracker_map(0, map, fields...);
+
+    std::size_t next = 0;
+    while (source.key(map, state, next))
+    {
+      switch (wire::sum(fields.try_read(source, next)...))
+      {
+      default:
+      case 0:
+        throw_exception(wire::error::schema::invalid_key, "bad map setup", nullptr);
+        break;
+      case 2:
+        --required; /* fallthrough */
+      case 1:
+        break;
+      }
+    }
+
+    if (required)
+    {
+      const char* missing[] = {fields.name_if_missing()...};
+      throw_exception(wire::error::schema::missing_key, "", missing);
+    }
+
+    source.end_object();
+  }
+} // wire_read
+
+namespace wire
+{
+  template<typename T>
+  inline void array(reader& source, T& dest)
+  {
+    wire_read::array(source, dest);
+  }
+  template<typename R, typename T>
+  inline enable_if<is_array<T>::value> read_bytes(R& source, T& dest)
+  {
+    wire_read::array(source, dest);
+  }
+
+  template<typename... T>
+  inline void object(reader& source, T... fields)
+  {
+    wire_read::object(source, wire_read::tracker<T>{std::move(fields)}...);
+  }
+}

src/wire/traits.h

@@ -0,0 +1,46 @@
+// Copyright (c) 2020, The Monero Project
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#pragma once
+
+#include <type_traits>
+#include <utility>
+
+namespace wire
+{
+  template<bool C>
+  using enable_if = typename std::enable_if<C>::type;
+
+  template<typename T>
+  struct is_array : std::false_type
+  {};
+
+  template<typename T>
+  struct is_blob : std::false_type
+  {};
+}
+

src/wire/vector.h

@@ -0,0 +1,41 @@
+// Copyright (c) 2020, The Monero Project
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#pragma once
+
+#include <type_traits>
+#include <vector>
+
+#include "wire/traits.h"
+
+namespace wire
+{
+  template<typename T>
+  struct is_array<std::vector<T>>
+    : std::true_type
+  {};
+}

src/wire/write.cpp

@@ -0,0 +1,31 @@
+// Copyright (c) 2020, The Monero Project
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "write.h"
+
+wire::writer::~writer() noexcept
+{}

src/wire/write.h

@@ -0,0 +1,224 @@
+// Copyright (c) 2020, The Monero Project
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without modification, are
+// permitted provided that the following conditions are met:
+//
+// 1. Redistributions of source code must retain the above copyright notice, this list of
+//    conditions and the following disclaimer.
+//
+// 2. Redistributions in binary form must reproduce the above copyright notice, this list
+//    of conditions and the following disclaimer in the documentation and/or other
+//    materials provided with the distribution.
+//
+// 3. Neither the name of the copyright holder nor the names of its contributors may be
+//    used to endorse or promote products derived from this software without specific
+//    prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
+// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#pragma once
+
+#include <array>
+#include <boost/utility/string_ref.hpp>
+#include <cstdint>
+#include <type_traits>
+
+#include "byte_slice.h" // monero/contrib/epee/include
+#include "span.h"       // monero/contrib/epee/include
+#include "wire/field.h"
+#include "wire/filters.h"
+#include "wire/traits.h"
+
+namespace wire
+{
+  //! Interface for converting C/C++ objects to "wire" (byte) formats.
+  struct writer
+  {
+    writer() = default;
+
+    virtual ~writer() noexcept;
+
+    virtual void integer(int) = 0;
+    virtual void integer(std::intmax_t) = 0;
+
+    virtual void unsigned_integer(unsigned) = 0;
+    virtual void unsigned_integer(std::uintmax_t) = 0;
+
+    virtual void real(double) = 0;
+
+    virtual void string(boost::string_ref) = 0;
+    virtual void binary(epee::span<const std::uint8_t> bytes) = 0;
+
+    virtual void enumeration(std::size_t index, epee::span<char const* const> enums) = 0;
+
+    virtual void start_array(std::size_t) = 0;
+    virtual void end_array() = 0;
+
+    virtual void start_object(std::size_t) = 0;
+    virtual void key(std::uintmax_t) = 0;
+    virtual void key(boost::string_ref) = 0;
+    virtual void key(unsigned, boost::string_ref) = 0; //!< Implementation should output fastest key
+    virtual void end_object() = 0;
+
+  protected:
+    writer(const writer&) = default;
+    writer(writer&&) = default;
+    writer& operator=(const writer&) = default;
+    writer& operator=(writer&&) = default;
+  };
+
+  // leave in header, compiler can de-virtualize when final type is given
+
+  inline void write_bytes(writer& dest, const int source)
+  {
+    dest.integer(source);
+  }
+  inline void write_bytes(writer& dest, const std::intmax_t source)
+  {
+    dest.integer(source);
+  }
+
+  inline void write_bytes(writer& dest, const unsigned source)
+  {
+    dest.unsigned_integer(source);
+  }
+  inline void write_bytes(writer& dest, const std::uintmax_t source)
+  {
+    dest.unsigned_integer(source);
+  }
+
+  inline void write_bytes(writer& dest, const double source)
+  {
+    dest.real(source);
+  }
+
+  inline void write_bytes(writer& dest, const boost::string_ref source)
+  {
+    dest.string(source);
+  }
+
+  template<typename T>
+  inline enable_if<is_blob<T>::value> write_bytes(writer& dest, const T& source)
+  {
+    dest.binary(epee::as_byte_span(source));
+  }
+
+  inline void write_bytes(writer& dest, const epee::span<const std::uint8_t> source)
+  {
+    dest.binary(source);
+  }
+}
+
+namespace wire_write
+{
+  /*! Don't add a function called `write_bytes` to this namespace, it will
+      prevent ADL lookup. ADL lookup delays the function searching until the
+      template is used instead of when its defined. This allows the unqualified
+      calls to `write_bytes` in this namespace to "find" user functions that are
+      declared after these functions. */
+
+  template<typename W, typename T>
+  inline epee::byte_slice to_bytes(const T& value)
+  {
+    W dest{};
+    write_bytes(dest, value);
+    return dest.take_bytes();
+  }
+
+  template<typename W, typename T, typename F = wire::identity_>
+  inline void array(W& dest, const T& source, const std::size_t count, F filter = F{})
+  {
+    using value_type = typename T::value_type;
+    static_assert(!std::is_same<value_type, char>::value, "write array of chars as binary");
+    static_assert(!std::is_same<value_type, std::uint8_t>::value, "write array of unsigned chars as binary");
+
+    dest.start_array(count);
+    for (const auto& elem : source)
+      write_bytes(dest, filter(elem));
+    dest.end_array();
+  }
+
+  template<typename W, typename T>
+  inline bool field(W& dest, const wire::field_<T, true> elem)
+  {
+    dest.key(0, elem.name);
+    write_bytes(dest, elem.get_value());
+    return true;
+  }
+
+  template<typename W, typename T>
+  inline bool field(W& dest, const wire::field_<T, false> elem)
+  {
+    if (bool(elem.get_value()))
+    {
+      dest.key(0, elem.name);
+      write_bytes(dest, *elem.get_value());
+    }
+    return true;
+  }
+
+  template<typename W, typename... T>
+  inline void object(W& dest, T... fields)
+  {
+    dest.start_object(wire::sum(std::size_t(wire::available(fields))...));
+    const bool dummy[] = {field(dest, std::move(fields))...};
+    dest.end_object();
+  }
+
+  template<typename W, typename T, typename F, typename G>
+  inline void dynamic_object(W& dest, const T& values, const std::size_t count, F key_filter, G value_filter)
+  {
+    dest.start_object(count);
+    for (const auto& elem : values)
+    {
+      dest.key(key_filter(elem.first));
+      write_bytes(dest, value_filter(elem.second));
+    }
+    dest.end_object();
+  }
+} // wire_write
+
+namespace wire
+{
+  template<typename T, typename F = identity_>
+  inline void array(writer& dest, const T& source, F filter = F{})
+  {
+    wire_write::array(dest, source, source.size(), std::move(filter));
+  }
+  template<typename T, typename F>
+  inline void write_bytes(writer& dest, as_array_<T, F> source)
+  {
+    wire::array(dest, source.get_value(), std::move(source.filter));
+  }
+  template<typename T>
+  inline enable_if<is_array<T>::value> write_bytes(writer& dest, const T& source)
+  {
+    wire::array(dest, source);
+  }
+
+  template<typename T, typename F = identity_, typename G = identity_>
+  inline void dynamic_object(writer& dest, const T& source, F key_filter = F{}, G value_filter = G{})
+  {
+    wire_write::dynamic_object(dest, source, source.size(), std::move(key_filter), std::move(value_filter));
+  }
+  template<typename T, typename F, typename G>
+  inline void write_bytes(writer& dest, as_object_<T, F, G> source)
+  {
+    wire::dynamic_object(dest, source.get_map(), std::move(source.key_filter), std::move(source.value_filter));
+  }
+
+  template<typename... T>
+  inline void object(writer& dest, T... fields)
+  {
+    wire_write::object(dest, std::move(fields)...);
+  }
+}