Ecosystem/monero-lws
1
0
Fork 0
mirror of https://github.com/vtnerd/monero-lws.git synced 2024-12-22 19:39:23 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Add (working draft) subaddress support (#83)

Browse source
This commit is contained in:
Lee *!* Clagett 2023-12-05 20:23:50 -05:00 committed by Lee *!* Clagett
parent e09d3d57e9
commit b4426b4a74
21 changed files with 1539 additions and 88 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

21
src/db/account.cpp
View file

@ -72,11 +72,11 @@ namespace lws
crypto::secret_key view_key;
};
account::account(std::shared_ptr<const internal> immutable, db::block_id height, std::vector<db::output_id> spendable, std::vector<crypto::public_key> pubs) noexcept
account::account(std::shared_ptr<const internal> immutable, db::block_id height, std::vector<std::pair<db::output_id, db::address_index>> spendable, std::vector<crypto::public_key> pubs) noexcept
: immutable_(std::move(immutable))
, spendable_(std::move(spendable))
, pubs_(std::move(pubs))
, spends_()
, spends_()
, outputs_()
, height_(height)
{}
@ -87,7 +87,7 @@ namespace lws
MONERO_THROW(::common_error::kInvalidArgument, "using moved from account");
}
account::account(db::account const& source, std::vector<db::output_id> spendable, std::vector<crypto::public_key> pubs)
account::account(db::account const& source, std::vector<std::pair<db::output_id, db::address_index>> spendable, std::vector<crypto::public_key> pubs)
: account(std::make_shared<internal>(source), source.scan_height, std::move(spendable), std::move(pubs))
{
std::sort(spendable_.begin(), spendable_.end());
@ -151,9 +151,15 @@ namespace lws
return immutable_->view_key;
}
bool account::has_spendable(db::output_id const& id) const noexcept
boost::optional<db::address_index> account::get_spendable(db::output_id const& id) const noexcept
{
return std::binary_search(spendable_.begin(), spendable_.end(), id);
const auto searchable =
std::make_pair(id, db::address_index{db::major_index::primary, db::minor_index::primary});
const auto account =
std::lower_bound(spendable_.begin(), spendable_.end(), searchable);
if (account == spendable_.end() || account->first != id)
return boost::none;
return account->second;
}
bool account::add_out(db::output const& out)
@ -163,9 +169,10 @@ namespace lws
return false;
pubs_.insert(existing_pub, out.pub);
auto spendable_value = std::make_pair(out.spend_meta.id, out.recipient);
spendable_.insert(
std::lower_bound(spendable_.begin(), spendable_.end(), out.spend_meta.id),
out.spend_meta.id
std::lower_bound(spendable_.begin(), spendable_.end(), spendable_value),
spendable_value
);
outputs_.push_back(out);
return true;

12
src/db/account.h
View file

@ -26,6 +26,7 @@
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#pragma once
#include <boost/optional/optional.hpp>
#include <cstdint>
#include <memory>
#include <string>
@ -33,6 +34,7 @@
#include "crypto/crypto.h"
#include "fwd.h"
#include "db/data.h"
#include "db/fwd.h"
namespace lws
@ -43,19 +45,19 @@ namespace lws
struct internal;
std::shared_ptr<const internal> immutable_;
std::vector<db::output_id> spendable_;
std::vector<std::pair<db::output_id, db::address_index>> spendable_;
std::vector<crypto::public_key> pubs_;
std::vector<db::spend> spends_;
std::vector<db::output> outputs_;
db::block_id height_;
explicit account(std::shared_ptr<const internal> immutable, db::block_id height, std::vector<db::output_id> spendable, std::vector<crypto::public_key> pubs) noexcept;
explicit account(std::shared_ptr<const internal> immutable, db::block_id height, std::vector<std::pair<db::output_id, db::address_index>> spendable, std::vector<crypto::public_key> pubs) noexcept;
void null_check() const;
public:
//! Construct an account from `source` and current `spendable` outputs.
explicit account(db::account const& source, std::vector<db::output_id> spendable, std::vector<crypto::public_key> pubs);
explicit account(db::account const& source, std::vector<std::pair<db::output_id, db::address_index>> spendable, std::vector<crypto::public_key> pubs);
/*!
\return False if this is a "moved-from" account (i.e. the internal memory
@ -96,8 +98,8 @@ namespace lws
//! \return Current scan height of `this`.
db::block_id scan_height() const noexcept { return height_; }
//! \return True iff `id` is spendable by `this`.
bool has_spendable(db::output_id const& id) const noexcept;
//! \return Subaddress index iff `id` is spendable by `this`.
boost::optional<db::address_index> get_spendable(db::output_id const& id) const noexcept;
//! \return Outputs matched during the latest scan.
std::vector<db::output> const& outputs() const noexcept { return outputs_; }

108
src/db/data.cpp
View file

@ -29,12 +29,18 @@
#include <cstring>
#include <memory>
#include "cryptonote_config.h" // monero/src
#include "db/string.h"
#include "int-util.h" // monero/contribe/epee/include
#include "ringct/rctOps.h" // monero/src
#include "ringct/rctTypes.h" // monero/src
#include "wire.h"
#include "wire/adapted/array.h"
#include "wire/crypto.h"
#include "wire/json/write.h"
#include "wire/msgpack.h"
#include "wire/uuid.h"
#include "wire/vector.h"
#include "wire/wrapper/defaulted.h"
namespace lws
@ -69,6 +75,102 @@ namespace db
}
WIRE_DEFINE_OBJECT(account_address, map_account_address);
namespace
{
template<typename F, typename T>
void map_subaddress_dict(F& format, T& self)
{
wire::object(format,
wire::field<0>("key", std::ref(self.first)),
wire::field<1>("value", std::ref(self.second))
);
}
}
bool check_subaddress_dict(const subaddress_dict& self)
{
bool is_first = true;
minor_index last = minor_index::primary;
for (const auto& elem : self.second)
{
if (elem[1] < elem[0])
{
MERROR("Invalid subaddress_range (last before first");
return false;
}
if (std::uint32_t(elem[0]) <= std::uint64_t(last) + 1 && !is_first)
{
MERROR("Invalid subaddress_range (overlapping with previous)");
return false;
}
is_first = false;
last = elem[1];
}
return true;
}
void read_bytes(wire::reader& source, subaddress_dict& dest)
{
map_subaddress_dict(source, dest);
if (!check_subaddress_dict(dest))
WIRE_DLOG_THROW_(wire::error::schema::array);
}
void write_bytes(wire::writer& dest, const subaddress_dict& source)
{
if (!check_subaddress_dict(source))
WIRE_DLOG_THROW_(wire::error::schema::array);
map_subaddress_dict(dest, source);
}
namespace
{
template<typename F, typename T>
void map_address_index(F& format, T& self)
{
wire::object(format, WIRE_FIELD_ID(0, maj_i), WIRE_FIELD_ID(1, min_i));
}
crypto::secret_key get_subaddress_secret_key(const crypto::secret_key &a, const std::uint32_t major, const std::uint32_t minor)
{
char data[sizeof(config::HASH_KEY_SUBADDRESS) + sizeof(crypto::secret_key) + 2 * sizeof(uint32_t)];
memcpy(data, config::HASH_KEY_SUBADDRESS, sizeof(config::HASH_KEY_SUBADDRESS));
memcpy(data + sizeof(config::HASH_KEY_SUBADDRESS), &a, sizeof(crypto::secret_key));
std::uint32_t idx = SWAP32LE(major);
memcpy(data + sizeof(config::HASH_KEY_SUBADDRESS) + sizeof(crypto::secret_key), &idx, sizeof(uint32_t));
idx = SWAP32LE(minor);
memcpy(data + sizeof(config::HASH_KEY_SUBADDRESS) + sizeof(crypto::secret_key) + sizeof(uint32_t), &idx, sizeof(uint32_t));
crypto::secret_key m;
crypto::hash_to_scalar(data, sizeof(data), m);
return m;
}
}
WIRE_DEFINE_OBJECT(address_index, map_address_index);
crypto::public_key address_index::get_spend_public(account_address const& base, crypto::secret_key const& view) const
{
if (is_zero())
return base.spend_public;
// m = Hs(a || index_major || index_minor)
crypto::secret_key m = get_subaddress_secret_key(view, std::uint32_t(maj_i), std::uint32_t(min_i));
// M = m*G
crypto::public_key M;
crypto::secret_key_to_public_key(m, M);
// D = B + M
return rct::rct2pk(rct::addKeys(rct::pk2rct(base.spend_public), rct::pk2rct(M)));
}
namespace
{
template<typename F, typename T>
void map_subaddress_map(F& format, T& self)
{
wire::object(format, WIRE_FIELD_ID(0, subaddress), WIRE_FIELD_ID(1, index));
}
}
WIRE_DEFINE_OBJECT(subaddress_map, map_subaddress_map);
void write_bytes(wire::writer& dest, const account& self, const bool show_key)
{
view_key const* const key =
@ -144,7 +246,8 @@ namespace db
wire::field<10>("unlock_time", self.unlock_time),
wire::field<11>("mixin_count", self.spend_meta.mixin_count),
wire::field<12>("coinbase", coinbase),
wire::field<13>("fee", self.fee)
wire::field<13>("fee", self.fee),
wire::field<14>("recipient", self.recipient)
);
}
@ -161,7 +264,8 @@ namespace db
WIRE_FIELD(timestamp),
WIRE_FIELD(unlock_time),
WIRE_FIELD(mixin_count),
wire::optional_field("payment_id", std::ref(payment_id))
wire::optional_field("payment_id", std::ref(payment_id)),
WIRE_FIELD(sender)
);
}
}

64
src/db/data.h
View file

@ -26,13 +26,16 @@
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#pragma once
#include <array>
#include <boost/uuid/uuid.hpp>
#include <cassert>
#include <cstdint>
#include <iosfwd>
#include <limits>
#include <string>
#include <type_traits>
#include <utility>
#include <vector>
#include "crypto/crypto.h"
#include "lmdb/util.h"
@ -122,6 +125,49 @@ namespace db
static_assert(sizeof(account_address) == 64, "padding in account_address");
WIRE_DECLARE_OBJECT(account_address);
//! Major index of a subaddress
enum class major_index : std::uint32_t { primary = 0 };
WIRE_AS_INTEGER(major_index);
//! Minor index of a subaddress
enum class minor_index : std::uint32_t { primary = 0 };
WIRE_AS_INTEGER(minor_index);
//! Range within a major index
using index_range = std::array<minor_index, 2>;
//! Ranges within a major index
using index_ranges = std::vector<index_range>;
//! Compatible with msgpack_table
using subaddress_dict = std::pair<major_index, index_ranges>;
bool check_subaddress_dict(const subaddress_dict&);
WIRE_DECLARE_OBJECT(subaddress_dict);
//! A specific (sub)address index
struct address_index
{
major_index maj_i;
minor_index min_i;
crypto::public_key get_spend_public(account_address const& base, crypto::secret_key const& view) const;
constexpr bool is_zero() const noexcept
{
return maj_i == major_index::primary && min_i == minor_index::primary;
}
};
static_assert(sizeof(address_index) == 4 * 2, "padding in address_index");
WIRE_DECLARE_OBJECT(address_index);
//! Maps a subaddress pubkey to its index values
struct subaddress_map
{
crypto::public_key subaddress; //!< Must be first for LMDB optimzations
address_index index;
};
static_assert(sizeof(subaddress_map) == 32 + 4 * 2, "padding in subaddress_map");
WIRE_DECLARE_OBJECT(subaddress_map);
struct account
{
account_id id; //!< Must be first for LMDB optimizations
@ -205,9 +251,10 @@ namespace db
crypto::hash long_; //!< Long version of payment id (always decrypted)
} payment_id;
std::uint64_t fee; //!< Total fee for transaction
address_index recipient;
};
static_assert(
sizeof(output) == 8 + 32 + (8 * 3) + (4 * 2) + 32 + (8 * 2) + (32 * 3) + 7 + 1 + 32 + 8,
sizeof(output) == 8 + 32 + (8 * 3) + (4 * 2) + 32 + (8 * 2) + (32 * 3) + 7 + 1 + 32 + 8 + 2 * 4,
"padding in output"
);
void write_bytes(wire::writer&, const output&);
@ -225,8 +272,9 @@ namespace db
char reserved[3];
std::uint8_t length; //!< Length of `payment_id` field (0..32).
crypto::hash payment_id; //!< Unencrypted only, can't decrypt spend
address_index sender;
};
static_assert(sizeof(spend) == 8 + 32 * 2 + 8 * 4 + 4 + 3 + 1 + 32, "padding in spend");
static_assert(sizeof(spend) == 8 + 32 * 2 + 8 * 4 + 4 + 3 + 1 + 32 + 2 * 4, "padding in spend");
WIRE_DECLARE_OBJECT(spend);
//! Key image and info needed to retrieve primary `spend` data.
@ -325,6 +373,18 @@ namespace db
};
void write_bytes(wire::writer&, const webhook_new_account&);
inline constexpr bool operator==(address_index const& left, address_index const& right) noexcept
{
return left.maj_i == right.maj_i && left.min_i == right.min_i;
}
inline constexpr bool operator<(address_index const& left, address_index const& right) noexcept
{
return left.maj_i == right.maj_i ?
left.min_i < right.min_i : left.maj_i < right.maj_i;
}
bool operator==(transaction_link const& left, transaction_link const& right) noexcept;
bool operator<(transaction_link const& left, transaction_link const& right) noexcept;
bool operator<=(transaction_link const& left, transaction_link const& right) noexcept;

12
src/db/fwd.h
View file

@ -39,10 +39,14 @@ namespace db
enum class block_id : std::uint64_t;
enum extra : std::uint8_t;
enum class extra_and_length : std::uint8_t;
enum class major_index : std::uint32_t;
enum class minor_index : std::uint32_t;
enum class request : std::uint8_t;
enum class webhook_type : std::uint8_t;
struct account;
struct account_address;
struct address_index;
struct block_info;
struct key_image;
struct output;
@ -50,7 +54,15 @@ namespace db
struct request_info;
struct spend;
class storage;
struct subaddress_map;
struct transaction_link;
struct view_key;
struct webhook_data;
struct webhook_dupsort;
struct webhook_event;
struct webhook_key;
struct webhook_new_account;
struct webhook_output;
struct webhook_tx_confirmation;
} // db
} // lws

329
src/db/storage.cpp
View file

@ -57,6 +57,7 @@
#include "lmdb/value_stream.h"
#include "net/net_parse_helpers.h" // monero/contrib/epee/include
#include "span.h"
#include "wire/adapted/array.h"
#include "wire/filters.h"
#include "wire/json.h"
#include "wire/vector.h"
@ -102,8 +103,63 @@ namespace db
sizeof(output) == 8 + 32 + (8 * 3) + (4 * 2) + 32 + (8 * 2) + (32 * 3) + 7 + 1 + 32,
"padding in output"
);
//! Original db value, with no subaddress
struct spend
{
transaction_link link; //!< Orders and links `spend` to `output`.
crypto::key_image image; //!< Unique ID for the spend
// `link` and `image` must in this order for LMDB optimizations
output_id source; //!< The output being spent
std::uint64_t timestamp; //!< Timestamp of spend
std::uint64_t unlock_time;//!< Unlock time of spend
std::uint32_t mixin_count;//!< Ring-size of TX output
char reserved[3];
std::uint8_t length; //!< Length of `payment_id` field (0..32).
crypto::hash payment_id; //!< Unencrypted only, can't decrypt spend
};
static_assert(sizeof(spend) == 8 + 32 * 2 + 8 * 4 + 4 + 3 + 1 + 32, "padding in spend");
}
namespace v1
{
//! Second DB value, with no subaddress
struct output
{
transaction_link link; //! Orders and links `output` to `spend`s.
//! Data that a linked `spend` needs in some REST endpoints.
struct spend_meta_
{
output_id id; //!< Unique id for output within monero
// `link` and `id` must be in this order for LMDB optimizations
std::uint64_t amount;
std::uint32_t mixin_count;//!< Ring-size of TX
std::uint32_t index; //!< Offset within a tx
crypto::public_key tx_public;
} spend_meta;
std::uint64_t timestamp;
std::uint64_t unlock_time; //!< Not always a timestamp; mirrors chain value.
crypto::hash tx_prefix_hash;
crypto::public_key pub; //!< One-time spendable public key.
rct::key ringct_mask; //!< Unencrypted CT mask
char reserved[7];
extra_and_length extra; //!< Extra info + length of payment id
union payment_id_
{
crypto::hash8 short_; //!< Decrypted short payment id
crypto::hash long_; //!< Long version of payment id (always decrypted)
} payment_id;
std::uint64_t fee; //!< Total fee for transaction
};
static_assert(
sizeof(output) == 8 + 32 + (8 * 3) + (4 * 2) + 32 + (8 * 2) + (32 * 3) + 7 + 1 + 32 + 8,
"padding in output"
);
}
namespace
{
//! Used for finding `account` instances by other indexes.
@ -243,11 +299,17 @@ namespace db
constexpr const lmdb::basic_table<account_id, v0::output> outputs_v0{
"outputs_by_account_id,block_id,tx_hash,output_id", MDB_DUPSORT, &output_compare
};
constexpr const lmdb::basic_table<account_id, v1::output> outputs_v1{
"outputs_v1_by_account_id,block_id,tx_hash,output_id", MDB_DUPSORT, &output_compare
};
constexpr const lmdb::basic_table<account_id, output> outputs{
"outputs_v1_by_account_id,block_id,tx_hash,output_id", (MDB_CREATE | MDB_DUPSORT), &output_compare
"outputs_v2_by_account_id,block_id,tx_hash,output_id", (MDB_CREATE | MDB_DUPSORT), &output_compare
};
constexpr const lmdb::basic_table<account_id, v0::spend> spends_v0{
"spends_by_account_id,block_id,tx_hash,image", MDB_DUPSORT, &spend_compare
};
constexpr const lmdb::basic_table<account_id, spend> spends{
"spends_by_account_id,block_id,tx_hash,image", (MDB_CREATE | MDB_DUPSORT), &spend_compare
"spends_v1_by_account_id,block_id,tx_hash,image", (MDB_CREATE | MDB_DUPSORT), &spend_compare
};
constexpr const lmdb::basic_table<output_id, db::key_image> images{
"key_images_by_output_id,image", (MDB_CREATE | MDB_DUPSORT), MONERO_COMPARE(db::key_image, value)
@ -261,6 +323,12 @@ namespace db
constexpr const lmdb::basic_table<account_id, webhook_event> events_by_account_id{
"webhook_events_by_account_id,type,block_id,tx_hash,output_id,payment_id,event_id", (MDB_CREATE | MDB_DUPSORT), &lmdb::less<webhook_event>
};
constexpr const lmdb::msgpack_table<account_id, major_index, index_ranges> subaddress_ranges{
"subaddress_ranges_by_account_id,major_index", (MDB_CREATE | MDB_DUPSORT), &lmdb::less<db::major_index>
};
constexpr const lmdb::basic_table<account_id, subaddress_map> subaddress_indexes{
"subaddress_indexes_by_account_id,public_key", (MDB_CREATE | MDB_DUPSORT), MONERO_COMPARE(subaddress_map, subaddress)
};
template<typename D>
expect<void> check_cursor(MDB_txn& txn, MDB_dbi tbl, std::unique_ptr<MDB_cursor, D>& cur) noexcept
@ -553,6 +621,8 @@ namespace db
MDB_dbi requests;
MDB_dbi webhooks;
MDB_dbi events;
MDB_dbi subaddress_ranges;
MDB_dbi subaddress_indexes;
} tables;
const unsigned create_queue_max;
@ -573,6 +643,8 @@ namespace db
tables.requests = requests.open(*txn).value();
tables.webhooks = webhooks.open(*txn).value();
tables.events = events_by_account_id.open(*txn).value();
tables.subaddress_ranges = subaddress_ranges.open(*txn).value();
tables.subaddress_indexes = subaddress_indexes.open(*txn).value();
const auto v0_outputs = outputs_v0.open(*txn);
if (v0_outputs)
@ -580,6 +652,18 @@ namespace db
else if (v0_outputs != lmdb::error(MDB_NOTFOUND))
MONERO_THROW(v0_outputs.error(), "Error opening old outputs table");
const auto v1_outputs = outputs_v1.open(*txn);
if (v1_outputs)
MONERO_UNWRAP(convert_table<v1::output, output>(*txn, *v1_outputs, tables.outputs));
else if (v1_outputs != lmdb::error(MDB_NOTFOUND))
MONERO_THROW(v1_outputs.error(), "Error opening old outputs table");
const auto v0_spends = spends_v0.open(*txn);
if (v0_spends)
MONERO_UNWRAP(convert_table<v0::spend, spend>(*txn, *v0_spends, tables.spends));
else if (v0_spends != lmdb::error(MDB_NOTFOUND))
MONERO_THROW(v0_spends.error(), "Error opening old spends table");
check_blockchain(*txn, tables.blocks);
MONERO_UNWRAP(this->commit(std::move(txn)));
@ -755,6 +839,52 @@ namespace db
return requests.get_value<request_info>(value);
}
expect<std::vector<subaddress_dict>>
storage_reader::get_subaddresses(account_id id, cursor::subaddress_ranges cur) noexcept
{
MONERO_PRECOND(txn != nullptr);
assert(db != nullptr);
MONERO_CHECK(check_cursor(*txn, db->tables.subaddress_ranges, cur));
MDB_val key = lmdb::to_val(id);
MDB_val value{};
std::vector<subaddress_dict> ranges{};
int err = mdb_cursor_get(cur.get(), &key, &value, MDB_SET_KEY);
if (!err)
{
std::size_t count = 0;
if (mdb_cursor_count(cur.get(), &count) == 0)
ranges.reserve(count);
}
for (;;)
{
if (err)
{
if (err == MDB_NOTFOUND)
break;
return {lmdb::error(err)};
}
ranges.push_back(MONERO_UNWRAP(subaddress_ranges.get_value(value)));
err = mdb_cursor_get(cur.get(), &key, &value, MDB_NEXT_DUP);
}
return {std::move(ranges)};
}
expect<address_index>
storage_reader::find_subaddress(account_id id, crypto::public_key const& address, cursor::subaddress_indexes& cur) noexcept
{
MONERO_PRECOND(txn != nullptr);
assert(db != nullptr);
MONERO_CHECK(check_cursor(*txn, db->tables.subaddress_indexes, cur));
MDB_val key = lmdb::to_val(id);
MDB_val value = lmdb::to_val(address);
MONERO_LMDB_CHECK(mdb_cursor_get(cur.get(), &key, &value, MDB_GET_BOTH));
return subaddress_indexes.get_value<MONERO_FIELD(subaddress_map, index)>(value);
}
expect<std::vector<webhook_value>>
storage_reader::find_webhook(webhook_key const& key, crypto::hash8 const& payment_id, cursor::webhooks cur)
{
@ -889,6 +1019,14 @@ namespace db
);
}
static void write_bytes(wire::json_writer& dest, const std::pair<lws::db::account_id, std::vector<std::pair<lws::db::major_index, std::vector<std::array<lws::db::minor_index, 2>>>>>& self)
{
wire::object(dest,
wire::field("id", std::cref(self.first)),
wire::field("subaddress_indexes", std::cref(self.second))
);
}
expect<void> storage_reader::json_debug(std::ostream& out, bool show_keys)
{
using boost::adaptors::reverse;
@ -909,6 +1047,8 @@ namespace db
cursor::requests requests_cur;
cursor::webhooks webhooks_cur;
cursor::webhooks events_cur;
cursor::subaddress_ranges ranges_cur;
cursor::subaddress_indexes indexes_cur;
MONERO_CHECK(check_cursor(*txn, db->tables.blocks, curs.blocks_cur));
MONERO_CHECK(check_cursor(*txn, db->tables.accounts, accounts_cur));
@ -920,6 +1060,8 @@ namespace db
MONERO_CHECK(check_cursor(*txn, db->tables.requests, requests_cur));
MONERO_CHECK(check_cursor(*txn, db->tables.webhooks, webhooks_cur));
MONERO_CHECK(check_cursor(*txn, db->tables.events, events_cur));
MONERO_CHECK(check_cursor(*txn, db->tables.subaddress_ranges, ranges_cur));
MONERO_CHECK(check_cursor(*txn, db->tables.subaddress_indexes, indexes_cur));
auto blocks_partial =
get_blocks<boost::container::static_vector<block_info, 12>>(*curs.blocks_cur, 0);
@ -958,6 +1100,14 @@ namespace db
if (!requests_stream)
return requests_stream.error();
const auto ranges_data = subaddress_ranges.get_all(*ranges_cur);
if (!ranges_data)
return ranges_data.error();
auto indexes_stream = subaddress_indexes.get_key_stream(std::move(indexes_cur));
if (!indexes_stream)
return indexes_stream.error();
// This list should be smaller ... ?
const auto webhooks_data = webhooks.get_all(*webhooks_cur);
if (!webhooks_data)
@ -978,6 +1128,8 @@ namespace db
wire::field(spends.name, wire::as_object(spends_stream->make_range(), wire::as_integer, wire::as_array)),
wire::field(images.name, wire::as_object(images_stream->make_range(), output_id_key{}, wire::as_array)),
wire::field(requests.name, wire::as_object(requests_stream->make_range(), wire::enum_as_string, toggle_keys_filter)),
wire::field(subaddress_ranges.name, std::cref(*ranges_data)),
wire::field(subaddress_indexes.name, wire::as_object(indexes_stream->make_range(), wire::as_integer, wire::as_array)),
wire::field(webhooks.name, std::cref(*webhooks_data)),
wire::field(events_by_account_id.name, wire::as_object(events_stream->make_range(), wire::as_integer, wire::as_array))
);
@ -2215,6 +2367,173 @@ namespace db
});
}
expect<std::vector<subaddress_dict>>
storage::upsert_subaddresses(const account_id id, const account_address& address, const crypto::secret_key& view_key, std::vector<subaddress_dict> subaddrs, const std::uint32_t max_subaddr)
{
MONERO_PRECOND(db != nullptr);
std::sort(subaddrs.begin(), subaddrs.end());
return db->try_write([this, id, &address, &view_key, &subaddrs, max_subaddr] (MDB_txn& txn) -> expect<std::vector<subaddress_dict>>
{
std::size_t subaddr_count = 0;
std::vector<subaddress_dict> out{};
index_ranges new_dict{};
const auto add_out = [&out] (major_index major, index_range minor)
{
if (out.empty() || out.back().first != major)
out.emplace_back(major, index_ranges{minor});
else
out.back().second.push_back(minor);
};
const auto check_max_range = [&subaddr_count, max_subaddr] (const index_range& range) -> bool
{
const auto more = std::uint32_t(range[1]) - std::uint32_t(range[0]);
if (max_subaddr - subaddr_count <= more)
return false;
subaddr_count += more + 1;
return true;
};
const auto check_max_ranges = [&check_max_range] (const index_ranges& ranges) -> bool
{
for (const auto& range : ranges)
{
if (!check_max_range(range))
return false;
}
return true;
};
cursor::subaddress_ranges ranges_cur;
cursor::subaddress_indexes indexes_cur;
MONERO_CHECK(check_cursor(txn, this->db->tables.subaddress_ranges, ranges_cur));
MONERO_CHECK(check_cursor(txn, this->db->tables.subaddress_indexes, indexes_cur));
MDB_val key = lmdb::to_val(id);
MDB_val value{};
int err = mdb_cursor_get(indexes_cur.get(), &key, &value, MDB_SET);
if (err)
{
if (err != MDB_NOTFOUND)
return {lmdb::error(err)};
}
else
{
MONERO_LMDB_CHECK(mdb_cursor_count(indexes_cur.get(), &subaddr_count));
if (max_subaddr < subaddr_count)
return {error::max_subaddresses};
}
for (auto& major_entry : subaddrs)
{
new_dict.clear();
if (!check_subaddress_dict(major_entry))
{
MERROR("Invalid subaddress_dict given to storage::upsert_subaddrs");
return {wire::error::schema::array};
}
value = lmdb::to_val(major_entry.first);
err = mdb_cursor_get(ranges_cur.get(), &key, &value, MDB_GET_BOTH);
if (err)
{
if (err != MDB_NOTFOUND)
return {lmdb::error(err)};
if (!check_max_ranges(major_entry.second))
return {error::max_subaddresses};
out.push_back(major_entry);
new_dict = std::move(major_entry.second);
}
else // merge new minor index ranges with old
{
auto old_dict = subaddress_ranges.get_value(value);
if (!old_dict)
return old_dict.error();
auto& old_range = old_dict->second;
const auto& new_range = major_entry.second;
auto old_loc = old_range.begin();
auto new_loc = new_range.begin();
for ( ; old_loc != old_range.end() && new_loc != new_range.end(); )
{
if (std::uint64_t(new_loc->at(1)) + 1 < std::uint32_t(old_loc->at(0)))
{ // new has no overlap with existing
if (!check_max_range(*new_loc))
return {error::max_subaddresses};
new_dict.push_back(*new_loc);
add_out(major_entry.first, *new_loc);
++new_loc;
}
else if (std::uint64_t(old_loc->at(1)) + 1 < std::uint32_t(new_loc->at(0)))
{ // existing has no overlap with new
new_dict.push_back(*old_loc);
++old_loc;
}
else if (old_loc->at(0) <= new_loc->at(0) && new_loc->at(1) <= old_loc->at(1))
{ // new is completely within existing
++new_loc;
}
else // new overlap at beginning, end, or both
{
if (new_loc->at(0) < old_loc->at(0))
{ // overlap at beginning
const index_range new_range{new_loc->at(0), minor_index(std::uint32_t(old_loc->at(0)) - 1)};
if (!check_max_range(new_range))
return {error::max_subaddresses};
add_out(major_entry.first, new_range);
old_loc->at(0) = new_loc->at(0);
}
if (old_loc->at(1) < new_loc->at(1))
{ // overlap at end
const index_range new_range{minor_index(std::uint32_t(old_loc->at(1)) + 1), new_loc->at(1)};
if (!check_max_range(new_range))
return {error::max_subaddresses};
add_out(major_entry.first, new_range);
old_loc->at(1) = new_loc->at(1);
}
++new_loc;
}
}
std::copy(old_loc, old_range.end(), std::back_inserter(new_dict));
for ( ; new_loc != new_range.end(); ++new_loc)
{
if (!check_max_range(*new_loc))
return {error::max_subaddresses};
new_dict.push_back(*new_loc);
add_out(major_entry.first, *new_loc);
}
}
for (const auto& new_indexes : new_dict)
{
for (std::uint64_t minor : boost::counting_range(std::uint64_t(new_indexes[0]), std::uint64_t(new_indexes[1]) + 1))
{
subaddress_map new_value{};
new_value.index = address_index{major_entry.first, minor_index(minor)};
new_value.subaddress = new_value.index.get_spend_public(address, view_key);
value = lmdb::to_val(new_value);
MONERO_LMDB_CHECK(mdb_cursor_put(indexes_cur.get(), &key, &value, 0));
}
}
const expect<epee::byte_slice> value_bytes =
subaddress_ranges.make_value(major_entry.first, new_dict);
if (!value_bytes)
return value_bytes.error();
value = MDB_val{value_bytes->size(), const_cast<void*>(static_cast<const void*>(value_bytes->data()))};
MONERO_LMDB_CHECK(mdb_cursor_put(ranges_cur.get(), &key, &value, 0));
}
return {std::move(out)};
});
}
expect<void> storage::add_webhook(const webhook_type type, const boost::optional<account_address>& address, const webhook_value& event)
{
if (event.second.url != "zmq")
@ -2253,8 +2572,10 @@ namespace db
return {error::bad_webhook};
lmkey = lmdb::to_val(key);
const epee::byte_slice value = webhooks.make_value(event.first, event.second);
lmvalue = MDB_val{value.size(), const_cast<void*>(static_cast<const void*>(value.data()))};
const expect<epee::byte_slice> value = webhooks.make_value(event.first, event.second);
if (!value)
return value.error();
lmvalue = MDB_val{value->size(), const_cast<void*>(static_cast<const void*>(value->data()))};
MONERO_LMDB_CHECK(mdb_cursor_put(webhooks_cur.get(), &lmkey, &lmvalue, 0));
return success();
});

27
src/db/storage.h
View file

@ -52,6 +52,8 @@ namespace db
MONERO_CURSOR(spends);
MONERO_CURSOR(images);
MONERO_CURSOR(requests);
MONERO_CURSOR(subaddress_ranges);
MONERO_CURSOR(subaddress_indexes);
MONERO_CURSOR(blocks);
MONERO_CURSOR(accounts_by_address);
@ -133,6 +135,13 @@ namespace db
expect<request_info>
get_request(request type, account_address const& address, cursor::requests cur = nullptr) noexcept;
//! \return All subaddresses activated for account `id`.
expect<std::vector<subaddress_dict>> get_subaddresses(account_id id, cursor::subaddress_ranges cur = nullptr) noexcept;
//! \return A specific subaddress index
expect<address_index>
find_subaddress(account_id id, crypto::public_key const& spend_public, cursor::subaddress_indexes& cur) noexcept;
//! \return All webhook values associated with user `key` and `payment_id`.
expect<std::vector<webhook_value>>
find_webhook(webhook_key const& key, crypto::hash8 const& payment_id, cursor::webhooks cur = nullptr);
@ -243,6 +252,24 @@ namespace db
expect<std::pair<std::size_t, std::vector<webhook_tx_confirmation>>>
update(block_id height, epee::span<const crypto::hash> chain, epee::span<const lws::account> accts);
/*!
Adds subaddresses to an account. Upon success, an account will
immediately begin tracking them in the scanner.
\param id of the account to associate new indexes
\param addresss of the account (needed to generate subaddress publc key)
\param view_key of the account (needed to generate subaddress public key)
\param subaddrs Range of subaddress indexes that need to be added to the
database. Indexes _may_ overlap with existing indexes.
\param max_subaddresses The maximum number of subaddresses allowed per
account.
\return The new ranges of subaddress indexes added to the database
(whereas `subaddrs` may overlap with existing indexes).
*/
expect<std::vector<subaddress_dict>>
upsert_subaddresses(account_id id, const account_address& address, const crypto::secret_key& view_key, std::vector<subaddress_dict> subaddrs, std::uint32_t max_subaddresses);
/*!
Add webhook to be tracked in the database. The webhook will "call"
the specified URL with JSON/msgpack information when the event occurs.

4
src/error.cpp
View file

@ -85,10 +85,14 @@ namespace lws
return "Unspecified error when retrieving exchange rates";
case error::http_server:
return "HTTP server failed";
case error::invalid_range:
return "Invalid subaddress range provided";
case error::json_rpc:
return "Error returned by JSON-RPC server";
case error::exchange_rates_old:
return "Exchange rates are older than cache interval";
case error::max_subaddresses:
return "Max subaddresses exceeded";
case error::not_enough_mixin:
return "Not enough outputs to meet requested mixin count";
case error::signal_abort_process:

2
src/error.h
View file

@ -57,7 +57,9 @@ namespace lws
exchange_rates_fetch, //!< Exchange rates fetching failed
exchange_rates_old, //!< Exchange rates are older than cache interval
http_server, //!< HTTP server failure (init or run)
invalid_range, //!< Invalid subaddress range provided
json_rpc, //!< Error returned by JSON-RPC server
max_subaddresses, //!< Max subaddresses exceeded
not_enough_mixin, //!< Not enough outputs to meet mixin count
signal_abort_process, //!< In process ZMQ PUB to abort the process was received
signal_abort_scan, //!< In process ZMQ PUB to abort the scan was received

16
src/lmdb/msgpack_table.h
View file

@ -33,13 +33,20 @@ namespace lmdb
return out;
}
static epee::byte_slice make_value(const fixed_value_type& val1, const msgpack_value_type& val2)
static expect<epee::byte_slice> make_value(const fixed_value_type& val1, const msgpack_value_type& val2)
{
epee::byte_stream initial;
initial.write({reinterpret_cast<const char*>(std::addressof(val1)), sizeof(val1)});
wire::msgpack_slice_writer dest{std::move(initial), true};
wire_write::bytes(dest, val2);
try
{
wire_write::bytes(dest, val2);
}
catch (const wire::exception& e)
{
return {e.code()};
}
return epee::byte_slice{dest.take_sink()};
}
@ -80,12 +87,9 @@ namespace lmdb
auto msgpack_bytes = lmdb::to_byte_span(value);
msgpack_bytes.remove_prefix(sizeof(out.first));
msgpack_value_type second{};
const std::error_code error = wire::msgpack::from_bytes(epee::byte_slice{{msgpack_bytes}}, second);
const std::error_code error = wire::msgpack::from_bytes(epee::byte_slice{{msgpack_bytes}}, out.second);
if (error)
return error;
out.second = std::move(second);
return out;
}

149
src/rest_server.cpp
View file

@ -27,6 +27,7 @@
#include "rest_server.h"
#include <algorithm>
#include <boost/range/counting_range.hpp>
#include <boost/utility/string_ref.hpp>
#include <cstring>
#include <limits>
@ -140,6 +141,7 @@ namespace lws
struct runtime_options
{
std::uint32_t max_subaddresses;
epee::net_utils::ssl_verification_t webhook_verify;
bool disable_admin_auth;
};
@ -482,6 +484,23 @@ namespace lws
}
};
struct get_subaddrs
{
using request = rpc::account_credentials;
using response = rpc::get_subaddrs_response;
static expect<response> handle(request const& req, db::storage disk, rpc::client const&, runtime_options const& options)
{
auto user = open_account(req, std::move(disk));
if (!user)
return user.error();
auto subaddrs = user->second.get_subaddresses(user->first.id);
if (!subaddrs)
return subaddrs.error();
return response{std::move(*subaddrs)};
}
};
struct get_unspent_outs
{
using request = rpc::get_unspent_outs_request;
@ -642,6 +661,68 @@ namespace lws
}
};
struct provision_subaddrs
{
using request = rpc::provision_subaddrs_request;
using response = rpc::new_subaddrs_response;
static expect<response> handle(request req, db::storage disk, rpc::client const&, runtime_options const& options)
{
if (!req.maj_i && !req.min_i && !req.n_min && !req.n_maj)
return {lws::error::invalid_range};
db::account_id id = db::account_id::invalid;
{
auto user = open_account(req.creds, disk.clone());
if (!user)
return user.error();
id = user->first.id;
}
const std::uint32_t major_i = req.maj_i.value_or(0);
const std::uint32_t minor_i = req.min_i.value_or(0);
const std::uint32_t n_major = req.n_maj.value_or(50);
const std::uint32_t n_minor = req.n_min.value_or(500);
const bool get_all = req.get_all.value_or(true);
if (std::numeric_limits<std::uint32_t>::max() / n_major < n_minor)
return {lws::error::max_subaddresses};
if (options.max_subaddresses < n_major * n_minor)
return {lws::error::max_subaddresses};
std::vector<db::subaddress_dict> new_ranges;
std::vector<db::subaddress_dict> all_ranges;
if (n_major && n_minor)
{
std::vector<db::subaddress_dict> ranges;
ranges.reserve(n_major);
for (std::uint64_t elem : boost::counting_range(std::uint64_t(major_i), std::uint64_t(major_i) + n_major))
{
ranges.emplace_back(
db::major_index(elem), db::index_ranges{db::index_range{db::minor_index(minor_i), db::minor_index(minor_i + n_minor - 1)}}
);
}
auto upserted = disk.upsert_subaddresses(id, req.creds.address, req.creds.key, ranges, options.max_subaddresses);
if (!upserted)
return upserted.error();
new_ranges = std::move(*upserted);
}
if (get_all)
{
// must start a new read after the last write
auto reader = disk.start_read();
if (!reader)
return reader.error();
auto rc = reader->get_subaddresses(id);
if (!rc)
return rc.error();
all_ranges = std::move(*rc);
}
return response{std::move(new_ranges), std::move(all_ranges)};
}
};
struct submit_raw_tx
{
using request = rpc::submit_raw_tx_request;
@ -672,6 +753,46 @@ namespace lws
}
};
struct upsert_subaddrs
{
using request = rpc::upsert_subaddrs_request;
using response = rpc::new_subaddrs_response;
static expect<response> handle(request req, db::storage disk, rpc::client const&, runtime_options const& options)
{
if (!options.max_subaddresses)
return {lws::error::max_subaddresses};
db::account_id id = db::account_id::invalid;
{
auto user = open_account(req.creds, disk.clone());
if (!user)
return user.error();
id = user->first.id;
}
const bool get_all = req.get_all.value_or(true);
std::vector<db::subaddress_dict> all_ranges;
auto new_ranges =
disk.upsert_subaddresses(id, req.creds.address, req.creds.key, req.subaddrs, options.max_subaddresses);
if (!new_ranges)
return new_ranges.error();
if (get_all)
{
auto reader = disk.start_read();
if (!reader)
return reader.error();
auto rc = reader->get_subaddresses(id);
if (!rc)
return rc.error();
all_ranges = std::move(*rc);
}
return response{std::move(*new_ranges), std::move(all_ranges)};
}
};
template<typename E>
expect<epee::byte_slice> call(std::string&& root, db::storage disk, const rpc::client& gclient, const runtime_options& options)
{
@ -758,14 +879,17 @@ namespace lws
constexpr const endpoint endpoints[] =
{
{"/get_address_info", call<get_address_info>, 2 * 1024},
{"/get_address_txs", call<get_address_txs>, 2 * 1024},
{"/get_random_outs", call<get_random_outs>, 2 * 1024},
{"/get_txt_records", nullptr, 0 },
{"/get_unspent_outs", call<get_unspent_outs>, 2 * 1024},
{"/import_wallet_request", call<import_request>, 2 * 1024},
{"/login", call<login>, 2 * 1024},
{"/submit_raw_tx", call<submit_raw_tx>, 50 * 1024}
{"/get_address_info", call<get_address_info>, 2 * 1024},
{"/get_address_txs", call<get_address_txs>, 2 * 1024},
{"/get_random_outs", call<get_random_outs>, 2 * 1024},
{"/get_subaddrs", call<get_subaddrs>, 2 * 1024},
{"/get_txt_records", nullptr, 0 },
{"/get_unspent_outs", call<get_unspent_outs>, 2 * 1024},
{"/import_wallet_request", call<import_request>, 2 * 1024},
{"/login", call<login>, 2 * 1024},
{"/provision_subaddrs", call<provision_subaddrs>, 2 * 1024},
{"/submit_raw_tx", call<submit_raw_tx>, 50 * 1024},
{"/upsert_subaddrs", call<upsert_subaddrs>, 10 * 1024}
};
constexpr const endpoint admin_endpoints[] =
@ -893,7 +1017,7 @@ namespace lws
{
MINFO(body.error().message() << " from " << ctx.m_remote_address.str() << " on " << handler->name);
if (body.error().category() == wire::error::rapidjson_category())
if (body.error().category() == wire::error::rapidjson_category() || body == lws::error::invalid_range)
{
response.m_response_code = 400;
response.m_response_comment = "Bad Request";
@ -903,6 +1027,11 @@ namespace lws
response.m_response_code = 403;
response.m_response_comment = "Forbidden";
}
else if (body == lws::error::max_subaddresses)
{
response.m_response_code = 409;
response.m_response_comment = "Conflict";
}
else if (body.matches(std::errc::timed_out) || body.matches(std::errc::no_lock_available))
{
response.m_response_code = 503;
@ -1017,7 +1146,7 @@ namespace lws
};
bool any_ssl = false;
const runtime_options options{config.webhook_verify, config.disable_admin_auth};
const runtime_options options{config.max_subaddresses, config.webhook_verify, config.disable_admin_auth};
for (const std::string& address : addresses)
{
ports_.emplace_back(io_service_, disk.clone(), MONERO_UNWRAP(client.clone()), options);

1
src/rest_server.h
View file

@ -53,6 +53,7 @@ namespace lws
epee::net_utils::ssl_authentication_t auth;
std::vector<std::string> access_controls;
std::size_t threads;
std::uint32_t max_subaddresses;
epee::net_utils::ssl_verification_t webhook_verify;
bool allow_external;
bool disable_admin_auth;

44
src/rpc/light_wallet.cpp
View file

@ -134,7 +134,8 @@ namespace
wire::field("timestamp", iso_timestamp(self.data.first.timestamp)),
wire::field("height", self.data.first.link.height),
wire::field("spend_key_images", std::cref(self.data.second)),
wire::optional_field("rct", optional_rct)
wire::optional_field("rct", optional_rct),
wire::field("recipient", std::cref(self.data.first.recipient))
);
}
@ -192,6 +193,12 @@ namespace lws
convert_address(address, self.address);
}
void rpc::write_bytes(wire::json_writer& dest, const new_subaddrs_response& self)
{
wire::object(dest, WIRE_FIELD(new_subaddrs), WIRE_FIELD(all_subaddrs));
}
void rpc::write_bytes(wire::json_writer& dest, const transaction_spend& self)
{
wire::object(dest,
@ -199,7 +206,8 @@ namespace lws
wire::field("key_image", std::cref(self.possible_spend.image)),
wire::field("tx_pub_key", std::cref(self.meta.tx_public)),
wire::field("out_index", self.meta.index),
wire::field("mixin", self.possible_spend.mixin_count)
wire::field("mixin", self.possible_spend.mixin_count),
wire::field("sender", std::cref(self.possible_spend.sender))
);
}
@ -278,6 +286,11 @@ namespace lws
wire::object(dest, WIRE_FIELD(amount_outs));
}
void rpc::write_bytes(wire::json_writer& dest, const get_subaddrs_response& self)
{
wire::object(dest, WIRE_FIELD(all_subaddrs));
}
void rpc::read_bytes(wire::json_reader& source, get_unspent_outs_request& self)
{
std::string address;
@ -331,6 +344,21 @@ namespace lws
wire::object(dest, WIRE_FIELD_COPY(new_address), WIRE_FIELD_COPY(generated_locally));
}
void rpc::read_bytes(wire::json_reader& source, provision_subaddrs_request& self)
{
std::string address;
wire::object(source,
wire::field("address", std::ref(address)),
wire::field("view_key", std::ref(unwrap(unwrap(self.creds.key)))),
WIRE_OPTIONAL_FIELD(maj_i),
WIRE_OPTIONAL_FIELD(min_i),
WIRE_OPTIONAL_FIELD(n_maj),
WIRE_OPTIONAL_FIELD(n_min),
WIRE_OPTIONAL_FIELD(get_all)
);
convert_address(address, self.creds.address);
}
void rpc::read_bytes(wire::json_reader& source, submit_raw_tx_request& self)
{
wire::object(source, WIRE_FIELD(tx));
@ -339,4 +367,16 @@ namespace lws
{
wire::object(dest, WIRE_FIELD_COPY(status));
}
void rpc::read_bytes(wire::json_reader& source, upsert_subaddrs_request& self)
{
std::string address;
wire::object(source,
wire::field("address", std::ref(address)),
wire::field("view_key", std::ref(unwrap(unwrap(self.creds.key)))),
WIRE_FIELD(subaddrs),
WIRE_OPTIONAL_FIELD(get_all)
);
convert_address(address, self.creds.address);
}
} // lws

40
src/rpc/light_wallet.h
View file

@ -65,6 +65,15 @@ namespace rpc
void read_bytes(wire::json_reader&, account_credentials&);
struct new_subaddrs_response
{
new_subaddrs_response() = delete;
std::vector<db::subaddress_dict> new_subaddrs;
std::vector<db::subaddress_dict> all_subaddrs;
};
void write_bytes(wire::json_writer&, const new_subaddrs_response&);
struct transaction_spend
{
transaction_spend() = delete;
@ -164,6 +173,14 @@ namespace rpc
void write_bytes(wire::json_writer&, const get_unspent_outs_response&);
struct get_subaddrs_response
{
get_subaddrs_response() = delete;
std::vector<db::subaddress_dict> all_subaddrs;
};
void write_bytes(wire::json_writer&, const get_subaddrs_response&);
struct import_response
{
import_response() = delete;
@ -193,6 +210,19 @@ namespace rpc
void write_bytes(wire::json_writer&, login_response);
struct provision_subaddrs_request
{
provision_subaddrs_request() = delete;
account_credentials creds;
boost::optional<std::uint32_t> maj_i;
boost::optional<std::uint32_t> min_i;
boost::optional<std::uint32_t> n_maj;
boost::optional<std::uint32_t> n_min;
boost::optional<bool> get_all;
};
void read_bytes(wire::json_reader&, provision_subaddrs_request&);
struct submit_raw_tx_request
{
submit_raw_tx_request() = delete;
@ -206,5 +236,15 @@ namespace rpc
const char* status;
};
void write_bytes(wire::json_writer&, submit_raw_tx_response);
struct upsert_subaddrs_request
{
upsert_subaddrs_request() = delete;
account_credentials creds;
std::vector<db::subaddress_dict> subaddrs;
boost::optional<bool> get_all;
};
void read_bytes(wire::json_reader&, upsert_subaddrs_request&);
} // rpc
} // lws

191
src/scanner.cpp
View file

@ -94,16 +94,22 @@ namespace lws
std::atomic<bool> update;
};
struct options
{
net::ssl_verification_t webhook_verify;
bool enable_subaddresses;
};
struct thread_data
{
explicit thread_data(rpc::client client, db::storage disk, std::vector<lws::account> users, net::ssl_verification_t webhook_verify)
: client(std::move(client)), disk(std::move(disk)), users(std::move(users)), webhook_verify(webhook_verify)
explicit thread_data(rpc::client client, db::storage disk, std::vector<lws::account> users, options opts)
: client(std::move(client)), disk(std::move(disk)), users(std::move(users)), opts(opts)
{}
rpc::client client;
db::storage disk;
std::vector<lws::account> users;
net::ssl_verification_t webhook_verify;
options opts;
};
// until we have a signal-handler safe notification system
@ -263,6 +269,23 @@ namespace lws
}
};
struct subaddress_reader
{
expect<db::storage_reader> reader;
db::cursor::subaddress_indexes cur;
subaddress_reader(db::storage const& disk, const bool enable_subaddresses)
: reader(common_error::kInvalidArgument), cur(nullptr)
{
if (enable_subaddresses)
{
reader = disk.start_read();
if (!reader)
MERROR("Subadress lookup failure: " << reader.error().message());
}
}
};
void scan_transaction_base(
epee::span<lws::account> users,
const db::block_id height,
@ -270,6 +293,7 @@ namespace lws
crypto::hash const& tx_hash,
cryptonote::transaction const& tx,
std::vector<std::uint64_t> const& out_ids,
subaddress_reader& reader,
std::function<void(lws::account&, const db::spend&)> spend_action,
std::function<bool(lws::account&, const db::output&)> output_action)
{
@ -280,6 +304,8 @@ namespace lws
boost::optional<crypto::hash> prefix_hash;
boost::optional<cryptonote::tx_extra_nonce> extra_nonce;
std::pair<std::uint8_t, db::output::payment_id_> payment_id;
cryptonote::tx_extra_additional_pub_keys additional_tx_pub_keys;
std::vector<crypto::key_derivation> additional_derivations;
{
std::vector<cryptonote::tx_extra_field> extra;
@ -297,6 +323,10 @@ namespace lws
}
else
extra_nonce = boost::none;
// additional tx pub keys present when there are 3+ outputs in a tx involving subaddresses
if (reader.reader)
cryptonote::find_tx_extra_field_by_type(extra, additional_tx_pub_keys);
} // destruct `extra` vector
for (account& user : users)
@ -308,6 +338,21 @@ namespace lws
if (!crypto::wallet::generate_key_derivation(key.pub_key, user.view_key(), derived))
continue; // to next user
if (reader.reader && additional_tx_pub_keys.data.size() == tx.vout.size())
{
additional_derivations.resize(tx.vout.size());
std::size_t index = -1;
for (auto const& out: tx.vout)
{
++index;
if (!crypto::wallet::generate_key_derivation(additional_tx_pub_keys.data[index], user.view_key(), additional_derivations[index]))
{
additional_derivations.clear();
break; // vout loop
}
}
}
db::extra ext{};
std::uint32_t mixin = 0;
for (auto const& in : tx.vin)
@ -324,23 +369,26 @@ namespace lws
for (std::uint64_t offset : in_data->key_offsets)
{
goffset += offset;
if (user.has_spendable(db::output_id{in_data->amount, goffset}))
{
spend_action(
user,
db::spend{
db::transaction_link{height, tx_hash},
in_data->k_image,
db::output_id{in_data->amount, goffset},
timestamp,
tx.unlock_time,
mixin,
{0, 0, 0}, // reserved
payment_id.first,
payment_id.second.long_
}
);
}
const boost::optional<db::address_index> subaccount =
user.get_spendable(db::output_id{in_data->amount, goffset});
if (!subaccount)
continue; // to next input
spend_action(
user,
db::spend{
db::transaction_link{height, tx_hash},
in_data->k_image,
db::output_id{in_data->amount, goffset},
timestamp,
tx.unlock_time,
mixin,
{0, 0, 0}, // reserved
payment_id.first,
payment_id.second.long_,
*subaccount
}
);
}
}
else if (boost::get<cryptonote::txin_gen>(std::addressof(in)))
@ -358,15 +406,57 @@ namespace lws
boost::optional<crypto::view_tag> view_tag_opt =
cryptonote::get_output_view_tag(out);
if (!cryptonote::out_can_be_to_acc(view_tag_opt, derived, index))
const bool found_tag =
(!additional_derivations.empty() && cryptonote::out_can_be_to_acc(view_tag_opt, additional_derivations.at(index), index)) ||
cryptonote::out_can_be_to_acc(view_tag_opt, derived, index);
if (!found_tag)
continue; // to next output
crypto::public_key derived_pub;
const bool received =
crypto::wallet::derive_subaddress_public_key(out_pub_key, derived, index, derived_pub) &&
derived_pub == user.spend_public();
bool found_pub = false;
db::address_index account_index{db::major_index::primary, db::minor_index::primary};
crypto::key_derivation active_derived;
if (!received)
// inspect the additional and traditional keys
for (std::size_t attempt = 0; attempt < 2; ++attempt)
{
if (attempt == 0)
active_derived = derived;
else if (!additional_derivations.empty())
active_derived = additional_derivations.at(index);
else
break; // inspection loop
crypto::public_key derived_pub;
if (!crypto::wallet::derive_subaddress_public_key(out_pub_key, active_derived, index, derived_pub))
continue; // to next available active_derived
if (user.spend_public() != derived_pub)
{
if (!reader.reader)
continue; // to next available active_derived
const expect<db::address_index> match =
reader.reader->find_subaddress(user.id(), derived_pub, reader.cur);
if (!match)
{
if (match != lmdb::error(MDB_NOTFOUND))
MERROR("Failure when doing subaddress search: " << match.error().message());
continue; // to next available active_derived
}
found_pub = true;
account_index = *match;
break; // additional_derivations loop
}
else
{
found_pub = true;
break; // additional_derivations loop
}
}
if (!found_pub)
continue; // to next output
if (!prefix_hash)
@ -381,7 +471,7 @@ namespace lws
{
const bool bulletproof2 = (rct::RCTTypeBulletproof2 <= tx.rct_signatures.type);
const auto decrypted = lws::decode_amount(
tx.rct_signatures.outPk.at(index).mask, tx.rct_signatures.ecdhInfo.at(index), derived, index, bulletproof2
tx.rct_signatures.outPk.at(index).mask, tx.rct_signatures.ecdhInfo.at(index), active_derived, index, bulletproof2
);
if (!decrypted)
{
@ -398,7 +488,7 @@ namespace lws
if (!payment_id.first && cryptonote::get_encrypted_payment_id_from_tx_extra_nonce(extra_nonce->nonce, payment_id.second.short_))
{
payment_id.first = sizeof(crypto::hash8);
lws::decrypt_payment_id(payment_id.second.short_, derived);
lws::decrypt_payment_id(payment_id.second.short_, active_derived);
}
}
@ -421,7 +511,8 @@ namespace lws
{0, 0, 0, 0, 0, 0, 0}, // reserved bytes
db::pack(ext, payment_id.first),
payment_id.second,
tx.rct_signatures.txnFee
tx.rct_signatures.txnFee,
account_index
}
);
@ -437,12 +528,13 @@ namespace lws
const std::uint64_t timestamp,
crypto::hash const& tx_hash,
cryptonote::transaction const& tx,
std::vector<std::uint64_t> const& out_ids)
std::vector<std::uint64_t> const& out_ids,
subaddress_reader& reader)
{
scan_transaction_base(users, height, timestamp, tx_hash, tx, out_ids, add_spend{}, add_output{});
scan_transaction_base(users, height, timestamp, tx_hash, tx, out_ids, reader, add_spend{}, add_output{});
}
void scan_transactions(std::string&& txpool_msg, epee::span<lws::account> users, db::storage const& disk, rpc::client& client, const net::ssl_verification_t verify_mode)
void scan_transactions(std::string&& txpool_msg, epee::span<lws::account> users, db::storage const& disk, rpc::client& client, const options& opts)
{
// uint64::max is for txpool
static const std::vector<std::uint64_t> fake_outs(
@ -459,9 +551,10 @@ namespace lws
const auto time =
boost::numeric_cast<std::uint64_t>(std::chrono::system_clock::to_time_t(std::chrono::system_clock::now()));
send_webhook sender{disk, client, verify_mode};
subaddress_reader reader{disk, opts.enable_subaddresses};
send_webhook sender{disk, client, opts.webhook_verify};
for (const auto& tx : parsed->txes)
scan_transaction_base(users, db::block_id::txpool, time, crypto::hash{}, tx, fake_outs, null_spend{}, sender);
scan_transaction_base(users, db::block_id::txpool, time, crypto::hash{}, tx, fake_outs, reader, null_spend{}, sender);
}
void update_rates(rpc::context& ctx)
@ -481,7 +574,7 @@ namespace lws
rpc::client client{std::move(data->client)};
db::storage disk{std::move(data->disk)};
std::vector<lws::account> users{std::move(data->users)};
const net::ssl_verification_t webhook_verify = data->webhook_verify;
const options opts = std::move(data->opts);
assert(!users.empty());
assert(std::is_sorted(users.begin(), users.end(), by_height{}));
@ -568,7 +661,7 @@ namespace lws
{
if (message->first != rpc::client::topic::txpool)
break; // inner for loop
scan_transactions(std::move(message->second), epee::to_mut_span(users), disk, client, webhook_verify);
scan_transactions(std::move(message->second), epee::to_mut_span(users), disk, client, opts);
}
for ( ; message != new_pubs->end(); ++message)
@ -605,6 +698,7 @@ namespace lws
else
fetched->start_height = 0;
subaddress_reader reader{disk, opts.enable_subaddresses};
for (auto block_data : boost::combine(blocks, indices))
{
++(fetched->start_height);
@ -629,7 +723,8 @@ namespace lws
block.timestamp,
miner_tx_hash,
block.miner_tx,
*(indices.begin())
*(indices.begin()),
reader
);
indices.remove_prefix(1);
@ -644,13 +739,15 @@ namespace lws
block.timestamp,
boost::get<0>(tx_data),
boost::get<1>(tx_data),
boost::get<2>(tx_data)
boost::get<2>(tx_data),
reader
);
}
blockchain.push_back(cryptonote::get_block_hash(block));
} // for each block
reader.reader = std::error_code{common_error::kInvalidArgument}; // cleanup reader before next write
auto updated = disk.update(
users.front().scan_height(), epee::to_span(blockchain), epee::to_span(users)
);
@ -665,7 +762,7 @@ namespace lws
}
MINFO("Processed " << blocks.size() << " block(s) against " << users.size() << " account(s)");
send_payment_hook(client, epee::to_span(updated->second), webhook_verify);
send_payment_hook(client, epee::to_span(updated->second), opts.webhook_verify);
if (updated->first != users.size())
{
MWARNING("Only updated " << updated->first << " account(s) out of " << users.size() << ", resetting");
@ -692,7 +789,7 @@ namespace lws
Launches `thread_count` threads to run `scan_loop`, and then polls for
active account changes in background
*/
void check_loop(db::storage disk, rpc::context& ctx, std::size_t thread_count, std::vector<lws::account> users, std::vector<db::account_id> active, const net::ssl_verification_t webhook_verify)
void check_loop(db::storage disk, rpc::context& ctx, std::size_t thread_count, std::vector<lws::account> users, std::vector<db::account_id> active, const options opts)
{
assert(0 < thread_count);
assert(0 < users.size());
@ -754,7 +851,7 @@ namespace lws
client.watch_scan_signals();
auto data = std::make_shared<thread_data>(
std::move(client), disk.clone(), std::move(thread_users), webhook_verify
std::move(client), disk.clone(), std::move(thread_users), opts
);
threads.emplace_back(attrs, std::bind(&scan_loop, std::ref(self), std::move(data)));
}
@ -765,7 +862,7 @@ namespace lws
client.watch_scan_signals();
auto data = std::make_shared<thread_data>(
std::move(client), disk.clone(), std::move(users), webhook_verify
std::move(client), disk.clone(), std::move(users), opts
);
threads.emplace_back(attrs, std::bind(&scan_loop, std::ref(self), std::move(data)));
}
@ -908,7 +1005,7 @@ namespace lws
return {std::move(client)};
}
void scanner::run(db::storage disk, rpc::context ctx, std::size_t thread_count, const epee::net_utils::ssl_verification_t webhook_verify)
void scanner::run(db::storage disk, rpc::context ctx, std::size_t thread_count, const epee::net_utils::ssl_verification_t webhook_verify, const bool enable_subaddresses)
{
thread_count = std::max(std::size_t(1), thread_count);
@ -931,7 +1028,7 @@ namespace lws
for (db::account user : accounts.make_range())
{
std::vector<db::output_id> receives{};
std::vector<std::pair<db::output_id, db::address_index>> receives{};
std::vector<crypto::public_key> pubs{};
auto receive_list = MONERO_UNWRAP(reader.get_outputs(user.id));
@ -941,7 +1038,9 @@ namespace lws
for (auto output = receive_list.make_iterator(); !output.is_end(); ++output)
{
receives.emplace_back(output.get_value<MONERO_FIELD(db::output, spend_meta.id)>());
auto id = output.get_value<MONERO_FIELD(db::output, spend_meta.id)>();
auto subaddr = output.get_value<MONERO_FIELD(db::output, recipient)>();
receives.emplace_back(std::move(id), std::move(subaddr));
pubs.emplace_back(output.get_value<MONERO_FIELD(db::output, pub)>());
}
@ -960,7 +1059,7 @@ namespace lws
checked_wait(account_poll_interval - (std::chrono::steady_clock::now() - last));
}
else
check_loop(disk.clone(), ctx, thread_count, std::move(users), std::move(active), webhook_verify);
check_loop(disk.clone(), ctx, thread_count, std::move(users), std::move(active), options{webhook_verify, enable_subaddresses});
if (!scanner::is_running())
return;

2
src/scanner.h
View file

@ -49,7 +49,7 @@ namespace lws
static expect<rpc::client> sync(db::storage disk, rpc::client client);
//! Poll daemon until `stop()` is called, using `thread_count` threads.
static void run(db::storage disk, rpc::context ctx, std::size_t thread_count, epee::net_utils::ssl_verification_t webhook_verify);
static void run(db::storage disk, rpc::context ctx, std::size_t thread_count, epee::net_utils::ssl_verification_t webhook_verify, bool enable_subaddresses);
//! \return True if `stop()` has never been called.
static bool is_running() noexcept { return running; }

9
src/server_main.cpp
View file

@ -78,6 +78,7 @@ namespace
const command_line::arg_descriptor<bool> disable_admin_auth;
const command_line::arg_descriptor<std::string> webhook_ssl_verification;
const command_line::arg_descriptor<std::string> config_file;
const command_line::arg_descriptor<std::uint32_t> max_subaddresses;
static std::string get_default_zmq()
{
@ -120,6 +121,7 @@ namespace
, disable_admin_auth{"disable-admin-auth", "Make auth field optional in HTTP-REST requests", false}
, webhook_ssl_verification{"webhook-ssl-verification", "[<none|system_ca>] specify SSL verification mode for webhooks", "system_ca"}
, config_file{"config-file", "Specify any option in a config file; <name>=<value> on separate lines"}
, max_subaddresses{"max-subaddresses", "Maximum number of subaddresses per primary account (defaults to 0)", 0}
{}
void prepare(boost::program_options::options_description& description) const
@ -150,6 +152,7 @@ namespace
command_line::add_arg(description, disable_admin_auth);
command_line::add_arg(description, webhook_ssl_verification);
command_line::add_arg(description, config_file);
command_line::add_arg(description, max_subaddresses);
}
};
@ -230,6 +233,7 @@ namespace
{command_line::get_arg(args, opts.rest_ssl_key), command_line::get_arg(args, opts.rest_ssl_cert)},
command_line::get_arg(args, opts.access_controls),
command_line::get_arg(args, opts.rest_threads),
command_line::get_arg(args, opts.max_subaddresses),
webhook_verify,
command_line::get_arg(args, opts.external_bind),
command_line::get_arg(args, opts.disable_admin_auth)
@ -250,7 +254,7 @@ namespace
command_line::get_arg(args, opts.webhook_ssl_verification),
std::chrono::minutes{command_line::get_arg(args, opts.rates_interval)},
command_line::get_arg(args, opts.scan_threads),
command_line::get_arg(args, opts.create_queue_max),
command_line::get_arg(args, opts.create_queue_max)
};
prog.rest_config.threads = std::max(std::size_t(1), prog.rest_config.threads);
@ -273,6 +277,7 @@ namespace
MINFO("Using monerod ZMQ RPC at " << ctx.daemon_address());
auto client = lws::scanner::sync(disk.clone(), ctx.connect().value()).value();
const auto enable_subaddresses = bool(prog.rest_config.max_subaddresses);
const auto webhook_verify = prog.rest_config.webhook_verify;
lws::rest_server server{
epee::to_span(prog.rest_servers), prog.admin_rest_servers, disk.clone(), std::move(client), std::move(prog.rest_config)
@ -283,7 +288,7 @@ namespace
MINFO("Listening for REST admin clients at " << address);
// blocks until SIGINT
lws::scanner::run(std::move(disk), std::move(ctx), prog.scan_threads, webhook_verify);
lws::scanner::run(std::move(disk), std::move(ctx), prog.scan_threads, webhook_verify, enable_subaddresses);
}
} // anonymous

96
src/wire/adapted/array.h Normal file
View file

@ -0,0 +1,96 @@
// Copyright (c) 2023, The Monero Project
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without modification, are
// permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this list of
// conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice, this list
// of conditions and the following disclaimer in the documentation and/or other
// materials provided with the distribution.
//
// 3. Neither the name of the copyright holder nor the names of its contributors may be
// used to endorse or promote products derived from this software without specific
// prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#pragma once
#include <cstdint>
#include "span.h"
#include "wire/error.h"
#include "wire/read.h"
namespace wire
{
// enable writing of std::array
template<typename T, std::size_t N>
struct is_array<std::array<T, N>>
: std::true_type
{};
// `std::array`s of `char` and `uint8_t` are not arrays
template<std::size_t N>
struct is_array<std::array<char, N>>
: std::false_type
{};
template<std::size_t N>
struct is_array<std::array<std::uint8_t, N>>
: std::false_type
{};
template<typename R, std::size_t N>
inline void read_bytes(R& source, std::array<char, N>& dest)
{
source.binary(epee::to_mut_span(dest));
}
template<typename R, std::size_t N>
inline void read_bytes(R& source, std::array<std::uint8_t, N>& dest)
{
source.binary(epee::to_mut_span(dest));
}
template<typename W, std::size_t N>
inline void write_bytes(W& dest, const std::array<char, N>& source)
{
source.binary(epee::to_span(source));
}
template<typename W, std::size_t N>
inline void write_bytes(W& dest, const std::array<std::uint8_t, N>& source)
{
source.binary(epee::to_span(source));
}
// Read a fixed sized array
template<typename R, typename T, std::size_t N>
inline void read_bytes(R& source, std::array<T, N>& dest)
{
std::size_t count = source.start_array();
const bool json = (count == 0);
if (!json && count != dest.size())
WIRE_DLOG_THROW(wire::error::schema::array, "Expected array of size " << dest.size());
for (auto& elem : dest)
{
if (json && source.is_array_end(count))
WIRE_DLOG_THROW(wire::error::schema::array, "Expected array of size " << dest.size());
wire_read::bytes(source, elem);
--count;
}
if (!source.is_array_end(count))
WIRE_DLOG_THROW(wire::error::schema::array, "Expected array of size " << dest.size());
source.end_array();
}
}

2
tests/unit/db/CMakeLists.txt
View file

@ -26,7 +26,7 @@
# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
add_library(monero-lws-unit-db OBJECT storage.test.cpp webhook.test.cpp)
add_library(monero-lws-unit-db OBJECT data.test.cpp storage.test.cpp subaddress.test.cpp webhook.test.cpp)
target_link_libraries(
monero-lws-unit-db
monero-lws-unit-framework

76
tests/unit/db/data.test.cpp Normal file
View file

@ -0,0 +1,76 @@
// Copyright (c) 2023, The Monero Project
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without modification, are
// permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this list of
// conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice, this list
// of conditions and the following disclaimer in the documentation and/or other
// materials provided with the distribution.
//
// 3. Neither the name of the copyright holder nor the names of its contributors may be
// used to endorse or promote products derived from this software without specific
// prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include "framework.test.h"
#include "db/data.h"
LWS_CASE("db::data::check_subaddress_dict")
{
EXPECT(lws::db::check_subaddress_dict({lws::db::major_index(0), lws::db::index_ranges{}}));
EXPECT(lws::db::check_subaddress_dict(
{
lws::db::major_index(0),
lws::db::index_ranges{lws::db::index_range{{lws::db::minor_index(0), lws::db::minor_index(0)}}}
}
));
EXPECT(lws::db::check_subaddress_dict(
{
lws::db::major_index(0),
lws::db::index_ranges{
lws::db::index_range{{lws::db::minor_index(0), lws::db::minor_index(0)}},
lws::db::index_range{{lws::db::minor_index(2), lws::db::minor_index(10)}}
}
}
));
EXPECT(!lws::db::check_subaddress_dict(
{
lws::db::major_index(0),
lws::db::index_ranges{lws::db::index_range{{lws::db::minor_index(1), lws::db::minor_index(0)}}}
}
));
EXPECT(!lws::db::check_subaddress_dict(
{
lws::db::major_index(0),
lws::db::index_ranges{
lws::db::index_range{{lws::db::minor_index(0), lws::db::minor_index(4)}},
lws::db::index_range{{lws::db::minor_index(1), lws::db::minor_index(10)}}
}
}
));
EXPECT(!lws::db::check_subaddress_dict(
{
lws::db::major_index(0),
lws::db::index_ranges{
lws::db::index_range{{lws::db::minor_index(0), lws::db::minor_index(0)}},
lws::db::index_range{{lws::db::minor_index(1), lws::db::minor_index(10)}}
}
}
));
}

422
tests/unit/db/subaddress.test.cpp Normal file
View file

@ -0,0 +1,422 @@
// Copyright (c) 2023, The Monero Project
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without modification, are
// permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this list of
// conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice, this list
// of conditions and the following disclaimer in the documentation and/or other
// materials provided with the distribution.
//
// 3. Neither the name of the copyright holder nor the names of its contributors may be
// used to endorse or promote products derived from this software without specific
// prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include "framework.test.h"
#include <boost/range/counting_range.hpp>
#include <boost/uuid/random_generator.hpp>
#include <cstdint>
#include "crypto/crypto.h" // monero/src
#include "db/data.h"
#include "db/storage.h"
#include "db/storage.test.h"
#include "error.h"
#include "wire/error.h"
namespace
{
struct user_account
{
lws::db::account_address account;
crypto::secret_key view;
user_account()
: account{}, view{}
{}
};
void check_address_map(lest::env& lest_env, lws::db::storage_reader& reader, const user_account& user, const std::vector<lws::db::subaddress_dict>& source)
{
SETUP("check_address_map")
{
lws::db::cursor::subaddress_indexes cur = nullptr;
for (const auto& major_entry : source)
{
for (const auto& minor_entry : major_entry.second)
{
for (std::uint64_t elem : boost::counting_range(std::uint64_t(minor_entry[0]), std::uint64_t(minor_entry[1]) + 1))
{
const lws::db::address_index index{major_entry.first, lws::db::minor_index(elem)};
auto result = reader.find_subaddress(lws::db::account_id(1), index.get_spend_public(user.account, user.view), cur);
EXPECT(result.has_value());
EXPECT(result == index);
}
}
}
}
}
}
LWS_CASE("db::storage::upsert_subaddresses")
{
user_account user{};
crypto::generate_keys(user.account.spend_public, user.view);
crypto::generate_keys(user.account.view_public, user.view);
SETUP("One Account DB")
{
lws::db::test::cleanup_db on_scope_exit{};
lws::db::storage db = lws::db::test::get_fresh_db();
const lws::db::block_info last_block =
MONERO_UNWRAP(MONERO_UNWRAP(db.start_read()).get_last_block());
MONERO_UNWRAP(db.add_account(user.account, user.view));
SECTION("Empty get_subaddresses")
{
lws::db::storage_reader reader = MONERO_UNWRAP(db.start_read());
EXPECT(MONERO_UNWRAP(reader.get_subaddresses(lws::db::account_id(1))).empty());
}
SECTION("Upsert Basic")
{
std::vector<lws::db::subaddress_dict> subs{};
subs.emplace_back(
lws::db::major_index(0),
lws::db::index_ranges{lws::db::index_range{lws::db::minor_index(1), lws::db::minor_index(100)}}
);
auto result = db.upsert_subaddresses(lws::db::account_id(1), user.account, user.view, subs, 100);
{
lws::db::storage_reader reader = MONERO_UNWRAP(db.start_read());
EXPECT(result.has_value());
EXPECT(result->size() == 1);
EXPECT(result->at(0).first == lws::db::major_index(0));
EXPECT(result->at(0).second.size() == 1);
EXPECT(result->at(0).second[0][0] == lws::db::minor_index(1));
EXPECT(result->at(0).second[0][1] == lws::db::minor_index(100));
check_address_map(lest_env, reader, user, subs);
}
subs.back().first = lws::db::major_index(1);
result = db.upsert_subaddresses(lws::db::account_id(1), user.account, user.view, subs, 199);
EXPECT(result.has_error());
EXPECT(result == lws::error::max_subaddresses);
lws::db::storage_reader reader = MONERO_UNWRAP(db.start_read());
const auto fetched = reader.get_subaddresses(lws::db::account_id(1));
EXPECT(fetched.has_value());
EXPECT(fetched->size() == 1);
EXPECT(fetched->at(0).first == lws::db::major_index(0));
EXPECT(fetched->at(0).second.size() == 1);
EXPECT(fetched->at(0).second[0][0] == lws::db::minor_index(1));
EXPECT(fetched->at(0).second[0][1] == lws::db::minor_index(100));
}
SECTION("Upsert Appended")
{
std::vector<lws::db::subaddress_dict> subs{};
subs.emplace_back(
lws::db::major_index(0),
lws::db::index_ranges{lws::db::index_range{lws::db::minor_index(1), lws::db::minor_index(100)}}
);
auto result = db.upsert_subaddresses(lws::db::account_id(1), user.account, user.view, subs, 100);
EXPECT(result.has_value());
EXPECT(result->size() == 1);
EXPECT(result->at(0).first == lws::db::major_index(0));
EXPECT(result->at(0).second.size() == 1);
EXPECT(result->at(0).second[0][0] == lws::db::minor_index(1));
EXPECT(result->at(0).second[0][1] == lws::db::minor_index(100));
{
auto reader = MONERO_UNWRAP(db.start_read());
check_address_map(lest_env, reader, user, subs);
}
subs.back().second =
lws::db::index_ranges{lws::db::index_range{lws::db::minor_index(101), lws::db::minor_index(200)}};
result = db.upsert_subaddresses(lws::db::account_id(1), user.account, user.view, subs, 200);
EXPECT(result.has_value());
EXPECT(result->size() == 1);
EXPECT(result->at(0).first == lws::db::major_index(0));
EXPECT(result->at(0).second.size() == 1);
EXPECT(result->at(0).second[0][0] == lws::db::minor_index(101));
EXPECT(result->at(0).second[0][1] == lws::db::minor_index(200));
{
auto reader = MONERO_UNWRAP(db.start_read());
check_address_map(lest_env, reader, user, subs);
}
subs.back().second =
lws::db::index_ranges{lws::db::index_range{lws::db::minor_index(201), lws::db::minor_index(201)}};
result = db.upsert_subaddresses(lws::db::account_id(1), user.account, user.view, subs, 200);
EXPECT(result.has_error());
EXPECT(result == lws::error::max_subaddresses);
auto reader = MONERO_UNWRAP(db.start_read());
const auto fetched = reader.get_subaddresses(lws::db::account_id(1));
EXPECT(fetched.has_value());
EXPECT(fetched->size() == 1);
EXPECT(fetched->at(0).first == lws::db::major_index(0));
EXPECT(fetched->at(0).second.size() == 1);
EXPECT(fetched->at(0).second[0][0] == lws::db::minor_index(1));
EXPECT(fetched->at(0).second[0][1] == lws::db::minor_index(200));
}
SECTION("Upsert Prepended")
{
std::vector<lws::db::subaddress_dict> subs{};
subs.emplace_back(
lws::db::major_index(0),
lws::db::index_ranges{lws::db::index_range{lws::db::minor_index(101), lws::db::minor_index(200)}}
);
auto result = db.upsert_subaddresses(lws::db::account_id(1), user.account, user.view, subs, 100);
EXPECT(result.has_value());
EXPECT(result->size() == 1);
EXPECT(result->at(0).first == lws::db::major_index(0));
EXPECT(result->at(0).second.size() == 1);
EXPECT(result->at(0).second[0][0] == lws::db::minor_index(101));
EXPECT(result->at(0).second[0][1] == lws::db::minor_index(200));
{
auto reader = MONERO_UNWRAP(db.start_read());
check_address_map(lest_env, reader, user, subs);
}
subs.back().second =
lws::db::index_ranges{lws::db::index_range{lws::db::minor_index(1), lws::db::minor_index(100)}};
result = db.upsert_subaddresses(lws::db::account_id(1), user.account, user.view, subs, 199);
EXPECT(result.has_error());
EXPECT(result == lws::error::max_subaddresses);
result = db.upsert_subaddresses(lws::db::account_id(1), user.account, user.view, subs, 200);
EXPECT(result.has_value());
EXPECT(result->size() == 1);
EXPECT(result->at(0).first == lws::db::major_index(0));
EXPECT(result->at(0).second.size() == 1);
EXPECT(result->at(0).second[0][0] == lws::db::minor_index(1));
EXPECT(result->at(0).second[0][1] == lws::db::minor_index(100));
lws::db::storage_reader reader = MONERO_UNWRAP(db.start_read());
check_address_map(lest_env, reader, user, subs);
const auto fetched = reader.get_subaddresses(lws::db::account_id(1));
EXPECT(fetched.has_value());
EXPECT(fetched->size() == 1);
EXPECT(fetched->at(0).first == lws::db::major_index(0));
EXPECT(fetched->at(0).second.size() == 1);
EXPECT(fetched->at(0).second[0][0] == lws::db::minor_index(1));
EXPECT(fetched->at(0).second[0][1] == lws::db::minor_index(200));
}
SECTION("Upsert Wrapped")
{
std::vector<lws::db::subaddress_dict> subs{};
subs.emplace_back(
lws::db::major_index(0),
lws::db::index_ranges{lws::db::index_range{lws::db::minor_index(101), lws::db::minor_index(200)}}
);
auto result = db.upsert_subaddresses(lws::db::account_id(1), user.account, user.view, subs, 100);
EXPECT(result.has_value());
EXPECT(result->size() == 1);
EXPECT(result->at(0).first == lws::db::major_index(0));
EXPECT(result->at(0).second.size() == 1);
EXPECT(result->at(0).second[0][0] == lws::db::minor_index(101));
EXPECT(result->at(0).second[0][1] == lws::db::minor_index(200));
{
auto reader = MONERO_UNWRAP(db.start_read());
check_address_map(lest_env, reader, user, subs);
}
subs.back().second =
lws::db::index_ranges{lws::db::index_range{lws::db::minor_index(1), lws::db::minor_index(300)}};
result = db.upsert_subaddresses(lws::db::account_id(1), user.account, user.view, subs, 299);
EXPECT(result.has_error());
EXPECT(result == lws::error::max_subaddresses);
result = db.upsert_subaddresses(lws::db::account_id(1), user.account, user.view, subs, 300);
EXPECT(result.has_value());
EXPECT(result->size() == 1);
EXPECT(result->at(0).first == lws::db::major_index(0));
EXPECT(result->at(0).second.size() == 2);
EXPECT(result->at(0).second[0][0] == lws::db::minor_index(1));
EXPECT(result->at(0).second[0][1] == lws::db::minor_index(100));
EXPECT(result->at(0).second[1][0] == lws::db::minor_index(201));
EXPECT(result->at(0).second[1][1] == lws::db::minor_index(300));
lws::db::storage_reader reader = MONERO_UNWRAP(db.start_read());
check_address_map(lest_env, reader, user, subs);
const auto fetched = reader.get_subaddresses(lws::db::account_id(1));
EXPECT(fetched.has_value());
EXPECT(fetched->size() == 1);
EXPECT(fetched->at(0).first == lws::db::major_index(0));
EXPECT(fetched->at(0).second.size() == 1);
EXPECT(fetched->at(0).second[0][0] == lws::db::minor_index(1));
EXPECT(fetched->at(0).second[0][1] == lws::db::minor_index(300));
}
SECTION("Upsert After")
{
std::vector<lws::db::subaddress_dict> subs{};
subs.emplace_back(
lws::db::major_index(0),
lws::db::index_ranges{lws::db::index_range{lws::db::minor_index(1), lws::db::minor_index(100)}}
);
auto result = db.upsert_subaddresses(lws::db::account_id(1), user.account, user.view, subs, 100);
EXPECT(result.has_value());
EXPECT(result->size() == 1);
EXPECT(result->at(0).first == lws::db::major_index(0));
EXPECT(result->at(0).second.size() == 1);
EXPECT(result->at(0).second[0][0] == lws::db::minor_index(1));
EXPECT(result->at(0).second[0][1] == lws::db::minor_index(100));
{
auto reader = MONERO_UNWRAP(db.start_read());
check_address_map(lest_env, reader, user, subs);
}
subs.back().second =
lws::db::index_ranges{lws::db::index_range{lws::db::minor_index(102), lws::db::minor_index(200)}};
result = db.upsert_subaddresses(lws::db::account_id(1), user.account, user.view, subs, 198);
EXPECT(result.has_error());
EXPECT(result == lws::error::max_subaddresses);
result = db.upsert_subaddresses(lws::db::account_id(1), user.account, user.view, subs, 199);
EXPECT(result.has_value());
EXPECT(result->size() == 1);
EXPECT(result->at(0).first == lws::db::major_index(0));
EXPECT(result->at(0).second.size() == 1);
EXPECT(result->at(0).second[0][0] == lws::db::minor_index(102));
EXPECT(result->at(0).second[0][1] == lws::db::minor_index(200));
auto reader = MONERO_UNWRAP(db.start_read());
check_address_map(lest_env, reader, user, subs);
const auto fetched = reader.get_subaddresses(lws::db::account_id(1));
EXPECT(fetched.has_value());
EXPECT(fetched->size() == 1);
EXPECT(fetched->at(0).first == lws::db::major_index(0));
EXPECT(fetched->at(0).second.size() == 2);
EXPECT(fetched->at(0).second[0][0] == lws::db::minor_index(1));
EXPECT(fetched->at(0).second[0][1] == lws::db::minor_index(100));
EXPECT(fetched->at(0).second[1][0] == lws::db::minor_index(102));
EXPECT(fetched->at(0).second[1][1] == lws::db::minor_index(200));
}
SECTION("Upsert Before")
{
std::vector<lws::db::subaddress_dict> subs{};
subs.emplace_back(
lws::db::major_index(0),
lws::db::index_ranges{lws::db::index_range{lws::db::minor_index(101), lws::db::minor_index(200)}}
);
auto result = db.upsert_subaddresses(lws::db::account_id(1), user.account, user.view, subs, 100);
EXPECT(result.has_value());
EXPECT(result->size() == 1);
EXPECT(result->at(0).first == lws::db::major_index(0));
EXPECT(result->at(0).second.size() == 1);
EXPECT(result->at(0).second[0][0] == lws::db::minor_index(101));
EXPECT(result->at(0).second[0][1] == lws::db::minor_index(200));
{
auto reader = MONERO_UNWRAP(db.start_read());
check_address_map(lest_env, reader, user, subs);
}
subs.back().second =
lws::db::index_ranges{lws::db::index_range{lws::db::minor_index(1), lws::db::minor_index(99)}};
result = db.upsert_subaddresses(lws::db::account_id(1), user.account, user.view, subs, 198);
EXPECT(result.has_error());
EXPECT(result == lws::error::max_subaddresses);
result = db.upsert_subaddresses(lws::db::account_id(1), user.account, user.view, subs, 199);
EXPECT(result.has_value());
EXPECT(result->size() == 1);
EXPECT(result->at(0).first == lws::db::major_index(0));
EXPECT(result->at(0).second.size() == 1);
EXPECT(result->at(0).second[0][0] == lws::db::minor_index(1));
EXPECT(result->at(0).second[0][1] == lws::db::minor_index(99));
auto reader = MONERO_UNWRAP(db.start_read());
check_address_map(lest_env, reader, user, subs);
const auto fetched = reader.get_subaddresses(lws::db::account_id(1));
EXPECT(fetched.has_value());
EXPECT(fetched->size() == 1);
EXPECT(fetched->at(0).first == lws::db::major_index(0));
EXPECT(fetched->at(0).second.size() == 2);
EXPECT(fetched->at(0).second[0][0] == lws::db::minor_index(1));
EXPECT(fetched->at(0).second[0][1] == lws::db::minor_index(99));
EXPECT(fetched->at(0).second[1][0] == lws::db::minor_index(101));
EXPECT(fetched->at(0).second[1][1] == lws::db::minor_index(200));
}
SECTION("Upsert Encapsulated")
{
std::vector<lws::db::subaddress_dict> subs{};
subs.emplace_back(
lws::db::major_index(0),
lws::db::index_ranges{lws::db::index_range{lws::db::minor_index(1), lws::db::minor_index(200)}}
);
auto result = db.upsert_subaddresses(lws::db::account_id(1), user.account, user.view, subs, 200);
EXPECT(result.has_value());
EXPECT(result->size() == 1);
EXPECT(result->at(0).first == lws::db::major_index(0));
EXPECT(result->at(0).second.size() == 1);
EXPECT(result->at(0).second[0][0] == lws::db::minor_index(1));
EXPECT(result->at(0).second[0][1] == lws::db::minor_index(200));
{
auto reader = MONERO_UNWRAP(db.start_read());
check_address_map(lest_env, reader, user, subs);
}
subs.back().second =
lws::db::index_ranges{lws::db::index_range{lws::db::minor_index(5), lws::db::minor_index(99)}};
result = db.upsert_subaddresses(lws::db::account_id(1), user.account, user.view, subs, 300);
EXPECT(result.has_value());
EXPECT(result->size() == 0);
auto reader = MONERO_UNWRAP(db.start_read());
check_address_map(lest_env, reader, user, subs);
const auto fetched = reader.get_subaddresses(lws::db::account_id(1));
EXPECT(fetched.has_value());
EXPECT(fetched->size() == 1);
EXPECT(fetched->at(0).first == lws::db::major_index(0));
EXPECT(fetched->at(0).second.size() == 1);
EXPECT(fetched->at(0).second[0][0] == lws::db::minor_index(1));
EXPECT(fetched->at(0).second[0][1] == lws::db::minor_index(200));
}
SECTION("Bad subaddress_dict")
{
std::vector<lws::db::subaddress_dict> subs{};
subs.emplace_back(
lws::db::major_index(0),
lws::db::index_ranges{lws::db::index_range{lws::db::minor_index(1), lws::db::minor_index(100)}}
);
subs.back().second.push_back(
lws::db::index_range{lws::db::minor_index(101), lws::db::minor_index(200)}
);
auto result = db.upsert_subaddresses(lws::db::account_id(1), user.account, user.view, subs, 100);
EXPECT(result.has_error());
EXPECT(result == wire::error::schema::array);
}
}
}