Ecosystem/monero-lws
1
0
Fork 0
mirror of https://github.com/vtnerd/monero-lws.git synced 2024-09-28 19:41:03 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Basic "chain hardening" for slightly untrusted daemons (#93)

Browse source
This commit is contained in:
Lee *!* Clagett 2024-03-07 17:39:18 -05:00 committed by Lee *!* Clagett
parent db66d410cd
commit 351ccaa872
15 changed files with 1063 additions and 82 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

37
src/db/data.cpp
View file

@ -201,6 +201,43 @@ namespace db
}
WIRE_DEFINE_OBJECT(block_info, map_block_info);
namespace
{
template<typename F, typename T>
void map_block_difficulty(F& format, T& self)
{
wire::object(format, WIRE_FIELD_ID(0, high), WIRE_FIELD_ID(1, low));
}
}
WIRE_DEFINE_OBJECT(block_difficulty, map_block_difficulty);
void block_difficulty::set_difficulty(const unsigned_int& in)
{
high = ((in >> 64) & 0xffffffffffffffff).convert_to<std::uint64_t>();
low = (in & 0xffffffffffffffff).convert_to<std::uint64_t>();
}
block_difficulty::unsigned_int block_difficulty::get_difficulty() const
{
unsigned_int out = high;
out <<= 64;
out += low;
return out;
}
namespace
{
template<typename F, typename T>
void map_block_pow(F& format, T& self)
{
wire::object(format,
WIRE_FIELD_ID(0, id),
WIRE_FIELD_ID(1, timestamp),
WIRE_FIELD_ID(2, cumulative_diff)
);
}
}
WIRE_DEFINE_OBJECT(block_pow, map_block_pow);
namespace
{
template<typename F, typename T>

32
src/db/data.h
View file

@ -27,6 +27,7 @@
#pragma once
#include <array>
#include <boost/multiprecision/cpp_int.hpp>
#include <boost/uuid/uuid.hpp>
#include <cassert>
#include <cstdint>
@ -183,6 +184,7 @@ namespace db
static_assert(sizeof(account) == (4 * 2) + 64 + 32 + (8 * 2) + (4 * 2), "padding in account");
void write_bytes(wire::writer&, const account&, bool show_key = false);
//! Used with quick and full sync mode
struct block_info
{
block_id id; //!< Must be first for LMDB optimizations
@ -191,6 +193,36 @@ namespace db
static_assert(sizeof(block_info) == 8 + 32, "padding in block_info");
WIRE_DECLARE_OBJECT(block_info);
struct block_difficulty
{
using unsigned_int = boost::multiprecision::uint128_t;
std::uint64_t high;
std::uint64_t low;
void set_difficulty(const unsigned_int& in);
unsigned_int get_difficulty() const;
};
static_assert(sizeof(block_difficulty) == 8 * 2, "padding in block_difficulty");
WIRE_DECLARE_OBJECT(block_difficulty);
//! Used with untrusted daemons / full sync mode
struct block_pow
{
block_id id;
std::uint64_t timestamp;
block_difficulty cumulative_diff;
};
static_assert(sizeof(block_pow) == 8 * 4, "padding in blow_pow");
WIRE_DECLARE_OBJECT(block_pow);
//! Used during sync "check-ins" if --untrusted-daemon
struct pow_sync
{
std::uint64_t timestamp;
block_difficulty cumulative_diff;
};
//! `output`s and `spend`s are sorted by these fields to make merging easier.
struct transaction_link
{

402
src/db/storage.cpp
View file

@ -181,6 +181,7 @@ namespace db
constexpr const unsigned blocks_version = 0;
constexpr const unsigned by_address_version = 0;
constexpr const unsigned pows_version = 0;
template<typename T>
int less(epee::span<const std::uint8_t> left, epee::span<const std::uint8_t> right) noexcept
@ -287,6 +288,9 @@ namespace db
constexpr const lmdb::basic_table<unsigned, block_info> blocks{
"blocks_by_id", (MDB_CREATE | MDB_DUPSORT), MONERO_SORT_BY(block_info, id)
};
constexpr const lmdb::basic_table<unsigned, block_pow> pows{
"pow_by_id", (MDB_CREATE | MDB_DUPSORT), MONERO_SORT_BY(block_pow, id)
};
constexpr const lmdb::basic_table<account_status, account> accounts{
"accounts_by_status,id", (MDB_CREATE | MDB_DUPSORT), MONERO_SORT_BY(account, id)
};
@ -348,7 +352,7 @@ namespace db
}
template<typename K, typename V>
expect<void> bulk_insert(MDB_cursor& cur, K const& key, epee::span<V> values) noexcept
expect<void> bulk_insert(MDB_cursor& cur, K const& key, epee::span<V> values, unsigned flags = MDB_NODUPDATA) noexcept
{
while (!values.empty())
{
@ -359,7 +363,7 @@ namespace db
};
int err = mdb_cursor_put(
&cur, &key_bytes, value_bytes, (MDB_NODUPDATA | MDB_MULTIPLE)
&cur, &key_bytes, value_bytes, (flags | MDB_MULTIPLE)
);
if (err && err != MDB_KEYEXIST)
return {lmdb::error(err)};
@ -472,18 +476,29 @@ namespace db
}
}
template<typename T>
expect<T> get_blocks(MDB_cursor& cur, std::size_t max_internal)
void check_pow(MDB_txn& txn, MDB_dbi tbl)
{
T out{};
max_internal = std::min(std::size_t(64), max_internal);
out.reserve(12 + max_internal);
cursor::pow cur = MONERO_UNWRAP(lmdb::open_cursor<cursor::close_pow>(txn, tbl));
MDB_val key = lmdb::to_val(blocks_version);
MDB_val value{};
MONERO_LMDB_CHECK(mdb_cursor_get(&cur, &key, &value, MDB_SET));
MONERO_LMDB_CHECK(mdb_cursor_get(&cur, &key, &value, MDB_LAST_DUP));
int err = mdb_cursor_get(cur.get(), &key, nullptr, MDB_SET);
if (err)
{
if (err != MDB_NOTFOUND)
MONERO_THROW(lmdb::error(err), "Unable to retrieve blockchain hashes");
// new database
block_pow checkpoint{block_id(0), 0u, block_difficulty{0u, 1u}};
MDB_val value = lmdb::to_val(checkpoint);
err = mdb_cursor_put(cur.get(), &key, &value, MDB_NODUPDATA);
if (err)
MONERO_THROW(lmdb::error(err), "Unable to add hash to local blockchain");
}
}
template<typename T>
expect<void> get_blocks_tail(T& out, MDB_cursor& cur, MDB_val value, std::size_t max_internal)
{
for (unsigned i = 0; i < 10; ++i)
{
expect<block_info> next = blocks.get_value<block_info>(value);
@ -492,6 +507,7 @@ namespace db
out.push_back(std::move(*next));
MDB_val key{};
const int err = mdb_cursor_get(&cur, &key, &value, MDB_PREV_DUP);
if (err)
{
@ -499,7 +515,7 @@ namespace db
return {lmdb::error(err)};
if (out.back().id != block_id(0))
return {lws::error::bad_blockchain};
return out;
return success();
}
}
@ -527,6 +543,73 @@ namespace db
MONERO_CHECK(add_block(checkpoint));
if (out.back().id != block_id(0))
MONERO_CHECK(add_block(0));
return success();
}
template<typename T>
expect<T> get_blocks(MDB_cursor& cur, std::size_t max_internal)
{
T out{};
max_internal = std::min(std::size_t(64), max_internal);
out.reserve(12 + max_internal);
MDB_val key = lmdb::to_val(blocks_version);
MDB_val value{};
MONERO_LMDB_CHECK(mdb_cursor_get(&cur, &key, &value, MDB_SET));
MONERO_LMDB_CHECK(mdb_cursor_get(&cur, &key, &value, MDB_LAST_DUP));
MONERO_CHECK(get_blocks_tail(out, cur, value, max_internal));
return out;
}
template<typename T>
expect<T> get_blocks_from_height(MDB_cursor& cur, std::size_t max_internal, block_id last_pow)
{
T out{};
max_internal = std::min(std::size_t(64), max_internal);
out.reserve(12 + max_internal);
MDB_val key = lmdb::to_val(blocks_version);
MDB_val value = lmdb::to_val(last_pow);
MONERO_LMDB_CHECK(mdb_cursor_get(&cur, &key, &value, MDB_GET_BOTH));
MONERO_CHECK(get_blocks_tail(out, cur, value, max_internal));
return out;
}
template<typename T>
expect<T> get_pow_blocks(MDB_cursor& cur, std::size_t max_internal)
{
T out{};
max_internal = std::min(std::size_t(64), max_internal);
out.reserve(max_internal);
MDB_val key = lmdb::to_val(pows_version);
MDB_val value{};
MONERO_LMDB_CHECK(mdb_cursor_get(&cur, &key, &value, MDB_SET));
MONERO_LMDB_CHECK(mdb_cursor_get(&cur, &key, &value, MDB_LAST_DUP));
for (unsigned i = 0; i < max_internal; ++i)
{
expect<block_pow> next = pows.get_value<block_pow>(value);
if (!next)
return next.error();
out.push_back(std::move(*next));
MDB_val key{};
const int err = mdb_cursor_get(&cur, &key, &value, MDB_PREV_DUP);
if (err)
{
if (err != MDB_NOTFOUND)
return {lmdb::error(err)};
if (out.back().id != block_id(0))
return {lws::error::bad_blockchain};
return out;
}
}
return out;
}
@ -566,6 +649,7 @@ namespace db
struct tables_
{
MDB_dbi blocks;
MDB_dbi pows;
MDB_dbi accounts;
MDB_dbi accounts_ba;
MDB_dbi accounts_bh;
@ -588,6 +672,7 @@ namespace db
assert(txn != nullptr);
tables.blocks = blocks.open(*txn).value();
tables.pows = pows.open(*txn).value();
tables.accounts = accounts.open(*txn).value();
tables.accounts_ba = accounts_by_address.open(*txn).value();
tables.accounts_bh = accounts_by_height.open(*txn).value();
@ -619,7 +704,7 @@ namespace db
MONERO_THROW(v0_spends.error(), "Error opening old spends table");
check_blockchain(*txn, tables.blocks);
check_pow(*txn, tables.pows);
MONERO_UNWRAP(this->commit(std::move(txn)));
}
};
@ -641,6 +726,22 @@ namespace db
return blocks.get_value<block_info>(value);
}
expect<block_pow> storage_reader::get_last_pow_block() noexcept
{
MONERO_PRECOND(txn != nullptr);
assert(db != nullptr);
cursor::pow pow_cur;
MONERO_CHECK(check_cursor(*txn, db->tables.pows, pow_cur));
MDB_val key = lmdb::to_val(pows_version);
MDB_val value{};
MONERO_LMDB_CHECK(mdb_cursor_get(pow_cur.get(), &key, &value, MDB_SET));
MONERO_LMDB_CHECK(mdb_cursor_get(pow_cur.get(), &key, &value, MDB_LAST_DUP));
return pows.get_value<block_pow>(value);
}
expect<crypto::hash> storage_reader::get_block_hash(const block_id height) noexcept
{
MONERO_PRECOND(txn != nullptr);
@ -667,6 +768,88 @@ namespace db
return out;
}
expect<std::list<crypto::hash>> storage_reader::get_pow_sync()
{
MONERO_PRECOND(txn != nullptr);
assert(db != nullptr);
cursor::pow pow_cur;
MONERO_CHECK(check_cursor(*txn, db->tables.pows, pow_cur));
MONERO_CHECK(check_cursor(*txn, db->tables.blocks, curs.blocks_cur));
MDB_val key = lmdb::to_val(pows_version);
MDB_val value{};
MONERO_LMDB_CHECK(mdb_cursor_get(pow_cur.get(), &key, &value, MDB_SET));
MONERO_LMDB_CHECK(mdb_cursor_get(pow_cur.get(), &key, &value, MDB_LAST_DUP));
const block_id pow_height =
MONERO_UNWRAP(pows.get_value<MONERO_FIELD(block_pow, id)>(value));
auto blocks = get_blocks_from_height<std::vector<block_info>>(*curs.blocks_cur, 64, pow_height);
if (!blocks)
return blocks.error();
std::list<crypto::hash> out{};
for (block_info const& block : *blocks)
out.push_back(block.hash);
return out;
}
expect<pow_window>storage_reader::get_pow_window(const db::block_id last)
{
MONERO_PRECOND(txn != nullptr);
assert(db != nullptr);
pow_window out{};
if (last == block_id(0))
return out;
std::uint64_t next = 0;
static_assert(1 <= DIFFICULTY_BLOCKS_COUNT, "invalid DIFFICULTY_BLOCKS_COUNT value");
if (block_id(DIFFICULTY_BLOCKS_COUNT) < last)
next = std::uint64_t(last) - (DIFFICULTY_BLOCKS_COUNT - 1);
cursor::pow pow_cur;
MONERO_CHECK(check_cursor(*txn, db->tables.pows, pow_cur));
MDB_val key = lmdb::to_val(pows_version);
MDB_val value = lmdb::to_val(next);
MONERO_LMDB_CHECK(mdb_cursor_get(pow_cur.get(), &key, &value, MDB_GET_BOTH));
for (;;)
{
const auto insert = MONERO_UNWRAP(pows.get_value<block_pow>(value));
out.pow_timestamps.push_back(insert.timestamp);
out.cumulative_diffs.push_back(insert.cumulative_diff.get_difficulty());
++next;
if (next == std::uint64_t(last) + 1)
break;
MONERO_LMDB_CHECK(mdb_cursor_get(pow_cur.get(), &key, &value, MDB_NEXT_DUP));
}
if (last < db::block_id(BLOCKCHAIN_TIMESTAMP_CHECK_WINDOW))
return out;
next = std::uint64_t(last) - (BLOCKCHAIN_TIMESTAMP_CHECK_WINDOW - 1);
key = lmdb::to_val(pows_version);
value = lmdb::to_val(next);
MONERO_LMDB_CHECK(mdb_cursor_get(pow_cur.get(), &key, &value, MDB_GET_BOTH));
for (;;)
{
out.median_timestamps.push_back(
MONERO_UNWRAP(pows.get_value<MONERO_FIELD(block_pow, timestamp)>(value))
);
++next;
if (next == std::uint64_t(last) + 1)
break;
MONERO_LMDB_CHECK(mdb_cursor_get(pow_cur.get(), &key, &value, MDB_NEXT_DUP));
}
return out;
}
expect<lmdb::key_stream<account_status, account, cursor::close_accounts>>
storage_reader::get_accounts(cursor::accounts cur) noexcept
{
@ -994,6 +1177,7 @@ namespace db
return std::make_pair(address_string(src.address), src.lookup);
};
cursor::pow pow_cur;
cursor::accounts accounts_cur;
cursor::outputs outputs_cur;
cursor::spends spends_cur;
@ -1005,6 +1189,7 @@ namespace db
cursor::subaddress_indexes indexes_cur;
MONERO_CHECK(check_cursor(*txn, db->tables.blocks, curs.blocks_cur));
MONERO_CHECK(check_cursor(*txn, db->tables.pows, pow_cur));
MONERO_CHECK(check_cursor(*txn, db->tables.accounts, accounts_cur));
MONERO_CHECK(check_cursor(*txn, db->tables.accounts_ba, curs.accounts_ba_cur));
MONERO_CHECK(check_cursor(*txn, db->tables.accounts_bh, curs.accounts_bh_cur));
@ -1022,6 +1207,11 @@ namespace db
if (!blocks_partial)
return blocks_partial.error();
auto pow_partial =
get_pow_blocks<boost::container::static_vector<block_pow, 12>>(*pow_cur, 12);
if (!pow_partial)
return pow_partial.error();
auto accounts_stream = accounts.get_key_stream(std::move(accounts_cur));
if (!accounts_stream)
return accounts_stream.error();
@ -1075,6 +1265,7 @@ namespace db
wire::json_stream_writer json_stream{out};
wire::object(json_stream,
wire::field(blocks.name, wire::array(reverse(*blocks_partial))),
wire::field(pows.name, wire::array(reverse(*pow_partial))),
wire::field(accounts.name, wire::as_object(accounts_stream->make_range(), wire::enum_as_string, toggle_keys_filter)),
wire::field(accounts_by_address.name, wire::as_object(transform(accounts_ba_stream->make_range(), address_as_key))),
wire::field(accounts_by_height.name, wire::array(accounts_bh_stream->make_range())),
@ -1154,6 +1345,16 @@ namespace db
return instance.data;
}
block_info storage::get_last_checkpoint()
{
const auto& checkpoints = get_checkpoints().get_points();
if (checkpoints.empty())
MONERO_THROW(error::bad_blockchain, "Checkpoints invalid");
const auto last = checkpoints.rbegin();
return block_info{block_id(last->first), last->second};
}
storage storage::open(const char* path, unsigned create_queue_max)
{
return {
@ -1364,6 +1565,27 @@ namespace db
err = mdb_cursor_get(&cur, &key, &value, MDB_NEXT_DUP);
} while (err == 0);
// rollback pow
{
cursor::pow pow_cur;
MONERO_CHECK(check_cursor(txn, tables.pows, pow_cur));
MDB_val key = lmdb::to_val(pows_version);
MDB_val value = lmdb::to_val(height);
int err = mdb_cursor_get(pow_cur.get(), &key, &value, MDB_GET_BOTH);
for (;;)
{
if (err)
{
if (err == MDB_NOTFOUND)
break;
return {lmdb::error(err)};
}
MONERO_LMDB_CHECK(mdb_cursor_del(pow_cur.get(), 0));
err = mdb_cursor_get(pow_cur.get(), &key, &value, MDB_NEXT_DUP);
}
}
if (err != MDB_NOTFOUND)
return {lmdb::error(err)};
@ -1382,7 +1604,8 @@ namespace db
{
if (current == chain.end() || hashes.size() == hashes.capacity())
{
MONERO_CHECK(bulk_insert(cur, blocks_version, epee::to_span(hashes)));
// always overwrite, for pow case (where pows is catching up to blocks)
MONERO_CHECK(bulk_insert(cur, blocks_version, epee::to_span(hashes), 0));
if (current == chain.end())
return success();
hashes.clear();
@ -1392,6 +1615,29 @@ namespace db
++height;
}
}
template<typename T>
expect<void> append_pow(MDB_cursor& cur, db::block_id first, T const& chain)
{
std::uint64_t height = std::uint64_t(first);
boost::container::static_vector<block_pow, 31> pows{};
static_assert(sizeof(pows) <= 1024, "using more stack space than expected");
for (auto current = chain.begin() ;; ++current)
{
if (current == chain.end() || pows.size() == pows.capacity())
{
MONERO_CHECK(bulk_insert(cur, pows_version, epee::to_span(pows)));
if (current == chain.end())
return success();
pows.clear();
}
pows.push_back(block_pow{db::block_id(height), current->timestamp, current->cumulative_diff});
++height;
}
}
} // anonymous
expect<void> storage::rollback(block_id height)
@ -1467,6 +1713,91 @@ namespace db
});
}
expect<void> storage::sync_pow(block_id height, epee::span<const crypto::hash> hashes, epee::span<const pow_sync> pow)
{
MONERO_PRECOND(!hashes.empty());
MONERO_PRECOND(hashes.size() == pow.size());
MONERO_PRECOND(db != nullptr);
return db->try_write([this, height, hashes, pow] (MDB_txn& txn) -> expect<void>
{
cursor::blocks blocks_cur;
MONERO_CHECK(check_cursor(txn, this->db->tables.blocks, blocks_cur));
expect<crypto::hash> hash = do_get_block_hash(*blocks_cur, height);
if (!hash)
return hash.error();
// the first entry should always match on in the DB
if (*hash != *(hashes.begin()))
return {lws::error::bad_blockchain};
MDB_val key{};
MDB_val value{};
std::uint64_t current = std::uint64_t(height) + 1;
auto first = hashes.begin();
auto chain = boost::make_iterator_range(++first, hashes.end());
const auto& checkpoints = get_checkpoints();
for ( ; !chain.empty(); chain.advance_begin(1), ++current)
{
// if while syncing from beginning, a checkpoint was missed
const auto checkpoint = checkpoints.get_points().find(current);
if (checkpoint != checkpoints.get_points().end() && checkpoint->second != chain.front())
{
MERROR("Missed a checkpoint during sync_pow");
return {error::bad_blockchain};
}
const int err = mdb_cursor_get(blocks_cur.get(), &key, &value, MDB_NEXT_DUP);
if (err == MDB_NOTFOUND)
break;
if (err)
return {lmdb::error(err)};
auto full_value = blocks.get_value<block_info>(value);
if (!full_value)
return full_value.error();
if (full_value->id != block_id(current)) // hit a checkpoint or other block that is ahead of pow
break;
if (full_value->hash != chain.front())
{
if (current <= checkpoints.get_max_height())
{
MERROR("Attempting rollback past last checkpoint; invalid daemon chain response");
return {lws::error::bad_blockchain};
}
MONERO_CHECK(rollback_chain(this->db->tables, txn, *blocks_cur, db::block_id(current)));
break;
}
}
// scan checkpoints, this is hardened mode!
{
std::uint64_t current_copy = current;
for (const auto& current_hash : chain)
{
// if while syncing from beginning, a checkpoint was missed
const auto checkpoint = checkpoints.get_points().find(current_copy);
if (checkpoint != checkpoints.get_points().end() && checkpoint->second != current_hash)
{
MERROR("Missed a checkpoint during sync_pow");
return {error::bad_blockchain};
}
++current_copy;
}
}
auto first_pow = pow.begin() + std::ptrdiff_t(chain.begin() - hashes.begin());
cursor::pow pow_cur;
MONERO_CHECK(check_cursor(txn, this->db->tables.pows, pow_cur));
MONERO_CHECK(append_block_hashes(*blocks_cur, db::block_id(current), chain));
return append_pow(*pow_cur, db::block_id(current), boost::make_iterator_range(first_pow, pow.end()));
});
}
namespace
{
expect<db::account_time> get_account_time() noexcept
@ -2233,16 +2564,19 @@ namespace db
}
} // anonymous
expect<std::pair<std::size_t, std::vector<webhook_tx_confirmation>>> storage::update(block_id height, epee::span<const crypto::hash> chain, epee::span<const lws::account> users)
expect<std::pair<std::size_t, std::vector<webhook_tx_confirmation>>> storage::update(block_id height, epee::span<const crypto::hash> chain, epee::span<const lws::account> users, epee::span<const pow_sync> pow)
{
if (users.empty() && chain.empty())
return {std::make_pair(0, std::vector<webhook_tx_confirmation>{})};
MONERO_PRECOND(!chain.empty());
MONERO_PRECOND(db != nullptr);
if (!pow.empty())
MONERO_PRECOND(chain.size() == pow.size());
return db->try_write([this, height, chain, users] (MDB_txn& txn) -> expect<std::pair<std::size_t, std::vector<webhook_tx_confirmation>>>
return db->try_write([this, height, chain, users, pow] (MDB_txn& txn) -> expect<std::pair<std::size_t, std::vector<webhook_tx_confirmation>>>
{
epee::span<const crypto::hash> chain_copy{chain};
epee::span<const pow_sync> pow_copy{pow};
const std::uint64_t last_update =
lmdb::to_native(height) + chain.size() - 1;
const std::uint64_t first_new = lmdb::to_native(height) + 1;
@ -2252,7 +2586,9 @@ namespace db
if (get_checkpoints().get_max_height() <= last_update)
{
cursor::blocks blocks_cur;
cursor::pow pow_cur;
MONERO_CHECK(check_cursor(txn, this->db->tables.blocks, blocks_cur));
MONERO_CHECK(check_cursor(txn, this->db->tables.pows, pow_cur));
MDB_val key = lmdb::to_val(blocks_version);
MDB_val value;
@ -2276,6 +2612,40 @@ namespace db
*blocks_cur, block_id(lmdb::to_native(height) + offset + 1), chain_copy
)
);
if (!pow_copy.empty())
{
pow_copy.remove_prefix(offset + 1);
MONERO_CHECK(
append_pow(*pow_cur, block_id(lmdb::to_native(height) + offset + 1), pow_copy)
);
}
}
else // perform chain/pow hardening via checkpoints (if available)
{
cursor::blocks blocks_cur;
MONERO_CHECK(check_cursor(txn, this->db->tables.blocks, blocks_cur));
MDB_val key = lmdb::to_val(blocks_version);
MDB_val value = lmdb::to_val(last_update);
int err = mdb_cursor_get(blocks_cur.get(), &key, &value, MDB_GET_BOTH);
// verify last block hash if available. If not availble, --untrusted-daemon was not used
if (err)
{
if (err != MDB_NOTFOUND)
return {lmdb::error(err)};
}
else
{
const auto cur_block = blocks.get_value<block_info>(value);
if (!cur_block)
return cur_block.error();
// If a reorg past a checkpoint is being attempted
if (chain[chain.size() - 1] != cur_block->hash)
return {error::bad_blockchain};
}
}
cursor::accounts accounts_cur;

24
src/db/storage.h
View file

@ -57,6 +57,7 @@ namespace db
MONERO_CURSOR(subaddress_indexes);
MONERO_CURSOR(blocks);
MONERO_CURSOR(pow);
MONERO_CURSOR(accounts_by_address);
MONERO_CURSOR(accounts_by_height);
@ -72,6 +73,13 @@ namespace db
cursor::accounts_by_height accounts_bh_cur;
};
struct pow_window
{
std::vector<std::uint64_t> pow_timestamps; //!< for pow calculation
std::vector<block_difficulty::unsigned_int> cumulative_diffs;
std::vector<std::uint64_t> median_timestamps; //!< for timestamp check
};
//! Wrapper for LMDB read access to on-disk storage of light-weight server data.
class storage_reader
{
@ -95,12 +103,21 @@ namespace db
//! \return Last known block.
expect<block_info> get_last_block() noexcept;
//! \return Last known pow block.
expect<block_pow> get_last_pow_block() noexcept;
//! \return "Our" block hash at `height`.
expect<crypto::hash> get_block_hash(const block_id height) noexcept;
//! \return List for `GetHashesFast` to sync blockchain with daemon.
expect<std::list<crypto::hash>> get_chain_sync();
//! \return List for GetBlocksFast` to sync blockchain+pow with daemon
expect<std::list<crypto::hash>> get_pow_sync();
//! \return Objects for use with cryptonote::next_difficulty and median timestamp check
expect<pow_window> get_pow_window(block_id last);
//! \return All registered `account`s.
expect<lmdb::key_stream<account_status, account, cursor::close_accounts>>
get_accounts(cursor::accounts cur = nullptr) noexcept;
@ -172,6 +189,9 @@ namespace db
//! \return A single instance of compiled-in checkpoints for lws
static cryptonote::checkpoints const& get_checkpoints();
//! \return Last hard-coded block checkpoint
static block_info get_last_checkpoint();
/*!
Open a light_wallet_server LDMB database.
@ -210,6 +230,8 @@ namespace db
*/
expect<void> sync_chain(block_id height, epee::span<const crypto::hash> hashes);
expect<void> sync_pow(block_id height, epee::span<const crypto::hash> hashes, epee::span<const pow_sync> pow);
//! Bump the last access time of `address` to the current time.
expect<void> update_access_time(account_address const& address) noexcept;
@ -255,7 +277,7 @@ namespace db
\return True iff LMDB successfully committed the update.
*/
expect<std::pair<std::size_t, std::vector<webhook_tx_confirmation>>>
update(block_id height, epee::span<const crypto::hash> chain, epee::span<const lws::account> accts);
update(block_id height, epee::span<const crypto::hash> chain, epee::span<const lws::account> accts, epee::span<const pow_sync> pow);
/*!
Adds subaddresses to an account. Upon success, an account will

124
src/rpc/daemon_zmq.cpp
View file

@ -58,20 +58,128 @@ namespace rct
wire::object(source, WIRE_FIELD(mask), WIRE_FIELD(amount));
}
static void read_bytes(wire::json_reader& source, clsag& self)
{
wire::object(source, WIRE_FIELD(s), WIRE_FIELD(c1), WIRE_FIELD(D));
}
static void read_bytes(wire::json_reader& source, mgSig& self)
{
wire::object(source, WIRE_FIELD(ss), WIRE_FIELD(cc));
}
static void read_bytes(wire::json_reader& source, BulletproofPlus& self)
{
wire::object(source,
WIRE_FIELD(V),
WIRE_FIELD(A),
WIRE_FIELD(A1),
WIRE_FIELD(B),
WIRE_FIELD(r1),
WIRE_FIELD(s1),
WIRE_FIELD(d1),
WIRE_FIELD(L),
WIRE_FIELD(R)
);
}
static void read_bytes(wire::json_reader& source, Bulletproof& self)
{
wire::object(source,
WIRE_FIELD(V),
WIRE_FIELD(A),
WIRE_FIELD(S),
WIRE_FIELD(T1),
WIRE_FIELD(T2),
WIRE_FIELD(taux),
WIRE_FIELD(mu),
WIRE_FIELD(L),
WIRE_FIELD(R),
WIRE_FIELD(a),
WIRE_FIELD(b),
WIRE_FIELD(t)
);
}
static void read_bytes(wire::json_reader& source, boroSig& self)
{
std::vector<rct::key> s0;
std::vector<rct::key> s1;
s0.reserve(64);
s1.reserve(64);
wire::object(source, wire::field("s0", std::ref(s0)), wire::field("s1", std::ref(s1)));
if (s0.size() != 64 || s1.size() != 64)
WIRE_DLOG_THROW(wire::error::schema::array, "Expected s0 and s1 to have 64 elements");
for (std::size_t i = 0; i < 64; ++i)
self.s0[i] = s0[i];
for (std::size_t i = 0; i < 64; ++i)
self.s1[i] = s1[i];
}
static void read_bytes(wire::json_reader& source, rangeSig& self)
{
std::vector<rct::key> keys{};
keys.reserve(64);
wire::object(source, WIRE_FIELD(asig), wire::field("Ci", std::ref(keys)));
if (keys.size() != 64)
WIRE_DLOG_THROW(wire::error::schema::array, "Expected 64 eleents in Ci");
for (std::size_t i = 0; i < 64; ++i)
{
self.Ci[i] = keys[i];
}
}
namespace
{
struct prunable_helper
{
rctSigPrunable prunable;
rct::keyV pseudo_outs;
};
void read_bytes(wire::json_reader& source, prunable_helper& self)
{
wire::object(source,
wire::field("range_proofs", std::ref(self.prunable.rangeSigs)),
wire::field("bulletproofs", std::ref(self.prunable.bulletproofs)),
wire::field("bulletproofs_plus", std::ref(self.prunable.bulletproofs_plus)),
wire::field("mlsags", std::ref(self.prunable.MGs)),
wire::field("clsags", std::ref(self.prunable.CLSAGs)),
wire::field("pseudo_outs", std::ref(self.pseudo_outs))
);
const bool pruned =
self.prunable.rangeSigs.empty() &&
self.prunable.bulletproofs.empty() &&
self.prunable.bulletproofs_plus.empty() &&
self.prunable.MGs.empty() &&
self.prunable.CLSAGs.empty() &&
self.pseudo_outs.empty();
if (pruned)
WIRE_DLOG_THROW(wire::error::schema::array, "Expected at least one prunable field");
}
} // anonymous
static void read_bytes(wire::json_reader& source, rctSig& self)
{
boost::optional<std::vector<ecdhTuple>> ecdhInfo;
boost::optional<ctkeyV> outPk;
boost::optional<xmr_amount> txnFee;
boost::optional<prunable_helper> prunable;
self.outPk.reserve(default_inputs);
wire::object(source,
WIRE_FIELD(type),
wire::optional_field("encrypted", std::ref(ecdhInfo)),
wire::optional_field("commitments", std::ref(outPk)),
wire::optional_field("fee", std::ref(txnFee))
wire::optional_field("fee", std::ref(txnFee)),
wire::optional_field("prunable", std::ref(prunable))
);
self.txnFee = 0;
if (self.type != RCTTypeNull)
{
if (!ecdhInfo || !outPk || !txnFee)
@ -82,6 +190,12 @@ namespace rct
}
else if (ecdhInfo || outPk || txnFee)
WIRE_DLOG_THROW(wire::error::schema::invalid_key, "Did not expected `encrypted`, `commitments`, or `fee`");
if (prunable)
{
self.p = std::move(prunable->prunable);
self.get_pseudo_outs() = std::move(prunable->pseudo_outs);
}
}
} // rct
@ -186,6 +300,12 @@ namespace cryptonote
} // rpc
} // cryptonote
void lws::rpc::read_bytes(wire::json_reader& source, get_hashes_fast_response& self)
{
self.hashes.reserve(default_blocks_fetched);
wire::object(source, WIRE_FIELD(hashes), WIRE_FIELD(start_height), WIRE_FIELD(current_height));
}
void lws::rpc::read_bytes(wire::json_reader& source, get_blocks_fast_response& self)
{
self.blocks.reserve(default_blocks_fetched);

20
src/rpc/daemon_zmq.h
View file

@ -75,6 +75,26 @@ namespace rpc
using response = get_blocks_fast_response;
};
void read_bytes(wire::json_reader&, get_blocks_fast_response&);
struct get_hashes_fast_request
{
get_hashes_fast_request() = delete;
std::vector<crypto::hash> known_hashes;
std::uint64_t start_height;
};
struct get_hashes_fast_response
{
get_hashes_fast_response() = delete;
std::vector<crypto::hash> hashes;
std::uint64_t start_height;
std::uint64_t current_height;
};
struct get_hashes_fast
{
using request = get_hashes_fast_request;
using response = get_hashes_fast_response;
};
void read_bytes(wire::json_reader&, get_hashes_fast_response&);
struct get_transaction_pool_request
{

348
src/scanner.cpp
View file

@ -1,4 +1,4 @@
// Copyright (c) 2018-2023, The Monero Project
// Copyright (c) 2018-2023, The Monero Project
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without modification, are
@ -42,13 +42,16 @@
#include <vector>
#include "common/error.h" // monero/src
#include "config.h"
#include "crypto/crypto.h" // monero/src
#include "crypto/wallet/crypto.h" // monero/src
#include "cryptonote_basic/cryptonote_basic.h" // monero/src
#include "cryptonote_basic/cryptonote_format_utils.h" // monero/src
#include "db/account.h"
#include "db/data.h"
#include "cryptonote_basic/difficulty.h" // monero/src
#include "error.h"
#include "hardforks/hardforks.h" // monero/src
#include "misc_log_ex.h" // monero/contrib/epee/include
#include "net/net_parse_helpers.h"
#include "net/net_ssl.h" // monero/contrib/epee/include
@ -57,6 +60,7 @@
#include "rpc/json.h"
#include "rpc/message_data_structs.h" // monero/src
#include "rpc/webhook.h"
#include "util/blocks.h"
#include "util/source_location.h"
#include "util/transactions.h"
@ -98,6 +102,7 @@ namespace lws
{
net::ssl_verification_t webhook_verify;
bool enable_subaddresses;
bool untrusted_daemon;
};
struct thread_data
@ -174,6 +179,43 @@ namespace lws
rpc::send_webhook(client, events, "json-full-payment_hook:", "msgpack-full-payment_hook:", std::chrono::seconds{5}, verify_mode);
}
std::size_t get_target_time(db::block_id height)
{
const hardfork_t* fork = nullptr;
switch (config::network)
{
case cryptonote::network_type::MAINNET:
if (num_mainnet_hard_forks < 2)
MONERO_THROW(error::bad_blockchain, "expected more mainnet forks");
fork = mainnet_hard_forks;
break;
case cryptonote::network_type::TESTNET:
if (num_testnet_hard_forks < 2)
MONERO_THROW(error::bad_blockchain, "expected more testnet forks");
fork = testnet_hard_forks;
break;
case cryptonote::network_type::STAGENET:
if (num_stagenet_hard_forks < 2)
MONERO_THROW(error::bad_blockchain, "expected more stagenet forks");
fork = stagenet_hard_forks;
break;
default:
MONERO_THROW(error::bad_blockchain, "chain type not support with full sync");
}
// this is hardfork version 2
return height < db::block_id(fork[1].height) ?
DIFFICULTY_TARGET_V1 : DIFFICULTY_TARGET_V2;
}
//! For difficulty vectors only
template<typename T>
void update_window(T& vec)
{
// should only have one to pop each time
while (DIFFICULTY_BLOCKS_COUNT < vec.size())
vec.erase(vec.begin());
};
struct by_height
{
bool operator()(account const& left, account const& right) const noexcept
@ -572,7 +614,7 @@ namespace lws
MINFO("Updated exchange rates: " << *(*new_rates));
}
void scan_loop(thread_sync& self, std::shared_ptr<thread_data> data) noexcept
void scan_loop(thread_sync& self, std::shared_ptr<thread_data> data, const bool untrusted_daemon, const bool leader_thread) noexcept
{
try
{
@ -602,17 +644,22 @@ namespace lws
cryptonote::rpc::GetBlocksFast::Request req{};
req.start_height = std::uint64_t(users.begin()->scan_height());
req.start_height = std::max(std::uint64_t(1), req.start_height);
req.prune = true;
req.prune = !untrusted_daemon;
epee::byte_slice block_request = rpc::client::make_message("get_blocks_fast", req);
if (!send(client, block_request.clone()))
return;
std::vector<crypto::hash> blockchain{};
std::vector<db::pow_sync> new_pow{};
db::pow_window pow_window{};
const db::block_info last_checkpoint = db::storage::get_last_checkpoint();
const db::block_id last_pow = MONERO_UNWRAP(MONERO_UNWRAP(disk.start_read()).get_last_pow_block()).id;
while (!self.update && scanner::is_running())
{
blockchain.clear();
new_pow.clear();
auto resp = client.get_message(block_rpc_timeout);
if (!resp)
@ -691,6 +738,8 @@ namespace lws
throw std::runtime_error{"Bad daemon response - need same number of blocks and indices"};
blockchain.push_back(cryptonote::get_block_hash(fetched->blocks.front().block));
if (untrusted_daemon)
new_pow.push_back(db::pow_sync{fetched->blocks.front().block.timestamp});
auto blocks = epee::to_span(fetched->blocks);
auto indices = epee::to_span(fetched->output_indices);
@ -704,7 +753,16 @@ namespace lws
else
fetched->start_height = 0;
if (untrusted_daemon)
{
pow_window = MONERO_UNWRAP(
MONERO_UNWRAP(disk.start_read()).get_pow_window(db::block_id(fetched->start_height))
);
}
subaddress_reader reader{disk, opts.enable_subaddresses};
db::block_difficulty::unsigned_int diff{};
const db::block_id initial_height = db::block_id(fetched->start_height);
for (auto block_data : boost::combine(blocks, indices))
{
++(fetched->start_height);
@ -733,12 +791,48 @@ namespace lws
reader
);
if (untrusted_daemon)
{
if (block.prev_id != blockchain.back())
MONERO_THROW(error::bad_blockchain, "A blocks prev_id does not match");
update_window(pow_window.pow_timestamps);
update_window(pow_window.cumulative_diffs);
while (BLOCKCHAIN_TIMESTAMP_CHECK_WINDOW < pow_window.median_timestamps.size())
pow_window.median_timestamps.erase(pow_window.median_timestamps.begin());
// longhash takes a while, check is_running
if (!scanner::is_running())
return;
diff = cryptonote::next_difficulty(pow_window.pow_timestamps, pow_window.cumulative_diffs, get_target_time(db::block_id(fetched->start_height)));
// skip POW hashing if done previously
if (last_pow < db::block_id(fetched->start_height))
{
if (!verify_timestamp(block.timestamp, pow_window.median_timestamps))
MONERO_THROW(error::bad_blockchain, "Block failed timestamp check - possible chain forgery");
const crypto::hash pow =
get_block_longhash(get_block_hashing_blob(block), db::block_id(fetched->start_height), block.major_version, disk, initial_height, epee::to_span(blockchain));
if (!cryptonote::check_hash(pow, diff))
MONERO_THROW(error::bad_blockchain, "Block had too low difficulty");
}
}
indices.remove_prefix(1);
if (txes.size() != indices.size())
throw std::runtime_error{"Bad daemon respnse - need same number of txes and indices"};
for (auto tx_data : boost::combine(block.tx_hashes, txes, indices))
{
if (untrusted_daemon)
{
if (cryptonote::get_transaction_hash(boost::get<1>(tx_data)) != boost::get<0>(tx_data))
MONERO_THROW(error::bad_blockchain, "Hash of transaction does not match hash in block");
}
scan_transaction(
epee::to_mut_span(users),
db::block_id(fetched->start_height),
@ -750,12 +844,24 @@ namespace lws
);
}
if (untrusted_daemon)
{
const auto last_difficulty =
pow_window.cumulative_diffs.empty() ?
db::block_difficulty::unsigned_int(0) : pow_window.cumulative_diffs.back();
pow_window.pow_timestamps.push_back(block.timestamp);
pow_window.median_timestamps.push_back(block.timestamp);
pow_window.cumulative_diffs.push_back(diff + last_difficulty);
new_pow.push_back(db::pow_sync{block.timestamp});
new_pow.back().cumulative_diff.set_difficulty(pow_window.cumulative_diffs.back());
}
blockchain.push_back(cryptonote::get_block_hash(block));
} // for each block
reader.reader = std::error_code{common_error::kInvalidArgument}; // cleanup reader before next write
auto updated = disk.update(
users.front().scan_height(), epee::to_span(blockchain), epee::to_span(users)
users.front().scan_height(), epee::to_span(blockchain), epee::to_span(users), epee::to_span(new_pow)
);
if (!updated)
{
@ -767,6 +873,11 @@ namespace lws
MONERO_THROW(updated.error(), "Failed to update accounts on disk");
}
if (untrusted_daemon && leader_thread && fetched->start_height % 4 == 0 && last_pow < db::block_id(fetched->start_height))
{
MINFO("On chain with hash " << blockchain.back() << " and difficulty " << diff << " at height " << fetched->start_height);
}
MINFO("Processed " << blocks.size() << " block(s) against " << users.size() << " account(s)");
send_payment_hook(client, epee::to_span(updated->second), opts.webhook_verify);
if (updated->first != users.size())
@ -844,6 +955,7 @@ namespace lws
MINFO("Starting scan loops on " << std::min(thread_count, users.size()) << " thread(s) with " << users.size() << " account(s)");
bool leader_thread = true;
while (!users.empty() && --thread_count)
{
const std::size_t per_thread = std::max(std::size_t(1), users.size() / (thread_count + 1));
@ -859,7 +971,8 @@ namespace lws
auto data = std::make_shared<thread_data>(
std::move(client), disk.clone(), std::move(thread_users), opts
);
threads.emplace_back(attrs, std::bind(&scan_loop, std::ref(self), std::move(data)));
threads.emplace_back(attrs, std::bind(&scan_loop, std::ref(self), std::move(data), opts.untrusted_daemon, leader_thread));
leader_thread = false;
}
if (!users.empty())
@ -870,7 +983,7 @@ namespace lws
auto data = std::make_shared<thread_data>(
std::move(client), disk.clone(), std::move(users), opts
);
threads.emplace_back(attrs, std::bind(&scan_loop, std::ref(self), std::move(data)));
threads.emplace_back(attrs, std::bind(&scan_loop, std::ref(self), std::move(data), opts.untrusted_daemon, false /*leader thread*/));
}
auto last_check = std::chrono::steady_clock::now();
@ -931,36 +1044,20 @@ namespace lws
accounts_cur = current_users.give_cursor();
} // while scanning
}
} // anonymous
expect<rpc::client> scanner::sync(db::storage disk, rpc::client client)
{
using get_hashes = cryptonote::rpc::GetHashesFast;
MINFO("Starting blockchain sync with daemon");
get_hashes::Request req{};
req.start_height = 0;
expect<std::list<crypto::hash>> get_chain_sync(expect<db::storage_reader> reader)
{
auto reader = disk.start_read();
if (!reader)
return reader.error();
auto chain = reader->get_chain_sync();
if (!chain)
return chain.error();
req.known_hashes = std::move(*chain);
return reader->get_chain_sync();
}
for (;;)
template<typename R, typename Q>
expect<typename R::response> fetch_chain(rpc::client& client, const char* endpoint, const Q& req)
{
if (req.known_hashes.empty())
return {lws::error::bad_blockchain};
expect<void> sent{lws::error::daemon_timeout};
epee::byte_slice msg = rpc::client::make_message("get_hashes_fast", req);
epee::byte_slice msg = rpc::client::make_message(endpoint, req);
auto start = std::chrono::steady_clock::now();
while (!(sent = client.send(std::move(msg), std::chrono::seconds{1})))
@ -975,10 +1072,10 @@ namespace lws
return sent.error();
}
expect<get_hashes::Response> resp{lws::error::daemon_timeout};
expect<std::string> resp{lws::error::daemon_timeout};
start = std::chrono::steady_clock::now();
while (!(resp = client.receive<get_hashes::Response>(std::chrono::seconds{1}, MLWS_CURRENT_LOCATION)))
while (!(resp = client.get_message(std::chrono::seconds{1})))
{
if (!scanner::is_running())
return {lws::error::signal_abort_process};
@ -989,29 +1086,180 @@ namespace lws
if (!resp.matches(std::errc::timed_out))
return resp.error();
}
//
// Exit loop if it appears we have synced to top of chain
//
if (resp->hashes.size() <= 1 || resp->hashes.back() == req.known_hashes.front())
return {std::move(client)};
MONERO_CHECK(disk.sync_chain(db::block_id(resp->start_height), epee::to_span(resp->hashes)));
req.known_hashes.erase(req.known_hashes.begin(), --(req.known_hashes.end()));
for (std::size_t num = 0; num < 10; ++num)
{
if (resp->hashes.empty())
break;
req.known_hashes.insert(--(req.known_hashes.end()), resp->hashes.back());
}
return rpc::parse_json_response<R>(std::move(*resp));
}
return {std::move(client)};
// does not validate blockchain hashes
expect<rpc::client> sync_quick(db::storage disk, rpc::client client)
{
MINFO("Starting blockchain sync with daemon");
cryptonote::rpc::GetHashesFast::Request req{};
req.start_height = 0;
req.known_hashes = MONERO_UNWRAP(MONERO_UNWRAP(disk.start_read()).get_chain_sync());
for (;;)
{
if (req.known_hashes.empty())
return {lws::error::bad_blockchain};
auto resp = fetch_chain<rpc::get_hashes_fast>(client, "get_hashes_fast", req);
if (!resp)
return resp.error();
//
// exit loop if it appears we have synced to top of chain
//
if (resp->hashes.size() <= 1 || resp->hashes.back() == req.known_hashes.front())
return {std::move(client)};
MONERO_CHECK(disk.sync_chain(db::block_id(resp->start_height), epee::to_span(resp->hashes)));
req.known_hashes.erase(req.known_hashes.begin(), --(req.known_hashes.end()));
for (std::size_t num = 0; num < 10; ++num)
{
if (resp->hashes.empty())
break;
req.known_hashes.insert(--(req.known_hashes.end()), resp->hashes.back());
}
}
return {std::move(client)};
}
// validates blockchain hashes
expect<rpc::client> sync_full(db::storage disk, rpc::client client)
{
MINFO("Starting blockchain sync with daemon");
cryptonote::rpc::GetBlocksFast::Request req{};
req.start_height = 0;
req.block_ids = MONERO_UNWRAP(MONERO_UNWRAP(disk.start_read()).get_pow_sync());
req.prune = true;
std::vector<crypto::hash> new_hashes{};
std::vector<db::pow_sync> new_pow{};
for (;;)
{
if (req.block_ids.empty())
return {lws::error::bad_blockchain};
auto resp = fetch_chain<rpc::get_blocks_fast>(client, "get_blocks_fast", req);
if (!resp)
return resp.error();
if (resp->blocks.empty())
return {error::bad_daemon_response};
crypto::hash hash{};
if (!cryptonote::get_block_hash(resp->blocks.front().block, hash))
return {lws::error::bad_blockchain};
//
// exit loop if it appears we have synced to top of chain
//
const db::block_info last_checkpoint = db::storage::get_last_checkpoint();
if (resp->blocks.size() <= 1)
{
// error if not past last checkpoint
const auto expected_hash =
MONERO_UNWRAP(disk.start_read()).get_block_hash(db::block_id(resp->start_height));
if (!expected_hash || *expected_hash != hash || db::block_id(resp->start_height) < last_checkpoint.id)
return {error::bad_daemon_response};
return {std::move(client)};
}
// genesis block must be present as last entry
req.block_ids.erase(req.block_ids.begin(), --(req.block_ids.end()));
auto pow_window =
MONERO_UNWRAP(MONERO_UNWRAP(disk.start_read()).get_pow_window(db::block_id(resp->start_height)));
// overlap check performed in db::storage::pow_sync
new_hashes.clear();
new_pow.clear();
new_hashes.reserve(resp->blocks.size());
new_pow.reserve(resp->blocks.size());
new_hashes.push_back(hash);
new_pow.push_back(db::pow_sync{resp->blocks.front().block.timestamp});
// skip overlap block
db::block_difficulty::unsigned_int diff = 0;
for (std::size_t i = 1; i < resp->blocks.size(); ++i)
{
const auto& block = resp->blocks[i].block;
const db::block_id height = db::block_id(resp->start_height + i);
// important check, ensure we haven't deviated from chain
if (block.prev_id != hash)
return {lws::error::bad_blockchain};
// compute block id hash
if (!cryptonote::get_block_hash(block, hash))
return {lws::error::bad_blockchain};
req.block_ids.push_front(hash);
update_window(pow_window.pow_timestamps);
update_window(pow_window.cumulative_diffs);
while (BLOCKCHAIN_TIMESTAMP_CHECK_WINDOW < pow_window.median_timestamps.size())
pow_window.median_timestamps.erase(pow_window.median_timestamps.begin());
// longhash takes a while, check is_running
if (!scanner::is_running())
return {error::signal_abort_process};
diff = cryptonote::next_difficulty(pow_window.pow_timestamps, pow_window.cumulative_diffs, get_target_time(height));
// skip POW hashing when sync is within checkpoint
// storage::sync_pow(...) currently verifies checkpoint hashes
if (last_checkpoint.id < height)
{
if (!verify_timestamp(block.timestamp, pow_window.median_timestamps))
{
MERROR("Block failed timestamp check - possible chain forgery");
return {error::bad_blockchain};
}
const crypto::hash pow =
get_block_longhash(get_block_hashing_blob(block), height, block.major_version, disk, db::block_id(resp->start_height), epee::to_span(new_hashes));
if (!cryptonote::check_hash(pow, diff))
{
MERROR("Block " << std::uint64_t(height) << "had too low difficulty");
return {error::bad_blockchain};
}
}
const auto last_difficulty =
pow_window.cumulative_diffs.empty() ?
db::block_difficulty::unsigned_int(0) : pow_window.cumulative_diffs.back();
pow_window.pow_timestamps.push_back(block.timestamp);
pow_window.median_timestamps.push_back(block.timestamp);
pow_window.cumulative_diffs.push_back(diff + last_difficulty);
new_hashes.push_back(hash);
new_pow.push_back(db::pow_sync{block.timestamp});
new_pow.back().cumulative_diff.set_difficulty(pow_window.cumulative_diffs.back());
} // for every tx in block
MONERO_CHECK(disk.sync_pow(db::block_id(resp->start_height), epee::to_span(new_hashes), epee::to_span(new_pow)));
MINFO("Verified up to block " << (resp->start_height + new_hashes.size() - 1) << " with hash " << hash << " and difficulty " << diff);
} // for until sync
return {std::move(client)};
}
} // anonymous
expect<rpc::client> scanner::sync(db::storage disk, rpc::client client, const bool untrusted_daemon)
{
if (untrusted_daemon)
return sync_full(std::move(disk), std::move(client));
return sync_quick(std::move(disk), std::move(client));
}
void scanner::run(db::storage disk, rpc::context ctx, std::size_t thread_count, const epee::net_utils::ssl_verification_t webhook_verify, const bool enable_subaddresses)
void scanner::run(db::storage disk, rpc::context ctx, std::size_t thread_count, const epee::net_utils::ssl_verification_t webhook_verify, const bool enable_subaddresses, const bool untrusted_daemon)
{
thread_count = std::max(std::size_t(1), thread_count);
@ -1065,7 +1313,7 @@ namespace lws
checked_wait(account_poll_interval - (std::chrono::steady_clock::now() - last));
}
else
check_loop(disk.clone(), ctx, thread_count, std::move(users), std::move(active), options{webhook_verify, enable_subaddresses});
check_loop(disk.clone(), ctx, thread_count, std::move(users), std::move(active), options{webhook_verify, enable_subaddresses, untrusted_daemon});
if (!scanner::is_running())
return;
@ -1073,7 +1321,7 @@ namespace lws
if (!client)
client = MONERO_UNWRAP(ctx.connect());
expect<rpc::client> synced = sync(disk.clone(), std::move(client));
expect<rpc::client> synced = sync(disk.clone(), std::move(client), untrusted_daemon);
if (!synced)
{
if (!synced.matches(std::errc::timed_out))

4
src/scanner.h
View file

@ -46,10 +46,10 @@ namespace lws
public:
//! Use `client` to sync blockchain data, and \return client if successful.
static expect<rpc::client> sync(db::storage disk, rpc::client client);
static expect<rpc::client> sync(db::storage disk, rpc::client client, const bool untrusted_daemon = false);
//! Poll daemon until `stop()` is called, using `thread_count` threads.
static void run(db::storage disk, rpc::context ctx, std::size_t thread_count, epee::net_utils::ssl_verification_t webhook_verify, bool enable_subaddresses);
static void run(db::storage disk, rpc::context ctx, std::size_t thread_count, epee::net_utils::ssl_verification_t webhook_verify, bool enable_subaddresses, bool untrusted_daemon = false);
//! \return True if `stop()` has never been called.
static bool is_running() noexcept { return running; }

11
src/server_main.cpp
View file

@ -80,6 +80,7 @@ namespace
const command_line::arg_descriptor<std::string> config_file;
const command_line::arg_descriptor<std::uint32_t> max_subaddresses;
const command_line::arg_descriptor<bool> auto_accept_creation;
const command_line::arg_descriptor<bool> untrusted_daemon;
static std::string get_default_zmq()
{
@ -124,6 +125,7 @@ namespace
, config_file{"config-file", "Specify any option in a config file; <name>=<value> on separate lines"}
, max_subaddresses{"max-subaddresses", "Maximum number of subaddresses per primary account (defaults to 0)", 0}
, auto_accept_creation{"auto-accept-creation", "New account creation requests are automatically accepted", false}
, untrusted_daemon{"untrusted-daemon", "Perform (expensive) chain-verification and PoW checks", false}
{}
void prepare(boost::program_options::options_description& description) const
@ -156,6 +158,7 @@ namespace
command_line::add_arg(description, config_file);
command_line::add_arg(description, max_subaddresses);
command_line::add_arg(description, auto_accept_creation);
command_line::add_arg(description, untrusted_daemon);
}
};
@ -173,6 +176,7 @@ namespace
std::chrono::minutes rates_interval;
std::size_t scan_threads;
unsigned create_queue_max;
bool untrusted_daemon;
};
void print_help(std::ostream& out)
@ -258,7 +262,8 @@ namespace
command_line::get_arg(args, opts.webhook_ssl_verification),
std::chrono::minutes{command_line::get_arg(args, opts.rates_interval)},
command_line::get_arg(args, opts.scan_threads),
command_line::get_arg(args, opts.create_queue_max)
command_line::get_arg(args, opts.create_queue_max),
command_line::get_arg(args, opts.untrusted_daemon)
};
prog.rest_config.threads = std::max(std::size_t(1), prog.rest_config.threads);
@ -279,7 +284,7 @@ namespace
auto ctx = lws::rpc::context::make(std::move(prog.daemon_rpc), std::move(prog.daemon_sub), std::move(prog.zmq_pub), std::move(prog.rmq), prog.rates_interval);
MINFO("Using monerod ZMQ RPC at " << ctx.daemon_address());
auto client = lws::scanner::sync(disk.clone(), ctx.connect().value()).value();
auto client = lws::scanner::sync(disk.clone(), ctx.connect().value(), prog.untrusted_daemon).value();
const auto enable_subaddresses = bool(prog.rest_config.max_subaddresses);
const auto webhook_verify = prog.rest_config.webhook_verify;
@ -292,7 +297,7 @@ namespace
MINFO("Listening for REST admin clients at " << address);
// blocks until SIGINT
lws::scanner::run(std::move(disk), std::move(ctx), prog.scan_threads, webhook_verify, enable_subaddresses);
lws::scanner::run(std::move(disk), std::move(ctx), prog.scan_threads, webhook_verify, enable_subaddresses, prog.untrusted_daemon);
}
} // anonymous

6
src/util/CMakeLists.txt
View file

@ -26,8 +26,8 @@
# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
set(monero-lws-util_sources gamma_picker.cpp random_outputs.cpp source_location.cpp transactions.cpp)
set(monero-lws-util_headers fwd.h gamma_picker.h http_server.h random_outputs.h source_location.h transactions.h)
set(monero-lws-util_sources blocks.cpp gamma_picker.cpp random_outputs.cpp source_location.cpp transactions.cpp)
set(monero-lws-util_headers blocks.h fwd.h gamma_picker.h http_server.h random_outputs.h source_location.h transactions.h)
add_library(monero-lws-util ${monero-lws-util_sources} ${monero-lws-util_headers})
target_link_libraries(monero-lws-util monero::libraries)
target_link_libraries(monero-lws-util monero::libraries monero-lws-db)

87
src/util/blocks.cpp Normal file
View file

@ -0,0 +1,87 @@
// Copyright (c) 2024, The Monero Project
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without modification, are
// permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this list of
// conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice, this list
// of conditions and the following disclaimer in the documentation and/or other
// materials provided with the distribution.
//
// 3. Neither the name of the copyright holder nor the names of its contributors may be
// used to endorse or promote products derived from this software without specific
// prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include "blocks.h"
#include "cryptonote_config.h" // monero/src
#include "crypto/hash-ops.h" // monero/src
#include "db/storage.h"
#include "error.h"
#include "misc_language.h"
#include "string_tools.h"
namespace lws
{
crypto::hash get_block_longhash(
const std::string& bd, const db::block_id height, const unsigned major_version, const db::storage& disk, const db::block_id cached_start, epee::span<const crypto::hash> cached)
{
crypto::hash result{};
// block 202612 bug workaround
if (height == db::block_id(202612))
{
static const std::string longhash_202612 = "84f64766475d51837ac9efbef1926486e58563c95a19fef4aec3254f03000000";
epee::string_tools::hex_to_pod(longhash_202612, result);
return result;
}
if (major_version >= RX_BLOCK_VERSION)
{
crypto::hash hash{};
if (height != db::block_id(0))
{
const uint64_t seed_height = crypto::rx_seedheight(std::uint64_t(height));
if (cached_start <= db::block_id(seed_height))
{
if (cached.size() <= seed_height - std::uint64_t(cached_start))
MONERO_THROW(error::bad_blockchain, "invalid seed_height for cache or DB");
hash = cached[seed_height - std::uint64_t(cached_start)];
}
else
hash = MONERO_UNWRAP(MONERO_UNWRAP(disk.start_read()).get_block_hash(db::block_id(seed_height)));
}
else
{
memset(&result, 0, sizeof(crypto::hash)); // only happens when generating genesis block
}
crypto::rx_slow_hash(hash.data, bd.data(), bd.size(), result.data);
} else {
const int pow_variant = major_version >= 7 ? major_version - 6 : 0;
crypto::cn_slow_hash(bd.data(), bd.size(), result, pow_variant, std::uint64_t(height));
}
return result;
}
bool verify_timestamp(std::uint64_t check, std::vector<std::uint64_t> timestamps)
{
if (timestamps.empty())
return true;
if(check < epee::misc_utils::median(timestamps))
return false;
return true;
}
}

40
src/util/blocks.h Normal file
View file

@ -0,0 +1,40 @@
// Copyright (c) 2024, The Monero Project
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without modification, are
// permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this list of
// conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice, this list
// of conditions and the following disclaimer in the documentation and/or other
// materials provided with the distribution.
//
// 3. Neither the name of the copyright holder nor the names of its contributors may be
// used to endorse or promote products derived from this software without specific
// prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include <string>
#include "crypto/hash.h"
#include "db/data.h"
#include "db/fwd.h"
#include "span.h" // in monero/contrib/epee/include
namespace lws
{
crypto::hash get_block_longhash(
const std::string& bd, const db::block_id height, const unsigned major_version, const db::storage& disk, db::block_id cached_start, epee::span<const crypto::hash> cached);
bool verify_timestamp(std::uint64_t verify, std::vector<std::uint64_t> timestamps);
}

2
tests/unit/db/chain.test.cpp
View file

@ -90,7 +90,7 @@ LWS_CASE("db::storage::sync_chain")
{
const lws::account accounts[1] = {lws::account{get_account(), {}, {}}};
EXPECT(accounts[0].scan_height() == last_block.id);
EXPECT(db.update(last_block.id, chain, accounts));
EXPECT(db.update(last_block.id, chain, accounts, nullptr));
EXPECT(get_account().scan_height == lws::db::block_id(std::uint64_t(last_block.id) + 4));
}

4
tests/unit/db/webhook.test.cpp
View file

@ -141,7 +141,7 @@ LWS_CASE("db::storage::*_webhook")
{
crypto::hash chain[2] = {head.hash, crypto::rand<crypto::hash>()};
auto updated = db.update(head.id, chain, {std::addressof(full_account), 1});
auto updated = db.update(head.id, chain, {std::addressof(full_account), 1}, nullptr);
EXPECT(!updated.has_error());
EXPECT(updated->first == 1);
if (i < 3)
@ -185,7 +185,7 @@ LWS_CASE("db::storage::*_webhook")
const std::vector<lws::db::output> outs = full_account.outputs();
EXPECT(outs.size() == 1);
const auto updated = db.update(last_block.id, chain, {std::addressof(full_account), 1});
const auto updated = db.update(last_block.id, chain, {std::addressof(full_account), 1}, nullptr);
EXPECT(!updated.has_error());
EXPECT(updated->first == 1);
EXPECT(updated->second.size() == 3);

4
tests/unit/scanner.test.cpp
View file

@ -425,8 +425,8 @@ LWS_CASE("lws::scanner::sync and lws::scanner::run")
EXPECT(result->at(0).second.at(0).size() == 2);
EXPECT(result->at(0).second.at(0).at(0) == lws::db::minor_index(1));
EXPECT(result->at(0).second.at(0).at(1) == lws::db::minor_index(2));
}
}
std::vector<cryptonote::tx_destination_entry> destinations;
destinations.emplace_back();
destinations.back().amount = 8000;