From 4ef2d8ad348df8018cbe7bf7790968f1804936fd Mon Sep 17 00:00:00 2001 From: Lee *!* Clagett Date: Thu, 7 Mar 2024 17:39:18 -0500 Subject: [PATCH] Basic "chain hardening" for slightly untrusted daemons (#93) --- src/db/data.cpp | 37 +++ src/db/data.h | 32 +++ src/db/storage.cpp | 402 +++++++++++++++++++++++++++++++-- src/db/storage.h | 24 +- src/rpc/daemon_zmq.cpp | 124 +++++++++- src/rpc/daemon_zmq.h | 20 ++ src/scanner.cpp | 348 ++++++++++++++++++++++++---- src/scanner.h | 4 +- src/server_main.cpp | 11 +- src/util/CMakeLists.txt | 6 +- src/util/blocks.cpp | 87 +++++++ src/util/blocks.h | 40 ++++ tests/unit/db/chain.test.cpp | 2 +- tests/unit/db/webhook.test.cpp | 4 +- tests/unit/scanner.test.cpp | 4 +- 15 files changed, 1063 insertions(+), 82 deletions(-) create mode 100644 src/util/blocks.cpp create mode 100644 src/util/blocks.h diff --git a/src/db/data.cpp b/src/db/data.cpp index cba0456..d78f8c1 100644 --- a/src/db/data.cpp +++ b/src/db/data.cpp @@ -201,6 +201,43 @@ namespace db } WIRE_DEFINE_OBJECT(block_info, map_block_info); + namespace + { + template + void map_block_difficulty(F& format, T& self) + { + wire::object(format, WIRE_FIELD_ID(0, high), WIRE_FIELD_ID(1, low)); + } + } + WIRE_DEFINE_OBJECT(block_difficulty, map_block_difficulty); + + void block_difficulty::set_difficulty(const unsigned_int& in) + { + high = ((in >> 64) & 0xffffffffffffffff).convert_to(); + low = (in & 0xffffffffffffffff).convert_to(); + } + block_difficulty::unsigned_int block_difficulty::get_difficulty() const + { + unsigned_int out = high; + out <<= 64; + out += low; + return out; + } + + namespace + { + template + void map_block_pow(F& format, T& self) + { + wire::object(format, + WIRE_FIELD_ID(0, id), + WIRE_FIELD_ID(1, timestamp), + WIRE_FIELD_ID(2, cumulative_diff) + ); + } + } + WIRE_DEFINE_OBJECT(block_pow, map_block_pow); + namespace { template diff --git a/src/db/data.h b/src/db/data.h index 0a55068..369e2bd 100644 --- a/src/db/data.h +++ b/src/db/data.h @@ -27,6 +27,7 @@ #pragma once #include +#include #include #include #include @@ -183,6 +184,7 @@ namespace db static_assert(sizeof(account) == (4 * 2) + 64 + 32 + (8 * 2) + (4 * 2), "padding in account"); void write_bytes(wire::writer&, const account&, bool show_key = false); + //! Used with quick and full sync mode struct block_info { block_id id; //!< Must be first for LMDB optimizations @@ -191,6 +193,36 @@ namespace db static_assert(sizeof(block_info) == 8 + 32, "padding in block_info"); WIRE_DECLARE_OBJECT(block_info); + struct block_difficulty + { + using unsigned_int = boost::multiprecision::uint128_t; + + std::uint64_t high; + std::uint64_t low; + + void set_difficulty(const unsigned_int& in); + unsigned_int get_difficulty() const; + }; + static_assert(sizeof(block_difficulty) == 8 * 2, "padding in block_difficulty"); + WIRE_DECLARE_OBJECT(block_difficulty); + + //! Used with untrusted daemons / full sync mode + struct block_pow + { + block_id id; + std::uint64_t timestamp; + block_difficulty cumulative_diff; + }; + static_assert(sizeof(block_pow) == 8 * 4, "padding in blow_pow"); + WIRE_DECLARE_OBJECT(block_pow); + + //! Used during sync "check-ins" if --untrusted-daemon + struct pow_sync + { + std::uint64_t timestamp; + block_difficulty cumulative_diff; + }; + //! `output`s and `spend`s are sorted by these fields to make merging easier. struct transaction_link { diff --git a/src/db/storage.cpp b/src/db/storage.cpp index bfa6b97..4a68e05 100644 --- a/src/db/storage.cpp +++ b/src/db/storage.cpp @@ -181,6 +181,7 @@ namespace db constexpr const unsigned blocks_version = 0; constexpr const unsigned by_address_version = 0; + constexpr const unsigned pows_version = 0; template int less(epee::span left, epee::span right) noexcept @@ -287,6 +288,9 @@ namespace db constexpr const lmdb::basic_table blocks{ "blocks_by_id", (MDB_CREATE | MDB_DUPSORT), MONERO_SORT_BY(block_info, id) }; + constexpr const lmdb::basic_table pows{ + "pow_by_id", (MDB_CREATE | MDB_DUPSORT), MONERO_SORT_BY(block_pow, id) + }; constexpr const lmdb::basic_table accounts{ "accounts_by_status,id", (MDB_CREATE | MDB_DUPSORT), MONERO_SORT_BY(account, id) }; @@ -348,7 +352,7 @@ namespace db } template - expect bulk_insert(MDB_cursor& cur, K const& key, epee::span values) noexcept + expect bulk_insert(MDB_cursor& cur, K const& key, epee::span values, unsigned flags = MDB_NODUPDATA) noexcept { while (!values.empty()) { @@ -359,7 +363,7 @@ namespace db }; int err = mdb_cursor_put( - &cur, &key_bytes, value_bytes, (MDB_NODUPDATA | MDB_MULTIPLE) + &cur, &key_bytes, value_bytes, (flags | MDB_MULTIPLE) ); if (err && err != MDB_KEYEXIST) return {lmdb::error(err)}; @@ -472,18 +476,29 @@ namespace db } } - template - expect get_blocks(MDB_cursor& cur, std::size_t max_internal) + void check_pow(MDB_txn& txn, MDB_dbi tbl) { - T out{}; - - max_internal = std::min(std::size_t(64), max_internal); - out.reserve(12 + max_internal); + cursor::pow cur = MONERO_UNWRAP(lmdb::open_cursor(txn, tbl)); MDB_val key = lmdb::to_val(blocks_version); - MDB_val value{}; - MONERO_LMDB_CHECK(mdb_cursor_get(&cur, &key, &value, MDB_SET)); - MONERO_LMDB_CHECK(mdb_cursor_get(&cur, &key, &value, MDB_LAST_DUP)); + int err = mdb_cursor_get(cur.get(), &key, nullptr, MDB_SET); + if (err) + { + if (err != MDB_NOTFOUND) + MONERO_THROW(lmdb::error(err), "Unable to retrieve blockchain hashes"); + + // new database + block_pow checkpoint{block_id(0), 0u, block_difficulty{0u, 1u}}; + MDB_val value = lmdb::to_val(checkpoint); + err = mdb_cursor_put(cur.get(), &key, &value, MDB_NODUPDATA); + if (err) + MONERO_THROW(lmdb::error(err), "Unable to add hash to local blockchain"); + } + } + + template + expect get_blocks_tail(T& out, MDB_cursor& cur, MDB_val value, std::size_t max_internal) + { for (unsigned i = 0; i < 10; ++i) { expect next = blocks.get_value(value); @@ -492,6 +507,7 @@ namespace db out.push_back(std::move(*next)); + MDB_val key{}; const int err = mdb_cursor_get(&cur, &key, &value, MDB_PREV_DUP); if (err) { @@ -499,7 +515,7 @@ namespace db return {lmdb::error(err)}; if (out.back().id != block_id(0)) return {lws::error::bad_blockchain}; - return out; + return success(); } } @@ -527,6 +543,73 @@ namespace db MONERO_CHECK(add_block(checkpoint)); if (out.back().id != block_id(0)) MONERO_CHECK(add_block(0)); + + return success(); + } + + template + expect get_blocks(MDB_cursor& cur, std::size_t max_internal) + { + T out{}; + + max_internal = std::min(std::size_t(64), max_internal); + out.reserve(12 + max_internal); + + MDB_val key = lmdb::to_val(blocks_version); + MDB_val value{}; + MONERO_LMDB_CHECK(mdb_cursor_get(&cur, &key, &value, MDB_SET)); + MONERO_LMDB_CHECK(mdb_cursor_get(&cur, &key, &value, MDB_LAST_DUP)); + MONERO_CHECK(get_blocks_tail(out, cur, value, max_internal)); + return out; + } + + template + expect get_blocks_from_height(MDB_cursor& cur, std::size_t max_internal, block_id last_pow) + { + T out{}; + + max_internal = std::min(std::size_t(64), max_internal); + out.reserve(12 + max_internal); + + MDB_val key = lmdb::to_val(blocks_version); + MDB_val value = lmdb::to_val(last_pow); + MONERO_LMDB_CHECK(mdb_cursor_get(&cur, &key, &value, MDB_GET_BOTH)); + MONERO_CHECK(get_blocks_tail(out, cur, value, max_internal)); + return out; + } + + template + expect get_pow_blocks(MDB_cursor& cur, std::size_t max_internal) + { + T out{}; + + max_internal = std::min(std::size_t(64), max_internal); + out.reserve(max_internal); + + MDB_val key = lmdb::to_val(pows_version); + MDB_val value{}; + MONERO_LMDB_CHECK(mdb_cursor_get(&cur, &key, &value, MDB_SET)); + MONERO_LMDB_CHECK(mdb_cursor_get(&cur, &key, &value, MDB_LAST_DUP)); + + for (unsigned i = 0; i < max_internal; ++i) + { + expect next = pows.get_value(value); + if (!next) + return next.error(); + + out.push_back(std::move(*next)); + + MDB_val key{}; + const int err = mdb_cursor_get(&cur, &key, &value, MDB_PREV_DUP); + if (err) + { + if (err != MDB_NOTFOUND) + return {lmdb::error(err)}; + if (out.back().id != block_id(0)) + return {lws::error::bad_blockchain}; + return out; + } + } return out; } @@ -566,6 +649,7 @@ namespace db struct tables_ { MDB_dbi blocks; + MDB_dbi pows; MDB_dbi accounts; MDB_dbi accounts_ba; MDB_dbi accounts_bh; @@ -588,6 +672,7 @@ namespace db assert(txn != nullptr); tables.blocks = blocks.open(*txn).value(); + tables.pows = pows.open(*txn).value(); tables.accounts = accounts.open(*txn).value(); tables.accounts_ba = accounts_by_address.open(*txn).value(); tables.accounts_bh = accounts_by_height.open(*txn).value(); @@ -619,7 +704,7 @@ namespace db MONERO_THROW(v0_spends.error(), "Error opening old spends table"); check_blockchain(*txn, tables.blocks); - + check_pow(*txn, tables.pows); MONERO_UNWRAP(this->commit(std::move(txn))); } }; @@ -641,6 +726,22 @@ namespace db return blocks.get_value(value); } + expect storage_reader::get_last_pow_block() noexcept + { + MONERO_PRECOND(txn != nullptr); + assert(db != nullptr); + + cursor::pow pow_cur; + MONERO_CHECK(check_cursor(*txn, db->tables.pows, pow_cur)); + + MDB_val key = lmdb::to_val(pows_version); + MDB_val value{}; + MONERO_LMDB_CHECK(mdb_cursor_get(pow_cur.get(), &key, &value, MDB_SET)); + MONERO_LMDB_CHECK(mdb_cursor_get(pow_cur.get(), &key, &value, MDB_LAST_DUP)); + + return pows.get_value(value); + } + expect storage_reader::get_block_hash(const block_id height) noexcept { MONERO_PRECOND(txn != nullptr); @@ -667,6 +768,88 @@ namespace db return out; } + expect> storage_reader::get_pow_sync() + { + MONERO_PRECOND(txn != nullptr); + assert(db != nullptr); + + cursor::pow pow_cur; + MONERO_CHECK(check_cursor(*txn, db->tables.pows, pow_cur)); + MONERO_CHECK(check_cursor(*txn, db->tables.blocks, curs.blocks_cur)); + + MDB_val key = lmdb::to_val(pows_version); + MDB_val value{}; + + MONERO_LMDB_CHECK(mdb_cursor_get(pow_cur.get(), &key, &value, MDB_SET)); + MONERO_LMDB_CHECK(mdb_cursor_get(pow_cur.get(), &key, &value, MDB_LAST_DUP)); + + const block_id pow_height = + MONERO_UNWRAP(pows.get_value(value)); + + auto blocks = get_blocks_from_height>(*curs.blocks_cur, 64, pow_height); + if (!blocks) + return blocks.error(); + + std::list out{}; + for (block_info const& block : *blocks) + out.push_back(block.hash); + return out; + } + + expectstorage_reader::get_pow_window(const db::block_id last) + { + MONERO_PRECOND(txn != nullptr); + assert(db != nullptr); + + pow_window out{}; + if (last == block_id(0)) + return out; + + std::uint64_t next = 0; + static_assert(1 <= DIFFICULTY_BLOCKS_COUNT, "invalid DIFFICULTY_BLOCKS_COUNT value"); + if (block_id(DIFFICULTY_BLOCKS_COUNT) < last) + next = std::uint64_t(last) - (DIFFICULTY_BLOCKS_COUNT - 1); + + cursor::pow pow_cur; + MONERO_CHECK(check_cursor(*txn, db->tables.pows, pow_cur)); + + MDB_val key = lmdb::to_val(pows_version); + MDB_val value = lmdb::to_val(next); + MONERO_LMDB_CHECK(mdb_cursor_get(pow_cur.get(), &key, &value, MDB_GET_BOTH)); + for (;;) + { + const auto insert = MONERO_UNWRAP(pows.get_value(value)); + out.pow_timestamps.push_back(insert.timestamp); + out.cumulative_diffs.push_back(insert.cumulative_diff.get_difficulty()); + + ++next; + if (next == std::uint64_t(last) + 1) + break; + + MONERO_LMDB_CHECK(mdb_cursor_get(pow_cur.get(), &key, &value, MDB_NEXT_DUP)); + } + + if (last < db::block_id(BLOCKCHAIN_TIMESTAMP_CHECK_WINDOW)) + return out; + + next = std::uint64_t(last) - (BLOCKCHAIN_TIMESTAMP_CHECK_WINDOW - 1); + key = lmdb::to_val(pows_version); + value = lmdb::to_val(next); + MONERO_LMDB_CHECK(mdb_cursor_get(pow_cur.get(), &key, &value, MDB_GET_BOTH)); + for (;;) + { + out.median_timestamps.push_back( + MONERO_UNWRAP(pows.get_value(value)) + ); + + ++next; + if (next == std::uint64_t(last) + 1) + break; + MONERO_LMDB_CHECK(mdb_cursor_get(pow_cur.get(), &key, &value, MDB_NEXT_DUP)); + } + return out; + } + expect> storage_reader::get_accounts(cursor::accounts cur) noexcept { @@ -994,6 +1177,7 @@ namespace db return std::make_pair(address_string(src.address), src.lookup); }; + cursor::pow pow_cur; cursor::accounts accounts_cur; cursor::outputs outputs_cur; cursor::spends spends_cur; @@ -1005,6 +1189,7 @@ namespace db cursor::subaddress_indexes indexes_cur; MONERO_CHECK(check_cursor(*txn, db->tables.blocks, curs.blocks_cur)); + MONERO_CHECK(check_cursor(*txn, db->tables.pows, pow_cur)); MONERO_CHECK(check_cursor(*txn, db->tables.accounts, accounts_cur)); MONERO_CHECK(check_cursor(*txn, db->tables.accounts_ba, curs.accounts_ba_cur)); MONERO_CHECK(check_cursor(*txn, db->tables.accounts_bh, curs.accounts_bh_cur)); @@ -1022,6 +1207,11 @@ namespace db if (!blocks_partial) return blocks_partial.error(); + auto pow_partial = + get_pow_blocks>(*pow_cur, 12); + if (!pow_partial) + return pow_partial.error(); + auto accounts_stream = accounts.get_key_stream(std::move(accounts_cur)); if (!accounts_stream) return accounts_stream.error(); @@ -1075,6 +1265,7 @@ namespace db wire::json_stream_writer json_stream{out}; wire::object(json_stream, wire::field(blocks.name, wire::array(reverse(*blocks_partial))), + wire::field(pows.name, wire::array(reverse(*pow_partial))), wire::field(accounts.name, wire::as_object(accounts_stream->make_range(), wire::enum_as_string, toggle_keys_filter)), wire::field(accounts_by_address.name, wire::as_object(transform(accounts_ba_stream->make_range(), address_as_key))), wire::field(accounts_by_height.name, wire::array(accounts_bh_stream->make_range())), @@ -1154,6 +1345,16 @@ namespace db return instance.data; } + block_info storage::get_last_checkpoint() + { + const auto& checkpoints = get_checkpoints().get_points(); + if (checkpoints.empty()) + MONERO_THROW(error::bad_blockchain, "Checkpoints invalid"); + + const auto last = checkpoints.rbegin(); + return block_info{block_id(last->first), last->second}; + } + storage storage::open(const char* path, unsigned create_queue_max) { return { @@ -1364,6 +1565,27 @@ namespace db err = mdb_cursor_get(&cur, &key, &value, MDB_NEXT_DUP); } while (err == 0); + // rollback pow + { + cursor::pow pow_cur; + MONERO_CHECK(check_cursor(txn, tables.pows, pow_cur)); + + MDB_val key = lmdb::to_val(pows_version); + MDB_val value = lmdb::to_val(height); + int err = mdb_cursor_get(pow_cur.get(), &key, &value, MDB_GET_BOTH); + for (;;) + { + if (err) + { + if (err == MDB_NOTFOUND) + break; + return {lmdb::error(err)}; + } + MONERO_LMDB_CHECK(mdb_cursor_del(pow_cur.get(), 0)); + err = mdb_cursor_get(pow_cur.get(), &key, &value, MDB_NEXT_DUP); + } + } + if (err != MDB_NOTFOUND) return {lmdb::error(err)}; @@ -1382,7 +1604,8 @@ namespace db { if (current == chain.end() || hashes.size() == hashes.capacity()) { - MONERO_CHECK(bulk_insert(cur, blocks_version, epee::to_span(hashes))); + // always overwrite, for pow case (where pows is catching up to blocks) + MONERO_CHECK(bulk_insert(cur, blocks_version, epee::to_span(hashes), 0)); if (current == chain.end()) return success(); hashes.clear(); @@ -1392,6 +1615,29 @@ namespace db ++height; } } + + template + expect append_pow(MDB_cursor& cur, db::block_id first, T const& chain) + { + std::uint64_t height = std::uint64_t(first); + boost::container::static_vector pows{}; + static_assert(sizeof(pows) <= 1024, "using more stack space than expected"); + + for (auto current = chain.begin() ;; ++current) + { + if (current == chain.end() || pows.size() == pows.capacity()) + { + MONERO_CHECK(bulk_insert(cur, pows_version, epee::to_span(pows))); + if (current == chain.end()) + return success(); + pows.clear(); + } + + pows.push_back(block_pow{db::block_id(height), current->timestamp, current->cumulative_diff}); + ++height; + } + } + } // anonymous expect storage::rollback(block_id height) @@ -1467,6 +1713,91 @@ namespace db }); } + expect storage::sync_pow(block_id height, epee::span hashes, epee::span pow) + { + MONERO_PRECOND(!hashes.empty()); + MONERO_PRECOND(hashes.size() == pow.size()); + MONERO_PRECOND(db != nullptr); + + return db->try_write([this, height, hashes, pow] (MDB_txn& txn) -> expect + { + cursor::blocks blocks_cur; + MONERO_CHECK(check_cursor(txn, this->db->tables.blocks, blocks_cur)); + + expect hash = do_get_block_hash(*blocks_cur, height); + if (!hash) + return hash.error(); + + // the first entry should always match on in the DB + if (*hash != *(hashes.begin())) + return {lws::error::bad_blockchain}; + + MDB_val key{}; + MDB_val value{}; + + std::uint64_t current = std::uint64_t(height) + 1; + auto first = hashes.begin(); + auto chain = boost::make_iterator_range(++first, hashes.end()); + const auto& checkpoints = get_checkpoints(); + for ( ; !chain.empty(); chain.advance_begin(1), ++current) + { + // if while syncing from beginning, a checkpoint was missed + const auto checkpoint = checkpoints.get_points().find(current); + if (checkpoint != checkpoints.get_points().end() && checkpoint->second != chain.front()) + { + MERROR("Missed a checkpoint during sync_pow"); + return {error::bad_blockchain}; + } + + const int err = mdb_cursor_get(blocks_cur.get(), &key, &value, MDB_NEXT_DUP); + if (err == MDB_NOTFOUND) + break; + if (err) + return {lmdb::error(err)}; + + auto full_value = blocks.get_value(value); + if (!full_value) + return full_value.error(); + if (full_value->id != block_id(current)) // hit a checkpoint or other block that is ahead of pow + break; + + if (full_value->hash != chain.front()) + { + if (current <= checkpoints.get_max_height()) + { + MERROR("Attempting rollback past last checkpoint; invalid daemon chain response"); + return {lws::error::bad_blockchain}; + } + MONERO_CHECK(rollback_chain(this->db->tables, txn, *blocks_cur, db::block_id(current))); + break; + } + } + + // scan checkpoints, this is hardened mode! + { + std::uint64_t current_copy = current; + for (const auto& current_hash : chain) + { + // if while syncing from beginning, a checkpoint was missed + const auto checkpoint = checkpoints.get_points().find(current_copy); + if (checkpoint != checkpoints.get_points().end() && checkpoint->second != current_hash) + { + MERROR("Missed a checkpoint during sync_pow"); + return {error::bad_blockchain}; + } + ++current_copy; + } + } + + auto first_pow = pow.begin() + std::ptrdiff_t(chain.begin() - hashes.begin()); + + cursor::pow pow_cur; + MONERO_CHECK(check_cursor(txn, this->db->tables.pows, pow_cur)); + MONERO_CHECK(append_block_hashes(*blocks_cur, db::block_id(current), chain)); + return append_pow(*pow_cur, db::block_id(current), boost::make_iterator_range(first_pow, pow.end())); + }); + } + namespace { expect get_account_time() noexcept @@ -2233,16 +2564,19 @@ namespace db } } // anonymous - expect>> storage::update(block_id height, epee::span chain, epee::span users) + expect>> storage::update(block_id height, epee::span chain, epee::span users, epee::span pow) { if (users.empty() && chain.empty()) return {std::make_pair(0, std::vector{})}; MONERO_PRECOND(!chain.empty()); MONERO_PRECOND(db != nullptr); + if (!pow.empty()) + MONERO_PRECOND(chain.size() == pow.size()); - return db->try_write([this, height, chain, users] (MDB_txn& txn) -> expect>> + return db->try_write([this, height, chain, users, pow] (MDB_txn& txn) -> expect>> { epee::span chain_copy{chain}; + epee::span pow_copy{pow}; const std::uint64_t last_update = lmdb::to_native(height) + chain.size() - 1; const std::uint64_t first_new = lmdb::to_native(height) + 1; @@ -2252,7 +2586,9 @@ namespace db if (get_checkpoints().get_max_height() <= last_update) { cursor::blocks blocks_cur; + cursor::pow pow_cur; MONERO_CHECK(check_cursor(txn, this->db->tables.blocks, blocks_cur)); + MONERO_CHECK(check_cursor(txn, this->db->tables.pows, pow_cur)); MDB_val key = lmdb::to_val(blocks_version); MDB_val value; @@ -2276,6 +2612,40 @@ namespace db *blocks_cur, block_id(lmdb::to_native(height) + offset + 1), chain_copy ) ); + + if (!pow_copy.empty()) + { + pow_copy.remove_prefix(offset + 1); + MONERO_CHECK( + append_pow(*pow_cur, block_id(lmdb::to_native(height) + offset + 1), pow_copy) + ); + } + } + else // perform chain/pow hardening via checkpoints (if available) + { + cursor::blocks blocks_cur; + MONERO_CHECK(check_cursor(txn, this->db->tables.blocks, blocks_cur)); + + MDB_val key = lmdb::to_val(blocks_version); + MDB_val value = lmdb::to_val(last_update); + int err = mdb_cursor_get(blocks_cur.get(), &key, &value, MDB_GET_BOTH); + + // verify last block hash if available. If not availble, --untrusted-daemon was not used + if (err) + { + if (err != MDB_NOTFOUND) + return {lmdb::error(err)}; + } + else + { + const auto cur_block = blocks.get_value(value); + if (!cur_block) + return cur_block.error(); + // If a reorg past a checkpoint is being attempted + if (chain[chain.size() - 1] != cur_block->hash) + return {error::bad_blockchain}; + + } } cursor::accounts accounts_cur; diff --git a/src/db/storage.h b/src/db/storage.h index d958348..a0a6937 100644 --- a/src/db/storage.h +++ b/src/db/storage.h @@ -57,6 +57,7 @@ namespace db MONERO_CURSOR(subaddress_indexes); MONERO_CURSOR(blocks); + MONERO_CURSOR(pow); MONERO_CURSOR(accounts_by_address); MONERO_CURSOR(accounts_by_height); @@ -72,6 +73,13 @@ namespace db cursor::accounts_by_height accounts_bh_cur; }; + struct pow_window + { + std::vector pow_timestamps; //!< for pow calculation + std::vector cumulative_diffs; + std::vector median_timestamps; //!< for timestamp check + }; + //! Wrapper for LMDB read access to on-disk storage of light-weight server data. class storage_reader { @@ -95,12 +103,21 @@ namespace db //! \return Last known block. expect get_last_block() noexcept; + //! \return Last known pow block. + expect get_last_pow_block() noexcept; + //! \return "Our" block hash at `height`. expect get_block_hash(const block_id height) noexcept; //! \return List for `GetHashesFast` to sync blockchain with daemon. expect> get_chain_sync(); + //! \return List for GetBlocksFast` to sync blockchain+pow with daemon + expect> get_pow_sync(); + + //! \return Objects for use with cryptonote::next_difficulty and median timestamp check + expect get_pow_window(block_id last); + //! \return All registered `account`s. expect> get_accounts(cursor::accounts cur = nullptr) noexcept; @@ -172,6 +189,9 @@ namespace db //! \return A single instance of compiled-in checkpoints for lws static cryptonote::checkpoints const& get_checkpoints(); + //! \return Last hard-coded block checkpoint + static block_info get_last_checkpoint(); + /*! Open a light_wallet_server LDMB database. @@ -210,6 +230,8 @@ namespace db */ expect sync_chain(block_id height, epee::span hashes); + expect sync_pow(block_id height, epee::span hashes, epee::span pow); + //! Bump the last access time of `address` to the current time. expect update_access_time(account_address const& address) noexcept; @@ -255,7 +277,7 @@ namespace db \return True iff LMDB successfully committed the update. */ expect>> - update(block_id height, epee::span chain, epee::span accts); + update(block_id height, epee::span chain, epee::span accts, epee::span pow); /*! Adds subaddresses to an account. Upon success, an account will diff --git a/src/rpc/daemon_zmq.cpp b/src/rpc/daemon_zmq.cpp index 1e62d5f..681d66f 100644 --- a/src/rpc/daemon_zmq.cpp +++ b/src/rpc/daemon_zmq.cpp @@ -58,20 +58,128 @@ namespace rct wire::object(source, WIRE_FIELD(mask), WIRE_FIELD(amount)); } + static void read_bytes(wire::json_reader& source, clsag& self) + { + wire::object(source, WIRE_FIELD(s), WIRE_FIELD(c1), WIRE_FIELD(D)); + } + + static void read_bytes(wire::json_reader& source, mgSig& self) + { + wire::object(source, WIRE_FIELD(ss), WIRE_FIELD(cc)); + } + + static void read_bytes(wire::json_reader& source, BulletproofPlus& self) + { + wire::object(source, + WIRE_FIELD(V), + WIRE_FIELD(A), + WIRE_FIELD(A1), + WIRE_FIELD(B), + WIRE_FIELD(r1), + WIRE_FIELD(s1), + WIRE_FIELD(d1), + WIRE_FIELD(L), + WIRE_FIELD(R) + ); + } + + static void read_bytes(wire::json_reader& source, Bulletproof& self) + { + wire::object(source, + WIRE_FIELD(V), + WIRE_FIELD(A), + WIRE_FIELD(S), + WIRE_FIELD(T1), + WIRE_FIELD(T2), + WIRE_FIELD(taux), + WIRE_FIELD(mu), + WIRE_FIELD(L), + WIRE_FIELD(R), + WIRE_FIELD(a), + WIRE_FIELD(b), + WIRE_FIELD(t) + ); + } + + static void read_bytes(wire::json_reader& source, boroSig& self) + { + std::vector s0; + std::vector s1; + s0.reserve(64); + s1.reserve(64); + wire::object(source, wire::field("s0", std::ref(s0)), wire::field("s1", std::ref(s1))); + + if (s0.size() != 64 || s1.size() != 64) + WIRE_DLOG_THROW(wire::error::schema::array, "Expected s0 and s1 to have 64 elements"); + + for (std::size_t i = 0; i < 64; ++i) + self.s0[i] = s0[i]; + for (std::size_t i = 0; i < 64; ++i) + self.s1[i] = s1[i]; + } + + static void read_bytes(wire::json_reader& source, rangeSig& self) + { + std::vector keys{}; + keys.reserve(64); + + wire::object(source, WIRE_FIELD(asig), wire::field("Ci", std::ref(keys))); + if (keys.size() != 64) + WIRE_DLOG_THROW(wire::error::schema::array, "Expected 64 eleents in Ci"); + for (std::size_t i = 0; i < 64; ++i) + { + self.Ci[i] = keys[i]; + } + } + + namespace + { + struct prunable_helper + { + rctSigPrunable prunable; + rct::keyV pseudo_outs; + }; + + void read_bytes(wire::json_reader& source, prunable_helper& self) + { + wire::object(source, + wire::field("range_proofs", std::ref(self.prunable.rangeSigs)), + wire::field("bulletproofs", std::ref(self.prunable.bulletproofs)), + wire::field("bulletproofs_plus", std::ref(self.prunable.bulletproofs_plus)), + wire::field("mlsags", std::ref(self.prunable.MGs)), + wire::field("clsags", std::ref(self.prunable.CLSAGs)), + wire::field("pseudo_outs", std::ref(self.pseudo_outs)) + ); + + const bool pruned = + self.prunable.rangeSigs.empty() && + self.prunable.bulletproofs.empty() && + self.prunable.bulletproofs_plus.empty() && + self.prunable.MGs.empty() && + self.prunable.CLSAGs.empty() && + self.pseudo_outs.empty(); + + if (pruned) + WIRE_DLOG_THROW(wire::error::schema::array, "Expected at least one prunable field"); + } + } // anonymous + static void read_bytes(wire::json_reader& source, rctSig& self) { boost::optional> ecdhInfo; boost::optional outPk; boost::optional txnFee; - + boost::optional prunable; self.outPk.reserve(default_inputs); wire::object(source, WIRE_FIELD(type), wire::optional_field("encrypted", std::ref(ecdhInfo)), wire::optional_field("commitments", std::ref(outPk)), - wire::optional_field("fee", std::ref(txnFee)) + wire::optional_field("fee", std::ref(txnFee)), + wire::optional_field("prunable", std::ref(prunable)) ); + self.txnFee = 0; if (self.type != RCTTypeNull) { if (!ecdhInfo || !outPk || !txnFee) @@ -82,6 +190,12 @@ namespace rct } else if (ecdhInfo || outPk || txnFee) WIRE_DLOG_THROW(wire::error::schema::invalid_key, "Did not expected `encrypted`, `commitments`, or `fee`"); + + if (prunable) + { + self.p = std::move(prunable->prunable); + self.get_pseudo_outs() = std::move(prunable->pseudo_outs); + } } } // rct @@ -186,6 +300,12 @@ namespace cryptonote } // rpc } // cryptonote +void lws::rpc::read_bytes(wire::json_reader& source, get_hashes_fast_response& self) +{ + self.hashes.reserve(default_blocks_fetched); + wire::object(source, WIRE_FIELD(hashes), WIRE_FIELD(start_height), WIRE_FIELD(current_height)); +} + void lws::rpc::read_bytes(wire::json_reader& source, get_blocks_fast_response& self) { self.blocks.reserve(default_blocks_fetched); diff --git a/src/rpc/daemon_zmq.h b/src/rpc/daemon_zmq.h index 3bd3896..dc1f62d 100644 --- a/src/rpc/daemon_zmq.h +++ b/src/rpc/daemon_zmq.h @@ -75,6 +75,26 @@ namespace rpc using response = get_blocks_fast_response; }; void read_bytes(wire::json_reader&, get_blocks_fast_response&); + + struct get_hashes_fast_request + { + get_hashes_fast_request() = delete; + std::vector known_hashes; + std::uint64_t start_height; + }; + struct get_hashes_fast_response + { + get_hashes_fast_response() = delete; + std::vector hashes; + std::uint64_t start_height; + std::uint64_t current_height; + }; + struct get_hashes_fast + { + using request = get_hashes_fast_request; + using response = get_hashes_fast_response; + }; + void read_bytes(wire::json_reader&, get_hashes_fast_response&); struct get_transaction_pool_request { diff --git a/src/scanner.cpp b/src/scanner.cpp index 8aa7fe8..7696770 100644 --- a/src/scanner.cpp +++ b/src/scanner.cpp @@ -1,4 +1,4 @@ - // Copyright (c) 2018-2023, The Monero Project +// Copyright (c) 2018-2023, The Monero Project // All rights reserved. // // Redistribution and use in source and binary forms, with or without modification, are @@ -42,13 +42,16 @@ #include #include "common/error.h" // monero/src +#include "config.h" #include "crypto/crypto.h" // monero/src #include "crypto/wallet/crypto.h" // monero/src #include "cryptonote_basic/cryptonote_basic.h" // monero/src #include "cryptonote_basic/cryptonote_format_utils.h" // monero/src #include "db/account.h" #include "db/data.h" +#include "cryptonote_basic/difficulty.h" // monero/src #include "error.h" +#include "hardforks/hardforks.h" // monero/src #include "misc_log_ex.h" // monero/contrib/epee/include #include "net/net_parse_helpers.h" #include "net/net_ssl.h" // monero/contrib/epee/include @@ -57,6 +60,7 @@ #include "rpc/json.h" #include "rpc/message_data_structs.h" // monero/src #include "rpc/webhook.h" +#include "util/blocks.h" #include "util/source_location.h" #include "util/transactions.h" @@ -98,6 +102,7 @@ namespace lws { net::ssl_verification_t webhook_verify; bool enable_subaddresses; + bool untrusted_daemon; }; struct thread_data @@ -174,6 +179,43 @@ namespace lws rpc::send_webhook(client, events, "json-full-payment_hook:", "msgpack-full-payment_hook:", std::chrono::seconds{5}, verify_mode); } + std::size_t get_target_time(db::block_id height) + { + const hardfork_t* fork = nullptr; + switch (config::network) + { + case cryptonote::network_type::MAINNET: + if (num_mainnet_hard_forks < 2) + MONERO_THROW(error::bad_blockchain, "expected more mainnet forks"); + fork = mainnet_hard_forks; + break; + case cryptonote::network_type::TESTNET: + if (num_testnet_hard_forks < 2) + MONERO_THROW(error::bad_blockchain, "expected more testnet forks"); + fork = testnet_hard_forks; + break; + case cryptonote::network_type::STAGENET: + if (num_stagenet_hard_forks < 2) + MONERO_THROW(error::bad_blockchain, "expected more stagenet forks"); + fork = stagenet_hard_forks; + break; + default: + MONERO_THROW(error::bad_blockchain, "chain type not support with full sync"); + } + // this is hardfork version 2 + return height < db::block_id(fork[1].height) ? + DIFFICULTY_TARGET_V1 : DIFFICULTY_TARGET_V2; + } + + //! For difficulty vectors only + template + void update_window(T& vec) + { + // should only have one to pop each time + while (DIFFICULTY_BLOCKS_COUNT < vec.size()) + vec.erase(vec.begin()); + }; + struct by_height { bool operator()(account const& left, account const& right) const noexcept @@ -572,7 +614,7 @@ namespace lws MINFO("Updated exchange rates: " << *(*new_rates)); } - void scan_loop(thread_sync& self, std::shared_ptr data) noexcept + void scan_loop(thread_sync& self, std::shared_ptr data, const bool untrusted_daemon, const bool leader_thread) noexcept { try { @@ -602,17 +644,22 @@ namespace lws cryptonote::rpc::GetBlocksFast::Request req{}; req.start_height = std::uint64_t(users.begin()->scan_height()); req.start_height = std::max(std::uint64_t(1), req.start_height); - req.prune = true; + req.prune = !untrusted_daemon; epee::byte_slice block_request = rpc::client::make_message("get_blocks_fast", req); if (!send(client, block_request.clone())) return; std::vector blockchain{}; + std::vector new_pow{}; + db::pow_window pow_window{}; + const db::block_info last_checkpoint = db::storage::get_last_checkpoint(); + const db::block_id last_pow = MONERO_UNWRAP(MONERO_UNWRAP(disk.start_read()).get_last_pow_block()).id; while (!self.update && scanner::is_running()) { blockchain.clear(); + new_pow.clear(); auto resp = client.get_message(block_rpc_timeout); if (!resp) @@ -691,6 +738,8 @@ namespace lws throw std::runtime_error{"Bad daemon response - need same number of blocks and indices"}; blockchain.push_back(cryptonote::get_block_hash(fetched->blocks.front().block)); + if (untrusted_daemon) + new_pow.push_back(db::pow_sync{fetched->blocks.front().block.timestamp}); auto blocks = epee::to_span(fetched->blocks); auto indices = epee::to_span(fetched->output_indices); @@ -704,7 +753,16 @@ namespace lws else fetched->start_height = 0; + if (untrusted_daemon) + { + pow_window = MONERO_UNWRAP( + MONERO_UNWRAP(disk.start_read()).get_pow_window(db::block_id(fetched->start_height)) + ); + } + subaddress_reader reader{disk, opts.enable_subaddresses}; + db::block_difficulty::unsigned_int diff{}; + const db::block_id initial_height = db::block_id(fetched->start_height); for (auto block_data : boost::combine(blocks, indices)) { ++(fetched->start_height); @@ -733,12 +791,48 @@ namespace lws reader ); + if (untrusted_daemon) + { + if (block.prev_id != blockchain.back()) + MONERO_THROW(error::bad_blockchain, "A blocks prev_id does not match"); + + update_window(pow_window.pow_timestamps); + update_window(pow_window.cumulative_diffs); + + while (BLOCKCHAIN_TIMESTAMP_CHECK_WINDOW < pow_window.median_timestamps.size()) + pow_window.median_timestamps.erase(pow_window.median_timestamps.begin()); + + // longhash takes a while, check is_running + if (!scanner::is_running()) + return; + + diff = cryptonote::next_difficulty(pow_window.pow_timestamps, pow_window.cumulative_diffs, get_target_time(db::block_id(fetched->start_height))); + + // skip POW hashing if done previously + if (last_pow < db::block_id(fetched->start_height)) + { + if (!verify_timestamp(block.timestamp, pow_window.median_timestamps)) + MONERO_THROW(error::bad_blockchain, "Block failed timestamp check - possible chain forgery"); + + const crypto::hash pow = + get_block_longhash(get_block_hashing_blob(block), db::block_id(fetched->start_height), block.major_version, disk, initial_height, epee::to_span(blockchain)); + if (!cryptonote::check_hash(pow, diff)) + MONERO_THROW(error::bad_blockchain, "Block had too low difficulty"); + } + } + indices.remove_prefix(1); if (txes.size() != indices.size()) throw std::runtime_error{"Bad daemon respnse - need same number of txes and indices"}; for (auto tx_data : boost::combine(block.tx_hashes, txes, indices)) { + if (untrusted_daemon) + { + if (cryptonote::get_transaction_hash(boost::get<1>(tx_data)) != boost::get<0>(tx_data)) + MONERO_THROW(error::bad_blockchain, "Hash of transaction does not match hash in block"); + } + scan_transaction( epee::to_mut_span(users), db::block_id(fetched->start_height), @@ -750,12 +844,24 @@ namespace lws ); } + if (untrusted_daemon) + { + const auto last_difficulty = + pow_window.cumulative_diffs.empty() ? + db::block_difficulty::unsigned_int(0) : pow_window.cumulative_diffs.back(); + + pow_window.pow_timestamps.push_back(block.timestamp); + pow_window.median_timestamps.push_back(block.timestamp); + pow_window.cumulative_diffs.push_back(diff + last_difficulty); + new_pow.push_back(db::pow_sync{block.timestamp}); + new_pow.back().cumulative_diff.set_difficulty(pow_window.cumulative_diffs.back()); + } blockchain.push_back(cryptonote::get_block_hash(block)); } // for each block reader.reader = std::error_code{common_error::kInvalidArgument}; // cleanup reader before next write auto updated = disk.update( - users.front().scan_height(), epee::to_span(blockchain), epee::to_span(users) + users.front().scan_height(), epee::to_span(blockchain), epee::to_span(users), epee::to_span(new_pow) ); if (!updated) { @@ -767,6 +873,11 @@ namespace lws MONERO_THROW(updated.error(), "Failed to update accounts on disk"); } + if (untrusted_daemon && leader_thread && fetched->start_height % 4 == 0 && last_pow < db::block_id(fetched->start_height)) + { + MINFO("On chain with hash " << blockchain.back() << " and difficulty " << diff << " at height " << fetched->start_height); + } + MINFO("Processed " << blocks.size() << " block(s) against " << users.size() << " account(s)"); send_payment_hook(client, epee::to_span(updated->second), opts.webhook_verify); if (updated->first != users.size()) @@ -844,6 +955,7 @@ namespace lws MINFO("Starting scan loops on " << std::min(thread_count, users.size()) << " thread(s) with " << users.size() << " account(s)"); + bool leader_thread = true; while (!users.empty() && --thread_count) { const std::size_t per_thread = std::max(std::size_t(1), users.size() / (thread_count + 1)); @@ -859,7 +971,8 @@ namespace lws auto data = std::make_shared( std::move(client), disk.clone(), std::move(thread_users), opts ); - threads.emplace_back(attrs, std::bind(&scan_loop, std::ref(self), std::move(data))); + threads.emplace_back(attrs, std::bind(&scan_loop, std::ref(self), std::move(data), opts.untrusted_daemon, leader_thread)); + leader_thread = false; } if (!users.empty()) @@ -870,7 +983,7 @@ namespace lws auto data = std::make_shared( std::move(client), disk.clone(), std::move(users), opts ); - threads.emplace_back(attrs, std::bind(&scan_loop, std::ref(self), std::move(data))); + threads.emplace_back(attrs, std::bind(&scan_loop, std::ref(self), std::move(data), opts.untrusted_daemon, false /*leader thread*/)); } auto last_check = std::chrono::steady_clock::now(); @@ -931,36 +1044,20 @@ namespace lws accounts_cur = current_users.give_cursor(); } // while scanning } - } // anonymous - expect scanner::sync(db::storage disk, rpc::client client) - { - using get_hashes = cryptonote::rpc::GetHashesFast; - - MINFO("Starting blockchain sync with daemon"); - - get_hashes::Request req{}; - req.start_height = 0; + expect> get_chain_sync(expect reader) { - auto reader = disk.start_read(); if (!reader) return reader.error(); - - auto chain = reader->get_chain_sync(); - if (!chain) - return chain.error(); - - req.known_hashes = std::move(*chain); + return reader->get_chain_sync(); } - for (;;) + template + expect fetch_chain(rpc::client& client, const char* endpoint, const Q& req) { - if (req.known_hashes.empty()) - return {lws::error::bad_blockchain}; - expect sent{lws::error::daemon_timeout}; - epee::byte_slice msg = rpc::client::make_message("get_hashes_fast", req); + epee::byte_slice msg = rpc::client::make_message(endpoint, req); auto start = std::chrono::steady_clock::now(); while (!(sent = client.send(std::move(msg), std::chrono::seconds{1}))) @@ -975,10 +1072,10 @@ namespace lws return sent.error(); } - expect resp{lws::error::daemon_timeout}; + expect resp{lws::error::daemon_timeout}; start = std::chrono::steady_clock::now(); - while (!(resp = client.receive(std::chrono::seconds{1}, MLWS_CURRENT_LOCATION))) + while (!(resp = client.get_message(std::chrono::seconds{1}))) { if (!scanner::is_running()) return {lws::error::signal_abort_process}; @@ -989,29 +1086,180 @@ namespace lws if (!resp.matches(std::errc::timed_out)) return resp.error(); } - - // - // Exit loop if it appears we have synced to top of chain - // - if (resp->hashes.size() <= 1 || resp->hashes.back() == req.known_hashes.front()) - return {std::move(client)}; - - MONERO_CHECK(disk.sync_chain(db::block_id(resp->start_height), epee::to_span(resp->hashes))); - - req.known_hashes.erase(req.known_hashes.begin(), --(req.known_hashes.end())); - for (std::size_t num = 0; num < 10; ++num) - { - if (resp->hashes.empty()) - break; - - req.known_hashes.insert(--(req.known_hashes.end()), resp->hashes.back()); - } + return rpc::parse_json_response(std::move(*resp)); } - return {std::move(client)}; + // does not validate blockchain hashes + expect sync_quick(db::storage disk, rpc::client client) + { + MINFO("Starting blockchain sync with daemon"); + + cryptonote::rpc::GetHashesFast::Request req{}; + req.start_height = 0; + req.known_hashes = MONERO_UNWRAP(MONERO_UNWRAP(disk.start_read()).get_chain_sync()); + + for (;;) + { + if (req.known_hashes.empty()) + return {lws::error::bad_blockchain}; + + auto resp = fetch_chain(client, "get_hashes_fast", req); + if (!resp) + return resp.error(); + + // + // exit loop if it appears we have synced to top of chain + // + if (resp->hashes.size() <= 1 || resp->hashes.back() == req.known_hashes.front()) + return {std::move(client)}; + + MONERO_CHECK(disk.sync_chain(db::block_id(resp->start_height), epee::to_span(resp->hashes))); + + req.known_hashes.erase(req.known_hashes.begin(), --(req.known_hashes.end())); + for (std::size_t num = 0; num < 10; ++num) + { + if (resp->hashes.empty()) + break; + + req.known_hashes.insert(--(req.known_hashes.end()), resp->hashes.back()); + } + } + + return {std::move(client)}; + } + + // validates blockchain hashes + expect sync_full(db::storage disk, rpc::client client) + { + MINFO("Starting blockchain sync with daemon"); + + cryptonote::rpc::GetBlocksFast::Request req{}; + req.start_height = 0; + req.block_ids = MONERO_UNWRAP(MONERO_UNWRAP(disk.start_read()).get_pow_sync()); + req.prune = true; + + std::vector new_hashes{}; + std::vector new_pow{}; + for (;;) + { + if (req.block_ids.empty()) + return {lws::error::bad_blockchain}; + + auto resp = fetch_chain(client, "get_blocks_fast", req); + if (!resp) + return resp.error(); + + if (resp->blocks.empty()) + return {error::bad_daemon_response}; + + crypto::hash hash{}; + if (!cryptonote::get_block_hash(resp->blocks.front().block, hash)) + return {lws::error::bad_blockchain}; + + // + // exit loop if it appears we have synced to top of chain + // + const db::block_info last_checkpoint = db::storage::get_last_checkpoint(); + if (resp->blocks.size() <= 1) + { + // error if not past last checkpoint + const auto expected_hash = + MONERO_UNWRAP(disk.start_read()).get_block_hash(db::block_id(resp->start_height)); + if (!expected_hash || *expected_hash != hash || db::block_id(resp->start_height) < last_checkpoint.id) + return {error::bad_daemon_response}; + return {std::move(client)}; + } + + // genesis block must be present as last entry + req.block_ids.erase(req.block_ids.begin(), --(req.block_ids.end())); + + auto pow_window = + MONERO_UNWRAP(MONERO_UNWRAP(disk.start_read()).get_pow_window(db::block_id(resp->start_height))); + + // overlap check performed in db::storage::pow_sync + new_hashes.clear(); + new_pow.clear(); + new_hashes.reserve(resp->blocks.size()); + new_pow.reserve(resp->blocks.size()); + new_hashes.push_back(hash); + new_pow.push_back(db::pow_sync{resp->blocks.front().block.timestamp}); + + // skip overlap block + db::block_difficulty::unsigned_int diff = 0; + for (std::size_t i = 1; i < resp->blocks.size(); ++i) + { + const auto& block = resp->blocks[i].block; + const db::block_id height = db::block_id(resp->start_height + i); + + // important check, ensure we haven't deviated from chain + if (block.prev_id != hash) + return {lws::error::bad_blockchain}; + + // compute block id hash + if (!cryptonote::get_block_hash(block, hash)) + return {lws::error::bad_blockchain}; + + req.block_ids.push_front(hash); + update_window(pow_window.pow_timestamps); + update_window(pow_window.cumulative_diffs); + + while (BLOCKCHAIN_TIMESTAMP_CHECK_WINDOW < pow_window.median_timestamps.size()) + pow_window.median_timestamps.erase(pow_window.median_timestamps.begin()); + + // longhash takes a while, check is_running + if (!scanner::is_running()) + return {error::signal_abort_process}; + + diff = cryptonote::next_difficulty(pow_window.pow_timestamps, pow_window.cumulative_diffs, get_target_time(height)); + + // skip POW hashing when sync is within checkpoint + // storage::sync_pow(...) currently verifies checkpoint hashes + if (last_checkpoint.id < height) + { + if (!verify_timestamp(block.timestamp, pow_window.median_timestamps)) + { + MERROR("Block failed timestamp check - possible chain forgery"); + return {error::bad_blockchain}; + } + const crypto::hash pow = + get_block_longhash(get_block_hashing_blob(block), height, block.major_version, disk, db::block_id(resp->start_height), epee::to_span(new_hashes)); + + if (!cryptonote::check_hash(pow, diff)) + { + MERROR("Block " << std::uint64_t(height) << "had too low difficulty"); + return {error::bad_blockchain}; + } + } + + const auto last_difficulty = + pow_window.cumulative_diffs.empty() ? + db::block_difficulty::unsigned_int(0) : pow_window.cumulative_diffs.back(); + + pow_window.pow_timestamps.push_back(block.timestamp); + pow_window.median_timestamps.push_back(block.timestamp); + pow_window.cumulative_diffs.push_back(diff + last_difficulty); + new_hashes.push_back(hash); + new_pow.push_back(db::pow_sync{block.timestamp}); + new_pow.back().cumulative_diff.set_difficulty(pow_window.cumulative_diffs.back()); + } // for every tx in block + + MONERO_CHECK(disk.sync_pow(db::block_id(resp->start_height), epee::to_span(new_hashes), epee::to_span(new_pow))); + MINFO("Verified up to block " << (resp->start_height + new_hashes.size() - 1) << " with hash " << hash << " and difficulty " << diff); + + } // for until sync + + return {std::move(client)}; + } + } // anonymous + + expect scanner::sync(db::storage disk, rpc::client client, const bool untrusted_daemon) + { + if (untrusted_daemon) + return sync_full(std::move(disk), std::move(client)); + return sync_quick(std::move(disk), std::move(client)); } - void scanner::run(db::storage disk, rpc::context ctx, std::size_t thread_count, const epee::net_utils::ssl_verification_t webhook_verify, const bool enable_subaddresses) + void scanner::run(db::storage disk, rpc::context ctx, std::size_t thread_count, const epee::net_utils::ssl_verification_t webhook_verify, const bool enable_subaddresses, const bool untrusted_daemon) { thread_count = std::max(std::size_t(1), thread_count); @@ -1065,7 +1313,7 @@ namespace lws checked_wait(account_poll_interval - (std::chrono::steady_clock::now() - last)); } else - check_loop(disk.clone(), ctx, thread_count, std::move(users), std::move(active), options{webhook_verify, enable_subaddresses}); + check_loop(disk.clone(), ctx, thread_count, std::move(users), std::move(active), options{webhook_verify, enable_subaddresses, untrusted_daemon}); if (!scanner::is_running()) return; @@ -1073,7 +1321,7 @@ namespace lws if (!client) client = MONERO_UNWRAP(ctx.connect()); - expect synced = sync(disk.clone(), std::move(client)); + expect synced = sync(disk.clone(), std::move(client), untrusted_daemon); if (!synced) { if (!synced.matches(std::errc::timed_out)) diff --git a/src/scanner.h b/src/scanner.h index d7d8db3..0694f9f 100644 --- a/src/scanner.h +++ b/src/scanner.h @@ -46,10 +46,10 @@ namespace lws public: //! Use `client` to sync blockchain data, and \return client if successful. - static expect sync(db::storage disk, rpc::client client); + static expect sync(db::storage disk, rpc::client client, const bool untrusted_daemon = false); //! Poll daemon until `stop()` is called, using `thread_count` threads. - static void run(db::storage disk, rpc::context ctx, std::size_t thread_count, epee::net_utils::ssl_verification_t webhook_verify, bool enable_subaddresses); + static void run(db::storage disk, rpc::context ctx, std::size_t thread_count, epee::net_utils::ssl_verification_t webhook_verify, bool enable_subaddresses, bool untrusted_daemon = false); //! \return True if `stop()` has never been called. static bool is_running() noexcept { return running; } diff --git a/src/server_main.cpp b/src/server_main.cpp index 95982ce..c3f83f4 100644 --- a/src/server_main.cpp +++ b/src/server_main.cpp @@ -80,6 +80,7 @@ namespace const command_line::arg_descriptor config_file; const command_line::arg_descriptor max_subaddresses; const command_line::arg_descriptor auto_accept_creation; + const command_line::arg_descriptor untrusted_daemon; static std::string get_default_zmq() { @@ -124,6 +125,7 @@ namespace , config_file{"config-file", "Specify any option in a config file; = on separate lines"} , max_subaddresses{"max-subaddresses", "Maximum number of subaddresses per primary account (defaults to 0)", 0} , auto_accept_creation{"auto-accept-creation", "New account creation requests are automatically accepted", false} + , untrusted_daemon{"untrusted-daemon", "Perform (expensive) chain-verification and PoW checks", false} {} void prepare(boost::program_options::options_description& description) const @@ -156,6 +158,7 @@ namespace command_line::add_arg(description, config_file); command_line::add_arg(description, max_subaddresses); command_line::add_arg(description, auto_accept_creation); + command_line::add_arg(description, untrusted_daemon); } }; @@ -173,6 +176,7 @@ namespace std::chrono::minutes rates_interval; std::size_t scan_threads; unsigned create_queue_max; + bool untrusted_daemon; }; void print_help(std::ostream& out) @@ -258,7 +262,8 @@ namespace command_line::get_arg(args, opts.webhook_ssl_verification), std::chrono::minutes{command_line::get_arg(args, opts.rates_interval)}, command_line::get_arg(args, opts.scan_threads), - command_line::get_arg(args, opts.create_queue_max) + command_line::get_arg(args, opts.create_queue_max), + command_line::get_arg(args, opts.untrusted_daemon) }; prog.rest_config.threads = std::max(std::size_t(1), prog.rest_config.threads); @@ -279,7 +284,7 @@ namespace auto ctx = lws::rpc::context::make(std::move(prog.daemon_rpc), std::move(prog.daemon_sub), std::move(prog.zmq_pub), std::move(prog.rmq), prog.rates_interval); MINFO("Using monerod ZMQ RPC at " << ctx.daemon_address()); - auto client = lws::scanner::sync(disk.clone(), ctx.connect().value()).value(); + auto client = lws::scanner::sync(disk.clone(), ctx.connect().value(), prog.untrusted_daemon).value(); const auto enable_subaddresses = bool(prog.rest_config.max_subaddresses); const auto webhook_verify = prog.rest_config.webhook_verify; @@ -292,7 +297,7 @@ namespace MINFO("Listening for REST admin clients at " << address); // blocks until SIGINT - lws::scanner::run(std::move(disk), std::move(ctx), prog.scan_threads, webhook_verify, enable_subaddresses); + lws::scanner::run(std::move(disk), std::move(ctx), prog.scan_threads, webhook_verify, enable_subaddresses, prog.untrusted_daemon); } } // anonymous diff --git a/src/util/CMakeLists.txt b/src/util/CMakeLists.txt index 009a8ff..2fbadf1 100644 --- a/src/util/CMakeLists.txt +++ b/src/util/CMakeLists.txt @@ -26,8 +26,8 @@ # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -set(monero-lws-util_sources gamma_picker.cpp random_outputs.cpp source_location.cpp transactions.cpp) -set(monero-lws-util_headers fwd.h gamma_picker.h http_server.h random_outputs.h source_location.h transactions.h) +set(monero-lws-util_sources blocks.cpp gamma_picker.cpp random_outputs.cpp source_location.cpp transactions.cpp) +set(monero-lws-util_headers blocks.h fwd.h gamma_picker.h http_server.h random_outputs.h source_location.h transactions.h) add_library(monero-lws-util ${monero-lws-util_sources} ${monero-lws-util_headers}) -target_link_libraries(monero-lws-util monero::libraries) +target_link_libraries(monero-lws-util monero::libraries monero-lws-db) diff --git a/src/util/blocks.cpp b/src/util/blocks.cpp new file mode 100644 index 0000000..ebf53d8 --- /dev/null +++ b/src/util/blocks.cpp @@ -0,0 +1,87 @@ +// Copyright (c) 2024, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "blocks.h" + +#include "cryptonote_config.h" // monero/src +#include "crypto/hash-ops.h" // monero/src +#include "db/storage.h" +#include "error.h" +#include "misc_language.h" +#include "string_tools.h" + +namespace lws +{ + crypto::hash get_block_longhash( + const std::string& bd, const db::block_id height, const unsigned major_version, const db::storage& disk, const db::block_id cached_start, epee::span cached) + { + crypto::hash result{}; + + // block 202612 bug workaround + if (height == db::block_id(202612)) + { + static const std::string longhash_202612 = "84f64766475d51837ac9efbef1926486e58563c95a19fef4aec3254f03000000"; + epee::string_tools::hex_to_pod(longhash_202612, result); + return result; + } + if (major_version >= RX_BLOCK_VERSION) + { + crypto::hash hash{}; + if (height != db::block_id(0)) + { + const uint64_t seed_height = crypto::rx_seedheight(std::uint64_t(height)); + if (cached_start <= db::block_id(seed_height)) + { + if (cached.size() <= seed_height - std::uint64_t(cached_start)) + MONERO_THROW(error::bad_blockchain, "invalid seed_height for cache or DB"); + hash = cached[seed_height - std::uint64_t(cached_start)]; + } + else + hash = MONERO_UNWRAP(MONERO_UNWRAP(disk.start_read()).get_block_hash(db::block_id(seed_height))); + } + else + { + memset(&result, 0, sizeof(crypto::hash)); // only happens when generating genesis block + } + crypto::rx_slow_hash(hash.data, bd.data(), bd.size(), result.data); + } else { + const int pow_variant = major_version >= 7 ? major_version - 6 : 0; + crypto::cn_slow_hash(bd.data(), bd.size(), result, pow_variant, std::uint64_t(height)); + } + return result; + } + + bool verify_timestamp(std::uint64_t check, std::vector timestamps) + { + if (timestamps.empty()) + return true; + if(check < epee::misc_utils::median(timestamps)) + return false; + return true; + } +} + diff --git a/src/util/blocks.h b/src/util/blocks.h new file mode 100644 index 0000000..c00dfa2 --- /dev/null +++ b/src/util/blocks.h @@ -0,0 +1,40 @@ +// Copyright (c) 2024, The Monero Project +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include +#include "crypto/hash.h" +#include "db/data.h" +#include "db/fwd.h" +#include "span.h" // in monero/contrib/epee/include + +namespace lws +{ + crypto::hash get_block_longhash( + const std::string& bd, const db::block_id height, const unsigned major_version, const db::storage& disk, db::block_id cached_start, epee::span cached); + + bool verify_timestamp(std::uint64_t verify, std::vector timestamps); +} diff --git a/tests/unit/db/chain.test.cpp b/tests/unit/db/chain.test.cpp index aed7da8..4fbd88e 100644 --- a/tests/unit/db/chain.test.cpp +++ b/tests/unit/db/chain.test.cpp @@ -90,7 +90,7 @@ LWS_CASE("db::storage::sync_chain") { const lws::account accounts[1] = {lws::account{get_account(), {}, {}}}; EXPECT(accounts[0].scan_height() == last_block.id); - EXPECT(db.update(last_block.id, chain, accounts)); + EXPECT(db.update(last_block.id, chain, accounts, nullptr)); EXPECT(get_account().scan_height == lws::db::block_id(std::uint64_t(last_block.id) + 4)); } diff --git a/tests/unit/db/webhook.test.cpp b/tests/unit/db/webhook.test.cpp index db2de3e..38424b3 100644 --- a/tests/unit/db/webhook.test.cpp +++ b/tests/unit/db/webhook.test.cpp @@ -141,7 +141,7 @@ LWS_CASE("db::storage::*_webhook") { crypto::hash chain[2] = {head.hash, crypto::rand()}; - auto updated = db.update(head.id, chain, {std::addressof(full_account), 1}); + auto updated = db.update(head.id, chain, {std::addressof(full_account), 1}, nullptr); EXPECT(!updated.has_error()); EXPECT(updated->first == 1); if (i < 3) @@ -185,7 +185,7 @@ LWS_CASE("db::storage::*_webhook") const std::vector outs = full_account.outputs(); EXPECT(outs.size() == 1); - const auto updated = db.update(last_block.id, chain, {std::addressof(full_account), 1}); + const auto updated = db.update(last_block.id, chain, {std::addressof(full_account), 1}, nullptr); EXPECT(!updated.has_error()); EXPECT(updated->first == 1); EXPECT(updated->second.size() == 3); diff --git a/tests/unit/scanner.test.cpp b/tests/unit/scanner.test.cpp index dd4ca0b..18faa8d 100644 --- a/tests/unit/scanner.test.cpp +++ b/tests/unit/scanner.test.cpp @@ -425,8 +425,8 @@ LWS_CASE("lws::scanner::sync and lws::scanner::run") EXPECT(result->at(0).second.at(0).size() == 2); EXPECT(result->at(0).second.at(0).at(0) == lws::db::minor_index(1)); EXPECT(result->at(0).second.at(0).at(1) == lws::db::minor_index(2)); - } - + } + std::vector destinations; destinations.emplace_back(); destinations.back().amount = 8000;