From f137a35984985232a3aae50cd4c7b47634866594 Mon Sep 17 00:00:00 2001 From: j-berman Date: Tue, 13 Dec 2022 16:08:56 -0800 Subject: [PATCH] Enforce restricted # pool txs served via RPC + optimize chunked reqs [release-v0.18] - `/getblocks.bin` respects the `RESTRICTED_TX_COUNT` (=100) when returning pool txs via a restricted RPC daemon. - A restricted RPC daemon includes a max of `RESTRICTED_TX_COUNT` txs in the `added_pool_txs` field, and returns any remaining pool hashes in the `remaining_added_pool_txids` field. The client then requests the remaining txs via `/gettransactions` in chunks. - `/gettransactions` no longer does expensive no-ops for ALL pool txs if the client requests a subset of pool txs. Instead it searches for the txs the client explicitly requests. - Reset `m_pool_info_query_time` when a user: (1) rescans the chain (so the wallet re-requests the whole pool) (2) changes the daemon their wallets points to (a new daemon would have a different view of the pool) - `/getblocks.bin` respects the `req.prune` field when returning pool txs. - Pool extension fields in response to `/getblocks.bin` are optional with default 0'd values. --- src/cryptonote_core/blockchain.cpp | 2 +- src/cryptonote_core/cryptonote_core.cpp | 9 +- src/cryptonote_core/cryptonote_core.h | 11 +- src/cryptonote_core/tx_pool.cpp | 78 ++++++---- src/cryptonote_core/tx_pool.h | 11 +- src/rpc/core_rpc_server.cpp | 55 +++---- src/rpc/core_rpc_server_commands_defs.h | 16 +- src/wallet/node_rpc_proxy.cpp | 30 ++++ src/wallet/node_rpc_proxy.h | 1 + src/wallet/wallet2.cpp | 193 +++++++++++++----------- src/wallet/wallet2.h | 3 +- src/wallet/wallet_rpc_server.cpp | 4 +- 12 files changed, 245 insertions(+), 168 deletions(-) diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index 35fa77688..2620010c4 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -2065,7 +2065,7 @@ bool Blockchain::handle_alternative_block(const block& b, const crypto::hash& id cryptonote::blobdata blob; if (m_tx_pool.have_tx(txid, relay_category::legacy)) { - if (m_tx_pool.get_transaction_info(txid, td)) + if (m_tx_pool.get_transaction_info(txid, td, true/*include_sensitive_data*/)) { bei.block_cumulative_weight += td.weight; } diff --git a/src/cryptonote_core/cryptonote_core.cpp b/src/cryptonote_core/cryptonote_core.cpp index b252f49de..d34c92723 100644 --- a/src/cryptonote_core/cryptonote_core.cpp +++ b/src/cryptonote_core/cryptonote_core.cpp @@ -1727,6 +1727,11 @@ namespace cryptonote return true; } //----------------------------------------------------------------------------------------------- + bool core::get_pool_transactions_info(const std::vector& txids, std::vector>& txs, bool include_sensitive_txes) const + { + return m_mempool.get_transactions_info(txids, txs, include_sensitive_txes); + } + //----------------------------------------------------------------------------------------------- bool core::get_pool_transactions(std::vector& txs, bool include_sensitive_data) const { m_mempool.get_transactions(txs, include_sensitive_data); @@ -1739,9 +1744,9 @@ namespace cryptonote return true; } //----------------------------------------------------------------------------------------------- - bool core::get_pool_info(time_t start_time, bool include_sensitive_txes, std::vector& added_txs, std::vector& removed_txs, bool& incremental) const + bool core::get_pool_info(time_t start_time, bool include_sensitive_txes, size_t max_tx_count, std::vector>& added_txs, std::vector& remaining_added_txids, std::vector& removed_txs, bool& incremental) const { - return m_mempool.get_pool_info(start_time, include_sensitive_txes, added_txs, removed_txs, incremental); + return m_mempool.get_pool_info(start_time, include_sensitive_txes, max_tx_count, added_txs, remaining_added_txids, removed_txs, incremental); } //----------------------------------------------------------------------------------------------- bool core::get_pool_transaction_stats(struct txpool_stats& stats, bool include_sensitive_data) const diff --git a/src/cryptonote_core/cryptonote_core.h b/src/cryptonote_core/cryptonote_core.h index b0d99e1ce..efab56405 100644 --- a/src/cryptonote_core/cryptonote_core.h +++ b/src/cryptonote_core/cryptonote_core.h @@ -509,13 +509,22 @@ namespace cryptonote */ bool get_pool_transaction_hashes(std::vector& txs, bool include_sensitive_txes = false) const; + /** + * @copydoc tx_memory_pool::get_pool_transactions_info + * @param include_sensitive_txes include private transactions + * + * @note see tx_memory_pool::get_pool_transactions_info + */ + bool get_pool_transactions_info(const std::vector& txids, std::vector>& txs, bool include_sensitive_txes = false) const; + /** * @copydoc tx_memory_pool::get_pool_info * @param include_sensitive_txes include private transactions + * @param max_tx_count max allowed added_txs in response * * @note see tx_memory_pool::get_pool_info */ - bool get_pool_info(time_t start_time, bool include_sensitive_txes, std::vector& added_txs, std::vector& removed_txs, bool& incremental) const; + bool get_pool_info(time_t start_time, bool include_sensitive_txes, size_t max_tx_count, std::vector>& added_txs, std::vector& remaining_added_txids, std::vector& removed_txs, bool& incremental) const; /** * @copydoc tx_memory_pool::get_transactions diff --git a/src/cryptonote_core/tx_pool.cpp b/src/cryptonote_core/tx_pool.cpp index f8fbaf373..36fea56a3 100644 --- a/src/cryptonote_core/tx_pool.cpp +++ b/src/cryptonote_core/tx_pool.cpp @@ -614,7 +614,7 @@ namespace cryptonote return true; } //--------------------------------------------------------------------------------- - bool tx_memory_pool::get_transaction_info(const crypto::hash &txid, tx_details &td) const + bool tx_memory_pool::get_transaction_info(const crypto::hash &txid, tx_details &td, bool include_sensitive_data, bool include_blob) const { PERF_TIMER(get_transaction_info); CRITICAL_REGION_LOCAL(m_transactions_lock); @@ -626,7 +626,12 @@ namespace cryptonote txpool_tx_meta_t meta; if (!m_blockchain.get_txpool_tx_meta(txid, meta)) { - MERROR("Failed to find tx in txpool"); + LOG_PRINT_L2("Failed to find tx in txpool: " << txid); + return false; + } + if (!include_sensitive_data && !meta.matches(relay_category::broadcasted)) + { + // We don't want sensitive data && the tx is sensitive, so no need to return it return false; } cryptonote::blobdata txblob = m_blockchain.get_txpool_tx_blob(txid, relay_category::all); @@ -652,12 +657,13 @@ namespace cryptonote td.kept_by_block = meta.kept_by_block; td.last_failed_height = meta.last_failed_height; td.last_failed_id = meta.last_failed_id; - td.receive_time = meta.receive_time; - td.last_relayed_time = meta.dandelionpp_stem ? 0 : meta.last_relayed_time; + td.receive_time = include_sensitive_data ? meta.receive_time : 0; + td.last_relayed_time = (include_sensitive_data && !meta.dandelionpp_stem) ? meta.last_relayed_time : 0; td.relayed = meta.relayed; td.do_not_relay = meta.do_not_relay; td.double_spend_seen = meta.double_spend_seen; - td.sensitive = !meta.matches(relay_category::broadcasted); + if (include_blob) + td.tx_blob = std::move(txblob); } catch (const std::exception &e) { @@ -667,6 +673,25 @@ namespace cryptonote return true; } + //------------------------------------------------------------------ + bool tx_memory_pool::get_transactions_info(const std::vector& txids, std::vector>& txs, bool include_sensitive) const + { + CRITICAL_REGION_LOCAL(m_transactions_lock); + CRITICAL_REGION_LOCAL1(m_blockchain); + + txs.clear(); + + for (const auto &it: txids) + { + tx_details details; + bool success = get_transaction_info(it, details, include_sensitive, true/*include_blob*/); + if (success) + { + txs.push_back(std::make_pair(it, std::move(details))); + } + } + return true; + } //--------------------------------------------------------------------------------- bool tx_memory_pool::get_complement(const std::vector &hashes, std::vector &txes) const { @@ -940,7 +965,7 @@ namespace cryptonote }, false, category); } //------------------------------------------------------------------ - bool tx_memory_pool::get_pool_info(time_t start_time, bool include_sensitive, std::vector& added_txs, std::vector& removed_txs, bool& incremental) const + bool tx_memory_pool::get_pool_info(time_t start_time, bool include_sensitive, size_t max_tx_count, std::vector>& added_txs, std::vector& remaining_added_txids, std::vector& removed_txs, bool& incremental) const { CRITICAL_REGION_LOCAL(m_transactions_lock); CRITICAL_REGION_LOCAL1(m_blockchain); @@ -968,46 +993,39 @@ namespace cryptonote } added_txs.clear(); + remaining_added_txids.clear(); removed_txs.clear(); + std::vector txids; if (!incremental) { + LOG_PRINT_L2("Giving back the whole pool"); // Give back the whole pool in 'added_txs'; because calling 'get_transaction_info' right inside the // anonymous method somehow results in an LMDB error with transactions we have to build a list of // ids first and get the full info afterwards - std::vector txids; - const relay_category category = include_sensitive ? relay_category::all : relay_category::broadcasted; - m_blockchain.for_all_txpool_txes([&txids](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata_ref *bd){ - txids.push_back(txid); - return true; - }, false, category); - tx_details details; - for (const auto &it: txids) + get_transaction_hashes(txids, include_sensitive); + if (txids.size() > max_tx_count) { - bool success = get_transaction_info(it, details); - if (success) - { - added_txs.push_back(std::move(details)); - } + remaining_added_txids = std::vector(txids.begin() + max_tx_count, txids.end()); + txids.erase(txids.begin() + max_tx_count, txids.end()); } + get_transactions_info(txids, added_txs, include_sensitive); return true; } // Give back incrementally, based on time of entry into the map - tx_details details; for (const auto &pit : m_added_txs_by_id) { if (pit.second >= start_time) - { - bool success = get_transaction_info(pit.first, details); - if (success) - { - if (include_sensitive || !details.sensitive) - { - added_txs.push_back(std::move(details)); - } - } - } + txids.push_back(pit.first); + } + get_transactions_info(txids, added_txs, include_sensitive); + if (added_txs.size() > max_tx_count) + { + remaining_added_txids.reserve(added_txs.size() - max_tx_count); + for (size_t i = max_tx_count; i < added_txs.size(); ++i) + remaining_added_txids.push_back(added_txs[i].first); + added_txs.erase(added_txs.begin() + max_tx_count, added_txs.end()); } std::multimap::const_iterator rit = m_removed_txs_by_time.lower_bound(start_time); diff --git a/src/cryptonote_core/tx_pool.h b/src/cryptonote_core/tx_pool.h index 3e099cf2d..23135ead1 100644 --- a/src/cryptonote_core/tx_pool.h +++ b/src/cryptonote_core/tx_pool.h @@ -428,6 +428,7 @@ namespace cryptonote struct tx_details { transaction tx; //!< the transaction + cryptonote::blobdata tx_blob; //!< the transaction's binary blob size_t blob_size; //!< the transaction's size size_t weight; //!< the transaction's weight uint64_t fee; //!< the transaction's fee amount @@ -461,13 +462,17 @@ namespace cryptonote bool do_not_relay; //!< to avoid relay this transaction to the network bool double_spend_seen; //!< true iff another tx was seen double spending this one - bool sensitive; }; /** * @brief get infornation about a single transaction */ - bool get_transaction_info(const crypto::hash &txid, tx_details &td) const; + bool get_transaction_info(const crypto::hash &txid, tx_details &td, bool include_sensitive_data, bool include_blob = false) const; + + /** + * @brief get information about multiple transactions + */ + bool get_transactions_info(const std::vector& txids, std::vector>& txs, bool include_sensitive_data = false) const; /** * @brief get transactions not in the passed set @@ -479,7 +484,7 @@ namespace cryptonote * * @return true on success, false on error */ - bool get_pool_info(time_t start_time, bool include_sensitive, std::vector& added_txs, std::vector& removed_txs, bool& incremental) const; + bool get_pool_info(time_t start_time, bool include_sensitive, size_t max_tx_count, std::vector>& added_txs, std::vector& remaining_added_txids, std::vector& removed_txs, bool& incremental) const; private: diff --git a/src/rpc/core_rpc_server.cpp b/src/rpc/core_rpc_server.cpp index e570ee109..981a207a7 100644 --- a/src/rpc/core_rpc_server.cpp +++ b/src/rpc/core_rpc_server.cpp @@ -630,31 +630,34 @@ namespace cryptonote const bool restricted = m_restricted && ctx; const bool request_has_rpc_origin = ctx != NULL; const bool allow_sensitive = !request_has_rpc_origin || !restricted; + const size_t max_tx_count = restricted ? RESTRICTED_TRANSACTIONS_COUNT : std::numeric_limits::max(); bool incremental; - std::vector added_pool_txs; - bool success = m_core.get_pool_info((time_t)req.pool_info_since, allow_sensitive, added_pool_txs, res.removed_pool_txids, incremental); + std::vector> added_pool_txs; + bool success = m_core.get_pool_info((time_t)req.pool_info_since, allow_sensitive, max_tx_count, added_pool_txs, res.remaining_added_pool_txids, res.removed_pool_txids, incremental); if (success) { res.added_pool_txs.clear(); if (m_rpc_payment) { - CHECK_PAYMENT_SAME_TS(req, res, added_pool_txs.size() * COST_PER_TX + res.removed_pool_txids.size() * COST_PER_POOL_HASH); + CHECK_PAYMENT_SAME_TS(req, res, added_pool_txs.size() * COST_PER_TX + (res.remaining_added_pool_txids.size() + res.removed_pool_txids.size()) * COST_PER_POOL_HASH); } - for (auto tx_detail: added_pool_txs) + for (const auto &added_pool_tx: added_pool_txs) { COMMAND_RPC_GET_BLOCKS_FAST::pool_tx_info info; - info.tx_hash = cryptonote::get_transaction_hash(tx_detail.tx); + info.tx_hash = added_pool_tx.first; std::stringstream oss; binary_archive ar(oss); - bool r = ::serialization::serialize(ar, tx_detail.tx); + bool r = req.prune + ? const_cast(added_pool_tx.second.tx).serialize_base(ar) + : ::serialization::serialize(ar, const_cast(added_pool_tx.second.tx)); if (!r) { res.status = "Failed to serialize transaction"; return true; } info.tx_blob = oss.str(); - info.double_spend_seen = tx_detail.double_spend_seen; + info.double_spend_seen = added_pool_tx.second.double_spend_seen; res.added_pool_txs.push_back(std::move(info)); } } @@ -993,17 +996,16 @@ namespace cryptonote // try the pool for any missing txes size_t found_in_pool = 0; std::unordered_set pool_tx_hashes; - std::unordered_map per_tx_pool_tx_info; + std::unordered_map per_tx_pool_tx_details; if (!missed_txs.empty()) { - std::vector pool_tx_info; - std::vector pool_key_image_info; - bool r = m_core.get_pool_transactions_and_spent_keys_info(pool_tx_info, pool_key_image_info, !request_has_rpc_origin || !restricted); + std::vector> pool_txs; + bool r = m_core.get_pool_transactions_info(missed_txs, pool_txs, !request_has_rpc_origin || !restricted); if(r) { // sort to match original request std::vector> sorted_txs; - std::vector::const_iterator i; + std::vector>::const_iterator i; unsigned txs_processed = 0; for (const crypto::hash &h: vh) { @@ -1023,36 +1025,23 @@ namespace cryptonote sorted_txs.push_back(std::move(txs[txs_processed])); ++txs_processed; } - else if ((i = std::find_if(pool_tx_info.begin(), pool_tx_info.end(), [h](const tx_info &txi) { return epee::string_tools::pod_to_hex(h) == txi.id_hash; })) != pool_tx_info.end()) + else if ((i = std::find_if(pool_txs.begin(), pool_txs.end(), [h](const std::pair &pt) { return h == pt.first; })) != pool_txs.end()) { - cryptonote::transaction tx; - if (!cryptonote::parse_and_validate_tx_from_blob(i->tx_blob, tx)) - { - res.status = "Failed to parse and validate tx from blob"; - return true; - } + const tx_memory_pool::tx_details &td = i->second; std::stringstream ss; binary_archive ba(ss); - bool r = const_cast(tx).serialize_base(ba); + bool r = const_cast(td.tx).serialize_base(ba); if (!r) { res.status = "Failed to serialize transaction base"; return true; } const cryptonote::blobdata pruned = ss.str(); - const crypto::hash prunable_hash = tx.version == 1 ? crypto::null_hash : get_transaction_prunable_hash(tx); - sorted_txs.push_back(std::make_tuple(h, pruned, prunable_hash, std::string(i->tx_blob, pruned.size()))); + const crypto::hash prunable_hash = td.tx.version == 1 ? crypto::null_hash : get_transaction_prunable_hash(td.tx); + sorted_txs.push_back(std::make_tuple(h, pruned, prunable_hash, std::string(td.tx_blob, pruned.size()))); missed_txs.erase(std::find(missed_txs.begin(), missed_txs.end(), h)); pool_tx_hashes.insert(h); - const std::string hash_string = epee::string_tools::pod_to_hex(h); - for (const auto &ti: pool_tx_info) - { - if (ti.id_hash == hash_string) - { - per_tx_pool_tx_info.insert(std::make_pair(h, ti)); - break; - } - } + per_tx_pool_tx_details.insert(std::make_pair(h, td)); ++found_in_pool; } } @@ -1148,8 +1137,8 @@ namespace cryptonote { e.block_height = e.block_timestamp = std::numeric_limits::max(); e.confirmations = 0; - auto it = per_tx_pool_tx_info.find(tx_hash); - if (it != per_tx_pool_tx_info.end()) + auto it = per_tx_pool_tx_details.find(tx_hash); + if (it != per_tx_pool_tx_details.end()) { e.double_spend_seen = it->second.double_spend_seen; e.relayed = it->second.relayed; diff --git a/src/rpc/core_rpc_server_commands_defs.h b/src/rpc/core_rpc_server_commands_defs.h index c52422e09..63f74108d 100644 --- a/src/rpc/core_rpc_server_commands_defs.h +++ b/src/rpc/core_rpc_server_commands_defs.h @@ -236,6 +236,7 @@ namespace cryptonote uint64_t daemon_time; uint8_t pool_info_extent; std::vector added_pool_txs; + std::vector remaining_added_pool_txids; std::vector removed_pool_txids; BEGIN_KV_SERIALIZE_MAP() @@ -244,10 +245,17 @@ namespace cryptonote KV_SERIALIZE(start_height) KV_SERIALIZE(current_height) KV_SERIALIZE(output_indices) - KV_SERIALIZE(daemon_time) - KV_SERIALIZE(pool_info_extent) - KV_SERIALIZE(added_pool_txs) - KV_SERIALIZE_CONTAINER_POD_AS_BLOB(removed_pool_txids) + KV_SERIALIZE_OPT(daemon_time, (uint64_t) 0) + KV_SERIALIZE_OPT(pool_info_extent, (uint8_t) 0) + if (pool_info_extent != POOL_INFO_EXTENT::NONE) + { + KV_SERIALIZE(added_pool_txs) + KV_SERIALIZE_CONTAINER_POD_AS_BLOB(remaining_added_pool_txids) + } + if (pool_info_extent == POOL_INFO_EXTENT::INCREMENTAL) + { + KV_SERIALIZE_CONTAINER_POD_AS_BLOB(removed_pool_txids) + } END_KV_SERIALIZE_MAP() }; typedef epee::misc_utils::struct_init response; diff --git a/src/wallet/node_rpc_proxy.cpp b/src/wallet/node_rpc_proxy.cpp index 44c52df62..aa0b4dfd3 100644 --- a/src/wallet/node_rpc_proxy.cpp +++ b/src/wallet/node_rpc_proxy.cpp @@ -392,6 +392,36 @@ boost::optional NodeRPCProxy::get_rpc_payment_info(bool mining, boo return boost::none; } +boost::optional NodeRPCProxy::get_transactions(const std::vector &txids, const std::function &f) +{ + const size_t SLICE_SIZE = 100; // RESTRICTED_TRANSACTIONS_COUNT as defined in rpc/core_rpc_server.cpp + for (size_t offset = 0; offset < txids.size(); offset += SLICE_SIZE) + { + cryptonote::COMMAND_RPC_GET_TRANSACTIONS::request req_t = AUTO_VAL_INIT(req_t); + cryptonote::COMMAND_RPC_GET_TRANSACTIONS::response resp_t = AUTO_VAL_INIT(resp_t); + + const size_t n_txids = std::min(SLICE_SIZE, txids.size() - offset); + for (size_t n = offset; n < (offset + n_txids); ++n) + req_t.txs_hashes.push_back(epee::string_tools::pod_to_hex(txids[n])); + MDEBUG("asking for " << req_t.txs_hashes.size() << " transactions"); + req_t.decode_as_json = false; + req_t.prune = true; + + bool r = false; + { + const boost::lock_guard lock{m_daemon_rpc_mutex}; + uint64_t pre_call_credits = m_rpc_payment_state.credits; + req_t.client = cryptonote::make_rpc_payment_signature(m_client_id_secret_key); + r = net_utils::invoke_http_json("/gettransactions", req_t, resp_t, m_http_client, rpc_timeout); + if (r && resp_t.status == CORE_RPC_STATUS_OK) + check_rpc_cost(m_rpc_payment_state, "/gettransactions", resp_t.credits, pre_call_credits, resp_t.txs.size() * COST_PER_TX); + } + + f(req_t, resp_t, r); + } + return boost::optional(); +} + boost::optional NodeRPCProxy::get_block_header_by_height(uint64_t height, cryptonote::block_header_response &block_header) { if (m_offline) diff --git a/src/wallet/node_rpc_proxy.h b/src/wallet/node_rpc_proxy.h index f78cd6427..0088090a8 100644 --- a/src/wallet/node_rpc_proxy.h +++ b/src/wallet/node_rpc_proxy.h @@ -59,6 +59,7 @@ public: boost::optional get_dynamic_base_fee_estimate_2021_scaling(uint64_t grace_blocks, std::vector &fees); boost::optional get_fee_quantization_mask(uint64_t &fee_quantization_mask); boost::optional get_rpc_payment_info(bool mining, bool &payment_required, uint64_t &credits, uint64_t &diff, uint64_t &credits_per_hash_found, cryptonote::blobdata &blob, uint64_t &height, uint64_t &seed_height, crypto::hash &seed_hash, crypto::hash &next_seed_hash, uint32_t &cookie); + boost::optional get_transactions(const std::vector &txids, const std::function &f); boost::optional get_block_header_by_height(uint64_t height, cryptonote::block_header_response &block_header); private: diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index 76d7b2a4f..1ea699ef1 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -1350,6 +1350,7 @@ bool wallet2::set_daemon(std::string daemon_address, boost::optional &txids, std::vector> &txs) +{ + if (r && res.status == CORE_RPC_STATUS_OK) + { + MDEBUG("Reading pool txs"); + if (res.txs.size() == req.txs_hashes.size()) + { + for (const auto &tx_entry: res.txs) + { + if (tx_entry.in_pool) + { + cryptonote::transaction tx; + cryptonote::blobdata bd; + crypto::hash tx_hash; + + if (get_pruned_tx(tx_entry, tx, tx_hash)) + { + const std::vector::const_iterator i = std::find_if(txids.begin(), txids.end(), + [tx_hash](const crypto::hash &e) { return e == tx_hash; }); + if (i != txids.end()) + { + txs.push_back(std::make_tuple(tx, tx_hash, tx_entry.double_spend_seen)); + } + else + { + MERROR("Got txid " << tx_hash << " which we did not ask for"); + } + } + else + { + LOG_PRINT_L0("Failed to parse transaction from daemon"); + } + } + else + { + LOG_PRINT_L1("Transaction from daemon was in pool, but is no more"); + } + } + } + else + { + LOG_PRINT_L0("Expected " << req.txs_hashes.size() << " out of " << txids.size() << " tx(es), got " << res.txs.size()); + } + } +} +//---------------------------------------------------------------------------------------------------- +void wallet2::process_pool_info_extent(const cryptonote::COMMAND_RPC_GET_BLOCKS_FAST::response &res, std::vector> &process_txs, bool refreshed) +{ + std::vector> added_pool_txs; + added_pool_txs.reserve(res.added_pool_txs.size() + res.remaining_added_pool_txids.size()); + + for (const auto &pool_tx: res.added_pool_txs) + { + cryptonote::transaction tx; + THROW_WALLET_EXCEPTION_IF(!cryptonote::parse_and_validate_tx_base_from_blob(pool_tx.tx_blob, tx), + error::wallet_internal_error, "Failed to validate transaction base from daemon"); + added_pool_txs.push_back(std::make_tuple(tx, pool_tx.tx_hash, pool_tx.double_spend_seen)); + } + + // getblocks.bin may return more added pool transactions than we're allowed to request in restricted mode + if (!res.remaining_added_pool_txids.empty()) + { + // request the remaining txs + m_node_rpc_proxy.get_transactions(res.remaining_added_pool_txids, + [this, &res, &added_pool_txs](const cryptonote::COMMAND_RPC_GET_TRANSACTIONS::request &req_t, const cryptonote::COMMAND_RPC_GET_TRANSACTIONS::response &resp_t, bool r) + { + read_pool_txs(req_t, resp_t, r, res.remaining_added_pool_txids, added_pool_txs); + if (!r || resp_t.status != CORE_RPC_STATUS_OK) + LOG_PRINT_L0("Error calling gettransactions daemon RPC: r " << r << ", status " << get_rpc_status(resp_t.status)); + } + ); + } + + update_pool_state_from_pool_data(res.pool_info_extent == COMMAND_RPC_GET_BLOCKS_FAST::INCREMENTAL, res.removed_pool_txids, added_pool_txs, process_txs, refreshed); +} +//---------------------------------------------------------------------------------------------------- void wallet2::pull_blocks(bool first, bool try_incremental, uint64_t start_height, uint64_t &blocks_start_height, const std::list &short_chain_history, std::vector &blocks, std::vector &o_indices, uint64_t ¤t_height) { cryptonote::COMMAND_RPC_GET_BLOCKS_FAST::request req = AUTO_VAL_INIT(req); @@ -2965,7 +3042,7 @@ void wallet2::pull_blocks(bool first, bool try_incremental, uint64_t start_heigh THROW_WALLET_EXCEPTION_IF(res.blocks.size() != res.output_indices.size(), error::wallet_internal_error, "mismatched blocks (" + boost::lexical_cast(res.blocks.size()) + ") and output_indices (" + boost::lexical_cast(res.output_indices.size()) + ") sizes from daemon"); - uint64_t pool_info_cost = res.added_pool_txs.size() * COST_PER_TX + res.removed_pool_txids.size() * COST_PER_POOL_HASH; + uint64_t pool_info_cost = res.added_pool_txs.size() * COST_PER_TX + (res.remaining_added_pool_txids.size() + res.removed_pool_txids.size()) * COST_PER_POOL_HASH; check_rpc_cost("/getblocks.bin", res.credits, pre_call_credits, 1 + res.blocks.size() * COST_PER_BLOCK + pool_info_cost); } @@ -2984,7 +3061,7 @@ void wallet2::pull_blocks(bool first, bool try_incremental, uint64_t start_heigh { if (res.pool_info_extent != COMMAND_RPC_GET_BLOCKS_FAST::NONE) { - update_pool_state_from_pool_data(res, m_process_pool_txs, true); + process_pool_info_extent(res, m_process_pool_txs, true); } else { @@ -3510,14 +3587,14 @@ void wallet2::update_pool_state(std::vector> txids; + std::vector txids; for (const auto &txid: res.tx_hashes) { if (accept_pool_tx_for_processing(txid)) - txids.push_back({txid, false}); + txids.push_back(txid); } - // get_transaction_pool_hashes.bin may return more transactions than we're allowed to request in restricted mode - const size_t SLICE_SIZE = 100; // RESTRICTED_TRANSACTIONS_COUNT as defined in rpc/core_rpc_server.cpp - for (size_t offset = 0; offset < txids.size(); offset += SLICE_SIZE) - { - cryptonote::COMMAND_RPC_GET_TRANSACTIONS::request req; - cryptonote::COMMAND_RPC_GET_TRANSACTIONS::response res; - - const size_t n_txids = std::min(SLICE_SIZE, txids.size() - offset); - for (size_t n = offset; n < (offset + n_txids); ++n) { - req.txs_hashes.push_back(epee::string_tools::pod_to_hex(txids.at(n).first)); - } - MDEBUG("asking for " << req.txs_hashes.size() << " transactions"); - req.decode_as_json = false; - req.prune = true; - - bool r; + m_node_rpc_proxy.get_transactions(txids, + [this, &txids, &process_txs](const cryptonote::COMMAND_RPC_GET_TRANSACTIONS::request &req_t, const cryptonote::COMMAND_RPC_GET_TRANSACTIONS::response &resp_t, bool r) { - const boost::lock_guard lock{m_daemon_rpc_mutex}; - uint64_t pre_call_credits = m_rpc_payment_state.credits; - req.client = get_client_signature(); - r = epee::net_utils::invoke_http_json("/gettransactions", req, res, *m_http_client, rpc_timeout); - if (r && res.status == CORE_RPC_STATUS_OK) - check_rpc_cost("/gettransactions", res.credits, pre_call_credits, res.txs.size() * COST_PER_TX); + read_pool_txs(req_t, resp_t, r, txids, process_txs); + if (!r || resp_t.status != CORE_RPC_STATUS_OK) + LOG_PRINT_L0("Error calling gettransactions daemon RPC: r " << r << ", status " << get_rpc_status(resp_t.status)); } + ); - MDEBUG("Got " << r << " and " << res.status); - if (r && res.status == CORE_RPC_STATUS_OK) - { - if (res.txs.size() == req.txs_hashes.size()) - { - for (const auto &tx_entry: res.txs) - { - if (tx_entry.in_pool) - { - cryptonote::transaction tx; - cryptonote::blobdata bd; - crypto::hash tx_hash; - - if (get_pruned_tx(tx_entry, tx, tx_hash)) - { - const std::vector>::const_iterator i = std::find_if(txids.begin(), txids.end(), - [tx_hash](const std::pair &e) { return e.first == tx_hash; }); - if (i != txids.end()) - { - process_txs.push_back(std::make_tuple(tx, tx_hash, tx_entry.double_spend_seen)); - } - else - { - MERROR("Got txid " << tx_hash << " which we did not ask for"); - } - } - else - { - LOG_PRINT_L0("Failed to parse transaction from daemon"); - } - } - else - { - LOG_PRINT_L1("Transaction from daemon was in pool, but is no more"); - } - } - } - else - { - LOG_PRINT_L0("Expected " << n_txids << " out of " << txids.size() << " tx(es), got " << res.txs.size()); - } - } - else - { - LOG_PRINT_L0("Error calling gettransactions daemon RPC: r " << r << ", status " << get_rpc_status(res.status)); - } - } MTRACE("update_pool_state_by_pool_query end"); } //---------------------------------------------------------------------------------------------------- @@ -3673,15 +3687,13 @@ void wallet2::update_pool_state_by_pool_query(std::vector> &process_txs, bool refreshed) +void wallet2::update_pool_state_from_pool_data(bool incremental, const std::vector &removed_pool_txids, const std::vector> &added_pool_txs, std::vector> &process_txs, bool refreshed) { MTRACE("update_pool_state_from_pool_data start"); auto keys_reencryptor = epee::misc_utils::create_scope_leave_handler([&, this]() { m_encrypt_keys_after_refresh.reset(); }); - bool incremental = res.pool_info_extent == COMMAND_RPC_GET_BLOCKS_FAST::INCREMENTAL; - if (refreshed) { if (incremental) @@ -3690,7 +3702,7 @@ void wallet2::update_pool_state_from_pool_data(const cryptonote::COMMAND_RPC_GET // pool; do so only after refresh to not delete too early and too eagerly; maybe we will find the tx // later in a block, or not, or find it again in the pool txs because it was first removed but then // somehow quickly "resurrected" - that all does not matter here, we retrace the removal - remove_obsolete_pool_txs(res.removed_pool_txids, true); + remove_obsolete_pool_txs(removed_pool_txids, true); } else { @@ -3698,10 +3710,10 @@ void wallet2::update_pool_state_from_pool_data(const cryptonote::COMMAND_RPC_GET // unfortunate that we have to build a new vector with ids first, but better than copying and // modifying the code of 'remove_obsolete_pool_txs' here std::vector txids; - txids.reserve(res.added_pool_txs.size()); - for (const auto &it: res.added_pool_txs) + txids.reserve(added_pool_txs.size()); + for (const auto &pool_tx: added_pool_txs) { - txids.push_back(it.tx_hash); + txids.push_back(std::get<1>(pool_tx)); } remove_obsolete_pool_txs(txids, false); } @@ -3715,9 +3727,9 @@ void wallet2::update_pool_state_from_pool_data(const cryptonote::COMMAND_RPC_GET const crypto::hash &txid = it->first; MDEBUG("Checking m_unconfirmed_txs entry " << txid); bool found = false; - for (const auto &it2: res.added_pool_txs) + for (const auto &pool_tx: added_pool_txs) { - if (it2.tx_hash == txid) + if (std::get<1>(pool_tx) == txid) { found = true; break; @@ -3732,16 +3744,11 @@ void wallet2::update_pool_state_from_pool_data(const cryptonote::COMMAND_RPC_GET // if we work incrementally and thus see only new pool txs since last time we asked it should // be rare that we know already about one of those, but check nevertheless process_txs.clear(); - for (const auto &pool_tx: res.added_pool_txs) + for (const auto &pool_tx: added_pool_txs) { - cryptonote::transaction tx; - THROW_WALLET_EXCEPTION_IF(!cryptonote::parse_and_validate_tx_from_blob(pool_tx.tx_blob, tx), - error::wallet_internal_error, "Failed to validate transaction from daemon"); - const crypto::hash &txid = pool_tx.tx_hash; - bool take = accept_pool_tx_for_processing(txid); - if (take) + if (accept_pool_tx_for_processing(std::get<1>(pool_tx))) { - process_txs.push_back(std::make_tuple(tx, txid, pool_tx.double_spend_seen)); + process_txs.push_back(pool_tx); } } @@ -4323,6 +4330,7 @@ bool wallet2::clear() m_subaddress_labels.clear(); m_multisig_rounds_passed = 0; m_device_last_key_image_sync = 0; + m_pool_info_query_time = 0; m_skip_to_height = 0; return true; } @@ -4340,6 +4348,7 @@ void wallet2::clear_soft(bool keep_key_images) m_unconfirmed_payments.clear(); m_scanned_pool_txs[0].clear(); m_scanned_pool_txs[1].clear(); + m_pool_info_query_time = 0; m_skip_to_height = 0; cryptonote::block b; diff --git a/src/wallet/wallet2.h b/src/wallet/wallet2.h index 589a077ab..f07e1c41a 100644 --- a/src/wallet/wallet2.h +++ b/src/wallet/wallet2.h @@ -1740,8 +1740,9 @@ private: void process_parsed_blocks(uint64_t start_height, const std::vector &blocks, const std::vector &parsed_blocks, uint64_t& blocks_added, std::map, size_t> *output_tracker_cache = NULL); bool accept_pool_tx_for_processing(const crypto::hash &txid); void process_unconfirmed_transfer(bool incremental, const crypto::hash &txid, wallet2::unconfirmed_transfer_details &tx_details, bool seen_in_pool, std::chrono::system_clock::time_point now, bool refreshed); + void process_pool_info_extent(const cryptonote::COMMAND_RPC_GET_BLOCKS_FAST::response &res, std::vector> &process_txs, bool refreshed); void update_pool_state_by_pool_query(std::vector> &process_txs, bool refreshed = false); - void update_pool_state_from_pool_data(const cryptonote::COMMAND_RPC_GET_BLOCKS_FAST::response &res, std::vector> &process_txs, bool refreshed); + void update_pool_state_from_pool_data(bool incremental, const std::vector &removed_pool_txids, const std::vector> &added_pool_txs, std::vector> &process_txs, bool refreshed); uint64_t select_transfers(uint64_t needed_money, std::vector unused_transfers_indices, std::vector& selected_transfers) const; bool prepare_file_names(const std::string& file_path); void process_unconfirmed(const crypto::hash &txid, const cryptonote::transaction& tx, uint64_t height); diff --git a/src/wallet/wallet_rpc_server.cpp b/src/wallet/wallet_rpc_server.cpp index 32628d65b..4f1b3bd21 100644 --- a/src/wallet/wallet_rpc_server.cpp +++ b/src/wallet/wallet_rpc_server.cpp @@ -149,8 +149,10 @@ namespace tools return true; if (boost::posix_time::microsec_clock::universal_time() < m_last_auto_refresh_time + boost::posix_time::seconds(m_auto_refresh_period)) return true; + uint64_t blocks_fetched = 0; try { - if (m_wallet) m_wallet->refresh(m_wallet->is_trusted_daemon()); + bool received_money = false; + if (m_wallet) m_wallet->refresh(m_wallet->is_trusted_daemon(), 0, blocks_fetched, received_money, true, true); } catch (const std::exception& ex) { LOG_ERROR("Exception at while refreshing, what=" << ex.what()); }