From b7140daea2b33d03f65b852de7f1ce4b99661adf Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@symas.com>
Date: Wed, 13 Apr 2016 23:45:02 +0100
Subject: [PATCH 1/3] Add GET_HASHES_FAST rpc, use it in wallet
When m_refresh_from_block_height has been set, only hashes will be
retrieved up to that height, instead of full blocks. The same will
be done for "refresh <height>" when the specified height is beyond
the current local blockchain.
---
src/rpc/core_rpc_server.cpp | 19 ++++++
src/rpc/core_rpc_server.h | 2 +
src/rpc/core_rpc_server_commands_defs.h | 30 +++++++++
src/wallet/wallet2.cpp | 87 +++++++++++++++++++++++++
src/wallet/wallet2.h | 2 +
src/wallet/wallet_errors.h | 5 ++
6 files changed, 145 insertions(+)
diff --git a/src/rpc/core_rpc_server.cpp b/src/rpc/core_rpc_server.cpp
index 9fcb4373b..9e8d1108e 100644
--- a/src/rpc/core_rpc_server.cpp
+++ b/src/rpc/core_rpc_server.cpp
@@ -166,6 +166,25 @@ namespace cryptonote
return true;
}
//------------------------------------------------------------------------------------------------------------------------------
+ bool core_rpc_server::on_get_hashes(const COMMAND_RPC_GET_HASHES_FAST::request& req, COMMAND_RPC_GET_HASHES_FAST::response& res)
+ {
+ CHECK_CORE_BUSY();
+ NOTIFY_RESPONSE_CHAIN_ENTRY::request resp;
+
+ resp.start_height = req.start_height;
+ if(!m_core.find_blockchain_supplement(req.block_ids, resp))
+ {
+ res.status = "Failed";
+ return false;
+ }
+ res.current_height = resp.total_height;
+ res.start_height = resp.start_height;
+ res.m_block_ids = std::move(resp.m_block_ids);
+
+ res.status = CORE_RPC_STATUS_OK;
+ return true;
+ }
+ //------------------------------------------------------------------------------------------------------------------------------
bool core_rpc_server::on_get_random_outs(const COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::request& req, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS::response& res)
{
CHECK_CORE_BUSY();
diff --git a/src/rpc/core_rpc_server.h b/src/rpc/core_rpc_server.h
index 5c3707209..4bed148fe 100644
--- a/src/rpc/core_rpc_server.h
+++ b/src/rpc/core_rpc_server.h
@@ -75,6 +75,7 @@ namespace cryptonote
BEGIN_URI_MAP2()
MAP_URI_AUTO_JON2("/getheight", on_get_height, COMMAND_RPC_GET_HEIGHT)
MAP_URI_AUTO_BIN2("/getblocks.bin", on_get_blocks, COMMAND_RPC_GET_BLOCKS_FAST)
+ MAP_URI_AUTO_BIN2("/gethashes.bin", on_get_hashes, COMMAND_RPC_GET_HASHES_FAST)
MAP_URI_AUTO_BIN2("/get_o_indexes.bin", on_get_indexes, COMMAND_RPC_GET_TX_GLOBAL_OUTPUTS_INDEXES)
MAP_URI_AUTO_BIN2("/getrandom_outs.bin", on_get_random_outs, COMMAND_RPC_GET_RANDOM_OUTPUTS_FOR_AMOUNTS)
MAP_URI_AUTO_JON2("/gettransactions", on_get_transactions, COMMAND_RPC_GET_TRANSACTIONS)
@@ -115,6 +116,7 @@ namespace cryptonote
bool on_get_height(const COMMAND_RPC_GET_HEIGHT::request& req, COMMAND_RPC_GET_HEIGHT::response& res);
bool on_get_blocks(const COMMAND_RPC_GET_BLOCKS_FAST::request& req, COMMAND_RPC_GET_BLOCKS_FAST::response& res);
+ bool on_get_hashes(const COMMAND_RPC_GET_HASHES_FAST::request& req, COMMAND_RPC_GET_HASHES_FAST::response& res);
bool on_get_transactions(const COMMAND_RPC_GET_TRANSACTIONS::request& req, COMMAND_RPC_GET_TRANSACTIONS::response& res);
bool on_is_key_image_spent(const COMMAND_RPC_IS_KEY_IMAGE_SPENT::request& req, COMMAND_RPC_IS_KEY_IMAGE_SPENT::response& res);
bool on_get_indexes(const COMMAND_RPC_GET_TX_GLOBAL_OUTPUTS_INDEXES::request& req, COMMAND_RPC_GET_TX_GLOBAL_OUTPUTS_INDEXES::response& res);
diff --git a/src/rpc/core_rpc_server_commands_defs.h b/src/rpc/core_rpc_server_commands_defs.h
index 6067a28b7..392c7501f 100644
--- a/src/rpc/core_rpc_server_commands_defs.h
+++ b/src/rpc/core_rpc_server_commands_defs.h
@@ -89,6 +89,36 @@ namespace cryptonote
END_KV_SERIALIZE_MAP()
};
};
+
+ struct COMMAND_RPC_GET_HASHES_FAST
+ {
+
+ struct request
+ {
+ std::list<crypto::hash> block_ids; //*first 10 blocks id goes sequential, next goes in pow(2,n) offset, like 2, 4, 8, 16, 32, 64 and so on, and the last one is always genesis block */
+ uint64_t start_height;
+ BEGIN_KV_SERIALIZE_MAP()
+ KV_SERIALIZE_CONTAINER_POD_AS_BLOB(block_ids)
+ KV_SERIALIZE(start_height)
+ END_KV_SERIALIZE_MAP()
+ };
+
+ struct response
+ {
+ std::list<crypto::hash> m_block_ids;
+ uint64_t start_height;
+ uint64_t current_height;
+ std::string status;
+
+ BEGIN_KV_SERIALIZE_MAP()
+ KV_SERIALIZE_CONTAINER_POD_AS_BLOB(m_block_ids)
+ KV_SERIALIZE(start_height)
+ KV_SERIALIZE(current_height)
+ KV_SERIALIZE(status)
+ END_KV_SERIALIZE_MAP()
+ };
+ };
+
//-----------------------------------------------
struct COMMAND_RPC_GET_TRANSACTIONS
{
diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp
index 14da8d7a2..1c1e386ca 100644
--- a/src/wallet/wallet2.cpp
+++ b/src/wallet/wallet2.cpp
@@ -578,6 +578,24 @@ void wallet2::pull_blocks(uint64_t start_height, uint64_t &blocks_start_height,
blocks = res.blocks;
}
//----------------------------------------------------------------------------------------------------
+void wallet2::pull_hashes(uint64_t start_height, uint64_t &blocks_start_height, const std::list<crypto::hash> &short_chain_history, std::list<crypto::hash> &hashes)
+{
+ cryptonote::COMMAND_RPC_GET_HASHES_FAST::request req = AUTO_VAL_INIT(req);
+ cryptonote::COMMAND_RPC_GET_HASHES_FAST::response res = AUTO_VAL_INIT(res);
+ req.block_ids = short_chain_history;
+
+ req.start_height = start_height;
+ m_daemon_rpc_mutex.lock();
+ bool r = net_utils::invoke_http_bin_remote_command2(m_daemon_address + "/gethashes.bin", req, res, m_http_client, WALLET_RCP_CONNECTION_TIMEOUT);
+ m_daemon_rpc_mutex.unlock();
+ THROW_WALLET_EXCEPTION_IF(!r, error::no_connection_to_daemon, "gethashes.bin");
+ THROW_WALLET_EXCEPTION_IF(res.status == CORE_RPC_STATUS_BUSY, error::daemon_busy, "gethashes.bin");
+ THROW_WALLET_EXCEPTION_IF(res.status != CORE_RPC_STATUS_OK, error::get_hashes_error, res.status);
+
+ blocks_start_height = res.start_height;
+ hashes = res.m_block_ids;
+}
+//----------------------------------------------------------------------------------------------------
void wallet2::process_blocks(uint64_t start_height, const std::list<cryptonote::block_complete_entry> &blocks, uint64_t& blocks_added)
{
size_t current_index = start_height;
@@ -771,6 +789,60 @@ void wallet2::check_pending_txes()
}
}
}
+//----------------------------------------------------------------------------------------------------
+void wallet2::fast_refresh(uint64_t stop_height, uint64_t &blocks_start_height, std::list<crypto::hash> &short_chain_history)
+{
+ std::list<crypto::hash> hashes;
+ size_t current_index = m_blockchain.size();
+ while(current_index < stop_height)
+ {
+ pull_hashes(0, blocks_start_height, short_chain_history, hashes);
+ if (hashes.size() < 3)
+ return;
+ if (hashes.size() + current_index < stop_height) {
+ std::list<crypto::hash>::iterator right;
+ // drop early 3 off, skipping the genesis block
+ if (short_chain_history.size() > 3) {
+ right = short_chain_history.end();
+ std::advance(right,-1);
+ std::list<crypto::hash>::iterator left = right;
+ std::advance(left, -3);
+ short_chain_history.erase(left, right);
+ }
+ right = hashes.end();
+ // prepend 3 more
+ for (int i = 0; i<3; i++) {
+ right--;
+ short_chain_history.push_front(*right);
+ }
+ }
+ current_index = blocks_start_height;
+ BOOST_FOREACH(auto& bl_id, hashes)
+ {
+ if(current_index >= m_blockchain.size())
+ {
+ LOG_PRINT_L2( "Skipped block by height: " << current_index);
+ m_blockchain.push_back(bl_id);
+ ++m_local_bc_height;
+
+ if (0 != m_callback)
+ { // FIXME: this isn't right, but simplewallet just logs that we got a block.
+ cryptonote::block dummy;
+ m_callback->on_new_block(current_index, dummy);
+ }
+ }
+ else if(bl_id != m_blockchain[current_index])
+ {
+ //split detected here !!!
+ return;
+ }
+ ++current_index;
+ if (current_index >= stop_height)
+ return;
+ }
+ }
+}
+
//----------------------------------------------------------------------------------------------------
void wallet2::refresh(uint64_t start_height, uint64_t & blocks_fetched, bool& received_money)
{
@@ -786,7 +858,22 @@ void wallet2::refresh(uint64_t start_height, uint64_t & blocks_fetched, bool& re
// pull the first set of blocks
get_short_chain_history(short_chain_history);
+ if (start_height > m_blockchain.size() || m_refresh_from_block_height > m_blockchain.size()) {
+ if (!start_height)
+ start_height = m_refresh_from_block_height;
+ // we can shortcut by only pulling hashes up to the start_height
+ fast_refresh(start_height, blocks_start_height, short_chain_history);
+ // regenerate the history now that we've got a full set of hashes
+ short_chain_history.clear();
+ get_short_chain_history(short_chain_history);
+ start_height = 0;
+ // and then fall through to regular refresh processing
+ }
+
pull_blocks(start_height, blocks_start_height, short_chain_history, blocks);
+ // always reset start_height to 0 to force short_chain_ history to be used on
+ // subsequent pulls in this refresh.
+ start_height = 0;
m_run.store(true, std::memory_order_relaxed);
while(m_run.load(std::memory_order_relaxed))
diff --git a/src/wallet/wallet2.h b/src/wallet/wallet2.h
index 179d1553e..10532daf9 100644
--- a/src/wallet/wallet2.h
+++ b/src/wallet/wallet2.h
@@ -387,6 +387,8 @@ namespace tools
bool is_transfer_unlocked(const transfer_details& td) const;
bool clear();
void pull_blocks(uint64_t start_height, uint64_t& blocks_start_height, const std::list<crypto::hash> &short_chain_history, std::list<cryptonote::block_complete_entry> &blocks);
+ void pull_hashes(uint64_t start_height, uint64_t& blocks_start_height, const std::list<crypto::hash> &short_chain_history, std::list<crypto::hash> &hashes);
+ void fast_refresh(uint64_t stop_height, uint64_t &blocks_start_height, std::list<crypto::hash> &short_chain_history);
void pull_next_blocks(uint64_t start_height, uint64_t &blocks_start_height, std::list<crypto::hash> &short_chain_history, const std::list<cryptonote::block_complete_entry> &prev_blocks, std::list<cryptonote::block_complete_entry> &blocks, bool &error);
void process_blocks(uint64_t start_height, const std::list<cryptonote::block_complete_entry> &blocks, uint64_t& blocks_added);
uint64_t select_transfers(uint64_t needed_money, std::vector<size_t> unused_transfers_indices, std::list<transfer_container::iterator>& selected_transfers, bool trusted_daemon);
diff --git a/src/wallet/wallet_errors.h b/src/wallet/wallet_errors.h
index 3de97f49d..184d8a2a1 100644
--- a/src/wallet/wallet_errors.h
+++ b/src/wallet/wallet_errors.h
@@ -60,6 +60,7 @@ namespace tools
// acc_outs_lookup_error
// block_parse_error
// get_blocks_error
+ // get_hashes_error
// get_out_indexes_error
// tx_parse_error
// get_tx_pool_error
@@ -107,12 +108,14 @@ namespace tools
//----------------------------------------------------------------------------------------------------
const char* const failed_rpc_request_messages[] = {
"failed to get blocks",
+ "failed to get hashes",
"failed to get out indices",
"failed to get random outs"
};
enum failed_rpc_request_message_indices
{
get_blocks_error_message_index,
+ get_hashes_error_message_index,
get_out_indices_error_message_index,
get_random_outs_error_message_index
};
@@ -291,6 +294,8 @@ namespace tools
//----------------------------------------------------------------------------------------------------
typedef failed_rpc_request<refresh_error, get_blocks_error_message_index> get_blocks_error;
//----------------------------------------------------------------------------------------------------
+ typedef failed_rpc_request<refresh_error, get_hashes_error_message_index> get_hashes_error;
+ //----------------------------------------------------------------------------------------------------
typedef failed_rpc_request<refresh_error, get_out_indices_error_message_index> get_out_indices_error;
//----------------------------------------------------------------------------------------------------
struct tx_parse_error : public refresh_error
From b6e42c3276994f25dbe271d68092fb97ee1c7424 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@symas.com>
Date: Fri, 15 Apr 2016 16:30:06 +0100
Subject: [PATCH 2/3] Speed up new wallet refresh
Use the current blockchain height as the refresh_from_block_height.
---
src/simplewallet/simplewallet.cpp | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/src/simplewallet/simplewallet.cpp b/src/simplewallet/simplewallet.cpp
index 29c92e049..46a8095fc 100644
--- a/src/simplewallet/simplewallet.cpp
+++ b/src/simplewallet/simplewallet.cpp
@@ -1410,6 +1410,12 @@ bool simple_wallet::new_wallet(const std::string &wallet_file, const std::string
}
m_wallet->init(m_daemon_address);
+ // for a totally new account, we don't care about older blocks.
+ if (!m_restore_deterministic_wallet)
+ {
+ std::string err;
+ m_wallet->set_refresh_from_block_height(get_daemon_blockchain_height(err));
+ }
// convert rng value to electrum-style word list
std::string electrum_words;
From 19fe8ae3ef1aa46ae8fdd4e4d6862510390ddab7 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@symas.com>
Date: Fri, 15 Apr 2016 19:10:20 +0100
Subject: [PATCH 3/3] Add --restore-height option
For specifying the block height from which to start a restore
---
src/simplewallet/simplewallet.cpp | 24 ++++++++++++++++++++++++
src/simplewallet/simplewallet.h | 1 +
2 files changed, 25 insertions(+)
diff --git a/src/simplewallet/simplewallet.cpp b/src/simplewallet/simplewallet.cpp
index 46a8095fc..f6e5f2cc6 100644
--- a/src/simplewallet/simplewallet.cpp
+++ b/src/simplewallet/simplewallet.cpp
@@ -97,6 +97,7 @@ namespace
const command_line::arg_descriptor<bool> arg_testnet = {"testnet", sw::tr("For testnet. Daemon must also be launched with --testnet flag"), false};
const command_line::arg_descriptor<bool> arg_restricted = {"restricted-rpc", sw::tr("Restricts RPC to view-only commands"), false};
const command_line::arg_descriptor<bool> arg_trusted_daemon = {"trusted-daemon", sw::tr("Enable commands which rely on a trusted daemon"), false};
+ const command_line::arg_descriptor<uint64_t> arg_restore_height = {"restore-height", sw::tr("Restore from specific blockchain height"), 0};
const command_line::arg_descriptor< std::vector<std::string> > arg_command = {"command", ""};
@@ -1136,6 +1137,22 @@ bool simple_wallet::init(const boost::program_options::variables_map& vm)
return false;
}
}
+ if (!m_restore_height)
+ {
+ std::string heightstr = command_line::input_line("Restore from specific blockchain height (optional, default 0): ");
+ if (std::cin.eof())
+ return false;
+ if (heightstr.size())
+ {
+ try {
+ m_restore_height = boost::lexical_cast<uint64_t>(heightstr);
+ }
+ catch (boost::bad_lexical_cast &) {
+ fail_msg_writer() << tr("bad m_restore_height parameter:") << " " << heightstr;
+ return false;
+ }
+ }
+ }
if (!m_generate_from_view_key.empty())
{
// parse address
@@ -1306,6 +1323,7 @@ bool simple_wallet::handle_command_line(const boost::program_options::variables_
m_restore_deterministic_wallet = command_line::get_arg(vm, arg_restore_deterministic_wallet);
m_non_deterministic = command_line::get_arg(vm, arg_non_deterministic);
m_trusted_daemon = command_line::get_arg(vm, arg_trusted_daemon);
+ m_restore_height = command_line::get_arg(vm, arg_restore_height);
return true;
}
@@ -1415,6 +1433,9 @@ bool simple_wallet::new_wallet(const std::string &wallet_file, const std::string
{
std::string err;
m_wallet->set_refresh_from_block_height(get_daemon_blockchain_height(err));
+ } else if (m_restore_height)
+ {
+ m_wallet->set_refresh_from_block_height(m_restore_height);
}
// convert rng value to electrum-style word list
@@ -1463,6 +1484,7 @@ bool simple_wallet::new_wallet(const std::string &wallet_file, const std::string
}
m_wallet->init(m_daemon_address);
+ m_wallet->set_refresh_from_block_height(m_restore_height);
return true;
}
@@ -1488,6 +1510,7 @@ bool simple_wallet::new_wallet(const std::string &wallet_file, const std::string
}
m_wallet->init(m_daemon_address);
+ m_wallet->set_refresh_from_block_height(m_restore_height);
return true;
}
@@ -2927,6 +2950,7 @@ int main(int argc, char* argv[])
command_line::add_arg(desc_params, arg_testnet);
command_line::add_arg(desc_params, arg_restricted);
command_line::add_arg(desc_params, arg_trusted_daemon);
+ command_line::add_arg(desc_params, arg_restore_height);
tools::wallet_rpc_server::init_options(desc_params);
po::positional_options_description positional_options;
diff --git a/src/simplewallet/simplewallet.h b/src/simplewallet/simplewallet.h
index 21bbfa566..87d502d67 100644
--- a/src/simplewallet/simplewallet.h
+++ b/src/simplewallet/simplewallet.h
@@ -231,6 +231,7 @@ namespace cryptonote
bool m_restore_deterministic_wallet; // recover flag
bool m_non_deterministic; // old 2-random generation
bool m_trusted_daemon;
+ uint64_t m_restore_height; // optional
std::string m_daemon_address;
std::string m_daemon_host;