monero-project/monero
monero-project/monero
1
0
Fork 0
mirror of https://github.com/monero-project/monero.git synced 2024-12-23 03:59:33 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

monerod can now sync from pruned blocks

Browse source 
If the peer (whether pruned or not itself) supports sending pruned blocks
to syncing nodes, the pruned version will be sent along with the hash
of the pruned data and the block weight. The original tx hashes can be
reconstructed from the pruned txes and theur prunable data hash. Those
hashes and the block weights are hashes and checked against the set of
precompiled hashes, ensuring the data we received is the original data.
It is currently not possible to use this system when not using the set
of precompiled hashes, since block weights can not otherwise be checked
for validity.

This is off by default for now, and is enabled by --sync-pruned-blocks
This commit is contained in:
moneromooo-monero 2019-09-16 12:18:34 +00:00
parent d0d76f771a
commit 8330e772f1
No known key found for this signature in database
GPG key ID: 686F07454D6CEFC3
30 changed files with 731 additions and 195 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
src/blockchain_db/blockchain_db.h
View file

@ -156,7 +156,8 @@ struct txpool_tx_meta_t
uint8_t relayed;
uint8_t do_not_relay;
uint8_t double_spend_seen: 1;
uint8_t bf_padding: 7;
uint8_t pruned: 1;
uint8_t bf_padding: 6;
uint8_t padding[76]; // till 192 bytes
};

6
src/blockchain_db/lmdb/db_lmdb.cpp
View file

@ -2056,7 +2056,7 @@ bool BlockchainLMDB::prune_worker(int mode, uint32_t pruning_seed)
++n_prunable_records;
result = mdb_cursor_get(c_txs_prunable, &k, &v, MDB_SET);
if (result == MDB_NOTFOUND)
MWARNING("Already pruned at height " << block_height << "/" << blockchain_height);
MDEBUG("Already pruned at height " << block_height << "/" << blockchain_height);
else if (result)
throw0(DB_ERROR(lmdb_error("Failed to find transaction prunable data: ", result).c_str()));
else
@ -2152,7 +2152,7 @@ bool BlockchainLMDB::prune_worker(int mode, uint32_t pruning_seed)
{
++n_prunable_records;
if (result == MDB_NOTFOUND)
MWARNING("Already pruned at height " << block_height << "/" << blockchain_height);
MDEBUG("Already pruned at height " << block_height << "/" << blockchain_height);
else
{
MDEBUG("Pruning at height " << block_height << "/" << blockchain_height);
@ -2994,6 +2994,8 @@ bool BlockchainLMDB::get_tx_blob(const crypto::hash& h, cryptonote::blobdata &bd
return false;
else if (get_result)
throw0(DB_ERROR(lmdb_error("DB error attempting to fetch tx from hash", get_result).c_str()));
else if (result1.mv_size == 0)
return false;
bd.assign(reinterpret_cast<char*>(result0.mv_data), result0.mv_size);
bd.append(reinterpret_cast<char*>(result1.mv_data), result1.mv_size);

16
src/blockchain_utilities/blockchain_import.cpp
View file

@ -191,7 +191,7 @@ int check_flush(cryptonote::core &core, std::vector<block_complete_entry> &block
}
hashes.push_back(cryptonote::get_block_hash(block));
}
core.prevalidate_block_hashes(core.get_blockchain_storage().get_db().height(), hashes);
core.prevalidate_block_hashes(core.get_blockchain_storage().get_db().height(), hashes, {});
std::vector<block> pblocks;
if (!core.prepare_handle_incoming_blocks(blocks, pblocks))
@ -217,7 +217,7 @@ int check_flush(cryptonote::core &core, std::vector<block_complete_entry> &block
if(tvc.m_verifivation_failed)
{
MERROR("transaction verification failed, tx_id = "
<< epee::string_tools::pod_to_hex(get_blob_hash(tx_blob)));
<< epee::string_tools::pod_to_hex(get_blob_hash(tx_blob.blob)));
core.cleanup_handle_incoming_blocks();
return 1;
}
@ -468,13 +468,17 @@ int import_from_file(cryptonote::core& core, const std::string& import_file_path
{
cryptonote::blobdata block;
cryptonote::block_to_blob(bp.block, block);
std::vector<cryptonote::blobdata> txs;
std::vector<tx_blob_entry> txs;
for (const auto &tx: bp.txs)
{
txs.push_back(cryptonote::blobdata());
cryptonote::tx_to_blob(tx, txs.back());
txs.push_back({cryptonote::blobdata(), crypto::null_hash});
cryptonote::tx_to_blob(tx, txs.back().blob);
}
blocks.push_back({block, txs});
block_complete_entry bce;
bce.pruned = false;
bce.block = std::move(block);
bce.txs = std::move(txs);
blocks.push_back(bce);
int ret = check_flush(core, blocks, false);
if (ret)
{

15
src/blockchain_utilities/blocksdat_file.cpp
View file

@ -99,17 +99,23 @@ bool BlocksdatFile::initialize_file(uint64_t block_stop)
return true;
}
void BlocksdatFile::write_block(const crypto::hash& block_hash)
void BlocksdatFile::write_block(const crypto::hash& block_hash, uint64_t weight)
{
m_hashes.push_back(block_hash);
m_weights.push_back(weight);
while (m_hashes.size() >= HASH_OF_HASHES_STEP)
{
crypto::hash hash;
crypto::cn_fast_hash(m_hashes.data(), HASH_OF_HASHES_STEP * sizeof(crypto::hash), hash);
memmove(m_hashes.data(), m_hashes.data() + HASH_OF_HASHES_STEP, (m_hashes.size() - HASH_OF_HASHES_STEP) * sizeof(crypto::hash));
m_hashes.resize(m_hashes.size() - HASH_OF_HASHES_STEP);
const std::string data(hash.data, sizeof(hash));
*m_raw_data_file << data;
const std::string data_hashes(hash.data, sizeof(hash));
*m_raw_data_file << data_hashes;
crypto::cn_fast_hash(m_weights.data(), HASH_OF_HASHES_STEP * sizeof(uint64_t), hash);
memmove(m_weights.data(), m_weights.data() + HASH_OF_HASHES_STEP, (m_weights.size() - HASH_OF_HASHES_STEP) * sizeof(uint64_t));
m_weights.resize(m_weights.size() - HASH_OF_HASHES_STEP);
const std::string data_weights(hash.data, sizeof(hash));
*m_raw_data_file << data_weights;
}
}
@ -154,7 +160,8 @@ bool BlocksdatFile::store_blockchain_raw(Blockchain* _blockchain_storage, tx_mem
{
// this method's height refers to 0-based height (genesis block = height 0)
crypto::hash hash = m_blockchain_storage->get_block_id_by_height(m_cur_height);
write_block(hash);
uint64_t weight = m_blockchain_storage->get_db().get_block_weight(m_cur_height);
write_block(hash, weight);
if (m_cur_height % NUM_BLOCKS_PER_CHUNK == 0) {
num_blocks_written += NUM_BLOCKS_PER_CHUNK;
}

3
src/blockchain_utilities/blocksdat_file.h
View file

@ -72,10 +72,11 @@ protected:
bool open_writer(const boost::filesystem::path& file_path, uint64_t block_stop);
bool initialize_file(uint64_t block_stop);
bool close();
void write_block(const crypto::hash &block_hash);
void write_block(const crypto::hash &block_hash, uint64_t weight);
private:
uint64_t m_cur_height; // tracks current height during export
std::vector<crypto::hash> m_hashes;
std::vector<uint64_t> m_weights;
};

BIN
src/blocks/checkpoints.dat
View file

Binary file not shown.

2
src/cryptonote_basic/connection_context.h
View file

@ -55,7 +55,7 @@ namespace cryptonote
};
state m_state;
std::vector<crypto::hash> m_needed_objects;
std::vector<std::pair<crypto::hash, uint64_t>> m_needed_objects;
std::unordered_set<crypto::hash> m_requested_objects;
uint64_t m_remote_blockchain_height;
uint64_t m_last_response_height;

74
src/cryptonote_basic/cryptonote_basic.h
View file

@ -195,6 +195,7 @@ namespace cryptonote
private:
// hash cash
mutable std::atomic<bool> hash_valid;
mutable std::atomic<bool> prunable_hash_valid;
mutable std::atomic<bool> blob_size_valid;
public:
@ -203,6 +204,7 @@ namespace cryptonote
// hash cash
mutable crypto::hash hash;
mutable crypto::hash prunable_hash;
mutable size_t blob_size;
bool pruned;
@ -211,22 +213,26 @@ namespace cryptonote
std::atomic<unsigned int> prefix_size;
transaction();
transaction(const transaction &t): transaction_prefix(t), hash_valid(false), blob_size_valid(false), signatures(t.signatures), rct_signatures(t.rct_signatures), pruned(t.pruned), unprunable_size(t.unprunable_size.load()), prefix_size(t.prefix_size.load()) { if (t.is_hash_valid()) { hash = t.hash; set_hash_valid(true); } if (t.is_blob_size_valid()) { blob_size = t.blob_size; set_blob_size_valid(true); } }
transaction &operator=(const transaction &t) { transaction_prefix::operator=(t); set_hash_valid(false); set_blob_size_valid(false); signatures = t.signatures; rct_signatures = t.rct_signatures; if (t.is_hash_valid()) { hash = t.hash; set_hash_valid(true); } if (t.is_blob_size_valid()) { blob_size = t.blob_size; set_blob_size_valid(true); } pruned = t.pruned; unprunable_size = t.unprunable_size.load(); prefix_size = t.prefix_size.load(); return *this; }
transaction(const transaction &t);
transaction &operator=(const transaction &t);
virtual ~transaction();
void set_null();
void invalidate_hashes();
bool is_hash_valid() const { return hash_valid.load(std::memory_order_acquire); }
void set_hash_valid(bool v) const { hash_valid.store(v,std::memory_order_release); }
bool is_prunable_hash_valid() const { return prunable_hash_valid.load(std::memory_order_acquire); }
void set_prunable_hash_valid(bool v) const { prunable_hash_valid.store(v,std::memory_order_release); }
bool is_blob_size_valid() const { return blob_size_valid.load(std::memory_order_acquire); }
void set_blob_size_valid(bool v) const { blob_size_valid.store(v,std::memory_order_release); }
void set_hash(const crypto::hash &h) { hash = h; set_hash_valid(true); }
void set_blob_size(size_t sz) { blob_size = sz; set_blob_size_valid(true); }
void set_hash(const crypto::hash &h) const { hash = h; set_hash_valid(true); }
void set_prunable_hash(const crypto::hash &h) const { prunable_hash = h; set_prunable_hash_valid(true); }
void set_blob_size(size_t sz) const { blob_size = sz; set_blob_size_valid(true); }
BEGIN_SERIALIZE_OBJECT()
if (!typename Archive<W>::is_saving())
{
set_hash_valid(false);
set_prunable_hash_valid(false);
set_blob_size_valid(false);
}
@ -327,6 +333,63 @@ namespace cryptonote
static size_t get_signature_size(const txin_v& tx_in);
};
inline transaction::transaction(const transaction &t):
transaction_prefix(t),
hash_valid(false),
prunable_hash_valid(false),
blob_size_valid(false),
signatures(t.signatures),
rct_signatures(t.rct_signatures),
pruned(t.pruned),
unprunable_size(t.unprunable_size.load()),
prefix_size(t.prefix_size.load())
{
if (t.is_hash_valid())
{
hash = t.hash;
set_hash_valid(true);
}
if (t.is_blob_size_valid())
{
blob_size = t.blob_size;
set_blob_size_valid(true);
}
if (t.is_prunable_hash_valid())
{
prunable_hash = t.prunable_hash;
set_prunable_hash_valid(true);
}
}
inline transaction &transaction::operator=(const transaction &t)
{
transaction_prefix::operator=(t);
set_hash_valid(false);
set_prunable_hash_valid(false);
set_blob_size_valid(false);
signatures = t.signatures;
rct_signatures = t.rct_signatures;
if (t.is_hash_valid())
{
hash = t.hash;
set_hash_valid(true);
}
if (t.is_prunable_hash_valid())
{
prunable_hash = t.prunable_hash;
set_prunable_hash_valid(true);
}
if (t.is_blob_size_valid())
{
blob_size = t.blob_size;
set_blob_size_valid(true);
}
pruned = t.pruned;
unprunable_size = t.unprunable_size.load();
prefix_size = t.prefix_size.load();
return *this;
}
inline
transaction::transaction()
@ -346,6 +409,7 @@ namespace cryptonote
signatures.clear();
rct_signatures.type = rct::RCTTypeNull;
set_hash_valid(false);
set_prunable_hash_valid(false);
set_blob_size_valid(false);
pruned = false;
unprunable_size = 0;
@ -356,6 +420,7 @@ namespace cryptonote
void transaction::invalidate_hashes()
{
set_hash_valid(false);
set_prunable_hash_valid(false);
set_blob_size_valid(false);
}
@ -408,6 +473,7 @@ namespace cryptonote
void invalidate_hashes() { set_hash_valid(false); }
bool is_hash_valid() const { return hash_valid.load(std::memory_order_acquire); }
void set_hash_valid(bool v) const { hash_valid.store(v,std::memory_order_release); }
void set_hash(const crypto::hash &h) const { hash = h; set_hash_valid(true); }
transaction miner_tx;
std::vector<crypto::hash> tx_hashes;

33
src/cryptonote_basic/cryptonote_format_utils.cpp
View file

@ -1011,7 +1011,19 @@ namespace cryptonote
crypto::hash get_transaction_prunable_hash(const transaction& t, const cryptonote::blobdata *blobdata)
{
crypto::hash res;
if (t.is_prunable_hash_valid())
{
#ifdef ENABLE_HASH_CASH_INTEGRITY_CHECK
CHECK_AND_ASSERT_THROW_MES(!calculate_transaction_prunable_hash(t, blobdata, res) || t.hash == res, "tx hash cash integrity failure");
#endif
res = t.prunable_hash;
++tx_hashes_cached_count;
return res;
}
++tx_hashes_calculated_count;
CHECK_AND_ASSERT_THROW_MES(calculate_transaction_prunable_hash(t, blobdata, res), "Failed to calculate tx prunable hash");
t.set_prunable_hash(res);
return res;
}
//---------------------------------------------------------------
@ -1047,11 +1059,14 @@ namespace cryptonote
// the tx hash is the hash of the 3 hashes
crypto::hash res = cn_fast_hash(hashes, sizeof(hashes));
t.set_hash(res);
return res;
}
//---------------------------------------------------------------
bool calculate_transaction_hash(const transaction& t, crypto::hash& res, size_t* blob_size)
{
CHECK_AND_ASSERT_MES(!t.pruned, false, "Cannot calculate the hash of a pruned transaction");
// v1 transactions hash the entire blob
if (t.version == 1)
{
@ -1091,8 +1106,7 @@ namespace cryptonote
{
if (!t.is_blob_size_valid())
{
t.blob_size = blob.size();
t.set_blob_size_valid(true);
t.set_blob_size(blob.size());
}
*blob_size = t.blob_size;
}
@ -1112,8 +1126,7 @@ namespace cryptonote
{
if (!t.is_blob_size_valid())
{
t.blob_size = get_object_blobsize(t);
t.set_blob_size_valid(true);
t.set_blob_size(get_object_blobsize(t));
}
*blob_size = t.blob_size;
}
@ -1124,12 +1137,10 @@ namespace cryptonote
bool ret = calculate_transaction_hash(t, res, blob_size);
if (!ret)
return false;
t.hash = res;
t.set_hash_valid(true);
t.set_hash(res);
if (blob_size)
{
t.blob_size = *blob_size;
t.set_blob_size_valid(true);
t.set_blob_size(*blob_size);
}
return true;
}
@ -1206,8 +1217,7 @@ namespace cryptonote
bool ret = calculate_block_hash(b, res);
if (!ret)
return false;
b.hash = res;
b.set_hash_valid(true);
b.set_hash(res);
return true;
}
//---------------------------------------------------------------
@ -1273,8 +1283,7 @@ namespace cryptonote
{
calculate_block_hash(b, *block_hash, &b_blob);
++block_hashes_calculated_count;
b.hash = *block_hash;
b.set_hash_valid(true);
b.set_hash(*block_hash);
}
return true;
}

2
src/cryptonote_config.h
View file

@ -163,7 +163,7 @@
#define PER_KB_FEE_QUANTIZATION_DECIMALS 8
#define HASH_OF_HASHES_STEP 256
#define HASH_OF_HASHES_STEP 512
#define DEFAULT_TXPOOL_MAX_WEIGHT 648000000ull // 3 days at 300000, in bytes

244
src/cryptonote_core/blockchain.cpp
View file

@ -1255,7 +1255,7 @@ bool Blockchain::validate_miner_transaction(const block& b, size_t cumulative_bl
}
if(base_reward + fee < money_in_use)
{
MERROR_VER("coinbase transaction spend too much money (" << print_money(money_in_use) << "). Block reward is " << print_money(base_reward + fee) << "(" << print_money(base_reward) << "+" << print_money(fee) << ")");
MERROR_VER("coinbase transaction spend too much money (" << print_money(money_in_use) << "). Block reward is " << print_money(base_reward + fee) << "(" << print_money(base_reward) << "+" << print_money(fee) << "), cumulative_block_weight " << cumulative_block_weight);
return false;
}
// From hard fork 2, we allow a miner to claim less block reward than is allowed, in case a miner wants less dust
@ -1907,8 +1907,9 @@ bool Blockchain::handle_get_objects(NOTIFY_REQUEST_GET_OBJECTS::request& arg, NO
std::vector<std::pair<cryptonote::blobdata,block>> blocks;
get_blocks(arg.blocks, blocks, rsp.missed_ids);
for (auto& bl: blocks)
for (size_t i = 0; i < blocks.size(); ++i)
{
auto& bl = blocks[i];
std::vector<crypto::hash> missed_tx_ids;
rsp.blocks.push_back(block_complete_entry());
@ -1916,8 +1917,8 @@ bool Blockchain::handle_get_objects(NOTIFY_REQUEST_GET_OBJECTS::request& arg, NO
// FIXME: s/rsp.missed_ids/missed_tx_id/ ? Seems like rsp.missed_ids
// is for missed blocks, not missed transactions as well.
get_transactions_blobs(bl.second.tx_hashes, e.txs, missed_tx_ids);
e.pruned = arg.prune;
get_transactions_blobs(bl.second.tx_hashes, e.txs, missed_tx_ids, arg.prune);
if (missed_tx_ids.size() != 0)
{
// do not display an error if the peer asked for an unpruned block which we are not meant to have
@ -1938,6 +1939,9 @@ bool Blockchain::handle_get_objects(NOTIFY_REQUEST_GET_OBJECTS::request& arg, NO
//pack block
e.block = std::move(bl.first);
e.block_weight = 0;
if (arg.prune && m_db->block_exists(arg.blocks[i]))
e.block_weight = m_db->get_block_weight(m_db->get_block_height(arg.blocks[i]));
}
return true;
@ -2216,23 +2220,95 @@ bool Blockchain::get_blocks(const t_ids_container& block_ids, t_blocks_container
return true;
}
//------------------------------------------------------------------
static bool fill(BlockchainDB *db, const crypto::hash &tx_hash, cryptonote::blobdata &tx, bool pruned)
{
if (pruned)
{
if (!db->get_pruned_tx_blob(tx_hash, tx))
{
MDEBUG("Pruned transaction blob not found for " << tx_hash);
return false;
}
}
else
{
if (!db->get_tx_blob(tx_hash, tx))
{
MDEBUG("Transaction blob not found for " << tx_hash);
return false;
}
}
return true;
}
//------------------------------------------------------------------
static bool fill(BlockchainDB *db, const crypto::hash &tx_hash, tx_blob_entry &tx, bool pruned)
{
if (!fill(db, tx_hash, tx.blob, pruned))
return false;
if (pruned)
{
if (is_v1_tx(tx.blob))
{
// v1 txes aren't pruned, so fetch the whole thing
cryptonote::blobdata prunable_blob;
if (!db->get_prunable_tx_blob(tx_hash, prunable_blob))
{
MDEBUG("Prunable transaction blob not found for " << tx_hash);
return false;
}
tx.blob.append(prunable_blob);
tx.prunable_hash = crypto::null_hash;
}
else
{
if (!db->get_prunable_tx_hash(tx_hash, tx.prunable_hash))
{
MDEBUG("Prunable transaction data hash not found for " << tx_hash);
return false;
}
}
}
return true;
}
//------------------------------------------------------------------
//TODO: return type should be void, throw on exception
// alternatively, return true only if no transactions missed
template<class t_ids_container, class t_tx_container, class t_missed_container>
bool Blockchain::get_transactions_blobs(const t_ids_container& txs_ids, t_tx_container& txs, t_missed_container& missed_txs, bool pruned) const
bool Blockchain::get_transactions_blobs(const std::vector<crypto::hash>& txs_ids, std::vector<cryptonote::blobdata>& txs, std::vector<crypto::hash>& missed_txs, bool pruned) const
{
LOG_PRINT_L3("Blockchain::" << __func__);
CRITICAL_REGION_LOCAL(m_blockchain_lock);
reserve_container(txs, txs_ids.size());
txs.reserve(txs_ids.size());
for (const auto& tx_hash : txs_ids)
{
try
{
cryptonote::blobdata tx;
if (pruned && m_db->get_pruned_tx_blob(tx_hash, tx))
if (fill(m_db, tx_hash, tx, pruned))
txs.push_back(std::move(tx));
else if (!pruned && m_db->get_tx_blob(tx_hash, tx))
else
missed_txs.push_back(tx_hash);
}
catch (const std::exception& e)
{
return false;
}
}
return true;
}
//------------------------------------------------------------------
bool Blockchain::get_transactions_blobs(const std::vector<crypto::hash>& txs_ids, std::vector<tx_blob_entry>& txs, std::vector<crypto::hash>& missed_txs, bool pruned) const
{
LOG_PRINT_L3("Blockchain::" << __func__);
CRITICAL_REGION_LOCAL(m_blockchain_lock);
txs.reserve(txs_ids.size());
for (const auto& tx_hash : txs_ids)
{
try
{
tx_blob_entry tx;
if (fill(m_db, tx_hash, tx, pruned))
txs.push_back(std::move(tx));
else
missed_txs.push_back(tx_hash);
@ -2325,7 +2401,7 @@ bool Blockchain::get_transactions(const t_ids_container& txs_ids, t_tx_container
// Find the split point between us and foreign blockchain and return
// (by reference) the most recent common block hash along with up to
// BLOCKS_IDS_SYNCHRONIZING_DEFAULT_COUNT additional (more recent) hashes.
bool Blockchain::find_blockchain_supplement(const std::list<crypto::hash>& qblock_ids, std::vector<crypto::hash>& hashes, uint64_t& start_height, uint64_t& current_height, bool clip_pruned) const
bool Blockchain::find_blockchain_supplement(const std::list<crypto::hash>& qblock_ids, std::vector<crypto::hash>& hashes, std::vector<uint64_t>* weights, uint64_t& start_height, uint64_t& current_height, bool clip_pruned) const
{
LOG_PRINT_L3("Blockchain::" << __func__);
CRITICAL_REGION_LOCAL(m_blockchain_lock);
@ -2342,25 +2418,34 @@ bool Blockchain::find_blockchain_supplement(const std::list<crypto::hash>& qbloc
if (clip_pruned)
{
const uint32_t pruning_seed = get_blockchain_pruning_seed();
start_height = tools::get_next_unpruned_block_height(start_height, current_height, pruning_seed);
if (start_height < tools::get_next_unpruned_block_height(start_height, current_height, pruning_seed))
{
MDEBUG("We only have a pruned version of the common ancestor");
return false;
}
stop_height = tools::get_next_pruned_block_height(start_height, current_height, pruning_seed);
}
size_t count = 0;
hashes.reserve(std::min((size_t)(stop_height - start_height), (size_t)BLOCKS_IDS_SYNCHRONIZING_DEFAULT_COUNT));
const size_t reserve = std::min((size_t)(stop_height - start_height), (size_t)BLOCKS_IDS_SYNCHRONIZING_DEFAULT_COUNT);
hashes.reserve(reserve);
if (weights)
weights->reserve(reserve);
for(size_t i = start_height; i < stop_height && count < BLOCKS_IDS_SYNCHRONIZING_DEFAULT_COUNT; i++, count++)
{
hashes.push_back(m_db->get_block_hash_from_height(i));
if (weights)
weights->push_back(m_db->get_block_weight(i));
}
return true;
}
bool Blockchain::find_blockchain_supplement(const std::list<crypto::hash>& qblock_ids, NOTIFY_RESPONSE_CHAIN_ENTRY::request& resp) const
bool Blockchain::find_blockchain_supplement(const std::list<crypto::hash>& qblock_ids, bool clip_pruned, NOTIFY_RESPONSE_CHAIN_ENTRY::request& resp) const
{
LOG_PRINT_L3("Blockchain::" << __func__);
CRITICAL_REGION_LOCAL(m_blockchain_lock);
bool result = find_blockchain_supplement(qblock_ids, resp.m_block_ids, resp.start_height, resp.total_height, true);
bool result = find_blockchain_supplement(qblock_ids, resp.m_block_ids, &resp.m_block_weights, resp.start_height, resp.total_height, clip_pruned);
if (result)
{
cryptonote::difficulty_type wide_cumulative_difficulty = m_db->get_block_cumulative_difficulty(resp.total_height - 1);
@ -2799,18 +2884,24 @@ bool Blockchain::expand_transaction_2(transaction &tx, const crypto::hash &tx_pr
// II
if (rv.type == rct::RCTTypeFull)
{
rv.p.MGs.resize(1);
rv.p.MGs[0].II.resize(tx.vin.size());
for (size_t n = 0; n < tx.vin.size(); ++n)
rv.p.MGs[0].II[n] = rct::ki2rct(boost::get<txin_to_key>(tx.vin[n]).k_image);
if (!tx.pruned)
{
rv.p.MGs.resize(1);
rv.p.MGs[0].II.resize(tx.vin.size());
for (size_t n = 0; n < tx.vin.size(); ++n)
rv.p.MGs[0].II[n] = rct::ki2rct(boost::get<txin_to_key>(tx.vin[n]).k_image);
}
}
else if (rv.type == rct::RCTTypeSimple || rv.type == rct::RCTTypeBulletproof || rv.type == rct::RCTTypeBulletproof2)
{
CHECK_AND_ASSERT_MES(rv.p.MGs.size() == tx.vin.size(), false, "Bad MGs size");
for (size_t n = 0; n < tx.vin.size(); ++n)
if (!tx.pruned)
{
rv.p.MGs[n].II.resize(1);
rv.p.MGs[n].II[0] = rct::ki2rct(boost::get<txin_to_key>(tx.vin[n]).k_image);
CHECK_AND_ASSERT_MES(rv.p.MGs.size() == tx.vin.size(), false, "Bad MGs size");
for (size_t n = 0; n < tx.vin.size(); ++n)
{
rv.p.MGs[n].II.resize(1);
rv.p.MGs[n].II[0] = rct::ki2rct(boost::get<txin_to_key>(tx.vin[n]).k_image);
}
}
}
else
@ -2836,6 +2927,10 @@ bool Blockchain::check_tx_inputs(transaction& tx, tx_verification_context &tvc,
if(pmax_used_block_height)
*pmax_used_block_height = 0;
// pruned txes are skipped, as they're only allowed in sync-pruned-blocks mode, which is within the builtin hashes
if (tx.pruned)
return true;
crypto::hash tx_prefix_hash = get_transaction_prefix_hash(tx);
const uint8_t hf_version = m_hardfork->get_current_version();
@ -3539,9 +3634,9 @@ bool Blockchain::flush_txes_from_pool(const std::vector<crypto::hash> &txids)
cryptonote::blobdata txblob;
size_t tx_weight;
uint64_t fee;
bool relayed, do_not_relay, double_spend_seen;
bool relayed, do_not_relay, double_spend_seen, pruned;
MINFO("Removing txid " << txid << " from the pool");
if(m_tx_pool.have_tx(txid) && !m_tx_pool.take_tx(txid, tx, txblob, tx_weight, fee, relayed, do_not_relay, double_spend_seen))
if(m_tx_pool.have_tx(txid) && !m_tx_pool.take_tx(txid, tx, txblob, tx_weight, fee, relayed, do_not_relay, double_spend_seen, pruned))
{
MERROR("Failed to remove txid " << txid << " from the pool");
res = false;
@ -3639,7 +3734,7 @@ leave:
#if defined(PER_BLOCK_CHECKPOINT)
if (blockchain_height < m_blocks_hash_check.size())
{
const auto &expected_hash = m_blocks_hash_check[blockchain_height];
const auto &expected_hash = m_blocks_hash_check[blockchain_height].first;
if (expected_hash != crypto::null_hash)
{
if (memcmp(&id, &expected_hash, sizeof(hash)) != 0)
@ -3713,6 +3808,7 @@ leave:
uint64_t t_exists = 0;
uint64_t t_pool = 0;
uint64_t t_dblspnd = 0;
uint64_t n_pruned = 0;
TIME_MEASURE_FINISH(t3);
// XXX old code adds miner tx here
@ -3728,7 +3824,7 @@ leave:
blobdata txblob;
size_t tx_weight = 0;
uint64_t fee = 0;
bool relayed = false, do_not_relay = false, double_spend_seen = false;
bool relayed = false, do_not_relay = false, double_spend_seen = false, pruned = false;
TIME_MEASURE_START(aa);
// XXX old code does not check whether tx exists
@ -3745,13 +3841,15 @@ leave:
TIME_MEASURE_START(bb);
// get transaction with hash <tx_id> from tx_pool
if(!m_tx_pool.take_tx(tx_id, tx_tmp, txblob, tx_weight, fee, relayed, do_not_relay, double_spend_seen))
if(!m_tx_pool.take_tx(tx_id, tx_tmp, txblob, tx_weight, fee, relayed, do_not_relay, double_spend_seen, pruned))
{
MERROR_VER("Block with id: " << id << " has at least one unknown transaction with id: " << tx_id);
bvc.m_verifivation_failed = true;
return_tx_to_pool(txs);
goto leave;
}
if (pruned)
++n_pruned;
TIME_MEASURE_FINISH(bb);
t_pool += bb;
@ -3822,6 +3920,17 @@ leave:
cumulative_block_weight += tx_weight;
}
// if we were syncing pruned blocks
if (n_pruned > 0)
{
if (blockchain_height >= m_blocks_hash_check.size() || m_blocks_hash_check[blockchain_height].second == 0)
{
MERROR("Block at " << blockchain_height << " is pruned, but we do not have a weight for it");
goto leave;
}
cumulative_block_weight = m_blocks_hash_check[blockchain_height].second;
}
m_blocks_txs_check.clear();
TIME_MEASURE_START(vmt);
@ -4262,11 +4371,13 @@ void Blockchain::output_scan_worker(const uint64_t amount, const std::vector<uin
}
}
uint64_t Blockchain::prevalidate_block_hashes(uint64_t height, const std::vector<crypto::hash> &hashes)
uint64_t Blockchain::prevalidate_block_hashes(uint64_t height, const std::vector<crypto::hash> &hashes, const std::vector<uint64_t> &weights)
{
// new: . . . . . X X X X X . . . . . .
// pre: A A A A B B B B C C C C D D D D
CHECK_AND_ASSERT_MES(weights.empty() || weights.size() == hashes.size(), 0, "Unexpected weights size");
// easy case: height >= hashes
if (height >= m_blocks_hash_of_hashes.size() * HASH_OF_HASHES_STEP)
return hashes.size();
@ -4285,8 +4396,11 @@ uint64_t Blockchain::prevalidate_block_hashes(uint64_t height, const std::vector
return hashes.size();
// build hashes vector to hash hashes together
std::vector<crypto::hash> data;
data.reserve(hashes.size() + HASH_OF_HASHES_STEP - 1); // may be a bit too much
std::vector<crypto::hash> data_hashes;
std::vector<uint64_t> data_weights;
data_hashes.reserve(hashes.size() + HASH_OF_HASHES_STEP - 1); // may be a bit too much
if (!weights.empty())
data_weights.reserve(data_hashes.size());
// we expect height to be either equal or a bit below db height
bool disconnected = (height > m_db->height());
@ -4301,18 +4415,24 @@ uint64_t Blockchain::prevalidate_block_hashes(uint64_t height, const std::vector
// we might need some already in the chain for the first part of the first hash
for (uint64_t h = first_index * HASH_OF_HASHES_STEP; h < height; ++h)
{
data.push_back(m_db->get_block_hash_from_height(h));
data_hashes.push_back(m_db->get_block_hash_from_height(h));
if (!weights.empty())
data_weights.push_back(m_db->get_block_weight(h));
}
pop = 0;
}
// push the data to check
for (const auto &h: hashes)
for (size_t i = 0; i < hashes.size(); ++i)
{
if (pop)
--pop;
else
data.push_back(h);
{
data_hashes.push_back(hashes[i]);
if (!weights.empty())
data_weights.push_back(weights[i]);
}
}
// hash and check
@ -4322,12 +4442,17 @@ uint64_t Blockchain::prevalidate_block_hashes(uint64_t height, const std::vector
if (n < m_blocks_hash_of_hashes.size())
{
// if the last index isn't fully filled, we can't tell if valid
if (data.size() < (n - first_index) * HASH_OF_HASHES_STEP + HASH_OF_HASHES_STEP)
if (data_hashes.size() < (n - first_index) * HASH_OF_HASHES_STEP + HASH_OF_HASHES_STEP)
break;
crypto::hash hash;
cn_fast_hash(data.data() + (n - first_index) * HASH_OF_HASHES_STEP, HASH_OF_HASHES_STEP * sizeof(crypto::hash), hash);
bool valid = hash == m_blocks_hash_of_hashes[n];
cn_fast_hash(data_hashes.data() + (n - first_index) * HASH_OF_HASHES_STEP, HASH_OF_HASHES_STEP * sizeof(crypto::hash), hash);
bool valid = hash == m_blocks_hash_of_hashes[n].first;
if (valid && !weights.empty())
{
cn_fast_hash(data_weights.data() + (n - first_index) * HASH_OF_HASHES_STEP, HASH_OF_HASHES_STEP * sizeof(uint64_t), hash);
valid &= hash == m_blocks_hash_of_hashes[n].second;
}
// add to the known hashes array
if (!valid)
@ -4339,9 +4464,15 @@ uint64_t Blockchain::prevalidate_block_hashes(uint64_t height, const std::vector
size_t end = n * HASH_OF_HASHES_STEP + HASH_OF_HASHES_STEP;
for (size_t i = n * HASH_OF_HASHES_STEP; i < end; ++i)
{
CHECK_AND_ASSERT_MES(m_blocks_hash_check[i] == crypto::null_hash || m_blocks_hash_check[i] == data[i - first_index * HASH_OF_HASHES_STEP],
CHECK_AND_ASSERT_MES(m_blocks_hash_check[i].first == crypto::null_hash || m_blocks_hash_check[i].first == data_hashes[i - first_index * HASH_OF_HASHES_STEP],
0, "Consistency failure in m_blocks_hash_check construction");
m_blocks_hash_check[i] = data[i - first_index * HASH_OF_HASHES_STEP];
m_blocks_hash_check[i].first = data_hashes[i - first_index * HASH_OF_HASHES_STEP];
if (!weights.empty())
{
CHECK_AND_ASSERT_MES(m_blocks_hash_check[i].second == 0 || m_blocks_hash_check[i].second == data_weights[i - first_index * HASH_OF_HASHES_STEP],
0, "Consistency failure in m_blocks_hash_check construction");
m_blocks_hash_check[i].second = data_weights[i - first_index * HASH_OF_HASHES_STEP];
}
}
usable += HASH_OF_HASHES_STEP;
}
@ -4358,6 +4489,18 @@ uint64_t Blockchain::prevalidate_block_hashes(uint64_t height, const std::vector
return usable;
}
bool Blockchain::has_block_weights(uint64_t height, uint64_t nblocks) const
{
CHECK_AND_ASSERT_MES(nblocks > 0, false, "nblocks is 0");
uint64_t last_block_height = height + nblocks - 1;
if (last_block_height >= m_blocks_hash_check.size())
return false;
for (uint64_t h = height; h <= last_block_height; ++h)
if (m_blocks_hash_check[h].second == 0)
return false;
return true;
}
//------------------------------------------------------------------
// ND: Speedups:
// 1. Thread long_hash computations if possible (m_max_prepare_blocks_threads = nthreads, default = 4)
@ -4399,7 +4542,7 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::vector<block_complete
bytes += entry.block.size();
for (const auto &tx_blob : entry.txs)
{
bytes += tx_blob.size();
bytes += tx_blob.blob.size();
}
total_txs += entry.txs.size();
}
@ -4555,7 +4698,7 @@ bool Blockchain::prepare_handle_incoming_blocks(const std::vector<block_complete
crypto::hash &tx_prefix_hash = txes[tx_index].second;
++tx_index;
if (!parse_and_validate_tx_base_from_blob(tx_blob, tx))
if (!parse_and_validate_tx_base_from_blob(tx_blob.blob, tx))
SCAN_TABLE_QUIT("Could not parse tx from incoming blocks.");
cryptonote::get_transaction_prefix_hash(tx, tx_prefix_hash);
@ -4887,7 +5030,7 @@ void Blockchain::cancel()
}
#if defined(PER_BLOCK_CHECKPOINT)
static const char expected_block_hashes_hash[] = "7dafb40b414a0e59bfced6682ef519f0b416bc914dd3d622b72e0dd1a47117c2";
static const char expected_block_hashes_hash[] = "95e60612c1a16f4cd992c335b66daabd98e2d351c2b02b66e43ced0296848d33";
void Blockchain::load_compiled_in_block_hashes(const GetCheckpointsCallback& get_checkpoints)
{
if (get_checkpoints == nullptr || !m_fast_sync)
@ -4931,19 +5074,21 @@ void Blockchain::load_compiled_in_block_hashes(const GetCheckpointsCallback& get
MERROR("Block hash data is too large");
return;
}
const size_t size_needed = 4 + nblocks * sizeof(crypto::hash);
const size_t size_needed = 4 + nblocks * (sizeof(crypto::hash) * 2);
if(nblocks > 0 && nblocks > (m_db->height() + HASH_OF_HASHES_STEP - 1) / HASH_OF_HASHES_STEP && checkpoints.size() >= size_needed)
{
p += sizeof(uint32_t);
m_blocks_hash_of_hashes.reserve(nblocks);
for (uint32_t i = 0; i < nblocks; i++)
{
crypto::hash hash;
memcpy(hash.data, p, sizeof(hash.data));
p += sizeof(hash.data);
m_blocks_hash_of_hashes.push_back(hash);
crypto::hash hash_hashes, hash_weights;
memcpy(hash_hashes.data, p, sizeof(hash_hashes.data));
p += sizeof(hash_hashes.data);
memcpy(hash_weights.data, p, sizeof(hash_weights.data));
p += sizeof(hash_weights.data);
m_blocks_hash_of_hashes.push_back(std::make_pair(hash_hashes, hash_weights));
}
m_blocks_hash_check.resize(m_blocks_hash_of_hashes.size() * HASH_OF_HASHES_STEP, crypto::null_hash);
m_blocks_hash_check.resize(m_blocks_hash_of_hashes.size() * HASH_OF_HASHES_STEP, std::make_pair(crypto::null_hash, 0));
MINFO(nblocks << " block hashes loaded");
// FIXME: clear tx_pool because the process might have been
@ -4958,13 +5103,13 @@ void Blockchain::load_compiled_in_block_hashes(const GetCheckpointsCallback& get
size_t tx_weight;
uint64_t fee;
bool relayed, do_not_relay, double_spend_seen;
bool relayed, do_not_relay, double_spend_seen, pruned;
transaction pool_tx;
blobdata txblob;
for(const transaction &tx : txs)
{
crypto::hash tx_hash = get_transaction_hash(tx);
m_tx_pool.take_tx(tx_hash, pool_tx, txblob, tx_weight, fee, relayed, do_not_relay, double_spend_seen);
m_tx_pool.take_tx(tx_hash, pool_tx, txblob, tx_weight, fee, relayed, do_not_relay, double_spend_seen, pruned);
}
}
}
@ -5037,6 +5182,5 @@ void Blockchain::cache_block_template(const block &b, const cryptonote::account_
namespace cryptonote {
template bool Blockchain::get_transactions(const std::vector<crypto::hash>&, std::vector<transaction>&, std::vector<crypto::hash>&) const;
template bool Blockchain::get_transactions_blobs(const std::vector<crypto::hash>&, std::vector<cryptonote::blobdata>&, std::vector<crypto::hash>&, bool) const;
template bool Blockchain::get_split_transactions_blobs(const std::vector<crypto::hash>&, std::vector<std::tuple<crypto::hash, cryptonote::blobdata, crypto::hash, cryptonote::blobdata>>&, std::vector<crypto::hash>&) const;
}

32
src/cryptonote_core/blockchain.h
View file

@ -381,13 +381,14 @@ namespace cryptonote
*
* @param qblock_ids the foreign chain's "short history" (see get_short_chain_history)
* @param hashes the hashes to be returned, return-by-reference
* @param weights the block weights to be returned, return-by-reference
* @param start_height the start height, return-by-reference
* @param current_height the current blockchain height, return-by-reference
* @param clip_pruned whether to constrain results to unpruned data
*
* @return true if a block found in common, else false
*/
bool find_blockchain_supplement(const std::list<crypto::hash>& qblock_ids, std::vector<crypto::hash>& hashes, uint64_t& start_height, uint64_t& current_height, bool clip_pruned) const;
bool find_blockchain_supplement(const std::list<crypto::hash>& qblock_ids, std::vector<crypto::hash>& hashes, std::vector<uint64_t>* weights, uint64_t& start_height, uint64_t& current_height, bool clip_pruned) const;
/**
* @brief get recent block hashes for a foreign chain
@ -397,11 +398,12 @@ namespace cryptonote
* BLOCKS_IDS_SYNCHRONIZING_DEFAULT_COUNT additional (more recent) hashes.
*
* @param qblock_ids the foreign chain's "short history" (see get_short_chain_history)
* @param clip_pruned clip pruned blocks if true, include them otherwise
* @param resp return-by-reference the split height and subsequent blocks' hashes
*
* @return true if a block found in common, else false
*/
bool find_blockchain_supplement(const std::list<crypto::hash>& qblock_ids, NOTIFY_RESPONSE_CHAIN_ENTRY::request& resp) const;
bool find_blockchain_supplement(const std::list<crypto::hash>& qblock_ids, bool clip_pruned, NOTIFY_RESPONSE_CHAIN_ENTRY::request& resp) const;
/**
* @brief find the most recent common point between ours and a foreign chain
@ -675,8 +677,8 @@ namespace cryptonote
*
* @return false if an unexpected exception occurs, else true
*/
template<class t_ids_container, class t_tx_container, class t_missed_container>
bool get_transactions_blobs(const t_ids_container& txs_ids, t_tx_container& txs, t_missed_container& missed_txs, bool pruned = false) const;
bool get_transactions_blobs(const std::vector<crypto::hash>& txs_ids, std::vector<cryptonote::blobdata>& txs, std::vector<crypto::hash>& missed_txs, bool pruned = false) const;
bool get_transactions_blobs(const std::vector<crypto::hash>& txs_ids, std::vector<tx_blob_entry>& txs, std::vector<crypto::hash>& missed_txs, bool pruned = false) const;
template<class t_ids_container, class t_tx_container, class t_missed_container>
bool get_split_transactions_blobs(const t_ids_container& txs_ids, t_tx_container& txs, t_missed_container& missed_txs) const;
template<class t_ids_container, class t_tx_container, class t_missed_container>
@ -963,9 +965,8 @@ namespace cryptonote
cryptonote::blobdata get_txpool_tx_blob(const crypto::hash& txid) const;
bool for_all_txpool_txes(std::function<bool(const crypto::hash&, const txpool_tx_meta_t&, const cryptonote::blobdata*)>, bool include_blob = false, bool include_unrelayed_txes = true) const;
bool is_within_compiled_block_hash_area(uint64_t height) const;
bool is_within_compiled_block_hash_area() const { return is_within_compiled_block_hash_area(m_db->height()); }
uint64_t prevalidate_block_hashes(uint64_t height, const std::vector<crypto::hash> &hashes);
uint64_t prevalidate_block_hashes(uint64_t height, const std::vector<crypto::hash> &hashes, const std::vector<uint64_t> &weights);
uint32_t get_blockchain_pruning_seed() const { return m_db->get_blockchain_pruning_seed(); }
bool prune_blockchain(uint32_t pruning_seed = 0);
bool update_blockchain_pruning();
@ -995,6 +996,21 @@ namespace cryptonote
*/
void pop_blocks(uint64_t nblocks);
/**
* @brief checks whether a given block height is included in the precompiled block hash area
*
* @param height the height to check for
*/
bool is_within_compiled_block_hash_area(uint64_t height) const;
/**
* @brief checks whether we have known weights for the given block heights
*
* @param height the start height to check for
* @param nblocks how many blocks to check from that height
*/
bool has_block_weights(uint64_t height, uint64_t nblocks) const;
#ifndef IN_UNIT_TESTS
private:
#endif
@ -1022,8 +1038,8 @@ namespace cryptonote
std::unordered_map<crypto::hash, crypto::hash> m_blocks_longhash_table;
// SHA-3 hashes for each block and for fast pow checking
std::vector<crypto::hash> m_blocks_hash_of_hashes;
std::vector<crypto::hash> m_blocks_hash_check;
std::vector<std::pair<crypto::hash, crypto::hash>> m_blocks_hash_of_hashes;
std::vector<std::pair<crypto::hash, uint64_t>> m_blocks_hash_check;
std::vector<crypto::hash> m_blocks_txs_check;
blockchain_db_sync_mode m_db_sync_mode;

75
src/cryptonote_core/cryptonote_core.cpp
View file

@ -113,6 +113,10 @@ namespace cryptonote
, "Set maximum size of block download queue in bytes (0 for default)"
, 0
};
const command_line::arg_descriptor<bool> arg_sync_pruned_blocks = {
"sync-pruned-blocks"
, "Allow syncing from nodes with only pruned blocks"
};
static const command_line::arg_descriptor<bool> arg_test_drop_download = {
"test-drop-download"
@ -323,6 +327,7 @@ namespace cryptonote
command_line::add_arg(desc, arg_offline);
command_line::add_arg(desc, arg_disable_dns_checkpoints);
command_line::add_arg(desc, arg_block_download_max_size);
command_line::add_arg(desc, arg_sync_pruned_blocks);
command_line::add_arg(desc, arg_max_txpool_weight);
command_line::add_arg(desc, arg_pad_transactions);
command_line::add_arg(desc, arg_block_notify);
@ -744,13 +749,13 @@ namespace cryptonote
return false;
}
//-----------------------------------------------------------------------------------------------
bool core::handle_incoming_tx_pre(const blobdata& tx_blob, tx_verification_context& tvc, cryptonote::transaction &tx, crypto::hash &tx_hash, bool keeped_by_block, bool relayed, bool do_not_relay)
bool core::handle_incoming_tx_pre(const tx_blob_entry& tx_blob, tx_verification_context& tvc, cryptonote::transaction &tx, crypto::hash &tx_hash, bool keeped_by_block, bool relayed, bool do_not_relay)
{
tvc = boost::value_initialized<tx_verification_context>();
if(tx_blob.size() > get_max_tx_size())
if(tx_blob.blob.size() > get_max_tx_size())
{
LOG_PRINT_L1("WRONG TRANSACTION BLOB, too big size " << tx_blob.size() << ", rejected");
LOG_PRINT_L1("WRONG TRANSACTION BLOB, too big size " << tx_blob.blob.size() << ", rejected");
tvc.m_verifivation_failed = true;
tvc.m_too_big = true;
return false;
@ -758,7 +763,23 @@ namespace cryptonote
tx_hash = crypto::null_hash;
if(!parse_tx_from_blob(tx, tx_hash, tx_blob))
bool r;
if (tx_blob.prunable_hash == crypto::null_hash)
{
r = parse_tx_from_blob(tx, tx_hash, tx_blob.blob);
}
else
{
r = parse_and_validate_tx_base_from_blob(tx_blob.blob, tx);
if (r)
{
tx.set_prunable_hash(tx_blob.prunable_hash);
tx_hash = cryptonote::get_pruned_transaction_hash(tx, tx_blob.prunable_hash);
tx.set_hash(tx_hash);
}
}
if (!r)
{
LOG_PRINT_L1("WRONG TRANSACTION BLOB, Failed to parse, rejected");
tvc.m_verifivation_failed = true;
@ -784,6 +805,7 @@ namespace cryptonote
if (tx.version == 0 || tx.version > max_tx_version)
{
// v2 is the latest one we know
MERROR_VER("Bad tx version (" << tx.version << ", max is " << max_tx_version << ")");
tvc.m_verifivation_failed = true;
return false;
}
@ -791,7 +813,7 @@ namespace cryptonote
return true;
}
//-----------------------------------------------------------------------------------------------
bool core::handle_incoming_tx_post(const blobdata& tx_blob, tx_verification_context& tvc, cryptonote::transaction &tx, crypto::hash &tx_hash, bool keeped_by_block, bool relayed, bool do_not_relay)
bool core::handle_incoming_tx_post(const tx_blob_entry& tx_blob, tx_verification_context& tvc, cryptonote::transaction &tx, crypto::hash &tx_hash, bool keeped_by_block, bool relayed, bool do_not_relay)
{
if(!check_tx_syntax(tx))
{
@ -920,7 +942,7 @@ namespace cryptonote
return ret;
}
//-----------------------------------------------------------------------------------------------
bool core::handle_incoming_txs(const std::vector<blobdata>& tx_blobs, std::vector<tx_verification_context>& tvc, bool keeped_by_block, bool relayed, bool do_not_relay)
bool core::handle_incoming_txs(const std::vector<tx_blob_entry>& tx_blobs, std::vector<tx_verification_context>& tvc, bool keeped_by_block, bool relayed, bool do_not_relay)
{
TRY_ENTRY();
CRITICAL_REGION_LOCAL(m_incoming_tx_lock);
@ -931,7 +953,7 @@ namespace cryptonote
tvc.resize(tx_blobs.size());
tools::threadpool& tpool = tools::threadpool::getInstance();
tools::threadpool::waiter waiter;
std::vector<blobdata>::const_iterator it = tx_blobs.begin();
std::vector<tx_blob_entry>::const_iterator it = tx_blobs.begin();
for (size_t i = 0; i < tx_blobs.size(); i++, ++it) {
tpool.submit(&waiter, [&, i, it] {
try
@ -1003,8 +1025,12 @@ namespace cryptonote
if (already_have[i])
continue;
const size_t weight = get_transaction_weight(results[i].tx, it->size());
ok &= add_new_tx(results[i].tx, results[i].hash, tx_blobs[i], weight, tvc[i], keeped_by_block, relayed, do_not_relay);
// if it's a pruned tx from an incoming block, we'll get a weight that's technically
// different from the actual transaction weight, but it's OK for our use. Those txes
// will be ignored when mining, and using that "pruned" weight seems appropriate for
// keeping the txpool size constrained
const uint64_t weight = results[i].tx.pruned ? 0 : get_transaction_weight(results[i].tx, it->blob.size());
ok &= add_new_tx(results[i].tx, results[i].hash, tx_blobs[i].blob, weight, tvc[i], keeped_by_block, relayed, do_not_relay);
if(tvc[i].m_verifivation_failed)
{MERROR_VER("Transaction verification failed: " << results[i].hash);}
else if(tvc[i].m_verifivation_impossible)
@ -1018,9 +1044,9 @@ namespace cryptonote
CATCH_ENTRY_L0("core::handle_incoming_txs()", false);
}
//-----------------------------------------------------------------------------------------------
bool core::handle_incoming_tx(const blobdata& tx_blob, tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay)
bool core::handle_incoming_tx(const tx_blob_entry& tx_blob, tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay)
{
std::vector<cryptonote::blobdata> tx_blobs;
std::vector<tx_blob_entry> tx_blobs;
tx_blobs.push_back(tx_blob);
std::vector<tx_verification_context> tvcv(1);
bool r = handle_incoming_txs(tx_blobs, tvcv, keeped_by_block, relayed, do_not_relay);
@ -1028,6 +1054,11 @@ namespace cryptonote
return r;
}
//-----------------------------------------------------------------------------------------------
bool core::handle_incoming_tx(const blobdata& tx_blob, tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay)
{
return handle_incoming_tx({tx_blob, crypto::null_hash}, tvc, keeped_by_block, relayed, do_not_relay);
}
//-----------------------------------------------------------------------------------------------
bool core::get_stat_info(core_stat_info& st_inf) const
{
st_inf.mining_speed = m_miner.get_speed();
@ -1290,9 +1321,9 @@ namespace cryptonote
return m_blockchain_storage.create_block_template(b, prev_block, adr, diffic, height, expected_reward, ex_nonce);
}
//-----------------------------------------------------------------------------------------------
bool core::find_blockchain_supplement(const std::list<crypto::hash>& qblock_ids, NOTIFY_RESPONSE_CHAIN_ENTRY::request& resp) const
bool core::find_blockchain_supplement(const std::list<crypto::hash>& qblock_ids, bool clip_pruned, NOTIFY_RESPONSE_CHAIN_ENTRY::request& resp) const
{
return m_blockchain_storage.find_blockchain_supplement(qblock_ids, resp);
return m_blockchain_storage.find_blockchain_supplement(qblock_ids, clip_pruned, resp);
}
//-----------------------------------------------------------------------------------------------
bool core::find_blockchain_supplement(const uint64_t req_start_block, const std::list<crypto::hash>& qblock_ids, std::vector<std::pair<std::pair<cryptonote::blobdata, crypto::hash>, std::vector<std::pair<crypto::hash, cryptonote::blobdata> > > >& blocks, uint64_t& total_height, uint64_t& start_height, bool pruned, bool get_miner_tx_hash, size_t max_count) const
@ -1338,7 +1369,7 @@ namespace cryptonote
{
cryptonote::blobdata txblob;
CHECK_AND_ASSERT_THROW_MES(pool.get_transaction(tx_hash, txblob), "Transaction not found in pool");
bce.txs.push_back(txblob);
bce.txs.push_back({txblob, crypto::null_hash});
}
return bce;
}
@ -1391,7 +1422,7 @@ namespace cryptonote
block_to_blob(b, arg.b.block);
//pack transactions
for(auto& tx: txs)
arg.b.txs.push_back(tx);
arg.b.txs.push_back({tx, crypto::null_hash});
m_pprotocol->relay_block(arg, exclude_context);
}
@ -1901,9 +1932,9 @@ namespace cryptonote
return m_target_blockchain_height;
}
//-----------------------------------------------------------------------------------------------
uint64_t core::prevalidate_block_hashes(uint64_t height, const std::vector<crypto::hash> &hashes)
uint64_t core::prevalidate_block_hashes(uint64_t height, const std::vector<crypto::hash> &hashes, const std::vector<uint64_t> &weights)
{
return get_blockchain_storage().prevalidate_block_hashes(height, hashes);
return get_blockchain_storage().prevalidate_block_hashes(height, hashes, weights);
}
//-----------------------------------------------------------------------------------------------
uint64_t core::get_free_space() const
@ -1923,6 +1954,16 @@ namespace cryptonote
return get_blockchain_storage().prune_blockchain(pruning_seed);
}
//-----------------------------------------------------------------------------------------------
bool core::is_within_compiled_block_hash_area(uint64_t height) const
{
return get_blockchain_storage().is_within_compiled_block_hash_area(height);
}
//-----------------------------------------------------------------------------------------------
bool core::has_block_weights(uint64_t height, uint64_t nblocks) const
{
return get_blockchain_storage().has_block_weights(height, nblocks);
}
//-----------------------------------------------------------------------------------------------
std::time_t core::get_start_time() const
{
return start_time;

24
src/cryptonote_core/cryptonote_core.h
View file

@ -64,6 +64,7 @@ namespace cryptonote
extern const command_line::arg_descriptor<difficulty_type> arg_fixed_difficulty;
extern const command_line::arg_descriptor<bool> arg_offline;
extern const command_line::arg_descriptor<size_t> arg_block_download_max_size;
extern const command_line::arg_descriptor<bool> arg_sync_pruned_blocks;
/************************************************************************/
/* */
@ -120,6 +121,7 @@ namespace cryptonote
*
* @return true if the transaction was accepted, false otherwise
*/
bool handle_incoming_tx(const tx_blob_entry& tx_blob, tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay);
bool handle_incoming_tx(const blobdata& tx_blob, tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay);
/**
@ -136,7 +138,7 @@ namespace cryptonote
*
* @return true if the transactions were accepted, false otherwise
*/
bool handle_incoming_txs(const std::vector<blobdata>& tx_blobs, std::vector<tx_verification_context>& tvc, bool keeped_by_block, bool relayed, bool do_not_relay);
bool handle_incoming_txs(const std::vector<tx_blob_entry>& tx_blobs, std::vector<tx_verification_context>& tvc, bool keeped_by_block, bool relayed, bool do_not_relay);
/**
* @brief handles an incoming block
@ -522,7 +524,7 @@ namespace cryptonote
*
* @note see Blockchain::find_blockchain_supplement(const std::list<crypto::hash>&, NOTIFY_RESPONSE_CHAIN_ENTRY::request&) const
*/
bool find_blockchain_supplement(const std::list<crypto::hash>& qblock_ids, NOTIFY_RESPONSE_CHAIN_ENTRY::request& resp) const;
bool find_blockchain_supplement(const std::list<crypto::hash>& qblock_ids, bool clip_pruned, NOTIFY_RESPONSE_CHAIN_ENTRY::request& resp) const;
/**
* @copydoc Blockchain::find_blockchain_supplement(const uint64_t, const std::list<crypto::hash>&, std::vector<std::pair<cryptonote::blobdata, std::vector<cryptonote::blobdata> > >&, uint64_t&, uint64_t&, size_t) const
@ -779,7 +781,7 @@ namespace cryptonote
*
* @return number of usable blocks
*/
uint64_t prevalidate_block_hashes(uint64_t height, const std::vector<crypto::hash> &hashes);
uint64_t prevalidate_block_hashes(uint64_t height, const std::vector<crypto::hash> &hashes, const std::vector<uint64_t> &weights);
/**
* @brief get free disk space on the blockchain partition
@ -825,6 +827,18 @@ namespace cryptonote
*/
bool check_blockchain_pruning();
/**
* @brief checks whether a given block height is included in the precompiled block hash area
*
* @param height the height to check for
*/
bool is_within_compiled_block_hash_area(uint64_t height) const;
/**
* @brief checks whether block weights are known for the given range
*/
bool has_block_weights(uint64_t height, uint64_t nblocks) const;
private:
/**
@ -910,8 +924,8 @@ namespace cryptonote
bool check_tx_semantic(const transaction& tx, bool keeped_by_block) const;
void set_semantics_failed(const crypto::hash &tx_hash);
bool handle_incoming_tx_pre(const blobdata& tx_blob, tx_verification_context& tvc, cryptonote::transaction &tx, crypto::hash &tx_hash, bool keeped_by_block, bool relayed, bool do_not_relay);
bool handle_incoming_tx_post(const blobdata& tx_blob, tx_verification_context& tvc, cryptonote::transaction &tx, crypto::hash &tx_hash, bool keeped_by_block, bool relayed, bool do_not_relay);
bool handle_incoming_tx_pre(const tx_blob_entry& tx_blob, tx_verification_context& tvc, cryptonote::transaction &tx, crypto::hash &tx_hash, bool keeped_by_block, bool relayed, bool do_not_relay);
bool handle_incoming_tx_post(const tx_blob_entry &tx_blob, tx_verification_context& tvc, cryptonote::transaction &tx, crypto::hash &tx_hash, bool keeped_by_block, bool relayed, bool do_not_relay);
struct tx_verification_batch_info { const cryptonote::transaction *tx; crypto::hash tx_hash; tx_verification_context &tvc; bool &result; };
bool handle_incoming_tx_accumulated_batch(std::vector<tx_verification_batch_info> &tx_info, bool keeped_by_block);

25
src/cryptonote_core/tx_pool.cpp
View file

@ -247,6 +247,7 @@ namespace cryptonote
meta.relayed = relayed;
meta.do_not_relay = do_not_relay;
meta.double_spend_seen = have_tx_keyimges_as_spent(tx);
meta.pruned = tx.pruned;
meta.bf_padding = 0;
memset(meta.padding, 0, sizeof(meta.padding));
try
@ -290,6 +291,7 @@ namespace cryptonote
meta.relayed = relayed;
meta.do_not_relay = do_not_relay;
meta.double_spend_seen = false;
meta.pruned = tx.pruned;
meta.bf_padding = 0;
memset(meta.padding, 0, sizeof(meta.padding));
@ -460,7 +462,7 @@ namespace cryptonote
return true;
}
//---------------------------------------------------------------------------------
bool tx_memory_pool::take_tx(const crypto::hash &id, transaction &tx, cryptonote::blobdata &txblob, size_t& tx_weight, uint64_t& fee, bool &relayed, bool &do_not_relay, bool &double_spend_seen)
bool tx_memory_pool::take_tx(const crypto::hash &id, transaction &tx, cryptonote::blobdata &txblob, size_t& tx_weight, uint64_t& fee, bool &relayed, bool &do_not_relay, bool &double_spend_seen, bool &pruned)
{
CRITICAL_REGION_LOCAL(m_transactions_lock);
CRITICAL_REGION_LOCAL1(m_blockchain);
@ -482,7 +484,7 @@ namespace cryptonote
{
tx = ci->second;
}
else if (!parse_and_validate_tx_from_blob(txblob, tx))
else if (!(meta.pruned ? parse_and_validate_tx_base_from_blob(txblob, tx) : parse_and_validate_tx_from_blob(txblob, tx)))
{
MERROR("Failed to parse tx from txpool");
return false;
@ -496,6 +498,7 @@ namespace cryptonote
relayed = meta.relayed;
do_not_relay = meta.do_not_relay;
double_spend_seen = meta.double_spend_seen;
pruned = meta.pruned;
// remove first, in case this throws, so key images aren't removed
m_blockchain.remove_txpool_tx(id);
@ -601,7 +604,7 @@ namespace cryptonote
txs.reserve(m_blockchain.get_txpool_tx_count());
m_blockchain.for_all_txpool_txes([this, now, &txs](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata *){
// 0 fee transactions are never relayed
if(meta.fee > 0 && !meta.do_not_relay && now - meta.last_relayed_time > get_relay_delay(now, meta.receive_time))
if(!meta.pruned && meta.fee > 0 && !meta.do_not_relay && now - meta.last_relayed_time > get_relay_delay(now, meta.receive_time))
{
// if the tx is older than half the max lifetime, we don't re-relay it, to avoid a problem
// mentioned by smooth where nodes would flush txes at slightly different times, causing
@ -667,7 +670,7 @@ namespace cryptonote
txs.reserve(m_blockchain.get_txpool_tx_count(include_unrelayed_txes));
m_blockchain.for_all_txpool_txes([&txs](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata *bd){
transaction tx;
if (!parse_and_validate_tx_from_blob(*bd, tx))
if (!(meta.pruned ? parse_and_validate_tx_base_from_blob(*bd, tx) : parse_and_validate_tx_from_blob(*bd, tx)))
{
MERROR("Failed to parse tx from txpool");
// continue
@ -791,7 +794,7 @@ namespace cryptonote
txi.id_hash = epee::string_tools::pod_to_hex(txid);
txi.tx_blob = *bd;
transaction tx;
if (!parse_and_validate_tx_from_blob(*bd, tx))
if (!(meta.pruned ? parse_and_validate_tx_base_from_blob(*bd, tx) : parse_and_validate_tx_from_blob(*bd, tx)))
{
MERROR("Failed to parse tx from txpool");
// continue
@ -863,7 +866,7 @@ namespace cryptonote
m_blockchain.for_all_txpool_txes([&tx_infos, key_image_infos](const crypto::hash &txid, const txpool_tx_meta_t &meta, const cryptonote::blobdata *bd){
cryptonote::rpc::tx_in_pool txi;
txi.tx_hash = txid;
if (!parse_and_validate_tx_from_blob(*bd, txi.tx))
if (!(meta.pruned ? parse_and_validate_tx_base_from_blob(*bd, txi.tx) : parse_and_validate_tx_from_blob(*bd, txi.tx)))
{
MERROR("Failed to parse tx from txpool");
// continue
@ -1143,7 +1146,7 @@ namespace cryptonote
ss << "id: " << txid << std::endl;
if (!short_format) {
cryptonote::transaction tx;
if (!parse_and_validate_tx_from_blob(*txblob, tx))
if (!(meta.pruned ? parse_and_validate_tx_base_from_blob(*txblob, tx) : parse_and_validate_tx_from_blob(*txblob, tx)))
{
MERROR("Failed to parse tx from txpool");
return true; // continue
@ -1199,6 +1202,12 @@ namespace cryptonote
}
LOG_PRINT_L2("Considering " << sorted_it->second << ", weight " << meta.weight << ", current block weight " << total_weight << "/" << max_total_weight << ", current coinbase " << print_money(best_coinbase));
if (meta.pruned)
{
LOG_PRINT_L2(" tx is pruned");
continue;
}
// Can not exceed maximum block weight
if (max_total_weight < total_weight + meta.weight)
{
@ -1322,7 +1331,7 @@ namespace cryptonote
{
cryptonote::blobdata txblob = m_blockchain.get_txpool_tx_blob(txid);
cryptonote::transaction tx;
if (!parse_and_validate_tx_from_blob(txblob, tx))
if (!parse_and_validate_tx_from_blob(txblob, tx)) // remove pruned ones on startup, they're meant to be temporary
{
MERROR("Failed to parse tx from txpool");
continue;

3
src/cryptonote_core/tx_pool.h
View file

@ -139,10 +139,11 @@ namespace cryptonote
* @param relayed return-by-reference was transaction relayed to us by the network?
* @param do_not_relay return-by-reference is transaction not to be relayed to the network?
* @param double_spend_seen return-by-reference was a double spend seen for that transaction?
* @param pruned return-by-reference is the tx pruned
*
* @return true unless the transaction cannot be found in the pool
*/
bool take_tx(const crypto::hash &id, transaction &tx, cryptonote::blobdata &txblob, size_t& tx_weight, uint64_t& fee, bool &relayed, bool &do_not_relay, bool &double_spend_seen);
bool take_tx(const crypto::hash &id, transaction &tx, cryptonote::blobdata &txblob, size_t& tx_weight, uint64_t& fee, bool &relayed, bool &do_not_relay, bool &double_spend_seen, bool &pruned);
/**
* @brief checks if the pool has a transaction with the given hash

43
src/cryptonote_protocol/block_queue.cpp
View file

@ -228,13 +228,14 @@ bool block_queue::have(const crypto::hash &hash) const
return have_blocks.find(hash) != have_blocks.end();
}
std::pair<uint64_t, uint64_t> block_queue::reserve_span(uint64_t first_block_height, uint64_t last_block_height, uint64_t max_blocks, const boost::uuids::uuid &connection_id, uint32_t pruning_seed, uint64_t blockchain_height, const std::vector<crypto::hash> &block_hashes, boost::posix_time::ptime time)
std::pair<uint64_t, uint64_t> block_queue::reserve_span(uint64_t first_block_height, uint64_t last_block_height, uint64_t max_blocks, const boost::uuids::uuid &connection_id, bool sync_pruned_blocks, uint32_t local_pruning_seed, uint32_t pruning_seed, uint64_t blockchain_height, const std::vector<std::pair<crypto::hash, uint64_t>> &block_hashes, boost::posix_time::ptime time)
{
boost::unique_lock<boost::recursive_mutex> lock(mutex);
MDEBUG("reserve_span: first_block_height " << first_block_height << ", last_block_height " << last_block_height
<< ", max " << max_blocks << ", seed " << epee::string_tools::to_string_hex(pruning_seed) << ", blockchain_height " <<
blockchain_height << ", block hashes size " << block_hashes.size());
<< ", max " << max_blocks << ", peer seed " << epee::string_tools::to_string_hex(pruning_seed) << ", blockchain_height " <<
blockchain_height << ", block hashes size " << block_hashes.size() << ", local seed " << epee::string_tools::to_string_hex(local_pruning_seed)
<< ", sync_pruned_blocks " << sync_pruned_blocks);
if (last_block_height < first_block_height || max_blocks == 0)
{
MDEBUG("reserve_span: early out: first_block_height " << first_block_height << ", last_block_height " << last_block_height << ", max_blocks " << max_blocks);
@ -248,22 +249,25 @@ std::pair<uint64_t, uint64_t> block_queue::reserve_span(uint64_t first_block_hei
// skip everything we've already requested
uint64_t span_start_height = last_block_height - block_hashes.size() + 1;
std::vector<crypto::hash>::const_iterator i = block_hashes.begin();
while (i != block_hashes.end() && requested_internal(*i))
std::vector<std::pair<crypto::hash, uint64_t>>::const_iterator i = block_hashes.begin();
while (i != block_hashes.end() && requested_internal((*i).first))
{
++i;
++span_start_height;
}
// if the peer's pruned for the starting block and its unpruned stripe comes next, start downloading from there
const uint32_t next_unpruned_height = tools::get_next_unpruned_block_height(span_start_height, blockchain_height, pruning_seed);
MDEBUG("reserve_span: next_unpruned_height " << next_unpruned_height << " from " << span_start_height << " and seed "
<< epee::string_tools::to_string_hex(pruning_seed) << ", limit " << span_start_height + CRYPTONOTE_PRUNING_STRIPE_SIZE);
if (next_unpruned_height > span_start_height && next_unpruned_height < span_start_height + CRYPTONOTE_PRUNING_STRIPE_SIZE)
if (!sync_pruned_blocks)
{
MDEBUG("We can download from next span: ideal height " << span_start_height << ", next unpruned height " << next_unpruned_height <<
"(+" << next_unpruned_height - span_start_height << "), current seed " << pruning_seed);
span_start_height = next_unpruned_height;
// if the peer's pruned for the starting block and its unpruned stripe comes next, start downloading from there
const uint32_t next_unpruned_height = tools::get_next_unpruned_block_height(span_start_height, blockchain_height, pruning_seed);
MDEBUG("reserve_span: next_unpruned_height " << next_unpruned_height << " from " << span_start_height << " and seed "
<< epee::string_tools::to_string_hex(pruning_seed) << ", limit " << span_start_height + CRYPTONOTE_PRUNING_STRIPE_SIZE);
if (next_unpruned_height > span_start_height && next_unpruned_height < span_start_height + CRYPTONOTE_PRUNING_STRIPE_SIZE)
{
MDEBUG("We can download from next span: ideal height " << span_start_height << ", next unpruned height " << next_unpruned_height <<
"(+" << next_unpruned_height - span_start_height << "), current seed " << pruning_seed);
span_start_height = next_unpruned_height;
}
}
MDEBUG("span_start_height: " <<span_start_height);
const uint64_t block_hashes_start_height = last_block_height - block_hashes.size() + 1;
@ -274,7 +278,7 @@ std::pair<uint64_t, uint64_t> block_queue::reserve_span(uint64_t first_block_hei
}
i = block_hashes.begin() + span_start_height - block_hashes_start_height;
while (i != block_hashes.end() && requested_internal(*i))
while (i != block_hashes.end() && requested_internal((*i).first))
{
++i;
++span_start_height;
@ -282,9 +286,16 @@ std::pair<uint64_t, uint64_t> block_queue::reserve_span(uint64_t first_block_hei
uint64_t span_length = 0;
std::vector<crypto::hash> hashes;
while (i != block_hashes.end() && span_length < max_blocks && tools::has_unpruned_block(span_start_height + span_length, blockchain_height, pruning_seed))
bool first_is_pruned = sync_pruned_blocks && !tools::has_unpruned_block(span_start_height + span_length, blockchain_height, local_pruning_seed);
while (i != block_hashes.end() && span_length < max_blocks && (sync_pruned_blocks || tools::has_unpruned_block(span_start_height + span_length, blockchain_height, pruning_seed)))
{
hashes.push_back(*i);
// if we want to sync pruned blocks, stop at the first block for which we need full data
if (sync_pruned_blocks && first_is_pruned == tools::has_unpruned_block(span_start_height + span_length, blockchain_height, local_pruning_seed))
{
MDEBUG("Stopping at " << span_start_height + span_length << " for peer on stripe " << tools::get_pruning_stripe(pruning_seed) << " as we need full data for " << tools::get_pruning_stripe(local_pruning_seed));
break;
}
hashes.push_back((*i).first);
++i;
++span_length;
}

2
src/cryptonote_protocol/block_queue.h
View file

@ -78,7 +78,7 @@ namespace cryptonote
void print() const;
std::string get_overview(uint64_t blockchain_height) const;
bool has_unpruned_height(uint64_t block_height, uint64_t blockchain_height, uint32_t pruning_seed) const;
std::pair<uint64_t, uint64_t> reserve_span(uint64_t first_block_height, uint64_t last_block_height, uint64_t max_blocks, const boost::uuids::uuid &connection_id, uint32_t pruning_seed, uint64_t blockchain_height, const std::vector<crypto::hash> &block_hashes, boost::posix_time::ptime time = boost::posix_time::microsec_clock::universal_time());
std::pair<uint64_t, uint64_t> reserve_span(uint64_t first_block_height, uint64_t last_block_height, uint64_t max_blocks, const boost::uuids::uuid &connection_id, bool sync_pruned_blocks, uint32_t local_pruning_seed, uint32_t pruning_seed, uint64_t blockchain_height, const std::vector<std::pair<crypto::hash, uint64_t>> &block_hashes, boost::posix_time::ptime time = boost::posix_time::microsec_clock::universal_time());
uint64_t get_next_needed_height(uint64_t blockchain_height) const;
std::pair<uint64_t, uint64_t> get_next_span_if_scheduled(std::vector<crypto::hash> &hashes, boost::uuids::uuid &connection_id, boost::posix_time::ptime &time) const;
void reset_next_span_time(boost::posix_time::ptime t = boost::posix_time::microsec_clock::universal_time());

48
src/cryptonote_protocol/cryptonote_protocol_defs.h
View file

@ -116,14 +116,51 @@ namespace cryptonote
/************************************************************************/
/* */
/************************************************************************/
struct tx_blob_entry
{
blobdata blob;
crypto::hash prunable_hash;
BEGIN_KV_SERIALIZE_MAP()
KV_SERIALIZE(blob)
KV_SERIALIZE_VAL_POD_AS_BLOB(prunable_hash)
END_KV_SERIALIZE_MAP()
tx_blob_entry(const blobdata &bd = {}, const crypto::hash &h = crypto::null_hash): blob(bd), prunable_hash(h) {}
};
struct block_complete_entry
{
bool pruned;
blobdata block;
std::vector<blobdata> txs;
uint64_t block_weight;
std::vector<tx_blob_entry> txs;
BEGIN_KV_SERIALIZE_MAP()
KV_SERIALIZE_OPT(pruned, false)
KV_SERIALIZE(block)
KV_SERIALIZE(txs)
KV_SERIALIZE_OPT(block_weight, (uint64_t)0)
if (this_ref.pruned)
{
KV_SERIALIZE(txs)
}
else
{
std::vector<blobdata> txs;
if (is_store)
{
txs.reserve(this_ref.txs.size());
for (const auto &e: this_ref.txs) txs.push_back(e.blob);
}
epee::serialization::selector<is_store>::serialize(txs, stg, hparent_section, "txs");
if (!is_store)
{
block_complete_entry &self = const_cast<block_complete_entry&>(this_ref);
self.txs.clear();
self.txs.reserve(txs.size());
for (auto &e: txs) self.txs.push_back({std::move(e), crypto::null_hash});
}
}
END_KV_SERIALIZE_MAP()
block_complete_entry(): pruned(false) {}
};
@ -176,8 +213,11 @@ namespace cryptonote
struct request_t
{
std::vector<crypto::hash> blocks;
bool prune;
BEGIN_KV_SERIALIZE_MAP()
KV_SERIALIZE_CONTAINER_POD_AS_BLOB(blocks)
KV_SERIALIZE_OPT(prune, false)
END_KV_SERIALIZE_MAP()
};
typedef epee::misc_utils::struct_init<request_t> request;
@ -229,9 +269,11 @@ namespace cryptonote
struct request_t
{
std::list<crypto::hash> block_ids; /*IDs of the first 10 blocks are sequential, next goes with pow(2,n) offset, like 2, 4, 8, 16, 32, 64 and so on, and the last one is always genesis block */
bool prune;
BEGIN_KV_SERIALIZE_MAP()
KV_SERIALIZE_CONTAINER_POD_AS_BLOB(block_ids)
KV_SERIALIZE_OPT(prune, false)
END_KV_SERIALIZE_MAP()
};
typedef epee::misc_utils::struct_init<request_t> request;
@ -248,6 +290,7 @@ namespace cryptonote
uint64_t cumulative_difficulty;
uint64_t cumulative_difficulty_top64;
std::vector<crypto::hash> m_block_ids;
std::vector<uint64_t> m_block_weights;
BEGIN_KV_SERIALIZE_MAP()
KV_SERIALIZE(start_height)
@ -255,6 +298,7 @@ namespace cryptonote
KV_SERIALIZE(cumulative_difficulty)
KV_SERIALIZE(cumulative_difficulty_top64)
KV_SERIALIZE_CONTAINER_POD_AS_BLOB(m_block_ids)
KV_SERIALIZE_CONTAINER_POD_AS_BLOB(m_block_weights)
END_KV_SERIALIZE_MAP()
};
typedef epee::misc_utils::struct_init<request_t> request;

2
src/cryptonote_protocol/cryptonote_protocol_handler.h
View file

@ -137,6 +137,7 @@ namespace cryptonote
size_t get_synchronizing_connections_count();
bool on_connection_synchronized();
bool should_download_next_span(cryptonote_connection_context& context, bool standby);
bool should_ask_for_pruned_data(cryptonote_connection_context& context, uint64_t first_block_height, uint64_t nblocks, bool check_block_weights) const;
void drop_connection(cryptonote_connection_context &context, bool add_fail, bool flush_all_spans);
bool kick_idle_peers();
bool check_standby_peers();
@ -164,6 +165,7 @@ namespace cryptonote
uint64_t m_sync_spans_downloaded, m_sync_old_spans_downloaded, m_sync_bad_spans_downloaded;
uint64_t m_sync_download_chain_size, m_sync_download_objects_size;
size_t m_block_download_max_size;
bool m_sync_pruned_blocks;
boost::mutex m_buffer_mutex;
double get_avg_block_size();

183
src/cryptonote_protocol/cryptonote_protocol_handler.inl
View file

@ -106,6 +106,7 @@ namespace cryptonote
m_sync_download_objects_size = 0;
m_block_download_max_size = command_line::get_arg(vm, cryptonote::arg_block_download_max_size);
m_sync_pruned_blocks = command_line::get_arg(vm, cryptonote::arg_sync_pruned_blocks);
return true;
}
@ -138,6 +139,7 @@ namespace cryptonote
context.m_needed_objects.clear();
m_core.get_short_chain_history(r.block_ids);
handler_request_blocks_history( r.block_ids ); // change the limit(?), sleep(?)
r.prune = m_sync_pruned_blocks;
MLOG_P2P_MESSAGE("-->>NOTIFY_REQUEST_CHAIN: m_block_ids.size()=" << r.block_ids.size() );
post_notify<NOTIFY_REQUEST_CHAIN>(r, context);
MLOG_PEER_STATE("requesting chain");
@ -493,6 +495,7 @@ namespace cryptonote
context.m_state = cryptonote_connection_context::state_synchronizing;
NOTIFY_REQUEST_CHAIN::request r = boost::value_initialized<NOTIFY_REQUEST_CHAIN::request>();
m_core.get_short_chain_history(r.block_ids);
r.prune = m_sync_pruned_blocks;
handler_request_blocks_history( r.block_ids ); // change the limit(?), sleep(?)
MLOG_P2P_MESSAGE("-->>NOTIFY_REQUEST_CHAIN: m_block_ids.size()=" << r.block_ids.size() );
post_notify<NOTIFY_REQUEST_CHAIN>(r, context);
@ -540,9 +543,9 @@ namespace cryptonote
return 1;
}
}
std::vector<blobdata> have_tx;
std::vector<tx_blob_entry> have_tx;
// Instead of requesting missing transactions by hash like BTC,
// we do it by index (thanks to a suggestion from moneromooo) because
// we're way cooler .. and also because they're smaller than hashes.
@ -556,7 +559,7 @@ namespace cryptonote
for(auto& tx_blob: arg.b.txs)
{
if(parse_and_validate_tx_from_blob(tx_blob, tx))
if(parse_and_validate_tx_from_blob(tx_blob.blob, tx))
{
try
{
@ -641,7 +644,7 @@ namespace cryptonote
LOG_ERROR_CCONTEXT
(
"sent wrong tx: failed to parse and validate transaction: "
<< epee::string_tools::buff_to_hex_nodelimer(tx_blob)
<< epee::string_tools::buff_to_hex_nodelimer(tx_blob.blob)
<< ", dropping connection"
);
@ -676,7 +679,7 @@ namespace cryptonote
cryptonote::blobdata txblob;
if(m_core.get_pool_transaction(tx_hash, txblob))
{
have_tx.push_back(txblob);
have_tx.push_back({txblob, crypto::null_hash});
}
else
{
@ -688,7 +691,7 @@ namespace cryptonote
{
if (txes.size() == 1)
{
have_tx.push_back(tx_to_blob(txes.front()));
have_tx.push_back({tx_to_blob(txes.front()), crypto::null_hash});
}
else
{
@ -771,6 +774,7 @@ namespace cryptonote
NOTIFY_REQUEST_CHAIN::request r = boost::value_initialized<NOTIFY_REQUEST_CHAIN::request>();
m_core.get_short_chain_history(r.block_ids);
handler_request_blocks_history( r.block_ids ); // change the limit(?), sleep(?)
r.prune = m_sync_pruned_blocks;
MLOG_P2P_MESSAGE("-->>NOTIFY_REQUEST_CHAIN: m_block_ids.size()=" << r.block_ids.size() );
post_notify<NOTIFY_REQUEST_CHAIN>(r, context);
MLOG_PEER_STATE("requesting chain");
@ -872,7 +876,7 @@ namespace cryptonote
for(auto& tx: txs)
{
fluffy_response.b.txs.push_back(t_serializable_object_to_blob(tx));
fluffy_response.b.txs.push_back({t_serializable_object_to_blob(tx), crypto::null_hash});
}
MLOG_P2P_MESSAGE
@ -910,7 +914,7 @@ namespace cryptonote
for (size_t i = 0; i < arg.txs.size(); ++i)
{
cryptonote::tx_verification_context tvc = AUTO_VAL_INIT(tvc);
m_core.handle_incoming_tx(arg.txs[i], tvc, false, true, false);
m_core.handle_incoming_tx({arg.txs[i], crypto::null_hash}, tvc, false, true, false);
if(tvc.m_verifivation_failed)
{
LOG_PRINT_CCONTEXT_L1("Tx verification failed, dropping connection");
@ -991,7 +995,7 @@ namespace cryptonote
for (const auto &element : arg.blocks) {
blocks_size += element.block.size();
for (const auto &tx : element.txs)
blocks_size += tx.size();
blocks_size += tx.blob.size();
}
size += blocks_size;
@ -1090,6 +1094,53 @@ namespace cryptonote
return 1;
}
const bool pruned_ok = should_ask_for_pruned_data(context, start_height, arg.blocks.size(), true);
if (!pruned_ok)
{
// if we don't want pruned data, check we did not get any
for (block_complete_entry& block_entry: arg.blocks)
{
if (block_entry.pruned)
{
MERROR(context << "returned a pruned block, dropping connection");
drop_connection(context, false, false);
++m_sync_bad_spans_downloaded;
return 1;
}
if (block_entry.block_weight)
{
MERROR(context << "returned a block weight for a non pruned block, dropping connection");
drop_connection(context, false, false);
++m_sync_bad_spans_downloaded;
return 1;
}
for (const tx_blob_entry &tx_entry: block_entry.txs)
{
if (tx_entry.prunable_hash != crypto::null_hash)
{
MERROR(context << "returned at least one pruned object which we did not expect, dropping connection");
drop_connection(context, false, false);
++m_sync_bad_spans_downloaded;
return 1;
}
}
}
}
else
{
// we accept pruned data, check that if we got some, then no weights are zero
for (block_complete_entry& block_entry: arg.blocks)
{
if (block_entry.block_weight == 0 && block_entry.pruned)
{
MERROR(context << "returned at least one pruned block with 0 weight, dropping connection");
drop_connection(context, false, false);
++m_sync_bad_spans_downloaded;
return 1;
}
}
}
{
MLOG_YELLOW(el::Level::Debug, context << " Got NEW BLOCKS inside of " << __FUNCTION__ << ": size: " << arg.blocks.size()
<< ", blocks: " << start_height << " - " << (start_height + arg.blocks.size() - 1) <<
@ -1273,18 +1324,32 @@ namespace cryptonote
if (tvc.size() != block_entry.txs.size())
{
LOG_ERROR_CCONTEXT("Internal error: tvc.size() != block_entry.txs.size()");
if (!m_core.cleanup_handle_incoming_blocks())
{
LOG_PRINT_CCONTEXT_L0("Failure in cleanup_handle_incoming_blocks");
return 1;
}
return 1;
}
std::vector<blobdata>::const_iterator it = block_entry.txs.begin();
std::vector<tx_blob_entry>::const_iterator it = block_entry.txs.begin();
for (size_t i = 0; i < tvc.size(); ++i, ++it)
{
if(tvc[i].m_verifivation_failed)
{
if (!m_p2p->for_connection(span_connection_id, [&](cryptonote_connection_context& context, nodetool::peerid_type peer_id, uint32_t f)->bool{
cryptonote::transaction tx;
parse_and_validate_tx_from_blob(*it, tx); // must succeed if we got here
crypto::hash txid;
if (it->prunable_hash == crypto::null_hash)
{
parse_and_validate_tx_from_blob(it->blob, tx, txid); // must succeed if we got here
}
else
{
parse_and_validate_tx_base_from_blob(it->blob, tx); // must succeed if we got here
txid = get_pruned_transaction_hash(tx, it->prunable_hash);
}
LOG_ERROR_CCONTEXT("transaction verification failed on NOTIFY_RESPONSE_GET_OBJECTS, tx_id = "
<< epee::string_tools::pod_to_hex(cryptonote::get_transaction_hash(tx)) << ", dropping connection");
<< epee::string_tools::pod_to_hex(txid) << ", dropping connection");
drop_connection(context, false, true);
return 1;
}))
@ -1543,7 +1608,7 @@ skip:
{
MLOG_P2P_MESSAGE("Received NOTIFY_REQUEST_CHAIN (" << arg.block_ids.size() << " blocks");
NOTIFY_RESPONSE_CHAIN_ENTRY::request r;
if(!m_core.find_blockchain_supplement(arg.block_ids, r))
if(!m_core.find_blockchain_supplement(arg.block_ids, !arg.prune, r))
{
LOG_ERROR_CCONTEXT("Failed to handle NOTIFY_REQUEST_CHAIN.");
drop_connection(context, false, false);
@ -1662,6 +1727,12 @@ skip:
MDEBUG(context << "This peer has needed stripe " << peer_stripe << ", not dropping");
return false;
}
const uint32_t local_stripe = tools::get_pruning_stripe(m_core.get_blockchain_pruning_seed());
if (m_sync_pruned_blocks && peer_stripe == local_stripe)
{
MDEBUG(context << "We can sync pruned blocks off this peer, not dropping");
return false;
}
if (!context.m_needed_objects.empty())
{
@ -1701,22 +1772,42 @@ skip:
{
// take out blocks we already have
size_t skip = 0;
while (skip < context.m_needed_objects.size() && (m_core.have_block(context.m_needed_objects[skip]) || (check_block_queue && m_block_queue.have(context.m_needed_objects[skip]))))
while (skip < context.m_needed_objects.size() && (m_core.have_block(context.m_needed_objects[skip].first) || (check_block_queue && m_block_queue.have(context.m_needed_objects[skip].first))))
{
// if we're popping the last hash, record it so we can ask again from that hash,
// this prevents never being able to progress on peers we get old hash lists from
if (skip + 1 == context.m_needed_objects.size())
context.m_last_known_hash = context.m_needed_objects[skip];
context.m_last_known_hash = context.m_needed_objects[skip].first;
++skip;
}
if (skip > 0)
{
MDEBUG(context << "skipping " << skip << "/" << context.m_needed_objects.size() << " blocks");
context.m_needed_objects = std::vector<crypto::hash>(context.m_needed_objects.begin() + skip, context.m_needed_objects.end());
context.m_needed_objects = std::vector<std::pair<crypto::hash, uint64_t>>(context.m_needed_objects.begin() + skip, context.m_needed_objects.end());
}
}
//------------------------------------------------------------------------------------------------------------------------
template<class t_core>
bool t_cryptonote_protocol_handler<t_core>::should_ask_for_pruned_data(cryptonote_connection_context& context, uint64_t first_block_height, uint64_t nblocks, bool check_block_weights) const
{
if (!m_sync_pruned_blocks)
return false;
if (!m_core.is_within_compiled_block_hash_area(first_block_height + nblocks - 1))
return false;
const uint32_t local_stripe = tools::get_pruning_stripe(m_core.get_blockchain_pruning_seed());
if (local_stripe == 0)
return false;
// assumes the span size is less or equal to the stripe size
bool full_data_needed = tools::get_pruning_stripe(first_block_height, context.m_remote_blockchain_height, CRYPTONOTE_PRUNING_LOG_STRIPES) == local_stripe
|| tools::get_pruning_stripe(first_block_height + nblocks - 1, context.m_remote_blockchain_height, CRYPTONOTE_PRUNING_LOG_STRIPES) == local_stripe;
if (full_data_needed)
return false;
if (check_block_weights && !m_core.has_block_weights(first_block_height, nblocks))
return false;
return true;
}
//------------------------------------------------------------------------------------------------------------------------
template<class t_core>
bool t_cryptonote_protocol_handler<t_core>::request_missing_objects(cryptonote_connection_context& context, bool check_having_blocks, bool force_next_span)
{
// flush stale spans
@ -1739,6 +1830,7 @@ skip:
const auto next_needed_pruning_stripe = get_next_needed_pruning_stripe();
const uint32_t add_stripe = tools::get_pruning_stripe(bc_height, context.m_remote_blockchain_height, CRYPTONOTE_PRUNING_LOG_STRIPES);
const uint32_t peer_stripe = tools::get_pruning_stripe(context.m_pruning_seed);
const uint32_t local_stripe = tools::get_pruning_stripe(m_core.get_blockchain_pruning_seed());
const size_t block_queue_size_threshold = m_block_download_max_size ? m_block_download_max_size : BLOCK_QUEUE_SIZE_THRESHOLD;
bool queue_proceed = nspans < BLOCK_QUEUE_NSPANS_THRESHOLD || size < block_queue_size_threshold;
// get rid of blocks we already requested, or already have
@ -1749,7 +1841,7 @@ skip:
next_block_height = next_needed_height;
else
next_block_height = context.m_last_response_height - context.m_needed_objects.size() + 1;
bool stripe_proceed_main = (add_stripe == 0 || peer_stripe == 0 || add_stripe == peer_stripe) && (next_block_height < bc_height + BLOCK_QUEUE_FORCE_DOWNLOAD_NEAR_BLOCKS || next_needed_height < bc_height + BLOCK_QUEUE_FORCE_DOWNLOAD_NEAR_BLOCKS);
bool stripe_proceed_main = ((m_sync_pruned_blocks && peer_stripe == local_stripe) || add_stripe == 0 || peer_stripe == 0 || add_stripe == peer_stripe) && (next_block_height < bc_height + BLOCK_QUEUE_FORCE_DOWNLOAD_NEAR_BLOCKS || next_needed_height < bc_height + BLOCK_QUEUE_FORCE_DOWNLOAD_NEAR_BLOCKS);
bool stripe_proceed_secondary = tools::has_unpruned_block(next_block_height, context.m_remote_blockchain_height, context.m_pruning_seed);
bool proceed = stripe_proceed_main || (queue_proceed && stripe_proceed_secondary);
if (!stripe_proceed_main && !stripe_proceed_secondary && should_drop_connection(context, tools::get_pruning_stripe(next_block_height, context.m_remote_blockchain_height, CRYPTONOTE_PRUNING_LOG_STRIPES)))
@ -1812,8 +1904,9 @@ skip:
{
const uint64_t now = tools::get_tick_count();
const uint64_t dt = now - m_last_add_end_time;
if (tools::ticks_to_ns(dt) >= DROP_ON_SYNC_WEDGE_THRESHOLD)
if (m_last_add_end_time && tools::ticks_to_ns(dt) >= DROP_ON_SYNC_WEDGE_THRESHOLD)
{
MDEBUG(context << "ns " << tools::ticks_to_ns(dt) << " from " << m_last_add_end_time << " and " << now);
MDEBUG(context << "Block addition seems to have wedged, dropping connection");
return false;
}
@ -1880,7 +1973,8 @@ skip:
skip_unneeded_hashes(context, false);
const uint64_t first_block_height = context.m_last_response_height - context.m_needed_objects.size() + 1;
span = m_block_queue.reserve_span(first_block_height, context.m_last_response_height, count_limit, context.m_connection_id, context.m_pruning_seed, context.m_remote_blockchain_height, context.m_needed_objects);
bool sync_pruned_blocks = m_sync_pruned_blocks && m_core.get_blockchain_pruning_seed();
span = m_block_queue.reserve_span(first_block_height, context.m_last_response_height, count_limit, context.m_connection_id, sync_pruned_blocks, m_core.get_blockchain_pruning_seed(), context.m_pruning_seed, context.m_remote_blockchain_height, context.m_needed_objects);
MDEBUG(context << " span from " << first_block_height << ": " << span.first << "/" << span.second);
if (span.second > 0)
{
@ -1910,7 +2004,8 @@ skip:
++count;
context.m_requested_objects.insert(hash);
// that's atrocious O(n) wise, but this is rare
auto i = std::find(context.m_needed_objects.begin(), context.m_needed_objects.end(), hash);
auto i = std::find_if(context.m_needed_objects.begin(), context.m_needed_objects.end(),
[&hash](const std::pair<crypto::hash, uint64_t> &o) { return o.first == hash; });
if (i != context.m_needed_objects.end())
context.m_needed_objects.erase(i);
}
@ -1929,7 +2024,7 @@ skip:
return false;
}
if (skip > 0)
context.m_needed_objects = std::vector<crypto::hash>(context.m_needed_objects.begin() + skip, context.m_needed_objects.end());
context.m_needed_objects = std::vector<std::pair<crypto::hash, uint64_t>>(context.m_needed_objects.begin() + skip, context.m_needed_objects.end());
if (context.m_needed_objects.size() < span.second)
{
MERROR("ERROR: span " << span.first << "/" << span.second << ", m_needed_objects " << context.m_needed_objects.size());
@ -1938,18 +2033,37 @@ skip:
for (size_t n = 0; n < span.second; ++n)
{
req.blocks.push_back(context.m_needed_objects[n]);
req.blocks.push_back(context.m_needed_objects[n].first);
++count;
context.m_requested_objects.insert(context.m_needed_objects[n]);
context.m_requested_objects.insert(context.m_needed_objects[n].first);
}
context.m_needed_objects = std::vector<crypto::hash>(context.m_needed_objects.begin() + span.second, context.m_needed_objects.end());
context.m_needed_objects = std::vector<std::pair<crypto::hash, uint64_t>>(context.m_needed_objects.begin() + span.second, context.m_needed_objects.end());
}
req.prune = should_ask_for_pruned_data(context, span.first, span.second, true);
// if we need to ask for full data and that peer does not have the right stripe, we can't ask it
if (!req.prune && context.m_pruning_seed)
{
const uint32_t peer_stripe = tools::get_pruning_stripe(context.m_pruning_seed);
const uint32_t first_stripe = tools::get_pruning_stripe(span.first, context.m_remote_blockchain_height, CRYPTONOTE_PRUNING_LOG_STRIPES);
const uint32_t last_stripe = tools::get_pruning_stripe(span.first + span.second - 1, context.m_remote_blockchain_height, CRYPTONOTE_PRUNING_LOG_STRIPES);
if ((first_stripe && peer_stripe != first_stripe) || (last_stripe && peer_stripe != last_stripe))
{
MDEBUG(context << "We need full data, but the peer does not have it, dropping peer");
return false;
}
}
context.m_last_request_time = boost::posix_time::microsec_clock::universal_time();
MLOG_P2P_MESSAGE("-->>NOTIFY_REQUEST_GET_OBJECTS: blocks.size()=" << req.blocks.size()
<< "requested blocks count=" << count << " / " << count_limit << " from " << span.first << ", first hash " << req.blocks.front());
//epee::net_utils::network_throttle_manager::get_global_throttle_inreq().logger_handle_net("log/dr-monero/net/req-all.data", sec, get_avg_block_size());
MDEBUG("Asking for " << (req.prune ? "pruned" : "full") << " data, start/end "
<< tools::get_pruning_stripe(span.first, context.m_remote_blockchain_height, CRYPTONOTE_PRUNING_LOG_STRIPES)
<< "/" << tools::get_pruning_stripe(span.first + span.second - 1, context.m_remote_blockchain_height, CRYPTONOTE_PRUNING_LOG_STRIPES)
<< ", ours " << tools::get_pruning_stripe(m_core.get_blockchain_pruning_seed()) << ", peer stripe " << tools::get_pruning_stripe(context.m_pruning_seed));
post_notify<NOTIFY_REQUEST_GET_OBJECTS>(req, context);
MLOG_PEER_STATE("requesting objects");
return true;
@ -1959,7 +2073,8 @@ skip:
// drop it to make space for other peers, or ask for a span further down the line
const uint32_t next_stripe = get_next_needed_pruning_stripe().first;
const uint32_t peer_stripe = tools::get_pruning_stripe(context.m_pruning_seed);
if (next_stripe && peer_stripe && next_stripe != peer_stripe)
const uint32_t local_stripe = tools::get_pruning_stripe(m_core.get_blockchain_pruning_seed());
if (!(m_sync_pruned_blocks && peer_stripe == local_stripe) && next_stripe && peer_stripe && next_stripe != peer_stripe)
{
// at this point, we have to either close the connection, or start getting blocks past the
// current point, or become dormant
@ -2022,6 +2137,7 @@ skip:
}
handler_request_blocks_history( r.block_ids ); // change the limit(?), sleep(?)
r.prune = m_sync_pruned_blocks;
//std::string blob; // for calculate size of request
//epee::serialization::store_t_to_binary(r, blob);
@ -2128,6 +2244,12 @@ skip:
drop_connection(context, true, false);
return 1;
}
if (!arg.m_block_weights.empty() && arg.m_block_weights.size() != arg.m_block_ids.size())
{
LOG_ERROR_CCONTEXT("sent invalid block weight array, dropping connection");
drop_connection(context, true, false);
return 1;
}
MDEBUG(context << "first block hash " << arg.m_block_ids.front() << ", last " << arg.m_block_ids.back());
if (arg.total_height >= CRYPTONOTE_MAX_BLOCK_NUMBER || arg.m_block_ids.size() >= CRYPTONOTE_MAX_BLOCK_NUMBER)
@ -2147,7 +2269,7 @@ skip:
return 1;
}
uint64_t n_use_blocks = m_core.prevalidate_block_hashes(arg.start_height, arg.m_block_ids);
uint64_t n_use_blocks = m_core.prevalidate_block_hashes(arg.start_height, arg.m_block_ids, arg.m_block_weights);
if (n_use_blocks + HASH_OF_HASHES_STEP <= arg.m_block_ids.size())
{
LOG_ERROR_CCONTEXT("Most blocks are invalid, dropping connection");
@ -2157,9 +2279,10 @@ skip:
context.m_needed_objects.clear();
uint64_t added = 0;
for(auto& bl_id: arg.m_block_ids)
for (size_t i = 0; i < arg.m_block_ids.size(); ++i)
{
context.m_needed_objects.push_back(bl_id);
const uint64_t block_weight = arg.m_block_weights.empty() ? 0 : arg.m_block_weights[i];
context.m_needed_objects.push_back(std::make_pair(arg.m_block_ids[i], block_weight));
if (++added == n_use_blocks)
break;
}
@ -2183,7 +2306,7 @@ skip:
{
NOTIFY_NEW_FLUFFY_BLOCK::request fluffy_arg = AUTO_VAL_INIT(fluffy_arg);
fluffy_arg.current_blockchain_height = arg.current_blockchain_height;
std::vector<blobdata> fluffy_txs;
std::vector<tx_blob_entry> fluffy_txs;
fluffy_arg.b = arg.b;
fluffy_arg.b.txs = fluffy_txs;

11
src/rpc/core_rpc_server.cpp
View file

@ -377,6 +377,7 @@ namespace cryptonote
for(auto& bd: bs)
{
res.blocks.resize(res.blocks.size()+1);
res.blocks.back().pruned = req.prune;
res.blocks.back().block = bd.first.first;
pruned_size += bd.first.first.size();
unpruned_size += bd.first.first.size();
@ -389,10 +390,10 @@ namespace cryptonote
for (std::vector<std::pair<crypto::hash, cryptonote::blobdata>>::iterator i = bd.second.begin(); i != bd.second.end(); ++i)
{
unpruned_size += i->second.size();
res.blocks.back().txs.push_back(std::move(i->second));
res.blocks.back().txs.push_back({std::move(i->second), crypto::null_hash});
i->second.clear();
i->second.shrink_to_fit();
pruned_size += res.blocks.back().txs.back().size();
pruned_size += res.blocks.back().txs.back().blob.size();
}
const size_t n_txes_to_lookup = bd.second.size() + (req.no_miner_tx ? 0 : 1);
@ -474,7 +475,7 @@ namespace cryptonote
res.blocks.resize(res.blocks.size() + 1);
res.blocks.back().block = block_to_blob(blk);
for (auto& tx : txs)
res.blocks.back().txs.push_back(tx_to_blob(tx));
res.blocks.back().txs.push_back({tx_to_blob(tx), crypto::null_hash});
}
res.status = CORE_RPC_STATUS_OK;
return true;
@ -488,7 +489,7 @@ namespace cryptonote
return r;
res.start_height = req.start_height;
if(!m_core.get_blockchain_storage().find_blockchain_supplement(req.block_ids, res.m_block_ids, res.start_height, res.current_height, false))
if(!m_core.get_blockchain_storage().find_blockchain_supplement(req.block_ids, res.m_block_ids, NULL, res.start_height, res.current_height, false))
{
res.status = "Failed";
add_host_fail(ctx);
@ -912,7 +913,7 @@ namespace cryptonote
cryptonote_connection_context fake_context = AUTO_VAL_INIT(fake_context);
tx_verification_context tvc = AUTO_VAL_INIT(tvc);
if(!m_core.handle_incoming_tx(tx_blob, tvc, false, false, req.do_not_relay) || tvc.m_verifivation_failed)
if(!m_core.handle_incoming_tx({tx_blob, crypto::null_hash}, tvc, false, false, req.do_not_relay) || tvc.m_verifivation_failed)
{
res.status = "Failed";
std::string reason = "";

4
src/rpc/daemon_handler.cpp
View file

@ -141,7 +141,7 @@ namespace rpc
auto& chain = m_core.get_blockchain_storage();
if (!chain.find_blockchain_supplement(req.known_hashes, res.hashes, res.start_height, res.current_height, false))
if (!chain.find_blockchain_supplement(req.known_hashes, res.hashes, NULL, res.start_height, res.current_height, false))
{
res.status = Message::STATUS_FAILED;
res.error_details = "Blockchain::find_blockchain_supplement() returned false";
@ -291,7 +291,7 @@ namespace rpc
cryptonote_connection_context fake_context = AUTO_VAL_INIT(fake_context);
tx_verification_context tvc = AUTO_VAL_INIT(tvc);
if(!m_core.handle_incoming_tx(tx_blob, tvc, false, false, !relay) || tvc.m_verifivation_failed)
if(!m_core.handle_incoming_tx({tx_blob, crypto::null_hash}, tvc, false, false, !relay) || tvc.m_verifivation_failed)
{
if (tvc.m_verifivation_failed)
{

19
src/serialization/json_object.cpp
View file

@ -627,6 +627,25 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::connection_info& inf
GET_FROM_JSON_OBJECT(val, info.current_upload, current_upload);
}
void toJsonValue(rapidjson::Document& doc, const cryptonote::tx_blob_entry& tx, rapidjson::Value& val)
{
val.SetObject();
INSERT_INTO_JSON_OBJECT(val, doc, blob, tx.blob);
INSERT_INTO_JSON_OBJECT(val, doc, prunable_hash, tx.prunable_hash);
}
void fromJsonValue(const rapidjson::Value& val, cryptonote::tx_blob_entry& tx)
{
if (!val.IsObject())
{
throw WRONG_TYPE("json object");
}
GET_FROM_JSON_OBJECT(val, tx.blob, blob);
GET_FROM_JSON_OBJECT(val, tx.prunable_hash, prunable_hash);
}
void toJsonValue(rapidjson::Document& doc, const cryptonote::block_complete_entry& blk, rapidjson::Value& val)
{
val.SetObject();

3
src/serialization/json_object.h
View file

@ -221,6 +221,9 @@ void fromJsonValue(const rapidjson::Value& val, cryptonote::tx_out& txout);
void toJsonValue(rapidjson::Document& doc, const cryptonote::connection_info& info, rapidjson::Value& val);
void fromJsonValue(const rapidjson::Value& val, cryptonote::connection_info& info);
void toJsonValue(rapidjson::Document& doc, const cryptonote::tx_blob_entry& tx, rapidjson::Value& val);
void fromJsonValue(const rapidjson::Value& val, cryptonote::tx_blob_entry& tx);
void toJsonValue(rapidjson::Document& doc, const cryptonote::block_complete_entry& blk, rapidjson::Value& val);
void fromJsonValue(const rapidjson::Value& val, cryptonote::block_complete_entry& blk);

2
src/wallet/wallet2.cpp
View file

@ -2735,7 +2735,7 @@ void wallet2::pull_and_parse_next_blocks(uint64_t start_height, uint64_t &blocks
for (size_t j = 0; j < blocks[i].txs.size(); ++j)
{
tpool.submit(&waiter, [&, i, j](){
if (!parse_and_validate_tx_base_from_blob(blocks[i].txs[j], parsed_blocks[i].txes[j]))
if (!parse_and_validate_tx_base_from_blob(blocks[i].txs[j].blob, parsed_blocks[i].txes[j]))
{
boost::unique_lock<boost::mutex> lock(error_lock);
error = true;

14
tests/core_proxy/core_proxy.cpp
View file

@ -160,7 +160,7 @@ string tx2str(const cryptonote::transaction& tx, const cryptonote::hash256& tx_h
return ss.str();
}*/
bool tests::proxy_core::handle_incoming_tx(const cryptonote::blobdata& tx_blob, cryptonote::tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay) {
bool tests::proxy_core::handle_incoming_tx(const cryptonote::tx_blob_entry& tx_blob, cryptonote::tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay) {
if (!keeped_by_block)
return true;
@ -168,7 +168,13 @@ bool tests::proxy_core::handle_incoming_tx(const cryptonote::blobdata& tx_blob,
crypto::hash tx_prefix_hash = null_hash;
transaction tx;
if (!parse_and_validate_tx_from_blob(tx_blob, tx, tx_hash, tx_prefix_hash)) {
if (tx_blob.prunable_hash != crypto::null_hash)
{
cerr << "WRONG TRANSACTION, pruned blob rejected" << endl;
return false;
}
if (!parse_and_validate_tx_from_blob(tx_blob.blob, tx, tx_hash, tx_prefix_hash)) {
cerr << "WRONG TRANSACTION BLOB, Failed to parse, rejected" << endl;
return false;
}
@ -176,7 +182,7 @@ bool tests::proxy_core::handle_incoming_tx(const cryptonote::blobdata& tx_blob,
cout << "TX " << endl << endl;
cout << tx_hash << endl;
cout << tx_prefix_hash << endl;
cout << tx_blob.size() << endl;
cout << tx_blob.blob.size() << endl;
//cout << string_tools::buff_to_hex_nodelimer(tx_blob) << endl << endl;
cout << obj_to_json_str(tx) << endl;
cout << endl << "ENDTX" << endl;
@ -184,7 +190,7 @@ bool tests::proxy_core::handle_incoming_tx(const cryptonote::blobdata& tx_blob,
return true;
}
bool tests::proxy_core::handle_incoming_txs(const std::vector<blobdata>& tx_blobs, std::vector<tx_verification_context>& tvc, bool keeped_by_block, bool relayed, bool do_not_relay)
bool tests::proxy_core::handle_incoming_txs(const std::vector<tx_blob_entry>& tx_blobs, std::vector<tx_verification_context>& tvc, bool keeped_by_block, bool relayed, bool do_not_relay)
{
tvc.resize(tx_blobs.size());
size_t i = 0;

10
tests/core_proxy/core_proxy.h
View file

@ -75,13 +75,13 @@ namespace tests
bool get_stat_info(cryptonote::core_stat_info& st_inf){return true;}
bool have_block(const crypto::hash& id);
void get_blockchain_top(uint64_t& height, crypto::hash& top_id);
bool handle_incoming_tx(const cryptonote::blobdata& tx_blob, cryptonote::tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay);
bool handle_incoming_txs(const std::vector<cryptonote::blobdata>& tx_blobs, std::vector<cryptonote::tx_verification_context>& tvc, bool keeped_by_block, bool relayed, bool do_not_relay);
bool handle_incoming_tx(const cryptonote::tx_blob_entry& tx_blob, cryptonote::tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay);
bool handle_incoming_txs(const std::vector<cryptonote::tx_blob_entry>& tx_blobs, std::vector<cryptonote::tx_verification_context>& tvc, bool keeped_by_block, bool relayed, bool do_not_relay);
bool handle_incoming_block(const cryptonote::blobdata& block_blob, const cryptonote::block *block, cryptonote::block_verification_context& bvc, bool update_miner_blocktemplate = true);
void pause_mine(){}
void resume_mine(){}
bool on_idle(){return true;}
bool find_blockchain_supplement(const std::list<crypto::hash>& qblock_ids, cryptonote::NOTIFY_RESPONSE_CHAIN_ENTRY::request& resp){return true;}
bool find_blockchain_supplement(const std::list<crypto::hash>& qblock_ids, bool clip_pruned, cryptonote::NOTIFY_RESPONSE_CHAIN_ENTRY::request& resp){return true;}
bool handle_get_objects(cryptonote::NOTIFY_REQUEST_GET_OBJECTS::request& arg, cryptonote::NOTIFY_RESPONSE_GET_OBJECTS::request& rsp, cryptonote::cryptonote_connection_context& context){return true;}
cryptonote::Blockchain &get_blockchain_storage() { throw std::runtime_error("Called invalid member function: please never call get_blockchain_storage on the TESTING class proxy_core."); }
bool get_test_drop_download() {return true;}
@ -103,7 +103,9 @@ namespace tests
uint64_t get_earliest_ideal_height_for_version(uint8_t version) const { return 0; }
cryptonote::difficulty_type get_block_cumulative_difficulty(uint64_t height) const { return 0; }
bool fluffy_blocks_enabled() const { return false; }
uint64_t prevalidate_block_hashes(uint64_t height, const std::vector<crypto::hash> &hashes) { return 0; }
uint64_t prevalidate_block_hashes(uint64_t height, const std::vector<crypto::hash> &hashes, const std::vector<uint64_t> &weights) { return 0; }
bool has_block_weights(uint64_t height, uint64_t nblocks) const { return false; }
bool is_within_compiled_block_hash_area(uint64_t height) const { return false; }
bool pad_transactions() const { return false; }
uint32_t get_blockchain_pruning_seed() const { return 0; }
bool prune_blockchain(uint32_t pruning_seed) const { return true; }

18
tests/core_tests/chaingen.h
View file

@ -544,7 +544,7 @@ public:
cryptonote::tx_verification_context tvc = AUTO_VAL_INIT(tvc);
size_t pool_size = m_c.get_pool_transactions_count();
m_c.handle_incoming_tx(t_serializable_object_to_blob(tx), tvc, m_txs_keeped_by_block, false, false);
m_c.handle_incoming_tx({t_serializable_object_to_blob(tx), crypto::null_hash}, tvc, m_txs_keeped_by_block, false, false);
bool tx_added = pool_size + 1 == m_c.get_pool_transactions_count();
bool r = m_validator.check_tx_verification_context(tvc, tx_added, m_ev_index, tx);
CHECK_AND_NO_ASSERT_MES(r, false, "tx verification context check failed");
@ -555,12 +555,12 @@ public:
{
log_event("cryptonote::transaction");
std::vector<cryptonote::blobdata> tx_blobs;
std::vector<cryptonote::tx_blob_entry> tx_blobs;
std::vector<cryptonote::tx_verification_context> tvcs;
cryptonote::tx_verification_context tvc0 = AUTO_VAL_INIT(tvc0);
for (const auto &tx: txs)
{
tx_blobs.push_back(t_serializable_object_to_blob(tx));
tx_blobs.push_back({t_serializable_object_to_blob(tx)});
tvcs.push_back(tvc0);
}
size_t pool_size = m_c.get_pool_transactions_count();
@ -578,7 +578,11 @@ public:
cryptonote::block_verification_context bvc = AUTO_VAL_INIT(bvc);
cryptonote::blobdata bd = t_serializable_object_to_blob(b);
std::vector<cryptonote::block> pblocks;
if (m_c.prepare_handle_incoming_blocks(std::vector<cryptonote::block_complete_entry>(1, {bd, {}}), pblocks))
cryptonote::block_complete_entry bce;
bce.pruned = false;
bce.block = bd;
bce.txs = {};
if (m_c.prepare_handle_incoming_blocks(std::vector<cryptonote::block_complete_entry>(1, bce), pblocks))
{
m_c.handle_incoming_block(bd, &b, bvc);
m_c.cleanup_handle_incoming_blocks();
@ -608,7 +612,11 @@ public:
cryptonote::block_verification_context bvc = AUTO_VAL_INIT(bvc);
std::vector<cryptonote::block> pblocks;
if (m_c.prepare_handle_incoming_blocks(std::vector<cryptonote::block_complete_entry>(1, {sr_block.data, {}}), pblocks))
cryptonote::block_complete_entry bce;
bce.pruned = false;
bce.block = sr_block.data;
bce.txs = {};
if (m_c.prepare_handle_incoming_blocks(std::vector<cryptonote::block_complete_entry>(1, bce), pblocks))
{
m_c.handle_incoming_block(sr_block.data, NULL, bvc);
m_c.cleanup_handle_incoming_blocks();

10
tests/unit_tests/ban.cpp
View file

@ -55,13 +55,13 @@ public:
bool get_stat_info(cryptonote::core_stat_info& st_inf) const {return true;}
bool have_block(const crypto::hash& id) const {return true;}
void get_blockchain_top(uint64_t& height, crypto::hash& top_id)const{height=0;top_id=crypto::null_hash;}
bool handle_incoming_tx(const cryptonote::blobdata& tx_blob, cryptonote::tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay) { return true; }
bool handle_incoming_txs(const std::vector<cryptonote::blobdata>& tx_blob, std::vector<cryptonote::tx_verification_context>& tvc, bool keeped_by_block, bool relayed, bool do_not_relay) { return true; }
bool handle_incoming_tx(const cryptonote::tx_blob_entry& tx_blob, cryptonote::tx_verification_context& tvc, bool keeped_by_block, bool relayed, bool do_not_relay) { return true; }
bool handle_incoming_txs(const std::vector<cryptonote::tx_blob_entry>& tx_blob, std::vector<cryptonote::tx_verification_context>& tvc, bool keeped_by_block, bool relayed, bool do_not_relay) { return true; }
bool handle_incoming_block(const cryptonote::blobdata& block_blob, const cryptonote::block *block, cryptonote::block_verification_context& bvc, bool update_miner_blocktemplate = true) { return true; }
void pause_mine(){}
void resume_mine(){}
bool on_idle(){return true;}
bool find_blockchain_supplement(const std::list<crypto::hash>& qblock_ids, cryptonote::NOTIFY_RESPONSE_CHAIN_ENTRY::request& resp){return true;}
bool find_blockchain_supplement(const std::list<crypto::hash>& qblock_ids, bool clip_pruned, cryptonote::NOTIFY_RESPONSE_CHAIN_ENTRY::request& resp){return true;}
bool handle_get_objects(cryptonote::NOTIFY_REQUEST_GET_OBJECTS::request& arg, cryptonote::NOTIFY_RESPONSE_GET_OBJECTS::request& rsp, cryptonote::cryptonote_connection_context& context){return true;}
cryptonote::blockchain_storage &get_blockchain_storage() { throw std::runtime_error("Called invalid member function: please never call get_blockchain_storage on the TESTING class test_core."); }
bool get_test_drop_download() const {return true;}
@ -83,10 +83,12 @@ public:
uint64_t get_earliest_ideal_height_for_version(uint8_t version) const { return 0; }
cryptonote::difficulty_type get_block_cumulative_difficulty(uint64_t height) const { return 0; }
bool fluffy_blocks_enabled() const { return false; }
uint64_t prevalidate_block_hashes(uint64_t height, const std::vector<crypto::hash> &hashes) { return 0; }
uint64_t prevalidate_block_hashes(uint64_t height, const std::vector<crypto::hash> &hashes, const std::vector<uint64_t> &weights) { return 0; }
bool pad_transactions() { return false; }
uint32_t get_blockchain_pruning_seed() const { return 0; }
bool prune_blockchain(uint32_t pruning_seed = 0) { return true; }
bool is_within_compiled_block_hash_area(uint64_t height) const { return false; }
bool has_block_weights(uint64_t height, uint64_t nblocks) const { return false; }
void stop() {}
};