From 4b21d38dfda29fe916700887c99d44edeb8fe06f Mon Sep 17 00:00:00 2001
From: moneromooo-monero <moneromooo-monero@users.noreply.github.com>
Date: Thu, 21 Feb 2019 00:13:21 +0000
Subject: [PATCH] blockchain: speed up getting N blocks weights/long term
weights
---
src/blockchain_db/blockchain_db.h | 22 +++++++
src/blockchain_db/lmdb/db_lmdb.cpp | 64 +++++++++++++++++++++
src/blockchain_db/lmdb/db_lmdb.h | 6 ++
src/blockchain_db/testdb.h | 2 +
src/cryptonote_core/blockchain.cpp | 24 ++++----
src/cryptonote_core/blockchain.h | 15 ++++-
tests/block_weight/block_weight.cpp | 12 ++++
tests/unit_tests/long_term_block_weight.cpp | 12 ++++
8 files changed, 142 insertions(+), 15 deletions(-)
diff --git a/src/blockchain_db/blockchain_db.h b/src/blockchain_db/blockchain_db.h
index ed13de5b5..8eb1d7fa5 100644
--- a/src/blockchain_db/blockchain_db.h
+++ b/src/blockchain_db/blockchain_db.h
@@ -946,6 +946,17 @@ public:
*/
virtual size_t get_block_weight(const uint64_t& height) const = 0;
+ /**
+ * @brief fetch the last N blocks' weights
+ *
+ * If there are fewer than N blocks, the returned array will be smaller than N
+ *
+ * @param count the number of blocks requested
+ *
+ * @return the weights
+ */
+ virtual std::vector<uint64_t> get_block_weights(uint64_t start_height, size_t count) const = 0;
+
/**
* @brief fetch a block's cumulative difficulty
*
@@ -999,6 +1010,17 @@ public:
*/
virtual uint64_t get_block_long_term_weight(const uint64_t& height) const = 0;
+ /**
+ * @brief fetch the last N blocks' long term weights
+ *
+ * If there are fewer than N blocks, the returned array will be smaller than N
+ *
+ * @param count the number of blocks requested
+ *
+ * @return the weights
+ */
+ virtual std::vector<uint64_t> get_long_term_block_weights(uint64_t start_height, size_t count) const = 0;
+
/**
* @brief fetch a block's hash
*
diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp
index e2db5e04e..2cc070ed8 100644
--- a/src/blockchain_db/lmdb/db_lmdb.cpp
+++ b/src/blockchain_db/lmdb/db_lmdb.cpp
@@ -2445,6 +2445,70 @@ size_t BlockchainLMDB::get_block_weight(const uint64_t& height) const
return ret;
}
+std::vector<uint64_t> BlockchainLMDB::get_block_info_64bit_fields(uint64_t start_height, size_t count, off_t offset) const
+{
+ LOG_PRINT_L3("BlockchainLMDB::" << __func__);
+ check_open();
+
+ TXN_PREFIX_RDONLY();
+ RCURSOR(block_info);
+
+ const uint64_t h = height();
+ if (start_height >= h)
+ throw0(DB_ERROR(("Height " + std::to_string(start_height) + " not in blockchain").c_str()));
+
+ std::vector<uint64_t> ret;
+ ret.reserve(count);
+
+ MDB_val v;
+ uint64_t range_begin = 0, range_end = 0;
+ for (uint64_t height = start_height; height < h && count--; ++height)
+ {
+ if (height >= range_begin && height < range_end)
+ {
+ // nothing to do
+ }
+ else
+ {
+ int result = 0;
+ if (range_end > 0)
+ {
+ MDB_val k2;
+ result = mdb_cursor_get(m_cur_block_info, &k2, &v, MDB_NEXT_MULTIPLE);
+ range_begin = ((const mdb_block_info*)v.mv_data)->bi_height;
+ range_end = range_begin + v.mv_size / sizeof(mdb_block_info); // whole records please
+ if (height < range_begin || height >= range_end)
+ throw0(DB_ERROR(("Height " + std::to_string(height) + " not included in multiple record range: " + std::to_string(range_begin) + "-" + std::to_string(range_end)).c_str()));
+ }
+ else
+ {
+ v.mv_size = sizeof(uint64_t);
+ v.mv_data = (void*)&height;
+ result = mdb_cursor_get(m_cur_block_info, (MDB_val *)&zerokval, &v, MDB_GET_BOTH);
+ range_begin = height;
+ range_end = range_begin + 1;
+ }
+ if (result)
+ throw0(DB_ERROR(lmdb_error("Error attempting to retrieve block_info from the db: ", result).c_str()));
+ }
+ const mdb_block_info *bi = ((const mdb_block_info *)v.mv_data) + (height - range_begin);
+ ret.push_back(*(const uint64_t*)(((const char*)bi) + offset));
+ }
+
+ TXN_POSTFIX_RDONLY();
+ return ret;
+}
+
+std::vector<uint64_t> BlockchainLMDB::get_block_weights(uint64_t start_height, size_t count) const
+{
+ return get_block_info_64bit_fields(start_height, count, offsetof(mdb_block_info, bi_weight));
+}
+
+std::vector<uint64_t> BlockchainLMDB::get_long_term_block_weights(uint64_t start_height, size_t count) const
+{
+ return get_block_info_64bit_fields(start_height, count, offsetof(mdb_block_info, bi_long_term_block_weight));
+}
+
difficulty_type BlockchainLMDB::get_block_cumulative_difficulty(const uint64_t& height) const
{
LOG_PRINT_L3("BlockchainLMDB::" << __func__ << " height: " << height);
diff --git a/src/blockchain_db/lmdb/db_lmdb.h b/src/blockchain_db/lmdb/db_lmdb.h
index 82016c17a..fcf87b646 100644
--- a/src/blockchain_db/lmdb/db_lmdb.h
+++ b/src/blockchain_db/lmdb/db_lmdb.h
@@ -219,6 +219,8 @@ public:
virtual size_t get_block_weight(const uint64_t& height) const;
+ virtual std::vector<uint64_t> get_block_weights(uint64_t start_height, size_t count) const;
+
virtual difficulty_type get_block_cumulative_difficulty(const uint64_t& height) const;
virtual difficulty_type get_block_difficulty(const uint64_t& height) const;
@@ -227,6 +229,8 @@ public:
virtual uint64_t get_block_long_term_weight(const uint64_t& height) const;
+ virtual std::vector<uint64_t> get_long_term_block_weights(uint64_t start_height, size_t count) const;
+
virtual crypto::hash get_block_hash_from_height(const uint64_t& height) const;
virtual std::vector<block> get_blocks_range(const uint64_t& h1, const uint64_t& h2) const;
@@ -394,6 +398,8 @@ private:
virtual uint64_t get_database_size() const;
+ std::vector<uint64_t> get_block_info_64bit_fields(uint64_t start_height, size_t count, off_t offset) const;
+
// fix up anything that may be wrong due to past bugs
virtual void fixup();
diff --git a/src/blockchain_db/testdb.h b/src/blockchain_db/testdb.h
index ac1849b5f..2ead91c3e 100644
--- a/src/blockchain_db/testdb.h
+++ b/src/blockchain_db/testdb.h
@@ -72,10 +72,12 @@ public:
virtual std::vector<uint64_t> get_block_cumulative_rct_outputs(const std::vector<uint64_t> &heights) const { return {}; }
virtual uint64_t get_top_block_timestamp() const { return 0; }
virtual size_t get_block_weight(const uint64_t& height) const { return 128; }
+ virtual std::vector<uint64_t> get_block_weights(uint64_t start_height, size_t count) const { return {}; }
virtual cryptonote::difficulty_type get_block_cumulative_difficulty(const uint64_t& height) const { return 10; }
virtual cryptonote::difficulty_type get_block_difficulty(const uint64_t& height) const { return 0; }
virtual uint64_t get_block_already_generated_coins(const uint64_t& height) const { return 10000000000; }
virtual uint64_t get_block_long_term_weight(const uint64_t& height) const { return 128; }
+ virtual std::vector<uint64_t> get_long_term_block_weights(uint64_t start_height, size_t count) const { return {}; }
virtual crypto::hash get_block_hash_from_height(const uint64_t& height) const { return crypto::hash(); }
virtual std::vector<cryptonote::block> get_blocks_range(const uint64_t& h1, const uint64_t& h2) const { return std::vector<cryptonote::block>(); }
virtual std::vector<crypto::hash> get_hashes_range(const uint64_t& h1, const uint64_t& h2) const { return std::vector<crypto::hash>(); }
diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp
index 50e0f1959..03725c665 100644
--- a/src/cryptonote_core/blockchain.cpp
+++ b/src/cryptonote_core/blockchain.cpp
@@ -1269,15 +1269,17 @@ void Blockchain::get_last_n_blocks_weights(std::vector<uint64_t>& weights, size_
if(h == 0)
return;
- m_db->block_txn_start(true);
// add weight of last <count> blocks to vector <weights> (or less, if blockchain size < count)
size_t start_offset = h - std::min<size_t>(h, count);
- weights.reserve(weights.size() + h - start_offset);
- for(size_t i = start_offset; i < h; i++)
- {
- weights.push_back(m_db->get_block_weight(i));
- }
- m_db->block_txn_stop();
+ weights = m_db->get_block_weights(start_offset, count);
+}
+//------------------------------------------------------------------
+void Blockchain::get_long_term_block_weights(std::vector<uint64_t>& weights, uint64_t start_height, size_t count) const
+{
+ LOG_PRINT_L3("Blockchain::" << __func__);
+ CRITICAL_REGION_LOCAL(m_blockchain_lock);
+
+ weights = m_db->get_long_term_block_weights(start_height, count);
}
//------------------------------------------------------------------
uint64_t Blockchain::get_current_cumulative_block_weight_limit() const
@@ -3804,9 +3806,7 @@ uint64_t Blockchain::get_next_long_term_block_weight(uint64_t block_weight) cons
return block_weight;
std::vector<uint64_t> weights;
- weights.resize(nblocks);
- for (uint64_t h = 0; h < nblocks; ++h)
- weights[h] = m_db->get_block_long_term_weight(db_height - nblocks + h);
+ get_long_term_block_weights(weights, db_height - nblocks, nblocks);
uint64_t long_term_median = epee::misc_utils::median(weights);
uint64_t long_term_effective_median_block_weight = std::max<uint64_t>(CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5, long_term_median);
@@ -3850,9 +3850,7 @@ bool Blockchain::update_next_cumulative_weight_limit(uint64_t *long_term_effecti
uint64_t nblocks = std::min<uint64_t>(m_long_term_block_weights_window, db_height);
if (nblocks == db_height)
--nblocks;
- weights.resize(nblocks);
- for (uint64_t h = 0; h < nblocks; ++h)
- weights[h] = m_db->get_block_long_term_weight(db_height - nblocks + h - 1);
+ get_long_term_block_weights(weights, db_height - nblocks - 1, nblocks);
new_weights = weights;
long_term_median = epee::misc_utils::median(weights);
}
diff --git a/src/cryptonote_core/blockchain.h b/src/cryptonote_core/blockchain.h
index 9d928e386..b20447c29 100644
--- a/src/cryptonote_core/blockchain.h
+++ b/src/cryptonote_core/blockchain.h
@@ -1288,13 +1288,24 @@ namespace cryptonote
/**
* @brief gets recent block weights for median calculation
*
- * get the block weights of the last <count> blocks, and return by reference <sz>.
+ * get the block weights of the last <count> blocks, and return by reference <weights>.
*
- * @param sz return-by-reference the list of weights
+ * @param weights return-by-reference the list of weights
* @param count the number of blocks to get weights for
*/
void get_last_n_blocks_weights(std::vector<uint64_t>& weights, size_t count) const;
+ /**
+ * @brief gets recent block long term weights for median calculation
+ *
+ * get the block long term weights of the last <count> blocks, and return by reference <weights>.
+ *
+ * @param weights return-by-reference the list of weights
+ * @param start_height the block height of the first block to query
+ * @param count the number of blocks to get weights for
+ */
+ void get_long_term_block_weights(std::vector<uint64_t>& weights, uint64_t start_height, size_t count) const;
+
/**
* @brief checks if a transaction is unlocked (its outputs spendable)
*
diff --git a/tests/block_weight/block_weight.cpp b/tests/block_weight/block_weight.cpp
index 57fcb497e..1e8974b78 100644
--- a/tests/block_weight/block_weight.cpp
+++ b/tests/block_weight/block_weight.cpp
@@ -72,6 +72,18 @@ public:
virtual uint64_t height() const override { return blocks.size(); }
virtual size_t get_block_weight(const uint64_t &h) const override { return blocks[h].weight; }
virtual uint64_t get_block_long_term_weight(const uint64_t &h) const override { return blocks[h].long_term_weight; }
+ virtual std::vector<uint64_t> get_block_weights(uint64_t start_height, size_t count) const override {
+ std::vector<uint64_t> ret;
+ ret.reserve(count);
+ while (count-- && start_height < blocks.size()) ret.push_back(blocks[start_height++].weight);
+ return ret;
+ }
+ virtual std::vector<uint64_t> get_long_term_block_weights(uint64_t start_height, size_t count) const override {
+ std::vector<uint64_t> ret;
+ ret.reserve(count);
+ while (count-- && start_height < blocks.size()) ret.push_back(blocks[start_height++].long_term_weight);
+ return ret;
+ }
virtual crypto::hash top_block_hash(uint64_t *block_height = NULL) const override {
uint64_t h = height();
crypto::hash top = crypto::null_hash;
diff --git a/tests/unit_tests/long_term_block_weight.cpp b/tests/unit_tests/long_term_block_weight.cpp
index f37b608b6..ce6ff3551 100644
--- a/tests/unit_tests/long_term_block_weight.cpp
+++ b/tests/unit_tests/long_term_block_weight.cpp
@@ -64,6 +64,18 @@ public:
virtual uint64_t height() const override { return blocks.size(); }
virtual size_t get_block_weight(const uint64_t &h) const override { return blocks[h].weight; }
virtual uint64_t get_block_long_term_weight(const uint64_t &h) const override { return blocks[h].long_term_weight; }
+ virtual std::vector<uint64_t> get_block_weights(uint64_t start_height, size_t count) const override {
+ std::vector<uint64_t> ret;
+ ret.reserve(count);
+ while (count-- && start_height < blocks.size()) ret.push_back(blocks[start_height++].weight);
+ return ret;
+ }
+ virtual std::vector<uint64_t> get_long_term_block_weights(uint64_t start_height, size_t count) const override {
+ std::vector<uint64_t> ret;
+ ret.reserve(count);
+ while (count-- && start_height < blocks.size()) ret.push_back(blocks[start_height++].long_term_weight);
+ return ret;
+ }
virtual crypto::hash top_block_hash(uint64_t *block_height = NULL) const override {
uint64_t h = height();
crypto::hash top = crypto::null_hash;