From bf22109d1636faead40ce8c71a2bd416afdd147f Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@symas.com>
Date: Sun, 14 Feb 2016 19:31:52 +0000
Subject: [PATCH 1/2] Win32 import batchsize tweaks
Reduce frequency of resizes: bump minimum increase from 128MB to 512MB
Use a bigger safety margin at small batch sizes
---
src/blockchain_db/lmdb/db_lmdb.cpp | 9 ++++++---
1 file changed, 6 insertions(+), 3 deletions(-)
diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp
index 0bee8aae9..1c1e595af 100644
--- a/src/blockchain_db/lmdb/db_lmdb.cpp
+++ b/src/blockchain_db/lmdb/db_lmdb.cpp
@@ -428,7 +428,7 @@ void BlockchainLMDB::check_and_resize_for_batch(uint64_t batch_num_blocks)
{
LOG_PRINT_L3("BlockchainLMDB::" << __func__);
LOG_PRINT_L1("[" << __func__ << "] " << "checking DB size");
- const uint64_t min_increase_size = 128 * (1 << 20);
+ const uint64_t min_increase_size = 512 * (1 << 20);
uint64_t threshold_size = 0;
uint64_t increase_size = 0;
if (batch_num_blocks > 0)
@@ -464,6 +464,7 @@ uint64_t BlockchainLMDB::get_estimated_batch_size(uint64_t batch_num_blocks) con
// batch size estimate * batch safety factor = final size estimate
// Takes into account "reasonable" block size increases in batch.
float batch_safety_factor = 1.7f;
+ float batch_fudge_factor = batch_safety_factor * batch_num_blocks;
// estimate of stored block expanded from raw block, including denormalization and db overhead.
// Note that this probably doesn't grow linearly with block size.
float db_expand_factor = 4.5f;
@@ -502,8 +503,10 @@ uint64_t BlockchainLMDB::get_estimated_batch_size(uint64_t batch_num_blocks) con
avg_block_size = min_block_size;
LOG_PRINT_L1("estimated average block size for batch: " << avg_block_size);
- threshold_size = avg_block_size * db_expand_factor * batch_num_blocks;
- threshold_size = threshold_size * batch_safety_factor;
+ // bigger safety margin on smaller block sizes
+ if (batch_fudge_factor < 5000.0)
+ batch_fudge_factor = 5000.0;
+ threshold_size = avg_block_size * db_expand_factor * batch_fudge_factor;
return threshold_size;
}
From d8f9bb380cf70a80963a12e27baeab66474b13a2 Mon Sep 17 00:00:00 2001
From: Howard Chu <hyc@symas.com>
Date: Sun, 14 Feb 2016 20:27:43 +0000
Subject: [PATCH 2/2] Keep a running blocksize count
Used in batch size estimation, avoids rereading already processed
blocks during import
---
src/blockchain_db/lmdb/db_lmdb.cpp | 13 +++++++++++++
src/blockchain_db/lmdb/db_lmdb.h | 2 ++
2 files changed, 15 insertions(+)
diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp
index 1c1e595af..a93079aa7 100644
--- a/src/blockchain_db/lmdb/db_lmdb.cpp
+++ b/src/blockchain_db/lmdb/db_lmdb.cpp
@@ -486,6 +486,13 @@ uint64_t BlockchainLMDB::get_estimated_batch_size(uint64_t batch_num_blocks) con
{
LOG_PRINT_L1("No existing blocks to check for average block size");
}
+ else if (m_cum_count)
+ {
+ avg_block_size = m_cum_size / m_cum_count;
+ LOG_PRINT_L1("average block size across recent " << m_cum_count << " blocks: " << avg_block_size);
+ m_cum_size = 0;
+ m_cum_count = 0;
+ }
else
{
for (uint64_t block_num = block_start; block_num <= block_stop; ++block_num)
@@ -573,6 +580,8 @@ void BlockchainLMDB::add_block(const block& blk, const size_t& block_size, const
if (result)
throw0(DB_ERROR(std::string("Failed to add block hash to db transaction: ").append(mdb_strerror(result)).c_str()));
+ m_cum_size += block_size;
+ m_cum_count++;
}
void BlockchainLMDB::remove_block()
@@ -948,6 +957,8 @@ BlockchainLMDB::BlockchainLMDB(bool batch_transactions)
m_write_batch_txn = nullptr;
m_batch_active = false;
m_height = 0;
+ m_cum_size = 0;
+ m_cum_count = 0;
m_hardfork = nullptr;
}
@@ -1211,6 +1222,8 @@ void BlockchainLMDB::reset()
txn.commit();
m_height = 0;
m_num_outputs = 0;
+ m_cum_size = 0;
+ m_cum_count = 0;
}
std::vector<std::string> BlockchainLMDB::get_filenames() const
diff --git a/src/blockchain_db/lmdb/db_lmdb.h b/src/blockchain_db/lmdb/db_lmdb.h
index e88bcd01b..1964bfe45 100644
--- a/src/blockchain_db/lmdb/db_lmdb.h
+++ b/src/blockchain_db/lmdb/db_lmdb.h
@@ -306,6 +306,8 @@ private:
uint64_t m_height;
uint64_t m_num_outputs;
+ mutable uint64_t m_cum_size; // used in batch size estimation
+ mutable int m_cum_count;
std::string m_folder;
mdb_txn_safe* m_write_txn; // may point to either a short-lived txn or a batch txn
mdb_txn_safe* m_write_batch_txn; // persist batch txn outside of BlockchainLMDB