diff --git a/contrib/epee/src/mlocker.cpp b/contrib/epee/src/mlocker.cpp index 5573d591a..2d6e64e7a 100644 --- a/contrib/epee/src/mlocker.cpp +++ b/contrib/epee/src/mlocker.cpp @@ -47,7 +47,6 @@ static size_t query_page_size() MERROR("Failed to determine page size"); return 0; } - MINFO("Page size: " << ret); return ret; #else #warning Missing query_page_size implementation diff --git a/src/blockchain_db/blockchain_db.cpp b/src/blockchain_db/blockchain_db.cpp index be0ffeac3..682857157 100644 --- a/src/blockchain_db/blockchain_db.cpp +++ b/src/blockchain_db/blockchain_db.cpp @@ -197,6 +197,7 @@ void BlockchainDB::add_transaction(const crypto::hash& blk_hash, const transacti uint64_t BlockchainDB::add_block( const block& blk , size_t block_weight + , uint64_t long_term_block_weight , const difficulty_type& cumulative_difficulty , const uint64_t& coins_generated , const std::vector& txs @@ -241,7 +242,7 @@ uint64_t BlockchainDB::add_block( const block& blk // call out to subclass implementation to add the block & metadata time1 = epee::misc_utils::get_tick_count(); - add_block(blk, block_weight, cumulative_difficulty, coins_generated, num_rct_outs, blk_hash); + add_block(blk, block_weight, long_term_block_weight, cumulative_difficulty, coins_generated, num_rct_outs, blk_hash); TIME_MEASURE_FINISH(time1); time_add_block1 += time1; diff --git a/src/blockchain_db/blockchain_db.h b/src/blockchain_db/blockchain_db.h index 396ae7544..6cd4cf346 100644 --- a/src/blockchain_db/blockchain_db.h +++ b/src/blockchain_db/blockchain_db.h @@ -359,12 +359,14 @@ private: * * @param blk the block to be added * @param block_weight the weight of the block (transactions and all) + * @param long_term_block_weight the long term block weight of the block (transactions and all) * @param cumulative_difficulty the accumulated difficulty after this block * @param coins_generated the number of coins generated total after this block * @param blk_hash the hash of the block */ virtual void add_block( const block& blk , size_t block_weight + , uint64_t long_term_block_weight , const difficulty_type& cumulative_difficulty , const uint64_t& coins_generated , uint64_t num_rct_outs @@ -376,7 +378,7 @@ private: * * The subclass implementing this will remove the block data from the top * block in the chain. The data to be removed is that which was added in - * BlockchainDB::add_block(const block& blk, size_t block_weight, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated, const crypto::hash& blk_hash) + * BlockchainDB::add_block(const block& blk, size_t block_weight, uint64_t long_term_block_weight, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated, const crypto::hash& blk_hash) * * If any of this cannot be done, the subclass should throw the corresponding * subclass of DB_EXCEPTION @@ -790,6 +792,7 @@ public: * * @param blk the block to be added * @param block_weight the size of the block (transactions and all) + * @param long_term_block_weight the long term weight of the block (transactions and all) * @param cumulative_difficulty the accumulated difficulty after this block * @param coins_generated the number of coins generated total after this block * @param txs the transactions in the block @@ -798,6 +801,7 @@ public: */ virtual uint64_t add_block( const block& blk , size_t block_weight + , uint64_t long_term_block_weight , const difficulty_type& cumulative_difficulty , const uint64_t& coins_generated , const std::vector& txs @@ -985,6 +989,17 @@ public: */ virtual uint64_t get_block_already_generated_coins(const uint64_t& height) const = 0; + /** + * @brief fetch a block's long term weight + * + * If the block does not exist, the subclass should throw BLOCK_DNE + * + * @param height the height requested + * + * @return the long term weight + */ + virtual uint64_t get_block_long_term_weight(const uint64_t& height) const = 0; + /** * @brief fetch a block's hash * diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index bd91f308a..747d30cf8 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -30,6 +30,7 @@ #include #include #include +#include #include // std::unique_ptr #include // memcpy #include @@ -54,7 +55,7 @@ using epee::string_tools::pod_to_hex; using namespace crypto; // Increase when the DB structure changes -#define VERSION 3 +#define VERSION 4 namespace { @@ -251,7 +252,7 @@ inline void lmdb_db_open(MDB_txn* txn, const char* name, int flags, MDB_dbi& dbi namespace cryptonote { -typedef struct mdb_block_info_old +typedef struct mdb_block_info_1 { uint64_t bi_height; uint64_t bi_timestamp; @@ -259,9 +260,9 @@ typedef struct mdb_block_info_old uint64_t bi_weight; // a size_t really but we need 32-bit compat difficulty_type bi_diff; crypto::hash bi_hash; -} mdb_block_info_old; +} mdb_block_info_1; -typedef struct mdb_block_info +typedef struct mdb_block_info_2 { uint64_t bi_height; uint64_t bi_timestamp; @@ -270,7 +271,21 @@ typedef struct mdb_block_info difficulty_type bi_diff; crypto::hash bi_hash; uint64_t bi_cum_rct; -} mdb_block_info; +} mdb_block_info_2; + +typedef struct mdb_block_info_3 +{ + uint64_t bi_height; + uint64_t bi_timestamp; + uint64_t bi_coins; + uint64_t bi_weight; // a size_t really but we need 32-bit compat + difficulty_type bi_diff; + crypto::hash bi_hash; + uint64_t bi_cum_rct; + uint64_t bi_long_term_block_weight; +} mdb_block_info_3; + +typedef mdb_block_info_3 mdb_block_info; typedef struct blk_height { crypto::hash bh_hash; @@ -677,7 +692,7 @@ estim: return threshold_size; } -void BlockchainLMDB::add_block(const block& blk, size_t block_weight, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated, +void BlockchainLMDB::add_block(const block& blk, size_t block_weight, uint64_t long_term_block_weight, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated, uint64_t num_rct_outs, const crypto::hash& blk_hash) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); @@ -736,6 +751,7 @@ void BlockchainLMDB::add_block(const block& blk, size_t block_weight, const diff const mdb_block_info *bi_prev = (const mdb_block_info*)h.mv_data; bi.bi_cum_rct += bi_prev->bi_cum_rct; } + bi.bi_long_term_block_weight = long_term_block_weight; MDB_val_set(val, bi); result = mdb_cursor_put(m_cur_block_info, (MDB_val *)&zerokval, &val, MDB_APPENDDUP); @@ -2105,6 +2121,29 @@ uint64_t BlockchainLMDB::get_block_already_generated_coins(const uint64_t& heigh return ret; } +uint64_t BlockchainLMDB::get_block_long_term_weight(const uint64_t& height) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + TXN_PREFIX_RDONLY(); + RCURSOR(block_info); + + MDB_val_set(result, height); + auto get_result = mdb_cursor_get(m_cur_block_info, (MDB_val *)&zerokval, &result, MDB_GET_BOTH); + if (get_result == MDB_NOTFOUND) + { + throw0(BLOCK_DNE(std::string("Attempt to get block long term weight from height ").append(boost::lexical_cast(height)).append(" failed -- block info not in db").c_str())); + } + else if (get_result) + throw0(DB_ERROR("Error attempting to retrieve a long term block weight from the db")); + + mdb_block_info *bi = (mdb_block_info *)result.mv_data; + uint64_t ret = bi->bi_long_term_block_weight; + TXN_POSTFIX_RDONLY(); + return ret; +} + crypto::hash BlockchainLMDB::get_block_hash_from_height(const uint64_t& height) const { LOG_PRINT_L3("BlockchainLMDB::" << __func__); @@ -3098,7 +3137,7 @@ void BlockchainLMDB::block_txn_abort() } } -uint64_t BlockchainLMDB::add_block(const block& blk, size_t block_weight, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated, +uint64_t BlockchainLMDB::add_block(const block& blk, size_t block_weight, uint64_t long_term_block_weight, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated, const std::vector& txs) { LOG_PRINT_L3("BlockchainLMDB::" << __func__); @@ -3117,7 +3156,7 @@ uint64_t BlockchainLMDB::add_block(const block& blk, size_t block_weight, const try { - BlockchainDB::add_block(blk, block_weight, cumulative_difficulty, coins_generated, txs); + BlockchainDB::add_block(blk, block_weight, long_term_block_weight, cumulative_difficulty, coins_generated, txs); } catch (const DB_ERROR_TXN_START& e) { @@ -3655,7 +3694,7 @@ void BlockchainLMDB::migrate_0_1() break; } MDB_dbi diffs, hashes, sizes, timestamps; - mdb_block_info_old bi; + mdb_block_info_1 bi; MDB_val_set(nv, bi); lmdb_db_open(txn, "block_diffs", 0, diffs, "Failed to open db handle for block_diffs"); @@ -4282,8 +4321,8 @@ void BlockchainLMDB::migrate_2_3() } else if (result) throw0(DB_ERROR(lmdb_error("Failed to get a record from block_info: ", result).c_str())); - const mdb_block_info_old *bi_old = (const mdb_block_info_old*)v.mv_data; - mdb_block_info bi; + const mdb_block_info_1 *bi_old = (const mdb_block_info_1*)v.mv_data; + mdb_block_info_2 bi; bi.bi_height = bi_old->bi_height; bi.bi_timestamp = bi_old->bi_timestamp; bi.bi_coins = bi_old->bi_coins; @@ -4336,6 +4375,161 @@ void BlockchainLMDB::migrate_2_3() txn.commit(); } +void BlockchainLMDB::migrate_3_4() +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + uint64_t i; + int result; + mdb_txn_safe txn(false); + MDB_val k, v; + char *ptr; + bool past_long_term_weight = false; + + MGINFO_YELLOW("Migrating blockchain from DB version 3 to 4 - this may take a while:"); + + do { + LOG_PRINT_L1("migrating block info:"); + + result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + + MDB_stat db_stats; + if ((result = mdb_stat(txn, m_blocks, &db_stats))) + throw0(DB_ERROR(lmdb_error("Failed to query m_blocks: ", result).c_str())); + const uint64_t blockchain_height = db_stats.ms_entries; + + boost::circular_buffer long_term_block_weights(CRYPTONOTE_LONG_TERM_BLOCK_WEIGHT_WINDOW_SIZE); + + /* the block_info table name is the same but the old version and new version + * have incompatible data. Create a new table. We want the name to be similar + * to the old name so that it will occupy the same location in the DB. + */ + MDB_dbi o_block_info = m_block_info; + lmdb_db_open(txn, "block_infn", MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_block_info, "Failed to open db handle for block_infn"); + mdb_set_dupsort(txn, m_block_info, compare_uint64); + + + MDB_cursor *c_blocks; + result = mdb_cursor_open(txn, m_blocks, &c_blocks); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for blocks: ", result).c_str())); + + MDB_cursor *c_old, *c_cur; + i = 0; + while(1) { + if (!(i % 1000)) { + if (i) { + LOGIF(el::Level::Info) { + std::cout << i << " / " << blockchain_height << " \r" << std::flush; + } + txn.commit(); + result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + } + result = mdb_cursor_open(txn, m_block_info, &c_cur); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for block_infn: ", result).c_str())); + result = mdb_cursor_open(txn, o_block_info, &c_old); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to open a cursor for block_info: ", result).c_str())); + if (!i) { + MDB_stat db_stat; + result = mdb_stat(txn, m_block_info, &db_stats); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to query m_block_info: ", result).c_str())); + i = db_stats.ms_entries; + } + } + result = mdb_cursor_get(c_old, &k, &v, MDB_NEXT); + if (result == MDB_NOTFOUND) { + txn.commit(); + break; + } + else if (result) + throw0(DB_ERROR(lmdb_error("Failed to get a record from block_info: ", result).c_str())); + const mdb_block_info_2 *bi_old = (const mdb_block_info_2*)v.mv_data; + mdb_block_info_3 bi; + bi.bi_height = bi_old->bi_height; + bi.bi_timestamp = bi_old->bi_timestamp; + bi.bi_coins = bi_old->bi_coins; + bi.bi_weight = bi_old->bi_weight; + bi.bi_diff = bi_old->bi_diff; + bi.bi_hash = bi_old->bi_hash; + bi.bi_cum_rct = bi_old->bi_cum_rct; + + // get block major version to determine which rule is in place + if (!past_long_term_weight) + { + MDB_val kb, vb; + result = mdb_cursor_get(c_blocks, &k, &v, MDB_NEXT); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to query m_blocks: ", result).c_str())); + if (vb.mv_size == 0) + throw0(DB_ERROR("Invalid data from m_blocks")); + const uint8_t block_major_version = *((const uint8_t*)vb.mv_data); + if (block_major_version >= HF_VERSION_LONG_TERM_BLOCK_WEIGHT) + past_long_term_weight = true; + } + + uint64_t long_term_block_weight; + if (past_long_term_weight) + { + std::vector weights(long_term_block_weights.begin(), long_term_block_weights.end()); + uint64_t long_term_effective_block_median_weight = std::max(CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5, epee::misc_utils::median(weights)); + long_term_block_weight = std::min(bi.bi_weight, long_term_effective_block_median_weight + long_term_effective_block_median_weight * 2 / 5); + } + else + { + long_term_block_weight = bi.bi_weight; + } + long_term_block_weights.push_back(long_term_block_weight); + bi.bi_long_term_block_weight = long_term_block_weight; + + MDB_val_set(nv, bi); + result = mdb_cursor_put(c_cur, (MDB_val *)&zerokval, &nv, MDB_APPENDDUP); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to put a record into block_infn: ", result).c_str())); + /* we delete the old records immediately, so the overall DB and mapsize should not grow. + * This is a little slower than just letting mdb_drop() delete it all at the end, but + * it saves a significant amount of disk space. + */ + result = mdb_cursor_del(c_old, 0); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to delete a record from block_info: ", result).c_str())); + i++; + } + + result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + /* Delete the old table */ + result = mdb_drop(txn, o_block_info, 1); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to delete old block_info table: ", result).c_str())); + + RENAME_DB("block_infn"); + + lmdb_db_open(txn, "block_info", MDB_INTEGERKEY | MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, m_block_info, "Failed to open db handle for block_infn"); + mdb_set_dupsort(txn, m_block_info, compare_uint64); + + txn.commit(); + } while(0); + + uint32_t version = 4; + v.mv_data = (void *)&version; + v.mv_size = sizeof(version); + MDB_val_copy vk("version"); + result = mdb_txn_begin(m_env, NULL, 0, txn); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to create a transaction for the db: ", result).c_str())); + result = mdb_put(txn, m_properties, &vk, &v, 0); + if (result) + throw0(DB_ERROR(lmdb_error("Failed to update version for the db: ", result).c_str())); + txn.commit(); +} + void BlockchainLMDB::migrate(const uint32_t oldversion) { switch(oldversion) { @@ -4345,6 +4539,8 @@ void BlockchainLMDB::migrate(const uint32_t oldversion) migrate_1_2(); /* FALLTHRU */ case 2: migrate_2_3(); /* FALLTHRU */ + case 3: + migrate_3_4(); /* FALLTHRU */ default: ; } diff --git a/src/blockchain_db/lmdb/db_lmdb.h b/src/blockchain_db/lmdb/db_lmdb.h index e1f748ed8..923507d06 100644 --- a/src/blockchain_db/lmdb/db_lmdb.h +++ b/src/blockchain_db/lmdb/db_lmdb.h @@ -213,6 +213,8 @@ public: virtual uint64_t get_block_already_generated_coins(const uint64_t& height) const; + virtual uint64_t get_block_long_term_weight(const uint64_t& height) const; + virtual crypto::hash get_block_hash_from_height(const uint64_t& height) const; virtual std::vector get_blocks_range(const uint64_t& h1, const uint64_t& h2) const; @@ -274,6 +276,7 @@ public: virtual uint64_t add_block( const block& blk , size_t block_weight + , uint64_t long_term_block_weight , const difficulty_type& cumulative_difficulty , const uint64_t& coins_generated , const std::vector& txs @@ -318,6 +321,7 @@ private: virtual void add_block( const block& blk , size_t block_weight + , uint64_t long_term_block_weight , const difficulty_type& cumulative_difficulty , const uint64_t& coins_generated , uint64_t num_rct_outs @@ -396,6 +400,9 @@ private: // migrate from DB version 2 to 3 void migrate_2_3(); + // migrate from DB version 3 to 4 + void migrate_3_4(); + void cleanup_batch(); private: diff --git a/src/blockchain_db/testdb.h b/src/blockchain_db/testdb.h new file mode 100644 index 000000000..0476e5e1b --- /dev/null +++ b/src/blockchain_db/testdb.h @@ -0,0 +1,152 @@ +// Copyright (c) 2014-2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers + +#pragma once + +#include +#include +#include + +#include "blockchain_db.h" + +namespace cryptonote +{ + +class BaseTestDB: public cryptonote::BlockchainDB { +public: + BaseTestDB() {} + virtual void open(const std::string& filename, const int db_flags = 0) { } + virtual void close() {} + virtual void sync() {} + virtual void safesyncmode(const bool onoff) {} + virtual void reset() {} + virtual std::vector get_filenames() const { return std::vector(); } + virtual bool remove_data_file(const std::string& folder) const { return true; } + virtual std::string get_db_name() const { return std::string(); } + virtual bool lock() { return true; } + virtual void unlock() { } + virtual bool batch_start(uint64_t batch_num_blocks=0, uint64_t batch_bytes=0) { return true; } + virtual void batch_stop() {} + virtual void set_batch_transactions(bool) {} + virtual void block_txn_start(bool readonly=false) {} + virtual void block_txn_stop() {} + virtual void block_txn_abort() {} + virtual void drop_hard_fork_info() {} + virtual bool block_exists(const crypto::hash& h, uint64_t *height) const { return false; } + virtual cryptonote::blobdata get_block_blob_from_height(const uint64_t& height) const { return cryptonote::t_serializable_object_to_blob(get_block_from_height(height)); } + virtual cryptonote::blobdata get_block_blob(const crypto::hash& h) const { return cryptonote::blobdata(); } + virtual bool get_tx_blob(const crypto::hash& h, cryptonote::blobdata &tx) const { return false; } + virtual bool get_pruned_tx_blob(const crypto::hash& h, cryptonote::blobdata &tx) const { return false; } + virtual bool get_prunable_tx_blob(const crypto::hash& h, cryptonote::blobdata &tx) const { return false; } + virtual bool get_prunable_tx_hash(const crypto::hash& tx_hash, crypto::hash &prunable_hash) const { return false; } + virtual uint64_t get_block_height(const crypto::hash& h) const { return 0; } + virtual cryptonote::block_header get_block_header(const crypto::hash& h) const { return cryptonote::block_header(); } + virtual uint64_t get_block_timestamp(const uint64_t& height) const { return 0; } + virtual std::vector get_block_cumulative_rct_outputs(const std::vector &heights) const { return {}; } + virtual uint64_t get_top_block_timestamp() const { return 0; } + virtual size_t get_block_weight(const uint64_t& height) const { return 128; } + virtual cryptonote::difficulty_type get_block_cumulative_difficulty(const uint64_t& height) const { return 10; } + virtual cryptonote::difficulty_type get_block_difficulty(const uint64_t& height) const { return 0; } + virtual uint64_t get_block_already_generated_coins(const uint64_t& height) const { return 10000000000; } + virtual uint64_t get_block_long_term_weight(const uint64_t& height) const { return 128; } + virtual crypto::hash get_block_hash_from_height(const uint64_t& height) const { return crypto::hash(); } + virtual std::vector get_blocks_range(const uint64_t& h1, const uint64_t& h2) const { return std::vector(); } + virtual std::vector get_hashes_range(const uint64_t& h1, const uint64_t& h2) const { return std::vector(); } + virtual crypto::hash top_block_hash() const { return crypto::hash(); } + virtual cryptonote::block get_top_block() const { return cryptonote::block(); } + virtual uint64_t height() const { return 1; } + virtual bool tx_exists(const crypto::hash& h) const { return false; } + virtual bool tx_exists(const crypto::hash& h, uint64_t& tx_index) const { return false; } + virtual uint64_t get_tx_unlock_time(const crypto::hash& h) const { return 0; } + virtual cryptonote::transaction get_tx(const crypto::hash& h) const { return cryptonote::transaction(); } + virtual bool get_tx(const crypto::hash& h, cryptonote::transaction &tx) const { return false; } + virtual uint64_t get_tx_count() const { return 0; } + virtual std::vector get_tx_list(const std::vector& hlist) const { return std::vector(); } + virtual uint64_t get_tx_block_height(const crypto::hash& h) const { return 0; } + virtual uint64_t get_num_outputs(const uint64_t& amount) const { return 1; } + virtual uint64_t get_indexing_base() const { return 0; } + virtual cryptonote::output_data_t get_output_key(const uint64_t& amount, const uint64_t& index) { return cryptonote::output_data_t(); } + virtual cryptonote::tx_out_index get_output_tx_and_index_from_global(const uint64_t& index) const { return cryptonote::tx_out_index(); } + virtual cryptonote::tx_out_index get_output_tx_and_index(const uint64_t& amount, const uint64_t& index) const { return cryptonote::tx_out_index(); } + virtual void get_output_tx_and_index(const uint64_t& amount, const std::vector &offsets, std::vector &indices) const {} + virtual void get_output_key(const uint64_t &amount, const std::vector &offsets, std::vector &outputs, bool allow_partial = false) {} + virtual bool can_thread_bulk_indices() const { return false; } + virtual std::vector get_tx_output_indices(const crypto::hash& h) const { return std::vector(); } + virtual std::vector get_tx_amount_output_indices(const uint64_t tx_index) const { return std::vector(); } + virtual bool has_key_image(const crypto::key_image& img) const { return false; } + virtual void remove_block() { } + virtual uint64_t add_transaction_data(const crypto::hash& blk_hash, const cryptonote::transaction& tx, const crypto::hash& tx_hash, const crypto::hash& tx_prunable_hash) {return 0;} + virtual void remove_transaction_data(const crypto::hash& tx_hash, const cryptonote::transaction& tx) {} + virtual uint64_t add_output(const crypto::hash& tx_hash, const cryptonote::tx_out& tx_output, const uint64_t& local_index, const uint64_t unlock_time, const rct::key *commitment) {return 0;} + virtual void add_tx_amount_output_indices(const uint64_t tx_index, const std::vector& amount_output_indices) {} + virtual void add_spent_key(const crypto::key_image& k_image) {} + virtual void remove_spent_key(const crypto::key_image& k_image) {} + + virtual bool for_all_key_images(std::function) const { return true; } + virtual bool for_blocks_range(const uint64_t&, const uint64_t&, std::function) const { return true; } + virtual bool for_all_transactions(std::function, bool pruned) const { return true; } + virtual bool for_all_outputs(std::function f) const { return true; } + virtual bool for_all_outputs(uint64_t amount, const std::function &f) const { return true; } + virtual bool is_read_only() const { return false; } + virtual std::map> get_output_histogram(const std::vector &amounts, bool unlocked, uint64_t recent_cutoff, uint64_t min_count) const { return std::map>(); } + virtual bool get_output_distribution(uint64_t amount, uint64_t from_height, uint64_t to_height, std::vector &distribution, uint64_t &base) const { return false; } + + virtual void add_txpool_tx(const cryptonote::transaction &tx, const cryptonote::txpool_tx_meta_t& details) {} + virtual void update_txpool_tx(const crypto::hash &txid, const cryptonote::txpool_tx_meta_t& details) {} + virtual uint64_t get_txpool_tx_count(bool include_unrelayed_txes = true) const { return 0; } + virtual bool txpool_has_tx(const crypto::hash &txid) const { return false; } + virtual void remove_txpool_tx(const crypto::hash& txid) {} + virtual bool get_txpool_tx_meta(const crypto::hash& txid, cryptonote::txpool_tx_meta_t &meta) const { return false; } + virtual bool get_txpool_tx_blob(const crypto::hash& txid, cryptonote::blobdata &bd) const { return false; } + virtual uint64_t get_database_size() const { return 0; } + virtual cryptonote::blobdata get_txpool_tx_blob(const crypto::hash& txid) const { return ""; } + virtual bool for_all_txpool_txes(std::function, bool include_blob = false, bool include_unrelayed_txes = false) const { return false; } + + virtual void add_block( const cryptonote::block& blk + , size_t block_weight + , uint64_t long_term_block_weight + , const cryptonote::difficulty_type& cumulative_difficulty + , const uint64_t& coins_generated + , uint64_t num_rct_outs + , const crypto::hash& blk_hash + ) { } + virtual cryptonote::block get_block_from_height(const uint64_t& height) const { return cryptonote::block(); } + virtual void set_hard_fork_version(uint64_t height, uint8_t version) {} + virtual uint8_t get_hard_fork_version(uint64_t height) const { return 0; } + virtual void check_hard_fork_info() {} + + virtual uint32_t get_blockchain_pruning_seed() const { return 0; } + virtual bool prune_blockchain(uint32_t pruning_seed = 0) { return true; } + virtual bool update_pruning() { return true; } + virtual bool check_pruning() { return true; } + virtual void prune_outputs(uint64_t amount) {} +}; + +} diff --git a/src/blockchain_utilities/blockchain_import.cpp b/src/blockchain_utilities/blockchain_import.cpp index 9ec768d26..b78b8b5c9 100644 --- a/src/blockchain_utilities/blockchain_import.cpp +++ b/src/blockchain_utilities/blockchain_import.cpp @@ -484,7 +484,8 @@ int import_from_file(cryptonote::core& core, const std::string& import_file_path try { - core.get_blockchain_storage().get_db().add_block(b, block_weight, cumulative_difficulty, coins_generated, txs); + uint64_t long_term_block_weight = core.get_blockchain_storage().get_next_long_term_block_weight(block_weight); + core.get_blockchain_storage().get_db().add_block(b, block_weight, long_term_block_weight, cumulative_difficulty, coins_generated, txs); } catch (const std::exception& e) { diff --git a/src/cryptonote_config.h b/src/cryptonote_config.h index a6858ce7c..021c8562c 100644 --- a/src/cryptonote_config.h +++ b/src/cryptonote_config.h @@ -58,6 +58,8 @@ #define CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V2 60000 //size of block (bytes) after which reward for block calculated using block size #define CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V1 20000 //size of block (bytes) after which reward for block calculated using block size - before first fork #define CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5 300000 //size of block (bytes) after which reward for block calculated using block size - second change, from v5 +#define CRYPTONOTE_LONG_TERM_BLOCK_WEIGHT_WINDOW_SIZE 100000 // size in blocks of the long term block weight median window +#define CRYPTONOTE_SHORT_TERM_BLOCK_WEIGHT_SURGE_FACTOR 50 #define CRYPTONOTE_COINBASE_BLOB_RESERVED_SIZE 600 #define CRYPTONOTE_DISPLAY_DECIMAL_POINT 12 // COIN - number of smallest units in one coin @@ -138,6 +140,7 @@ #define HF_VERSION_MIN_MIXIN_10 8 #define HF_VERSION_ENFORCE_RCT 6 #define HF_VERSION_PER_BYTE_FEE 8 +#define HF_VERSION_LONG_TERM_BLOCK_WEIGHT 10 #define PER_KB_FEE_QUANTIZATION_DECIMALS 8 diff --git a/src/cryptonote_core/blockchain.cpp b/src/cryptonote_core/blockchain.cpp index fa23b6bd2..9c6153212 100644 --- a/src/cryptonote_core/blockchain.cpp +++ b/src/cryptonote_core/blockchain.cpp @@ -167,6 +167,8 @@ static const struct { Blockchain::Blockchain(tx_memory_pool& tx_pool) : m_db(), m_tx_pool(tx_pool), m_hardfork(NULL), m_timestamps_and_difficulties_height(0), m_current_block_cumul_weight_limit(0), m_current_block_cumul_weight_median(0), m_enforce_dns_checkpoints(false), m_max_prepare_blocks_threads(4), m_db_sync_on_blocks(true), m_db_sync_threshold(1), m_db_sync_mode(db_async), m_db_default_sync(false), m_fast_sync(true), m_show_time_stats(false), m_sync_counter(0), m_bytes_to_sync(0), m_cancel(false), + m_long_term_block_weights(CRYPTONOTE_LONG_TERM_BLOCK_WEIGHT_WINDOW_SIZE), + m_long_term_block_weights_height(0), m_difficulty_for_next_block_top_hash(crypto::null_hash), m_difficulty_for_next_block(1), m_btc_valid(false) @@ -492,6 +494,17 @@ bool Blockchain::init(BlockchainDB* db, const network_type nettype, bool offline m_tx_pool.on_blockchain_dec(m_db->height()-1, get_tail_id()); } + if (test_options && test_options->long_term_block_weight_window) + m_long_term_block_weights.set_capacity(test_options->long_term_block_weight_window); + + // fill up the long term block weights to reach the required amount + const uint64_t db_height = m_db->height(); + const uint64_t nblocks = std::min(m_long_term_block_weights.capacity(), db_height); + while (m_long_term_block_weights.size() < nblocks) + { + m_long_term_block_weights.push_front(db_height - 1 - m_long_term_block_weights.size()); + } + update_next_cumulative_weight_limit(); return true; } @@ -606,6 +619,8 @@ block Blockchain::pop_block_from_blockchain() throw; } + pop_from_long_term_block_weights(); + // return transactions from popped block to the tx_pool for (transaction& tx : popped_txs) { @@ -655,6 +670,8 @@ bool Blockchain::reset_and_set_genesis_block(const block& b) m_timestamps_and_difficulties_height = 0; m_alternative_chains.clear(); invalidate_block_template_cache(); + m_long_term_block_weights.clear(); + m_long_term_block_weights_height = 0; m_db->reset(); m_hardfork->init(); @@ -2947,7 +2964,8 @@ bool Blockchain::check_fee(size_t tx_weight, uint64_t fee) const uint64_t needed_fee; if (version >= HF_VERSION_PER_BYTE_FEE) { - uint64_t fee_per_byte = get_dynamic_base_fee(base_reward, median, version); + const bool use_long_term_median_in_fee = version >= HF_VERSION_LONG_TERM_BLOCK_WEIGHT; + uint64_t fee_per_byte = get_dynamic_base_fee(base_reward, use_long_term_median_in_fee ? m_long_term_block_weights.back() : median, version); MDEBUG("Using " << print_money(fee_per_byte) << "/byte fee"); needed_fee = tx_weight * fee_per_byte; // quantize fee up to 8 decimals @@ -3010,7 +3028,8 @@ uint64_t Blockchain::get_dynamic_base_fee_estimate(uint64_t grace_blocks) const base_reward = BLOCK_REWARD_OVERESTIMATE; } - uint64_t fee = get_dynamic_base_fee(base_reward, median, version); + const bool use_long_term_median_in_fee = version >= HF_VERSION_LONG_TERM_BLOCK_WEIGHT; + uint64_t fee = get_dynamic_base_fee(base_reward, use_long_term_median_in_fee ? m_long_term_block_weights.back() : median, version); const bool per_byte = version < HF_VERSION_PER_BYTE_FEE; MDEBUG("Estimating " << grace_blocks << "-block fee at " << print_money(fee) << "/" << (per_byte ? "byte" : "kB")); return fee; @@ -3507,7 +3526,8 @@ leave: { try { - new_height = m_db->add_block(bl, block_weight, cumulative_difficulty, already_generated_coins, txs); + uint64_t long_term_block_weight = get_next_long_term_block_weight(block_weight); + new_height = m_db->add_block(bl, block_weight, long_term_block_weight, cumulative_difficulty, already_generated_coins, txs); } catch (const KEY_IMAGE_EXISTS& e) { @@ -3560,20 +3580,109 @@ leave: return true; } //------------------------------------------------------------------ -bool Blockchain::update_next_cumulative_weight_limit() +void Blockchain::pop_from_long_term_block_weights() { - uint64_t full_reward_zone = get_min_block_weight(get_current_hard_fork_version()); + m_long_term_block_weights.pop_back(); + --m_long_term_block_weights_height; + if (m_long_term_block_weights_height + 1 > m_long_term_block_weights.capacity()) + { + uint64_t block_height = m_long_term_block_weights_height - m_long_term_block_weights.capacity() + 1; + m_long_term_block_weights.push_front(block_height); + } +} +//------------------------------------------------------------------ +uint64_t Blockchain::get_next_long_term_block_weight(uint64_t block_weight) const +{ + PERF_TIMER(get_next_long_term_block_weight); + + const uint64_t nblocks = std::min(m_long_term_block_weights.capacity(), m_db->height()); + CHECK_AND_ASSERT_MES(m_long_term_block_weights.size() == nblocks, 0, + "m_long_term_block_weights is not the expected size"); + + const uint8_t hf_version = get_current_hard_fork_version(); + if (hf_version < HF_VERSION_LONG_TERM_BLOCK_WEIGHT) + return block_weight; + + std::vector weights(m_long_term_block_weights.begin(), m_long_term_block_weights.end()); + uint64_t long_term_median = epee::misc_utils::median(weights); + uint64_t long_term_effective_median_block_weight = std::max(CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5, long_term_median); + + uint64_t short_term_constraint = long_term_effective_median_block_weight + long_term_effective_median_block_weight * 2 / 5; + uint64_t long_term_block_weight = std::min(block_weight, short_term_constraint); + + return long_term_block_weight; +} +//------------------------------------------------------------------ +bool Blockchain::update_next_cumulative_weight_limit(uint64_t *long_term_effective_median_block_weight) +{ + PERF_TIMER(update_next_cumulative_weight_limit); LOG_PRINT_L3("Blockchain::" << __func__); - std::vector weights; - get_last_n_blocks_weights(weights, CRYPTONOTE_REWARD_BLOCKS_WINDOW); - uint64_t median = epee::misc_utils::median(weights); - m_current_block_cumul_weight_median = median; - if(median <= full_reward_zone) - median = full_reward_zone; + uint64_t local_long_term_effective_median_block_weight; + if (!long_term_effective_median_block_weight) + long_term_effective_median_block_weight = &local_long_term_effective_median_block_weight; + + // when we reach this, the last hf version is not yet written to the db + const uint64_t db_height = m_db->height(); + const uint8_t hf_version = (m_long_term_block_weights_height + 1 < m_db->height()) ? m_db->get_hard_fork_version(db_height - 1) : get_current_hard_fork_version(); + uint64_t full_reward_zone = get_min_block_weight(hf_version); + uint64_t long_term_block_weight; + + // recalc, or reorg ? pop the last one(s) and recompute + uint64_t pop = 0; + if (db_height && db_height <= m_long_term_block_weights_height) + { + pop = m_long_term_block_weights_height - db_height + 1; + while (pop--) + pop_from_long_term_block_weights(); + } + + if (hf_version < HF_VERSION_LONG_TERM_BLOCK_WEIGHT) + { + std::vector weights; + get_last_n_blocks_weights(weights, CRYPTONOTE_REWARD_BLOCKS_WINDOW); + m_current_block_cumul_weight_median = epee::misc_utils::median(weights); + long_term_block_weight = weights.back(); + } + else + { + const uint64_t block_weight = m_db->get_block_weight(db_height - 1); + + std::vector weights(m_long_term_block_weights.begin(), m_long_term_block_weights.end()); + uint64_t long_term_median = epee::misc_utils::median(weights); + *long_term_effective_median_block_weight = std::max(CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5, long_term_median); + + uint64_t short_term_constraint = *long_term_effective_median_block_weight + *long_term_effective_median_block_weight * 2 / 5; + long_term_block_weight = std::min(block_weight, short_term_constraint); + + weights = std::vector(m_long_term_block_weights.begin(), m_long_term_block_weights.end()); + if (weights.empty()) + weights.resize(1); + weights[0] = long_term_block_weight; + long_term_median = epee::misc_utils::median(weights); + *long_term_effective_median_block_weight = std::max(CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5, long_term_median); + short_term_constraint = *long_term_effective_median_block_weight + *long_term_effective_median_block_weight * 2 / 5; + + weights.clear(); + get_last_n_blocks_weights(weights, CRYPTONOTE_REWARD_BLOCKS_WINDOW); + + uint64_t short_term_median = epee::misc_utils::median(weights); + uint64_t effective_median_block_weight = std::min(std::max(CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5, short_term_median), CRYPTONOTE_SHORT_TERM_BLOCK_WEIGHT_SURGE_FACTOR * *long_term_effective_median_block_weight); + + m_current_block_cumul_weight_median = effective_median_block_weight; + } + + m_long_term_block_weights.push_back(long_term_block_weight); + m_long_term_block_weights_height = db_height; + const uint64_t nblocks = std::min(m_long_term_block_weights.capacity(), db_height); + CHECK_AND_ASSERT_MES(m_long_term_block_weights.size() == nblocks, false, "Bad m_long_term_block_weights size"); + + if (m_current_block_cumul_weight_median <= full_reward_zone) + m_current_block_cumul_weight_median = full_reward_zone; + + m_current_block_cumul_weight_limit = m_current_block_cumul_weight_median * 2; - m_current_block_cumul_weight_limit = median*2; return true; } //------------------------------------------------------------------ diff --git a/src/cryptonote_core/blockchain.h b/src/cryptonote_core/blockchain.h index ab66fac8b..463a37b64 100644 --- a/src/cryptonote_core/blockchain.h +++ b/src/cryptonote_core/blockchain.h @@ -37,6 +37,7 @@ #include #include #include +#include #include #include #include @@ -611,6 +612,13 @@ namespace cryptonote */ uint64_t get_current_cumulative_block_weight_limit() const; + /** + * @brief gets the long term block weight for a new block + * + * @return the long term block weight + */ + uint64_t get_next_long_term_block_weight(uint64_t block_weight) const; + /** * @brief gets the block weight median based on recent blocks (same window as for the limit) * @@ -952,7 +960,21 @@ namespace cryptonote */ void on_new_tx_from_block(const cryptonote::transaction &tx); + /** + * @brief returns the timestamps of the last N blocks + */ + std::vector get_last_block_timestamps(unsigned int blocks) const; + + /** + * @brief removes blocks from the top of the blockchain + * + * @param nblocks number of blocks to be removed + */ + void pop_blocks(uint64_t nblocks); + +#ifndef IN_UNIT_TESTS private: +#endif // TODO: evaluate whether or not each of these typedefs are left over from blockchain_storage typedef std::unordered_map blocks_by_id_index; @@ -1005,6 +1027,8 @@ namespace cryptonote std::vector m_timestamps; std::vector m_difficulties; uint64_t m_timestamps_and_difficulties_height; + boost::circular_buffer m_long_term_block_weights; + uint64_t m_long_term_block_weights_height; epee::critical_section m_difficulty_lock; crypto::hash m_difficulty_for_next_block_top_hash; @@ -1336,9 +1360,11 @@ namespace cryptonote /** * @brief calculate the block weight limit for the next block to be added * + * @param long_term_effective_median_block_weight optionally return that value + * * @return true */ - bool update_next_cumulative_weight_limit(); + bool update_next_cumulative_weight_limit(uint64_t *long_term_effective_median_block_weight = NULL); void return_tx_to_pool(std::vector &txs); /** @@ -1392,5 +1418,12 @@ namespace cryptonote * At some point, may be used to push an update to miners */ void cache_block_template(const block &b, const cryptonote::account_public_address &address, const blobdata &nonce, const difficulty_type &diff, uint64_t expected_reward, uint64_t pool_cookie); + + /** + * @brief pops an entry from long term block weights + * + * another is added at the other end if necessary + */ + void pop_from_long_term_block_weights(); }; } // namespace cryptonote diff --git a/src/cryptonote_core/cryptonote_core.cpp b/src/cryptonote_core/cryptonote_core.cpp index 2fec6b613..40541ba8c 100644 --- a/src/cryptonote_core/cryptonote_core.cpp +++ b/src/cryptonote_core/cryptonote_core.cpp @@ -564,7 +564,8 @@ namespace cryptonote const std::pair regtest_hard_forks[3] = {std::make_pair(1, 0), std::make_pair(Blockchain::get_hard_fork_heights(MAINNET).back().version, 1), std::make_pair(0, 0)}; const cryptonote::test_options regtest_test_options = { - regtest_hard_forks + regtest_hard_forks, + 0 }; const difficulty_type fixed_difficulty = command_line::get_arg(vm, arg_fixed_difficulty); r = m_blockchain_storage.init(db.release(), m_nettype, m_offline, regtest ? ®test_test_options : test_options, fixed_difficulty); diff --git a/src/cryptonote_core/cryptonote_core.h b/src/cryptonote_core/cryptonote_core.h index 58fe5b7b5..c74c244b0 100644 --- a/src/cryptonote_core/cryptonote_core.h +++ b/src/cryptonote_core/cryptonote_core.h @@ -55,6 +55,7 @@ namespace cryptonote { struct test_options { const std::pair *hard_forks; + const size_t long_term_block_weight_window; }; extern const command_line::arg_descriptor arg_data_dir; diff --git a/src/daemon/rpc_command_executor.cpp b/src/daemon/rpc_command_executor.cpp index 6464d372f..3348989f2 100644 --- a/src/daemon/rpc_command_executor.cpp +++ b/src/daemon/rpc_command_executor.cpp @@ -77,6 +77,7 @@ namespace { << "POW hash: " << header.pow_hash << std::endl << "block size: " << header.block_size << std::endl << "block weight: " << header.block_weight << std::endl + << "long term weight: " << header.long_term_weight << std::endl << "num txes: " << header.num_txes << std::endl << "reward: " << cryptonote::print_money(header.reward); } diff --git a/src/rpc/core_rpc_server.cpp b/src/rpc/core_rpc_server.cpp index bc33dc77f..8098b62ad 100644 --- a/src/rpc/core_rpc_server.cpp +++ b/src/rpc/core_rpc_server.cpp @@ -1264,6 +1264,7 @@ namespace cryptonote response.block_size = response.block_weight = m_core.get_blockchain_storage().get_db().get_block_weight(height); response.num_txes = blk.tx_hashes.size(); response.pow_hash = fill_pow_hash ? string_tools::pod_to_hex(get_block_longhash(blk, height)) : ""; + response.long_term_weight = m_core.get_blockchain_storage().get_db().get_block_long_term_weight(height); return true; } //------------------------------------------------------------------------------------------------------------------------------ diff --git a/src/rpc/core_rpc_server_commands_defs.h b/src/rpc/core_rpc_server_commands_defs.h index 3b654d4cb..8ac1279a2 100644 --- a/src/rpc/core_rpc_server_commands_defs.h +++ b/src/rpc/core_rpc_server_commands_defs.h @@ -1128,6 +1128,7 @@ namespace cryptonote uint64_t block_weight; uint64_t num_txes; std::string pow_hash; + uint64_t long_term_weight; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(major_version) @@ -1146,6 +1147,7 @@ namespace cryptonote KV_SERIALIZE_OPT(block_weight, (uint64_t)0) KV_SERIALIZE(num_txes) KV_SERIALIZE(pow_hash) + KV_SERIALIZE_OPT(long_term_weight, (uint64_t)0) END_KV_SERIALIZE_MAP() }; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 20afd4203..89d77cf32 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -85,6 +85,7 @@ add_subdirectory(performance_tests) add_subdirectory(core_proxy) add_subdirectory(unit_tests) add_subdirectory(difficulty) +add_subdirectory(block_weight) add_subdirectory(hash) add_subdirectory(net_load_tests) if (BUILD_GUI_DEPS) @@ -115,6 +116,7 @@ add_test( set(enabled_tests core_tests difficulty + block_weight hash performance_tests core_proxy diff --git a/tests/block_weight/CMakeLists.txt b/tests/block_weight/CMakeLists.txt new file mode 100644 index 000000000..b0d716ea0 --- /dev/null +++ b/tests/block_weight/CMakeLists.txt @@ -0,0 +1,45 @@ +# Copyright (c) 2014-2018, The Monero Project +# +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without modification, are +# permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this list of +# conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, this list +# of conditions and the following disclaimer in the documentation and/or other +# materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors may be +# used to endorse or promote products derived from this software without specific +# prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +# THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +set(block_weight_sources + block_weight.cpp) + +set(block_weight_headers) + +add_executable(block_weight + ${block_weight_sources} + ${block_weight_headers}) +target_link_libraries(block_weight + PRIVATE + cryptonote_core + blockchain_db + ${EXTRA_LIBRARIES}) + +add_test( + NAME block_weight + COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/compare.py ${CMAKE_CURRENT_SOURCE_DIR}/block_weight.py ${CMAKE_CURRENT_BINARY_DIR}/block_weight) diff --git a/tests/block_weight/block_weight.cpp b/tests/block_weight/block_weight.cpp new file mode 100644 index 000000000..9b7343ca3 --- /dev/null +++ b/tests/block_weight/block_weight.cpp @@ -0,0 +1,192 @@ +// Copyright (c) 2019, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#define IN_UNIT_TESTS + +#include +#include +#include "cryptonote_core/blockchain.h" +#include "cryptonote_core/tx_pool.h" +#include "cryptonote_core/cryptonote_core.h" +#include "blockchain_db/testdb.h" + +#define LONG_TERM_BLOCK_WEIGHT_WINDOW 5000 + +enum test_t +{ + test_max = 0, + test_lcg = 1, + test_min = 2, +}; + +namespace +{ + +class TestDB: public cryptonote::BaseTestDB +{ +private: + struct block_t + { + size_t weight; + uint64_t long_term_weight; + }; + +public: + TestDB() { m_open = true; } + + virtual void add_block( const cryptonote::block& blk + , size_t block_weight + , uint64_t long_term_block_weight + , const cryptonote::difficulty_type& cumulative_difficulty + , const uint64_t& coins_generated + , uint64_t num_rct_outs + , const crypto::hash& blk_hash + ) override { + blocks.push_back({block_weight, long_term_block_weight}); + } + virtual uint64_t height() const override { return blocks.size(); } + virtual size_t get_block_weight(const uint64_t &h) const override { return blocks[h].weight; } + virtual uint64_t get_block_long_term_weight(const uint64_t &h) const override { return blocks[h].long_term_weight; } + virtual crypto::hash top_block_hash() const override { + uint64_t h = height(); + crypto::hash top = crypto::null_hash; + if (h) + *(uint64_t*)&top = h - 1; + return top; + } + virtual void pop_block(cryptonote::block &blk, std::vector &txs) override { blocks.pop_back(); } + virtual void set_hard_fork_version(uint64_t height, uint8_t version) override { if (height >= hf.size()) hf.resize(height + 1); hf[height] = version; } + virtual uint8_t get_hard_fork_version(uint64_t height) const override { if (height >= hf.size()) return 255; return hf[height]; } + +private: + std::vector blocks; + std::vector hf; +}; + +} + +#define PREFIX_WINDOW(hf_version,window) \ + std::unique_ptr bc; \ + cryptonote::tx_memory_pool txpool(*bc); \ + bc.reset(new cryptonote::Blockchain(txpool)); \ + struct get_test_options { \ + const std::pair hard_forks[3]; \ + const cryptonote::test_options test_options = { \ + hard_forks, \ + window, \ + }; \ + get_test_options(): hard_forks{std::make_pair(1, (uint64_t)0), std::make_pair((uint8_t)hf_version, (uint64_t)LONG_TERM_BLOCK_WEIGHT_WINDOW), std::make_pair((uint8_t)0, (uint64_t)0)} {} \ + } opts; \ + cryptonote::Blockchain *blockchain = bc.get(); \ + bool r = blockchain->init(new TestDB(), cryptonote::FAKECHAIN, true, &opts.test_options, 0); \ + if (!r) \ + { \ + fprintf(stderr, "Failed to init blockchain\n"); \ + exit(1); \ + } + +#define PREFIX(hf_version) PREFIX_WINDOW(hf_version, LONG_TERM_BLOCK_WEIGHT_WINDOW) + +static uint32_t lcg_seed = 0; + +static uint32_t lcg() +{ + lcg_seed = (lcg_seed * 0x100000001b3 + 0xcbf29ce484222325) & 0xffffffff; + return lcg_seed; +} + +static void test(test_t t, uint64_t blocks) +{ + PREFIX(10); + + for (uint64_t h = 0; h < LONG_TERM_BLOCK_WEIGHT_WINDOW; ++h) + { + cryptonote::block b; + b.major_version = 1; + b.minor_version = 1; + bc->get_db().add_block(std::move(b), 300000, 300000, bc->get_db().height(), bc->get_db().height(), {}); + if (!bc->update_next_cumulative_weight_limit()) + { + fprintf(stderr, "Failed to update cumulative weight limit 1\n"); + exit(1); + } + } + + for (uint64_t h = 0; h < blocks; ++h) + { + uint64_t w; + uint64_t effective_block_weight_median = bc->get_current_cumulative_block_weight_median(); + switch (t) + { + case test_lcg: + { + uint32_t r = lcg(); + int64_t wi = 90 + r % 500000 + 250000 + sin(h / 200.) * 350000; + w = wi < 90 ? 90 : wi; + break; + } + case test_max: + w = bc->get_current_cumulative_block_weight_limit(); + break; + case test_min: + w = 90; + break; + default: + exit(1); + } + uint64_t ltw = bc->get_next_long_term_block_weight(w); + cryptonote::block b; + b.major_version = 10; + b.minor_version = 10; + bc->get_db().add_block(std::move(b), w, ltw, bc->get_db().height(), bc->get_db().height(), {}); + + if (!bc->update_next_cumulative_weight_limit()) + { + fprintf(stderr, "Failed to update cumulative weight limit 2\n"); + exit(1); + } + if(0)if (!bc->update_next_cumulative_weight_limit()) + { + fprintf(stderr, "Failed to update cumulative weight limit 2\n"); + exit(1); + } +uint64_t last = bc->get_db().get_block_long_term_weight(bc->get_db().height() - 1); +if (last != ltw) std::cout << "Inconsistency at " << h << std::endl; + std::cout << "H " << h << ", BW " << w << ", EMBW " << effective_block_weight_median << ", LTBW " << ltw << std::endl; + } +} + +int main() +{ + mlog_configure("", false); + mlog_set_categories(""); + test(test_max, 2 * LONG_TERM_BLOCK_WEIGHT_WINDOW); + test(test_lcg, 9 * LONG_TERM_BLOCK_WEIGHT_WINDOW); + test(test_min, 1 * LONG_TERM_BLOCK_WEIGHT_WINDOW); + return 0; +} diff --git a/tests/block_weight/block_weight.py b/tests/block_weight/block_weight.py new file mode 100755 index 000000000..06aaabb02 --- /dev/null +++ b/tests/block_weight/block_weight.py @@ -0,0 +1,74 @@ +#!/usr/bin/python +# Simulate a maximal block attack on the Monero network +# This uses the scheme proposed by ArticMine +# Written by Sarang Nother +# Copyright (c) 2019 The Monero Project +import sys +import math + +MEDIAN_WINDOW_SMALL = 100 # number of recent blocks for median computation +MEDIAN_WINDOW_BIG = 5000 +MULTIPLIER_SMALL = 1.4 # multipliers for determining weights +MULTIPLIER_BIG = 50.0 +MEDIAN_THRESHOLD = 300*1000 # initial value for median (scaled kB -> B) +lcg_seed = 0 +embw = MEDIAN_THRESHOLD +ltembw = MEDIAN_THRESHOLD + +weights = [MEDIAN_THRESHOLD]*MEDIAN_WINDOW_SMALL # weights of recent blocks (B), with index -1 most recent +lt_weights = [MEDIAN_THRESHOLD]*MEDIAN_WINDOW_BIG # long-term weights + +# Compute the median of a list +def get_median(vec): + #temp = vec + temp = sorted(vec) + if len(temp) % 2 == 1: + return temp[len(temp)/2] + else: + return int((temp[len(temp)/2]+temp[len(temp)/2-1])/2) + +def LCG(): + global lcg_seed + lcg_seed = (lcg_seed * 0x100000001b3 + 0xcbf29ce484222325) & 0xffffffff + return lcg_seed + +def run(t, blocks): + global embw + global ltembw + + weights = [MEDIAN_THRESHOLD]*MEDIAN_WINDOW_SMALL # weights of recent blocks (B), with index -1 most recent + lt_weights = [MEDIAN_THRESHOLD]*MEDIAN_WINDOW_BIG # long-term weights + + for block in range(blocks): + # determine the long-term effective weight + ltmedian = get_median(lt_weights[-MEDIAN_WINDOW_BIG:]) + ltembw = max(MEDIAN_THRESHOLD,ltmedian) + + # determine the effective weight + stmedian = get_median(weights[-MEDIAN_WINDOW_SMALL:]) + embw = min(max(MEDIAN_THRESHOLD,stmedian),int(MULTIPLIER_BIG*ltembw)) + + # drop the lowest values + weights = weights[1:] + lt_weights = lt_weights[1:] + + # add a block of max weight + if t == 0: + max_weight = 2 * embw + elif t == 1: + r = LCG() + max_weight = int(90 + r % 500000 + 250000 + math.sin(block / 200.) * 350000) + if max_weight < 90: max_weight = 90 + elif t == 2: + max_weight = 90 + else: + sys.exit(1) + weights.append(max_weight) + lt_weights.append(min(max_weight,int(ltembw + int(ltembw * 2 / 5)))) + + #print "H %u, r %u, BW %u, EMBW %u, LTBW %u, LTEMBW %u, ltmedian %u" % (block, r, max_weight, embw, lt_weights[-1], ltembw, ltmedian) + print "H %u, BW %u, EMBW %u, LTBW %u" % (block, max_weight, embw, lt_weights[-1]) + +run(0, 2 * MEDIAN_WINDOW_BIG) +run(1, 9 * MEDIAN_WINDOW_BIG) +run(2, 1 * MEDIAN_WINDOW_BIG) diff --git a/tests/block_weight/compare.py b/tests/block_weight/compare.py new file mode 100755 index 000000000..c6be05206 --- /dev/null +++ b/tests/block_weight/compare.py @@ -0,0 +1,13 @@ +#!/usr/bin/python + +import sys +import subprocess + +print 'running: ', sys.argv[1] +S0 = subprocess.check_output(sys.argv[1], stderr=subprocess.STDOUT) +print 'running: ', sys.argv[2] +S1 = subprocess.check_output(sys.argv[2], stderr=subprocess.STDOUT) +print 'comparing' +if S0 != S1: + sys.exit(1) +sys.exit(0) diff --git a/tests/unit_tests/CMakeLists.txt b/tests/unit_tests/CMakeLists.txt index d6bcbde46..87496a1b8 100644 --- a/tests/unit_tests/CMakeLists.txt +++ b/tests/unit_tests/CMakeLists.txt @@ -54,6 +54,7 @@ set(unit_tests_sources hashchain.cpp http.cpp keccak.cpp + long_term_block_weight.cpp main.cpp memwipe.cpp mlocker.cpp diff --git a/tests/unit_tests/blockchain_db.cpp b/tests/unit_tests/blockchain_db.cpp index 7e7ce9bf7..9cf8c5fbe 100644 --- a/tests/unit_tests/blockchain_db.cpp +++ b/tests/unit_tests/blockchain_db.cpp @@ -277,10 +277,10 @@ TYPED_TEST(BlockchainDBTest, AddBlock) // TODO: need at least one more block to make this reasonable, as the // BlockchainDB implementation should not check for parent if // no blocks have been added yet (because genesis has no parent). - //ASSERT_THROW(this->m_db->add_block(this->m_blocks[1], t_sizes[1], t_diffs[1], t_coins[1], this->m_txs[1]), BLOCK_PARENT_DNE); + //ASSERT_THROW(this->m_db->add_block(this->m_blocks[1], t_sizes[1], t_sizes[1], t_diffs[1], t_coins[1], this->m_txs[1]), BLOCK_PARENT_DNE); - ASSERT_NO_THROW(this->m_db->add_block(this->m_blocks[0], t_sizes[0], t_diffs[0], t_coins[0], this->m_txs[0])); - ASSERT_NO_THROW(this->m_db->add_block(this->m_blocks[1], t_sizes[1], t_diffs[1], t_coins[1], this->m_txs[1])); + ASSERT_NO_THROW(this->m_db->add_block(this->m_blocks[0], t_sizes[0], t_sizes[0], t_diffs[0], t_coins[0], this->m_txs[0])); + ASSERT_NO_THROW(this->m_db->add_block(this->m_blocks[1], t_sizes[1], t_sizes[1], t_diffs[1], t_coins[1], this->m_txs[1])); block b; ASSERT_TRUE(this->m_db->block_exists(get_block_hash(this->m_blocks[0]))); @@ -293,7 +293,7 @@ TYPED_TEST(BlockchainDBTest, AddBlock) ASSERT_TRUE(compare_blocks(this->m_blocks[0], b)); // assert that we can't add the same block twice - ASSERT_THROW(this->m_db->add_block(this->m_blocks[0], t_sizes[0], t_diffs[0], t_coins[0], this->m_txs[0]), TX_EXISTS); + ASSERT_THROW(this->m_db->add_block(this->m_blocks[0], t_sizes[0], t_sizes[0], t_diffs[0], t_coins[0], this->m_txs[0]), TX_EXISTS); for (auto& h : this->m_blocks[0].tx_hashes) { @@ -317,14 +317,14 @@ TYPED_TEST(BlockchainDBTest, RetrieveBlockData) this->get_filenames(); this->init_hard_fork(); - ASSERT_NO_THROW(this->m_db->add_block(this->m_blocks[0], t_sizes[0], t_diffs[0], t_coins[0], this->m_txs[0])); + ASSERT_NO_THROW(this->m_db->add_block(this->m_blocks[0], t_sizes[0], t_sizes[0], t_diffs[0], t_coins[0], this->m_txs[0])); ASSERT_EQ(t_sizes[0], this->m_db->get_block_weight(0)); ASSERT_EQ(t_diffs[0], this->m_db->get_block_cumulative_difficulty(0)); ASSERT_EQ(t_diffs[0], this->m_db->get_block_difficulty(0)); ASSERT_EQ(t_coins[0], this->m_db->get_block_already_generated_coins(0)); - ASSERT_NO_THROW(this->m_db->add_block(this->m_blocks[1], t_sizes[1], t_diffs[1], t_coins[1], this->m_txs[1])); + ASSERT_NO_THROW(this->m_db->add_block(this->m_blocks[1], t_sizes[1], t_sizes[1], t_diffs[1], t_coins[1], this->m_txs[1])); ASSERT_EQ(t_diffs[1] - t_diffs[0], this->m_db->get_block_difficulty(1)); ASSERT_HASH_EQ(get_block_hash(this->m_blocks[0]), this->m_db->get_block_hash_from_height(0)); diff --git a/tests/unit_tests/hardfork.cpp b/tests/unit_tests/hardfork.cpp index 47177db1c..5dd93dfaa 100644 --- a/tests/unit_tests/hardfork.cpp +++ b/tests/unit_tests/hardfork.cpp @@ -34,6 +34,7 @@ #include "blockchain_db/blockchain_db.h" #include "cryptonote_basic/cryptonote_format_utils.h" #include "cryptonote_basic/hardfork.h" +#include "blockchain_db/testdb.h" using namespace cryptonote; @@ -41,7 +42,7 @@ using namespace cryptonote; #define SECONDS_PER_YEAR 31557600 -class TestDB: public BlockchainDB { +class TestDB: public cryptonote::BaseTestDB { public: TestDB() {}; virtual void open(const std::string& filename, const int db_flags = 0) { } @@ -131,6 +132,7 @@ public: virtual void add_block( const block& blk , size_t block_weight + , uint64_t long_term_block_weight , const difficulty_type& cumulative_difficulty , const uint64_t& coins_generated , uint64_t num_rct_outs @@ -186,20 +188,20 @@ TEST(major, Only) ASSERT_FALSE(hf.add(mkblock(0, 2), 0)); ASSERT_FALSE(hf.add(mkblock(2, 2), 0)); ASSERT_TRUE(hf.add(mkblock(1, 2), 0)); - db.add_block(mkblock(1, 1), 0, 0, 0, 0, crypto::hash()); + db.add_block(mkblock(1, 1), 0, 0, 0, 0, 0, crypto::hash()); // block height 1, only version 1 is accepted ASSERT_FALSE(hf.add(mkblock(0, 2), 1)); ASSERT_FALSE(hf.add(mkblock(2, 2), 1)); ASSERT_TRUE(hf.add(mkblock(1, 2), 1)); - db.add_block(mkblock(1, 1), 0, 0, 0, 0, crypto::hash()); + db.add_block(mkblock(1, 1), 0, 0, 0, 0, 0, crypto::hash()); // block height 2, only version 2 is accepted ASSERT_FALSE(hf.add(mkblock(0, 2), 2)); ASSERT_FALSE(hf.add(mkblock(1, 2), 2)); ASSERT_FALSE(hf.add(mkblock(3, 2), 2)); ASSERT_TRUE(hf.add(mkblock(2, 2), 2)); - db.add_block(mkblock(2, 1), 0, 0, 0, 0, crypto::hash()); + db.add_block(mkblock(2, 1), 0, 0, 0, 0, 0, crypto::hash()); } TEST(empty_hardforks, Success) @@ -213,7 +215,7 @@ TEST(empty_hardforks, Success) ASSERT_TRUE(hf.get_state(time(NULL) + 3600*24*400) == HardFork::Ready); for (uint64_t h = 0; h <= 10; ++h) { - db.add_block(mkblock(hf, h, 1), 0, 0, 0, 0, crypto::hash()); + db.add_block(mkblock(hf, h, 1), 0, 0, 0, 0, 0, crypto::hash()); ASSERT_TRUE(hf.add(db.get_block_from_height(h), h)); } ASSERT_EQ(hf.get(0), 1); @@ -247,14 +249,14 @@ TEST(check_for_height, Success) for (uint64_t h = 0; h <= 4; ++h) { ASSERT_TRUE(hf.check_for_height(mkblock(1, 1), h)); ASSERT_FALSE(hf.check_for_height(mkblock(2, 2), h)); // block version is too high - db.add_block(mkblock(hf, h, 1), 0, 0, 0, 0, crypto::hash()); + db.add_block(mkblock(hf, h, 1), 0, 0, 0, 0, 0, crypto::hash()); ASSERT_TRUE(hf.add(db.get_block_from_height(h), h)); } for (uint64_t h = 5; h <= 10; ++h) { ASSERT_FALSE(hf.check_for_height(mkblock(1, 1), h)); // block version is too low ASSERT_TRUE(hf.check_for_height(mkblock(2, 2), h)); - db.add_block(mkblock(hf, h, 2), 0, 0, 0, 0, crypto::hash()); + db.add_block(mkblock(hf, h, 2), 0, 0, 0, 0, 0, crypto::hash()); ASSERT_TRUE(hf.add(db.get_block_from_height(h), h)); } } @@ -271,19 +273,19 @@ TEST(get, next_version) for (uint64_t h = 0; h <= 4; ++h) { ASSERT_EQ(2, hf.get_next_version()); - db.add_block(mkblock(hf, h, 1), 0, 0, 0, 0, crypto::hash()); + db.add_block(mkblock(hf, h, 1), 0, 0, 0, 0, 0, crypto::hash()); ASSERT_TRUE(hf.add(db.get_block_from_height(h), h)); } for (uint64_t h = 5; h <= 9; ++h) { ASSERT_EQ(4, hf.get_next_version()); - db.add_block(mkblock(hf, h, 2), 0, 0, 0, 0, crypto::hash()); + db.add_block(mkblock(hf, h, 2), 0, 0, 0, 0, 0, crypto::hash()); ASSERT_TRUE(hf.add(db.get_block_from_height(h), h)); } for (uint64_t h = 10; h <= 15; ++h) { ASSERT_EQ(4, hf.get_next_version()); - db.add_block(mkblock(hf, h, 4), 0, 0, 0, 0, crypto::hash()); + db.add_block(mkblock(hf, h, 4), 0, 0, 0, 0, 0, crypto::hash()); ASSERT_TRUE(hf.add(db.get_block_from_height(h), h)); } } @@ -324,7 +326,7 @@ TEST(steps_asap, Success) hf.init(); for (uint64_t h = 0; h < 10; ++h) { - db.add_block(mkblock(hf, h, 9), 0, 0, 0, 0, crypto::hash()); + db.add_block(mkblock(hf, h, 9), 0, 0, 0, 0, 0, crypto::hash()); ASSERT_TRUE(hf.add(db.get_block_from_height(h), h)); } @@ -351,7 +353,7 @@ TEST(steps_1, Success) hf.init(); for (uint64_t h = 0 ; h < 10; ++h) { - db.add_block(mkblock(hf, h, h+1), 0, 0, 0, 0, crypto::hash()); + db.add_block(mkblock(hf, h, h+1), 0, 0, 0, 0, 0, crypto::hash()); ASSERT_TRUE(hf.add(db.get_block_from_height(h), h)); } @@ -376,7 +378,7 @@ TEST(reorganize, Same) // index 0 1 2 3 4 5 6 7 8 9 static const uint8_t block_versions[] = { 1, 1, 4, 4, 7, 7, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9 }; for (uint64_t h = 0; h < 20; ++h) { - db.add_block(mkblock(hf, h, block_versions[h]), 0, 0, 0, 0, crypto::hash()); + db.add_block(mkblock(hf, h, block_versions[h]), 0, 0, 0, 0, 0, crypto::hash()); ASSERT_TRUE(hf.add(db.get_block_from_height(h), h)); } @@ -407,7 +409,7 @@ TEST(reorganize, Changed) static const uint8_t block_versions[] = { 1, 1, 4, 4, 7, 7, 9, 9, 9, 9, 9, 9, 9, 9, 9, 9 }; static const uint8_t expected_versions[] = { 1, 1, 1, 1, 1, 1, 4, 4, 7, 7, 9, 9, 9, 9, 9, 9 }; for (uint64_t h = 0; h < 16; ++h) { - db.add_block(mkblock(hf, h, block_versions[h]), 0, 0, 0, 0, crypto::hash()); + db.add_block(mkblock(hf, h, block_versions[h]), 0, 0, 0, 0, 0, crypto::hash()); ASSERT_TRUE (hf.add(db.get_block_from_height(h), h)); } @@ -427,7 +429,7 @@ TEST(reorganize, Changed) ASSERT_EQ(db.height(), 3); hf.reorganize_from_block_height(2); for (uint64_t h = 3; h < 16; ++h) { - db.add_block(mkblock(hf, h, block_versions_new[h]), 0, 0, 0, 0, crypto::hash()); + db.add_block(mkblock(hf, h, block_versions_new[h]), 0, 0, 0, 0, 0, crypto::hash()); bool ret = hf.add(db.get_block_from_height(h), h); ASSERT_EQ (ret, h < 15); } @@ -451,7 +453,7 @@ TEST(voting, threshold) for (uint64_t h = 0; h <= 8; ++h) { uint8_t v = 1 + !!(h % 8); - db.add_block(mkblock(hf, h, v), 0, 0, 0, 0, crypto::hash()); + db.add_block(mkblock(hf, h, v), 0, 0, 0, 0, 0, crypto::hash()); bool ret = hf.add(db.get_block_from_height(h), h); if (h >= 8 && threshold == 87) { // for threshold 87, we reach the treshold at height 7, so from height 8, hard fork to version 2, but 8 tries to add 1 @@ -485,7 +487,7 @@ TEST(voting, different_thresholds) static const uint8_t expected_versions[] = { 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4 }; for (uint64_t h = 0; h < sizeof(block_versions) / sizeof(block_versions[0]); ++h) { - db.add_block(mkblock(hf, h, block_versions[h]), 0, 0, 0, 0, crypto::hash()); + db.add_block(mkblock(hf, h, block_versions[h]), 0, 0, 0, 0, 0, crypto::hash()); bool ret = hf.add(db.get_block_from_height(h), h); ASSERT_EQ(ret, true); } @@ -539,7 +541,7 @@ TEST(voting, info) ASSERT_EQ(expected_thresholds[h], threshold); ASSERT_EQ(4, voting); - db.add_block(mkblock(hf, h, block_versions[h]), 0, 0, 0, 0, crypto::hash()); + db.add_block(mkblock(hf, h, block_versions[h]), 0, 0, 0, 0, 0, crypto::hash()); ASSERT_TRUE(hf.add(db.get_block_from_height(h), h)); } } @@ -602,7 +604,7 @@ TEST(reorganize, changed) #define ADD(v, h, a) \ do { \ cryptonote::block b = mkblock(hf, h, v); \ - db.add_block(b, 0, 0, 0, 0, crypto::hash()); \ + db.add_block(b, 0, 0, 0, 0, 0, crypto::hash()); \ ASSERT_##a(hf.add(b, h)); \ } while(0) #define ADD_TRUE(v, h) ADD(v, h, TRUE) diff --git a/tests/unit_tests/long_term_block_weight.cpp b/tests/unit_tests/long_term_block_weight.cpp new file mode 100644 index 000000000..9bfd95e0e --- /dev/null +++ b/tests/unit_tests/long_term_block_weight.cpp @@ -0,0 +1,384 @@ +// Copyright (c) 2019, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#define IN_UNIT_TESTS + +#include "gtest/gtest.h" +#include "cryptonote_core/blockchain.h" +#include "cryptonote_core/tx_pool.h" +#include "cryptonote_core/cryptonote_core.h" +#include "blockchain_db/testdb.h" + +#define TEST_LONG_TERM_BLOCK_WEIGHT_WINDOW 5000 + +namespace +{ + +class TestDB: public cryptonote::BaseTestDB +{ +private: + struct block_t + { + size_t weight; + uint64_t long_term_weight; + }; + +public: + TestDB() { m_open = true; } + + virtual void add_block( const cryptonote::block& blk + , size_t block_weight + , uint64_t long_term_block_weight + , const cryptonote::difficulty_type& cumulative_difficulty + , const uint64_t& coins_generated + , uint64_t num_rct_outs + , const crypto::hash& blk_hash + ) override { + blocks.push_back({block_weight, long_term_block_weight}); + } + virtual uint64_t height() const override { return blocks.size(); } + virtual size_t get_block_weight(const uint64_t &h) const override { return blocks[h].weight; } + virtual uint64_t get_block_long_term_weight(const uint64_t &h) const override { return blocks[h].long_term_weight; } + virtual crypto::hash top_block_hash() const override { + uint64_t h = height(); + crypto::hash top = crypto::null_hash; + if (h) + *(uint64_t*)&top = h - 1; + return top; + } + virtual void pop_block(cryptonote::block &blk, std::vector &txs) override { blocks.pop_back(); } + +private: + std::vector blocks; +}; + +static uint32_t lcg_seed = 0; + +static uint32_t lcg() +{ + lcg_seed = (lcg_seed * 0x100000001b3 + 0xcbf29ce484222325) & 0xffffffff; + return lcg_seed; +} + +} + +#define PREFIX_WINDOW(hf_version,window) \ + std::unique_ptr bc; \ + cryptonote::tx_memory_pool txpool(*bc); \ + bc.reset(new cryptonote::Blockchain(txpool)); \ + struct get_test_options { \ + const std::pair hard_forks[3]; \ + const cryptonote::test_options test_options = { \ + hard_forks, \ + window, \ + }; \ + get_test_options(): hard_forks{std::make_pair(1, (uint64_t)0), std::make_pair((uint8_t)hf_version, (uint64_t)1), std::make_pair((uint8_t)0, (uint64_t)0)} {} \ + } opts; \ + cryptonote::Blockchain *blockchain = bc.get(); \ + bool r = blockchain->init(new TestDB(), cryptonote::FAKECHAIN, true, &opts.test_options, 0); \ + ASSERT_TRUE(r) + +#define PREFIX(hf_version) PREFIX_WINDOW(hf_version, TEST_LONG_TERM_BLOCK_WEIGHT_WINDOW) + +TEST(long_term_block_weight, empty_short) +{ + PREFIX(9); + + ASSERT_TRUE(bc->update_next_cumulative_weight_limit()); + + ASSERT_EQ(bc->get_current_cumulative_block_weight_median(), CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5); + ASSERT_EQ(bc->get_current_cumulative_block_weight_limit(), CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5 * 2); +} + +TEST(long_term_block_weight, identical_before_fork) +{ + PREFIX(9); + + for (uint64_t h = 1; h < 10 * TEST_LONG_TERM_BLOCK_WEIGHT_WINDOW; ++h) + { + size_t w = h < CRYPTONOTE_REWARD_BLOCKS_WINDOW ? CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5 : bc->get_current_cumulative_block_weight_limit(); + uint64_t ltw = bc->get_next_long_term_block_weight(w); + bc->get_db().add_block(cryptonote::block(), w, ltw, h, h, {}); + ASSERT_TRUE(bc->update_next_cumulative_weight_limit()); + } + for (uint64_t h = 0; h < 10 * TEST_LONG_TERM_BLOCK_WEIGHT_WINDOW; ++h) + { + ASSERT_EQ(bc->get_db().get_block_long_term_weight(h), bc->get_db().get_block_weight(h)); + } +} + +TEST(long_term_block_weight, identical_after_fork_before_long_term_window) +{ + PREFIX(10); + + for (uint64_t h = 1; h <= TEST_LONG_TERM_BLOCK_WEIGHT_WINDOW; ++h) + { + size_t w = h < TEST_LONG_TERM_BLOCK_WEIGHT_WINDOW ? CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5 : bc->get_current_cumulative_block_weight_limit(); + uint64_t ltw = bc->get_next_long_term_block_weight(w); + bc->get_db().add_block(cryptonote::block(), w, ltw, h, h, {}); + ASSERT_TRUE(bc->update_next_cumulative_weight_limit()); + } + for (uint64_t h = 0; h < TEST_LONG_TERM_BLOCK_WEIGHT_WINDOW; ++h) + { + ASSERT_EQ(bc->get_db().get_block_long_term_weight(h), bc->get_db().get_block_weight(h)); + } +} + +TEST(long_term_block_weight, ceiling_at_30000000) +{ + PREFIX(10); + + for (uint64_t h = 0; h < TEST_LONG_TERM_BLOCK_WEIGHT_WINDOW + TEST_LONG_TERM_BLOCK_WEIGHT_WINDOW / 2 - 1; ++h) + { + size_t w = h < TEST_LONG_TERM_BLOCK_WEIGHT_WINDOW ? CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5 : bc->get_current_cumulative_block_weight_limit(); + uint64_t ltw = bc->get_next_long_term_block_weight(w); + bc->get_db().add_block(cryptonote::block(), w, ltw, h, h, {}); + ASSERT_TRUE(bc->update_next_cumulative_weight_limit()); + } + ASSERT_EQ(bc->get_current_cumulative_block_weight_median(), 15000000); + ASSERT_EQ(bc->get_current_cumulative_block_weight_limit(), 30000000); +} + +TEST(long_term_block_weight, multi_pop) +{ + PREFIX(10); + + for (uint64_t h = 1; h <= TEST_LONG_TERM_BLOCK_WEIGHT_WINDOW + 20; ++h) + { + size_t w = h < TEST_LONG_TERM_BLOCK_WEIGHT_WINDOW ? CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5 : bc->get_current_cumulative_block_weight_limit(); + uint64_t ltw = bc->get_next_long_term_block_weight(w); + bc->get_db().add_block(cryptonote::block(), w, ltw, h, h, {}); + ASSERT_TRUE(bc->update_next_cumulative_weight_limit()); + } + + const uint64_t effective_median = bc->get_current_cumulative_block_weight_median(); + const uint64_t effective_limit = bc->get_current_cumulative_block_weight_limit(); + + for (uint64_t h = 0; h < 4; ++h) + { + size_t w = h < TEST_LONG_TERM_BLOCK_WEIGHT_WINDOW ? CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5 : bc->get_current_cumulative_block_weight_limit(); + uint64_t ltw = bc->get_next_long_term_block_weight(w); + bc->get_db().add_block(cryptonote::block(), w, ltw, h, h, {}); + ASSERT_TRUE(bc->update_next_cumulative_weight_limit()); + } + + cryptonote::block b; + std::vector txs; + bc->get_db().pop_block(b, txs); + bc->get_db().pop_block(b, txs); + bc->get_db().pop_block(b, txs); + bc->get_db().pop_block(b, txs); + ASSERT_TRUE(bc->update_next_cumulative_weight_limit()); + + ASSERT_EQ(effective_median, bc->get_current_cumulative_block_weight_median()); + ASSERT_EQ(effective_limit, bc->get_current_cumulative_block_weight_limit()); +} + +TEST(long_term_block_weight, multiple_updates) +{ + PREFIX(10); + + for (uint64_t h = 1; h <= 3 * TEST_LONG_TERM_BLOCK_WEIGHT_WINDOW; ++h) + { + size_t w = h < TEST_LONG_TERM_BLOCK_WEIGHT_WINDOW ? CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5 : bc->get_current_cumulative_block_weight_limit(); + uint64_t ltw = bc->get_next_long_term_block_weight(w); + bc->get_db().add_block(cryptonote::block(), w, ltw, h, h, {}); + ASSERT_TRUE(bc->update_next_cumulative_weight_limit()); + const uint64_t effective_median = bc->get_current_cumulative_block_weight_median(); + const uint64_t effective_limit = bc->get_current_cumulative_block_weight_limit(); + ASSERT_TRUE(bc->update_next_cumulative_weight_limit()); + ASSERT_EQ(effective_median, bc->get_current_cumulative_block_weight_median()); + ASSERT_EQ(effective_limit, bc->get_current_cumulative_block_weight_limit()); + ASSERT_TRUE(bc->update_next_cumulative_weight_limit()); + ASSERT_EQ(effective_median, bc->get_current_cumulative_block_weight_median()); + ASSERT_EQ(effective_limit, bc->get_current_cumulative_block_weight_limit()); + ASSERT_TRUE(bc->update_next_cumulative_weight_limit()); + ASSERT_EQ(effective_median, bc->get_current_cumulative_block_weight_median()); + ASSERT_EQ(effective_limit, bc->get_current_cumulative_block_weight_limit()); + } +} + +TEST(long_term_block_weight, pop_invariant_max) +{ + PREFIX(10); + + for (uint64_t h = 1; h < TEST_LONG_TERM_BLOCK_WEIGHT_WINDOW - 10; ++h) + { + size_t w = h < TEST_LONG_TERM_BLOCK_WEIGHT_WINDOW ? CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5 : bc->get_current_cumulative_block_weight_limit(); + uint64_t ltw = bc->get_next_long_term_block_weight(w); + bc->get_db().add_block(cryptonote::block(), w, ltw, h, h, {}); + ASSERT_TRUE(bc->update_next_cumulative_weight_limit()); + } + + for (int n = 0; n < 1000; ++n) + { + // pop some blocks, then add some more + int remove = 1 + (n * 17) % 8; + int add = (n * 23) % 12; + + // save long term block weights we're about to remove + uint64_t old_ltbw[16], h0 = bc->get_db().height() - remove - 1; + for (int i = -2; i < remove; ++i) + { + old_ltbw[i + 2] = bc->get_db().get_block_long_term_weight(h0 + i); + } + + for (int i = 0; i < remove; ++i) + { + cryptonote::block b; + std::vector txs; + bc->get_db().pop_block(b, txs); + ASSERT_TRUE(bc->update_next_cumulative_weight_limit()); + } + for (int i = 0; i < add; ++i) + { + size_t w = bc->get_db().height() < TEST_LONG_TERM_BLOCK_WEIGHT_WINDOW ? CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5 : bc->get_current_cumulative_block_weight_limit(); + uint64_t ltw = bc->get_next_long_term_block_weight(w); + bc->get_db().add_block(cryptonote::block(), w, ltw, bc->get_db().height(), bc->get_db().height(), {}); + ASSERT_TRUE(bc->update_next_cumulative_weight_limit()); + } + + // check the new values are the same as the old ones + for (int i = -2; i < std::min(add, remove); ++i) + { + ASSERT_EQ(bc->get_db().get_block_long_term_weight(h0 + i), old_ltbw[i + 2]); + } + } +} + +TEST(long_term_block_weight, pop_invariant_random) +{ + PREFIX(10); + + for (uint64_t h = 1; h < TEST_LONG_TERM_BLOCK_WEIGHT_WINDOW - 10; ++h) + { + size_t w = h < TEST_LONG_TERM_BLOCK_WEIGHT_WINDOW ? CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5 : bc->get_current_cumulative_block_weight_limit(); + uint64_t ltw = bc->get_next_long_term_block_weight(w); + bc->get_db().add_block(cryptonote::block(), w, ltw, h, h, {}); + ASSERT_TRUE(bc->update_next_cumulative_weight_limit()); + } + + for (int n = 0; n < 1000; ++n) + { + // pop some blocks, then add some more + int remove = 1 + (n * 17) % 8; + int add = (n * 23) % 12; + + // save long term block weights we're about to remove + uint64_t old_ltbw[16], h0 = bc->get_db().height() - remove - 1; + for (int i = -2; i < remove; ++i) + { + old_ltbw[i + 2] = bc->get_db().get_block_long_term_weight(h0 + i); + } + + for (int i = 0; i < remove; ++i) + { + cryptonote::block b; + std::vector txs; + bc->get_db().pop_block(b, txs); + ASSERT_TRUE(bc->update_next_cumulative_weight_limit()); + const uint64_t effective_median = bc->get_current_cumulative_block_weight_median(); + const uint64_t effective_limit = bc->get_current_cumulative_block_weight_limit(); + ASSERT_TRUE(bc->update_next_cumulative_weight_limit()); + ASSERT_EQ(effective_median, bc->get_current_cumulative_block_weight_median()); + ASSERT_EQ(effective_limit, bc->get_current_cumulative_block_weight_limit()); + } + for (int i = 0; i < add; ++i) + { + lcg_seed = bc->get_db().height(); + uint32_t r = lcg(); + size_t w = bc->get_db().height() < TEST_LONG_TERM_BLOCK_WEIGHT_WINDOW ? CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5 : (r % bc->get_current_cumulative_block_weight_limit()); + uint64_t ltw = bc->get_next_long_term_block_weight(w); + bc->get_db().add_block(cryptonote::block(), w, ltw, bc->get_db().height(), bc->get_db().height(), {}); + ASSERT_TRUE(bc->update_next_cumulative_weight_limit()); + const uint64_t effective_median = bc->get_current_cumulative_block_weight_median(); + const uint64_t effective_limit = bc->get_current_cumulative_block_weight_limit(); + ASSERT_TRUE(bc->update_next_cumulative_weight_limit()); + ASSERT_EQ(effective_median, bc->get_current_cumulative_block_weight_median()); + ASSERT_EQ(effective_limit, bc->get_current_cumulative_block_weight_limit()); + } + + // check the new values are the same as the old ones + for (int i = -2; i < std::min(add, remove); ++i) + { + ASSERT_EQ(bc->get_db().get_block_long_term_weight(h0 + i), old_ltbw[i + 2]); + } + } +} + +TEST(long_term_block_weight, long_growth_spike_and_drop) +{ + PREFIX(10); + + uint64_t long_term_effective_median_block_weight; + + // constant init + for (uint64_t h = 0; h < TEST_LONG_TERM_BLOCK_WEIGHT_WINDOW; ++h) + { + size_t w = CRYPTONOTE_BLOCK_GRANTED_FULL_REWARD_ZONE_V5; + uint64_t ltw = bc->get_next_long_term_block_weight(w); + bc->get_db().add_block(cryptonote::block(), w, ltw, h, h, {}); + ASSERT_TRUE(bc->update_next_cumulative_weight_limit(&long_term_effective_median_block_weight)); + } + ASSERT_EQ(long_term_effective_median_block_weight, 300000); + + // slow 10% yearly for a year (scaled down by 100000 / TEST_LONG_TERM_BLOCK_WEIGHT_WINDOW) -> 8% change + for (uint64_t h = 0; h < 365 * 720 * TEST_LONG_TERM_BLOCK_WEIGHT_WINDOW / 100000; ++h) + { + //size_t w = bc->get_current_cumulative_block_weight_median() * rate; + float t = h / float(365 * 720 * TEST_LONG_TERM_BLOCK_WEIGHT_WINDOW / 100000); + size_t w = 300000 + t * 30000; + uint64_t ltw = bc->get_next_long_term_block_weight(w); + bc->get_db().add_block(cryptonote::block(), w, ltw, h, h, {}); + ASSERT_TRUE(bc->update_next_cumulative_weight_limit(&long_term_effective_median_block_weight)); + } + ASSERT_GT(long_term_effective_median_block_weight, 300000 * 1.07); + ASSERT_LT(long_term_effective_median_block_weight, 300000 * 1.09); + + // spike over three weeks - does not move much + for (uint64_t h = 0; h < 21 * 720 * TEST_LONG_TERM_BLOCK_WEIGHT_WINDOW / 100000; ++h) + { + size_t w = bc->get_current_cumulative_block_weight_limit(); + uint64_t ltw = bc->get_next_long_term_block_weight(w); + bc->get_db().add_block(cryptonote::block(), w, ltw, h, h, {}); + ASSERT_TRUE(bc->update_next_cumulative_weight_limit(&long_term_effective_median_block_weight)); + } + ASSERT_GT(long_term_effective_median_block_weight, 300000 * 1.07); + ASSERT_LT(long_term_effective_median_block_weight, 300000 * 1.09); + + // drop - does not move much + for (uint64_t h = 0; h < 21 * 720 * TEST_LONG_TERM_BLOCK_WEIGHT_WINDOW / 100000; ++h) + { + size_t w = bc->get_current_cumulative_block_weight_median() * .25; + uint64_t ltw = bc->get_next_long_term_block_weight(w); + bc->get_db().add_block(cryptonote::block(), w, ltw, h, h, {}); + ASSERT_TRUE(bc->update_next_cumulative_weight_limit(&long_term_effective_median_block_weight)); + } + ASSERT_GT(long_term_effective_median_block_weight, 300000 * 1.07); + ASSERT_LT(long_term_effective_median_block_weight, 300000 * 1.09); +}