From d6488719661f82461a24351ab41bfa2fcf3e2eb9 Mon Sep 17 00:00:00 2001 From: Boog900 <54e72d8a-345f-4599-bd90-c6b9bc7d0ec5@aleeas.com> Date: Thu, 29 Aug 2024 18:44:34 +0100 Subject: [PATCH 01/15] add new tables & types --- storage/blockchain/src/lib.rs | 2 +- storage/blockchain/src/tables.rs | 25 ++++++-- storage/blockchain/src/types.rs | 101 ++++++++++++++++++++++++++++++- types/src/types.rs | 3 +- 4 files changed, 123 insertions(+), 8 deletions(-) diff --git a/storage/blockchain/src/lib.rs b/storage/blockchain/src/lib.rs index e544a69e..0dea345b 100644 --- a/storage/blockchain/src/lib.rs +++ b/storage/blockchain/src/lib.rs @@ -52,7 +52,7 @@ unused_crate_dependencies, unused_doc_comments, unused_mut, - missing_docs, + //missing_docs, deprecated, unused_comparisons, nonstandard_style diff --git a/storage/blockchain/src/tables.rs b/storage/blockchain/src/tables.rs index 122ac31b..6db76816 100644 --- a/storage/blockchain/src/tables.rs +++ b/storage/blockchain/src/tables.rs @@ -16,11 +16,7 @@ //! accessing _all_ tables defined here at once. //---------------------------------------------------------------------------------------------------- Import -use crate::types::{ - Amount, AmountIndex, AmountIndices, BlockBlob, BlockHash, BlockHeight, BlockInfo, KeyImage, - Output, PreRctOutputId, PrunableBlob, PrunableHash, PrunedBlob, RctOutput, TxBlob, TxHash, - TxId, UnlockTime, -}; +use crate::types::{Amount, AmountIndex, AmountIndices, BlockBlob, BlockHash, BlockHeight, BlockInfo, KeyImage, Output, PreRctOutputId, PrunableBlob, PrunableHash, PrunedBlob, RctOutput, TxBlob, TxHash, TxId, UnlockTime, RawChainId, AltChainInfo, AltBlockHeight, CompactAltBlockInfo, AltTransactionInfo}; //---------------------------------------------------------------------------------------------------- Tables // Notes: @@ -129,6 +125,25 @@ cuprate_database::define_tables! { /// Transactions without unlock times will not exist in this table. 14 => TxUnlockTime, TxId => UnlockTime, + + 15 => AltChainInfos, + RawChainId => AltChainInfo, + + 16 => AltBlockHeights, + BlockHash => AltBlockHeight, + + 17 => AltBlocksInfo, + AltBlockHeight => CompactAltBlockInfo, + + 18 => AltBlockBlobs, + AltBlockHeight => BlockBlob, + + 19 => AltTransactionBlobs, + TxHash => TxBlob, + + 20 => AltTransactionInfos, + TxHash => AltTransactionInfo, + } //---------------------------------------------------------------------------------------------------- Tests diff --git a/storage/blockchain/src/types.rs b/storage/blockchain/src/types.rs index eb1dc647..73c7614f 100644 --- a/storage/blockchain/src/types.rs +++ b/storage/blockchain/src/types.rs @@ -41,13 +41,15 @@ #![forbid(unsafe_code)] // if you remove this line i will steal your monero //---------------------------------------------------------------------------------------------------- Import +use std::num::NonZero; + use bytemuck::{Pod, Zeroable}; #[cfg(feature = "serde")] use serde::{Deserialize, Serialize}; use cuprate_database::{Key, StorableVec}; - +use cuprate_types::{Chain, ChainId}; //---------------------------------------------------------------------------------------------------- Aliases // These type aliases exist as many Monero-related types are the exact same. // For clarity, they're given type aliases as to not confuse them. @@ -324,6 +326,103 @@ pub struct RctOutput { } // TODO: local_index? +//---------------------------------------------------------------------------------------------------- RawChain +#[derive(Copy, Clone, Debug, PartialEq, PartialOrd, Eq, Ord, Hash, Pod, Zeroable)] +#[repr(transparent)] +pub struct RawChain(u64); + +impl From for RawChain { + fn from(value: Chain) -> Self { + match value { + Chain::Main => RawChain(0), + Chain::Alt(chain_id) => RawChain(chain_id.0.get()), + } + } +} + +impl From for Chain { + fn from(value: RawChain) -> Self { + NonZero::new(value.0) + .map(|id| Chain::Alt(ChainId(id))) + .unwrap_or(Chain::Main) + } +} + +//---------------------------------------------------------------------------------------------------- RawChainId +#[derive(Copy, Clone, Debug, PartialEq, PartialOrd, Eq, Ord, Hash, Pod, Zeroable)] +#[repr(transparent)] +pub struct RawChainId(u64); + +impl From for RawChainId { + fn from(value: ChainId) -> Self { + RawChainId(value.0.get()) + } +} + +impl From for ChainId { + fn from(value: RawChainId) -> Self { + ChainId(NonZero::new(value.0).expect("RawChainId mut not have a value of `0`")) + } +} + +impl Key for RawChainId {} + +//---------------------------------------------------------------------------------------------------- AltChainInfo +#[derive(Copy, Clone, Debug, PartialEq, PartialOrd, Eq, Ord, Hash, Pod, Zeroable)] +#[repr(C)] +pub struct AltChainInfo { + parent_chain: RawChain, + common_ancestor_height: u64 +} + +//---------------------------------------------------------------------------------------------------- AltBlockHeight +#[derive(Copy, Clone, Debug, PartialEq, PartialOrd, Eq, Ord, Hash, Pod, Zeroable)] +#[repr(C)] +pub struct AltBlockHeight { + chain_id: u64, + height: u64, +} + +impl Key for AltBlockHeight {} + +//---------------------------------------------------------------------------------------------------- CompactAltBlockInfo +#[derive(Copy, Clone, Debug, PartialEq, PartialOrd, Eq, Ord, Hash, Pod, Zeroable)] +#[repr(C)] +pub struct CompactAltBlockInfo { + /// The block's hash. + /// + /// [`Block::hash`]. + pub block_hash: [u8; 32], + /// The block's proof-of-work hash. + pub pow_hash: [u8; 32], + /// The block's height. + pub height: u64, + /// The adjusted block size, in bytes. + pub weight: usize, + /// The long term block weight, which is the weight factored in with previous block weights. + pub long_term_weight: usize, + /// The cumulative difficulty of all blocks up until and including this block. + pub cumulative_difficulty_low: u64, + pub cumulative_difficulty_high: u64, + +} + +//---------------------------------------------------------------------------------------------------- AltTransactionInfo +#[derive(Copy, Clone, Debug, PartialEq, PartialOrd, Eq, Ord, Hash, Pod, Zeroable)] +#[repr(C)] +pub struct AltTransactionInfo { + /// The transaction's weight. + /// + /// [`Transaction::weight`]. + pub tx_weight: usize, + /// The transaction's total fees. + pub fee: u64, + /// The transaction's hash. + /// + /// [`Transaction::hash`]. + pub tx_hash: [u8; 32], +} + //---------------------------------------------------------------------------------------------------- Tests #[cfg(test)] mod test { diff --git a/types/src/types.rs b/types/src/types.rs index 4b6e2e12..da4422a1 100644 --- a/types/src/types.rs +++ b/types/src/types.rs @@ -1,5 +1,6 @@ //! Various shared data types in Cuprate. +use std::num::NonZero; //---------------------------------------------------------------------------------------------------- Import use curve25519_dalek::edwards::EdwardsPoint; use monero_serai::{ @@ -97,7 +98,7 @@ pub struct VerifiedBlockInformation { /// /// The inner value is meaningless. #[derive(Debug, Copy, Clone, Ord, PartialOrd, Eq, PartialEq, Hash)] -pub struct ChainId(pub u64); +pub struct ChainId(pub NonZero); //---------------------------------------------------------------------------------------------------- Chain /// An identifier for a chain. From e1ae84836911acab79fe3eb853725ca94fc4ff3e Mon Sep 17 00:00:00 2001 From: Boog900 <54e72d8a-345f-4599-bd90-c6b9bc7d0ec5@aleeas.com> Date: Fri, 30 Aug 2024 03:01:23 +0100 Subject: [PATCH 02/15] add function to fully add an alt block --- storage/blockchain/src/ops/alt_block.rs | 104 ++++++++++++++++++++++++ storage/blockchain/src/ops/mod.rs | 1 + storage/blockchain/src/tables.rs | 7 +- storage/blockchain/src/types.rs | 19 +++-- 4 files changed, 124 insertions(+), 7 deletions(-) create mode 100644 storage/blockchain/src/ops/alt_block.rs diff --git a/storage/blockchain/src/ops/alt_block.rs b/storage/blockchain/src/ops/alt_block.rs new file mode 100644 index 00000000..d04899ef --- /dev/null +++ b/storage/blockchain/src/ops/alt_block.rs @@ -0,0 +1,104 @@ +use bytemuck::TransparentWrapper; + +use cuprate_database::{DatabaseRw, RuntimeError, StorableVec, DatabaseRo}; +use cuprate_helper::map::split_u128_into_low_high_bits; +use cuprate_types::{AltBlockInformation, Chain, VerifiedTransactionInformation}; + +use crate::{ + tables::TablesMut, + types::{AltBlockHeight, AltChainInfo, AltTransactionInfo, BlockHash, CompactAltBlockInfo}, +}; + +pub fn add_alt_block( + alt_block: &AltBlockInformation, + tables: &mut impl TablesMut, +) -> Result<(), RuntimeError> { + let alt_block_height = AltBlockHeight { + chain_id: alt_block.chain_id.into(), + height: alt_block.height, + }; + + tables + .alt_block_heights_mut() + .put(&alt_block.block_hash, &alt_block_height)?; + + check_add_alt_chain_info(&alt_block_height, &alt_block.block.header.previous, tables)?; + + let (cumulative_difficulty_low, cumulative_difficulty_high) = + split_u128_into_low_high_bits(alt_block.cumulative_difficulty); + + let alt_block_info = CompactAltBlockInfo { + block_hash: alt_block.block_hash, + pow_hash: alt_block.pow_hash, + height: alt_block.height, + weight: alt_block.weight, + long_term_weight: alt_block.long_term_weight, + cumulative_difficulty_low, + cumulative_difficulty_high, + }; + + tables + .alt_blocks_info_mut() + .put(&alt_block_height, &alt_block_info)?; + + tables.alt_block_blobs_mut().put( + &alt_block_height, + StorableVec::wrap_ref(&alt_block.block_blob), + )?; + + for tx in &alt_block.txs { + add_alt_transaction(&tx, tables)?; + } + + Ok(()) +} + +pub fn add_alt_transaction( + tx: &VerifiedTransactionInformation, + tables: &mut impl TablesMut, +) -> Result<(), RuntimeError> { + if tables.tx_ids().get(&tx.tx_hash).is_ok() + || tables.alt_transaction_infos().get(&tx.tx_hash).is_ok() + { + return Ok(()); + } + + tables.alt_transaction_infos_mut().put( + &tx.tx_hash, + &AltTransactionInfo { + tx_weight: tx.tx_weight, + fee: tx.fee, + tx_hash: tx.tx_hash, + }, + )?; + + tables + .alt_transaction_blobs_mut() + .put(&tx.tx_hash, StorableVec::wrap_ref(&tx.tx_blob)) +} + +pub fn check_add_alt_chain_info( + alt_block_height: &AltBlockHeight, + prev_hash: &BlockHash, + tables: &mut impl TablesMut, +) -> Result<(), RuntimeError> { + match tables.alt_chain_infos().get(&alt_block_height.chain_id) { + Ok(_) => return Ok(()), + Err(RuntimeError::KeyNotFound) => (), + Err(e) => return Err(e), + } + + let parent_chain = match tables.alt_block_heights().get(prev_hash) { + Ok(alt_parent_height) => Chain::Alt(alt_parent_height.chain_id.into()), + Err(RuntimeError::KeyNotFound) => Chain::Main, + Err(e) => return Err(e), + }; + + tables.alt_chain_infos_mut().put( + &alt_block_height.chain_id, + &AltChainInfo { + parent_chain: parent_chain.into(), + common_ancestor_height: alt_block_height.height - 1, + }, + ) +} diff --git a/storage/blockchain/src/ops/mod.rs b/storage/blockchain/src/ops/mod.rs index 4ff7dff1..1ec9c237 100644 --- a/storage/blockchain/src/ops/mod.rs +++ b/storage/blockchain/src/ops/mod.rs @@ -108,5 +108,6 @@ pub mod key_image; pub mod output; pub mod property; pub mod tx; +pub mod alt_block; mod macros; diff --git a/storage/blockchain/src/tables.rs b/storage/blockchain/src/tables.rs index 6db76816..381430d7 100644 --- a/storage/blockchain/src/tables.rs +++ b/storage/blockchain/src/tables.rs @@ -16,7 +16,12 @@ //! accessing _all_ tables defined here at once. //---------------------------------------------------------------------------------------------------- Import -use crate::types::{Amount, AmountIndex, AmountIndices, BlockBlob, BlockHash, BlockHeight, BlockInfo, KeyImage, Output, PreRctOutputId, PrunableBlob, PrunableHash, PrunedBlob, RctOutput, TxBlob, TxHash, TxId, UnlockTime, RawChainId, AltChainInfo, AltBlockHeight, CompactAltBlockInfo, AltTransactionInfo}; +use crate::types::{ + AltBlockHeight, AltChainInfo, AltTransactionInfo, Amount, AmountIndex, AmountIndices, + BlockBlob, BlockHash, BlockHeight, BlockInfo, CompactAltBlockInfo, KeyImage, Output, + PreRctOutputId, PrunableBlob, PrunableHash, PrunedBlob, RawChainId, RctOutput, TxBlob, TxHash, + TxId, UnlockTime, +}; //---------------------------------------------------------------------------------------------------- Tables // Notes: diff --git a/storage/blockchain/src/types.rs b/storage/blockchain/src/types.rs index 73c7614f..88ece10b 100644 --- a/storage/blockchain/src/types.rs +++ b/storage/blockchain/src/types.rs @@ -348,6 +348,14 @@ impl From for Chain { } } +impl From for RawChain { + fn from(value: RawChainId) -> Self { + assert_ne!(value.0, 0); + + RawChain(value.0) + } +} + //---------------------------------------------------------------------------------------------------- RawChainId #[derive(Copy, Clone, Debug, PartialEq, PartialOrd, Eq, Ord, Hash, Pod, Zeroable)] #[repr(transparent)] @@ -371,16 +379,16 @@ impl Key for RawChainId {} #[derive(Copy, Clone, Debug, PartialEq, PartialOrd, Eq, Ord, Hash, Pod, Zeroable)] #[repr(C)] pub struct AltChainInfo { - parent_chain: RawChain, - common_ancestor_height: u64 + pub parent_chain: RawChain, + pub common_ancestor_height: usize, } //---------------------------------------------------------------------------------------------------- AltBlockHeight #[derive(Copy, Clone, Debug, PartialEq, PartialOrd, Eq, Ord, Hash, Pod, Zeroable)] #[repr(C)] pub struct AltBlockHeight { - chain_id: u64, - height: u64, + pub chain_id: RawChainId, + pub height: usize, } impl Key for AltBlockHeight {} @@ -396,7 +404,7 @@ pub struct CompactAltBlockInfo { /// The block's proof-of-work hash. pub pow_hash: [u8; 32], /// The block's height. - pub height: u64, + pub height: usize, /// The adjusted block size, in bytes. pub weight: usize, /// The long term block weight, which is the weight factored in with previous block weights. @@ -404,7 +412,6 @@ pub struct CompactAltBlockInfo { /// The cumulative difficulty of all blocks up until and including this block. pub cumulative_difficulty_low: u64, pub cumulative_difficulty_high: u64, - } //---------------------------------------------------------------------------------------------------- AltTransactionInfo From ed887a7c859d45fce68cbf4e70f56e4b46449492 Mon Sep 17 00:00:00 2001 From: Boog900 <54e72d8a-345f-4599-bd90-c6b9bc7d0ec5@aleeas.com> Date: Fri, 30 Aug 2024 23:06:30 +0100 Subject: [PATCH 03/15] resolve current todo!s --- storage/blockchain/src/lib.rs | 2 + storage/blockchain/src/ops/alt_block.rs | 137 ++++++++++++++++++++++-- storage/blockchain/src/ops/block.rs | 10 +- storage/blockchain/src/ops/mod.rs | 2 +- storage/blockchain/src/service/read.rs | 43 +++++++- 5 files changed, 178 insertions(+), 16 deletions(-) diff --git a/storage/blockchain/src/lib.rs b/storage/blockchain/src/lib.rs index 0dea345b..8a6f96b6 100644 --- a/storage/blockchain/src/lib.rs +++ b/storage/blockchain/src/lib.rs @@ -98,6 +98,8 @@ clippy::too_many_lines ) )] +extern crate core; + // Only allow building 64-bit targets. // // This allows us to assume 64-bit diff --git a/storage/blockchain/src/ops/alt_block.rs b/storage/blockchain/src/ops/alt_block.rs index d04899ef..82a37ba8 100644 --- a/storage/blockchain/src/ops/alt_block.rs +++ b/storage/blockchain/src/ops/alt_block.rs @@ -1,12 +1,21 @@ -use bytemuck::TransparentWrapper; +use std::cmp::max; -use cuprate_database::{DatabaseRw, RuntimeError, StorableVec, DatabaseRo}; -use cuprate_helper::map::split_u128_into_low_high_bits; -use cuprate_types::{AltBlockInformation, Chain, VerifiedTransactionInformation}; +use bytemuck::TransparentWrapper; +use cuprate_database::{DatabaseRo, DatabaseRw, RuntimeError, StorableVec}; +use cuprate_helper::map::{combine_low_high_bits_to_u128, split_u128_into_low_high_bits}; +use cuprate_types::{ + AltBlockInformation, Chain, ChainId, ExtendedBlockHeader, HardFork, + VerifiedTransactionInformation, +}; +use monero_serai::block::BlockHeader; use crate::{ - tables::TablesMut, - types::{AltBlockHeight, AltChainInfo, AltTransactionInfo, BlockHash, CompactAltBlockInfo}, + ops::block::{get_block_extended_header_from_height, get_block_info}, + tables::{Tables, TablesMut}, + types::{ + AltBlockHeight, AltChainInfo, AltTransactionInfo, BlockHash, BlockHeight, + CompactAltBlockInfo, + }, }; pub fn add_alt_block( @@ -102,3 +111,119 @@ pub fn check_add_alt_chain_info( }, ) } + +pub fn alt_block_hash( + block_height: &BlockHeight, + alt_chain: ChainId, + tables: &mut impl Tables, +) -> Result { + let alt_chains = tables.alt_chain_infos(); + + let original_chain = { + let mut chain = alt_chain.into(); + loop { + let chain_info = alt_chains.get(&chain)?; + + if chain_info.common_ancestor_height < *block_height { + break Chain::Alt(chain.into()); + } + + match chain_info.parent_chain.into() { + Chain::Main => break Chain::Main, + Chain::Alt(alt_chain_id) => { + chain = alt_chain_id.into(); + continue; + } + } + } + }; + + match original_chain { + Chain::Main => { + get_block_info(&block_height, tables.block_infos()).map(|info| info.block_hash) + } + Chain::Alt(chain_id) => tables + .alt_blocks_info() + .get(&AltBlockHeight { + chain_id: chain_id.into(), + height: *block_height, + }) + .map(|info| info.block_hash), + } +} + +pub fn alt_extended_headers_in_range( + range: std::ops::Range, + alt_chain: ChainId, + tables: &impl Tables, +) -> Result, RuntimeError> { + // TODO: this function does not use rayon, however it probably should. + + let mut ranges = Vec::with_capacity(5); + let alt_chains = tables.alt_chain_infos(); + + let mut i = range.end; + let mut current_chain_id = alt_chain.into(); + while i > range.start { + let chain_info = alt_chains.get(¤t_chain_id)?; + + let start_height = max(range.start, chain_info.common_ancestor_height + 1); + + ranges.push((chain_info.parent_chain.into(), start_height..i)); + i = chain_info.common_ancestor_height; + + match chain_info.parent_chain.into() { + Chain::Main => { + ranges.push((Chain::Main, range.start..i)); + break; + } + Chain::Alt(alt_chain_id) => { + current_chain_id = alt_chain_id.into(); + continue; + } + } + } + + let res = ranges + .into_iter() + .rev() + .map(|(chain, range)| { + range.into_iter().map(move |height| match chain { + Chain::Main => get_block_extended_header_from_height(&height, tables), + Chain::Alt(chain_id) => get_alt_block_extended_header_from_height( + &AltBlockHeight { + chain_id: chain_id.into(), + height, + }, + tables, + ), + }) + }) + .flatten() + .collect::>()?; + + Ok(res) +} + +pub fn get_alt_block_extended_header_from_height( + height: &AltBlockHeight, + table: &impl Tables, +) -> Result { + let block_info = table.alt_blocks_info().get(height)?; + + let block_blob = table.alt_block_blobs().get(height)?.0; + + let block_header = BlockHeader::read(&mut block_blob.as_slice())?; + + Ok(ExtendedBlockHeader { + version: HardFork::from_version(0).expect("Block in DB must have correct version"), + vote: block_header.hardfork_version, + timestamp: block_header.timestamp, + cumulative_difficulty: combine_low_high_bits_to_u128( + block_info.cumulative_difficulty_low, + block_info.cumulative_difficulty_high, + ), + block_weight: block_info.weight, + long_term_weight: block_info.long_term_weight, + }) +} diff --git a/storage/blockchain/src/ops/block.rs b/storage/blockchain/src/ops/block.rs index 4f77d736..2e110fed 100644 --- a/storage/blockchain/src/ops/block.rs +++ b/storage/blockchain/src/ops/block.rs @@ -2,7 +2,7 @@ //---------------------------------------------------------------------------------------------------- Import use bytemuck::TransparentWrapper; -use monero_serai::block::Block; +use monero_serai::block::{Block, BlockHeader}; use cuprate_database::{ RuntimeError, StorableVec, {DatabaseRo, DatabaseRw}, @@ -190,7 +190,7 @@ pub fn get_block_extended_header_from_height( ) -> Result { let block_info = tables.block_infos().get(block_height)?; let block_blob = tables.block_blobs().get(block_height)?.0; - let block = Block::read(&mut block_blob.as_slice())?; + let block_header = BlockHeader::read(&mut block_blob.as_slice())?; let cumulative_difficulty = combine_low_high_bits_to_u128( block_info.cumulative_difficulty_low, @@ -201,10 +201,10 @@ pub fn get_block_extended_header_from_height( #[allow(clippy::cast_possible_truncation)] Ok(ExtendedBlockHeader { cumulative_difficulty, - version: HardFork::from_version(block.header.hardfork_version) + version: HardFork::from_version(block_header.hardfork_version) .expect("Stored block must have a valid hard-fork"), - vote: block.header.hardfork_signal, - timestamp: block.header.timestamp, + vote: block_header.hardfork_signal, + timestamp: block_header.timestamp, block_weight: block_info.weight as usize, long_term_weight: block_info.long_term_weight as usize, }) diff --git a/storage/blockchain/src/ops/mod.rs b/storage/blockchain/src/ops/mod.rs index 1ec9c237..8a8f0f15 100644 --- a/storage/blockchain/src/ops/mod.rs +++ b/storage/blockchain/src/ops/mod.rs @@ -102,12 +102,12 @@ //! # Ok(()) } //! ``` +pub mod alt_block; pub mod block; pub mod blockchain; pub mod key_image; pub mod output; pub mod property; pub mod tx; -pub mod alt_block; mod macros; diff --git a/storage/blockchain/src/service/read.rs b/storage/blockchain/src/service/read.rs index 207da416..eef40b5e 100644 --- a/storage/blockchain/src/service/read.rs +++ b/storage/blockchain/src/service/read.rs @@ -22,6 +22,7 @@ use cuprate_types::{ use crate::{ ops::{ + alt_block::{alt_block_hash, alt_extended_headers_in_range}, block::{ block_exists, get_block_extended_header_from_height, get_block_height, get_block_info, }, @@ -33,7 +34,7 @@ use crate::{ free::{compact_history_genesis_not_included, compact_history_index_to_height_offset}, types::{BlockchainReadHandle, ResponseResult}, }, - tables::{BlockHeights, BlockInfos, OpenTables, Tables}, + tables::{AltBlockHeights, BlockHeights, BlockInfos, OpenTables, Tables}, types::{Amount, AmountIndex, BlockHash, BlockHeight, KeyImage, PreRctOutputId}, }; @@ -87,7 +88,7 @@ fn map_request( match request { R::BlockExtendedHeader(block) => block_extended_header(env, block), R::BlockHash(block, chain) => block_hash(env, block, chain), - R::FindBlock(_) => todo!("Add alt blocks to DB"), + R::FindBlock(block_hash) => find_block(env, block_hash), R::FilterUnknownHashes(hashes) => filter_unknown_hashes(env, hashes), R::BlockExtendedHeaderInRange(range, chain) => { block_extended_header_in_range(env, range, chain) @@ -198,12 +199,39 @@ fn block_hash(env: &ConcreteEnv, block_height: BlockHeight, chain: Chain) -> Res let block_hash = match chain { Chain::Main => get_block_info(&block_height, &table_block_infos)?.block_hash, - Chain::Alt(_) => todo!("Add alt blocks to DB"), + Chain::Alt(chain) => { + alt_block_hash(&block_height, chain, &mut env_inner.open_tables(&tx_ro)?)? + } }; Ok(BlockchainResponse::BlockHash(block_hash)) } +/// [`BlockchainReadRequest::FindBlock`] +fn find_block(env: &ConcreteEnv, block_hash: BlockHash) -> ResponseResult { + // Single-threaded, no `ThreadLocal` required. + let env_inner = env.env_inner(); + let tx_ro = env_inner.tx_ro()?; + + let table_block_heights = env_inner.open_db_ro::(&tx_ro)?; + + // Check the main chain first. + match table_block_heights.get(&block_hash) { + Ok(height) => return Ok(BlockchainResponse::FindBlock(Some((Chain::Main, height)))), + Err(RuntimeError::KeyNotFound) => (), + Err(e) => return Err(e), + } + + let table_alt_block_heights = env_inner.open_db_ro::(&tx_ro)?; + + let height = table_alt_block_heights.get(&block_hash)?; + + Ok(BlockchainResponse::FindBlock(Some(( + Chain::Alt(height.chain_id.into()), + height.height, + )))) +} + /// [`BlockchainReadRequest::FilterUnknownHashes`]. #[inline] fn filter_unknown_hashes(env: &ConcreteEnv, mut hashes: HashSet) -> ResponseResult { @@ -254,7 +282,14 @@ fn block_extended_header_in_range( get_block_extended_header_from_height(&block_height, tables) }) .collect::, RuntimeError>>()?, - Chain::Alt(_) => todo!("Add alt blocks to DB"), + Chain::Alt(chain_id) => { + let tx_ro = tx_ro.get_or_try(|| env_inner.tx_ro())?; + alt_extended_headers_in_range( + range, + chain_id, + get_tables!(env_inner, tx_ro, tables)?.as_ref(), + )? + } }; Ok(BlockchainResponse::BlockExtendedHeaderInRange(vec)) From bc619b61eb06d54647dbba14530ebba6e110697f Mon Sep 17 00:00:00 2001 From: Boog900 <54e72d8a-345f-4599-bd90-c6b9bc7d0ec5@aleeas.com> Date: Sat, 31 Aug 2024 01:22:30 +0100 Subject: [PATCH 04/15] add new requests --- storage/blockchain/src/service/write.rs | 4 +++ types/src/blockchain.rs | 40 +++++++++++++++++++++---- 2 files changed, 39 insertions(+), 5 deletions(-) diff --git a/storage/blockchain/src/service/write.rs b/storage/blockchain/src/service/write.rs index 816afc4f..a2cc71c7 100644 --- a/storage/blockchain/src/service/write.rs +++ b/storage/blockchain/src/service/write.rs @@ -29,6 +29,10 @@ fn handle_blockchain_request( ) -> Result { match req { BlockchainWriteRequest::WriteBlock(block) => write_block(env, block), + BlockchainWriteRequest::WriteAltBlock(_) => todo!(), + BlockchainWriteRequest::StartReorg(_) => todo!(), + BlockchainWriteRequest::ReverseReorg(_) => todo!(), + BlockchainWriteRequest::FlushAltBlocks => todo!(), } } diff --git a/types/src/blockchain.rs b/types/src/blockchain.rs index b502c3fa..48eab292 100644 --- a/types/src/blockchain.rs +++ b/types/src/blockchain.rs @@ -8,7 +8,7 @@ use std::{ collections::{HashMap, HashSet}, ops::Range, }; - +use crate::{AltBlockInformation, ChainId}; use crate::types::{Chain, ExtendedBlockHeader, OutputOnChain, VerifiedBlockInformation}; //---------------------------------------------------------------------------------------------------- ReadRequest @@ -112,6 +112,27 @@ pub enum BlockchainWriteRequest { /// /// Input is an already verified block. WriteBlock(VerifiedBlockInformation), + /// Write an alternative block to the database, + /// + /// Input is the alternative block. + WriteAltBlock(AltBlockInformation), + /// A request to start the re-org process. + /// + /// The inner value is the [`ChainId`] of the alt-chain we want to re-org to. + /// + /// This will: + /// - pop blocks from the main chain + /// - retrieve all alt-blocks in this alt-chain + /// - flush all other alt blocks + StartReorg(ChainId), + /// A request to reverse the re-org process. + /// + /// The inner value is the [`ChainId`] of the old main chain. + /// + /// It is invalid to call this with a [`ChainId`] that was not returned from [`BlockchainWriteRequest::StartReorg`]. + ReverseReorg(ChainId), + /// A request to flush all alternative blocks. + FlushAltBlocks, } //---------------------------------------------------------------------------------------------------- Response @@ -198,11 +219,20 @@ pub enum BlockchainResponse { FindFirstUnknown(Option<(usize, usize)>), //------------------------------------------------------ Writes - /// Response to [`BlockchainWriteRequest::WriteBlock`]. + /// A generic Ok response to indicate a request was successfully handled. /// - /// This response indicates that the requested block has - /// successfully been written to the database without error. - WriteBlockOk, + /// currently the response for: + /// - [`BlockchainWriteRequest::WriteBlock`] + /// - [`BlockchainWriteRequest::ReverseReorg`] + /// - [`BlockchainWriteRequest::FlushAltBlocks`] + Ok, + /// The response for [`BlockchainWriteRequest::StartReorg`]. + StartReorg { + /// The [`ChainId`] of the old main chain blocks that were popped. + old_main_chain_id: ChainId, + /// The next alt chain blocks. + alt_chain: Vec + }, } //---------------------------------------------------------------------------------------------------- Tests From 029f439f0b5e48230c34fb7598a2f7d4a149f11d Mon Sep 17 00:00:00 2001 From: Boog900 <54e72d8a-345f-4599-bd90-c6b9bc7d0ec5@aleeas.com> Date: Sun, 1 Sep 2024 02:15:16 +0100 Subject: [PATCH 05/15] WIP: starting re-orgs --- .../ops/{alt_block.rs => alt_block/block.rs} | 130 ++++++------------ storage/blockchain/src/ops/alt_block/chain.rs | 63 +++++++++ storage/blockchain/src/ops/alt_block/mod.rs | 7 + storage/blockchain/src/ops/alt_block/tx.rs | 35 +++++ storage/blockchain/src/service/read.rs | 6 +- storage/blockchain/src/service/write.rs | 57 +++++++- storage/blockchain/src/tables.rs | 4 - types/src/blockchain.rs | 7 +- types/src/types.rs | 5 +- 9 files changed, 212 insertions(+), 102 deletions(-) rename storage/blockchain/src/ops/{alt_block.rs => alt_block/block.rs} (61%) create mode 100644 storage/blockchain/src/ops/alt_block/chain.rs create mode 100644 storage/blockchain/src/ops/alt_block/mod.rs create mode 100644 storage/blockchain/src/ops/alt_block/tx.rs diff --git a/storage/blockchain/src/ops/alt_block.rs b/storage/blockchain/src/ops/alt_block/block.rs similarity index 61% rename from storage/blockchain/src/ops/alt_block.rs rename to storage/blockchain/src/ops/alt_block/block.rs index 82a37ba8..171cdd7b 100644 --- a/storage/blockchain/src/ops/alt_block.rs +++ b/storage/blockchain/src/ops/alt_block/block.rs @@ -1,22 +1,20 @@ -use std::cmp::max; - +use crate::ops::alt_block::{ + add_alt_transaction_blob, check_add_alt_chain_info, get_alt_chain_history_ranges, + get_alt_transaction_blob, +}; +use crate::ops::block::{get_block_extended_header_from_height, get_block_info}; +use crate::tables::{Tables, TablesMut}; +use crate::types::{ + AltBlockHeight, AltTransactionInfo, BlockHash, BlockHeight, CompactAltBlockInfo, +}; use bytemuck::TransparentWrapper; -use cuprate_database::{DatabaseRo, DatabaseRw, RuntimeError, StorableVec}; +use cuprate_database::{RuntimeError, StorableVec}; use cuprate_helper::map::{combine_low_high_bits_to_u128, split_u128_into_low_high_bits}; use cuprate_types::{ AltBlockInformation, Chain, ChainId, ExtendedBlockHeader, HardFork, VerifiedTransactionInformation, }; -use monero_serai::block::BlockHeader; - -use crate::{ - ops::block::{get_block_extended_header_from_height, get_block_info}, - tables::{Tables, TablesMut}, - types::{ - AltBlockHeight, AltChainInfo, AltTransactionInfo, BlockHash, BlockHeight, - CompactAltBlockInfo, - }, -}; +use monero_serai::block::{Block, BlockHeader}; pub fn add_alt_block( alt_block: &AltBlockInformation, @@ -55,64 +53,48 @@ pub fn add_alt_block( StorableVec::wrap_ref(&alt_block.block_blob), )?; - for tx in &alt_block.txs { - add_alt_transaction(&tx, tables)?; + assert_eq!(alt_block.txs.len(), alt_block.block.transactions.len()); + for (tx, tx_hash) in alt_block.txs.iter().zip(&alt_block.block.transactions) { + add_alt_transaction_blob(tx_hash, StorableVec::wrap_ref(tx), tables)?; } Ok(()) } -pub fn add_alt_transaction( - tx: &VerifiedTransactionInformation, - tables: &mut impl TablesMut, -) -> Result<(), RuntimeError> { - if tables.tx_ids().get(&tx.tx_hash).is_ok() - || tables.alt_transaction_infos().get(&tx.tx_hash).is_ok() - { - return Ok(()); - } - - tables.alt_transaction_infos_mut().put( - &tx.tx_hash, - &AltTransactionInfo { - tx_weight: tx.tx_weight, - fee: tx.fee, - tx_hash: tx.tx_hash, - }, - )?; - - tables - .alt_transaction_blobs_mut() - .put(&tx.tx_hash, StorableVec::wrap_ref(&tx.tx_blob)) -} - -pub fn check_add_alt_chain_info( +pub fn get_alt_block( alt_block_height: &AltBlockHeight, - prev_hash: &BlockHash, - tables: &mut impl TablesMut, -) -> Result<(), RuntimeError> { - match tables.alt_chain_infos().get(&alt_block_height.chain_id) { - Ok(_) => return Ok(()), - Err(RuntimeError::KeyNotFound) => (), - Err(e) => return Err(e), - } + tables: &impl Tables, +) -> Result { + let block_info = tables.alt_blocks_info().get(alt_block_height)?; - let parent_chain = match tables.alt_block_heights().get(prev_hash) { - Ok(alt_parent_height) => Chain::Alt(alt_parent_height.chain_id.into()), - Err(RuntimeError::KeyNotFound) => Chain::Main, - Err(e) => return Err(e), - }; + let block_blob = tables.alt_block_blobs().get(alt_block_height)?.0; - tables.alt_chain_infos_mut().put( - &alt_block_height.chain_id, - &AltChainInfo { - parent_chain: parent_chain.into(), - common_ancestor_height: alt_block_height.height - 1, - }, - ) + let block = Block::read(&mut block_blob.as_slice())?; + + let txs = block + .transactions + .iter() + .map(|tx_hash| get_alt_transaction_blob(tx_hash, tables)) + .collect()?; + + Ok(AltBlockInformation { + block, + block_blob, + txs, + block_hash: block_info.block_hash, + pow_hash: block_info.pow_hash, + height: block_info.height, + weight: block_info.weight, + long_term_weight: block_info.long_term_weight, + cumulative_difficulty: combine_low_high_bits_to_u128( + block_info.cumulative_difficulty_low, + block_info.cumulative_difficulty_high, + ), + chain_id: alt_block_height.chain_id.into(), + }) } -pub fn alt_block_hash( +pub fn get_alt_block_hash( block_height: &BlockHeight, alt_chain: ChainId, tables: &mut impl Tables, @@ -152,37 +134,15 @@ pub fn alt_block_hash( } } -pub fn alt_extended_headers_in_range( +pub fn get_alt_extended_headers_in_range( range: std::ops::Range, alt_chain: ChainId, tables: &impl Tables, ) -> Result, RuntimeError> { // TODO: this function does not use rayon, however it probably should. - let mut ranges = Vec::with_capacity(5); let alt_chains = tables.alt_chain_infos(); - - let mut i = range.end; - let mut current_chain_id = alt_chain.into(); - while i > range.start { - let chain_info = alt_chains.get(¤t_chain_id)?; - - let start_height = max(range.start, chain_info.common_ancestor_height + 1); - - ranges.push((chain_info.parent_chain.into(), start_height..i)); - i = chain_info.common_ancestor_height; - - match chain_info.parent_chain.into() { - Chain::Main => { - ranges.push((Chain::Main, range.start..i)); - break; - } - Chain::Alt(alt_chain_id) => { - current_chain_id = alt_chain_id.into(); - continue; - } - } - } + let ranges = get_alt_chain_history_ranges(range, alt_chain, alt_chains)?; let res = ranges .into_iter() diff --git a/storage/blockchain/src/ops/alt_block/chain.rs b/storage/blockchain/src/ops/alt_block/chain.rs new file mode 100644 index 00000000..4259d4dc --- /dev/null +++ b/storage/blockchain/src/ops/alt_block/chain.rs @@ -0,0 +1,63 @@ +use crate::tables::{AltChainInfos, TablesMut}; +use crate::types::{AltBlockHeight, AltChainInfo, BlockHash, BlockHeight}; +use cuprate_database::{DatabaseRo, RuntimeError}; +use cuprate_types::{Chain, ChainId}; +use std::cmp::max; + +pub fn check_add_alt_chain_info( + alt_block_height: &AltBlockHeight, + prev_hash: &BlockHash, + tables: &mut impl TablesMut, +) -> Result<(), RuntimeError> { + match tables.alt_chain_infos().get(&alt_block_height.chain_id) { + Ok(_) => return Ok(()), + Err(RuntimeError::KeyNotFound) => (), + Err(e) => return Err(e), + } + + let parent_chain = match tables.alt_block_heights().get(prev_hash) { + Ok(alt_parent_height) => Chain::Alt(alt_parent_height.chain_id.into()), + Err(RuntimeError::KeyNotFound) => Chain::Main, + Err(e) => return Err(e), + }; + + tables.alt_chain_infos_mut().put( + &alt_block_height.chain_id, + &AltChainInfo { + parent_chain: parent_chain.into(), + common_ancestor_height: alt_block_height.height - 1, + }, + ) +} + +pub fn get_alt_chain_history_ranges( + range: std::ops::Range, + alt_chain: ChainId, + alt_chain_infos: &impl DatabaseRo, +) -> Result)>, RuntimeError> { + let mut ranges = Vec::with_capacity(5); + + let mut i = range.end; + let mut current_chain_id = alt_chain.into(); + while i > range.start { + let chain_info = alt_chain_infos.get(¤t_chain_id)?; + + let start_height = max(range.start, chain_info.common_ancestor_height + 1); + + ranges.push((chain_info.parent_chain.into(), start_height..i)); + i = chain_info.common_ancestor_height; + + match chain_info.parent_chain.into() { + Chain::Main => { + ranges.push((Chain::Main, range.start..i)); + break; + } + Chain::Alt(alt_chain_id) => { + current_chain_id = alt_chain_id.into(); + continue; + } + } + } + + Ok(ranges) +} diff --git a/storage/blockchain/src/ops/alt_block/mod.rs b/storage/blockchain/src/ops/alt_block/mod.rs new file mode 100644 index 00000000..8b2d1f17 --- /dev/null +++ b/storage/blockchain/src/ops/alt_block/mod.rs @@ -0,0 +1,7 @@ +mod block; +mod chain; +mod tx; + +pub use block::*; +pub use chain::*; +pub use tx::*; diff --git a/storage/blockchain/src/ops/alt_block/tx.rs b/storage/blockchain/src/ops/alt_block/tx.rs new file mode 100644 index 00000000..aad4dc3d --- /dev/null +++ b/storage/blockchain/src/ops/alt_block/tx.rs @@ -0,0 +1,35 @@ +use crate::tables::{Tables, TablesMut}; +use crate::types::{AltTransactionInfo, TxHash}; +use bytemuck::TransparentWrapper; +use cuprate_database::{RuntimeError, StorableVec}; +use cuprate_types::VerifiedTransactionInformation; + +pub fn add_alt_transaction_blob( + tx_hash: &TxHash, + tx_block: &StorableVec, + tables: &mut impl TablesMut, +) -> Result<(), RuntimeError> { + if tables.tx_ids().get(&tx_hash).is_ok() || tables.alt_transaction_blobs().get(&tx_hash).is_ok() + { + return Ok(()); + } + + tables.alt_transaction_blobs_mut().put(&tx_hash, tx_block) +} + +pub fn get_alt_transaction_blob( + tx_hash: &TxHash, + tables: &impl Tables, +) -> Result, RuntimeError> { + match tables.alt_transaction_blobs().get(tx_hash) { + Ok(blob) => Ok(blob.0), + Err(RuntimeError::KeyNotFound) => { + let tx_id = tables.tx_ids().get(tx_hash)?; + + let blob = tables.tx_blobs().get(&tx_id)?; + + Ok(blob.0) + } + Err(e) => return Err(e), + } +} diff --git a/storage/blockchain/src/service/read.rs b/storage/blockchain/src/service/read.rs index eef40b5e..70da01b3 100644 --- a/storage/blockchain/src/service/read.rs +++ b/storage/blockchain/src/service/read.rs @@ -22,7 +22,7 @@ use cuprate_types::{ use crate::{ ops::{ - alt_block::{alt_block_hash, alt_extended_headers_in_range}, + alt_block::{get_alt_block_hash, get_alt_extended_headers_in_range}, block::{ block_exists, get_block_extended_header_from_height, get_block_height, get_block_info, }, @@ -200,7 +200,7 @@ fn block_hash(env: &ConcreteEnv, block_height: BlockHeight, chain: Chain) -> Res let block_hash = match chain { Chain::Main => get_block_info(&block_height, &table_block_infos)?.block_hash, Chain::Alt(chain) => { - alt_block_hash(&block_height, chain, &mut env_inner.open_tables(&tx_ro)?)? + get_alt_block_hash(&block_height, chain, &mut env_inner.open_tables(&tx_ro)?)? } }; @@ -284,7 +284,7 @@ fn block_extended_header_in_range( .collect::, RuntimeError>>()?, Chain::Alt(chain_id) => { let tx_ro = tx_ro.get_or_try(|| env_inner.tx_ro())?; - alt_extended_headers_in_range( + get_alt_extended_headers_in_range( range, chain_id, get_tables!(env_inner, tx_ro, tables)?.as_ref(), diff --git a/storage/blockchain/src/service/write.rs b/storage/blockchain/src/service/write.rs index a2cc71c7..067ba7f1 100644 --- a/storage/blockchain/src/service/write.rs +++ b/storage/blockchain/src/service/write.rs @@ -7,7 +7,7 @@ use cuprate_database::{ConcreteEnv, Env, EnvInner, RuntimeError, TxRw}; use cuprate_database_service::DatabaseWriteHandle; use cuprate_types::{ blockchain::{BlockchainResponse, BlockchainWriteRequest}, - VerifiedBlockInformation, + AltBlockInformation, VerifiedBlockInformation, }; use crate::{ @@ -29,10 +29,10 @@ fn handle_blockchain_request( ) -> Result { match req { BlockchainWriteRequest::WriteBlock(block) => write_block(env, block), - BlockchainWriteRequest::WriteAltBlock(_) => todo!(), + BlockchainWriteRequest::WriteAltBlock(alt_block) => write_alt_block(env, alt_block), BlockchainWriteRequest::StartReorg(_) => todo!(), BlockchainWriteRequest::ReverseReorg(_) => todo!(), - BlockchainWriteRequest::FlushAltBlocks => todo!(), + BlockchainWriteRequest::FlushAltBlocks => flush_alt_blocks(env), } } @@ -59,7 +59,56 @@ fn write_block(env: &ConcreteEnv, block: &VerifiedBlockInformation) -> ResponseR match result { Ok(()) => { TxRw::commit(tx_rw)?; - Ok(BlockchainResponse::WriteBlockOk) + Ok(BlockchainResponse::Ok) + } + Err(e) => { + // INVARIANT: ensure database atomicity by aborting + // the transaction on `add_block()` failures. + TxRw::abort(tx_rw) + .expect("could not maintain database atomicity by aborting write transaction"); + Err(e) + } + } +} + +/// [`BlockchainWriteRequest::WriteAltBlock`]. +#[inline] +fn write_alt_block(env: &ConcreteEnv, block: &AltBlockInformation) -> ResponseResult { + let env_inner = env.env_inner(); + let tx_rw = env_inner.tx_rw()?; + + let result = { + let mut tables_mut = env_inner.open_tables_mut(&tx_rw)?; + crate::ops::alt_block::add_alt_block(block, &mut tables_mut) + }; + + match result { + Ok(()) => { + TxRw::commit(tx_rw)?; + Ok(BlockchainResponse::Ok) + } + Err(e) => { + // INVARIANT: ensure database atomicity by aborting + // the transaction on `add_block()` failures. + TxRw::abort(tx_rw) + .expect("could not maintain database atomicity by aborting write transaction"); + Err(e) + } + } +} + +/// [`BlockchainWriteRequest::FlushAltBlocks`]. +#[inline] +fn flush_alt_blocks(env: &ConcreteEnv) -> ResponseResult { + let env_inner = env.env_inner(); + let mut tx_rw = env_inner.tx_rw()?; + + let result = { crate::ops::alt_block::flush_alt_blocks(&env_inner, &mut tx_rw) }; + + match result { + Ok(()) => { + TxRw::commit(tx_rw)?; + Ok(BlockchainResponse::Ok) } Err(e) => { // INVARIANT: ensure database atomicity by aborting diff --git a/storage/blockchain/src/tables.rs b/storage/blockchain/src/tables.rs index 381430d7..deb957ea 100644 --- a/storage/blockchain/src/tables.rs +++ b/storage/blockchain/src/tables.rs @@ -145,10 +145,6 @@ cuprate_database::define_tables! { 19 => AltTransactionBlobs, TxHash => TxBlob, - - 20 => AltTransactionInfos, - TxHash => AltTransactionInfo, - } //---------------------------------------------------------------------------------------------------- Tests diff --git a/types/src/blockchain.rs b/types/src/blockchain.rs index 48eab292..33c3e8bd 100644 --- a/types/src/blockchain.rs +++ b/types/src/blockchain.rs @@ -4,12 +4,12 @@ //! responses are also tested in Cuprate's blockchain database crate. //---------------------------------------------------------------------------------------------------- Import +use crate::types::{Chain, ExtendedBlockHeader, OutputOnChain, VerifiedBlockInformation}; +use crate::{AltBlockInformation, ChainId}; use std::{ collections::{HashMap, HashSet}, ops::Range, }; -use crate::{AltBlockInformation, ChainId}; -use crate::types::{Chain, ExtendedBlockHeader, OutputOnChain, VerifiedBlockInformation}; //---------------------------------------------------------------------------------------------------- ReadRequest /// A read request to the blockchain database. @@ -223,6 +223,7 @@ pub enum BlockchainResponse { /// /// currently the response for: /// - [`BlockchainWriteRequest::WriteBlock`] + /// - [`BlockchainWriteRequest::WriteAltBlock`] /// - [`BlockchainWriteRequest::ReverseReorg`] /// - [`BlockchainWriteRequest::FlushAltBlocks`] Ok, @@ -231,7 +232,7 @@ pub enum BlockchainResponse { /// The [`ChainId`] of the old main chain blocks that were popped. old_main_chain_id: ChainId, /// The next alt chain blocks. - alt_chain: Vec + alt_chain: Vec, }, } diff --git a/types/src/types.rs b/types/src/types.rs index da4422a1..cc4543e6 100644 --- a/types/src/types.rs +++ b/types/src/types.rs @@ -39,8 +39,7 @@ pub struct ExtendedBlockHeader { //---------------------------------------------------------------------------------------------------- VerifiedTransactionInformation /// Verified information of a transaction. /// -/// - If this is in a [`VerifiedBlockInformation`] this represents a valid transaction -/// - If this is in an [`AltBlockInformation`] this represents a potentially valid transaction +/// This represents a valid transaction #[derive(Clone, Debug, PartialEq, Eq)] pub struct VerifiedTransactionInformation { /// The transaction itself. @@ -121,7 +120,7 @@ pub struct AltBlockInformation { /// [`Block::serialize`]. pub block_blob: Vec, /// All the transactions in the block, excluding the [`Block::miner_transaction`]. - pub txs: Vec, + pub txs: Vec>, /// The block's hash. /// /// [`Block::hash`]. From 6927b05f81c0de3a82a406d11ff365bb3a4aa826 Mon Sep 17 00:00:00 2001 From: Boog900 <54e72d8a-345f-4599-bd90-c6b9bc7d0ec5@aleeas.com> Date: Fri, 6 Sep 2024 00:23:55 +0100 Subject: [PATCH 06/15] add last service request --- Cargo.lock | 1 + storage/blockchain/Cargo.toml | 3 +- storage/blockchain/src/free.rs | 32 +++++ storage/blockchain/src/ops/alt_block/block.rs | 21 ++-- storage/blockchain/src/ops/alt_block/chain.rs | 3 +- storage/blockchain/src/ops/alt_block/mod.rs | 17 +++ storage/blockchain/src/ops/alt_block/tx.rs | 44 +++++-- storage/blockchain/src/ops/block.rs | 61 ++++++--- storage/blockchain/src/service/free.rs | 37 +++++- storage/blockchain/src/service/write.rs | 118 +++++++++++++++++- storage/blockchain/src/tables.rs | 3 + storage/blockchain/src/types.rs | 5 +- types/src/blockchain.rs | 24 ++-- types/src/types.rs | 3 +- 14 files changed, 306 insertions(+), 66 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 77531897..d5d64902 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -531,6 +531,7 @@ dependencies = [ "monero-serai", "pretty_assertions", "proptest", + "rand", "rayon", "serde", "tempfile", diff --git a/storage/blockchain/Cargo.toml b/storage/blockchain/Cargo.toml index 7e79305a..b03ef038 100644 --- a/storage/blockchain/Cargo.toml +++ b/storage/blockchain/Cargo.toml @@ -25,11 +25,12 @@ cuprate-database = { path = "../database" } cuprate-database-service = { path = "../service" } cuprate-helper = { path = "../../helper", features = ["fs", "thread", "map"] } cuprate-types = { path = "../../types", features = ["blockchain"] } +cuprate-pruning = { path = "../../pruning" } bitflags = { workspace = true, features = ["std", "serde", "bytemuck"] } bytemuck = { workspace = true, features = ["must_cast", "derive", "min_const_generics", "extern_crate_alloc"] } curve25519-dalek = { workspace = true } -cuprate-pruning = { path = "../../pruning" } +rand = { workspace = true } monero-serai = { workspace = true, features = ["std"] } serde = { workspace = true, optional = true } diff --git a/storage/blockchain/src/free.rs b/storage/blockchain/src/free.rs index 8288e65f..20d56226 100644 --- a/storage/blockchain/src/free.rs +++ b/storage/blockchain/src/free.rs @@ -1,5 +1,6 @@ //! General free functions (related to the database). +use monero_serai::transaction::{Input, Transaction}; //---------------------------------------------------------------------------------------------------- Import use cuprate_database::{ConcreteEnv, Env, EnvInner, InitError, RuntimeError, TxRw}; @@ -61,6 +62,37 @@ pub fn open(config: Config) -> Result { Ok(env) } +//---------------------------------------------------------------------------------------------------- Tx Fee +/// Calculates the fee of the [`Transaction`]. +/// +/// # Panics +/// This will panic if the inputs overflow or the transaction outputs too much. +pub(crate) fn tx_fee(tx: &Transaction) -> u64 { + let mut fee = 0_u64; + + match &tx { + Transaction::V1 { prefix, .. } => { + for input in &prefix.inputs { + match input { + Input::Gen(_) => return 0, + Input::ToKey { amount, .. } => { + fee = fee.checked_add(amount.unwrap_or(0)).unwrap(); + } + } + } + + for output in &prefix.outputs { + fee.checked_sub(output.amount.unwrap_or(0)).unwrap(); + } + } + Transaction::V2 { proofs, .. } => { + fee = proofs.as_ref().unwrap().base.fee; + } + }; + + fee +} + //---------------------------------------------------------------------------------------------------- Tests #[cfg(test)] mod test { diff --git a/storage/blockchain/src/ops/alt_block/block.rs b/storage/blockchain/src/ops/alt_block/block.rs index 171cdd7b..a429eab8 100644 --- a/storage/blockchain/src/ops/alt_block/block.rs +++ b/storage/blockchain/src/ops/alt_block/block.rs @@ -1,19 +1,14 @@ use crate::ops::alt_block::{ add_alt_transaction_blob, check_add_alt_chain_info, get_alt_chain_history_ranges, - get_alt_transaction_blob, + get_alt_transaction, }; use crate::ops::block::{get_block_extended_header_from_height, get_block_info}; use crate::tables::{Tables, TablesMut}; -use crate::types::{ - AltBlockHeight, AltTransactionInfo, BlockHash, BlockHeight, CompactAltBlockInfo, -}; +use crate::types::{AltBlockHeight, BlockHash, BlockHeight, CompactAltBlockInfo}; use bytemuck::TransparentWrapper; -use cuprate_database::{RuntimeError, StorableVec}; +use cuprate_database::{DatabaseRo, DatabaseRw, RuntimeError, StorableVec}; use cuprate_helper::map::{combine_low_high_bits_to_u128, split_u128_into_low_high_bits}; -use cuprate_types::{ - AltBlockInformation, Chain, ChainId, ExtendedBlockHeader, HardFork, - VerifiedTransactionInformation, -}; +use cuprate_types::{AltBlockInformation, Chain, ChainId, ExtendedBlockHeader, HardFork}; use monero_serai::block::{Block, BlockHeader}; pub fn add_alt_block( @@ -54,8 +49,8 @@ pub fn add_alt_block( )?; assert_eq!(alt_block.txs.len(), alt_block.block.transactions.len()); - for (tx, tx_hash) in alt_block.txs.iter().zip(&alt_block.block.transactions) { - add_alt_transaction_blob(tx_hash, StorableVec::wrap_ref(tx), tables)?; + for tx in alt_block.txs.iter() { + add_alt_transaction_blob(tx, tables)?; } Ok(()) @@ -74,8 +69,8 @@ pub fn get_alt_block( let txs = block .transactions .iter() - .map(|tx_hash| get_alt_transaction_blob(tx_hash, tables)) - .collect()?; + .map(|tx_hash| get_alt_transaction(tx_hash, tables)) + .collect::>()?; Ok(AltBlockInformation { block, diff --git a/storage/blockchain/src/ops/alt_block/chain.rs b/storage/blockchain/src/ops/alt_block/chain.rs index 4259d4dc..1162a9cd 100644 --- a/storage/blockchain/src/ops/alt_block/chain.rs +++ b/storage/blockchain/src/ops/alt_block/chain.rs @@ -1,6 +1,6 @@ use crate::tables::{AltChainInfos, TablesMut}; use crate::types::{AltBlockHeight, AltChainInfo, BlockHash, BlockHeight}; -use cuprate_database::{DatabaseRo, RuntimeError}; +use cuprate_database::{DatabaseRo, DatabaseRw, RuntimeError}; use cuprate_types::{Chain, ChainId}; use std::cmp::max; @@ -26,6 +26,7 @@ pub fn check_add_alt_chain_info( &AltChainInfo { parent_chain: parent_chain.into(), common_ancestor_height: alt_block_height.height - 1, + chain_height: alt_block_height.height, }, ) } diff --git a/storage/blockchain/src/ops/alt_block/mod.rs b/storage/blockchain/src/ops/alt_block/mod.rs index 8b2d1f17..72e0933e 100644 --- a/storage/blockchain/src/ops/alt_block/mod.rs +++ b/storage/blockchain/src/ops/alt_block/mod.rs @@ -5,3 +5,20 @@ mod tx; pub use block::*; pub use chain::*; pub use tx::*; + +pub fn flush_alt_blocks<'a, E: cuprate_database::EnvInner<'a>>( + env_inner: &E, + tx_rw: &mut E::Rw<'_>, +) -> Result<(), cuprate_database::RuntimeError> { + use crate::tables::{ + AltBlockBlobs, AltBlockHeights, AltBlocksInfo, AltChainInfos, AltTransactionBlobs, + AltTransactionInfos, + }; + + env_inner.clear_db::(tx_rw)?; + env_inner.clear_db::(tx_rw)?; + env_inner.clear_db::(tx_rw)?; + env_inner.clear_db::(tx_rw)?; + env_inner.clear_db::(tx_rw)?; + env_inner.clear_db::(tx_rw) +} diff --git a/storage/blockchain/src/ops/alt_block/tx.rs b/storage/blockchain/src/ops/alt_block/tx.rs index aad4dc3d..a49c72ae 100644 --- a/storage/blockchain/src/ops/alt_block/tx.rs +++ b/storage/blockchain/src/ops/alt_block/tx.rs @@ -1,35 +1,57 @@ use crate::tables::{Tables, TablesMut}; use crate::types::{AltTransactionInfo, TxHash}; use bytemuck::TransparentWrapper; -use cuprate_database::{RuntimeError, StorableVec}; +use cuprate_database::{DatabaseRo, DatabaseRw, RuntimeError, StorableVec}; use cuprate_types::VerifiedTransactionInformation; +use monero_serai::transaction::Transaction; pub fn add_alt_transaction_blob( - tx_hash: &TxHash, - tx_block: &StorableVec, + tx: &VerifiedTransactionInformation, tables: &mut impl TablesMut, ) -> Result<(), RuntimeError> { - if tables.tx_ids().get(&tx_hash).is_ok() || tables.alt_transaction_blobs().get(&tx_hash).is_ok() + tables.alt_transaction_infos_mut().put( + &tx.tx_hash, + &AltTransactionInfo { + tx_weight: tx.tx_weight, + fee: tx.fee, + tx_hash: tx.tx_hash, + }, + )?; + + if tables.tx_ids().get(&tx.tx_hash).is_ok() + || tables.alt_transaction_blobs().get(&tx.tx_hash).is_ok() { return Ok(()); } - tables.alt_transaction_blobs_mut().put(&tx_hash, tx_block) + tables + .alt_transaction_blobs_mut() + .put(&tx.tx_hash, StorableVec::wrap_ref(&tx.tx_blob)) } -pub fn get_alt_transaction_blob( +pub fn get_alt_transaction( tx_hash: &TxHash, tables: &impl Tables, -) -> Result, RuntimeError> { - match tables.alt_transaction_blobs().get(tx_hash) { - Ok(blob) => Ok(blob.0), +) -> Result { + let tx_info = tables.alt_transaction_infos().get(tx_hash)?; + + let tx_blob = match tables.alt_transaction_blobs().get(tx_hash) { + Ok(blob) => blob.0, Err(RuntimeError::KeyNotFound) => { let tx_id = tables.tx_ids().get(tx_hash)?; let blob = tables.tx_blobs().get(&tx_id)?; - Ok(blob.0) + blob.0 } Err(e) => return Err(e), - } + }; + + Ok(VerifiedTransactionInformation { + tx: Transaction::read(&mut tx_blob.as_slice()).unwrap(), + tx_blob, + tx_weight: tx_info.tx_weight, + fee: tx_info.fee, + tx_hash: tx_info.tx_hash, + }) } diff --git a/storage/blockchain/src/ops/block.rs b/storage/blockchain/src/ops/block.rs index 2e110fed..5cb3b4bc 100644 --- a/storage/blockchain/src/ops/block.rs +++ b/storage/blockchain/src/ops/block.rs @@ -8,8 +8,13 @@ use cuprate_database::{ RuntimeError, StorableVec, {DatabaseRo, DatabaseRw}, }; use cuprate_helper::map::{combine_low_high_bits_to_u128, split_u128_into_low_high_bits}; -use cuprate_types::{ExtendedBlockHeader, HardFork, VerifiedBlockInformation}; +use cuprate_types::{ + AltBlockInformation, ChainId, ExtendedBlockHeader, HardFork, VerifiedBlockInformation, + VerifiedTransactionInformation, +}; +use crate::free::tx_fee; +use crate::ops::alt_block; use crate::{ ops::{ blockchain::{chain_height, cumulative_generated_coins}, @@ -106,9 +111,8 @@ pub fn add_block( cumulative_rct_outs, timestamp: block.block.header.timestamp, block_hash: block.block_hash, - // INVARIANT: #[cfg] @ lib.rs asserts `usize == u64` - weight: block.weight as u64, - long_term_weight: block.long_term_weight as u64, + weight: block.weight, + long_term_weight: block.long_term_weight, }, )?; @@ -135,17 +139,15 @@ pub fn add_block( /// will be returned if there are no blocks left. // no inline, too big pub fn pop_block( + move_to_alt_chain: Option, tables: &mut impl TablesMut, ) -> Result<(BlockHeight, BlockHash, Block), RuntimeError> { //------------------------------------------------------ Block Info // Remove block data from tables. - let (block_height, block_hash) = { - let (block_height, block_info) = tables.block_infos_mut().pop_last()?; - (block_height, block_info.block_hash) - }; + let (block_height, block_info) = tables.block_infos_mut().pop_last()?; // Block heights. - tables.block_heights_mut().delete(&block_hash)?; + tables.block_heights_mut().delete(&block_info.block_hash)?; // Block blobs. // We deserialize the block blob into a `Block`, such @@ -154,12 +156,42 @@ pub fn pop_block( let block = Block::read(&mut block_blob.as_slice())?; //------------------------------------------------------ Transaction / Outputs / Key Images + let mut txs = Vec::with_capacity(block.transactions.len()); + remove_tx(&block.miner_transaction.hash(), tables)?; for tx_hash in &block.transactions { - remove_tx(tx_hash, tables)?; + let (_, tx) = remove_tx(tx_hash, tables)?; + + if move_to_alt_chain.is_some() { + txs.push(VerifiedTransactionInformation { + tx_weight: tx.weight(), + tx_blob: tx.serialize(), + tx_hash: tx.hash(), + fee: tx_fee(&tx), + tx, + }) + } } - Ok((block_height, block_hash, block)) + if let Some(chain_id) = move_to_alt_chain { + alt_block::add_alt_block( + &AltBlockInformation { + block: block.clone(), + block_blob, + txs, + block_hash: block_info.block_hash, + pow_hash: [255; 32], + height: block_height, + weight: block_info.weight, + long_term_weight: block_info.long_term_weight, + cumulative_difficulty: 0, + chain_id, + }, + tables, + )?; + } + + Ok((block_height, block_info.block_hash, block)) } //---------------------------------------------------------------------------------------------------- `get_block_extended_header_*` @@ -205,8 +237,8 @@ pub fn get_block_extended_header_from_height( .expect("Stored block must have a valid hard-fork"), vote: block_header.hardfork_signal, timestamp: block_header.timestamp, - block_weight: block_info.weight as usize, - long_term_weight: block_info.long_term_weight as usize, + block_weight: block_info.weight, + long_term_weight: block_info.long_term_weight, }) } @@ -412,7 +444,8 @@ mod test { for block_hash in block_hashes.into_iter().rev() { println!("pop_block(): block_hash: {}", hex::encode(block_hash)); - let (_popped_height, popped_hash, _popped_block) = pop_block(&mut tables).unwrap(); + let (_popped_height, popped_hash, _popped_block) = + pop_block(None, &mut tables).unwrap(); assert_eq!(block_hash, popped_hash); diff --git a/storage/blockchain/src/service/free.rs b/storage/blockchain/src/service/free.rs index e748bbbe..aa8238f9 100644 --- a/storage/blockchain/src/service/free.rs +++ b/storage/blockchain/src/service/free.rs @@ -3,13 +3,13 @@ //---------------------------------------------------------------------------------------------------- Import use std::sync::Arc; -use cuprate_database::{ConcreteEnv, InitError}; - use crate::service::{init_read_service, init_write_service}; use crate::{ config::Config, service::types::{BlockchainReadHandle, BlockchainWriteHandle}, }; +use cuprate_database::{ConcreteEnv, InitError}; +use cuprate_types::{AltBlockInformation, VerifiedBlockInformation}; //---------------------------------------------------------------------------------------------------- Init #[cold] @@ -81,6 +81,39 @@ pub(super) const fn compact_history_genesis_not_included INITIAL_BLOCKS && !(top_block_height - INITIAL_BLOCKS + 2).is_power_of_two() } +//---------------------------------------------------------------------------------------------------- Compact history +pub(super) fn map_valid_alt_block_to_verified_block( + alt_block: AltBlockInformation, +) -> VerifiedBlockInformation { + let total_fees = alt_block.txs.iter().map(|tx| tx.fee).sum::(); + let total_miner_output = alt_block + .block + .miner_transaction + .prefix() + .outputs + .iter() + .map(|out| out.amount.unwrap_or(0)) + .sum::(); + + VerifiedBlockInformation { + block: alt_block.block, + block_blob: alt_block.block_blob, + txs: alt_block + .txs + .into_iter() + .map(TryInto::try_into) + .collect::>() + .unwrap(), + block_hash: alt_block.block_hash, + pow_hash: alt_block.pow_hash, + height: alt_block.height, + generated_coins: total_miner_output - total_fees, + weight: alt_block.weight, + long_term_weight: alt_block.long_term_weight, + cumulative_difficulty: alt_block.cumulative_difficulty, + } +} + //---------------------------------------------------------------------------------------------------- Tests #[cfg(test)] diff --git a/storage/blockchain/src/service/write.rs b/storage/blockchain/src/service/write.rs index 067ba7f1..95124d41 100644 --- a/storage/blockchain/src/service/write.rs +++ b/storage/blockchain/src/service/write.rs @@ -1,18 +1,20 @@ //! Database writer thread definitions and logic. - //---------------------------------------------------------------------------------------------------- Import use std::sync::Arc; -use cuprate_database::{ConcreteEnv, Env, EnvInner, RuntimeError, TxRw}; +use cuprate_database::{ConcreteEnv, DatabaseRo, DatabaseRw, Env, EnvInner, RuntimeError, TxRw}; use cuprate_database_service::DatabaseWriteHandle; use cuprate_types::{ blockchain::{BlockchainResponse, BlockchainWriteRequest}, - AltBlockInformation, VerifiedBlockInformation, + AltBlockInformation, Chain, ChainId, VerifiedBlockInformation, }; +use crate::service::free::map_valid_alt_block_to_verified_block; +use crate::types::AltBlockHeight; use crate::{ service::types::{BlockchainWriteHandle, ResponseResult}, - tables::OpenTables, + tables::{OpenTables, Tables, TablesMut}, + types::AltChainInfo, }; //---------------------------------------------------------------------------------------------------- init_write_service @@ -30,8 +32,10 @@ fn handle_blockchain_request( match req { BlockchainWriteRequest::WriteBlock(block) => write_block(env, block), BlockchainWriteRequest::WriteAltBlock(alt_block) => write_alt_block(env, alt_block), - BlockchainWriteRequest::StartReorg(_) => todo!(), - BlockchainWriteRequest::ReverseReorg(_) => todo!(), + BlockchainWriteRequest::PopBlocks(numb_blocks) => pop_blocks(env, *numb_blocks), + BlockchainWriteRequest::ReverseReorg(old_main_chain_id) => { + reverse_reorg(env, *old_main_chain_id) + } BlockchainWriteRequest::FlushAltBlocks => flush_alt_blocks(env), } } @@ -97,6 +101,108 @@ fn write_alt_block(env: &ConcreteEnv, block: &AltBlockInformation) -> ResponseRe } } +/// [`BlockchainWriteRequest::PopBlocks`]. +fn pop_blocks(env: &ConcreteEnv, numb_blocks: usize) -> ResponseResult { + let env_inner = env.env_inner(); + let mut tx_rw = env_inner.tx_rw()?; + + let result = { + crate::ops::alt_block::flush_alt_blocks(&env_inner, &mut tx_rw)?; + + let mut tables_mut = env_inner.open_tables_mut(&tx_rw)?; + + let old_main_chain_id = ChainId(rand::random()); + + let mut last_block_height = 0; + for _ in 0..numb_blocks { + (last_block_height, _, _) = + crate::ops::block::pop_block(Some(old_main_chain_id), &mut tables_mut)?; + } + + tables_mut.alt_chain_infos_mut().put( + &old_main_chain_id.into(), + &AltChainInfo { + parent_chain: Chain::Main.into(), + common_ancestor_height: last_block_height - 1, + chain_height: last_block_height + numb_blocks, + }, + )?; + + Ok(old_main_chain_id) + }; + + match result { + Ok(old_main_chain_id) => { + TxRw::commit(tx_rw)?; + Ok(BlockchainResponse::PopBlocks(old_main_chain_id)) + } + Err(e) => { + // INVARIANT: ensure database atomicity by aborting + // the transaction on `add_block()` failures. + TxRw::abort(tx_rw) + .expect("could not maintain database atomicity by aborting write transaction"); + Err(e) + } + } +} + +/// [`BlockchainWriteRequest::ReverseReorg`]. +fn reverse_reorg(env: &ConcreteEnv, chain_id: ChainId) -> ResponseResult { + let env_inner = env.env_inner(); + let tx_rw = env_inner.tx_rw()?; + + let result = { + let mut tables_mut = env_inner.open_tables_mut(&tx_rw)?; + + let chain_info = tables_mut.alt_chain_infos().get(&chain_id.into())?; + assert_eq!(Chain::from(chain_info.parent_chain), Chain::Main); + + let tob_block_height = + crate::ops::blockchain::top_block_height(tables_mut.block_heights())?; + + for _ in chain_info.common_ancestor_height..tob_block_height { + crate::ops::block::pop_block(None, &mut tables_mut)?; + } + + // Rust borrow rules requires us to collect into a Vec first before looping over the Vec. + let alt_blocks = (chain_info.common_ancestor_height..chain_info.chain_height) + .map(|height| { + crate::ops::alt_block::get_alt_block( + &AltBlockHeight { + chain_id: chain_id.into(), + height, + }, + &tables_mut, + ) + }) + .collect::>(); + + for res_alt_block in alt_blocks { + let alt_block = res_alt_block?; + + let verified_block = map_valid_alt_block_to_verified_block(alt_block); + + crate::ops::block::add_block(&verified_block, &mut tables_mut)?; + } + + Ok(()) + }; + + match result { + Ok(()) => { + TxRw::commit(tx_rw)?; + Ok(BlockchainResponse::Ok) + } + Err(e) => { + // INVARIANT: ensure database atomicity by aborting + // the transaction on `add_block()` failures. + TxRw::abort(tx_rw) + .expect("could not maintain database atomicity by aborting write transaction"); + Err(e) + } + } +} + /// [`BlockchainWriteRequest::FlushAltBlocks`]. #[inline] fn flush_alt_blocks(env: &ConcreteEnv) -> ResponseResult { diff --git a/storage/blockchain/src/tables.rs b/storage/blockchain/src/tables.rs index deb957ea..fa568ae6 100644 --- a/storage/blockchain/src/tables.rs +++ b/storage/blockchain/src/tables.rs @@ -145,6 +145,9 @@ cuprate_database::define_tables! { 19 => AltTransactionBlobs, TxHash => TxBlob, + + 20 => AltTransactionInfos, + TxHash => AltTransactionInfo, } //---------------------------------------------------------------------------------------------------- Tests diff --git a/storage/blockchain/src/types.rs b/storage/blockchain/src/types.rs index 88ece10b..14917249 100644 --- a/storage/blockchain/src/types.rs +++ b/storage/blockchain/src/types.rs @@ -189,7 +189,7 @@ pub struct BlockInfo { /// The adjusted block size, in bytes. /// /// See [`block_weight`](https://monero-book.cuprate.org/consensus_rules/blocks/weights.html#blocks-weight). - pub weight: u64, + pub weight: usize, /// Least-significant 64 bits of the 128-bit cumulative difficulty. pub cumulative_difficulty_low: u64, /// Most-significant 64 bits of the 128-bit cumulative difficulty. @@ -201,7 +201,7 @@ pub struct BlockInfo { /// The long term block weight, based on the median weight of the preceding `100_000` blocks. /// /// See [`long_term_weight`](https://monero-book.cuprate.org/consensus_rules/blocks/weights.html#long-term-block-weight). - pub long_term_weight: u64, + pub long_term_weight: usize, } //---------------------------------------------------------------------------------------------------- OutputFlags @@ -381,6 +381,7 @@ impl Key for RawChainId {} pub struct AltChainInfo { pub parent_chain: RawChain, pub common_ancestor_height: usize, + pub chain_height: usize, } //---------------------------------------------------------------------------------------------------- AltBlockHeight diff --git a/types/src/blockchain.rs b/types/src/blockchain.rs index 33c3e8bd..c2a5517d 100644 --- a/types/src/blockchain.rs +++ b/types/src/blockchain.rs @@ -116,20 +116,17 @@ pub enum BlockchainWriteRequest { /// /// Input is the alternative block. WriteAltBlock(AltBlockInformation), - /// A request to start the re-org process. + /// A request to pop some blocks from the top of the main chain /// - /// The inner value is the [`ChainId`] of the alt-chain we want to re-org to. + /// Input is the amount of blocks to pop. /// - /// This will: - /// - pop blocks from the main chain - /// - retrieve all alt-blocks in this alt-chain - /// - flush all other alt blocks - StartReorg(ChainId), + /// This request flush all alt-chains from the cache before adding the popped blocks to the alt cache. + PopBlocks(usize), /// A request to reverse the re-org process. /// /// The inner value is the [`ChainId`] of the old main chain. /// - /// It is invalid to call this with a [`ChainId`] that was not returned from [`BlockchainWriteRequest::StartReorg`]. + /// It is invalid to call this with a [`ChainId`] that was not returned from [`BlockchainWriteRequest::PopBlocks`]. ReverseReorg(ChainId), /// A request to flush all alternative blocks. FlushAltBlocks, @@ -227,13 +224,10 @@ pub enum BlockchainResponse { /// - [`BlockchainWriteRequest::ReverseReorg`] /// - [`BlockchainWriteRequest::FlushAltBlocks`] Ok, - /// The response for [`BlockchainWriteRequest::StartReorg`]. - StartReorg { - /// The [`ChainId`] of the old main chain blocks that were popped. - old_main_chain_id: ChainId, - /// The next alt chain blocks. - alt_chain: Vec, - }, + /// The response for [`BlockchainWriteRequest::PopBlocks`]. + /// + /// The inner value is the alt-chain ID for the old main chain blocks. + PopBlocks(ChainId), } //---------------------------------------------------------------------------------------------------- Tests diff --git a/types/src/types.rs b/types/src/types.rs index cc4543e6..c6e83d09 100644 --- a/types/src/types.rs +++ b/types/src/types.rs @@ -79,6 +79,7 @@ pub struct VerifiedBlockInformation { /// [`Block::hash`]. pub block_hash: [u8; 32], /// The block's proof-of-work hash. + // TODO: make this an option. pub pow_hash: [u8; 32], /// The block's height. pub height: usize, @@ -120,7 +121,7 @@ pub struct AltBlockInformation { /// [`Block::serialize`]. pub block_blob: Vec, /// All the transactions in the block, excluding the [`Block::miner_transaction`]. - pub txs: Vec>, + pub txs: Vec, /// The block's hash. /// /// [`Block::hash`]. From 21e4b3a20a56323961f65e100ca826f0677a41e5 Mon Sep 17 00:00:00 2001 From: Boog900 <54e72d8a-345f-4599-bd90-c6b9bc7d0ec5@aleeas.com> Date: Fri, 6 Sep 2024 00:26:36 +0100 Subject: [PATCH 07/15] commit Cargo.lock --- Cargo.lock | 39 +++++++++++++++++++++++++++------------ 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d5d64902..e277e3f7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,6 +17,12 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +[[package]] +name = "adler2" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" + [[package]] name = "ahash" version = "0.8.11" @@ -160,7 +166,7 @@ dependencies = [ "cc", "cfg-if", "libc", - "miniz_oxide", + "miniz_oxide 0.7.3", "object", "rustc-demangle", ] @@ -1080,12 +1086,12 @@ checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" [[package]] name = "flate2" -version = "1.0.30" +version = "1.0.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f54427cfd1c7829e2a139fcefea601bf088ebca651d2bf53ebc600eac295dae" +checksum = "324a1be68054ef05ad64b861cc9eaf1d623d2d8cb25b4bf2cb9cdd902b4bf253" dependencies = [ "crc32fast", - "miniz_oxide", + "miniz_oxide 0.8.0", ] [[package]] @@ -1239,9 +1245,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa82e28a107a8cc405f0839610bdc9b15f1e25ec7d696aa5cf173edbcb1486ab" +checksum = "524e8ac6999421f49a846c2d4411f337e53497d8ec55d67753beffa43c5d9205" dependencies = [ "atomic-waker", "bytes", @@ -1735,6 +1741,15 @@ dependencies = [ "adler", ] +[[package]] +name = "miniz_oxide" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2d80299ef12ff69b16a84bb182e3b9df68b5a91574d3d4fa6e41b65deec4df1" +dependencies = [ + "adler2", +] + [[package]] name = "mio" version = "0.8.11" @@ -2397,9 +2412,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.7.0" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "976295e77ce332211c0d24d92c0e83e50f5c5f046d11082cea19f3df13a3562d" +checksum = "fc0a2ce646f8655401bb81e7927b812614bd5d91dbc968696be50603510fcaf0" [[package]] name = "rustls-webpki" @@ -2958,9 +2973,9 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "ureq" -version = "2.10.0" +version = "2.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72139d247e5f97a3eff96229a7ae85ead5328a39efe76f8bf5a06313d505b6ea" +checksum = "b74fc6b57825be3373f7054754755f03ac3a8f5d70015ccad699ba2029956f4a" dependencies = [ "base64", "flate2", @@ -3083,9 +3098,9 @@ checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" [[package]] name = "webpki-roots" -version = "0.26.3" +version = "0.26.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd7c23921eeb1713a4e851530e9b9756e4fb0e89978582942612524cf09f01cd" +checksum = "0bd24728e5af82c6c4ec1b66ac4844bdf8156257fccda846ec58b42cd0cdbe6a" dependencies = [ "rustls-pki-types", ] From 123aedd6a986f7884efdc922eca046c86b7169ae Mon Sep 17 00:00:00 2001 From: Boog900 <54e72d8a-345f-4599-bd90-c6b9bc7d0ec5@aleeas.com> Date: Fri, 6 Sep 2024 02:39:40 +0100 Subject: [PATCH 08/15] add test --- storage/blockchain/src/ops/alt_block/block.rs | 100 +++++++++++++++++- storage/blockchain/src/ops/alt_block/chain.rs | 2 +- storage/blockchain/src/ops/block.rs | 2 +- storage/blockchain/src/ops/mod.rs | 2 +- storage/blockchain/src/service/mod.rs | 2 +- storage/blockchain/src/service/tests.rs | 2 +- storage/blockchain/src/tests.rs | 16 +++ 7 files changed, 120 insertions(+), 6 deletions(-) diff --git a/storage/blockchain/src/ops/alt_block/block.rs b/storage/blockchain/src/ops/alt_block/block.rs index a429eab8..83d04ba7 100644 --- a/storage/blockchain/src/ops/alt_block/block.rs +++ b/storage/blockchain/src/ops/alt_block/block.rs @@ -171,7 +171,7 @@ pub fn get_alt_block_extended_header_from_height( let block_header = BlockHeader::read(&mut block_blob.as_slice())?; Ok(ExtendedBlockHeader { - version: HardFork::from_version(0).expect("Block in DB must have correct version"), + version: HardFork::from_version(block_header.hardfork_version).expect("Block in DB must have correct version"), vote: block_header.hardfork_version, timestamp: block_header.timestamp, cumulative_difficulty: combine_low_high_bits_to_u128( @@ -182,3 +182,101 @@ pub fn get_alt_block_extended_header_from_height( long_term_weight: block_info.long_term_weight, }) } + +#[cfg(test)] +mod tests { + use std::num::NonZero; + use cuprate_database::{Env, EnvInner, TxRw}; + use cuprate_test_utils::data::{BLOCK_V1_TX2, BLOCK_V9_TX3, BLOCK_V16_TX0}; + use cuprate_types::ChainId; + use crate::ops::alt_block::{add_alt_block, flush_alt_blocks, get_alt_block, get_alt_extended_headers_in_range}; + use crate::ops::block::{add_block, pop_block}; + use crate::tables::OpenTables; + use crate::tests::{assert_all_tables_are_empty, map_verified_block_to_alt, tmp_concrete_env}; + use crate::types::AltBlockHeight; + + #[test] + fn all_alt_blocks() { + let (env, _tmp) = tmp_concrete_env(); + let env_inner = env.env_inner(); + assert_all_tables_are_empty(&env); + + let chain_id = ChainId(NonZero::new(1).unwrap()).into(); + + // Add initial block. + { + let tx_rw = env_inner.tx_rw().unwrap(); + let mut tables = env_inner.open_tables_mut(&tx_rw).unwrap(); + + let mut initial_block = BLOCK_V1_TX2.clone(); + initial_block.height = 0; + + add_block(&initial_block, &mut tables).unwrap(); + + drop(tables); + TxRw::commit(tx_rw).unwrap(); + } + + let alt_blocks = [ + map_verified_block_to_alt(BLOCK_V9_TX3.clone(), chain_id), + map_verified_block_to_alt(BLOCK_V16_TX0.clone(), chain_id), + ]; + + // Add alt-blocks + { + let tx_rw = env_inner.tx_rw().unwrap(); + let mut tables = env_inner.open_tables_mut(&tx_rw).unwrap(); + + let mut prev_hash = BLOCK_V1_TX2.block_hash; + for (i, mut alt_block) in alt_blocks.into_iter().enumerate() { + let height = i + 1; + + alt_block.height = height; + alt_block.block.header.previous = prev_hash; + alt_block.block_blob = alt_block.block.serialize(); + + add_alt_block(&alt_block, &mut tables).unwrap(); + + let alt_height = AltBlockHeight { + chain_id: chain_id.into(), + height, + }; + + let alt_block_2 = get_alt_block(&alt_height, &tables).unwrap(); + assert_eq!(alt_block.block, alt_block_2.block); + + let headers = get_alt_extended_headers_in_range(0..(height + 1), chain_id, &tables).unwrap(); + assert_eq!(headers.len(), height); + + let last_header = headers.last().unwrap(); + assert_eq!(last_header.timestamp, alt_block.block.header.timestamp); + assert_eq!(last_header.block_weight, alt_block.weight); + assert_eq!(last_header.long_term_weight, alt_block.long_term_weight); + assert_eq!(last_header.cumulative_difficulty, alt_block.cumulative_difficulty); + assert_eq!(last_header.version.as_u8(), alt_block.block.header.hardfork_version); + assert_eq!(last_header.vote, alt_block.block.header.hardfork_signal); + + prev_hash = alt_block.block_hash; + } + + drop(tables); + TxRw::commit(tx_rw).unwrap(); + } + + + { + let mut tx_rw = env_inner.tx_rw().unwrap(); + + flush_alt_blocks(&env_inner, &mut tx_rw).unwrap(); + + let mut tables = env_inner.open_tables_mut(&tx_rw).unwrap(); + pop_block(None, &mut tables).unwrap(); + + drop(tables); + TxRw::commit(tx_rw).unwrap(); + } + + assert_all_tables_are_empty(&env); + } + +} diff --git a/storage/blockchain/src/ops/alt_block/chain.rs b/storage/blockchain/src/ops/alt_block/chain.rs index 1162a9cd..166a294a 100644 --- a/storage/blockchain/src/ops/alt_block/chain.rs +++ b/storage/blockchain/src/ops/alt_block/chain.rs @@ -45,7 +45,7 @@ pub fn get_alt_chain_history_ranges( let start_height = max(range.start, chain_info.common_ancestor_height + 1); - ranges.push((chain_info.parent_chain.into(), start_height..i)); + ranges.push((Chain::Alt(current_chain_id.into()), start_height..i)); i = chain_info.common_ancestor_height; match chain_info.parent_chain.into() { diff --git a/storage/blockchain/src/ops/block.rs b/storage/blockchain/src/ops/block.rs index 229d35e1..b0997f76 100644 --- a/storage/blockchain/src/ops/block.rs +++ b/storage/blockchain/src/ops/block.rs @@ -180,7 +180,7 @@ pub fn pop_block( block_blob, txs, block_hash: block_info.block_hash, - pow_hash: [255; 32], + pow_hash: [0; 32], height: block_height, weight: block_info.weight, long_term_weight: block_info.long_term_weight, diff --git a/storage/blockchain/src/ops/mod.rs b/storage/blockchain/src/ops/mod.rs index 8a8f0f15..285aa244 100644 --- a/storage/blockchain/src/ops/mod.rs +++ b/storage/blockchain/src/ops/mod.rs @@ -94,7 +94,7 @@ //! // Read the data, assert it is correct. //! let tx_rw = env_inner.tx_rw()?; //! let mut tables = env_inner.open_tables_mut(&tx_rw)?; -//! let (height, hash, serai_block) = pop_block(&mut tables)?; +//! let (height, hash, serai_block) = pop_block(None, &mut tables)?; //! //! assert_eq!(height, 0); //! assert_eq!(serai_block, block.block); diff --git a/storage/blockchain/src/service/mod.rs b/storage/blockchain/src/service/mod.rs index c774ee49..aa322d06 100644 --- a/storage/blockchain/src/service/mod.rs +++ b/storage/blockchain/src/service/mod.rs @@ -98,7 +98,7 @@ //! //! // Block write was OK. //! let response = response_channel.await?; -//! assert_eq!(response, BlockchainResponse::WriteBlockOk); +//! assert_eq!(response, BlockchainResponse::Ok); //! //! // Now, let's try getting the block hash //! // of the block we just wrote. diff --git a/storage/blockchain/src/service/tests.rs b/storage/blockchain/src/service/tests.rs index b68b5444..78cb6944 100644 --- a/storage/blockchain/src/service/tests.rs +++ b/storage/blockchain/src/service/tests.rs @@ -84,7 +84,7 @@ async fn test_template( let request = BlockchainWriteRequest::WriteBlock(block); let response_channel = writer.call(request); let response = response_channel.await.unwrap(); - assert_eq!(response, BlockchainResponse::WriteBlockOk); + assert_eq!(response, BlockchainResponse::Ok); } //----------------------------------------------------------------------- Reset the transaction diff --git a/storage/blockchain/src/tests.rs b/storage/blockchain/src/tests.rs index 65527e10..60239127 100644 --- a/storage/blockchain/src/tests.rs +++ b/storage/blockchain/src/tests.rs @@ -10,6 +10,7 @@ use std::{borrow::Cow, fmt::Debug}; use pretty_assertions::assert_eq; use cuprate_database::{ConcreteEnv, DatabaseRo, Env, EnvInner}; +use cuprate_types::{AltBlockInformation, ChainId, VerifiedBlockInformation}; use crate::{ config::ConfigBuilder, @@ -88,3 +89,18 @@ pub(crate) fn assert_all_tables_are_empty(env: &ConcreteEnv) { assert!(tables.all_tables_empty().unwrap()); assert_eq!(crate::ops::tx::get_num_tx(tables.tx_ids()).unwrap(), 0); } + +pub(crate) fn map_verified_block_to_alt(verified_block: VerifiedBlockInformation, chain_id: ChainId) -> AltBlockInformation { + AltBlockInformation { + block: verified_block.block, + block_blob: verified_block.block_blob, + txs: verified_block.txs, + block_hash: verified_block.block_hash, + pow_hash: verified_block.pow_hash, + height: verified_block.height, + weight: verified_block.weight, + long_term_weight: verified_block.long_term_weight, + cumulative_difficulty: verified_block.cumulative_difficulty, + chain_id, + } +} \ No newline at end of file From ba5c5ac45d958f43382df21dd12eeca520a7c870 Mon Sep 17 00:00:00 2001 From: Boog900 <54e72d8a-345f-4599-bd90-c6b9bc7d0ec5@aleeas.com> Date: Sat, 7 Sep 2024 02:02:19 +0100 Subject: [PATCH 09/15] more docs + cleanup + alt blocks request --- storage/blockchain/src/ops/alt_block/block.rs | 166 +++++++++++------- storage/blockchain/src/ops/alt_block/chain.rs | 49 ++++-- storage/blockchain/src/ops/alt_block/mod.rs | 59 ++++++- storage/blockchain/src/ops/alt_block/tx.rs | 23 ++- storage/blockchain/src/ops/block.rs | 23 +-- storage/blockchain/src/ops/macros.rs | 21 +++ storage/blockchain/src/service/read.rs | 98 +++++++++-- storage/blockchain/src/service/write.rs | 19 +- storage/blockchain/src/tests.rs | 7 +- types/src/blockchain.rs | 20 ++- 10 files changed, 368 insertions(+), 117 deletions(-) diff --git a/storage/blockchain/src/ops/alt_block/block.rs b/storage/blockchain/src/ops/alt_block/block.rs index 83d04ba7..07878d55 100644 --- a/storage/blockchain/src/ops/alt_block/block.rs +++ b/storage/blockchain/src/ops/alt_block/block.rs @@ -1,16 +1,35 @@ -use crate::ops::alt_block::{ - add_alt_transaction_blob, check_add_alt_chain_info, get_alt_chain_history_ranges, - get_alt_transaction, -}; -use crate::ops::block::{get_block_extended_header_from_height, get_block_info}; -use crate::tables::{Tables, TablesMut}; -use crate::types::{AltBlockHeight, BlockHash, BlockHeight, CompactAltBlockInfo}; use bytemuck::TransparentWrapper; +use monero_serai::block::{Block, BlockHeader}; + use cuprate_database::{DatabaseRo, DatabaseRw, RuntimeError, StorableVec}; use cuprate_helper::map::{combine_low_high_bits_to_u128, split_u128_into_low_high_bits}; use cuprate_types::{AltBlockInformation, Chain, ChainId, ExtendedBlockHeader, HardFork}; -use monero_serai::block::{Block, BlockHeader}; +use crate::{ + ops::{ + alt_block::{add_alt_transaction_blob, get_alt_transaction, update_alt_chain_info}, + block::get_block_info, + macros::doc_error, + }, + tables::{Tables, TablesMut}, + types::{AltBlockHeight, BlockHash, BlockHeight, CompactAltBlockInfo}, +}; + +/// Add a [`AltBlockInformation`] to the database. +/// +/// This extracts all the data from the input block and +/// maps/adds them to the appropriate database tables. +/// +#[doc = doc_error!()] +/// +/// # Panics +/// This function will panic if: +/// - `block.height` is == `0` +/// +/// # Already exists +/// This function will operate normally even if `block` already +/// exists, i.e., this function will not return `Err` even if you +/// call this function infinitely with the same block. pub fn add_alt_block( alt_block: &AltBlockInformation, tables: &mut impl TablesMut, @@ -24,7 +43,7 @@ pub fn add_alt_block( .alt_block_heights_mut() .put(&alt_block.block_hash, &alt_block_height)?; - check_add_alt_chain_info(&alt_block_height, &alt_block.block.header.previous, tables)?; + update_alt_chain_info(&alt_block_height, &alt_block.block.header.previous, tables)?; let (cumulative_difficulty_low, cumulative_difficulty_high) = split_u128_into_low_high_bits(alt_block.cumulative_difficulty); @@ -49,13 +68,18 @@ pub fn add_alt_block( )?; assert_eq!(alt_block.txs.len(), alt_block.block.transactions.len()); - for tx in alt_block.txs.iter() { + for tx in &alt_block.txs { add_alt_transaction_blob(tx, tables)?; } Ok(()) } +/// Retrieves an [`AltBlockInformation`] from the database. +/// +/// This function will look at only the blocks with the given [`AltBlockHeight::chain_id`], no others +/// even if they are technically part of this chain. +#[doc = doc_error!()] pub fn get_alt_block( alt_block_height: &AltBlockHeight, tables: &impl Tables, @@ -89,13 +113,21 @@ pub fn get_alt_block( }) } +/// Retrieves the hash of the block at the given `block_height` on the alt chain with +/// the given [`ChainId`]. +/// +/// This function will get blocks from the whole chain, for example if you were to ask for height +/// `0` with any [`ChainId`] (as long that chain actually exists) you will get the main chain genesis. +/// +#[doc = doc_error!()] pub fn get_alt_block_hash( block_height: &BlockHeight, alt_chain: ChainId, - tables: &mut impl Tables, + tables: &impl Tables, ) -> Result { let alt_chains = tables.alt_chain_infos(); + // First find what [`ChainId`] this block would be stored under. let original_chain = { let mut chain = alt_chain.into(); loop { @@ -115,9 +147,10 @@ pub fn get_alt_block_hash( } }; + // Get the block hash. match original_chain { Chain::Main => { - get_block_info(&block_height, tables.block_infos()).map(|info| info.block_hash) + get_block_info(block_height, tables.block_infos()).map(|info| info.block_hash) } Chain::Alt(chain_id) => tables .alt_blocks_info() @@ -129,37 +162,12 @@ pub fn get_alt_block_hash( } } -pub fn get_alt_extended_headers_in_range( - range: std::ops::Range, - alt_chain: ChainId, - tables: &impl Tables, -) -> Result, RuntimeError> { - // TODO: this function does not use rayon, however it probably should. - - let alt_chains = tables.alt_chain_infos(); - let ranges = get_alt_chain_history_ranges(range, alt_chain, alt_chains)?; - - let res = ranges - .into_iter() - .rev() - .map(|(chain, range)| { - range.into_iter().map(move |height| match chain { - Chain::Main => get_block_extended_header_from_height(&height, tables), - Chain::Alt(chain_id) => get_alt_block_extended_header_from_height( - &AltBlockHeight { - chain_id: chain_id.into(), - height, - }, - tables, - ), - }) - }) - .flatten() - .collect::>()?; - - Ok(res) -} - +/// Retrieves the [`ExtendedBlockHeader`] of the alt-block with an exact [`AltBlockHeight`]. +/// +/// This function will look at only the blocks with the given [`AltBlockHeight::chain_id`], no others +/// even if they are technically part of this chain. +/// +#[doc = doc_error!()] pub fn get_alt_block_extended_header_from_height( height: &AltBlockHeight, table: &impl Tables, @@ -171,7 +179,8 @@ pub fn get_alt_block_extended_header_from_height( let block_header = BlockHeader::read(&mut block_blob.as_slice())?; Ok(ExtendedBlockHeader { - version: HardFork::from_version(block_header.hardfork_version).expect("Block in DB must have correct version"), + version: HardFork::from_version(block_header.hardfork_version) + .expect("Block in DB must have correct version"), vote: block_header.hardfork_version, timestamp: block_header.timestamp, cumulative_difficulty: combine_low_high_bits_to_u128( @@ -186,22 +195,33 @@ pub fn get_alt_block_extended_header_from_height( #[cfg(test)] mod tests { use std::num::NonZero; - use cuprate_database::{Env, EnvInner, TxRw}; - use cuprate_test_utils::data::{BLOCK_V1_TX2, BLOCK_V9_TX3, BLOCK_V16_TX0}; - use cuprate_types::ChainId; - use crate::ops::alt_block::{add_alt_block, flush_alt_blocks, get_alt_block, get_alt_extended_headers_in_range}; - use crate::ops::block::{add_block, pop_block}; - use crate::tables::OpenTables; - use crate::tests::{assert_all_tables_are_empty, map_verified_block_to_alt, tmp_concrete_env}; - use crate::types::AltBlockHeight; + use cuprate_database::{Env, EnvInner, TxRw}; + use cuprate_test_utils::data::{BLOCK_V16_TX0, BLOCK_V1_TX2, BLOCK_V9_TX3}; + use cuprate_types::{Chain, ChainId}; + + use crate::{ + ops::{ + alt_block::{ + add_alt_block, flush_alt_blocks, get_alt_block, + get_alt_block_extended_header_from_height, get_alt_block_hash, + get_alt_chain_history_ranges, + }, + block::{add_block, pop_block}, + }, + tables::{OpenTables, Tables}, + tests::{assert_all_tables_are_empty, map_verified_block_to_alt, tmp_concrete_env}, + types::AltBlockHeight, + }; + + #[allow(clippy::range_plus_one)] #[test] fn all_alt_blocks() { let (env, _tmp) = tmp_concrete_env(); let env_inner = env.env_inner(); assert_all_tables_are_empty(&env); - let chain_id = ChainId(NonZero::new(1).unwrap()).into(); + let chain_id = ChainId(NonZero::new(1).unwrap()); // Add initial block. { @@ -245,25 +265,44 @@ mod tests { let alt_block_2 = get_alt_block(&alt_height, &tables).unwrap(); assert_eq!(alt_block.block, alt_block_2.block); - let headers = get_alt_extended_headers_in_range(0..(height + 1), chain_id, &tables).unwrap(); - assert_eq!(headers.len(), height); + let headers = get_alt_chain_history_ranges( + 0..(height + 1), + chain_id, + tables.alt_chain_infos(), + ) + .unwrap(); - let last_header = headers.last().unwrap(); - assert_eq!(last_header.timestamp, alt_block.block.header.timestamp); - assert_eq!(last_header.block_weight, alt_block.weight); - assert_eq!(last_header.long_term_weight, alt_block.long_term_weight); - assert_eq!(last_header.cumulative_difficulty, alt_block.cumulative_difficulty); - assert_eq!(last_header.version.as_u8(), alt_block.block.header.hardfork_version); - assert_eq!(last_header.vote, alt_block.block.header.hardfork_signal); + assert_eq!(headers.len(), 2); + assert_eq!(headers[1], (Chain::Main, 0..1)); + assert_eq!(headers[0], (Chain::Alt(chain_id), 1..(height + 1))); prev_hash = alt_block.block_hash; + + let header = + get_alt_block_extended_header_from_height(&alt_height, &tables).unwrap(); + + assert_eq!(header.timestamp, alt_block.block.header.timestamp); + assert_eq!(header.block_weight, alt_block.weight); + assert_eq!(header.long_term_weight, alt_block.long_term_weight); + assert_eq!( + header.cumulative_difficulty, + alt_block.cumulative_difficulty + ); + assert_eq!( + header.version.as_u8(), + alt_block.block.header.hardfork_version + ); + assert_eq!(header.vote, alt_block.block.header.hardfork_signal); + + let block_hash = get_alt_block_hash(&height, chain_id, &tables).unwrap(); + + assert_eq!(block_hash, alt_block.block_hash); } drop(tables); TxRw::commit(tx_rw).unwrap(); } - { let mut tx_rw = env_inner.tx_rw().unwrap(); @@ -278,5 +317,4 @@ mod tests { assert_all_tables_are_empty(&env); } - } diff --git a/storage/blockchain/src/ops/alt_block/chain.rs b/storage/blockchain/src/ops/alt_block/chain.rs index 166a294a..3e27d7d4 100644 --- a/storage/blockchain/src/ops/alt_block/chain.rs +++ b/storage/blockchain/src/ops/alt_block/chain.rs @@ -1,20 +1,43 @@ -use crate::tables::{AltChainInfos, TablesMut}; -use crate::types::{AltBlockHeight, AltChainInfo, BlockHash, BlockHeight}; -use cuprate_database::{DatabaseRo, DatabaseRw, RuntimeError}; -use cuprate_types::{Chain, ChainId}; use std::cmp::max; -pub fn check_add_alt_chain_info( +use cuprate_database::{DatabaseRo, DatabaseRw, RuntimeError}; +use cuprate_types::{Chain, ChainId}; + +use crate::{ + ops::macros::{doc_add_alt_block_inner_invariant, doc_error}, + tables::{AltChainInfos, TablesMut}, + types::{AltBlockHeight, AltChainInfo, BlockHash, BlockHeight}, +}; + +/// Updates the [`AltChainInfo`] with information on a new alt-block. +/// +#[doc = doc_add_alt_block_inner_invariant!()] +#[doc = doc_error!()] +/// +/// # Panics +/// +/// This will panic if [`AltBlockHeight::height`] == `0`. +pub fn update_alt_chain_info( alt_block_height: &AltBlockHeight, prev_hash: &BlockHash, tables: &mut impl TablesMut, ) -> Result<(), RuntimeError> { - match tables.alt_chain_infos().get(&alt_block_height.chain_id) { - Ok(_) => return Ok(()), + // try update the info if one exists for this chain. + let update = tables + .alt_chain_infos_mut() + .update(&alt_block_height.chain_id, |mut info| { + info.chain_height = alt_block_height.height + 1; + Some(info) + }); + + match update { + Ok(()) => return Ok(()), Err(RuntimeError::KeyNotFound) => (), Err(e) => return Err(e), } + // If one doesn't already exist add it. + let parent_chain = match tables.alt_block_heights().get(prev_hash) { Ok(alt_parent_height) => Chain::Alt(alt_parent_height.chain_id.into()), Err(RuntimeError::KeyNotFound) => Chain::Main, @@ -25,12 +48,18 @@ pub fn check_add_alt_chain_info( &alt_block_height.chain_id, &AltChainInfo { parent_chain: parent_chain.into(), - common_ancestor_height: alt_block_height.height - 1, - chain_height: alt_block_height.height, + common_ancestor_height: alt_block_height.height.checked_sub(1).unwrap(), + chain_height: alt_block_height.height + 1, }, ) } +/// Get the height history of an alt-chain in reverse chronological order. +/// +/// Height history is a list of height ranges with the corresponding [`Chain`] they are stored under. +/// For example if your range goes from height `0` the last entry in the list will be [`Chain::Main`] +/// upto the height where the first split occurs. +#[doc = doc_error!()] pub fn get_alt_chain_history_ranges( range: std::ops::Range, alt_chain: ChainId, @@ -46,7 +75,7 @@ pub fn get_alt_chain_history_ranges( let start_height = max(range.start, chain_info.common_ancestor_height + 1); ranges.push((Chain::Alt(current_chain_id.into()), start_height..i)); - i = chain_info.common_ancestor_height; + i = chain_info.common_ancestor_height + 1; match chain_info.parent_chain.into() { Chain::Main => { diff --git a/storage/blockchain/src/ops/alt_block/mod.rs b/storage/blockchain/src/ops/alt_block/mod.rs index 72e0933e..36e4768e 100644 --- a/storage/blockchain/src/ops/alt_block/mod.rs +++ b/storage/blockchain/src/ops/alt_block/mod.rs @@ -1,11 +1,62 @@ -mod block; -mod chain; -mod tx; - +//! Alternative Block/Chain Ops +//! +//! Alternative chains are chains that potentially have more proof-of-work than the main-chain +//! which we are tracking to potentially re-org to. +//! +//! Cuprate uses an ID system for alt-chains. When a split is made from the main-chain we generate +//! a random [`ChainID`](cuprate_types::ChainId) and assign it to the chain: +//! +//! ```text +//! | +//! | +//! | split +//! |------------- +//! | | +//! | | +//! \|/ \|/ +//! main-chain ChainID(X) +//! ``` +//! +//! In that example if we were to receive an alt-block which immediately follows the top block of `ChainID(X)` +//! then that block will also be stored under `ChainID(X)`. However if it follows from another block from `ChainID(X)` +//! we will split into a chain with a different ID. +//! +//! ```text +//! | +//! | +//! | split +//! |------------- +//! | | split +//! | |-------------| +//! | | | +//! | | | +//! | | | +//! \|/ \|/ \|/ +//! main-chain ChainID(X) ChainID(Z) +//! ``` +//! +//! As you can see if we wanted to get all the alt-blocks in `ChainID(Z)` that now includes some blocks from `ChainID(X)` as well. +//! [`get_alt_chain_history_ranges`] covers this and is the method to get the ranges of heights needed from each [`ChainID`](cuprate_types::ChainId) +//! to get all the alt-blocks in a given [`ChainID`](cuprate_types::ChainId). +//! +//! Although this should be kept in mind as a possibility because Cuprate's block downloader will only track a single chain it is +//! unlikely that we will be tracking [`ChainID`](cuprate_types::ChainId) that don't immediately connect to the main-chain. +//! +//! ## Why not use block's previous field? +//! +//! Although that would be easier, it makes getting a range of block extremely slow, as we have to build the weight cache to verify +//! blocks, roughly 100,000 block headers needed, this cost was seen as too high. pub use block::*; pub use chain::*; pub use tx::*; +mod block; +mod chain; +mod tx; + +/// Flush all alt-block data from all the alt-block tables. +/// +/// This function completely empties the alt block tables. pub fn flush_alt_blocks<'a, E: cuprate_database::EnvInner<'a>>( env_inner: &E, tx_rw: &mut E::Rw<'_>, diff --git a/storage/blockchain/src/ops/alt_block/tx.rs b/storage/blockchain/src/ops/alt_block/tx.rs index a49c72ae..5671e4b4 100644 --- a/storage/blockchain/src/ops/alt_block/tx.rs +++ b/storage/blockchain/src/ops/alt_block/tx.rs @@ -1,10 +1,22 @@ -use crate::tables::{Tables, TablesMut}; -use crate::types::{AltTransactionInfo, TxHash}; use bytemuck::TransparentWrapper; -use cuprate_database::{DatabaseRo, DatabaseRw, RuntimeError, StorableVec}; -use cuprate_types::VerifiedTransactionInformation; use monero_serai::transaction::Transaction; +use cuprate_database::{DatabaseRo, DatabaseRw, RuntimeError, StorableVec}; +use cuprate_types::VerifiedTransactionInformation; + +use crate::ops::macros::{doc_add_alt_block_inner_invariant, doc_error}; +use crate::tables::{Tables, TablesMut}; +use crate::types::{AltTransactionInfo, TxHash}; + +/// Adds a [`VerifiedTransactionInformation`] form an alt-block to the DB, if +/// that transaction is not already in the DB. +/// +/// If the transaction is in the main-chain this function will still fill in the +/// [`AltTransactionInfos`](crate::tables::AltTransactionInfos) table, as that +/// table holds data which we don't keep around for main-chain txs. +/// +#[doc = doc_add_alt_block_inner_invariant!()] +#[doc = doc_error!()] pub fn add_alt_transaction_blob( tx: &VerifiedTransactionInformation, tables: &mut impl TablesMut, @@ -29,6 +41,9 @@ pub fn add_alt_transaction_blob( .put(&tx.tx_hash, StorableVec::wrap_ref(&tx.tx_blob)) } +/// Retrieve a [`VerifiedTransactionInformation`] from the database. +/// +#[doc = doc_error!()] pub fn get_alt_transaction( tx_hash: &TxHash, tables: &impl Tables, diff --git a/storage/blockchain/src/ops/block.rs b/storage/blockchain/src/ops/block.rs index b0997f76..45bab41c 100644 --- a/storage/blockchain/src/ops/block.rs +++ b/storage/blockchain/src/ops/block.rs @@ -38,11 +38,6 @@ use crate::{ /// This function will panic if: /// - `block.height > u32::MAX` (not normally possible) /// - `block.height` is not != [`chain_height`] -/// -/// # Already exists -/// This function will operate normally even if `block` already -/// exists, i.e., this function will not return `Err` even if you -/// call this function infinitely with the same block. // no inline, too big. pub fn add_block( block: &VerifiedBlockInformation, @@ -133,6 +128,9 @@ pub fn add_block( /// Remove the top/latest block from the database. /// /// The removed block's data is returned. +/// +/// If a [`ChainId`] is specified the popped block will be added to the alt block tables under +/// that [`ChainId`]. Otherwise, the block will be completely removed from the DB. #[doc = doc_error!()] /// /// In `pop_block()`'s case, [`RuntimeError::KeyNotFound`] @@ -169,7 +167,7 @@ pub fn pop_block( tx_hash: tx.hash(), fee: tx_fee(&tx), tx, - }) + }); } } @@ -180,11 +178,16 @@ pub fn pop_block( block_blob, txs, block_hash: block_info.block_hash, + // We know the PoW is valid for this block so just set it so it will always verify as + // valid. pow_hash: [0; 32], height: block_height, weight: block_info.weight, long_term_weight: block_info.long_term_weight, - cumulative_difficulty: 0, + cumulative_difficulty: combine_low_high_bits_to_u128( + block_info.cumulative_difficulty_low, + block_info.cumulative_difficulty_high, + ), chain_id, }, tables, @@ -229,8 +232,6 @@ pub fn get_block_extended_header_from_height( block_info.cumulative_difficulty_high, ); - // INVARIANT: #[cfg] @ lib.rs asserts `usize == u64` - #[allow(clippy::cast_possible_truncation)] Ok(ExtendedBlockHeader { cumulative_difficulty, version: HardFork::from_version(block_header.hardfork_version) @@ -302,14 +303,14 @@ mod test { use cuprate_database::{Env, EnvInner, TxRw}; use cuprate_test_utils::data::{BLOCK_V16_TX0, BLOCK_V1_TX2, BLOCK_V9_TX3}; - use super::*; - use crate::{ ops::tx::{get_tx, tx_exists}, tables::OpenTables, tests::{assert_all_tables_are_empty, tmp_concrete_env, AssertTableLen}, }; + use super::*; + /// Tests all above block functions. /// /// Note that this doesn't test the correctness of values added, as the diff --git a/storage/blockchain/src/ops/macros.rs b/storage/blockchain/src/ops/macros.rs index b7cdba47..e547c7f2 100644 --- a/storage/blockchain/src/ops/macros.rs +++ b/storage/blockchain/src/ops/macros.rs @@ -31,3 +31,24 @@ When calling this function, ensure that either: }; } pub(super) use doc_add_block_inner_invariant; + +// This is pretty much the same as [`doc_add_block_inner_invariant`], it's not worth the effort to reduce +// the duplication. +/// Generate `# Invariant` documentation for internal alt block `fn`'s +/// that should be called directly with caution. +macro_rules! doc_add_alt_block_inner_invariant { + () => { + r#"# ⚠️ Invariant ⚠️ +This function mainly exists to be used internally by the parent function [`crate::ops::alt_block::add_alt_block`]. + +`add_alt_block()` makes sure all data related to the input is mutated, while +this function _does not_, it specifically mutates _particular_ tables. + +This is usually undesired - although this function is still available to call directly. + +When calling this function, ensure that either: +1. This effect (incomplete database mutation) is what is desired, or that... +2. ...the other tables will also be mutated to a correct state"# + }; +} +pub(super) use doc_add_alt_block_inner_invariant; diff --git a/storage/blockchain/src/service/read.rs b/storage/blockchain/src/service/read.rs index 70da01b3..416e6137 100644 --- a/storage/blockchain/src/service/read.rs +++ b/storage/blockchain/src/service/read.rs @@ -1,28 +1,32 @@ //! Database reader thread-pool definitions and logic. -//---------------------------------------------------------------------------------------------------- Import use std::{ collections::{HashMap, HashSet}, sync::Arc, }; +//---------------------------------------------------------------------------------------------------- Import use rayon::{ iter::{IntoParallelIterator, ParallelIterator}, + prelude::*, ThreadPool, }; use thread_local::ThreadLocal; use cuprate_database::{ConcreteEnv, DatabaseRo, Env, EnvInner, RuntimeError}; -use cuprate_database_service::{init_thread_pool, DatabaseReadService, ReaderThreads}; +use cuprate_database_service::{DatabaseReadService, init_thread_pool, ReaderThreads}; use cuprate_helper::map::combine_low_high_bits_to_u128; use cuprate_types::{ blockchain::{BlockchainReadRequest, BlockchainResponse}, - Chain, ExtendedBlockHeader, OutputOnChain, + Chain, ChainId, ExtendedBlockHeader, OutputOnChain, }; use crate::{ ops::{ - alt_block::{get_alt_block_hash, get_alt_extended_headers_in_range}, + alt_block::{ + get_alt_block_extended_header_from_height, get_alt_block_hash, + get_alt_chain_history_ranges, + }, block::{ block_exists, get_block_extended_header_from_height, get_block_height, get_block_info, }, @@ -35,8 +39,11 @@ use crate::{ types::{BlockchainReadHandle, ResponseResult}, }, tables::{AltBlockHeights, BlockHeights, BlockInfos, OpenTables, Tables}, - types::{Amount, AmountIndex, BlockHash, BlockHeight, KeyImage, PreRctOutputId}, + types::{ + AltBlockHeight, Amount, AmountIndex, BlockHash, BlockHeight, KeyImage, PreRctOutputId, + }, }; +use crate::ops::alt_block::get_alt_block; //---------------------------------------------------------------------------------------------------- init_read_service /// Initialize the [`BlockchainReadHandle`] thread-pool backed by [`rayon`]. @@ -100,6 +107,7 @@ fn map_request( R::KeyImagesSpent(set) => key_images_spent(env, set), R::CompactChainHistory => compact_chain_history(env), R::FindFirstUnknown(block_ids) => find_first_unknown(env, &block_ids), + R::AltBlocksInChain(chain_id) => alt_blocks_in_chain(env, chain_id), } /* SOMEDAY: post-request handling, run some code for each request? */ @@ -200,7 +208,7 @@ fn block_hash(env: &ConcreteEnv, block_height: BlockHeight, chain: Chain) -> Res let block_hash = match chain { Chain::Main => get_block_info(&block_height, &table_block_infos)?.block_hash, Chain::Alt(chain) => { - get_alt_block_hash(&block_height, chain, &mut env_inner.open_tables(&tx_ro)?)? + get_alt_block_hash(&block_height, chain, &env_inner.open_tables(&tx_ro)?)? } }; @@ -283,12 +291,36 @@ fn block_extended_header_in_range( }) .collect::, RuntimeError>>()?, Chain::Alt(chain_id) => { - let tx_ro = tx_ro.get_or_try(|| env_inner.tx_ro())?; - get_alt_extended_headers_in_range( - range, - chain_id, - get_tables!(env_inner, tx_ro, tables)?.as_ref(), - )? + let ranges = { + let tx_ro = tx_ro.get_or_try(|| env_inner.tx_ro())?; + let tables = get_tables!(env_inner, tx_ro, tables)?.as_ref(); + let alt_chains = tables.alt_chain_infos(); + + get_alt_chain_history_ranges(range, chain_id, alt_chains)? + }; + + ranges + .par_iter() + .rev() + .map(|(chain, range)| { + range.clone().into_par_iter().map(|height| { + let tx_ro = tx_ro.get_or_try(|| env_inner.tx_ro())?; + let tables = get_tables!(env_inner, tx_ro, tables)?.as_ref(); + + match *chain { + Chain::Main => get_block_extended_header_from_height(&height, tables), + Chain::Alt(chain_id) => get_alt_block_extended_header_from_height( + &AltBlockHeight { + chain_id: chain_id.into(), + height, + }, + tables, + ), + } + }) + }) + .flatten() + .collect::, _>>()? } }; @@ -524,3 +556,45 @@ fn find_first_unknown(env: &ConcreteEnv, block_ids: &[BlockHash]) -> ResponseRes BlockchainResponse::FindFirstUnknown(Some((idx, last_known_height + 1))) }) } + +/// [`BlockchainReadRequest::AltBlocksInChain`] +fn alt_blocks_in_chain(env: &ConcreteEnv, chain_id: ChainId) -> ResponseResult { + // Prepare tx/tables in `ThreadLocal`. + let env_inner = env.env_inner(); + let tx_ro = thread_local(env); + let tables = thread_local(env); + + // Get the history of this alt-chain. + let history = { + let tx_ro = tx_ro.get_or_try(|| env_inner.tx_ro())?; + let tables = get_tables!(env_inner, tx_ro, tables)?.as_ref(); + get_alt_chain_history_ranges(0..usize::MAX, chain_id, tables.alt_chain_infos())? + }; + + // Get all the blocks until we join the main-chain. + let blocks = history + .par_iter() + .rev() + .skip(1) + .flat_map(|(chain_id, range)| { + let Chain::Alt(chain_id) = chain_id else { + panic!("Should not have main chain blocks here we skipped last range"); + }; + + range.clone().into_par_iter().map(|height| { + let tx_ro = tx_ro.get_or_try(|| env_inner.tx_ro())?; + let tables = get_tables!(env_inner, tx_ro, tables)?.as_ref(); + + get_alt_block( + &AltBlockHeight { + chain_id: (*chain_id).into(), + height, + }, + tables, + ) + }) + }) + .collect::>()?; + + Ok(BlockchainResponse::AltBlocksInChain(blocks)) +} diff --git a/storage/blockchain/src/service/write.rs b/storage/blockchain/src/service/write.rs index 95124d41..849a3030 100644 --- a/storage/blockchain/src/service/write.rs +++ b/storage/blockchain/src/service/write.rs @@ -9,12 +9,13 @@ use cuprate_types::{ AltBlockInformation, Chain, ChainId, VerifiedBlockInformation, }; -use crate::service::free::map_valid_alt_block_to_verified_block; -use crate::types::AltBlockHeight; use crate::{ - service::types::{BlockchainWriteHandle, ResponseResult}, + service::{ + free::map_valid_alt_block_to_verified_block, + types::{BlockchainWriteHandle, ResponseResult}, + }, tables::{OpenTables, Tables, TablesMut}, - types::AltChainInfo, + types::{AltBlockHeight, AltChainInfo}, }; //---------------------------------------------------------------------------------------------------- init_write_service @@ -106,19 +107,23 @@ fn pop_blocks(env: &ConcreteEnv, numb_blocks: usize) -> ResponseResult { let env_inner = env.env_inner(); let mut tx_rw = env_inner.tx_rw()?; + // TODO: try blocks let result = { + // flush all the current alt blocks as they may reference blocks to be popped. crate::ops::alt_block::flush_alt_blocks(&env_inner, &mut tx_rw)?; let mut tables_mut = env_inner.open_tables_mut(&tx_rw)?; - + // generate a `ChainId` for the popped blocks. let old_main_chain_id = ChainId(rand::random()); + // pop the blocks let mut last_block_height = 0; for _ in 0..numb_blocks { (last_block_height, _, _) = crate::ops::block::pop_block(Some(old_main_chain_id), &mut tables_mut)?; } + // Update the alt_chain_info with the correct information. tables_mut.alt_chain_infos_mut().put( &old_main_chain_id.into(), &AltChainInfo { @@ -155,11 +160,14 @@ fn reverse_reorg(env: &ConcreteEnv, chain_id: ChainId) -> ResponseResult { let mut tables_mut = env_inner.open_tables_mut(&tx_rw)?; let chain_info = tables_mut.alt_chain_infos().get(&chain_id.into())?; + // Although this doesn't guarantee the chain was popped from the main-chain, it's an easy + // thing for us to check. assert_eq!(Chain::from(chain_info.parent_chain), Chain::Main); let tob_block_height = crate::ops::blockchain::top_block_height(tables_mut.block_heights())?; + // pop any blocks that were added as part of a re-org. for _ in chain_info.common_ancestor_height..tob_block_height { crate::ops::block::pop_block(None, &mut tables_mut)?; } @@ -177,6 +185,7 @@ fn reverse_reorg(env: &ConcreteEnv, chain_id: ChainId) -> ResponseResult { }) .collect::>(); + // Add the old main chain blocks back to the main chain. for res_alt_block in alt_blocks { let alt_block = res_alt_block?; diff --git a/storage/blockchain/src/tests.rs b/storage/blockchain/src/tests.rs index 60239127..d57a3715 100644 --- a/storage/blockchain/src/tests.rs +++ b/storage/blockchain/src/tests.rs @@ -90,7 +90,10 @@ pub(crate) fn assert_all_tables_are_empty(env: &ConcreteEnv) { assert_eq!(crate::ops::tx::get_num_tx(tables.tx_ids()).unwrap(), 0); } -pub(crate) fn map_verified_block_to_alt(verified_block: VerifiedBlockInformation, chain_id: ChainId) -> AltBlockInformation { +pub(crate) fn map_verified_block_to_alt( + verified_block: VerifiedBlockInformation, + chain_id: ChainId, +) -> AltBlockInformation { AltBlockInformation { block: verified_block.block, block_blob: verified_block.block_blob, @@ -103,4 +106,4 @@ pub(crate) fn map_verified_block_to_alt(verified_block: VerifiedBlockInformation cumulative_difficulty: verified_block.cumulative_difficulty, chain_id, } -} \ No newline at end of file +} diff --git a/types/src/blockchain.rs b/types/src/blockchain.rs index c2a5517d..9f79c3a3 100644 --- a/types/src/blockchain.rs +++ b/types/src/blockchain.rs @@ -3,14 +3,15 @@ //! Tests that assert particular requests lead to particular //! responses are also tested in Cuprate's blockchain database crate. -//---------------------------------------------------------------------------------------------------- Import -use crate::types::{Chain, ExtendedBlockHeader, OutputOnChain, VerifiedBlockInformation}; -use crate::{AltBlockInformation, ChainId}; use std::{ collections::{HashMap, HashSet}, ops::Range, }; +use crate::{AltBlockInformation, ChainId}; +//---------------------------------------------------------------------------------------------------- Import +use crate::types::{Chain, ExtendedBlockHeader, OutputOnChain, VerifiedBlockInformation}; + //---------------------------------------------------------------------------------------------------- ReadRequest /// A read request to the blockchain database. /// @@ -92,12 +93,14 @@ pub enum BlockchainReadRequest { CompactChainHistory, /// A request to find the first unknown block ID in a list of block IDs. - //// + /// /// # Invariant /// The [`Vec`] containing the block IDs must be sorted in chronological block /// order, or else the returned response is unspecified and meaningless, /// as this request performs a binary search. FindFirstUnknown(Vec<[u8; 32]>), + /// A request for all alt blocks in the chain with the given [`ChainId`]. + AltBlocksInChain(ChainId), } //---------------------------------------------------------------------------------------------------- WriteRequest @@ -120,12 +123,14 @@ pub enum BlockchainWriteRequest { /// /// Input is the amount of blocks to pop. /// - /// This request flush all alt-chains from the cache before adding the popped blocks to the alt cache. + /// This request flushes all alt-chains from the cache before adding the popped blocks to the + /// alt cache. PopBlocks(usize), /// A request to reverse the re-org process. /// /// The inner value is the [`ChainId`] of the old main chain. /// + /// # Invariant /// It is invalid to call this with a [`ChainId`] that was not returned from [`BlockchainWriteRequest::PopBlocks`]. ReverseReorg(ChainId), /// A request to flush all alternative blocks. @@ -215,6 +220,11 @@ pub enum BlockchainResponse { /// This will be [`None`] if all blocks were known. FindFirstUnknown(Option<(usize, usize)>), + /// The response for [`BlockchainReadRequest::AltBlocksInChain`]. + /// + /// Contains all the alt blocks in the alt-chain in chronological order. + AltBlocksInChain(Vec), + //------------------------------------------------------ Writes /// A generic Ok response to indicate a request was successfully handled. /// From f92375f6a61d72fa802e4a75ca7cba3f8d968363 Mon Sep 17 00:00:00 2001 From: Boog900 <54e72d8a-345f-4599-bd90-c6b9bc7d0ec5@aleeas.com> Date: Sat, 7 Sep 2024 02:07:23 +0100 Subject: [PATCH 10/15] clippy + fmt --- storage/blockchain/src/service/read.rs | 4 ++-- storage/blockchain/src/types.rs | 16 +++++++--------- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/storage/blockchain/src/service/read.rs b/storage/blockchain/src/service/read.rs index 416e6137..835f2d6a 100644 --- a/storage/blockchain/src/service/read.rs +++ b/storage/blockchain/src/service/read.rs @@ -14,13 +14,14 @@ use rayon::{ use thread_local::ThreadLocal; use cuprate_database::{ConcreteEnv, DatabaseRo, Env, EnvInner, RuntimeError}; -use cuprate_database_service::{DatabaseReadService, init_thread_pool, ReaderThreads}; +use cuprate_database_service::{init_thread_pool, DatabaseReadService, ReaderThreads}; use cuprate_helper::map::combine_low_high_bits_to_u128; use cuprate_types::{ blockchain::{BlockchainReadRequest, BlockchainResponse}, Chain, ChainId, ExtendedBlockHeader, OutputOnChain, }; +use crate::ops::alt_block::get_alt_block; use crate::{ ops::{ alt_block::{ @@ -43,7 +44,6 @@ use crate::{ AltBlockHeight, Amount, AmountIndex, BlockHash, BlockHeight, KeyImage, PreRctOutputId, }, }; -use crate::ops::alt_block::get_alt_block; //---------------------------------------------------------------------------------------------------- init_read_service /// Initialize the [`BlockchainReadHandle`] thread-pool backed by [`rayon`]. diff --git a/storage/blockchain/src/types.rs b/storage/blockchain/src/types.rs index 14917249..13c44994 100644 --- a/storage/blockchain/src/types.rs +++ b/storage/blockchain/src/types.rs @@ -44,12 +44,12 @@ use std::num::NonZero; use bytemuck::{Pod, Zeroable}; - #[cfg(feature = "serde")] use serde::{Deserialize, Serialize}; use cuprate_database::{Key, StorableVec}; use cuprate_types::{Chain, ChainId}; + //---------------------------------------------------------------------------------------------------- Aliases // These type aliases exist as many Monero-related types are the exact same. // For clarity, they're given type aliases as to not confuse them. @@ -334,17 +334,15 @@ pub struct RawChain(u64); impl From for RawChain { fn from(value: Chain) -> Self { match value { - Chain::Main => RawChain(0), - Chain::Alt(chain_id) => RawChain(chain_id.0.get()), + Chain::Main => Self(0), + Chain::Alt(chain_id) => Self(chain_id.0.get()), } } } impl From for Chain { fn from(value: RawChain) -> Self { - NonZero::new(value.0) - .map(|id| Chain::Alt(ChainId(id))) - .unwrap_or(Chain::Main) + NonZero::new(value.0).map_or(Self::Main, |id| Self::Alt(ChainId(id))) } } @@ -352,7 +350,7 @@ impl From for RawChain { fn from(value: RawChainId) -> Self { assert_ne!(value.0, 0); - RawChain(value.0) + Self(value.0) } } @@ -363,13 +361,13 @@ pub struct RawChainId(u64); impl From for RawChainId { fn from(value: ChainId) -> Self { - RawChainId(value.0.get()) + Self(value.0.get()) } } impl From for ChainId { fn from(value: RawChainId) -> Self { - ChainId(NonZero::new(value.0).expect("RawChainId mut not have a value of `0`")) + Self(NonZero::new(value.0).expect("RawChainId mut not have a value of `0`")) } } From a864f934be9f2e42cbda24d66dfb83ee9ff84f91 Mon Sep 17 00:00:00 2001 From: Boog900 <54e72d8a-345f-4599-bd90-c6b9bc7d0ec5@aleeas.com> Date: Sat, 7 Sep 2024 02:45:45 +0100 Subject: [PATCH 11/15] document types --- storage/blockchain/src/types.rs | 164 ++++++++++++++++++++++++++++++-- types/src/blockchain.rs | 4 - 2 files changed, 157 insertions(+), 11 deletions(-) diff --git a/storage/blockchain/src/types.rs b/storage/blockchain/src/types.rs index 13c44994..e9665695 100644 --- a/storage/blockchain/src/types.rs +++ b/storage/blockchain/src/types.rs @@ -327,6 +327,29 @@ pub struct RctOutput { // TODO: local_index? //---------------------------------------------------------------------------------------------------- RawChain +/// [`Chain`] in a format which can be stored in the DB. +/// +/// Implements [`Into`] and [`From`] for [`Chain`]. +/// +/// ```rust +/// # use std::borrow::*; +/// # use cuprate_blockchain::{*, types::*}; +/// use cuprate_database::Storable; +/// use cuprate_types::Chain; +/// +/// // Assert Storable is correct. +/// let a: RawChain = Chain::Main.into(); +/// let b = Storable::as_bytes(&a); +/// let c: RawChain = Storable::from_bytes(b); +/// assert_eq!(a, c); +/// ``` +/// +/// # Size & Alignment +/// ```rust +/// # use cuprate_blockchain::types::*; +/// assert_eq!(size_of::(), 8); +/// assert_eq!(align_of::(), 8); +/// ``` #[derive(Copy, Clone, Debug, PartialEq, PartialOrd, Eq, Ord, Hash, Pod, Zeroable)] #[repr(transparent)] pub struct RawChain(u64); @@ -348,6 +371,7 @@ impl From for Chain { impl From for RawChain { fn from(value: RawChainId) -> Self { + // A [`ChainID`] with an inner value of `0` is invalid. assert_ne!(value.0, 0); Self(value.0) @@ -355,6 +379,29 @@ impl From for RawChain { } //---------------------------------------------------------------------------------------------------- RawChainId +/// [`ChainId`] in a format which can be stored in the DB. +/// +/// Implements [`Into`] and [`From`] for [`ChainId`]. +/// +/// ```rust +/// # use std::borrow::*; +/// # use cuprate_blockchain::{*, types::*}; +/// use cuprate_database::Storable; +/// use cuprate_types::ChainId; +/// +/// // Assert Storable is correct. +/// let a: RawChainId = ChainId(10.try_into().unwrap()).into(); +/// let b = Storable::as_bytes(&a); +/// let c: RawChainId = Storable::from_bytes(b); +/// assert_eq!(a, c); +/// ``` +/// +/// # Size & Alignment +/// ```rust +/// # use cuprate_blockchain::types::*; +/// assert_eq!(size_of::(), 8); +/// assert_eq!(align_of::(), 8); +/// ``` #[derive(Copy, Clone, Debug, PartialEq, PartialOrd, Eq, Ord, Hash, Pod, Zeroable)] #[repr(transparent)] pub struct RawChainId(u64); @@ -374,31 +421,112 @@ impl From for ChainId { impl Key for RawChainId {} //---------------------------------------------------------------------------------------------------- AltChainInfo +/// Information on an alternative chain. +/// +/// ```rust +/// # use std::borrow::*; +/// # use cuprate_blockchain::{*, types::*}; +/// use cuprate_database::Storable; +/// use cuprate_types::Chain; +/// +/// // Assert Storable is correct. +/// let a: AltChainInfo = AltChainInfo { +/// parent_chain: Chain::Main.into(), +/// common_ancestor_height: 0, +/// chain_height: 1, +/// }; +/// let b = Storable::as_bytes(&a); +/// let c: AltChainInfo = Storable::from_bytes(b); +/// assert_eq!(a, c); +/// ``` +/// +/// # Size & Alignment +/// ```rust +/// # use cuprate_blockchain::types::*; +/// assert_eq!(size_of::(), 24); +/// assert_eq!(align_of::(), 8); +/// ``` #[derive(Copy, Clone, Debug, PartialEq, PartialOrd, Eq, Ord, Hash, Pod, Zeroable)] #[repr(C)] pub struct AltChainInfo { + /// The chain this alt chain forks from. pub parent_chain: RawChain, + /// The height of the first block we share with the parent chain. pub common_ancestor_height: usize, + /// The chain height of the blocks in this alt chain. pub chain_height: usize, } //---------------------------------------------------------------------------------------------------- AltBlockHeight +/// Represents the height of a block on an alt-chain. +/// +/// ```rust +/// # use std::borrow::*; +/// # use cuprate_blockchain::{*, types::*}; +/// use cuprate_database::Storable; +/// use cuprate_types::ChainId; +/// +/// // Assert Storable is correct. +/// let a: AltBlockHeight = AltBlockHeight { +/// chain_id: ChainId(1.try_into().unwrap()).into(), +/// height: 1, +/// }; +/// let b = Storable::as_bytes(&a); +/// let c: AltBlockHeight = Storable::from_bytes(b); +/// assert_eq!(a, c); +/// ``` +/// +/// # Size & Alignment +/// ```rust +/// # use cuprate_blockchain::types::*; +/// assert_eq!(size_of::(), 16); +/// assert_eq!(align_of::(), 8); +/// ``` #[derive(Copy, Clone, Debug, PartialEq, PartialOrd, Eq, Ord, Hash, Pod, Zeroable)] #[repr(C)] pub struct AltBlockHeight { + /// The [`ChainId`] of the chain this alt block is on, in raw form. pub chain_id: RawChainId, + /// The height of this alt-block. pub height: usize, } impl Key for AltBlockHeight {} //---------------------------------------------------------------------------------------------------- CompactAltBlockInfo +/// Represents information on an alt-chain. +/// +/// ```rust +/// # use std::borrow::*; +/// # use cuprate_blockchain::{*, types::*}; +/// use cuprate_database::Storable; +/// +/// // Assert Storable is correct. +/// let a: CompactAltBlockInfo = CompactAltBlockInfo { +/// block_hash: [1; 32], +/// pow_hash: [2; 32], +/// height: 10, +/// weight: 20, +/// long_term_weight: 30, +/// cumulative_difficulty_low: 40, +/// cumulative_difficulty_high: 50, +/// }; +/// +/// let b = Storable::as_bytes(&a); +/// let c: CompactAltBlockInfo = Storable::from_bytes(b); +/// assert_eq!(a, c); +/// ``` +/// +/// # Size & Alignment +/// ```rust +/// # use cuprate_blockchain::types::*; +/// assert_eq!(size_of::(), 104); +/// assert_eq!(align_of::(), 8); +/// ``` #[derive(Copy, Clone, Debug, PartialEq, PartialOrd, Eq, Ord, Hash, Pod, Zeroable)] #[repr(C)] pub struct CompactAltBlockInfo { /// The block's hash. - /// - /// [`Block::hash`]. pub block_hash: [u8; 32], /// The block's proof-of-work hash. pub pow_hash: [u8; 32], @@ -408,24 +536,46 @@ pub struct CompactAltBlockInfo { pub weight: usize, /// The long term block weight, which is the weight factored in with previous block weights. pub long_term_weight: usize, - /// The cumulative difficulty of all blocks up until and including this block. + /// The low 64 bits of the cumulative difficulty. pub cumulative_difficulty_low: u64, + /// The high 64 bits of the cumulative difficulty. pub cumulative_difficulty_high: u64, } //---------------------------------------------------------------------------------------------------- AltTransactionInfo +/// Represents information on an alt transaction. +/// +/// ```rust +/// # use std::borrow::*; +/// # use cuprate_blockchain::{*, types::*}; +/// use cuprate_database::Storable; +/// +/// // Assert Storable is correct. +/// let a: AltTransactionInfo = AltTransactionInfo { +/// tx_weight: 1, +/// fee: 6, +/// tx_hash: [6; 32], +/// }; +/// +/// let b = Storable::as_bytes(&a); +/// let c: AltTransactionInfo = Storable::from_bytes(b); +/// assert_eq!(a, c); +/// ``` +/// +/// # Size & Alignment +/// ```rust +/// # use cuprate_blockchain::types::*; +/// assert_eq!(size_of::(), 48); +/// assert_eq!(align_of::(), 8); +/// ``` #[derive(Copy, Clone, Debug, PartialEq, PartialOrd, Eq, Ord, Hash, Pod, Zeroable)] #[repr(C)] pub struct AltTransactionInfo { /// The transaction's weight. - /// - /// [`Transaction::weight`]. pub tx_weight: usize, /// The transaction's total fees. pub fee: u64, /// The transaction's hash. - /// - /// [`Transaction::hash`]. pub tx_hash: [u8; 32], } diff --git a/types/src/blockchain.rs b/types/src/blockchain.rs index 9f79c3a3..6c7ecf37 100644 --- a/types/src/blockchain.rs +++ b/types/src/blockchain.rs @@ -105,10 +105,6 @@ pub enum BlockchainReadRequest { //---------------------------------------------------------------------------------------------------- WriteRequest /// A write request to the blockchain database. -/// -/// There is currently only 1 write request to the database, -/// as such, the only valid [`BlockchainResponse`] to this request is -/// the proper response for a [`BlockchainResponse::WriteBlockOk`]. #[derive(Debug, Clone, PartialEq, Eq)] pub enum BlockchainWriteRequest { /// Request that a block be written to the database. From 6119972fe81ff9969a64cc07cbcb2f8bd7fc9220 Mon Sep 17 00:00:00 2001 From: Boog900 <54e72d8a-345f-4599-bd90-c6b9bc7d0ec5@aleeas.com> Date: Sun, 8 Sep 2024 02:08:20 +0100 Subject: [PATCH 12/15] move tx_fee to helper --- helper/Cargo.toml | 4 ++- helper/src/lib.rs | 2 ++ helper/src/tx_utils.rs | 34 ++++++++++++++++++++++++ storage/blockchain/src/free.rs | 32 ----------------------- storage/blockchain/src/ops/block.rs | 8 +++--- test-utils/Cargo.toml | 2 +- test-utils/src/data/statics.rs | 40 ++++------------------------- 7 files changed, 50 insertions(+), 72 deletions(-) create mode 100644 helper/src/tx_utils.rs diff --git a/helper/Cargo.toml b/helper/Cargo.toml index c74e40fd..4bd17ca8 100644 --- a/helper/Cargo.toml +++ b/helper/Cargo.toml @@ -9,8 +9,9 @@ repository = "https://github.com/Cuprate/cuprate/tree/main/consensus" [features] +# TODO: I don't think this is a good idea # All features on by default. -default = ["std", "atomic", "asynch", "cast", "fs", "num", "map", "time", "thread", "constants"] +default = ["std", "atomic", "asynch", "cast", "fs", "num", "map", "time", "thread", "constants", "tx-utils"] std = [] atomic = ["dep:crossbeam"] asynch = ["dep:futures", "dep:rayon"] @@ -21,6 +22,7 @@ num = [] map = ["cast", "dep:monero-serai"] time = ["dep:chrono", "std"] thread = ["std", "dep:target_os_lib"] +tx-utils = ["dep:monero-serai"] [dependencies] crossbeam = { workspace = true, optional = true } diff --git a/helper/src/lib.rs b/helper/src/lib.rs index de0d9555..e82ec827 100644 --- a/helper/src/lib.rs +++ b/helper/src/lib.rs @@ -31,6 +31,8 @@ pub mod thread; #[cfg(feature = "time")] pub mod time; +#[cfg(feature = "tx-utils")] +pub mod tx_utils; //---------------------------------------------------------------------------------------------------- Private Usage //---------------------------------------------------------------------------------------------------- diff --git a/helper/src/tx_utils.rs b/helper/src/tx_utils.rs new file mode 100644 index 00000000..aeccf32b --- /dev/null +++ b/helper/src/tx_utils.rs @@ -0,0 +1,34 @@ +//! Utils for working with [`Transaction`] + +use monero_serai::transaction::{Input, Transaction}; + +/// Calculates the fee of the [`Transaction`]. +/// +/// # Panics +/// This will panic if the inputs overflow or the transaction outputs too much, so should only +/// be used on known to be valid txs. +pub fn tx_fee(tx: &Transaction) -> u64 { + let mut fee = 0_u64; + + match &tx { + Transaction::V1 { prefix, .. } => { + for input in &prefix.inputs { + match input { + Input::Gen(_) => return 0, + Input::ToKey { amount, .. } => { + fee = fee.checked_add(amount.unwrap_or(0)).unwrap(); + } + } + } + + for output in &prefix.outputs { + fee.checked_sub(output.amount.unwrap_or(0)).unwrap(); + } + } + Transaction::V2 { proofs, .. } => { + fee = proofs.as_ref().unwrap().base.fee; + } + }; + + fee +} diff --git a/storage/blockchain/src/free.rs b/storage/blockchain/src/free.rs index 20d56226..8288e65f 100644 --- a/storage/blockchain/src/free.rs +++ b/storage/blockchain/src/free.rs @@ -1,6 +1,5 @@ //! General free functions (related to the database). -use monero_serai::transaction::{Input, Transaction}; //---------------------------------------------------------------------------------------------------- Import use cuprate_database::{ConcreteEnv, Env, EnvInner, InitError, RuntimeError, TxRw}; @@ -62,37 +61,6 @@ pub fn open(config: Config) -> Result { Ok(env) } -//---------------------------------------------------------------------------------------------------- Tx Fee -/// Calculates the fee of the [`Transaction`]. -/// -/// # Panics -/// This will panic if the inputs overflow or the transaction outputs too much. -pub(crate) fn tx_fee(tx: &Transaction) -> u64 { - let mut fee = 0_u64; - - match &tx { - Transaction::V1 { prefix, .. } => { - for input in &prefix.inputs { - match input { - Input::Gen(_) => return 0, - Input::ToKey { amount, .. } => { - fee = fee.checked_add(amount.unwrap_or(0)).unwrap(); - } - } - } - - for output in &prefix.outputs { - fee.checked_sub(output.amount.unwrap_or(0)).unwrap(); - } - } - Transaction::V2 { proofs, .. } => { - fee = proofs.as_ref().unwrap().base.fee; - } - }; - - fee -} - //---------------------------------------------------------------------------------------------------- Tests #[cfg(test)] mod test { diff --git a/storage/blockchain/src/ops/block.rs b/storage/blockchain/src/ops/block.rs index 45bab41c..29586925 100644 --- a/storage/blockchain/src/ops/block.rs +++ b/storage/blockchain/src/ops/block.rs @@ -7,16 +7,18 @@ use monero_serai::block::{Block, BlockHeader}; use cuprate_database::{ RuntimeError, StorableVec, {DatabaseRo, DatabaseRw}, }; -use cuprate_helper::map::{combine_low_high_bits_to_u128, split_u128_into_low_high_bits}; +use cuprate_helper::{ + map::{combine_low_high_bits_to_u128, split_u128_into_low_high_bits}, + tx_utils::tx_fee, +}; use cuprate_types::{ AltBlockInformation, ChainId, ExtendedBlockHeader, HardFork, VerifiedBlockInformation, VerifiedTransactionInformation, }; -use crate::free::tx_fee; -use crate::ops::alt_block; use crate::{ ops::{ + alt_block, blockchain::{chain_height, cumulative_generated_coins}, macros::doc_error, output::get_rct_num_outputs, diff --git a/test-utils/Cargo.toml b/test-utils/Cargo.toml index a96a9cfc..9c64bd81 100644 --- a/test-utils/Cargo.toml +++ b/test-utils/Cargo.toml @@ -7,7 +7,7 @@ authors = ["Boog900", "hinto-janai"] [dependencies] cuprate-types = { path = "../types" } -cuprate-helper = { path = "../helper", features = ["map"] } +cuprate-helper = { path = "../helper", features = ["map", "tx-utils"] } cuprate-wire = { path = "../net/wire" } cuprate-p2p-core = { path = "../p2p/p2p-core", features = ["borsh"] } diff --git a/test-utils/src/data/statics.rs b/test-utils/src/data/statics.rs index 8b98171a..a45cc13f 100644 --- a/test-utils/src/data/statics.rs +++ b/test-utils/src/data/statics.rs @@ -8,12 +8,12 @@ //---------------------------------------------------------------------------------------------------- Import use std::sync::LazyLock; -use cuprate_helper::map::combine_low_high_bits_to_u128; -use cuprate_types::{VerifiedBlockInformation, VerifiedTransactionInformation}; use hex_literal::hex; -use monero_serai::transaction::Input; use monero_serai::{block::Block, transaction::Transaction}; +use cuprate_helper::{map::combine_low_high_bits_to_u128, tx_utils::tx_fee}; +use cuprate_types::{VerifiedBlockInformation, VerifiedTransactionInformation}; + use crate::data::constants::{ BLOCK_43BD1F, BLOCK_5ECB7E, BLOCK_F91043, TX_2180A8, TX_3BC7FF, TX_84D48D, TX_9E3F73, TX_B6B439, TX_D7FEBD, TX_E2D393, TX_E57440, @@ -110,36 +110,6 @@ fn to_tx_verification_data(tx_blob: impl AsRef<[u8]>) -> VerifiedTransactionInfo } } -/// Calculates the fee of the [`Transaction`]. -/// -/// # Panics -/// This will panic if the inputs overflow or the transaction outputs too much. -pub fn tx_fee(tx: &Transaction) -> u64 { - let mut fee = 0_u64; - - match &tx { - Transaction::V1 { prefix, .. } => { - for input in &prefix.inputs { - match input { - Input::Gen(_) => return 0, - Input::ToKey { amount, .. } => { - fee = fee.checked_add(amount.unwrap_or(0)).unwrap(); - } - } - } - - for output in &prefix.outputs { - fee.checked_sub(output.amount.unwrap_or(0)).unwrap(); - } - } - Transaction::V2 { proofs, .. } => { - fee = proofs.as_ref().unwrap().base.fee; - } - }; - - fee -} - //---------------------------------------------------------------------------------------------------- Blocks /// Generate a `static LazyLock`. /// @@ -311,12 +281,12 @@ transaction_verification_data! { //---------------------------------------------------------------------------------------------------- TESTS #[cfg(test)] mod tests { - use super::*; - use pretty_assertions::assert_eq; use crate::rpc::client::HttpRpcClient; + use super::*; + /// Assert the defined blocks are the same compared to ones received from a local RPC call. #[ignore] // FIXME: doesn't work in CI, we need a real unrestricted node #[tokio::test] From b211210fa24c220dac1c46824430e1a1accaaebf Mon Sep 17 00:00:00 2001 From: Boog900 <54e72d8a-345f-4599-bd90-c6b9bc7d0ec5@aleeas.com> Date: Sun, 8 Sep 2024 15:34:30 +0100 Subject: [PATCH 13/15] more doc updates --- storage/blockchain/src/ops/alt_block/block.rs | 7 ++----- storage/blockchain/src/ops/alt_block/mod.rs | 6 +++--- storage/blockchain/src/ops/alt_block/tx.rs | 8 +++++--- storage/blockchain/src/service/free.rs | 14 ++++++++++---- storage/blockchain/src/service/read.rs | 7 +++---- storage/blockchain/src/service/write.rs | 17 +++++++++-------- storage/blockchain/src/tables.rs | 16 ++++++++++++++++ test-utils/src/data/mod.rs | 6 ++---- test-utils/src/rpc/client.rs | 13 ++++++------- 9 files changed, 56 insertions(+), 38 deletions(-) diff --git a/storage/blockchain/src/ops/alt_block/block.rs b/storage/blockchain/src/ops/alt_block/block.rs index 07878d55..5bc052b7 100644 --- a/storage/blockchain/src/ops/alt_block/block.rs +++ b/storage/blockchain/src/ops/alt_block/block.rs @@ -24,12 +24,9 @@ use crate::{ /// /// # Panics /// This function will panic if: -/// - `block.height` is == `0` +/// - `alt_block.height` is == `0` +/// - `alt_block.txs.len()` != `alt_block.block.transactions.len()` /// -/// # Already exists -/// This function will operate normally even if `block` already -/// exists, i.e., this function will not return `Err` even if you -/// call this function infinitely with the same block. pub fn add_alt_block( alt_block: &AltBlockInformation, tables: &mut impl TablesMut, diff --git a/storage/blockchain/src/ops/alt_block/mod.rs b/storage/blockchain/src/ops/alt_block/mod.rs index 36e4768e..96c1cecf 100644 --- a/storage/blockchain/src/ops/alt_block/mod.rs +++ b/storage/blockchain/src/ops/alt_block/mod.rs @@ -18,8 +18,8 @@ //! ``` //! //! In that example if we were to receive an alt-block which immediately follows the top block of `ChainID(X)` -//! then that block will also be stored under `ChainID(X)`. However if it follows from another block from `ChainID(X)` -//! we will split into a chain with a different ID. +//! then that block will also be stored under `ChainID(X)`. However, if it follows from another block from `ChainID(X)` +//! we will split into a chain with a different ID: //! //! ```text //! | @@ -39,7 +39,7 @@ //! [`get_alt_chain_history_ranges`] covers this and is the method to get the ranges of heights needed from each [`ChainID`](cuprate_types::ChainId) //! to get all the alt-blocks in a given [`ChainID`](cuprate_types::ChainId). //! -//! Although this should be kept in mind as a possibility because Cuprate's block downloader will only track a single chain it is +//! Although this should be kept in mind as a possibility, because Cuprate's block downloader will only track a single chain it is //! unlikely that we will be tracking [`ChainID`](cuprate_types::ChainId) that don't immediately connect to the main-chain. //! //! ## Why not use block's previous field? diff --git a/storage/blockchain/src/ops/alt_block/tx.rs b/storage/blockchain/src/ops/alt_block/tx.rs index 5671e4b4..aa2e82f4 100644 --- a/storage/blockchain/src/ops/alt_block/tx.rs +++ b/storage/blockchain/src/ops/alt_block/tx.rs @@ -4,9 +4,11 @@ use monero_serai::transaction::Transaction; use cuprate_database::{DatabaseRo, DatabaseRw, RuntimeError, StorableVec}; use cuprate_types::VerifiedTransactionInformation; -use crate::ops::macros::{doc_add_alt_block_inner_invariant, doc_error}; -use crate::tables::{Tables, TablesMut}; -use crate::types::{AltTransactionInfo, TxHash}; +use crate::{ + ops::macros::{doc_add_alt_block_inner_invariant, doc_error}, + tables::{Tables, TablesMut}, + types::{AltTransactionInfo, TxHash}, +}; /// Adds a [`VerifiedTransactionInformation`] form an alt-block to the DB, if /// that transaction is not already in the DB. diff --git a/storage/blockchain/src/service/free.rs b/storage/blockchain/src/service/free.rs index 7a4e6ce6..d12844f3 100644 --- a/storage/blockchain/src/service/free.rs +++ b/storage/blockchain/src/service/free.rs @@ -3,13 +3,14 @@ //---------------------------------------------------------------------------------------------------- Import use std::sync::Arc; -use crate::service::{init_read_service, init_write_service}; +use cuprate_database::{ConcreteEnv, InitError}; +use cuprate_types::{AltBlockInformation, VerifiedBlockInformation}; + use crate::{ config::Config, service::types::{BlockchainReadHandle, BlockchainWriteHandle}, }; -use cuprate_database::{ConcreteEnv, InitError}; -use cuprate_types::{AltBlockInformation, VerifiedBlockInformation}; +use crate::service::{init_read_service, init_write_service}; //---------------------------------------------------------------------------------------------------- Init #[cold] @@ -81,7 +82,12 @@ pub(super) const fn compact_history_genesis_not_included INITIAL_BLOCKS && !(top_block_height - INITIAL_BLOCKS + 2).is_power_of_two() } -//---------------------------------------------------------------------------------------------------- Compact history +//---------------------------------------------------------------------------------------------------- Map Block +/// Maps [`AltBlockInformation`] to [`VerifiedBlockInformation`] +/// +/// # Panics +/// This will panic if the block is invalid, so should only be used on blocks that have been popped from +/// the main-chain. pub(super) fn map_valid_alt_block_to_verified_block( alt_block: AltBlockInformation, ) -> VerifiedBlockInformation { diff --git a/storage/blockchain/src/service/read.rs b/storage/blockchain/src/service/read.rs index 835f2d6a..3d0379e0 100644 --- a/storage/blockchain/src/service/read.rs +++ b/storage/blockchain/src/service/read.rs @@ -1,11 +1,11 @@ //! Database reader thread-pool definitions and logic. +//---------------------------------------------------------------------------------------------------- Import use std::{ collections::{HashMap, HashSet}, sync::Arc, }; -//---------------------------------------------------------------------------------------------------- Import use rayon::{ iter::{IntoParallelIterator, ParallelIterator}, prelude::*, @@ -14,18 +14,17 @@ use rayon::{ use thread_local::ThreadLocal; use cuprate_database::{ConcreteEnv, DatabaseRo, Env, EnvInner, RuntimeError}; -use cuprate_database_service::{init_thread_pool, DatabaseReadService, ReaderThreads}; +use cuprate_database_service::{DatabaseReadService, init_thread_pool, ReaderThreads}; use cuprate_helper::map::combine_low_high_bits_to_u128; use cuprate_types::{ blockchain::{BlockchainReadRequest, BlockchainResponse}, Chain, ChainId, ExtendedBlockHeader, OutputOnChain, }; -use crate::ops::alt_block::get_alt_block; use crate::{ ops::{ alt_block::{ - get_alt_block_extended_header_from_height, get_alt_block_hash, + get_alt_block, get_alt_block_extended_header_from_height, get_alt_block_hash, get_alt_chain_history_ranges, }, block::{ diff --git a/storage/blockchain/src/service/write.rs b/storage/blockchain/src/service/write.rs index 849a3030..723843f5 100644 --- a/storage/blockchain/src/service/write.rs +++ b/storage/blockchain/src/service/write.rs @@ -5,8 +5,8 @@ use std::sync::Arc; use cuprate_database::{ConcreteEnv, DatabaseRo, DatabaseRw, Env, EnvInner, RuntimeError, TxRw}; use cuprate_database_service::DatabaseWriteHandle; use cuprate_types::{ - blockchain::{BlockchainResponse, BlockchainWriteRequest}, - AltBlockInformation, Chain, ChainId, VerifiedBlockInformation, + AltBlockInformation, + blockchain::{BlockchainResponse, BlockchainWriteRequest}, Chain, ChainId, VerifiedBlockInformation, }; use crate::{ @@ -107,8 +107,8 @@ fn pop_blocks(env: &ConcreteEnv, numb_blocks: usize) -> ResponseResult { let env_inner = env.env_inner(); let mut tx_rw = env_inner.tx_rw()?; - // TODO: try blocks - let result = { + // TODO: turn this function into a try block once stable. + let mut result = || { // flush all the current alt blocks as they may reference blocks to be popped. crate::ops::alt_block::flush_alt_blocks(&env_inner, &mut tx_rw)?; @@ -136,7 +136,7 @@ fn pop_blocks(env: &ConcreteEnv, numb_blocks: usize) -> ResponseResult { Ok(old_main_chain_id) }; - match result { + match result() { Ok(old_main_chain_id) => { TxRw::commit(tx_rw)?; Ok(BlockchainResponse::PopBlocks(old_main_chain_id)) @@ -156,7 +156,8 @@ fn reverse_reorg(env: &ConcreteEnv, chain_id: ChainId) -> ResponseResult { let env_inner = env.env_inner(); let tx_rw = env_inner.tx_rw()?; - let result = { + // TODO: turn this function into a try block once stable. + let result = || { let mut tables_mut = env_inner.open_tables_mut(&tx_rw)?; let chain_info = tables_mut.alt_chain_infos().get(&chain_id.into())?; @@ -197,7 +198,7 @@ fn reverse_reorg(env: &ConcreteEnv, chain_id: ChainId) -> ResponseResult { Ok(()) }; - match result { + match result() { Ok(()) => { TxRw::commit(tx_rw)?; Ok(BlockchainResponse::Ok) @@ -218,7 +219,7 @@ fn flush_alt_blocks(env: &ConcreteEnv) -> ResponseResult { let env_inner = env.env_inner(); let mut tx_rw = env_inner.tx_rw()?; - let result = { crate::ops::alt_block::flush_alt_blocks(&env_inner, &mut tx_rw) }; + let result = crate::ops::alt_block::flush_alt_blocks(&env_inner, &mut tx_rw); match result { Ok(()) => { diff --git a/storage/blockchain/src/tables.rs b/storage/blockchain/src/tables.rs index fa568ae6..74d2e591 100644 --- a/storage/blockchain/src/tables.rs +++ b/storage/blockchain/src/tables.rs @@ -131,21 +131,37 @@ cuprate_database::define_tables! { 14 => TxUnlockTime, TxId => UnlockTime, + /// Information on alt-chains. 15 => AltChainInfos, RawChainId => AltChainInfo, + /// Alt-block heights. + /// + /// Contains the height of all alt-blocks. 16 => AltBlockHeights, BlockHash => AltBlockHeight, + /// Alt-block information. + /// + /// Contains information on all alt-blocks. 17 => AltBlocksInfo, AltBlockHeight => CompactAltBlockInfo, + /// Alt-block blobs. + /// + /// Contains the raw bytes of all alt-blocks. 18 => AltBlockBlobs, AltBlockHeight => BlockBlob, + /// Alt-Block transactions blobs. + /// + /// Contains the raw bytes of alt transactions, if those transactions are not in the main-chain. 19 => AltTransactionBlobs, TxHash => TxBlob, + /// Alt-Block transactions information. + /// + /// Contains information on all alt transactions, even if they are in the main-chain. 20 => AltTransactionInfos, TxHash => AltTransactionInfo, } diff --git a/test-utils/src/data/mod.rs b/test-utils/src/data/mod.rs index b9d42fb8..3be409fe 100644 --- a/test-utils/src/data/mod.rs +++ b/test-utils/src/data/mod.rs @@ -25,13 +25,11 @@ //! let tx: VerifiedTransactionInformation = TX_V1_SIG0.clone(); //! ``` -mod constants; pub use constants::{ BLOCK_43BD1F, BLOCK_5ECB7E, BLOCK_BBD604, BLOCK_F91043, TX_2180A8, TX_3BC7FF, TX_84D48D, TX_9E3F73, TX_B6B439, TX_D7FEBD, TX_E2D393, TX_E57440, }; +pub use statics::{BLOCK_V16_TX0, BLOCK_V1_TX2, BLOCK_V9_TX3, TX_V1_SIG0, TX_V1_SIG2, TX_V2_RCT3}; +mod constants; mod statics; -pub use statics::{ - tx_fee, BLOCK_V16_TX0, BLOCK_V1_TX2, BLOCK_V9_TX3, TX_V1_SIG0, TX_V1_SIG2, TX_V2_RCT3, -}; diff --git a/test-utils/src/rpc/client.rs b/test-utils/src/rpc/client.rs index fbe6fb9e..3711334b 100644 --- a/test-utils/src/rpc/client.rs +++ b/test-utils/src/rpc/client.rs @@ -1,18 +1,16 @@ //! HTTP RPC client. //---------------------------------------------------------------------------------------------------- Use +use monero_rpc::Rpc; +use monero_serai::block::Block; +use monero_simple_request_rpc::SimpleRequestRpc; use serde::Deserialize; use serde_json::json; use tokio::task::spawn_blocking; -use monero_rpc::Rpc; -use monero_serai::block::Block; -use monero_simple_request_rpc::SimpleRequestRpc; - +use cuprate_helper::tx_utils::tx_fee; use cuprate_types::{VerifiedBlockInformation, VerifiedTransactionInformation}; -use crate::data::tx_fee; - //---------------------------------------------------------------------------------------------------- Constants /// The default URL used for Monero RPC connections. pub const LOCALHOST_RPC_URL: &str = "http://127.0.0.1:18081"; @@ -184,9 +182,10 @@ impl HttpRpcClient { //---------------------------------------------------------------------------------------------------- TESTS #[cfg(test)] mod tests { - use super::*; use hex_literal::hex; + use super::*; + /// Assert the default address is localhost. #[tokio::test] async fn localhost() { From 68807e7563a86ac189b005797a687dd513a3b10d Mon Sep 17 00:00:00 2001 From: Boog900 <54e72d8a-345f-4599-bd90-c6b9bc7d0ec5@aleeas.com> Date: Sun, 8 Sep 2024 15:37:01 +0100 Subject: [PATCH 14/15] fmt --- storage/blockchain/src/service/free.rs | 6 ++++-- storage/blockchain/src/service/read.rs | 2 +- storage/blockchain/src/service/write.rs | 4 ++-- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/storage/blockchain/src/service/free.rs b/storage/blockchain/src/service/free.rs index d12844f3..d8a878c9 100644 --- a/storage/blockchain/src/service/free.rs +++ b/storage/blockchain/src/service/free.rs @@ -8,9 +8,11 @@ use cuprate_types::{AltBlockInformation, VerifiedBlockInformation}; use crate::{ config::Config, - service::types::{BlockchainReadHandle, BlockchainWriteHandle}, + service::{ + init_read_service, init_write_service, + types::{BlockchainReadHandle, BlockchainWriteHandle}, + }, }; -use crate::service::{init_read_service, init_write_service}; //---------------------------------------------------------------------------------------------------- Init #[cold] diff --git a/storage/blockchain/src/service/read.rs b/storage/blockchain/src/service/read.rs index 3d0379e0..73b2d220 100644 --- a/storage/blockchain/src/service/read.rs +++ b/storage/blockchain/src/service/read.rs @@ -14,7 +14,7 @@ use rayon::{ use thread_local::ThreadLocal; use cuprate_database::{ConcreteEnv, DatabaseRo, Env, EnvInner, RuntimeError}; -use cuprate_database_service::{DatabaseReadService, init_thread_pool, ReaderThreads}; +use cuprate_database_service::{init_thread_pool, DatabaseReadService, ReaderThreads}; use cuprate_helper::map::combine_low_high_bits_to_u128; use cuprate_types::{ blockchain::{BlockchainReadRequest, BlockchainResponse}, diff --git a/storage/blockchain/src/service/write.rs b/storage/blockchain/src/service/write.rs index 723843f5..4038d94a 100644 --- a/storage/blockchain/src/service/write.rs +++ b/storage/blockchain/src/service/write.rs @@ -5,8 +5,8 @@ use std::sync::Arc; use cuprate_database::{ConcreteEnv, DatabaseRo, DatabaseRw, Env, EnvInner, RuntimeError, TxRw}; use cuprate_database_service::DatabaseWriteHandle; use cuprate_types::{ - AltBlockInformation, - blockchain::{BlockchainResponse, BlockchainWriteRequest}, Chain, ChainId, VerifiedBlockInformation, + blockchain::{BlockchainResponse, BlockchainWriteRequest}, + AltBlockInformation, Chain, ChainId, VerifiedBlockInformation, }; use crate::{ From c03065bcd5569a631cc28687667445bfd0e5e151 Mon Sep 17 00:00:00 2001 From: Boog900 <54e72d8a-345f-4599-bd90-c6b9bc7d0ec5@aleeas.com> Date: Sun, 8 Sep 2024 18:42:25 +0100 Subject: [PATCH 15/15] fix imports --- types/src/blockchain.rs | 10 ++++++---- types/src/types.rs | 3 ++- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/types/src/blockchain.rs b/types/src/blockchain.rs index 6c7ecf37..f246e59a 100644 --- a/types/src/blockchain.rs +++ b/types/src/blockchain.rs @@ -2,15 +2,17 @@ //! //! Tests that assert particular requests lead to particular //! responses are also tested in Cuprate's blockchain database crate. - +//! +//---------------------------------------------------------------------------------------------------- Import use std::{ collections::{HashMap, HashSet}, ops::Range, }; -use crate::{AltBlockInformation, ChainId}; -//---------------------------------------------------------------------------------------------------- Import -use crate::types::{Chain, ExtendedBlockHeader, OutputOnChain, VerifiedBlockInformation}; +use crate::{ + types::{Chain, ExtendedBlockHeader, OutputOnChain, VerifiedBlockInformation}, + AltBlockInformation, ChainId, +}; //---------------------------------------------------------------------------------------------------- ReadRequest /// A read request to the blockchain database. diff --git a/types/src/types.rs b/types/src/types.rs index c6e83d09..a60ce6c6 100644 --- a/types/src/types.rs +++ b/types/src/types.rs @@ -1,7 +1,8 @@ //! Various shared data types in Cuprate. -use std::num::NonZero; //---------------------------------------------------------------------------------------------------- Import +use std::num::NonZero; + use curve25519_dalek::edwards::EdwardsPoint; use monero_serai::{ block::Block,