From a97367167b0ec392089708e208c449e9d1a7cda9 Mon Sep 17 00:00:00 2001 From: "hinto.janai" Date: Sun, 15 Dec 2024 23:12:52 -0500 Subject: [PATCH] combines tests into `compat` --- Cargo.lock | 87 +++++- Cargo.toml | 5 +- tests/{pow => compat}/Cargo.toml | 23 +- tests/compat/src/cli.rs | 30 ++ tests/compat/src/constants.rs | 10 + tests/{pow => compat}/src/cryptonight.rs | 8 +- tests/compat/src/main.rs | 53 ++++ tests/{pow => compat}/src/randomx.rs | 4 +- tests/{pow => compat}/src/rpc.rs | 115 ++++++-- tests/compat/src/types.rs | 69 +++++ tests/compat/src/verify.rs | 212 ++++++++++++++ tests/monero-serai/Cargo.toml | 17 -- tests/monero-serai/src/main.rs | 85 ------ tests/monero-serai/src/rpc.rs | 337 ----------------------- tests/pow/src/main.rs | 73 ----- tests/pow/src/verify.rs | 105 ------- 16 files changed, 558 insertions(+), 675 deletions(-) rename tests/{pow => compat}/Cargo.toml (50%) create mode 100644 tests/compat/src/cli.rs create mode 100644 tests/compat/src/constants.rs rename tests/{pow => compat}/src/cryptonight.rs (87%) create mode 100644 tests/compat/src/main.rs rename tests/{pow => compat}/src/randomx.rs (91%) rename tests/{pow => compat}/src/rpc.rs (52%) create mode 100644 tests/compat/src/types.rs create mode 100644 tests/compat/src/verify.rs delete mode 100644 tests/monero-serai/Cargo.toml delete mode 100644 tests/monero-serai/src/main.rs delete mode 100644 tests/monero-serai/src/rpc.rs delete mode 100644 tests/pow/src/main.rs delete mode 100644 tests/pow/src/verify.rs diff --git a/Cargo.lock b/Cargo.lock index ff298eb..6efecf0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -59,12 +59,55 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" +[[package]] +name = "anstream" +version = "0.6.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8acc5369981196006228e28809f761875c0327210a891e941f4c683b3a99529b" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + [[package]] name = "anstyle" version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9" +[[package]] +name = "anstyle-parse" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b2d16507662817a6a20a9ea92df6652ee4f94f914589377d69f3b21bc5798a9" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c" +dependencies = [ + "windows-sys 0.59.0", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2109dbce0e72be3ec00bed26e6a7479ca384ad226efdd66db8fa2e3a38c83125" +dependencies = [ + "anstyle", + "windows-sys 0.59.0", +] + [[package]] name = "anyhow" version = "1.0.92" @@ -444,8 +487,10 @@ version = "4.5.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19bc80abd44e4bed93ca373a0704ccbd1b710dc5749406201bb018272808dc54" dependencies = [ + "anstream", "anstyle", "clap_lex", + "strsim", "terminal_size", ] @@ -467,6 +512,12 @@ version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1462739cb27611015575c0c11df5df7601141071f07518d56fcc1be504cbec97" +[[package]] +name = "colorchoice" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" + [[package]] name = "const_format" version = "0.2.33" @@ -1898,6 +1949,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "is_terminal_polyfill" +version = "1.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" + [[package]] name = "itertools" version = "0.10.5" @@ -3192,6 +3249,12 @@ dependencies = [ "spin", ] +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + [[package]] name = "strum" version = "0.26.3" @@ -3318,23 +3381,10 @@ dependencies = [ ] [[package]] -name = "tests-monero-serai" -version = "0.0.0" -dependencies = [ - "futures", - "hex", - "monero-serai", - "rayon", - "reqwest", - "serde", - "serde_json", - "tokio", -] - -[[package]] -name = "tests-pow" +name = "tests-compat" version = "0.0.0" dependencies = [ + "clap", "crossbeam", "cuprate-consensus-rules", "cuprate-cryptonight", @@ -3343,6 +3393,7 @@ dependencies = [ "hex-literal", "monero-serai", "randomx-rs", + "rayon", "reqwest", "serde", "serde_json", @@ -3730,6 +3781,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + [[package]] name = "valuable" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index f173551..65ea523 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -54,12 +54,11 @@ members = [ "types", # Tests - "tests/pow", - "tests/monero-serai", + "tests/compat", ] [profile.release] -panic = "abort" +panic = "abort" lto = true # Build with LTO strip = "none" # Keep panic stack traces codegen-units = 1 # Optimize for binary speed over compile times diff --git a/tests/pow/Cargo.toml b/tests/compat/Cargo.toml similarity index 50% rename from tests/pow/Cargo.toml rename to tests/compat/Cargo.toml index 8544793..15d8bc8 100644 --- a/tests/pow/Cargo.toml +++ b/tests/compat/Cargo.toml @@ -1,15 +1,23 @@ [package] -name = "tests-pow" -version = "0.0.0" -edition = "2021" +name = "tests-compat" +version = "0.0.0" +edition = "2021" +description = "Compatability tests between `cuprated` and `monerod`" +license = "MIT" +authors = ["hinto-janai"] +repository = "https://github.com/Cuprate/cuprate/tree/main/tests/compat" +keywords = ["cuprate", "tests", "compat"] + [dependencies] cuprate-consensus-rules = { workspace = true } -cuprate-cryptonight = { workspace = true } +cuprate-cryptonight = { workspace = true } +clap = { workspace = true, features = ["cargo", "derive", "default"] } crossbeam = { workspace = true, features = ["std"] } futures = { workspace = true, features = ["std"] } monero-serai = { workspace = true } +rayon = { workspace = true } hex = { workspace = true, features = ["serde", "std"] } hex-literal = { workspace = true } serde = { workspace = true, features = ["derive"] } @@ -20,3 +28,10 @@ randomx-rs = { workspace = true } [lints] workspace = true + +[profile.release] +panic = "unwind" +lto = true +strip = "none" +codegen-units = 1 +opt-level = 3 \ No newline at end of file diff --git a/tests/compat/src/cli.rs b/tests/compat/src/cli.rs new file mode 100644 index 0000000..5cd89c1 --- /dev/null +++ b/tests/compat/src/cli.rs @@ -0,0 +1,30 @@ +use std::num::{NonZeroU64, NonZeroUsize}; + +use clap::Parser; + +/// `cuprate` <-> `monerod` compatability tester. +#[derive(Parser, Debug)] +#[command(about, long_about = None)] +pub struct Args { + /// Name of the person to greet + #[arg(short, long, default_value_t = String::from("http://127.0.0.1:18081"))] + pub rpc_url: String, + + /// Amount of verifying threads to spawn. + #[arg(short, long, default_value_t = std::thread::available_parallelism().unwrap())] + pub threads: NonZeroUsize, + + /// Print an update every `update` amount of blocks. + #[arg(short, long, default_value_t = NonZeroU64::new(500).unwrap())] + pub update: NonZeroU64, +} + +impl Args { + pub fn get() -> Self { + let this = Self::parse(); + + println!("{this:#?}"); + + this + } +} diff --git a/tests/compat/src/constants.rs b/tests/compat/src/constants.rs new file mode 100644 index 0000000..157c265 --- /dev/null +++ b/tests/compat/src/constants.rs @@ -0,0 +1,10 @@ +use std::sync::atomic::{AtomicU64, AtomicUsize}; + +/// Height at which RandomX activated. +pub const RANDOMX_START_HEIGHT: u64 = 1978433; + +/// Total amount of blocks tested, used as a global counter. +pub static TESTED_BLOCK_COUNT: AtomicU64 = AtomicU64::new(0); + +/// Total amount of transactions tested, used as a global counter. +pub static TESTED_TX_COUNT: AtomicUsize = AtomicUsize::new(0); diff --git a/tests/pow/src/cryptonight.rs b/tests/compat/src/cryptonight.rs similarity index 87% rename from tests/pow/src/cryptonight.rs rename to tests/compat/src/cryptonight.rs index bab5e30..9711a49 100644 --- a/tests/pow/src/cryptonight.rs +++ b/tests/compat/src/cryptonight.rs @@ -3,7 +3,7 @@ use std::fmt::Display; use hex_literal::hex; #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] -pub(crate) enum CryptoNightHash { +pub enum CryptoNightHash { V0, V1, V2, @@ -12,7 +12,7 @@ pub(crate) enum CryptoNightHash { impl CryptoNightHash { /// The last height this hash function is used for proof-of-work. - pub(crate) const fn from_height(height: u64) -> Self { + pub const fn from_height(height: u64) -> Self { if height < 1546000 { Self::V0 } else if height < 1685555 { @@ -26,7 +26,7 @@ impl CryptoNightHash { } } - pub(crate) fn hash(data: &[u8], height: u64) -> (&'static str, [u8; 32]) { + pub fn hash(data: &[u8], height: u64) -> (&'static str, [u8; 32]) { let this = Self::from_height(height); let hash = match Self::from_height(height) { @@ -45,7 +45,7 @@ impl CryptoNightHash { (this.as_str(), hash) } - pub(crate) const fn as_str(self) -> &'static str { + pub const fn as_str(self) -> &'static str { match self { Self::V0 => "cryptonight_v0", Self::V1 => "cryptonight_v1", diff --git a/tests/compat/src/main.rs b/tests/compat/src/main.rs new file mode 100644 index 0000000..1b8034e --- /dev/null +++ b/tests/compat/src/main.rs @@ -0,0 +1,53 @@ +#![allow( + clippy::doc_markdown, + reason = "TODO: add exception to doc clippy for `RandomX`" +)] +#![allow(unreachable_pub, reason = "This is a binary, everything `pub` is ok")] + +mod cli; +mod constants; +mod cryptonight; +mod randomx; +mod rpc; +mod types; +mod verify; + +use std::{ + sync::atomic::Ordering, + time::{Duration, Instant}, +}; + +#[tokio::main] +async fn main() { + let now = Instant::now(); + + // Parse CLI args. + let cli::Args { + rpc_url, + update, + threads, + } = cli::Args::get(); + + // Set-up RPC client. + let client = rpc::RpcClient::new(rpc_url).await; + let top_height = client.top_height; + println!("top_height: {top_height}"); + println!(); + + // Test. + let (tx, rx) = crossbeam::channel::unbounded(); + verify::spawn_verify_pool(threads, update, top_height, rx); + client.test(top_height, tx).await; + + // Wait for other threads to finish. + loop { + let count = constants::TESTED_BLOCK_COUNT.load(Ordering::Acquire); + + if top_height == count { + println!("finished, took {}s", now.elapsed().as_secs()); + std::process::exit(0); + } + + std::thread::sleep(Duration::from_secs(1)); + } +} diff --git a/tests/pow/src/randomx.rs b/tests/compat/src/randomx.rs similarity index 91% rename from tests/pow/src/randomx.rs rename to tests/compat/src/randomx.rs index d4d76a6..7fcb92f 100644 --- a/tests/pow/src/randomx.rs +++ b/tests/compat/src/randomx.rs @@ -1,7 +1,7 @@ use randomx_rs::{RandomXCache, RandomXDataset, RandomXFlag, RandomXVM}; /// Returns a [`RandomXVM`] with no optimization flags (default, light-verification). -pub(crate) fn randomx_vm_default(seed_hash: &[u8; 32]) -> RandomXVM { +pub fn randomx_vm_default(seed_hash: &[u8; 32]) -> RandomXVM { const FLAG: RandomXFlag = RandomXFlag::FLAG_DEFAULT; let cache = RandomXCache::new(FLAG, seed_hash).unwrap(); @@ -9,7 +9,7 @@ pub(crate) fn randomx_vm_default(seed_hash: &[u8; 32]) -> RandomXVM { } /// Returns a [`RandomXVM`] with most optimization flags. -pub(crate) fn randomx_vm_optimized(seed_hash: &[u8; 32]) -> RandomXVM { +pub fn randomx_vm_optimized(seed_hash: &[u8; 32]) -> RandomXVM { // TODO: conditional FLAG_LARGE_PAGES, FLAG_JIT let mut vm_flag = RandomXFlag::FLAG_FULL_MEM; diff --git a/tests/pow/src/rpc.rs b/tests/compat/src/rpc.rs similarity index 52% rename from tests/pow/src/rpc.rs rename to tests/compat/src/rpc.rs index e84fe58..189e4d8 100644 --- a/tests/pow/src/rpc.rs +++ b/tests/compat/src/rpc.rs @@ -1,7 +1,6 @@ -use std::time::Duration; - use crossbeam::channel::Sender; -use hex::serde::deserialize; +use monero_serai::{block::Block, transaction::Transaction}; +use rayon::iter::{IndexedParallelIterator, IntoParallelIterator, ParallelIterator}; use reqwest::{ header::{HeaderMap, HeaderValue}, Client, ClientBuilder, @@ -9,37 +8,21 @@ use reqwest::{ use serde::Deserialize; use serde_json::{json, Value}; -use crate::{VerifyData, RANDOMX_START_HEIGHT}; - -#[derive(Debug, Clone, Deserialize)] -struct JsonRpcResponse { - result: GetBlockResponse, -} - -#[derive(Debug, Clone, Deserialize)] -pub struct GetBlockResponse { - #[serde(deserialize_with = "deserialize")] - pub blob: Vec, - pub block_header: BlockHeader, -} - -#[derive(Debug, Clone, Deserialize)] -pub struct BlockHeader { - #[serde(deserialize_with = "deserialize")] - pub pow_hash: Vec, - #[serde(deserialize_with = "deserialize")] - pub hash: Vec, -} +use crate::{ + constants::RANDOMX_START_HEIGHT, + types::{GetBlockResponse, JsonRpcResponse, RpcBlockData, RpcTxData}, +}; #[derive(Debug, Clone)] -pub(crate) struct RpcClient { +pub struct RpcClient { client: Client, json_rpc_url: String, + get_transactions_url: String, pub top_height: u64, } impl RpcClient { - pub(crate) async fn new(rpc_url: String) -> Self { + pub async fn new(rpc_url: String) -> Self { let headers = { let mut h = HeaderMap::new(); h.insert("Content-Type", HeaderValue::from_static("application/json")); @@ -59,6 +42,7 @@ impl RpcClient { }); let json_rpc_url = format!("{rpc_url}/json_rpc"); + let get_transactions_url = format!("{rpc_url}/get_transactions"); let top_height = client .get(&json_rpc_url) @@ -83,6 +67,7 @@ impl RpcClient { Self { client, json_rpc_url, + get_transactions_url, top_height, } } @@ -107,16 +92,85 @@ impl RpcClient { .result } - pub(crate) async fn test(self, top_height: u64, tx: Sender) { + async fn get_transactions(&self, tx_hashes: Vec<[u8; 32]>) -> Vec { + assert!(!tx_hashes.is_empty()); + + #[derive(Debug, Clone, Deserialize)] + struct GetTransactionsResponse { + txs: Vec, + } + + #[derive(Debug, Clone, Deserialize)] + struct Tx { + as_hex: String, + pruned_as_hex: String, + } + + let txs_hashes = tx_hashes.iter().map(hex::encode).collect::>(); + let request = json!({"txs_hashes":txs_hashes}); + + let txs = self + .client + .get(&self.get_transactions_url) + .json(&request) + .send() + .await + .unwrap() + .json::() + .await + .unwrap() + .txs; + + assert_eq!(txs.len(), tx_hashes.len()); + + txs.into_par_iter() + .zip(tx_hashes) + .map(|(r, tx_hash)| { + let tx_blob = hex::decode(if r.as_hex.is_empty() { + r.pruned_as_hex + } else { + r.as_hex + }) + .unwrap(); + + let tx = Transaction::read(&mut tx_blob.as_slice()).unwrap(); + + RpcTxData { + tx, + tx_blob, + tx_hash, + } + }) + .collect() + } + + pub async fn test(self, top_height: u64, tx: Sender) { use futures::StreamExt; let iter = (0..top_height).map(|height| { - let this = &self; + let this = self.clone(); let tx = tx.clone(); async move { let get_block_response = this.get_block(height).await; + let (this, get_block_response, block, txs) = + tokio::task::spawn_blocking(move || async move { + // Deserialize the block. + let block = Block::read(&mut get_block_response.blob.as_slice()).unwrap(); + + // Fetch and deserialize all transactions. + let mut tx_hashes = Vec::with_capacity(block.transactions.len() + 1); + tx_hashes.push(block.miner_transaction.hash()); + tx_hashes.extend(block.transactions.iter()); + let txs = this.get_transactions(tx_hashes).await; + + (this, get_block_response, block, txs) + }) + .await + .unwrap() + .await; + let (seed_height, seed_hash) = if height < RANDOMX_START_HEIGHT { (0, [0; 32]) } else { @@ -137,11 +191,12 @@ impl RpcClient { (seed_height, seed_hash) }; - let data = VerifyData { + let data = RpcBlockData { get_block_response, - height, + block, seed_height, seed_hash, + txs, }; tx.send(data).unwrap(); diff --git a/tests/compat/src/types.rs b/tests/compat/src/types.rs new file mode 100644 index 0000000..5277b50 --- /dev/null +++ b/tests/compat/src/types.rs @@ -0,0 +1,69 @@ +use hex::serde::deserialize; +use monero_serai::{block::Block, transaction::Transaction}; +use serde::Deserialize; + +/// Data of a single block from RPC. +#[derive(Debug)] +pub struct RpcBlockData { + /// Subset of JSON-RPC `get_block` data. + pub get_block_response: GetBlockResponse, + + /// The block itself. + pub block: Block, + + /// The correct seed height needed for this block for `RandomX`. + pub seed_height: u64, + /// The correct seed hash needed for this block for `RandomX`. + pub seed_hash: [u8; 32], + + /// All transactions in the block. + /// This vec is: + /// - the original transaction blobs + pub txs: Vec, +} + +/// Data of a transaction. +#[derive(Debug)] +pub struct RpcTxData { + /// The transactions itself. + pub tx: Transaction, + /// The transactions blob. + pub tx_blob: Vec, + /// The transaction's hash. + pub tx_hash: [u8; 32], +} + +/// Subset of JSON-RPC `get_block` response. +#[derive(Debug, Clone, Deserialize)] +pub struct JsonRpcResponse { + pub result: GetBlockResponse, +} + +/// Subset of JSON-RPC `get_block` data. +#[derive(Debug, Clone, Deserialize)] +pub struct GetBlockResponse { + #[serde(deserialize_with = "deserialize")] + pub blob: Vec, + pub block_header: BlockHeader, +} + +#[derive(Debug, Clone, Deserialize)] +pub(crate) struct BlockHeader { + #[serde(deserialize_with = "deserialize")] + pub hash: [u8; 32], + #[serde(deserialize_with = "deserialize")] + pub pow_hash: [u8; 32], + #[serde(deserialize_with = "deserialize")] + pub miner_tx_hash: [u8; 32], + #[serde(deserialize_with = "deserialize")] + pub prev_hash: [u8; 32], + + pub block_weight: usize, + pub height: u64, + pub major_version: u8, + pub minor_version: u8, + pub nonce: u32, + pub num_txes: usize, + pub reward: u64, + pub timestamp: u64, +} diff --git a/tests/compat/src/verify.rs b/tests/compat/src/verify.rs new file mode 100644 index 0000000..b482ebf --- /dev/null +++ b/tests/compat/src/verify.rs @@ -0,0 +1,212 @@ +use std::{ + collections::HashSet, + num::{NonZeroU64, NonZeroUsize}, + sync::{atomic::Ordering, LazyLock, Mutex}, + time::Instant, +}; + +use crossbeam::channel::Receiver; + +use crate::{ + constants::{RANDOMX_START_HEIGHT, TESTED_BLOCK_COUNT, TESTED_TX_COUNT}, + cryptonight::CryptoNightHash, + types::{BlockHeader, GetBlockResponse, RpcBlockData, RpcTxData}, +}; + +#[expect( + clippy::needless_pass_by_value, + clippy::cast_precision_loss, + clippy::cast_possible_truncation, + clippy::cast_sign_loss, + clippy::significant_drop_tightening +)] +pub fn spawn_verify_pool( + thread_count: NonZeroUsize, + update: NonZeroU64, + top_height: u64, + rx: Receiver, +) { + let now = Instant::now(); + + for i in 0..thread_count.get() { + let rx = rx.clone(); + + std::thread::spawn(move || { + let mut current_seed_hash = [0; 32]; + let mut randomx_vm = None; + + loop { + let Ok(data) = rx.recv() else { + println!("Exiting verify thread {i}/{thread_count}"); + return; + }; + + // Panic info. + let p = format!("data: {data:#?}"); + + let RpcBlockData { + get_block_response, + block, + seed_height, + seed_hash, + txs, + } = data; + let GetBlockResponse { blob, block_header } = get_block_response; + let BlockHeader { + block_weight, + hash, + pow_hash, + height, + major_version, + minor_version, + miner_tx_hash, + nonce, + num_txes, + prev_hash, + reward, + timestamp, + } = block_header; + + // Test block properties. + assert_eq!(blob, block.serialize(), "{p:#?}"); + + assert!( + !block.miner_transaction.prefix().outputs.is_empty(), + "miner_tx has no outputs\n{p:#?}" + ); + + let block_reward = block + .miner_transaction + .prefix() + .outputs + .iter() + .map(|o| o.amount.unwrap()) + .sum::(); + assert_ne!(block_reward, 0, "block reward is 0\n{p:#?}"); + + let total_block_weight = txs + .iter() + .map(|RpcTxData { tx, .. }| tx.weight()) + .sum::(); + + // Test all transactions are unique. + { + static TX_SET: LazyLock>> = + LazyLock::new(|| Mutex::new(HashSet::new())); + + let mut tx_set = TX_SET.lock().unwrap(); + + for tx_hash in txs.iter().map(|RpcTxData { tx_hash, .. }| tx_hash) { + assert!( + tx_set.insert(*tx_hash), + "duplicated tx_hash: {}, {p:#?}", + hex::encode(tx_hash), + ); + } + } + + // Test transaction properties. + for RpcTxData { + tx, + tx_blob, + tx_hash, + } in txs + { + assert_eq!(tx_hash, tx.hash(), "{p:#?}, tx: {tx:#?}"); + assert_ne!(tx.weight(), 0, "{p:#?}, tx: {tx:#?}"); + assert!(!tx.prefix().inputs.is_empty(), "{p:#?}, tx: {tx:#?}"); + assert_eq!(tx_blob, tx.serialize(), "{p:#?}, tx: {tx:#?}"); + assert!(matches!(tx.version(), 1 | 2), "{p:#?}, tx: {tx:#?}"); + } + + // Test block fields are correct. + assert_eq!(block_weight, total_block_weight, "{p:#?}"); + assert_ne!(block.miner_transaction.weight(), 0, "{p:#?}"); + assert_eq!(hash, block.hash(), "{p:#?}"); + assert_eq!( + height, + u64::try_from(block.number().unwrap()).unwrap(), + "{p:#?}" + ); + assert_eq!(major_version, block.header.hardfork_version, "{p:#?}"); + assert_eq!(minor_version, block.header.hardfork_signal, "{p:#?}"); + assert_eq!(miner_tx_hash, block.miner_transaction.hash(), "{p:#?}"); + assert_eq!(nonce, block.header.nonce, "{p:#?}"); + assert_eq!(num_txes, block.transactions.len(), "{p:#?}"); + assert_eq!(prev_hash, block.header.previous, "{p:#?}"); + assert_eq!(reward, block_reward, "{p:#?}"); + assert_eq!(timestamp, block.header.timestamp, "{p:#?}"); + + // + let pow_data = block.serialize_pow_hash(); + + let (algo, calculated_pow_hash) = if height < RANDOMX_START_HEIGHT { + CryptoNightHash::hash(&pow_data, height) + } else { + if current_seed_hash != seed_hash { + randomx_vm = None; + } + + let randomx_vm = randomx_vm.get_or_insert_with(|| { + current_seed_hash = seed_hash; + // crate::randomx::randomx_vm_optimized(&seed_hash) + crate::randomx::randomx_vm_default(&seed_hash) + }); + + let pow_hash = randomx_vm + .calculate_hash(&pow_data) + .unwrap() + .try_into() + .unwrap(); + + ("randomx", pow_hash) + }; + + assert_eq!(calculated_pow_hash, pow_hash, "{p:#?}",); + + let count = TESTED_BLOCK_COUNT.fetch_add(1, Ordering::Release) + 1; + let total_tx_count = TESTED_TX_COUNT.fetch_add(num_txes, Ordering::Release) + 1; + + if count % update.get() != 0 { + continue; + } + + let pow_hash = hex::encode(pow_hash); + let seed_hash = hex::encode(seed_hash); + let percent = (count as f64 / top_height as f64) * 100.0; + + let elapsed = now.elapsed().as_secs_f64(); + let secs_per_hash = elapsed / count as f64; + let bps = count as f64 / elapsed; + let remaining_secs = (top_height as f64 - count as f64) * secs_per_hash; + let h = (remaining_secs / 60.0 / 60.0) as u64; + let m = (remaining_secs / 60.0 % 60.0) as u64; + let s = (remaining_secs % 60.0) as u64; + + let block_hash = hex::encode(hash); + let miner_tx_hash = hex::encode(miner_tx_hash); + let prev_hash = hex::encode(prev_hash); + let miner_tx_weight = block.miner_transaction.weight(); + + println!("progress | {count}/{top_height} ({percent:.2}%, {algo}, {bps:.2} blocks/sec, {h}h {m}m {s}s left) +seed_hash | {seed_hash} +pow_hash | {pow_hash} +block_hash | {block_hash} +miner_tx_hash | {miner_tx_hash} +prev_hash | {prev_hash} +reward | {reward} +timestamp | {timestamp} +nonce | {nonce} +total_tx_count | {total_tx_count} +height | {height} +seed_height | {seed_height} +block_weight | {block_weight} +miner_tx_weight | {miner_tx_weight} +major_version | {major_version} +minor_version | {minor_version} +num_txes | {num_txes}\n", + ); + } + }); + } +} diff --git a/tests/monero-serai/Cargo.toml b/tests/monero-serai/Cargo.toml deleted file mode 100644 index 0088e32..0000000 --- a/tests/monero-serai/Cargo.toml +++ /dev/null @@ -1,17 +0,0 @@ -[package] -name = "tests-monero-serai" -version = "0.0.0" -edition = "2021" - -[dependencies] -monero-serai = { workspace = true } -hex = { workspace = true, features = ["serde", "std"] } -serde = { workspace = true, features = ["derive"] } -serde_json = { workspace = true, features = ["std"] } -tokio = { workspace = true, features = ["full"] } -reqwest = { workspace = true, features = ["json"] } -rayon = { workspace = true } -futures = { workspace = true, features = ["std"] } - -[lints] -workspace = true diff --git a/tests/monero-serai/src/main.rs b/tests/monero-serai/src/main.rs deleted file mode 100644 index 9dd8590..0000000 --- a/tests/monero-serai/src/main.rs +++ /dev/null @@ -1,85 +0,0 @@ -use std::{ - sync::atomic::{AtomicUsize, Ordering}, - time::{Duration, Instant}, -}; - -mod rpc; - -pub static TESTED_BLOCK_COUNT: AtomicUsize = AtomicUsize::new(0); -pub static TESTED_TX_COUNT: AtomicUsize = AtomicUsize::new(0); - -#[tokio::main] -async fn main() { - let now = Instant::now(); - - let rpc_url = if let Ok(url) = std::env::var("RPC_URL") { - println!("RPC_URL (found): {url}"); - url - } else { - let rpc_url = "http://127.0.0.1:18081".to_string(); - println!("RPC_URL (off, using default): {rpc_url}"); - rpc_url - }; - if std::env::var("VERBOSE").is_ok() { - println!("VERBOSE: true"); - } else { - println!("VERBOSE: false"); - } - - let mut client = rpc::RpcClient::new(rpc_url).await; - - let top_height = if let Ok(Ok(h)) = std::env::var("TOP_HEIGHT").map(|s| s.parse()) { - client.top_height = h; - println!("TOP_HEIGHT (found): {h}"); - h - } else { - println!("TOP_HEIGHT (off, using latest): {}", client.top_height); - client.top_height - }; - - let ranges = (0..top_height) - .collect::>() - .chunks(100_000) - .map(<[usize]>::to_vec) - .collect::>>(); - - println!("ranges: ["); - for range in &ranges { - println!( - " ({}..{}),", - range.first().unwrap(), - range.last().unwrap() - ); - } - - println!("]\n"); - - let iter = ranges.into_iter().map(move |range| { - let c = client.clone(); - async move { - tokio::task::spawn_blocking(move || async move { - c.get_block_test_batch(range.into_iter().collect()).await; - }) - .await - .unwrap() - .await; - } - }); - - futures::future::join_all(iter).await; - - loop { - let block_count = TESTED_BLOCK_COUNT.load(Ordering::Acquire); - let tx_count = TESTED_TX_COUNT.load(Ordering::Acquire); - - if top_height == block_count { - println!( - "finished processing: blocks: {block_count}/{top_height}, txs: {tx_count}, took {}s", - now.elapsed().as_secs() - ); - std::process::exit(0); - } - - std::thread::sleep(Duration::from_secs(1)); - } -} diff --git a/tests/monero-serai/src/rpc.rs b/tests/monero-serai/src/rpc.rs deleted file mode 100644 index 62774bb..0000000 --- a/tests/monero-serai/src/rpc.rs +++ /dev/null @@ -1,337 +0,0 @@ -use std::{ - collections::{BTreeSet, HashSet}, - sync::{atomic::Ordering, LazyLock}, - time::Instant, -}; - -use hex::serde::deserialize; -use monero_serai::{block::Block, transaction::Transaction}; -use rayon::iter::{IndexedParallelIterator, IntoParallelIterator, ParallelIterator}; -use reqwest::{ - header::{HeaderMap, HeaderValue}, - Client, ClientBuilder, -}; -use serde::Deserialize; -use serde_json::json; -use tokio::sync::Mutex; - -use crate::{TESTED_BLOCK_COUNT, TESTED_TX_COUNT}; - -#[derive(Debug, Clone, Deserialize)] -pub(crate) struct BlockHeader { - #[serde(deserialize_with = "deserialize")] - pub hash: Vec, - #[serde(deserialize_with = "deserialize")] - pub miner_tx_hash: Vec, - #[serde(deserialize_with = "deserialize")] - pub prev_hash: Vec, - - pub block_weight: usize, - pub height: usize, - pub major_version: u8, - pub minor_version: u8, - pub nonce: u32, - pub num_txes: usize, - pub reward: u64, - pub timestamp: u64, -} - -#[derive(Debug, Clone)] -pub(crate) struct RpcClient { - client: Client, - rpc_url: String, - json_rpc_url: String, - get_transactions_url: String, - pub top_height: usize, -} - -impl RpcClient { - pub(crate) async fn new(rpc_url: String) -> Self { - let headers = { - let mut h = HeaderMap::new(); - h.insert("Content-Type", HeaderValue::from_static("application/json")); - h - }; - - let client = ClientBuilder::new() - .default_headers(headers) - .build() - .unwrap(); - - #[derive(Debug, Clone, Deserialize)] - struct JsonRpcResponse { - result: GetLastBlockHeaderResponse, - } - - #[derive(Debug, Clone, Deserialize)] - pub(crate) struct GetLastBlockHeaderResponse { - pub block_header: BlockHeader, - } - - let request = json!({ - "jsonrpc": "2.0", - "id": 0, - "method": "get_last_block_header", - "params": {} - }); - - let json_rpc_url = format!("{rpc_url}/json_rpc"); - let get_transactions_url = format!("{rpc_url}/get_transactions"); - - let top_height = client - .get(&json_rpc_url) - .json(&request) - .send() - .await - .unwrap() - .json::() - .await - .unwrap() - .result - .block_header - .height; - - assert!(top_height > 3301441, "node is behind"); - - Self { - client, - rpc_url, - json_rpc_url, - get_transactions_url, - top_height, - } - } - - async fn get_transactions(&self, tx_hashes: Vec<[u8; 32]>) -> Vec<(Transaction, Vec)> { - assert!(!tx_hashes.is_empty()); - - #[derive(Debug, Clone, Deserialize)] - pub(crate) struct GetTransactionsResponse { - pub txs: Vec, - } - - #[derive(Debug, Clone, Deserialize)] - pub(crate) struct Tx { - pub as_hex: String, - pub pruned_as_hex: String, - } - - let txs_hashes = tx_hashes - .into_iter() - .map(hex::encode) - .collect::>(); - - let request = json!({"txs_hashes":txs_hashes}); - - let txs = self - .client - .get(&self.get_transactions_url) - .json(&request) - .send() - .await - .unwrap() - .json::() - .await - .unwrap() - .txs; - - txs.into_par_iter() - .map(|r| { - let blob = hex::decode(if r.as_hex.is_empty() { - r.pruned_as_hex - } else { - r.as_hex - }) - .unwrap(); - - (Transaction::read(&mut blob.as_slice()).unwrap(), blob) - }) - .collect() - } - - #[expect( - clippy::cast_possible_truncation, - clippy::cast_sign_loss, - clippy::significant_drop_tightening - )] - pub(crate) async fn get_block_test_batch(&self, heights: BTreeSet) { - #[derive(Debug, Clone, Deserialize)] - struct JsonRpcResponse { - result: GetBlockResponse, - } - - #[derive(Debug, Clone, Deserialize)] - pub(crate) struct GetBlockResponse { - #[serde(deserialize_with = "deserialize")] - pub blob: Vec, - pub block_header: BlockHeader, - } - - let now = Instant::now(); - - let tasks = heights.into_iter().map(|height| { - let request = json!({ - "jsonrpc": "2.0", - "id": 0, - "method": "get_block", - "params": {"height": height} - }); - - let task = - tokio::task::spawn(self.client.get(&self.json_rpc_url).json(&request).send()); - - (height, task) - }); - - for (height, task) in tasks { - let resp = task - .await - .unwrap() - .unwrap() - .json::() - .await - .unwrap() - .result; - - let info = format!("\nheight: {height}\nresponse: {resp:#?}"); - - // Test block deserialization. - let block = match Block::read(&mut resp.blob.as_slice()) { - Ok(b) => b, - Err(e) => panic!("{e:?}\n{info}"), - }; - - // Fetch all transactions. - let mut tx_hashes = vec![block.miner_transaction.hash()]; - tx_hashes.extend(block.transactions.iter()); - let txs = self.get_transactions(tx_hashes.clone()).await; - assert_eq!(tx_hashes.len(), txs.len()); - - // Test all transactions are unique. - { - static TX_SET: LazyLock>> = - LazyLock::new(|| Mutex::new(HashSet::new())); - - let tx_hashes = tx_hashes.clone(); - let mut tx_set = TX_SET.lock().await; - - for hash in tx_hashes { - assert!( - tx_set.insert(hash), - "duplicated tx hash: {}\n{info}", - hex::encode(hash), - ); - } - } - - let top_height = self.top_height; - - #[expect(clippy::cast_precision_loss)] - rayon::spawn(move || { - // Test block properties. - assert_eq!(resp.blob, block.serialize(), "{info}"); - - assert!( - !block.miner_transaction.prefix().outputs.is_empty(), - "miner_tx has no outputs\n{info}" - ); - - let block_reward = block - .miner_transaction - .prefix() - .outputs - .iter() - .map(|o| o.amount.unwrap()) - .sum::(); - assert_ne!(block_reward, 0, "block reward is 0\n{info}"); - - let BlockHeader { - block_weight, - hash, - height, - major_version, - minor_version, - miner_tx_hash, - nonce, - num_txes, - prev_hash, - reward, - timestamp, - } = resp.block_header; - - let total_block_weight = txs.iter().map(|(tx, _)| tx.weight()).sum::(); - - // Test transaction properties. - txs.into_par_iter() - .zip(tx_hashes) - .for_each(|((tx, blob), hash)| { - assert_eq!(hash, tx.hash(), "{info}, tx: {tx:#?}"); - assert_ne!(tx.weight(), 0, "{info}, tx: {tx:#?}"); - assert!(!tx.prefix().inputs.is_empty(), "{info}, tx: {tx:#?}"); - assert_eq!(blob, tx.serialize(), "{info}, tx: {tx:#?}"); - assert!(matches!(tx.version(), 1 | 2), "{info}, tx: {tx:#?}"); - }); - - // Test block fields are correct. - assert_eq!(block_weight, total_block_weight, "{info}"); - assert_ne!(block.miner_transaction.weight(), 0, "{info}"); - assert_eq!(hash, block.hash(), "{info}"); - assert_eq!(height, block.number().unwrap(), "{info}"); - assert_eq!(major_version, block.header.hardfork_version, "{info}"); - assert_eq!(minor_version, block.header.hardfork_signal, "{info}"); - assert_eq!(miner_tx_hash, block.miner_transaction.hash(), "{info}"); - assert_eq!(nonce, block.header.nonce, "{info}"); - assert_eq!(num_txes, block.transactions.len(), "{info}"); - assert_eq!(prev_hash, block.header.previous, "{info}"); - assert_eq!(reward, block_reward, "{info}"); - assert_eq!(timestamp, block.header.timestamp, "{info}"); - - let progress = TESTED_BLOCK_COUNT.fetch_add(1, Ordering::Release) + 1; - let tx_count = TESTED_TX_COUNT.fetch_add(num_txes, Ordering::Release) + 1; - - if std::env::var("VERBOSE").is_err() && progress % 1000 != 0 { - return; - } - - let percent = (progress as f64 / top_height as f64) * 100.0; - - let elapsed = now.elapsed().as_secs_f64(); - let secs_per_hash = elapsed / progress as f64; - let bps = progress as f64 / elapsed; - let remaining_secs = (top_height as f64 - progress as f64) * secs_per_hash; - let h = (remaining_secs / 60.0 / 60.0) as u64; - let m = (remaining_secs / 60.0 % 60.0) as u64; - let s = (remaining_secs % 60.0) as u64; - - println!( - "progress | {progress}/{top_height} ({percent:.2}%, {bps:.2} blocks/sec, {h}h {m}m {s}s left) -tx_count | {tx_count} -hash | {} -miner_tx_hash | {} -prev_hash | {} -reward | {} -timestamp | {} -nonce | {} -height | {} -block_weight | {} -miner_tx_weight | {} -major_version | {} -minor_version | {} -num_txes | {}\n", - hex::encode(hash), - hex::encode(miner_tx_hash), - hex::encode(prev_hash), - reward, - timestamp, - nonce, - height, - block_weight, - block.miner_transaction.weight(), - major_version, - minor_version, - num_txes, - ); - }); - } - } -} diff --git a/tests/pow/src/main.rs b/tests/pow/src/main.rs deleted file mode 100644 index 149203f..0000000 --- a/tests/pow/src/main.rs +++ /dev/null @@ -1,73 +0,0 @@ -mod cryptonight; -mod randomx; -mod rpc; -mod verify; - -use std::{ - sync::atomic::{AtomicU64, Ordering}, - time::{Duration, Instant}, -}; - -use crate::rpc::GetBlockResponse; - -pub const RANDOMX_START_HEIGHT: u64 = 1978433; -pub static TESTED_BLOCK_COUNT: AtomicU64 = AtomicU64::new(0); - -#[derive(Debug)] -pub struct VerifyData { - pub get_block_response: GetBlockResponse, - pub height: u64, - pub seed_height: u64, - pub seed_hash: [u8; 32], -} - -#[tokio::main] -async fn main() { - let now = Instant::now(); - - let rpc_url = if let Ok(url) = std::env::var("RPC_URL") { - println!("RPC_URL (found): {url}"); - url - } else { - let rpc_url = "http://127.0.0.1:18081".to_string(); - println!("RPC_URL (off, using default): {rpc_url}"); - rpc_url - }; - if std::env::var("VERBOSE").is_ok() { - println!("VERBOSE: true"); - } else { - println!("VERBOSE: false"); - } - - let client = rpc::RpcClient::new(rpc_url).await; - let top_height = client.top_height; - println!("top_height: {top_height}"); - - let threads = if let Ok(Ok(c)) = std::env::var("THREADS").map(|s| s.parse()) { - println!("THREADS (found): {c}"); - c - } else { - let c = std::thread::available_parallelism().unwrap().get(); - println!("THREADS (off): {c}"); - c - }; - - println!(); - - // Test RandomX. - let (tx, rx) = crossbeam::channel::unbounded(); - verify::spawn_verify_pool(threads, top_height, rx); - client.test(top_height, tx).await; - - // Wait for other threads to finish. - loop { - let count = TESTED_BLOCK_COUNT.load(Ordering::Acquire); - - if top_height == count { - println!("finished, took {}s", now.elapsed().as_secs()); - std::process::exit(0); - } - - std::thread::sleep(Duration::from_secs(1)); - } -} diff --git a/tests/pow/src/verify.rs b/tests/pow/src/verify.rs deleted file mode 100644 index 0ed7fdd..0000000 --- a/tests/pow/src/verify.rs +++ /dev/null @@ -1,105 +0,0 @@ -use std::{sync::atomic::Ordering, time::Instant}; - -use crossbeam::channel::Receiver; -use monero_serai::block::Block; - -use crate::{ - cryptonight::CryptoNightHash, rpc::GetBlockResponse, VerifyData, RANDOMX_START_HEIGHT, - TESTED_BLOCK_COUNT, -}; - -#[expect( - clippy::needless_pass_by_value, - clippy::cast_precision_loss, - clippy::cast_possible_truncation, - clippy::cast_sign_loss -)] -pub(crate) fn spawn_verify_pool(thread_count: usize, top_height: u64, rx: Receiver) { - let now = Instant::now(); - - for i in 0..thread_count { - let rx = rx.clone(); - - std::thread::spawn(move || { - let mut current_seed_hash = [0; 32]; - let mut randomx_vm = None; - - loop { - let Ok(data) = rx.recv() else { - println!("Exiting verify thread {i}/{thread_count}"); - return; - }; - - let VerifyData { - get_block_response, - height, - seed_height, - seed_hash, - } = data; - - let GetBlockResponse { blob, block_header } = get_block_response; - let header = block_header; - - let block = match Block::read(&mut blob.as_slice()) { - Ok(b) => b, - Err(e) => panic!("{e:?}\nblob: {blob:?}, header: {header:?}"), - }; - - let pow_data = block.serialize_pow_hash(); - - let (algo, pow_hash) = if height < RANDOMX_START_HEIGHT { - CryptoNightHash::hash(&pow_data, height) - } else { - if current_seed_hash != seed_hash { - randomx_vm = None; - } - - let randomx_vm = randomx_vm.get_or_insert_with(|| { - current_seed_hash = seed_hash; - // crate::randomx::randomx_vm_optimized(&seed_hash) - crate::randomx::randomx_vm_default(&seed_hash) - }); - - let pow_hash = randomx_vm - .calculate_hash(&pow_data) - .unwrap() - .try_into() - .unwrap(); - - ("randomx", pow_hash) - }; - - assert_eq!( - header.pow_hash, pow_hash, - "\nheight: {height}\nheader: {header:#?}\nblock: {block:#?}", - ); - - let count = TESTED_BLOCK_COUNT.fetch_add(1, Ordering::Release) + 1; - - if std::env::var("VERBOSE").is_err() && count % 500 != 0 { - continue; - } - - let pow_hash = hex::encode(pow_hash); - let seed_hash = hex::encode(seed_hash); - let percent = (count as f64 / top_height as f64) * 100.0; - - let elapsed = now.elapsed().as_secs_f64(); - let secs_per_hash = elapsed / count as f64; - let bps = count as f64 / elapsed; - let remaining_secs = (top_height as f64 - count as f64) * secs_per_hash; - let h = (remaining_secs / 60.0 / 60.0) as u64; - let m = (remaining_secs / 60.0 % 60.0) as u64; - let s = (remaining_secs % 60.0) as u64; - - println!( - "progress | {count}/{top_height} ({percent:.2}%, {bps:.2} blocks/sec, {h}h {m}m {s}s left) -algo | {algo} -seed_height | {seed_height} -seed_hash | {seed_hash} -pow_hash | {pow_hash}\n" - ); - } - }); - } -}