From 1f94a90c459aafbe231f193f945f16230a82ab7e Mon Sep 17 00:00:00 2001 From: Boog900 <54e72d8a-345f-4599-bd90-c6b9bc7d0ec5@aleeas.com> Date: Tue, 31 Dec 2024 16:57:17 +0000 Subject: [PATCH] fix small issue in block downloader more misc clean up --- binaries/cuprated/Cuprated.toml | 4 +- binaries/cuprated/src/config.rs | 6 +-- binaries/cuprated/src/config/default.rs | 44 ------------------- binaries/cuprated/src/config/p2p.rs | 4 +- binaries/cuprated/src/config/rayon.rs | 3 ++ binaries/cuprated/src/config/storage.rs | 1 + binaries/cuprated/src/config/tokio.rs | 1 + .../cuprated/src/config/tracing_config.rs | 3 +- p2p/address-book/src/book.rs | 6 --- .../src/block_downloader/download_batch.rs | 11 ++--- 10 files changed, 18 insertions(+), 65 deletions(-) delete mode 100644 binaries/cuprated/src/config/default.rs diff --git a/binaries/cuprated/Cuprated.toml b/binaries/cuprated/Cuprated.toml index e2c87cc..549e1ae 100644 --- a/binaries/cuprated/Cuprated.toml +++ b/binaries/cuprated/Cuprated.toml @@ -19,7 +19,7 @@ file = { level = "debug", max_log_files = 7 } ## Clear-net config. [p2p.clear_net] ## The number of outbound connections we should make and maintain. -outbound_connections = 32 +outbound_connections = 64 ## The number of extra connections we should make under load from the rest of Cuprate, i.e. when syncing. extra_outbound_connections = 8 ## The maximum number of incoming we should allow. @@ -47,7 +47,7 @@ buffer_bytes = 50_000_000 ## The size of the queue of blocks which are waiting for a parent block to be downloaded (bytes). in_progress_queue_bytes = 50_000_000 ## The target size of a batch of blocks (bytes), must not exceed 100MB. -target_batch_bytes = 15_000_000 +target_batch_bytes = 10_000_000 ## The amount of time between checking the pool of connected peers for free peers to download blocks. check_client_pool_interval = { secs = 30, nanos = 0 } diff --git a/binaries/cuprated/src/config.rs b/binaries/cuprated/src/config.rs index f3f55bf..c218c1d 100644 --- a/binaries/cuprated/src/config.rs +++ b/binaries/cuprated/src/config.rs @@ -25,11 +25,11 @@ mod storage; mod tokio; mod tracing_config; -use crate::config::fs::FileSystemConfig; -use crate::config::rayon::RayonConfig; -use crate::config::tokio::TokioConfig; +use fs::FileSystemConfig; use p2p::P2PConfig; +use rayon::RayonConfig; use storage::StorageConfig; +use tokio::TokioConfig; use tracing_config::TracingConfig; /// Reads the args & config file, returning a [`Config`]. diff --git a/binaries/cuprated/src/config/default.rs b/binaries/cuprated/src/config/default.rs deleted file mode 100644 index 8c9642b..0000000 --- a/binaries/cuprated/src/config/default.rs +++ /dev/null @@ -1,44 +0,0 @@ -use std::{ - io::Write, - path::{Path, PathBuf}, - str::from_utf8, -}; - -use cuprate_helper::fs::{CUPRATE_CACHE_DIR, DEFAULT_CONFIG_FILE_NAME}; - -use crate::constants::EXAMPLE_CONFIG; - -/// Creates a config file which will be named [`DEFAULT_CONFIG_FILE_NAME`] in the directory given in [`Path`]. -/// -/// This will always terminate the program, on success and failure. -pub fn create_default_config_file(path: &Path) -> ! { - let config_file = path.join(DEFAULT_CONFIG_FILE_NAME); - - tracing::info!("Attempting to create new config file here: {config_file:?}"); - - let mut file = match std::fs::OpenOptions::new() - .write(true) - .create_new(true) - .open(&config_file) - { - Ok(file) => file, - Err(e) => { - tracing::error!("Failed to create config file, got error: {e}"); - std::process::exit(1); - } - }; - - let config = EXAMPLE_CONFIG; - file.write_all(config.as_bytes()).unwrap(); - - std::process::exit(0); -} - -#[cfg(test)] -mod tests { - use crate::{config::Config, constants::EXAMPLE_CONFIG}; - #[test] - fn generate_config_text_is_valid() { - let config: Config = toml::from_str(EXAMPLE_CONFIG).unwrap(); - } -} diff --git a/binaries/cuprated/src/config/p2p.rs b/binaries/cuprated/src/config/p2p.rs index b9a4257..da9c7bc 100644 --- a/binaries/cuprated/src/config/p2p.rs +++ b/binaries/cuprated/src/config/p2p.rs @@ -50,7 +50,7 @@ impl Default for BlockDownloaderConfig { buffer_bytes: 50_000_000, in_progress_queue_bytes: 50_000_000, check_client_pool_interval: Duration::from_secs(30), - target_batch_bytes: 15_000_000, + target_batch_bytes: 10_000_000, } } } @@ -111,7 +111,7 @@ impl SharedNetConfig { impl Default for SharedNetConfig { fn default() -> Self { Self { - outbound_connections: 32, + outbound_connections: 64, extra_outbound_connections: 8, max_inbound_connections: 128, gray_peers_percent: 0.7, diff --git a/binaries/cuprated/src/config/rayon.rs b/binaries/cuprated/src/config/rayon.rs index a216fb7..83c69a0 100644 --- a/binaries/cuprated/src/config/rayon.rs +++ b/binaries/cuprated/src/config/rayon.rs @@ -1,14 +1,17 @@ use serde::{Deserialize, Serialize}; +/// The [`rayon`] config. #[derive(Debug, Deserialize, Serialize, Eq, PartialEq)] #[serde(deny_unknown_fields, default)] pub struct RayonConfig { + /// The number of threads to use for the [`rayon::ThreadPool`]. pub threads: usize, } impl Default for RayonConfig { fn default() -> Self { Self { + // 75% available threads. threads: (std::thread::available_parallelism().unwrap().get() * 3).div_ceil(4), } } diff --git a/binaries/cuprated/src/config/storage.rs b/binaries/cuprated/src/config/storage.rs index b09dd14..4914b6e 100644 --- a/binaries/cuprated/src/config/storage.rs +++ b/binaries/cuprated/src/config/storage.rs @@ -21,6 +21,7 @@ pub struct StorageConfig { impl Default for StorageConfig { fn default() -> Self { Self { + // 25% available threads. reader_threads: std::thread::available_parallelism() .unwrap() .get() diff --git a/binaries/cuprated/src/config/tokio.rs b/binaries/cuprated/src/config/tokio.rs index 0063ddf..defd34e 100644 --- a/binaries/cuprated/src/config/tokio.rs +++ b/binaries/cuprated/src/config/tokio.rs @@ -11,6 +11,7 @@ pub struct TokioConfig { impl Default for TokioConfig { fn default() -> Self { Self { + // 75% available threads. threads: (std::thread::available_parallelism().unwrap().get() * 3).div_ceil(4), } } diff --git a/binaries/cuprated/src/config/tracing_config.rs b/binaries/cuprated/src/config/tracing_config.rs index bd96e29..2608951 100644 --- a/binaries/cuprated/src/config/tracing_config.rs +++ b/binaries/cuprated/src/config/tracing_config.rs @@ -31,7 +31,8 @@ pub struct FileTracingConfig { /// The default minimum log level. #[serde(with = "level_filter_serde")] pub level: LevelFilter, - + /// The maximum amount of log files to keep, once this number is passed the oldest file + /// will be deleted. pub max_log_files: usize, } diff --git a/p2p/address-book/src/book.rs b/p2p/address-book/src/book.rs index 0b1421d..66811db 100644 --- a/p2p/address-book/src/book.rs +++ b/p2p/address-book/src/book.rs @@ -302,12 +302,6 @@ impl AddressBook { if peb.pruning_seed != peer.pruning_seed { return Err(AddressBookError::PeersDataChanged("Pruning seed")); } - /* - if Z::CHECK_NODE_ID && peb.id != peer.id { - return Err(AddressBookError::PeersDataChanged("peer ID")); - } - - */ // TODO: cuprate doesn't need last seen timestamps but should we have them anyway? peb.last_seen = 0; peb.rpc_port = peer.rpc_port; diff --git a/p2p/p2p/src/block_downloader/download_batch.rs b/p2p/p2p/src/block_downloader/download_batch.rs index 7b6e4c9..dd95218 100644 --- a/p2p/p2p/src/block_downloader/download_batch.rs +++ b/p2p/p2p/src/block_downloader/download_batch.rs @@ -2,7 +2,6 @@ use std::collections::HashSet; use monero_serai::{block::Block, transaction::Transaction}; use rayon::prelude::*; -use tokio::time::timeout; use tower::{Service, ServiceExt}; use tracing::instrument; @@ -16,7 +15,7 @@ use cuprate_wire::protocol::{GetObjectsRequest, GetObjectsResponse}; use crate::{ block_downloader::{BlockBatch, BlockDownloadError, BlockDownloadTaskResponse}, - constants::{BLOCK_DOWNLOADER_REQUEST_TIMEOUT, MAX_TRANSACTION_BLOB_SIZE, MEDIUM_BAN}, + constants::{MAX_TRANSACTION_BLOB_SIZE, MEDIUM_BAN}, peer_set::ClientDropGuard, }; @@ -60,17 +59,15 @@ async fn request_batch_from_peer( })); // Request the blocks and add a timeout to the request - let blocks_response = timeout(BLOCK_DOWNLOADER_REQUEST_TIMEOUT, async { + let blocks_response = { let PeerResponse::Protocol(ProtocolResponse::GetObjects(blocks_response)) = client.ready().await?.call(request).await? else { panic!("Connection task returned wrong response."); }; - Ok::<_, BlockDownloadError>(blocks_response) - }) - .await - .map_err(|_| BlockDownloadError::TimedOut)??; + blocks_response + }; // Initial sanity checks if blocks_response.blocks.len() > ids.len() {