From e3a2877882c22384e413cddf0c17e962f0325dba Mon Sep 17 00:00:00 2001 From: Boog900 <54e72d8a-345f-4599-bd90-c6b9bc7d0ec5@aleeas.com> Date: Wed, 18 Dec 2024 23:08:33 +0000 Subject: [PATCH] manually set numb threads for each pool --- Cargo.lock | 99 +++++++++++++++++---- binaries/cuprated/Cuprated.toml | 2 +- binaries/cuprated/src/blockchain/manager.rs | 4 - binaries/cuprated/src/config.rs | 8 ++ binaries/cuprated/src/config/rayon.rs | 17 ++++ binaries/cuprated/src/config/storage.rs | 12 ++- binaries/cuprated/src/config/tokio.rs | 17 ++++ binaries/cuprated/src/main.rs | 25 ++++-- storage/blockchain/src/service/free.rs | 6 +- storage/txpool/src/service/free.rs | 3 +- storage/txpool/src/service/read.rs | 5 +- 11 files changed, 164 insertions(+), 34 deletions(-) create mode 100644 binaries/cuprated/src/config/rayon.rs create mode 100644 binaries/cuprated/src/config/tokio.rs diff --git a/Cargo.lock b/Cargo.lock index fda0fce4..2826f26d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1234,6 +1234,7 @@ dependencies = [ "toml", "tower 0.5.1", "tracing", + "tracing-appender", "tracing-subscriber", ] @@ -1269,7 +1270,7 @@ dependencies = [ [[package]] name = "dalek-ff-group" version = "0.4.1" -source = "git+https://github.com/Cuprate/serai.git?rev=d5205ce#d5205ce2319e09414eb91d12cf38e83a08165f79" +source = "git+https://github.com/Cuprate/serai.git?rev=e6fdef6#e6fdef6d0b4481932ac9647796eb3fa56197ed66" dependencies = [ "crypto-bigint", "curve25519-dalek", @@ -1296,6 +1297,15 @@ dependencies = [ "parking_lot_core", ] +[[package]] +name = "deranged" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" +dependencies = [ + "powerfmt", +] + [[package]] name = "diff" version = "0.1.13" @@ -1425,7 +1435,7 @@ dependencies = [ [[package]] name = "flexible-transcript" version = "0.3.2" -source = "git+https://github.com/Cuprate/serai.git?rev=d5205ce#d5205ce2319e09414eb91d12cf38e83a08165f79" +source = "git+https://github.com/Cuprate/serai.git?rev=e6fdef6#e6fdef6d0b4481932ac9647796eb3fa56197ed66" dependencies = [ "blake2", "digest", @@ -2175,7 +2185,7 @@ dependencies = [ [[package]] name = "monero-address" version = "0.1.0" -source = "git+https://github.com/Cuprate/serai.git?rev=d5205ce#d5205ce2319e09414eb91d12cf38e83a08165f79" +source = "git+https://github.com/Cuprate/serai.git?rev=e6fdef6#e6fdef6d0b4481932ac9647796eb3fa56197ed66" dependencies = [ "curve25519-dalek", "monero-io", @@ -2188,7 +2198,7 @@ dependencies = [ [[package]] name = "monero-borromean" version = "0.1.0" -source = "git+https://github.com/Cuprate/serai.git?rev=d5205ce#d5205ce2319e09414eb91d12cf38e83a08165f79" +source = "git+https://github.com/Cuprate/serai.git?rev=e6fdef6#e6fdef6d0b4481932ac9647796eb3fa56197ed66" dependencies = [ "curve25519-dalek", "monero-generators", @@ -2201,7 +2211,7 @@ dependencies = [ [[package]] name = "monero-bulletproofs" version = "0.1.0" -source = "git+https://github.com/Cuprate/serai.git?rev=d5205ce#d5205ce2319e09414eb91d12cf38e83a08165f79" +source = "git+https://github.com/Cuprate/serai.git?rev=e6fdef6#e6fdef6d0b4481932ac9647796eb3fa56197ed66" dependencies = [ "curve25519-dalek", "monero-generators", @@ -2216,7 +2226,7 @@ dependencies = [ [[package]] name = "monero-clsag" version = "0.1.0" -source = "git+https://github.com/Cuprate/serai.git?rev=d5205ce#d5205ce2319e09414eb91d12cf38e83a08165f79" +source = "git+https://github.com/Cuprate/serai.git?rev=e6fdef6#e6fdef6d0b4481932ac9647796eb3fa56197ed66" dependencies = [ "curve25519-dalek", "dalek-ff-group", @@ -2236,7 +2246,7 @@ dependencies = [ [[package]] name = "monero-generators" version = "0.4.0" -source = "git+https://github.com/Cuprate/serai.git?rev=d5205ce#d5205ce2319e09414eb91d12cf38e83a08165f79" +source = "git+https://github.com/Cuprate/serai.git?rev=e6fdef6#e6fdef6d0b4481932ac9647796eb3fa56197ed66" dependencies = [ "curve25519-dalek", "dalek-ff-group", @@ -2250,7 +2260,7 @@ dependencies = [ [[package]] name = "monero-io" version = "0.1.0" -source = "git+https://github.com/Cuprate/serai.git?rev=d5205ce#d5205ce2319e09414eb91d12cf38e83a08165f79" +source = "git+https://github.com/Cuprate/serai.git?rev=e6fdef6#e6fdef6d0b4481932ac9647796eb3fa56197ed66" dependencies = [ "curve25519-dalek", "std-shims", @@ -2259,7 +2269,7 @@ dependencies = [ [[package]] name = "monero-mlsag" version = "0.1.0" -source = "git+https://github.com/Cuprate/serai.git?rev=d5205ce#d5205ce2319e09414eb91d12cf38e83a08165f79" +source = "git+https://github.com/Cuprate/serai.git?rev=e6fdef6#e6fdef6d0b4481932ac9647796eb3fa56197ed66" dependencies = [ "curve25519-dalek", "monero-generators", @@ -2273,7 +2283,7 @@ dependencies = [ [[package]] name = "monero-primitives" version = "0.1.0" -source = "git+https://github.com/Cuprate/serai.git?rev=d5205ce#d5205ce2319e09414eb91d12cf38e83a08165f79" +source = "git+https://github.com/Cuprate/serai.git?rev=e6fdef6#e6fdef6d0b4481932ac9647796eb3fa56197ed66" dependencies = [ "curve25519-dalek", "monero-generators", @@ -2286,9 +2296,8 @@ dependencies = [ [[package]] name = "monero-rpc" version = "0.1.0" -source = "git+https://github.com/Cuprate/serai.git?rev=d5205ce#d5205ce2319e09414eb91d12cf38e83a08165f79" +source = "git+https://github.com/Cuprate/serai.git?rev=e6fdef6#e6fdef6d0b4481932ac9647796eb3fa56197ed66" dependencies = [ - "async-trait", "curve25519-dalek", "hex", "monero-address", @@ -2303,7 +2312,7 @@ dependencies = [ [[package]] name = "monero-serai" version = "0.1.4-alpha" -source = "git+https://github.com/Cuprate/serai.git?rev=d5205ce#d5205ce2319e09414eb91d12cf38e83a08165f79" +source = "git+https://github.com/Cuprate/serai.git?rev=e6fdef6#e6fdef6d0b4481932ac9647796eb3fa56197ed66" dependencies = [ "curve25519-dalek", "hex-literal", @@ -2321,9 +2330,8 @@ dependencies = [ [[package]] name = "monero-simple-request-rpc" version = "0.1.0" -source = "git+https://github.com/Cuprate/serai.git?rev=d5205ce#d5205ce2319e09414eb91d12cf38e83a08165f79" +source = "git+https://github.com/Cuprate/serai.git?rev=e6fdef6#e6fdef6d0b4481932ac9647796eb3fa56197ed66" dependencies = [ - "async-trait", "digest_auth", "hex", "monero-rpc", @@ -2341,6 +2349,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + [[package]] name = "num-traits" version = "0.2.19" @@ -2537,6 +2551,12 @@ dependencies = [ "plotters-backend", ] +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + [[package]] name = "ppv-lite86" version = "0.2.20" @@ -3088,7 +3108,7 @@ dependencies = [ [[package]] name = "simple-request" version = "0.1.0" -source = "git+https://github.com/Cuprate/serai.git?rev=d5205ce#d5205ce2319e09414eb91d12cf38e83a08165f79" +source = "git+https://github.com/Cuprate/serai.git?rev=e6fdef6#e6fdef6d0b4481932ac9647796eb3fa56197ed66" dependencies = [ "http-body-util", "hyper", @@ -3154,7 +3174,7 @@ checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" [[package]] name = "std-shims" version = "0.1.1" -source = "git+https://github.com/Cuprate/serai.git?rev=d5205ce#d5205ce2319e09414eb91d12cf38e83a08165f79" +source = "git+https://github.com/Cuprate/serai.git?rev=e6fdef6#e6fdef6d0b4481932ac9647796eb3fa56197ed66" dependencies = [ "hashbrown 0.14.5", "spin", @@ -3296,6 +3316,37 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a693d0c8cf16973fac5a93fbe47b8c6452e7097d4fcac49f3d7a18e39c76e62e" +[[package]] +name = "time" +version = "0.3.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35e7868883861bd0e56d9ac6efcaaca0d6d5d82a2a7ec8209ff492c07cf37b21" +dependencies = [ + "deranged", + "itoa", + "num-conv", + "powerfmt", + "serde", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" + +[[package]] +name = "time-macros" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2834e6017e3e5e4b9834939793b282bc03b37a3336245fa820e35e233e2a85de" +dependencies = [ + "num-conv", + "time-core", +] + [[package]] name = "tinystr" version = "0.7.6" @@ -3389,6 +3440,8 @@ dependencies = [ "bytes", "futures-core", "futures-sink", + "futures-util", + "hashbrown 0.14.5", "pin-project-lite", "slab", "tokio", @@ -3495,6 +3548,18 @@ dependencies = [ "tracing-core", ] +[[package]] +name = "tracing-appender" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3566e8ce28cc0a3fe42519fc80e6b4c943cc4c8cef275620eb8dac2d3d4e06cf" +dependencies = [ + "crossbeam-channel", + "thiserror", + "time", + "tracing-subscriber", +] + [[package]] name = "tracing-attributes" version = "0.1.28" diff --git a/binaries/cuprated/Cuprated.toml b/binaries/cuprated/Cuprated.toml index c9d9de6e..7c465985 100644 --- a/binaries/cuprated/Cuprated.toml +++ b/binaries/cuprated/Cuprated.toml @@ -52,7 +52,7 @@ check_client_pool_interval = { secs = 30, nanos = 0 } ## Storage config [storage] ## The amount of reader threads to spawn. -reader_threads = "OnePerThread" +reader_threads = "Percent(0.25)" ## Txpool storage config. [storage.txpool] diff --git a/binaries/cuprated/src/blockchain/manager.rs b/binaries/cuprated/src/blockchain/manager.rs index 609634c9..210c078a 100644 --- a/binaries/cuprated/src/blockchain/manager.rs +++ b/binaries/cuprated/src/blockchain/manager.rs @@ -133,15 +133,11 @@ impl BlockchainManager { ) { loop { tokio::select! { - /* Some(batch) = block_batch_rx.recv() => { self.handle_incoming_block_batch( batch, ).await; } - - */ - Some(incoming_command) = command_rx.recv() => { self.handle_command(incoming_command).await; } diff --git a/binaries/cuprated/src/config.rs b/binaries/cuprated/src/config.rs index 36d5092a..f78abc74 100644 --- a/binaries/cuprated/src/config.rs +++ b/binaries/cuprated/src/config.rs @@ -21,12 +21,16 @@ mod args; mod fs; mod p2p; mod storage; +mod tokio; mod tracing_config; +mod rayon; use crate::config::fs::FileSystemConfig; +use crate::config::tokio::TokioConfig; use p2p::P2PConfig; use storage::StorageConfig; use tracing_config::TracingConfig; +use crate::config::rayon::RayonConfig; /// Reads the args & config file, returning a [`Config`]. pub fn read_config_and_args() -> Config { @@ -74,6 +78,10 @@ pub struct Config { /// [`tracing`] config. pub tracing: TracingConfig, + pub tokio: TokioConfig, + + pub rayon: RayonConfig, + /// The P2P network config. p2p: P2PConfig, diff --git a/binaries/cuprated/src/config/rayon.rs b/binaries/cuprated/src/config/rayon.rs new file mode 100644 index 00000000..2b830bda --- /dev/null +++ b/binaries/cuprated/src/config/rayon.rs @@ -0,0 +1,17 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Deserialize, Serialize)] +#[serde(deny_unknown_fields, default)] +pub struct RayonConfig { + pub threads: usize, +} + +impl Default for RayonConfig { + fn default() -> Self { + Self { + threads: (std::thread::available_parallelism().unwrap().get() * 3).div_ceil(4), + } + } +} + + diff --git a/binaries/cuprated/src/config/storage.rs b/binaries/cuprated/src/config/storage.rs index b3e3c1f4..df390fba 100644 --- a/binaries/cuprated/src/config/storage.rs +++ b/binaries/cuprated/src/config/storage.rs @@ -7,7 +7,7 @@ use cuprate_database_service::ReaderThreads; use cuprate_helper::fs::CUPRATE_DATA_DIR; /// The storage config. -#[derive(Default, Deserialize, Serialize)] +#[derive(Deserialize, Serialize)] #[serde(deny_unknown_fields, default)] pub struct StorageConfig { /// The amount of reader threads to spawn between the tx-pool and blockchain. @@ -18,6 +18,16 @@ pub struct StorageConfig { pub blockchain: BlockchainConfig, } +impl Default for StorageConfig { + fn default() -> Self { + Self { + reader_threads: ReaderThreads::Percent(0.25), + txpool: Default::default(), + blockchain: Default::default(), + } + } +} + /// The blockchain config. #[derive(Deserialize, Serialize)] #[serde(deny_unknown_fields, default)] diff --git a/binaries/cuprated/src/config/tokio.rs b/binaries/cuprated/src/config/tokio.rs new file mode 100644 index 00000000..dd215889 --- /dev/null +++ b/binaries/cuprated/src/config/tokio.rs @@ -0,0 +1,17 @@ +use serde::{Deserialize, Serialize}; + +/// [`tokio`] config. +#[derive(Deserialize, Serialize)] +#[serde(deny_unknown_fields, default)] +pub struct TokioConfig { + /// The amount of threads to spawn for the async thread-pool + pub threads: usize, +} + +impl Default for TokioConfig { + fn default() -> Self { + Self { + threads: (std::thread::available_parallelism().unwrap().get() * 3).div_ceil(4), + } + } +} diff --git a/binaries/cuprated/src/main.rs b/binaries/cuprated/src/main.rs index 75db7408..52349c0a 100644 --- a/binaries/cuprated/src/main.rs +++ b/binaries/cuprated/src/main.rs @@ -43,14 +43,22 @@ fn main() { let config = config::read_config_and_args(); + init_global_rayon_pool(&config); + logging::init_logging(&config); - let rt = init_tokio_rt(); + let rt = init_tokio_rt(&config); - let db_thread_pool = cuprate_database_service::init_thread_pool(cuprate_database_service::ReaderThreads::Percent(0.3)); + let db_thread_pool = cuprate_database_service::init_thread_pool( + cuprate_database_service::ReaderThreads::Percent(0.3), + ); let (mut blockchain_read_handle, mut blockchain_write_handle, _) = - cuprate_blockchain::service::init_with_pool(config.blockchain_config(), db_thread_pool.clone()).unwrap(); + cuprate_blockchain::service::init_with_pool( + config.blockchain_config(), + db_thread_pool.clone(), + ) + .unwrap(); let (txpool_read_handle, txpool_write_handle, _) = cuprate_txpool::service::init_with_pool(config.txpool_config(), db_thread_pool).unwrap(); @@ -105,14 +113,21 @@ fn main() { }); } -fn init_tokio_rt() -> tokio::runtime::Runtime { +fn init_tokio_rt(config: &Config) -> tokio::runtime::Runtime { tokio::runtime::Builder::new_multi_thread() - .worker_threads(13) + .worker_threads(config.tokio.threads) .enable_all() .build() .unwrap() } +fn init_global_rayon_pool(config: &Config) { + rayon::ThreadPoolBuilder::new() + .num_threads(config.rayon.threads) + .build_global() + .unwrap() +} + async fn io_loop(mut incoming_commands: mpsc::Receiver) -> ! { while let Some(command) = incoming_commands.recv().await { match command { diff --git a/storage/blockchain/src/service/free.rs b/storage/blockchain/src/service/free.rs index 75e98616..20a72527 100644 --- a/storage/blockchain/src/service/free.rs +++ b/storage/blockchain/src/service/free.rs @@ -1,15 +1,15 @@ //! General free functions used (related to `cuprate_blockchain::service`). //---------------------------------------------------------------------------------------------------- Import -use std::sync::Arc; -use rayon::ThreadPool; use cuprate_database::{ConcreteEnv, InitError}; use cuprate_types::{AltBlockInformation, VerifiedBlockInformation}; +use rayon::ThreadPool; +use std::sync::Arc; use crate::{ config::Config, service::{ - init_read_service, init_write_service, init_read_service_with_pool, + init_read_service, init_read_service_with_pool, init_write_service, types::{BlockchainReadHandle, BlockchainWriteHandle}, }, }; diff --git a/storage/txpool/src/service/free.rs b/storage/txpool/src/service/free.rs index 67e656e6..0971125e 100644 --- a/storage/txpool/src/service/free.rs +++ b/storage/txpool/src/service/free.rs @@ -4,6 +4,7 @@ use rayon::ThreadPool; use cuprate_database::{ConcreteEnv, InitError}; +use crate::service::read::init_read_service_with_pool; use crate::{ service::{ read::init_read_service, @@ -12,7 +13,6 @@ use crate::{ }, Config, }; -use crate::service::read::init_read_service_with_pool; //---------------------------------------------------------------------------------------------------- Init #[cold] @@ -52,4 +52,3 @@ pub fn init_with_pool( Ok((readers, writer, db)) } - diff --git a/storage/txpool/src/service/read.rs b/storage/txpool/src/service/read.rs index 8e6b47e3..5fc20dd1 100644 --- a/storage/txpool/src/service/read.rs +++ b/storage/txpool/src/service/read.rs @@ -44,7 +44,10 @@ pub(super) fn init_read_service(env: Arc, threads: ReaderThreads) - /// Should be called _once_ per actual database. #[cold] #[inline(never)] // Only called once. -pub(super) fn init_read_service_with_pool(env: Arc, pool: Arc) -> TxpoolReadHandle { +pub(super) fn init_read_service_with_pool( + env: Arc, + pool: Arc, +) -> TxpoolReadHandle { DatabaseReadService::new(env, pool, map_request) }