SyntheticBird/cuprate-hinto-janai
1
0
Fork 0
mirror of https://github.com/hinto-janai/cuprate.git synced 2025-02-02 03:06:35 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

database: remove ReaderThreads, make db_directory mandatory
Some checks are pending
Audit / audit (push) Waiting to run
Details
Deny / audit (push) Waiting to run
Details

Browse source
This commit is contained in:
hinto.janai 2024-06-13 17:21:51 -04:00
parent 7024517cf4
commit 98413dfacd
No known key found for this signature in database
GPG key ID: D47CE05FA175A499
6 changed files with 48 additions and 279 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
storage/database/README.md
View file

@ -98,9 +98,7 @@ use cuprate_database::{
# fn main() -> Result<(), Box<dyn std::error::Error>> {
// Create a configuration for the database environment.
let db_dir = tempfile::tempdir()?;
let config = ConfigBuilder::new()
.db_directory(db_dir.path().to_path_buf())
.build();
let config = ConfigBuilder::new(db_dir.path().to_path_buf()).build();
// Initialize the database environment.
let env = ConcreteEnv::open(config)?;

3
storage/database/src/backend/heed/env.rs
View file

@ -182,8 +182,7 @@ impl Env for ConcreteEnv {
// For now:
// - No other program using our DB exists
// - Almost no-one has a 126+ thread CPU
let reader_threads =
u32::try_from(config.reader_threads.as_threads().get()).unwrap_or(u32::MAX);
let reader_threads = u32::try_from(config.reader_threads.get()).unwrap_or(u32::MAX);
env_open_options.max_readers(if reader_threads < 110 {
126
} else {

112
storage/database/src/config/config.rs
View file

@ -3,18 +3,25 @@
//---------------------------------------------------------------------------------------------------- Import
use std::{
borrow::Cow,
num::NonZeroUsize,
path::{Path, PathBuf},
};
#[cfg(feature = "serde")]
use serde::{Deserialize, Serialize};
use cuprate_helper::fs::cuprate_blockchain_dir;
use crate::{config::SyncMode, constants::DATABASE_DATA_FILENAME, resize::ResizeAlgorithm};
use crate::{
config::{ReaderThreads, SyncMode},
constants::DATABASE_DATA_FILENAME,
resize::ResizeAlgorithm,
//---------------------------------------------------------------------------------------------------- Constants
/// Default value for [`Config::reader_threads`].
///
/// ```rust
/// use cuprate_database::config::*;
/// assert_eq!(READER_THREADS_DEFAULT.get(), 126);
/// ```
pub const READER_THREADS_DEFAULT: NonZeroUsize = match NonZeroUsize::new(126) {
Some(n) => n,
None => unreachable!(),
};
//---------------------------------------------------------------------------------------------------- ConfigBuilder
@ -25,13 +32,13 @@ use crate::{
#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
pub struct ConfigBuilder {
/// [`Config::db_directory`].
db_directory: Option<Cow<'static, Path>>,
db_directory: Cow<'static, Path>,
/// [`Config::sync_mode`].
sync_mode: Option<SyncMode>,
/// [`Config::reader_threads`].
reader_threads: Option<ReaderThreads>,
reader_threads: Option<NonZeroUsize>,
/// [`Config::resize_algorithm`].
resize_algorithm: Option<ResizeAlgorithm>,
@ -42,11 +49,11 @@ impl ConfigBuilder {
///
/// [`ConfigBuilder::build`] can be called immediately
/// after this function to use default values.
pub const fn new() -> Self {
pub const fn new(db_directory: PathBuf) -> Self {
Self {
db_directory: None,
db_directory: Cow::Owned(db_directory),
sync_mode: None,
reader_threads: None,
reader_threads: Some(READER_THREADS_DEFAULT),
resize_algorithm: None,
}
}
@ -54,41 +61,25 @@ impl ConfigBuilder {
/// Build into a [`Config`].
///
/// # Default values
/// If [`ConfigBuilder::db_directory`] was not called,
/// the default [`cuprate_blockchain_dir`] will be used.
///
/// For all other values, [`Default::default`] is used.
/// - [`READER_THREADS_DEFAULT`] is used ofr [`Config::reader_threads`]
/// - [`Default::default`] is used for all other values (except the `db_directory`)
pub fn build(self) -> Config {
// INVARIANT: all PATH safety checks are done
// in `helper::fs`. No need to do them here.
// TODO: fix me
let db_directory = self
.db_directory
.unwrap_or_else(|| Cow::Borrowed(cuprate_blockchain_dir()));
// Add the database filename to the directory.
let db_file = {
let mut db_file = db_directory.to_path_buf();
let mut db_file = self.db_directory.to_path_buf();
db_file.push(DATABASE_DATA_FILENAME);
Cow::Owned(db_file)
};
Config {
db_directory,
db_directory: self.db_directory,
db_file,
sync_mode: self.sync_mode.unwrap_or_default(),
reader_threads: self.reader_threads.unwrap_or_default(),
reader_threads: self.reader_threads.unwrap_or(READER_THREADS_DEFAULT),
resize_algorithm: self.resize_algorithm.unwrap_or_default(),
}
}
/// Set a custom database directory (and file) [`Path`].
#[must_use]
pub fn db_directory(mut self, db_directory: PathBuf) -> Self {
self.db_directory = Some(Cow::Owned(db_directory));
self
}
/// Tune the [`ConfigBuilder`] for the highest performing,
/// but also most resource-intensive & maybe risky settings.
///
@ -96,7 +87,6 @@ impl ConfigBuilder {
#[must_use]
pub fn fast(mut self) -> Self {
self.sync_mode = Some(SyncMode::Fast);
self.reader_threads = Some(ReaderThreads::OnePerThread);
self.resize_algorithm = Some(ResizeAlgorithm::default());
self
}
@ -108,7 +98,6 @@ impl ConfigBuilder {
#[must_use]
pub fn low_power(mut self) -> Self {
self.sync_mode = Some(SyncMode::default());
self.reader_threads = Some(ReaderThreads::One);
self.resize_algorithm = Some(ResizeAlgorithm::default());
self
}
@ -120,9 +109,9 @@ impl ConfigBuilder {
self
}
/// Set a custom [`ReaderThreads`].
/// Set a custom [`Config::reader_threads`].
#[must_use]
pub const fn reader_threads(mut self, reader_threads: ReaderThreads) -> Self {
pub const fn reader_threads(mut self, reader_threads: NonZeroUsize) -> Self {
self.reader_threads = Some(reader_threads);
self
}
@ -135,25 +124,13 @@ impl ConfigBuilder {
}
}
impl Default for ConfigBuilder {
fn default() -> Self {
Self {
// TODO: fix me
db_directory: Some(Cow::Borrowed(cuprate_blockchain_dir())),
sync_mode: Some(SyncMode::default()),
reader_threads: Some(ReaderThreads::default()),
resize_algorithm: Some(ResizeAlgorithm::default()),
}
}
}
//---------------------------------------------------------------------------------------------------- Config
/// Database [`Env`](crate::Env) configuration.
///
/// This is the struct passed to [`Env::open`](crate::Env::open) that
/// allows the database to be configured in various ways.
///
/// For construction, either use [`ConfigBuilder`] or [`Config::default`].
/// For construction, use [`ConfigBuilder`].
///
// SOMEDAY: there's are many more options to add in the future.
#[derive(Debug, Clone, PartialEq, PartialOrd)]
@ -179,7 +156,14 @@ pub struct Config {
pub sync_mode: SyncMode,
/// Database reader thread count.
pub reader_threads: ReaderThreads,
///
/// Set the number of slots in the reader table.
///
/// This is only used in LMDB, see
/// <https://github.com/LMDB/lmdb/blob/b8e54b4c31378932b69f1298972de54a565185b1/libraries/liblmdb/mdb.c#L794-L799>.
///
/// By default, this value is [`READER_THREADS_DEFAULT`].
pub reader_threads: NonZeroUsize,
/// Database memory map resizing algorithm.
///
@ -192,27 +176,25 @@ pub struct Config {
impl Config {
/// Create a new [`Config`] with sane default settings.
///
/// The [`Config::db_directory`] will be [`cuprate_blockchain_dir`].
/// The [`Config::db_directory`] must be passed.
///
/// All other values will be [`Default::default`].
///
/// Same as [`Config::default`].
///
/// ```rust
/// use cuprate_database::{config::*, resize::*, DATABASE_DATA_FILENAME};
/// use cuprate_helper::fs::*;
///
/// let config = Config::new();
/// let db_directory = tempfile::tempdir().unwrap();
/// let config = Config::new(db_directory.path().into());
///
/// assert_eq!(config.db_directory(), cuprate_blockchain_dir());
/// assert!(config.db_file().starts_with(cuprate_blockchain_dir()));
/// assert_eq!(config.db_directory(), db_directory.path());
/// assert!(config.db_file().starts_with(db_directory));
/// assert!(config.db_file().ends_with(DATABASE_DATA_FILENAME));
/// assert_eq!(config.sync_mode, SyncMode::default());
/// assert_eq!(config.reader_threads, ReaderThreads::default());
/// assert_eq!(config.reader_threads, READER_THREADS_DEFAULT);
/// assert_eq!(config.resize_algorithm, ResizeAlgorithm::default());
/// ```
pub fn new() -> Self {
ConfigBuilder::default().build()
pub fn new(db_directory: PathBuf) -> Self {
ConfigBuilder::new(db_directory).build()
}
/// Return the absolute [`Path`] to the database directory.
@ -225,15 +207,3 @@ impl Config {
&self.db_file
}
}
impl Default for Config {
/// Same as [`Config::new`].
///
/// ```rust
/// # use cuprate_database::config::*;
/// assert_eq!(Config::default(), Config::new());
/// ```
fn default() -> Self {
Self::new()
}
}

15
storage/database/src/config/mod.rs
View file

@ -14,23 +14,19 @@
//! ```rust
//! use cuprate_database::{
//! ConcreteEnv, Env,
//! config::{ConfigBuilder, ReaderThreads, SyncMode}
//! config::{ConfigBuilder, SyncMode}
//! };
//!
//! # fn main() -> Result<(), Box<dyn std::error::Error>> {
//! let db_dir = tempfile::tempdir()?;
//!
//! let config = ConfigBuilder::new()
//! // Use a custom database directory.
//! .db_directory(db_dir.path().to_path_buf())
//! // Use as many reader threads as possible (when using `service`).
//! .reader_threads(ReaderThreads::OnePerThread)
//! let config = ConfigBuilder::new(db_dir.path().to_path_buf())
//! // Use the fastest sync mode.
//! .sync_mode(SyncMode::Fast)
//! // Build into `Config`
//! .build();
//!
//! // Open the database `service` using this configuration.
//! // Open the database using this configuration.
//! let env = ConcreteEnv::open(config.clone())?;
//! // It's using the config we provided.
//! assert_eq!(env.config(), &config);
@ -38,10 +34,7 @@
//! ```
mod config;
pub use config::{Config, ConfigBuilder};
mod reader_threads;
pub use reader_threads::ReaderThreads;
pub use config::{Config, ConfigBuilder, READER_THREADS_DEFAULT};
mod sync_mode;
pub use sync_mode::SyncMode;

190
storage/database/src/config/reader_threads.rs
View file

@ -1,190 +0,0 @@
//! Database [`Env`](crate::Env) configuration.
//!
//! This module contains the main [`Config`]uration struct
//! for the database [`Env`](crate::Env)ironment, and data
//! structures related to any configuration setting.
//!
//! These configurations are processed at runtime, meaning
//! the `Env` can/will dynamically adjust its behavior
//! based on these values.
//---------------------------------------------------------------------------------------------------- Import
use std::num::NonZeroUsize;
#[cfg(feature = "serde")]
use serde::{Deserialize, Serialize};
//---------------------------------------------------------------------------------------------------- ReaderThreads
/// TODO: fix me
// Amount of database reader threads to spawn when using [`service`](crate::service).
///
/// This controls how many reader thread `service`'s
/// thread-pool will spawn to receive and send requests/responses.
///
/// It does nothing outside of `service`.
///
/// It will always be at least 1, up until the amount of threads on the machine.
///
/// The main function used to extract an actual
/// usable thread count out of this is [`ReaderThreads::as_threads`].
#[derive(Copy, Clone, Debug, Default, PartialEq, PartialOrd)]
#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
pub enum ReaderThreads {
#[default]
/// Spawn 1 reader thread per available thread on the machine.
///
/// For example, a `32-thread` system will spawn
/// `32` reader threads using this setting.
OnePerThread,
/// Only spawn 1 reader thread.
One,
/// Spawn a specified amount of reader threads.
///
/// Note that no matter how large this value, it will be
/// ultimately capped at the amount of system threads.
///
/// # `0`
/// `ReaderThreads::Number(0)` represents "use maximum value",
/// as such, it is equal to [`ReaderThreads::OnePerThread`].
///
/// ```rust
/// # use cuprate_database::config::*;
/// let reader_threads = ReaderThreads::from(0_usize);
/// assert!(matches!(reader_threads, ReaderThreads::OnePerThread));
/// ```
Number(usize),
/// Spawn a specified % of reader threads.
///
/// This must be a value in-between `0.0..1.0`
/// where `1.0` represents [`ReaderThreads::OnePerThread`].
///
/// # Example
/// For example, using a `16-core, 32-thread` Ryzen 5950x CPU:
///
/// | Input | Total thread used |
/// |------------------------------------|-------------------|
/// | `ReaderThreads::Percent(0.0)` | 32 (maximum value)
/// | `ReaderThreads::Percent(0.5)` | 16
/// | `ReaderThreads::Percent(0.75)` | 24
/// | `ReaderThreads::Percent(1.0)` | 32
/// | `ReaderThreads::Percent(2.0)` | 32 (saturating)
/// | `ReaderThreads::Percent(f32::NAN)` | 32 (non-normal default)
///
/// # `0.0`
/// `ReaderThreads::Percent(0.0)` represents "use maximum value",
/// as such, it is equal to [`ReaderThreads::OnePerThread`].
///
/// # Not quite `0.0`
/// If the thread count multiplied by the percentage ends up being
/// non-zero, but not 1 thread, the minimum value 1 will be returned.
///
/// ```rust
/// # use cuprate_database::config::*;
/// assert_eq!(ReaderThreads::Percent(0.000000001).as_threads().get(), 1);
/// ```
Percent(f32),
}
impl ReaderThreads {
/// This converts [`ReaderThreads`] into a safe, usable
/// number representing how many threads to spawn.
///
/// This function will always return a number in-between `1..=total_thread_count`.
///
/// It uses [`cuprate_helper::thread::threads()`] internally to determine the total thread count.
///
/// # Example
/// ```rust
/// use cuprate_database::config::ReaderThreads as Rt;
///
/// let total_threads: std::num::NonZeroUsize =
/// cuprate_helper::thread::threads();
///
/// assert_eq!(Rt::OnePerThread.as_threads(), total_threads);
///
/// assert_eq!(Rt::One.as_threads().get(), 1);
///
/// assert_eq!(Rt::Number(0).as_threads(), total_threads);
/// assert_eq!(Rt::Number(1).as_threads().get(), 1);
/// assert_eq!(Rt::Number(usize::MAX).as_threads(), total_threads);
///
/// assert_eq!(Rt::Percent(0.01).as_threads().get(), 1);
/// assert_eq!(Rt::Percent(0.0).as_threads(), total_threads);
/// assert_eq!(Rt::Percent(1.0).as_threads(), total_threads);
/// assert_eq!(Rt::Percent(f32::NAN).as_threads(), total_threads);
/// assert_eq!(Rt::Percent(f32::INFINITY).as_threads(), total_threads);
/// assert_eq!(Rt::Percent(f32::NEG_INFINITY).as_threads(), total_threads);
///
/// // Percentage only works on more than 1 thread.
/// if total_threads.get() > 1 {
/// assert_eq!(
/// Rt::Percent(0.5).as_threads().get(),
/// (total_threads.get() as f32 / 2.0) as usize,
/// );
/// }
/// ```
//
// INVARIANT:
// LMDB will error if we input zero, so don't allow that.
// <https://github.com/LMDB/lmdb/blob/b8e54b4c31378932b69f1298972de54a565185b1/libraries/liblmdb/mdb.c#L4687>
pub fn as_threads(&self) -> NonZeroUsize {
let total_threads = cuprate_helper::thread::threads();
match self {
Self::OnePerThread => total_threads, // use all threads
Self::One => NonZeroUsize::MIN, // one
Self::Number(n) => match NonZeroUsize::new(*n) {
Some(n) => std::cmp::min(n, total_threads), // saturate at total threads
None => total_threads, // 0 == maximum value
},
// We handle the casting loss.
#[allow(
clippy::cast_precision_loss,
clippy::cast_possible_truncation,
clippy::cast_sign_loss
)]
Self::Percent(f) => {
// If non-normal float, use the default (all threads).
if !f.is_normal() || !(0.0..=1.0).contains(f) {
return total_threads;
}
// 0.0 == maximum value.
if *f == 0.0 {
return total_threads;
}
// Calculate percentage of total threads.
let thread_percent = (total_threads.get() as f32) * f;
match NonZeroUsize::new(thread_percent as usize) {
Some(n) => std::cmp::min(n, total_threads), // saturate at total threads.
None => {
// We checked for `0.0` above, so what this
// being 0 means that the percentage was _so_
// low it made our thread count something like
// 0.99. In this case, just use 1 thread.
NonZeroUsize::MIN
}
}
}
}
}
}
impl<T: Into<usize>> From<T> for ReaderThreads {
/// Create a [`ReaderThreads::Number`].
///
/// If `value` is `0`, this will return [`ReaderThreads::OnePerThread`].
fn from(value: T) -> Self {
let u: usize = value.into();
if u == 0 {
Self::OnePerThread
} else {
Self::Number(u)
}
}
}

3
storage/database/src/tests.rs
View file

@ -24,8 +24,7 @@ impl Table for TestTable {
/// FIXME: changing this to `-> impl Env` causes lifetime errors...
pub(crate) fn tmp_concrete_env() -> (ConcreteEnv, tempfile::TempDir) {
let tempdir = tempfile::tempdir().unwrap();
let config = ConfigBuilder::new()
.db_directory(tempdir.path().into())
let config = ConfigBuilder::new(tempdir.path().into())
.low_power()
.build();
let env = ConcreteEnv::open(config).unwrap();