From 0a390a362ab8906b0c8a611d3334e262d5f1988d Mon Sep 17 00:00:00 2001 From: hinto-janai Date: Fri, 12 Jul 2024 17:15:02 -0400 Subject: [PATCH] storage: doc fixes (#228) * database: doc fixes * blockchain: doc fixes * database: fix doc test * database: readme fixes * blockchain: ops fix * blockchain: readme fix --- storage/blockchain/README.md | 8 ++- .../blockchain/src/config/reader_threads.rs | 33 ++++++----- storage/blockchain/src/free.rs | 2 +- storage/blockchain/src/ops/block.rs | 2 +- storage/blockchain/src/ops/mod.rs | 8 +-- storage/database/README.md | 18 +++--- storage/database/src/config/config.rs | 2 +- storage/database/src/config/sync_mode.rs | 4 +- storage/database/src/env.rs | 56 ++++++++++++++----- storage/database/src/error.rs | 4 +- storage/database/src/storable.rs | 2 +- storage/database/src/table.rs | 2 + storage/database/src/tables.rs | 20 ++++--- storage/database/src/transaction.rs | 4 +- 14 files changed, 100 insertions(+), 65 deletions(-) diff --git a/storage/blockchain/README.md b/storage/blockchain/README.md index 58c06e0..4800546 100644 --- a/storage/blockchain/README.md +++ b/storage/blockchain/README.md @@ -5,6 +5,10 @@ This documentation is mostly for practical usage of `cuprate_blockchain`. For a high-level overview, see the database section in [Cuprate's architecture book](https://architecture.cuprate.org). +If you're looking for a database crate, consider using the lower-level +[`cuprate-database`](https://doc.cuprate.org/cuprate_database) +crate that this crate is built on-top of. + # Purpose This crate does 3 things: 1. Uses [`cuprate_database`] as a base database layer @@ -47,11 +51,11 @@ there are some things that must be kept in mind when doing so. Failing to uphold these invariants may cause panics. 1. `LMDB` requires the user to resize the memory map resizing (see [`cuprate_database::RuntimeError::ResizeNeeded`] -1. `LMDB` has a maximum reader transaction count, currently it is set to `128` +1. `LMDB` has a maximum reader transaction count, currently, [it is set to `126`](https://github.com/LMDB/lmdb/blob/b8e54b4c31378932b69f1298972de54a565185b1/libraries/liblmdb/mdb.c#L794-L799) 1. `LMDB` has [maximum key/value byte size](http://www.lmdb.tech/doc/group__internal.html#gac929399f5d93cef85f874b9e9b1d09e0) which must not be exceeded # Examples -The below is an example of using `cuprate_blockchain` +The below is an example of using `cuprate_blockchain`'s lowest API, i.e. using a mix of this crate and `cuprate_database`'s traits directly - **this is NOT recommended.** diff --git a/storage/blockchain/src/config/reader_threads.rs b/storage/blockchain/src/config/reader_threads.rs index 04216e3..d4dd6ac 100644 --- a/storage/blockchain/src/config/reader_threads.rs +++ b/storage/blockchain/src/config/reader_threads.rs @@ -20,12 +20,11 @@ use serde::{Deserialize, Serialize}; /// This controls how many reader thread `service`'s /// thread-pool will spawn to receive and send requests/responses. /// -/// It does nothing outside of `service`. -/// -/// It will always be at least 1, up until the amount of threads on the machine. -/// +/// # Invariant /// The main function used to extract an actual /// usable thread count out of this is [`ReaderThreads::as_threads`]. +/// +/// This will always return at least 1, up until the amount of threads on the machine. #[derive(Copy, Clone, Debug, Default, PartialEq, PartialOrd)] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] pub enum ReaderThreads { @@ -97,30 +96,30 @@ impl ReaderThreads { /// /// # Example /// ```rust - /// use cuprate_blockchain::config::ReaderThreads as Rt; + /// use cuprate_blockchain::config::ReaderThreads as R; /// /// let total_threads: std::num::NonZeroUsize = /// cuprate_helper::thread::threads(); /// - /// assert_eq!(Rt::OnePerThread.as_threads(), total_threads); + /// assert_eq!(R::OnePerThread.as_threads(), total_threads); /// - /// assert_eq!(Rt::One.as_threads().get(), 1); + /// assert_eq!(R::One.as_threads().get(), 1); /// - /// assert_eq!(Rt::Number(0).as_threads(), total_threads); - /// assert_eq!(Rt::Number(1).as_threads().get(), 1); - /// assert_eq!(Rt::Number(usize::MAX).as_threads(), total_threads); + /// assert_eq!(R::Number(0).as_threads(), total_threads); + /// assert_eq!(R::Number(1).as_threads().get(), 1); + /// assert_eq!(R::Number(usize::MAX).as_threads(), total_threads); /// - /// assert_eq!(Rt::Percent(0.01).as_threads().get(), 1); - /// assert_eq!(Rt::Percent(0.0).as_threads(), total_threads); - /// assert_eq!(Rt::Percent(1.0).as_threads(), total_threads); - /// assert_eq!(Rt::Percent(f32::NAN).as_threads(), total_threads); - /// assert_eq!(Rt::Percent(f32::INFINITY).as_threads(), total_threads); - /// assert_eq!(Rt::Percent(f32::NEG_INFINITY).as_threads(), total_threads); + /// assert_eq!(R::Percent(0.01).as_threads().get(), 1); + /// assert_eq!(R::Percent(0.0).as_threads(), total_threads); + /// assert_eq!(R::Percent(1.0).as_threads(), total_threads); + /// assert_eq!(R::Percent(f32::NAN).as_threads(), total_threads); + /// assert_eq!(R::Percent(f32::INFINITY).as_threads(), total_threads); + /// assert_eq!(R::Percent(f32::NEG_INFINITY).as_threads(), total_threads); /// /// // Percentage only works on more than 1 thread. /// if total_threads.get() > 1 { /// assert_eq!( - /// Rt::Percent(0.5).as_threads().get(), + /// R::Percent(0.5).as_threads().get(), /// (total_threads.get() as f32 / 2.0) as usize, /// ); /// } diff --git a/storage/blockchain/src/free.rs b/storage/blockchain/src/free.rs index 87e63d7..8288e65 100644 --- a/storage/blockchain/src/free.rs +++ b/storage/blockchain/src/free.rs @@ -6,7 +6,7 @@ use cuprate_database::{ConcreteEnv, Env, EnvInner, InitError, RuntimeError, TxRw use crate::{config::Config, tables::OpenTables}; //---------------------------------------------------------------------------------------------------- Free functions -/// Open the blockchain database, using the passed [`Config`]. +/// Open the blockchain database using the passed [`Config`]. /// /// This calls [`cuprate_database::Env::open`] and prepares the /// database to be ready for blockchain-related usage, e.g. diff --git a/storage/blockchain/src/ops/block.rs b/storage/blockchain/src/ops/block.rs index 7e0284d..b0eb013 100644 --- a/storage/blockchain/src/ops/block.rs +++ b/storage/blockchain/src/ops/block.rs @@ -1,4 +1,4 @@ -//! Blocks functions. +//! Block functions. //---------------------------------------------------------------------------------------------------- Import use bytemuck::TransparentWrapper; diff --git a/storage/blockchain/src/ops/mod.rs b/storage/blockchain/src/ops/mod.rs index 7210ae7..2699fc8 100644 --- a/storage/blockchain/src/ops/mod.rs +++ b/storage/blockchain/src/ops/mod.rs @@ -5,14 +5,14 @@ //! database operations. //! //! # `impl Table` -//! `ops/` functions take [`Tables`](crate::tables::Tables) and +//! Functions in this module take [`Tables`](crate::tables::Tables) and //! [`TablesMut`](crate::tables::TablesMut) directly - these are //! _already opened_ database tables. //! -//! As such, the function puts the responsibility -//! of transactions, tables, etc on the caller. +//! As such, the responsibility of +//! transactions, tables, etc, are on the caller. //! -//! This does mean these functions are mostly as lean +//! Notably, this means that these functions are as lean //! as possible, so calling them in a loop should be okay. //! //! # Atomicity diff --git a/storage/database/README.md b/storage/database/README.md index 5bcec60..aed738e 100644 --- a/storage/database/README.md +++ b/storage/database/README.md @@ -6,10 +6,10 @@ For a high-level overview, see the database section in [Cuprate's architecture book](https://architecture.cuprate.org). If you need blockchain specific capabilities, consider using the higher-level -`cuprate-blockchain` crate which builds upon this one. +[`cuprate-blockchain`](https://doc.cuprate.org/cuprate_blockchain) crate which builds upon this one. # Purpose -This crate abstracts various database backends with traits. The databases are: +This crate abstracts various database backends with traits. All backends have the following attributes: - [Embedded](https://en.wikipedia.org/wiki/Embedded_database) @@ -19,6 +19,10 @@ All backends have the following attributes: - Are table oriented (`"table_name" -> (key, value)`) - Allows concurrent readers +The currently implemented backends are: +- [`heed`](https://github.com/meilisearch/heed) (LMDB) +- [`redb`](https://github.com/cberner/redb) + # Terminology To be more clear on some terms used in this crate: @@ -26,17 +30,17 @@ To be more clear on some terms used in this crate: |------------------|--------------------------------------| | `Env` | The 1 database environment, the "whole" thing | `DatabaseR{o,w}` | A _actively open_ readable/writable `key/value` store -| `Table` | Solely the metadata of a `cuprate_database` (the `key` and `value` types, and the name) +| `Table` | Solely the metadata of a `Database` (the `key` and `value` types, and the name) | `TxR{o,w}` | A read/write transaction -| `Storable` | A data that type can be stored in the database +| `Storable` | A data type that can be stored in the database -The dataflow is `Env` -> `Tx` -> `cuprate_database` +The flow is `Env` -> `Tx` -> `Database` Which reads as: 1. You have a database `Environment` 1. You open up a `Transaction` -1. You open a particular `Table` from that `Environment`, getting a `cuprate_database` -1. You can now read/write data from/to that `cuprate_database` +1. You open a particular `Table` from that `Environment`, getting a `Database` +1. You can now read/write data from/to that `Database` # Concrete types You should _not_ rely on the concrete type of any abstracted backend. diff --git a/storage/database/src/config/config.rs b/storage/database/src/config/config.rs index a5ecbb2..8a4ddbf 100644 --- a/storage/database/src/config/config.rs +++ b/storage/database/src/config/config.rs @@ -160,7 +160,7 @@ pub struct Config { /// Set the number of slots in the reader table. /// /// This is only used in LMDB, see - /// . + /// [here](https://github.com/LMDB/lmdb/blob/b8e54b4c31378932b69f1298972de54a565185b1/libraries/liblmdb/mdb.c#L794-L799). /// /// By default, this value is [`READER_THREADS_DEFAULT`]. pub reader_threads: NonZeroUsize, diff --git a/storage/database/src/config/sync_mode.rs b/storage/database/src/config/sync_mode.rs index e000462..5a0cba5 100644 --- a/storage/database/src/config/sync_mode.rs +++ b/storage/database/src/config/sync_mode.rs @@ -127,8 +127,8 @@ pub enum SyncMode { /// In the case of a system crash, the database /// may become corrupted when using this option. /// - /// [^1]: - /// Semantically, this variant would actually map to + /// + /// [^1]: Semantically, this variant would actually map to /// [`redb::Durability::None`](https://docs.rs/redb/1.5.0/redb/enum.Durability.html#variant.None), /// however due to [`#149`](https://github.com/Cuprate/cuprate/issues/149), /// this is not possible. As such, when using the `redb` backend, diff --git a/storage/database/src/env.rs b/storage/database/src/env.rs index de094a9..cae4973 100644 --- a/storage/database/src/env.rs +++ b/storage/database/src/env.rs @@ -24,15 +24,14 @@ use crate::{ /// /// # Lifetimes /// The lifetimes associated with `Env` have a sequential flow: -/// 1. `ConcreteEnv` -/// 2. `'env` -/// 3. `'tx` -/// 4. `'db` +/// ```text +/// Env -> Tx -> Database +/// ``` /// /// As in: /// - open database tables only live as long as... /// - transactions which only live as long as the... -/// - environment ([`EnvInner`]) +/// - database environment pub trait Env: Sized { //------------------------------------------------ Constants /// Does the database backend need to be manually @@ -202,18 +201,19 @@ Subsequent table opens will follow the flags/ordering, but only if /// Note that when opening tables with [`EnvInner::open_db_ro`], /// they must be created first or else it will return error. /// -/// Note that when opening tables with [`EnvInner::open_db_ro`], -/// they must be created first or else it will return error. -/// /// See [`EnvInner::create_db`] for creating tables. /// /// # Invariant #[doc = doc_heed_create_db_invariant!()] pub trait EnvInner<'env> { /// The read-only transaction type of the backend. - type Ro<'a>: TxRo<'a>; + /// + /// `'tx` is the lifetime of the transaction itself. + type Ro<'tx>: TxRo<'tx>; /// The read-write transaction type of the backend. - type Rw<'a>: TxRw<'a>; + /// + /// `'tx` is the lifetime of the transaction itself. + type Rw<'tx>: TxRw<'tx>; /// Create a read-only transaction. /// @@ -235,11 +235,37 @@ pub trait EnvInner<'env> { /// This will open the database [`Table`] /// passed as a generic to this function. /// - /// ```rust,ignore - /// let db = env.open_db_ro::(&tx_ro); - /// // ^ ^ - /// // database table table metadata - /// // (name, key/value type) + /// ```rust + /// # use cuprate_database::{ + /// # ConcreteEnv, + /// # config::ConfigBuilder, + /// # Env, EnvInner, + /// # DatabaseRo, DatabaseRw, TxRo, TxRw, + /// # }; + /// # fn main() -> Result<(), Box> { + /// # let tmp_dir = tempfile::tempdir()?; + /// # let db_dir = tmp_dir.path().to_owned(); + /// # let config = ConfigBuilder::new(db_dir.into()).build(); + /// # let env = ConcreteEnv::open(config)?; + /// # + /// # struct Table; + /// # impl cuprate_database::Table for Table { + /// # const NAME: &'static str = "table"; + /// # type Key = u8; + /// # type Value = u64; + /// # } + /// # + /// # let env_inner = env.env_inner(); + /// # let tx_rw = env_inner.tx_rw()?; + /// # env_inner.create_db::
(&tx_rw)?; + /// # TxRw::commit(tx_rw); + /// # + /// # let tx_ro = env_inner.tx_ro()?; + /// let db = env_inner.open_db_ro::
(&tx_ro); + /// // ^ ^ + /// // database table table metadata + /// // (name, key/value type) + /// # Ok(()) } /// ``` /// /// # Errors diff --git a/storage/database/src/error.rs b/storage/database/src/error.rs index 386091d..3471ac7 100644 --- a/storage/database/src/error.rs +++ b/storage/database/src/error.rs @@ -59,18 +59,16 @@ pub enum InitError { } //---------------------------------------------------------------------------------------------------- RuntimeError -/// Errors that occur _after_ successful ([`Env::open`](crate::env::Env::open)). +/// Errors that occur _after_ successful [`Env::open`](crate::env::Env::open). /// /// There are no errors for: /// 1. Missing tables /// 2. (De)serialization -/// 3. Shutdown errors /// /// as `cuprate_database` upholds the invariant that: /// /// 1. All tables exist /// 2. (De)serialization never fails -/// 3. The database (thread-pool) only shuts down when all channels are dropped #[derive(thiserror::Error, Debug)] pub enum RuntimeError { /// The given key already existed in the database. diff --git a/storage/database/src/storable.rs b/storage/database/src/storable.rs index 8842af7..100ed44 100644 --- a/storage/database/src/storable.rs +++ b/storage/database/src/storable.rs @@ -129,7 +129,7 @@ where /// /// Slice types are owned both: /// - when returned from the database -/// - in `put()` +/// - in [`crate::DatabaseRw::put()`] /// /// This is needed as `impl Storable for Vec` runs into impl conflicts. /// diff --git a/storage/database/src/table.rs b/storage/database/src/table.rs index 56e84dd..3ad0e79 100644 --- a/storage/database/src/table.rs +++ b/storage/database/src/table.rs @@ -8,6 +8,8 @@ use crate::{key::Key, storable::Storable}; /// Database table metadata. /// /// Purely compile time information for database tables. +/// +/// See [`crate::define_tables`] for bulk table generation. pub trait Table: 'static { /// Name of the database table. const NAME: &'static str; diff --git a/storage/database/src/tables.rs b/storage/database/src/tables.rs index c2ac821..83a00e1 100644 --- a/storage/database/src/tables.rs +++ b/storage/database/src/tables.rs @@ -349,11 +349,18 @@ macro_rules! define_tables { /// Note that this is already implemented on [`cuprate_database::EnvInner`], thus: /// - You don't need to implement this /// - It can be called using `env_inner.open_tables()` notation + /// + /// # Creation before opening + /// As [`cuprate_database::EnvInner`] documentation states, + /// tables must be created before they are opened. + /// + /// I.e. [`OpenTables::create_tables`] must be called before + /// [`OpenTables::open_tables`] or else panics may occur. pub trait OpenTables<'env> { /// The read-only transaction type of the backend. - type Ro<'a>; + type Ro<'tx>; /// The read-write transaction type of the backend. - type Rw<'a>; + type Rw<'tx>; /// Open all tables in read/iter mode. /// @@ -362,11 +369,6 @@ macro_rules! define_tables { /// /// # Errors /// This will only return [`cuprate_database::RuntimeError::Io`] if it errors. - /// - /// # Invariant - /// All tables should be created with a crate-specific open function. - /// - /// TODO: explain why fn open_tables(&self, tx_ro: &Self::Ro<'_>) -> Result; /// Open all tables in read-write mode. @@ -391,8 +393,8 @@ macro_rules! define_tables { where Ei: $crate::EnvInner<'env>, { - type Ro<'a> = >::Ro<'a>; - type Rw<'a> = >::Rw<'a>; + type Ro<'tx> = >::Ro<'tx>; + type Rw<'tx> = >::Rw<'tx>; fn open_tables(&self, tx_ro: &Self::Ro<'_>) -> Result { Ok(($( diff --git a/storage/database/src/transaction.rs b/storage/database/src/transaction.rs index e4c310a..8f33983 100644 --- a/storage/database/src/transaction.rs +++ b/storage/database/src/transaction.rs @@ -11,7 +11,7 @@ use crate::error::RuntimeError; /// # Commit /// It's recommended but may not be necessary to call [`TxRo::commit`] in certain cases: /// - -pub trait TxRo<'env> { +pub trait TxRo<'tx> { /// Commit the read-only transaction. /// /// # Errors @@ -23,7 +23,7 @@ pub trait TxRo<'env> { /// Read/write database transaction. /// /// Returned from [`EnvInner::tx_rw`](crate::EnvInner::tx_rw). -pub trait TxRw<'env> { +pub trait TxRw<'tx> { /// Commit the read/write transaction. /// /// Note that this doesn't necessarily sync the database caches to disk.