SyntheticBird/cuprate-hinto-janai
1
0
Fork 0
mirror of https://github.com/hinto-janai/cuprate.git synced 2024-11-16 15:58:14 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

database: fix open_db_ro, open_db_rw, create_db behavior

Browse source
This commit is contained in:
hinto.janai 2024-06-13 16:14:57 -04:00
parent bb81d34868
commit 4a04625a8b
No known key found for this signature in database
GPG key ID: D47CE05FA175A499
18 changed files with 223 additions and 301 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
storage/database/Cargo.toml
View file

@ -1,6 +1,6 @@
[package] [package]
name = "database" name = "cuprate-database"
version = "0.0.0" version = "0.0.1"
edition = "2021" edition = "2021"
description = "Cuprate's database abstraction" description = "Cuprate's database abstraction"
license = "MIT" license = "MIT"
@ -9,8 +9,8 @@ repository = "https://github.com/Cuprate/cuprate/tree/main/storage/database"
keywords = ["cuprate", "database"] keywords = ["cuprate", "database"]
[features] [features]
# default = ["heed", "redb"] default = ["heed"]
default = ["redb"] # default = ["redb"]
# default = ["redb-memory"] # default = ["redb-memory"]
heed = ["dep:heed"] heed = ["dep:heed"]
redb = ["dep:redb"] redb = ["dep:redb"]

21
storage/database/README.md
View file

@ -8,7 +8,7 @@ For a high-level overview, see the database section in
[Cuprate's architecture book](https://architecture.cuprate.org). [Cuprate's architecture book](https://architecture.cuprate.org).
# Purpose # Purpose
This crate does 3 things abstracts various database backends with traits. This crate abstracts various database backends with traits.
If you need blockchain specific capabilities, consider using the higher-level If you need blockchain specific capabilities, consider using the higher-level
`cuprate-blockchain` crate which builds upon this one. `cuprate-blockchain` crate which builds upon this one.
@ -64,9 +64,6 @@ generic-backed dynamic runtime selection of the database backend, i.e.
the user can select which database backend they use. --> the user can select which database backend they use. -->
# Feature flags # Feature flags
The `service` module requires the `service` feature to be enabled.
See the module for more documentation.
Different database backends are enabled by the feature flags: Different database backends are enabled by the feature flags:
- `heed` (LMDB) - `heed` (LMDB)
- `redb` - `redb`
@ -77,10 +74,10 @@ The default is `heed`.
<!-- FIXME: tracing should be behind a feature flag --> <!-- FIXME: tracing should be behind a feature flag -->
# Examples # Examples
The below is an example of using `database`. The below is an example of using `cuprate-database`.
```rust ```rust
use database::{ use cuprate_database::{
ConcreteEnv, ConcreteEnv,
config::ConfigBuilder, config::ConfigBuilder,
Env, EnvInner, Env, EnvInner,
@ -97,17 +94,21 @@ let config = ConfigBuilder::new()
// Initialize the database environment. // Initialize the database environment.
let env = ConcreteEnv::open(config)?; let env = ConcreteEnv::open(config)?;
// Open up a transaction + tables for writing. // Define metadata for a table.
struct Table; struct Table;
impl database::Table for Table { impl cuprate_database::Table for Table {
// The name of the table is "table".
const NAME: &'static str = "table"; const NAME: &'static str = "table";
// The key type is a `u8`.
type Key = u8; type Key = u8;
type Value = u8; // The key type is a `u64`.
type Value = u64;
} }
// Open up a transaction + tables for writing.
let env_inner = env.env_inner(); let env_inner = env.env_inner();
let tx_rw = env_inner.tx_rw()?; let tx_rw = env_inner.tx_rw()?;
env_inner.create_db::<Table>(&tx_rw)?; env_inner.create_db::<Table>(&tx_rw)?; // we must create it or the next line will panic.
let mut table = env_inner.open_db_rw::<Table>(&tx_rw)?; let mut table = env_inner.open_db_rw::<Table>(&tx_rw)?;
// Write data to the table. // Write data to the table.

19
storage/database/src/backend/heed/env.rs
View file

@ -25,7 +25,7 @@ use crate::{
//---------------------------------------------------------------------------------------------------- Consts //---------------------------------------------------------------------------------------------------- Consts
/// Panic message when there's a table missing. /// Panic message when there's a table missing.
const PANIC_MSG_MISSING_TABLE: &str = const PANIC_MSG_MISSING_TABLE: &str =
"database::Env should uphold the invariant that all tables are already created"; "cuprate_database::Env should uphold the invariant that all tables are already created";
//---------------------------------------------------------------------------------------------------- ConcreteEnv //---------------------------------------------------------------------------------------------------- ConcreteEnv
/// A strongly typed, concrete database environment, backed by `heed`. /// A strongly typed, concrete database environment, backed by `heed`.
@ -272,7 +272,7 @@ where
Ok(HeedTableRo { Ok(HeedTableRo {
db: self db: self
.open_database(tx_ro, Some(T::NAME))? .open_database(tx_ro, Some(T::NAME))?
.expect(PANIC_MSG_MISSING_TABLE), .ok_or(RuntimeError::TableNotFound)?,
tx_ro, tx_ro,
}) })
} }
@ -282,25 +282,16 @@ where
&self, &self,
tx_rw: &RefCell<heed::RwTxn<'env>>, tx_rw: &RefCell<heed::RwTxn<'env>>,
) -> Result<impl DatabaseRw<T>, RuntimeError> { ) -> Result<impl DatabaseRw<T>, RuntimeError> {
let tx_ro = tx_rw.borrow();
// Open up a read/write database using our table's const metadata. // Open up a read/write database using our table's const metadata.
Ok(HeedTableRw { Ok(HeedTableRw {
db: self db: self.create_database(&mut tx_rw.borrow_mut(), Some(T::NAME))?,
.open_database(&tx_ro, Some(T::NAME))?
.expect(PANIC_MSG_MISSING_TABLE),
tx_rw, tx_rw,
}) })
} }
fn create_db<T: Table>(&self, tx_rw: &RefCell<heed::RwTxn<'env>>) -> Result<(), RuntimeError> { fn create_db<T: Table>(&self, tx_rw: &RefCell<heed::RwTxn<'env>>) -> Result<(), RuntimeError> {
use crate::backend::heed::storable::StorableHeed; // INVARIANT: `heed` creates tables with `open_database` if they don't exist.
self.open_db_rw::<T>(tx_rw)?;
self.create_database::<StorableHeed<<T as Table>::Key>, StorableHeed<<T as Table>::Value>>(
&mut tx_rw.borrow_mut(),
Some(T::NAME),
)?;
Ok(()) Ok(())
} }

4
storage/database/src/backend/heed/error.rs
View file

@ -134,12 +134,12 @@ impl From<heed::Error> for crate::RuntimeError {
// Don't use a key that is `>511` bytes. // Don't use a key that is `>511` bytes.
// <http://www.lmdb.tech/doc/group__mdb.html#gaaf0be004f33828bf2fb09d77eb3cef94> // <http://www.lmdb.tech/doc/group__mdb.html#gaaf0be004f33828bf2fb09d77eb3cef94>
| E2::BadValSize | E2::BadValSize
=> panic!("fix the database code! {mdb_error:#?}"), => panic!("E2: fix the database code! {mdb_error:#?}"),
}, },
// Only if we write incorrect code. // Only if we write incorrect code.
E1::DatabaseClosing | E1::BadOpenOptions { .. } | E1::Encoding(_) | E1::Decoding(_) => { E1::DatabaseClosing | E1::BadOpenOptions { .. } | E1::Encoding(_) | E1::Decoding(_) => {
panic!("fix the database code! {error:#?}") panic!("E1: fix the database code! {error:#?}")
} }
} }
} }

4
storage/database/src/backend/heed/storable.rs
View file

@ -1,4 +1,4 @@
//! `database::Storable` <-> `heed` serde trait compatibility layer. //! `cuprate_database::Storable` <-> `heed` serde trait compatibility layer.
//---------------------------------------------------------------------------------------------------- Use //---------------------------------------------------------------------------------------------------- Use
use std::{borrow::Cow, marker::PhantomData}; use std::{borrow::Cow, marker::PhantomData};
@ -9,7 +9,7 @@ use crate::storable::Storable;
//---------------------------------------------------------------------------------------------------- StorableHeed //---------------------------------------------------------------------------------------------------- StorableHeed
/// The glue struct that implements `heed`'s (de)serialization /// The glue struct that implements `heed`'s (de)serialization
/// traits on any type that implements `database::Storable`. /// traits on any type that implements `cuprate_database::Storable`.
/// ///
/// Never actually gets constructed, just used for trait bound translations. /// Never actually gets constructed, just used for trait bound translations.
pub(super) struct StorableHeed<T>(PhantomData<T>) pub(super) struct StorableHeed<T>(PhantomData<T>)

7
storage/database/src/backend/redb/env.rs
View file

@ -21,7 +21,7 @@ pub struct ConcreteEnv {
/// (and in current use). /// (and in current use).
config: Config, config: Config,
/// A cached, redb version of `database::config::SyncMode`. /// A cached, redb version of `cuprate_database::config::SyncMode`.
/// `redb` needs the sync mode to be set _per_ TX, so we /// `redb` needs the sync mode to be set _per_ TX, so we
/// will continue to use this value every `Env::tx_rw`. /// will continue to use this value every `Env::tx_rw`.
durability: redb::Durability, durability: redb::Durability,
@ -148,7 +148,6 @@ where
let table: redb::TableDefinition<'static, StorableRedb<T::Key>, StorableRedb<T::Value>> = let table: redb::TableDefinition<'static, StorableRedb<T::Key>, StorableRedb<T::Value>> =
redb::TableDefinition::new(T::NAME); redb::TableDefinition::new(T::NAME);
// INVARIANT: Our `?` error conversion will panic if the table does not exist.
Ok(tx_ro.open_table(table)?) Ok(tx_ro.open_table(table)?)
} }
@ -161,13 +160,13 @@ where
let table: redb::TableDefinition<'static, StorableRedb<T::Key>, StorableRedb<T::Value>> = let table: redb::TableDefinition<'static, StorableRedb<T::Key>, StorableRedb<T::Value>> =
redb::TableDefinition::new(T::NAME); redb::TableDefinition::new(T::NAME);
// `redb` creates tables if they don't exist, so this should never panic. // `redb` creates tables if they don't exist, so this shouldn't return `RuntimeError::TableNotFound`.
// <https://docs.rs/redb/latest/redb/struct.WriteTransaction.html#method.open_table> // <https://docs.rs/redb/latest/redb/struct.WriteTransaction.html#method.open_table>
Ok(tx_rw.open_table(table)?) Ok(tx_rw.open_table(table)?)
} }
fn create_db<T: Table>(&self, tx_rw: &redb::WriteTransaction) -> Result<(), RuntimeError> { fn create_db<T: Table>(&self, tx_rw: &redb::WriteTransaction) -> Result<(), RuntimeError> {
// `redb` creates tables if they don't exist. // INVARIANT: `redb` creates tables if they don't exist.
self.open_db_rw::<T>(tx_rw)?; self.open_db_rw::<T>(tx_rw)?;
Ok(()) Ok(())
} }

15
storage/database/src/backend/redb/error.rs
View file

@ -12,13 +12,13 @@ use crate::{
//---------------------------------------------------------------------------------------------------- InitError //---------------------------------------------------------------------------------------------------- InitError
impl From<redb::DatabaseError> for InitError { impl From<redb::DatabaseError> for InitError {
/// Created by `redb` in: /// Created by `redb` in:
/// - [`redb::Database::open`](https://docs.rs/redb/1.5.0/redb/struct.Database.html#method.open). /// - [`redb::cuprate_database::open`](https://docs.rs/redb/1.5.0/redb/struct.Database.html#method.open).
fn from(error: redb::DatabaseError) -> Self { fn from(error: redb::DatabaseError) -> Self {
use redb::DatabaseError as E; use redb::DatabaseError as E;
use redb::StorageError as E2; use redb::StorageError as E2;
// Reference of all possible errors `redb` will return // Reference of all possible errors `redb` will return
// upon using `redb::Database::open`: // upon using `redb::cuprate_database::open`:
// <https://docs.rs/redb/1.5.0/src/redb/db.rs.html#908-923> // <https://docs.rs/redb/1.5.0/src/redb/db.rs.html#908-923>
match error { match error {
E::RepairAborted => Self::Corrupt, E::RepairAborted => Self::Corrupt,
@ -39,7 +39,7 @@ impl From<redb::DatabaseError> for InitError {
impl From<redb::StorageError> for InitError { impl From<redb::StorageError> for InitError {
/// Created by `redb` in: /// Created by `redb` in:
/// - [`redb::Database::open`](https://docs.rs/redb/1.5.0/redb/struct.Database.html#method.check_integrity) /// - [`redb::cuprate_database::open`](https://docs.rs/redb/1.5.0/redb/struct.Database.html#method.check_integrity)
fn from(error: redb::StorageError) -> Self { fn from(error: redb::StorageError) -> Self {
use redb::StorageError as E; use redb::StorageError as E;
@ -54,7 +54,7 @@ impl From<redb::StorageError> for InitError {
impl From<redb::TransactionError> for InitError { impl From<redb::TransactionError> for InitError {
/// Created by `redb` in: /// Created by `redb` in:
/// - [`redb::Database::begin_write`](https://docs.rs/redb/1.5.0/redb/struct.Database.html#method.begin_write) /// - [`redb::cuprate_database::begin_write`](https://docs.rs/redb/1.5.0/redb/struct.Database.html#method.begin_write)
fn from(error: redb::TransactionError) -> Self { fn from(error: redb::TransactionError) -> Self {
match error { match error {
redb::TransactionError::Storage(error) => error.into(), redb::TransactionError::Storage(error) => error.into(),
@ -94,8 +94,8 @@ impl From<redb::CommitError> for InitError {
#[allow(clippy::fallible_impl_from)] // We need to panic sometimes. #[allow(clippy::fallible_impl_from)] // We need to panic sometimes.
impl From<redb::TransactionError> for RuntimeError { impl From<redb::TransactionError> for RuntimeError {
/// Created by `redb` in: /// Created by `redb` in:
/// - [`redb::Database::begin_write`](https://docs.rs/redb/1.5.0/redb/struct.Database.html#method.begin_write) /// - [`redb::cuprate_database::begin_write`](https://docs.rs/redb/1.5.0/redb/struct.Database.html#method.begin_write)
/// - [`redb::Database::begin_read`](https://docs.rs/redb/1.5.0/redb/struct.Database.html#method.begin_read) /// - [`redb::cuprate_database::begin_read`](https://docs.rs/redb/1.5.0/redb/struct.Database.html#method.begin_read)
fn from(error: redb::TransactionError) -> Self { fn from(error: redb::TransactionError) -> Self {
match error { match error {
redb::TransactionError::Storage(error) => error.into(), redb::TransactionError::Storage(error) => error.into(),
@ -131,12 +131,13 @@ impl From<redb::TableError> for RuntimeError {
match error { match error {
E::Storage(error) => error.into(), E::Storage(error) => error.into(),
E::TableDoesNotExist(_) => Self::TableNotFound,
// Only if we write incorrect code. // Only if we write incorrect code.
E::TableTypeMismatch { .. } E::TableTypeMismatch { .. }
| E::TableIsMultimap(_) | E::TableIsMultimap(_)
| E::TableIsNotMultimap(_) | E::TableIsNotMultimap(_)
| E::TypeDefinitionChanged { .. } | E::TypeDefinitionChanged { .. }
| E::TableDoesNotExist(_)
| E::TableAlreadyOpen(..) => panic!("fix the database code! {error:#?}"), | E::TableAlreadyOpen(..) => panic!("fix the database code! {error:#?}"),
// HACK: Handle new errors as `redb` adds them. // HACK: Handle new errors as `redb` adds them.

4
storage/database/src/backend/redb/storable.rs
View file

@ -1,4 +1,4 @@
//! `database::Storable` <-> `redb` serde trait compatibility layer. //! `cuprate_database::Storable` <-> `redb` serde trait compatibility layer.
//---------------------------------------------------------------------------------------------------- Use //---------------------------------------------------------------------------------------------------- Use
use std::{cmp::Ordering, fmt::Debug, marker::PhantomData}; use std::{cmp::Ordering, fmt::Debug, marker::PhantomData};
@ -9,7 +9,7 @@ use crate::{key::Key, storable::Storable};
//---------------------------------------------------------------------------------------------------- StorableRedb //---------------------------------------------------------------------------------------------------- StorableRedb
/// The glue structs that implements `redb`'s (de)serialization /// The glue structs that implements `redb`'s (de)serialization
/// traits on any type that implements `database::Key`. /// traits on any type that implements `cuprate_database::Key`.
/// ///
/// Never actually get constructed, just used for trait bound translations. /// Never actually get constructed, just used for trait bound translations.
#[derive(Debug)] #[derive(Debug)]

241
storage/database/src/backend/tests.rs
View file

@ -1,149 +1,126 @@
// //! Tests for `database`'s backends. //! Tests for `database`'s backends.
// //! //!
// //! These tests are fully trait-based, meaning there //! These tests are fully trait-based, meaning there
// //! is no reference to `backend/`-specific types. //! is no reference to `backend/`-specific types.
// //! //!
// //! As such, which backend is tested is //! As such, which backend is tested is
// //! dependant on the feature flags used. //! dependant on the feature flags used.
// //! //!
// //! | Feature flag | Tested backend | //! | Feature flag | Tested backend |
// //! |---------------|----------------| //! |---------------|----------------|
// //! | Only `redb` | `redb` //! | Only `redb` | `redb`
// //! | Anything else | `heed` //! | Anything else | `heed`
// //! //!
// //! `redb`, and it only must be enabled for it to be tested. //! `redb`, and it only must be enabled for it to be tested.
// //---------------------------------------------------------------------------------------------------- Import //---------------------------------------------------------------------------------------------------- Import
// use crate::{ use crate::{
// database::{DatabaseIter, DatabaseRo, DatabaseRw}, database::{DatabaseIter, DatabaseRo, DatabaseRw},
// env::{Env, EnvInner}, env::{Env, EnvInner},
// error::RuntimeError, error::RuntimeError,
// resize::ResizeAlgorithm, resize::ResizeAlgorithm,
// storable::StorableVec, storable::StorableVec,
// tables::{ tests::{tmp_concrete_env, TestTable},
// BlockBlobs, BlockHeights, BlockInfos, KeyImages, NumOutputs, Outputs, PrunableHashes, transaction::{TxRo, TxRw},
// PrunableTxBlobs, PrunedTxBlobs, RctOutputs, TxBlobs, TxHeights, TxIds, TxOutputs, ConcreteEnv,
// TxUnlockTime, };
// },
// tables::{TablesIter, TablesMut},
// tests::tmp_concrete_env,
// transaction::{TxRo, TxRw},
// types::{
// Amount, AmountIndex, AmountIndices, BlockBlob, BlockHash, BlockHeight, BlockInfo, KeyImage,
// Output, OutputFlags, PreRctOutputId, PrunableBlob, PrunableHash, PrunedBlob, RctOutput,
// TxBlob, TxHash, TxId, UnlockTime,
// },
// ConcreteEnv,
// };
// //---------------------------------------------------------------------------------------------------- Tests //---------------------------------------------------------------------------------------------------- Tests
// /// Simply call [`Env::open`]. If this fails, something is really wrong. /// Simply call [`Env::open`]. If this fails, something is really wrong.
// #[test] #[test]
// fn open() { fn open() {
// tmp_concrete_env(); tmp_concrete_env();
// } }
// /// Create database transactions, but don't write any data. /// Create database transactions, but don't write any data.
// #[test] #[test]
// fn tx() { fn tx() {
// let (env, _tempdir) = tmp_concrete_env(); let (env, _tempdir) = tmp_concrete_env();
// let env_inner = env.env_inner(); let env_inner = env.env_inner();
// TxRo::commit(env_inner.tx_ro().unwrap()).unwrap(); TxRo::commit(env_inner.tx_ro().unwrap()).unwrap();
// TxRw::commit(env_inner.tx_rw().unwrap()).unwrap(); TxRw::commit(env_inner.tx_rw().unwrap()).unwrap();
// TxRw::abort(env_inner.tx_rw().unwrap()).unwrap(); TxRw::abort(env_inner.tx_rw().unwrap()).unwrap();
// } }
// /// Open (and verify) that all database tables /// Test [`Env::open`] and creating/opening tables.
// /// exist already after calling [`Env::open`]. #[test]
// #[test] fn open_db() {
// fn open_db() { let (env, _tempdir) = tmp_concrete_env();
// let (env, _tempdir) = tmp_concrete_env(); let env_inner = env.env_inner();
// let env_inner = env.env_inner();
// let tx_ro = env_inner.tx_ro().unwrap();
// let tx_rw = env_inner.tx_rw().unwrap();
// // Open all tables in read-only mode. // Create table.
// // This should be updated when tables are modified. {
// env_inner.open_db_ro::<BlockBlobs>(&tx_ro).unwrap(); let tx_rw = env_inner.tx_rw().unwrap();
// env_inner.open_db_ro::<BlockHeights>(&tx_ro).unwrap(); env_inner.create_db::<TestTable>(&tx_rw).unwrap();
// env_inner.open_db_ro::<BlockInfos>(&tx_ro).unwrap(); TxRw::commit(tx_rw).unwrap();
// env_inner.open_db_ro::<KeyImages>(&tx_ro).unwrap(); }
// env_inner.open_db_ro::<NumOutputs>(&tx_ro).unwrap();
// env_inner.open_db_ro::<Outputs>(&tx_ro).unwrap();
// env_inner.open_db_ro::<PrunableHashes>(&tx_ro).unwrap();
// env_inner.open_db_ro::<PrunableTxBlobs>(&tx_ro).unwrap();
// env_inner.open_db_ro::<PrunedTxBlobs>(&tx_ro).unwrap();
// env_inner.open_db_ro::<RctOutputs>(&tx_ro).unwrap();
// env_inner.open_db_ro::<TxBlobs>(&tx_ro).unwrap();
// env_inner.open_db_ro::<TxHeights>(&tx_ro).unwrap();
// env_inner.open_db_ro::<TxIds>(&tx_ro).unwrap();
// env_inner.open_db_ro::<TxOutputs>(&tx_ro).unwrap();
// env_inner.open_db_ro::<TxUnlockTime>(&tx_ro).unwrap();
// TxRo::commit(tx_ro).unwrap();
// // Open all tables in read/write mode. let tx_ro = env_inner.tx_ro().unwrap();
// env_inner.open_db_rw::<BlockBlobs>(&tx_rw).unwrap(); let tx_rw = env_inner.tx_rw().unwrap();
// env_inner.open_db_rw::<BlockHeights>(&tx_rw).unwrap();
// env_inner.open_db_rw::<BlockInfos>(&tx_rw).unwrap();
// env_inner.open_db_rw::<KeyImages>(&tx_rw).unwrap();
// env_inner.open_db_rw::<NumOutputs>(&tx_rw).unwrap();
// env_inner.open_db_rw::<Outputs>(&tx_rw).unwrap();
// env_inner.open_db_rw::<PrunableHashes>(&tx_rw).unwrap();
// env_inner.open_db_rw::<PrunableTxBlobs>(&tx_rw).unwrap();
// env_inner.open_db_rw::<PrunedTxBlobs>(&tx_rw).unwrap();
// env_inner.open_db_rw::<RctOutputs>(&tx_rw).unwrap();
// env_inner.open_db_rw::<TxBlobs>(&tx_rw).unwrap();
// env_inner.open_db_rw::<TxHeights>(&tx_rw).unwrap();
// env_inner.open_db_rw::<TxIds>(&tx_rw).unwrap();
// env_inner.open_db_rw::<TxOutputs>(&tx_rw).unwrap();
// env_inner.open_db_rw::<TxUnlockTime>(&tx_rw).unwrap();
// TxRw::commit(tx_rw).unwrap();
// }
// /// Test `Env` resizes. // Open table in read-only mode.
// #[test] env_inner.open_db_ro::<TestTable>(&tx_ro).unwrap();
// fn resize() { TxRo::commit(tx_ro).unwrap();
// // This test is only valid for `Env`'s that need to resize manually.
// if !ConcreteEnv::MANUAL_RESIZE {
// return;
// }
// let (env, _tempdir) = tmp_concrete_env(); // Open table in read/write mode.
env_inner.open_db_rw::<TestTable>(&tx_rw).unwrap();
TxRw::commit(tx_rw).unwrap();
}
// // Resize by the OS page size. /// Assert that opening a read-only table before creating errors.
// let page_size = crate::resize::page_size(); #[test]
// let old_size = env.current_map_size(); fn open_uncreated_table() {
// env.resize_map(Some(ResizeAlgorithm::FixedBytes(page_size))); let (env, _tempdir) = tmp_concrete_env();
let env_inner = env.env_inner();
let tx_ro = env_inner.tx_ro().unwrap();
// // Assert it resized exactly by the OS page size. // Open uncreated table.
// let new_size = env.current_map_size(); let error = env_inner.open_db_ro::<TestTable>(&tx_ro);
// assert_eq!(new_size, old_size + page_size.get()); assert!(matches!(error, Err(RuntimeError::TableNotFound)));
// } }
// /// Test that `Env`'s that don't manually resize. /// Test `Env` resizes.
// #[test] #[test]
// #[should_panic = "unreachable"] fn resize() {
// fn non_manual_resize_1() { // This test is only valid for `Env`'s that need to resize manually.
// if ConcreteEnv::MANUAL_RESIZE { if !ConcreteEnv::MANUAL_RESIZE {
// unreachable!(); return;
// } else { }
// let (env, _tempdir) = tmp_concrete_env();
// env.resize_map(None);
// }
// }
// #[test] let (env, _tempdir) = tmp_concrete_env();
// #[should_panic = "unreachable"]
// fn non_manual_resize_2() { // Resize by the OS page size.
// if ConcreteEnv::MANUAL_RESIZE { let page_size = crate::resize::page_size();
// unreachable!(); let old_size = env.current_map_size();
// } else { env.resize_map(Some(ResizeAlgorithm::FixedBytes(page_size)));
// let (env, _tempdir) = tmp_concrete_env();
// env.current_map_size(); // Assert it resized exactly by the OS page size.
// } let new_size = env.current_map_size();
// } assert_eq!(new_size, old_size + page_size.get());
}
/// Test that `Env`'s that don't manually resize.
#[test]
#[should_panic = "unreachable"]
fn non_manual_resize_1() {
if ConcreteEnv::MANUAL_RESIZE {
unreachable!();
}
let (env, _tempdir) = tmp_concrete_env();
env.resize_map(None);
}
#[test]
#[should_panic = "unreachable"]
fn non_manual_resize_2() {
if ConcreteEnv::MANUAL_RESIZE {
unreachable!();
}
let (env, _tempdir) = tmp_concrete_env();
env.current_map_size();
}
// /// Test all `DatabaseR{o,w}` operations. // /// Test all `DatabaseR{o,w}` operations.
// #[test] // #[test]

4
storage/database/src/config/config.rs
View file

@ -199,7 +199,7 @@ impl Config {
/// Same as [`Config::default`]. /// Same as [`Config::default`].
/// ///
/// ```rust /// ```rust
/// use database::{config::*, resize::*, DATABASE_DATA_FILENAME}; /// use cuprate_database::{config::*, resize::*, DATABASE_DATA_FILENAME};
/// use cuprate_helper::fs::*; /// use cuprate_helper::fs::*;
/// ///
/// let config = Config::new(); /// let config = Config::new();
@ -230,7 +230,7 @@ impl Default for Config {
/// Same as [`Config::new`]. /// Same as [`Config::new`].
/// ///
/// ```rust /// ```rust
/// # use database::config::*; /// # use cuprate_database::config::*;
/// assert_eq!(Config::default(), Config::new()); /// assert_eq!(Config::default(), Config::new());
/// ``` /// ```
fn default() -> Self { fn default() -> Self {

2
storage/database/src/config/mod.rs
View file

@ -12,7 +12,7 @@
//! //!
//! # Example //! # Example
//! ```rust //! ```rust
//! use database::{ //! use cuprate_database::{
//! ConcreteEnv, Env, //! ConcreteEnv, Env,
//! config::{ConfigBuilder, ReaderThreads, SyncMode} //! config::{ConfigBuilder, ReaderThreads, SyncMode}
//! }; //! };

6
storage/database/src/config/reader_threads.rs
View file

@ -50,7 +50,7 @@ pub enum ReaderThreads {
/// as such, it is equal to [`ReaderThreads::OnePerThread`]. /// as such, it is equal to [`ReaderThreads::OnePerThread`].
/// ///
/// ```rust /// ```rust
/// # use database::config::*; /// # use cuprate_database::config::*;
/// let reader_threads = ReaderThreads::from(0_usize); /// let reader_threads = ReaderThreads::from(0_usize);
/// assert!(matches!(reader_threads, ReaderThreads::OnePerThread)); /// assert!(matches!(reader_threads, ReaderThreads::OnePerThread));
/// ``` /// ```
@ -82,7 +82,7 @@ pub enum ReaderThreads {
/// non-zero, but not 1 thread, the minimum value 1 will be returned. /// non-zero, but not 1 thread, the minimum value 1 will be returned.
/// ///
/// ```rust /// ```rust
/// # use database::config::*; /// # use cuprate_database::config::*;
/// assert_eq!(ReaderThreads::Percent(0.000000001).as_threads().get(), 1); /// assert_eq!(ReaderThreads::Percent(0.000000001).as_threads().get(), 1);
/// ``` /// ```
Percent(f32), Percent(f32),
@ -98,7 +98,7 @@ impl ReaderThreads {
/// ///
/// # Example /// # Example
/// ```rust /// ```rust
/// use database::config::ReaderThreads as Rt; /// use cuprate_database::config::ReaderThreads as Rt;
/// ///
/// let total_threads: std::num::NonZeroUsize = /// let total_threads: std::num::NonZeroUsize =
/// cuprate_helper::thread::threads(); /// cuprate_helper::thread::threads();

50
storage/database/src/env.rs
View file

@ -80,14 +80,13 @@ pub trait Env: Sized {
/// Open the database environment, using the passed [`Config`]. /// Open the database environment, using the passed [`Config`].
/// ///
/// # Invariants /// # Invariants
/// TODO: fix me /// This function does not create any tables.
// This function **must** create all tables listed in [`crate::tables`].
/// ///
/// The rest of the functions depend on the fact /// You must create all possible tables with [`EnvInner::create_db`]
/// they already exist, or else they will panic. /// before attempting to open any.
/// ///
/// # Errors /// # Errors
/// This will error if the database could not be opened. /// This will error if the database file could not be opened.
/// ///
/// This is the only [`Env`] function that will return /// This is the only [`Env`] function that will return
/// an [`InitError`] instead of a [`RuntimeError`]. /// an [`InitError`] instead of a [`RuntimeError`].
@ -180,10 +179,14 @@ pub trait Env: Sized {
macro_rules! doc_table_error { macro_rules! doc_table_error {
() => { () => {
r"# Errors r"# Errors
This will only return [`RuntimeError::Io`] if it errors. This will only return [`RuntimeError::Io`] on normal errors.
As all tables are created upon [`Env::open`], If the specified table is not created upon before this function is called,
this function will never error because a table doesn't exist." this will return an error.
Implementation detail you should NOT rely on:
- This only panics on `heed`
- `redb` will create the table if it does not exist"
}; };
} }
@ -229,7 +232,11 @@ where
/// // (name, key/value type) /// // (name, key/value type)
/// ``` /// ```
/// ///
#[doc = doc_table_error!()] /// # Errors
/// This will only return [`RuntimeError::Io`] on normal errors.
///
/// If the specified table is not created upon before this function is called,
/// this will return [`RuntimeError::TableNotFound`].
fn open_db_ro<T: Table>( fn open_db_ro<T: Table>(
&self, &self,
tx_ro: &Ro, tx_ro: &Ro,
@ -246,30 +253,23 @@ where
/// This will open the database [`Table`] /// This will open the database [`Table`]
/// passed as a generic to this function. /// passed as a generic to this function.
/// ///
#[doc = doc_table_error!()] /// # Errors
/// This will only return [`RuntimeError::Io`] on errors.
///
/// Implementation details: Both `heed` & `redb` backends create
/// the table with this function if it does not already exist. For safety and
/// clear intent, you should still consider using [`EnvInner::create_db`] instead.
fn open_db_rw<T: Table>(&self, tx_rw: &Rw) -> Result<impl DatabaseRw<T>, RuntimeError>; fn open_db_rw<T: Table>(&self, tx_rw: &Rw) -> Result<impl DatabaseRw<T>, RuntimeError>;
// TODO: make equivalent in `cuprate-blockchain`.
/// Create a database table. /// Create a database table.
/// ///
/// This will create the database [`Table`] /// This will create the database [`Table`]
/// passed as a generic to this function. /// passed as a generic to this function.
#[doc = doc_table_error!()] ///
/// # Errors
/// This will only return [`RuntimeError::Io`] on errors.
fn create_db<T: Table>(&self, tx_rw: &Rw) -> Result<(), RuntimeError>; fn create_db<T: Table>(&self, tx_rw: &Rw) -> Result<(), RuntimeError>;
// /// Open all tables in read-write mode.
// ///
// /// This calls [`EnvInner::open_db_rw`] on all database tables
// /// and returns a structure that allows access to all tables.
// ///
// #[doc = doc_table_error!()]
// fn open_tables_mut(&self, tx_rw: &Rw) -> Result<impl TablesMut, RuntimeError> {
// call_fn_on_all_tables_or_early_return! {
// Self::open_db_rw(self, tx_rw)
// }
// }
/// Clear all `(key, value)`'s from a database table. /// Clear all `(key, value)`'s from a database table.
/// ///
/// This will delete all key and values in the passed /// This will delete all key and values in the passed

4
storage/database/src/error.rs
View file

@ -88,6 +88,10 @@ pub enum RuntimeError {
#[error("database memory map must be resized")] #[error("database memory map must be resized")]
ResizeNeeded, ResizeNeeded,
/// The given table did not exist in the database.
#[error("database table did not exist")]
TableNotFound,
/// A [`std::io::Error`]. /// A [`std::io::Error`].
#[error("I/O error: {0}")] #[error("I/O error: {0}")]
Io(#[from] std::io::Error), Io(#[from] std::io::Error),

2
storage/database/src/key.rs
View file

@ -23,7 +23,7 @@ pub trait Key: Storable + Sized {
/// not a comparison of the key's value. /// not a comparison of the key's value.
/// ///
/// ```rust /// ```rust
/// # use database::*; /// # use cuprate_database::*;
/// assert_eq!( /// assert_eq!(
/// <u64 as Key>::compare([0].as_slice(), [1].as_slice()), /// <u64 as Key>::compare([0].as_slice(), [1].as_slice()),
/// std::cmp::Ordering::Less, /// std::cmp::Ordering::Less,

16
storage/database/src/resize.rs
View file

@ -50,7 +50,7 @@ impl ResizeAlgorithm {
/// Returns [`Self::Monero`]. /// Returns [`Self::Monero`].
/// ///
/// ```rust /// ```rust
/// # use database::resize::*; /// # use cuprate_database::resize::*;
/// assert!(matches!(ResizeAlgorithm::new(), ResizeAlgorithm::Monero)); /// assert!(matches!(ResizeAlgorithm::new(), ResizeAlgorithm::Monero));
/// ``` /// ```
#[inline] #[inline]
@ -75,7 +75,7 @@ impl Default for ResizeAlgorithm {
/// Calls [`Self::new`]. /// Calls [`Self::new`].
/// ///
/// ```rust /// ```rust
/// # use database::resize::*; /// # use cuprate_database::resize::*;
/// assert_eq!(ResizeAlgorithm::new(), ResizeAlgorithm::default()); /// assert_eq!(ResizeAlgorithm::new(), ResizeAlgorithm::default());
/// ``` /// ```
#[inline] #[inline]
@ -113,7 +113,7 @@ pub fn page_size() -> NonZeroUsize {
/// [^2]: `1_073_745_920` /// [^2]: `1_073_745_920`
/// ///
/// ```rust /// ```rust
/// # use database::resize::*; /// # use cuprate_database::resize::*;
/// // The value this function will increment by /// // The value this function will increment by
/// // (assuming page multiple of 4096). /// // (assuming page multiple of 4096).
/// const N: usize = 1_073_741_824; /// const N: usize = 1_073_741_824;
@ -129,7 +129,7 @@ pub fn page_size() -> NonZeroUsize {
/// This function will panic if adding onto `current_size_bytes` overflows [`usize::MAX`]. /// This function will panic if adding onto `current_size_bytes` overflows [`usize::MAX`].
/// ///
/// ```rust,should_panic /// ```rust,should_panic
/// # use database::resize::*; /// # use cuprate_database::resize::*;
/// // Ridiculous large numbers panic. /// // Ridiculous large numbers panic.
/// monero(usize::MAX); /// monero(usize::MAX);
/// ``` /// ```
@ -166,7 +166,7 @@ pub fn monero(current_size_bytes: usize) -> NonZeroUsize {
/// and then round up to nearest OS page size. /// and then round up to nearest OS page size.
/// ///
/// ```rust /// ```rust
/// # use database::resize::*; /// # use cuprate_database::resize::*;
/// let page_size: usize = page_size().get(); /// let page_size: usize = page_size().get();
/// ///
/// // Anything below the page size will round up to the page size. /// // Anything below the page size will round up to the page size.
@ -185,7 +185,7 @@ pub fn monero(current_size_bytes: usize) -> NonZeroUsize {
/// This function will panic if adding onto `current_size_bytes` overflows [`usize::MAX`]. /// This function will panic if adding onto `current_size_bytes` overflows [`usize::MAX`].
/// ///
/// ```rust,should_panic /// ```rust,should_panic
/// # use database::resize::*; /// # use cuprate_database::resize::*;
/// // Ridiculous large numbers panic. /// // Ridiculous large numbers panic.
/// fixed_bytes(1, usize::MAX); /// fixed_bytes(1, usize::MAX);
/// ``` /// ```
@ -221,7 +221,7 @@ pub fn fixed_bytes(current_size_bytes: usize, add_bytes: usize) -> NonZeroUsize
/// (rounded up to the OS page size). /// (rounded up to the OS page size).
/// ///
/// ```rust /// ```rust
/// # use database::resize::*; /// # use cuprate_database::resize::*;
/// let page_size: usize = page_size().get(); /// let page_size: usize = page_size().get();
/// ///
/// // Anything below the page size will round up to the page size. /// // Anything below the page size will round up to the page size.
@ -247,7 +247,7 @@ pub fn fixed_bytes(current_size_bytes: usize, add_bytes: usize) -> NonZeroUsize
/// is closer to [`usize::MAX`] than the OS page size. /// is closer to [`usize::MAX`] than the OS page size.
/// ///
/// ```rust,should_panic /// ```rust,should_panic
/// # use database::resize::*; /// # use cuprate_database::resize::*;
/// // Ridiculous large numbers panic. /// // Ridiculous large numbers panic.
/// percent(usize::MAX, 1.001); /// percent(usize::MAX, 1.001);
/// ``` /// ```

8
storage/database/src/storable.rs
View file

@ -30,7 +30,7 @@ use bytes::Bytes;
/// See [`StorableVec`] & [`StorableBytes`] for storing slices of `T: Storable`. /// See [`StorableVec`] & [`StorableBytes`] for storing slices of `T: Storable`.
/// ///
/// ```rust /// ```rust
/// # use database::*; /// # use cuprate_database::*;
/// # use std::borrow::*; /// # use std::borrow::*;
/// let number: u64 = 0; /// let number: u64 = 0;
/// ///
@ -78,7 +78,7 @@ pub trait Storable: Debug {
/// ///
/// # Examples /// # Examples
/// ```rust /// ```rust
/// # use database::*; /// # use cuprate_database::*;
/// assert_eq!(<()>::BYTE_LENGTH, Some(0)); /// assert_eq!(<()>::BYTE_LENGTH, Some(0));
/// assert_eq!(u8::BYTE_LENGTH, Some(1)); /// assert_eq!(u8::BYTE_LENGTH, Some(1));
/// assert_eq!(u16::BYTE_LENGTH, Some(2)); /// assert_eq!(u16::BYTE_LENGTH, Some(2));
@ -137,7 +137,7 @@ where
/// ///
/// # Example /// # Example
/// ```rust /// ```rust
/// # use database::*; /// # use cuprate_database::*;
/// //---------------------------------------------------- u8 /// //---------------------------------------------------- u8
/// let vec: StorableVec<u8> = StorableVec(vec![0,1]); /// let vec: StorableVec<u8> = StorableVec(vec![0,1]);
/// ///
@ -203,7 +203,7 @@ impl<T> Borrow<[T]> for StorableVec<T> {
/// A [`Storable`] version of [`Bytes`]. /// A [`Storable`] version of [`Bytes`].
/// ///
/// ```rust /// ```rust
/// # use database::*; /// # use cuprate_database::*;
/// # use bytes::Bytes; /// # use bytes::Bytes;
/// let bytes: StorableBytes = StorableBytes(Bytes::from_static(&[0,1])); /// let bytes: StorableBytes = StorableBytes(Bytes::from_static(&[0,1]));
/// ///

109
storage/database/src/tests.rs
View file

@ -1,85 +1,34 @@
// //! Utilities for `database` testing. //! Utilities for `database` testing.
// //! //!
// //! These types/fn's are only: //! These types/fn's are only:
// //! - enabled on #[cfg(test)] //! - enabled on #[cfg(test)]
// //! - only used internally //! - only used internally
// //---------------------------------------------------------------------------------------------------- Import //---------------------------------------------------------------------------------------------------- Import
// use std::fmt::Debug; use crate::{config::ConfigBuilder, table::Table, ConcreteEnv, Env};
// use pretty_assertions::assert_eq; //---------------------------------------------------------------------------------------------------- struct
/// A test table.
pub(crate) struct TestTable;
// use crate::{config::ConfigBuilder, tables::Tables, ConcreteEnv, DatabaseRo, Env, EnvInner}; impl Table for TestTable {
const NAME: &'static str = "test_table";
type Key = u8;
type Value = u8;
}
// //---------------------------------------------------------------------------------------------------- Struct //---------------------------------------------------------------------------------------------------- fn
// /// Named struct to assert the length of all tables. /// Create an `Env` in a temporarily directory.
// /// /// The directory is automatically removed after the `TempDir` is dropped.
// /// This is a struct with fields instead of a function ///
// /// so that callers can name arguments, otherwise the call-site /// FIXME: changing this to `-> impl Env` causes lifetime errors...
// /// is a little confusing, i.e. `assert_table_len(0, 25, 1, 123)`. pub(crate) fn tmp_concrete_env() -> (ConcreteEnv, tempfile::TempDir) {
// #[derive(Copy, Clone, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Hash)] let tempdir = tempfile::tempdir().unwrap();
// pub(crate) struct AssertTableLen { let config = ConfigBuilder::new()
// pub(crate) block_infos: u64, .db_directory(tempdir.path().into())
// pub(crate) block_blobs: u64, .low_power()
// pub(crate) block_heights: u64, .build();
// pub(crate) key_images: u64, let env = ConcreteEnv::open(config).unwrap();
// pub(crate) num_outputs: u64,
// pub(crate) pruned_tx_blobs: u64,
// pub(crate) prunable_hashes: u64,
// pub(crate) outputs: u64,
// pub(crate) prunable_tx_blobs: u64,
// pub(crate) rct_outputs: u64,
// pub(crate) tx_blobs: u64,
// pub(crate) tx_ids: u64,
// pub(crate) tx_heights: u64,
// pub(crate) tx_unlock_time: u64,
// }
// impl AssertTableLen { (env, tempdir)
// /// Assert the length of all tables. }
// pub(crate) fn assert(self, tables: &impl Tables) {
// let other = Self {
// block_infos: tables.block_infos().len().unwrap(),
// block_blobs: tables.block_blobs().len().unwrap(),
// block_heights: tables.block_heights().len().unwrap(),
// key_images: tables.key_images().len().unwrap(),
// num_outputs: tables.num_outputs().len().unwrap(),
// pruned_tx_blobs: tables.pruned_tx_blobs().len().unwrap(),
// prunable_hashes: tables.prunable_hashes().len().unwrap(),
// outputs: tables.outputs().len().unwrap(),
// prunable_tx_blobs: tables.prunable_tx_blobs().len().unwrap(),
// rct_outputs: tables.rct_outputs().len().unwrap(),
// tx_blobs: tables.tx_blobs().len().unwrap(),
// tx_ids: tables.tx_ids().len().unwrap(),
// tx_heights: tables.tx_heights().len().unwrap(),
// tx_unlock_time: tables.tx_unlock_time().len().unwrap(),
// };
// assert_eq!(self, other);
// }
// }
// //---------------------------------------------------------------------------------------------------- fn
// /// Create an `Env` in a temporarily directory.
// /// The directory is automatically removed after the `TempDir` is dropped.
// ///
// /// FIXME: changing this to `-> impl Env` causes lifetime errors...
// pub(crate) fn tmp_concrete_env() -> (ConcreteEnv, tempfile::TempDir) {
// let tempdir = tempfile::tempdir().unwrap();
// let config = ConfigBuilder::new()
// .db_directory(tempdir.path().into())
// .low_power()
// .build();
// let env = ConcreteEnv::open(config).unwrap();
// (env, tempdir)
// }
// /// Assert all the tables in the environment are empty.
// pub(crate) fn assert_all_tables_are_empty(env: &ConcreteEnv) {
// let env_inner = env.env_inner();
// let tx_ro = env_inner.tx_ro().unwrap();
// let tables = env_inner.open_tables(&tx_ro).unwrap();
// assert!(tables.all_tables_empty().unwrap());
// assert_eq!(crate::ops::tx::get_num_tx(tables.tx_ids()).unwrap(), 0);
// }