SyntheticBird/cuprate-hinto-janai
1
0
Fork 0
mirror of https://github.com/hinto-janai/cuprate.git synced 2024-11-16 15:58:14 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

initial cuprate-blockchain split

Browse source
This commit is contained in:
hinto.janai 2024-06-13 20:26:01 -04:00
parent 98413dfacd
commit 6e48c76d80
No known key found for this signature in database
GPG key ID: D47CE05FA175A499
47 changed files with 221 additions and 4066 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
Cargo.lock generated
View file

@ -486,12 +486,12 @@ dependencies = [
"bytes", "bytes",
"cfg-if", "cfg-if",
"crossbeam", "crossbeam",
"cuprate-database",
"cuprate-helper", "cuprate-helper",
"cuprate-test-utils", "cuprate-test-utils",
"cuprate-types", "cuprate-types",
"curve25519-dalek", "curve25519-dalek",
"futures", "futures",
"heed",
"hex", "hex",
"hex-literal", "hex-literal",
"monero-pruning", "monero-pruning",
@ -500,8 +500,6 @@ dependencies = [
"paste", "paste",
"pretty_assertions", "pretty_assertions",
"rayon", "rayon",
"redb",
"serde",
"tempfile", "tempfile",
"thiserror", "thiserror",
"thread_local", "thread_local",

12
storage/cuprate-blockchain/Cargo.toml
View file

@ -12,9 +12,9 @@ keywords = ["cuprate", "blockchain", "database"]
default = ["heed", "redb", "service"] default = ["heed", "redb", "service"]
# default = ["redb", "service"] # default = ["redb", "service"]
# default = ["redb-memory", "service"] # default = ["redb-memory", "service"]
heed = ["dep:heed"] heed = ["cuprate-database/heed"]
redb = ["dep:redb"] redb = ["cuprate-database/redb"]
redb-memory = ["redb"] redb-memory = ["cuprate-database/redb-memory"]
service = ["dep:crossbeam", "dep:futures", "dep:tokio", "dep:tokio-util", "dep:tower", "dep:rayon"] service = ["dep:crossbeam", "dep:futures", "dep:tokio", "dep:tokio-util", "dep:tower", "dep:rayon"]
[dependencies] [dependencies]
@ -25,6 +25,7 @@ cfg-if = { workspace = true }
# FIXME: # FIXME:
# We only need the `thread` feature if `service` is enabled. # We only need the `thread` feature if `service` is enabled.
# Figure out how to enable features of an already pulled in dependency conditionally. # Figure out how to enable features of an already pulled in dependency conditionally.
cuprate-database = { path = "../database" }
cuprate-helper = { path = "../../helper", features = ["fs", "thread", "map"] } cuprate-helper = { path = "../../helper", features = ["fs", "thread", "map"] }
cuprate-types = { path = "../../types", features = ["blockchain"] } cuprate-types = { path = "../../types", features = ["blockchain"] }
curve25519-dalek = { workspace = true } curve25519-dalek = { workspace = true }
@ -43,11 +44,6 @@ tower = { workspace = true, features = ["full"], optional = true }
thread_local = { workspace = true } thread_local = { workspace = true }
rayon = { workspace = true, optional = true } rayon = { workspace = true, optional = true }
# Optional features.
heed = { version = "0.20.0", features = ["read-txn-no-tls"], optional = true }
redb = { version = "2.1.0", optional = true }
serde = { workspace = true, optional = true }
[dev-dependencies] [dev-dependencies]
bytemuck = { version = "1.14.3", features = ["must_cast", "derive", "min_const_generics", "extern_crate_alloc"] } bytemuck = { version = "1.14.3", features = ["must_cast", "derive", "min_const_generics", "extern_crate_alloc"] }
cuprate-helper = { path = "../../helper", features = ["thread"] } cuprate-helper = { path = "../../helper", features = ["thread"] }

261
storage/cuprate-blockchain/src/backend/heed/database.rs
View file

@ -1,261 +0,0 @@
//! Implementation of `trait Database` for `heed`.
//---------------------------------------------------------------------------------------------------- Import
use std::{cell::RefCell, ops::RangeBounds};
use crate::{
backend::heed::types::HeedDb,
database::{DatabaseIter, DatabaseRo, DatabaseRw},
error::RuntimeError,
table::Table,
};
//---------------------------------------------------------------------------------------------------- Heed Database Wrappers
// Q. Why does `HeedTableR{o,w}` exist?
// A. These wrapper types combine `heed`'s database/table
// types with its transaction types. It exists to match
// `redb`, which has this behavior built-in.
//
// `redb` forces us to abstract read/write semantics
// at the _opened table_ level, so, we must match that in `heed`,
// which abstracts it at the transaction level.
//
// We must also maintain the ability for
// write operations to also read, aka, `Rw`.
/// An opened read-only database associated with a transaction.
///
/// Matches `redb::ReadOnlyTable`.
pub(super) struct HeedTableRo<'tx, T: Table> {
/// An already opened database table.
pub(super) db: HeedDb<T::Key, T::Value>,
/// The associated read-only transaction that opened this table.
pub(super) tx_ro: &'tx heed::RoTxn<'tx>,
}
/// An opened read/write database associated with a transaction.
///
/// Matches `redb::Table` (read & write).
pub(super) struct HeedTableRw<'env, 'tx, T: Table> {
/// An already opened database table.
pub(super) db: HeedDb<T::Key, T::Value>,
/// The associated read/write transaction that opened this table.
pub(super) tx_rw: &'tx RefCell<heed::RwTxn<'env>>,
}
//---------------------------------------------------------------------------------------------------- Shared functions
// FIXME: we cannot just deref `HeedTableRw -> HeedTableRo` and
// call the functions since the database is held by value, so
// just use these generic functions that both can call instead.
/// Shared [`DatabaseRo::get()`].
#[inline]
fn get<T: Table>(
db: &HeedDb<T::Key, T::Value>,
tx_ro: &heed::RoTxn<'_>,
key: &T::Key,
) -> Result<T::Value, RuntimeError> {
db.get(tx_ro, key)?.ok_or(RuntimeError::KeyNotFound)
}
/// Shared [`DatabaseRo::len()`].
#[inline]
fn len<T: Table>(
db: &HeedDb<T::Key, T::Value>,
tx_ro: &heed::RoTxn<'_>,
) -> Result<u64, RuntimeError> {
Ok(db.len(tx_ro)?)
}
/// Shared [`DatabaseRo::first()`].
#[inline]
fn first<T: Table>(
db: &HeedDb<T::Key, T::Value>,
tx_ro: &heed::RoTxn<'_>,
) -> Result<(T::Key, T::Value), RuntimeError> {
db.first(tx_ro)?.ok_or(RuntimeError::KeyNotFound)
}
/// Shared [`DatabaseRo::last()`].
#[inline]
fn last<T: Table>(
db: &HeedDb<T::Key, T::Value>,
tx_ro: &heed::RoTxn<'_>,
) -> Result<(T::Key, T::Value), RuntimeError> {
db.last(tx_ro)?.ok_or(RuntimeError::KeyNotFound)
}
/// Shared [`DatabaseRo::is_empty()`].
#[inline]
fn is_empty<T: Table>(
db: &HeedDb<T::Key, T::Value>,
tx_ro: &heed::RoTxn<'_>,
) -> Result<bool, RuntimeError> {
Ok(db.is_empty(tx_ro)?)
}
//---------------------------------------------------------------------------------------------------- DatabaseIter Impl
impl<T: Table> DatabaseIter<T> for HeedTableRo<'_, T> {
#[inline]
fn get_range<'a, Range>(
&'a self,
range: Range,
) -> Result<impl Iterator<Item = Result<T::Value, RuntimeError>> + 'a, RuntimeError>
where
Range: RangeBounds<T::Key> + 'a,
{
Ok(self.db.range(self.tx_ro, &range)?.map(|res| Ok(res?.1)))
}
#[inline]
fn iter(
&self,
) -> Result<impl Iterator<Item = Result<(T::Key, T::Value), RuntimeError>> + '_, RuntimeError>
{
Ok(self.db.iter(self.tx_ro)?.map(|res| Ok(res?)))
}
#[inline]
fn keys(
&self,
) -> Result<impl Iterator<Item = Result<T::Key, RuntimeError>> + '_, RuntimeError> {
Ok(self.db.iter(self.tx_ro)?.map(|res| Ok(res?.0)))
}
#[inline]
fn values(
&self,
) -> Result<impl Iterator<Item = Result<T::Value, RuntimeError>> + '_, RuntimeError> {
Ok(self.db.iter(self.tx_ro)?.map(|res| Ok(res?.1)))
}
}
//---------------------------------------------------------------------------------------------------- DatabaseRo Impl
// SAFETY: `HeedTableRo: !Send` as it holds a reference to `heed::RoTxn: Send + !Sync`.
unsafe impl<T: Table> DatabaseRo<T> for HeedTableRo<'_, T> {
#[inline]
fn get(&self, key: &T::Key) -> Result<T::Value, RuntimeError> {
get::<T>(&self.db, self.tx_ro, key)
}
#[inline]
fn len(&self) -> Result<u64, RuntimeError> {
len::<T>(&self.db, self.tx_ro)
}
#[inline]
fn first(&self) -> Result<(T::Key, T::Value), RuntimeError> {
first::<T>(&self.db, self.tx_ro)
}
#[inline]
fn last(&self) -> Result<(T::Key, T::Value), RuntimeError> {
last::<T>(&self.db, self.tx_ro)
}
#[inline]
fn is_empty(&self) -> Result<bool, RuntimeError> {
is_empty::<T>(&self.db, self.tx_ro)
}
}
//---------------------------------------------------------------------------------------------------- DatabaseRw Impl
// SAFETY: The `Send` bound only applies to `HeedTableRo`.
// `HeedTableRw`'s write transaction is `!Send`.
unsafe impl<T: Table> DatabaseRo<T> for HeedTableRw<'_, '_, T> {
#[inline]
fn get(&self, key: &T::Key) -> Result<T::Value, RuntimeError> {
get::<T>(&self.db, &self.tx_rw.borrow(), key)
}
#[inline]
fn len(&self) -> Result<u64, RuntimeError> {
len::<T>(&self.db, &self.tx_rw.borrow())
}
#[inline]
fn first(&self) -> Result<(T::Key, T::Value), RuntimeError> {
first::<T>(&self.db, &self.tx_rw.borrow())
}
#[inline]
fn last(&self) -> Result<(T::Key, T::Value), RuntimeError> {
last::<T>(&self.db, &self.tx_rw.borrow())
}
#[inline]
fn is_empty(&self) -> Result<bool, RuntimeError> {
is_empty::<T>(&self.db, &self.tx_rw.borrow())
}
}
impl<T: Table> DatabaseRw<T> for HeedTableRw<'_, '_, T> {
#[inline]
fn put(&mut self, key: &T::Key, value: &T::Value) -> Result<(), RuntimeError> {
Ok(self.db.put(&mut self.tx_rw.borrow_mut(), key, value)?)
}
#[inline]
fn delete(&mut self, key: &T::Key) -> Result<(), RuntimeError> {
self.db.delete(&mut self.tx_rw.borrow_mut(), key)?;
Ok(())
}
#[inline]
fn take(&mut self, key: &T::Key) -> Result<T::Value, RuntimeError> {
// LMDB/heed does not return the value on deletion.
// So, fetch it first - then delete.
let value = get::<T>(&self.db, &self.tx_rw.borrow(), key)?;
match self.db.delete(&mut self.tx_rw.borrow_mut(), key) {
Ok(true) => Ok(value),
Err(e) => Err(e.into()),
// We just `get()`'ed the value - it is
// incorrect for it to suddenly not exist.
Ok(false) => unreachable!(),
}
}
#[inline]
fn pop_first(&mut self) -> Result<(T::Key, T::Value), RuntimeError> {
let tx_rw = &mut self.tx_rw.borrow_mut();
// Get the value first...
let Some((key, value)) = self.db.first(tx_rw)? else {
return Err(RuntimeError::KeyNotFound);
};
// ...then remove it.
match self.db.delete(tx_rw, &key) {
Ok(true) => Ok((key, value)),
Err(e) => Err(e.into()),
// We just `get()`'ed the value - it is
// incorrect for it to suddenly not exist.
Ok(false) => unreachable!(),
}
}
#[inline]
fn pop_last(&mut self) -> Result<(T::Key, T::Value), RuntimeError> {
let tx_rw = &mut self.tx_rw.borrow_mut();
// Get the value first...
let Some((key, value)) = self.db.last(tx_rw)? else {
return Err(RuntimeError::KeyNotFound);
};
// ...then remove it.
match self.db.delete(tx_rw, &key) {
Ok(true) => Ok((key, value)),
Err(e) => Err(e.into()),
// We just `get()`'ed the value - it is
// incorrect for it to suddenly not exist.
Ok(false) => unreachable!(),
}
}
}
//---------------------------------------------------------------------------------------------------- Tests
#[cfg(test)]
mod test {
// use super::*;
}

347
storage/cuprate-blockchain/src/backend/heed/env.rs
View file

@ -1,347 +0,0 @@
//! Implementation of `trait Env` for `heed`.
//---------------------------------------------------------------------------------------------------- Import
use std::{
cell::RefCell,
num::NonZeroUsize,
sync::{RwLock, RwLockReadGuard},
};
use heed::{DatabaseOpenOptions, EnvFlags, EnvOpenOptions};
use crate::{
backend::heed::{
database::{HeedTableRo, HeedTableRw},
storable::StorableHeed,
types::HeedDb,
},
config::{Config, SyncMode},
database::{DatabaseIter, DatabaseRo, DatabaseRw},
env::{Env, EnvInner},
error::{InitError, RuntimeError},
resize::ResizeAlgorithm,
table::Table,
tables::call_fn_on_all_tables_or_early_return,
};
//---------------------------------------------------------------------------------------------------- Consts
/// Panic message when there's a table missing.
const PANIC_MSG_MISSING_TABLE: &str =
"cuprate_blockchain::Env should uphold the invariant that all tables are already created";
//---------------------------------------------------------------------------------------------------- ConcreteEnv
/// A strongly typed, concrete database environment, backed by `heed`.
pub struct ConcreteEnv {
/// The actual database environment.
///
/// # Why `RwLock`?
/// We need mutual exclusive access to the environment for resizing.
///
/// Using 2 atomics for mutual exclusion was considered:
/// - `currently_resizing: AtomicBool`
/// - `reader_count: AtomicUsize`
///
/// This is how `monerod` does it:
/// <https://github.com/monero-project/monero/blob/059028a30a8ae9752338a7897329fe8012a310d5/src/blockchain_db/lmdb/db_lmdb.cpp#L354-L355>
///
/// `currently_resizing` would be set to `true` on resizes and
/// `reader_count` would be spinned on until 0, at which point
/// we are safe to resize.
///
/// Although, 3 atomic operations (check atomic bool, `reader_count++`, `reader_count--`)
/// turns out to be roughly as expensive as acquiring a non-contended `RwLock`,
/// the CPU sleeping instead of spinning is much better too.
///
/// # `unwrap()`
/// This will be [`unwrap()`]ed everywhere.
///
/// If lock is poisoned, we want all of Cuprate to panic.
env: RwLock<heed::Env>,
/// The configuration we were opened with
/// (and in current use).
pub(super) config: Config,
}
impl Drop for ConcreteEnv {
fn drop(&mut self) {
// INVARIANT: drop(ConcreteEnv) must sync.
//
// SOMEDAY:
// "if the environment has the MDB_NOSYNC flag set the flushes will be omitted,
// and with MDB_MAPASYNC they will be asynchronous."
// <http://www.lmdb.tech/doc/group__mdb.html#ga85e61f05aa68b520cc6c3b981dba5037>
//
// We need to do `mdb_env_set_flags(&env, MDB_NOSYNC|MDB_ASYNCMAP, 0)`
// to clear the no sync and async flags such that the below `self.sync()`
// _actually_ synchronously syncs.
if let Err(_e) = crate::Env::sync(self) {
// TODO: log error?
}
// TODO: log that we are dropping the database.
// TODO: use tracing.
// <https://github.com/LMDB/lmdb/blob/b8e54b4c31378932b69f1298972de54a565185b1/libraries/liblmdb/lmdb.h#L49-L61>
let result = self.env.read().unwrap().clear_stale_readers();
match result {
Ok(n) => println!("LMDB stale readers cleared: {n}"),
Err(e) => println!("LMDB stale reader clear error: {e:?}"),
}
}
}
//---------------------------------------------------------------------------------------------------- Env Impl
impl Env for ConcreteEnv {
const MANUAL_RESIZE: bool = true;
const SYNCS_PER_TX: bool = false;
type EnvInner<'env> = RwLockReadGuard<'env, heed::Env>;
type TxRo<'tx> = heed::RoTxn<'tx>;
/// HACK:
/// `heed::RwTxn` is wrapped in `RefCell` to allow:
/// - opening a database with only a `&` to it
/// - allowing 1 write tx to open multiple tables
///
/// Our mutable accesses are safe and will not panic as:
/// - Write transactions are `!Sync`
/// - A table operation does not hold a reference to the inner cell
/// once the call is over
/// - The function to manipulate the table takes the same type
/// of reference that the `RefCell` gets for that function
///
/// Also see:
/// - <https://github.com/Cuprate/cuprate/pull/102#discussion_r1548695610>
/// - <https://github.com/Cuprate/cuprate/pull/104>
type TxRw<'tx> = RefCell<heed::RwTxn<'tx>>;
#[cold]
#[inline(never)] // called once.
fn open(config: Config) -> Result<Self, InitError> {
// <https://github.com/monero-project/monero/blob/059028a30a8ae9752338a7897329fe8012a310d5/src/blockchain_db/lmdb/db_lmdb.cpp#L1324>
let mut env_open_options = EnvOpenOptions::new();
// Map our `Config` sync mode to the LMDB environment flags.
//
// <https://github.com/monero-project/monero/blob/059028a30a8ae9752338a7897329fe8012a310d5/src/blockchain_db/lmdb/db_lmdb.cpp#L1324>
let flags = match config.sync_mode {
SyncMode::Safe => EnvFlags::empty(),
SyncMode::Async => EnvFlags::MAP_ASYNC,
SyncMode::Fast => EnvFlags::NO_SYNC | EnvFlags::WRITE_MAP | EnvFlags::MAP_ASYNC,
// SOMEDAY: dynamic syncs are not implemented.
SyncMode::FastThenSafe | SyncMode::Threshold(_) => unimplemented!(),
};
// SAFETY: the flags we're setting are 'unsafe'
// from a data durability perspective, although,
// the user config wanted this.
//
// MAYBE: We may need to open/create tables with certain flags
// <https://github.com/monero-project/monero/blob/059028a30a8ae9752338a7897329fe8012a310d5/src/blockchain_db/lmdb/db_lmdb.cpp#L1324>
// MAYBE: Set comparison functions for certain tables
// <https://github.com/monero-project/monero/blob/059028a30a8ae9752338a7897329fe8012a310d5/src/blockchain_db/lmdb/db_lmdb.cpp#L1324>
unsafe {
env_open_options.flags(flags);
}
// Set the memory map size to
// (current disk size) + (a bit of leeway)
// to account for empty databases where we
// need to write same tables.
#[allow(clippy::cast_possible_truncation)] // only 64-bit targets
let disk_size_bytes = match std::fs::File::open(&config.db_file) {
Ok(file) => file.metadata()?.len() as usize,
// The database file doesn't exist, 0 bytes.
Err(io_err) if io_err.kind() == std::io::ErrorKind::NotFound => 0,
Err(io_err) => return Err(io_err.into()),
};
// Add leeway space.
let memory_map_size = crate::resize::fixed_bytes(disk_size_bytes, 1_000_000 /* 1MB */);
env_open_options.map_size(memory_map_size.get());
// Set the max amount of database tables.
// We know at compile time how many tables there are.
// SOMEDAY: ...how many?
env_open_options.max_dbs(32);
// LMDB documentation:
// ```
// Number of slots in the reader table.
// This value was chosen somewhat arbitrarily. 126 readers plus a
// couple mutexes fit exactly into 8KB on my development machine.
// ```
// <https://github.com/LMDB/lmdb/blob/b8e54b4c31378932b69f1298972de54a565185b1/libraries/liblmdb/mdb.c#L794-L799>
//
// So, we're going to be following these rules:
// - Use at least 126 reader threads
// - Add 16 extra reader threads if <126
//
// FIXME: This behavior is from `monerod`:
// <https://github.com/monero-project/monero/blob/059028a30a8ae9752338a7897329fe8012a310d5/src/blockchain_db/lmdb/db_lmdb.cpp#L1324>
// I believe this could be adjusted percentage-wise so very high
// thread PCs can benefit from something like (cuprated + anything that uses the DB in the future).
// For now:
// - No other program using our DB exists
// - Almost no-one has a 126+ thread CPU
let reader_threads =
u32::try_from(config.reader_threads.as_threads().get()).unwrap_or(u32::MAX);
env_open_options.max_readers(if reader_threads < 110 {
126
} else {
reader_threads.saturating_add(16)
});
// Create the database directory if it doesn't exist.
std::fs::create_dir_all(config.db_directory())?;
// Open the environment in the user's PATH.
// SAFETY: LMDB uses a memory-map backed file.
// <https://docs.rs/heed/0.20.0/heed/struct.EnvOpenOptions.html#method.open>
let env = unsafe { env_open_options.open(config.db_directory())? };
/// Function that creates the tables based off the passed `T: Table`.
fn create_table<T: Table>(
env: &heed::Env,
tx_rw: &mut heed::RwTxn<'_>,
) -> Result<(), InitError> {
DatabaseOpenOptions::new(env)
.name(<T as Table>::NAME)
.types::<StorableHeed<<T as Table>::Key>, StorableHeed<<T as Table>::Value>>()
.create(tx_rw)?;
Ok(())
}
let mut tx_rw = env.write_txn()?;
// Create all tables.
// FIXME: this macro is kinda awkward.
{
let env = &env;
let tx_rw = &mut tx_rw;
match call_fn_on_all_tables_or_early_return!(create_table(env, tx_rw)) {
Ok(_) => (),
Err(e) => return Err(e),
}
}
// INVARIANT: this should never return `ResizeNeeded` due to adding
// some tables since we added some leeway to the memory map above.
tx_rw.commit()?;
Ok(Self {
env: RwLock::new(env),
config,
})
}
fn config(&self) -> &Config {
&self.config
}
fn sync(&self) -> Result<(), RuntimeError> {
Ok(self.env.read().unwrap().force_sync()?)
}
fn resize_map(&self, resize_algorithm: Option<ResizeAlgorithm>) -> NonZeroUsize {
let resize_algorithm = resize_algorithm.unwrap_or_else(|| self.config().resize_algorithm);
let current_size_bytes = self.current_map_size();
let new_size_bytes = resize_algorithm.resize(current_size_bytes);
// SAFETY:
// Resizing requires that we have
// exclusive access to the database environment.
// Our `heed::Env` is wrapped within a `RwLock`,
// and we have a WriteGuard to it, so we're safe.
//
// <http://www.lmdb.tech/doc/group__mdb.html#gaa2506ec8dab3d969b0e609cd82e619e5>
unsafe {
// INVARIANT: `resize()` returns a valid `usize` to resize to.
self.env
.write()
.unwrap()
.resize(new_size_bytes.get())
.unwrap();
}
new_size_bytes
}
#[inline]
fn current_map_size(&self) -> usize {
self.env.read().unwrap().info().map_size
}
#[inline]
fn env_inner(&self) -> Self::EnvInner<'_> {
self.env.read().unwrap()
}
}
//---------------------------------------------------------------------------------------------------- EnvInner Impl
impl<'env> EnvInner<'env, heed::RoTxn<'env>, RefCell<heed::RwTxn<'env>>>
for RwLockReadGuard<'env, heed::Env>
where
Self: 'env,
{
#[inline]
fn tx_ro(&'env self) -> Result<heed::RoTxn<'env>, RuntimeError> {
Ok(self.read_txn()?)
}
#[inline]
fn tx_rw(&'env self) -> Result<RefCell<heed::RwTxn<'env>>, RuntimeError> {
Ok(RefCell::new(self.write_txn()?))
}
#[inline]
fn open_db_ro<T: Table>(
&self,
tx_ro: &heed::RoTxn<'env>,
) -> Result<impl DatabaseRo<T> + DatabaseIter<T>, RuntimeError> {
// Open up a read-only database using our table's const metadata.
Ok(HeedTableRo {
db: self
.open_database(tx_ro, Some(T::NAME))?
.expect(PANIC_MSG_MISSING_TABLE),
tx_ro,
})
}
#[inline]
fn open_db_rw<T: Table>(
&self,
tx_rw: &RefCell<heed::RwTxn<'env>>,
) -> Result<impl DatabaseRw<T>, RuntimeError> {
let tx_ro = tx_rw.borrow();
// Open up a read/write database using our table's const metadata.
Ok(HeedTableRw {
db: self
.open_database(&tx_ro, Some(T::NAME))?
.expect(PANIC_MSG_MISSING_TABLE),
tx_rw,
})
}
#[inline]
fn clear_db<T: Table>(
&self,
tx_rw: &mut RefCell<heed::RwTxn<'env>>,
) -> Result<(), RuntimeError> {
let tx_rw = tx_rw.get_mut();
// Open the table first...
let db: HeedDb<T::Key, T::Value> = self
.open_database(tx_rw, Some(T::NAME))?
.expect(PANIC_MSG_MISSING_TABLE);
// ...then clear it.
Ok(db.clear(tx_rw)?)
}
}
//---------------------------------------------------------------------------------------------------- Tests
#[cfg(test)]
mod test {
// use super::*;
}

152
storage/cuprate-blockchain/src/backend/heed/error.rs
View file

@ -1,152 +0,0 @@
//! Conversion from `heed::Error` -> `cuprate_blockchain`'s errors.
//---------------------------------------------------------------------------------------------------- Use
use crate::constants::DATABASE_CORRUPT_MSG;
//---------------------------------------------------------------------------------------------------- InitError
impl From<heed::Error> for crate::InitError {
fn from(error: heed::Error) -> Self {
use heed::Error as E1;
use heed::MdbError as E2;
// Reference of all possible errors `heed` will return
// upon using [`heed::EnvOpenOptions::open`]:
// <https://docs.rs/heed/latest/src/heed/env.rs.html#149-219>
match error {
E1::Io(io_error) => Self::Io(io_error),
E1::DatabaseClosing => Self::ShuttingDown,
// LMDB errors.
E1::Mdb(mdb_error) => match mdb_error {
E2::Invalid => Self::Invalid,
E2::VersionMismatch => Self::InvalidVersion,
// "Located page was wrong type".
// <https://docs.rs/heed/latest/heed/enum.MdbError.html#variant.Corrupted>
//
// "Requested page not found - this usually indicates corruption."
// <https://docs.rs/heed/latest/heed/enum.MdbError.html#variant.PageNotFound>
E2::Corrupted | E2::PageNotFound => Self::Corrupt,
// These errors shouldn't be returned on database init.
E2::Incompatible
| E2::Other(_)
| E2::BadTxn
| E2::Problem
| E2::KeyExist
| E2::NotFound
| E2::MapFull
| E2::ReadersFull
| E2::PageFull
| E2::DbsFull
| E2::TlsFull
| E2::TxnFull
| E2::CursorFull
| E2::MapResized
| E2::BadRslot
| E2::BadValSize
| E2::BadDbi
| E2::Panic => Self::Unknown(Box::new(mdb_error)),
},
E1::BadOpenOptions { .. } | E1::Encoding(_) | E1::Decoding(_) => {
Self::Unknown(Box::new(error))
}
}
}
}
//---------------------------------------------------------------------------------------------------- RuntimeError
#[allow(clippy::fallible_impl_from)] // We need to panic sometimes.
impl From<heed::Error> for crate::RuntimeError {
/// # Panics
/// This will panic on unrecoverable errors for safety.
fn from(error: heed::Error) -> Self {
use heed::Error as E1;
use heed::MdbError as E2;
match error {
// I/O errors.
E1::Io(io_error) => Self::Io(io_error),
// LMDB errors.
E1::Mdb(mdb_error) => match mdb_error {
E2::KeyExist => Self::KeyExists,
E2::NotFound => Self::KeyNotFound,
E2::MapFull => Self::ResizeNeeded,
// Corruption errors, these have special panic messages.
//
// "Located page was wrong type".
// <https://docs.rs/heed/latest/heed/enum.MdbError.html#variant.Corrupted>
//
// "Requested page not found - this usually indicates corruption."
// <https://docs.rs/heed/latest/heed/enum.MdbError.html#variant.PageNotFound>
E2::Corrupted | E2::PageNotFound => panic!("{mdb_error:#?}\n{DATABASE_CORRUPT_MSG}"),
// These errors should not occur, and if they do,
// the best thing `cuprate_blockchain` can do for
// safety is to panic right here.
E2::Panic
| E2::PageFull
| E2::Other(_)
| E2::BadTxn
| E2::Problem
| E2::Invalid
| E2::TlsFull
| E2::TxnFull
| E2::BadRslot
| E2::VersionMismatch
| E2::BadDbi => panic!("{mdb_error:#?}"),
// These errors are the same as above, but instead
// of being errors we can't control, these are errors
// that only happen if we write incorrect code.
// "Database contents grew beyond environment mapsize."
// We should be resizing the map when needed, this error
// occurring indicates we did _not_ do that, which is a bug
// and we should panic.
//
// FIXME: This can also mean _another_ process wrote to our
// LMDB file and increased the size. I don't think we need to accommodate for this.
// <http://www.lmdb.tech/doc/group__mdb.html#gaa2506ec8dab3d969b0e609cd82e619e5>
// Although `monerod` reacts to that instead of `MDB_MAP_FULL`
// which is what `mdb_put()` returns so... idk?
// <https://github.com/monero-project/monero/blob/059028a30a8ae9752338a7897329fe8012a310d5/src/blockchain_db/lmdb/db_lmdb.cpp#L526>
| E2::MapResized
// We should be setting `heed::EnvOpenOptions::max_readers()`
// with our reader thread value in [`crate::config::Config`],
// thus this error should never occur.
// <http://www.lmdb.tech/doc/group__mdb.html#gae687966c24b790630be2a41573fe40e2>
| E2::ReadersFull
// Do not open more database tables than we initially started with.
// We know this number at compile time (amount of `Table`'s) so this
// should never happen.
// <https://docs.rs/heed/0.20.0-alpha.9/heed/struct.EnvOpenOptions.html#method.max_dbs>
// <https://docs.rs/heed/0.20.0-alpha.9/src/heed/env.rs.html#251>
| E2::DbsFull
// Don't do crazy multi-nested LMDB cursor stuff.
| E2::CursorFull
// <https://docs.rs/heed/0.20.0-alpha.9/heed/enum.MdbError.html#variant.Incompatible>
| E2::Incompatible
// Unsupported size of key/DB name/data, or wrong DUP_FIXED size.
// Don't use a key that is `>511` bytes.
// <http://www.lmdb.tech/doc/group__mdb.html#gaaf0be004f33828bf2fb09d77eb3cef94>
| E2::BadValSize
=> panic!("fix the database code! {mdb_error:#?}"),
},
// Only if we write incorrect code.
E1::DatabaseClosing | E1::BadOpenOptions { .. } | E1::Encoding(_) | E1::Decoding(_) => {
panic!("fix the database code! {error:#?}")
}
}
}
}
//---------------------------------------------------------------------------------------------------- Tests
#[cfg(test)]
mod test {
// use super::*;
}

10
storage/cuprate-blockchain/src/backend/heed/mod.rs
View file

@ -1,10 +0,0 @@
//! Database backend implementation backed by `heed`.
mod env;
pub use env::ConcreteEnv;
mod database;
mod error;
mod storable;
mod transaction;
mod types;

122
storage/cuprate-blockchain/src/backend/heed/storable.rs
View file

@ -1,122 +0,0 @@
//! `cuprate_blockchain::Storable` <-> `heed` serde trait compatibility layer.
//---------------------------------------------------------------------------------------------------- Use
use std::{borrow::Cow, marker::PhantomData};
use heed::{BoxedError, BytesDecode, BytesEncode};
use crate::storable::Storable;
//---------------------------------------------------------------------------------------------------- StorableHeed
/// The glue struct that implements `heed`'s (de)serialization
/// traits on any type that implements `cuprate_blockchain::Storable`.
///
/// Never actually gets constructed, just used for trait bound translations.
pub(super) struct StorableHeed<T>(PhantomData<T>)
where
T: Storable + ?Sized;
//---------------------------------------------------------------------------------------------------- BytesDecode
impl<'a, T> BytesDecode<'a> for StorableHeed<T>
where
T: Storable + 'static,
{
type DItem = T;
#[inline]
/// This function is infallible (will always return `Ok`).
fn bytes_decode(bytes: &'a [u8]) -> Result<Self::DItem, BoxedError> {
Ok(T::from_bytes(bytes))
}
}
//---------------------------------------------------------------------------------------------------- BytesEncode
impl<'a, T> BytesEncode<'a> for StorableHeed<T>
where
T: Storable + ?Sized + 'a,
{
type EItem = T;
#[inline]
/// This function is infallible (will always return `Ok`).
fn bytes_encode(item: &'a Self::EItem) -> Result<Cow<'a, [u8]>, BoxedError> {
Ok(Cow::Borrowed(item.as_bytes()))
}
}
//---------------------------------------------------------------------------------------------------- Tests
#[cfg(test)]
mod test {
use std::fmt::Debug;
use super::*;
use crate::{StorableBytes, StorableVec};
// Each `#[test]` function has a `test()` to:
// - log
// - simplify trait bounds
// - make sure the right function is being called
#[test]
/// Assert `BytesEncode::bytes_encode` is accurate.
fn bytes_encode() {
fn test<T>(t: &T, expected: &[u8])
where
T: Storable + ?Sized,
{
println!("t: {t:?}, expected: {expected:?}");
assert_eq!(
<StorableHeed::<T> as BytesEncode>::bytes_encode(t).unwrap(),
expected
);
}
test::<()>(&(), &[]);
test::<u8>(&0, &[0]);
test::<u16>(&1, &[1, 0]);
test::<u32>(&2, &[2, 0, 0, 0]);
test::<u64>(&3, &[3, 0, 0, 0, 0, 0, 0, 0]);
test::<i8>(&-1, &[255]);
test::<i16>(&-2, &[254, 255]);
test::<i32>(&-3, &[253, 255, 255, 255]);
test::<i64>(&-4, &[252, 255, 255, 255, 255, 255, 255, 255]);
test::<StorableVec<u8>>(&StorableVec(vec![1, 2]), &[1, 2]);
test::<StorableBytes>(&StorableBytes(bytes::Bytes::from_static(&[1, 2])), &[1, 2]);
test::<[u8; 0]>(&[], &[]);
test::<[u8; 1]>(&[255], &[255]);
test::<[u8; 2]>(&[111, 0], &[111, 0]);
test::<[u8; 3]>(&[1, 0, 1], &[1, 0, 1]);
}
#[test]
/// Assert `BytesDecode::bytes_decode` is accurate.
fn bytes_decode() {
fn test<T>(bytes: &[u8], expected: &T)
where
T: Storable + PartialEq + ToOwned + Debug + 'static,
T::Owned: Debug,
{
println!("bytes: {bytes:?}, expected: {expected:?}");
assert_eq!(
&<StorableHeed::<T> as BytesDecode>::bytes_decode(bytes).unwrap(),
expected
);
}
test::<()>([].as_slice(), &());
test::<u8>([0].as_slice(), &0);
test::<u16>([1, 0].as_slice(), &1);
test::<u32>([2, 0, 0, 0].as_slice(), &2);
test::<u64>([3, 0, 0, 0, 0, 0, 0, 0].as_slice(), &3);
test::<i8>([255].as_slice(), &-1);
test::<i16>([254, 255].as_slice(), &-2);
test::<i32>([253, 255, 255, 255].as_slice(), &-3);
test::<i64>([252, 255, 255, 255, 255, 255, 255, 255].as_slice(), &-4);
test::<StorableVec<u8>>(&[1, 2], &StorableVec(vec![1, 2]));
test::<StorableBytes>(&[1, 2], &StorableBytes(bytes::Bytes::from_static(&[1, 2])));
test::<[u8; 0]>([].as_slice(), &[]);
test::<[u8; 1]>([255].as_slice(), &[255]);
test::<[u8; 2]>([111, 0].as_slice(), &[111, 0]);
test::<[u8; 3]>([1, 0, 1].as_slice(), &[1, 0, 1]);
}
}

41
storage/cuprate-blockchain/src/backend/heed/transaction.rs
View file

@ -1,41 +0,0 @@
//! Implementation of `trait TxRo/TxRw` for `heed`.
use std::cell::RefCell;
//---------------------------------------------------------------------------------------------------- Import
use crate::{
error::RuntimeError,
transaction::{TxRo, TxRw},
};
//---------------------------------------------------------------------------------------------------- TxRo
impl TxRo<'_> for heed::RoTxn<'_> {
fn commit(self) -> Result<(), RuntimeError> {
Ok(heed::RoTxn::commit(self)?)
}
}
//---------------------------------------------------------------------------------------------------- TxRw
impl TxRo<'_> for RefCell<heed::RwTxn<'_>> {
fn commit(self) -> Result<(), RuntimeError> {
TxRw::commit(self)
}
}
impl TxRw<'_> for RefCell<heed::RwTxn<'_>> {
fn commit(self) -> Result<(), RuntimeError> {
Ok(heed::RwTxn::commit(self.into_inner())?)
}
/// This function is infallible.
fn abort(self) -> Result<(), RuntimeError> {
heed::RwTxn::abort(self.into_inner());
Ok(())
}
}
//---------------------------------------------------------------------------------------------------- Tests
#[cfg(test)]
mod test {
// use super::*;
}

8
storage/cuprate-blockchain/src/backend/heed/types.rs
View file

@ -1,8 +0,0 @@
//! `heed` type aliases.
//---------------------------------------------------------------------------------------------------- Use
use crate::backend::heed::storable::StorableHeed;
//---------------------------------------------------------------------------------------------------- Types
/// The concrete database type for `heed`, usable for reads and writes.
pub(super) type HeedDb<K, V> = heed::Database<StorableHeed<K>, StorableHeed<V>>;

16
storage/cuprate-blockchain/src/backend/mod.rs
View file

@ -1,16 +0,0 @@
//! Database backends.
cfg_if::cfg_if! {
// If both backends are enabled, fallback to `heed`.
// This is useful when using `--all-features`.
if #[cfg(all(feature = "redb", not(feature = "heed")))] {
mod redb;
pub use redb::ConcreteEnv;
} else {
mod heed;
pub use heed::ConcreteEnv;
}
}
#[cfg(test)]
mod tests;

213
storage/cuprate-blockchain/src/backend/redb/database.rs
View file

@ -1,213 +0,0 @@
//! Implementation of `trait DatabaseR{o,w}` for `redb`.
//---------------------------------------------------------------------------------------------------- Import
use std::ops::RangeBounds;
use redb::ReadableTable;
use crate::{
backend::redb::{
storable::StorableRedb,
types::{RedbTableRo, RedbTableRw},
},
database::{DatabaseIter, DatabaseRo, DatabaseRw},
error::RuntimeError,
table::Table,
};
//---------------------------------------------------------------------------------------------------- Shared functions
// FIXME: we cannot just deref `RedbTableRw -> RedbTableRo` and
// call the functions since the database is held by value, so
// just use these generic functions that both can call instead.
/// Shared [`DatabaseRo::get()`].
#[inline]
fn get<T: Table + 'static>(
db: &impl redb::ReadableTable<StorableRedb<T::Key>, StorableRedb<T::Value>>,
key: &T::Key,
) -> Result<T::Value, RuntimeError> {
Ok(db.get(key)?.ok_or(RuntimeError::KeyNotFound)?.value())
}
/// Shared [`DatabaseRo::len()`].
#[inline]
fn len<T: Table>(
db: &impl redb::ReadableTable<StorableRedb<T::Key>, StorableRedb<T::Value>>,
) -> Result<u64, RuntimeError> {
Ok(db.len()?)
}
/// Shared [`DatabaseRo::first()`].
#[inline]
fn first<T: Table>(
db: &impl redb::ReadableTable<StorableRedb<T::Key>, StorableRedb<T::Value>>,
) -> Result<(T::Key, T::Value), RuntimeError> {
let (key, value) = db.first()?.ok_or(RuntimeError::KeyNotFound)?;
Ok((key.value(), value.value()))
}
/// Shared [`DatabaseRo::last()`].
#[inline]
fn last<T: Table>(
db: &impl redb::ReadableTable<StorableRedb<T::Key>, StorableRedb<T::Value>>,
) -> Result<(T::Key, T::Value), RuntimeError> {
let (key, value) = db.last()?.ok_or(RuntimeError::KeyNotFound)?;
Ok((key.value(), value.value()))
}
/// Shared [`DatabaseRo::is_empty()`].
#[inline]
fn is_empty<T: Table>(
db: &impl redb::ReadableTable<StorableRedb<T::Key>, StorableRedb<T::Value>>,
) -> Result<bool, RuntimeError> {
Ok(db.is_empty()?)
}
//---------------------------------------------------------------------------------------------------- DatabaseIter
impl<T: Table + 'static> DatabaseIter<T> for RedbTableRo<T::Key, T::Value> {
#[inline]
fn get_range<'a, Range>(
&'a self,
range: Range,
) -> Result<impl Iterator<Item = Result<T::Value, RuntimeError>> + 'a, RuntimeError>
where
Range: RangeBounds<T::Key> + 'a,
{
Ok(ReadableTable::range(self, range)?.map(|result| {
let (_key, value) = result?;
Ok(value.value())
}))
}
#[inline]
fn iter(
&self,
) -> Result<impl Iterator<Item = Result<(T::Key, T::Value), RuntimeError>> + '_, RuntimeError>
{
Ok(ReadableTable::iter(self)?.map(|result| {
let (key, value) = result?;
Ok((key.value(), value.value()))
}))
}
#[inline]
fn keys(
&self,
) -> Result<impl Iterator<Item = Result<T::Key, RuntimeError>> + '_, RuntimeError> {
Ok(ReadableTable::iter(self)?.map(|result| {
let (key, _value) = result?;
Ok(key.value())
}))
}
#[inline]
fn values(
&self,
) -> Result<impl Iterator<Item = Result<T::Value, RuntimeError>> + '_, RuntimeError> {
Ok(ReadableTable::iter(self)?.map(|result| {
let (_key, value) = result?;
Ok(value.value())
}))
}
}
//---------------------------------------------------------------------------------------------------- DatabaseRo
// SAFETY: Both `redb`'s transaction and table types are `Send + Sync`.
unsafe impl<T: Table + 'static> DatabaseRo<T> for RedbTableRo<T::Key, T::Value> {
#[inline]
fn get(&self, key: &T::Key) -> Result<T::Value, RuntimeError> {
get::<T>(self, key)
}
#[inline]
fn len(&self) -> Result<u64, RuntimeError> {
len::<T>(self)
}
#[inline]
fn first(&self) -> Result<(T::Key, T::Value), RuntimeError> {
first::<T>(self)
}
#[inline]
fn last(&self) -> Result<(T::Key, T::Value), RuntimeError> {
last::<T>(self)
}
#[inline]
fn is_empty(&self) -> Result<bool, RuntimeError> {
is_empty::<T>(self)
}
}
//---------------------------------------------------------------------------------------------------- DatabaseRw
// SAFETY: Both `redb`'s transaction and table types are `Send + Sync`.
unsafe impl<T: Table + 'static> DatabaseRo<T> for RedbTableRw<'_, T::Key, T::Value> {
#[inline]
fn get(&self, key: &T::Key) -> Result<T::Value, RuntimeError> {
get::<T>(self, key)
}
#[inline]
fn len(&self) -> Result<u64, RuntimeError> {
len::<T>(self)
}
#[inline]
fn first(&self) -> Result<(T::Key, T::Value), RuntimeError> {
first::<T>(self)
}
#[inline]
fn last(&self) -> Result<(T::Key, T::Value), RuntimeError> {
last::<T>(self)
}
#[inline]
fn is_empty(&self) -> Result<bool, RuntimeError> {
is_empty::<T>(self)
}
}
impl<T: Table + 'static> DatabaseRw<T> for RedbTableRw<'_, T::Key, T::Value> {
// `redb` returns the value after function calls so we end with Ok(()) instead.
#[inline]
fn put(&mut self, key: &T::Key, value: &T::Value) -> Result<(), RuntimeError> {
redb::Table::insert(self, key, value)?;
Ok(())
}
#[inline]
fn delete(&mut self, key: &T::Key) -> Result<(), RuntimeError> {
redb::Table::remove(self, key)?;
Ok(())
}
#[inline]
fn take(&mut self, key: &T::Key) -> Result<T::Value, RuntimeError> {
if let Some(value) = redb::Table::remove(self, key)? {
Ok(value.value())
} else {
Err(RuntimeError::KeyNotFound)
}
}
#[inline]
fn pop_first(&mut self) -> Result<(T::Key, T::Value), RuntimeError> {
let (key, value) = redb::Table::pop_first(self)?.ok_or(RuntimeError::KeyNotFound)?;
Ok((key.value(), value.value()))
}
#[inline]
fn pop_last(&mut self) -> Result<(T::Key, T::Value), RuntimeError> {
let (key, value) = redb::Table::pop_last(self)?.ok_or(RuntimeError::KeyNotFound)?;
Ok((key.value(), value.value()))
}
}
//---------------------------------------------------------------------------------------------------- Tests
#[cfg(test)]
mod test {
// use super::*;
}

226
storage/cuprate-blockchain/src/backend/redb/env.rs
View file

@ -1,226 +0,0 @@
//! Implementation of `trait Env` for `redb`.
//---------------------------------------------------------------------------------------------------- Import
use crate::{
backend::redb::storable::StorableRedb,
config::{Config, SyncMode},
database::{DatabaseIter, DatabaseRo, DatabaseRw},
env::{Env, EnvInner},
error::{InitError, RuntimeError},
table::Table,
tables::call_fn_on_all_tables_or_early_return,
TxRw,
};
//---------------------------------------------------------------------------------------------------- ConcreteEnv
/// A strongly typed, concrete database environment, backed by `redb`.
pub struct ConcreteEnv {
/// The actual database environment.
env: redb::Database,
/// The configuration we were opened with
/// (and in current use).
config: Config,
/// A cached, redb version of `cuprate_blockchain::config::SyncMode`.
/// `redb` needs the sync mode to be set _per_ TX, so we
/// will continue to use this value every `Env::tx_rw`.
durability: redb::Durability,
}
impl Drop for ConcreteEnv {
fn drop(&mut self) {
// INVARIANT: drop(ConcreteEnv) must sync.
if let Err(e) = self.sync() {
// TODO: use tracing
println!("{e:#?}");
}
// TODO: log that we are dropping the database.
}
}
//---------------------------------------------------------------------------------------------------- Env Impl
impl Env for ConcreteEnv {
const MANUAL_RESIZE: bool = false;
const SYNCS_PER_TX: bool = false;
type EnvInner<'env> = (&'env redb::Database, redb::Durability);
type TxRo<'tx> = redb::ReadTransaction;
type TxRw<'tx> = redb::WriteTransaction;
#[cold]
#[inline(never)] // called once.
fn open(config: Config) -> Result<Self, InitError> {
// SOMEDAY: dynamic syncs are not implemented.
let durability = match config.sync_mode {
// FIXME: There's also `redb::Durability::Paranoid`:
// <https://docs.rs/redb/1.5.0/redb/enum.Durability.html#variant.Paranoid>
// should we use that instead of Immediate?
SyncMode::Safe => redb::Durability::Immediate,
SyncMode::Async => redb::Durability::Eventual,
SyncMode::Fast => redb::Durability::None,
// SOMEDAY: dynamic syncs are not implemented.
SyncMode::FastThenSafe | SyncMode::Threshold(_) => unimplemented!(),
};
let env_builder = redb::Builder::new();
// FIXME: we can set cache sizes with:
// env_builder.set_cache(bytes);
// Use the in-memory backend if the feature is enabled.
let mut env = if cfg!(feature = "redb-memory") {
env_builder.create_with_backend(redb::backends::InMemoryBackend::new())?
} else {
// Create the database directory if it doesn't exist.
std::fs::create_dir_all(config.db_directory())?;
// Open the database file, create if needed.
let db_file = std::fs::OpenOptions::new()
.read(true)
.write(true)
.create(true)
.truncate(false)
.open(config.db_file())?;
env_builder.create_file(db_file)?
};
// Create all database tables.
// `redb` creates tables if they don't exist.
// <https://docs.rs/redb/latest/redb/struct.WriteTransaction.html#method.open_table>
/// Function that creates the tables based off the passed `T: Table`.
fn create_table<T: Table>(tx_rw: &redb::WriteTransaction) -> Result<(), InitError> {
let table: redb::TableDefinition<
'static,
StorableRedb<<T as Table>::Key>,
StorableRedb<<T as Table>::Value>,
> = redb::TableDefinition::new(<T as Table>::NAME);
// `redb` creates tables on open if not already created.
tx_rw.open_table(table)?;
Ok(())
}
// Create all tables.
// FIXME: this macro is kinda awkward.
let mut tx_rw = env.begin_write()?;
{
let tx_rw = &mut tx_rw;
match call_fn_on_all_tables_or_early_return!(create_table(tx_rw)) {
Ok(_) => (),
Err(e) => return Err(e),
}
}
tx_rw.commit()?;
// Check for file integrity.
// FIXME: should we do this? is it slow?
env.check_integrity()?;
Ok(Self {
env,
config,
durability,
})
}
fn config(&self) -> &Config {
&self.config
}
fn sync(&self) -> Result<(), RuntimeError> {
// `redb`'s syncs are tied with write transactions,
// so just create one, don't do anything and commit.
let mut tx_rw = self.env.begin_write()?;
tx_rw.set_durability(redb::Durability::Paranoid);
TxRw::commit(tx_rw)
}
fn env_inner(&self) -> Self::EnvInner<'_> {
(&self.env, self.durability)
}
}
//---------------------------------------------------------------------------------------------------- EnvInner Impl
impl<'env> EnvInner<'env, redb::ReadTransaction, redb::WriteTransaction>
for (&'env redb::Database, redb::Durability)
where
Self: 'env,
{
#[inline]
fn tx_ro(&'env self) -> Result<redb::ReadTransaction, RuntimeError> {
Ok(self.0.begin_read()?)
}
#[inline]
fn tx_rw(&'env self) -> Result<redb::WriteTransaction, RuntimeError> {
// `redb` has sync modes on the TX level, unlike heed,
// which sets it at the Environment level.
//
// So, set the durability here before returning the TX.
let mut tx_rw = self.0.begin_write()?;
tx_rw.set_durability(self.1);
Ok(tx_rw)
}
#[inline]
fn open_db_ro<T: Table>(
&self,
tx_ro: &redb::ReadTransaction,
) -> Result<impl DatabaseRo<T> + DatabaseIter<T>, RuntimeError> {
// Open up a read-only database using our `T: Table`'s const metadata.
let table: redb::TableDefinition<'static, StorableRedb<T::Key>, StorableRedb<T::Value>> =
redb::TableDefinition::new(T::NAME);
// INVARIANT: Our `?` error conversion will panic if the table does not exist.
Ok(tx_ro.open_table(table)?)
}
#[inline]
fn open_db_rw<T: Table>(
&self,
tx_rw: &redb::WriteTransaction,
) -> Result<impl DatabaseRw<T>, RuntimeError> {
// Open up a read/write database using our `T: Table`'s const metadata.
let table: redb::TableDefinition<'static, StorableRedb<T::Key>, StorableRedb<T::Value>> =
redb::TableDefinition::new(T::NAME);
// `redb` creates tables if they don't exist, so this should never panic.
// <https://docs.rs/redb/latest/redb/struct.WriteTransaction.html#method.open_table>
Ok(tx_rw.open_table(table)?)
}
#[inline]
fn clear_db<T: Table>(&self, tx_rw: &mut redb::WriteTransaction) -> Result<(), RuntimeError> {
let table: redb::TableDefinition<
'static,
StorableRedb<<T as Table>::Key>,
StorableRedb<<T as Table>::Value>,
> = redb::TableDefinition::new(<T as Table>::NAME);
// INVARIANT:
// This `delete_table()` will not run into this `TableAlreadyOpen` error:
// <https://docs.rs/redb/2.0.0/src/redb/transactions.rs.html#382>
// which will panic in the `From` impl, as:
//
// 1. Only 1 `redb::WriteTransaction` can exist at a time
// 2. We have exclusive access to it
// 3. So it's not being used to open a table since that needs `&tx_rw`
//
// Reader-open tables do not affect this, if they're open the below is still OK.
redb::WriteTransaction::delete_table(tx_rw, table)?;
// Re-create the table.
// `redb` creates tables if they don't exist, so this should never panic.
redb::WriteTransaction::open_table(tx_rw, table)?;
Ok(())
}
}
//---------------------------------------------------------------------------------------------------- Tests
#[cfg(test)]
mod test {
// use super::*;
}

172
storage/cuprate-blockchain/src/backend/redb/error.rs
View file

@ -1,172 +0,0 @@
//! Conversion from `redb`'s errors -> `cuprate_blockchain`'s errors.
//!
//! HACK: There's a lot of `_ =>` usage here because
//! `redb`'s errors are `#[non_exhaustive]`...
//---------------------------------------------------------------------------------------------------- Import
use crate::{
constants::DATABASE_CORRUPT_MSG,
error::{InitError, RuntimeError},
};
//---------------------------------------------------------------------------------------------------- InitError
impl From<redb::DatabaseError> for InitError {
/// Created by `redb` in:
/// - [`redb::Database::open`](https://docs.rs/redb/1.5.0/redb/struct.Database.html#method.open).
fn from(error: redb::DatabaseError) -> Self {
use redb::DatabaseError as E;
use redb::StorageError as E2;
// Reference of all possible errors `redb` will return
// upon using `redb::Database::open`:
// <https://docs.rs/redb/1.5.0/src/redb/db.rs.html#908-923>
match error {
E::RepairAborted => Self::Corrupt,
E::UpgradeRequired(_) => Self::InvalidVersion,
E::Storage(s_error) => match s_error {
E2::Io(e) => Self::Io(e),
E2::Corrupted(_) => Self::Corrupt,
// HACK: Handle new errors as `redb` adds them.
_ => Self::Unknown(Box::new(s_error)),
},
// HACK: Handle new errors as `redb` adds them.
_ => Self::Unknown(Box::new(error)),
}
}
}
impl From<redb::StorageError> for InitError {
/// Created by `redb` in:
/// - [`redb::Database::open`](https://docs.rs/redb/1.5.0/redb/struct.Database.html#method.check_integrity)
fn from(error: redb::StorageError) -> Self {
use redb::StorageError as E;
match error {
E::Io(e) => Self::Io(e),
E::Corrupted(_) => Self::Corrupt,
// HACK: Handle new errors as `redb` adds them.
_ => Self::Unknown(Box::new(error)),
}
}
}
impl From<redb::TransactionError> for InitError {
/// Created by `redb` in:
/// - [`redb::Database::begin_write`](https://docs.rs/redb/1.5.0/redb/struct.Database.html#method.begin_write)
fn from(error: redb::TransactionError) -> Self {
match error {
redb::TransactionError::Storage(error) => error.into(),
// HACK: Handle new errors as `redb` adds them.
_ => Self::Unknown(Box::new(error)),
}
}
}
impl From<redb::TableError> for InitError {
/// Created by `redb` in:
/// - [`redb::WriteTransaction::open_table`](https://docs.rs/redb/1.5.0/redb/struct.WriteTransaction.html#method.open_table)
fn from(error: redb::TableError) -> Self {
use redb::TableError as E;
match error {
E::Storage(error) => error.into(),
// HACK: Handle new errors as `redb` adds them.
_ => Self::Unknown(Box::new(error)),
}
}
}
impl From<redb::CommitError> for InitError {
/// Created by `redb` in:
/// - [`redb::WriteTransaction::commit`](https://docs.rs/redb/1.5.0/redb/struct.WriteTransaction.html#method.commit)
fn from(error: redb::CommitError) -> Self {
match error {
redb::CommitError::Storage(error) => error.into(),
// HACK: Handle new errors as `redb` adds them.
_ => Self::Unknown(Box::new(error)),
}
}
}
//---------------------------------------------------------------------------------------------------- RuntimeError
#[allow(clippy::fallible_impl_from)] // We need to panic sometimes.
impl From<redb::TransactionError> for RuntimeError {
/// Created by `redb` in:
/// - [`redb::Database::begin_write`](https://docs.rs/redb/1.5.0/redb/struct.Database.html#method.begin_write)
/// - [`redb::Database::begin_read`](https://docs.rs/redb/1.5.0/redb/struct.Database.html#method.begin_read)
fn from(error: redb::TransactionError) -> Self {
match error {
redb::TransactionError::Storage(error) => error.into(),
// HACK: Handle new errors as `redb` adds them.
_ => unreachable!(),
}
}
}
#[allow(clippy::fallible_impl_from)] // We need to panic sometimes.
impl From<redb::CommitError> for RuntimeError {
/// Created by `redb` in:
/// - [`redb::WriteTransaction::commit`](https://docs.rs/redb/1.5.0/redb/struct.WriteTransaction.html#method.commit)
fn from(error: redb::CommitError) -> Self {
match error {
redb::CommitError::Storage(error) => error.into(),
// HACK: Handle new errors as `redb` adds them.
_ => unreachable!(),
}
}
}
#[allow(clippy::fallible_impl_from)] // We need to panic sometimes.
impl From<redb::TableError> for RuntimeError {
/// Created by `redb` in:
/// - [`redb::WriteTransaction::open_table`](https://docs.rs/redb/1.5.0/redb/struct.WriteTransaction.html#method.open_table)
/// - [`redb::ReadTransaction::open_table`](https://docs.rs/redb/1.5.0/redb/struct.ReadTransaction.html#method.open_table)
fn from(error: redb::TableError) -> Self {
use redb::TableError as E;
match error {
E::Storage(error) => error.into(),
// Only if we write incorrect code.
E::TableTypeMismatch { .. }
| E::TableIsMultimap(_)
| E::TableIsNotMultimap(_)
| E::TypeDefinitionChanged { .. }
| E::TableDoesNotExist(_)
| E::TableAlreadyOpen(..) => panic!("fix the database code! {error:#?}"),
// HACK: Handle new errors as `redb` adds them.
_ => unreachable!(),
}
}
}
#[allow(clippy::fallible_impl_from)] // We need to panic sometimes.
impl From<redb::StorageError> for RuntimeError {
/// Created by `redb` in:
/// - [`redb::Table`](https://docs.rs/redb/1.5.0/redb/struct.Table.html) functions
/// - [`redb::ReadOnlyTable`](https://docs.rs/redb/1.5.0/redb/struct.ReadOnlyTable.html) functions
fn from(error: redb::StorageError) -> Self {
use redb::StorageError as E;
match error {
E::Io(e) => Self::Io(e),
E::Corrupted(s) => panic!("{s:#?}\n{DATABASE_CORRUPT_MSG}"),
E::ValueTooLarge(s) => panic!("fix the database code! {s:#?}"),
E::LockPoisoned(s) => panic!("{s:#?}"),
// HACK: Handle new errors as `redb` adds them.
_ => unreachable!(),
}
}
}
//---------------------------------------------------------------------------------------------------- Tests
#[cfg(test)]
mod test {
// use super::*;
}

9
storage/cuprate-blockchain/src/backend/redb/mod.rs
View file

@ -1,9 +0,0 @@
//! Database backend implementation backed by `sanakirja`.
mod env;
pub use env::ConcreteEnv;
mod database;
mod error;
mod storable;
mod transaction;
mod types;

221
storage/cuprate-blockchain/src/backend/redb/storable.rs
View file

@ -1,221 +0,0 @@
//! `cuprate_blockchain::Storable` <-> `redb` serde trait compatibility layer.
//---------------------------------------------------------------------------------------------------- Use
use std::{cmp::Ordering, fmt::Debug, marker::PhantomData};
use redb::TypeName;
use crate::{key::Key, storable::Storable};
//---------------------------------------------------------------------------------------------------- StorableRedb
/// The glue structs that implements `redb`'s (de)serialization
/// traits on any type that implements `cuprate_blockchain::Key`.
///
/// Never actually get constructed, just used for trait bound translations.
#[derive(Debug)]
pub(super) struct StorableRedb<T>(PhantomData<T>)
where
T: Storable;
//---------------------------------------------------------------------------------------------------- redb::Key
// If `Key` is also implemented, this can act as a `redb::Key`.
impl<T> redb::Key for StorableRedb<T>
where
T: Key + 'static,
{
#[inline]
fn compare(left: &[u8], right: &[u8]) -> Ordering {
<T as Key>::compare(left, right)
}
}
//---------------------------------------------------------------------------------------------------- redb::Value
impl<T> redb::Value for StorableRedb<T>
where
T: Storable + 'static,
{
type SelfType<'a> = T where Self: 'a;
type AsBytes<'a> = &'a [u8] where Self: 'a;
#[inline]
fn fixed_width() -> Option<usize> {
<T as Storable>::BYTE_LENGTH
}
#[inline]
fn from_bytes<'a>(data: &'a [u8]) -> Self::SelfType<'static>
where
Self: 'a,
{
<T as Storable>::from_bytes(data)
}
#[inline]
fn as_bytes<'a, 'b: 'a>(value: &'a Self::SelfType<'b>) -> &'a [u8]
where
Self: 'a + 'b,
{
<T as Storable>::as_bytes(value)
}
#[inline]
fn type_name() -> TypeName {
TypeName::new(std::any::type_name::<T>())
}
}
//---------------------------------------------------------------------------------------------------- Tests
#[cfg(test)]
#[allow(clippy::needless_pass_by_value)]
mod test {
use super::*;
use crate::{StorableBytes, StorableVec};
// Each `#[test]` function has a `test()` to:
// - log
// - simplify trait bounds
// - make sure the right function is being called
#[test]
/// Assert `redb::Key::compare` works for `StorableRedb`.
fn compare() {
fn test<T>(left: T, right: T, expected: Ordering)
where
T: Key + 'static,
{
println!("left: {left:?}, right: {right:?}, expected: {expected:?}");
assert_eq!(
<StorableRedb::<T> as redb::Key>::compare(
<StorableRedb::<T> as redb::Value>::as_bytes(&left),
<StorableRedb::<T> as redb::Value>::as_bytes(&right)
),
expected
);
}
test::<i64>(-1, 2, Ordering::Greater); // bytes are greater, not the value
test::<u64>(0, 1, Ordering::Less);
test::<[u8; 2]>([1, 1], [1, 0], Ordering::Greater);
test::<[u8; 3]>([1, 2, 3], [1, 2, 3], Ordering::Equal);
}
#[test]
/// Assert `redb::Key::fixed_width` is accurate.
fn fixed_width() {
fn test<T>(expected: Option<usize>)
where
T: Storable + 'static,
{
assert_eq!(<StorableRedb::<T> as redb::Value>::fixed_width(), expected);
}
test::<()>(Some(0));
test::<u8>(Some(1));
test::<u16>(Some(2));
test::<u32>(Some(4));
test::<u64>(Some(8));
test::<i8>(Some(1));
test::<i16>(Some(2));
test::<i32>(Some(4));
test::<i64>(Some(8));
test::<StorableVec<u8>>(None);
test::<StorableBytes>(None);
test::<[u8; 0]>(Some(0));
test::<[u8; 1]>(Some(1));
test::<[u8; 2]>(Some(2));
test::<[u8; 3]>(Some(3));
}
#[test]
/// Assert `redb::Key::as_bytes` is accurate.
fn as_bytes() {
fn test<T>(t: &T, expected: &[u8])
where
T: Storable + 'static,
{
println!("t: {t:?}, expected: {expected:?}");
assert_eq!(<StorableRedb::<T> as redb::Value>::as_bytes(t), expected);
}
test::<()>(&(), &[]);
test::<u8>(&0, &[0]);
test::<u16>(&1, &[1, 0]);
test::<u32>(&2, &[2, 0, 0, 0]);
test::<u64>(&3, &[3, 0, 0, 0, 0, 0, 0, 0]);
test::<i8>(&-1, &[255]);
test::<i16>(&-2, &[254, 255]);
test::<i32>(&-3, &[253, 255, 255, 255]);
test::<i64>(&-4, &[252, 255, 255, 255, 255, 255, 255, 255]);
test::<StorableVec<u8>>(&StorableVec([1, 2].to_vec()), &[1, 2]);
test::<StorableBytes>(&StorableBytes(bytes::Bytes::from_static(&[1, 2])), &[1, 2]);
test::<[u8; 0]>(&[], &[]);
test::<[u8; 1]>(&[255], &[255]);
test::<[u8; 2]>(&[111, 0], &[111, 0]);
test::<[u8; 3]>(&[1, 0, 1], &[1, 0, 1]);
}
#[test]
/// Assert `redb::Key::from_bytes` is accurate.
fn from_bytes() {
fn test<T>(bytes: &[u8], expected: &T)
where
T: Storable + PartialEq + 'static,
{
println!("bytes: {bytes:?}, expected: {expected:?}");
assert_eq!(
&<StorableRedb::<T> as redb::Value>::from_bytes(bytes),
expected
);
}
test::<()>([].as_slice(), &());
test::<u8>([0].as_slice(), &0);
test::<u16>([1, 0].as_slice(), &1);
test::<u32>([2, 0, 0, 0].as_slice(), &2);
test::<u64>([3, 0, 0, 0, 0, 0, 0, 0].as_slice(), &3);
test::<i8>([255].as_slice(), &-1);
test::<i16>([254, 255].as_slice(), &-2);
test::<i32>([253, 255, 255, 255].as_slice(), &-3);
test::<i64>([252, 255, 255, 255, 255, 255, 255, 255].as_slice(), &-4);
test::<StorableVec<u8>>(&[1, 2], &StorableVec(vec![1, 2]));
test::<StorableBytes>(&[1, 2], &StorableBytes(bytes::Bytes::from_static(&[1, 2])));
test::<[u8; 0]>([].as_slice(), &[]);
test::<[u8; 1]>([255].as_slice(), &[255]);
test::<[u8; 2]>([111, 0].as_slice(), &[111, 0]);
test::<[u8; 3]>([1, 0, 1].as_slice(), &[1, 0, 1]);
}
#[test]
/// Assert `redb::Key::type_name` returns unique names.
/// The name itself isn't tested, the invariant is that
/// they are all unique.
fn type_name() {
// Can't use a proper set because `redb::TypeName: !Ord`.
let set = [
<StorableRedb<()> as redb::Value>::type_name(),
<StorableRedb<u8> as redb::Value>::type_name(),
<StorableRedb<u16> as redb::Value>::type_name(),
<StorableRedb<u32> as redb::Value>::type_name(),
<StorableRedb<u64> as redb::Value>::type_name(),
<StorableRedb<i8> as redb::Value>::type_name(),
<StorableRedb<i16> as redb::Value>::type_name(),
<StorableRedb<i32> as redb::Value>::type_name(),
<StorableRedb<i64> as redb::Value>::type_name(),
<StorableRedb<StorableVec<u8>> as redb::Value>::type_name(),
<StorableRedb<StorableBytes> as redb::Value>::type_name(),
<StorableRedb<[u8; 0]> as redb::Value>::type_name(),
<StorableRedb<[u8; 1]> as redb::Value>::type_name(),
<StorableRedb<[u8; 2]> as redb::Value>::type_name(),
<StorableRedb<[u8; 3]> as redb::Value>::type_name(),
];
// Check every permutation is unique.
for (index, i) in set.iter().enumerate() {
for (index2, j) in set.iter().enumerate() {
if index != index2 {
assert_ne!(i, j);
}
}
}
}
}

38
storage/cuprate-blockchain/src/backend/redb/transaction.rs
View file

@ -1,38 +0,0 @@
//! Implementation of `trait TxRo/TxRw` for `redb`.
//---------------------------------------------------------------------------------------------------- Import
use crate::{
error::RuntimeError,
transaction::{TxRo, TxRw},
};
//---------------------------------------------------------------------------------------------------- TxRo
impl TxRo<'_> for redb::ReadTransaction {
/// This function is infallible.
fn commit(self) -> Result<(), RuntimeError> {
// `redb`'s read transactions cleanup automatically when all references are dropped.
//
// There is `close()`:
// <https://docs.rs/redb/2.0.0/redb/struct.ReadTransaction.html#method.close>
// but this will error if there are outstanding references, i.e. an open table.
// This is unwanted behavior in our case, so we don't call this.
Ok(())
}
}
//---------------------------------------------------------------------------------------------------- TxRw
impl TxRw<'_> for redb::WriteTransaction {
fn commit(self) -> Result<(), RuntimeError> {
Ok(self.commit()?)
}
fn abort(self) -> Result<(), RuntimeError> {
Ok(self.abort()?)
}
}
//---------------------------------------------------------------------------------------------------- Tests
#[cfg(test)]
mod test {
// use super::*;
}

11
storage/cuprate-blockchain/src/backend/redb/types.rs
View file

@ -1,11 +0,0 @@
//! `redb` type aliases.
//---------------------------------------------------------------------------------------------------- Types
use crate::backend::redb::storable::StorableRedb;
//---------------------------------------------------------------------------------------------------- Types
/// The concrete type for readable `redb` tables.
pub(super) type RedbTableRo<K, V> = redb::ReadOnlyTable<StorableRedb<K>, StorableRedb<V>>;
/// The concrete type for readable/writable `redb` tables.
pub(super) type RedbTableRw<'tx, K, V> = redb::Table<'tx, StorableRedb<K>, StorableRedb<V>>;

550
storage/cuprate-blockchain/src/backend/tests.rs
View file

@ -1,550 +0,0 @@
//! Tests for `cuprate_blockchain`'s backends.
//!
//! These tests are fully trait-based, meaning there
//! is no reference to `backend/`-specific types.
//!
//! As such, which backend is tested is
//! dependant on the feature flags used.
//!
//! | Feature flag | Tested backend |
//! |---------------|----------------|
//! | Only `redb` | `redb`
//! | Anything else | `heed`
//!
//! `redb`, and it only must be enabled for it to be tested.
//---------------------------------------------------------------------------------------------------- Import
use crate::{
database::{DatabaseIter, DatabaseRo, DatabaseRw},
env::{Env, EnvInner},
error::RuntimeError,
resize::ResizeAlgorithm,
storable::StorableVec,
tables::{
BlockBlobs, BlockHeights, BlockInfos, KeyImages, NumOutputs, Outputs, PrunableHashes,
PrunableTxBlobs, PrunedTxBlobs, RctOutputs, TxBlobs, TxHeights, TxIds, TxOutputs,
TxUnlockTime,
},
tables::{TablesIter, TablesMut},
tests::tmp_concrete_env,
transaction::{TxRo, TxRw},
types::{
Amount, AmountIndex, AmountIndices, BlockBlob, BlockHash, BlockHeight, BlockInfo, KeyImage,
Output, OutputFlags, PreRctOutputId, PrunableBlob, PrunableHash, PrunedBlob, RctOutput,
TxBlob, TxHash, TxId, UnlockTime,
},
ConcreteEnv,
};
//---------------------------------------------------------------------------------------------------- Tests
/// Simply call [`Env::open`]. If this fails, something is really wrong.
#[test]
fn open() {
tmp_concrete_env();
}
/// Create database transactions, but don't write any data.
#[test]
fn tx() {
let (env, _tempdir) = tmp_concrete_env();
let env_inner = env.env_inner();
TxRo::commit(env_inner.tx_ro().unwrap()).unwrap();
TxRw::commit(env_inner.tx_rw().unwrap()).unwrap();
TxRw::abort(env_inner.tx_rw().unwrap()).unwrap();
}
/// Open (and verify) that all database tables
/// exist already after calling [`Env::open`].
#[test]
fn open_db() {
let (env, _tempdir) = tmp_concrete_env();
let env_inner = env.env_inner();
let tx_ro = env_inner.tx_ro().unwrap();
let tx_rw = env_inner.tx_rw().unwrap();
// Open all tables in read-only mode.
// This should be updated when tables are modified.
env_inner.open_db_ro::<BlockBlobs>(&tx_ro).unwrap();
env_inner.open_db_ro::<BlockHeights>(&tx_ro).unwrap();
env_inner.open_db_ro::<BlockInfos>(&tx_ro).unwrap();
env_inner.open_db_ro::<KeyImages>(&tx_ro).unwrap();
env_inner.open_db_ro::<NumOutputs>(&tx_ro).unwrap();
env_inner.open_db_ro::<Outputs>(&tx_ro).unwrap();
env_inner.open_db_ro::<PrunableHashes>(&tx_ro).unwrap();
env_inner.open_db_ro::<PrunableTxBlobs>(&tx_ro).unwrap();
env_inner.open_db_ro::<PrunedTxBlobs>(&tx_ro).unwrap();
env_inner.open_db_ro::<RctOutputs>(&tx_ro).unwrap();
env_inner.open_db_ro::<TxBlobs>(&tx_ro).unwrap();
env_inner.open_db_ro::<TxHeights>(&tx_ro).unwrap();
env_inner.open_db_ro::<TxIds>(&tx_ro).unwrap();
env_inner.open_db_ro::<TxOutputs>(&tx_ro).unwrap();
env_inner.open_db_ro::<TxUnlockTime>(&tx_ro).unwrap();
TxRo::commit(tx_ro).unwrap();
// Open all tables in read/write mode.
env_inner.open_db_rw::<BlockBlobs>(&tx_rw).unwrap();
env_inner.open_db_rw::<BlockHeights>(&tx_rw).unwrap();
env_inner.open_db_rw::<BlockInfos>(&tx_rw).unwrap();
env_inner.open_db_rw::<KeyImages>(&tx_rw).unwrap();
env_inner.open_db_rw::<NumOutputs>(&tx_rw).unwrap();
env_inner.open_db_rw::<Outputs>(&tx_rw).unwrap();
env_inner.open_db_rw::<PrunableHashes>(&tx_rw).unwrap();
env_inner.open_db_rw::<PrunableTxBlobs>(&tx_rw).unwrap();
env_inner.open_db_rw::<PrunedTxBlobs>(&tx_rw).unwrap();
env_inner.open_db_rw::<RctOutputs>(&tx_rw).unwrap();
env_inner.open_db_rw::<TxBlobs>(&tx_rw).unwrap();
env_inner.open_db_rw::<TxHeights>(&tx_rw).unwrap();
env_inner.open_db_rw::<TxIds>(&tx_rw).unwrap();
env_inner.open_db_rw::<TxOutputs>(&tx_rw).unwrap();
env_inner.open_db_rw::<TxUnlockTime>(&tx_rw).unwrap();
TxRw::commit(tx_rw).unwrap();
}
/// Test `Env` resizes.
#[test]
fn resize() {
// This test is only valid for `Env`'s that need to resize manually.
if !ConcreteEnv::MANUAL_RESIZE {
return;
}
let (env, _tempdir) = tmp_concrete_env();
// Resize by the OS page size.
let page_size = crate::resize::page_size();
let old_size = env.current_map_size();
env.resize_map(Some(ResizeAlgorithm::FixedBytes(page_size)));
// Assert it resized exactly by the OS page size.
let new_size = env.current_map_size();
assert_eq!(new_size, old_size + page_size.get());
}
/// Test that `Env`'s that don't manually resize.
#[test]
#[should_panic = "unreachable"]
fn non_manual_resize_1() {
if ConcreteEnv::MANUAL_RESIZE {
unreachable!();
} else {
let (env, _tempdir) = tmp_concrete_env();
env.resize_map(None);
}
}
#[test]
#[should_panic = "unreachable"]
fn non_manual_resize_2() {
if ConcreteEnv::MANUAL_RESIZE {
unreachable!();
} else {
let (env, _tempdir) = tmp_concrete_env();
env.current_map_size();
}
}
/// Test all `DatabaseR{o,w}` operations.
#[test]
fn db_read_write() {
let (env, _tempdir) = tmp_concrete_env();
let env_inner = env.env_inner();
let tx_rw = env_inner.tx_rw().unwrap();
let mut table = env_inner.open_db_rw::<Outputs>(&tx_rw).unwrap();
/// The (1st) key.
const KEY: PreRctOutputId = PreRctOutputId {
amount: 1,
amount_index: 123,
};
/// The expected value.
const VALUE: Output = Output {
key: [35; 32],
height: 45_761_798,
output_flags: OutputFlags::empty(),
tx_idx: 2_353_487,
};
/// How many `(key, value)` pairs will be inserted.
const N: u64 = 100;
/// Assert 2 `Output`'s are equal, and that accessing
/// their fields don't result in an unaligned panic.
fn assert_same(output: Output) {
assert_eq!(output, VALUE);
assert_eq!(output.key, VALUE.key);
assert_eq!(output.height, VALUE.height);
assert_eq!(output.output_flags, VALUE.output_flags);
assert_eq!(output.tx_idx, VALUE.tx_idx);
}
assert!(table.is_empty().unwrap());
// Insert keys.
let mut key = KEY;
for _ in 0..N {
table.put(&key, &VALUE).unwrap();
key.amount += 1;
}
assert_eq!(table.len().unwrap(), N);
// Assert the first/last `(key, value)`s are there.
{
assert!(table.contains(&KEY).unwrap());
let get: Output = table.get(&KEY).unwrap();
assert_same(get);
let first: Output = table.first().unwrap().1;
assert_same(first);
let last: Output = table.last().unwrap().1;
assert_same(last);
}
// Commit transactions, create new ones.
drop(table);
TxRw::commit(tx_rw).unwrap();
let tx_ro = env_inner.tx_ro().unwrap();
let table_ro = env_inner.open_db_ro::<Outputs>(&tx_ro).unwrap();
let tx_rw = env_inner.tx_rw().unwrap();
let mut table = env_inner.open_db_rw::<Outputs>(&tx_rw).unwrap();
// Assert the whole range is there.
{
let range = table_ro.get_range(..).unwrap();
let mut i = 0;
for result in range {
let value: Output = result.unwrap();
assert_same(value);
i += 1;
}
assert_eq!(i, N);
}
// `get_range()` tests.
let mut key = KEY;
key.amount += N;
let range = KEY..key;
// Assert count is correct.
assert_eq!(
N as usize,
table_ro.get_range(range.clone()).unwrap().count()
);
// Assert each returned value from the iterator is owned.
{
let mut iter = table_ro.get_range(range.clone()).unwrap();
let value: Output = iter.next().unwrap().unwrap(); // 1. take value out
drop(iter); // 2. drop the `impl Iterator + 'a`
assert_same(value); // 3. assert even without the iterator, the value is alive
}
// Assert each value is the same.
{
let mut iter = table_ro.get_range(range).unwrap();
for _ in 0..N {
let value: Output = iter.next().unwrap().unwrap();
assert_same(value);
}
}
// Assert `update()` works.
{
const HEIGHT: u32 = 999;
assert_ne!(table.get(&KEY).unwrap().height, HEIGHT);
table
.update(&KEY, |mut value| {
value.height = HEIGHT;
Some(value)
})
.unwrap();
assert_eq!(table.get(&KEY).unwrap().height, HEIGHT);
}
// Assert deleting works.
{
table.delete(&KEY).unwrap();
let value = table.get(&KEY);
assert!(!table.contains(&KEY).unwrap());
assert!(matches!(value, Err(RuntimeError::KeyNotFound)));
// Assert the other `(key, value)` pairs are still there.
let mut key = KEY;
key.amount += N - 1; // we used inclusive `0..N`
let value = table.get(&key).unwrap();
assert_same(value);
}
// Assert `take()` works.
{
let mut key = KEY;
key.amount += 1;
let value = table.take(&key).unwrap();
assert_eq!(value, VALUE);
let get = table.get(&KEY);
assert!(!table.contains(&key).unwrap());
assert!(matches!(get, Err(RuntimeError::KeyNotFound)));
// Assert the other `(key, value)` pairs are still there.
key.amount += 1;
let value = table.get(&key).unwrap();
assert_same(value);
}
drop(table);
TxRw::commit(tx_rw).unwrap();
// Assert `clear_db()` works.
{
let mut tx_rw = env_inner.tx_rw().unwrap();
env_inner.clear_db::<Outputs>(&mut tx_rw).unwrap();
let table = env_inner.open_db_rw::<Outputs>(&tx_rw).unwrap();
assert!(table.is_empty().unwrap());
for n in 0..N {
let mut key = KEY;
key.amount += n;
let value = table.get(&key);
assert!(matches!(value, Err(RuntimeError::KeyNotFound)));
assert!(!table.contains(&key).unwrap());
}
// Reader still sees old value.
assert!(!table_ro.is_empty().unwrap());
// Writer sees updated value (nothing).
assert!(table.is_empty().unwrap());
}
}
/// Assert that `key`'s in database tables are sorted in
/// an ordered B-Tree fashion, i.e. `min_value -> max_value`.
#[test]
fn tables_are_sorted() {
let (env, _tmp) = tmp_concrete_env();
let env_inner = env.env_inner();
let tx_rw = env_inner.tx_rw().unwrap();
let mut tables_mut = env_inner.open_tables_mut(&tx_rw).unwrap();
// Insert `{5, 4, 3, 2, 1, 0}`, assert each new
// number inserted is the minimum `first()` value.
for key in (0..6).rev() {
tables_mut.num_outputs_mut().put(&key, &123).unwrap();
let (first, _) = tables_mut.num_outputs_mut().first().unwrap();
assert_eq!(first, key);
}
drop(tables_mut);
TxRw::commit(tx_rw).unwrap();
let tx_rw = env_inner.tx_rw().unwrap();
// Assert iterators are ordered.
{
let tx_ro = env_inner.tx_ro().unwrap();
let tables = env_inner.open_tables(&tx_ro).unwrap();
let t = tables.num_outputs_iter();
let iter = t.iter().unwrap();
let keys = t.keys().unwrap();
for ((i, iter), key) in (0..6).zip(iter).zip(keys) {
let (iter, _) = iter.unwrap();
let key = key.unwrap();
assert_eq!(i, iter);
assert_eq!(iter, key);
}
}
let mut tables_mut = env_inner.open_tables_mut(&tx_rw).unwrap();
let t = tables_mut.num_outputs_mut();
// Assert the `first()` values are the minimum, i.e. `{0, 1, 2}`
for key in 0..3 {
let (first, _) = t.first().unwrap();
assert_eq!(first, key);
t.delete(&key).unwrap();
}
// Assert the `last()` values are the maximum, i.e. `{5, 4, 3}`
for key in (3..6).rev() {
let (last, _) = tables_mut.num_outputs_mut().last().unwrap();
assert_eq!(last, key);
tables_mut.num_outputs_mut().delete(&key).unwrap();
}
}
//---------------------------------------------------------------------------------------------------- Table Tests
/// Test multiple tables and their key + values.
///
/// Each one of these tests:
/// - Opens a specific table
/// - Essentially does the `db_read_write` test
macro_rules! test_tables {
($(
$table:ident, // Table type
$key_type:ty => // Key (type)
$value_type:ty, // Value (type)
$key:expr => // Key (the value)
$value:expr, // Value (the value)
)* $(,)?) => { paste::paste! { $(
// Test function's name is the table type in `snake_case`.
#[test]
fn [<$table:snake>]() {
// Open the database env and table.
let (env, _tempdir) = tmp_concrete_env();
let env_inner = env.env_inner();
let mut tx_rw = env_inner.tx_rw().unwrap();
let mut table = env_inner.open_db_rw::<$table>(&mut tx_rw).unwrap();
/// The expected key.
const KEY: $key_type = $key;
// The expected value.
let value: $value_type = $value;
// Assert a passed value is equal to the const value.
let assert_eq = |v: &$value_type| {
assert_eq!(v, &value);
};
// Insert the key.
table.put(&KEY, &value).unwrap();
// Assert key is there.
{
let value: $value_type = table.get(&KEY).unwrap();
assert_eq(&value);
}
assert!(table.contains(&KEY).unwrap());
assert_eq!(table.len().unwrap(), 1);
// Commit transactions, create new ones.
drop(table);
TxRw::commit(tx_rw).unwrap();
let mut tx_rw = env_inner.tx_rw().unwrap();
let tx_ro = env_inner.tx_ro().unwrap();
let mut table = env_inner.open_db_rw::<$table>(&tx_rw).unwrap();
let table_ro = env_inner.open_db_ro::<$table>(&tx_ro).unwrap();
// Assert `get_range()` works.
{
let range = KEY..;
assert_eq!(1, table_ro.get_range(range.clone()).unwrap().count());
let mut iter = table_ro.get_range(range).unwrap();
let value = iter.next().unwrap().unwrap();
assert_eq(&value);
}
// Assert deleting works.
{
table.delete(&KEY).unwrap();
let value = table.get(&KEY);
assert!(matches!(value, Err(RuntimeError::KeyNotFound)));
assert!(!table.contains(&KEY).unwrap());
assert_eq!(table.len().unwrap(), 0);
}
table.put(&KEY, &value).unwrap();
// Assert `clear_db()` works.
{
drop(table);
env_inner.clear_db::<$table>(&mut tx_rw).unwrap();
let table = env_inner.open_db_rw::<$table>(&mut tx_rw).unwrap();
let value = table.get(&KEY);
assert!(matches!(value, Err(RuntimeError::KeyNotFound)));
assert!(!table.contains(&KEY).unwrap());
assert_eq!(table.len().unwrap(), 0);
}
}
)*}};
}
// Notes:
// - Keep this sorted A-Z (by table name)
test_tables! {
BlockBlobs, // Table type
BlockHeight => BlockBlob, // Key type => Value type
123 => StorableVec(vec![1,2,3,4,5,6,7,8]), // Actual key => Actual value
BlockHeights,
BlockHash => BlockHeight,
[32; 32] => 123,
BlockInfos,
BlockHeight => BlockInfo,
123 => BlockInfo {
timestamp: 1,
cumulative_generated_coins: 123,
weight: 321,
cumulative_difficulty_low: 111,
cumulative_difficulty_high: 111,
block_hash: [54; 32],
cumulative_rct_outs: 2389,
long_term_weight: 2389,
},
KeyImages,
KeyImage => (),
[32; 32] => (),
NumOutputs,
Amount => AmountIndex,
123 => 123,
TxBlobs,
TxId => TxBlob,
123 => StorableVec(vec![1,2,3,4,5,6,7,8]),
TxIds,
TxHash => TxId,
[32; 32] => 123,
TxHeights,
TxId => BlockHeight,
123 => 123,
TxOutputs,
TxId => AmountIndices,
123 => StorableVec(vec![1,2,3,4,5,6,7,8]),
TxUnlockTime,
TxId => UnlockTime,
123 => 123,
Outputs,
PreRctOutputId => Output,
PreRctOutputId {
amount: 1,
amount_index: 2,
} => Output {
key: [1; 32],
height: 1,
output_flags: OutputFlags::empty(),
tx_idx: 3,
},
PrunedTxBlobs,
TxId => PrunedBlob,
123 => StorableVec(vec![1,2,3,4,5,6,7,8]),
PrunableTxBlobs,
TxId => PrunableBlob,
123 => StorableVec(vec![1,2,3,4,5,6,7,8]),
PrunableHashes,
TxId => PrunableHash,
123 => [32; 32],
RctOutputs,
AmountIndex => RctOutput,
123 => RctOutput {
key: [1; 32],
height: 1,
output_flags: OutputFlags::empty(),
tx_idx: 3,
commitment: [3; 32],
},
}

129
storage/cuprate-blockchain/src/config/config.rs
View file

@ -1,21 +1,15 @@
//! The main [`Config`] struct, holding all configurable values. //! The main [`Config`] struct, holding all configurable values.
//---------------------------------------------------------------------------------------------------- Import //---------------------------------------------------------------------------------------------------- Import
use std::{ use std::{borrow::Cow, path::Path};
borrow::Cow,
path::{Path, PathBuf},
};
#[cfg(feature = "serde")] #[cfg(feature = "serde")]
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use cuprate_database::{config::SyncMode, resize::ResizeAlgorithm};
use cuprate_helper::fs::cuprate_blockchain_dir; use cuprate_helper::fs::cuprate_blockchain_dir;
use crate::{ use crate::config::ReaderThreads;
config::{ReaderThreads, SyncMode},
constants::DATABASE_DATA_FILENAME,
resize::ResizeAlgorithm,
};
//---------------------------------------------------------------------------------------------------- ConfigBuilder //---------------------------------------------------------------------------------------------------- ConfigBuilder
/// Builder for [`Config`]. /// Builder for [`Config`].
@ -27,14 +21,11 @@ pub struct ConfigBuilder {
/// [`Config::db_directory`]. /// [`Config::db_directory`].
db_directory: Option<Cow<'static, Path>>, db_directory: Option<Cow<'static, Path>>,
/// [`Config::sync_mode`]. /// [`Config::cuprate_database_config`].
sync_mode: Option<SyncMode>, db_config: cuprate_database::config::ConfigBuilder,
/// [`Config::reader_threads`]. /// [`Config::reader_threads`].
reader_threads: Option<ReaderThreads>, reader_threads: Option<ReaderThreads>,
/// [`Config::resize_algorithm`].
resize_algorithm: Option<ResizeAlgorithm>,
} }
impl ConfigBuilder { impl ConfigBuilder {
@ -42,12 +33,13 @@ impl ConfigBuilder {
/// ///
/// [`ConfigBuilder::build`] can be called immediately /// [`ConfigBuilder::build`] can be called immediately
/// after this function to use default values. /// after this function to use default values.
pub const fn new() -> Self { pub fn new() -> Self {
Self { Self {
db_directory: None, db_directory: None,
sync_mode: None, db_config: cuprate_database::config::ConfigBuilder::new(Cow::Borrowed(
cuprate_blockchain_dir(),
)),
reader_threads: None, reader_threads: None,
resize_algorithm: None,
} }
} }
@ -65,26 +57,23 @@ impl ConfigBuilder {
.db_directory .db_directory
.unwrap_or_else(|| Cow::Borrowed(cuprate_blockchain_dir())); .unwrap_or_else(|| Cow::Borrowed(cuprate_blockchain_dir()));
// Add the database filename to the directory. let reader_threads = self.reader_threads.unwrap_or_default();
let db_file = { let db_config = self
let mut db_file = db_directory.to_path_buf(); .db_config
db_file.push(DATABASE_DATA_FILENAME); .db_directory(db_directory)
Cow::Owned(db_file) .reader_threads(reader_threads.as_threads())
}; .build();
Config { Config {
db_directory, db_config,
db_file, reader_threads,
sync_mode: self.sync_mode.unwrap_or_default(),
reader_threads: self.reader_threads.unwrap_or_default(),
resize_algorithm: self.resize_algorithm.unwrap_or_default(),
} }
} }
/// Set a custom database directory (and file) [`Path`]. /// Set a custom database directory (and file) [`Path`].
#[must_use] #[must_use]
pub fn db_directory(mut self, db_directory: PathBuf) -> Self { pub fn db_directory(mut self, db_directory: Cow<'static, Path>) -> Self {
self.db_directory = Some(Cow::Owned(db_directory)); self.db_directory = Some(db_directory);
self self
} }
@ -94,9 +83,12 @@ impl ConfigBuilder {
/// Good default for testing, and resource-available machines. /// Good default for testing, and resource-available machines.
#[must_use] #[must_use]
pub fn fast(mut self) -> Self { pub fn fast(mut self) -> Self {
self.sync_mode = Some(SyncMode::Fast); self.db_config =
cuprate_database::config::ConfigBuilder::new(Cow::Borrowed(cuprate_blockchain_dir()))
.sync_mode(SyncMode::Fast)
.resize_algorithm(ResizeAlgorithm::default());
self.reader_threads = Some(ReaderThreads::OnePerThread); self.reader_threads = Some(ReaderThreads::OnePerThread);
self.resize_algorithm = Some(ResizeAlgorithm::default());
self self
} }
@ -106,16 +98,12 @@ impl ConfigBuilder {
/// Good default for resource-limited machines, e.g. a cheap VPS. /// Good default for resource-limited machines, e.g. a cheap VPS.
#[must_use] #[must_use]
pub fn low_power(mut self) -> Self { pub fn low_power(mut self) -> Self {
self.sync_mode = Some(SyncMode::default()); self.db_config =
self.reader_threads = Some(ReaderThreads::One); cuprate_database::config::ConfigBuilder::new(Cow::Borrowed(cuprate_blockchain_dir()))
self.resize_algorithm = Some(ResizeAlgorithm::default()); .sync_mode(SyncMode::default())
self .resize_algorithm(ResizeAlgorithm::default());
}
/// Set a custom [`SyncMode`]. self.reader_threads = Some(ReaderThreads::One);
#[must_use]
pub const fn sync_mode(mut self, sync_mode: SyncMode) -> Self {
self.sync_mode = Some(sync_mode);
self self
} }
@ -125,66 +113,35 @@ impl ConfigBuilder {
self.reader_threads = Some(reader_threads); self.reader_threads = Some(reader_threads);
self self
} }
/// Set a custom [`ResizeAlgorithm`].
#[must_use]
pub const fn resize_algorithm(mut self, resize_algorithm: ResizeAlgorithm) -> Self {
self.resize_algorithm = Some(resize_algorithm);
self
}
} }
impl Default for ConfigBuilder { impl Default for ConfigBuilder {
fn default() -> Self { fn default() -> Self {
let db_directory = Cow::Borrowed(cuprate_blockchain_dir());
Self { Self {
db_directory: Some(Cow::Borrowed(cuprate_blockchain_dir())), db_directory: Some(db_directory.clone()),
sync_mode: Some(SyncMode::default()), db_config: cuprate_database::config::ConfigBuilder::new(db_directory),
reader_threads: Some(ReaderThreads::default()), reader_threads: Some(ReaderThreads::default()),
resize_algorithm: Some(ResizeAlgorithm::default()),
} }
} }
} }
//---------------------------------------------------------------------------------------------------- Config //---------------------------------------------------------------------------------------------------- Config
/// Database [`Env`](crate::Env) configuration. /// `cuprate_blockchain` configuration.
/// ///
/// This is the struct passed to [`Env::open`](crate::Env::open) that /// This is a configuration built on-top of [`cuprate_database::config::Config`].
/// allows the database to be configured in various ways. ///
/// It contains configuration specific to this crate, plus the database config.
/// ///
/// For construction, either use [`ConfigBuilder`] or [`Config::default`]. /// For construction, either use [`ConfigBuilder`] or [`Config::default`].
///
// SOMEDAY: there's are many more options to add in the future.
#[derive(Debug, Clone, PartialEq, PartialOrd)] #[derive(Debug, Clone, PartialEq, PartialOrd)]
#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
pub struct Config { pub struct Config {
//------------------------ Database PATHs /// The database configuration.
// These are private since we don't want pub db_config: cuprate_database::config::Config,
// users messing with them after construction.
/// The directory used to store all database files.
///
/// By default, if no value is provided in the [`Config`]
/// constructor functions, this will be [`cuprate_blockchain_dir`].
///
// SOMEDAY: we should also support `/etc/cuprated.conf`.
// This could be represented with an `enum DbPath { Default, Custom, Etc, }`
pub(crate) db_directory: Cow<'static, Path>,
/// The actual database data file.
///
/// This is private, and created from the above `db_directory`.
pub(crate) db_file: Cow<'static, Path>,
/// Disk synchronization mode.
pub sync_mode: SyncMode,
/// Database reader thread count. /// Database reader thread count.
pub reader_threads: ReaderThreads, pub reader_threads: ReaderThreads,
/// Database memory map resizing algorithm.
///
/// This is used as the default fallback, but
/// custom algorithms can be used as well with
/// [`Env::resize_map`](crate::Env::resize_map).
pub resize_algorithm: ResizeAlgorithm,
} }
impl Config { impl Config {
@ -212,16 +169,6 @@ impl Config {
pub fn new() -> Self { pub fn new() -> Self {
ConfigBuilder::default().build() ConfigBuilder::default().build()
} }
/// Return the absolute [`Path`] to the database directory.
pub const fn db_directory(&self) -> &Cow<'_, Path> {
&self.db_directory
}
/// Return the absolute [`Path`] to the database data file.
pub const fn db_file(&self) -> &Cow<'_, Path> {
&self.db_file
}
} }
impl Default for Config { impl Default for Config {

51
storage/cuprate-blockchain/src/constants.rs
View file

@ -30,57 +30,6 @@ TODO: instructions on:
3. General advice for preventing corruption 3. General advice for preventing corruption
4. etc"; 4. etc";
//---------------------------------------------------------------------------------------------------- Misc
/// Static string of the `crate` being used as the database backend.
///
/// | Backend | Value |
/// |---------|-------|
/// | `heed` | `"heed"`
/// | `redb` | `"redb"`
pub const DATABASE_BACKEND: &str = {
cfg_if! {
if #[cfg(all(feature = "redb", not(feature = "heed")))] {
"redb"
} else {
"heed"
}
}
};
/// Cuprate's database filename.
///
/// Used in [`Config::db_file`](crate::config::Config::db_file).
///
/// | Backend | Value |
/// |---------|-------|
/// | `heed` | `"data.mdb"`
/// | `redb` | `"data.redb"`
pub const DATABASE_DATA_FILENAME: &str = {
cfg_if! {
if #[cfg(all(feature = "redb", not(feature = "heed")))] {
"data.redb"
} else {
"data.mdb"
}
}
};
/// Cuprate's database lock filename.
///
/// | Backend | Value |
/// |---------|-------|
/// | `heed` | `Some("lock.mdb")`
/// | `redb` | `None` (redb doesn't use a file lock)
pub const DATABASE_LOCK_FILENAME: Option<&str> = {
cfg_if! {
if #[cfg(all(feature = "redb", not(feature = "heed")))] {
None
} else {
Some("lock.mdb")
}
}
};
//---------------------------------------------------------------------------------------------------- Tests //---------------------------------------------------------------------------------------------------- Tests
#[cfg(test)] #[cfg(test)]
mod test {} mod test {}

216
storage/cuprate-blockchain/src/database.rs
View file

@ -1,216 +0,0 @@
//! Abstracted database table operations; `trait DatabaseRo` & `trait DatabaseRw`.
//---------------------------------------------------------------------------------------------------- Import
use std::ops::RangeBounds;
use crate::{error::RuntimeError, table::Table};
//---------------------------------------------------------------------------------------------------- DatabaseIter
/// Generic post-fix documentation for `DatabaseIter` methods.
macro_rules! doc_iter {
() => {
r"Although the returned iterator itself is tied to the lifetime
of `&self`, the returned values from the iterator are _owned_.
# Errors
The construction of the iterator itself may error.
Each iteration of the iterator has the potential to error as well."
};
}
/// Database (key-value store) read-only iteration abstraction.
///
/// These are read-only iteration-related operations that
/// can only be called from [`DatabaseRo`] objects.
///
/// # Hack
/// This is a HACK to get around the fact [`DatabaseRw`] tables
/// cannot safely return values returning lifetimes, as such,
/// only read-only tables implement this trait.
///
/// - <https://github.com/Cuprate/cuprate/pull/102#discussion_r1548695610>
/// - <https://github.com/Cuprate/cuprate/pull/104>
pub trait DatabaseIter<T: Table> {
/// Get an [`Iterator`] of value's corresponding to a range of keys.
///
/// For example:
/// ```rust,ignore
/// // This will return all 100 values corresponding
/// // to the keys `{0, 1, 2, ..., 100}`.
/// self.get_range(0..100);
/// ```
///
/// Although the returned iterator itself is tied to the lifetime
/// of `&'a self`, the returned values from the iterator are _owned_.
///
#[doc = doc_iter!()]
fn get_range<'a, Range>(
&'a self,
range: Range,
) -> Result<impl Iterator<Item = Result<T::Value, RuntimeError>> + 'a, RuntimeError>
where
Range: RangeBounds<T::Key> + 'a;
/// Get an [`Iterator`] that returns the `(key, value)` types for this database.
#[doc = doc_iter!()]
#[allow(clippy::iter_not_returning_iterator)]
fn iter(
&self,
) -> Result<impl Iterator<Item = Result<(T::Key, T::Value), RuntimeError>> + '_, RuntimeError>;
/// Get an [`Iterator`] that returns _only_ the `key` type for this database.
#[doc = doc_iter!()]
fn keys(&self)
-> Result<impl Iterator<Item = Result<T::Key, RuntimeError>> + '_, RuntimeError>;
/// Get an [`Iterator`] that returns _only_ the `value` type for this database.
#[doc = doc_iter!()]
fn values(
&self,
) -> Result<impl Iterator<Item = Result<T::Value, RuntimeError>> + '_, RuntimeError>;
}
//---------------------------------------------------------------------------------------------------- DatabaseRo
/// Generic post-fix documentation for `DatabaseR{o,w}` methods.
macro_rules! doc_database {
() => {
r"# Errors
This will return [`RuntimeError::KeyNotFound`] if:
- Input does not exist OR
- Database is empty"
};
}
/// Database (key-value store) read abstraction.
///
/// This is a read-only database table,
/// write operations are defined in [`DatabaseRw`].
///
/// # Safety
/// The table type that implements this MUST be `Send`.
///
/// However if the table holds a reference to a transaction:
/// - only the transaction only has to be `Send`
/// - the table cannot implement `Send`
///
/// For example:
///
/// `heed`'s transactions are `Send` but `HeedTableRo` contains a `&`
/// to the transaction, as such, if `Send` were implemented on `HeedTableRo`
/// then 1 transaction could be used to open multiple tables, then sent to
/// other threads - this would be a soundness hole against `HeedTableRo`.
///
/// `&T` is only `Send` if `T: Sync`.
///
/// `heed::RoTxn: !Sync`, therefore our table
/// holding `&heed::RoTxn` must NOT be `Send`.
///
/// - <https://doc.rust-lang.org/std/marker/trait.Sync.html>
/// - <https://doc.rust-lang.org/nomicon/send-and-sync.html>
pub unsafe trait DatabaseRo<T: Table> {
/// Get the value corresponding to a key.
#[doc = doc_database!()]
fn get(&self, key: &T::Key) -> Result<T::Value, RuntimeError>;
/// Returns `true` if the database contains a value for the specified key.
///
/// # Errors
/// Note that this will _never_ return `Err(RuntimeError::KeyNotFound)`,
/// as in that case, `Ok(false)` will be returned.
///
/// Other errors may still occur.
fn contains(&self, key: &T::Key) -> Result<bool, RuntimeError> {
match self.get(key) {
Ok(_) => Ok(true),
Err(RuntimeError::KeyNotFound) => Ok(false),
Err(e) => Err(e),
}
}
/// Returns the number of `(key, value)` pairs in the database.
///
/// # Errors
/// This will never return [`RuntimeError::KeyNotFound`].
fn len(&self) -> Result<u64, RuntimeError>;
/// Returns the first `(key, value)` pair in the database.
#[doc = doc_database!()]
fn first(&self) -> Result<(T::Key, T::Value), RuntimeError>;
/// Returns the last `(key, value)` pair in the database.
#[doc = doc_database!()]
fn last(&self) -> Result<(T::Key, T::Value), RuntimeError>;
/// Returns `true` if the database contains no `(key, value)` pairs.
///
/// # Errors
/// This can only return [`RuntimeError::Io`] on errors.
fn is_empty(&self) -> Result<bool, RuntimeError>;
}
//---------------------------------------------------------------------------------------------------- DatabaseRw
/// Database (key-value store) read/write abstraction.
///
/// All [`DatabaseRo`] functions are also callable by [`DatabaseRw`].
pub trait DatabaseRw<T: Table>: DatabaseRo<T> {
/// Insert a key-value pair into the database.
///
/// This will overwrite any existing key-value pairs.
///
#[doc = doc_database!()]
///
/// This will never [`RuntimeError::KeyExists`].
fn put(&mut self, key: &T::Key, value: &T::Value) -> Result<(), RuntimeError>;
/// Delete a key-value pair in the database.
///
/// This will return `Ok(())` if the key does not exist.
///
#[doc = doc_database!()]
///
/// This will never [`RuntimeError::KeyExists`].
fn delete(&mut self, key: &T::Key) -> Result<(), RuntimeError>;
/// Delete and return a key-value pair in the database.
///
/// This is the same as [`DatabaseRw::delete`], however,
/// it will serialize the `T::Value` and return it.
///
#[doc = doc_database!()]
fn take(&mut self, key: &T::Key) -> Result<T::Value, RuntimeError>;
/// Fetch the value, and apply a function to it - or delete the entry.
///
/// This will call [`DatabaseRo::get`] and call your provided function `f` on it.
///
/// The [`Option`] `f` returns will dictate whether `update()`:
/// - Updates the current value OR
/// - Deletes the `(key, value)` pair
///
/// - If `f` returns `Some(value)`, that will be [`DatabaseRw::put`] as the new value
/// - If `f` returns `None`, the entry will be [`DatabaseRw::delete`]d
///
#[doc = doc_database!()]
fn update<F>(&mut self, key: &T::Key, mut f: F) -> Result<(), RuntimeError>
where
F: FnMut(T::Value) -> Option<T::Value>,
{
let value = DatabaseRo::get(self, key)?;
match f(value) {
Some(value) => DatabaseRw::put(self, key, &value),
None => DatabaseRw::delete(self, key),
}
}
/// Removes and returns the first `(key, value)` pair in the database.
///
#[doc = doc_database!()]
fn pop_first(&mut self) -> Result<(T::Key, T::Value), RuntimeError>;
/// Removes and returns the last `(key, value)` pair in the database.
///
#[doc = doc_database!()]
fn pop_last(&mut self) -> Result<(T::Key, T::Value), RuntimeError>;
}

286
storage/cuprate-blockchain/src/env.rs
View file

@ -1,286 +0,0 @@
//! Abstracted database environment; `trait Env`.
//---------------------------------------------------------------------------------------------------- Import
use std::num::NonZeroUsize;
use crate::{
config::Config,
database::{DatabaseIter, DatabaseRo, DatabaseRw},
error::{InitError, RuntimeError},
resize::ResizeAlgorithm,
table::Table,
tables::{call_fn_on_all_tables_or_early_return, TablesIter, TablesMut},
transaction::{TxRo, TxRw},
};
//---------------------------------------------------------------------------------------------------- Env
/// Database environment abstraction.
///
/// Essentially, the functions that can be called on [`ConcreteEnv`](crate::ConcreteEnv).
///
/// # `Drop`
/// Objects that implement [`Env`] _should_ probably
/// [`Env::sync`] in their drop implementations,
/// although, no invariant relies on this (yet).
///
/// # Lifetimes
/// The lifetimes associated with `Env` have a sequential flow:
/// 1. `ConcreteEnv`
/// 2. `'env`
/// 3. `'tx`
/// 4. `'db`
///
/// As in:
/// - open database tables only live as long as...
/// - transactions which only live as long as the...
/// - environment ([`EnvInner`])
pub trait Env: Sized {
//------------------------------------------------ Constants
/// Does the database backend need to be manually
/// resized when the memory-map is full?
///
/// # Invariant
/// If this is `false`, that means this [`Env`]
/// must _never_ return a [`RuntimeError::ResizeNeeded`].
///
/// If this is `true`, [`Env::resize_map`] & [`Env::current_map_size`]
/// _must_ be re-implemented, as it just panics by default.
const MANUAL_RESIZE: bool;
/// Does the database backend forcefully sync/flush
/// to disk on every transaction commit?
///
/// This is used as an optimization.
const SYNCS_PER_TX: bool;
//------------------------------------------------ Types
/// The struct representing the actual backend's database environment.
///
/// This is used as the `self` in [`EnvInner`] functions, so whatever
/// this type is, is what will be accessible from those functions.
///
// # HACK
// For `heed`, this is just `heed::Env`, for `redb` this is
// `(redb::Database, redb::Durability)` as each transaction
// needs the sync mode set during creation.
type EnvInner<'env>: EnvInner<'env, Self::TxRo<'env>, Self::TxRw<'env>>
where
Self: 'env;
/// The read-only transaction type of the backend.
type TxRo<'env>: TxRo<'env> + 'env
where
Self: 'env;
/// The read/write transaction type of the backend.
type TxRw<'env>: TxRw<'env> + 'env
where
Self: 'env;
//------------------------------------------------ Required
/// Open the database environment, using the passed [`Config`].
///
/// # Invariants
/// This function **must** create all tables listed in [`crate::tables`].
///
/// The rest of the functions depend on the fact
/// they already exist, or else they will panic.
///
/// # Errors
/// This will error if the database could not be opened.
///
/// This is the only [`Env`] function that will return
/// an [`InitError`] instead of a [`RuntimeError`].
fn open(config: Config) -> Result<Self, InitError>;
/// Return the [`Config`] that this database was [`Env::open`]ed with.
fn config(&self) -> &Config;
/// Fully sync the database caches to disk.
///
/// # Invariant
/// This must **fully** and **synchronously** flush the database data to disk.
///
/// I.e., after this function returns, there must be no doubts
/// that the data isn't synced yet, it _must_ be synced.
///
// FIXME: either this invariant or `sync()` itself will most
// likely be removed/changed after `SyncMode` is finalized.
///
/// # Errors
/// If there is a synchronization error, this should return an error.
fn sync(&self) -> Result<(), RuntimeError>;
/// Resize the database's memory map to a
/// new (bigger) size using a [`ResizeAlgorithm`].
///
/// By default, this function will use the `ResizeAlgorithm` in [`Env::config`].
///
/// If `resize_algorithm` is `Some`, that will be used instead.
///
/// This function returns the _new_ memory map size in bytes.
///
/// # Invariant
/// This function _must_ be re-implemented if [`Env::MANUAL_RESIZE`] is `true`.
///
/// Otherwise, this function will panic with `unreachable!()`.
#[allow(unused_variables)]
fn resize_map(&self, resize_algorithm: Option<ResizeAlgorithm>) -> NonZeroUsize {
unreachable!()
}
/// What is the _current_ size of the database's memory map in bytes?
///
/// # Invariant
/// 1. This function _must_ be re-implemented if [`Env::MANUAL_RESIZE`] is `true`.
/// 2. This function must be accurate, as [`Env::resize_map()`] may depend on it.
fn current_map_size(&self) -> usize {
unreachable!()
}
/// Return the [`Env::EnvInner`].
///
/// # Locking behavior
/// When using the `heed` backend, [`Env::EnvInner`] is a
/// `RwLockReadGuard`, i.e., calling this function takes a
/// read lock on the `heed::Env`.
///
/// Be aware of this, as other functions (currently only
/// [`Env::resize_map`]) will take a _write_ lock.
fn env_inner(&self) -> Self::EnvInner<'_>;
//------------------------------------------------ Provided
/// Return the amount of actual of bytes the database is taking up on disk.
///
/// This is the current _disk_ value in bytes, not the memory map.
///
/// # Errors
/// This will error if either:
///
/// - [`std::fs::File::open`]
/// - [`std::fs::File::metadata`]
///
/// failed on the database file on disk.
fn disk_size_bytes(&self) -> std::io::Result<u64> {
// We have the direct PATH to the file,
// no need to use backend-specific functions.
//
// SAFETY: as we are only accessing the metadata of
// the file and not reading the bytes, it should be
// fine even with a memory mapped file being actively
// written to.
Ok(std::fs::File::open(&self.config().db_file)?
.metadata()?
.len())
}
}
//---------------------------------------------------------------------------------------------------- DatabaseRo
/// Document errors when opening tables in [`EnvInner`].
macro_rules! doc_table_error {
() => {
r"# Errors
This will only return [`RuntimeError::Io`] if it errors.
As all tables are created upon [`Env::open`],
this function will never error because a table doesn't exist."
};
}
/// The inner [`Env`] type.
///
/// This type is created with [`Env::env_inner`] and represents
/// the type able to generate transactions and open tables.
///
/// # Locking behavior
/// As noted in `Env::env_inner`, this is a `RwLockReadGuard`
/// when using the `heed` backend, be aware of this and do
/// not hold onto an `EnvInner` for a long time.
pub trait EnvInner<'env, Ro, Rw>
where
Self: 'env,
Ro: TxRo<'env>,
Rw: TxRw<'env>,
{
/// Create a read-only transaction.
///
/// # Errors
/// This will only return [`RuntimeError::Io`] if it errors.
fn tx_ro(&'env self) -> Result<Ro, RuntimeError>;
/// Create a read/write transaction.
///
/// # Errors
/// This will only return [`RuntimeError::Io`] if it errors.
fn tx_rw(&'env self) -> Result<Rw, RuntimeError>;
/// Open a database in read-only mode.
///
/// The returned value can have [`DatabaseRo`]
/// & [`DatabaseIter`] functions called on it.
///
/// This will open the database [`Table`]
/// passed as a generic to this function.
///
/// ```rust,ignore
/// let db = env.open_db_ro::<Table>(&tx_ro);
/// // ^ ^
/// // database table table metadata
/// // (name, key/value type)
/// ```
///
#[doc = doc_table_error!()]
fn open_db_ro<T: Table>(
&self,
tx_ro: &Ro,
) -> Result<impl DatabaseRo<T> + DatabaseIter<T>, RuntimeError>;
/// Open a database in read/write mode.
///
/// All [`DatabaseRo`] functions are also callable
/// with the returned [`DatabaseRw`] structure.
///
/// Note that [`DatabaseIter`] functions are _not_
/// available to [`DatabaseRw`] structures.
///
/// This will open the database [`Table`]
/// passed as a generic to this function.
///
#[doc = doc_table_error!()]
fn open_db_rw<T: Table>(&self, tx_rw: &Rw) -> Result<impl DatabaseRw<T>, RuntimeError>;
/// Open all tables in read/iter mode.
///
/// This calls [`EnvInner::open_db_ro`] on all database tables
/// and returns a structure that allows access to all tables.
///
#[doc = doc_table_error!()]
fn open_tables(&self, tx_ro: &Ro) -> Result<impl TablesIter, RuntimeError> {
call_fn_on_all_tables_or_early_return! {
Self::open_db_ro(self, tx_ro)
}
}
/// Open all tables in read-write mode.
///
/// This calls [`EnvInner::open_db_rw`] on all database tables
/// and returns a structure that allows access to all tables.
///
#[doc = doc_table_error!()]
fn open_tables_mut(&self, tx_rw: &Rw) -> Result<impl TablesMut, RuntimeError> {
call_fn_on_all_tables_or_early_return! {
Self::open_db_rw(self, tx_rw)
}
}
/// Clear all `(key, value)`'s from a database table.
///
/// This will delete all key and values in the passed
/// `T: Table`, but the table itself will continue to exist.
///
/// Note that this operation is tied to `tx_rw`, as such this
/// function's effects can be aborted using [`TxRw::abort`].
///
#[doc = doc_table_error!()]
fn clear_db<T: Table>(&self, tx_rw: &mut Rw) -> Result<(), RuntimeError>;
}

94
storage/cuprate-blockchain/src/error.rs
View file

@ -1,94 +0,0 @@
//! Database error types.
//---------------------------------------------------------------------------------------------------- Import
use std::fmt::Debug;
//---------------------------------------------------------------------------------------------------- Types
/// Alias for a thread-safe boxed error.
type BoxError = Box<dyn std::error::Error + Send + Sync + 'static>;
//---------------------------------------------------------------------------------------------------- InitError
/// Errors that occur during ([`Env::open`](crate::env::Env::open)).
///
/// # Handling
/// As this is a database initialization error, the correct
/// way to handle any of these occurring is probably just to
/// exit the program.
///
/// There is not much we as Cuprate can do
/// to recover from any of these errors.
#[derive(thiserror::Error, Debug)]
pub enum InitError {
/// The given `Path/File` existed and was accessible,
/// but was not a valid database file.
#[error("database file exists but is not valid")]
Invalid,
/// The given `Path/File` existed, was a valid
/// database, but the version is incorrect.
#[error("database file is valid, but version is incorrect")]
InvalidVersion,
/// I/O error.
#[error("database I/O error: {0}")]
Io(#[from] std::io::Error),
/// The given `Path/File` existed,
/// was a valid database, but it is corrupt.
#[error("database file is corrupt")]
Corrupt,
/// The database is currently in the process
/// of shutting down and cannot respond.
///
/// # Notes
/// This error can only occur with the `heed` backend when
/// the database environment is opened _right_ at the same time
/// another thread/process is closing it.
///
/// This will never occur with other backends.
#[error("database is shutting down")]
ShuttingDown,
/// An unknown error occurred.
///
/// This is for errors that cannot be recovered from,
/// but we'd still like to panic gracefully.
#[error("unknown error: {0}")]
Unknown(BoxError),
}
//---------------------------------------------------------------------------------------------------- RuntimeError
/// Errors that occur _after_ successful ([`Env::open`](crate::env::Env::open)).
///
/// There are no errors for:
/// 1. Missing tables
/// 2. (De)serialization
/// 3. Shutdown errors
///
/// as `cuprate_blockchain` upholds the invariant that:
///
/// 1. All tables exist
/// 2. (De)serialization never fails
/// 3. The database (thread-pool) only shuts down when all channels are dropped
#[derive(thiserror::Error, Debug)]
pub enum RuntimeError {
/// The given key already existed in the database.
#[error("key already existed")]
KeyExists,
/// The given key did not exist in the database.
#[error("key/value pair was not found")]
KeyNotFound,
/// The database memory map is full and needs a resize.
///
/// # Invariant
/// This error can only occur if [`Env::MANUAL_RESIZE`](crate::Env::MANUAL_RESIZE) is `true`.
#[error("database memory map must be resized")]
ResizeNeeded,
/// A [`std::io::Error`].
#[error("I/O error: {0}")]
Io(#[from] std::io::Error),
}

58
storage/cuprate-blockchain/src/key.rs
View file

@ -1,58 +0,0 @@
//! Database key abstraction; `trait Key`.
//---------------------------------------------------------------------------------------------------- Import
use std::cmp::Ordering;
use crate::storable::Storable;
//---------------------------------------------------------------------------------------------------- Table
/// Database [`Table`](crate::table::Table) key metadata.
///
/// Purely compile time information for database table keys.
//
// FIXME: this doesn't need to exist right now but
// may be used if we implement getting values using ranges.
// <https://github.com/Cuprate/cuprate/pull/117#discussion_r1589378104>
pub trait Key: Storable + Sized {
/// The primary key type.
type Primary: Storable;
/// Compare 2 [`Key`]'s against each other.
///
/// By default, this does a straight _byte_ comparison,
/// not a comparison of the key's value.
///
/// ```rust
/// # use cuprate_blockchain::*;
/// assert_eq!(
/// <u64 as Key>::compare([0].as_slice(), [1].as_slice()),
/// std::cmp::Ordering::Less,
/// );
/// assert_eq!(
/// <u64 as Key>::compare([1].as_slice(), [1].as_slice()),
/// std::cmp::Ordering::Equal,
/// );
/// assert_eq!(
/// <u64 as Key>::compare([2].as_slice(), [1].as_slice()),
/// std::cmp::Ordering::Greater,
/// );
/// ```
#[inline]
fn compare(left: &[u8], right: &[u8]) -> Ordering {
left.cmp(right)
}
}
//---------------------------------------------------------------------------------------------------- Impl
impl<T> Key for T
where
T: Storable + Sized,
{
type Primary = Self;
}
//---------------------------------------------------------------------------------------------------- Tests
#[cfg(test)]
mod test {
// use super::*;
}

37
storage/cuprate-blockchain/src/lib.rs
View file

@ -247,48 +247,19 @@ compile_error!("Cuprate is only compatible with 64-bit CPUs");
// //
// Documentation for each module is located in the respective file. // Documentation for each module is located in the respective file.
mod backend;
pub use backend::ConcreteEnv;
pub mod config; pub mod config;
mod constants; mod constants;
pub use constants::{ pub use constants::{DATABASE_CORRUPT_MSG, DATABASE_VERSION};
DATABASE_BACKEND, DATABASE_CORRUPT_MSG, DATABASE_DATA_FILENAME, DATABASE_LOCK_FILENAME,
DATABASE_VERSION,
};
mod database; mod open_tables;
pub use database::{DatabaseIter, DatabaseRo, DatabaseRw}; pub use open_tables::OpenTables;
mod env;
pub use env::{Env, EnvInner};
mod error;
pub use error::{InitError, RuntimeError};
pub(crate) mod free;
pub mod resize;
mod key;
pub use key::Key;
mod storable;
pub use storable::{Storable, StorableBytes, StorableVec};
pub mod free;
pub mod ops; pub mod ops;
mod table;
pub use table::Table;
pub mod tables; pub mod tables;
pub mod types; pub mod types;
mod transaction;
pub use transaction::{TxRo, TxRw};
//---------------------------------------------------------------------------------------------------- Feature-gated //---------------------------------------------------------------------------------------------------- Feature-gated
#[cfg(feature = "service")] #[cfg(feature = "service")]
pub mod service; pub mod service;

89
storage/cuprate-blockchain/src/open_tables.rs Normal file
View file

@ -0,0 +1,89 @@
//! TODO
//---------------------------------------------------------------------------------------------------- Import
use cuprate_database::{EnvInner, RuntimeError, TxRo, TxRw};
use crate::tables::{TablesIter, TablesMut};
//---------------------------------------------------------------------------------------------------- Table function macro
/// `crate`-private macro for callings functions on all tables.
///
/// This calls the function `$fn` with the optional
/// arguments `$args` on all tables - returning early
/// (within whatever scope this is called) if any
/// of the function calls error.
///
/// Else, it evaluates to an `Ok((tuple, of, all, table, types, ...))`,
/// i.e., an `impl Table[Mut]` wrapped in `Ok`.
macro_rules! call_fn_on_all_tables_or_early_return {
(
$($fn:ident $(::)?)*
(
$($arg:ident),* $(,)?
)
) => {{
Ok((
$($fn ::)*<$crate::tables::BlockInfos>($($arg),*)?,
$($fn ::)*<$crate::tables::BlockBlobs>($($arg),*)?,
$($fn ::)*<$crate::tables::BlockHeights>($($arg),*)?,
$($fn ::)*<$crate::tables::KeyImages>($($arg),*)?,
$($fn ::)*<$crate::tables::NumOutputs>($($arg),*)?,
$($fn ::)*<$crate::tables::PrunedTxBlobs>($($arg),*)?,
$($fn ::)*<$crate::tables::PrunableHashes>($($arg),*)?,
$($fn ::)*<$crate::tables::Outputs>($($arg),*)?,
$($fn ::)*<$crate::tables::PrunableTxBlobs>($($arg),*)?,
$($fn ::)*<$crate::tables::RctOutputs>($($arg),*)?,
$($fn ::)*<$crate::tables::TxBlobs>($($arg),*)?,
$($fn ::)*<$crate::tables::TxIds>($($arg),*)?,
$($fn ::)*<$crate::tables::TxHeights>($($arg),*)?,
$($fn ::)*<$crate::tables::TxOutputs>($($arg),*)?,
$($fn ::)*<$crate::tables::TxUnlockTime>($($arg),*)?,
))
}};
}
//---------------------------------------------------------------------------------------------------- OpenTables
/// TODO
pub trait OpenTables<'env, Ro, Rw>
where
Self: 'env,
Ro: TxRo<'env>,
Rw: TxRw<'env>,
{
/// TODO
///
/// # Errors
/// TODO
fn open_tables(&'env self, tx_ro: &Ro) -> Result<impl TablesIter, RuntimeError>;
/// TODO
///
/// # Errors
/// TODO
fn open_tables_mut(&'env self, tx_rw: &Rw) -> Result<impl TablesMut, RuntimeError>;
}
impl<'env, Ei, Ro, Rw> OpenTables<'env, Ro, Rw> for Ei
where
Ei: EnvInner<'env, Ro, Rw>,
Ro: TxRo<'env>,
Rw: TxRw<'env>,
{
fn open_tables(&'env self, tx_ro: &Ro) -> Result<impl TablesIter, RuntimeError> {
call_fn_on_all_tables_or_early_return! {
Self::open_db_ro(self, tx_ro)
}
}
fn open_tables_mut(&'env self, tx_rw: &Rw) -> Result<impl TablesMut, RuntimeError> {
call_fn_on_all_tables_or_early_return! {
Self::open_db_rw(self, tx_rw)
}
}
}
//---------------------------------------------------------------------------------------------------- Tests
#[cfg(test)]
mod test {
// use super::*;
}

11
storage/cuprate-blockchain/src/ops/block.rs
View file

@ -4,12 +4,13 @@
use bytemuck::TransparentWrapper; use bytemuck::TransparentWrapper;
use monero_serai::block::Block; use monero_serai::block::Block;
use cuprate_database::{
RuntimeError, StorableVec, {DatabaseRo, DatabaseRw},
};
use cuprate_helper::map::{combine_low_high_bits_to_u128, split_u128_into_low_high_bits}; use cuprate_helper::map::{combine_low_high_bits_to_u128, split_u128_into_low_high_bits};
use cuprate_types::{ExtendedBlockHeader, VerifiedBlockInformation}; use cuprate_types::{ExtendedBlockHeader, VerifiedBlockInformation};
use crate::{ use crate::{
database::{DatabaseRo, DatabaseRw},
error::RuntimeError,
ops::{ ops::{
blockchain::{chain_height, cumulative_generated_coins}, blockchain::{chain_height, cumulative_generated_coins},
macros::doc_error, macros::doc_error,
@ -18,7 +19,6 @@ use crate::{
}, },
tables::{BlockHeights, BlockInfos, Tables, TablesMut}, tables::{BlockHeights, BlockInfos, Tables, TablesMut},
types::{BlockHash, BlockHeight, BlockInfo}, types::{BlockHash, BlockHeight, BlockInfo},
StorableVec,
}; };
//---------------------------------------------------------------------------------------------------- `add_block_*` //---------------------------------------------------------------------------------------------------- `add_block_*`
@ -265,14 +265,15 @@ pub fn block_exists(
mod test { mod test {
use pretty_assertions::assert_eq; use pretty_assertions::assert_eq;
use cuprate_database::{Env, EnvInner, TxRw};
use cuprate_test_utils::data::{block_v16_tx0, block_v1_tx2, block_v9_tx3}; use cuprate_test_utils::data::{block_v16_tx0, block_v1_tx2, block_v9_tx3};
use super::*; use super::*;
use crate::{ use crate::{
open_tables::OpenTables,
ops::tx::{get_tx, tx_exists}, ops::tx::{get_tx, tx_exists},
tests::{assert_all_tables_are_empty, tmp_concrete_env, AssertTableLen}, tests::{assert_all_tables_are_empty, tmp_concrete_env, AssertTableLen},
transaction::TxRw,
Env, EnvInner,
}; };
/// Tests all above block functions. /// Tests all above block functions.

9
storage/cuprate-blockchain/src/ops/blockchain.rs
View file

@ -1,9 +1,9 @@
//! Blockchain functions - chain height, generated coins, etc. //! Blockchain functions - chain height, generated coins, etc.
//---------------------------------------------------------------------------------------------------- Import //---------------------------------------------------------------------------------------------------- Import
use cuprate_database::{DatabaseRo, RuntimeError};
use crate::{ use crate::{
database::DatabaseRo,
error::RuntimeError,
ops::macros::doc_error, ops::macros::doc_error,
tables::{BlockHeights, BlockInfos}, tables::{BlockHeights, BlockInfos},
types::BlockHeight, types::BlockHeight,
@ -81,15 +81,16 @@ pub fn cumulative_generated_coins(
mod test { mod test {
use pretty_assertions::assert_eq; use pretty_assertions::assert_eq;
use cuprate_database::{Env, EnvInner, TxRw};
use cuprate_test_utils::data::{block_v16_tx0, block_v1_tx2, block_v9_tx3}; use cuprate_test_utils::data::{block_v16_tx0, block_v1_tx2, block_v9_tx3};
use super::*; use super::*;
use crate::{ use crate::{
open_tables::OpenTables,
ops::block::add_block, ops::block::add_block,
tables::Tables, tables::Tables,
tests::{assert_all_tables_are_empty, tmp_concrete_env, AssertTableLen}, tests::{assert_all_tables_are_empty, tmp_concrete_env, AssertTableLen},
transaction::TxRw,
Env, EnvInner,
}; };
/// Tests all above functions. /// Tests all above functions.

10
storage/cuprate-blockchain/src/ops/key_image.rs
View file

@ -1,9 +1,9 @@
//! Key image functions. //! Key image functions.
//---------------------------------------------------------------------------------------------------- Import //---------------------------------------------------------------------------------------------------- Import
use cuprate_database::{DatabaseRo, DatabaseRw, RuntimeError};
use crate::{ use crate::{
database::{DatabaseRo, DatabaseRw},
error::RuntimeError,
ops::macros::{doc_add_block_inner_invariant, doc_error}, ops::macros::{doc_add_block_inner_invariant, doc_error},
tables::KeyImages, tables::KeyImages,
types::KeyImage, types::KeyImage,
@ -47,12 +47,14 @@ pub fn key_image_exists(
mod test { mod test {
use hex_literal::hex; use hex_literal::hex;
use cuprate_database::{Env, EnvInner, TxRw};
use super::*; use super::*;
use crate::{ use crate::{
open_tables::OpenTables,
tables::{Tables, TablesMut}, tables::{Tables, TablesMut},
tests::{assert_all_tables_are_empty, tmp_concrete_env, AssertTableLen}, tests::{assert_all_tables_are_empty, tmp_concrete_env, AssertTableLen},
transaction::TxRw,
Env, EnvInner,
}; };
/// Tests all above key-image functions. /// Tests all above key-image functions.

14
storage/cuprate-blockchain/src/ops/output.rs
View file

@ -4,12 +4,13 @@
use curve25519_dalek::{constants::ED25519_BASEPOINT_POINT, edwards::CompressedEdwardsY, Scalar}; use curve25519_dalek::{constants::ED25519_BASEPOINT_POINT, edwards::CompressedEdwardsY, Scalar};
use monero_serai::{transaction::Timelock, H}; use monero_serai::{transaction::Timelock, H};
use cuprate_database::{
RuntimeError, {DatabaseRo, DatabaseRw},
};
use cuprate_helper::map::u64_to_timelock; use cuprate_helper::map::u64_to_timelock;
use cuprate_types::OutputOnChain; use cuprate_types::OutputOnChain;
use crate::{ use crate::{
database::{DatabaseRo, DatabaseRw},
error::RuntimeError,
ops::macros::{doc_add_block_inner_invariant, doc_error}, ops::macros::{doc_add_block_inner_invariant, doc_error},
tables::{Outputs, RctOutputs, Tables, TablesMut, TxUnlockTime}, tables::{Outputs, RctOutputs, Tables, TablesMut, TxUnlockTime},
types::{Amount, AmountIndex, Output, OutputFlags, PreRctOutputId, RctOutput}, types::{Amount, AmountIndex, Output, OutputFlags, PreRctOutputId, RctOutput},
@ -247,15 +248,18 @@ pub fn id_to_output_on_chain(
#[cfg(test)] #[cfg(test)]
mod test { mod test {
use super::*; use super::*;
use pretty_assertions::assert_eq;
use cuprate_database::{Env, EnvInner};
use crate::{ use crate::{
open_tables::OpenTables,
tables::{Tables, TablesMut}, tables::{Tables, TablesMut},
tests::{assert_all_tables_are_empty, tmp_concrete_env, AssertTableLen}, tests::{assert_all_tables_are_empty, tmp_concrete_env, AssertTableLen},
types::OutputFlags, types::OutputFlags,
Env, EnvInner,
}; };
use pretty_assertions::assert_eq;
/// Dummy `Output`. /// Dummy `Output`.
const OUTPUT: Output = Output { const OUTPUT: Output = Output {
key: [44; 32], key: [44; 32],

5
storage/cuprate-blockchain/src/ops/property.rs
View file

@ -5,7 +5,10 @@
//---------------------------------------------------------------------------------------------------- Import //---------------------------------------------------------------------------------------------------- Import
use monero_pruning::PruningSeed; use monero_pruning::PruningSeed;
use crate::{error::RuntimeError, ops::macros::doc_error}; use cuprate_database::RuntimeError;
use crate::ops::macros::doc_error;
//---------------------------------------------------------------------------------------------------- Free Functions //---------------------------------------------------------------------------------------------------- Free Functions
/// SOMEDAY /// SOMEDAY
/// ///

16
storage/cuprate-blockchain/src/ops/tx.rs
View file

@ -5,9 +5,9 @@ use bytemuck::TransparentWrapper;
use curve25519_dalek::{constants::ED25519_BASEPOINT_POINT, Scalar}; use curve25519_dalek::{constants::ED25519_BASEPOINT_POINT, Scalar};
use monero_serai::transaction::{Input, Timelock, Transaction}; use monero_serai::transaction::{Input, Timelock, Transaction};
use cuprate_database::{DatabaseRo, DatabaseRw, RuntimeError, StorableVec};
use crate::{ use crate::{
database::{DatabaseRo, DatabaseRw},
error::RuntimeError,
ops::{ ops::{
key_image::{add_key_image, remove_key_image}, key_image::{add_key_image, remove_key_image},
macros::{doc_add_block_inner_invariant, doc_error}, macros::{doc_add_block_inner_invariant, doc_error},
@ -17,7 +17,6 @@ use crate::{
}, },
tables::{TablesMut, TxBlobs, TxIds}, tables::{TablesMut, TxBlobs, TxIds},
types::{BlockHeight, Output, OutputFlags, PreRctOutputId, RctOutput, TxHash, TxId}, types::{BlockHeight, Output, OutputFlags, PreRctOutputId, RctOutput, TxHash, TxId},
StorableVec,
}; };
//---------------------------------------------------------------------------------------------------- Private //---------------------------------------------------------------------------------------------------- Private
@ -325,14 +324,17 @@ pub fn tx_exists(
#[cfg(test)] #[cfg(test)]
mod test { mod test {
use super::*; use super::*;
use pretty_assertions::assert_eq;
use cuprate_database::{Env, EnvInner, TxRw};
use cuprate_test_utils::data::{tx_v1_sig0, tx_v1_sig2, tx_v2_rct3};
use crate::{ use crate::{
open_tables::OpenTables,
tables::Tables, tables::Tables,
tests::{assert_all_tables_are_empty, tmp_concrete_env, AssertTableLen}, tests::{assert_all_tables_are_empty, tmp_concrete_env, AssertTableLen},
transaction::TxRw,
Env, EnvInner,
}; };
use cuprate_test_utils::data::{tx_v1_sig0, tx_v1_sig2, tx_v2_rct3};
use pretty_assertions::assert_eq;
/// Tests all above tx functions when only inputting `Transaction` data (no Block). /// Tests all above tx functions when only inputting `Transaction` data (no Block).
#[test] #[test]

307
storage/cuprate-blockchain/src/resize.rs
View file

@ -1,307 +0,0 @@
//! Database memory map resizing algorithms.
//!
//! This modules contains [`ResizeAlgorithm`] which determines how the
//! [`ConcreteEnv`](crate::ConcreteEnv) resizes its memory map when needing more space.
//! This value is in [`Config`](crate::config::Config) and can be selected at runtime.
//!
//! Although, it is only used by `ConcreteEnv` if [`Env::MANUAL_RESIZE`](crate::env::Env::MANUAL_RESIZE) is `true`.
//!
//! The algorithms are available as free functions in this module as well.
//!
//! # Page size
//! All free functions in this module will
//! return a multiple of the OS page size ([`page_size()`]),
//! [LMDB will error](http://www.lmdb.tech/doc/group__mdb.html#gaa2506ec8dab3d969b0e609cd82e619e5)
//! if this is not the case.
//!
//! # Invariants
//! All returned [`NonZeroUsize`] values of the free functions in this module
//! (including [`ResizeAlgorithm::resize`]) uphold the following invariants:
//! 1. It will always be `>=` the input `current_size_bytes`
//! 2. It will always be a multiple of [`page_size()`]
//---------------------------------------------------------------------------------------------------- Import
use std::{num::NonZeroUsize, sync::OnceLock};
//---------------------------------------------------------------------------------------------------- ResizeAlgorithm
/// The function/algorithm used by the
/// database when resizing the memory map.
///
// # SOMEDAY
// We could test around with different algorithms.
// Calling `heed::Env::resize` is surprisingly fast,
// around `0.0000082s` on my machine. We could probably
// get away with smaller and more frequent resizes.
// **With the caveat being we are taking a `WriteGuard` to a `RwLock`.**
#[derive(Copy, Clone, Debug, PartialEq, PartialOrd)]
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
pub enum ResizeAlgorithm {
/// Uses [`monero`].
Monero,
/// Uses [`fixed_bytes`].
FixedBytes(NonZeroUsize),
/// Uses [`percent`].
Percent(f32),
}
impl ResizeAlgorithm {
/// Returns [`Self::Monero`].
///
/// ```rust
/// # use cuprate_blockchain::resize::*;
/// assert!(matches!(ResizeAlgorithm::new(), ResizeAlgorithm::Monero));
/// ```
#[inline]
pub const fn new() -> Self {
Self::Monero
}
/// Maps the `self` variant to the free functions in [`crate::resize`].
///
/// This function returns the _new_ memory map size in bytes.
#[inline]
pub fn resize(&self, current_size_bytes: usize) -> NonZeroUsize {
match self {
Self::Monero => monero(current_size_bytes),
Self::FixedBytes(add_bytes) => fixed_bytes(current_size_bytes, add_bytes.get()),
Self::Percent(f) => percent(current_size_bytes, *f),
}
}
}
impl Default for ResizeAlgorithm {
/// Calls [`Self::new`].
///
/// ```rust
/// # use cuprate_blockchain::resize::*;
/// assert_eq!(ResizeAlgorithm::new(), ResizeAlgorithm::default());
/// ```
#[inline]
fn default() -> Self {
Self::new()
}
}
//---------------------------------------------------------------------------------------------------- Free functions
/// This function retrieves the system’s memory page size.
///
/// It is just [`page_size::get`](https://docs.rs/page_size) internally.
///
/// This caches the result, so this function is cheap after the 1st call.
///
/// # Panics
/// This function will panic if the OS returns of page size of `0` (impossible?).
#[inline]
pub fn page_size() -> NonZeroUsize {
/// Cached result of [`page_size()`].
static PAGE_SIZE: OnceLock<NonZeroUsize> = OnceLock::new();
*PAGE_SIZE
.get_or_init(|| NonZeroUsize::new(page_size::get()).expect("page_size::get() returned 0"))
}
/// Memory map resize closely matching `monerod`.
///
/// # Method
/// This function mostly matches `monerod`'s current resize implementation[^1],
/// and will increase `current_size_bytes` by `1 << 30`[^2] exactly then
/// rounded to the nearest multiple of the OS page size.
///
/// [^1]: <https://github.com/monero-project/monero/blob/059028a30a8ae9752338a7897329fe8012a310d5/src/blockchain_db/lmdb/db_lmdb.cpp#L549>
///
/// [^2]: `1_073_745_920`
///
/// ```rust
/// # use cuprate_blockchain::resize::*;
/// // The value this function will increment by
/// // (assuming page multiple of 4096).
/// const N: usize = 1_073_741_824;
///
/// // 0 returns the minimum value.
/// assert_eq!(monero(0).get(), N);
///
/// // Rounds up to nearest OS page size.
/// assert_eq!(monero(1).get(), N + page_size().get());
/// ```
///
/// # Panics
/// This function will panic if adding onto `current_size_bytes` overflows [`usize::MAX`].
///
/// ```rust,should_panic
/// # use cuprate_blockchain::resize::*;
/// // Ridiculous large numbers panic.
/// monero(usize::MAX);
/// ```
pub fn monero(current_size_bytes: usize) -> NonZeroUsize {
/// The exact expression used by `monerod`
/// when calculating how many bytes to add.
///
/// The nominal value is `1_073_741_824`.
/// Not actually 1 GB but close enough I guess.
///
/// <https://github.com/monero-project/monero/blob/059028a30a8ae9752338a7897329fe8012a310d5/src/blockchain_db/lmdb/db_lmdb.cpp#L553>
const ADD_SIZE: usize = 1_usize << 30;
let page_size = page_size().get();
let new_size_bytes = current_size_bytes + ADD_SIZE;
// Round up the new size to the
// nearest multiple of the OS page size.
let remainder = new_size_bytes % page_size;
// INVARIANT: minimum is always at least `ADD_SIZE`.
NonZeroUsize::new(if remainder == 0 {
new_size_bytes
} else {
(new_size_bytes + page_size) - remainder
})
.unwrap()
}
/// Memory map resize by a fixed amount of bytes.
///
/// # Method
/// This function will `current_size_bytes + add_bytes`
/// and then round up to nearest OS page size.
///
/// ```rust
/// # use cuprate_blockchain::resize::*;
/// let page_size: usize = page_size().get();
///
/// // Anything below the page size will round up to the page size.
/// for i in 0..=page_size {
/// assert_eq!(fixed_bytes(0, i).get(), page_size);
/// }
///
/// // (page_size + 1) will round up to (page_size * 2).
/// assert_eq!(fixed_bytes(page_size, 1).get(), page_size * 2);
///
/// // (page_size + page_size) doesn't require any rounding.
/// assert_eq!(fixed_bytes(page_size, page_size).get(), page_size * 2);
/// ```
///
/// # Panics
/// This function will panic if adding onto `current_size_bytes` overflows [`usize::MAX`].
///
/// ```rust,should_panic
/// # use cuprate_blockchain::resize::*;
/// // Ridiculous large numbers panic.
/// fixed_bytes(1, usize::MAX);
/// ```
pub fn fixed_bytes(current_size_bytes: usize, add_bytes: usize) -> NonZeroUsize {
let page_size = page_size();
let new_size_bytes = current_size_bytes + add_bytes;
// Guard against < page_size.
if new_size_bytes <= page_size.get() {
return page_size;
}
// Round up the new size to the
// nearest multiple of the OS page size.
let remainder = new_size_bytes % page_size;
// INVARIANT: we guarded against < page_size above.
NonZeroUsize::new(if remainder == 0 {
new_size_bytes
} else {
(new_size_bytes + page_size.get()) - remainder
})
.unwrap()
}
/// Memory map resize by a percentage.
///
/// # Method
/// This function will multiply `current_size_bytes` by `percent`.
///
/// Any input `<= 1.0` or non-normal float ([`f32::NAN`], [`f32::INFINITY`])
/// will make the returning `NonZeroUsize` the same as `current_size_bytes`
/// (rounded up to the OS page size).
///
/// ```rust
/// # use cuprate_blockchain::resize::*;
/// let page_size: usize = page_size().get();
///
/// // Anything below the page size will round up to the page size.
/// for i in 0..=page_size {
/// assert_eq!(percent(i, 1.0).get(), page_size);
/// }
///
/// // Same for 2 page sizes.
/// for i in (page_size + 1)..=(page_size * 2) {
/// assert_eq!(percent(i, 1.0).get(), page_size * 2);
/// }
///
/// // Weird floats do nothing.
/// assert_eq!(percent(page_size, f32::NAN).get(), page_size);
/// assert_eq!(percent(page_size, f32::INFINITY).get(), page_size);
/// assert_eq!(percent(page_size, f32::NEG_INFINITY).get(), page_size);
/// assert_eq!(percent(page_size, -1.0).get(), page_size);
/// assert_eq!(percent(page_size, 0.999).get(), page_size);
/// ```
///
/// # Panics
/// This function will panic if `current_size_bytes * percent`
/// is closer to [`usize::MAX`] than the OS page size.
///
/// ```rust,should_panic
/// # use cuprate_blockchain::resize::*;
/// // Ridiculous large numbers panic.
/// percent(usize::MAX, 1.001);
/// ```
pub fn percent(current_size_bytes: usize, percent: f32) -> NonZeroUsize {
// Guard against bad floats.
use std::num::FpCategory;
let percent = match percent.classify() {
FpCategory::Normal => {
if percent <= 1.0 {
1.0
} else {
percent
}
}
_ => 1.0,
};
let page_size = page_size();
// INVARIANT: Allow `f32` <-> `usize` casting, we handle all cases.
#[allow(
clippy::cast_possible_truncation,
clippy::cast_sign_loss,
clippy::cast_precision_loss
)]
let new_size_bytes = ((current_size_bytes as f32) * percent) as usize;
// Panic if rounding up to the nearest page size would overflow.
let new_size_bytes = if new_size_bytes > (usize::MAX - page_size.get()) {
panic!("new_size_bytes is percent() near usize::MAX");
} else {
new_size_bytes
};
// Guard against < page_size.
if new_size_bytes <= page_size.get() {
return page_size;
}
// Round up the new size to the
// nearest multiple of the OS page size.
let remainder = new_size_bytes % page_size;
// INVARIANT: we guarded against < page_size above.
NonZeroUsize::new(if remainder == 0 {
new_size_bytes
} else {
(new_size_bytes + page_size.get()) - remainder
})
.unwrap()
}
//---------------------------------------------------------------------------------------------------- Tests
#[cfg(test)]
mod test {
// use super::*;
}

11
storage/cuprate-blockchain/src/service/free.rs
View file

@ -3,11 +3,11 @@
//---------------------------------------------------------------------------------------------------- Import //---------------------------------------------------------------------------------------------------- Import
use std::sync::Arc; use std::sync::Arc;
use cuprate_database::{ConcreteEnv, Env, InitError};
use crate::{ use crate::{
config::Config, config::Config,
error::InitError,
service::{DatabaseReadHandle, DatabaseWriteHandle}, service::{DatabaseReadHandle, DatabaseWriteHandle},
ConcreteEnv, Env,
}; };
//---------------------------------------------------------------------------------------------------- Init //---------------------------------------------------------------------------------------------------- Init
@ -21,10 +21,13 @@ use crate::{
/// # Errors /// # Errors
/// This will forward the error if [`Env::open`] failed. /// This will forward the error if [`Env::open`] failed.
pub fn init(config: Config) -> Result<(DatabaseReadHandle, DatabaseWriteHandle), InitError> { pub fn init(config: Config) -> Result<(DatabaseReadHandle, DatabaseWriteHandle), InitError> {
let reader_threads = config.reader_threads; let Config {
db_config,
reader_threads,
} = config;
// Initialize the database itself. // Initialize the database itself.
let db = Arc::new(ConcreteEnv::open(config)?); let db = Arc::new(ConcreteEnv::open(db_config)?);
// Spawn the Reader thread pool and Writer. // Spawn the Reader thread pool and Writer.
let readers = DatabaseReadHandle::init(&db, reader_threads); let readers = DatabaseReadHandle::init(&db, reader_threads);

6
storage/cuprate-blockchain/src/service/read.rs
View file

@ -13,6 +13,7 @@ use thread_local::ThreadLocal;
use tokio::sync::{OwnedSemaphorePermit, Semaphore}; use tokio::sync::{OwnedSemaphorePermit, Semaphore};
use tokio_util::sync::PollSemaphore; use tokio_util::sync::PollSemaphore;
use cuprate_database::{ConcreteEnv, DatabaseRo, Env, EnvInner, RuntimeError};
use cuprate_helper::asynch::InfallibleOneshotReceiver; use cuprate_helper::asynch::InfallibleOneshotReceiver;
use cuprate_types::{ use cuprate_types::{
blockchain::{BCReadRequest, BCResponse}, blockchain::{BCReadRequest, BCResponse},
@ -21,7 +22,7 @@ use cuprate_types::{
use crate::{ use crate::{
config::ReaderThreads, config::ReaderThreads,
error::RuntimeError, open_tables::OpenTables,
ops::block::block_exists, ops::block::block_exists,
ops::{ ops::{
block::{get_block_extended_header_from_height, get_block_info}, block::{get_block_extended_header_from_height, get_block_info},
@ -33,7 +34,6 @@ use crate::{
tables::{BlockHeights, BlockInfos, Tables}, tables::{BlockHeights, BlockInfos, Tables},
types::BlockHash, types::BlockHash,
types::{Amount, AmountIndex, BlockHeight, KeyImage, PreRctOutputId}, types::{Amount, AmountIndex, BlockHeight, KeyImage, PreRctOutputId},
ConcreteEnv, DatabaseRo, Env, EnvInner,
}; };
//---------------------------------------------------------------------------------------------------- DatabaseReadHandle //---------------------------------------------------------------------------------------------------- DatabaseReadHandle
@ -233,7 +233,7 @@ fn map_request(
/// <https://github.com/Cuprate/cuprate/pull/113#discussion_r1576762346> /// <https://github.com/Cuprate/cuprate/pull/113#discussion_r1576762346>
#[inline] #[inline]
fn thread_local<T: Send>(env: &impl Env) -> ThreadLocal<T> { fn thread_local<T: Send>(env: &impl Env) -> ThreadLocal<T> {
ThreadLocal::with_capacity(env.config().reader_threads.as_threads().get()) ThreadLocal::with_capacity(env.config().reader_threads.get())
} }
/// Take in a `ThreadLocal<impl Tables>` and return an `&impl Tables + Send`. /// Take in a `ThreadLocal<impl Tables>` and return an `&impl Tables + Send`.

6
storage/cuprate-blockchain/src/service/tests.rs
View file

@ -7,6 +7,7 @@
//---------------------------------------------------------------------------------------------------- Use //---------------------------------------------------------------------------------------------------- Use
use std::{ use std::{
borrow::Cow,
collections::{HashMap, HashSet}, collections::{HashMap, HashSet},
sync::Arc, sync::Arc,
}; };
@ -14,6 +15,7 @@ use std::{
use pretty_assertions::assert_eq; use pretty_assertions::assert_eq;
use tower::{Service, ServiceExt}; use tower::{Service, ServiceExt};
use cuprate_database::{ConcreteEnv, DatabaseIter, DatabaseRo, Env, EnvInner, RuntimeError};
use cuprate_test_utils::data::{block_v16_tx0, block_v1_tx2, block_v9_tx3}; use cuprate_test_utils::data::{block_v16_tx0, block_v1_tx2, block_v9_tx3};
use cuprate_types::{ use cuprate_types::{
blockchain::{BCReadRequest, BCResponse, BCWriteRequest}, blockchain::{BCReadRequest, BCResponse, BCWriteRequest},
@ -22,6 +24,7 @@ use cuprate_types::{
use crate::{ use crate::{
config::ConfigBuilder, config::ConfigBuilder,
open_tables::OpenTables,
ops::{ ops::{
block::{get_block_extended_header_from_height, get_block_info}, block::{get_block_extended_header_from_height, get_block_info},
blockchain::chain_height, blockchain::chain_height,
@ -31,7 +34,6 @@ use crate::{
tables::{Tables, TablesIter}, tables::{Tables, TablesIter},
tests::AssertTableLen, tests::AssertTableLen,
types::{Amount, AmountIndex, PreRctOutputId}, types::{Amount, AmountIndex, PreRctOutputId},
ConcreteEnv, DatabaseIter, DatabaseRo, Env, EnvInner, RuntimeError,
}; };
//---------------------------------------------------------------------------------------------------- Helper functions //---------------------------------------------------------------------------------------------------- Helper functions
@ -44,7 +46,7 @@ fn init_service() -> (
) { ) {
let tempdir = tempfile::tempdir().unwrap(); let tempdir = tempfile::tempdir().unwrap();
let config = ConfigBuilder::new() let config = ConfigBuilder::new()
.db_directory(tempdir.path().into()) .db_directory(Cow::Owned(tempdir.path().into()))
.low_power() .low_power()
.build(); .build();
let (reader, writer) = init(config).unwrap(); let (reader, writer) = init(config).unwrap();

3
storage/cuprate-blockchain/src/service/types.rs
View file

@ -5,11 +5,10 @@
//---------------------------------------------------------------------------------------------------- Use //---------------------------------------------------------------------------------------------------- Use
use futures::channel::oneshot::Sender; use futures::channel::oneshot::Sender;
use cuprate_database::RuntimeError;
use cuprate_helper::asynch::InfallibleOneshotReceiver; use cuprate_helper::asynch::InfallibleOneshotReceiver;
use cuprate_types::blockchain::BCResponse; use cuprate_types::blockchain::BCResponse;
use crate::error::RuntimeError;
//---------------------------------------------------------------------------------------------------- Types //---------------------------------------------------------------------------------------------------- Types
/// The actual type of the response. /// The actual type of the response.
/// ///

6
storage/cuprate-blockchain/src/service/write.rs
View file

@ -8,6 +8,7 @@ use std::{
use futures::channel::oneshot; use futures::channel::oneshot;
use cuprate_database::{ConcreteEnv, Env, EnvInner, RuntimeError, TxRw};
use cuprate_helper::asynch::InfallibleOneshotReceiver; use cuprate_helper::asynch::InfallibleOneshotReceiver;
use cuprate_types::{ use cuprate_types::{
blockchain::{BCResponse, BCWriteRequest}, blockchain::{BCResponse, BCWriteRequest},
@ -15,11 +16,8 @@ use cuprate_types::{
}; };
use crate::{ use crate::{
env::{Env, EnvInner}, open_tables::OpenTables,
error::RuntimeError,
service::types::{ResponseReceiver, ResponseResult, ResponseSender}, service::types::{ResponseReceiver, ResponseResult, ResponseSender},
transaction::TxRw,
ConcreteEnv,
}; };
//---------------------------------------------------------------------------------------------------- Constants //---------------------------------------------------------------------------------------------------- Constants

347
storage/cuprate-blockchain/src/storable.rs
View file

@ -1,347 +0,0 @@
//! (De)serialization for table keys & values.
//---------------------------------------------------------------------------------------------------- Import
use std::{borrow::Borrow, fmt::Debug};
use bytemuck::Pod;
use bytes::Bytes;
//---------------------------------------------------------------------------------------------------- Storable
/// A type that can be stored in the database.
///
/// All keys and values in the database must be able
/// to be (de)serialized into/from raw bytes (`[u8]`).
///
/// This trait represents types that can be **perfectly**
/// casted/represented as raw bytes.
///
/// ## `bytemuck`
/// Any type that implements:
/// - [`bytemuck::Pod`]
/// - [`Debug`]
///
/// will automatically implement [`Storable`].
///
/// This includes:
/// - Most primitive types
/// - All types in [`tables`](crate::tables)
///
/// See [`StorableVec`] & [`StorableBytes`] for storing slices of `T: Storable`.
///
/// ```rust
/// # use cuprate_blockchain::*;
/// # use std::borrow::*;
/// let number: u64 = 0;
///
/// // Into bytes.
/// let into = Storable::as_bytes(&number);
/// assert_eq!(into, &[0; 8]);
///
/// // From bytes.
/// let from: u64 = Storable::from_bytes(&into);
/// assert_eq!(from, number);
/// ```
///
/// ## Invariants
/// No function in this trait is expected to panic.
///
/// The byte conversions must execute flawlessly.
///
/// ## Endianness
/// This trait doesn't currently care about endianness.
///
/// Bytes are (de)serialized as-is, and `bytemuck`
/// types are architecture-dependant.
///
/// Most likely, the bytes are little-endian, however
/// that cannot be relied upon when using this trait.
pub trait Storable: Debug {
/// Is this type fixed width in byte length?
///
/// I.e., when converting `Self` to bytes, is it
/// represented with a fixed length array of bytes?
///
/// # `Some`
/// This should be `Some(usize)` on types like:
/// - `u8`
/// - `u64`
/// - `i32`
///
/// where the byte length is known.
///
/// # `None`
/// This should be `None` on any variable-length type like:
/// - `str`
/// - `[u8]`
/// - `Vec<u8>`
///
/// # Examples
/// ```rust
/// # use cuprate_blockchain::*;
/// assert_eq!(<()>::BYTE_LENGTH, Some(0));
/// assert_eq!(u8::BYTE_LENGTH, Some(1));
/// assert_eq!(u16::BYTE_LENGTH, Some(2));
/// assert_eq!(u32::BYTE_LENGTH, Some(4));
/// assert_eq!(u64::BYTE_LENGTH, Some(8));
/// assert_eq!(i8::BYTE_LENGTH, Some(1));
/// assert_eq!(i16::BYTE_LENGTH, Some(2));
/// assert_eq!(i32::BYTE_LENGTH, Some(4));
/// assert_eq!(i64::BYTE_LENGTH, Some(8));
/// assert_eq!(StorableVec::<u8>::BYTE_LENGTH, None);
/// assert_eq!(StorableVec::<u64>::BYTE_LENGTH, None);
/// ```
const BYTE_LENGTH: Option<usize>;
/// Return `self` in byte form.
fn as_bytes(&self) -> &[u8];
/// Create an owned [`Self`] from bytes.
///
/// # Blanket implementation
/// The blanket implementation that covers all types used
/// by `cuprate_blockchain` will simply bitwise copy `bytes`
/// into `Self`.
///
/// The bytes do not have be correctly aligned.
fn from_bytes(bytes: &[u8]) -> Self;
}
impl<T> Storable for T
where
Self: Pod + Debug,
{
const BYTE_LENGTH: Option<usize> = Some(std::mem::size_of::<T>());
#[inline]
fn as_bytes(&self) -> &[u8] {
bytemuck::bytes_of(self)
}
#[inline]
fn from_bytes(bytes: &[u8]) -> T {
bytemuck::pod_read_unaligned(bytes)
}
}
//---------------------------------------------------------------------------------------------------- StorableVec
/// A [`Storable`] vector of `T: Storable`.
///
/// This is a wrapper around `Vec<T> where T: Storable`.
///
/// Slice types are owned both:
/// - when returned from the database
/// - in `put()`
///
/// This is needed as `impl Storable for Vec<T>` runs into impl conflicts.
///
/// # Example
/// ```rust
/// # use cuprate_blockchain::*;
/// //---------------------------------------------------- u8
/// let vec: StorableVec<u8> = StorableVec(vec![0,1]);
///
/// // Into bytes.
/// let into = Storable::as_bytes(&vec);
/// assert_eq!(into, &[0,1]);
///
/// // From bytes.
/// let from: StorableVec<u8> = Storable::from_bytes(&into);
/// assert_eq!(from, vec);
///
/// //---------------------------------------------------- u64
/// let vec: StorableVec<u64> = StorableVec(vec![0,1]);
///
/// // Into bytes.
/// let into = Storable::as_bytes(&vec);
/// assert_eq!(into, &[0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0]);
///
/// // From bytes.
/// let from: StorableVec<u64> = Storable::from_bytes(&into);
/// assert_eq!(from, vec);
/// ```
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, bytemuck::TransparentWrapper)]
#[repr(transparent)]
pub struct StorableVec<T>(pub Vec<T>);
impl<T> Storable for StorableVec<T>
where
T: Pod + Debug,
{
const BYTE_LENGTH: Option<usize> = None;
/// Casts the inner `Vec<T>` directly as bytes.
#[inline]
fn as_bytes(&self) -> &[u8] {
bytemuck::must_cast_slice(&self.0)
}
/// This always allocates a new `Vec<T>`,
/// casting `bytes` into a vector of type `T`.
#[inline]
fn from_bytes(bytes: &[u8]) -> Self {
Self(bytemuck::pod_collect_to_vec(bytes))
}
}
impl<T> std::ops::Deref for StorableVec<T> {
type Target = [T];
#[inline]
fn deref(&self) -> &[T] {
&self.0
}
}
impl<T> Borrow<[T]> for StorableVec<T> {
#[inline]
fn borrow(&self) -> &[T] {
&self.0
}
}
//---------------------------------------------------------------------------------------------------- StorableBytes
/// A [`Storable`] version of [`Bytes`].
///
/// ```rust
/// # use cuprate_blockchain::*;
/// # use bytes::Bytes;
/// let bytes: StorableBytes = StorableBytes(Bytes::from_static(&[0,1]));
///
/// // Into bytes.
/// let into = Storable::as_bytes(&bytes);
/// assert_eq!(into, &[0,1]);
///
/// // From bytes.
/// let from: StorableBytes = Storable::from_bytes(&into);
/// assert_eq!(from, bytes);
/// ```
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
#[repr(transparent)]
pub struct StorableBytes(pub Bytes);
impl Storable for StorableBytes {
const BYTE_LENGTH: Option<usize> = None;
#[inline]
fn as_bytes(&self) -> &[u8] {
&self.0
}
/// This always allocates a new `Bytes`.
#[inline]
fn from_bytes(bytes: &[u8]) -> Self {
Self(Bytes::copy_from_slice(bytes))
}
}
impl std::ops::Deref for StorableBytes {
type Target = [u8];
#[inline]
fn deref(&self) -> &[u8] {
&self.0
}
}
impl Borrow<[u8]> for StorableBytes {
#[inline]
fn borrow(&self) -> &[u8] {
&self.0
}
}
//---------------------------------------------------------------------------------------------------- Tests
#[cfg(test)]
mod test {
use super::*;
/// Serialize, deserialize, and compare that
/// the intermediate/end results are correct.
fn test_storable<const LEN: usize, T>(
// The primitive number function that
// converts the number into little endian bytes,
// e.g `u8::to_le_bytes`.
to_le_bytes: fn(T) -> [u8; LEN],
// A `Vec` of the numbers to test.
t: Vec<T>,
) where
T: Storable + Debug + Copy + PartialEq,
{
for t in t {
let expected_bytes = to_le_bytes(t);
println!("testing: {t:?}, expected_bytes: {expected_bytes:?}");
// (De)serialize.
let se: &[u8] = Storable::as_bytes(&t);
let de = <T as Storable>::from_bytes(se);
println!("serialized: {se:?}, deserialized: {de:?}\n");
// Assert we wrote correct amount of bytes.
if T::BYTE_LENGTH.is_some() {
assert_eq!(se.len(), expected_bytes.len());
}
// Assert the data is the same.
assert_eq!(de, t);
}
}
/// Create all the float tests.
macro_rules! test_float {
($(
$float:ident // The float type.
),* $(,)?) => {
$(
#[test]
fn $float() {
test_storable(
$float::to_le_bytes,
vec![
-1.0,
0.0,
1.0,
$float::MIN,
$float::MAX,
$float::INFINITY,
$float::NEG_INFINITY,
],
);
}
)*
};
}
test_float! {
f32,
f64,
}
/// Create all the (un)signed number tests.
/// u8 -> u128, i8 -> i128.
macro_rules! test_unsigned {
($(
$number:ident // The integer type.
),* $(,)?) => {
$(
#[test]
fn $number() {
test_storable($number::to_le_bytes, vec![$number::MIN, 0, 1, $number::MAX]);
}
)*
};
}
test_unsigned! {
u8,
u16,
u32,
u64,
u128,
usize,
i8,
i16,
i32,
i64,
i128,
isize,
}
}

31
storage/cuprate-blockchain/src/table.rs
View file

@ -1,31 +0,0 @@
//! Database table abstraction; `trait Table`.
//---------------------------------------------------------------------------------------------------- Import
use crate::{key::Key, storable::Storable};
//---------------------------------------------------------------------------------------------------- Table
/// Database table metadata.
///
/// Purely compile time information for database tables.
///
/// ## Sealed
/// This trait is [`Sealed`](https://rust-lang.github.io/api-guidelines/future-proofing.html#sealed-traits-protect-against-downstream-implementations-c-sealed).
///
/// It is only implemented on the types inside [`tables`][crate::tables].
pub trait Table: crate::tables::private::Sealed + 'static {
/// Name of the database table.
const NAME: &'static str;
/// Primary key type.
type Key: Key + 'static;
/// Value type.
type Value: Storable + 'static;
}
//---------------------------------------------------------------------------------------------------- Tests
#[cfg(test)]
mod test {
// use super::*;
}

56
storage/cuprate-blockchain/src/tables.rs
View file

@ -18,14 +18,12 @@
//! For example, this is the object returned by [`EnvInner::open_tables`](crate::EnvInner::open_tables). //! For example, this is the object returned by [`EnvInner::open_tables`](crate::EnvInner::open_tables).
//---------------------------------------------------------------------------------------------------- Import //---------------------------------------------------------------------------------------------------- Import
use crate::{ use cuprate_database::{DatabaseIter, DatabaseRo, DatabaseRw, Table};
database::{DatabaseIter, DatabaseRo, DatabaseRw},
table::Table, use crate::types::{
types::{ Amount, AmountIndex, AmountIndices, BlockBlob, BlockHash, BlockHeight, BlockInfo, KeyImage,
Amount, AmountIndex, AmountIndices, BlockBlob, BlockHash, BlockHeight, BlockInfo, KeyImage, Output, PreRctOutputId, PrunableBlob, PrunableHash, PrunedBlob, RctOutput, TxBlob, TxHash,
Output, PreRctOutputId, PrunableBlob, PrunableHash, PrunedBlob, RctOutput, TxBlob, TxHash, TxId, UnlockTime,
TxId, UnlockTime,
},
}; };
//---------------------------------------------------------------------------------------------------- Sealed //---------------------------------------------------------------------------------------------------- Sealed
@ -98,7 +96,7 @@ macro_rules! define_trait_tables {
/// ///
/// # Errors /// # Errors
/// This returns errors on regular database errors. /// This returns errors on regular database errors.
fn all_tables_empty(&self) -> Result<bool, $crate::error::RuntimeError>; fn all_tables_empty(&self) -> Result<bool, cuprate_database::RuntimeError>;
} }
/// Object containing all opened [`Table`]s in read + iter mode. /// Object containing all opened [`Table`]s in read + iter mode.
@ -183,7 +181,7 @@ macro_rules! define_trait_tables {
} }
)* )*
fn all_tables_empty(&self) -> Result<bool, $crate::error::RuntimeError> { fn all_tables_empty(&self) -> Result<bool, cuprate_database::RuntimeError> {
$( $(
if !DatabaseRo::is_empty(&self.$index)? { if !DatabaseRo::is_empty(&self.$index)? {
return Ok(false); return Ok(false);
@ -265,44 +263,6 @@ define_trait_tables! {
TxUnlockTime => 14, TxUnlockTime => 14,
} }
//---------------------------------------------------------------------------------------------------- Table function macro
/// `crate`-private macro for callings functions on all tables.
///
/// This calls the function `$fn` with the optional
/// arguments `$args` on all tables - returning early
/// (within whatever scope this is called) if any
/// of the function calls error.
///
/// Else, it evaluates to an `Ok((tuple, of, all, table, types, ...))`,
/// i.e., an `impl Table[Mut]` wrapped in `Ok`.
macro_rules! call_fn_on_all_tables_or_early_return {
(
$($fn:ident $(::)?)*
(
$($arg:ident),* $(,)?
)
) => {{
Ok((
$($fn ::)*<$crate::tables::BlockInfos>($($arg),*)?,
$($fn ::)*<$crate::tables::BlockBlobs>($($arg),*)?,
$($fn ::)*<$crate::tables::BlockHeights>($($arg),*)?,
$($fn ::)*<$crate::tables::KeyImages>($($arg),*)?,
$($fn ::)*<$crate::tables::NumOutputs>($($arg),*)?,
$($fn ::)*<$crate::tables::PrunedTxBlobs>($($arg),*)?,
$($fn ::)*<$crate::tables::PrunableHashes>($($arg),*)?,
$($fn ::)*<$crate::tables::Outputs>($($arg),*)?,
$($fn ::)*<$crate::tables::PrunableTxBlobs>($($arg),*)?,
$($fn ::)*<$crate::tables::RctOutputs>($($arg),*)?,
$($fn ::)*<$crate::tables::TxBlobs>($($arg),*)?,
$($fn ::)*<$crate::tables::TxIds>($($arg),*)?,
$($fn ::)*<$crate::tables::TxHeights>($($arg),*)?,
$($fn ::)*<$crate::tables::TxOutputs>($($arg),*)?,
$($fn ::)*<$crate::tables::TxUnlockTime>($($arg),*)?,
))
}};
}
pub(crate) use call_fn_on_all_tables_or_early_return;
//---------------------------------------------------------------------------------------------------- Table macro //---------------------------------------------------------------------------------------------------- Table macro
/// Create all tables, should be used _once_. /// Create all tables, should be used _once_.
/// ///

10
storage/cuprate-blockchain/src/tests.rs
View file

@ -5,11 +5,13 @@
//! - only used internally //! - only used internally
//---------------------------------------------------------------------------------------------------- Import //---------------------------------------------------------------------------------------------------- Import
use std::fmt::Debug; use std::{borrow::Cow, fmt::Debug};
use pretty_assertions::assert_eq; use pretty_assertions::assert_eq;
use crate::{config::ConfigBuilder, tables::Tables, ConcreteEnv, DatabaseRo, Env, EnvInner}; use cuprate_database::{ConcreteEnv, DatabaseRo, Env, EnvInner};
use crate::{config::ConfigBuilder, open_tables::OpenTables, tables::Tables};
//---------------------------------------------------------------------------------------------------- Struct //---------------------------------------------------------------------------------------------------- Struct
/// Named struct to assert the length of all tables. /// Named struct to assert the length of all tables.
@ -67,10 +69,10 @@ impl AssertTableLen {
pub(crate) fn tmp_concrete_env() -> (ConcreteEnv, tempfile::TempDir) { pub(crate) fn tmp_concrete_env() -> (ConcreteEnv, tempfile::TempDir) {
let tempdir = tempfile::tempdir().unwrap(); let tempdir = tempfile::tempdir().unwrap();
let config = ConfigBuilder::new() let config = ConfigBuilder::new()
.db_directory(tempdir.path().into()) .db_directory(Cow::Owned(tempdir.path().into()))
.low_power() .low_power()
.build(); .build();
let env = ConcreteEnv::open(config).unwrap(); let env = ConcreteEnv::open(config.db_config).unwrap();
(env, tempdir) (env, tempdir)
} }

43
storage/cuprate-blockchain/src/transaction.rs
View file

@ -1,43 +0,0 @@
//! Database transaction abstraction; `trait TxRo`, `trait TxRw`.
//---------------------------------------------------------------------------------------------------- Import
use crate::error::RuntimeError;
//---------------------------------------------------------------------------------------------------- TxRo
/// Read-only database transaction.
///
/// Returned from [`EnvInner::tx_ro`](crate::EnvInner::tx_ro).
///
/// # Commit
/// It's recommended but may not be necessary to call [`TxRo::commit`] in certain cases:
/// - <https://docs.rs/heed/0.20.0-alpha.9/heed/struct.RoTxn.html#method.commit>
pub trait TxRo<'env> {
/// Commit the read-only transaction.
///
/// # Errors
/// This operation will always return `Ok(())` with the `redb` backend.
fn commit(self) -> Result<(), RuntimeError>;
}
//---------------------------------------------------------------------------------------------------- TxRw
/// Read/write database transaction.
///
/// Returned from [`EnvInner::tx_rw`](crate::EnvInner::tx_rw).
pub trait TxRw<'env> {
/// Commit the read/write transaction.
///
/// Note that this doesn't necessarily sync the database caches to disk.
///
/// # Errors
/// This operation will always return `Ok(())` with the `redb` backend.
///
/// If `Env::MANUAL_RESIZE == true`,
/// [`RuntimeError::ResizeNeeded`] may be returned.
fn commit(self) -> Result<(), RuntimeError>;
/// Abort the transaction, erasing any writes that have occurred.
///
/// # Errors
/// This operation will always return `Ok(())` with the `heed` backend.
fn abort(self) -> Result<(), RuntimeError>;
}

2
storage/cuprate-blockchain/src/types.rs
View file

@ -46,7 +46,7 @@ use bytemuck::{Pod, Zeroable};
#[cfg(feature = "serde")] #[cfg(feature = "serde")]
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use crate::storable::StorableVec; use cuprate_database::StorableVec;
//---------------------------------------------------------------------------------------------------- Aliases //---------------------------------------------------------------------------------------------------- Aliases
// These type aliases exist as many Monero-related types are the exact same. // These type aliases exist as many Monero-related types are the exact same.

2
storage/database/README.md
View file

@ -1,5 +1,3 @@
# `database`
Cuprate's database abstraction. Cuprate's database abstraction.
This documentation is mostly for practical usage of `database`. This documentation is mostly for practical usage of `database`.

15
storage/database/src/config/config.rs
View file

@ -49,9 +49,9 @@ impl ConfigBuilder {
/// ///
/// [`ConfigBuilder::build`] can be called immediately /// [`ConfigBuilder::build`] can be called immediately
/// after this function to use default values. /// after this function to use default values.
pub const fn new(db_directory: PathBuf) -> Self { pub const fn new(db_directory: Cow<'static, Path>) -> Self {
Self { Self {
db_directory: Cow::Owned(db_directory), db_directory,
sync_mode: None, sync_mode: None,
reader_threads: Some(READER_THREADS_DEFAULT), reader_threads: Some(READER_THREADS_DEFAULT),
resize_algorithm: None, resize_algorithm: None,
@ -61,7 +61,7 @@ impl ConfigBuilder {
/// Build into a [`Config`]. /// Build into a [`Config`].
/// ///
/// # Default values /// # Default values
/// - [`READER_THREADS_DEFAULT`] is used ofr [`Config::reader_threads`] /// - [`READER_THREADS_DEFAULT`] is used for [`Config::reader_threads`]
/// - [`Default::default`] is used for all other values (except the `db_directory`) /// - [`Default::default`] is used for all other values (except the `db_directory`)
pub fn build(self) -> Config { pub fn build(self) -> Config {
// Add the database filename to the directory. // Add the database filename to the directory.
@ -80,6 +80,13 @@ impl ConfigBuilder {
} }
} }
/// Set a custom database directory (and file) [`Path`].
#[must_use]
pub fn db_directory(mut self, db_directory: Cow<'static, Path>) -> Self {
self.db_directory = db_directory;
self
}
/// Tune the [`ConfigBuilder`] for the highest performing, /// Tune the [`ConfigBuilder`] for the highest performing,
/// but also most resource-intensive & maybe risky settings. /// but also most resource-intensive & maybe risky settings.
/// ///
@ -193,7 +200,7 @@ impl Config {
/// assert_eq!(config.reader_threads, READER_THREADS_DEFAULT); /// assert_eq!(config.reader_threads, READER_THREADS_DEFAULT);
/// assert_eq!(config.resize_algorithm, ResizeAlgorithm::default()); /// assert_eq!(config.resize_algorithm, ResizeAlgorithm::default());
/// ``` /// ```
pub fn new(db_directory: PathBuf) -> Self { pub fn new(db_directory: Cow<'static, Path>) -> Self {
ConfigBuilder::new(db_directory).build() ConfigBuilder::new(db_directory).build()
} }

4
storage/database/src/tests.rs
View file

@ -5,6 +5,8 @@
//! - only used internally //! - only used internally
//---------------------------------------------------------------------------------------------------- Import //---------------------------------------------------------------------------------------------------- Import
use std::borrow::Cow;
use crate::{config::ConfigBuilder, table::Table, ConcreteEnv, Env}; use crate::{config::ConfigBuilder, table::Table, ConcreteEnv, Env};
//---------------------------------------------------------------------------------------------------- struct //---------------------------------------------------------------------------------------------------- struct
@ -24,7 +26,7 @@ impl Table for TestTable {
/// FIXME: changing this to `-> impl Env` causes lifetime errors... /// FIXME: changing this to `-> impl Env` causes lifetime errors...
pub(crate) fn tmp_concrete_env() -> (ConcreteEnv, tempfile::TempDir) { pub(crate) fn tmp_concrete_env() -> (ConcreteEnv, tempfile::TempDir) {
let tempdir = tempfile::tempdir().unwrap(); let tempdir = tempfile::tempdir().unwrap();
let config = ConfigBuilder::new(tempdir.path().into()) let config = ConfigBuilder::new(Cow::Owned(tempdir.path().into()))
.low_power() .low_power()
.build(); .build();
let env = ConcreteEnv::open(config).unwrap(); let env = ConcreteEnv::open(config).unwrap();