From 240e579066d377076b2fed5e0e51f97affb07116 Mon Sep 17 00:00:00 2001 From: hinto-janai Date: Thu, 29 Feb 2024 12:40:15 -0500 Subject: [PATCH] database: replace `sanakirja` with `redb` (#80) * cargo: replace `sanakirja` with `redb` * database: update docs `sanakirja` -> `redb` * lib: add TODO for `ConcreteEnv` generic replacement * database: split `trait Database` -> `trait Database{Read,Write}` * heed: add `struct HeedTable{Ro,Rw}` to match `redb` behavior * ops: remove imports for now * env: fix `&mut` bound on RwTx * database: impl `redb`, type-checks * fix heed trait impls, `Database{Read,Write}` -> `Database{Ro,Rw}` * redb: impl `From<_>` for `RuntimeError` * update readme * heed: document `HeedTableR{o,w}` types * env: doc `sync()` invariant * database: document data & lock filenames * misc docs, `redb` durability impl, `'db` -> `'env` * redb: fixes * misc docs and fixes * Update database/README.md Co-authored-by: Boog900 --------- Co-authored-by: Boog900 --- Cargo.lock | 95 ++------------ database/Cargo.toml | 32 ++--- database/README.md | 41 +++++- database/src/backend/heed/database.rs | 84 ++++++++++-- database/src/backend/heed/env.rs | 52 +++++--- database/src/backend/heed/error.rs | 10 +- database/src/backend/heed/transaction.rs | 21 ++- database/src/backend/heed/types.rs | 2 +- database/src/backend/mod.rs | 6 +- database/src/backend/redb/database.rs | 61 +++++++++ database/src/backend/redb/env.rs | 120 ++++++++++++++++++ database/src/backend/redb/error.rs | 107 ++++++++++++++++ .../src/backend/{sanakirja => redb}/mod.rs | 0 .../{sanakirja => redb}/transaction.rs | 14 +- database/src/backend/redb/types.rs | 10 ++ database/src/backend/sanakirja/database.rs | 49 ------- database/src/backend/sanakirja/env.rs | 93 -------------- database/src/backend/sanakirja/error.rs | 51 -------- database/src/backend/sanakirja/types.rs | 6 - database/src/config.rs | 55 ++++++-- database/src/constants.rs | 69 +++++++--- database/src/database.rs | 69 ++++------ database/src/env.rs | 59 ++++++--- database/src/lib.rs | 44 ++++--- database/src/ops/alt_block.rs | 8 -- database/src/ops/block.rs | 8 -- database/src/ops/blockchain.rs | 8 -- database/src/ops/output.rs | 8 -- database/src/ops/property.rs | 8 -- database/src/ops/spent_key.rs | 8 -- database/src/ops/tx.rs | 8 -- database/src/resize.rs | 2 +- database/src/transaction.rs | 12 +- 33 files changed, 689 insertions(+), 531 deletions(-) create mode 100644 database/src/backend/redb/database.rs create mode 100644 database/src/backend/redb/env.rs create mode 100644 database/src/backend/redb/error.rs rename database/src/backend/{sanakirja => redb}/mod.rs (100%) rename database/src/backend/{sanakirja => redb}/transaction.rs (69%) create mode 100644 database/src/backend/redb/types.rs delete mode 100644 database/src/backend/sanakirja/database.rs delete mode 100644 database/src/backend/sanakirja/env.rs delete mode 100644 database/src/backend/sanakirja/error.rs delete mode 100644 database/src/backend/sanakirja/types.rs diff --git a/Cargo.lock b/Cargo.lock index da5282cd..1591dc2b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -578,7 +578,7 @@ dependencies = [ "heed", "page_size", "paste", - "sanakirja", + "redb", "serde", "tempfile", "thiserror", @@ -838,7 +838,7 @@ checksum = "1ee447700ac8aa0b2f2bd7bc4462ad686ba06baa6727ac149a2d6277f0d240fd" dependencies = [ "cfg-if", "libc", - "redox_syscall 0.4.1", + "redox_syscall", "windows-sys 0.52.0", ] @@ -903,16 +903,6 @@ dependencies = [ "percent-encoding", ] -[[package]] -name = "fs2" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213" -dependencies = [ - "libc", - "winapi", -] - [[package]] name = "funty" version = "2.0.0" @@ -1371,15 +1361,6 @@ dependencies = [ "generic-array", ] -[[package]] -name = "instant" -version = "0.1.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" -dependencies = [ - "cfg-if", -] - [[package]] name = "ipnet" version = "2.9.0" @@ -1445,7 +1426,7 @@ checksum = "85c833ca1e66078851dba29046874e38f08b2c883700aa29a03ddd3b23814ee8" dependencies = [ "bitflags 2.4.2", "libc", - "redox_syscall 0.4.1", + "redox_syscall", ] [[package]] @@ -1496,16 +1477,6 @@ version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149" -[[package]] -name = "memmap" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6585fd95e7bb50d6cc31e20d4cf9afb4e2ba16c5846fc76793f11218da9c475b" -dependencies = [ - "libc", - "winapi", -] - [[package]] name = "merlin" version = "3.0.0" @@ -1827,17 +1798,6 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bb813b8af86854136c6922af0598d719255ecb2179515e6e7730d468f05c9cae" -[[package]] -name = "parking_lot" -version = "0.11.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" -dependencies = [ - "instant", - "lock_api", - "parking_lot_core 0.8.6", -] - [[package]] name = "parking_lot" version = "0.12.1" @@ -1845,21 +1805,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" dependencies = [ "lock_api", - "parking_lot_core 0.9.9", -] - -[[package]] -name = "parking_lot_core" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc" -dependencies = [ - "cfg-if", - "instant", - "libc", - "redox_syscall 0.2.16", - "smallvec", - "winapi", + "parking_lot_core", ] [[package]] @@ -1870,7 +1816,7 @@ checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e" dependencies = [ "cfg-if", "libc", - "redox_syscall 0.4.1", + "redox_syscall", "smallvec", "windows-targets 0.48.5", ] @@ -2204,12 +2150,12 @@ dependencies = [ ] [[package]] -name = "redox_syscall" -version = "0.2.16" +name = "redb" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" +checksum = "72623e6275cd430215b741f41ebda34db93a13ebde253f908b70871c46afc5ba" dependencies = [ - "bitflags 1.3.2", + "libc", ] [[package]] @@ -2427,27 +2373,6 @@ version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e86697c916019a8588c99b5fac3cead74ec0b4b819707a682fd4d23fa0ce1ba1" -[[package]] -name = "sanakirja" -version = "1.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c385eb43079aa7dc6204e473b68b4305ceaea8048dda3a985a339bbb57cde72" -dependencies = [ - "fs2", - "log", - "memmap", - "parking_lot 0.11.2", - "sanakirja-core", - "serde", - "thiserror", -] - -[[package]] -name = "sanakirja-core" -version = "1.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8376db34ae3eac6e7bd91168bc638450073b708ce9fb46940de676f552238bf5" - [[package]] name = "schannel" version = "0.1.23" @@ -2851,7 +2776,7 @@ dependencies = [ "libc", "mio", "num_cpus", - "parking_lot 0.12.1", + "parking_lot", "pin-project-lite", "signal-hook-registry", "socket2", diff --git a/database/Cargo.toml b/database/Cargo.toml index 605ad800..812e70b3 100644 --- a/database/Cargo.toml +++ b/database/Cargo.toml @@ -10,27 +10,27 @@ keywords = ["cuprate", "database"] [features] default = ["heed", "service"] -# default = ["sanakirja", "service"] # For testing `sanakirja`. -heed = ["dep:heed"] -sanakirja = ["dep:sanakirja"] -service = ["dep:crossbeam", "dep:tokio", "dep:tower"] +# default = ["redb", "service"] # For testing `redb`. +heed = ["dep:heed"] +redb = ["dep:redb"] +service = ["dep:crossbeam", "dep:tokio", "dep:tower"] [dependencies] -cfg-if = { workspace = true } +cfg-if = { workspace = true } # FIXME: # We only need the `thread` feature if `service` is enabled. # Figure out how to enable features of an already pulled in dependency conditionally. cuprate-helper = { path = "../helper", features = ["fs", "thread"] } -paste = { workspace = true } +paste = { workspace = true } # Needed for database resizes. # They must be a multiple of the OS page size. -page_size = { version = "0.6.0" } -thiserror = { workspace = true } +page_size = { version = "0.6.0" } +thiserror = { workspace = true } # `service` feature. -crossbeam = { workspace = true, features = ["std"], optional = true } -tokio = { workspace = true, features = ["full"], optional = true } -tower = { workspace = true, features = ["full"], optional = true } +crossbeam = { workspace = true, features = ["std"], optional = true } +tokio = { workspace = true, features = ["full"], optional = true } +tower = { workspace = true, features = ["full"], optional = true } # SOMEDAY: could be used in `service` as # the database mutual exclusive `RwLock`. # @@ -40,12 +40,12 @@ tower = { workspace = true, features = ["full"], optional = true } # parking_lot = { workspace = true, optional = true } # Optional features. -borsh = { workspace = true, optional = true } -heed = { git = "https://github.com/Cuprate/heed", rev = "5aa75b7", optional = true } -sanakirja = { version = "1.4.0", optional = true } -serde = { workspace = true, optional = true } +borsh = { workspace = true, optional = true } +heed = { git = "https://github.com/Cuprate/heed", rev = "5aa75b7", optional = true } +redb = { version = "1.5.0", optional = true } +serde = { workspace = true, optional = true } [dev-dependencies] cuprate-helper = { path = "../helper", features = ["thread"] } page_size = { version = "0.6.0" } -tempfile = { version = "3.10.0" } \ No newline at end of file +tempfile = { version = "3.10.0" } \ No newline at end of file diff --git a/database/README.md b/database/README.md index 4279972d..a5dd7c34 100644 --- a/database/README.md +++ b/database/README.md @@ -10,7 +10,9 @@ Cuprate's database implementation. - [`src/backend/`](#src-backend) 1. [Backends](#backends) - [`heed`](#heed) + - [`redb`](#redb) - [`sanakirja`](#sanakirja) + - [`MDBX`](#mdbx) 1. [Layers](#layers) - [Database](#database) - [Trait](#trait) @@ -68,7 +70,7 @@ The top-level `src/` files. |------------------|---------| | `config.rs` | Database `Env` configuration | `constants.rs` | General constants used throughout `cuprate-database` -| `database.rs` | Abstracted database; `trait Database` +| `database.rs` | Abstracted database; `trait DatabaseR{o,w}` | `env.rs` | Abstracted database environment; `trait Env` | `error.rs` | Database error types | `free.rs` | General free functions (related to the database) @@ -76,7 +78,7 @@ The top-level `src/` files. | `pod.rs` | Data (de)serialization; `trait Pod` | `table.rs` | Database table abstraction; `trait Table` | `tables.rs` | All the table definitions used by `cuprate-database` -| `transaction.rs` | Database transaction abstraction; `trait RoTx`, `trait RwTx` +| `transaction.rs` | Database transaction abstraction; `trait TxR{o,w}` ## `src/ops/` This folder contains the `cupate_database::ops` module. @@ -119,10 +121,10 @@ All backends follow the same file structure: | File | Purpose | |------------------|---------| -| `database.rs` | Implementation of `trait Database` +| `database.rs` | Implementation of `trait DatabaseR{o,w}` | `env.rs` | Implementation of `trait Env` | `error.rs` | Implementation of backend's errors to `cuprate_database`'s error types -| `transaction.rs` | Implementation of `trait RoTx/RwTx` +| `transaction.rs` | Implementation of `trait TxR{o,w}` | `types.rs` | Type aliases for long backend-specific types # Backends @@ -131,7 +133,7 @@ All backends follow the same file structure: Each database's implementation is located in its respective file in `src/backend/${DATABASE_NAME}.rs`. ## `heed` -The default database used is a modified fork of [`heed`](https://github.com/meilisearch/heed), located at [`Cuprate/heed`](https://github.com/Cuprate/heed). +The default database used is a modified fork of [`heed`](https://github.com/meilisearch/heed) (LMDB), located at [`Cuprate/heed`](https://github.com/Cuprate/heed). To generate documentation of the fork for local use: ```bash @@ -140,10 +142,37 @@ cargo doc ``` `LMDB` should not need to be installed as `heed` has a build script that pulls it in automatically. +`heed`'s filenames inside Cuprate's database folder (`~/.local/share/cuprate/database/`) are: + +| Filename | Purpose | +|------------|---------| +| `data.mdb` | Main data file +| `lock.mdb` | Database lock file + TODO: document max readers limit: https://github.com/monero-project/monero/blob/059028a30a8ae9752338a7897329fe8012a310d5/src/blockchain_db/lmdb/db_lmdb.cpp#L1372. Other potential processes (e.g. `xmrblocks`) that are also reading the `data.mdb` file need to be accounted for. +## `redb` +The 2nd database backend is the 100% Rust [`redb`](https://github.com/cberner/redb). + +The upstream versions from [`crates.io`](https://crates.io/crates/redb) are used. + +`redb`'s filenames inside Cuprate's database folder (`~/.local/share/cuprate/database/`) are: + +| Filename | Purpose | +|-------------|---------| +| `data.redb` | Main data file + ## `sanakirja` -TODO +[`sanakirja`](https://docs.rs/sanakirja) was a candidate as a backend, however there were problems with maximum value sizes. + +The default maximum value size is [1012 bytes](https://docs.rs/sanakirja/1.4.1/sanakirja/trait.Storable.html) which was too small for our requirements. Using [`sanakirja::Slice`](https://docs.rs/sanakirja/1.4.1/sanakirja/union.Slice.html) and [sanakirja::UnsizedStorage](https://docs.rs/sanakirja/1.4.1/sanakirja/trait.UnsizedStorable.html) was attempted, but there were bugs found when inserting a value in-between `512..=4096` bytes. + +As such, it is not implemented. + +## `MDBX` +[`MDBX`](https://erthink.github.io/libmdbx) was a candidate as a backend, however MDBX deprecated the custom key/value comparison functions, this makes it a bit trickier to implement dup tables. It is also quite similar to the main backend LMDB (of which it was originally a fork of). + +As such, it is not implemented (yet). # Layers TODO: update with accurate information when ready, update image. diff --git a/database/src/backend/heed/database.rs b/database/src/backend/heed/database.rs index 3a310fbe..eaae70c4 100644 --- a/database/src/backend/heed/database.rs +++ b/database/src/backend/heed/database.rs @@ -1,41 +1,97 @@ //! Implementation of `trait Database` for `heed`. //---------------------------------------------------------------------------------------------------- Import -use crate::{backend::heed::types::HeedDb, database::Database, error::RuntimeError, table::Table}; +use std::marker::PhantomData; -//---------------------------------------------------------------------------------------------------- Database Impls -impl Database for HeedDb { - type RoTx<'db> = heed::RoTxn<'db>; - type RwTx<'db> = heed::RwTxn<'db>; +use crate::{ + backend::heed::types::HeedDb, + database::{DatabaseRo, DatabaseRw}, + error::RuntimeError, + table::Table, +}; - fn get(&self, ro_tx: &Self::RoTx<'_>, key: &T::Key) -> Result, RuntimeError> { +//---------------------------------------------------------------------------------------------------- Heed Database Wrappers +// Q. Why does `HeedTableR{o,w}` exist? +// A. These wrapper types combine `heed`'s database/table +// types with its transaction types. It exists to match +// `redb`, which has this behavior built-in. +// +// `redb` forces us to abstract read/write semantics +// at the _opened table_ level, so, we must match that in `heed`, +// which abstracts it at the transaction level. +// +// We must also maintain the ability for +// write operations to also read, aka, `Rw`. +// +// TODO: do we need the `T: Table` phantom bound? +// It allows us to reference the `Table` info. + +/// An opened read-only database associated with a transaction. +/// +/// Matches `redb::ReadOnlyTable`. +pub(super) struct HeedTableRo<'env, T: Table> { + /// An already opened database table. + db: HeedDb, + /// The associated read-only transaction that opened this table. + tx_ro: &'env heed::RoTxn<'env>, + /// TODO: do we need this? + _table: PhantomData, +} + +/// An opened read/write database associated with a transaction. +/// +/// Matches `redb::Table` (read & write). +pub(super) struct HeedTableRw<'env, T: Table> { + /// TODO + db: HeedDb, + /// The associated read/write transaction that opened this table. + tx_rw: &'env mut heed::RwTxn<'env>, + /// TODO: do we need this? + _table: PhantomData, +} + +//---------------------------------------------------------------------------------------------------- DatabaseRo Impl +impl DatabaseRo for HeedTableRo<'_, T> { + fn get(&self, key: &T::Key) -> Result, RuntimeError> { todo!() } fn get_range( &self, - ro_tx: &Self::RoTx<'_>, key: &T::Key, amount: usize, ) -> Result, RuntimeError> { let iter: std::vec::Drain<'_, T::Value> = todo!(); Ok(iter) } +} - fn put( - &mut self, - rw_tx: &mut Self::RwTx<'_>, +//---------------------------------------------------------------------------------------------------- DatabaseRw Impl +impl DatabaseRo for HeedTableRw<'_, T> { + fn get(&self, key: &T::Key) -> Result, RuntimeError> { + todo!() + } + + fn get_range( + &self, key: &T::Key, - value: &T::Value, - ) -> Result<(), RuntimeError> { + amount: usize, + ) -> Result, RuntimeError> { + let iter: std::vec::Drain<'_, T::Value> = todo!(); + Ok(iter) + } +} + +impl DatabaseRw for HeedTableRw<'_, T> { + fn put(&mut self, key: &T::Key, value: &T::Value) -> Result<(), RuntimeError> { todo!() } - fn clear(&mut self, rw_tx: &mut Self::RwTx<'_>) -> Result<(), RuntimeError> { + fn clear(&mut self) -> Result<(), RuntimeError> { todo!() } - fn delete(&mut self, rw_tx: &mut Self::RwTx<'_>, key: &T::Key) -> Result { + fn delete(&mut self, key: &T::Key) -> Result { todo!() } } diff --git a/database/src/backend/heed/env.rs b/database/src/backend/heed/env.rs index bb7deb9f..18cf85db 100644 --- a/database/src/backend/heed/env.rs +++ b/database/src/backend/heed/env.rs @@ -4,9 +4,9 @@ use std::sync::RwLock; use crate::{ - backend::heed::types::HeedDb, + backend::heed::database::{HeedTableRo, HeedTableRw}, config::Config, - database::Database, + database::{DatabaseRo, DatabaseRw}, env::Env, error::{InitError, RuntimeError}, resize::ResizeAlgorithm, @@ -49,6 +49,14 @@ pub struct ConcreteEnv { impl Drop for ConcreteEnv { fn drop(&mut self) { + // TODO: + // "if the environment has the MDB_NOSYNC flag set the flushes will be omitted, + // and with MDB_MAPASYNC they will be asynchronous." + // + // + // We need to do `mdb_env_set_flags(&env, MDB_NOSYNC|MDB_ASYNCMAP, 0)` + // to clear the no sync and async flags such that the below `self.sync()` + // _actually_ synchronously syncs. if let Err(e) = self.sync() { // TODO: log error? } @@ -61,8 +69,8 @@ impl Drop for ConcreteEnv { impl Env for ConcreteEnv { const MANUAL_RESIZE: bool = true; const SYNCS_PER_TX: bool = false; - type RoTx<'db> = heed::RoTxn<'db>; - type RwTx<'db> = heed::RwTxn<'db>; + type TxRo<'env> = heed::RoTxn<'env>; + type TxRw<'env> = heed::RwTxn<'env>; #[cold] #[inline(never)] // called once. @@ -80,6 +88,12 @@ impl Env for ConcreteEnv { todo!() } + #[cold] + #[inline(never)] // called once in [`Env::open`]? + fn create_tables(&self, tx_rw: &mut Self::TxRw<'_>) -> Result<(), RuntimeError> { + todo!() + } + fn config(&self) -> &Config { &self.config } @@ -116,30 +130,30 @@ impl Env for ConcreteEnv { } #[inline] - fn ro_tx(&self) -> Result, RuntimeError> { + fn tx_ro(&self) -> Result, RuntimeError> { todo!() } #[inline] - fn rw_tx(&self) -> Result, RuntimeError> { - todo!() - } - - #[cold] - #[inline(never)] // called infrequently?. - fn create_tables_if_needed( - &self, - tx_rw: &mut Self::RwTx<'_>, - ) -> Result<(), RuntimeError> { + fn tx_rw(&self) -> Result, RuntimeError> { todo!() } #[inline] - fn open_database( + fn open_db_ro( &self, - to_rw: &Self::RoTx<'_>, - ) -> Result, RuntimeError> { - let tx: HeedDb = todo!(); + tx_ro: &Self::TxRo<'_>, + ) -> Result, RuntimeError> { + let tx: HeedTableRo = todo!(); + Ok(tx) + } + + #[inline] + fn open_db_rw( + &self, + tx_rw: &mut Self::TxRw<'_>, + ) -> Result, RuntimeError> { + let tx: HeedTableRw = todo!(); Ok(tx) } } diff --git a/database/src/backend/heed/error.rs b/database/src/backend/heed/error.rs index 4bd4c166..65b781b2 100644 --- a/database/src/backend/heed/error.rs +++ b/database/src/backend/heed/error.rs @@ -1,4 +1,4 @@ -//! Conversion from `heed::Error` -> `cuprate_database::RuntimeError`. +//! Conversion from `heed::Error` -> `cuprate_database`'s errors. //---------------------------------------------------------------------------------------------------- Use use crate::constants::DATABASE_CORRUPT_MSG; @@ -83,7 +83,7 @@ impl From for crate::RuntimeError { // // "Requested page not found - this usually indicates corruption." // - E2::Corrupted | E2::PageNotFound => panic!("{mdb_error:?}\n{DATABASE_CORRUPT_MSG}"), + E2::Corrupted | E2::PageNotFound => panic!("{mdb_error:#?}\n{DATABASE_CORRUPT_MSG}"), // These errors should not occur, and if they do, // the best thing `cuprate_database` can do for @@ -98,7 +98,7 @@ impl From for crate::RuntimeError { | E2::TxnFull | E2::BadRslot | E2::VersionMismatch - | E2::BadDbi => panic!("{mdb_error:?}"), + | E2::BadDbi => panic!("{mdb_error:#?}"), // These errors are the same as above, but instead // of being errors we can't control, these are errors @@ -135,7 +135,7 @@ impl From for crate::RuntimeError { // Don't use a key that is `>511` bytes. // | E2::BadValSize - => panic!("fix the database code! {mdb_error:?}"), + => panic!("fix the database code! {mdb_error:#?}"), }, // Only if we write incorrect code. @@ -143,7 +143,7 @@ impl From for crate::RuntimeError { | E1::DatabaseClosing | E1::BadOpenOptions { .. } | E1::Encoding(_) - | E1::Decoding(_) => panic!("fix the database code! {error:?}"), + | E1::Decoding(_) => panic!("fix the database code! {error:#?}"), } } } diff --git a/database/src/backend/heed/transaction.rs b/database/src/backend/heed/transaction.rs index b5f66011..655ba8b1 100644 --- a/database/src/backend/heed/transaction.rs +++ b/database/src/backend/heed/transaction.rs @@ -1,20 +1,29 @@ -//! Implementation of `trait RoTx/RwTx` for `heed`. +//! Implementation of `trait TxRo/TxRw` for `heed`. //---------------------------------------------------------------------------------------------------- Import use crate::{ error::RuntimeError, - transaction::{RoTx, RwTx}, + transaction::{TxRo, TxRw}, }; -//---------------------------------------------------------------------------------------------------- RoTx -impl RoTx<'_> for heed::RoTxn<'_> { +//---------------------------------------------------------------------------------------------------- TxRo +impl TxRo<'_> for heed::RoTxn<'_> { fn commit(self) -> Result<(), RuntimeError> { todo!() } } -//---------------------------------------------------------------------------------------------------- RwTx -impl RwTx<'_> for heed::RwTxn<'_> { +//---------------------------------------------------------------------------------------------------- TxRw +impl TxRo<'_> for heed::RwTxn<'_> { + /// TODO + /// # Errors + /// TODO + fn commit(self) -> Result<(), RuntimeError> { + todo!() + } +} + +impl TxRw<'_> for heed::RwTxn<'_> { /// TODO /// # Errors /// TODO diff --git a/database/src/backend/heed/types.rs b/database/src/backend/heed/types.rs index e1237676..48f57824 100644 --- a/database/src/backend/heed/types.rs +++ b/database/src/backend/heed/types.rs @@ -4,5 +4,5 @@ use heed::{types::Bytes, Database}; //---------------------------------------------------------------------------------------------------- Types -/// The concrete database type for `heed`. +/// The concrete database type for `heed`, usable for reads and writes. pub(super) type HeedDb = Database; diff --git a/database/src/backend/mod.rs b/database/src/backend/mod.rs index 249c4de2..3b458d8c 100644 --- a/database/src/backend/mod.rs +++ b/database/src/backend/mod.rs @@ -12,9 +12,9 @@ cfg_if::cfg_if! { // If both backends are enabled, fallback to `heed`. // This is useful when using `--all-features`. - if #[cfg(all(feature = "sanakirja", not(feature = "heed")))] { - mod sanakirja; - pub use sanakirja::ConcreteEnv; + if #[cfg(all(feature = "redb", not(feature = "heed")))] { + mod redb; + pub use redb::ConcreteEnv; } else { mod heed; pub use heed::ConcreteEnv; diff --git a/database/src/backend/redb/database.rs b/database/src/backend/redb/database.rs new file mode 100644 index 00000000..834ca645 --- /dev/null +++ b/database/src/backend/redb/database.rs @@ -0,0 +1,61 @@ +//! Implementation of `trait DatabaseR{o,w}` for `redb`. + +//---------------------------------------------------------------------------------------------------- Import +use crate::{ + backend::redb::types::{RedbTableRo, RedbTableRw}, + database::{DatabaseRo, DatabaseRw}, + error::RuntimeError, + table::Table, +}; + +//---------------------------------------------------------------------------------------------------- DatabaseRo +impl DatabaseRo for RedbTableRo<'_> { + fn get(&self, key: &T::Key) -> Result, RuntimeError> { + todo!() + } + + fn get_range( + &self, + key: &T::Key, + amount: usize, + ) -> Result, RuntimeError> { + let iter: std::vec::Drain<'_, T::Value> = todo!(); + Ok(iter) + } +} + +//---------------------------------------------------------------------------------------------------- DatabaseRw +impl DatabaseRo for RedbTableRw<'_, '_> { + fn get(&self, key: &T::Key) -> Result, RuntimeError> { + todo!() + } + + fn get_range( + &self, + key: &T::Key, + amount: usize, + ) -> Result, RuntimeError> { + let iter: std::vec::Drain<'_, T::Value> = todo!(); + Ok(iter) + } +} + +impl DatabaseRw for RedbTableRw<'_, '_> { + fn put(&mut self, key: &T::Key, value: &T::Value) -> Result<(), RuntimeError> { + todo!() + } + + fn clear(&mut self) -> Result<(), RuntimeError> { + todo!() + } + + fn delete(&mut self, key: &T::Key) -> Result { + todo!() + } +} + +//---------------------------------------------------------------------------------------------------- Tests +#[cfg(test)] +mod test { + // use super::*; +} diff --git a/database/src/backend/redb/env.rs b/database/src/backend/redb/env.rs new file mode 100644 index 00000000..0e3381ec --- /dev/null +++ b/database/src/backend/redb/env.rs @@ -0,0 +1,120 @@ +//! Implementation of `trait Env` for `redb`. + +//---------------------------------------------------------------------------------------------------- Import +use std::{path::Path, sync::Arc}; + +use crate::{ + backend::redb::types::{RedbTableRo, RedbTableRw}, + config::{Config, SyncMode}, + database::{DatabaseRo, DatabaseRw}, + env::Env, + error::{InitError, RuntimeError}, + table::Table, +}; + +//---------------------------------------------------------------------------------------------------- ConcreteEnv +/// A strongly typed, concrete database environment, backed by `redb`. +pub struct ConcreteEnv { + /// The actual database environment. + env: redb::Database, + + /// The configuration we were opened with + /// (and in current use). + config: Config, + + /// A cached, redb version of `cuprate_database::config::SyncMode`. + /// `redb` needs the sync mode to be set _per_ TX, so we + /// will continue to use this value every `Env::tx_rw`. + durability: redb::Durability, +} + +impl Drop for ConcreteEnv { + fn drop(&mut self) { + if let Err(e) = self.sync() { + // TODO: log error? + } + + // TODO: log that we are dropping the database. + } +} + +//---------------------------------------------------------------------------------------------------- Env Impl +impl Env for ConcreteEnv { + const MANUAL_RESIZE: bool = false; + const SYNCS_PER_TX: bool = false; + + type TxRo<'env> = redb::ReadTransaction<'env>; + type TxRw<'env> = redb::WriteTransaction<'env>; + + #[cold] + #[inline(never)] // called once. + fn open(config: Config) -> Result { + // TODO: dynamic syncs are not implemented. + let durability = match config.sync_mode { + // TODO: There's also `redb::Durability::Paranoid`: + // + // should we use that instead of Immediate? + SyncMode::Safe => redb::Durability::Immediate, + SyncMode::Async => redb::Durability::Eventual, + SyncMode::Fast => redb::Durability::None, + // TODO: dynamic syncs are not implemented. + SyncMode::FastThenSafe | SyncMode::Threshold(_) => unimplemented!(), + }; + + todo!() + } + + #[cold] + #[inline(never)] // called once in [`Env::open`]?` + fn create_tables(&self, tx_rw: &mut Self::TxRw<'_>) -> Result<(), RuntimeError> { + todo!() + } + + fn config(&self) -> &Config { + &self.config + } + + fn sync(&self) -> Result<(), RuntimeError> { + todo!() + } + + #[inline] + fn tx_ro(&self) -> Result, RuntimeError> { + todo!() + } + + #[inline] + fn tx_rw(&self) -> Result, RuntimeError> { + // `redb` has sync modes on the TX level, unlike heed, + // which sets it at the Environment level. + // + // So, set the durability here before returning the TX. + let mut tx_rw = self.env.begin_write()?; + tx_rw.set_durability(self.durability); + Ok(tx_rw) + } + + #[inline] + fn open_db_ro( + &self, + tx_ro: &Self::TxRo<'_>, + ) -> Result, RuntimeError> { + let tx: RedbTableRo = todo!(); + Ok(tx) + } + + #[inline] + fn open_db_rw( + &self, + tx_rw: &mut Self::TxRw<'_>, + ) -> Result, RuntimeError> { + let tx: RedbTableRw = todo!(); + Ok(tx) + } +} + +//---------------------------------------------------------------------------------------------------- Tests +#[cfg(test)] +mod test { + // use super::*; +} diff --git a/database/src/backend/redb/error.rs b/database/src/backend/redb/error.rs new file mode 100644 index 00000000..025c5411 --- /dev/null +++ b/database/src/backend/redb/error.rs @@ -0,0 +1,107 @@ +//! Conversion from `redb`'s errors -> `cuprate_database`'s errors. +//! +//! HACK: There's a lot of `_ =>` usage here because +//! `redb`'s errors are `#[non_exhaustive]`... + +//---------------------------------------------------------------------------------------------------- Import +use crate::constants::DATABASE_CORRUPT_MSG; + +//---------------------------------------------------------------------------------------------------- DatabaseError +impl From for crate::InitError { + /// Created by `redb` in: + /// - [`redb::Database::open`](https://docs.rs/redb/1.5.0/redb/struct.Database.html#method.open). + fn from(error: redb::DatabaseError) -> Self { + use redb::DatabaseError as E; + use redb::StorageError as E2; + + // Reference of all possible errors `redb` will return + // upon using `redb::Database::open`: + // + match error { + E::RepairAborted => Self::Corrupt, + E::UpgradeRequired(_) => Self::InvalidVersion, + E::Storage(s_error) => match s_error { + E2::Io(e) => Self::Io(e), + E2::Corrupted(_) => Self::Corrupt, + + // HACK: Handle new errors as `redb` adds them. + _ => Self::Unknown(Box::new(s_error)), + }, + + // HACK: Handle new errors as `redb` adds them. + _ => Self::Unknown(Box::new(error)), + } + } +} + +//---------------------------------------------------------------------------------------------------- TransactionError +#[allow(clippy::fallible_impl_from)] // We need to panic sometimes. +impl From for crate::RuntimeError { + /// Created by `redb` in: + /// - [`redb::Database::begin_write`](https://docs.rs/redb/1.5.0/redb/struct.Database.html#method.begin_write) + /// - [`redb::Database::begin_read`](https://docs.rs/redb/1.5.0/redb/struct.Database.html#method.begin_read) + fn from(error: redb::TransactionError) -> Self { + use redb::StorageError as E; + + match error { + redb::TransactionError::Storage(error) => error.into(), + + // HACK: Handle new errors as `redb` adds them. + _ => unreachable!(), + } + } +} + +//---------------------------------------------------------------------------------------------------- TableError +#[allow(clippy::fallible_impl_from)] // We need to panic sometimes. +impl From for crate::RuntimeError { + /// Created by `redb` in: + /// - [`redb::WriteTransaction::open_table`](https://docs.rs/redb/1.5.0/redb/struct.WriteTransaction.html#method.open_table) + /// - [`redb::ReadTransaction::open_table`](https://docs.rs/redb/1.5.0/redb/struct.ReadTransaction.html#method.open_table) + fn from(error: redb::TableError) -> Self { + use redb::StorageError as E2; + use redb::TableError as E; + + match error { + E::Storage(error) => error.into(), + + // Only if we write incorrect code. + E::TableTypeMismatch { .. } + | E::TableIsMultimap(_) + | E::TableIsNotMultimap(_) + | E::TypeDefinitionChanged { .. } + | E::TableDoesNotExist(_) + | E::TableAlreadyOpen(..) => panic!("fix the database code! {error:#?}"), + + // HACK: Handle new errors as `redb` adds them. + _ => unreachable!(), + } + } +} + +//---------------------------------------------------------------------------------------------------- StorageError +#[allow(clippy::fallible_impl_from)] // We need to panic sometimes. +impl From for crate::RuntimeError { + /// Created by `redb` in: + /// - [`redb::Table`](https://docs.rs/redb/1.5.0/redb/struct.Table.html) functions + /// - [`redb::ReadOnlyTable`](https://docs.rs/redb/1.5.0/redb/struct.ReadOnlyTable.html) functions + fn from(error: redb::StorageError) -> Self { + use redb::StorageError as E; + + match error { + E::Io(e) => Self::Io(e), + E::Corrupted(s) => panic!("{s:#?}\n{DATABASE_CORRUPT_MSG}"), + E::ValueTooLarge(s) => panic!("fix the database code! {s:#?}"), + E::LockPoisoned(s) => panic!("{s:#?}"), + + // HACK: Handle new errors as `redb` adds them. + _ => unreachable!(), + } + } +} + +//---------------------------------------------------------------------------------------------------- Tests +#[cfg(test)] +mod test { + // use super::*; +} diff --git a/database/src/backend/sanakirja/mod.rs b/database/src/backend/redb/mod.rs similarity index 100% rename from database/src/backend/sanakirja/mod.rs rename to database/src/backend/redb/mod.rs diff --git a/database/src/backend/sanakirja/transaction.rs b/database/src/backend/redb/transaction.rs similarity index 69% rename from database/src/backend/sanakirja/transaction.rs rename to database/src/backend/redb/transaction.rs index 7d9ae669..0af3c696 100644 --- a/database/src/backend/sanakirja/transaction.rs +++ b/database/src/backend/redb/transaction.rs @@ -1,20 +1,22 @@ -//! Implementation of `trait RoTx/RwTx` for `sanakirja`. +//! Implementation of `trait TxRo/TxRw` for `redb`. //---------------------------------------------------------------------------------------------------- Import use crate::{ + config::SyncMode, + env::Env, error::RuntimeError, - transaction::{RoTx, RwTx}, + transaction::{TxRo, TxRw}, }; -//---------------------------------------------------------------------------------------------------- RoTx -impl RoTx<'_> for sanakirja::Txn<&'_ sanakirja::Env> { +//---------------------------------------------------------------------------------------------------- TxRo +impl TxRo<'_> for redb::ReadTransaction<'_> { fn commit(self) -> Result<(), RuntimeError> { todo!() } } -//---------------------------------------------------------------------------------------------------- RwTx -impl RwTx<'_> for sanakirja::MutTxn<&'_ sanakirja::Env, ()> { +//---------------------------------------------------------------------------------------------------- TxRw +impl TxRw<'_> for redb::WriteTransaction<'_> { /// TODO /// # Errors /// TODO diff --git a/database/src/backend/redb/types.rs b/database/src/backend/redb/types.rs new file mode 100644 index 00000000..2525de53 --- /dev/null +++ b/database/src/backend/redb/types.rs @@ -0,0 +1,10 @@ +//! `redb` type aliases. + +//---------------------------------------------------------------------------------------------------- Types +// TODO: replace `()` with a byte container. + +/// The concrete type for readable `redb` tables. +pub(super) type RedbTableRo<'env> = redb::ReadOnlyTable<'env, (), ()>; + +/// The concrete type for readable/writable `redb` tables. +pub(super) type RedbTableRw<'env, 'tx> = redb::Table<'env, 'tx, (), ()>; diff --git a/database/src/backend/sanakirja/database.rs b/database/src/backend/sanakirja/database.rs deleted file mode 100644 index 9c9b8915..00000000 --- a/database/src/backend/sanakirja/database.rs +++ /dev/null @@ -1,49 +0,0 @@ -//! Implementation of `trait Database` for `sanakirja`. - -//---------------------------------------------------------------------------------------------------- Import -use crate::{ - backend::sanakirja::types::SanakirjaDb, database::Database, error::RuntimeError, table::Table, -}; - -//---------------------------------------------------------------------------------------------------- Database Impls -impl Database for SanakirjaDb { - type RoTx<'db> = sanakirja::Txn<&'db sanakirja::Env>; - type RwTx<'db> = sanakirja::MutTxn<&'db sanakirja::Env, ()>; - - fn get(&self, ro_tx: &Self::RoTx<'_>, key: &T::Key) -> Result, RuntimeError> { - todo!() - } - - fn get_range( - &self, - ro_tx: &Self::RoTx<'_>, - key: &T::Key, - amount: usize, - ) -> Result, RuntimeError> { - let iter: std::vec::Drain<'_, T::Value> = todo!(); - Ok(iter) - } - - fn put( - &mut self, - rx_tx: &mut Self::RwTx<'_>, - key: &T::Key, - value: &T::Value, - ) -> Result<(), RuntimeError> { - todo!() - } - - fn clear(&mut self, rx_tx: &mut Self::RwTx<'_>) -> Result<(), RuntimeError> { - todo!() - } - - fn delete(&mut self, rx_tx: &mut Self::RwTx<'_>, key: &T::Key) -> Result { - todo!() - } -} - -//---------------------------------------------------------------------------------------------------- Tests -#[cfg(test)] -mod test { - // use super::*; -} diff --git a/database/src/backend/sanakirja/env.rs b/database/src/backend/sanakirja/env.rs deleted file mode 100644 index 42e6e9a0..00000000 --- a/database/src/backend/sanakirja/env.rs +++ /dev/null @@ -1,93 +0,0 @@ -//! Implementation of `trait Env` for `sanakirja`. - -//---------------------------------------------------------------------------------------------------- Import -use std::{path::Path, sync::Arc}; - -use crate::{ - backend::sanakirja::types::SanakirjaDb, - config::Config, - database::Database, - env::Env, - error::{InitError, RuntimeError}, - table::Table, -}; - -//---------------------------------------------------------------------------------------------------- ConcreteEnv -/// A strongly typed, concrete database environment, backed by `sanakirja`. -pub struct ConcreteEnv { - /// The actual database environment. - env: sanakirja::Env, - - /// The configuration we were opened with - /// (and in current use). - config: Config, -} - -impl Drop for ConcreteEnv { - fn drop(&mut self) { - if let Err(e) = self.sync() { - // TODO: log error? - } - - // TODO: log that we are dropping the database. - } -} - -//---------------------------------------------------------------------------------------------------- Env Impl -impl Env for ConcreteEnv { - const MANUAL_RESIZE: bool = false; - const SYNCS_PER_TX: bool = true; - /// FIXME: - /// We could also implement `Borrow for ConcreteEnv` - /// instead of this reference. - type RoTx<'db> = sanakirja::Txn<&'db sanakirja::Env>; - type RwTx<'db> = sanakirja::MutTxn<&'db sanakirja::Env, ()>; - - #[cold] - #[inline(never)] // called once. - fn open(config: Config) -> Result { - todo!() - } - - fn config(&self) -> &Config { - &self.config - } - - fn sync(&self) -> Result<(), RuntimeError> { - todo!() - } - - #[inline] - fn ro_tx(&self) -> Result, RuntimeError> { - todo!() - } - - #[inline] - fn rw_tx(&self) -> Result, RuntimeError> { - todo!() - } - - #[cold] - #[inline(never)] // called infrequently?. - fn create_tables_if_needed( - &self, - tx_rw: &mut Self::RwTx<'_>, - ) -> Result<(), RuntimeError> { - todo!() - } - - #[inline] - fn open_database( - &self, - to_rw: &Self::RoTx<'_>, - ) -> Result, RuntimeError> { - let tx: SanakirjaDb = todo!(); - Ok(tx) - } -} - -//---------------------------------------------------------------------------------------------------- Tests -#[cfg(test)] -mod test { - // use super::*; -} diff --git a/database/src/backend/sanakirja/error.rs b/database/src/backend/sanakirja/error.rs deleted file mode 100644 index 9fa0bb1c..00000000 --- a/database/src/backend/sanakirja/error.rs +++ /dev/null @@ -1,51 +0,0 @@ -//! Conversion from `sanakirja::Error` -> `cuprate_database::RuntimeError`. - -//---------------------------------------------------------------------------------------------------- Import -use crate::constants::DATABASE_CORRUPT_MSG; - -//---------------------------------------------------------------------------------------------------- InitError -impl From for crate::InitError { - fn from(error: sanakirja::Error) -> Self { - use sanakirja::Error as E; - - match error { - E::IO(io_error) => Self::Io(io_error), - E::VersionMismatch => Self::InvalidVersion, - - // A CRC failure essentially means a `sanakirja` page was corrupt. - // - E::Corrupt(_) | E::CRC(_) => Self::Corrupt, - - // A database lock was poisoned. - // - E::Poison => Self::Unknown(Box::new(error)), - } - } -} - -//---------------------------------------------------------------------------------------------------- RuntimeError -#[allow(clippy::fallible_impl_from)] // We need to panic sometimes. -impl From for crate::RuntimeError { - fn from(error: sanakirja::Error) -> Self { - use sanakirja::Error as E; - - match error { - E::IO(io_error) => Self::Io(io_error), - - // A CRC failure essentially means a `sanakirja` page was corrupt. - // - E::Corrupt(_) | E::CRC(_) => panic!("{error:?}\n{DATABASE_CORRUPT_MSG}"), - - // These errors should not occur, and if they do, - // the best thing `cuprate_database` can do for - // safety is to panic right here. - E::Poison | E::VersionMismatch => panic!("{error:?}"), - } - } -} - -//---------------------------------------------------------------------------------------------------- Tests -#[cfg(test)] -mod test { - // use super::*; -} diff --git a/database/src/backend/sanakirja/types.rs b/database/src/backend/sanakirja/types.rs deleted file mode 100644 index d3383e57..00000000 --- a/database/src/backend/sanakirja/types.rs +++ /dev/null @@ -1,6 +0,0 @@ -//! `sanakirja` type aliases. - -//---------------------------------------------------------------------------------------------------- Types -/// The concrete database type for `sanakirja`. -pub(super) type SanakirjaDb = - sanakirja::btree::Db_<[u8], [u8], sanakirja::btree::page_unsized::Page<[u8], [u8]>>; diff --git a/database/src/config.rs b/database/src/config.rs index 7541c4c7..25d62367 100644 --- a/database/src/config.rs +++ b/database/src/config.rs @@ -17,7 +17,7 @@ use std::{ use cuprate_helper::fs::cuprate_database_dir; -use crate::{constants::DATABASE_FILENAME, resize::ResizeAlgorithm}; +use crate::{constants::DATABASE_DATA_FILENAME, resize::ResizeAlgorithm}; //---------------------------------------------------------------------------------------------------- Config /// Database [`Env`](crate::Env) configuration. @@ -72,7 +72,7 @@ impl Config { // Add the database filename to the directory. let mut db_file = db_directory.to_path_buf(); - db_file.push(DATABASE_FILENAME); + db_file.push(DATABASE_DATA_FILENAME); (db_directory, Cow::Owned(db_file)) } @@ -175,15 +175,15 @@ impl Default for Config { /// will always cause it to fully sync to disk. /// /// # Sync vs Async -/// All invariants except [`SyncMode::Fast`] are `synchronous`, -/// as in the database will wait until the OS has finished syncing -/// all the data to disk before continuing. +/// All invariants except [`SyncMode::Async`] & [`SyncMode::Fast`] +/// are `synchronous`, as in the database will wait until the OS has +/// finished syncing all the data to disk before continuing. /// -/// `SyncMode::Fast` is `asynchronous`, meaning the database will _NOT_ -/// wait until the data is fully synced to disk before continuing. -/// Note that this doesn't mean the database itself won't be synchronized -/// between readers/writers, but rather that the data _on disk_ may not -/// be immediately synchronized after a write. +/// `SyncMode::Async` & `SyncMode::Fast` are `asynchronous`, meaning +/// the database will _NOT_ wait until the data is fully synced to disk +/// before continuing. Note that this doesn't mean the database itself +/// won't be synchronized between readers/writers, but rather that the +/// data _on disk_ may not be immediately synchronized after a write. /// /// Something like: /// ```rust,ignore @@ -191,6 +191,17 @@ impl Default for Config { /// db.get("key"); /// ``` /// will be fine, most likely pulling from memory instead of disk. +/// +/// # TODO +/// Dynamic sync's are not yet supported. +/// +/// Only: +/// +/// - [`SyncMode::Safe`] +/// - [`SyncMode::Async`] +/// - [`SyncMode::Fast`] +/// +/// are supported, all other variants will panic on [`crate::Env::open`]. #[derive(Copy, Clone, Debug, Default, PartialEq, PartialOrd, Eq, Ord, Hash)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] #[cfg_attr( @@ -226,11 +237,27 @@ pub enum SyncMode { /// /// Every database transaction commit will /// fully sync all data to disk, _synchronously_, - /// so the database halts until synced. + /// so the database (writer) halts until synced. /// /// This is expected to be very slow. + /// + /// This matches: + /// - LMDB without any special sync flags + /// - [`redb::Durability::Immediate`](https://docs.rs/redb/1.5.0/redb/enum.Durability.html#variant.Immediate) Safe, + /// Asynchrously sync to disk per transaction. + /// + /// This is the same as [`SyncMode::Safe`], + /// but the syncs will be asynchronous, i.e. + /// each transaction commit will sync to disk, + /// but only eventually, not necessarily immediately. + /// + /// This matches: + /// - [`MDB_MAPASYNC`](http://www.lmdb.tech/doc/group__mdb__env.html#gab034ed0d8e5938090aef5ee0997f7e94) + /// - [`redb::Durability::Eventual`](https://docs.rs/redb/1.5.0/redb/enum.Durability.html#variant.Eventual) + Async, + /// Fully sync to disk after we cross this transaction threshold. /// /// After committing [`usize`] amount of database @@ -247,6 +274,12 @@ pub enum SyncMode { /// It will cause the database to never _actively_ sync, /// letting the OS decide when to flush data to disk. /// + /// This matches: + /// - [`MDB_NOSYNC`](http://www.lmdb.tech/doc/group__mdb__env.html#ga5791dd1adb09123f82dd1f331209e12e) + [`MDB_MAPASYNC`](http://www.lmdb.tech/doc/group__mdb__env.html#gab034ed0d8e5938090aef5ee0997f7e94) + /// - [`redb::Durability::None`](https://docs.rs/redb/1.5.0/redb/enum.Durability.html#variant.None) + /// + /// `monerod` reference: + /// /// # Corruption /// In the case of a system crash, the database /// may become corrupted when using this option. diff --git a/database/src/constants.rs b/database/src/constants.rs index 48df2551..37a3efaf 100644 --- a/database/src/constants.rs +++ b/database/src/constants.rs @@ -1,6 +1,7 @@ //! General constants used throughout `cuprate-database`. //---------------------------------------------------------------------------------------------------- Import +use cfg_if::cfg_if; //---------------------------------------------------------------------------------------------------- Error Messages /// Corrupt database error message. @@ -18,27 +19,55 @@ TODO: instructions on: 4. etc"; //---------------------------------------------------------------------------------------------------- Misc -cfg_if::cfg_if! { - // If both backends are enabled, fallback to `heed`. - // This is useful when using `--all-features`. - if #[cfg(all(feature = "sanakirja", not(feature = "heed")))] { - /// Static string of the `crate` being used as the database backend. - pub const DATABASE_BACKEND: &str = "sanakirja"; - - /// Cuprate's database filename. - /// - /// This is the filename for Cuprate's database, used in [`Config::db_file`](crate::config::Config::db_file). - pub const DATABASE_FILENAME: &str = "data.san"; // TODO: pick a name + extension. - } else { - /// Static string of the `crate` being used as the database backend. - pub const DATABASE_BACKEND: &str = "heed"; - - /// Cuprate's database filename. - /// - /// This is the filename for Cuprate's database, used in [`Config::db_file`](crate::config::Config::db_file). - pub const DATABASE_FILENAME: &str = "data.mdb"; +/// Static string of the `crate` being used as the database backend. +/// +/// | Backend | Value | +/// |---------|-------| +/// | `heed` | "heed" +/// | `redb` | "redb" +pub const DATABASE_BACKEND: &str = { + cfg_if! { + if #[cfg(all(feature = "redb", not(feature = "heed")))] { + "redb" + } else { + "heed" + } } -} +}; + +/// Cuprate's database filename. +/// +/// Used in [`Config::db_file`](crate::config::Config::db_file). +/// +/// | Backend | Value | +/// |---------|-------| +/// | `heed` | "data.mdb" +/// | `redb` | "data.redb" +pub const DATABASE_DATA_FILENAME: &str = { + cfg_if! { + if #[cfg(all(feature = "redb", not(feature = "heed")))] { + "data.redb" + } else { + "data.mdb" + } + } +}; + +/// Cuprate's database lock filename. +/// +/// | Backend | Value | +/// |---------|-------| +/// | `heed` | Some("lock.mdb") +/// | `redb` | None (redb doesn't use a file lock) +pub const DATABASE_LOCK_FILENAME: Option<&str> = { + cfg_if! { + if #[cfg(all(feature = "redb", not(feature = "heed")))] { + None + } else { + Some("lock.mdb") + } + } +}; //---------------------------------------------------------------------------------------------------- Tests #[cfg(test)] diff --git a/database/src/database.rs b/database/src/database.rs index def9645a..2ff3e2cf 100644 --- a/database/src/database.rs +++ b/database/src/database.rs @@ -1,58 +1,45 @@ -//! Abstracted database; `trait Database`. +//! Abstracted database; `trait DatabaseRo` & `trait DatabaseRw`. //---------------------------------------------------------------------------------------------------- Import -use crate::{ - error::RuntimeError, - table::Table, - transaction::{RoTx, RwTx}, -}; +use crate::{error::RuntimeError, table::Table}; -//---------------------------------------------------------------------------------------------------- Database -/// Database (key-value store) abstraction. +//---------------------------------------------------------------------------------------------------- DatabaseRo +/// Database (key-value store) read abstraction. /// -/// TODO -pub trait Database { - //------------------------------------------------ Types - /// TODO - type RoTx<'db>: RoTx<'db>; - - /// TODO - type RwTx<'db>: RwTx<'db>; - - //-------------------------------------------------------- Read +/// TODO: document relation between `DatabaseRo` <-> `DatabaseRw`. +pub trait DatabaseRo { /// TODO /// # Errors /// TODO - fn get(&self, ro_tx: &Self::RoTx<'_>, key: &T::Key) -> Result, RuntimeError>; + fn get(&self, key: &T::Key) -> Result, RuntimeError>; /// TODO /// # Errors /// TODO fn get_range( &self, - ro_tx: &Self::RoTx<'_>, key: &T::Key, amount: usize, ) -> Result, RuntimeError>; - - //-------------------------------------------------------- Write - /// TODO - /// # Errors - /// TODO - fn put( - &mut self, - rw_tx: &mut Self::RwTx<'_>, - key: &T::Key, - value: &T::Value, - ) -> Result<(), RuntimeError>; - - /// TODO - /// # Errors - /// TODO - fn clear(&mut self, rw_tx: &mut Self::RwTx<'_>) -> Result<(), RuntimeError>; - - /// TODO - /// # Errors - /// TODO - fn delete(&mut self, rw_tx: &mut Self::RwTx<'_>, key: &T::Key) -> Result; +} + +//---------------------------------------------------------------------------------------------------- DatabaseRw +/// Database (key-value store) read/write abstraction. +/// +/// TODO: document relation between `DatabaseRo` <-> `DatabaseRw`. +pub trait DatabaseRw: DatabaseRo { + /// TODO + /// # Errors + /// TODO + fn put(&mut self, key: &T::Key, value: &T::Value) -> Result<(), RuntimeError>; + + /// TODO + /// # Errors + /// TODO + fn clear(&mut self) -> Result<(), RuntimeError>; + + /// TODO + /// # Errors + /// TODO + fn delete(&mut self, key: &T::Key) -> Result; } diff --git a/database/src/env.rs b/database/src/env.rs index 4b434a5e..989aedb7 100644 --- a/database/src/env.rs +++ b/database/src/env.rs @@ -3,11 +3,11 @@ //---------------------------------------------------------------------------------------------------- Import use crate::{ config::Config, - database::Database, + database::{DatabaseRo, DatabaseRw}, error::{InitError, RuntimeError}, resize::ResizeAlgorithm, table::Table, - transaction::{RoTx, RwTx}, + transaction::{TxRo, TxRw}, }; //---------------------------------------------------------------------------------------------------- Env @@ -40,10 +40,10 @@ pub trait Env: Sized { //------------------------------------------------ Types /// TODO - type RoTx<'db>: RoTx<'db>; + type TxRo<'env>: TxRo<'env>; /// TODO - type RwTx<'db>: RwTx<'db>; + type TxRw<'env>: TxRw<'env>; //------------------------------------------------ Required /// TODO @@ -51,6 +51,11 @@ pub trait Env: Sized { /// TODO fn open(config: Config) -> Result; + /// TODO + /// # Errors + /// TODO + fn create_tables(&self, tx_rw: &mut Self::TxRw<'_>) -> Result<(), RuntimeError>; + /// Return the [`Config`] that this database was [`Env::open`]ed with. fn config(&self) -> &Config; @@ -79,6 +84,16 @@ pub trait Env: Sized { } /// TODO + /// + /// # Invariant + /// This must **fully** and **synchronously** flush the database data to disk. + /// + /// I.e., after this function returns, there must be no doubts + /// that the data isn't synced yet, it _must_ be synced. + /// + /// TODO: either this invariant or `sync()` itself will most + /// likely be removed/changed after `SyncMode` is finalized. + /// /// # Errors /// TODO fn sync(&self) -> Result<(), RuntimeError>; @@ -110,36 +125,44 @@ pub trait Env: Sized { /// TODO /// # Errors /// TODO - fn ro_tx(&self) -> Result, RuntimeError>; + fn tx_ro(&self) -> Result, RuntimeError>; /// TODO /// # Errors /// TODO - fn rw_tx(&self) -> Result, RuntimeError>; - - /// TODO - /// # Errors - /// TODO - fn create_tables_if_needed( - &self, - rw_tx: &mut Self::RwTx<'_>, - ) -> Result<(), RuntimeError>; + fn tx_rw(&self) -> Result, RuntimeError>; /// TODO /// /// # TODO: Invariant /// This should never panic the database because the table doesn't exist. /// - /// Opening/using the database env should have an invariant + /// Opening/using the database [`Env`] should have an invariant /// that it creates all the tables we need, such that this /// never returns `None`. /// /// # Errors /// TODO - fn open_database( + fn open_db_ro( &self, - ro_tx: &Self::RoTx<'_>, - ) -> Result, RuntimeError>; + tx_ro: &Self::TxRo<'_>, + ) -> Result, RuntimeError>; + + /// TODO + /// + /// # TODO: Invariant + /// This should never panic the database because the table doesn't exist. + /// + /// Opening/using the database [`Env`] should have an invariant + /// that it creates all the tables we need, such that this + /// never returns `None`. + /// + /// # Errors + /// TODO + fn open_db_rw( + &self, + tx_rw: &mut Self::TxRw<'_>, + ) -> Result, RuntimeError>; //------------------------------------------------ Provided } diff --git a/database/src/lib.rs b/database/src/lib.rs index 3f0b7aac..f350b75d 100644 --- a/database/src/lib.rs +++ b/database/src/lib.rs @@ -7,28 +7,29 @@ //! //! # Purpose //! This crate does 3 things: -//! 1. Abstracts various databases with the [`Env`], [`Database`], [`Table`], [`Key`], [`RoTx`], and [`RwTx`] traits +//! 1. Abstracts various database backends with traits //! 2. Implements various `Monero` related [functions](ops) & [`tables`] //! 3. Exposes a [`tower::Service`] backed by a thread-pool //! //! # Terminology //! To be more clear on some terms used in this crate: //! -//! | Term | Meaning | -//! |------------|--------------------------------------| -//! | `Env` | The 1 database environment, the "whole" thing -//! | `Database` | A `key/value` store -//! | `Table` | Solely the metadata of a `Database` (the `key` and `value` types, and the name) -//! | `RoTx` | Read only transaction -//! | `RwTx` | Read/write transaction +//! | Term | Meaning | +//! |---------------|--------------------------------------| +//! | `Env` | The 1 database environment, the "whole" thing +//! | `DatabaseRo` | A read-only `key/value` store +//! | `DatabaseRw` | A readable/writable `key/value` store +//! | `Table` | Solely the metadata of a `Database` (the `key` and `value` types, and the name) +//! | `TxRo` | Read only transaction +//! | `TxRw` | Read/write transaction //! //! The dataflow is `Env` -> `Tx` -> `Database` //! //! Which reads as: //! 1. You have a database `Environment` -//! 2. You open up a `Transaction` -//! 2. You get a particular `Database` from that `Environment` -//! 3. You can now read/write data from/to that `Database` +//! 1. You open up a `Transaction` +//! 1. You get a particular `Database` from that `Environment` +//! 1. You can now read/write data from/to that `Database` //! //! # `ConcreteEnv` //! This crate exposes [`ConcreteEnv`], which is a non-generic/non-dynamic, @@ -60,13 +61,18 @@ //! going to be storing any databases in structs, to lessen //! the generic `` pain. //! +//! TODO: we could replace `ConcreteEnv` with `fn Env::open() -> impl Env`/ +//! and use `` everywhere it is stored instead. This would allow +//! generic-backed dynamic runtime selection of the database backend, i.e. +//! the user can select which database backend they use. +//! //! # Feature flags //! The `service` module requires the `service` feature to be enabled. //! See the module for more documentation. //! //! Different database backends are enabled by the feature flags: -//! - `heed` -//! - `sanakirja` +//! - `heed` (LMDB) +//! - `redb` //! //! The default is `heed`. //! @@ -92,7 +98,7 @@ //! use cuprate_database::{ //! config::Config, //! ConcreteEnv, -//! Env, Key, RoTx, RwTx, +//! Env, Key, TxRo, TxRw, //! service::{ReadRequest, WriteRequest, Response}, //! }; //! @@ -166,7 +172,7 @@ unused_comparisons, nonstandard_style )] -#![allow(unreachable_code, unused_variables, dead_code)] // TODO: remove +#![allow(unreachable_code, unused_variables, dead_code, unused_imports)] // TODO: remove #![allow( // FIXME: this lint affects crates outside of // `database/` for some reason, allow for now. @@ -289,10 +295,12 @@ pub use backend::ConcreteEnv; pub mod config; mod constants; -pub use constants::{DATABASE_BACKEND, DATABASE_CORRUPT_MSG, DATABASE_FILENAME}; +pub use constants::{ + DATABASE_BACKEND, DATABASE_CORRUPT_MSG, DATABASE_DATA_FILENAME, DATABASE_LOCK_FILENAME, +}; mod database; -pub use database::Database; +pub use database::{DatabaseRo, DatabaseRw}; mod env; pub use env::Env; @@ -320,7 +328,7 @@ pub use table::Table; pub mod tables; mod transaction; -pub use transaction::{RoTx, RwTx}; +pub use transaction::{TxRo, TxRw}; //---------------------------------------------------------------------------------------------------- Feature-gated #[cfg(feature = "service")] diff --git a/database/src/ops/alt_block.rs b/database/src/ops/alt_block.rs index 87663ca3..82f33aaf 100644 --- a/database/src/ops/alt_block.rs +++ b/database/src/ops/alt_block.rs @@ -1,14 +1,6 @@ //! Alternative blocks. //---------------------------------------------------------------------------------------------------- Import -#[allow(unused_imports)] // FIXME: these traits will be eventually in the function impls. -use crate::{ - database::Database, - env::Env, - table::Table, - transaction::{RoTx, RwTx}, - ConcreteEnv, -}; //---------------------------------------------------------------------------------------------------- Free Functions /// TODO diff --git a/database/src/ops/block.rs b/database/src/ops/block.rs index d9a9d811..361b391b 100644 --- a/database/src/ops/block.rs +++ b/database/src/ops/block.rs @@ -1,14 +1,6 @@ //! Blocks. //---------------------------------------------------------------------------------------------------- Import -#[allow(unused_imports)] // FIXME: these traits will be eventually in the function impls. -use crate::{ - database::Database, - env::Env, - table::Table, - transaction::{RoTx, RwTx}, - ConcreteEnv, -}; //---------------------------------------------------------------------------------------------------- Free Functions /// TODO diff --git a/database/src/ops/blockchain.rs b/database/src/ops/blockchain.rs index 876a23f3..28c2284c 100644 --- a/database/src/ops/blockchain.rs +++ b/database/src/ops/blockchain.rs @@ -1,14 +1,6 @@ //! Blockchain. //---------------------------------------------------------------------------------------------------- Import -#[allow(unused_imports)] // FIXME: these traits will be eventually in the function impls. -use crate::{ - database::Database, - env::Env, - table::Table, - transaction::{RoTx, RwTx}, - ConcreteEnv, -}; //---------------------------------------------------------------------------------------------------- Free Functions /// TODO diff --git a/database/src/ops/output.rs b/database/src/ops/output.rs index ae7871e4..e0db143f 100644 --- a/database/src/ops/output.rs +++ b/database/src/ops/output.rs @@ -1,14 +1,6 @@ //! Outputs. //---------------------------------------------------------------------------------------------------- Import -#[allow(unused_imports)] // FIXME: these traits will be eventually in the function impls. -use crate::{ - database::Database, - env::Env, - table::Table, - transaction::{RoTx, RwTx}, - ConcreteEnv, -}; //---------------------------------------------------------------------------------------------------- Free Functions /// TODO diff --git a/database/src/ops/property.rs b/database/src/ops/property.rs index 3a963b64..8801ae8f 100644 --- a/database/src/ops/property.rs +++ b/database/src/ops/property.rs @@ -1,14 +1,6 @@ //! Properties. //---------------------------------------------------------------------------------------------------- Import -#[allow(unused_imports)] // FIXME: these traits will be eventually in the function impls. -use crate::{ - database::Database, - env::Env, - table::Table, - transaction::{RoTx, RwTx}, - ConcreteEnv, -}; //---------------------------------------------------------------------------------------------------- Free Functions /// TODO diff --git a/database/src/ops/spent_key.rs b/database/src/ops/spent_key.rs index ef02d850..a8e6fe02 100644 --- a/database/src/ops/spent_key.rs +++ b/database/src/ops/spent_key.rs @@ -1,14 +1,6 @@ //! Spent keys. //---------------------------------------------------------------------------------------------------- Import -#[allow(unused_imports)] // FIXME: these traits will be eventually in the function impls. -use crate::{ - database::Database, - env::Env, - table::Table, - transaction::{RoTx, RwTx}, - ConcreteEnv, -}; //---------------------------------------------------------------------------------------------------- Free Functions /// TODO diff --git a/database/src/ops/tx.rs b/database/src/ops/tx.rs index 511c28d0..9acafc9c 100644 --- a/database/src/ops/tx.rs +++ b/database/src/ops/tx.rs @@ -1,14 +1,6 @@ //! Transactions. //---------------------------------------------------------------------------------------------------- Import -#[allow(unused_imports)] // FIXME: these traits will be eventually in the function impls. -use crate::{ - database::Database, - env::Env, - table::Table, - transaction::{RoTx, RwTx}, - ConcreteEnv, -}; //---------------------------------------------------------------------------------------------------- Free Functions /// TODO diff --git a/database/src/resize.rs b/database/src/resize.rs index c77c1ee2..1540c690 100644 --- a/database/src/resize.rs +++ b/database/src/resize.rs @@ -29,7 +29,7 @@ use std::{num::NonZeroUsize, sync::OnceLock}; /// /// # TODO /// We could test around with different algorithms. -/// Calling [`heed::Env::resize`] is surprisingly fast, +/// Calling `heed::Env::resize` is surprisingly fast, /// around `0.0000082s` on my machine. We could probably /// get away with smaller and more frequent resizes. /// **With the caveat being we are taking a `WriteGuard` to a `RwLock`.** diff --git a/database/src/transaction.rs b/database/src/transaction.rs index fe3dc72a..74678836 100644 --- a/database/src/transaction.rs +++ b/database/src/transaction.rs @@ -1,24 +1,24 @@ -//! Database transaction abstraction; `trait RoTx`, `trait RwTx`. +//! Database transaction abstraction; `trait TxRo`, `trait TxRw`. //---------------------------------------------------------------------------------------------------- Import -use crate::error::RuntimeError; +use crate::{config::SyncMode, env::Env, error::RuntimeError}; -//---------------------------------------------------------------------------------------------------- RoTx +//---------------------------------------------------------------------------------------------------- TxRo /// Read-only database transaction. /// /// TODO -pub trait RoTx<'db> { +pub trait TxRo<'env> { /// TODO /// # Errors /// TODO fn commit(self) -> Result<(), RuntimeError>; } -//---------------------------------------------------------------------------------------------------- RwTx +//---------------------------------------------------------------------------------------------------- TxRw /// Read/write database transaction. /// /// TODO -pub trait RwTx<'db> { +pub trait TxRw<'env> { /// TODO /// # Errors /// TODO