SyntheticBird/cuprate-hinto-janai
1
0
Fork 0
mirror of https://github.com/hinto-janai/cuprate.git synced 2024-12-22 11:39:30 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

database: replace sanakirja with redb (#80)

Browse source 
* cargo: replace `sanakirja` with `redb`

* database: update docs `sanakirja` -> `redb`

* lib: add TODO for `ConcreteEnv` generic replacement

* database: split `trait Database` -> `trait Database{Read,Write}`

* heed: add `struct HeedTable{Ro,Rw}` to match `redb` behavior

* ops: remove imports for now

* env: fix `&mut` bound on RwTx

* database: impl `redb`, type-checks

* fix heed trait impls, `Database{Read,Write}` -> `Database{Ro,Rw}`

* redb: impl `From<_>` for `RuntimeError`

* update readme

* heed: document `HeedTableR{o,w}` types

* env: doc `sync()` invariant

* database: document data & lock filenames

* misc docs, `redb` durability impl, `'db` -> `'env`

* redb: fixes

* misc docs and fixes

* Update database/README.md

Co-authored-by: Boog900 <boog900@tutanota.com>

---------

Co-authored-by: Boog900 <boog900@tutanota.com>
This commit is contained in:
hinto-janai 2024-02-29 12:40:15 -05:00 committed by GitHub
parent 312e29f0bf
commit 240e579066
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
33 changed files with 689 additions and 531 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

95
Cargo.lock generated
View file

@ -578,7 +578,7 @@ dependencies = [
"heed",
"page_size",
"paste",
"sanakirja",
"redb",
"serde",
"tempfile",
"thiserror",
@ -838,7 +838,7 @@ checksum = "1ee447700ac8aa0b2f2bd7bc4462ad686ba06baa6727ac149a2d6277f0d240fd"
dependencies = [
"cfg-if",
"libc",
"redox_syscall 0.4.1",
"redox_syscall",
"windows-sys 0.52.0",
]
@ -903,16 +903,6 @@ dependencies = [
"percent-encoding",
]
[[package]]
name = "fs2"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213"
dependencies = [
"libc",
"winapi",
]
[[package]]
name = "funty"
version = "2.0.0"
@ -1371,15 +1361,6 @@ dependencies = [
"generic-array",
]
[[package]]
name = "instant"
version = "0.1.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c"
dependencies = [
"cfg-if",
]
[[package]]
name = "ipnet"
version = "2.9.0"
@ -1445,7 +1426,7 @@ checksum = "85c833ca1e66078851dba29046874e38f08b2c883700aa29a03ddd3b23814ee8"
dependencies = [
"bitflags 2.4.2",
"libc",
"redox_syscall 0.4.1",
"redox_syscall",
]
[[package]]
@ -1496,16 +1477,6 @@ version = "2.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149"
[[package]]
name = "memmap"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6585fd95e7bb50d6cc31e20d4cf9afb4e2ba16c5846fc76793f11218da9c475b"
dependencies = [
"libc",
"winapi",
]
[[package]]
name = "merlin"
version = "3.0.0"
@ -1827,17 +1798,6 @@ version = "2.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bb813b8af86854136c6922af0598d719255ecb2179515e6e7730d468f05c9cae"
[[package]]
name = "parking_lot"
version = "0.11.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99"
dependencies = [
"instant",
"lock_api",
"parking_lot_core 0.8.6",
]
[[package]]
name = "parking_lot"
version = "0.12.1"
@ -1845,21 +1805,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f"
dependencies = [
"lock_api",
"parking_lot_core 0.9.9",
]
[[package]]
name = "parking_lot_core"
version = "0.8.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc"
dependencies = [
"cfg-if",
"instant",
"libc",
"redox_syscall 0.2.16",
"smallvec",
"winapi",
"parking_lot_core",
]
[[package]]
@ -1870,7 +1816,7 @@ checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e"
dependencies = [
"cfg-if",
"libc",
"redox_syscall 0.4.1",
"redox_syscall",
"smallvec",
"windows-targets 0.48.5",
]
@ -2204,12 +2150,12 @@ dependencies = [
]
[[package]]
name = "redox_syscall"
version = "0.2.16"
name = "redb"
version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a"
checksum = "72623e6275cd430215b741f41ebda34db93a13ebde253f908b70871c46afc5ba"
dependencies = [
"bitflags 1.3.2",
"libc",
]
[[package]]
@ -2427,27 +2373,6 @@ version = "1.0.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e86697c916019a8588c99b5fac3cead74ec0b4b819707a682fd4d23fa0ce1ba1"
[[package]]
name = "sanakirja"
version = "1.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7c385eb43079aa7dc6204e473b68b4305ceaea8048dda3a985a339bbb57cde72"
dependencies = [
"fs2",
"log",
"memmap",
"parking_lot 0.11.2",
"sanakirja-core",
"serde",
"thiserror",
]
[[package]]
name = "sanakirja-core"
version = "1.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8376db34ae3eac6e7bd91168bc638450073b708ce9fb46940de676f552238bf5"
[[package]]
name = "schannel"
version = "0.1.23"
@ -2851,7 +2776,7 @@ dependencies = [
"libc",
"mio",
"num_cpus",
"parking_lot 0.12.1",
"parking_lot",
"pin-project-lite",
"signal-hook-registry",
"socket2",

32
database/Cargo.toml
View file

@ -10,27 +10,27 @@ keywords = ["cuprate", "database"]
[features]
default = ["heed", "service"]
# default = ["sanakirja", "service"] # For testing `sanakirja`.
heed = ["dep:heed"]
sanakirja = ["dep:sanakirja"]
service = ["dep:crossbeam", "dep:tokio", "dep:tower"]
# default = ["redb", "service"] # For testing `redb`.
heed = ["dep:heed"]
redb = ["dep:redb"]
service = ["dep:crossbeam", "dep:tokio", "dep:tower"]
[dependencies]
cfg-if = { workspace = true }
cfg-if = { workspace = true }
# FIXME:
# We only need the `thread` feature if `service` is enabled.
# Figure out how to enable features of an already pulled in dependency conditionally.
cuprate-helper = { path = "../helper", features = ["fs", "thread"] }
paste = { workspace = true }
paste = { workspace = true }
# Needed for database resizes.
# They must be a multiple of the OS page size.
page_size = { version = "0.6.0" }
thiserror = { workspace = true }
page_size = { version = "0.6.0" }
thiserror = { workspace = true }
# `service` feature.
crossbeam = { workspace = true, features = ["std"], optional = true }
tokio = { workspace = true, features = ["full"], optional = true }
tower = { workspace = true, features = ["full"], optional = true }
crossbeam = { workspace = true, features = ["std"], optional = true }
tokio = { workspace = true, features = ["full"], optional = true }
tower = { workspace = true, features = ["full"], optional = true }
# SOMEDAY: could be used in `service` as
# the database mutual exclusive `RwLock`.
#
@ -40,12 +40,12 @@ tower = { workspace = true, features = ["full"], optional = true }
# parking_lot = { workspace = true, optional = true }
# Optional features.
borsh = { workspace = true, optional = true }
heed = { git = "https://github.com/Cuprate/heed", rev = "5aa75b7", optional = true }
sanakirja = { version = "1.4.0", optional = true }
serde = { workspace = true, optional = true }
borsh = { workspace = true, optional = true }
heed = { git = "https://github.com/Cuprate/heed", rev = "5aa75b7", optional = true }
redb = { version = "1.5.0", optional = true }
serde = { workspace = true, optional = true }
[dev-dependencies]
cuprate-helper = { path = "../helper", features = ["thread"] }
page_size = { version = "0.6.0" }
tempfile = { version = "3.10.0" }
tempfile = { version = "3.10.0" }

41
database/README.md
View file

@ -10,7 +10,9 @@ Cuprate's database implementation.
- [`src/backend/`](#src-backend)
1. [Backends](#backends)
- [`heed`](#heed)
- [`redb`](#redb)
- [`sanakirja`](#sanakirja)
- [`MDBX`](#mdbx)
1. [Layers](#layers)
- [Database](#database)
- [Trait](#trait)
@ -68,7 +70,7 @@ The top-level `src/` files.
|------------------|---------|
| `config.rs` | Database `Env` configuration
| `constants.rs` | General constants used throughout `cuprate-database`
| `database.rs` | Abstracted database; `trait Database`
| `database.rs` | Abstracted database; `trait DatabaseR{o,w}`
| `env.rs` | Abstracted database environment; `trait Env`
| `error.rs` | Database error types
| `free.rs` | General free functions (related to the database)
@ -76,7 +78,7 @@ The top-level `src/` files.
| `pod.rs` | Data (de)serialization; `trait Pod`
| `table.rs` | Database table abstraction; `trait Table`
| `tables.rs` | All the table definitions used by `cuprate-database`
| `transaction.rs` | Database transaction abstraction; `trait RoTx`, `trait RwTx`
| `transaction.rs` | Database transaction abstraction; `trait TxR{o,w}`
## `src/ops/`
This folder contains the `cupate_database::ops` module.
@ -119,10 +121,10 @@ All backends follow the same file structure:
| File | Purpose |
|------------------|---------|
| `database.rs` | Implementation of `trait Database`
| `database.rs` | Implementation of `trait DatabaseR{o,w}`
| `env.rs` | Implementation of `trait Env`
| `error.rs` | Implementation of backend's errors to `cuprate_database`'s error types
| `transaction.rs` | Implementation of `trait RoTx/RwTx`
| `transaction.rs` | Implementation of `trait TxR{o,w}`
| `types.rs` | Type aliases for long backend-specific types
# Backends
@ -131,7 +133,7 @@ All backends follow the same file structure:
Each database's implementation is located in its respective file in `src/backend/${DATABASE_NAME}.rs`.
## `heed`
The default database used is a modified fork of [`heed`](https://github.com/meilisearch/heed), located at [`Cuprate/heed`](https://github.com/Cuprate/heed).
The default database used is a modified fork of [`heed`](https://github.com/meilisearch/heed) (LMDB), located at [`Cuprate/heed`](https://github.com/Cuprate/heed).
To generate documentation of the fork for local use:
```bash
@ -140,10 +142,37 @@ cargo doc
```
`LMDB` should not need to be installed as `heed` has a build script that pulls it in automatically.
`heed`'s filenames inside Cuprate's database folder (`~/.local/share/cuprate/database/`) are:
| Filename | Purpose |
|------------|---------|
| `data.mdb` | Main data file
| `lock.mdb` | Database lock file
TODO: document max readers limit: https://github.com/monero-project/monero/blob/059028a30a8ae9752338a7897329fe8012a310d5/src/blockchain_db/lmdb/db_lmdb.cpp#L1372. Other potential processes (e.g. `xmrblocks`) that are also reading the `data.mdb` file need to be accounted for.
## `redb`
The 2nd database backend is the 100% Rust [`redb`](https://github.com/cberner/redb).
The upstream versions from [`crates.io`](https://crates.io/crates/redb) are used.
`redb`'s filenames inside Cuprate's database folder (`~/.local/share/cuprate/database/`) are:
| Filename | Purpose |
|-------------|---------|
| `data.redb` | Main data file
## `sanakirja`
TODO
[`sanakirja`](https://docs.rs/sanakirja) was a candidate as a backend, however there were problems with maximum value sizes.
The default maximum value size is [1012 bytes](https://docs.rs/sanakirja/1.4.1/sanakirja/trait.Storable.html) which was too small for our requirements. Using [`sanakirja::Slice`](https://docs.rs/sanakirja/1.4.1/sanakirja/union.Slice.html) and [sanakirja::UnsizedStorage](https://docs.rs/sanakirja/1.4.1/sanakirja/trait.UnsizedStorable.html) was attempted, but there were bugs found when inserting a value in-between `512..=4096` bytes.
As such, it is not implemented.
## `MDBX`
[`MDBX`](https://erthink.github.io/libmdbx) was a candidate as a backend, however MDBX deprecated the custom key/value comparison functions, this makes it a bit trickier to implement dup tables. It is also quite similar to the main backend LMDB (of which it was originally a fork of).
As such, it is not implemented (yet).
# Layers
TODO: update with accurate information when ready, update image.

84
database/src/backend/heed/database.rs
View file

@ -1,41 +1,97 @@
//! Implementation of `trait Database` for `heed`.
//---------------------------------------------------------------------------------------------------- Import
use crate::{backend::heed::types::HeedDb, database::Database, error::RuntimeError, table::Table};
use std::marker::PhantomData;
//---------------------------------------------------------------------------------------------------- Database Impls
impl<T: Table> Database<T> for HeedDb {
type RoTx<'db> = heed::RoTxn<'db>;
type RwTx<'db> = heed::RwTxn<'db>;
use crate::{
backend::heed::types::HeedDb,
database::{DatabaseRo, DatabaseRw},
error::RuntimeError,
table::Table,
};
fn get(&self, ro_tx: &Self::RoTx<'_>, key: &T::Key) -> Result<Option<T::Value>, RuntimeError> {
//---------------------------------------------------------------------------------------------------- Heed Database Wrappers
// Q. Why does `HeedTableR{o,w}` exist?
// A. These wrapper types combine `heed`'s database/table
// types with its transaction types. It exists to match
// `redb`, which has this behavior built-in.
//
// `redb` forces us to abstract read/write semantics
// at the _opened table_ level, so, we must match that in `heed`,
// which abstracts it at the transaction level.
//
// We must also maintain the ability for
// write operations to also read, aka, `Rw`.
//
// TODO: do we need the `T: Table` phantom bound?
// It allows us to reference the `Table` info.
/// An opened read-only database associated with a transaction.
///
/// Matches `redb::ReadOnlyTable`.
pub(super) struct HeedTableRo<'env, T: Table> {
/// An already opened database table.
db: HeedDb,
/// The associated read-only transaction that opened this table.
tx_ro: &'env heed::RoTxn<'env>,
/// TODO: do we need this?
_table: PhantomData<T>,
}
/// An opened read/write database associated with a transaction.
///
/// Matches `redb::Table` (read & write).
pub(super) struct HeedTableRw<'env, T: Table> {
/// TODO
db: HeedDb,
/// The associated read/write transaction that opened this table.
tx_rw: &'env mut heed::RwTxn<'env>,
/// TODO: do we need this?
_table: PhantomData<T>,
}
//---------------------------------------------------------------------------------------------------- DatabaseRo Impl
impl<T: Table> DatabaseRo<T> for HeedTableRo<'_, T> {
fn get(&self, key: &T::Key) -> Result<Option<T::Value>, RuntimeError> {
todo!()
}
fn get_range(
&self,
ro_tx: &Self::RoTx<'_>,
key: &T::Key,
amount: usize,
) -> Result<impl Iterator<Item = T::Value>, RuntimeError> {
let iter: std::vec::Drain<'_, T::Value> = todo!();
Ok(iter)
}
}
fn put(
&mut self,
rw_tx: &mut Self::RwTx<'_>,
//---------------------------------------------------------------------------------------------------- DatabaseRw Impl
impl<T: Table> DatabaseRo<T> for HeedTableRw<'_, T> {
fn get(&self, key: &T::Key) -> Result<Option<T::Value>, RuntimeError> {
todo!()
}
fn get_range(
&self,
key: &T::Key,
value: &T::Value,
) -> Result<(), RuntimeError> {
amount: usize,
) -> Result<impl Iterator<Item = T::Value>, RuntimeError> {
let iter: std::vec::Drain<'_, T::Value> = todo!();
Ok(iter)
}
}
impl<T: Table> DatabaseRw<T> for HeedTableRw<'_, T> {
fn put(&mut self, key: &T::Key, value: &T::Value) -> Result<(), RuntimeError> {
todo!()
}
fn clear(&mut self, rw_tx: &mut Self::RwTx<'_>) -> Result<(), RuntimeError> {
fn clear(&mut self) -> Result<(), RuntimeError> {
todo!()
}
fn delete(&mut self, rw_tx: &mut Self::RwTx<'_>, key: &T::Key) -> Result<bool, RuntimeError> {
fn delete(&mut self, key: &T::Key) -> Result<bool, RuntimeError> {
todo!()
}
}

52
database/src/backend/heed/env.rs
View file

@ -4,9 +4,9 @@
use std::sync::RwLock;
use crate::{
backend::heed::types::HeedDb,
backend::heed::database::{HeedTableRo, HeedTableRw},
config::Config,
database::Database,
database::{DatabaseRo, DatabaseRw},
env::Env,
error::{InitError, RuntimeError},
resize::ResizeAlgorithm,
@ -49,6 +49,14 @@ pub struct ConcreteEnv {
impl Drop for ConcreteEnv {
fn drop(&mut self) {
// TODO:
// "if the environment has the MDB_NOSYNC flag set the flushes will be omitted,
// and with MDB_MAPASYNC they will be asynchronous."
// <http://www.lmdb.tech/doc/group__mdb.html#ga85e61f05aa68b520cc6c3b981dba5037>
//
// We need to do `mdb_env_set_flags(&env, MDB_NOSYNC|MDB_ASYNCMAP, 0)`
// to clear the no sync and async flags such that the below `self.sync()`
// _actually_ synchronously syncs.
if let Err(e) = self.sync() {
// TODO: log error?
}
@ -61,8 +69,8 @@ impl Drop for ConcreteEnv {
impl Env for ConcreteEnv {
const MANUAL_RESIZE: bool = true;
const SYNCS_PER_TX: bool = false;
type RoTx<'db> = heed::RoTxn<'db>;
type RwTx<'db> = heed::RwTxn<'db>;
type TxRo<'env> = heed::RoTxn<'env>;
type TxRw<'env> = heed::RwTxn<'env>;
#[cold]
#[inline(never)] // called once.
@ -80,6 +88,12 @@ impl Env for ConcreteEnv {
todo!()
}
#[cold]
#[inline(never)] // called once in [`Env::open`]?
fn create_tables<T: Table>(&self, tx_rw: &mut Self::TxRw<'_>) -> Result<(), RuntimeError> {
todo!()
}
fn config(&self) -> &Config {
&self.config
}
@ -116,30 +130,30 @@ impl Env for ConcreteEnv {
}
#[inline]
fn ro_tx(&self) -> Result<Self::RoTx<'_>, RuntimeError> {
fn tx_ro(&self) -> Result<Self::TxRo<'_>, RuntimeError> {
todo!()
}
#[inline]
fn rw_tx(&self) -> Result<Self::RwTx<'_>, RuntimeError> {
todo!()
}
#[cold]
#[inline(never)] // called infrequently?.
fn create_tables_if_needed<T: Table>(
&self,
tx_rw: &mut Self::RwTx<'_>,
) -> Result<(), RuntimeError> {
fn tx_rw(&self) -> Result<Self::TxRw<'_>, RuntimeError> {
todo!()
}
#[inline]
fn open_database<T: Table>(
fn open_db_ro<T: Table>(
&self,
to_rw: &Self::RoTx<'_>,
) -> Result<impl Database<T>, RuntimeError> {
let tx: HeedDb = todo!();
tx_ro: &Self::TxRo<'_>,
) -> Result<impl DatabaseRo<T>, RuntimeError> {
let tx: HeedTableRo<T> = todo!();
Ok(tx)
}
#[inline]
fn open_db_rw<T: Table>(
&self,
tx_rw: &mut Self::TxRw<'_>,
) -> Result<impl DatabaseRw<T>, RuntimeError> {
let tx: HeedTableRw<T> = todo!();
Ok(tx)
}
}

10
database/src/backend/heed/error.rs
View file

@ -1,4 +1,4 @@
//! Conversion from `heed::Error` -> `cuprate_database::RuntimeError`.
//! Conversion from `heed::Error` -> `cuprate_database`'s errors.
//---------------------------------------------------------------------------------------------------- Use
use crate::constants::DATABASE_CORRUPT_MSG;
@ -83,7 +83,7 @@ impl From<heed::Error> for crate::RuntimeError {
//
// "Requested page not found - this usually indicates corruption."
// <https://docs.rs/heed/latest/heed/enum.MdbError.html#variant.PageNotFound>
E2::Corrupted | E2::PageNotFound => panic!("{mdb_error:?}\n{DATABASE_CORRUPT_MSG}"),
E2::Corrupted | E2::PageNotFound => panic!("{mdb_error:#?}\n{DATABASE_CORRUPT_MSG}"),
// These errors should not occur, and if they do,
// the best thing `cuprate_database` can do for
@ -98,7 +98,7 @@ impl From<heed::Error> for crate::RuntimeError {
| E2::TxnFull
| E2::BadRslot
| E2::VersionMismatch
| E2::BadDbi => panic!("{mdb_error:?}"),
| E2::BadDbi => panic!("{mdb_error:#?}"),
// These errors are the same as above, but instead
// of being errors we can't control, these are errors
@ -135,7 +135,7 @@ impl From<heed::Error> for crate::RuntimeError {
// Don't use a key that is `>511` bytes.
// <http://www.lmdb.tech/doc/group__mdb.html#gaaf0be004f33828bf2fb09d77eb3cef94>
| E2::BadValSize
=> panic!("fix the database code! {mdb_error:?}"),
=> panic!("fix the database code! {mdb_error:#?}"),
},
// Only if we write incorrect code.
@ -143,7 +143,7 @@ impl From<heed::Error> for crate::RuntimeError {
| E1::DatabaseClosing
| E1::BadOpenOptions { .. }
| E1::Encoding(_)
| E1::Decoding(_) => panic!("fix the database code! {error:?}"),
| E1::Decoding(_) => panic!("fix the database code! {error:#?}"),
}
}
}

21
database/src/backend/heed/transaction.rs
View file

@ -1,20 +1,29 @@
//! Implementation of `trait RoTx/RwTx` for `heed`.
//! Implementation of `trait TxRo/TxRw` for `heed`.
//---------------------------------------------------------------------------------------------------- Import
use crate::{
error::RuntimeError,
transaction::{RoTx, RwTx},
transaction::{TxRo, TxRw},
};
//---------------------------------------------------------------------------------------------------- RoTx
impl RoTx<'_> for heed::RoTxn<'_> {
//---------------------------------------------------------------------------------------------------- TxRo
impl TxRo<'_> for heed::RoTxn<'_> {
fn commit(self) -> Result<(), RuntimeError> {
todo!()
}
}
//---------------------------------------------------------------------------------------------------- RwTx
impl RwTx<'_> for heed::RwTxn<'_> {
//---------------------------------------------------------------------------------------------------- TxRw
impl TxRo<'_> for heed::RwTxn<'_> {
/// TODO
/// # Errors
/// TODO
fn commit(self) -> Result<(), RuntimeError> {
todo!()
}
}
impl TxRw<'_> for heed::RwTxn<'_> {
/// TODO
/// # Errors
/// TODO

2
database/src/backend/heed/types.rs
View file

@ -4,5 +4,5 @@
use heed::{types::Bytes, Database};
//---------------------------------------------------------------------------------------------------- Types
/// The concrete database type for `heed`.
/// The concrete database type for `heed`, usable for reads and writes.
pub(super) type HeedDb = Database<Bytes, Bytes>;

6
database/src/backend/mod.rs
View file

@ -12,9 +12,9 @@
cfg_if::cfg_if! {
// If both backends are enabled, fallback to `heed`.
// This is useful when using `--all-features`.
if #[cfg(all(feature = "sanakirja", not(feature = "heed")))] {
mod sanakirja;
pub use sanakirja::ConcreteEnv;
if #[cfg(all(feature = "redb", not(feature = "heed")))] {
mod redb;
pub use redb::ConcreteEnv;
} else {
mod heed;
pub use heed::ConcreteEnv;

61
database/src/backend/redb/database.rs Normal file
View file

@ -0,0 +1,61 @@
//! Implementation of `trait DatabaseR{o,w}` for `redb`.
//---------------------------------------------------------------------------------------------------- Import
use crate::{
backend::redb::types::{RedbTableRo, RedbTableRw},
database::{DatabaseRo, DatabaseRw},
error::RuntimeError,
table::Table,
};
//---------------------------------------------------------------------------------------------------- DatabaseRo
impl<T: Table> DatabaseRo<T> for RedbTableRo<'_> {
fn get(&self, key: &T::Key) -> Result<Option<T::Value>, RuntimeError> {
todo!()
}
fn get_range(
&self,
key: &T::Key,
amount: usize,
) -> Result<impl Iterator<Item = T::Value>, RuntimeError> {
let iter: std::vec::Drain<'_, T::Value> = todo!();
Ok(iter)
}
}
//---------------------------------------------------------------------------------------------------- DatabaseRw
impl<T: Table> DatabaseRo<T> for RedbTableRw<'_, '_> {
fn get(&self, key: &T::Key) -> Result<Option<T::Value>, RuntimeError> {
todo!()
}
fn get_range(
&self,
key: &T::Key,
amount: usize,
) -> Result<impl Iterator<Item = T::Value>, RuntimeError> {
let iter: std::vec::Drain<'_, T::Value> = todo!();
Ok(iter)
}
}
impl<T: Table> DatabaseRw<T> for RedbTableRw<'_, '_> {
fn put(&mut self, key: &T::Key, value: &T::Value) -> Result<(), RuntimeError> {
todo!()
}
fn clear(&mut self) -> Result<(), RuntimeError> {
todo!()
}
fn delete(&mut self, key: &T::Key) -> Result<bool, RuntimeError> {
todo!()
}
}
//---------------------------------------------------------------------------------------------------- Tests
#[cfg(test)]
mod test {
// use super::*;
}

120
database/src/backend/redb/env.rs Normal file
View file

@ -0,0 +1,120 @@
//! Implementation of `trait Env` for `redb`.
//---------------------------------------------------------------------------------------------------- Import
use std::{path::Path, sync::Arc};
use crate::{
backend::redb::types::{RedbTableRo, RedbTableRw},
config::{Config, SyncMode},
database::{DatabaseRo, DatabaseRw},
env::Env,
error::{InitError, RuntimeError},
table::Table,
};
//---------------------------------------------------------------------------------------------------- ConcreteEnv
/// A strongly typed, concrete database environment, backed by `redb`.
pub struct ConcreteEnv {
/// The actual database environment.
env: redb::Database,
/// The configuration we were opened with
/// (and in current use).
config: Config,
/// A cached, redb version of `cuprate_database::config::SyncMode`.
/// `redb` needs the sync mode to be set _per_ TX, so we
/// will continue to use this value every `Env::tx_rw`.
durability: redb::Durability,
}
impl Drop for ConcreteEnv {
fn drop(&mut self) {
if let Err(e) = self.sync() {
// TODO: log error?
}
// TODO: log that we are dropping the database.
}
}
//---------------------------------------------------------------------------------------------------- Env Impl
impl Env for ConcreteEnv {
const MANUAL_RESIZE: bool = false;
const SYNCS_PER_TX: bool = false;
type TxRo<'env> = redb::ReadTransaction<'env>;
type TxRw<'env> = redb::WriteTransaction<'env>;
#[cold]
#[inline(never)] // called once.
fn open(config: Config) -> Result<Self, InitError> {
// TODO: dynamic syncs are not implemented.
let durability = match config.sync_mode {
// TODO: There's also `redb::Durability::Paranoid`:
// <https://docs.rs/redb/1.5.0/redb/enum.Durability.html#variant.Paranoid>
// should we use that instead of Immediate?
SyncMode::Safe => redb::Durability::Immediate,
SyncMode::Async => redb::Durability::Eventual,
SyncMode::Fast => redb::Durability::None,
// TODO: dynamic syncs are not implemented.
SyncMode::FastThenSafe | SyncMode::Threshold(_) => unimplemented!(),
};
todo!()
}
#[cold]
#[inline(never)] // called once in [`Env::open`]?`
fn create_tables<T: Table>(&self, tx_rw: &mut Self::TxRw<'_>) -> Result<(), RuntimeError> {
todo!()
}
fn config(&self) -> &Config {
&self.config
}
fn sync(&self) -> Result<(), RuntimeError> {
todo!()
}
#[inline]
fn tx_ro(&self) -> Result<Self::TxRo<'_>, RuntimeError> {
todo!()
}
#[inline]
fn tx_rw(&self) -> Result<Self::TxRw<'_>, RuntimeError> {
// `redb` has sync modes on the TX level, unlike heed,
// which sets it at the Environment level.
//
// So, set the durability here before returning the TX.
let mut tx_rw = self.env.begin_write()?;
tx_rw.set_durability(self.durability);
Ok(tx_rw)
}
#[inline]
fn open_db_ro<T: Table>(
&self,
tx_ro: &Self::TxRo<'_>,
) -> Result<impl DatabaseRo<T>, RuntimeError> {
let tx: RedbTableRo = todo!();
Ok(tx)
}
#[inline]
fn open_db_rw<T: Table>(
&self,
tx_rw: &mut Self::TxRw<'_>,
) -> Result<impl DatabaseRw<T>, RuntimeError> {
let tx: RedbTableRw = todo!();
Ok(tx)
}
}
//---------------------------------------------------------------------------------------------------- Tests
#[cfg(test)]
mod test {
// use super::*;
}

107
database/src/backend/redb/error.rs Normal file
View file

@ -0,0 +1,107 @@
//! Conversion from `redb`'s errors -> `cuprate_database`'s errors.
//!
//! HACK: There's a lot of `_ =>` usage here because
//! `redb`'s errors are `#[non_exhaustive]`...
//---------------------------------------------------------------------------------------------------- Import
use crate::constants::DATABASE_CORRUPT_MSG;
//---------------------------------------------------------------------------------------------------- DatabaseError
impl From<redb::DatabaseError> for crate::InitError {
/// Created by `redb` in:
/// - [`redb::Database::open`](https://docs.rs/redb/1.5.0/redb/struct.Database.html#method.open).
fn from(error: redb::DatabaseError) -> Self {
use redb::DatabaseError as E;
use redb::StorageError as E2;
// Reference of all possible errors `redb` will return
// upon using `redb::Database::open`:
// <https://docs.rs/redb/1.5.0/src/redb/db.rs.html#908-923>
match error {
E::RepairAborted => Self::Corrupt,
E::UpgradeRequired(_) => Self::InvalidVersion,
E::Storage(s_error) => match s_error {
E2::Io(e) => Self::Io(e),
E2::Corrupted(_) => Self::Corrupt,
// HACK: Handle new errors as `redb` adds them.
_ => Self::Unknown(Box::new(s_error)),
},
// HACK: Handle new errors as `redb` adds them.
_ => Self::Unknown(Box::new(error)),
}
}
}
//---------------------------------------------------------------------------------------------------- TransactionError
#[allow(clippy::fallible_impl_from)] // We need to panic sometimes.
impl From<redb::TransactionError> for crate::RuntimeError {
/// Created by `redb` in:
/// - [`redb::Database::begin_write`](https://docs.rs/redb/1.5.0/redb/struct.Database.html#method.begin_write)
/// - [`redb::Database::begin_read`](https://docs.rs/redb/1.5.0/redb/struct.Database.html#method.begin_read)
fn from(error: redb::TransactionError) -> Self {
use redb::StorageError as E;
match error {
redb::TransactionError::Storage(error) => error.into(),
// HACK: Handle new errors as `redb` adds them.
_ => unreachable!(),
}
}
}
//---------------------------------------------------------------------------------------------------- TableError
#[allow(clippy::fallible_impl_from)] // We need to panic sometimes.
impl From<redb::TableError> for crate::RuntimeError {
/// Created by `redb` in:
/// - [`redb::WriteTransaction::open_table`](https://docs.rs/redb/1.5.0/redb/struct.WriteTransaction.html#method.open_table)
/// - [`redb::ReadTransaction::open_table`](https://docs.rs/redb/1.5.0/redb/struct.ReadTransaction.html#method.open_table)
fn from(error: redb::TableError) -> Self {
use redb::StorageError as E2;
use redb::TableError as E;
match error {
E::Storage(error) => error.into(),
// Only if we write incorrect code.
E::TableTypeMismatch { .. }
| E::TableIsMultimap(_)
| E::TableIsNotMultimap(_)
| E::TypeDefinitionChanged { .. }
| E::TableDoesNotExist(_)
| E::TableAlreadyOpen(..) => panic!("fix the database code! {error:#?}"),
// HACK: Handle new errors as `redb` adds them.
_ => unreachable!(),
}
}
}
//---------------------------------------------------------------------------------------------------- StorageError
#[allow(clippy::fallible_impl_from)] // We need to panic sometimes.
impl From<redb::StorageError> for crate::RuntimeError {
/// Created by `redb` in:
/// - [`redb::Table`](https://docs.rs/redb/1.5.0/redb/struct.Table.html) functions
/// - [`redb::ReadOnlyTable`](https://docs.rs/redb/1.5.0/redb/struct.ReadOnlyTable.html) functions
fn from(error: redb::StorageError) -> Self {
use redb::StorageError as E;
match error {
E::Io(e) => Self::Io(e),
E::Corrupted(s) => panic!("{s:#?}\n{DATABASE_CORRUPT_MSG}"),
E::ValueTooLarge(s) => panic!("fix the database code! {s:#?}"),
E::LockPoisoned(s) => panic!("{s:#?}"),
// HACK: Handle new errors as `redb` adds them.
_ => unreachable!(),
}
}
}
//---------------------------------------------------------------------------------------------------- Tests
#[cfg(test)]
mod test {
// use super::*;
}

0
database/src/backend/sanakirja/mod.rs → database/src/backend/redb/mod.rs
View file

14
database/src/backend/sanakirja/transaction.rs → database/src/backend/redb/transaction.rs
View file

@ -1,20 +1,22 @@
//! Implementation of `trait RoTx/RwTx` for `sanakirja`.
//! Implementation of `trait TxRo/TxRw` for `redb`.
//---------------------------------------------------------------------------------------------------- Import
use crate::{
config::SyncMode,
env::Env,
error::RuntimeError,
transaction::{RoTx, RwTx},
transaction::{TxRo, TxRw},
};
//---------------------------------------------------------------------------------------------------- RoTx
impl RoTx<'_> for sanakirja::Txn<&'_ sanakirja::Env> {
//---------------------------------------------------------------------------------------------------- TxRo
impl TxRo<'_> for redb::ReadTransaction<'_> {
fn commit(self) -> Result<(), RuntimeError> {
todo!()
}
}
//---------------------------------------------------------------------------------------------------- RwTx
impl RwTx<'_> for sanakirja::MutTxn<&'_ sanakirja::Env, ()> {
//---------------------------------------------------------------------------------------------------- TxRw
impl TxRw<'_> for redb::WriteTransaction<'_> {
/// TODO
/// # Errors
/// TODO

10
database/src/backend/redb/types.rs Normal file
View file

@ -0,0 +1,10 @@
//! `redb` type aliases.
//---------------------------------------------------------------------------------------------------- Types
// TODO: replace `()` with a byte container.
/// The concrete type for readable `redb` tables.
pub(super) type RedbTableRo<'env> = redb::ReadOnlyTable<'env, (), ()>;
/// The concrete type for readable/writable `redb` tables.
pub(super) type RedbTableRw<'env, 'tx> = redb::Table<'env, 'tx, (), ()>;

49
database/src/backend/sanakirja/database.rs
View file

@ -1,49 +0,0 @@
//! Implementation of `trait Database` for `sanakirja`.
//---------------------------------------------------------------------------------------------------- Import
use crate::{
backend::sanakirja::types::SanakirjaDb, database::Database, error::RuntimeError, table::Table,
};
//---------------------------------------------------------------------------------------------------- Database Impls
impl<T: Table> Database<T> for SanakirjaDb {
type RoTx<'db> = sanakirja::Txn<&'db sanakirja::Env>;
type RwTx<'db> = sanakirja::MutTxn<&'db sanakirja::Env, ()>;
fn get(&self, ro_tx: &Self::RoTx<'_>, key: &T::Key) -> Result<Option<T::Value>, RuntimeError> {
todo!()
}
fn get_range(
&self,
ro_tx: &Self::RoTx<'_>,
key: &T::Key,
amount: usize,
) -> Result<impl Iterator<Item = T::Value>, RuntimeError> {
let iter: std::vec::Drain<'_, T::Value> = todo!();
Ok(iter)
}
fn put(
&mut self,
rx_tx: &mut Self::RwTx<'_>,
key: &T::Key,
value: &T::Value,
) -> Result<(), RuntimeError> {
todo!()
}
fn clear(&mut self, rx_tx: &mut Self::RwTx<'_>) -> Result<(), RuntimeError> {
todo!()
}
fn delete(&mut self, rx_tx: &mut Self::RwTx<'_>, key: &T::Key) -> Result<bool, RuntimeError> {
todo!()
}
}
//---------------------------------------------------------------------------------------------------- Tests
#[cfg(test)]
mod test {
// use super::*;
}

93
database/src/backend/sanakirja/env.rs
View file

@ -1,93 +0,0 @@
//! Implementation of `trait Env` for `sanakirja`.
//---------------------------------------------------------------------------------------------------- Import
use std::{path::Path, sync::Arc};
use crate::{
backend::sanakirja::types::SanakirjaDb,
config::Config,
database::Database,
env::Env,
error::{InitError, RuntimeError},
table::Table,
};
//---------------------------------------------------------------------------------------------------- ConcreteEnv
/// A strongly typed, concrete database environment, backed by `sanakirja`.
pub struct ConcreteEnv {
/// The actual database environment.
env: sanakirja::Env,
/// The configuration we were opened with
/// (and in current use).
config: Config,
}
impl Drop for ConcreteEnv {
fn drop(&mut self) {
if let Err(e) = self.sync() {
// TODO: log error?
}
// TODO: log that we are dropping the database.
}
}
//---------------------------------------------------------------------------------------------------- Env Impl
impl Env for ConcreteEnv {
const MANUAL_RESIZE: bool = false;
const SYNCS_PER_TX: bool = true;
/// FIXME:
/// We could also implement `Borrow<sanakirja::Env> for ConcreteEnv`
/// instead of this reference.
type RoTx<'db> = sanakirja::Txn<&'db sanakirja::Env>;
type RwTx<'db> = sanakirja::MutTxn<&'db sanakirja::Env, ()>;
#[cold]
#[inline(never)] // called once.
fn open(config: Config) -> Result<Self, InitError> {
todo!()
}
fn config(&self) -> &Config {
&self.config
}
fn sync(&self) -> Result<(), RuntimeError> {
todo!()
}
#[inline]
fn ro_tx(&self) -> Result<Self::RoTx<'_>, RuntimeError> {
todo!()
}
#[inline]
fn rw_tx(&self) -> Result<Self::RwTx<'_>, RuntimeError> {
todo!()
}
#[cold]
#[inline(never)] // called infrequently?.
fn create_tables_if_needed<T: Table>(
&self,
tx_rw: &mut Self::RwTx<'_>,
) -> Result<(), RuntimeError> {
todo!()
}
#[inline]
fn open_database<T: Table>(
&self,
to_rw: &Self::RoTx<'_>,
) -> Result<impl Database<T>, RuntimeError> {
let tx: SanakirjaDb = todo!();
Ok(tx)
}
}
//---------------------------------------------------------------------------------------------------- Tests
#[cfg(test)]
mod test {
// use super::*;
}

51
database/src/backend/sanakirja/error.rs
View file

@ -1,51 +0,0 @@
//! Conversion from `sanakirja::Error` -> `cuprate_database::RuntimeError`.
//---------------------------------------------------------------------------------------------------- Import
use crate::constants::DATABASE_CORRUPT_MSG;
//---------------------------------------------------------------------------------------------------- InitError
impl From<sanakirja::Error> for crate::InitError {
fn from(error: sanakirja::Error) -> Self {
use sanakirja::Error as E;
match error {
E::IO(io_error) => Self::Io(io_error),
E::VersionMismatch => Self::InvalidVersion,
// A CRC failure essentially means a `sanakirja` page was corrupt.
// <https://docs.rs/sanakirja/latest/sanakirja/enum.Error.html#variant.CRC>
E::Corrupt(_) | E::CRC(_) => Self::Corrupt,
// A database lock was poisoned.
// <https://docs.rs/sanakirja/latest/sanakirja/enum.Error.html#variant.Poison>
E::Poison => Self::Unknown(Box::new(error)),
}
}
}
//---------------------------------------------------------------------------------------------------- RuntimeError
#[allow(clippy::fallible_impl_from)] // We need to panic sometimes.
impl From<sanakirja::Error> for crate::RuntimeError {
fn from(error: sanakirja::Error) -> Self {
use sanakirja::Error as E;
match error {
E::IO(io_error) => Self::Io(io_error),
// A CRC failure essentially means a `sanakirja` page was corrupt.
// <https://docs.rs/sanakirja/latest/sanakirja/enum.Error.html#variant.CRC>
E::Corrupt(_) | E::CRC(_) => panic!("{error:?}\n{DATABASE_CORRUPT_MSG}"),
// These errors should not occur, and if they do,
// the best thing `cuprate_database` can do for
// safety is to panic right here.
E::Poison | E::VersionMismatch => panic!("{error:?}"),
}
}
}
//---------------------------------------------------------------------------------------------------- Tests
#[cfg(test)]
mod test {
// use super::*;
}

6
database/src/backend/sanakirja/types.rs
View file

@ -1,6 +0,0 @@
//! `sanakirja` type aliases.
//---------------------------------------------------------------------------------------------------- Types
/// The concrete database type for `sanakirja`.
pub(super) type SanakirjaDb =
sanakirja::btree::Db_<[u8], [u8], sanakirja::btree::page_unsized::Page<[u8], [u8]>>;

55
database/src/config.rs
View file

@ -17,7 +17,7 @@ use std::{
use cuprate_helper::fs::cuprate_database_dir;
use crate::{constants::DATABASE_FILENAME, resize::ResizeAlgorithm};
use crate::{constants::DATABASE_DATA_FILENAME, resize::ResizeAlgorithm};
//---------------------------------------------------------------------------------------------------- Config
/// Database [`Env`](crate::Env) configuration.
@ -72,7 +72,7 @@ impl Config {
// Add the database filename to the directory.
let mut db_file = db_directory.to_path_buf();
db_file.push(DATABASE_FILENAME);
db_file.push(DATABASE_DATA_FILENAME);
(db_directory, Cow::Owned(db_file))
}
@ -175,15 +175,15 @@ impl Default for Config {
/// will always cause it to fully sync to disk.
///
/// # Sync vs Async
/// All invariants except [`SyncMode::Fast`] are `synchronous`,
/// as in the database will wait until the OS has finished syncing
/// all the data to disk before continuing.
/// All invariants except [`SyncMode::Async`] & [`SyncMode::Fast`]
/// are `synchronous`, as in the database will wait until the OS has
/// finished syncing all the data to disk before continuing.
///
/// `SyncMode::Fast` is `asynchronous`, meaning the database will _NOT_
/// wait until the data is fully synced to disk before continuing.
/// Note that this doesn't mean the database itself won't be synchronized
/// between readers/writers, but rather that the data _on disk_ may not
/// be immediately synchronized after a write.
/// `SyncMode::Async` & `SyncMode::Fast` are `asynchronous`, meaning
/// the database will _NOT_ wait until the data is fully synced to disk
/// before continuing. Note that this doesn't mean the database itself
/// won't be synchronized between readers/writers, but rather that the
/// data _on disk_ may not be immediately synchronized after a write.
///
/// Something like:
/// ```rust,ignore
@ -191,6 +191,17 @@ impl Default for Config {
/// db.get("key");
/// ```
/// will be fine, most likely pulling from memory instead of disk.
///
/// # TODO
/// Dynamic sync's are not yet supported.
///
/// Only:
///
/// - [`SyncMode::Safe`]
/// - [`SyncMode::Async`]
/// - [`SyncMode::Fast`]
///
/// are supported, all other variants will panic on [`crate::Env::open`].
#[derive(Copy, Clone, Debug, Default, PartialEq, PartialOrd, Eq, Ord, Hash)]
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
#[cfg_attr(
@ -226,11 +237,27 @@ pub enum SyncMode {
///
/// Every database transaction commit will
/// fully sync all data to disk, _synchronously_,
/// so the database halts until synced.
/// so the database (writer) halts until synced.
///
/// This is expected to be very slow.
///
/// This matches:
/// - LMDB without any special sync flags
/// - [`redb::Durability::Immediate`](https://docs.rs/redb/1.5.0/redb/enum.Durability.html#variant.Immediate)
Safe,
/// Asynchrously sync to disk per transaction.
///
/// This is the same as [`SyncMode::Safe`],
/// but the syncs will be asynchronous, i.e.
/// each transaction commit will sync to disk,
/// but only eventually, not necessarily immediately.
///
/// This matches:
/// - [`MDB_MAPASYNC`](http://www.lmdb.tech/doc/group__mdb__env.html#gab034ed0d8e5938090aef5ee0997f7e94)
/// - [`redb::Durability::Eventual`](https://docs.rs/redb/1.5.0/redb/enum.Durability.html#variant.Eventual)
Async,
/// Fully sync to disk after we cross this transaction threshold.
///
/// After committing [`usize`] amount of database
@ -247,6 +274,12 @@ pub enum SyncMode {
/// It will cause the database to never _actively_ sync,
/// letting the OS decide when to flush data to disk.
///
/// This matches:
/// - [`MDB_NOSYNC`](http://www.lmdb.tech/doc/group__mdb__env.html#ga5791dd1adb09123f82dd1f331209e12e) + [`MDB_MAPASYNC`](http://www.lmdb.tech/doc/group__mdb__env.html#gab034ed0d8e5938090aef5ee0997f7e94)
/// - [`redb::Durability::None`](https://docs.rs/redb/1.5.0/redb/enum.Durability.html#variant.None)
///
/// `monerod` reference: <https://github.com/monero-project/monero/blob/7b7958bbd9d76375c47dc418b4adabba0f0b1785/src/blockchain_db/lmdb/db_lmdb.cpp#L1380-L1381>
///
/// # Corruption
/// In the case of a system crash, the database
/// may become corrupted when using this option.

69
database/src/constants.rs
View file

@ -1,6 +1,7 @@
//! General constants used throughout `cuprate-database`.
//---------------------------------------------------------------------------------------------------- Import
use cfg_if::cfg_if;
//---------------------------------------------------------------------------------------------------- Error Messages
/// Corrupt database error message.
@ -18,27 +19,55 @@ TODO: instructions on:
4. etc";
//---------------------------------------------------------------------------------------------------- Misc
cfg_if::cfg_if! {
// If both backends are enabled, fallback to `heed`.
// This is useful when using `--all-features`.
if #[cfg(all(feature = "sanakirja", not(feature = "heed")))] {
/// Static string of the `crate` being used as the database backend.
pub const DATABASE_BACKEND: &str = "sanakirja";
/// Cuprate's database filename.
///
/// This is the filename for Cuprate's database, used in [`Config::db_file`](crate::config::Config::db_file).
pub const DATABASE_FILENAME: &str = "data.san"; // TODO: pick a name + extension.
} else {
/// Static string of the `crate` being used as the database backend.
pub const DATABASE_BACKEND: &str = "heed";
/// Cuprate's database filename.
///
/// This is the filename for Cuprate's database, used in [`Config::db_file`](crate::config::Config::db_file).
pub const DATABASE_FILENAME: &str = "data.mdb";
/// Static string of the `crate` being used as the database backend.
///
/// | Backend | Value |
/// |---------|-------|
/// | `heed` | "heed"
/// | `redb` | "redb"
pub const DATABASE_BACKEND: &str = {
cfg_if! {
if #[cfg(all(feature = "redb", not(feature = "heed")))] {
"redb"
} else {
"heed"
}
}
}
};
/// Cuprate's database filename.
///
/// Used in [`Config::db_file`](crate::config::Config::db_file).
///
/// | Backend | Value |
/// |---------|-------|
/// | `heed` | "data.mdb"
/// | `redb` | "data.redb"
pub const DATABASE_DATA_FILENAME: &str = {
cfg_if! {
if #[cfg(all(feature = "redb", not(feature = "heed")))] {
"data.redb"
} else {
"data.mdb"
}
}
};
/// Cuprate's database lock filename.
///
/// | Backend | Value |
/// |---------|-------|
/// | `heed` | Some("lock.mdb")
/// | `redb` | None (redb doesn't use a file lock)
pub const DATABASE_LOCK_FILENAME: Option<&str> = {
cfg_if! {
if #[cfg(all(feature = "redb", not(feature = "heed")))] {
None
} else {
Some("lock.mdb")
}
}
};
//---------------------------------------------------------------------------------------------------- Tests
#[cfg(test)]

69
database/src/database.rs
View file

@ -1,58 +1,45 @@
//! Abstracted database; `trait Database`.
//! Abstracted database; `trait DatabaseRo` & `trait DatabaseRw`.
//---------------------------------------------------------------------------------------------------- Import
use crate::{
error::RuntimeError,
table::Table,
transaction::{RoTx, RwTx},
};
use crate::{error::RuntimeError, table::Table};
//---------------------------------------------------------------------------------------------------- Database
/// Database (key-value store) abstraction.
//---------------------------------------------------------------------------------------------------- DatabaseRo
/// Database (key-value store) read abstraction.
///
/// TODO
pub trait Database<T: Table> {
//------------------------------------------------ Types
/// TODO
type RoTx<'db>: RoTx<'db>;
/// TODO
type RwTx<'db>: RwTx<'db>;
//-------------------------------------------------------- Read
/// TODO: document relation between `DatabaseRo` <-> `DatabaseRw`.
pub trait DatabaseRo<T: Table> {
/// TODO
/// # Errors
/// TODO
fn get(&self, ro_tx: &Self::RoTx<'_>, key: &T::Key) -> Result<Option<T::Value>, RuntimeError>;
fn get(&self, key: &T::Key) -> Result<Option<T::Value>, RuntimeError>;
/// TODO
/// # Errors
/// TODO
fn get_range(
&self,
ro_tx: &Self::RoTx<'_>,
key: &T::Key,
amount: usize,
) -> Result<impl Iterator<Item = T::Value>, RuntimeError>;
//-------------------------------------------------------- Write
/// TODO
/// # Errors
/// TODO
fn put(
&mut self,
rw_tx: &mut Self::RwTx<'_>,
key: &T::Key,
value: &T::Value,
) -> Result<(), RuntimeError>;
/// TODO
/// # Errors
/// TODO
fn clear(&mut self, rw_tx: &mut Self::RwTx<'_>) -> Result<(), RuntimeError>;
/// TODO
/// # Errors
/// TODO
fn delete(&mut self, rw_tx: &mut Self::RwTx<'_>, key: &T::Key) -> Result<bool, RuntimeError>;
}
//---------------------------------------------------------------------------------------------------- DatabaseRw
/// Database (key-value store) read/write abstraction.
///
/// TODO: document relation between `DatabaseRo` <-> `DatabaseRw`.
pub trait DatabaseRw<T: Table>: DatabaseRo<T> {
/// TODO
/// # Errors
/// TODO
fn put(&mut self, key: &T::Key, value: &T::Value) -> Result<(), RuntimeError>;
/// TODO
/// # Errors
/// TODO
fn clear(&mut self) -> Result<(), RuntimeError>;
/// TODO
/// # Errors
/// TODO
fn delete(&mut self, key: &T::Key) -> Result<bool, RuntimeError>;
}

59
database/src/env.rs
View file

@ -3,11 +3,11 @@
//---------------------------------------------------------------------------------------------------- Import
use crate::{
config::Config,
database::Database,
database::{DatabaseRo, DatabaseRw},
error::{InitError, RuntimeError},
resize::ResizeAlgorithm,
table::Table,
transaction::{RoTx, RwTx},
transaction::{TxRo, TxRw},
};
//---------------------------------------------------------------------------------------------------- Env
@ -40,10 +40,10 @@ pub trait Env: Sized {
//------------------------------------------------ Types
/// TODO
type RoTx<'db>: RoTx<'db>;
type TxRo<'env>: TxRo<'env>;
/// TODO
type RwTx<'db>: RwTx<'db>;
type TxRw<'env>: TxRw<'env>;
//------------------------------------------------ Required
/// TODO
@ -51,6 +51,11 @@ pub trait Env: Sized {
/// TODO
fn open(config: Config) -> Result<Self, InitError>;
/// TODO
/// # Errors
/// TODO
fn create_tables<T: Table>(&self, tx_rw: &mut Self::TxRw<'_>) -> Result<(), RuntimeError>;
/// Return the [`Config`] that this database was [`Env::open`]ed with.
fn config(&self) -> &Config;
@ -79,6 +84,16 @@ pub trait Env: Sized {
}
/// TODO
///
/// # Invariant
/// This must **fully** and **synchronously** flush the database data to disk.
///
/// I.e., after this function returns, there must be no doubts
/// that the data isn't synced yet, it _must_ be synced.
///
/// TODO: either this invariant or `sync()` itself will most
/// likely be removed/changed after `SyncMode` is finalized.
///
/// # Errors
/// TODO
fn sync(&self) -> Result<(), RuntimeError>;
@ -110,36 +125,44 @@ pub trait Env: Sized {
/// TODO
/// # Errors
/// TODO
fn ro_tx(&self) -> Result<Self::RoTx<'_>, RuntimeError>;
fn tx_ro(&self) -> Result<Self::TxRo<'_>, RuntimeError>;
/// TODO
/// # Errors
/// TODO
fn rw_tx(&self) -> Result<Self::RwTx<'_>, RuntimeError>;
/// TODO
/// # Errors
/// TODO
fn create_tables_if_needed<T: Table>(
&self,
rw_tx: &mut Self::RwTx<'_>,
) -> Result<(), RuntimeError>;
fn tx_rw(&self) -> Result<Self::TxRw<'_>, RuntimeError>;
/// TODO
///
/// # TODO: Invariant
/// This should never panic the database because the table doesn't exist.
///
/// Opening/using the database env should have an invariant
/// Opening/using the database [`Env`] should have an invariant
/// that it creates all the tables we need, such that this
/// never returns `None`.
///
/// # Errors
/// TODO
fn open_database<T: Table>(
fn open_db_ro<T: Table>(
&self,
ro_tx: &Self::RoTx<'_>,
) -> Result<impl Database<T>, RuntimeError>;
tx_ro: &Self::TxRo<'_>,
) -> Result<impl DatabaseRo<T>, RuntimeError>;
/// TODO
///
/// # TODO: Invariant
/// This should never panic the database because the table doesn't exist.
///
/// Opening/using the database [`Env`] should have an invariant
/// that it creates all the tables we need, such that this
/// never returns `None`.
///
/// # Errors
/// TODO
fn open_db_rw<T: Table>(
&self,
tx_rw: &mut Self::TxRw<'_>,
) -> Result<impl DatabaseRw<T>, RuntimeError>;
//------------------------------------------------ Provided
}

44
database/src/lib.rs
View file

@ -7,28 +7,29 @@
//!
//! # Purpose
//! This crate does 3 things:
//! 1. Abstracts various databases with the [`Env`], [`Database`], [`Table`], [`Key`], [`RoTx`], and [`RwTx`] traits
//! 1. Abstracts various database backends with traits
//! 2. Implements various `Monero` related [functions](ops) & [`tables`]
//! 3. Exposes a [`tower::Service`] backed by a thread-pool
//!
//! # Terminology
//! To be more clear on some terms used in this crate:
//!
//! | Term | Meaning |
//! |------------|--------------------------------------|
//! | `Env` | The 1 database environment, the "whole" thing
//! | `Database` | A `key/value` store
//! | `Table` | Solely the metadata of a `Database` (the `key` and `value` types, and the name)
//! | `RoTx` | Read only transaction
//! | `RwTx` | Read/write transaction
//! | Term | Meaning |
//! |---------------|--------------------------------------|
//! | `Env` | The 1 database environment, the "whole" thing
//! | `DatabaseRo` | A read-only `key/value` store
//! | `DatabaseRw` | A readable/writable `key/value` store
//! | `Table` | Solely the metadata of a `Database` (the `key` and `value` types, and the name)
//! | `TxRo` | Read only transaction
//! | `TxRw` | Read/write transaction
//!
//! The dataflow is `Env` -> `Tx` -> `Database`
//!
//! Which reads as:
//! 1. You have a database `Environment`
//! 2. You open up a `Transaction`
//! 2. You get a particular `Database` from that `Environment`
//! 3. You can now read/write data from/to that `Database`
//! 1. You open up a `Transaction`
//! 1. You get a particular `Database` from that `Environment`
//! 1. You can now read/write data from/to that `Database`
//!
//! # `ConcreteEnv`
//! This crate exposes [`ConcreteEnv`], which is a non-generic/non-dynamic,
@ -60,13 +61,18 @@
//! going to be storing any databases in structs, to lessen
//! the generic `<D: Database>` pain.
//!
//! TODO: we could replace `ConcreteEnv` with `fn Env::open() -> impl Env`/
//! and use `<E: Env>` everywhere it is stored instead. This would allow
//! generic-backed dynamic runtime selection of the database backend, i.e.
//! the user can select which database backend they use.
//!
//! # Feature flags
//! The `service` module requires the `service` feature to be enabled.
//! See the module for more documentation.
//!
//! Different database backends are enabled by the feature flags:
//! - `heed`
//! - `sanakirja`
//! - `heed` (LMDB)
//! - `redb`
//!
//! The default is `heed`.
//!
@ -92,7 +98,7 @@
//! use cuprate_database::{
//! config::Config,
//! ConcreteEnv,
//! Env, Key, RoTx, RwTx,
//! Env, Key, TxRo, TxRw,
//! service::{ReadRequest, WriteRequest, Response},
//! };
//!
@ -166,7 +172,7 @@
unused_comparisons,
nonstandard_style
)]
#![allow(unreachable_code, unused_variables, dead_code)] // TODO: remove
#![allow(unreachable_code, unused_variables, dead_code, unused_imports)] // TODO: remove
#![allow(
// FIXME: this lint affects crates outside of
// `database/` for some reason, allow for now.
@ -289,10 +295,12 @@ pub use backend::ConcreteEnv;
pub mod config;
mod constants;
pub use constants::{DATABASE_BACKEND, DATABASE_CORRUPT_MSG, DATABASE_FILENAME};
pub use constants::{
DATABASE_BACKEND, DATABASE_CORRUPT_MSG, DATABASE_DATA_FILENAME, DATABASE_LOCK_FILENAME,
};
mod database;
pub use database::Database;
pub use database::{DatabaseRo, DatabaseRw};
mod env;
pub use env::Env;
@ -320,7 +328,7 @@ pub use table::Table;
pub mod tables;
mod transaction;
pub use transaction::{RoTx, RwTx};
pub use transaction::{TxRo, TxRw};
//---------------------------------------------------------------------------------------------------- Feature-gated
#[cfg(feature = "service")]

8
database/src/ops/alt_block.rs
View file

@ -1,14 +1,6 @@
//! Alternative blocks.
//---------------------------------------------------------------------------------------------------- Import
#[allow(unused_imports)] // FIXME: these traits will be eventually in the function impls.
use crate::{
database::Database,
env::Env,
table::Table,
transaction::{RoTx, RwTx},
ConcreteEnv,
};
//---------------------------------------------------------------------------------------------------- Free Functions
/// TODO

8
database/src/ops/block.rs
View file

@ -1,14 +1,6 @@
//! Blocks.
//---------------------------------------------------------------------------------------------------- Import
#[allow(unused_imports)] // FIXME: these traits will be eventually in the function impls.
use crate::{
database::Database,
env::Env,
table::Table,
transaction::{RoTx, RwTx},
ConcreteEnv,
};
//---------------------------------------------------------------------------------------------------- Free Functions
/// TODO

8
database/src/ops/blockchain.rs
View file

@ -1,14 +1,6 @@
//! Blockchain.
//---------------------------------------------------------------------------------------------------- Import
#[allow(unused_imports)] // FIXME: these traits will be eventually in the function impls.
use crate::{
database::Database,
env::Env,
table::Table,
transaction::{RoTx, RwTx},
ConcreteEnv,
};
//---------------------------------------------------------------------------------------------------- Free Functions
/// TODO

8
database/src/ops/output.rs
View file

@ -1,14 +1,6 @@
//! Outputs.
//---------------------------------------------------------------------------------------------------- Import
#[allow(unused_imports)] // FIXME: these traits will be eventually in the function impls.
use crate::{
database::Database,
env::Env,
table::Table,
transaction::{RoTx, RwTx},
ConcreteEnv,
};
//---------------------------------------------------------------------------------------------------- Free Functions
/// TODO

8
database/src/ops/property.rs
View file

@ -1,14 +1,6 @@
//! Properties.
//---------------------------------------------------------------------------------------------------- Import
#[allow(unused_imports)] // FIXME: these traits will be eventually in the function impls.
use crate::{
database::Database,
env::Env,
table::Table,
transaction::{RoTx, RwTx},
ConcreteEnv,
};
//---------------------------------------------------------------------------------------------------- Free Functions
/// TODO

8
database/src/ops/spent_key.rs
View file

@ -1,14 +1,6 @@
//! Spent keys.
//---------------------------------------------------------------------------------------------------- Import
#[allow(unused_imports)] // FIXME: these traits will be eventually in the function impls.
use crate::{
database::Database,
env::Env,
table::Table,
transaction::{RoTx, RwTx},
ConcreteEnv,
};
//---------------------------------------------------------------------------------------------------- Free Functions
/// TODO

8
database/src/ops/tx.rs
View file

@ -1,14 +1,6 @@
//! Transactions.
//---------------------------------------------------------------------------------------------------- Import
#[allow(unused_imports)] // FIXME: these traits will be eventually in the function impls.
use crate::{
database::Database,
env::Env,
table::Table,
transaction::{RoTx, RwTx},
ConcreteEnv,
};
//---------------------------------------------------------------------------------------------------- Free Functions
/// TODO

2
database/src/resize.rs
View file

@ -29,7 +29,7 @@ use std::{num::NonZeroUsize, sync::OnceLock};
///
/// # TODO
/// We could test around with different algorithms.
/// Calling [`heed::Env::resize`] is surprisingly fast,
/// Calling `heed::Env::resize` is surprisingly fast,
/// around `0.0000082s` on my machine. We could probably
/// get away with smaller and more frequent resizes.
/// **With the caveat being we are taking a `WriteGuard` to a `RwLock`.**

12
database/src/transaction.rs
View file

@ -1,24 +1,24 @@
//! Database transaction abstraction; `trait RoTx`, `trait RwTx`.
//! Database transaction abstraction; `trait TxRo`, `trait TxRw`.
//---------------------------------------------------------------------------------------------------- Import
use crate::error::RuntimeError;
use crate::{config::SyncMode, env::Env, error::RuntimeError};
//---------------------------------------------------------------------------------------------------- RoTx
//---------------------------------------------------------------------------------------------------- TxRo
/// Read-only database transaction.
///
/// TODO
pub trait RoTx<'db> {
pub trait TxRo<'env> {
/// TODO
/// # Errors
/// TODO
fn commit(self) -> Result<(), RuntimeError>;
}
//---------------------------------------------------------------------------------------------------- RwTx
//---------------------------------------------------------------------------------------------------- TxRw
/// Read/write database transaction.
///
/// TODO
pub trait RwTx<'db> {
pub trait TxRw<'env> {
/// TODO
/// # Errors
/// TODO