mirror of
https://github.com/Cuprate/cuprate.git
synced 2024-09-28 20:51:03 +00:00
database: replace Pod with bytemuck (#81)
* database: add `bytemuck`
* database: add `types` module, replace `Pod` with `bytemuck` traits
* types: docs
* types: more docs
* types: align safety msg
* types: docs
* misc docs
* add `storable.rs`
* add `slice.rs`
* storable: impl `impl_storable_checked_bit_pattern!()`
* database: TODO: fix `DatabaseRo::get_range` lifetimes
* key/table: trait bound fixes
* misc fixes
* remove `borsh`
- Doesn't work on must types
- Probably won't use it anyway
- Most things impl `serde`
* key: add `new_with_max_secondary()`
* key: add `new_with_max_secondary()`
* heed: add `StorableHeed` for `Storable` compat
* redb: add `StorableRedb{Key,Value}` for `Storable` compat
* storable: add `Debug` bound and `fixed_width()`
* redb: fix `'static` bound
* storable: docs
* `pod.rs` tests -> `storable.rs`
* redb: add `Storable` tests
* storable: add doc tests
* redb: simplify `Storable` tests
* heed: add `Storable` tests
* misc docs/fixes
* cargo: switch from forked `heed` -> `heed 0.20.0-alpha.9`
* update readme
* docs
* fix README
* table: remove `CONSTANT_SIZE`
* database: `get()/delete() -> Err(KeyNotFound)` instead of `Option`
This commit is contained in:
hinto-janai
GitHub
parent
28aa0b5552
commit
272ef18eb6
24 changed files with 875 additions and 760 deletions
33
Cargo.lock
generated
33
Cargo.lock
generated
|
|
@ -272,6 +272,20 @@ name = "bytemuck"
|
|||
version = "1.14.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a2ef034f05691a48569bd920a96c81b9d91bbad1ab5ac7c4616c1f6ef36cb79f"
|
||||
dependencies = [
|
||||
"bytemuck_derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bytemuck_derive"
|
||||
version = "1.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "965ab7eb5f8f97d2a083c799f3a1b994fc397b2fe2da5d1da1626ce15a39f2b1"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.52",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "byteorder"
|
||||
|
|
@ -571,7 +585,7 @@ dependencies = [
|
|||
name = "cuprate-database"
|
||||
version = "0.0.0"
|
||||
dependencies = [
|
||||
"borsh",
|
||||
"bytemuck",
|
||||
"cfg-if",
|
||||
"crossbeam",
|
||||
"cuprate-helper",
|
||||
|
|
@ -735,9 +749,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "doxygen-rs"
|
||||
version = "0.4.2"
|
||||
version = "0.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "415b6ec780d34dcf624666747194393603d0373b7141eef01d12ee58881507d9"
|
||||
checksum = "bff670ea0c9bbb8414e7efa6e23ebde2b8f520a7eef78273a3918cf1903e7505"
|
||||
dependencies = [
|
||||
"phf",
|
||||
]
|
||||
|
|
@ -1077,7 +1091,8 @@ checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8"
|
|||
[[package]]
|
||||
name = "heed"
|
||||
version = "0.20.0-alpha.9"
|
||||
source = "git+https://github.com/Cuprate/heed?rev=5aa75b7#5aa75b7a44c8e572cf4957c35526e604e7a692ac"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9648a50991c86df7d00c56c268c27754fcf4c80be2ba57fc4a00dc928c6fe934"
|
||||
dependencies = [
|
||||
"bitflags 2.4.2",
|
||||
"bytemuck",
|
||||
|
|
@ -1096,12 +1111,14 @@ dependencies = [
|
|||
[[package]]
|
||||
name = "heed-traits"
|
||||
version = "0.20.0-alpha.9"
|
||||
source = "git+https://github.com/Cuprate/heed?rev=5aa75b7#5aa75b7a44c8e572cf4957c35526e604e7a692ac"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5ab0b7d9cde969ad36dde692e487dc89d97f7168bf6a7bd3b894ad4bf7278298"
|
||||
|
||||
[[package]]
|
||||
name = "heed-types"
|
||||
version = "0.20.0-alpha.9"
|
||||
source = "git+https://github.com/Cuprate/heed?rev=5aa75b7#5aa75b7a44c8e572cf4957c35526e604e7a692ac"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f0cb3567a7363f28b597bf6e9897b9466397951dd0e52df2c8196dd8a71af44a"
|
||||
dependencies = [
|
||||
"bincode",
|
||||
"byteorder",
|
||||
|
|
@ -1438,11 +1455,13 @@ checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c"
|
|||
[[package]]
|
||||
name = "lmdb-master-sys"
|
||||
version = "0.1.0"
|
||||
source = "git+https://github.com/Cuprate/heed?rev=5aa75b7#5aa75b7a44c8e572cf4957c35526e604e7a692ac"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "629c123f5321b48fa4f8f4d3b868165b748d9ba79c7103fb58e3a94f736bcedd"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"doxygen-rs",
|
||||
"libc",
|
||||
"pkg-config",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
|
|||
|
|
@ -16,7 +16,8 @@ redb = ["dep:redb"]
|
|||
service = ["dep:crossbeam", "dep:tokio", "dep:tower"]
|
||||
|
||||
[dependencies]
|
||||
cfg-if = { workspace = true }
|
||||
bytemuck = { version = "1.14.3", features = ["must_cast", "derive", "min_const_generics", "extern_crate_alloc"] }
|
||||
cfg-if = { workspace = true }
|
||||
# FIXME:
|
||||
# We only need the `thread` feature if `service` is enabled.
|
||||
# Figure out how to enable features of an already pulled in dependency conditionally.
|
||||
|
|
@ -40,12 +41,12 @@ tower = { workspace = true, features = ["full"], optional = true }
|
|||
# parking_lot = { workspace = true, optional = true }
|
||||
|
||||
# Optional features.
|
||||
borsh = { workspace = true, optional = true }
|
||||
heed = { git = "https://github.com/Cuprate/heed", rev = "5aa75b7", optional = true }
|
||||
heed = { version = "0.20.0-alpha.9", optional = true }
|
||||
redb = { version = "1.5.0", optional = true }
|
||||
serde = { workspace = true, optional = true }
|
||||
|
||||
[dev-dependencies]
|
||||
bytemuck = { version = "1.14.3", features = ["must_cast", "derive", "min_const_generics", "extern_crate_alloc"] }
|
||||
cuprate-helper = { path = "../helper", features = ["thread"] }
|
||||
page_size = { version = "0.6.0" }
|
||||
tempfile = { version = "3.10.0" }
|
||||
tempfile = { version = "3.10.0" }
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ Cuprate's database implementation.
|
|||
1. [Layers](#layers)
|
||||
- [Database](#database)
|
||||
- [Trait](#trait)
|
||||
- [ConcreteEnv](#concreteenvConcreteEnv
|
||||
- [ConcreteEnv](#concreteenv)
|
||||
- [Thread-pool](#thread-pool)
|
||||
- [Service](#service)
|
||||
1. [Resizing](#resizing)
|
||||
|
|
@ -75,10 +75,12 @@ The top-level `src/` files.
|
|||
| `error.rs` | Database error types
|
||||
| `free.rs` | General free functions (related to the database)
|
||||
| `key.rs` | Abstracted database keys; `trait Key`
|
||||
| `pod.rs` | Data (de)serialization; `trait Pod`
|
||||
| `resize.rs` | Database resizing algorithms
|
||||
| `storable.rs` | Data (de)serialization; `trait Storable`
|
||||
| `table.rs` | Database table abstraction; `trait Table`
|
||||
| `tables.rs` | All the table definitions used by `cuprate-database`
|
||||
| `transaction.rs` | Database transaction abstraction; `trait TxR{o,w}`
|
||||
| `types.rs` | Database table schema types
|
||||
|
||||
## `src/ops/`
|
||||
This folder contains the `cupate_database::ops` module.
|
||||
|
|
@ -126,6 +128,7 @@ All backends follow the same file structure:
|
|||
| `error.rs` | Implementation of backend's errors to `cuprate_database`'s error types
|
||||
| `transaction.rs` | Implementation of `trait TxR{o,w}`
|
||||
| `types.rs` | Type aliases for long backend-specific types
|
||||
| `storable.rs` | Compatibility layer between `cuprate_database::Storable` and backend-specific (de)serialization
|
||||
|
||||
# Backends
|
||||
`cuprate-database`'s `trait`s abstract over various actual databases.
|
||||
|
|
@ -133,13 +136,8 @@ All backends follow the same file structure:
|
|||
Each database's implementation is located in its respective file in `src/backend/${DATABASE_NAME}.rs`.
|
||||
|
||||
## `heed`
|
||||
The default database used is a modified fork of [`heed`](https://github.com/meilisearch/heed) (LMDB), located at [`Cuprate/heed`](https://github.com/Cuprate/heed).
|
||||
The default database used is [`heed`](https://github.com/meilisearch/heed) (LMDB).
|
||||
|
||||
To generate documentation of the fork for local use:
|
||||
```bash
|
||||
git clone --recursive https://github.com/Cuprate/heed
|
||||
cargo doc
|
||||
```
|
||||
`LMDB` should not need to be installed as `heed` has a build script that pulls it in automatically.
|
||||
|
||||
`heed`'s filenames inside Cuprate's database folder (`~/.local/share/cuprate/database/`) are:
|
||||
|
|
@ -151,6 +149,8 @@ cargo doc
|
|||
|
||||
TODO: document max readers limit: https://github.com/monero-project/monero/blob/059028a30a8ae9752338a7897329fe8012a310d5/src/blockchain_db/lmdb/db_lmdb.cpp#L1372. Other potential processes (e.g. `xmrblocks`) that are also reading the `data.mdb` file need to be accounted for.
|
||||
|
||||
TODO: document DB on remote filesystem: https://github.com/LMDB/lmdb/blob/b8e54b4c31378932b69f1298972de54a565185b1/libraries/liblmdb/lmdb.h#L129.
|
||||
|
||||
## `redb`
|
||||
The 2nd database backend is the 100% Rust [`redb`](https://github.com/cberner/redb).
|
||||
|
||||
|
|
@ -162,6 +162,8 @@ The upstream versions from [`crates.io`](https://crates.io/crates/redb) are used
|
|||
|-------------|---------|
|
||||
| `data.redb` | Main data file
|
||||
|
||||
TODO: document DB on remote filesystem (does redb allow this?)
|
||||
|
||||
## `sanakirja`
|
||||
[`sanakirja`](https://docs.rs/sanakirja) was a candidate as a backend, however there were problems with maximum value sizes.
|
||||
|
||||
|
|
|
|||
|
|
@ -22,20 +22,15 @@ use crate::{
|
|||
//
|
||||
// We must also maintain the ability for
|
||||
// write operations to also read, aka, `Rw`.
|
||||
//
|
||||
// TODO: do we need the `T: Table` phantom bound?
|
||||
// It allows us to reference the `Table` info.
|
||||
|
||||
/// An opened read-only database associated with a transaction.
|
||||
///
|
||||
/// Matches `redb::ReadOnlyTable`.
|
||||
pub(super) struct HeedTableRo<'env, T: Table> {
|
||||
/// An already opened database table.
|
||||
db: HeedDb,
|
||||
db: HeedDb<T::Key, T::Value>,
|
||||
/// The associated read-only transaction that opened this table.
|
||||
tx_ro: &'env heed::RoTxn<'env>,
|
||||
/// TODO: do we need this?
|
||||
_table: PhantomData<T>,
|
||||
}
|
||||
|
||||
/// An opened read/write database associated with a transaction.
|
||||
|
|
@ -43,41 +38,45 @@ pub(super) struct HeedTableRo<'env, T: Table> {
|
|||
/// Matches `redb::Table` (read & write).
|
||||
pub(super) struct HeedTableRw<'env, T: Table> {
|
||||
/// TODO
|
||||
db: HeedDb,
|
||||
db: HeedDb<T::Key, T::Value>,
|
||||
/// The associated read/write transaction that opened this table.
|
||||
tx_rw: &'env mut heed::RwTxn<'env>,
|
||||
/// TODO: do we need this?
|
||||
_table: PhantomData<T>,
|
||||
}
|
||||
|
||||
//---------------------------------------------------------------------------------------------------- DatabaseRo Impl
|
||||
impl<T: Table> DatabaseRo<T> for HeedTableRo<'_, T> {
|
||||
fn get(&self, key: &T::Key) -> Result<Option<T::Value>, RuntimeError> {
|
||||
fn get(&self, key: &T::Key) -> Result<&T::Value, RuntimeError> {
|
||||
todo!()
|
||||
}
|
||||
|
||||
fn get_range(
|
||||
&self,
|
||||
key: &T::Key,
|
||||
fn get_range<'a>(
|
||||
&'a self,
|
||||
key: &'a T::Key,
|
||||
amount: usize,
|
||||
) -> Result<impl Iterator<Item = T::Value>, RuntimeError> {
|
||||
let iter: std::vec::Drain<'_, T::Value> = todo!();
|
||||
) -> Result<impl Iterator<Item = &'a T::Value>, RuntimeError>
|
||||
where
|
||||
<T as Table>::Value: 'a,
|
||||
{
|
||||
let iter: std::vec::Drain<'_, &T::Value> = todo!();
|
||||
Ok(iter)
|
||||
}
|
||||
}
|
||||
|
||||
//---------------------------------------------------------------------------------------------------- DatabaseRw Impl
|
||||
impl<T: Table> DatabaseRo<T> for HeedTableRw<'_, T> {
|
||||
fn get(&self, key: &T::Key) -> Result<Option<T::Value>, RuntimeError> {
|
||||
fn get(&self, key: &T::Key) -> Result<&T::Value, RuntimeError> {
|
||||
todo!()
|
||||
}
|
||||
|
||||
fn get_range(
|
||||
&self,
|
||||
key: &T::Key,
|
||||
fn get_range<'a>(
|
||||
&'a self,
|
||||
key: &'a T::Key,
|
||||
amount: usize,
|
||||
) -> Result<impl Iterator<Item = T::Value>, RuntimeError> {
|
||||
let iter: std::vec::Drain<'_, T::Value> = todo!();
|
||||
) -> Result<impl Iterator<Item = &'a T::Value>, RuntimeError>
|
||||
where
|
||||
<T as Table>::Value: 'a,
|
||||
{
|
||||
let iter: std::vec::Drain<'_, &T::Value> = todo!();
|
||||
Ok(iter)
|
||||
}
|
||||
}
|
||||
|
|
@ -91,7 +90,7 @@ impl<T: Table> DatabaseRw<T> for HeedTableRw<'_, T> {
|
|||
todo!()
|
||||
}
|
||||
|
||||
fn delete(&mut self, key: &T::Key) -> Result<bool, RuntimeError> {
|
||||
fn delete(&mut self, key: &T::Key) -> Result<(), RuntimeError> {
|
||||
todo!()
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -62,6 +62,14 @@ impl Drop for ConcreteEnv {
|
|||
}
|
||||
|
||||
// TODO: log that we are dropping the database.
|
||||
|
||||
// TODO: use tracing.
|
||||
// <https://github.com/LMDB/lmdb/blob/b8e54b4c31378932b69f1298972de54a565185b1/libraries/liblmdb/lmdb.h#L49-L61>
|
||||
let result = self.env.read().unwrap().clear_stale_readers();
|
||||
match result {
|
||||
Ok(n) => println!("LMDB stale readers cleared: {n}"),
|
||||
Err(e) => println!("LMDB stale reader clear error: {e:?}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -85,6 +93,7 @@ impl Env for ConcreteEnv {
|
|||
// <https://github.com/monero-project/monero/blob/059028a30a8ae9752338a7897329fe8012a310d5/src/blockchain_db/lmdb/db_lmdb.cpp#L1372>
|
||||
|
||||
// <https://github.com/monero-project/monero/blob/059028a30a8ae9752338a7897329fe8012a310d5/src/blockchain_db/lmdb/db_lmdb.cpp#L1324>
|
||||
|
||||
todo!()
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -5,5 +5,6 @@ pub use env::ConcreteEnv;
|
|||
|
||||
mod database;
|
||||
mod error;
|
||||
mod storable;
|
||||
mod transaction;
|
||||
mod types;
|
||||
|
|
|
|||
102
database/src/backend/heed/storable.rs
Normal file
102
database/src/backend/heed/storable.rs
Normal file
|
|
@ -0,0 +1,102 @@
|
|||
//! `cuprate_database::Storable` <-> `heed` serde trait compatibility layer.
|
||||
|
||||
//---------------------------------------------------------------------------------------------------- Use
|
||||
use std::{borrow::Cow, marker::PhantomData};
|
||||
|
||||
use heed::{types::Bytes, BoxedError, BytesDecode, BytesEncode, Database};
|
||||
|
||||
use crate::storable::Storable;
|
||||
|
||||
//---------------------------------------------------------------------------------------------------- StorableHeed
|
||||
/// The glue struct that implements `heed`'s (de)serialization
|
||||
/// traits on any type that implements `cuprate_database::Storable`.
|
||||
///
|
||||
/// Never actually gets constructed, just used for trait bound translations.
|
||||
pub(super) struct StorableHeed<T: Storable + ?Sized>(PhantomData<T>);
|
||||
|
||||
//---------------------------------------------------------------------------------------------------- BytesDecode
|
||||
impl<'a, T: Storable + ?Sized + 'a> BytesDecode<'a> for StorableHeed<T> {
|
||||
type DItem = &'a T;
|
||||
|
||||
#[inline]
|
||||
/// This function is infallible (will always return `Ok`).
|
||||
fn bytes_decode(bytes: &'a [u8]) -> Result<Self::DItem, BoxedError> {
|
||||
Ok(T::from_bytes(bytes))
|
||||
}
|
||||
}
|
||||
|
||||
//---------------------------------------------------------------------------------------------------- BytesEncode
|
||||
impl<'a, T: Storable + ?Sized + 'a> BytesEncode<'a> for StorableHeed<T> {
|
||||
type EItem = T;
|
||||
|
||||
#[inline]
|
||||
/// This function is infallible (will always return `Ok`).
|
||||
fn bytes_encode(item: &'a Self::EItem) -> Result<Cow<'a, [u8]>, BoxedError> {
|
||||
Ok(Cow::Borrowed(item.as_bytes()))
|
||||
}
|
||||
}
|
||||
|
||||
//---------------------------------------------------------------------------------------------------- Tests
|
||||
#[cfg(test)]
|
||||
mod test {
|
||||
use super::*;
|
||||
|
||||
// Each `#[test]` function has a `test()` to:
|
||||
// - log
|
||||
// - simplify trait bounds
|
||||
// - make sure the right function is being called
|
||||
|
||||
#[test]
|
||||
/// Assert `BytesEncode::bytes_encode` is accurate.
|
||||
fn bytes_encode() {
|
||||
fn test<T: Storable + ?Sized>(t: &T, expected: &[u8]) {
|
||||
println!("t: {t:?}, expected: {expected:?}");
|
||||
assert_eq!(
|
||||
<StorableHeed::<T> as BytesEncode>::bytes_encode(t).unwrap(),
|
||||
expected
|
||||
);
|
||||
}
|
||||
|
||||
test::<()>(&(), &[]);
|
||||
test::<u8>(&0, &[0]);
|
||||
test::<u16>(&1, &[1, 0]);
|
||||
test::<u32>(&2, &[2, 0, 0, 0]);
|
||||
test::<u64>(&3, &[3, 0, 0, 0, 0, 0, 0, 0]);
|
||||
test::<i8>(&-1, &[255]);
|
||||
test::<i16>(&-2, &[254, 255]);
|
||||
test::<i32>(&-3, &[253, 255, 255, 255]);
|
||||
test::<i64>(&-4, &[252, 255, 255, 255, 255, 255, 255, 255]);
|
||||
test::<[u8]>(&[1, 2], &[1, 2]);
|
||||
test::<[u8; 0]>(&[], &[]);
|
||||
test::<[u8; 1]>(&[255], &[255]);
|
||||
test::<[u8; 2]>(&[111, 0], &[111, 0]);
|
||||
test::<[u8; 3]>(&[1, 0, 1], &[1, 0, 1]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
/// Assert `BytesDecode::bytes_decode` is accurate.
|
||||
fn bytes_decode() {
|
||||
fn test<T: Storable + ?Sized + PartialEq>(bytes: &[u8], expected: &T) {
|
||||
println!("bytes: {bytes:?}, expected: {expected:?}");
|
||||
assert_eq!(
|
||||
<StorableHeed::<T> as BytesDecode>::bytes_decode(bytes).unwrap(),
|
||||
expected
|
||||
);
|
||||
}
|
||||
|
||||
test::<()>([].as_slice(), &());
|
||||
test::<u8>([0].as_slice(), &0);
|
||||
test::<u16>([1, 0].as_slice(), &1);
|
||||
test::<u32>([2, 0, 0, 0].as_slice(), &2);
|
||||
test::<u64>([3, 0, 0, 0, 0, 0, 0, 0].as_slice(), &3);
|
||||
test::<i8>([255].as_slice(), &-1);
|
||||
test::<i16>([254, 255].as_slice(), &-2);
|
||||
test::<i32>([253, 255, 255, 255].as_slice(), &-3);
|
||||
test::<i64>([252, 255, 255, 255, 255, 255, 255, 255].as_slice(), &-4);
|
||||
test::<[u8]>([1, 2].as_slice(), &[1, 2]);
|
||||
test::<[u8; 0]>([].as_slice(), &[]);
|
||||
test::<[u8; 1]>([255].as_slice(), &[255]);
|
||||
test::<[u8; 2]>([111, 0].as_slice(), &[111, 0]);
|
||||
test::<[u8; 3]>([1, 0, 1].as_slice(), &[1, 0, 1]);
|
||||
}
|
||||
}
|
||||
|
|
@ -1,8 +1,8 @@
|
|||
//! `heed` type aliases.
|
||||
|
||||
//---------------------------------------------------------------------------------------------------- Use
|
||||
use heed::{types::Bytes, Database};
|
||||
use crate::backend::heed::storable::StorableHeed;
|
||||
|
||||
//---------------------------------------------------------------------------------------------------- Types
|
||||
/// The concrete database type for `heed`, usable for reads and writes.
|
||||
pub(super) type HeedDb = Database<Bytes, Bytes>;
|
||||
pub(super) type HeedDb<K, V> = heed::Database<StorableHeed<K>, StorableHeed<V>>;
|
||||
|
|
|
|||
|
|
@ -9,38 +9,44 @@ use crate::{
|
|||
};
|
||||
|
||||
//---------------------------------------------------------------------------------------------------- DatabaseRo
|
||||
impl<T: Table> DatabaseRo<T> for RedbTableRo<'_> {
|
||||
fn get(&self, key: &T::Key) -> Result<Option<T::Value>, RuntimeError> {
|
||||
impl<T: Table> DatabaseRo<T> for RedbTableRo<'_, T::Key, T::Value> {
|
||||
fn get(&self, key: &T::Key) -> Result<&T::Value, RuntimeError> {
|
||||
todo!()
|
||||
}
|
||||
|
||||
fn get_range(
|
||||
&self,
|
||||
key: &T::Key,
|
||||
fn get_range<'a>(
|
||||
&'a self,
|
||||
key: &'a T::Key,
|
||||
amount: usize,
|
||||
) -> Result<impl Iterator<Item = T::Value>, RuntimeError> {
|
||||
let iter: std::vec::Drain<'_, T::Value> = todo!();
|
||||
) -> Result<impl Iterator<Item = &'a T::Value>, RuntimeError>
|
||||
where
|
||||
<T as Table>::Value: 'a,
|
||||
{
|
||||
let iter: std::vec::Drain<'_, &T::Value> = todo!();
|
||||
Ok(iter)
|
||||
}
|
||||
}
|
||||
|
||||
//---------------------------------------------------------------------------------------------------- DatabaseRw
|
||||
impl<T: Table> DatabaseRo<T> for RedbTableRw<'_, '_> {
|
||||
fn get(&self, key: &T::Key) -> Result<Option<T::Value>, RuntimeError> {
|
||||
impl<T: Table> DatabaseRo<T> for RedbTableRw<'_, '_, T::Key, T::Value> {
|
||||
fn get(&self, key: &T::Key) -> Result<&T::Value, RuntimeError> {
|
||||
todo!()
|
||||
}
|
||||
|
||||
fn get_range(
|
||||
&self,
|
||||
key: &T::Key,
|
||||
fn get_range<'a>(
|
||||
&'a self,
|
||||
key: &'a T::Key,
|
||||
amount: usize,
|
||||
) -> Result<impl Iterator<Item = T::Value>, RuntimeError> {
|
||||
let iter: std::vec::Drain<'_, T::Value> = todo!();
|
||||
) -> Result<impl Iterator<Item = &'a T::Value>, RuntimeError>
|
||||
where
|
||||
<T as Table>::Value: 'a,
|
||||
{
|
||||
let iter: std::vec::Drain<'_, &T::Value> = todo!();
|
||||
Ok(iter)
|
||||
}
|
||||
}
|
||||
|
||||
impl<T: Table> DatabaseRw<T> for RedbTableRw<'_, '_> {
|
||||
impl<T: Table> DatabaseRw<T> for RedbTableRw<'_, '_, T::Key, T::Value> {
|
||||
fn put(&mut self, key: &T::Key, value: &T::Value) -> Result<(), RuntimeError> {
|
||||
todo!()
|
||||
}
|
||||
|
|
@ -49,7 +55,7 @@ impl<T: Table> DatabaseRw<T> for RedbTableRw<'_, '_> {
|
|||
todo!()
|
||||
}
|
||||
|
||||
fn delete(&mut self, key: &T::Key) -> Result<bool, RuntimeError> {
|
||||
fn delete(&mut self, key: &T::Key) -> Result<(), RuntimeError> {
|
||||
todo!()
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -99,7 +99,7 @@ impl Env for ConcreteEnv {
|
|||
&self,
|
||||
tx_ro: &Self::TxRo<'_>,
|
||||
) -> Result<impl DatabaseRo<T>, RuntimeError> {
|
||||
let tx: RedbTableRo = todo!();
|
||||
let tx: RedbTableRo<'_, T::Key, T::Value> = todo!();
|
||||
Ok(tx)
|
||||
}
|
||||
|
||||
|
|
@ -108,7 +108,7 @@ impl Env for ConcreteEnv {
|
|||
&self,
|
||||
tx_rw: &mut Self::TxRw<'_>,
|
||||
) -> Result<impl DatabaseRw<T>, RuntimeError> {
|
||||
let tx: RedbTableRw = todo!();
|
||||
let tx: RedbTableRw<'_, '_, T::Key, T::Value> = todo!();
|
||||
Ok(tx)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,11 +2,8 @@
|
|||
|
||||
mod env;
|
||||
pub use env::ConcreteEnv;
|
||||
|
||||
mod error;
|
||||
|
||||
mod database;
|
||||
|
||||
mod error;
|
||||
mod storable;
|
||||
mod transaction;
|
||||
|
||||
mod types;
|
||||
|
|
|
|||
196
database/src/backend/redb/storable.rs
Normal file
196
database/src/backend/redb/storable.rs
Normal file
|
|
@ -0,0 +1,196 @@
|
|||
//! `cuprate_database::Storable` <-> `redb` serde trait compatibility layer.
|
||||
|
||||
//---------------------------------------------------------------------------------------------------- Use
|
||||
use std::{any::Any, borrow::Cow, cmp::Ordering, marker::PhantomData};
|
||||
|
||||
use redb::{RedbKey, RedbValue, TypeName};
|
||||
|
||||
use crate::{key::Key, storable::Storable};
|
||||
|
||||
//---------------------------------------------------------------------------------------------------- StorableRedb
|
||||
/// The glue struct that implements `redb`'s (de)serialization
|
||||
/// traits on any type that implements `cuprate_database::Key`.
|
||||
///
|
||||
/// Never actually gets constructed, just used for trait bound translations.
|
||||
#[derive(Debug)]
|
||||
pub(super) struct StorableRedb<T: Storable + ?Sized>(PhantomData<T>);
|
||||
|
||||
//---------------------------------------------------------------------------------------------------- RedbKey
|
||||
// If `Key` is also implemented, this can act as a `RedbKey`.
|
||||
impl<T: Key + ?Sized> RedbKey for StorableRedb<T> {
|
||||
#[inline]
|
||||
fn compare(left: &[u8], right: &[u8]) -> Ordering {
|
||||
<T as Key>::compare(left, right)
|
||||
}
|
||||
}
|
||||
|
||||
//---------------------------------------------------------------------------------------------------- RedbValue
|
||||
impl<T: Storable + ?Sized> RedbValue for StorableRedb<T> {
|
||||
type SelfType<'a> = &'a T where Self: 'a;
|
||||
type AsBytes<'a> = &'a [u8] where Self: 'a;
|
||||
|
||||
#[inline]
|
||||
fn fixed_width() -> Option<usize> {
|
||||
<T as Storable>::BYTE_LENGTH
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn from_bytes<'a>(data: &'a [u8]) -> &'a T
|
||||
where
|
||||
Self: 'a,
|
||||
{
|
||||
<T as Storable>::from_bytes(data)
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn as_bytes<'a, 'b: 'a>(value: &'a Self::SelfType<'b>) -> &'a [u8]
|
||||
where
|
||||
Self: 'a + 'b,
|
||||
{
|
||||
<T as Storable>::as_bytes(value)
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn type_name() -> TypeName {
|
||||
TypeName::new(std::any::type_name::<T>())
|
||||
}
|
||||
}
|
||||
|
||||
//---------------------------------------------------------------------------------------------------- Tests
|
||||
#[cfg(test)]
|
||||
#[allow(clippy::needless_pass_by_value)]
|
||||
mod test {
|
||||
use super::*;
|
||||
|
||||
// Each `#[test]` function has a `test()` to:
|
||||
// - log
|
||||
// - simplify trait bounds
|
||||
// - make sure the right function is being called
|
||||
|
||||
#[test]
|
||||
/// Assert `RedbKey::compare` works for `StorableRedb`.
|
||||
fn compare() {
|
||||
fn test<T: Key>(left: T, right: T, expected: Ordering) {
|
||||
println!("left: {left:?}, right: {right:?}, expected: {expected:?}");
|
||||
assert_eq!(
|
||||
<StorableRedb::<T> as RedbKey>::compare(
|
||||
<StorableRedb::<T> as RedbValue>::as_bytes(&&left),
|
||||
<StorableRedb::<T> as RedbValue>::as_bytes(&&right)
|
||||
),
|
||||
expected
|
||||
);
|
||||
}
|
||||
|
||||
test::<i64>(-1, 2, Ordering::Greater); // bytes are greater, not the value
|
||||
test::<u64>(0, 1, Ordering::Less);
|
||||
test::<[u8; 2]>([1, 1], [1, 0], Ordering::Greater);
|
||||
test::<[u8; 3]>([1, 2, 3], [1, 2, 3], Ordering::Equal);
|
||||
}
|
||||
|
||||
#[test]
|
||||
/// Assert `RedbKey::fixed_width` is accurate.
|
||||
fn fixed_width() {
|
||||
fn test<T: Storable + ?Sized>(expected: Option<usize>) {
|
||||
assert_eq!(<StorableRedb::<T> as RedbValue>::fixed_width(), expected);
|
||||
}
|
||||
|
||||
test::<()>(Some(0));
|
||||
test::<u8>(Some(1));
|
||||
test::<u16>(Some(2));
|
||||
test::<u32>(Some(4));
|
||||
test::<u64>(Some(8));
|
||||
test::<i8>(Some(1));
|
||||
test::<i16>(Some(2));
|
||||
test::<i32>(Some(4));
|
||||
test::<i64>(Some(8));
|
||||
test::<[u8]>(None);
|
||||
test::<[u8; 0]>(Some(0));
|
||||
test::<[u8; 1]>(Some(1));
|
||||
test::<[u8; 2]>(Some(2));
|
||||
test::<[u8; 3]>(Some(3));
|
||||
}
|
||||
|
||||
#[test]
|
||||
/// Assert `RedbKey::as_bytes` is accurate.
|
||||
fn as_bytes() {
|
||||
fn test<T: Storable + ?Sized>(t: &T, expected: &[u8]) {
|
||||
println!("t: {t:?}, expected: {expected:?}");
|
||||
assert_eq!(<StorableRedb::<T> as RedbValue>::as_bytes(&t), expected);
|
||||
}
|
||||
|
||||
test::<()>(&(), &[]);
|
||||
test::<u8>(&0, &[0]);
|
||||
test::<u16>(&1, &[1, 0]);
|
||||
test::<u32>(&2, &[2, 0, 0, 0]);
|
||||
test::<u64>(&3, &[3, 0, 0, 0, 0, 0, 0, 0]);
|
||||
test::<i8>(&-1, &[255]);
|
||||
test::<i16>(&-2, &[254, 255]);
|
||||
test::<i32>(&-3, &[253, 255, 255, 255]);
|
||||
test::<i64>(&-4, &[252, 255, 255, 255, 255, 255, 255, 255]);
|
||||
test::<[u8]>(&[1, 2], &[1, 2]);
|
||||
test::<[u8; 0]>(&[], &[]);
|
||||
test::<[u8; 1]>(&[255], &[255]);
|
||||
test::<[u8; 2]>(&[111, 0], &[111, 0]);
|
||||
test::<[u8; 3]>(&[1, 0, 1], &[1, 0, 1]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
/// Assert `RedbKey::from_bytes` is accurate.
|
||||
fn from_bytes() {
|
||||
fn test<T: Storable + ?Sized + PartialEq>(bytes: &[u8], expected: &T) {
|
||||
println!("bytes: {bytes:?}, expected: {expected:?}");
|
||||
assert_eq!(
|
||||
<StorableRedb::<T> as RedbValue>::from_bytes(bytes),
|
||||
expected
|
||||
);
|
||||
}
|
||||
|
||||
test::<()>([].as_slice(), &());
|
||||
test::<u8>([0].as_slice(), &0);
|
||||
test::<u16>([1, 0].as_slice(), &1);
|
||||
test::<u32>([2, 0, 0, 0].as_slice(), &2);
|
||||
test::<u64>([3, 0, 0, 0, 0, 0, 0, 0].as_slice(), &3);
|
||||
test::<i8>([255].as_slice(), &-1);
|
||||
test::<i16>([254, 255].as_slice(), &-2);
|
||||
test::<i32>([253, 255, 255, 255].as_slice(), &-3);
|
||||
test::<i64>([252, 255, 255, 255, 255, 255, 255, 255].as_slice(), &-4);
|
||||
test::<[u8]>([1, 2].as_slice(), &[1, 2]);
|
||||
test::<[u8; 0]>([].as_slice(), &[]);
|
||||
test::<[u8; 1]>([255].as_slice(), &[255]);
|
||||
test::<[u8; 2]>([111, 0].as_slice(), &[111, 0]);
|
||||
test::<[u8; 3]>([1, 0, 1].as_slice(), &[1, 0, 1]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
/// Assert `RedbKey::type_name` returns unique names.
|
||||
/// The name itself isn't tested, the invariant is that
|
||||
/// they are all unique.
|
||||
fn type_name() {
|
||||
// Can't use a proper set because `redb::TypeName: !Ord`.
|
||||
let set = [
|
||||
<StorableRedb<()> as RedbValue>::type_name(),
|
||||
<StorableRedb<u8> as RedbValue>::type_name(),
|
||||
<StorableRedb<u16> as RedbValue>::type_name(),
|
||||
<StorableRedb<u32> as RedbValue>::type_name(),
|
||||
<StorableRedb<u64> as RedbValue>::type_name(),
|
||||
<StorableRedb<i8> as RedbValue>::type_name(),
|
||||
<StorableRedb<i16> as RedbValue>::type_name(),
|
||||
<StorableRedb<i32> as RedbValue>::type_name(),
|
||||
<StorableRedb<i64> as RedbValue>::type_name(),
|
||||
<StorableRedb<[u8]> as RedbValue>::type_name(),
|
||||
<StorableRedb<[u8; 0]> as RedbValue>::type_name(),
|
||||
<StorableRedb<[u8; 1]> as RedbValue>::type_name(),
|
||||
<StorableRedb<[u8; 2]> as RedbValue>::type_name(),
|
||||
<StorableRedb<[u8; 3]> as RedbValue>::type_name(),
|
||||
];
|
||||
|
||||
// Check every permutation is unique.
|
||||
for (index, i) in set.iter().enumerate() {
|
||||
for (index2, j) in set.iter().enumerate() {
|
||||
if index != index2 {
|
||||
assert_ne!(i, j);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -17,14 +17,10 @@ impl TxRo<'_> for redb::ReadTransaction<'_> {
|
|||
|
||||
//---------------------------------------------------------------------------------------------------- TxRw
|
||||
impl TxRw<'_> for redb::WriteTransaction<'_> {
|
||||
/// TODO
|
||||
/// # Errors
|
||||
/// TODO
|
||||
fn commit(self) -> Result<(), RuntimeError> {
|
||||
todo!()
|
||||
}
|
||||
|
||||
/// TODO
|
||||
fn abort(self) {
|
||||
todo!()
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,10 +1,13 @@
|
|||
//! `redb` type aliases.
|
||||
|
||||
//---------------------------------------------------------------------------------------------------- Types
|
||||
// TODO: replace `()` with a byte container.
|
||||
use crate::{backend::redb::storable::StorableRedb, table::Table};
|
||||
|
||||
//---------------------------------------------------------------------------------------------------- Types
|
||||
/// The concrete type for readable `redb` tables.
|
||||
pub(super) type RedbTableRo<'env> = redb::ReadOnlyTable<'env, (), ()>;
|
||||
pub(super) type RedbTableRo<'env, K, V> =
|
||||
redb::ReadOnlyTable<'env, StorableRedb<K>, StorableRedb<V>>;
|
||||
|
||||
/// The concrete type for readable/writable `redb` tables.
|
||||
pub(super) type RedbTableRw<'env, 'tx> = redb::Table<'env, 'tx, (), ()>;
|
||||
pub(super) type RedbTableRw<'env, 'tx, K, V> =
|
||||
redb::Table<'env, 'tx, StorableRedb<K>, StorableRedb<V>>;
|
||||
|
|
|
|||
|
|
@ -15,6 +15,9 @@ use std::{
|
|||
path::{Path, PathBuf},
|
||||
};
|
||||
|
||||
#[cfg(feature = "serde")]
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use cuprate_helper::fs::cuprate_database_dir;
|
||||
|
||||
use crate::{constants::DATABASE_DATA_FILENAME, resize::ResizeAlgorithm};
|
||||
|
|
@ -27,7 +30,7 @@ use crate::{constants::DATABASE_DATA_FILENAME, resize::ResizeAlgorithm};
|
|||
///
|
||||
/// TODO: there's probably more options to add.
|
||||
#[derive(Debug, Clone, PartialEq, PartialOrd)]
|
||||
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
|
||||
#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
|
||||
pub struct Config {
|
||||
//------------------------ Database PATHs
|
||||
// These are private since we don't want
|
||||
|
|
@ -203,11 +206,7 @@ impl Default for Config {
|
|||
///
|
||||
/// are supported, all other variants will panic on [`crate::Env::open`].
|
||||
#[derive(Copy, Clone, Debug, Default, PartialEq, PartialOrd, Eq, Ord, Hash)]
|
||||
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
|
||||
#[cfg_attr(
|
||||
feature = "borsh",
|
||||
derive(borsh::BorshSerialize, borsh::BorshDeserialize)
|
||||
)]
|
||||
#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
|
||||
pub enum SyncMode {
|
||||
/// Use [`SyncMode::Fast`] until fully synced,
|
||||
/// then use [`SyncMode::Safe`].
|
||||
|
|
@ -302,11 +301,7 @@ pub enum SyncMode {
|
|||
/// The main function used to extract an actual
|
||||
/// usable thread count out of this is [`ReaderThreads::as_threads`].
|
||||
#[derive(Copy, Clone, Debug, Default, PartialEq, PartialOrd)]
|
||||
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
|
||||
#[cfg_attr(
|
||||
feature = "borsh",
|
||||
derive(borsh::BorshSerialize, borsh::BorshDeserialize)
|
||||
)]
|
||||
#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
|
||||
pub enum ReaderThreads {
|
||||
#[default]
|
||||
/// Spawn 1 reader thread per available thread on the machine.
|
||||
|
|
|
|||
|
|
@ -11,16 +11,23 @@ pub trait DatabaseRo<T: Table> {
|
|||
/// TODO
|
||||
/// # Errors
|
||||
/// TODO
|
||||
fn get(&self, key: &T::Key) -> Result<Option<T::Value>, RuntimeError>;
|
||||
///
|
||||
/// This will return [`RuntimeError::KeyNotFound`] wrapped in [`Err`] if `key` does not exist.
|
||||
fn get(&self, key: &T::Key) -> Result<&T::Value, RuntimeError>;
|
||||
|
||||
/// TODO
|
||||
/// # Errors
|
||||
/// TODO
|
||||
fn get_range(
|
||||
&self,
|
||||
key: &T::Key,
|
||||
//
|
||||
// TODO: (Iterators + ?Sized + lifetimes) == bad time
|
||||
// fix this later.
|
||||
fn get_range<'a>(
|
||||
&'a self,
|
||||
key: &'a T::Key,
|
||||
amount: usize,
|
||||
) -> Result<impl Iterator<Item = T::Value>, RuntimeError>;
|
||||
) -> Result<impl Iterator<Item = &'a T::Value>, RuntimeError>
|
||||
where
|
||||
<T as Table>::Value: 'a;
|
||||
}
|
||||
|
||||
//---------------------------------------------------------------------------------------------------- DatabaseRw
|
||||
|
|
@ -41,5 +48,7 @@ pub trait DatabaseRw<T: Table>: DatabaseRo<T> {
|
|||
/// TODO
|
||||
/// # Errors
|
||||
/// TODO
|
||||
fn delete(&mut self, key: &T::Key) -> Result<bool, RuntimeError>;
|
||||
///
|
||||
/// This will return [`RuntimeError::KeyNotFound`] wrapped in [`Err`] if `key` does not exist.
|
||||
fn delete(&mut self, key: &T::Key) -> Result<(), RuntimeError>;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,60 +1,83 @@
|
|||
//! Database key abstraction; `trait Key`.
|
||||
|
||||
//---------------------------------------------------------------------------------------------------- Import
|
||||
use crate::pod::Pod;
|
||||
use std::cmp::Ordering;
|
||||
|
||||
use bytemuck::Pod;
|
||||
|
||||
use crate::storable::{self, Storable};
|
||||
|
||||
//---------------------------------------------------------------------------------------------------- Table
|
||||
/// Database [`Table`](crate::table::Table) key metadata.
|
||||
///
|
||||
/// Purely compile time information for database table keys, supporting duplicate keys.
|
||||
pub trait Key {
|
||||
pub trait Key: Storable + Sized {
|
||||
/// Does this [`Key`] require multiple keys to reach a value?
|
||||
///
|
||||
/// If [`Key::DUPLICATE`] is `true`, [`Key::Secondary`] will contain
|
||||
/// the "subkey", or secondary key needed to access the actual value.
|
||||
///
|
||||
/// If [`Key::DUPLICATE`] is `false`, [`Key::Secondary`] is ignored.
|
||||
/// Consider using [`std::convert::Infallible`] as the type.
|
||||
/// # Invariant
|
||||
/// - If [`Key::DUPLICATE`] is `true`, [`Key::primary_secondary`] MUST be re-implemented.
|
||||
/// - If [`Key::DUPLICATE`] is `true`, [`Key::new_with_max_secondary`] MUST be re-implemented.
|
||||
const DUPLICATE: bool;
|
||||
|
||||
/// Does this [`Key`] have a custom comparison function?
|
||||
///
|
||||
/// # Invariant
|
||||
/// If [`Key::CUSTOM_COMPARE`] is `true`, [`Key::compare`] MUST be re-implemented.
|
||||
const CUSTOM_COMPARE: bool;
|
||||
|
||||
/// The primary key type.
|
||||
type Primary: Pod;
|
||||
type Primary: Storable;
|
||||
|
||||
/// The secondary key type.
|
||||
type Secondary: Pod;
|
||||
|
||||
/// Acquire [`Key::Primary`].
|
||||
fn primary(self) -> Self::Primary;
|
||||
|
||||
/// Acquire [`Self::Primary`] & [`Self::Secondary`].
|
||||
/// Acquire [`Self::Primary`] and the secondary key.
|
||||
///
|
||||
/// This only needs to be implemented on types that are [`Self::DUPLICATE`].
|
||||
///
|
||||
/// Consider using [`unreachable!()`] on non-duplicate key tables.
|
||||
fn primary_secondary(self) -> (Self::Primary, Self::Secondary);
|
||||
}
|
||||
/// # TODO: doc test
|
||||
fn primary_secondary(self) -> (Self::Primary, u64) {
|
||||
unreachable!()
|
||||
}
|
||||
|
||||
/// Duplicate key container.
|
||||
///
|
||||
/// This is a generic container to use alongside [`Key`] to support
|
||||
/// tables that require more than 1 key to access the value.
|
||||
#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
|
||||
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
|
||||
#[cfg_attr(
|
||||
feature = "borsh",
|
||||
derive(borsh::BorshSerialize, borsh::BorshDeserialize)
|
||||
)]
|
||||
pub struct DupKey<P, S> {
|
||||
/// Primary key type.
|
||||
pub primary: P,
|
||||
/// Secondary key type.
|
||||
pub secondary: S,
|
||||
/// Compare 2 [`Key`]'s against each other.
|
||||
///
|
||||
/// By default, this does a straight _byte_ comparison,
|
||||
/// not a comparison of the key's value.
|
||||
///
|
||||
/// ```rust
|
||||
/// # use cuprate_database::*;
|
||||
/// assert_eq!(
|
||||
/// <u64 as Key>::compare([0].as_slice(), [1].as_slice()),
|
||||
/// std::cmp::Ordering::Less,
|
||||
/// );
|
||||
/// assert_eq!(
|
||||
/// <u64 as Key>::compare([1].as_slice(), [1].as_slice()),
|
||||
/// std::cmp::Ordering::Equal,
|
||||
/// );
|
||||
/// assert_eq!(
|
||||
/// <u64 as Key>::compare([2].as_slice(), [1].as_slice()),
|
||||
/// std::cmp::Ordering::Greater,
|
||||
/// );
|
||||
/// ```
|
||||
fn compare(left: &[u8], right: &[u8]) -> Ordering {
|
||||
left.cmp(right)
|
||||
}
|
||||
|
||||
/// Create a new [`Key`] from the [`Key::Primary`] type,
|
||||
/// with the secondary key type set to the maximum value.
|
||||
///
|
||||
/// # Invariant
|
||||
/// Secondary key must be the max value of the type.
|
||||
///
|
||||
/// # TODO: doc test
|
||||
fn new_with_max_secondary(primary: Self::Primary) -> Self {
|
||||
unreachable!()
|
||||
}
|
||||
}
|
||||
|
||||
//---------------------------------------------------------------------------------------------------- Impl
|
||||
/// TODO: remove after we finalize tables.
|
||||
///
|
||||
/// Implement `Key` on most primitive types.
|
||||
///
|
||||
/// `Key::DUPLICATE` is always `false`.
|
||||
/// - `Key::DUPLICATE` is always `false`.
|
||||
/// - `Key::CUSTOM_COMPARE` is always `false`.
|
||||
macro_rules! impl_key {
|
||||
(
|
||||
$(
|
||||
|
|
@ -64,30 +87,13 @@ macro_rules! impl_key {
|
|||
$(
|
||||
impl Key for $t {
|
||||
const DUPLICATE: bool = false;
|
||||
const CUSTOM_COMPARE: bool = false;
|
||||
|
||||
type Primary = $t;
|
||||
|
||||
// This 0 variant enum is unconstructable,
|
||||
// and "has the same role as the ! “never” type":
|
||||
// <https://doc.rust-lang.org/std/convert/enum.Infallible.html#future-compatibility>.
|
||||
//
|
||||
// FIXME: Use the `!` type when stable.
|
||||
type Secondary = std::convert::Infallible;
|
||||
|
||||
#[inline(always)]
|
||||
fn primary(self) -> Self::Primary {
|
||||
self
|
||||
}
|
||||
|
||||
#[cold] #[inline(never)]
|
||||
fn primary_secondary(self) -> (Self::Primary, Self::Secondary) {
|
||||
unreachable!();
|
||||
}
|
||||
}
|
||||
)*
|
||||
};
|
||||
}
|
||||
|
||||
// Implement `Key` for primitives.
|
||||
impl_key! {
|
||||
u8,
|
||||
|
|
@ -100,25 +106,11 @@ impl_key! {
|
|||
i64,
|
||||
}
|
||||
|
||||
// Implement `Key` for any [`DupKey`] using [`Copy`] types.
|
||||
impl<P, S> Key for DupKey<P, S>
|
||||
where
|
||||
P: Pod + Copy,
|
||||
S: Pod + Copy,
|
||||
{
|
||||
const DUPLICATE: bool = true;
|
||||
type Primary = P;
|
||||
type Secondary = S;
|
||||
impl<const N: usize, T: Key + Pod> Key for [T; N] {
|
||||
const DUPLICATE: bool = false;
|
||||
const CUSTOM_COMPARE: bool = false;
|
||||
|
||||
#[inline]
|
||||
fn primary(self) -> Self::Primary {
|
||||
self.primary
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn primary_secondary(self) -> (Self::Primary, Self::Secondary) {
|
||||
(self.primary, self.secondary)
|
||||
}
|
||||
type Primary = [T; N];
|
||||
}
|
||||
|
||||
//---------------------------------------------------------------------------------------------------- Tests
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@
|
|||
//! # Purpose
|
||||
//! This crate does 3 things:
|
||||
//! 1. Abstracts various database backends with traits
|
||||
//! 2. Implements various `Monero` related [functions](ops) & [`tables`]
|
||||
//! 2. Implements various `Monero` related [functions](ops) & [tables] & [types]
|
||||
//! 3. Exposes a [`tower::Service`] backed by a thread-pool
|
||||
//!
|
||||
//! # Terminology
|
||||
|
|
@ -22,6 +22,7 @@
|
|||
//! | `Table` | Solely the metadata of a `Database` (the `key` and `value` types, and the name)
|
||||
//! | `TxRo` | Read only transaction
|
||||
//! | `TxRw` | Read/write transaction
|
||||
//! | `Storable` | A data that type can be stored in the database
|
||||
//!
|
||||
//! The dataflow is `Env` -> `Tx` -> `Database`
|
||||
//!
|
||||
|
|
@ -313,20 +314,22 @@ mod free;
|
|||
pub mod resize;
|
||||
|
||||
mod key;
|
||||
pub use key::{DupKey, Key};
|
||||
pub use key::Key;
|
||||
|
||||
mod macros;
|
||||
|
||||
pub mod ops;
|
||||
mod storable;
|
||||
pub use storable::Storable;
|
||||
|
||||
mod pod;
|
||||
pub use pod::Pod;
|
||||
pub mod ops;
|
||||
|
||||
mod table;
|
||||
pub use table::Table;
|
||||
|
||||
pub mod tables;
|
||||
|
||||
pub mod types;
|
||||
|
||||
mod transaction;
|
||||
pub use transaction::{TxRo, TxRw};
|
||||
|
||||
|
|
|
|||
|
|
@ -1,518 +0,0 @@
|
|||
//! (De)serialization for table keys & values.
|
||||
//!
|
||||
//! All keys and values in database tables must be able
|
||||
//! to be (de)serialized into/from raw bytes ([u8]).
|
||||
|
||||
//---------------------------------------------------------------------------------------------------- Import
|
||||
use std::{
|
||||
borrow::Cow,
|
||||
io::{Read, Write},
|
||||
sync::Arc,
|
||||
};
|
||||
|
||||
//---------------------------------------------------------------------------------------------------- Pod
|
||||
/// Plain Old Data.
|
||||
///
|
||||
/// Trait representing very simple types that can be
|
||||
/// (de)serialized into/from bytes.
|
||||
///
|
||||
/// Reference: <https://docs.rs/bytemuck/latest/bytemuck/trait.Pod.html>
|
||||
///
|
||||
/// ## Endianness
|
||||
/// As `bytemuck` provides everything needed here + more, it could be used,
|
||||
/// _but_, its `Pod` is endian dependant. We need to ensure bytes are the
|
||||
/// exact same such that the database stores the same bytes on different machines;
|
||||
/// so we use little endian functions instead, e.g. [`u8::to_le_bytes`].
|
||||
///
|
||||
/// This also means an `INVARIANT` of this trait is that
|
||||
/// implementors must use little endian bytes when applicable.
|
||||
///
|
||||
/// Slice types (just raw `[u8]` bytes) are (de)serialized as-is.
|
||||
///
|
||||
/// ## Sealed
|
||||
/// This trait is [`Sealed`](https://rust-lang.github.io/api-guidelines/future-proofing.html#sealed-traits-protect-against-downstream-implementations-c-sealed).
|
||||
///
|
||||
/// It cannot be implemented outside this crate,
|
||||
/// and is only implemented on specific types.
|
||||
///
|
||||
/// # TODO
|
||||
/// This could be implemented on `bytes::Bytes` if needed.
|
||||
///
|
||||
/// Maybe under a `bytes` feature flag.
|
||||
pub trait Pod: Sized + private::Sealed {
|
||||
/// Return `self` in byte form.
|
||||
///
|
||||
/// The returned bytes can be any form of array,
|
||||
/// - `[u8]`
|
||||
/// - `[u8; N]`
|
||||
/// - [`Vec<u8>`]
|
||||
///
|
||||
/// ..etc.
|
||||
///
|
||||
/// This is used on slice types (`Vec<u8>`, `[u8; N]`, etc) for cheap conversions.
|
||||
///
|
||||
/// Integer types ([`u8`], [`f32`], [`i8`], etc) return a fixed-sized array.
|
||||
fn as_bytes(&self) -> impl AsRef<[u8]>;
|
||||
|
||||
/// TODO
|
||||
fn into_bytes(self) -> Cow<'static, [u8]>;
|
||||
|
||||
/// Create [`Self`] from bytes.
|
||||
///
|
||||
/// # Panics
|
||||
/// This function should be infallible.
|
||||
///
|
||||
/// If `bytes` is invalid, this should panic.
|
||||
fn from_bytes(bytes: &[u8]) -> Self;
|
||||
|
||||
/// Convert [`Self`] into bytes, and write those bytes into a [`Write`]r.
|
||||
///
|
||||
/// The `usize` returned should be how many bytes were written.
|
||||
///
|
||||
/// TODO: do we ever actually need how many bytes were written?
|
||||
///
|
||||
/// # Panics
|
||||
/// This function should be infallible.
|
||||
///
|
||||
/// If the `writer` errors, this should panic.
|
||||
fn to_writer<W: Write>(self, writer: &mut W) -> usize;
|
||||
|
||||
/// Create [`Self`] by reading bytes from a [`Read`]er.
|
||||
///
|
||||
/// # Panics
|
||||
/// This function should be infallible.
|
||||
///
|
||||
/// If the `reader` errors, this should panic.
|
||||
fn from_reader<R: Read>(reader: &mut R) -> Self;
|
||||
}
|
||||
|
||||
/// Private module, should not be accessible outside this crate.
|
||||
///
|
||||
/// Used to block outsiders implementing [`Pod`].
|
||||
/// All [`Pod`] types must also implement [`Sealed`].
|
||||
mod private {
|
||||
/// Private sealed trait.
|
||||
///
|
||||
/// Cannot be implemented outside this crate.
|
||||
pub trait Sealed {}
|
||||
|
||||
/// Implement `Sealed`.
|
||||
macro_rules! impl_sealed {
|
||||
($(
|
||||
$t:ty // The type to implement for.
|
||||
),* $(,)?) => {
|
||||
$(
|
||||
impl Sealed for $t {}
|
||||
)*
|
||||
};
|
||||
}
|
||||
|
||||
// Special case cause of generic.
|
||||
impl<const N: usize> Sealed for [u8; N] {}
|
||||
|
||||
impl_sealed! {
|
||||
std::convert::Infallible,
|
||||
Vec<u8>,
|
||||
Box<[u8]>,
|
||||
std::sync::Arc<[u8]>,
|
||||
f32,
|
||||
f64,
|
||||
u8,
|
||||
u16,
|
||||
u32,
|
||||
u64,
|
||||
u128,
|
||||
usize,
|
||||
i8,
|
||||
i16,
|
||||
i32,
|
||||
i64,
|
||||
i128,
|
||||
isize,
|
||||
}
|
||||
}
|
||||
|
||||
//---------------------------------------------------------------------------------------------------- Pod Impl (bytes)
|
||||
// Implement for `Infallible`.
|
||||
// This type is `!` and should never be constructable,
|
||||
// so all these functions will just panic.
|
||||
impl Pod for std::convert::Infallible {
|
||||
#[cold]
|
||||
#[inline(never)]
|
||||
fn as_bytes(&self) -> impl AsRef<[u8]> {
|
||||
let bytes: &[u8] = unreachable!();
|
||||
bytes
|
||||
}
|
||||
|
||||
#[cold]
|
||||
#[inline(never)]
|
||||
fn into_bytes(self) -> Cow<'static, [u8]> {
|
||||
unreachable!()
|
||||
}
|
||||
|
||||
#[cold]
|
||||
#[inline(never)]
|
||||
fn from_bytes(bytes: &[u8]) -> Self {
|
||||
unreachable!()
|
||||
}
|
||||
|
||||
#[cold]
|
||||
#[inline(never)]
|
||||
fn from_reader<R: Read>(reader: &mut R) -> Self {
|
||||
unreachable!()
|
||||
}
|
||||
|
||||
#[cold]
|
||||
#[inline(never)]
|
||||
fn to_writer<W: Write>(self, writer: &mut W) -> usize {
|
||||
unreachable!()
|
||||
}
|
||||
}
|
||||
|
||||
// Implement for owned `Vec` bytes.
|
||||
impl Pod for Vec<u8> {
|
||||
#[inline]
|
||||
fn as_bytes(&self) -> impl AsRef<[u8]> {
|
||||
self
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn into_bytes(self) -> Cow<'static, [u8]> {
|
||||
Cow::Owned(self)
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn from_bytes(bytes: &[u8]) -> Self {
|
||||
bytes.to_vec()
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn from_reader<R: Read>(reader: &mut R) -> Self {
|
||||
// FIXME: Could be `Vec::with_capacity(likely_size)`?
|
||||
let mut vec = vec![];
|
||||
|
||||
reader
|
||||
.read_to_end(&mut vec)
|
||||
.expect("Pod::<Vec<u8>>::read_to_end() failed");
|
||||
|
||||
vec
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn to_writer<W: Write>(self, writer: &mut W) -> usize {
|
||||
writer
|
||||
.write_all(&self)
|
||||
.expect("Pod::<Vec<u8>>::write_all() failed");
|
||||
|
||||
self.len()
|
||||
}
|
||||
}
|
||||
|
||||
// Implement for any sized stack array.
|
||||
impl<const N: usize> Pod for [u8; N] {
|
||||
#[inline]
|
||||
fn as_bytes(&self) -> impl AsRef<[u8]> {
|
||||
self
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn into_bytes(self) -> Cow<'static, [u8]> {
|
||||
Cow::Owned(self.to_vec())
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn from_bytes(bytes: &[u8]) -> Self {
|
||||
// Return if the bytes are too short/long.
|
||||
let bytes_len = bytes.len();
|
||||
assert_eq!(
|
||||
bytes_len, N,
|
||||
"Pod::<[u8; {N}]>::from_bytes() failed, expected_len: {N}, found_len: {bytes_len}",
|
||||
);
|
||||
|
||||
let mut array = [0_u8; N];
|
||||
// INVARIANT: we checked the length is valid above.
|
||||
array.copy_from_slice(bytes);
|
||||
|
||||
array
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn from_reader<R: Read>(reader: &mut R) -> Self {
|
||||
let mut bytes = [0_u8; N];
|
||||
reader
|
||||
.read_exact(&mut bytes)
|
||||
.expect("Pod::<[u8; {N}]>::read_exact() failed");
|
||||
|
||||
bytes
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn to_writer<W: Write>(self, writer: &mut W) -> usize {
|
||||
writer
|
||||
.write_all(&self)
|
||||
.expect("Pod::<[u8; {N}]>::write_all() failed");
|
||||
self.len()
|
||||
}
|
||||
}
|
||||
|
||||
// Implement for any sized boxed array.
|
||||
//
|
||||
// In-case `[u8; N]` is too big and would
|
||||
// overflow the stack, this can be used.
|
||||
//
|
||||
// The benefit over `Vec<u8>` is that the capacity & length are static.
|
||||
//
|
||||
// The weird constructions of `Box` below are on purpose to avoid this:
|
||||
// <https://github.com/rust-lang/rust/issues/53827>
|
||||
impl Pod for Box<[u8]> {
|
||||
#[inline]
|
||||
fn as_bytes(&self) -> impl AsRef<[u8]> {
|
||||
self
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn into_bytes(self) -> Cow<'static, [u8]> {
|
||||
Cow::Owned(self.into())
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn from_bytes(bytes: &[u8]) -> Self {
|
||||
Self::from(bytes)
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn from_reader<R: Read>(reader: &mut R) -> Self {
|
||||
let mut bytes = vec![];
|
||||
reader
|
||||
.read_to_end(bytes.as_mut())
|
||||
.expect("Pod::<Box<[u8]>>::read_to_end() failed");
|
||||
bytes.into_boxed_slice()
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn to_writer<W: Write>(self, writer: &mut W) -> usize {
|
||||
writer
|
||||
.write_all(&self)
|
||||
.expect("Pod::<Box<[u8]>>::write_all() failed");
|
||||
self.len()
|
||||
}
|
||||
}
|
||||
|
||||
// Implement for any Arc bytes.
|
||||
impl Pod for Arc<[u8]> {
|
||||
#[inline]
|
||||
fn as_bytes(&self) -> impl AsRef<[u8]> {
|
||||
self
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn into_bytes(self) -> Cow<'static, [u8]> {
|
||||
Cow::Owned(self.to_vec())
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn from_bytes(bytes: &[u8]) -> Self {
|
||||
Self::from(bytes)
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn from_reader<R: Read>(reader: &mut R) -> Self {
|
||||
let mut bytes = vec![];
|
||||
reader
|
||||
.read_to_end(bytes.as_mut())
|
||||
.expect("Pod::<Arc<[u8]>>::read_to_end() failed");
|
||||
Self::from(bytes)
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn to_writer<W: Write>(self, writer: &mut W) -> usize {
|
||||
writer
|
||||
.write_all(&self)
|
||||
.expect("Pod::<Arc<[u8]>>::write_all() failed");
|
||||
self.len()
|
||||
}
|
||||
}
|
||||
|
||||
//---------------------------------------------------------------------------------------------------- Pod Impl (numbers)
|
||||
/// Implement `Pod` on primitive numbers.
|
||||
///
|
||||
/// This will always use little endian representations.
|
||||
macro_rules! impl_pod_le_bytes {
|
||||
($(
|
||||
$number:ident => // The number type.
|
||||
$length:literal // The length of `u8`'s this type takes up.
|
||||
),* $(,)?) => {
|
||||
$(
|
||||
impl Pod for $number {
|
||||
#[inline]
|
||||
fn as_bytes(&self) -> impl AsRef<[u8]> {
|
||||
$number::to_le_bytes(*self)
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn into_bytes(self) -> Cow<'static, [u8]> {
|
||||
Cow::Owned(self.as_bytes().as_ref().to_vec())
|
||||
}
|
||||
|
||||
#[inline]
|
||||
/// This function returns [`Err`] if `bytes`'s length is not
|
||||
#[doc = concat!(" ", stringify!($length), ".")]
|
||||
fn from_bytes(bytes: &[u8]) -> Self {
|
||||
// Return if the bytes are too short/long.
|
||||
let bytes_len = bytes.len();
|
||||
assert_eq!(
|
||||
bytes_len, $length,
|
||||
"Pod::<[u8; {0}]>::from_bytes() failed, expected_len: {0}, found_len: {bytes_len}",
|
||||
$length,
|
||||
);
|
||||
|
||||
let mut array = [0_u8; $length];
|
||||
// INVARIANT: we checked the length is valid above.
|
||||
array.copy_from_slice(bytes);
|
||||
|
||||
$number::from_le_bytes(array)
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn to_writer<W: Write>(self, writer: &mut W) -> usize {
|
||||
writer.write_all(self.as_bytes().as_ref()).expect(concat!(
|
||||
"Pod::<",
|
||||
stringify!($number),
|
||||
">::write_all() failed",
|
||||
));
|
||||
$length
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn from_reader<R: Read>(reader: &mut R) -> Self {
|
||||
let mut bytes = [0_u8; $length];
|
||||
|
||||
// Read exactly the bytes required.
|
||||
reader.read_exact(&mut bytes).expect(concat!(
|
||||
"Pod::<",
|
||||
stringify!($number),
|
||||
">::react_exact() failed",
|
||||
));
|
||||
|
||||
// INVARIANT: we checked the length is valid above.
|
||||
$number::from_le_bytes(bytes)
|
||||
}
|
||||
}
|
||||
)*
|
||||
};
|
||||
}
|
||||
|
||||
impl_pod_le_bytes! {
|
||||
f32 => 4,
|
||||
f64 => 8,
|
||||
|
||||
u8 => 1,
|
||||
u16 => 2,
|
||||
u32 => 4,
|
||||
u64 => 8,
|
||||
usize => 8,
|
||||
u128 => 16,
|
||||
|
||||
i8 => 1,
|
||||
i16 => 2,
|
||||
i32 => 4,
|
||||
i64 => 8,
|
||||
isize => 8,
|
||||
i128 => 16,
|
||||
}
|
||||
|
||||
//---------------------------------------------------------------------------------------------------- Tests
|
||||
#[cfg(test)]
|
||||
mod test {
|
||||
use super::*;
|
||||
|
||||
/// Serialize, deserialize, and compare that
|
||||
/// the intermediate/end results are correct.
|
||||
fn test_serde<const LEN: usize, T: Pod + Copy + PartialEq + std::fmt::Debug>(
|
||||
// The primitive number function that
|
||||
// converts the number into little endian bytes,
|
||||
// e.g `u8::to_le_bytes`.
|
||||
to_le_bytes: fn(T) -> [u8; LEN],
|
||||
// A `Vec` of the numbers to test.
|
||||
t: Vec<T>,
|
||||
) {
|
||||
for t in t {
|
||||
let expected_bytes = to_le_bytes(t);
|
||||
|
||||
println!("testing: {t:?}, expected_bytes: {expected_bytes:?}");
|
||||
|
||||
let mut bytes = vec![];
|
||||
|
||||
// (De)serialize.
|
||||
let se: usize = t.to_writer::<Vec<u8>>(bytes.as_mut());
|
||||
let de: T = T::from_reader::<&[u8]>(&mut bytes.as_slice());
|
||||
|
||||
println!("written: {se}, deserialize_t: {de:?}, bytes: {bytes:?}\n");
|
||||
|
||||
// Assert we wrote correct amount of bytes
|
||||
// and deserialized correctly.
|
||||
assert_eq!(se, expected_bytes.len());
|
||||
assert_eq!(de, t);
|
||||
}
|
||||
}
|
||||
|
||||
/// Create all the float tests.
|
||||
macro_rules! test_float {
|
||||
($(
|
||||
$float:ident // The float type.
|
||||
),* $(,)?) => {
|
||||
$(
|
||||
#[test]
|
||||
fn $float() {
|
||||
test_serde(
|
||||
$float::to_le_bytes,
|
||||
vec![
|
||||
-1.0,
|
||||
0.0,
|
||||
1.0,
|
||||
$float::MIN,
|
||||
$float::MAX,
|
||||
$float::INFINITY,
|
||||
$float::NEG_INFINITY,
|
||||
],
|
||||
);
|
||||
}
|
||||
)*
|
||||
};
|
||||
}
|
||||
|
||||
test_float! {
|
||||
f32,
|
||||
f64,
|
||||
}
|
||||
|
||||
/// Create all the (un)signed number tests.
|
||||
/// u8 -> u128, i8 -> i128.
|
||||
macro_rules! test_unsigned {
|
||||
($(
|
||||
$number:ident // The integer type.
|
||||
),* $(,)?) => {
|
||||
$(
|
||||
#[test]
|
||||
fn $number() {
|
||||
test_serde($number::to_le_bytes, vec![$number::MIN, 0, 1, $number::MAX]);
|
||||
}
|
||||
)*
|
||||
};
|
||||
}
|
||||
|
||||
test_unsigned! {
|
||||
u8,
|
||||
u16,
|
||||
u32,
|
||||
u64,
|
||||
u128,
|
||||
usize,
|
||||
i8,
|
||||
i16,
|
||||
i32,
|
||||
i64,
|
||||
i128,
|
||||
isize,
|
||||
}
|
||||
}
|
||||
|
|
@ -35,10 +35,6 @@ use std::{num::NonZeroUsize, sync::OnceLock};
|
|||
/// **With the caveat being we are taking a `WriteGuard` to a `RwLock`.**
|
||||
#[derive(Copy, Clone, Debug, PartialEq, PartialOrd)]
|
||||
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
|
||||
#[cfg_attr(
|
||||
feature = "borsh",
|
||||
derive(borsh::BorshSerialize, borsh::BorshDeserialize)
|
||||
)]
|
||||
pub enum ResizeAlgorithm {
|
||||
/// Uses [`monero`].
|
||||
Monero,
|
||||
|
|
|
|||
227
database/src/storable.rs
Normal file
227
database/src/storable.rs
Normal file
|
|
@ -0,0 +1,227 @@
|
|||
//! (De)serialization for table keys & values.
|
||||
|
||||
//---------------------------------------------------------------------------------------------------- Import
|
||||
use std::{
|
||||
borrow::Cow,
|
||||
fmt::Debug,
|
||||
io::{Read, Write},
|
||||
sync::Arc,
|
||||
};
|
||||
|
||||
use bytemuck::{AnyBitPattern, NoUninit};
|
||||
|
||||
//---------------------------------------------------------------------------------------------------- Storable
|
||||
/// A type that can be stored in the database.
|
||||
///
|
||||
/// All keys and values in the database must be able
|
||||
/// to be (de)serialized into/from raw bytes (`[u8]`).
|
||||
///
|
||||
/// This trait represents types that can be **perfectly**
|
||||
/// casted/represented as raw bytes.
|
||||
///
|
||||
/// ## `bytemuck`
|
||||
/// Any type that implements `bytemuck`'s [`NoUninit`] + [`AnyBitPattern`]
|
||||
/// (and [Debug]) will automatically implement [`Storable`].
|
||||
///
|
||||
/// This includes:
|
||||
/// - Most primitive types
|
||||
/// - All types in [`tables`](crate::tables)
|
||||
/// - Slices, e.g, `[T] where T: Storable`
|
||||
///
|
||||
/// ```rust
|
||||
/// # use cuprate_database::*;
|
||||
/// let number: u64 = 0;
|
||||
///
|
||||
/// // Into bytes.
|
||||
/// let into = Storable::as_bytes(&number);
|
||||
/// assert_eq!(into, &[0; 8]);
|
||||
///
|
||||
/// // From bytes.
|
||||
/// let from: &u64 = Storable::from_bytes(&into);
|
||||
/// assert_eq!(from, &number);
|
||||
/// ```
|
||||
///
|
||||
/// ## Invariants
|
||||
/// No function in this trait is expected to panic.
|
||||
///
|
||||
/// The byte conversions must execute flawlessly.
|
||||
///
|
||||
/// ## Endianness
|
||||
/// This trait doesn't currently care about endianness.
|
||||
///
|
||||
/// Bytes are (de)serialized as-is, and `bytemuck`
|
||||
/// types are architecture-dependant.
|
||||
///
|
||||
/// Most likely, the bytes are little-endian, however
|
||||
/// that cannot be relied upon when using this trait.
|
||||
pub trait Storable: Debug {
|
||||
/// Is this type fixed width in byte length?
|
||||
///
|
||||
/// I.e., when converting `Self` to bytes, is it
|
||||
/// represented with a fixed length array of bytes?
|
||||
///
|
||||
/// # `Some`
|
||||
/// This should be `Some(usize)` on types like:
|
||||
/// - `u8`
|
||||
/// - `u64`
|
||||
/// - `i32`
|
||||
///
|
||||
/// where the byte length is known.
|
||||
///
|
||||
/// # `None`
|
||||
/// This should be `None` on any variable-length type like:
|
||||
/// - `str`
|
||||
/// - `[u8]`
|
||||
/// - `Vec<u8>`
|
||||
///
|
||||
/// # Examples
|
||||
/// ```rust
|
||||
/// # use cuprate_database::Storable;
|
||||
/// assert_eq!(<()>::BYTE_LENGTH, Some(0));
|
||||
/// assert_eq!(u8::BYTE_LENGTH, Some(1));
|
||||
/// assert_eq!(u16::BYTE_LENGTH, Some(2));
|
||||
/// assert_eq!(u32::BYTE_LENGTH, Some(4));
|
||||
/// assert_eq!(u64::BYTE_LENGTH, Some(8));
|
||||
/// assert_eq!(i8::BYTE_LENGTH, Some(1));
|
||||
/// assert_eq!(i16::BYTE_LENGTH, Some(2));
|
||||
/// assert_eq!(i32::BYTE_LENGTH, Some(4));
|
||||
/// assert_eq!(i64::BYTE_LENGTH, Some(8));
|
||||
/// assert_eq!(<[u8]>::BYTE_LENGTH, None);
|
||||
/// assert_eq!(<[u8; 0]>::BYTE_LENGTH, Some(0));
|
||||
/// assert_eq!(<[u8; 1]>::BYTE_LENGTH, Some(1));
|
||||
/// assert_eq!(<[u8; 2]>::BYTE_LENGTH, Some(2));
|
||||
/// assert_eq!(<[u8; 3]>::BYTE_LENGTH, Some(3));
|
||||
/// ```
|
||||
const BYTE_LENGTH: Option<usize>;
|
||||
|
||||
/// Return `self` in byte form.
|
||||
fn as_bytes(&self) -> &[u8];
|
||||
|
||||
/// Create [`Self`] from bytes.
|
||||
fn from_bytes(bytes: &[u8]) -> &Self;
|
||||
}
|
||||
|
||||
//---------------------------------------------------------------------------------------------------- Impl
|
||||
impl<T: NoUninit + AnyBitPattern + Debug> Storable for T {
|
||||
const BYTE_LENGTH: Option<usize> = Some(std::mem::size_of::<T>());
|
||||
|
||||
#[inline]
|
||||
fn as_bytes(&self) -> &[u8] {
|
||||
bytemuck::bytes_of(self)
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn from_bytes(bytes: &[u8]) -> &Self {
|
||||
bytemuck::from_bytes(bytes)
|
||||
}
|
||||
}
|
||||
|
||||
impl<T: NoUninit + AnyBitPattern + Debug> Storable for [T] {
|
||||
const BYTE_LENGTH: Option<usize> = None;
|
||||
|
||||
#[inline]
|
||||
fn as_bytes(&self) -> &[u8] {
|
||||
bytemuck::must_cast_slice(self)
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn from_bytes(bytes: &[u8]) -> &Self {
|
||||
bytemuck::must_cast_slice(bytes)
|
||||
}
|
||||
}
|
||||
|
||||
//---------------------------------------------------------------------------------------------------- Tests
|
||||
#[cfg(test)]
|
||||
mod test {
|
||||
use super::*;
|
||||
|
||||
/// Serialize, deserialize, and compare that
|
||||
/// the intermediate/end results are correct.
|
||||
fn test_storable<const LEN: usize, T: Storable + Copy + PartialEq>(
|
||||
// The primitive number function that
|
||||
// converts the number into little endian bytes,
|
||||
// e.g `u8::to_le_bytes`.
|
||||
to_le_bytes: fn(T) -> [u8; LEN],
|
||||
// A `Vec` of the numbers to test.
|
||||
t: Vec<T>,
|
||||
) {
|
||||
for t in t {
|
||||
let expected_bytes = to_le_bytes(t);
|
||||
|
||||
println!("testing: {t:?}, expected_bytes: {expected_bytes:?}");
|
||||
|
||||
// (De)serialize.
|
||||
let se: &[u8] = Storable::as_bytes(&t);
|
||||
let de: &T = Storable::from_bytes(se);
|
||||
|
||||
println!("serialized: {se:?}, deserialized: {de:?}\n");
|
||||
|
||||
// Assert we wrote correct amount of bytes.
|
||||
if let Some(len) = T::BYTE_LENGTH {
|
||||
assert_eq!(se.len(), expected_bytes.len());
|
||||
}
|
||||
// Assert the data is the same.
|
||||
assert_eq!(de, &t);
|
||||
}
|
||||
}
|
||||
|
||||
/// Create all the float tests.
|
||||
macro_rules! test_float {
|
||||
($(
|
||||
$float:ident // The float type.
|
||||
),* $(,)?) => {
|
||||
$(
|
||||
#[test]
|
||||
fn $float() {
|
||||
test_storable(
|
||||
$float::to_le_bytes,
|
||||
vec![
|
||||
-1.0,
|
||||
0.0,
|
||||
1.0,
|
||||
$float::MIN,
|
||||
$float::MAX,
|
||||
$float::INFINITY,
|
||||
$float::NEG_INFINITY,
|
||||
],
|
||||
);
|
||||
}
|
||||
)*
|
||||
};
|
||||
}
|
||||
|
||||
test_float! {
|
||||
f32,
|
||||
f64,
|
||||
}
|
||||
|
||||
/// Create all the (un)signed number tests.
|
||||
/// u8 -> u128, i8 -> i128.
|
||||
macro_rules! test_unsigned {
|
||||
($(
|
||||
$number:ident // The integer type.
|
||||
),* $(,)?) => {
|
||||
$(
|
||||
#[test]
|
||||
fn $number() {
|
||||
test_storable($number::to_le_bytes, vec![$number::MIN, 0, 1, $number::MAX]);
|
||||
}
|
||||
)*
|
||||
};
|
||||
}
|
||||
|
||||
test_unsigned! {
|
||||
u8,
|
||||
u16,
|
||||
u32,
|
||||
u64,
|
||||
u128,
|
||||
usize,
|
||||
i8,
|
||||
i16,
|
||||
i32,
|
||||
i64,
|
||||
i128,
|
||||
isize,
|
||||
}
|
||||
}
|
||||
|
|
@ -1,7 +1,7 @@
|
|||
//! Database table abstraction; `trait Table`.
|
||||
|
||||
//---------------------------------------------------------------------------------------------------- Import
|
||||
use crate::{key::Key, pod::Pod};
|
||||
use crate::{key::Key, storable::Storable};
|
||||
|
||||
//---------------------------------------------------------------------------------------------------- Table
|
||||
/// Database table metadata.
|
||||
|
|
@ -13,25 +13,28 @@ use crate::{key::Key, pod::Pod};
|
|||
///
|
||||
/// It is, and can only be implemented on the types inside [`tables`][crate::tables].
|
||||
pub trait Table: crate::tables::private::Sealed {
|
||||
// TODO:
|
||||
//
|
||||
// Add K/V comparison `type`s that define
|
||||
// how this table will be stored.
|
||||
//
|
||||
// type KeyComparator: fn(&Self::Key, &Self::Key) -> Ordering;
|
||||
// type ValueComparator: fn(&Self::Value, &Self::Value) -> Ordering;
|
||||
|
||||
/// Name of the database table.
|
||||
const NAME: &'static str;
|
||||
|
||||
/// Whether the table's values are all the same size or not.
|
||||
const CONSTANT_SIZE: bool;
|
||||
// TODO:
|
||||
//
|
||||
// `redb` requires `K/V` is `'static`:
|
||||
// - <https://docs.rs/redb/1.5.0/redb/struct.ReadOnlyTable.html>
|
||||
// - <https://docs.rs/redb/1.5.0/redb/struct.Table.html>
|
||||
//
|
||||
// ...but kinda not really?
|
||||
// "Note that the lifetime of the K and V type parameters does not impact
|
||||
// the lifetimes of the data that is stored or retrieved from the table"
|
||||
// <https://docs.rs/redb/1.5.0/redb/struct.TableDefinition.html>
|
||||
//
|
||||
// This might be incompatible with `heed`. We'll see
|
||||
// after function bodies are actually implemented...
|
||||
|
||||
/// Primary key type.
|
||||
type Key: Key;
|
||||
type Key: Key + 'static;
|
||||
|
||||
/// Value type.
|
||||
type Value: Pod;
|
||||
type Value: Storable + ?Sized + 'static;
|
||||
}
|
||||
|
||||
//---------------------------------------------------------------------------------------------------- Tests
|
||||
|
|
|
|||
|
|
@ -1,10 +1,12 @@
|
|||
//! Database tables.
|
||||
//!
|
||||
//! This module contains all the table definitions used by `cuprate-database`
|
||||
//! and [`Tables`], an `enum` containing all [`Table`]s.
|
||||
//! This module contains all the table definitions used by `cuprate-database`.
|
||||
|
||||
//---------------------------------------------------------------------------------------------------- Import
|
||||
use crate::table::Table;
|
||||
use crate::{
|
||||
table::Table,
|
||||
types::{TestType, TestType2},
|
||||
};
|
||||
|
||||
//---------------------------------------------------------------------------------------------------- Tables
|
||||
/// Private module, should not be accessible outside this crate.
|
||||
|
|
@ -18,50 +20,6 @@ pub(super) mod private {
|
|||
pub trait Sealed {}
|
||||
}
|
||||
|
||||
//---------------------------------------------------------------------------------------------------- Tables
|
||||
/// An enumeration of _all_ database tables.
|
||||
///
|
||||
/// TODO: I don't think we need this.
|
||||
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
|
||||
#[cfg_attr(
|
||||
feature = "borsh",
|
||||
derive(borsh::BorshSerialize, borsh::BorshDeserialize)
|
||||
)]
|
||||
#[derive(Copy, Clone, Debug, PartialEq, PartialOrd, Eq, Ord, Hash)]
|
||||
#[allow(missing_docs)]
|
||||
pub enum Tables {
|
||||
TestTable(TestTable),
|
||||
TestTable2(TestTable2),
|
||||
}
|
||||
|
||||
impl Tables {
|
||||
/// Get the [`Table::NAME`].
|
||||
pub const fn name(&self) -> &'static str {
|
||||
/// Hack to access associated trait constant via a variable.
|
||||
const fn get<T: Table>(t: &T) -> &'static str {
|
||||
T::NAME
|
||||
}
|
||||
|
||||
match self {
|
||||
Self::TestTable(t) => get(t),
|
||||
Self::TestTable2(t) => get(t),
|
||||
}
|
||||
}
|
||||
|
||||
/// Get the [`Table::CONSTANT_SIZE`].
|
||||
pub const fn constant_size(&self) -> bool {
|
||||
/// Hack to access associated trait constant via a variable.
|
||||
const fn get<T: Table>(t: &T) -> bool {
|
||||
T::CONSTANT_SIZE
|
||||
}
|
||||
|
||||
match self {
|
||||
Self::TestTable(t) => get(t),
|
||||
Self::TestTable2(t) => get(t),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
//---------------------------------------------------------------------------------------------------- Table macro
|
||||
/// Create all tables, should be used _once_.
|
||||
///
|
||||
|
|
@ -98,7 +56,6 @@ macro_rules! tables {
|
|||
)]
|
||||
/// ```
|
||||
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
|
||||
#[cfg_attr(feature = "borsh", derive(borsh::BorshSerialize, borsh::BorshDeserialize))]
|
||||
#[derive(Copy,Clone,Debug,PartialEq,PartialOrd,Eq,Ord,Hash)]
|
||||
pub struct [<$table:camel>];
|
||||
|
||||
|
|
@ -109,17 +66,9 @@ macro_rules! tables {
|
|||
// Table trait impl.
|
||||
impl Table for [<$table:camel>] {
|
||||
const NAME: &'static str = stringify!([<$table:snake>]);
|
||||
const CONSTANT_SIZE: bool = $size;
|
||||
type Key = $key;
|
||||
type Value = $value;
|
||||
}
|
||||
|
||||
// Table enum.
|
||||
impl From<[<$table:camel>]> for Tables {
|
||||
fn from(table: [<$table:camel>]) -> Self {
|
||||
Self::[<$table:camel>](table)
|
||||
}
|
||||
}
|
||||
)* }
|
||||
};
|
||||
}
|
||||
|
|
@ -129,12 +78,12 @@ tables! {
|
|||
/// Test documentation.
|
||||
TestTable,
|
||||
true,
|
||||
i64 => u64,
|
||||
i64 => TestType,
|
||||
|
||||
/// Test documentation 2.
|
||||
TestTable2,
|
||||
true,
|
||||
u8 => i8,
|
||||
u8 => TestType2,
|
||||
}
|
||||
|
||||
//---------------------------------------------------------------------------------------------------- Tests
|
||||
|
|
|
|||
128
database/src/types.rs
Normal file
128
database/src/types.rs
Normal file
|
|
@ -0,0 +1,128 @@
|
|||
//! Database [table](crate::tables) types.
|
||||
//!
|
||||
//! This module contains all types used by the database tables.
|
||||
//!
|
||||
//! TODO: Add schema here or a link to it.
|
||||
//!
|
||||
//! ## `*Bits`
|
||||
//! The non-documented items ending in `Bits` can be ignored,
|
||||
//! they are helper structs generated by `bytemuck`.
|
||||
|
||||
/*
|
||||
* <============================================> VERY BIG SCARY SAFETY MESSAGE <============================================>
|
||||
* DO NOT IGNORE --- DO NOT IGNORE --- DO NOT IGNORE --- DO NOT IGNORE --- DO NOT IGNORE --- DO NOT IGNORE --- DO NOT IGNORE
|
||||
* DO NOT IGNORE --- DO NOT IGNORE --- DO NOT IGNORE --- DO NOT IGNORE --- DO NOT IGNORE --- DO NOT IGNORE --- DO NOT IGNORE
|
||||
* DO NOT IGNORE --- DO NOT IGNORE --- DO NOT IGNORE --- DO NOT IGNORE --- DO NOT IGNORE --- DO NOT IGNORE --- DO NOT IGNORE
|
||||
* DO NOT IGNORE --- DO NOT IGNORE --- DO NOT IGNORE --- DO NOT IGNORE --- DO NOT IGNORE --- DO NOT IGNORE --- DO NOT IGNORE
|
||||
* DO NOT IGNORE --- DO NOT IGNORE --- DO NOT IGNORE --- DO NOT IGNORE --- DO NOT IGNORE --- DO NOT IGNORE --- DO NOT IGNORE
|
||||
* DO NOT IGNORE --- DO NOT IGNORE --- DO NOT IGNORE --- DO NOT IGNORE --- DO NOT IGNORE --- DO NOT IGNORE --- DO NOT IGNORE
|
||||
*
|
||||
*
|
||||
*
|
||||
* We use `bytemuck` to (de)serialize data types in the database.
|
||||
* We are SAFELY casting bytes, but to do so, we must uphold some invariants.
|
||||
* When editing this file, there is only 1 commandment that MUST be followed:
|
||||
*
|
||||
* 1. Thou shall only utilize `bytemuck`'s derive macros
|
||||
*
|
||||
* The derive macros will fail at COMPILE time if something is incorrect.
|
||||
* <https://docs.rs/bytemuck/latest/bytemuck/derive.Pod.html>
|
||||
* If you submit a PR that breaks this I will come and find you.
|
||||
*
|
||||
*
|
||||
*
|
||||
* DO NOT IGNORE --- DO NOT IGNORE --- DO NOT IGNORE --- DO NOT IGNORE --- DO NOT IGNORE --- DO NOT IGNORE --- DO NOT IGNORE
|
||||
* DO NOT IGNORE --- DO NOT IGNORE --- DO NOT IGNORE --- DO NOT IGNORE --- DO NOT IGNORE --- DO NOT IGNORE --- DO NOT IGNORE
|
||||
* DO NOT IGNORE --- DO NOT IGNORE --- DO NOT IGNORE --- DO NOT IGNORE --- DO NOT IGNORE --- DO NOT IGNORE --- DO NOT IGNORE
|
||||
* DO NOT IGNORE --- DO NOT IGNORE --- DO NOT IGNORE --- DO NOT IGNORE --- DO NOT IGNORE --- DO NOT IGNORE --- DO NOT IGNORE
|
||||
* DO NOT IGNORE --- DO NOT IGNORE --- DO NOT IGNORE --- DO NOT IGNORE --- DO NOT IGNORE --- DO NOT IGNORE --- DO NOT IGNORE
|
||||
* DO NOT IGNORE --- DO NOT IGNORE --- DO NOT IGNORE --- DO NOT IGNORE --- DO NOT IGNORE --- DO NOT IGNORE --- DO NOT IGNORE
|
||||
* <============================================> VERY BIG SCARY SAFETY MESSAGE <============================================>
|
||||
*/
|
||||
// actually i still don't trust you. no unsafe.
|
||||
#![forbid(unsafe_code)] // if you remove this line i will steal your monero
|
||||
#![allow(missing_docs)] // bytemuck auto-generates some non-documented structs
|
||||
|
||||
//---------------------------------------------------------------------------------------------------- Import
|
||||
use bytemuck::{AnyBitPattern, NoUninit, Pod, Zeroable};
|
||||
|
||||
#[cfg(feature = "serde")]
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
//---------------------------------------------------------------------------------------------------- TestType
|
||||
/// TEST
|
||||
///
|
||||
/// ```rust
|
||||
/// # use cuprate_database::types::*;
|
||||
/// let a = TestType { u: 1, b: 255, _pad: [0; 7] }; // original struct
|
||||
/// let b = bytemuck::must_cast::<TestType, [u8; 16]>(a); // cast into bytes
|
||||
/// let c = bytemuck::checked::cast::<[u8; 16], TestType>(b); // cast back into struct
|
||||
///
|
||||
/// assert_eq!(a, c);
|
||||
/// assert_eq!(c.u, 1);
|
||||
/// assert_eq!(c.b, 255);
|
||||
/// assert_eq!(c._pad, [0; 7]);
|
||||
/// ```
|
||||
///
|
||||
/// # Size & Alignment
|
||||
/// ```rust
|
||||
/// # use cuprate_database::types::*;
|
||||
/// # use std::mem::*;
|
||||
/// assert_eq!(size_of::<TestType>(), 16);
|
||||
/// assert_eq!(align_of::<TestType>(), 8);
|
||||
/// ```
|
||||
#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
|
||||
#[derive(Copy, Clone, Debug, PartialEq, PartialOrd, Eq, Ord, Hash, Pod, Zeroable)]
|
||||
#[repr(C)]
|
||||
pub struct TestType {
|
||||
/// TEST
|
||||
pub u: usize,
|
||||
/// TEST
|
||||
pub b: u8,
|
||||
/// TEST
|
||||
///
|
||||
/// TODO: is there a cheaper way (CPU instruction wise)
|
||||
/// to add padding to structs over 0 filled arrays?
|
||||
///
|
||||
/// TODO: this is basically leeway to
|
||||
/// add more things to our structs too,
|
||||
/// because otherwise this space is wasted.
|
||||
pub _pad: [u8; 7],
|
||||
}
|
||||
|
||||
//---------------------------------------------------------------------------------------------------- TestType2
|
||||
/// TEST2
|
||||
///
|
||||
/// ```rust
|
||||
/// # use cuprate_database::types::*;
|
||||
/// let a = TestType2 { u: 1, b: [1; 32] }; // original struct
|
||||
/// let b = bytemuck::must_cast::<TestType2, [u8; 40]>(a); // cast into bytes
|
||||
/// let c = bytemuck::must_cast::<[u8; 40], TestType2>(b); // cast back into struct
|
||||
///
|
||||
/// assert_eq!(a, c);
|
||||
/// assert_eq!(c.u, 1);
|
||||
/// assert_eq!(c.b, [1; 32]);
|
||||
/// ```
|
||||
///
|
||||
/// # Size & Alignment
|
||||
/// ```rust
|
||||
/// # use cuprate_database::types::*;
|
||||
/// # use std::mem::*;
|
||||
/// assert_eq!(size_of::<TestType2>(), 40);
|
||||
/// assert_eq!(align_of::<TestType2>(), 8);
|
||||
/// ```
|
||||
#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
|
||||
#[derive(Copy, Clone, Debug, PartialEq, PartialOrd, Eq, Ord, Hash, Pod, Zeroable)]
|
||||
#[repr(C)]
|
||||
pub struct TestType2 {
|
||||
/// TEST
|
||||
pub u: usize,
|
||||
/// TEST
|
||||
pub b: [u8; 32],
|
||||
}
|
||||
|
||||
//---------------------------------------------------------------------------------------------------- Tests
|
||||
#[cfg(test)]
|
||||
mod test {
|
||||
// use super::*;
|
||||
}
|
||||
Loading…
Reference in a new issue