Merge branch 'main' into test

Cargo.lock

@@ -725,7 +725,9 @@ dependencies = [
  "bytes",
  "bzip2",
  "futures",
+ "hex",
  "monero-p2p",
+ "monero-serai",
  "monero-wire",
  "reqwest",
  "tar",
@@ -735,6 +737,17 @@ dependencies = [
  "zip",
 ]
 
+[[package]]
+name = "cuprate-types"
+version = "0.0.0"
+dependencies = [
+ "borsh",
+ "cfg-if",
+ "curve25519-dalek",
+ "monero-serai",
+ "serde",
+]
+
 [[package]]
 name = "curve25519-dalek"
 version = "4.1.2"
@@ -1143,9 +1156,9 @@ dependencies = [
 
 [[package]]
 name = "h2"
-version = "0.3.24"
+version = "0.3.26"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bb2c4422095b67ee78da96fbb51a4cc413b3b25883c7717ff7ca1ab31022c9c9"
+checksum = "81fe527a889e1532da5c525686d96d4c2e74cdd345badf8dfef9f6b39dd5f5e8"
 dependencies = [
  "bytes",
  "fnv",

Cargo.toml

@@ -16,6 +16,7 @@ members = [
 	"p2p/address-book",
 	"pruning",
 	"test-utils",
+	"types",
 ]
 
 [profile.release]
@@ -41,6 +42,7 @@ anyhow                = { version = "1.0.81", default-features = false }
 async-trait           = { version = "0.1.74", default-features = false }
 bitflags              = { version = "2.4.2", default-features = false }
 borsh                 = { version = "1.2.1", default-features = false }
+bytemuck              = { version = "1.14.3", default-features = false }
 bytes                 = { version = "1.5.0", default-features = false }
 cfg-if                = { version = "1.0.0", default-features = false }
 clap                  = { version = "4.5.4", default-features = false }
@@ -92,7 +94,6 @@ proptest-derive       = { version = "0.4.0" }
 # open      = { version = "5.0.0" }  # Open PATH/URL, probably for binaries | https://github.com/byron/open-rs
 # regex     = { version = "1.10.2" } # Regular expressions                  | https://github.com/rust-lang/regex
 # ryu       = { version = "1.0.15" } # Fast float to string formatting      | https://github.com/dtolnay/ryu
-# strum     = { version = "0.25.0" } # Enum macros/traits                   | https://github.com/Peternator7/strum
 
 # Maybe one day.
 # disk     = { version = "*" } # (De)serialization to/from disk with various file formats | https://github.com/hinto-janai/disk

database/Cargo.toml

@@ -9,10 +9,12 @@ repository  = "https://github.com/Cuprate/cuprate/tree/main/database"
 keywords    = ["cuprate", "database"]
 
 [features]
-default   = ["heed", "redb", "service"]
+# default   = ["heed", "redb", "service"]
 # default   = ["redb", "service"]
+default   = ["redb-memory", "service"]
 heed       = ["dep:heed"]
 redb       = ["dep:redb"]
+redb-memory = ["redb"]
 service     = ["dep:crossbeam", "dep:futures", "dep:tokio", "dep:tokio-util", "dep:tower", "dep:rayon"]
 
 [dependencies]

database/README.md

@@ -12,6 +12,7 @@ Cuprate's database implementation.
 1. [Backends](#backends)
     - [`heed`](#heed)
     - [`redb`](#redb)
+    - [`redb-memory`](#redb-memory)
     - [`sanakirja`](#sanakirja)
     - [`MDBX`](#mdbx)
 1. [Layers](#layers)
@@ -171,6 +172,11 @@ The upstream versions from [`crates.io`](https://crates.io/crates/redb) are used
 
 TODO: document DB on remote filesystem (does redb allow this?)
 
+## `redb-memory`
+This backend is 100% the same as `redb`, although, it uses `redb::backend::InMemoryBackend` which is a key-value store that completely resides in memory instead of a file.
+
+All other details about this should be the same as the normal `redb` backend.
+
 ## `sanakirja`
 [`sanakirja`](https://docs.rs/sanakirja) was a candidate as a backend, however there were problems with maximum value sizes.
 

database/src/backend/heed/database.rs

@@ -3,6 +3,7 @@
 //---------------------------------------------------------------------------------------------------- Import
 use std::{
     borrow::{Borrow, Cow},
+    cell::RefCell,
     fmt::Debug,
     ops::RangeBounds,
     sync::RwLockReadGuard,
@@ -10,7 +11,7 @@ use std::{
 
 use crate::{
     backend::heed::{storable::StorableHeed, types::HeedDb},
-    database::{DatabaseRo, DatabaseRw},
+    database::{DatabaseIter, DatabaseRo, DatabaseRw},
     error::RuntimeError,
     table::Table,
 };
@@ -42,10 +43,10 @@ pub(super) struct HeedTableRo<'tx, T: Table> {
 ///
 /// Matches `redb::Table` (read & write).
 pub(super) struct HeedTableRw<'env, 'tx, T: Table> {
-    /// TODO
+    /// An already opened database table.
     pub(super) db: HeedDb<T::Key, T::Value>,
     /// The associated read/write transaction that opened this table.
-    pub(super) tx_rw: &'tx mut heed::RwTxn<'env>,
+    pub(super) tx_rw: &'tx RefCell<heed::RwTxn<'env>>,
 }
 
 //---------------------------------------------------------------------------------------------------- Shared functions
@@ -53,7 +54,7 @@ pub(super) struct HeedTableRw<'env, 'tx, T: Table> {
 // call the functions since the database is held by value, so
 // just use these generic functions that both can call instead.
 
-/// Shared generic `get()` between `HeedTableR{o,w}`.
+/// Shared [`DatabaseRo::get()`].
 #[inline]
 fn get<T: Table>(
     db: &HeedDb<T::Key, T::Value>,
@@ -63,17 +64,76 @@ fn get<T: Table>(
     db.get(tx_ro, key)?.ok_or(RuntimeError::KeyNotFound)
 }
 
-/// Shared generic `get_range()` between `HeedTableR{o,w}`.
+/// Shared [`DatabaseRo::len()`].
 #[inline]
-fn get_range<'a, T: Table, Range>(
+fn len<T: Table>(
-    db: &'a HeedDb<T::Key, T::Value>,
+    db: &HeedDb<T::Key, T::Value>,
-    tx_ro: &'a heed::RoTxn<'_>,
+    tx_ro: &heed::RoTxn<'_>,
+) -> Result<u64, RuntimeError> {
+    Ok(db.len(tx_ro)?)
+}
+
+/// Shared [`DatabaseRo::first()`].
+#[inline]
+fn first<T: Table>(
+    db: &HeedDb<T::Key, T::Value>,
+    tx_ro: &heed::RoTxn<'_>,
+) -> Result<(T::Key, T::Value), RuntimeError> {
+    db.first(tx_ro)?.ok_or(RuntimeError::KeyNotFound)
+}
+
+/// Shared [`DatabaseRo::last()`].
+#[inline]
+fn last<T: Table>(
+    db: &HeedDb<T::Key, T::Value>,
+    tx_ro: &heed::RoTxn<'_>,
+) -> Result<(T::Key, T::Value), RuntimeError> {
+    db.last(tx_ro)?.ok_or(RuntimeError::KeyNotFound)
+}
+
+/// Shared [`DatabaseRo::is_empty()`].
+#[inline]
+fn is_empty<T: Table>(
+    db: &HeedDb<T::Key, T::Value>,
+    tx_ro: &heed::RoTxn<'_>,
+) -> Result<bool, RuntimeError> {
+    Ok(db.is_empty(tx_ro)?)
+}
+
+//---------------------------------------------------------------------------------------------------- DatabaseIter Impl
+impl<T: Table> DatabaseIter<T> for HeedTableRo<'_, T> {
+    #[inline]
+    fn get_range<'a, Range>(
+        &'a self,
         range: Range,
-) -> Result<impl Iterator<Item = Result<T::Value, RuntimeError>> + 'a, RuntimeError>
+    ) -> Result<impl Iterator<Item = Result<T::Value, RuntimeError>> + 'a, RuntimeError>
-where
+    where
         Range: RangeBounds<T::Key> + 'a,
-{
+    {
-    Ok(db.range(tx_ro, &range)?.map(|res| Ok(res?.1)))
+        Ok(self.db.range(self.tx_ro, &range)?.map(|res| Ok(res?.1)))
+    }
+
+    #[inline]
+    fn iter(
+        &self,
+    ) -> Result<impl Iterator<Item = Result<(T::Key, T::Value), RuntimeError>> + '_, RuntimeError>
+    {
+        Ok(self.db.iter(self.tx_ro)?.map(|res| Ok(res?)))
+    }
+
+    #[inline]
+    fn keys(
+        &self,
+    ) -> Result<impl Iterator<Item = Result<T::Key, RuntimeError>> + '_, RuntimeError> {
+        Ok(self.db.iter(self.tx_ro)?.map(|res| Ok(res?.0)))
+    }
+
+    #[inline]
+    fn values(
+        &self,
+    ) -> Result<impl Iterator<Item = Result<T::Value, RuntimeError>> + '_, RuntimeError> {
+        Ok(self.db.iter(self.tx_ro)?.map(|res| Ok(res?.1)))
+    }
 }
 
 //---------------------------------------------------------------------------------------------------- DatabaseRo Impl
@@ -84,14 +144,23 @@ impl<T: Table> DatabaseRo<T> for HeedTableRo<'_, T> {
     }
 
     #[inline]
-    fn get_range<'a, Range>(
+    fn len(&self) -> Result<u64, RuntimeError> {
-        &'a self,
+        len::<T>(&self.db, self.tx_ro)
-        range: Range,
+    }
-    ) -> Result<impl Iterator<Item = Result<T::Value, RuntimeError>> + 'a, RuntimeError>
+
-    where
+    #[inline]
-        Range: RangeBounds<T::Key> + 'a,
+    fn first(&self) -> Result<(T::Key, T::Value), RuntimeError> {
-    {
+        first::<T>(&self.db, self.tx_ro)
-        get_range::<T, Range>(&self.db, self.tx_ro, range)
+    }
+
+    #[inline]
+    fn last(&self) -> Result<(T::Key, T::Value), RuntimeError> {
+        last::<T>(&self.db, self.tx_ro)
+    }
+
+    #[inline]
+    fn is_empty(&self) -> Result<bool, RuntimeError> {
+        is_empty::<T>(&self.db, self.tx_ro)
     }
 }
 
@@ -99,32 +168,92 @@ impl<T: Table> DatabaseRo<T> for HeedTableRo<'_, T> {
 impl<T: Table> DatabaseRo<T> for HeedTableRw<'_, '_, T> {
     #[inline]
     fn get(&self, key: &T::Key) -> Result<T::Value, RuntimeError> {
-        get::<T>(&self.db, self.tx_rw, key)
+        get::<T>(&self.db, &self.tx_rw.borrow(), key)
     }
 
     #[inline]
-    fn get_range<'a, Range>(
+    fn len(&self) -> Result<u64, RuntimeError> {
-        &'a self,
+        len::<T>(&self.db, &self.tx_rw.borrow())
-        range: Range,
+    }
-    ) -> Result<impl Iterator<Item = Result<T::Value, RuntimeError>> + 'a, RuntimeError>
+
-    where
+    #[inline]
-        Range: RangeBounds<T::Key> + 'a,
+    fn first(&self) -> Result<(T::Key, T::Value), RuntimeError> {
-    {
+        first::<T>(&self.db, &self.tx_rw.borrow())
-        get_range::<T, Range>(&self.db, self.tx_rw, range)
+    }
+
+    #[inline]
+    fn last(&self) -> Result<(T::Key, T::Value), RuntimeError> {
+        last::<T>(&self.db, &self.tx_rw.borrow())
+    }
+
+    #[inline]
+    fn is_empty(&self) -> Result<bool, RuntimeError> {
+        is_empty::<T>(&self.db, &self.tx_rw.borrow())
     }
 }
 
 impl<T: Table> DatabaseRw<T> for HeedTableRw<'_, '_, T> {
     #[inline]
     fn put(&mut self, key: &T::Key, value: &T::Value) -> Result<(), RuntimeError> {
-        Ok(self.db.put(self.tx_rw, key, value)?)
+        Ok(self.db.put(&mut self.tx_rw.borrow_mut(), key, value)?)
     }
 
     #[inline]
     fn delete(&mut self, key: &T::Key) -> Result<(), RuntimeError> {
-        self.db.delete(self.tx_rw, key)?;
+        self.db.delete(&mut self.tx_rw.borrow_mut(), key)?;
         Ok(())
     }
+
+    #[inline]
+    fn pop_first(&mut self) -> Result<(T::Key, T::Value), RuntimeError> {
+        let tx_rw = &mut self.tx_rw.borrow_mut();
+
+        // Get the first value first...
+        let Some(first) = self.db.first(tx_rw)? else {
+            return Err(RuntimeError::KeyNotFound);
+        };
+
+        // ...then remove it.
+        //
+        // We use an iterator because we want to semantically
+        // remove the _first_ and only the first `(key, value)`.
+        // `delete()` removes all keys including duplicates which
+        // is slightly different behavior.
+        let mut iter = self.db.iter_mut(tx_rw)?;
+
+        // SAFETY:
+        // It is undefined behavior to keep a reference of
+        // a value from this database while modifying it.
+        // We are deleting the value and never accessing
+        // the iterator again so this should be safe.
+        unsafe {
+            iter.del_current()?;
+        }
+
+        Ok(first)
+    }
+
+    #[inline]
+    fn pop_last(&mut self) -> Result<(T::Key, T::Value), RuntimeError> {
+        let tx_rw = &mut self.tx_rw.borrow_mut();
+
+        let Some(first) = self.db.last(tx_rw)? else {
+            return Err(RuntimeError::KeyNotFound);
+        };
+
+        let mut iter = self.db.rev_iter_mut(tx_rw)?;
+
+        // SAFETY:
+        // It is undefined behavior to keep a reference of
+        // a value from this database while modifying it.
+        // We are deleting the value and never accessing
+        // the iterator again so this should be safe.
+        unsafe {
+            iter.del_current()?;
+        }
+
+        Ok(first)
+    }
 }
 
 //---------------------------------------------------------------------------------------------------- Tests

database/src/backend/heed/env.rs

@@ -2,6 +2,7 @@
 
 //---------------------------------------------------------------------------------------------------- Import
 use std::{
+    cell::RefCell,
     fmt::Debug,
     ops::Deref,
     sync::{RwLock, RwLockReadGuard, RwLockWriteGuard},
@@ -16,7 +17,7 @@ use crate::{
         types::HeedDb,
     },
     config::{Config, SyncMode},
-    database::{DatabaseRo, DatabaseRw},
+    database::{DatabaseIter, DatabaseRo, DatabaseRw},
     env::{Env, EnvInner},
     error::{InitError, RuntimeError},
     resize::ResizeAlgorithm,
@@ -96,7 +97,23 @@ impl Env for ConcreteEnv {
     const SYNCS_PER_TX: bool = false;
     type EnvInner<'env> = RwLockReadGuard<'env, heed::Env>;
     type TxRo<'tx> = heed::RoTxn<'tx>;
-    type TxRw<'tx> = heed::RwTxn<'tx>;
+
+    /// HACK:
+    /// `heed::RwTxn` is wrapped in `RefCell` to allow:
+    /// - opening a database with only a `&` to it
+    /// - allowing 1 write tx to open multiple tables
+    ///
+    /// Our mutable accesses are safe and will not panic as:
+    /// - Write transactions are `!Sync`
+    /// - A table operation does not hold a reference to the inner cell
+    ///   once the call is over
+    /// - The function to manipulate the table takes the same type
+    ///   of reference that the `RefCell` gets for that function
+    ///
+    /// Also see:
+    /// - <https://github.com/Cuprate/cuprate/pull/102#discussion_r1548695610>
+    /// - <https://github.com/Cuprate/cuprate/pull/104>
+    type TxRw<'tx> = RefCell<heed::RwTxn<'tx>>;
 
     #[cold]
     #[inline(never)] // called once.
@@ -263,7 +280,8 @@ impl Env for ConcreteEnv {
 }
 
 //---------------------------------------------------------------------------------------------------- EnvInner Impl
-impl<'env> EnvInner<'env, heed::RoTxn<'env>, heed::RwTxn<'env>> for RwLockReadGuard<'env, heed::Env>
+impl<'env> EnvInner<'env, heed::RoTxn<'env>, RefCell<heed::RwTxn<'env>>>
+    for RwLockReadGuard<'env, heed::Env>
 where
     Self: 'env,
 {
@@ -273,15 +291,15 @@ where
     }
 
     #[inline]
-    fn tx_rw(&'env self) -> Result<heed::RwTxn<'env>, RuntimeError> {
+    fn tx_rw(&'env self) -> Result<RefCell<heed::RwTxn<'env>>, RuntimeError> {
-        Ok(self.write_txn()?)
+        Ok(RefCell::new(self.write_txn()?))
     }
 
     #[inline]
     fn open_db_ro<T: Table>(
         &self,
         tx_ro: &heed::RoTxn<'env>,
-    ) -> Result<impl DatabaseRo<T>, RuntimeError> {
+    ) -> Result<impl DatabaseRo<T> + DatabaseIter<T>, RuntimeError> {
         // Open up a read-only database using our table's const metadata.
         Ok(HeedTableRo {
             db: self
@@ -294,16 +312,34 @@ where
     #[inline]
     fn open_db_rw<T: Table>(
         &self,
-        tx_rw: &mut heed::RwTxn<'env>,
+        tx_rw: &RefCell<heed::RwTxn<'env>>,
     ) -> Result<impl DatabaseRw<T>, RuntimeError> {
+        let tx_ro = tx_rw.borrow();
+
         // Open up a read/write database using our table's const metadata.
         Ok(HeedTableRw {
             db: self
-                .open_database(tx_rw, Some(T::NAME))?
+                .open_database(&tx_ro, Some(T::NAME))?
                 .expect(PANIC_MSG_MISSING_TABLE),
             tx_rw,
         })
     }
+
+    #[inline]
+    fn clear_db<T: Table>(
+        &self,
+        tx_rw: &mut RefCell<heed::RwTxn<'env>>,
+    ) -> Result<(), RuntimeError> {
+        let tx_rw = tx_rw.get_mut();
+
+        // Open the table first...
+        let db: HeedDb<T::Key, T::Value> = self
+            .open_database(tx_rw, Some(T::NAME))?
+            .expect(PANIC_MSG_MISSING_TABLE);
+
+        // ...then clear it.
+        Ok(db.clear(tx_rw)?)
+    }
 }
 
 //---------------------------------------------------------------------------------------------------- Tests

database/src/backend/heed/transaction.rs

@@ -1,6 +1,6 @@
 //! Implementation of `trait TxRo/TxRw` for `heed`.
 
-use std::{ops::Deref, sync::RwLockReadGuard};
+use std::{cell::RefCell, ops::Deref, sync::RwLockReadGuard};
 
 //---------------------------------------------------------------------------------------------------- Import
 use crate::{
@@ -11,25 +11,25 @@ use crate::{
 //---------------------------------------------------------------------------------------------------- TxRo
 impl TxRo<'_> for heed::RoTxn<'_> {
     fn commit(self) -> Result<(), RuntimeError> {
-        Ok(self.commit()?)
+        Ok(heed::RoTxn::commit(self)?)
     }
 }
 
 //---------------------------------------------------------------------------------------------------- TxRw
-impl TxRo<'_> for heed::RwTxn<'_> {
+impl TxRo<'_> for RefCell<heed::RwTxn<'_>> {
     fn commit(self) -> Result<(), RuntimeError> {
-        Ok(self.commit()?)
+        TxRw::commit(self)
     }
 }
 
-impl TxRw<'_> for heed::RwTxn<'_> {
+impl TxRw<'_> for RefCell<heed::RwTxn<'_>> {
     fn commit(self) -> Result<(), RuntimeError> {
-        Ok(self.commit()?)
+        Ok(heed::RwTxn::commit(self.into_inner())?)
     }
 
     /// This function is infallible.
     fn abort(self) -> Result<(), RuntimeError> {
-        self.abort();
+        heed::RwTxn::abort(self.into_inner());
         Ok(())
     }
 }

database/src/backend/redb/database.rs

@@ -8,12 +8,14 @@ use std::{
     ops::{Bound, Deref, RangeBounds},
 };
 
+use redb::ReadableTable;
+
 use crate::{
     backend::redb::{
         storable::StorableRedb,
         types::{RedbTableRo, RedbTableRw},
     },
-    database::{DatabaseRo, DatabaseRw},
+    database::{DatabaseIter, DatabaseRo, DatabaseRw},
     error::RuntimeError,
     storable::Storable,
     table::Table,
@@ -24,7 +26,7 @@ use crate::{
 // call the functions since the database is held by value, so
 // just use these generic functions that both can call instead.
 
-/// Shared generic `get()` between `RedbTableR{o,w}`.
+/// Shared [`DatabaseRo::get()`].
 #[inline]
 fn get<T: Table + 'static>(
     db: &impl redb::ReadableTable<StorableRedb<T::Key>, StorableRedb<T::Value>>,
@@ -33,19 +35,86 @@ fn get<T: Table + 'static>(
     Ok(db.get(key)?.ok_or(RuntimeError::KeyNotFound)?.value())
 }
 
-/// Shared generic `get_range()` between `RedbTableR{o,w}`.
+/// Shared [`DatabaseRo::len()`].
 #[inline]
-fn get_range<'a, T: Table, Range>(
+fn len<T: Table>(
-    db: &'a impl redb::ReadableTable<StorableRedb<T::Key>, StorableRedb<T::Value>>,
+    db: &impl redb::ReadableTable<StorableRedb<T::Key>, StorableRedb<T::Value>>,
+) -> Result<u64, RuntimeError> {
+    Ok(db.len()?)
+}
+
+/// Shared [`DatabaseRo::first()`].
+#[inline]
+fn first<T: Table>(
+    db: &impl redb::ReadableTable<StorableRedb<T::Key>, StorableRedb<T::Value>>,
+) -> Result<(T::Key, T::Value), RuntimeError> {
+    let (key, value) = db.first()?.ok_or(RuntimeError::KeyNotFound)?;
+    Ok((key.value(), value.value()))
+}
+
+/// Shared [`DatabaseRo::last()`].
+#[inline]
+fn last<T: Table>(
+    db: &impl redb::ReadableTable<StorableRedb<T::Key>, StorableRedb<T::Value>>,
+) -> Result<(T::Key, T::Value), RuntimeError> {
+    let (key, value) = db.last()?.ok_or(RuntimeError::KeyNotFound)?;
+    Ok((key.value(), value.value()))
+}
+
+/// Shared [`DatabaseRo::is_empty()`].
+#[inline]
+fn is_empty<T: Table>(
+    db: &impl redb::ReadableTable<StorableRedb<T::Key>, StorableRedb<T::Value>>,
+) -> Result<bool, RuntimeError> {
+    Ok(db.is_empty()?)
+}
+
+//---------------------------------------------------------------------------------------------------- DatabaseIter
+impl<T: Table + 'static> DatabaseIter<T> for RedbTableRo<T::Key, T::Value> {
+    #[inline]
+    fn get_range<'a, Range>(
+        &'a self,
         range: Range,
-) -> Result<impl Iterator<Item = Result<T::Value, RuntimeError>> + 'a, RuntimeError>
+    ) -> Result<impl Iterator<Item = Result<T::Value, RuntimeError>> + 'a, RuntimeError>
-where
+    where
         Range: RangeBounds<T::Key> + 'a,
-{
+    {
-    Ok(db.range(range)?.map(|result| {
+        Ok(ReadableTable::range(self, range)?.map(|result| {
-        let (_key, value_guard) = result?;
+            let (_key, value) = result?;
-        Ok(value_guard.value())
+            Ok(value.value())
         }))
+    }
+
+    #[inline]
+    fn iter(
+        &self,
+    ) -> Result<impl Iterator<Item = Result<(T::Key, T::Value), RuntimeError>> + '_, RuntimeError>
+    {
+        Ok(ReadableTable::iter(self)?.map(|result| {
+            let (key, value) = result?;
+            Ok((key.value(), value.value()))
+        }))
+    }
+
+    #[inline]
+    fn keys(
+        &self,
+    ) -> Result<impl Iterator<Item = Result<T::Key, RuntimeError>> + '_, RuntimeError> {
+        Ok(ReadableTable::iter(self)?.map(|result| {
+            let (key, _value) = result?;
+            Ok(key.value())
+        }))
+    }
+
+    #[inline]
+    fn values(
+        &self,
+    ) -> Result<impl Iterator<Item = Result<T::Value, RuntimeError>> + '_, RuntimeError> {
+        Ok(ReadableTable::iter(self)?.map(|result| {
+            let (_key, value) = result?;
+            Ok(value.value())
+        }))
+    }
 }
 
 //---------------------------------------------------------------------------------------------------- DatabaseRo
@@ -56,14 +125,23 @@ impl<T: Table + 'static> DatabaseRo<T> for RedbTableRo<T::Key, T::Value> {
     }
 
     #[inline]
-    fn get_range<'a, Range>(
+    fn len(&self) -> Result<u64, RuntimeError> {
-        &'a self,
+        len::<T>(self)
-        range: Range,
+    }
-    ) -> Result<impl Iterator<Item = Result<T::Value, RuntimeError>> + 'a, RuntimeError>
+
-    where
+    #[inline]
-        Range: RangeBounds<T::Key> + 'a,
+    fn first(&self) -> Result<(T::Key, T::Value), RuntimeError> {
-    {
+        first::<T>(self)
-        get_range::<T, Range>(self, range)
+    }
+
+    #[inline]
+    fn last(&self) -> Result<(T::Key, T::Value), RuntimeError> {
+        last::<T>(self)
+    }
+
+    #[inline]
+    fn is_empty(&self) -> Result<bool, RuntimeError> {
+        is_empty::<T>(self)
     }
 }
 
@@ -75,32 +153,52 @@ impl<T: Table + 'static> DatabaseRo<T> for RedbTableRw<'_, T::Key, T::Value> {
     }
 
     #[inline]
-    fn get_range<'a, Range>(
+    fn len(&self) -> Result<u64, RuntimeError> {
-        &'a self,
+        len::<T>(self)
-        range: Range,
+    }
-    ) -> Result<impl Iterator<Item = Result<T::Value, RuntimeError>> + 'a, RuntimeError>
+
-    where
+    #[inline]
-        Range: RangeBounds<T::Key> + 'a,
+    fn first(&self) -> Result<(T::Key, T::Value), RuntimeError> {
-    {
+        first::<T>(self)
-        get_range::<T, Range>(self, range)
+    }
+
+    #[inline]
+    fn last(&self) -> Result<(T::Key, T::Value), RuntimeError> {
+        last::<T>(self)
+    }
+
+    #[inline]
+    fn is_empty(&self) -> Result<bool, RuntimeError> {
+        is_empty::<T>(self)
     }
 }
 
 impl<T: Table + 'static> DatabaseRw<T> for RedbTableRw<'_, T::Key, T::Value> {
-    // `redb` returns the value after `insert()/remove()`
+    // `redb` returns the value after function calls so we end with Ok(()) instead.
-    // we end with Ok(()) instead.
 
     #[inline]
     fn put(&mut self, key: &T::Key, value: &T::Value) -> Result<(), RuntimeError> {
-        self.insert(key, value)?;
+        redb::Table::insert(self, key, value)?;
         Ok(())
     }
 
     #[inline]
     fn delete(&mut self, key: &T::Key) -> Result<(), RuntimeError> {
-        self.remove(key)?;
+        redb::Table::remove(self, key)?;
         Ok(())
     }
+
+    #[inline]
+    fn pop_first(&mut self) -> Result<(T::Key, T::Value), RuntimeError> {
+        let (key, value) = redb::Table::pop_first(self)?.ok_or(RuntimeError::KeyNotFound)?;
+        Ok((key.value(), value.value()))
+    }
+
+    #[inline]
+    fn pop_last(&mut self) -> Result<(T::Key, T::Value), RuntimeError> {
+        let (key, value) = redb::Table::pop_last(self)?.ok_or(RuntimeError::KeyNotFound)?;
+        Ok((key.value(), value.value()))
+    }
 }
 
 //---------------------------------------------------------------------------------------------------- Tests

database/src/backend/redb/env.rs

@@ -9,7 +9,7 @@ use crate::{
         types::{RedbTableRo, RedbTableRw},
     },
     config::{Config, SyncMode},
-    database::{DatabaseRo, DatabaseRw},
+    database::{DatabaseIter, DatabaseRo, DatabaseRw},
     env::{Env, EnvInner},
     error::{InitError, RuntimeError},
     table::Table,
@@ -72,6 +72,10 @@ impl Env for ConcreteEnv {
         // TODO: we can set cache sizes with:
         // env_builder.set_cache(bytes);
 
+        // Use the in-memory backend if the feature is enabled.
+        let mut env = if cfg!(feature = "redb-memory") {
+            env_builder.create_with_backend(redb::backends::InMemoryBackend::new())?
+        } else {
             // Create the database directory if it doesn't exist.
             std::fs::create_dir_all(config.db_directory())?;
 
@@ -81,7 +85,9 @@ impl Env for ConcreteEnv {
                 .write(true)
                 .create(true)
                 .open(config.db_file())?;
-        let mut env = env_builder.create_file(db_file)?;
+
+            env_builder.create_file(db_file)?
+        };
 
         // Create all database tables.
         // `redb` creates tables if they don't exist.
@@ -180,7 +186,7 @@ where
     fn open_db_ro<T: Table>(
         &self,
         tx_ro: &redb::ReadTransaction,
-    ) -> Result<impl DatabaseRo<T>, RuntimeError> {
+    ) -> Result<impl DatabaseRo<T> + DatabaseIter<T>, RuntimeError> {
         // Open up a read-only database using our `T: Table`'s const metadata.
         let table: redb::TableDefinition<'static, StorableRedb<T::Key>, StorableRedb<T::Value>> =
             redb::TableDefinition::new(T::NAME);
@@ -192,7 +198,7 @@ where
     #[inline]
     fn open_db_rw<T: Table>(
         &self,
-        tx_rw: &mut redb::WriteTransaction,
+        tx_rw: &redb::WriteTransaction,
     ) -> Result<impl DatabaseRw<T>, RuntimeError> {
         // Open up a read/write database using our `T: Table`'s const metadata.
         let table: redb::TableDefinition<'static, StorableRedb<T::Key>, StorableRedb<T::Value>> =
@@ -202,6 +208,32 @@ where
         // <https://docs.rs/redb/latest/redb/struct.WriteTransaction.html#method.open_table>
         Ok(tx_rw.open_table(table)?)
     }
+
+    #[inline]
+    fn clear_db<T: Table>(&self, tx_rw: &mut redb::WriteTransaction) -> Result<(), RuntimeError> {
+        let table: redb::TableDefinition<
+            'static,
+            StorableRedb<<T as Table>::Key>,
+            StorableRedb<<T as Table>::Value>,
+        > = redb::TableDefinition::new(<T as Table>::NAME);
+
+        // INVARIANT:
+        // This `delete_table()` will not run into this `TableAlreadyOpen` error:
+        // <https://docs.rs/redb/2.0.0/src/redb/transactions.rs.html#382>
+        // which will panic in the `From` impl, as:
+        //
+        // 1. Only 1 `redb::WriteTransaction` can exist at a time
+        // 2. We have exclusive access to it
+        // 3. So it's not being used to open a table since that needs `&tx_rw`
+        //
+        // Reader-open tables do not affect this, if they're open the below is still OK.
+        redb::WriteTransaction::delete_table(tx_rw, table)?;
+        // Re-create the table.
+        // `redb` creates tables if they don't exist, so this should never panic.
+        redb::WriteTransaction::open_table(tx_rw, table)?;
+
+        Ok(())
+    }
 }
 
 //---------------------------------------------------------------------------------------------------- Tests

database/src/backend/tests.rs

@@ -13,14 +13,18 @@
 //!
 //! `redb`, and it only must be enabled for it to be tested.
 
-#![allow(clippy::items_after_statements, clippy::significant_drop_tightening)]
+#![allow(
+    clippy::items_after_statements,
+    clippy::significant_drop_tightening,
+    clippy::cast_possible_truncation
+)]
 
 //---------------------------------------------------------------------------------------------------- Import
 use std::borrow::{Borrow, Cow};
 
 use crate::{
     config::{Config, SyncMode},
-    database::{DatabaseRo, DatabaseRw},
+    database::{DatabaseIter, DatabaseRo, DatabaseRw},
     env::{Env, EnvInner},
     error::{InitError, RuntimeError},
     resize::ResizeAlgorithm,
@@ -77,7 +81,7 @@ fn open_db() {
     let (env, _tempdir) = tmp_concrete_env();
     let env_inner = env.env_inner();
     let tx_ro = env_inner.tx_ro().unwrap();
-    let mut tx_rw = env_inner.tx_rw().unwrap();
+    let tx_rw = env_inner.tx_rw().unwrap();
 
     // Open all tables in read-only mode.
     // This should be updated when tables are modified.
@@ -99,21 +103,21 @@ fn open_db() {
     TxRo::commit(tx_ro).unwrap();
 
     // Open all tables in read/write mode.
-    env_inner.open_db_rw::<BlockBlobs>(&mut tx_rw).unwrap();
+    env_inner.open_db_rw::<BlockBlobs>(&tx_rw).unwrap();
-    env_inner.open_db_rw::<BlockHeights>(&mut tx_rw).unwrap();
+    env_inner.open_db_rw::<BlockHeights>(&tx_rw).unwrap();
-    env_inner.open_db_rw::<BlockInfoV1s>(&mut tx_rw).unwrap();
+    env_inner.open_db_rw::<BlockInfoV1s>(&tx_rw).unwrap();
-    env_inner.open_db_rw::<BlockInfoV2s>(&mut tx_rw).unwrap();
+    env_inner.open_db_rw::<BlockInfoV2s>(&tx_rw).unwrap();
-    env_inner.open_db_rw::<BlockInfoV3s>(&mut tx_rw).unwrap();
+    env_inner.open_db_rw::<BlockInfoV3s>(&tx_rw).unwrap();
-    env_inner.open_db_rw::<KeyImages>(&mut tx_rw).unwrap();
+    env_inner.open_db_rw::<KeyImages>(&tx_rw).unwrap();
-    env_inner.open_db_rw::<NumOutputs>(&mut tx_rw).unwrap();
+    env_inner.open_db_rw::<NumOutputs>(&tx_rw).unwrap();
-    env_inner.open_db_rw::<Outputs>(&mut tx_rw).unwrap();
+    env_inner.open_db_rw::<Outputs>(&tx_rw).unwrap();
-    env_inner.open_db_rw::<PrunableHashes>(&mut tx_rw).unwrap();
+    env_inner.open_db_rw::<PrunableHashes>(&tx_rw).unwrap();
-    env_inner.open_db_rw::<PrunableTxBlobs>(&mut tx_rw).unwrap();
+    env_inner.open_db_rw::<PrunableTxBlobs>(&tx_rw).unwrap();
-    env_inner.open_db_rw::<PrunedTxBlobs>(&mut tx_rw).unwrap();
+    env_inner.open_db_rw::<PrunedTxBlobs>(&tx_rw).unwrap();
-    env_inner.open_db_rw::<RctOutputs>(&mut tx_rw).unwrap();
+    env_inner.open_db_rw::<RctOutputs>(&tx_rw).unwrap();
-    env_inner.open_db_rw::<TxHeights>(&mut tx_rw).unwrap();
+    env_inner.open_db_rw::<TxHeights>(&tx_rw).unwrap();
-    env_inner.open_db_rw::<TxIds>(&mut tx_rw).unwrap();
+    env_inner.open_db_rw::<TxIds>(&tx_rw).unwrap();
-    env_inner.open_db_rw::<TxUnlockTime>(&mut tx_rw).unwrap();
+    env_inner.open_db_rw::<TxUnlockTime>(&tx_rw).unwrap();
     TxRw::commit(tx_rw).unwrap();
 }
 
@@ -162,11 +166,12 @@ fn non_manual_resize_2() {
 
 /// Test all `DatabaseR{o,w}` operations.
 #[test]
+#[allow(clippy::too_many_lines)]
 fn db_read_write() {
     let (env, _tempdir) = tmp_concrete_env();
     let env_inner = env.env_inner();
-    let mut tx_rw = env_inner.tx_rw().unwrap();
+    let tx_rw = env_inner.tx_rw().unwrap();
-    let mut table = env_inner.open_db_rw::<Outputs>(&mut tx_rw).unwrap();
+    let mut table = env_inner.open_db_rw::<Outputs>(&tx_rw).unwrap();
 
     /// The (1st) key.
     const KEY: PreRctOutputId = PreRctOutputId {
@@ -180,6 +185,8 @@ fn db_read_write() {
         output_flags: 0,
         tx_idx: 2_353_487,
     };
+    /// How many `(key, value)` pairs will be inserted.
+    const N: u64 = 100;
 
     /// Assert 2 `Output`'s are equal, and that accessing
     /// their fields don't result in an unaligned panic.
@@ -191,22 +198,41 @@ fn db_read_write() {
         assert_eq!(output.tx_idx, VALUE.tx_idx);
     }
 
-    // Insert `0..100` keys.
+    assert!(table.is_empty().unwrap());
+
+    // Insert keys.
     let mut key = KEY;
-    for i in 0..100 {
+    for i in 0..N {
         table.put(&key, &VALUE).unwrap();
         key.amount += 1;
     }
 
-    // Assert the 1st key is there.
+    assert_eq!(table.len().unwrap(), N);
+
+    // Assert the first/last `(key, value)`s are there.
     {
-        let value: Output = table.get(&KEY).unwrap();
+        assert!(table.contains(&KEY).unwrap());
-        assert_same(value);
+        let get: Output = table.get(&KEY).unwrap();
+        assert_same(get);
+
+        let first: Output = table.first().unwrap().1;
+        assert_same(first);
+
+        let last: Output = table.last().unwrap().1;
+        assert_same(last);
     }
 
+    // Commit transactions, create new ones.
+    drop(table);
+    TxRw::commit(tx_rw).unwrap();
+    let tx_ro = env_inner.tx_ro().unwrap();
+    let table_ro = env_inner.open_db_ro::<Outputs>(&tx_ro).unwrap();
+    let tx_rw = env_inner.tx_rw().unwrap();
+    let mut table = env_inner.open_db_rw::<Outputs>(&tx_rw).unwrap();
+
     // Assert the whole range is there.
     {
-        let range = table.get_range(..).unwrap();
+        let range = table_ro.get_range(..).unwrap();
         let mut i = 0;
         for result in range {
             let value: Output = result.unwrap();
@@ -214,20 +240,23 @@ fn db_read_write() {
 
             i += 1;
         }
-        assert_eq!(i, 100);
+        assert_eq!(i, N);
     }
 
     // `get_range()` tests.
     let mut key = KEY;
-    key.amount += 100;
+    key.amount += N;
     let range = KEY..key;
 
     // Assert count is correct.
-    assert_eq!(100, table.get_range(range.clone()).unwrap().count());
+    assert_eq!(
+        N as usize,
+        table_ro.get_range(range.clone()).unwrap().count()
+    );
 
     // Assert each returned value from the iterator is owned.
     {
-        let mut iter = table.get_range(range.clone()).unwrap();
+        let mut iter = table_ro.get_range(range.clone()).unwrap();
         let value: Output = iter.next().unwrap().unwrap(); // 1. take value out
         drop(iter); // 2. drop the `impl Iterator + 'a`
         assert_same(value); // 3. assert even without the iterator, the value is alive
@@ -235,17 +264,49 @@ fn db_read_write() {
 
     // Assert each value is the same.
     {
-        let mut iter = table.get_range(range).unwrap();
+        let mut iter = table_ro.get_range(range).unwrap();
-        for _ in 0..100 {
+        for _ in 0..N {
             let value: Output = iter.next().unwrap().unwrap();
             assert_same(value);
         }
     }
 
     // Assert deleting works.
+    {
         table.delete(&KEY).unwrap();
         let value = table.get(&KEY);
+        assert!(!table.contains(&KEY).unwrap());
         assert!(matches!(value, Err(RuntimeError::KeyNotFound)));
+        // Assert the other `(key, value)` pairs are still there.
+        let mut key = KEY;
+        key.amount += N - 1; // we used inclusive `0..N`
+        let value = table.get(&key).unwrap();
+        assert_same(value);
+    }
+
+    drop(table);
+    TxRw::commit(tx_rw).unwrap();
+
+    // Assert `clear_db()` works.
+    {
+        let mut tx_rw = env_inner.tx_rw().unwrap();
+        env_inner.clear_db::<Outputs>(&mut tx_rw).unwrap();
+        let table = env_inner.open_db_rw::<Outputs>(&tx_rw).unwrap();
+        assert!(table.is_empty().unwrap());
+        for n in 0..N {
+            let mut key = KEY;
+            key.amount += n;
+            let value = table.get(&key);
+            assert!(matches!(value, Err(RuntimeError::KeyNotFound)));
+            assert!(!table.contains(&key).unwrap());
+        }
+
+        // Reader still sees old value.
+        assert!(!table_ro.is_empty().unwrap());
+
+        // Writer sees updated value (nothing).
+        assert!(table.is_empty().unwrap());
+    }
 }
 
 //---------------------------------------------------------------------------------------------------- Table Tests
@@ -253,11 +314,7 @@ fn db_read_write() {
 ///
 /// Each one of these tests:
 /// - Opens a specific table
-/// - Inserts a key + value
+/// - Essentially does the `db_read_write` test
-/// - Retrieves the key + value
-/// - Asserts it is the same
-/// - Tests `get_range()`
-/// - Tests `delete()`
 macro_rules! test_tables {
     ($(
         $table:ident,    // Table type
@@ -293,19 +350,47 @@ macro_rules! test_tables {
                 assert_eq(&value);
             }
 
+            assert!(table.contains(&KEY).unwrap());
+            assert_eq!(table.len().unwrap(), 1);
+
+            // Commit transactions, create new ones.
+            drop(table);
+            TxRw::commit(tx_rw).unwrap();
+            let mut tx_rw = env_inner.tx_rw().unwrap();
+            let tx_ro = env_inner.tx_ro().unwrap();
+            let mut table = env_inner.open_db_rw::<$table>(&tx_rw).unwrap();
+            let table_ro = env_inner.open_db_ro::<$table>(&tx_ro).unwrap();
+
             // Assert `get_range()` works.
             {
                 let range = KEY..;
-                assert_eq!(1, table.get_range(range.clone()).unwrap().count());
+                assert_eq!(1, table_ro.get_range(range.clone()).unwrap().count());
-                let mut iter = table.get_range(range).unwrap();
+                let mut iter = table_ro.get_range(range).unwrap();
                 let value = iter.next().unwrap().unwrap();
                 assert_eq(&value);
             }
 
             // Assert deleting works.
+            {
                 table.delete(&KEY).unwrap();
                 let value = table.get(&KEY);
                 assert!(matches!(value, Err(RuntimeError::KeyNotFound)));
+                assert!(!table.contains(&KEY).unwrap());
+                assert_eq!(table.len().unwrap(), 0);
+            }
+
+            table.put(&KEY, &value).unwrap();
+
+            // Assert `clear_db()` works.
+            {
+                drop(table);
+                env_inner.clear_db::<$table>(&mut tx_rw).unwrap();
+                let table = env_inner.open_db_rw::<$table>(&mut tx_rw).unwrap();
+                let value = table.get(&KEY);
+                assert!(matches!(value, Err(RuntimeError::KeyNotFound)));
+                assert!(!table.contains(&KEY).unwrap());
+                assert_eq!(table.len().unwrap(), 0);
+            }
         }
     )*}};
 }

database/src/database.rs

@@ -13,6 +13,70 @@ use crate::{
     transaction::{TxRo, TxRw},
 };
 
+//---------------------------------------------------------------------------------------------------- DatabaseRoIter
+/// Database (key-value store) read-only iteration abstraction.
+///
+/// These are read-only iteration-related operations that
+/// can only be called from [`DatabaseRo`] objects.
+///
+/// # Hack
+/// This is a HACK to get around the fact our read/write tables
+/// cannot safely return values returning lifetimes, as such,
+/// only read-only tables implement this trait.
+///
+/// - <https://github.com/Cuprate/cuprate/pull/102#discussion_r1548695610>
+/// - <https://github.com/Cuprate/cuprate/pull/104>
+pub trait DatabaseIter<T: Table> {
+    /// Get an iterator of value's corresponding to a range of keys.
+    ///
+    /// For example:
+    /// ```rust,ignore
+    /// // This will return all 100 values corresponding
+    /// // to the keys `{0, 1, 2, ..., 100}`.
+    /// self.get_range(0..100);
+    /// ```
+    ///
+    /// Although the returned iterator itself is tied to the lifetime
+    /// of `&'a self`, the returned values from the iterator are _owned_.
+    ///
+    /// # Errors
+    /// Each key in the `range` has the potential to error, for example,
+    /// if a particular key in the `range` does not exist,
+    /// [`RuntimeError::KeyNotFound`] wrapped in [`Err`] will be returned
+    /// from the iterator.
+    #[allow(clippy::iter_not_returning_iterator)]
+    fn get_range<'a, Range>(
+        &'a self,
+        range: Range,
+    ) -> Result<impl Iterator<Item = Result<T::Value, RuntimeError>> + 'a, RuntimeError>
+    where
+        Range: RangeBounds<T::Key> + 'a;
+
+    /// TODO
+    ///
+    /// # Errors
+    /// TODO
+    #[allow(clippy::iter_not_returning_iterator)]
+    fn iter(
+        &self,
+    ) -> Result<impl Iterator<Item = Result<(T::Key, T::Value), RuntimeError>> + '_, RuntimeError>;
+
+    /// TODO
+    ///
+    /// # Errors
+    /// TODO
+    fn keys(&self)
+        -> Result<impl Iterator<Item = Result<T::Key, RuntimeError>> + '_, RuntimeError>;
+
+    /// TODO
+    ///
+    /// # Errors
+    /// TODO
+    fn values(
+        &self,
+    ) -> Result<impl Iterator<Item = Result<T::Value, RuntimeError>> + '_, RuntimeError>;
+}
+
 //---------------------------------------------------------------------------------------------------- DatabaseRo
 /// Database (key-value store) read abstraction.
 ///
@@ -29,22 +93,41 @@ pub trait DatabaseRo<T: Table> {
     /// It will return other [`RuntimeError`]'s on things like IO errors as well.
     fn get(&self, key: &T::Key) -> Result<T::Value, RuntimeError>;
 
-    /// Get an iterator of values corresponding to a range of keys.
+    /// TODO
-    ///
-    /// Although the returned iterator itself is tied to the lifetime
-    /// of `&'a self`, the returned values from the iterator are _owned_.
     ///
     /// # Errors
-    /// Each key in the `range` has the potential to error, for example,
+    /// TODO
-    /// if a particular key in the `range` does not exist,
+    fn contains(&self, key: &T::Key) -> Result<bool, RuntimeError> {
-    /// [`RuntimeError::KeyNotFound`] wrapped in [`Err`] will be returned
+        match self.get(key) {
-    /// from the iterator.
+            Ok(_) => Ok(true),
-    fn get_range<'a, Range>(
+            Err(RuntimeError::KeyNotFound) => Ok(false),
-        &'a self,
+            Err(e) => Err(e),
-        range: Range,
+        }
-    ) -> Result<impl Iterator<Item = Result<T::Value, RuntimeError>> + 'a, RuntimeError>
+    }
-    where
+
-        Range: RangeBounds<T::Key> + 'a;
+    /// TODO
+    ///
+    /// # Errors
+    /// TODO
+    fn len(&self) -> Result<u64, RuntimeError>;
+
+    /// TODO
+    ///
+    /// # Errors
+    /// TODO
+    fn first(&self) -> Result<(T::Key, T::Value), RuntimeError>;
+
+    /// TODO
+    ///
+    /// # Errors
+    /// TODO
+    fn last(&self) -> Result<(T::Key, T::Value), RuntimeError>;
+
+    /// TODO
+    ///
+    /// # Errors
+    /// TODO
+    fn is_empty(&self) -> Result<bool, RuntimeError>;
 }
 
 //---------------------------------------------------------------------------------------------------- DatabaseRw
@@ -65,4 +148,16 @@ pub trait DatabaseRw<T: Table>: DatabaseRo<T> {
     /// # Errors
     /// This will return [`RuntimeError::KeyNotFound`] wrapped in [`Err`] if `key` does not exist.
     fn delete(&mut self, key: &T::Key) -> Result<(), RuntimeError>;
+
+    /// TODO
+    ///
+    /// # Errors
+    /// TODO
+    fn pop_first(&mut self) -> Result<(T::Key, T::Value), RuntimeError>;
+
+    /// TODO
+    ///
+    /// # Errors
+    /// TODO
+    fn pop_last(&mut self) -> Result<(T::Key, T::Value), RuntimeError>;
 }

database/src/env.rs

@@ -5,7 +5,7 @@ use std::{fmt::Debug, ops::Deref};
 
 use crate::{
     config::Config,
-    database::{DatabaseRo, DatabaseRw},
+    database::{DatabaseIter, DatabaseRo, DatabaseRw},
     error::{InitError, RuntimeError},
     resize::ResizeAlgorithm,
     table::Table,
@@ -196,10 +196,15 @@ where
     /// ```
     ///
     /// # Errors
+    /// This function errors upon internal database/IO errors.
+    ///
     /// As [`Table`] is `Sealed`, and all tables are created
     /// upon [`Env::open`], this function will never error because
     /// a table doesn't exist.
-    fn open_db_ro<T: Table>(&self, tx_ro: &Ro) -> Result<impl DatabaseRo<T>, RuntimeError>;
+    fn open_db_ro<T: Table>(
+        &self,
+        tx_ro: &Ro,
+    ) -> Result<impl DatabaseRo<T> + DatabaseIter<T>, RuntimeError>;
 
     /// Open a database in read/write mode.
     ///
@@ -210,8 +215,26 @@ where
     /// passed as a generic to this function.
     ///
     /// # Errors
+    /// This function errors upon internal database/IO errors.
+    ///
     /// As [`Table`] is `Sealed`, and all tables are created
     /// upon [`Env::open`], this function will never error because
     /// a table doesn't exist.
-    fn open_db_rw<T: Table>(&self, tx_rw: &mut Rw) -> Result<impl DatabaseRw<T>, RuntimeError>;
+    fn open_db_rw<T: Table>(&self, tx_rw: &Rw) -> Result<impl DatabaseRw<T>, RuntimeError>;
+
+    /// Clear all `(key, value)`'s from a database table.
+    ///
+    /// This will delete all key and values in the passed
+    /// `T: Table`, but the table itself will continue to exist.
+    ///
+    /// Note that this operation is tied to `tx_rw`, as such this
+    /// function's effects can be aborted using [`TxRw::abort`].
+    ///
+    /// # Errors
+    /// This function errors upon internal database/IO errors.
+    ///
+    /// As [`Table`] is `Sealed`, and all tables are created
+    /// upon [`Env::open`], this function will never error because
+    /// a table doesn't exist.
+    fn clear_db<T: Table>(&self, tx_rw: &mut Rw) -> Result<(), RuntimeError>;
 }

database/src/lib.rs

@@ -222,7 +222,7 @@ pub use constants::{
 };
 
 mod database;
-pub use database::{DatabaseRo, DatabaseRw};
+pub use database::{DatabaseIter, DatabaseRo, DatabaseRw};
 
 mod env;
 pub use env::{Env, EnvInner};

test-utils/Cargo.toml

@@ -9,6 +9,7 @@ authors = ["Boog900"]
 monero-wire = {path = "../net/monero-wire"}
 monero-p2p = {path = "../p2p/monero-p2p", features = ["borsh"] }
 
+monero-serai = { workspace = true }
 futures = { workspace = true, features = ["std"] }
 async-trait = { workspace = true }
 tokio = { workspace = true, features = ["full"] }
@@ -25,3 +26,6 @@ bzip2 = "0.4.4"
 
 [target.'cfg(windows)'.dependencies]
 zip = "0.6"
+
+[dev-dependencies]
+hex = { workspace = true }

test-utils/README.MD

@@ -3,5 +3,6 @@
 This crate contains code that can be shared across multiple Cuprate crates tests, this crate should not be included in any
 Cuprate crate, only in tests.
 
-It currently contains code to spawn monerod instances and a testing network zone.
+It currently contains:
-
+- Code to spawn monerod instances and a testing network zone
+- Real raw and typed Monero data, e.g. `Block, Transaction`

test-utils/src/data/README.md

@@ -0,0 +1,14 @@
+# Data
+This module contains:
+- Raw binary, hex, or JSON data for testing purposes
+- Functions to access that data, either raw or typed
+
+- `.bin` is a data blob, directly deserializable into types, e.g. `monero_serai::block::Block::read::<&[u8]>(&mut blob)`
+- `.hex` is just a hex string of the blob
+- `.json` is just the data in regular JSON form (as it would be from a JSON-RPC response)
+
+# Actual data
+| Directory | File naming scheme           | Example |
+|-----------|------------------------------|---------|
+| `block/`  | `$block_hash.{bin,hex,json}` | `bbd604d2ba11ba27935e006ed39c9bfdd99b76bf4a50654bc1e1e61217962698.bin`
+| `tx/`     | `$tx_hash.{bin,hex,json}`    | `84d48dc11ec91950f8b70a85af9db91fe0c8abef71ef5db08304f7344b99ea66.bin`

test-utils/src/data/constants.rs

@@ -0,0 +1,150 @@
+//! Constants holding raw Monero data.
+
+//---------------------------------------------------------------------------------------------------- Import
+
+//---------------------------------------------------------------------------------------------------- Block
+/// Block with height `202612` and hash `bbd604d2ba11ba27935e006ed39c9bfdd99b76bf4a50654bc1e1e61217962698`.
+///
+/// ```rust
+/// use monero_serai::{block::Block, transaction::Input};
+///
+/// let block = Block::read(&mut
+///     cuprate_test_utils::data::BLOCK_BBD604
+/// ).unwrap();
+///
+/// assert_eq!(block.header.major_version, 1);
+/// assert_eq!(block.header.minor_version, 0);
+/// assert_eq!(block.header.timestamp, 1409804570);
+/// assert_eq!(block.header.nonce, 1073744198);
+/// assert!(matches!(block.miner_tx.prefix.inputs[0], Input::Gen(202612)));
+/// assert_eq!(block.txs.len(), 513);
+///
+/// assert_eq!(
+///     hex::encode(block.hash()),
+///     "bbd604d2ba11ba27935e006ed39c9bfdd99b76bf4a50654bc1e1e61217962698",
+/// );
+/// ```
+pub const BLOCK_BBD604: &[u8] =
+    include_bytes!("block/bbd604d2ba11ba27935e006ed39c9bfdd99b76bf4a50654bc1e1e61217962698.bin");
+
+/// Block with height `2751506` and hash `f910435a5477ca27be1986c080d5476aeab52d0c07cf3d9c72513213350d25d4`.
+///
+/// ```rust
+/// use monero_serai::{block::Block, transaction::Input};
+///
+/// let block = Block::read(&mut
+///     cuprate_test_utils::data::BLOCK_F91043
+/// ).unwrap();
+///
+/// assert_eq!(block.header.major_version, 9);
+/// assert_eq!(block.header.minor_version, 9);
+/// assert_eq!(block.header.timestamp, 1545423190);
+/// assert_eq!(block.header.nonce, 4123173351);
+/// assert!(matches!(block.miner_tx.prefix.inputs[0], Input::Gen(1731606)));
+/// assert_eq!(block.txs.len(), 3);
+///
+/// assert_eq!(
+///     hex::encode(block.hash()),
+///     "f910435a5477ca27be1986c080d5476aeab52d0c07cf3d9c72513213350d25d4",
+/// );
+/// ```
+pub const BLOCK_F91043: &[u8] =
+    include_bytes!("block/f910435a5477ca27be1986c080d5476aeab52d0c07cf3d9c72513213350d25d4.bin");
+
+/// Block with height `2751506` and hash `43bd1f2b6556dcafa413d8372974af59e4e8f37dbf74dc6b2a9b7212d0577428`.
+///
+/// ```rust
+/// use monero_serai::{block::Block, transaction::Input};
+///
+/// let block = Block::read(&mut
+///     cuprate_test_utils::data::BLOCK_43BD1F
+/// ).unwrap();
+///
+/// assert_eq!(block.header.major_version, 16);
+/// assert_eq!(block.header.minor_version, 16);
+/// assert_eq!(block.header.timestamp, 1667941829);
+/// assert_eq!(block.header.nonce, 4110909056);
+/// assert!(matches!(block.miner_tx.prefix.inputs[0], Input::Gen(2751506)));
+/// assert_eq!(block.txs.len(), 0);
+///
+/// assert_eq!(
+///     hex::encode(block.hash()),
+///     "43bd1f2b6556dcafa413d8372974af59e4e8f37dbf74dc6b2a9b7212d0577428",
+/// );
+/// ```
+pub const BLOCK_43BD1F: &[u8] =
+    include_bytes!("block/43bd1f2b6556dcafa413d8372974af59e4e8f37dbf74dc6b2a9b7212d0577428.bin");
+
+//---------------------------------------------------------------------------------------------------- Transaction
+/// Transaction with hash `3bc7ff015b227e7313cc2e8668bfbb3f3acbee274a9c201d6211cf681b5f6bb1`.
+///
+/// ```rust
+/// use monero_serai::transaction::{Transaction, Timelock};
+///
+/// let tx = Transaction::read(&mut
+///     cuprate_test_utils::data::TX_3BC7FF
+/// ).unwrap();
+///
+/// assert_eq!(tx.prefix.version, 1);
+/// assert_eq!(tx.prefix.timelock, Timelock::Block(100_081));
+/// assert_eq!(tx.prefix.inputs.len(), 1);
+/// assert_eq!(tx.prefix.outputs.len(), 5);
+/// assert_eq!(tx.signatures.len(), 0);
+///
+/// assert_eq!(
+///     hex::encode(tx.hash()),
+///     "3bc7ff015b227e7313cc2e8668bfbb3f3acbee274a9c201d6211cf681b5f6bb1",
+/// );
+/// ```
+pub const TX_3BC7FF: &[u8] =
+    include_bytes!("tx/3bc7ff015b227e7313cc2e8668bfbb3f3acbee274a9c201d6211cf681b5f6bb1.bin");
+
+/// Transaction with hash `9e3f73e66d7c7293af59c59c1ff5d6aae047289f49e5884c66caaf4aea49fb34`.
+///
+/// ```rust
+/// use monero_serai::transaction::{Transaction, Timelock};
+///
+/// let tx = Transaction::read(&mut
+///     cuprate_test_utils::data::TX_9E3F73
+/// ).unwrap();
+///
+/// assert_eq!(tx.prefix.version, 1);
+/// assert_eq!(tx.prefix.timelock, Timelock::None);
+/// assert_eq!(tx.prefix.inputs.len(), 2);
+/// assert_eq!(tx.prefix.outputs.len(), 5);
+/// assert_eq!(tx.signatures.len(), 2);
+///
+/// assert_eq!(
+///     hex::encode(tx.hash()),
+///     "9e3f73e66d7c7293af59c59c1ff5d6aae047289f49e5884c66caaf4aea49fb34",
+/// );
+/// ```
+pub const TX_9E3F73: &[u8] =
+    include_bytes!("tx/9e3f73e66d7c7293af59c59c1ff5d6aae047289f49e5884c66caaf4aea49fb34.bin");
+
+/// Transaction with hash `84d48dc11ec91950f8b70a85af9db91fe0c8abef71ef5db08304f7344b99ea66`.
+///
+/// ```rust
+/// use monero_serai::transaction::{Transaction, Timelock};
+///
+/// let tx = Transaction::read(&mut
+///     cuprate_test_utils::data::TX_84D48D
+/// ).unwrap();
+///
+/// assert_eq!(tx.prefix.version, 2);
+/// assert_eq!(tx.prefix.timelock, Timelock::None);
+/// assert_eq!(tx.prefix.inputs.len(), 2);
+/// assert_eq!(tx.prefix.outputs.len(), 2);
+/// assert_eq!(tx.signatures.len(), 0);
+///
+/// assert_eq!(
+///     hex::encode(tx.hash()),
+///     "84d48dc11ec91950f8b70a85af9db91fe0c8abef71ef5db08304f7344b99ea66",
+/// );
+/// ```
+pub const TX_84D48D: &[u8] =
+    include_bytes!("tx/84d48dc11ec91950f8b70a85af9db91fe0c8abef71ef5db08304f7344b99ea66.bin");
+
+//---------------------------------------------------------------------------------------------------- Tests
+#[cfg(test)]
+mod test {}

test-utils/src/data/free.rs

@@ -0,0 +1,110 @@
+//! Free functions to access data.
+
+#![allow(
+    const_item_mutation, // `R: Read` needs `&mut self`
+    clippy::missing_panics_doc, // These functions shouldn't panic
+)]
+
+//---------------------------------------------------------------------------------------------------- Import
+use std::sync::OnceLock;
+
+use monero_serai::{block::Block, transaction::Transaction};
+
+use crate::data::constants::{
+    BLOCK_43BD1F, BLOCK_BBD604, BLOCK_F91043, TX_3BC7FF, TX_84D48D, TX_9E3F73,
+};
+
+//---------------------------------------------------------------------------------------------------- Blocks
+/// Return [`BLOCK_BBD604`] as a [`Block`].
+///
+/// ```rust
+/// assert_eq!(
+///     &cuprate_test_utils::data::block_v1_tx513().serialize(),
+///     cuprate_test_utils::data::BLOCK_BBD604
+/// );
+/// ```
+pub fn block_v1_tx513() -> Block {
+    /// `OnceLock` holding the data.
+    static BLOCK: OnceLock<Block> = OnceLock::new();
+    BLOCK
+        .get_or_init(|| Block::read(&mut BLOCK_BBD604).unwrap())
+        .clone()
+}
+
+/// Return [`BLOCK_F91043`] as a [`Block`].
+///
+/// ```rust
+/// assert_eq!(
+///     &cuprate_test_utils::data::block_v9_tx3().serialize(),
+///     cuprate_test_utils::data::BLOCK_F91043
+/// );
+/// ```
+pub fn block_v9_tx3() -> Block {
+    /// `OnceLock` holding the data.
+    static BLOCK: OnceLock<Block> = OnceLock::new();
+    BLOCK
+        .get_or_init(|| Block::read(&mut BLOCK_F91043).unwrap())
+        .clone()
+}
+
+/// Return [`BLOCK_43BD1F`] as a [`Block`].
+///
+/// ```rust
+/// assert_eq!(
+///     &cuprate_test_utils::data::block_v16_tx0().serialize(),
+///     cuprate_test_utils::data::BLOCK_43BD1F
+/// );
+/// ```
+pub fn block_v16_tx0() -> Block {
+    /// `OnceLock` holding the data.
+    static BLOCK: OnceLock<Block> = OnceLock::new();
+    BLOCK
+        .get_or_init(|| Block::read(&mut BLOCK_43BD1F).unwrap())
+        .clone()
+}
+
+//---------------------------------------------------------------------------------------------------- Transactions
+/// Return [`TX_3BC7FF`] as a [`Transaction`].
+///
+/// ```rust
+/// assert_eq!(
+///     &cuprate_test_utils::data::tx_v1_sig0().serialize(),
+///     cuprate_test_utils::data::TX_3BC7FF
+/// );
+/// ```
+pub fn tx_v1_sig0() -> Transaction {
+    /// `OnceLock` holding the data.
+    static TX: OnceLock<Transaction> = OnceLock::new();
+    TX.get_or_init(|| Transaction::read(&mut TX_3BC7FF).unwrap())
+        .clone()
+}
+
+/// Return [`TX_9E3F73`] as a [`Transaction`].
+///
+/// ```rust
+/// assert_eq!(
+///     &cuprate_test_utils::data::tx_v1_sig2().serialize(),
+///     cuprate_test_utils::data::TX_9E3F73
+/// );
+/// ```
+pub fn tx_v1_sig2() -> Transaction {
+    /// `OnceLock` holding the data.
+    static TX: OnceLock<Transaction> = OnceLock::new();
+    TX.get_or_init(|| Transaction::read(&mut TX_9E3F73).unwrap())
+        .clone()
+}
+
+/// Return [`TX_84D48D`] as a [`Transaction`].
+///
+/// ```rust
+/// assert_eq!(
+///     &cuprate_test_utils::data::tx_v2_rct3().serialize(),
+///     cuprate_test_utils::data::TX_84D48D
+/// );
+/// ```
+pub fn tx_v2_rct3() -> Transaction {
+    /// `OnceLock` holding the data.
+    static TX: OnceLock<Transaction> = OnceLock::new();
+    TX.get_or_init(|| Transaction::read(&mut TX_84D48D).unwrap())
+        .clone()
+}

test-utils/src/data/mod.rs

@@ -0,0 +1,9 @@
+//! Testing data and utilities.
+//!
+//! Raw data is found in `data/`.
+
+mod constants;
+pub use constants::{BLOCK_43BD1F, BLOCK_BBD604, BLOCK_F91043, TX_3BC7FF, TX_84D48D, TX_9E3F73};
+
+mod free;
+pub use free::{block_v16_tx0, block_v1_tx513, block_v9_tx3, tx_v1_sig0, tx_v1_sig2, tx_v2_rct3};

test-utils/src/lib.rs

@@ -1,2 +1,7 @@
+//! Cuprate testing utilities.
+//!
+//! See the `README.md` for more info.
+
+pub mod data;
 pub mod monerod;
 pub mod test_netzone;

test-utils/src/monerod.rs

@@ -17,11 +17,17 @@ use tokio::{task::yield_now, time::timeout};
 
 mod download;
 
+/// IPv4 local host.
 const LOCALHOST: IpAddr = IpAddr::V4(Ipv4Addr::LOCALHOST);
+
+/// The `monerod` version to use.
 const MONEROD_VERSION: &str = "v0.18.3.1";
+
+/// The log line `monerod` emits indicated it has successfully started up.
 const MONEROD_STARTUP_TEXT: &str =
     "The daemon will start synchronizing with the network. This may take a long time to complete.";
 
+/// The log line `monerod` emits indicated it has stopped.
 const MONEROD_SHUTDOWN_TEXT: &str = "Stopping cryptonote protocol";
 
 /// Spawns monerod and returns [`SpawnedMoneroD`].
@@ -66,9 +72,7 @@ pub async fn monerod<T: AsRef<OsStr>>(flags: impl IntoIterator<Item = T>) -> Spa
 
             if logs.contains(MONEROD_SHUTDOWN_TEXT) {
                 panic!("Failed to start monerod, logs: \n {logs}");
-            }
+            } else if logs.contains(MONEROD_STARTUP_TEXT) {
-
-            if logs.contains(MONEROD_STARTUP_TEXT) {
                 break;
             }
             // this is blocking code but as this is for tests performance isn't a priority. However we should still yield so
@@ -88,6 +92,7 @@ pub async fn monerod<T: AsRef<OsStr>>(flags: impl IntoIterator<Item = T>) -> Spa
     }
 }
 
+/// Fetch an available TCP port on the machine for `monerod` to bind to.
 fn get_available_port(already_taken: &[u16]) -> u16 {
     loop {
         // Using `0` makes the OS return a random available port.
@@ -119,12 +124,12 @@ pub struct SpawnedMoneroD {
 
 impl SpawnedMoneroD {
     /// Returns the p2p port of the spawned monerod
-    pub fn p2p_addr(&self) -> SocketAddr {
+    pub const fn p2p_addr(&self) -> SocketAddr {
         SocketAddr::new(LOCALHOST, self.p2p_port)
     }
 
     /// Returns the RPC port of the spawned monerod
-    pub fn rpc_port(&self) -> SocketAddr {
+    pub const fn rpc_port(&self) -> SocketAddr {
         SocketAddr::new(LOCALHOST, self.rpc_port)
     }
 }
@@ -135,7 +140,7 @@ impl Drop for SpawnedMoneroD {
 
         if self.process.kill().is_err() {
             error = true;
-            println!("Failed to kill monerod, process id: {}", self.process.id())
+            println!("Failed to kill monerod, process id: {}", self.process.id());
         }
 
         if panicking() {

test-utils/src/monerod/download.rs

@@ -24,23 +24,23 @@ static DOWNLOAD_MONEROD_MUTEX: Mutex<()> = Mutex::const_new(());
 /// Returns the file name to download and the expected extracted folder name.
 fn file_name(version: &str) -> (String, String) {
     let download_file = match (OS, ARCH) {
-        ("windows", "x64") | ("windows", "x86_64") => format!("monero-win-x64-{}.zip", version),
+        ("windows", "x64" | "x86_64") => format!("monero-win-x64-{version}.zip"),
-        ("windows", "x86") => format!("monero-win-x86-{}.zip", version),
+        ("windows", "x86") => format!("monero-win-x86-{version}.zip"),
-        ("linux", "x64") | ("linux", "x86_64") => format!("monero-linux-x64-{}.tar.bz2", version),
+        ("linux", "x64" | "x86_64") => format!("monero-linux-x64-{version}.tar.bz2"),
-        ("linux", "x86") => format!("monero-linux-x86-{}.tar.bz2", version),
+        ("linux", "x86") => format!("monero-linux-x86-{version}.tar.bz2"),
-        ("macos", "x64") | ("macos", "x86_64") => format!("monero-mac-x64-{}.tar.bz2", version),
+        ("macos", "x64" | "x86_64") => format!("monero-mac-x64-{version}.tar.bz2"),
         _ => panic!("Can't get monerod for {OS}, {ARCH}."),
     };
 
     let extracted_dir = match (OS, ARCH) {
-        ("windows", "x64") | ("windows", "x86_64") => {
+        ("windows", "x64" | "x86_64") => {
-            format!("monero-x86_64-w64-mingw32-{}", version)
+            format!("monero-x86_64-w64-mingw32-{version}")
         }
-        ("windows", "x86") => format!("monero-i686-w64-mingw32-{}", version),
+        ("windows", "x86") => format!("monero-i686-w64-mingw32-{version}"),
-        ("linux", "x64") | ("linux", "x86_64") => format!("monero-x86_64-linux-gnu-{}", version),
+        ("linux", "x64" | "x86_64") => format!("monero-x86_64-linux-gnu-{version}"),
-        ("linux", "x86") => format!("monero-i686-linux-gnu-{}", version),
+        ("linux", "x86") => format!("monero-i686-linux-gnu-{version}"),
-        ("macos", "x64") | ("macos", "x86_64") => {
+        ("macos", "x64" | "x86_64") => {
-            format!("monero-x86_64-apple-darwin11-{}", version)
+            format!("monero-x86_64-apple-darwin11-{version}")
         }
         _ => panic!("Can't get monerod for {OS}, {ARCH}."),
     };
@@ -50,7 +50,7 @@ fn file_name(version: &str) -> (String, String) {
 
 /// Downloads the monerod file provided, extracts it and puts the extracted folder into `path_to_store`.
 async fn download_monerod(file_name: &str, path_to_store: &Path) -> Result<(), ReqError> {
-    let res = get(format!("https://downloads.getmonero.org/cli/{}", file_name)).await?;
+    let res = get(format!("https://downloads.getmonero.org/cli/{file_name}")).await?;
     let monerod_archive = res.bytes().await.unwrap();
 
     #[cfg(unix)]
@@ -83,7 +83,7 @@ fn find_target() -> PathBuf {
 }
 
 /// Checks if we have monerod or downloads it if we don't and then returns the path to it.
-pub async fn check_download_monerod() -> Result<PathBuf, ReqError> {
+pub(crate) async fn check_download_monerod() -> Result<PathBuf, ReqError> {
     // make sure no other threads are downloading monerod at the same time.
     let _guard = DOWNLOAD_MONEROD_MUTEX.lock().await;
 

test-utils/src/test_netzone.rs

@@ -1,4 +1,4 @@
-//! Test NetZone
+//! Test net zone.
 //!
 //! This module contains a test network zone, this network zone use channels as the network layer to simulate p2p
 //! communication.
@@ -48,7 +48,7 @@ impl std::fmt::Display for TestNetZoneAddr {
 
 impl From<TestNetZoneAddr> for NetworkAddress {
     fn from(value: TestNetZoneAddr) -> Self {
-        NetworkAddress::Clear(SocketAddr::new(Ipv4Addr::from(value.0).into(), 18080))
+        Self::Clear(SocketAddr::new(Ipv4Addr::from(value.0).into(), 18080))
     }
 }
 
@@ -58,13 +58,14 @@ impl TryFrom<NetworkAddress> for TestNetZoneAddr {
     fn try_from(value: NetworkAddress) -> Result<Self, Self::Error> {
         match value {
             NetworkAddress::Clear(soc) => match soc {
-                SocketAddr::V4(v4) => Ok(TestNetZoneAddr(u32::from_be_bytes(v4.ip().octets()))),
+                SocketAddr::V4(v4) => Ok(Self(u32::from_be_bytes(v4.ip().octets()))),
-                _ => panic!("None v4 address in test code"),
+                SocketAddr::V6(_) => panic!("None v4 address in test code"),
             },
         }
     }
 }
 
+/// TODO
 #[derive(Debug, Clone, Copy, Eq, PartialEq)]
 pub struct TestNetZone<const ALLOW_SYNC: bool, const DANDELION_PP: bool, const CHECK_NODE_ID: bool>;
 

types/Cargo.toml

@@ -0,0 +1,22 @@
+[package]
+name        = "cuprate-types"
+version     = "0.0.0"
+edition     = "2021"
+description = "Cuprate data types"
+license     = "MIT"
+authors     = ["hinto-janai"]
+repository  = "https://github.com/Cuprate/cuprate/tree/main/types"
+keywords    = ["cuprate", "types"]
+
+[features]
+default = ["service"]
+service = []
+
+[dependencies]
+borsh            = { workspace = true, optional = true }
+cfg-if           = { workspace = true }
+curve25519-dalek = { workspace = true }
+monero-serai     = { workspace = true }
+serde            = { workspace = true, optional = true }
+
+[dev-dependencies]

types/README.md

@@ -0,0 +1,21 @@
+# `cuprate-types`
+Various data types shared by Cuprate.
+
+<!-- Did you know markdown automatically increments number lists, even if they are all 1...? -->
+1. [File Structure](#file-structure)
+    - [`src/`](#src)
+
+---
+
+# File Structure
+A quick reference of the structure of the folders & files in `cuprate-types`.
+
+Note that `lib.rs/mod.rs` files are purely for re-exporting/visibility/lints, and contain no code. Each sub-directory has a corresponding `mod.rs`.
+
+## `src/`
+The top-level `src/` files.
+
+| File                | Purpose |
+|---------------------|---------|
+| `service.rs`        | Types used in database requests; `enum {Request,Response}`
+| `types.rs`          | Various general types used by Cuprate

types/src/lib.rs

@@ -0,0 +1,101 @@
+//! Cuprate shared data types.
+//!
+//! TODO
+
+//---------------------------------------------------------------------------------------------------- Lints
+// Forbid lints.
+// Our code, and code generated (e.g macros) cannot overrule these.
+#![forbid(
+	// `unsafe` is allowed but it _must_ be
+	// commented with `SAFETY: reason`.
+	clippy::undocumented_unsafe_blocks,
+
+	// Never.
+	unused_unsafe,
+	redundant_semicolons,
+	unused_allocation,
+	coherence_leak_check,
+	single_use_lifetimes,
+	while_true,
+	clippy::missing_docs_in_private_items,
+
+	// Maybe can be put into `#[deny]`.
+	unconditional_recursion,
+	for_loops_over_fallibles,
+	unused_braces,
+	unused_doc_comments,
+	unused_labels,
+	keyword_idents,
+	non_ascii_idents,
+	variant_size_differences,
+
+	// Probably can be put into `#[deny]`.
+	future_incompatible,
+	let_underscore,
+	break_with_label_and_loop,
+	duplicate_macro_attributes,
+	exported_private_dependencies,
+	large_assignments,
+	overlapping_range_endpoints,
+	semicolon_in_expressions_from_macros,
+	noop_method_call,
+	unreachable_pub,
+)]
+// Deny lints.
+// Some of these are `#[allow]`'ed on a per-case basis.
+#![deny(
+    clippy::all,
+    clippy::correctness,
+    clippy::suspicious,
+    clippy::style,
+    clippy::complexity,
+    clippy::perf,
+    clippy::pedantic,
+    clippy::nursery,
+    clippy::cargo,
+    unused_mut,
+    missing_docs,
+    deprecated,
+    unused_comparisons,
+    nonstandard_style
+)]
+#![allow(unreachable_code, unused_variables, dead_code, unused_imports)] // TODO: remove
+#![allow(
+	// FIXME: this lint affects crates outside of
+	// `database/` for some reason, allow for now.
+	clippy::cargo_common_metadata,
+
+	// FIXME: adding `#[must_use]` onto everything
+	// might just be more annoying than useful...
+	// although it is sometimes nice.
+	clippy::must_use_candidate,
+
+	// TODO: should be removed after all `todo!()`'s are gone.
+	clippy::diverging_sub_expression,
+
+	clippy::module_name_repetitions,
+	clippy::module_inception,
+	clippy::redundant_pub_crate,
+	clippy::option_if_let_else,
+)]
+// Allow some lints when running in debug mode.
+#![cfg_attr(debug_assertions, allow(clippy::todo, clippy::multiple_crate_versions))]
+
+//---------------------------------------------------------------------------------------------------- Public API
+// Import private modules, export public types.
+//
+// Documentation for each module is located in the respective file.
+
+mod types;
+pub use types::{
+    ExtendedBlockHeader, OutputOnChain, TransactionVerificationData, VerifiedBlockInformation,
+};
+
+//---------------------------------------------------------------------------------------------------- Feature-gated
+cfg_if::cfg_if! {
+    if #[cfg(feature = "service")] {
+        pub mod service;
+    }
+}
+
+//---------------------------------------------------------------------------------------------------- Private

types/src/service.rs

@@ -0,0 +1,88 @@
+//! Database [`ReadRequest`]s, [`WriteRequest`]s, and [`Response`]s.
+
+//---------------------------------------------------------------------------------------------------- Import
+use std::{
+    collections::{HashMap, HashSet},
+    ops::Range,
+};
+
+use monero_serai::{block::Block, transaction::Transaction};
+
+#[cfg(feature = "borsh")]
+use borsh::{BorshDeserialize, BorshSerialize};
+#[cfg(feature = "serde")]
+use serde::{Deserialize, Serialize};
+
+use crate::types::{ExtendedBlockHeader, OutputOnChain, VerifiedBlockInformation};
+
+//---------------------------------------------------------------------------------------------------- ReadRequest
+/// A read request to the database.
+#[derive(Debug, Clone, PartialEq, Eq)]
+#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
+#[cfg_attr(feature = "borsh", derive(BorshSerialize, BorshDeserialize))]
+pub enum ReadRequest {
+    /// TODO
+    BlockExtendedHeader(u64),
+    /// TODO
+    BlockHash(u64),
+    /// TODO
+    BlockExtendedHeaderInRange(Range<u64>),
+    /// TODO
+    ChainHeight,
+    /// TODO
+    GeneratedCoins,
+    /// TODO
+    Outputs(HashMap<u64, HashSet<u64>>),
+    /// TODO
+    NumberOutputsWithAmount(Vec<u64>),
+    /// TODO
+    CheckKIsNotSpent(HashSet<[u8; 32]>),
+    /// TODO
+    BlockBatchInRange(Range<u64>),
+}
+
+//---------------------------------------------------------------------------------------------------- WriteRequest
+/// A write request to the database.
+#[derive(Debug, Clone, PartialEq, Eq)]
+// #[cfg_attr(feature = "borsh", derive(BorshSerialize, BorshDeserialize))]
+pub enum WriteRequest {
+    /// TODO
+    WriteBlock(VerifiedBlockInformation),
+}
+
+//---------------------------------------------------------------------------------------------------- Response
+/// A response from the database.
+#[derive(Debug, Clone, PartialEq, Eq)]
+// #[cfg_attr(feature = "borsh", derive(BorshSerialize, BorshDeserialize))]
+pub enum Response {
+    //------------------------------------------------------ Reads
+    /// TODO
+    BlockExtendedHeader(ExtendedBlockHeader),
+    /// TODO
+    BlockHash([u8; 32]),
+    /// TODO
+    BlockExtendedHeaderInRange(Vec<ExtendedBlockHeader>),
+    /// TODO
+    ChainHeight(u64, [u8; 32]),
+    /// TODO
+    GeneratedCoins(u64),
+    /// TODO
+    Outputs(HashMap<u64, HashMap<u64, OutputOnChain>>),
+    /// TODO
+    NumberOutputsWithAmount(HashMap<u64, usize>),
+    /// TODO
+    /// returns true if key images are spent
+    CheckKIsNotSpent(bool),
+    /// TODO
+    BlockBatchInRange(Vec<(Block, Vec<Transaction>)>),
+
+    //------------------------------------------------------ Writes
+    /// TODO
+    WriteBlockOk,
+}
+
+//---------------------------------------------------------------------------------------------------- Tests
+#[cfg(test)]
+mod test {
+    // use super::*;
+}

types/src/types.rs

@@ -0,0 +1,103 @@
+//! TODO
+
+//---------------------------------------------------------------------------------------------------- Import
+use std::sync::Arc;
+
+use curve25519_dalek::edwards::EdwardsPoint;
+use monero_serai::{
+    block::Block,
+    transaction::{Timelock, Transaction},
+};
+
+#[cfg(feature = "borsh")]
+use borsh::{BorshDeserialize, BorshSerialize};
+#[cfg(feature = "serde")]
+use serde::{Deserialize, Serialize};
+
+//---------------------------------------------------------------------------------------------------- ExtendedBlockHeader
+/// TODO
+#[derive(Copy, Clone, Default, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
+#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
+#[cfg_attr(feature = "borsh", derive(BorshSerialize, BorshDeserialize))]
+pub struct ExtendedBlockHeader {
+    /// TODO
+    /// This is a `cuprate_consensus::HardFork`.
+    pub version: u8,
+    /// TODO
+    /// This is a `cuprate_consensus::HardFork`.
+    pub vote: u8,
+    /// TODO
+    pub timestamp: u64,
+    /// TODO
+    pub cumulative_difficulty: u128,
+    /// TODO
+    pub block_weight: usize,
+    /// TODO
+    pub long_term_weight: usize,
+}
+
+//---------------------------------------------------------------------------------------------------- TransactionVerificationData
+/// TODO
+#[derive(Clone, Debug, PartialEq, Eq)]
+// #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] // FIXME: monero_serai
+// #[cfg_attr(feature = "borsh", derive(BorshSerialize, BorshDeserialize))]
+pub struct TransactionVerificationData {
+    /// TODO
+    pub tx: Transaction,
+    /// TODO
+    pub tx_blob: Vec<u8>,
+    /// TODO
+    pub tx_weight: usize,
+    /// TODO
+    pub fee: u64,
+    /// TODO
+    pub tx_hash: [u8; 32],
+}
+
+//---------------------------------------------------------------------------------------------------- VerifiedBlockInformation
+/// TODO
+#[derive(Clone, Debug, PartialEq, Eq)]
+// #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] // FIXME: monero_serai
+// #[cfg_attr(feature = "borsh", derive(BorshSerialize, BorshDeserialize))]
+pub struct VerifiedBlockInformation {
+    /// TODO
+    pub block: Block,
+    /// TODO
+    pub txs: Vec<Arc<TransactionVerificationData>>,
+    /// TODO
+    pub block_hash: [u8; 32],
+    /// TODO
+    pub pow_hash: [u8; 32],
+    /// TODO
+    pub height: u64,
+    /// TODO
+    pub generated_coins: u64,
+    /// TODO
+    pub weight: usize,
+    /// TODO
+    pub long_term_weight: usize,
+    /// TODO
+    pub cumulative_difficulty: u128,
+}
+
+//---------------------------------------------------------------------------------------------------- OutputOnChain
+/// An already approved previous transaction output.
+#[derive(Clone, Debug, PartialEq, Eq)]
+// #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] // FIXME: monero_serai
+// #[cfg_attr(feature = "borsh", derive(BorshSerialize, BorshDeserialize))]
+pub struct OutputOnChain {
+    /// TODO
+    pub height: u64,
+    /// TODO
+    pub time_lock: Timelock,
+    /// TODO
+    pub key: Option<EdwardsPoint>,
+    /// TODO
+    pub commitment: EdwardsPoint,
+}
+
+//---------------------------------------------------------------------------------------------------- Tests
+#[cfg(test)]
+mod test {
+    // use super::*;
+}