rpc: use ByteArrayVec and ContainerAsBlob (#227)

* fixed-bytes: add `serde`, document feature flags

* fixed-bytes: add derives

* rpc: add `as _` syntax to macro

* rpc: use `ByteArrayVec` and `ContainerAsBlob` for binary types

* fixed-bytes: re-add derives

* rpc-types: dedup default value within macro

* readme: fixed bytes section

Cargo.lock

@@ -762,6 +762,7 @@ name = "cuprate-rpc-types"
 version = "0.0.0"
 dependencies = [
  "cuprate-epee-encoding",
+ "cuprate-fixed-bytes",
  "monero-serai",
  "paste",
  "serde",

net/fixed-bytes/Cargo.toml

@@ -16,4 +16,4 @@ bytes = { workspace = true }
 serde = { workspace = true, features = ["derive"], optional = true }
 
 [dev-dependencies]
-serde_json = { workspace = true, features = ["std"] }
+serde_json = { workspace = true, features = ["std"] }

net/fixed-bytes/src/lib.rs

@@ -12,6 +12,7 @@ use serde::{Deserialize, Deserializer, Serialize};
 
 #[cfg_attr(feature = "std", derive(thiserror::Error))]
 #[cfg_attr(feature = "serde", derive(Deserialize, Serialize))]
+#[derive(Clone, Eq, PartialEq, PartialOrd, Ord, Hash)]
 pub enum FixedByteError {
     #[cfg_attr(
         feature = "std",
@@ -48,7 +49,7 @@ impl Debug for FixedByteError {
 ///
 /// Internally this is just a wrapper around [`Bytes`], with the constructors checking that the length is equal to `N`.
 /// This implements [`Deref`] with the target being `[u8; N]`.
-#[derive(Debug, Clone, Eq, PartialEq)]
+#[derive(Debug, Default, Clone, Eq, PartialEq, PartialOrd, Ord, Hash)]
 #[cfg_attr(feature = "serde", derive(Serialize))]
 #[cfg_attr(feature = "serde", serde(transparent))]
 #[repr(transparent)]
@@ -115,7 +116,7 @@ impl<const N: usize> TryFrom<Vec<u8>> for ByteArray<N> {
     }
 }
 
-#[derive(Debug, Clone, Eq, PartialEq)]
+#[derive(Debug, Default, Clone, Eq, PartialEq, PartialOrd, Ord, Hash)]
 #[cfg_attr(feature = "serde", derive(Serialize))]
 #[cfg_attr(feature = "serde", serde(transparent))]
 #[repr(transparent)]

rpc/types/Cargo.toml

@@ -10,11 +10,12 @@ keywords    = ["cuprate", "rpc", "types", "monero"]
 
 [features]
 default = ["serde", "epee"]
-serde   = ["dep:serde"]
+serde   = ["dep:serde", "cuprate-fixed-bytes/serde"]
 epee    = ["dep:cuprate-epee-encoding"]
 
 [dependencies]
 cuprate-epee-encoding = { path = "../../net/epee-encoding", optional = true }
+cuprate-fixed-bytes   = { path = "../../net/fixed-bytes" }
 
 monero-serai = { workspace = true }
 paste        = { workspace = true }

rpc/types/README.md

@@ -64,6 +64,20 @@ These mixed types are:
 
 TODO: we need to figure out a type that (de)serializes correctly, `String` errors with `serde_json`
 
+# Fixed byte containers
+TODO
+
+<!--
+
+Some fields within requests/responses are containers, but fixed in size.
+
+For example, [`crate::json::GetBlockTemplateResponse::prev_hash`] is always a 32-byte hash.
+
+In these cases, stack allocated types like `cuprate_fixed_bytes::StrArray`
+will be used instead of a more typical [`String`] for optimization reasons.
+
+-->
+
 # Feature flags
 List of feature flags for `cuprate-rpc-types`.
 

rpc/types/src/bin.rs

@@ -3,6 +3,11 @@
 //! All types are originally defined in [`rpc/core_rpc_server_commands_defs.h`](https://github.com/monero-project/monero/blob/cc73fe71162d564ffda8e549b79a350bca53c454/src/rpc/core_rpc_server_commands_defs.h).
 
 //---------------------------------------------------------------------------------------------------- Import
+use cuprate_fixed_bytes::ByteArrayVec;
+
+#[cfg(feature = "epee")]
+use cuprate_epee_encoding::container_as_blob::ContainerAsBlob;
+
 use crate::{
     base::{AccessResponseBase, ResponseBase},
     defaults::{default_false, default_height, default_string, default_vec, default_zero},
@@ -22,16 +27,13 @@ define_request_and_response! {
     core_rpc_server_commands_defs.h => 162..=262,
     GetBlocks,
     Request {
-        #[cfg_attr(feature = "serde", serde(default = "default_zero"))]
+        requested_info: u8 = default_zero(), "default_zero",
-        requested_info: u8 = default_zero(),
+        // FIXME: This is a `std::list` in `monerod` because...?
-        // TODO: This is a `std::list` in `monerod` because...?
+        block_ids: ByteArrayVec<32>,
-        block_ids: Vec<[u8; 32]>,
         start_height: u64,
         prune: bool,
-        #[cfg_attr(feature = "serde", serde(default = "default_false"))]
+        no_miner_tx: bool = default_false(), "default_false",
-        no_miner_tx: bool = default_false(),
+        pool_info_since: u64 = default_zero(), "default_zero",
-        #[cfg_attr(feature = "serde", serde(default = "default_zero"))]
-        pool_info_since: u64 = default_zero(),
     },
     // TODO: this has custom epee (de)serialization.
     // <https://github.com/monero-project/monero/blob/cc73fe71162d564ffda8e549b79a350bca53c454/src/rpc/core_rpc_server_commands_defs.h#L242-L259>
@@ -67,16 +69,17 @@ define_request_and_response! {
     core_rpc_server_commands_defs.h => 309..=338,
     GetHashes,
     Request {
-        block_ids: Vec<[u8; 32]>,
+        block_ids: ByteArrayVec<32>,
         start_height: u64,
     },
     AccessResponseBase {
-        m_blocks_ids: Vec<[u8; 32]>,
+        m_blocks_ids: ByteArrayVec<32>,
         start_height: u64,
         current_height: u64,
     }
 }
 
+#[cfg(not(feature = "epee"))]
 define_request_and_response! {
     get_o_indexesbin,
     cc73fe71162d564ffda8e549b79a350bca53c454 =>
@@ -91,6 +94,21 @@ define_request_and_response! {
     }
 }
 
+#[cfg(feature = "epee")]
+define_request_and_response! {
+    get_o_indexesbin,
+    cc73fe71162d564ffda8e549b79a350bca53c454 =>
+    core_rpc_server_commands_defs.h => 487..=510,
+    GetOutputIndexes,
+    #[derive(Copy)]
+    Request {
+        txid: [u8; 32],
+    },
+    AccessResponseBase {
+        o_indexes: Vec<u64> as ContainerAsBlob<u64>,
+    }
+}
+
 define_request_and_response! {
     get_outsbin,
     cc73fe71162d564ffda8e549b79a350bca53c454 =>
@@ -98,8 +116,7 @@ define_request_and_response! {
     GetOuts,
     Request {
         outputs: Vec<GetOutputsOut>,
-        #[cfg_attr(feature = "serde", serde(default = "default_false"))]
+        get_txid: bool = default_false(), "default_false",
-        get_txid: bool = default_false(),
     },
     AccessResponseBase {
         outs: Vec<OutKeyBin>,
@@ -113,7 +130,7 @@ define_request_and_response! {
     GetTransactionPoolHashes,
     Request {},
     AccessResponseBase {
-        tx_hashes: Vec<[u8; 32]>,
+        tx_hashes: ByteArrayVec<32>,
     }
 }
 

rpc/types/src/json.rs

@@ -49,8 +49,33 @@ define_request_and_response! {
         // $FIELD_NAME: $FIELD_TYPE,
         // ```
         // The struct generated and all fields are `pub`.
-        extra_nonce: String,
+
-        prev_block: String,
+        // This optional expression can be placed after
+        // a `field: field_type`. this indicates to the
+        // macro to (de)serialize this field using this
+        // default expression if it doesn't exist in epee.
+        //
+        // See `cuprate_epee_encoding::epee_object` for info.
+        //
+        // The default function must be specified twice:
+        //
+        // 1. As an expression
+        // 2. As a string literal
+        //
+        // For example: `extra_nonce: String /* = default_string(), "default_string" */,`
+        //
+        // This is a HACK since `serde`'s default attribute only takes in
+        // string literals and macros (stringify) within attributes do not work.
+        extra_nonce: String /* = default_expression, "default_literal" */,
+
+        // Another optional expression:
+        // This indicates to the macro to (de)serialize
+        // this field as another type in epee.
+        //
+        // See `cuprate_epee_encoding::epee_object` for info.
+        prev_block: String /* as Type */,
+
+        // Regular fields.
         reserve_size: u64,
         wallet_address: String,
     },
@@ -197,8 +222,7 @@ define_request_and_response! {
     GetLastBlockHeader,
     #[derive(Copy)]
     Request {
-        #[cfg_attr(feature = "serde", serde(default = "default_false"))]
+        fill_pow_hash: bool = default_false(), "default_false",
-        fill_pow_hash: bool = default_false(),
     },
     AccessResponseBase {
         block_header: BlockHeader,
@@ -213,8 +237,7 @@ define_request_and_response! {
     Request {
         hash: String,
         hashes: Vec<String>,
-        #[cfg_attr(feature = "serde", serde(default = "default_false"))]
+        fill_pow_hash: bool = default_false(), "default_false",
-        fill_pow_hash: bool = default_false(),
     },
     AccessResponseBase {
         block_header: BlockHeader,
@@ -230,8 +253,7 @@ define_request_and_response! {
     #[derive(Copy)]
     Request {
         height: u64,
-        #[cfg_attr(feature = "serde", serde(default = "default_false"))]
+        fill_pow_hash: bool = default_false(), "default_false",
-        fill_pow_hash: bool = default_false(),
     },
     AccessResponseBase {
         block_header: BlockHeader,
@@ -247,8 +269,7 @@ define_request_and_response! {
     Request {
         start_height: u64,
         end_height: u64,
-        #[cfg_attr(feature = "serde", serde(default = "default_false"))]
+        fill_pow_hash: bool = default_false(), "default_false",
-        fill_pow_hash: bool = default_false(),
     },
     AccessResponseBase {
         headers: Vec<BlockHeader>,
@@ -264,12 +285,9 @@ define_request_and_response! {
         // `monerod` has both `hash` and `height` fields.
         // In the RPC handler, if `hash.is_empty()`, it will use it, else, it uses `height`.
         // <https://github.com/monero-project/monero/blob/cc73fe71162d564ffda8e549b79a350bca53c454/src/rpc/core_rpc_server.cpp#L2674>
-        #[cfg_attr(feature = "serde", serde(default = "default_string"))]
+        hash: String = default_string(), "default_string",
-        hash: String = default_string(),
+        height: u64 = default_height(), "default_height",
-        #[cfg_attr(feature = "serde", serde(default = "default_height"))]
+        fill_pow_hash: bool = default_false(), "default_false",
-        height: u64 = default_height(),
-        #[cfg_attr(feature = "serde", serde(default = "default_false"))]
-        fill_pow_hash: bool = default_false(),
     },
     AccessResponseBase {
         blob: String,
@@ -287,7 +305,7 @@ define_request_and_response! {
     GetConnections,
     Request {},
     ResponseBase {
-        // TODO: This is a `std::list` in `monerod` because...?
+        // FIXME: This is a `std::list` in `monerod` because...?
         connections: Vec<ConnectionInfo>,
     }
 }
@@ -405,8 +423,7 @@ define_request_and_response! {
     core_rpc_server_commands_defs.h => 2096..=2116,
     FlushTransactionPool,
     Request {
-        #[cfg_attr(feature = "serde", serde(default = "default_vec"))]
+        txids: Vec<String> = default_vec::<String>(), "default_vec",
-        txids: Vec<String> = default_vec::<String>(),
     },
     #[derive(Copy)]
     #[cfg_attr(feature = "serde", serde(transparent))]
@@ -461,12 +478,12 @@ define_request_and_response! {
     ResponseBase {
         version: u32,
         release: bool,
-        #[serde(skip_serializing_if = "is_zero", default = "default_zero")]
+        #[serde(skip_serializing_if = "is_zero")]
-        current_height: u64 = default_zero(),
+        current_height: u64 = default_zero(), "default_zero",
-        #[serde(skip_serializing_if = "is_zero", default = "default_zero")]
+        #[serde(skip_serializing_if = "is_zero")]
-        target_height: u64 = default_zero(),
+        target_height: u64 = default_zero(), "default_zero",
-        #[serde(skip_serializing_if = "Vec::is_empty", default = "default_vec")]
+        #[serde(skip_serializing_if = "Vec::is_empty")]
-        hard_forks: Vec<HardforkEntry> = default_vec(),
+        hard_forks: Vec<HardforkEntry> = default_vec(), "default_vec",
     }
 }
 
@@ -521,9 +538,9 @@ define_request_and_response! {
         height: u64,
         next_needed_pruning_seed: u32,
         overview: String,
-        // TODO: This is a `std::list` in `monerod` because...?
+        // FIXME: This is a `std::list` in `monerod` because...?
         peers: Vec<SyncInfoPeer>,
-        // TODO: This is a `std::list` in `monerod` because...?
+        // FIXME: This is a `std::list` in `monerod` because...?
         spans: Vec<Span>,
         target_height: u64,
     }
@@ -588,8 +605,7 @@ define_request_and_response! {
     PruneBlockchain,
     #[derive(Copy)]
     Request {
-        #[cfg_attr(feature = "serde", serde(default = "default_false"))]
+        check: bool = default_false(), "default_false",
-        check: bool = default_false(),
     },
     #[derive(Copy)]
     ResponseBase {
@@ -623,10 +639,8 @@ define_request_and_response! {
     FlushCache,
     #[derive(Copy)]
     Request {
-        #[cfg_attr(feature = "serde", serde(default = "default_false"))]
+        bad_txs: bool = default_false(), "default_false",
-        bad_txs: bool = default_false(),
+        bad_blocks: bool = default_false(), "default_false",
-        #[cfg_attr(feature = "serde", serde(default = "default_false"))]
-        bad_blocks: bool = default_false(),
     },
     ResponseBase {}
 }

rpc/types/src/macros.rs

@@ -45,7 +45,7 @@
 /// would trigger the different branches.
 macro_rules! define_request_and_response {
     (
-        // The markdown tag for Monero RPC documentation. Not necessarily the endpoint.
+        // The markdown tag for Monero daemon RPC documentation. Not necessarily the endpoint.
         $monero_daemon_rpc_doc_link:ident,
 
         // The commit hash and  `$file.$extension` in which this type is defined in
@@ -67,8 +67,10 @@ macro_rules! define_request_and_response {
         Request {
             // And any fields.
             $(
-                $( #[$request_field_attr:meta] )*
+                $( #[$request_field_attr:meta] )* // Field attribute.
-                $request_field:ident: $request_field_type:ty $(= $request_field_type_default:expr)?,
+                $request_field:ident: $request_field_type:ty // field_name: field type
+                $(as $request_field_type_as:ty)? // (optional) alternative type (de)serialization
+                $(= $request_field_type_default:expr, $request_field_type_default_string:literal)?, // (optional) default value
             )*
         },
 
@@ -78,7 +80,9 @@ macro_rules! define_request_and_response {
             // And any fields.
             $(
                 $( #[$response_field_attr:meta] )*
-                $response_field:ident: $response_field_type:ty $(= $response_field_type_default:expr)?,
+                $response_field:ident: $response_field_type:ty
+                $(as $response_field_type_as:ty)?
+                $(= $response_field_type_default:expr, $response_field_type_default_string:literal)?,
             )*
         }
     ) => { paste::paste! {
@@ -99,7 +103,9 @@ macro_rules! define_request_and_response {
             [<$type_name Request>] {
                 $(
                     $( #[$request_field_attr] )*
-                    $request_field: $request_field_type $(= $request_field_type_default)?,
+                    $request_field: $request_field_type
+                    $(as $request_field_type_as)?
+                    $(= $request_field_type_default, $request_field_type_default_string)?,
                 )*
             }
         }
@@ -125,7 +131,9 @@ macro_rules! define_request_and_response {
             $response_base_type => [<$type_name Response>] {
                 $(
                     $( #[$response_field_attr] )*
-                    $response_field: $response_field_type $(= $response_field_type_default)?,
+                    $response_field: $response_field_type
+                    $(as $response_field_type_as)?
+                    $(= $response_field_type_default, $response_field_type_default_string)?,
                 )*
             }
         }
@@ -166,7 +174,9 @@ macro_rules! __define_request {
             $(
                 $( #[$field_attr:meta] )* // field attributes
                 // field_name: FieldType
-                $field:ident: $field_type:ty $(= $field_default:expr)?,
+                $field:ident: $field_type:ty
+                $(as $field_as:ty)?
+                $(= $field_default:expr, $field_default_string:literal)?,
                 // The $field_default is an optional extra token that represents
                 // a default value to pass to [`cuprate_epee_encoding::epee_object`],
                 // see it for usage.
@@ -180,6 +190,7 @@ macro_rules! __define_request {
         pub struct $t {
             $(
                 $( #[$field_attr] )*
+                $(#[cfg_attr(feature = "serde", serde(default = $field_default_string))])?
                 pub $field: $field_type,
             )*
         }
@@ -188,7 +199,9 @@ macro_rules! __define_request {
         ::cuprate_epee_encoding::epee_object! {
             $t,
             $(
-                $field: $field_type $(= $field_default)?,
+                $field: $field_type
+                $(as $field_as)?
+                $(= $field_default)?,
             )*
         }
     };
@@ -218,7 +231,9 @@ macro_rules! __define_response {
             // See [`__define_request`] for docs, this does the same thing.
             $(
                 $( #[$field_attr:meta] )*
-                $field:ident: $field_type:ty $(= $field_default:expr)?,
+                $field:ident: $field_type:ty
+                $(as $field_as:ty)?
+                $(= $field_default:expr, $field_default_string:literal)?,
             )*
         }
     ) => {
@@ -226,6 +241,7 @@ macro_rules! __define_response {
         pub struct $t {
             $(
                 $( #[$field_attr] )*
+                $(#[cfg_attr(feature = "serde", serde(default = $field_default_string))])?
                 pub $field: $field_type,
             )*
         }
@@ -234,7 +250,9 @@ macro_rules! __define_response {
         ::cuprate_epee_encoding::epee_object! {
             $t,
             $(
-                $field: $field_type $($field_default)?,
+                $field: $field_type
+                $(as $field_as)?
+                $(= $field_default)?,
             )*
         }
     };
@@ -250,7 +268,9 @@ macro_rules! __define_response {
             // See [`__define_request`] for docs, this does the same thing.
             $(
                 $( #[$field_attr:meta] )*
-                $field:ident: $field_type:ty $(= $field_default:expr)?,
+                $field:ident: $field_type:ty
+                $(as $field_as:ty)?
+                $(= $field_default:expr, $field_default_string:literal)?,
             )*
         }
     ) => {
@@ -261,6 +281,7 @@ macro_rules! __define_response {
 
             $(
                 $( #[$field_attr] )*
+                $(#[cfg_attr(feature = "serde", serde(default = $field_default_string))])?
                 pub $field: $field_type,
             )*
         }
@@ -269,7 +290,9 @@ macro_rules! __define_response {
         ::cuprate_epee_encoding::epee_object! {
             $t,
             $(
-                $field: $field_type $(= $field_default)?,
+                $field: $field_type
+                $(as $field_as)?
+                $(= $field_default)?,
             )*
             !flatten: base: $base,
         }

rpc/types/src/other.rs

@@ -36,12 +36,9 @@ define_request_and_response! {
         // FIXME: this is documented as optional but it isn't serialized as an optional
         // but it is set _somewhere_ to false in `monerod`
         // <https://github.com/monero-project/monero/blob/cc73fe71162d564ffda8e549b79a350bca53c454/src/rpc/core_rpc_server_commands_defs.h#L382>
-        #[cfg_attr(feature = "serde", serde(default = "default_false"))]
+        decode_as_json: bool = default_false(), "default_false",
-        decode_as_json: bool = default_false(),
+        prune: bool = default_false(), "default_false",
-        #[cfg_attr(feature = "serde", serde(default = "default_false"))]
+        split: bool = default_false(), "default_false",
-        prune: bool = default_false(),
-        #[cfg_attr(feature = "serde", serde(default = "default_false"))]
-        split: bool = default_false(),
     },
     AccessResponseBase {
         txs_as_hex: Vec<String>,
@@ -82,10 +79,8 @@ define_request_and_response! {
     SendRawTransaction,
     Request {
         tx_as_hex: String,
-        #[cfg_attr(feature = "serde", serde(default = "default_false"))]
+        do_not_relay: bool = default_false(), "default_false",
-        do_not_relay: bool = default_false(),
+        do_sanity_checks: bool = default_true(), "default_true",
-        #[cfg_attr(feature = "serde", serde(default = "default_true"))]
-        do_sanity_checks: bool = default_true(),
     },
     AccessResponseBase {
         double_spend: bool,
@@ -167,10 +162,8 @@ define_request_and_response! {
     core_rpc_server_commands_defs.h => 1369..=1417,
     GetPeerList,
     Request {
-        #[cfg_attr(feature = "serde", serde(default = "default_true"))]
+        public_only: bool = default_true(), "default_true",
-        public_only: bool = default_true(),
+        include_blocked: bool = default_false(), "default_false",
-        #[cfg_attr(feature = "serde", serde(default = "default_false"))]
-        include_blocked: bool = default_false(),
     },
     ResponseBase {
         white_list: Vec<Peer>,
@@ -208,8 +201,7 @@ define_request_and_response! {
     core_rpc_server_commands_defs.h => 1494..=1517,
     SetLogCategories,
     Request {
-        #[cfg_attr(feature = "serde", serde(default = "default_string"))]
+        categories: String = default_string(), "default_string",
-        categories: String = default_string(),
     },
     ResponseBase {
         categories: String,
@@ -300,8 +292,7 @@ define_request_and_response! {
     core_rpc_server_commands_defs.h => 1876..=1903,
     OutPeers,
     Request {
-        #[cfg_attr(feature = "serde", serde(default = "default_true"))]
+        set: bool = default_true(), "default_true",
-        set: bool = default_true(),
         out_peers: u32,
     },
     ResponseBase {
@@ -345,8 +336,7 @@ define_request_and_response! {
     Update,
     Request {
         command: String,
-        #[cfg_attr(feature = "serde", serde(default = "default_string"))]
+        path: String = default_string(), "default_string",
-        path: String = default_string(),
     },
     ResponseBase {
         auto_uri: String,
@@ -402,12 +392,9 @@ define_request_and_response! {
     core_rpc_server_commands_defs.h => 1419..=1448,
     GetPublicNodes,
     Request {
-        #[cfg_attr(feature = "serde", serde(default = "default_false"))]
+        gray: bool = default_false(), "default_false",
-        gray: bool = default_false(),
+        white: bool = default_true(), "default_true",
-        #[cfg_attr(feature = "serde", serde(default = "default_true"))]
+        include_blocked: bool = default_false(), "default_false",
-        white: bool = default_true(),
-        #[cfg_attr(feature = "serde", serde(default = "default_false"))]
-        include_blocked: bool = default_false(),
     },
     ResponseBase {
         gray: Vec<PublicNode>,

storage/blockchain/README.md

@@ -5,6 +5,10 @@ This documentation is mostly for practical usage of `cuprate_blockchain`.
 For a high-level overview, see the database section in
 [Cuprate's architecture book](https://architecture.cuprate.org).
 
+If you're looking for a database crate, consider using the lower-level
+[`cuprate-database`](https://doc.cuprate.org/cuprate_database)
+crate that this crate is built on-top of.
+
 # Purpose
 This crate does 3 things:
 1. Uses [`cuprate_database`] as a base database layer
@@ -47,11 +51,11 @@ there are some things that must be kept in mind when doing so.
 Failing to uphold these invariants may cause panics.
 
 1. `LMDB` requires the user to resize the memory map resizing (see [`cuprate_database::RuntimeError::ResizeNeeded`]
-1. `LMDB` has a maximum reader transaction count, currently it is set to `128`
+1. `LMDB` has a maximum reader transaction count, currently, [it is set to `126`](https://github.com/LMDB/lmdb/blob/b8e54b4c31378932b69f1298972de54a565185b1/libraries/liblmdb/mdb.c#L794-L799)
 1. `LMDB` has [maximum key/value byte size](http://www.lmdb.tech/doc/group__internal.html#gac929399f5d93cef85f874b9e9b1d09e0) which must not be exceeded
 
 # Examples
-The below is an example of using `cuprate_blockchain`
+The below is an example of using `cuprate_blockchain`'s
 lowest API, i.e. using a mix of this crate and `cuprate_database`'s traits directly -
 **this is NOT recommended.**
 

storage/blockchain/src/config/reader_threads.rs

@@ -20,12 +20,11 @@ use serde::{Deserialize, Serialize};
 /// This controls how many reader thread `service`'s
 /// thread-pool will spawn to receive and send requests/responses.
 ///
-/// It does nothing outside of `service`.
+/// # Invariant
-///
-/// It will always be at least 1, up until the amount of threads on the machine.
-///
 /// The main function used to extract an actual
 /// usable thread count out of this is [`ReaderThreads::as_threads`].
+///
+/// This will always return at least 1, up until the amount of threads on the machine.
 #[derive(Copy, Clone, Debug, Default, PartialEq, PartialOrd)]
 #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
 pub enum ReaderThreads {
@@ -97,30 +96,30 @@ impl ReaderThreads {
     ///
     /// # Example
     /// ```rust
-    /// use cuprate_blockchain::config::ReaderThreads as Rt;
+    /// use cuprate_blockchain::config::ReaderThreads as R;
     ///
     /// let total_threads: std::num::NonZeroUsize =
     ///     cuprate_helper::thread::threads();
     ///
-    /// assert_eq!(Rt::OnePerThread.as_threads(), total_threads);
+    /// assert_eq!(R::OnePerThread.as_threads(), total_threads);
     ///
-    /// assert_eq!(Rt::One.as_threads().get(), 1);
+    /// assert_eq!(R::One.as_threads().get(), 1);
     ///
-    /// assert_eq!(Rt::Number(0).as_threads(), total_threads);
+    /// assert_eq!(R::Number(0).as_threads(), total_threads);
-    /// assert_eq!(Rt::Number(1).as_threads().get(), 1);
+    /// assert_eq!(R::Number(1).as_threads().get(), 1);
-    /// assert_eq!(Rt::Number(usize::MAX).as_threads(), total_threads);
+    /// assert_eq!(R::Number(usize::MAX).as_threads(), total_threads);
     ///
-    /// assert_eq!(Rt::Percent(0.01).as_threads().get(), 1);
+    /// assert_eq!(R::Percent(0.01).as_threads().get(), 1);
-    /// assert_eq!(Rt::Percent(0.0).as_threads(), total_threads);
+    /// assert_eq!(R::Percent(0.0).as_threads(), total_threads);
-    /// assert_eq!(Rt::Percent(1.0).as_threads(), total_threads);
+    /// assert_eq!(R::Percent(1.0).as_threads(), total_threads);
-    /// assert_eq!(Rt::Percent(f32::NAN).as_threads(), total_threads);
+    /// assert_eq!(R::Percent(f32::NAN).as_threads(), total_threads);
-    /// assert_eq!(Rt::Percent(f32::INFINITY).as_threads(), total_threads);
+    /// assert_eq!(R::Percent(f32::INFINITY).as_threads(), total_threads);
-    /// assert_eq!(Rt::Percent(f32::NEG_INFINITY).as_threads(), total_threads);
+    /// assert_eq!(R::Percent(f32::NEG_INFINITY).as_threads(), total_threads);
     ///
     /// // Percentage only works on more than 1 thread.
     /// if total_threads.get() > 1 {
     ///     assert_eq!(
-    ///         Rt::Percent(0.5).as_threads().get(),
+    ///         R::Percent(0.5).as_threads().get(),
     ///         (total_threads.get() as f32 / 2.0) as usize,
     ///     );
     /// }

storage/blockchain/src/free.rs

@@ -6,7 +6,7 @@ use cuprate_database::{ConcreteEnv, Env, EnvInner, InitError, RuntimeError, TxRw
 use crate::{config::Config, tables::OpenTables};
 
 //---------------------------------------------------------------------------------------------------- Free functions
-/// Open the blockchain database, using the passed [`Config`].
+/// Open the blockchain database using the passed [`Config`].
 ///
 /// This calls [`cuprate_database::Env::open`] and prepares the
 /// database to be ready for blockchain-related usage, e.g.

storage/blockchain/src/ops/block.rs

@@ -1,4 +1,4 @@
-//! Blocks functions.
+//! Block functions.
 
 //---------------------------------------------------------------------------------------------------- Import
 use bytemuck::TransparentWrapper;

storage/blockchain/src/ops/mod.rs

@@ -5,14 +5,14 @@
 //! database operations.
 //!
 //! # `impl Table`
-//! `ops/` functions take [`Tables`](crate::tables::Tables) and
+//! Functions in this module take [`Tables`](crate::tables::Tables) and
 //! [`TablesMut`](crate::tables::TablesMut) directly - these are
 //! _already opened_ database tables.
 //!
-//! As such, the function puts the responsibility
+//! As such, the responsibility of
-//! of transactions, tables, etc on the caller.
+//! transactions, tables, etc, are on the caller.
 //!
-//! This does mean these functions are mostly as lean
+//! Notably, this means that these functions are as lean
 //! as possible, so calling them in a loop should be okay.
 //!
 //! # Atomicity

storage/database/README.md

@@ -6,10 +6,10 @@ For a high-level overview, see the database section in
 [Cuprate's architecture book](https://architecture.cuprate.org).
 
 If you need blockchain specific capabilities, consider using the higher-level
-`cuprate-blockchain` crate which builds upon this one.
+[`cuprate-blockchain`](https://doc.cuprate.org/cuprate_blockchain) crate which builds upon this one.
 
 # Purpose
-This crate abstracts various database backends with traits. The databases are:
+This crate abstracts various database backends with traits.
 
 All backends have the following attributes:
 - [Embedded](https://en.wikipedia.org/wiki/Embedded_database)
@@ -19,6 +19,10 @@ All backends have the following attributes:
 - Are table oriented (`"table_name" -> (key, value)`)
 - Allows concurrent readers
 
+The currently implemented backends are:
+- [`heed`](https://github.com/meilisearch/heed) (LMDB)
+- [`redb`](https://github.com/cberner/redb)
+
 # Terminology
 To be more clear on some terms used in this crate:
 
@@ -26,17 +30,17 @@ To be more clear on some terms used in this crate:
 |------------------|--------------------------------------|
 | `Env`            | The 1 database environment, the "whole" thing
 | `DatabaseR{o,w}` | A _actively open_ readable/writable `key/value` store
-| `Table`          | Solely the metadata of a `cuprate_database` (the `key` and `value` types, and the name)
+| `Table`          | Solely the metadata of a `Database` (the `key` and `value` types, and the name)
 | `TxR{o,w}`       | A read/write transaction
-| `Storable`       | A data that type can be stored in the database
+| `Storable`       | A data type that can be stored in the database
 
-The dataflow is `Env` -> `Tx` -> `cuprate_database`
+The flow is `Env` -> `Tx` -> `Database`
 
 Which reads as:
 1. You have a database `Environment`
 1. You open up a `Transaction`
-1. You open a particular `Table` from that `Environment`, getting a `cuprate_database`
+1. You open a particular `Table` from that `Environment`, getting a `Database`
-1. You can now read/write data from/to that `cuprate_database`
+1. You can now read/write data from/to that `Database`
 
 # Concrete types
 You should _not_ rely on the concrete type of any abstracted backend.

storage/database/src/config/config.rs

@@ -160,7 +160,7 @@ pub struct Config {
     /// Set the number of slots in the reader table.
     ///
     /// This is only used in LMDB, see
-    /// <https://github.com/LMDB/lmdb/blob/b8e54b4c31378932b69f1298972de54a565185b1/libraries/liblmdb/mdb.c#L794-L799>.
+    /// [here](https://github.com/LMDB/lmdb/blob/b8e54b4c31378932b69f1298972de54a565185b1/libraries/liblmdb/mdb.c#L794-L799).
     ///
     /// By default, this value is [`READER_THREADS_DEFAULT`].
     pub reader_threads: NonZeroUsize,

storage/database/src/config/sync_mode.rs

@@ -127,8 +127,8 @@ pub enum SyncMode {
     /// In the case of a system crash, the database
     /// may become corrupted when using this option.
     ///
-    /// [^1]:
+    ///
-    /// Semantically, this variant would actually map to
+    /// [^1]: Semantically, this variant would actually map to
     /// [`redb::Durability::None`](https://docs.rs/redb/1.5.0/redb/enum.Durability.html#variant.None),
     /// however due to [`#149`](https://github.com/Cuprate/cuprate/issues/149),
     /// this is not possible. As such, when using the `redb` backend,

storage/database/src/env.rs

@@ -24,15 +24,14 @@ use crate::{
 ///
 /// # Lifetimes
 /// The lifetimes associated with `Env` have a sequential flow:
-/// 1. `ConcreteEnv`
+/// ```text
-/// 2. `'env`
+/// Env -> Tx -> Database
-/// 3. `'tx`
+/// ```
-/// 4. `'db`
 ///
 /// As in:
 /// - open database tables only live as long as...
 /// - transactions which only live as long as the...
-/// - environment ([`EnvInner`])
+/// - database environment
 pub trait Env: Sized {
     //------------------------------------------------ Constants
     /// Does the database backend need to be manually
@@ -202,18 +201,19 @@ Subsequent table opens will follow the flags/ordering, but only if
 /// Note that when opening tables with [`EnvInner::open_db_ro`],
 /// they must be created first or else it will return error.
 ///
-/// Note that when opening tables with [`EnvInner::open_db_ro`],
-/// they must be created first or else it will return error.
-///
 /// See [`EnvInner::create_db`] for creating tables.
 ///
 /// # Invariant
 #[doc = doc_heed_create_db_invariant!()]
 pub trait EnvInner<'env> {
     /// The read-only transaction type of the backend.
-    type Ro<'a>: TxRo<'a>;
+    ///
+    /// `'tx` is the lifetime of the transaction itself.
+    type Ro<'tx>: TxRo<'tx>;
     /// The read-write transaction type of the backend.
-    type Rw<'a>: TxRw<'a>;
+    ///
+    /// `'tx` is the lifetime of the transaction itself.
+    type Rw<'tx>: TxRw<'tx>;
 
     /// Create a read-only transaction.
     ///
@@ -235,11 +235,37 @@ pub trait EnvInner<'env> {
     /// This will open the database [`Table`]
     /// passed as a generic to this function.
     ///
-    /// ```rust,ignore
+    /// ```rust
-    /// let db = env.open_db_ro::<Table>(&tx_ro);
+    /// # use cuprate_database::{
-    /// //  ^                     ^
+    /// #     ConcreteEnv,
-    /// // database table       table metadata
+    /// #     config::ConfigBuilder,
-    /// //                      (name, key/value type)
+    /// #     Env, EnvInner,
+    /// #     DatabaseRo, DatabaseRw, TxRo, TxRw,
+    /// # };
+    /// # fn main() -> Result<(), Box<dyn std::error::Error>> {
+    /// # let tmp_dir = tempfile::tempdir()?;
+    /// # let db_dir = tmp_dir.path().to_owned();
+    /// # let config = ConfigBuilder::new(db_dir.into()).build();
+    /// # let env = ConcreteEnv::open(config)?;
+    /// #
+    /// # struct Table;
+    /// # impl cuprate_database::Table for Table {
+    /// #     const NAME: &'static str = "table";
+    /// #     type Key = u8;
+    /// #     type Value = u64;
+    /// # }
+    /// #
+    /// # let env_inner = env.env_inner();
+    /// # let tx_rw = env_inner.tx_rw()?;
+    /// # env_inner.create_db::<Table>(&tx_rw)?;
+    /// # TxRw::commit(tx_rw);
+    /// #
+    /// # let tx_ro = env_inner.tx_ro()?;
+    /// let db = env_inner.open_db_ro::<Table>(&tx_ro);
+    /// //  ^                           ^
+    /// // database table             table metadata
+    /// //                            (name, key/value type)
+    /// # Ok(()) }
     /// ```
     ///
     /// # Errors

storage/database/src/error.rs

@@ -59,18 +59,16 @@ pub enum InitError {
 }
 
 //---------------------------------------------------------------------------------------------------- RuntimeError
-/// Errors that occur _after_ successful ([`Env::open`](crate::env::Env::open)).
+/// Errors that occur _after_ successful [`Env::open`](crate::env::Env::open).
 ///
 /// There are no errors for:
 /// 1. Missing tables
 /// 2. (De)serialization
-/// 3. Shutdown errors
 ///
 /// as `cuprate_database` upholds the invariant that:
 ///
 /// 1. All tables exist
 /// 2. (De)serialization never fails
-/// 3. The database (thread-pool) only shuts down when all channels are dropped
 #[derive(thiserror::Error, Debug)]
 pub enum RuntimeError {
     /// The given key already existed in the database.

storage/database/src/storable.rs

@@ -129,7 +129,7 @@ where
 ///
 /// Slice types are owned both:
 /// - when returned from the database
-/// - in `put()`
+/// - in [`crate::DatabaseRw::put()`]
 ///
 /// This is needed as `impl Storable for Vec<T>` runs into impl conflicts.
 ///

storage/database/src/table.rs

@@ -8,6 +8,8 @@ use crate::{key::Key, storable::Storable};
 /// Database table metadata.
 ///
 /// Purely compile time information for database tables.
+///
+/// See [`crate::define_tables`] for bulk table generation.
 pub trait Table: 'static {
     /// Name of the database table.
     const NAME: &'static str;

storage/database/src/tables.rs

@@ -349,11 +349,18 @@ macro_rules! define_tables {
         /// Note that this is already implemented on [`cuprate_database::EnvInner`], thus:
         /// - You don't need to implement this
         /// - It can be called using `env_inner.open_tables()` notation
+        ///
+        /// # Creation before opening
+        /// As [`cuprate_database::EnvInner`] documentation states,
+        /// tables must be created before they are opened.
+        ///
+        /// I.e. [`OpenTables::create_tables`] must be called before
+        /// [`OpenTables::open_tables`] or else panics may occur.
         pub trait OpenTables<'env> {
             /// The read-only transaction type of the backend.
-            type Ro<'a>;
+            type Ro<'tx>;
             /// The read-write transaction type of the backend.
-            type Rw<'a>;
+            type Rw<'tx>;
 
             /// Open all tables in read/iter mode.
             ///
@@ -362,11 +369,6 @@ macro_rules! define_tables {
             ///
             /// # Errors
             /// This will only return [`cuprate_database::RuntimeError::Io`] if it errors.
-            ///
-            /// # Invariant
-            /// All tables should be created with a crate-specific open function.
-            ///
-            /// TODO: explain why
             fn open_tables(&self, tx_ro: &Self::Ro<'_>) -> Result<impl TablesIter, $crate::RuntimeError>;
 
             /// Open all tables in read-write mode.
@@ -391,8 +393,8 @@ macro_rules! define_tables {
         where
             Ei: $crate::EnvInner<'env>,
         {
-            type Ro<'a> = <Ei as $crate::EnvInner<'env>>::Ro<'a>;
+            type Ro<'tx> = <Ei as $crate::EnvInner<'env>>::Ro<'tx>;
-            type Rw<'a> = <Ei as $crate::EnvInner<'env>>::Rw<'a>;
+            type Rw<'tx> = <Ei as $crate::EnvInner<'env>>::Rw<'tx>;
 
             fn open_tables(&self, tx_ro: &Self::Ro<'_>) -> Result<impl TablesIter, $crate::RuntimeError> {
                 Ok(($(

storage/database/src/transaction.rs

@@ -11,7 +11,7 @@ use crate::error::RuntimeError;
 /// # Commit
 /// It's recommended but may not be necessary to call [`TxRo::commit`] in certain cases:
 /// - <https://docs.rs/heed/0.20.0-alpha.9/heed/struct.RoTxn.html#method.commit>
-pub trait TxRo<'env> {
+pub trait TxRo<'tx> {
     /// Commit the read-only transaction.
     ///
     /// # Errors
@@ -23,7 +23,7 @@ pub trait TxRo<'env> {
 /// Read/write database transaction.
 ///
 /// Returned from [`EnvInner::tx_rw`](crate::EnvInner::tx_rw).
-pub trait TxRw<'env> {
+pub trait TxRw<'tx> {
     /// Commit the read/write transaction.
     ///
     /// Note that this doesn't necessarily sync the database caches to disk.