From fb3d41ccbb51af54b72151fe60325d8bca8d095e Mon Sep 17 00:00:00 2001 From: hinto-janai Date: Sun, 5 May 2024 10:21:28 -0400 Subject: [PATCH] database: final docs + cleanup (#117) * re-apply 'main' merge + doc patches * fix redb lints * update readme * add `lib.rs, ops, service` doc-test examples * docs for `config`, `ops`, add doc-tests * remove merge error incorrect leftover code from previous merge * doc top-level types * docs: error, tables, types * misc docs, TODO, FIXME, SOMEDAY fixes * change clippy lints * tests: add `tables_are_sorted()` * move `tables_are_sorted()` test to `backend/tests.rs` * readme formatting * small fixes * readme fixes * docs: `helper/` * docs: `types/` * database/README.md fixes * doc fixes * types: doc fixes * fixes * all review changes --- database/README.md | 365 ++++++++++++++++------- database/src/backend/heed/database.rs | 10 +- database/src/backend/heed/env.rs | 81 +++-- database/src/backend/heed/error.rs | 4 +- database/src/backend/heed/storable.rs | 8 +- database/src/backend/heed/transaction.rs | 2 +- database/src/backend/mod.rs | 9 - database/src/backend/redb/database.rs | 8 +- database/src/backend/redb/env.rs | 56 ++-- database/src/backend/redb/error.rs | 12 +- database/src/backend/redb/storable.rs | 2 +- database/src/backend/redb/transaction.rs | 2 - database/src/backend/redb/types.rs | 2 +- database/src/backend/tests.rs | 69 ++++- database/src/config/backend.rs | 8 +- database/src/config/config.rs | 246 +++++++++------ database/src/config/mod.rs | 41 ++- database/src/config/reader_threads.rs | 20 +- database/src/config/sync_mode.rs | 49 ++- database/src/constants.rs | 12 +- database/src/database.rs | 125 ++++---- database/src/env.rs | 101 ++++--- database/src/error.rs | 9 +- database/src/key.rs | 92 +----- database/src/lib.rs | 101 ++++--- database/src/macros.rs | 17 -- database/src/ops/alt_block.rs | 29 -- database/src/ops/block.rs | 66 ++-- database/src/ops/blockchain.rs | 34 +-- database/src/ops/key_image.rs | 29 +- database/src/ops/macros.rs | 2 +- database/src/ops/mod.rs | 69 ++++- database/src/ops/output.rs | 38 +-- database/src/ops/property.rs | 38 +-- database/src/ops/tx.rs | 42 +-- database/src/resize.rs | 14 +- database/src/service/free.rs | 14 +- database/src/service/mod.rs | 60 +++- database/src/service/read.rs | 29 +- database/src/service/tests.rs | 40 +-- database/src/service/write.rs | 15 +- database/src/storable.rs | 17 +- database/src/table.rs | 3 +- database/src/tables.rs | 103 +++++-- database/src/tests.rs | 21 +- database/src/transaction.rs | 23 +- database/src/types.rs | 109 ++++--- database/src/unsafe_sendable.rs | 3 +- helper/src/fs.rs | 30 +- types/README.md | 11 +- types/src/lib.rs | 10 +- types/src/service.rs | 134 +++++++-- types/src/types.rs | 109 ++++--- 53 files changed, 1449 insertions(+), 1094 deletions(-) delete mode 100644 database/src/macros.rs delete mode 100644 database/src/ops/alt_block.rs diff --git a/database/README.md b/database/README.md index 6d47d2b6..7aa59cef 100644 --- a/database/README.md +++ b/database/README.md @@ -1,33 +1,34 @@ # Database Cuprate's database implementation. - -1. [Documentation](#documentation) -1. [File Structure](#file-structure) - - [`src/`](#src) - - [`src/ops`](#src-ops) - - [`src/service/`](#src-service) - - [`src/backend/`](#src-backend) -1. [Backends](#backends) - - [`heed`](#heed) - - [`redb`](#redb) - - [`redb-memory`](#redb-memory) - - [`sanakirja`](#sanakirja) - - [`MDBX`](#mdbx) -1. [Layers](#layers) - - [Database](#database) - - [Trait](#trait) - - [ConcreteEnv](#concreteenv) - - [Thread-pool](#thread-pool) - - [Service](#service) -1. [Resizing](#resizing) -1. [Flushing](#flushing) -1. [(De)serialization](#deserialization) +- [1. Documentation](#1-documentation) +- [2. File Structure](#2-file-structure) + - [2.1 `src/`](#21-src) + - [2.2 `src/backend/`](#22-srcbackend) + - [2.3 `src/config`](#23-srcconfig) + - [2.4 `src/ops`](#24-srcops) + - [2.5 `src/service/`](#25-srcservice) +- [3. Backends](#3-backends) + - [3.1 heed](#31-heed) + - [3.2 redb](#32-redb) + - [3.3 redb-memory](#33-redb-memory) + - [3.4 sanakirja](#34-sanakirja) + - [3.5 MDBX](#35-mdbx) +- [4. Layers](#4-layers) + - [4.1 Backend](#41-backend) + - [4.2 Trait](#42-trait) + - [4.3 ConcreteEnv](#43-concreteenv) + - [4.4 `ops`](#44-ops) + - [4.5 `service`](#45-service) +- [5. Syncing](#5-Syncing) +- [6. Thread model](#6-thread-model) +- [7. Resizing](#7-resizing) +- [8. (De)serialization](#8-deserialization) --- -# Documentation -In general, documentation for `database/` is split into 3: +## 1. Documentation +Documentation for `database/` is split into 3 locations: | Documentation location | Purpose | |---------------------------|---------| @@ -59,65 +60,41 @@ The code within `src/` is also littered with some `grep`-able comments containin | `TODO` | This must be implemented; There should be 0 of these in production code | `SOMEDAY` | This should be implemented... someday -# File Structure +## 2. File Structure A quick reference of the structure of the folders & files in `cuprate-database`. Note that `lib.rs/mod.rs` files are purely for re-exporting/visibility/lints, and contain no code. Each sub-directory has a corresponding `mod.rs`. -## `src/` +### 2.1 `src/` The top-level `src/` files. -| File | Purpose | -|---------------------|---------| -| `config.rs` | Database `Env` configuration -| `constants.rs` | General constants used throughout `cuprate-database` -| `database.rs` | Abstracted database; `trait DatabaseR{o,w}` -| `env.rs` | Abstracted database environment; `trait Env` -| `error.rs` | Database error types -| `free.rs` | General free functions (related to the database) -| `key.rs` | Abstracted database keys; `trait Key` -| `resize.rs` | Database resizing algorithms -| `storable.rs` | Data (de)serialization; `trait Storable` -| `table.rs` | Database table abstraction; `trait Table` -| `tables.rs` | All the table definitions used by `cuprate-database` -| `transaction.rs` | Database transaction abstraction; `trait TxR{o,w}` -| `types.rs` | Database table schema types +| File | Purpose | +|------------------------|---------| +| `constants.rs` | General constants used throughout `cuprate-database` +| `database.rs` | Abstracted database; `trait DatabaseR{o,w}` +| `env.rs` | Abstracted database environment; `trait Env` +| `error.rs` | Database error types +| `free.rs` | General free functions (related to the database) +| `key.rs` | Abstracted database keys; `trait Key` +| `resize.rs` | Database resizing algorithms +| `storable.rs` | Data (de)serialization; `trait Storable` +| `table.rs` | Database table abstraction; `trait Table` +| `tables.rs` | All the table definitions used by `cuprate-database` +| `tests.rs` | Utilities for `cuprate_database` testing +| `transaction.rs` | Database transaction abstraction; `trait TxR{o,w}` +| `types.rs` | Database-specific types +| `unsafe_unsendable.rs` | Marker type to impl `Send` for objects not `Send` -## `src/ops/` -This folder contains the `cupate_database::ops` module. - -TODO: more detailed descriptions. - -| File | Purpose | -|-----------------|---------| -| `alt_block.rs` | Alternative blocks -| `block.rs` | Blocks -| `blockchain.rs` | Blockchain-related -| `output.rs` | Outputs -| `property.rs` | Properties -| `spent_key.rs` | Spent keys -| `tx.rs` | Transactions - -## `src/service/` -This folder contains the `cupate_database::service` module. - -| File | Purpose | -|----------------|---------| -| `free.rs` | General free functions used (related to `cuprate_database::service`) -| `read.rs` | Read thread-pool definitions and logic -| `tests.rs` | Thread-pool tests and test helper functions -| `types.rs` | `cuprate_database::service`-related type aliases -| `write.rs` | Write thread-pool definitions and logic - -## `src/backend/` -This folder contains the actual database crates used as the backend for `cuprate-database`. +### 2.2 `src/backend/` +This folder contains the implementation for actual databases used as the backend for `cuprate-database`. Each backend has its own folder. -| Folder | Purpose | -|--------------|---------| -| `heed/` | Backend using using forked [`heed`](https://github.com/Cuprate/heed) -| `sanakirja/` | Backend using [`sanakirja`](https://docs.rs/sanakirja) +| Folder/File | Purpose | +|-------------|---------| +| `heed/` | Backend using using [`heed`](https://github.com/meilisearch/heed) (LMDB) +| `redb/` | Backend using [`redb`](https://github.com/cberner/redb) +| `tests.rs` | Backend-agnostic tests All backends follow the same file structure: @@ -127,18 +104,56 @@ All backends follow the same file structure: | `env.rs` | Implementation of `trait Env` | `error.rs` | Implementation of backend's errors to `cuprate_database`'s error types | `storable.rs` | Compatibility layer between `cuprate_database::Storable` and backend-specific (de)serialization -| `tests.rs` | Tests for the specific backend | `transaction.rs` | Implementation of `trait TxR{o,w}` | `types.rs` | Type aliases for long backend-specific types -# Backends -`cuprate-database`'s `trait`s abstract over various actual databases. +### 2.3 `src/config/` +This folder contains the `cupate_database::config` module; configuration options for the database. -Each database's implementation is located in its respective file in `src/backend/${DATABASE_NAME}.rs`. +| File | Purpose | +|---------------------|---------| +| `config.rs` | Main database `Config` struct +| `reader_threads.rs` | Reader thread configuration for `service` thread-pool +| `sync_mode.rs` | Disk sync configuration for backends -## `heed` +### 2.4 `src/ops/` +This folder contains the `cupate_database::ops` module. + +These are higher-level functions abstracted over the database, that are Monero-related. + +| File | Purpose | +|-----------------|---------| +| `block.rs` | Block related (main functions) +| `blockchain.rs` | Blockchain related (height, cumulative values, etc) +| `key_image.rs` | Key image related +| `macros.rs` | Macros specific to `ops/` +| `output.rs` | Output related +| `property.rs` | Database properties (pruned, version, etc) +| `tx.rs` | Transaction related + +### 2.5 `src/service/` +This folder contains the `cupate_database::service` module. + +The `async`hronous request/response API other Cuprate crates use instead of managing the database directly themselves. + +| File | Purpose | +|----------------|---------| +| `free.rs` | General free functions used (related to `cuprate_database::service`) +| `read.rs` | Read thread-pool definitions and logic +| `tests.rs` | Thread-pool tests and test helper functions +| `types.rs` | `cuprate_database::service`-related type aliases +| `write.rs` | Writer thread definitions and logic + +## 3. Backends +`cuprate-database`'s `trait`s allow abstracting over the actual database, such that any backend in particular could be used. + +Each database's implementation for those `trait`'s are located in its respective folder in `src/backend/${DATABASE_NAME}/`. + +### 3.1 heed The default database used is [`heed`](https://github.com/meilisearch/heed) (LMDB). +The upstream versions from [`crates.io`](https://crates.io/crates/heed) are used. + `LMDB` should not need to be installed as `heed` has a build script that pulls it in automatically. `heed`'s filenames inside Cuprate's database folder (`~/.local/share/cuprate/database/`) are: @@ -148,11 +163,11 @@ The default database used is [`heed`](https://github.com/meilisearch/heed) (LMDB | `data.mdb` | Main data file | `lock.mdb` | Database lock file -TODO: document max readers limit: https://github.com/monero-project/monero/blob/059028a30a8ae9752338a7897329fe8012a310d5/src/blockchain_db/lmdb/db_lmdb.cpp#L1372. Other potential processes (e.g. `xmrblocks`) that are also reading the `data.mdb` file need to be accounted for. +`heed`-specific notes: +- [There is a maximum reader limit](https://github.com/monero-project/monero/blob/059028a30a8ae9752338a7897329fe8012a310d5/src/blockchain_db/lmdb/db_lmdb.cpp#L1372). Other potential processes (e.g. `xmrblocks`) that are also reading the `data.mdb` file need to be accounted for. +- [LMDB does not work on remote filesystem](https://github.com/LMDB/lmdb/blob/b8e54b4c31378932b69f1298972de54a565185b1/libraries/liblmdb/lmdb.h#L129). -TODO: document DB on remote filesystem: https://github.com/LMDB/lmdb/blob/b8e54b4c31378932b69f1298972de54a565185b1/libraries/liblmdb/lmdb.h#L129. - -## `redb` +### 3.2 redb The 2nd database backend is the 100% Rust [`redb`](https://github.com/cberner/redb). The upstream versions from [`crates.io`](https://crates.io/crates/redb) are used. @@ -163,45 +178,187 @@ The upstream versions from [`crates.io`](https://crates.io/crates/redb) are used |-------------|---------| | `data.redb` | Main data file -TODO: document DB on remote filesystem (does redb allow this?) + -## `redb-memory` +### 3.3 redb-memory This backend is 100% the same as `redb`, although, it uses `redb::backend::InMemoryBackend` which is a key-value store that completely resides in memory instead of a file. All other details about this should be the same as the normal `redb` backend. -## `sanakirja` +### 3.4 sanakirja [`sanakirja`](https://docs.rs/sanakirja) was a candidate as a backend, however there were problems with maximum value sizes. The default maximum value size is [1012 bytes](https://docs.rs/sanakirja/1.4.1/sanakirja/trait.Storable.html) which was too small for our requirements. Using [`sanakirja::Slice`](https://docs.rs/sanakirja/1.4.1/sanakirja/union.Slice.html) and [sanakirja::UnsizedStorage](https://docs.rs/sanakirja/1.4.1/sanakirja/trait.UnsizedStorable.html) was attempted, but there were bugs found when inserting a value in-between `512..=4096` bytes. As such, it is not implemented. -## `MDBX` +### 3.5 MDBX [`MDBX`](https://erthink.github.io/libmdbx) was a candidate as a backend, however MDBX deprecated the custom key/value comparison functions, this makes it a bit trickier to implement duplicate tables. It is also quite similar to the main backend LMDB (of which it was originally a fork of). As such, it is not implemented (yet). -# Layers -TODO: update with accurate information when ready, update image. +## 4. Layers +`cuprate_database` is logically abstracted into 5 layers, starting from the lowest: +1. Backend +2. Trait +3. ConcreteEnv +4. `ops` +5. `service` -## Database -## Trait -## ConcreteEnv -## Thread -## Service +Each layer is built upon the last. -# Resizing -TODO: document resize algorithm: -- Exactly when it occurs -- How much bytes are added + -All backends follow the same algorithm. +### 4.1 Backend +This is the actual database backend implementation (or a Rust shim over one). -# Flushing -TODO: document disk flushing behavior. -- Config options -- Backend-specific behavior +Examples: +- `heed` (LMDB) +- `redb` -# (De)serialization -TODO: document `Storable` and how databases (de)serialize types when storing/fetching. +`cuprate_database` itself just uses a backend, it does not implement one. + +All backends have the following attributes: +- [Embedded](https://en.wikipedia.org/wiki/Embedded_database) +- [Multiversion concurrency control](https://en.wikipedia.org/wiki/Multiversion_concurrency_control) +- [ACID](https://en.wikipedia.org/wiki/ACID) +- Are `(key, value)` oriented and have the expected API (`get()`, `insert()`, `delete()`) +- Are table oriented (`"table_name" -> (key, value)`) +- Allows concurrent readers + +### 4.2 Trait +`cuprate_database` provides a set of `trait`s that abstract over the various database backends. + +This allows the function signatures and behavior to stay the same but allows for swapping out databases in an easier fashion. + +All common behavior of the backend's are encapsulated here and used instead of using the backend directly. + +Examples: +- [`trait Env`](https://github.com/Cuprate/cuprate/blob/2ac90420c658663564a71b7ecb52d74f3c2c9d0f/database/src/env.rs) +- [`trait {TxRo, TxRw}`](https://github.com/Cuprate/cuprate/blob/2ac90420c658663564a71b7ecb52d74f3c2c9d0f/database/src/transaction.rs) +- [`trait {DatabaseRo, DatabaseRw}`](https://github.com/Cuprate/cuprate/blob/2ac90420c658663564a71b7ecb52d74f3c2c9d0f/database/src/database.rs) + +For example, instead of calling `LMDB` or `redb`'s `get()` function directly, `DatabaseRo::get()` is called. + +### 4.3 ConcreteEnv +This is the non-generic, concrete `struct` provided by `cuprate_database` that contains all the data necessary to operate the database. The actual database backend `ConcreteEnv` will use internally depends on which backend feature is used. + +`ConcreteEnv` implements `trait Env`, which opens the door to all the other traits. + +The equivalent objects in the backends themselves are: +- [`heed::Env`](https://docs.rs/heed/0.20.0/heed/struct.Env.html) +- [`redb::Database`](https://docs.rs/redb/2.1.0/redb/struct.Database.html) + +This is the main object used when handling the database directly, although that is not strictly necessary as a user if the `service` layer is used. + +### 4.4 `ops` +These are Monero-specific functions that use the abstracted `trait` forms of the database. + +Instead of dealing with the database directly (`get()`, `delete()`), the `ops` layer provides more abstract functions that deal with commonly used Monero operations (`add_block()`, `pop_block()`). + +### 4.5 `service` +The final layer abstracts the database completely into a [Monero-specific `async` request/response API](https://github.com/Cuprate/cuprate/blob/2ac90420c658663564a71b7ecb52d74f3c2c9d0f/types/src/service.rs#L18-L78), using [`tower::Service`](https://docs.rs/tower/latest/tower/trait.Service.html). + +It handles the database using a separate writer thread & reader thread-pool, and uses the previously mentioned `ops` functions when responding to requests. + +Instead of handling the database directly, this layer provides read/write handles that allow: +- Sending requests for data (e.g. Outputs) +- Receiving responses + +For more information on the backing thread-pool, see [`Thread model`](#6-thread-model). + +## 5. Syncing +`cuprate_database`'s database has 5 disk syncing modes. + +1. FastThenSafe +1. Safe +1. Async +1. Threshold +1. Fast + +The default mode is `Safe`. + +This means that upon each transaction commit, all the data that was written will be fully synced to disk. This is the slowest, but safest mode of operation. + +Note that upon any database `Drop`, whether via `service` or dropping the database directly, the current implementation will sync to disk regardless of any configuration. + +For more information on the other modes, read the documentation [here](https://github.com/Cuprate/cuprate/blob/2ac90420c658663564a71b7ecb52d74f3c2c9d0f/database/src/config/sync_mode.rs#L63-L144). + +## 6. Thread model +As noted in the [`Layers`](#layers) section, the base database abstractions themselves are not concerned with parallelism, they are mostly functions to be called from a single-thread. + +However, the actual API `cuprate_database` exposes for practical usage for the main `cuprated` binary (and other `async` use-cases) is the asynchronous `service` API, which _does_ have a thread model backing it. + +As such, when [`cuprate_database::service`'s initialization function](https://github.com/Cuprate/cuprate/blob/9c27ba5791377d639cb5d30d0f692c228568c122/database/src/service/free.rs#L33-L44) is called, threads will be spawned and maintained until the user drops (disconnects) the returned handles. + +The current behavior is: +- [1 writer thread](https://github.com/Cuprate/cuprate/blob/9c27ba5791377d639cb5d30d0f692c228568c122/database/src/service/write.rs#L52-L66) +- [As many reader threads as there are system threads](https://github.com/Cuprate/cuprate/blob/9c27ba5791377d639cb5d30d0f692c228568c122/database/src/service/read.rs#L104-L126) + +For example, on a system with 32-threads, `cuprate_database` will spawn: +- 1 writer thread +- 32 reader threads + +whose sole responsibility is to listen for database requests, access the database (potentially in parallel), and return a response. + +Note that the `1 system thread = 1 reader thread` model is only the default setting, the reader thread count can be configured by the user to be any number between `1 .. amount_of_system_threads`. + +The reader threads are managed by [`rayon`](https://docs.rs/rayon). + +For an example of where multiple reader threads are used: given a request that asks if any key-image within a set already exists, `cuprate_database` will [split that work between the threads with `rayon`](https://github.com/Cuprate/cuprate/blob/9c27ba5791377d639cb5d30d0f692c228568c122/database/src/service/read.rs#L490-L503). + +Once the [handles](https://github.com/Cuprate/cuprate/blob/9c27ba5791377d639cb5d30d0f692c228568c122/database/src/service/free.rs#L33) to these threads are `Drop`ed, the backing thread(pool) will gracefully exit, automatically. + +## 7. Resizing +Database backends that require manually resizing will, by default, use a similar algorithm as `monerod`'s. + +Note that this only relates to the `service` module, where the database is handled by `cuprate_database` itself, not the user. In the case of a user directly using `cuprate_database`, it is up to them on how to resize. + +Within `service`, the resizing logic defined [here](https://github.com/Cuprate/cuprate/blob/2ac90420c658663564a71b7ecb52d74f3c2c9d0f/database/src/service/write.rs#L139-L201) does the following: + +- If there's not enough space to fit a write request's data, start a resize +- Each resize adds around [`1_073_745_920`](https://github.com/Cuprate/cuprate/blob/2ac90420c658663564a71b7ecb52d74f3c2c9d0f/database/src/resize.rs#L104-L160) bytes to the current map size +- A resize will be attempted `3` times before failing + +There are other [resizing algorithms](https://github.com/Cuprate/cuprate/blob/2ac90420c658663564a71b7ecb52d74f3c2c9d0f/database/src/resize.rs#L38-L47) that define how the database's memory map grows, although currently the behavior of [`monerod`](https://github.com/Cuprate/cuprate/blob/2ac90420c658663564a71b7ecb52d74f3c2c9d0f/database/src/resize.rs#L104-L160) is closely followed. + +## 8. (De)serialization +All types stored inside the database are either bytes already, or are perfectly bitcast-able. + +As such, they do not incur heavy (de)serialization costs when storing/fetching them from the database. The main (de)serialization used is [`bytemuck`](https://docs.rs/bytemuck)'s traits and casting functions. + +Note that the data stored in the tables are still type-safe; we still refer to the key and values within our tables by the type. + +The main deserialization `trait` for database storage is: [`cuprate_database::Storable`](https://github.com/Cuprate/cuprate/blob/2ac90420c658663564a71b7ecb52d74f3c2c9d0f/database/src/storable.rs#L16-L115). + +- Before storage, the type is [simply cast into bytes](https://github.com/Cuprate/cuprate/blob/2ac90420c658663564a71b7ecb52d74f3c2c9d0f/database/src/storable.rs#L125) +- When fetching, the bytes are [simply cast into the type](https://github.com/Cuprate/cuprate/blob/2ac90420c658663564a71b7ecb52d74f3c2c9d0f/database/src/storable.rs#L130) + +When a type is casted into bytes, [the reference is casted](https://docs.rs/bytemuck/latest/bytemuck/fn.bytes_of.html), i.e. this is zero-cost serialization. + +However, it is worth noting that when bytes are casted into the type, [it is copied](https://docs.rs/bytemuck/latest/bytemuck/fn.pod_read_unaligned.html). This is due to byte alignment guarantee issues with both backends, see: +- https://github.com/AltSysrq/lmdb-zero/issues/8 +- https://github.com/cberner/redb/issues/360 + +Without this, `bytemuck` will panic with [`TargetAlignmentGreaterAndInputNotAligned`](https://docs.rs/bytemuck/latest/bytemuck/enum.PodCastError.html#variant.TargetAlignmentGreaterAndInputNotAligned) when casting. + +Copying the bytes fixes this problem, although it is more costly than necessary. However, in the main use-case for `cuprate_database` (the `service` module) the bytes would need to be owned regardless as the `Request/Response` API uses owned data types (`T`, `Vec`, `HashMap`, etc). + +Practically speaking, this means lower-level database functions that normally look like such: +```rust +fn get(key: &Key) -> &Value; +``` +end up looking like this in `cuprate_database`: +```rust +fn get(key: &Key) -> Value; +``` + +Since each backend has its own (de)serialization methods, our types are wrapped in compatibility types that map our `Storable` functions into whatever is required for the backend, e.g: +- [`StorableHeed`](https://github.com/Cuprate/cuprate/blob/2ac90420c658663564a71b7ecb52d74f3c2c9d0f/database/src/backend/heed/storable.rs#L11-L45) +- [`StorableRedb`](https://github.com/Cuprate/cuprate/blob/2ac90420c658663564a71b7ecb52d74f3c2c9d0f/database/src/backend/redb/storable.rs#L11-L30) + +Compatibility structs also exist for any `Storable` containers: +- [`StorableVec`](https://github.com/Cuprate/cuprate/blob/2ac90420c658663564a71b7ecb52d74f3c2c9d0f/database/src/storable.rs#L135-L191) +- [`StorableBytes`](https://github.com/Cuprate/cuprate/blob/2ac90420c658663564a71b7ecb52d74f3c2c9d0f/database/src/storable.rs#L208-L241) + +Again, it's unfortunate that these must be owned, although in `service`'s use-case, they would have to be owned anyway. \ No newline at end of file diff --git a/database/src/backend/heed/database.rs b/database/src/backend/heed/database.rs index 573d32ca..c985d0de 100644 --- a/database/src/backend/heed/database.rs +++ b/database/src/backend/heed/database.rs @@ -1,16 +1,10 @@ //! Implementation of `trait Database` for `heed`. //---------------------------------------------------------------------------------------------------- Import -use std::{ - borrow::{Borrow, Cow}, - cell::RefCell, - fmt::Debug, - ops::RangeBounds, - sync::RwLockReadGuard, -}; +use std::{cell::RefCell, ops::RangeBounds}; use crate::{ - backend::heed::{storable::StorableHeed, types::HeedDb}, + backend::heed::types::HeedDb, database::{DatabaseIter, DatabaseRo, DatabaseRw}, error::RuntimeError, table::Table, diff --git a/database/src/backend/heed/env.rs b/database/src/backend/heed/env.rs index 10ee012c..56064849 100644 --- a/database/src/backend/heed/env.rs +++ b/database/src/backend/heed/env.rs @@ -3,10 +3,8 @@ //---------------------------------------------------------------------------------------------------- Import use std::{ cell::RefCell, - fmt::Debug, num::NonZeroUsize, - ops::Deref, - sync::{RwLock, RwLockReadGuard, RwLockWriteGuard}, + sync::{RwLock, RwLockReadGuard}, }; use heed::{DatabaseOpenOptions, EnvFlags, EnvOpenOptions}; @@ -23,10 +21,11 @@ use crate::{ error::{InitError, RuntimeError}, resize::ResizeAlgorithm, table::Table, + tables::call_fn_on_all_tables_or_early_return, }; //---------------------------------------------------------------------------------------------------- Consts -/// TODO +/// Panic message when there's a table missing. const PANIC_MSG_MISSING_TABLE: &str = "cuprate_database::Env should uphold the invariant that all tables are already created"; @@ -49,7 +48,7 @@ pub struct ConcreteEnv { /// `reader_count` would be spinned on until 0, at which point /// we are safe to resize. /// - /// Although, 3 atomic operations (check atomic bool, reader_count++, reader_count--) + /// Although, 3 atomic operations (check atomic bool, `reader_count++`, `reader_count--`) /// turns out to be roughly as expensive as acquiring a non-contended `RwLock`, /// the CPU sleeping instead of spinning is much better too. /// @@ -68,7 +67,7 @@ impl Drop for ConcreteEnv { fn drop(&mut self) { // INVARIANT: drop(ConcreteEnv) must sync. // - // TODO: + // SOMEDAY: // "if the environment has the MDB_NOSYNC flag set the flushes will be omitted, // and with MDB_MAPASYNC they will be asynchronous." // @@ -76,7 +75,7 @@ impl Drop for ConcreteEnv { // We need to do `mdb_env_set_flags(&env, MDB_NOSYNC|MDB_ASYNCMAP, 0)` // to clear the no sync and async flags such that the below `self.sync()` // _actually_ synchronously syncs. - if let Err(e) = crate::Env::sync(self) { + if let Err(_e) = crate::Env::sync(self) { // TODO: log error? } @@ -118,10 +117,11 @@ impl Env for ConcreteEnv { #[cold] #[inline(never)] // called once. - #[allow(clippy::items_after_statements)] fn open(config: Config) -> Result { // + let mut env_open_options = EnvOpenOptions::new(); + // Map our `Config` sync mode to the LMDB environment flags. // // @@ -129,11 +129,21 @@ impl Env for ConcreteEnv { SyncMode::Safe => EnvFlags::empty(), SyncMode::Async => EnvFlags::MAP_ASYNC, SyncMode::Fast => EnvFlags::NO_SYNC | EnvFlags::WRITE_MAP | EnvFlags::MAP_ASYNC, - // TODO: dynamic syncs are not implemented. + // SOMEDAY: dynamic syncs are not implemented. SyncMode::FastThenSafe | SyncMode::Threshold(_) => unimplemented!(), }; - let mut env_open_options = EnvOpenOptions::new(); + // SAFETY: the flags we're setting are 'unsafe' + // from a data durability perspective, although, + // the user config wanted this. + // + // MAYBE: We may need to open/create tables with certain flags + // + // MAYBE: Set comparison functions for certain tables + // + unsafe { + env_open_options.flags(flags); + } // Set the memory map size to // (current disk size) + (a bit of leeway) @@ -152,7 +162,7 @@ impl Env for ConcreteEnv { // Set the max amount of database tables. // We know at compile time how many tables there are. - // TODO: ...how many? + // SOMEDAY: ...how many? env_open_options.max_dbs(32); // LMDB documentation: @@ -167,19 +177,19 @@ impl Env for ConcreteEnv { // - Use at least 126 reader threads // - Add 16 extra reader threads if <126 // - // TODO: This behavior is from `monerod`: + // FIXME: This behavior is from `monerod`: // // I believe this could be adjusted percentage-wise so very high // thread PCs can benefit from something like (cuprated + anything that uses the DB in the future). // For now: // - No other program using our DB exists // - Almost no-one has a 126+ thread CPU - #[allow(clippy::cast_possible_truncation)] // no-one has `u32::MAX`+ threads - let reader_threads = config.reader_threads.as_threads().get() as u32; + let reader_threads = + u32::try_from(config.reader_threads.as_threads().get()).unwrap_or(u32::MAX); env_open_options.max_readers(if reader_threads < 110 { 126 } else { - reader_threads + 16 + reader_threads.saturating_add(16) }); // Create the database directory if it doesn't exist. @@ -189,18 +199,11 @@ impl Env for ConcreteEnv { // let env = unsafe { env_open_options.open(config.db_directory())? }; - // TODO: Open/create tables with certain flags - // - // `heed` creates the database if it didn't exist. - // - /// Function that creates the tables based off the passed `T: Table`. fn create_table( env: &heed::Env, tx_rw: &mut heed::RwTxn<'_>, ) -> Result<(), InitError> { - println!("create_table(): {}", T::NAME); // TODO: use tracing. - DatabaseOpenOptions::new(env) .name(::NAME) .types::::Key>, StorableHeed<::Value>>() @@ -208,31 +211,17 @@ impl Env for ConcreteEnv { Ok(()) } - use crate::tables::{ - BlockBlobs, BlockHeights, BlockInfos, KeyImages, NumOutputs, Outputs, PrunableHashes, - PrunableTxBlobs, PrunedTxBlobs, RctOutputs, TxBlobs, TxHeights, TxIds, TxOutputs, - TxUnlockTime, - }; - let mut tx_rw = env.write_txn()?; - create_table::(&env, &mut tx_rw)?; - create_table::(&env, &mut tx_rw)?; - create_table::(&env, &mut tx_rw)?; - create_table::(&env, &mut tx_rw)?; - create_table::(&env, &mut tx_rw)?; - create_table::(&env, &mut tx_rw)?; - create_table::(&env, &mut tx_rw)?; - create_table::(&env, &mut tx_rw)?; - create_table::(&env, &mut tx_rw)?; - create_table::(&env, &mut tx_rw)?; - create_table::(&env, &mut tx_rw)?; - create_table::(&env, &mut tx_rw)?; - create_table::(&env, &mut tx_rw)?; - create_table::(&env, &mut tx_rw)?; - create_table::(&env, &mut tx_rw)?; - - // TODO: Set dupsort and comparison functions for certain tables - // + // Create all tables. + // FIXME: this macro is kinda awkward. + { + let env = &env; + let tx_rw = &mut tx_rw; + match call_fn_on_all_tables_or_early_return!(create_table(env, tx_rw)) { + Ok(_) => (), + Err(e) => return Err(e), + } + } // INVARIANT: this should never return `ResizeNeeded` due to adding // some tables since we added some leeway to the memory map above. diff --git a/database/src/backend/heed/error.rs b/database/src/backend/heed/error.rs index bb591164..c47bd908 100644 --- a/database/src/backend/heed/error.rs +++ b/database/src/backend/heed/error.rs @@ -20,7 +20,6 @@ impl From for crate::InitError { E1::Mdb(mdb_error) => match mdb_error { E2::Invalid => Self::Invalid, E2::VersionMismatch => Self::InvalidVersion, - E2::Other(c_int) => Self::Unknown(Box::new(mdb_error)), // "Located page was wrong type". // @@ -31,6 +30,7 @@ impl From for crate::InitError { // These errors shouldn't be returned on database init. E2::Incompatible + | E2::Other(_) | E2::BadTxn | E2::Problem | E2::KeyExist @@ -108,7 +108,7 @@ impl From for crate::RuntimeError { // occurring indicates we did _not_ do that, which is a bug // and we should panic. // - // TODO: This can also mean _another_ process wrote to our + // FIXME: This can also mean _another_ process wrote to our // LMDB file and increased the size. I don't think we need to accommodate for this. // // Although `monerod` reacts to that instead of `MDB_MAP_FULL` diff --git a/database/src/backend/heed/storable.rs b/database/src/backend/heed/storable.rs index 0d180c29..83442212 100644 --- a/database/src/backend/heed/storable.rs +++ b/database/src/backend/heed/storable.rs @@ -1,11 +1,11 @@ //! `cuprate_database::Storable` <-> `heed` serde trait compatibility layer. //---------------------------------------------------------------------------------------------------- Use -use std::{borrow::Cow, fmt::Debug, marker::PhantomData}; +use std::{borrow::Cow, marker::PhantomData}; -use heed::{types::Bytes, BoxedError, BytesDecode, BytesEncode, Database}; +use heed::{BoxedError, BytesDecode, BytesEncode}; -use crate::{storable::Storable, storable::StorableVec}; +use crate::storable::Storable; //---------------------------------------------------------------------------------------------------- StorableHeed /// The glue struct that implements `heed`'s (de)serialization @@ -47,6 +47,8 @@ where //---------------------------------------------------------------------------------------------------- Tests #[cfg(test)] mod test { + use std::fmt::Debug; + use super::*; use crate::{StorableBytes, StorableVec}; diff --git a/database/src/backend/heed/transaction.rs b/database/src/backend/heed/transaction.rs index 43096980..d32f3707 100644 --- a/database/src/backend/heed/transaction.rs +++ b/database/src/backend/heed/transaction.rs @@ -1,6 +1,6 @@ //! Implementation of `trait TxRo/TxRw` for `heed`. -use std::{cell::RefCell, ops::Deref, sync::RwLockReadGuard}; +use std::cell::RefCell; //---------------------------------------------------------------------------------------------------- Import use crate::{ diff --git a/database/src/backend/mod.rs b/database/src/backend/mod.rs index 2d6800e1..11ae40b8 100644 --- a/database/src/backend/mod.rs +++ b/database/src/backend/mod.rs @@ -1,13 +1,4 @@ //! Database backends. -//! -//! TODO: -//! Create a test backend backed by `std::collections::HashMap`. -//! -//! The full type could be something like `HashMap<&'static str, HashMap>`. -//! where the `str` is the table name, and the containing hashmap are are the -//! key and values. -//! -//! Not sure how duplicate keys will work. cfg_if::cfg_if! { // If both backends are enabled, fallback to `heed`. diff --git a/database/src/backend/redb/database.rs b/database/src/backend/redb/database.rs index 3224c24e..cd9a0be9 100644 --- a/database/src/backend/redb/database.rs +++ b/database/src/backend/redb/database.rs @@ -1,12 +1,7 @@ //! Implementation of `trait DatabaseR{o,w}` for `redb`. //---------------------------------------------------------------------------------------------------- Import -use std::{ - borrow::{Borrow, Cow}, - fmt::Debug, - marker::PhantomData, - ops::{Bound, Deref, RangeBounds}, -}; +use std::ops::RangeBounds; use redb::ReadableTable; @@ -17,7 +12,6 @@ use crate::{ }, database::{DatabaseIter, DatabaseRo, DatabaseRw}, error::RuntimeError, - storable::Storable, table::Table, }; diff --git a/database/src/backend/redb/env.rs b/database/src/backend/redb/env.rs index 554333a8..e552d454 100644 --- a/database/src/backend/redb/env.rs +++ b/database/src/backend/redb/env.rs @@ -1,18 +1,14 @@ //! Implementation of `trait Env` for `redb`. //---------------------------------------------------------------------------------------------------- Import -use std::{fmt::Debug, ops::Deref, path::Path, sync::Arc}; - use crate::{ - backend::redb::{ - storable::StorableRedb, - types::{RedbTableRo, RedbTableRw}, - }, + backend::redb::storable::StorableRedb, config::{Config, SyncMode}, database::{DatabaseIter, DatabaseRo, DatabaseRw}, env::{Env, EnvInner}, error::{InitError, RuntimeError}, table::Table, + tables::call_fn_on_all_tables_or_early_return, TxRw, }; @@ -36,7 +32,8 @@ impl Drop for ConcreteEnv { fn drop(&mut self) { // INVARIANT: drop(ConcreteEnv) must sync. if let Err(e) = self.sync() { - // TODO: log error? + // TODO: use tracing + println!("{e:#?}"); } // TODO: log that we are dropping the database. @@ -53,23 +50,22 @@ impl Env for ConcreteEnv { #[cold] #[inline(never)] // called once. - #[allow(clippy::items_after_statements)] fn open(config: Config) -> Result { - // TODO: dynamic syncs are not implemented. + // SOMEDAY: dynamic syncs are not implemented. let durability = match config.sync_mode { - // TODO: There's also `redb::Durability::Paranoid`: + // FIXME: There's also `redb::Durability::Paranoid`: // // should we use that instead of Immediate? SyncMode::Safe => redb::Durability::Immediate, SyncMode::Async => redb::Durability::Eventual, SyncMode::Fast => redb::Durability::None, - // TODO: dynamic syncs are not implemented. + // SOMEDAY: dynamic syncs are not implemented. SyncMode::FastThenSafe | SyncMode::Threshold(_) => unimplemented!(), }; let env_builder = redb::Builder::new(); - // TODO: we can set cache sizes with: + // FIXME: we can set cache sizes with: // env_builder.set_cache(bytes); // Use the in-memory backend if the feature is enabled. @@ -96,8 +92,6 @@ impl Env for ConcreteEnv { /// Function that creates the tables based off the passed `T: Table`. fn create_table(tx_rw: &redb::WriteTransaction) -> Result<(), InitError> { - println!("create_table(): {}", T::NAME); // TODO: use tracing. - let table: redb::TableDefinition< 'static, StorableRedb<::Key>, @@ -109,32 +103,20 @@ impl Env for ConcreteEnv { Ok(()) } - use crate::tables::{ - BlockBlobs, BlockHeights, BlockInfos, KeyImages, NumOutputs, Outputs, PrunableHashes, - PrunableTxBlobs, PrunedTxBlobs, RctOutputs, TxBlobs, TxHeights, TxIds, TxOutputs, - TxUnlockTime, - }; - - let tx_rw = env.begin_write()?; - create_table::(&tx_rw)?; - create_table::(&tx_rw)?; - create_table::(&tx_rw)?; - create_table::(&tx_rw)?; - create_table::(&tx_rw)?; - create_table::(&tx_rw)?; - create_table::(&tx_rw)?; - create_table::(&tx_rw)?; - create_table::(&tx_rw)?; - create_table::(&tx_rw)?; - create_table::(&tx_rw)?; - create_table::(&tx_rw)?; - create_table::(&tx_rw)?; - create_table::(&tx_rw)?; - create_table::(&tx_rw)?; + // Create all tables. + // FIXME: this macro is kinda awkward. + let mut tx_rw = env.begin_write()?; + { + let tx_rw = &mut tx_rw; + match call_fn_on_all_tables_or_early_return!(create_table(tx_rw)) { + Ok(_) => (), + Err(e) => return Err(e), + } + } tx_rw.commit()?; // Check for file integrity. - // TODO: should we do this? is it slow? + // FIXME: should we do this? is it slow? env.check_integrity()?; Ok(Self { diff --git a/database/src/backend/redb/error.rs b/database/src/backend/redb/error.rs index 28cd6b7e..4d40dbd9 100644 --- a/database/src/backend/redb/error.rs +++ b/database/src/backend/redb/error.rs @@ -45,7 +45,7 @@ impl From for InitError { match error { E::Io(e) => Self::Io(e), - E::Corrupted(s) => Self::Corrupt, + E::Corrupted(_) => Self::Corrupt, // HACK: Handle new errors as `redb` adds them. _ => Self::Unknown(Box::new(error)), } @@ -56,8 +56,6 @@ impl From for InitError { /// Created by `redb` in: /// - [`redb::Database::begin_write`](https://docs.rs/redb/1.5.0/redb/struct.Database.html#method.begin_write) fn from(error: redb::TransactionError) -> Self { - use redb::StorageError as E; - match error { redb::TransactionError::Storage(error) => error.into(), // HACK: Handle new errors as `redb` adds them. @@ -70,7 +68,6 @@ impl From for InitError { /// Created by `redb` in: /// - [`redb::WriteTransaction::open_table`](https://docs.rs/redb/1.5.0/redb/struct.WriteTransaction.html#method.open_table) fn from(error: redb::TableError) -> Self { - use redb::StorageError as E2; use redb::TableError as E; match error { @@ -85,8 +82,6 @@ impl From for InitError { /// Created by `redb` in: /// - [`redb::WriteTransaction::commit`](https://docs.rs/redb/1.5.0/redb/struct.WriteTransaction.html#method.commit) fn from(error: redb::CommitError) -> Self { - use redb::StorageError as E; - match error { redb::CommitError::Storage(error) => error.into(), // HACK: Handle new errors as `redb` adds them. @@ -102,8 +97,6 @@ impl From for RuntimeError { /// - [`redb::Database::begin_write`](https://docs.rs/redb/1.5.0/redb/struct.Database.html#method.begin_write) /// - [`redb::Database::begin_read`](https://docs.rs/redb/1.5.0/redb/struct.Database.html#method.begin_read) fn from(error: redb::TransactionError) -> Self { - use redb::StorageError as E; - match error { redb::TransactionError::Storage(error) => error.into(), @@ -118,8 +111,6 @@ impl From for RuntimeError { /// Created by `redb` in: /// - [`redb::WriteTransaction::commit`](https://docs.rs/redb/1.5.0/redb/struct.WriteTransaction.html#method.commit) fn from(error: redb::CommitError) -> Self { - use redb::StorageError as E; - match error { redb::CommitError::Storage(error) => error.into(), @@ -135,7 +126,6 @@ impl From for RuntimeError { /// - [`redb::WriteTransaction::open_table`](https://docs.rs/redb/1.5.0/redb/struct.WriteTransaction.html#method.open_table) /// - [`redb::ReadTransaction::open_table`](https://docs.rs/redb/1.5.0/redb/struct.ReadTransaction.html#method.open_table) fn from(error: redb::TableError) -> Self { - use redb::StorageError as E2; use redb::TableError as E; match error { diff --git a/database/src/backend/redb/storable.rs b/database/src/backend/redb/storable.rs index 64e7d062..6735fec0 100644 --- a/database/src/backend/redb/storable.rs +++ b/database/src/backend/redb/storable.rs @@ -1,7 +1,7 @@ //! `cuprate_database::Storable` <-> `redb` serde trait compatibility layer. //---------------------------------------------------------------------------------------------------- Use -use std::{any::Any, borrow::Cow, cmp::Ordering, fmt::Debug, marker::PhantomData}; +use std::{cmp::Ordering, fmt::Debug, marker::PhantomData}; use redb::TypeName; diff --git a/database/src/backend/redb/transaction.rs b/database/src/backend/redb/transaction.rs index 21a22ec4..5048851d 100644 --- a/database/src/backend/redb/transaction.rs +++ b/database/src/backend/redb/transaction.rs @@ -2,8 +2,6 @@ //---------------------------------------------------------------------------------------------------- Import use crate::{ - config::SyncMode, - env::Env, error::RuntimeError, transaction::{TxRo, TxRw}, }; diff --git a/database/src/backend/redb/types.rs b/database/src/backend/redb/types.rs index 1890af17..f3534c55 100644 --- a/database/src/backend/redb/types.rs +++ b/database/src/backend/redb/types.rs @@ -1,7 +1,7 @@ //! `redb` type aliases. //---------------------------------------------------------------------------------------------------- Types -use crate::{backend::redb::storable::StorableRedb, table::Table}; +use crate::backend::redb::storable::StorableRedb; //---------------------------------------------------------------------------------------------------- Types /// The concrete type for readable `redb` tables. diff --git a/database/src/backend/tests.rs b/database/src/backend/tests.rs index 0c530548..03d06c69 100644 --- a/database/src/backend/tests.rs +++ b/database/src/backend/tests.rs @@ -13,28 +13,20 @@ //! //! `redb`, and it only must be enabled for it to be tested. -#![allow( - clippy::items_after_statements, - clippy::significant_drop_tightening, - clippy::cast_possible_truncation -)] - //---------------------------------------------------------------------------------------------------- Import -use std::borrow::{Borrow, Cow}; use crate::{ - config::{Config, SyncMode}, database::{DatabaseIter, DatabaseRo, DatabaseRw}, env::{Env, EnvInner}, - error::{InitError, RuntimeError}, + error::RuntimeError, resize::ResizeAlgorithm, storable::StorableVec, - table::Table, tables::{ BlockBlobs, BlockHeights, BlockInfos, KeyImages, NumOutputs, Outputs, PrunableHashes, PrunableTxBlobs, PrunedTxBlobs, RctOutputs, TxBlobs, TxHeights, TxIds, TxOutputs, TxUnlockTime, }, + tables::{TablesIter, TablesMut}, tests::tmp_concrete_env, transaction::{TxRo, TxRw}, types::{ @@ -155,7 +147,6 @@ fn non_manual_resize_2() { /// Test all `DatabaseR{o,w}` operations. #[test] -#[allow(clippy::too_many_lines)] fn db_read_write() { let (env, _tempdir) = tmp_concrete_env(); let env_inner = env.env_inner(); @@ -191,7 +182,7 @@ fn db_read_write() { // Insert keys. let mut key = KEY; - for i in 0..N { + for _ in 0..N { table.put(&key, &VALUE).unwrap(); key.amount += 1; } @@ -331,6 +322,60 @@ fn db_read_write() { } } +/// Assert that `key`'s in database tables are sorted in +/// an ordered B-Tree fashion, i.e. `min_value -> max_value`. +#[test] +fn tables_are_sorted() { + let (env, _tmp) = tmp_concrete_env(); + let env_inner = env.env_inner(); + let tx_rw = env_inner.tx_rw().unwrap(); + let mut tables_mut = env_inner.open_tables_mut(&tx_rw).unwrap(); + + // Insert `{5, 4, 3, 2, 1, 0}`, assert each new + // number inserted is the minimum `first()` value. + for key in (0..6).rev() { + tables_mut.num_outputs_mut().put(&key, &123).unwrap(); + let (first, _) = tables_mut.num_outputs_mut().first().unwrap(); + assert_eq!(first, key); + } + + drop(tables_mut); + TxRw::commit(tx_rw).unwrap(); + let tx_rw = env_inner.tx_rw().unwrap(); + + // Assert iterators are ordered. + { + let tx_ro = env_inner.tx_ro().unwrap(); + let tables = env_inner.open_tables(&tx_ro).unwrap(); + let t = tables.num_outputs_iter(); + let iter = t.iter().unwrap(); + let keys = t.keys().unwrap(); + for ((i, iter), key) in (0..6).zip(iter).zip(keys) { + let (iter, _) = iter.unwrap(); + let key = key.unwrap(); + assert_eq!(i, iter); + assert_eq!(iter, key); + } + } + + let mut tables_mut = env_inner.open_tables_mut(&tx_rw).unwrap(); + let t = tables_mut.num_outputs_mut(); + + // Assert the `first()` values are the minimum, i.e. `{0, 1, 2}` + for key in 0..3 { + let (first, _) = t.first().unwrap(); + assert_eq!(first, key); + t.delete(&key).unwrap(); + } + + // Assert the `last()` values are the maximum, i.e. `{5, 4, 3}` + for key in (3..6).rev() { + let (last, _) = tables_mut.num_outputs_mut().last().unwrap(); + assert_eq!(last, key); + tables_mut.num_outputs_mut().delete(&key).unwrap(); + } +} + //---------------------------------------------------------------------------------------------------- Table Tests /// Test multiple tables and their key + values. /// diff --git a/database/src/config/backend.rs b/database/src/config/backend.rs index ed826344..4bbb12ca 100644 --- a/database/src/config/backend.rs +++ b/database/src/config/backend.rs @@ -1,4 +1,4 @@ -//! TODO +//! SOMEDAY //---------------------------------------------------------------------------------------------------- Import use std::{ @@ -19,13 +19,13 @@ use crate::{ }; //---------------------------------------------------------------------------------------------------- Backend -/// TODO +/// SOMEDAY: allow runtime hot-swappable backends. #[derive(Copy, Clone, Debug, Default, PartialEq, PartialOrd, Eq, Ord, Hash)] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] pub enum Backend { #[default] - /// TODO + /// SOMEDAY Heed, - /// TODO + /// SOMEDAY Redb, } diff --git a/database/src/config/config.rs b/database/src/config/config.rs index 1791a540..d712cb69 100644 --- a/database/src/config/config.rs +++ b/database/src/config/config.rs @@ -1,17 +1,8 @@ -//! Database [`Env`](crate::Env) configuration. -//! -//! This module contains the main [`Config`]uration struct -//! for the database [`Env`](crate::Env)ironment, and data -//! structures related to any configuration setting. -//! -//! These configurations are processed at runtime, meaning -//! the `Env` can/will dynamically adjust its behavior -//! based on these values. +//! The main [`Config`] struct, holding all configurable values. //---------------------------------------------------------------------------------------------------- Import use std::{ borrow::Cow, - num::NonZeroUsize, path::{Path, PathBuf}, }; @@ -26,13 +17,143 @@ use crate::{ resize::ResizeAlgorithm, }; +//---------------------------------------------------------------------------------------------------- ConfigBuilder +/// Builder for [`Config`]. +/// +// SOMEDAY: there's are many more options to add in the future. +#[derive(Debug, Clone, PartialEq, PartialOrd)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +pub struct ConfigBuilder { + /// [`Config::db_directory`]. + db_directory: Option>, + + /// [`Config::sync_mode`]. + sync_mode: Option, + + /// [`Config::reader_threads`]. + reader_threads: Option, + + /// [`Config::resize_algorithm`]. + resize_algorithm: Option, +} + +impl ConfigBuilder { + /// Create a new [`ConfigBuilder`]. + /// + /// [`ConfigBuilder::build`] can be called immediately + /// after this function to use default values. + pub const fn new() -> Self { + Self { + db_directory: None, + sync_mode: None, + reader_threads: None, + resize_algorithm: None, + } + } + + /// Build into a [`Config`]. + /// + /// # Default values + /// If [`ConfigBuilder::db_directory`] was not called, + /// the default [`cuprate_database_dir`] will be used. + /// + /// For all other values, [`Default::default`] is used. + pub fn build(self) -> Config { + // INVARIANT: all PATH safety checks are done + // in `helper::fs`. No need to do them here. + let db_directory = self + .db_directory + .unwrap_or_else(|| Cow::Borrowed(cuprate_database_dir())); + + // Add the database filename to the directory. + let db_file = { + let mut db_file = db_directory.to_path_buf(); + db_file.push(DATABASE_DATA_FILENAME); + Cow::Owned(db_file) + }; + + Config { + db_directory, + db_file, + sync_mode: self.sync_mode.unwrap_or_default(), + reader_threads: self.reader_threads.unwrap_or_default(), + resize_algorithm: self.resize_algorithm.unwrap_or_default(), + } + } + + /// Set a custom database directory (and file) [`Path`]. + #[must_use] + pub fn db_directory(mut self, db_directory: PathBuf) -> Self { + self.db_directory = Some(Cow::Owned(db_directory)); + self + } + + /// Tune the [`ConfigBuilder`] for the highest performing, + /// but also most resource-intensive & maybe risky settings. + /// + /// Good default for testing, and resource-available machines. + #[must_use] + pub fn fast(mut self) -> Self { + self.sync_mode = Some(SyncMode::Fast); + self.reader_threads = Some(ReaderThreads::OnePerThread); + self.resize_algorithm = Some(ResizeAlgorithm::default()); + self + } + + /// Tune the [`ConfigBuilder`] for the lowest performing, + /// but also least resource-intensive settings. + /// + /// Good default for resource-limited machines, e.g. a cheap VPS. + #[must_use] + pub fn low_power(mut self) -> Self { + self.sync_mode = Some(SyncMode::default()); + self.reader_threads = Some(ReaderThreads::One); + self.resize_algorithm = Some(ResizeAlgorithm::default()); + self + } + + /// Set a custom [`SyncMode`]. + #[must_use] + pub const fn sync_mode(mut self, sync_mode: SyncMode) -> Self { + self.sync_mode = Some(sync_mode); + self + } + + /// Set a custom [`ReaderThreads`]. + #[must_use] + pub const fn reader_threads(mut self, reader_threads: ReaderThreads) -> Self { + self.reader_threads = Some(reader_threads); + self + } + + /// Set a custom [`ResizeAlgorithm`]. + #[must_use] + pub const fn resize_algorithm(mut self, resize_algorithm: ResizeAlgorithm) -> Self { + self.resize_algorithm = Some(resize_algorithm); + self + } +} + +impl Default for ConfigBuilder { + fn default() -> Self { + Self { + db_directory: Some(Cow::Borrowed(cuprate_database_dir())), + sync_mode: Some(SyncMode::default()), + reader_threads: Some(ReaderThreads::default()), + resize_algorithm: Some(ResizeAlgorithm::default()), + } + } +} + //---------------------------------------------------------------------------------------------------- Config /// Database [`Env`](crate::Env) configuration. /// /// This is the struct passed to [`Env::open`](crate::Env::open) that /// allows the database to be configured in various ways. /// -/// TODO: there's probably more options to add. +/// For construction, either use [`ConfigBuilder`] or [`Config::default`]. +/// +// SOMEDAY: there's are many more options to add in the future. #[derive(Debug, Clone, PartialEq, PartialOrd)] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] pub struct Config { @@ -44,8 +165,8 @@ pub struct Config { /// By default, if no value is provided in the [`Config`] /// constructor functions, this will be [`cuprate_database_dir`]. /// - /// TODO: we should also support `/etc/cuprated.conf`. - /// This could be represented with an `enum DbPath { Default, Custom, Etc, }` + // SOMEDAY: we should also support `/etc/cuprated.conf`. + // This could be represented with an `enum DbPath { Default, Custom, Etc, }` pub(crate) db_directory: Cow<'static, Path>, /// The actual database data file. /// @@ -67,111 +188,50 @@ pub struct Config { } impl Config { - /// Private function to acquire [`Config::db_file`] - /// from the user provided (or default) [`Config::db_directory`]. - /// - /// As the database data file PATH is just the directory + the filename, - /// we only need the directory from the user/Config, and can add it here. - fn return_db_dir_and_file( - db_directory: Option, - ) -> (Cow<'static, Path>, Cow<'static, Path>) { - // INVARIANT: all PATH safety checks are done - // in `helper::fs`. No need to do them here. - let db_directory = - db_directory.map_or_else(|| Cow::Borrowed(cuprate_database_dir()), Cow::Owned); - - // Add the database filename to the directory. - let mut db_file = db_directory.to_path_buf(); - db_file.push(DATABASE_DATA_FILENAME); - - (db_directory, Cow::Owned(db_file)) - } - /// Create a new [`Config`] with sane default settings. /// - /// # `db_directory` - /// If this is `Some`, it will be used as the - /// directory that contains all database files. + /// The [`Config::db_directory`] will be [`cuprate_database_dir`]. /// - /// If `None`, it will use the default directory [`cuprate_database_dir`]. - pub fn new(db_directory: Option) -> Self { - let (db_directory, db_file) = Self::return_db_dir_and_file(db_directory); - Self { - db_directory, - db_file, - sync_mode: SyncMode::default(), - reader_threads: ReaderThreads::OnePerThread, - resize_algorithm: ResizeAlgorithm::default(), - } - } - - /// Create a [`Config`] with the highest performing, - /// but also most resource-intensive & maybe risky settings. + /// All other values will be [`Default::default`]. /// - /// Good default for testing, and resource-available machines. + /// Same as [`Config::default`]. /// - /// # `db_directory` - /// If this is `Some`, it will be used as the - /// directory that contains all database files. + /// ```rust + /// use cuprate_database::{config::*, resize::*, DATABASE_DATA_FILENAME}; + /// use cuprate_helper::fs::*; /// - /// If `None`, it will use the default directory [`cuprate_database_dir`]. - pub fn fast(db_directory: Option) -> Self { - let (db_directory, db_file) = Self::return_db_dir_and_file(db_directory); - Self { - db_directory, - db_file, - sync_mode: SyncMode::Fast, - reader_threads: ReaderThreads::OnePerThread, - resize_algorithm: ResizeAlgorithm::default(), - } - } - - /// Create a [`Config`] with the lowest performing, - /// but also least resource-intensive settings. + /// let config = Config::new(); /// - /// Good default for resource-limited machines, e.g. a cheap VPS. - /// - /// # `db_directory` - /// If this is `Some`, it will be used as the - /// directory that contains all database files. - /// - /// If `None`, it will use the default directory [`cuprate_database_dir`]. - pub fn low_power(db_directory: Option) -> Self { - let (db_directory, db_file) = Self::return_db_dir_and_file(db_directory); - Self { - db_directory, - db_file, - sync_mode: SyncMode::default(), - reader_threads: ReaderThreads::One, - resize_algorithm: ResizeAlgorithm::default(), - } + /// assert_eq!(config.db_directory(), cuprate_database_dir()); + /// assert!(config.db_file().starts_with(cuprate_database_dir())); + /// assert!(config.db_file().ends_with(DATABASE_DATA_FILENAME)); + /// assert_eq!(config.sync_mode, SyncMode::default()); + /// assert_eq!(config.reader_threads, ReaderThreads::default()); + /// assert_eq!(config.resize_algorithm, ResizeAlgorithm::default()); + /// ``` + pub fn new() -> Self { + ConfigBuilder::default().build() } /// Return the absolute [`Path`] to the database directory. - /// - /// This will be the `db_directory` given - /// (or default) during [`Config`] construction. pub const fn db_directory(&self) -> &Cow<'_, Path> { &self.db_directory } /// Return the absolute [`Path`] to the database data file. - /// - /// This will be based off the `db_directory` given - /// (or default) during [`Config`] construction. pub const fn db_file(&self) -> &Cow<'_, Path> { &self.db_file } } impl Default for Config { - /// Same as `Self::new(None)`. + /// Same as [`Config::new`]. /// /// ```rust /// # use cuprate_database::config::*; - /// assert_eq!(Config::default(), Config::new(None)); + /// assert_eq!(Config::default(), Config::new()); /// ``` fn default() -> Self { - Self::new(None) + Self::new() } } diff --git a/database/src/config/mod.rs b/database/src/config/mod.rs index a8da828c..dfa4f674 100644 --- a/database/src/config/mod.rs +++ b/database/src/config/mod.rs @@ -1,7 +1,44 @@ -//! TODO +//! Database [`Env`](crate::Env) configuration. +//! +//! This module contains the main [`Config`]uration struct +//! for the database [`Env`](crate::Env)ironment, and types +//! related to configuration settings. +//! +//! The main constructor is the [`ConfigBuilder`]. +//! +//! These configurations are processed at runtime, meaning +//! the `Env` can/will dynamically adjust its behavior +//! based on these values. +//! +//! # Example +//! ```rust +//! use cuprate_database::{ +//! Env, +//! config::{ConfigBuilder, ReaderThreads, SyncMode} +//! }; +//! +//! # fn main() -> Result<(), Box> { +//! let db_dir = tempfile::tempdir()?; +//! +//! let config = ConfigBuilder::new() +//! // Use a custom database directory. +//! .db_directory(db_dir.path().to_path_buf()) +//! // Use as many reader threads as possible (when using `service`). +//! .reader_threads(ReaderThreads::OnePerThread) +//! // Use the fastest sync mode. +//! .sync_mode(SyncMode::Fast) +//! // Build into `Config` +//! .build(); +//! +//! // Start a database `service` using this configuration. +//! let (reader_handle, _) = cuprate_database::service::init(config.clone())?; +//! // It's using the config we provided. +//! assert_eq!(reader_handle.env().config(), &config); +//! # Ok(()) } +//! ``` mod config; -pub use config::Config; +pub use config::{Config, ConfigBuilder}; mod reader_threads; pub use reader_threads::ReaderThreads; diff --git a/database/src/config/reader_threads.rs b/database/src/config/reader_threads.rs index 0dc35581..34b20a88 100644 --- a/database/src/config/reader_threads.rs +++ b/database/src/config/reader_threads.rs @@ -9,25 +9,19 @@ //! based on these values. //---------------------------------------------------------------------------------------------------- Import -use std::{ - borrow::Cow, - num::NonZeroUsize, - path::{Path, PathBuf}, -}; +use std::num::NonZeroUsize; #[cfg(feature = "serde")] use serde::{Deserialize, Serialize}; -use cuprate_helper::fs::cuprate_database_dir; - -use crate::{constants::DATABASE_DATA_FILENAME, resize::ResizeAlgorithm}; - //---------------------------------------------------------------------------------------------------- ReaderThreads -/// Amount of database reader threads to spawn. +/// Amount of database reader threads to spawn when using [`service`](crate::service). /// -/// This controls how many reader thread [`crate::service`]'s +/// This controls how many reader thread `service`'s /// thread-pool will spawn to receive and send requests/responses. /// +/// It does nothing outside of `service`. +/// /// It will always be at least 1, up until the amount of threads on the machine. /// /// The main function used to extract an actual @@ -38,8 +32,8 @@ pub enum ReaderThreads { #[default] /// Spawn 1 reader thread per available thread on the machine. /// - /// For example, a `16-core, 32-thread` Ryzen 5950x will - /// spawn `32` reader threads using this setting. + /// For example, a `32-thread` system will spawn + /// `32` reader threads using this setting. OnePerThread, /// Only spawn 1 reader thread. diff --git a/database/src/config/sync_mode.rs b/database/src/config/sync_mode.rs index 7dba062a..1d203396 100644 --- a/database/src/config/sync_mode.rs +++ b/database/src/config/sync_mode.rs @@ -9,19 +9,10 @@ //! based on these values. //---------------------------------------------------------------------------------------------------- Import -use std::{ - borrow::Cow, - num::NonZeroUsize, - path::{Path, PathBuf}, -}; #[cfg(feature = "serde")] use serde::{Deserialize, Serialize}; -use cuprate_helper::fs::cuprate_database_dir; - -use crate::{constants::DATABASE_DATA_FILENAME, resize::ResizeAlgorithm}; - //---------------------------------------------------------------------------------------------------- SyncMode /// Disk synchronization mode. /// @@ -48,7 +39,7 @@ use crate::{constants::DATABASE_DATA_FILENAME, resize::ResizeAlgorithm}; /// ``` /// will be fine, most likely pulling from memory instead of disk. /// -/// # TODO +/// # SOMEDAY /// Dynamic sync's are not yet supported. /// /// Only: @@ -64,24 +55,24 @@ pub enum SyncMode { /// Use [`SyncMode::Fast`] until fully synced, /// then use [`SyncMode::Safe`]. /// - /// # TODO: how to implement this? - /// ref: - /// monerod-solution: - /// cuprate-issue: - /// - /// We could: - /// ```rust,ignore - /// if current_db_block <= top_block.saturating_sub(N) { - /// // don't sync() - /// } else { - /// // sync() - /// } - /// ``` - /// where N is some threshold we pick that is _close_ enough - /// to being synced where we want to start being safer. - /// - /// Essentially, when we are in a certain % range of being finished, - /// switch to safe mode, until then, go fast. + // # SOMEDAY: how to implement this? + // ref: + // monerod-solution: + // cuprate-issue: + // + // We could: + // ```rust,ignore + // if current_db_block <= top_block.saturating_sub(N) { + // // don't sync() + // } else { + // // sync() + // } + // ``` + // where N is some threshold we pick that is _close_ enough + // to being synced where we want to start being safer. + // + // Essentially, when we are in a certain % range of being finished, + // switch to safe mode, until then, go fast. FastThenSafe, #[default] @@ -136,7 +127,7 @@ pub enum SyncMode { /// In the case of a system crash, the database /// may become corrupted when using this option. // - // TODO: we could call this `unsafe` + // FIXME: we could call this `unsafe` // and use that terminology in the config file // so users know exactly what they are getting // themselves into. diff --git a/database/src/constants.rs b/database/src/constants.rs index a5d810dc..667e36cb 100644 --- a/database/src/constants.rs +++ b/database/src/constants.rs @@ -35,8 +35,8 @@ TODO: instructions on: /// /// | Backend | Value | /// |---------|-------| -/// | `heed` | "heed" -/// | `redb` | "redb" +/// | `heed` | `"heed"` +/// | `redb` | `"redb"` pub const DATABASE_BACKEND: &str = { cfg_if! { if #[cfg(all(feature = "redb", not(feature = "heed")))] { @@ -53,8 +53,8 @@ pub const DATABASE_BACKEND: &str = { /// /// | Backend | Value | /// |---------|-------| -/// | `heed` | "data.mdb" -/// | `redb` | "data.redb" +/// | `heed` | `"data.mdb"` +/// | `redb` | `"data.redb"` pub const DATABASE_DATA_FILENAME: &str = { cfg_if! { if #[cfg(all(feature = "redb", not(feature = "heed")))] { @@ -69,8 +69,8 @@ pub const DATABASE_DATA_FILENAME: &str = { /// /// | Backend | Value | /// |---------|-------| -/// | `heed` | Some("lock.mdb") -/// | `redb` | None (redb doesn't use a file lock) +/// | `heed` | `Some("lock.mdb")` +/// | `redb` | `None` (redb doesn't use a file lock) pub const DATABASE_LOCK_FILENAME: Option<&str> = { cfg_if! { if #[cfg(all(feature = "redb", not(feature = "heed")))] { diff --git a/database/src/database.rs b/database/src/database.rs index 8273a925..4a45f7cc 100644 --- a/database/src/database.rs +++ b/database/src/database.rs @@ -1,33 +1,38 @@ -//! Abstracted database; `trait DatabaseRo` & `trait DatabaseRw`. +//! Abstracted database table operations; `trait DatabaseRo` & `trait DatabaseRw`. //---------------------------------------------------------------------------------------------------- Import -use std::{ - borrow::{Borrow, Cow}, - fmt::Debug, - ops::{Deref, RangeBounds}, -}; +use std::ops::RangeBounds; -use crate::{ - error::RuntimeError, - table::Table, - transaction::{TxRo, TxRw}, -}; +use crate::{error::RuntimeError, table::Table}; //---------------------------------------------------------------------------------------------------- DatabaseIter +/// Generic post-fix documentation for `DatabaseIter` methods. +macro_rules! doc_iter { + () => { + r"Although the returned iterator itself is tied to the lifetime +of `&self`, the returned values from the iterator are _owned_. + +# Errors +The construction of the iterator itself may error. + +Each iteration of the iterator has the potential to error as well." + }; +} + /// Database (key-value store) read-only iteration abstraction. /// /// These are read-only iteration-related operations that /// can only be called from [`DatabaseRo`] objects. /// /// # Hack -/// This is a HACK to get around the fact our read/write tables +/// This is a HACK to get around the fact [`DatabaseRw`] tables /// cannot safely return values returning lifetimes, as such, /// only read-only tables implement this trait. /// /// - /// - pub trait DatabaseIter { - /// Get an iterator of value's corresponding to a range of keys. + /// Get an [`Iterator`] of value's corresponding to a range of keys. /// /// For example: /// ```rust,ignore @@ -39,12 +44,7 @@ pub trait DatabaseIter { /// Although the returned iterator itself is tied to the lifetime /// of `&'a self`, the returned values from the iterator are _owned_. /// - /// # Errors - /// Each key in the `range` has the potential to error, for example, - /// if a particular key in the `range` does not exist, - /// [`RuntimeError::KeyNotFound`] wrapped in [`Err`] will be returned - /// from the iterator. - #[allow(clippy::iter_not_returning_iterator)] + #[doc = doc_iter!()] fn get_range<'a, Range>( &'a self, range: Range, @@ -52,32 +52,36 @@ pub trait DatabaseIter { where Range: RangeBounds + 'a; - /// TODO - /// - /// # Errors - /// TODO + /// Get an [`Iterator`] that returns the `(key, value)` types for this database. + #[doc = doc_iter!()] #[allow(clippy::iter_not_returning_iterator)] fn iter( &self, ) -> Result> + '_, RuntimeError>; - /// TODO - /// - /// # Errors - /// TODO + /// Get an [`Iterator`] that returns _only_ the `key` type for this database. + #[doc = doc_iter!()] fn keys(&self) -> Result> + '_, RuntimeError>; - /// TODO - /// - /// # Errors - /// TODO + /// Get an [`Iterator`] that returns _only_ the `value` type for this database. + #[doc = doc_iter!()] fn values( &self, ) -> Result> + '_, RuntimeError>; } //---------------------------------------------------------------------------------------------------- DatabaseRo +/// Generic post-fix documentation for `DatabaseR{o,w}` methods. +macro_rules! doc_database { + () => { + r"# Errors +This will return [`RuntimeError::KeyNotFound`] if: +- Input does not exist OR +- Database is empty" + }; +} + /// Database (key-value store) read abstraction. /// /// This is a read-only database table, @@ -106,19 +110,16 @@ pub trait DatabaseIter { /// - pub unsafe trait DatabaseRo { /// Get the value corresponding to a key. - /// - /// The returned value is _owned_. - /// - /// # Errors - /// This will return [`RuntimeError::KeyNotFound`] wrapped in [`Err`] if `key` does not exist. - /// - /// It will return other [`RuntimeError`]'s on things like IO errors as well. + #[doc = doc_database!()] fn get(&self, key: &T::Key) -> Result; - /// TODO + /// Returns `true` if the database contains a value for the specified key. /// /// # Errors - /// TODO + /// Note that this will _never_ return `Err(RuntimeError::KeyNotFound)`, + /// as in that case, `Ok(false)` will be returned. + /// + /// Other errors may still occur. fn contains(&self, key: &T::Key) -> Result { match self.get(key) { Ok(_) => Ok(true), @@ -127,28 +128,24 @@ pub unsafe trait DatabaseRo { } } - /// TODO + /// Returns the number of `(key, value)` pairs in the database. /// /// # Errors - /// TODO + /// This will never return [`RuntimeError::KeyNotFound`]. fn len(&self) -> Result; - /// TODO - /// - /// # Errors - /// TODO + /// Returns the first `(key, value)` pair in the database. + #[doc = doc_database!()] fn first(&self) -> Result<(T::Key, T::Value), RuntimeError>; - /// TODO - /// - /// # Errors - /// TODO + /// Returns the last `(key, value)` pair in the database. + #[doc = doc_database!()] fn last(&self) -> Result<(T::Key, T::Value), RuntimeError>; - /// TODO + /// Returns `true` if the database contains no `(key, value)` pairs. /// /// # Errors - /// TODO + /// This can only return [`RuntimeError::Io`] on errors. fn is_empty(&self) -> Result; } @@ -161,7 +158,8 @@ pub trait DatabaseRw: DatabaseRo { /// /// This will overwrite any existing key-value pairs. /// - /// # Errors + #[doc = doc_database!()] + /// /// This will never [`RuntimeError::KeyExists`]. fn put(&mut self, key: &T::Key, value: &T::Value) -> Result<(), RuntimeError>; @@ -169,8 +167,9 @@ pub trait DatabaseRw: DatabaseRo { /// /// This will return `Ok(())` if the key does not exist. /// - /// # Errors - /// This will never [`RuntimeError::KeyNotFound`]. + #[doc = doc_database!()] + /// + /// This will never [`RuntimeError::KeyExists`]. fn delete(&mut self, key: &T::Key) -> Result<(), RuntimeError>; /// Delete and return a key-value pair in the database. @@ -178,8 +177,7 @@ pub trait DatabaseRw: DatabaseRo { /// This is the same as [`DatabaseRw::delete`], however, /// it will serialize the `T::Value` and return it. /// - /// # Errors - /// This will return [`RuntimeError::KeyNotFound`] wrapped in [`Err`] if `key` does not exist. + #[doc = doc_database!()] fn take(&mut self, key: &T::Key) -> Result; /// Fetch the value, and apply a function to it - or delete the entry. @@ -193,8 +191,7 @@ pub trait DatabaseRw: DatabaseRo { /// - If `f` returns `Some(value)`, that will be [`DatabaseRw::put`] as the new value /// - If `f` returns `None`, the entry will be [`DatabaseRw::delete`]d /// - /// # Errors - /// This will return [`RuntimeError::KeyNotFound`] wrapped in [`Err`] if `key` does not exist. + #[doc = doc_database!()] fn update(&mut self, key: &T::Key, mut f: F) -> Result<(), RuntimeError> where F: FnMut(T::Value) -> Option, @@ -207,15 +204,13 @@ pub trait DatabaseRw: DatabaseRo { } } - /// TODO + /// Removes and returns the first `(key, value)` pair in the database. /// - /// # Errors - /// TODO + #[doc = doc_database!()] fn pop_first(&mut self) -> Result<(T::Key, T::Value), RuntimeError>; - /// TODO + /// Removes and returns the last `(key, value)` pair in the database. /// - /// # Errors - /// TODO + #[doc = doc_database!()] fn pop_last(&mut self) -> Result<(T::Key, T::Value), RuntimeError>; } diff --git a/database/src/env.rs b/database/src/env.rs index 437928a6..3a32666b 100644 --- a/database/src/env.rs +++ b/database/src/env.rs @@ -1,7 +1,7 @@ //! Abstracted database environment; `trait Env`. //---------------------------------------------------------------------------------------------------- Import -use std::{fmt::Debug, num::NonZeroUsize, ops::Deref}; +use std::num::NonZeroUsize; use crate::{ config::Config, @@ -9,11 +9,7 @@ use crate::{ error::{InitError, RuntimeError}, resize::ResizeAlgorithm, table::Table, - tables::{ - call_fn_on_all_tables_or_early_return, BlockBlobs, BlockHeights, BlockInfos, KeyImages, - NumOutputs, Outputs, PrunableHashes, PrunableTxBlobs, PrunedTxBlobs, RctOutputs, Tables, - TablesIter, TablesMut, TxHeights, TxIds, TxUnlockTime, - }, + tables::{call_fn_on_all_tables_or_early_return, TablesIter, TablesMut}, transaction::{TxRo, TxRw}, }; @@ -28,8 +24,16 @@ use crate::{ /// although, no invariant relies on this (yet). /// /// # Lifetimes -/// TODO: Explain the very sequential lifetime pipeline: -/// - `ConcreteEnv` -> `'env` -> `'tx` -> `impl DatabaseR{o,w}` +/// The lifetimes associated with `Env` have a sequential flow: +/// 1. `ConcreteEnv` +/// 2. `'env` +/// 3. `'tx` +/// 4. `'db` +/// +/// As in: +/// - open database tables only live as long as... +/// - transactions which only live as long as the... +/// - environment ([`EnvInner`]) pub trait Env: Sized { //------------------------------------------------ Constants /// Does the database backend need to be manually @@ -37,7 +41,7 @@ pub trait Env: Sized { /// /// # Invariant /// If this is `false`, that means this [`Env`] - /// can _never_ return a [`RuntimeError::ResizeNeeded`]. + /// must _never_ return a [`RuntimeError::ResizeNeeded`]. /// /// If this is `true`, [`Env::resize_map`] & [`Env::current_map_size`] /// _must_ be re-implemented, as it just panics by default. @@ -55,10 +59,10 @@ pub trait Env: Sized { /// This is used as the `self` in [`EnvInner`] functions, so whatever /// this type is, is what will be accessible from those functions. /// - /// # Explanation (not needed for practical use) - /// For `heed`, this is just `heed::Env`, for `redb` this is - /// `(redb::Database, redb::Durability)` as each transaction - /// needs the sync mode set during creation. + // # HACK + // For `heed`, this is just `heed::Env`, for `redb` this is + // `(redb::Database, redb::Durability)` as each transaction + // needs the sync mode set during creation. type EnvInner<'env>: EnvInner<'env, Self::TxRo<'env>, Self::TxRw<'env>> where Self: 'env; @@ -100,11 +104,11 @@ pub trait Env: Sized { /// I.e., after this function returns, there must be no doubts /// that the data isn't synced yet, it _must_ be synced. /// - /// TODO: either this invariant or `sync()` itself will most - /// likely be removed/changed after `SyncMode` is finalized. + // FIXME: either this invariant or `sync()` itself will most + // likely be removed/changed after `SyncMode` is finalized. /// /// # Errors - /// TODO + /// If there is a synchronization error, this should return an error. fn sync(&self) -> Result<(), RuntimeError>; /// Resize the database's memory map to a @@ -120,6 +124,7 @@ pub trait Env: Sized { /// This function _must_ be re-implemented if [`Env::MANUAL_RESIZE`] is `true`. /// /// Otherwise, this function will panic with `unreachable!()`. + #[allow(unused_variables)] fn resize_map(&self, resize_algorithm: Option) -> NonZeroUsize { unreachable!() } @@ -171,7 +176,26 @@ pub trait Env: Sized { } //---------------------------------------------------------------------------------------------------- DatabaseRo -/// TODO +/// Document errors when opening tables in [`EnvInner`]. +macro_rules! doc_table_error { + () => { + r"# Errors +This will only return [`RuntimeError::Io`] if it errors. + +As all tables are created upon [`Env::open`], +this function will never error because a table doesn't exist." + }; +} + +/// The inner [`Env`] type. +/// +/// This type is created with [`Env::env_inner`] and represents +/// the type able to generate transactions and open tables. +/// +/// # Locking behavior +/// As noted in `Env::env_inner`, this is a `RwLockReadGuard` +/// when using the `heed` backend, be aware of this and do +/// not hold onto an `EnvInner` for a long time. pub trait EnvInner<'env, Ro, Rw> where Self: 'env, @@ -192,6 +216,9 @@ where /// Open a database in read-only mode. /// + /// The returned value can have [`DatabaseRo`] + /// & [`DatabaseIter`] functions called on it. + /// /// This will open the database [`Table`] /// passed as a generic to this function. /// @@ -202,12 +229,7 @@ where /// // (name, key/value type) /// ``` /// - /// # Errors - /// This function errors upon internal database/IO errors. - /// - /// As [`Table`] is `Sealed`, and all tables are created - /// upon [`Env::open`], this function will never error because - /// a table doesn't exist. + #[doc = doc_table_error!()] fn open_db_ro( &self, tx_ro: &Ro, @@ -218,31 +240,33 @@ where /// All [`DatabaseRo`] functions are also callable /// with the returned [`DatabaseRw`] structure. /// + /// Note that [`DatabaseIter`] functions are _not_ + /// available to [`DatabaseRw`] structures. + /// /// This will open the database [`Table`] /// passed as a generic to this function. /// - /// # Errors - /// This function errors upon internal database/IO errors. - /// - /// As [`Table`] is `Sealed`, and all tables are created - /// upon [`Env::open`], this function will never error because - /// a table doesn't exist. + #[doc = doc_table_error!()] fn open_db_rw(&self, tx_rw: &Rw) -> Result, RuntimeError>; - /// TODO + /// Open all tables in read/iter mode. /// - /// # Errors - /// TODO + /// This calls [`EnvInner::open_db_ro`] on all database tables + /// and returns a structure that allows access to all tables. + /// + #[doc = doc_table_error!()] fn open_tables(&self, tx_ro: &Ro) -> Result { call_fn_on_all_tables_or_early_return! { Self::open_db_ro(self, tx_ro) } } - /// TODO + /// Open all tables in read-write mode. /// - /// # Errors - /// TODO + /// This calls [`EnvInner::open_db_rw`] on all database tables + /// and returns a structure that allows access to all tables. + /// + #[doc = doc_table_error!()] fn open_tables_mut(&self, tx_rw: &Rw) -> Result { call_fn_on_all_tables_or_early_return! { Self::open_db_rw(self, tx_rw) @@ -257,11 +281,6 @@ where /// Note that this operation is tied to `tx_rw`, as such this /// function's effects can be aborted using [`TxRw::abort`]. /// - /// # Errors - /// This function errors upon internal database/IO errors. - /// - /// As [`Table`] is `Sealed`, and all tables are created - /// upon [`Env::open`], this function will never error because - /// a table doesn't exist. + #[doc = doc_table_error!()] fn clear_db(&self, tx_rw: &mut Rw) -> Result<(), RuntimeError>; } diff --git a/database/src/error.rs b/database/src/error.rs index 33c64346..e47634f6 100644 --- a/database/src/error.rs +++ b/database/src/error.rs @@ -1,5 +1,4 @@ //! Database error types. -//! TODO: `InitError/RuntimeError` are maybe bad names. //---------------------------------------------------------------------------------------------------- Import use std::fmt::Debug; @@ -42,8 +41,12 @@ pub enum InitError { /// The database is currently in the process /// of shutting down and cannot respond. /// - /// TODO: This might happen if we try to open - /// while we are shutting down, `unreachable!()`? + /// # Notes + /// This error can only occur with the `heed` backend when + /// the database environment is opened _right_ at the same time + /// another thread/process is closing it. + /// + /// This will never occur with other backends. #[error("database is shutting down")] ShuttingDown, diff --git a/database/src/key.rs b/database/src/key.rs index f88455ca..13f7cede 100644 --- a/database/src/key.rs +++ b/database/src/key.rs @@ -1,40 +1,22 @@ //! Database key abstraction; `trait Key`. //---------------------------------------------------------------------------------------------------- Import -use std::{cmp::Ordering, fmt::Debug}; +use std::cmp::Ordering; -use bytemuck::Pod; - -use crate::storable::{self, Storable}; +use crate::storable::Storable; //---------------------------------------------------------------------------------------------------- Table /// Database [`Table`](crate::table::Table) key metadata. /// -/// Purely compile time information for database table keys, supporting duplicate keys. +/// Purely compile time information for database table keys. +// +// FIXME: this doesn't need to exist right now but +// may be used if we implement getting values using ranges. +// pub trait Key: Storable + Sized { - /// Does this [`Key`] require multiple keys to reach a value? - /// - /// # Invariant - /// - If [`Key::DUPLICATE`] is `true`, [`Key::primary_secondary`] MUST be re-implemented. - /// - If [`Key::DUPLICATE`] is `true`, [`Key::new_with_max_secondary`] MUST be re-implemented. - const DUPLICATE: bool; - - /// Does this [`Key`] have a custom comparison function? - /// - /// # Invariant - /// If [`Key::CUSTOM_COMPARE`] is `true`, [`Key::compare`] MUST be re-implemented. - const CUSTOM_COMPARE: bool; - /// The primary key type. type Primary: Storable; - /// Acquire [`Self::Primary`] and the secondary key. - /// - /// # TODO: doc test - fn primary_secondary(self) -> (Self::Primary, u64) { - unreachable!() - } - /// Compare 2 [`Key`]'s against each other. /// /// By default, this does a straight _byte_ comparison, @@ -55,67 +37,17 @@ pub trait Key: Storable + Sized { /// std::cmp::Ordering::Greater, /// ); /// ``` + #[inline] fn compare(left: &[u8], right: &[u8]) -> Ordering { left.cmp(right) } - - /// Create a new [`Key`] from the [`Key::Primary`] type, - /// with the secondary key type set to the maximum value. - /// - /// # Invariant - /// Secondary key must be the max value of the type. - /// - /// # TODO: doc test - fn new_with_max_secondary(primary: Self::Primary) -> Self { - unreachable!() - } } //---------------------------------------------------------------------------------------------------- Impl -/// TODO: remove after we finalize tables. -/// -/// Implement `Key` on most primitive types. -/// -/// - `Key::DUPLICATE` is always `false`. -/// - `Key::CUSTOM_COMPARE` is always `false`. -macro_rules! impl_key { - ( - $( - $t:ident // Key type. - ),* $(,)? - ) => { - $( - impl Key for $t { - const DUPLICATE: bool = false; - const CUSTOM_COMPARE: bool = false; - - type Primary = $t; - } - )* - }; -} -// Implement `Key` for primitives. -impl_key! { - u8, - u16, - u32, - u64, - i8, - i16, - i32, - i64, -} - -impl Key for [T; N] { - const DUPLICATE: bool = false; - const CUSTOM_COMPARE: bool = false; - type Primary = Self; -} - -// TODO: temporary for now for `Key` bound, remove later. -impl Key for crate::types::PreRctOutputId { - const DUPLICATE: bool = false; - const CUSTOM_COMPARE: bool = false; +impl Key for T +where + T: Storable + Sized, +{ type Primary = Self; } diff --git a/database/src/lib.rs b/database/src/lib.rs index 4d0fe87f..f1d2b2eb 100644 --- a/database/src/lib.rs +++ b/database/src/lib.rs @@ -13,8 +13,8 @@ //! //! Each layer builds on-top of the previous. //! -//! As a user of `cuprate_database`, consider using the higher-level [`service`], -//! or at the very least [`ops`] instead of interacting with the database traits directly. +//! As a user of `cuprate_database`, consider using the higher-level [`service`] module, +//! or at the very least the [`ops`] module instead of interacting with the database traits directly. //! //! With that said, many database traits and internals (like [`DatabaseRo::get`]) are exposed. //! @@ -63,14 +63,10 @@ //! Note that `ConcreteEnv` itself is not a clonable type, //! it should be wrapped in [`std::sync::Arc`]. //! -//! TODO: we could also expose `ConcreteDatabase` if we're -//! going to be storing any databases in structs, to lessen -//! the generic `` pain. -//! -//! TODO: we could replace `ConcreteEnv` with `fn Env::open() -> impl Env`/ +//! //! //! # Feature flags //! The `service` module requires the `service` feature to be enabled. @@ -82,45 +78,66 @@ //! //! The default is `heed`. //! +//! `tracing` is always enabled and cannot be disabled via feature-flag. +//! +//! //! # Invariants when not using `service` //! `cuprate_database` can be used without the `service` feature enabled but -//! there are some things that must be kept in mind when doing so: +//! there are some things that must be kept in mind when doing so. //! -//! TODO: make pretty. these will need to be updated -//! as things change and as more backends are added. +//! Failing to uphold these invariants may cause panics. //! -//! 1. Memory map resizing (must resize as needed) -//! 1. Must not exceed `Config`'s maximum reader count -//! 1. Avoid many nested transactions -//! 1. `heed::MdbError::BadValSize` -//! 1. `heed::Error::InvalidDatabaseTyping` -//! 1. `heed::Error::BadOpenOptions` -//! 1. Encoding/decoding into `[u8]` +//! 1. `LMDB` requires the user to resize the memory map resizing (see [`RuntimeError::ResizeNeeded`] +//! 1. `LMDB` has a maximum reader transaction count, currently it is set to `128` +//! 1. `LMDB` has [maximum key/value byte size](http://www.lmdb.tech/doc/group__internal.html#gac929399f5d93cef85f874b9e9b1d09e0) which must not be exceeded //! -//! # Example -//! Simple usage of this crate. +//! # Examples +//! The below is an example of using `cuprate_database`'s +//! lowest API, i.e. using the database directly. +//! +//! For examples of the higher-level APIs, see: +//! - [`ops`] +//! - [`service`] //! //! ```rust //! use cuprate_database::{ -//! config::Config, //! ConcreteEnv, -//! Env, Key, TxRo, TxRw, -//! }; -//! use cuprate_types::{ -//! service::{ReadRequest, WriteRequest, Response}, +//! config::ConfigBuilder, +//! Env, EnvInner, +//! tables::{Tables, TablesMut}, +//! DatabaseRo, DatabaseRw, TxRo, TxRw, //! }; //! +//! # fn main() -> Result<(), Box> { //! // Create a configuration for the database environment. -//! let db_dir = tempfile::tempdir().unwrap(); -//! let config = Config::new(Some(db_dir.path().to_path_buf())); +//! let db_dir = tempfile::tempdir()?; +//! let config = ConfigBuilder::new() +//! .db_directory(db_dir.path().to_path_buf()) +//! .build(); //! -//! // Initialize the database thread-pool. +//! // Initialize the database environment. +//! let env = ConcreteEnv::open(config)?; //! -//! // TODO: -//! // 1. let (read_handle, write_handle) = cuprate_database::service::init(config).unwrap(); -//! // 2. Send write/read requests -//! // 3. Use some other `Env` functions -//! // 4. Shutdown +//! // Open up a transaction + tables for writing. +//! let env_inner = env.env_inner(); +//! let tx_rw = env_inner.tx_rw()?; +//! let mut tables = env_inner.open_tables_mut(&tx_rw)?; +//! +//! // ⚠️ Write data to the tables directly. +//! // (not recommended, use `ops` or `service`). +//! const KEY_IMAGE: [u8; 32] = [88; 32]; +//! tables.key_images_mut().put(&KEY_IMAGE, &())?; +//! +//! // Commit the data written. +//! drop(tables); +//! TxRw::commit(tx_rw)?; +//! +//! // Read the data, assert it is correct. +//! let tx_ro = env_inner.tx_ro()?; +//! let tables = env_inner.open_tables(&tx_ro)?; +//! let (key_image, _) = tables.key_images().first()?; +//! assert_eq!(key_image, KEY_IMAGE); +//! # Ok(()) } //! ``` //---------------------------------------------------------------------------------------------------- Lints @@ -180,7 +197,6 @@ unused_comparisons, nonstandard_style )] -#![allow(unreachable_code, unused_variables, dead_code, unused_imports)] // TODO: remove #![allow( // FIXME: this lint affects crates outside of // `database/` for some reason, allow for now. @@ -195,8 +211,8 @@ // with our `Env` + `RwLock` setup. clippy::significant_drop_tightening, - // TODO: should be removed after all `todo!()`'s are gone. - clippy::diverging_sub_expression, + // FIXME: good lint but is less clear in most cases. + clippy::items_after_statements, clippy::module_name_repetitions, clippy::module_inception, @@ -205,7 +221,16 @@ )] // Allow some lints when running in debug mode. #![cfg_attr(debug_assertions, allow(clippy::todo, clippy::multiple_crate_versions))] - +// Allow some lints in tests. +#![cfg_attr( + test, + allow( + clippy::cognitive_complexity, + clippy::needless_pass_by_value, + clippy::cast_possible_truncation, + clippy::too_many_lines + ) +)] // Only allow building 64-bit targets. // // This allows us to assume 64-bit @@ -249,8 +274,6 @@ pub mod resize; mod key; pub use key::Key; -mod macros; - mod storable; pub use storable::{Storable, StorableBytes, StorableVec}; diff --git a/database/src/macros.rs b/database/src/macros.rs deleted file mode 100644 index 7d4c2047..00000000 --- a/database/src/macros.rs +++ /dev/null @@ -1,17 +0,0 @@ -//! General macros used throughout `cuprate-database`. - -//---------------------------------------------------------------------------------------------------- Import - -//---------------------------------------------------------------------------------------------------- Constants - -//---------------------------------------------------------------------------------------------------- TYPE - -//---------------------------------------------------------------------------------------------------- IMPL - -//---------------------------------------------------------------------------------------------------- Trait Impl - -//---------------------------------------------------------------------------------------------------- Tests -#[cfg(test)] -mod test { - // use super::*; -} diff --git a/database/src/ops/alt_block.rs b/database/src/ops/alt_block.rs deleted file mode 100644 index 82f33aaf..00000000 --- a/database/src/ops/alt_block.rs +++ /dev/null @@ -1,29 +0,0 @@ -//! Alternative blocks. - -//---------------------------------------------------------------------------------------------------- Import - -//---------------------------------------------------------------------------------------------------- Free Functions -/// TODO -pub fn add_alt_block() { - todo!() -} - -/// TODO -pub fn get_alt_block() { - todo!() -} - -/// TODO -pub fn remove_alt_block() { - todo!() -} - -/// TODO -pub fn get_alt_block_count() { - todo!() -} - -/// TODO -pub fn drop_alt_blocks() { - todo!() -} diff --git a/database/src/ops/block.rs b/database/src/ops/block.rs index 77a80918..4f16cfde 100644 --- a/database/src/ops/block.rs +++ b/database/src/ops/block.rs @@ -1,41 +1,23 @@ -//! Blocks. +//! Blocks functions. //---------------------------------------------------------------------------------------------------- Import -use std::sync::Arc; - use bytemuck::TransparentWrapper; -use curve25519_dalek::{constants::ED25519_BASEPOINT_POINT, Scalar}; -use monero_serai::{ - block::Block, - transaction::{Input, Timelock, Transaction}, -}; +use monero_serai::block::Block; use cuprate_helper::map::{combine_low_high_bits_to_u128, split_u128_into_low_high_bits}; -use cuprate_types::{ExtendedBlockHeader, TransactionVerificationData, VerifiedBlockInformation}; +use cuprate_types::{ExtendedBlockHeader, VerifiedBlockInformation}; use crate::{ database::{DatabaseRo, DatabaseRw}, - env::EnvInner, error::RuntimeError, ops::{ blockchain::{chain_height, cumulative_generated_coins}, - key_image::{add_key_image, remove_key_image}, macros::doc_error, - output::{ - add_output, add_rct_output, get_rct_num_outputs, remove_output, remove_rct_output, - }, - tx::{add_tx, get_num_tx, remove_tx}, - }, - tables::{ - BlockBlobs, BlockHeights, BlockInfos, KeyImages, NumOutputs, Outputs, PrunableHashes, - PrunableTxBlobs, PrunedTxBlobs, RctOutputs, Tables, TablesMut, TxHeights, TxIds, - TxUnlockTime, - }, - transaction::{TxRo, TxRw}, - types::{ - AmountIndex, BlockHash, BlockHeight, BlockInfo, KeyImage, Output, OutputFlags, - PreRctOutputId, RctOutput, TxHash, + output::get_rct_num_outputs, + tx::{add_tx, remove_tx}, }, + tables::{BlockHeights, BlockInfos, Tables, TablesMut}, + types::{BlockHash, BlockHeight, BlockInfo}, StorableVec, }; @@ -66,9 +48,11 @@ pub fn add_block( // Cast height to `u32` for storage (handled at top of function). // Panic (should never happen) instead of allowing DB corruption. // - let Ok(height) = u32::try_from(block.height) else { - panic!("block.height ({}) > u32::MAX", block.height); - }; + assert!( + u32::try_from(block.height).is_ok(), + "block.height ({}) > u32::MAX", + block.height, + ); let chain_height = chain_height(tables.block_heights())?; assert_eq!( @@ -144,8 +128,11 @@ pub fn add_block( //---------------------------------------------------------------------------------------------------- `pop_block` /// Remove the top/latest block from the database. /// -/// The removed block's height and hash are returned. +/// The removed block's data is returned. #[doc = doc_error!()] +/// +/// In `pop_block()`'s case, [`RuntimeError::KeyNotFound`] +/// will be returned if there are no blocks left. // no inline, too big pub fn pop_block( tables: &mut impl TablesMut, @@ -254,7 +241,12 @@ pub fn get_block_height( } /// Check if a block exists in the database. -#[doc = doc_error!()] +/// +/// # Errors +/// Note that this will never return `Err(RuntimeError::KeyNotFound)`, +/// as in that case, `Ok(false)` will be returned. +/// +/// Other errors may still occur. #[inline] pub fn block_exists( block_hash: &BlockHash, @@ -271,16 +263,16 @@ pub fn block_exists( clippy::too_many_lines )] mod test { - use hex_literal::hex; use pretty_assertions::assert_eq; - use cuprate_test_utils::data::{block_v16_tx0, block_v1_tx2, block_v9_tx3, tx_v2_rct3}; + use cuprate_test_utils::data::{block_v16_tx0, block_v1_tx2, block_v9_tx3}; use super::*; use crate::{ ops::tx::{get_tx, tx_exists}, tests::{assert_all_tables_are_empty, tmp_concrete_env, AssertTableLen}, - Env, + transaction::TxRw, + Env, EnvInner, }; /// Tests all above block functions. @@ -292,7 +284,7 @@ mod test { /// stored and retrieved is the same. #[test] fn all_block_functions() { - let (env, tmp) = tmp_concrete_env(); + let (env, _tmp) = tmp_concrete_env(); let env_inner = env.env_inner(); assert_all_tables_are_empty(&env); @@ -417,7 +409,7 @@ mod test { for block_hash in block_hashes.into_iter().rev() { println!("pop_block(): block_hash: {}", hex::encode(block_hash)); - let (popped_height, popped_hash, popped_block) = pop_block(&mut tables).unwrap(); + let (_popped_height, popped_hash, _popped_block) = pop_block(&mut tables).unwrap(); assert_eq!(block_hash, popped_hash); @@ -438,7 +430,7 @@ mod test { #[test] #[should_panic(expected = "block.height (4294967296) > u32::MAX")] fn block_height_gt_u32_max() { - let (env, tmp) = tmp_concrete_env(); + let (env, _tmp) = tmp_concrete_env(); let env_inner = env.env_inner(); assert_all_tables_are_empty(&env); @@ -457,7 +449,7 @@ mod test { expected = "assertion `left == right` failed: block.height (123) != chain_height (1)\n left: 123\n right: 1" )] fn block_height_not_chain_height() { - let (env, tmp) = tmp_concrete_env(); + let (env, _tmp) = tmp_concrete_env(); let env_inner = env.env_inner(); assert_all_tables_are_empty(&env); diff --git a/database/src/ops/blockchain.rs b/database/src/ops/blockchain.rs index 92089956..ce9cd69d 100644 --- a/database/src/ops/blockchain.rs +++ b/database/src/ops/blockchain.rs @@ -1,22 +1,12 @@ -//! Blockchain. +//! Blockchain functions - chain height, generated coins, etc. //---------------------------------------------------------------------------------------------------- Import -use monero_serai::transaction::Timelock; - -use cuprate_types::VerifiedBlockInformation; - use crate::{ - database::{DatabaseRo, DatabaseRw}, - env::EnvInner, + database::DatabaseRo, error::RuntimeError, ops::macros::doc_error, - tables::{ - BlockBlobs, BlockHeights, BlockInfos, KeyImages, NumOutputs, Outputs, PrunableHashes, - PrunableTxBlobs, PrunedTxBlobs, RctOutputs, Tables, TablesMut, TxHeights, TxIds, - TxUnlockTime, - }, - transaction::{TxRo, TxRw}, - types::{BlockHash, BlockHeight, BlockInfo, KeyImage, Output, PreRctOutputId, RctOutput}, + tables::{BlockHeights, BlockInfos}, + types::BlockHeight, }; //---------------------------------------------------------------------------------------------------- Free Functions @@ -88,21 +78,18 @@ pub fn cumulative_generated_coins( //---------------------------------------------------------------------------------------------------- Tests #[cfg(test)] -#[allow(clippy::significant_drop_tightening)] mod test { - use hex_literal::hex; use pretty_assertions::assert_eq; - use cuprate_test_utils::data::{block_v16_tx0, block_v1_tx2, block_v9_tx3, tx_v2_rct3}; + use cuprate_test_utils::data::{block_v16_tx0, block_v1_tx2, block_v9_tx3}; use super::*; use crate::{ - ops::{ - block::add_block, - tx::{get_tx, tx_exists}, - }, + ops::block::add_block, + tables::Tables, tests::{assert_all_tables_are_empty, tmp_concrete_env, AssertTableLen}, - Env, + transaction::TxRw, + Env, EnvInner, }; /// Tests all above functions. @@ -113,9 +100,8 @@ mod test { /// It simply tests if the proper tables are mutated, and if the data /// stored and retrieved is the same. #[test] - #[allow(clippy::cognitive_complexity, clippy::cast_possible_truncation)] fn all_blockchain_functions() { - let (env, tmp) = tmp_concrete_env(); + let (env, _tmp) = tmp_concrete_env(); let env_inner = env.env_inner(); assert_all_tables_are_empty(&env); diff --git a/database/src/ops/key_image.rs b/database/src/ops/key_image.rs index 0938447f..5d0786c3 100644 --- a/database/src/ops/key_image.rs +++ b/database/src/ops/key_image.rs @@ -1,24 +1,12 @@ -//! Spent keys. +//! Key image functions. //---------------------------------------------------------------------------------------------------- Import -use monero_serai::transaction::{Timelock, Transaction}; - -use cuprate_types::{OutputOnChain, VerifiedBlockInformation}; - use crate::{ - database::{DatabaseIter, DatabaseRo, DatabaseRw}, - env::EnvInner, + database::{DatabaseRo, DatabaseRw}, error::RuntimeError, ops::macros::{doc_add_block_inner_invariant, doc_error}, - tables::{ - BlockBlobs, BlockHeights, BlockInfos, KeyImages, NumOutputs, Outputs, PrunableHashes, - PrunableTxBlobs, PrunedTxBlobs, RctOutputs, Tables, TablesMut, TxHeights, TxIds, - TxUnlockTime, - }, - transaction::{TxRo, TxRw}, - types::{ - BlockHash, BlockHeight, BlockInfo, KeyImage, Output, PreRctOutputId, RctOutput, TxHash, - }, + tables::KeyImages, + types::KeyImage, }; //---------------------------------------------------------------------------------------------------- Key image functions @@ -56,16 +44,15 @@ pub fn key_image_exists( //---------------------------------------------------------------------------------------------------- Tests #[cfg(test)] -#[allow(clippy::significant_drop_tightening, clippy::cognitive_complexity)] mod test { use hex_literal::hex; - use pretty_assertions::assert_eq; use super::*; use crate::{ - ops::tx::{get_tx, tx_exists}, + tables::{Tables, TablesMut}, tests::{assert_all_tables_are_empty, tmp_concrete_env, AssertTableLen}, - Env, + transaction::TxRw, + Env, EnvInner, }; /// Tests all above key-image functions. @@ -77,7 +64,7 @@ mod test { /// stored and retrieved is the same. #[test] fn all_key_image_functions() { - let (env, tmp) = tmp_concrete_env(); + let (env, _tmp) = tmp_concrete_env(); let env_inner = env.env_inner(); assert_all_tables_are_empty(&env); diff --git a/database/src/ops/macros.rs b/database/src/ops/macros.rs index 6a219e6d..b7cdba47 100644 --- a/database/src/ops/macros.rs +++ b/database/src/ops/macros.rs @@ -8,7 +8,7 @@ macro_rules! doc_error { () => { r#"# Errors -This function returns [`RuntimeError::KeyNotFound`] if the input doesn't exist or other `RuntimeError`'s on database errors."# +This function returns [`RuntimeError::KeyNotFound`] if the input (if applicable) doesn't exist or other `RuntimeError`'s on database errors."# }; } pub(super) use doc_error; diff --git a/database/src/ops/mod.rs b/database/src/ops/mod.rs index 658be45f..9f48bd65 100644 --- a/database/src/ops/mod.rs +++ b/database/src/ops/mod.rs @@ -31,16 +31,75 @@ //! # Sub-functions //! The main functions within this module are mostly within the [`block`] module. //! -//! Practically speaking, you should only be using 2 functions: +//! Practically speaking, you should only be using 2 functions for mutation: //! - [`add_block`](block::add_block) //! - [`pop_block`](block::pop_block) //! //! The `block` functions are "parent" functions, calling other -//! sub-functions such as [`add_output()`](output::add_output). `add_output()` -//! itself only modifies output-related tables, while the `block` "parent" functions -//! (like `add_block` and `pop_block`) modify _everything_ that is required. +//! sub-functions such as [`add_output()`](output::add_output). +//! +//! `add_output()` itself only modifies output-related tables, while the `block` "parent" +//! functions (like `add_block` and `pop_block`) modify all tables required. +//! +//! `add_block()` makes sure all data related to the input is mutated, while +//! this sub-function _do not_, it specifically mutates _particular_ tables. +//! +//! When calling this sub-functions, ensure that either: +//! 1. This effect (incomplete database mutation) is what is desired, or that... +//! 2. ...the other tables will also be mutated to a correct state +//! +//! # Example +//! Simple usage of `ops`. +//! +//! ```rust +//! use hex_literal::hex; +//! +//! use cuprate_test_utils::data::block_v16_tx0; +//! +//! use cuprate_database::{ +//! ConcreteEnv, +//! config::ConfigBuilder, +//! Env, EnvInner, +//! tables::{Tables, TablesMut}, +//! DatabaseRo, DatabaseRw, TxRo, TxRw, +//! ops::block::{add_block, pop_block}, +//! }; +//! +//! # fn main() -> Result<(), Box> { +//! // Create a configuration for the database environment. +//! let db_dir = tempfile::tempdir()?; +//! let config = ConfigBuilder::new() +//! .db_directory(db_dir.path().to_path_buf()) +//! .build(); +//! +//! // Initialize the database environment. +//! let env = ConcreteEnv::open(config)?; +//! +//! // Open up a transaction + tables for writing. +//! let env_inner = env.env_inner(); +//! let tx_rw = env_inner.tx_rw()?; +//! let mut tables = env_inner.open_tables_mut(&tx_rw)?; +//! +//! // Write a block to the database. +//! let mut block = block_v16_tx0().clone(); +//! # block.height = 0; +//! add_block(&block, &mut tables)?; +//! +//! // Commit the data written. +//! drop(tables); +//! TxRw::commit(tx_rw)?; +//! +//! // Read the data, assert it is correct. +//! let tx_rw = env_inner.tx_rw()?; +//! let mut tables = env_inner.open_tables_mut(&tx_rw)?; +//! let (height, hash, serai_block) = pop_block(&mut tables)?; +//! +//! assert_eq!(height, 0); +//! assert_eq!(serai_block, block.block); +//! assert_eq!(hash, hex!("43bd1f2b6556dcafa413d8372974af59e4e8f37dbf74dc6b2a9b7212d0577428")); +//! # Ok(()) } +//! ``` -// pub mod alt_block; // TODO: is this needed? pub mod block; pub mod blockchain; pub mod key_image; diff --git a/database/src/ops/output.rs b/database/src/ops/output.rs index e2902458..5b7620e4 100644 --- a/database/src/ops/output.rs +++ b/database/src/ops/output.rs @@ -1,30 +1,18 @@ -//! Outputs. +//! Output functions. + +//---------------------------------------------------------------------------------------------------- Import +use curve25519_dalek::{constants::ED25519_BASEPOINT_POINT, edwards::CompressedEdwardsY, Scalar}; +use monero_serai::{transaction::Timelock, H}; use cuprate_helper::map::u64_to_timelock; -use curve25519_dalek::{constants::ED25519_BASEPOINT_POINT, edwards::CompressedEdwardsY, Scalar}; -//---------------------------------------------------------------------------------------------------- Import -use monero_serai::{ - transaction::{Timelock, Transaction}, - H, -}; - -use cuprate_types::{OutputOnChain, VerifiedBlockInformation}; +use cuprate_types::OutputOnChain; use crate::{ - database::{DatabaseIter, DatabaseRo, DatabaseRw}, - env::EnvInner, + database::{DatabaseRo, DatabaseRw}, error::RuntimeError, ops::macros::{doc_add_block_inner_invariant, doc_error}, - tables::{ - BlockBlobs, BlockHeights, BlockInfos, KeyImages, NumOutputs, Outputs, PrunableHashes, - PrunableTxBlobs, PrunedTxBlobs, RctOutputs, Tables, TablesMut, TxHeights, TxIds, - TxUnlockTime, - }, - transaction::{TxRo, TxRw}, - types::{ - Amount, AmountIndex, BlockHash, BlockHeight, BlockInfo, KeyImage, Output, OutputFlags, - PreRctOutputId, RctOutput, TxHash, - }, + tables::{Outputs, RctOutputs, Tables, TablesMut, TxUnlockTime}, + types::{Amount, AmountIndex, Output, OutputFlags, PreRctOutputId, RctOutput}, }; //---------------------------------------------------------------------------------------------------- Pre-RCT Outputs @@ -257,15 +245,15 @@ pub fn id_to_output_on_chain( //---------------------------------------------------------------------------------------------------- Tests #[cfg(test)] -#[allow(clippy::significant_drop_tightening, clippy::cognitive_complexity)] mod test { use super::*; use crate::{ + tables::{Tables, TablesMut}, tests::{assert_all_tables_are_empty, tmp_concrete_env, AssertTableLen}, types::OutputFlags, - Env, + Env, EnvInner, }; - use cuprate_test_utils::data::{tx_v1_sig2, tx_v2_rct3}; + use pretty_assertions::assert_eq; /// Dummy `Output`. @@ -297,7 +285,7 @@ mod test { /// stored and retrieved is the same. #[test] fn all_output_functions() { - let (env, tmp) = tmp_concrete_env(); + let (env, _tmp) = tmp_concrete_env(); let env_inner = env.env_inner(); assert_all_tables_are_empty(&env); diff --git a/database/src/ops/property.rs b/database/src/ops/property.rs index 563bc8c0..279c3552 100644 --- a/database/src/ops/property.rs +++ b/database/src/ops/property.rs @@ -1,57 +1,39 @@ -//! Properties. +//! Database properties functions - version, pruning, etc. +//! +//! SOMEDAY: the database `properties` table is not yet implemented. //---------------------------------------------------------------------------------------------------- Import use monero_pruning::PruningSeed; -use monero_serai::transaction::{Timelock, Transaction}; -use cuprate_types::{OutputOnChain, VerifiedBlockInformation}; - -use crate::{ - database::{DatabaseIter, DatabaseRo, DatabaseRw}, - env::EnvInner, - error::RuntimeError, - ops::macros::{doc_add_block_inner_invariant, doc_error}, - tables::{ - BlockBlobs, BlockHeights, BlockInfos, KeyImages, NumOutputs, Outputs, PrunableHashes, - PrunableTxBlobs, PrunedTxBlobs, RctOutputs, Tables, TablesMut, TxHeights, TxIds, - TxUnlockTime, - }, - transaction::{TxRo, TxRw}, - types::{ - BlockHash, BlockHeight, BlockInfo, KeyImage, Output, PreRctOutputId, RctOutput, TxHash, - TxId, - }, -}; +use crate::{error::RuntimeError, ops::macros::doc_error}; //---------------------------------------------------------------------------------------------------- Free Functions -/// TODO +/// SOMEDAY /// -#[doc = doc_add_block_inner_invariant!()] #[doc = doc_error!()] /// /// # Example /// ```rust /// # use cuprate_database::{*, tables::*, ops::block::*, ops::tx::*}; -/// // TODO +/// // SOMEDAY /// ``` #[inline] pub const fn get_blockchain_pruning_seed() -> Result { - // TODO: impl pruning. + // SOMEDAY: impl pruning. // We need a DB properties table. Ok(PruningSeed::NotPruned) } -/// TODO +/// SOMEDAY /// -#[doc = doc_add_block_inner_invariant!()] #[doc = doc_error!()] /// /// # Example /// ```rust /// # use cuprate_database::{*, tables::*, ops::block::*, ops::tx::*}; -/// // TODO +/// // SOMEDAY /// ``` #[inline] pub const fn db_version() -> Result { - // TODO: We need a DB properties table. + // SOMEDAY: We need a DB properties table. Ok(crate::constants::DATABASE_VERSION) } diff --git a/database/src/ops/tx.rs b/database/src/ops/tx.rs index a76de399..b4f2984b 100644 --- a/database/src/ops/tx.rs +++ b/database/src/ops/tx.rs @@ -1,40 +1,25 @@ -//! Transactions. +//! Transaction functions. //---------------------------------------------------------------------------------------------------- Import use bytemuck::TransparentWrapper; use curve25519_dalek::{constants::ED25519_BASEPOINT_POINT, Scalar}; use monero_serai::transaction::{Input, Timelock, Transaction}; -use cuprate_types::{OutputOnChain, TransactionVerificationData, VerifiedBlockInformation}; -use monero_pruning::PruningSeed; - use crate::{ - database::{DatabaseIter, DatabaseRo, DatabaseRw}, - env::EnvInner, + database::{DatabaseRo, DatabaseRw}, error::RuntimeError, ops::{ - blockchain::chain_height, + key_image::{add_key_image, remove_key_image}, macros::{doc_add_block_inner_invariant, doc_error}, - property::get_blockchain_pruning_seed, - }, - tables::{ - BlockBlobs, BlockHeights, BlockInfos, KeyImages, NumOutputs, Outputs, PrunableHashes, - PrunableTxBlobs, PrunedTxBlobs, RctOutputs, Tables, TablesMut, TxBlobs, TxHeights, TxIds, - TxUnlockTime, - }, - transaction::{TxRo, TxRw}, - types::{ - AmountIndices, BlockHash, BlockHeight, BlockInfo, KeyImage, Output, OutputFlags, - PreRctOutputId, RctOutput, TxBlob, TxHash, TxId, + output::{ + add_output, add_rct_output, get_rct_num_outputs, remove_output, remove_rct_output, + }, }, + tables::{TablesMut, TxBlobs, TxIds}, + types::{BlockHeight, Output, OutputFlags, PreRctOutputId, RctOutput, TxHash, TxId}, StorableVec, }; -use super::{ - key_image::{add_key_image, remove_key_image}, - output::{add_output, add_rct_output, get_rct_num_outputs, remove_output, remove_rct_output}, -}; - //---------------------------------------------------------------------------------------------------- Private /// Add a [`Transaction`] (and related data) to the database. /// @@ -196,7 +181,7 @@ pub fn add_tx( /// Remove a transaction from the database with its [`TxHash`]. /// -/// This returns the [`TxId`] and [`TxBlob`] of the removed transaction. +/// This returns the [`TxId`] and [`TxBlob`](crate::types::TxBlob) of the removed transaction. /// #[doc = doc_add_block_inner_invariant!()] /// @@ -256,7 +241,7 @@ pub fn remove_tx( //------------------------------------------------------ Outputs // Remove each output in the transaction. - for (i, output) in tx.prefix.outputs.iter().enumerate() { + for output in &tx.prefix.outputs { // Outputs with clear amounts. if let Some(amount) = output.amount { // RingCT miner outputs. @@ -338,12 +323,13 @@ pub fn tx_exists( //---------------------------------------------------------------------------------------------------- Tests #[cfg(test)] -#[allow(clippy::significant_drop_tightening)] mod test { use super::*; use crate::{ + tables::Tables, tests::{assert_all_tables_are_empty, tmp_concrete_env, AssertTableLen}, - Env, + transaction::TxRw, + Env, EnvInner, }; use cuprate_test_utils::data::{tx_v1_sig0, tx_v1_sig2, tx_v2_rct3}; use pretty_assertions::assert_eq; @@ -351,7 +337,7 @@ mod test { /// Tests all above tx functions when only inputting `Transaction` data (no Block). #[test] fn all_tx_functions() { - let (env, tmp) = tmp_concrete_env(); + let (env, _tmp) = tmp_concrete_env(); let env_inner = env.env_inner(); assert_all_tables_are_empty(&env); diff --git a/database/src/resize.rs b/database/src/resize.rs index ac54781c..cf185029 100644 --- a/database/src/resize.rs +++ b/database/src/resize.rs @@ -1,7 +1,7 @@ //! Database memory map resizing algorithms. //! //! This modules contains [`ResizeAlgorithm`] which determines how the -//! [`ConcreteEnv`](crate::ConcreteEnv) resizes it's memory map when needing more space. +//! [`ConcreteEnv`](crate::ConcreteEnv) resizes its memory map when needing more space. //! This value is in [`Config`](crate::config::Config) and can be selected at runtime. //! //! Although, it is only used by `ConcreteEnv` if [`Env::MANUAL_RESIZE`](crate::env::Env::MANUAL_RESIZE) is `true`. @@ -27,12 +27,12 @@ use std::{num::NonZeroUsize, sync::OnceLock}; /// The function/algorithm used by the /// database when resizing the memory map. /// -/// # TODO -/// We could test around with different algorithms. -/// Calling `heed::Env::resize` is surprisingly fast, -/// around `0.0000082s` on my machine. We could probably -/// get away with smaller and more frequent resizes. -/// **With the caveat being we are taking a `WriteGuard` to a `RwLock`.** +// # SOMEDAY +// We could test around with different algorithms. +// Calling `heed::Env::resize` is surprisingly fast, +// around `0.0000082s` on my machine. We could probably +// get away with smaller and more frequent resizes. +// **With the caveat being we are taking a `WriteGuard` to a `RwLock`.** #[derive(Copy, Clone, Debug, PartialEq, PartialOrd)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] pub enum ResizeAlgorithm { diff --git a/database/src/service/free.rs b/database/src/service/free.rs index 53d0d095..fb40a065 100644 --- a/database/src/service/free.rs +++ b/database/src/service/free.rs @@ -6,7 +6,7 @@ use std::sync::Arc; use crate::{ config::Config, error::InitError, - service::{write::DatabaseWriter, DatabaseReadHandle, DatabaseWriteHandle}, + service::{DatabaseReadHandle, DatabaseWriteHandle}, ConcreteEnv, Env, }; @@ -20,21 +20,11 @@ use crate::{ /// /// # Errors /// This will forward the error if [`Env::open`] failed. -// -// INVARIANT: -// `cuprate_database` depends on the fact that this is the only -// function that hands out the handles. After that, they can be -// cloned, however they must eventually be dropped and shouldn't -// be leaked. -// -// As the reader thread-pool and writer thread both rely on the -// disconnection (drop) of these channels for shutdown behavior, -// leaking these handles could cause data to not get flushed to disk. pub fn init(config: Config) -> Result<(DatabaseReadHandle, DatabaseWriteHandle), InitError> { let reader_threads = config.reader_threads; // Initialize the database itself. - let db: Arc = Arc::new(ConcreteEnv::open(config)?); + let db = Arc::new(ConcreteEnv::open(config)?); // Spawn the Reader thread pool and Writer. let readers = DatabaseReadHandle::init(&db, reader_threads); diff --git a/database/src/service/mod.rs b/database/src/service/mod.rs index 88773052..ca5c9e6f 100644 --- a/database/src/service/mod.rs +++ b/database/src/service/mod.rs @@ -50,13 +50,69 @@ //! This channel can be `.await`ed upon to (eventually) receive //! the corresponding `Response` to your `Request`. //! -//! -//! //! [req_r]: cuprate_types::service::ReadRequest //! //! [req_w]: cuprate_types::service::WriteRequest //! //! [resp]: cuprate_types::service::Response +//! +//! # Example +//! Simple usage of `service`. +//! +//! ```rust +//! use hex_literal::hex; +//! use tower::{Service, ServiceExt}; +//! +//! use cuprate_types::service::{ReadRequest, WriteRequest, Response}; +//! use cuprate_test_utils::data::block_v16_tx0; +//! +//! use cuprate_database::{ConcreteEnv, config::ConfigBuilder, Env}; +//! +//! # #[tokio::main] +//! # async fn main() -> Result<(), Box> { +//! // Create a configuration for the database environment. +//! let db_dir = tempfile::tempdir()?; +//! let config = ConfigBuilder::new() +//! .db_directory(db_dir.path().to_path_buf()) +//! .build(); +//! +//! // Initialize the database thread-pool. +//! let (mut read_handle, mut write_handle) = cuprate_database::service::init(config)?; +//! +//! // Prepare a request to write block. +//! let mut block = block_v16_tx0().clone(); +//! # block.height = 0 as u64; // must be 0th height or panic in `add_block()` +//! let request = WriteRequest::WriteBlock(block); +//! +//! // Send the request. +//! // We receive back an `async` channel that will +//! // eventually yield the result when `service` +//! // is done writing the block. +//! let response_channel = write_handle.ready().await?.call(request); +//! +//! // Block write was OK. +//! let response = response_channel.await?; +//! assert_eq!(response, Response::WriteBlockOk); +//! +//! // Now, let's try getting the block hash +//! // of the block we just wrote. +//! let request = ReadRequest::BlockHash(0); +//! let response_channel = read_handle.ready().await?.call(request); +//! let response = response_channel.await?; +//! assert_eq!( +//! response, +//! Response::BlockHash( +//! hex!("43bd1f2b6556dcafa413d8372974af59e4e8f37dbf74dc6b2a9b7212d0577428") +//! ) +//! ); +//! +//! // This causes the writer thread on the +//! // other side of this handle to exit... +//! drop(write_handle); +//! // ...and this causes the reader thread-pool to exit. +//! drop(read_handle); +//! # Ok(()) } +//! ``` mod read; pub use read::DatabaseReadHandle; diff --git a/database/src/service/read.rs b/database/src/service/read.rs index b178bd10..e53c7f88 100644 --- a/database/src/service/read.rs +++ b/database/src/service/read.rs @@ -3,18 +3,12 @@ //---------------------------------------------------------------------------------------------------- Import use std::{ collections::{HashMap, HashSet}, - num::NonZeroUsize, - ops::Range, - sync::{Arc, RwLock}, + sync::Arc, task::{Context, Poll}, }; -use cfg_if::cfg_if; -use crossbeam::channel::Receiver; -use curve25519_dalek::{constants::ED25519_BASEPOINT_POINT, edwards::CompressedEdwardsY, Scalar}; use futures::{channel::oneshot, ready}; -use monero_serai::{transaction::Timelock, H}; -use rayon::iter::{IntoParallelIterator, IntoParallelRefIterator, ParallelIterator}; +use rayon::iter::{IntoParallelIterator, ParallelIterator}; use thread_local::ThreadLocal; use tokio::sync::{OwnedSemaphorePermit, Semaphore}; use tokio_util::sync::PollSemaphore; @@ -27,21 +21,16 @@ use cuprate_types::{ use crate::{ config::ReaderThreads, - constants::DATABASE_CORRUPT_MSG, error::RuntimeError, ops::{ block::{get_block_extended_header_from_height, get_block_info}, blockchain::{cumulative_generated_coins, top_block_height}, key_image::key_image_exists, - output::{ - get_output, get_rct_output, id_to_output_on_chain, output_to_output_on_chain, - rct_output_to_output_on_chain, - }, + output::id_to_output_on_chain, }, service::types::{ResponseReceiver, ResponseResult, ResponseSender}, - tables::{BlockHeights, BlockInfos, KeyImages, NumOutputs, Outputs, Tables}, - types::{Amount, AmountIndex, BlockHeight, KeyImage, OutputFlags, PreRctOutputId}, - unsafe_sendable::UnsafeSendable, + tables::{BlockHeights, BlockInfos, Tables}, + types::{Amount, AmountIndex, BlockHeight, KeyImage, PreRctOutputId}, ConcreteEnv, DatabaseRo, Env, EnvInner, }; @@ -208,7 +197,7 @@ fn map_request( ) { use ReadRequest as R; - /* TODO: pre-request handling, run some code for each request? */ + /* SOMEDAY: pre-request handling, run some code for each request? */ let response = match request { R::BlockExtendedHeader(block) => block_extended_header(env, block), @@ -226,7 +215,7 @@ fn map_request( println!("database reader failed to send response: {e:?}"); } - /* TODO: post-request handling, run some code for each request? */ + /* SOMEDAY: post-request handling, run some code for each request? */ } //---------------------------------------------------------------------------------------------------- Thread Local @@ -294,7 +283,7 @@ macro_rules! get_tables { // All functions below assume that this is the case, such that // `par_*()` functions will not block the _global_ rayon thread-pool. -// TODO: implement multi-transaction read atomicity. +// FIXME: implement multi-transaction read atomicity. // . /// [`ReadRequest::BlockExtendedHeader`]. @@ -481,7 +470,7 @@ fn check_k_is_not_spent(env: &ConcreteEnv, key_images: HashSet) -> Res key_image_exists(&key_image, tables.key_images()) }; - // TODO: + // FIXME: // Create/use `enum cuprate_types::Exist { Does, DoesNot }` // or similar instead of `bool` for clarity. // diff --git a/database/src/service/tests.rs b/database/src/service/tests.rs index 5f09b197..77c10cdd 100644 --- a/database/src/service/tests.rs +++ b/database/src/service/tests.rs @@ -1,25 +1,14 @@ //! `crate::service` tests. //! //! This module contains general tests for the `service` implementation. -//! -//! Testing a thread-pool is slightly more complicated, -//! so this file provides TODO. // This is only imported on `#[cfg(test)]` in `mod.rs`. - -#![allow( - clippy::significant_drop_tightening, - clippy::await_holding_lock, - clippy::too_many_lines -)] +#![allow(clippy::await_holding_lock, clippy::too_many_lines)] //---------------------------------------------------------------------------------------------------- Use use std::{ - collections::{hash_map::Entry, HashMap, HashSet}, - sync::{ - atomic::{AtomicU64, Ordering}, - Arc, - }, + collections::{HashMap, HashSet}, + sync::Arc, }; use pretty_assertions::assert_eq; @@ -28,20 +17,20 @@ use tower::{Service, ServiceExt}; use cuprate_test_utils::data::{block_v16_tx0, block_v1_tx2, block_v9_tx3}; use cuprate_types::{ service::{ReadRequest, Response, WriteRequest}, - ExtendedBlockHeader, OutputOnChain, VerifiedBlockInformation, + OutputOnChain, VerifiedBlockInformation, }; use crate::{ - config::Config, + config::ConfigBuilder, ops::{ block::{get_block_extended_header_from_height, get_block_info}, - blockchain::{chain_height, top_block_height}, - output::{get_output, id_to_output_on_chain, output_to_output_on_chain}, + blockchain::chain_height, + output::id_to_output_on_chain, }, service::{init, DatabaseReadHandle, DatabaseWriteHandle}, - tables::{KeyImages, Tables, TablesIter}, + tables::{Tables, TablesIter}, tests::AssertTableLen, - types::{Amount, AmountIndex, KeyImage, PreRctOutputId}, + types::{Amount, AmountIndex, PreRctOutputId}, ConcreteEnv, DatabaseIter, DatabaseRo, Env, EnvInner, RuntimeError, }; @@ -54,7 +43,10 @@ fn init_service() -> ( tempfile::TempDir, ) { let tempdir = tempfile::tempdir().unwrap(); - let config = Config::low_power(Some(tempdir.path().into())); + let config = ConfigBuilder::new() + .db_directory(tempdir.path().into()) + .low_power() + .build(); let (reader, writer) = init(config).unwrap(); let env = reader.env().clone(); (reader, writer, env, tempdir) @@ -169,7 +161,7 @@ async fn test_template( #[allow(clippy::cast_possible_truncation)] Ok(count) => (*amount, count as usize), Err(RuntimeError::KeyNotFound) => (*amount, 0), - Err(e) => panic!(), + Err(e) => panic!("{e:?}"), }) .collect::>(), )); @@ -196,7 +188,7 @@ async fn test_template( println!("response: {response:#?}, expected_response: {expected_response:#?}"); match response { Ok(resp) => assert_eq!(resp, expected_response.unwrap()), - Err(ref e) => assert!(matches!(response, expected_response)), + Err(_) => assert!(matches!(response, _expected_response)), } } @@ -303,7 +295,7 @@ async fn test_template( /// If this test fails, something is very wrong. #[test] fn init_drop() { - let (reader, writer, env, _tempdir) = init_service(); + let (_reader, _writer, _env, _tempdir) = init_service(); } /// Assert write/read correctness of [`block_v1_tx2`]. diff --git a/database/src/service/write.rs b/database/src/service/write.rs index 3cd7e80c..d6747e97 100644 --- a/database/src/service/write.rs +++ b/database/src/service/write.rs @@ -15,7 +15,6 @@ use cuprate_types::{ }; use crate::{ - constants::DATABASE_CORRUPT_MSG, env::{Env, EnvInner}, error::RuntimeError, service::types::{ResponseReceiver, ResponseResult, ResponseSender}, @@ -136,7 +135,6 @@ impl DatabaseWriter { /// How many times should we retry handling the request on resize errors? /// /// This is 1 on automatically resizing databases, meaning there is only 1 iteration. - #[allow(clippy::items_after_statements)] const REQUEST_RETRY_LIMIT: usize = if ConcreteEnv::MANUAL_RESIZE { 3 } else { 1 }; // Map [`Request`]'s to specific database functions. @@ -152,7 +150,7 @@ impl DatabaseWriter { // to represent this retry logic with recursive // functions instead of a loop. 'retry: for retry in 0..REQUEST_RETRY_LIMIT { - // TODO: will there be more than 1 write request? + // FIXME: will there be more than 1 write request? // this won't have to be an enum. let response = match &request { WriteRequest::WriteBlock(block) => write_block(&self.env, block), @@ -208,12 +206,6 @@ impl DatabaseWriter { // - ...retry until panic unreachable!(); } - - // The only case the ['main] loop breaks should be a: - // - direct function return - // - panic - // anything below should be unreachable. - unreachable!(); } } @@ -239,14 +231,13 @@ fn write_block(env: &ConcreteEnv, block: &VerifiedBlockInformation) -> ResponseR match result { Ok(()) => { - tx_rw.commit()?; + TxRw::commit(tx_rw)?; Ok(Response::WriteBlockOk) } Err(e) => { // INVARIANT: ensure database atomicity by aborting // the transaction on `add_block()` failures. - tx_rw - .abort() + TxRw::abort(tx_rw) .expect("could not maintain database atomicity by aborting write transaction"); Err(e) } diff --git a/database/src/storable.rs b/database/src/storable.rs index ac3f263a..f259523f 100644 --- a/database/src/storable.rs +++ b/database/src/storable.rs @@ -1,15 +1,9 @@ //! (De)serialization for table keys & values. //---------------------------------------------------------------------------------------------------- Import -use std::{ - borrow::{Borrow, Cow}, - char::ToLowercase, - fmt::Debug, - io::{Read, Write}, - sync::Arc, -}; +use std::{borrow::Borrow, fmt::Debug}; -use bytemuck::{Pod, Zeroable}; +use bytemuck::Pod; use bytes::Bytes; //---------------------------------------------------------------------------------------------------- Storable @@ -25,16 +19,14 @@ use bytes::Bytes; /// Any type that implements: /// - [`bytemuck::Pod`] /// - [`Debug`] -/// - [`ToOwned`] /// /// will automatically implement [`Storable`]. /// /// This includes: /// - Most primitive types /// - All types in [`tables`](crate::tables) -/// - Slices, e.g, `[T] where T: Storable` /// -/// See [`StorableVec`] for storing slices of `T: Storable`. +/// See [`StorableVec`] & [`StorableBytes`] for storing slices of `T: Storable`. /// /// ```rust /// # use cuprate_database::*; @@ -142,6 +134,7 @@ where /// /// This is needed as `impl Storable for Vec` runs into impl conflicts. /// +/// # Example /// ```rust /// # use cuprate_database::*; /// //---------------------------------------------------- u8 @@ -284,7 +277,7 @@ mod test { println!("serialized: {se:?}, deserialized: {de:?}\n"); // Assert we wrote correct amount of bytes. - if let Some(len) = T::BYTE_LENGTH { + if T::BYTE_LENGTH.is_some() { assert_eq!(se.len(), expected_bytes.len()); } // Assert the data is the same. diff --git a/database/src/table.rs b/database/src/table.rs index e117dc15..966a9873 100644 --- a/database/src/table.rs +++ b/database/src/table.rs @@ -1,7 +1,6 @@ //! Database table abstraction; `trait Table`. //---------------------------------------------------------------------------------------------------- Import -use std::fmt::Debug; use crate::{key::Key, storable::Storable}; @@ -13,7 +12,7 @@ use crate::{key::Key, storable::Storable}; /// ## Sealed /// This trait is [`Sealed`](https://rust-lang.github.io/api-guidelines/future-proofing.html#sealed-traits-protect-against-downstream-implementations-c-sealed). /// -/// It is, and can only be implemented on the types inside [`tables`][crate::tables]. +/// It is only implemented on the types inside [`tables`][crate::tables]. pub trait Table: crate::tables::private::Sealed + 'static { /// Name of the database table. const NAME: &'static str; diff --git a/database/src/tables.rs b/database/src/tables.rs index 6ac71dfb..9a425de1 100644 --- a/database/src/tables.rs +++ b/database/src/tables.rs @@ -1,6 +1,21 @@ //! Database tables. //! -//! This module contains all the table definitions used by `cuprate-database`. +//! # Table marker structs +//! This module contains all the table definitions used by `cuprate_database`. +//! +//! The zero-sized structs here represents the table type; +//! they all are essentially marker types that implement [`Table`]. +//! +//! Table structs are `CamelCase`, and their static string +//! names used by the actual database backend are `snake_case`. +//! +//! For example: [`BlockBlobs`] -> `block_blobs`. +//! +//! # Traits +//! This module also contains a set of traits for +//! accessing _all_ tables defined here at once. +//! +//! For example, this is the object returned by [`EnvInner::open_tables`](crate::EnvInner::open_tables). //---------------------------------------------------------------------------------------------------- Import use crate::{ @@ -25,6 +40,7 @@ pub(super) mod private { //---------------------------------------------------------------------------------------------------- `trait Tables[Mut]` /// Creates: /// - `pub trait Tables` +/// - `pub trait TablesIter` /// - `pub trait TablesMut` /// - Blanket implementation for `(tuples, containing, all, open, database, tables, ...)` /// @@ -54,10 +70,14 @@ macro_rules! define_trait_tables { /// ```rust,ignore /// let tables = open_tables(); /// - /// // The accessor function `block_info_v1s()` returns the field - /// // containing an open database table for `BlockInfoV1s`. - /// let _ = tables.block_info_v1s(); + /// // The accessor function `block_infos()` returns the field + /// // containing an open database table for `BlockInfos`. + /// let _ = tables.block_infos(); /// ``` + /// + /// See also: + /// - [`TablesMut`] + /// - [`TablesIter`] pub trait Tables: private::Sealed { // This expands to creating `fn field_accessor_functions()` // for each passed `$table` type. @@ -85,6 +105,9 @@ macro_rules! define_trait_tables { /// /// This is the same as [`Tables`] but includes `_iter()` variants. /// + /// Note that this trait is a supertrait of `Tables`, + /// as in it can use all of its functions as well. + /// /// See [`Tables`] for documentation - this trait exists for the same reasons. pub trait TablesIter: private::Sealed + Tables { $( @@ -99,6 +122,9 @@ macro_rules! define_trait_tables { /// /// This is the same as [`Tables`] but for mutable accesses. /// + /// Note that this trait is a supertrait of `Tables`, + /// as in it can use all of its functions as well. + /// /// See [`Tables`] for documentation - this trait exists for the same reasons. pub trait TablesMut: private::Sealed + Tables { $( @@ -207,14 +233,20 @@ macro_rules! define_trait_tables { }}; } -// Format: $table_type => $index +// Input format: $table_type => $index // // The $index: // - Simply increments by 1 for each table // - Must be 0.. -// - Must end at the total amount of table types +// - Must end at the total amount of table types - 1 // // Compile errors will occur if these aren't satisfied. +// +// $index is just the `tuple.$index`, as the above [`define_trait_tables`] +// macro has a blanket impl for `(all, table, types, ...)` and we must map +// each type to a tuple index explicitly. +// +// FIXME: there's definitely an automatic way to this :) define_trait_tables! { BlockInfos => 0, BlockBlobs => 1, @@ -294,6 +326,9 @@ macro_rules! tables { // Table struct. $(#[$attr])* // The below test show the `snake_case` table name in cargo docs. + #[doc = concat!("- Key: [`", stringify!($key), "`]")] + #[doc = concat!("- Value: [`", stringify!($value), "`]")] + /// /// ## Table Name /// ```rust /// # use cuprate_database::{*,tables::*}; @@ -332,19 +367,30 @@ macro_rules! tables { // b) `Env::open` to make sure it creates the table (for all backends) // c) `call_fn_on_all_tables_or_early_return!()` macro defined in this file tables! { - /// TODO + /// Serialized block blobs (bytes). + /// + /// Contains the serialized version of all blocks. BlockBlobs, BlockHeight => BlockBlob, - /// TODO + /// Block heights. + /// + /// Contains the height of all blocks. BlockHeights, BlockHash => BlockHeight, - /// TODO + /// Block information. + /// + /// Contains metadata of all blocks. BlockInfos, BlockHeight => BlockInfo, - /// TODO + /// Set of key images. + /// + /// Contains all the key images known to be spent. + /// + /// This table has `()` as the value type, as in, + /// it is a set of key images. KeyImages, KeyImage => (), @@ -355,18 +401,26 @@ tables! { NumOutputs, Amount => u64, - /// TODO + /// Pruned transaction blobs (bytes). + /// + /// Contains the pruned portion of serialized transaction data. PrunedTxBlobs, TxId => PrunedBlob, - /// TODO + /// Pre-RCT output data. Outputs, PreRctOutputId => Output, + /// Prunable transaction blobs (bytes). + /// + /// Contains the prunable portion of serialized transaction data. // SOMEDAY: impl when `monero-serai` supports pruning PrunableTxBlobs, TxId => PrunableBlob, + /// Prunable transaction hashes. + /// + /// Contains the prunable portion of transaction hashes. // SOMEDAY: impl when `monero-serai` supports pruning PrunableHashes, TxId => PrunableHash, @@ -377,27 +431,40 @@ tables! { // Properties, // StorableString => StorableVec, - /// TODO + /// RCT output data. RctOutputs, AmountIndex => RctOutput, - /// SOMEDAY: remove when `monero-serai` supports pruning + /// Transaction blobs (bytes). + /// + /// Contains the serialized version of all transactions. + // SOMEDAY: remove when `monero-serai` supports pruning TxBlobs, TxId => TxBlob, - /// TODO + /// Transaction indices. + /// + /// Contains the indices all transactions. TxIds, TxHash => TxId, - /// TODO + /// Transaction heights. + /// + /// Contains the block height associated with all transactions. TxHeights, TxId => BlockHeight, - /// TODO + /// Transaction outputs. + /// + /// Contains the list of `AmountIndex`'s of the + /// outputs associated with all transactions. TxOutputs, TxId => AmountIndices, - /// TODO + /// Transaction unlock time. + /// + /// Contains the unlock time of transactions IF they have one. + /// Transactions without unlock times will not exist in this table. TxUnlockTime, TxId => UnlockTime, } diff --git a/database/src/tests.rs b/database/src/tests.rs index dbebbe93..ba5e8550 100644 --- a/database/src/tests.rs +++ b/database/src/tests.rs @@ -4,24 +4,12 @@ //! - enabled on #[cfg(test)] //! - only used internally -#![allow(clippy::significant_drop_tightening)] - //---------------------------------------------------------------------------------------------------- Import -use std::{ - fmt::Debug, - sync::{Arc, OnceLock}, -}; +use std::fmt::Debug; -use monero_serai::{ - ringct::{RctPrunable, RctSignatures}, - transaction::{Timelock, Transaction, TransactionPrefix}, -}; use pretty_assertions::assert_eq; -use crate::{ - config::Config, key::Key, storable::Storable, tables::Tables, transaction::TxRo, ConcreteEnv, - DatabaseRo, Env, EnvInner, -}; +use crate::{config::ConfigBuilder, tables::Tables, ConcreteEnv, DatabaseRo, Env, EnvInner}; //---------------------------------------------------------------------------------------------------- Struct /// Named struct to assert the length of all tables. @@ -78,7 +66,10 @@ impl AssertTableLen { /// FIXME: changing this to `-> impl Env` causes lifetime errors... pub(crate) fn tmp_concrete_env() -> (ConcreteEnv, tempfile::TempDir) { let tempdir = tempfile::tempdir().unwrap(); - let config = Config::low_power(Some(tempdir.path().into())); + let config = ConfigBuilder::new() + .db_directory(tempdir.path().into()) + .low_power() + .build(); let env = ConcreteEnv::open(config).unwrap(); (env, tempdir) diff --git a/database/src/transaction.rs b/database/src/transaction.rs index 6dffae27..e4c310a0 100644 --- a/database/src/transaction.rs +++ b/database/src/transaction.rs @@ -1,21 +1,21 @@ //! Database transaction abstraction; `trait TxRo`, `trait TxRw`. //---------------------------------------------------------------------------------------------------- Import -use crate::{config::SyncMode, env::Env, error::RuntimeError}; +use crate::error::RuntimeError; //---------------------------------------------------------------------------------------------------- TxRo /// Read-only database transaction. /// /// Returned from [`EnvInner::tx_ro`](crate::EnvInner::tx_ro). /// -/// # TODO -/// I don't think we need this, we can just drop the `tx_ro`? -/// +/// # Commit +/// It's recommended but may not be necessary to call [`TxRo::commit`] in certain cases: +/// - pub trait TxRo<'env> { /// Commit the read-only transaction. /// /// # Errors - /// This operation is infallible (will always return `Ok(())`) with the `redb` backend. + /// This operation will always return `Ok(())` with the `redb` backend. fn commit(self) -> Result<(), RuntimeError>; } @@ -29,20 +29,15 @@ pub trait TxRw<'env> { /// Note that this doesn't necessarily sync the database caches to disk. /// /// # Errors - /// This operation is infallible (will always return `Ok(())`) with the `redb` backend. + /// This operation will always return `Ok(())` with the `redb` backend. /// - /// Else, this will only return: - /// - [`RuntimeError::ResizeNeeded`] (if `Env::MANUAL_RESIZE == true`) - /// - [`RuntimeError::Io`] + /// If `Env::MANUAL_RESIZE == true`, + /// [`RuntimeError::ResizeNeeded`] may be returned. fn commit(self) -> Result<(), RuntimeError>; /// Abort the transaction, erasing any writes that have occurred. /// /// # Errors - /// This operation is infallible (will always return `Ok(())`) with the `heed` backend. - /// - /// Else, this will only return: - /// - [`RuntimeError::ResizeNeeded`] (if `Env::MANUAL_RESIZE == true`) - /// - [`RuntimeError::Io`] + /// This operation will always return `Ok(())` with the `heed` backend. fn abort(self) -> Result<(), RuntimeError>; } diff --git a/database/src/types.rs b/database/src/types.rs index 6ff06a3c..5d89d4c4 100644 --- a/database/src/types.rs +++ b/database/src/types.rs @@ -1,8 +1,10 @@ //! Database [table](crate::tables) types. //! -//! This module contains all types used by the database tables. +//! This module contains all types used by the database tables, +//! and aliases for common Monero-related types that use the +//! same underlying primitive type. //! -//! TODO: Add schema here or a link to it. +//! /* * <============================================> VERY BIG SCARY SAFETY MESSAGE <============================================> @@ -39,7 +41,7 @@ #![forbid(unsafe_code)] // if you remove this line i will steal your monero //---------------------------------------------------------------------------------------------------- Import -use bytemuck::{AnyBitPattern, NoUninit, Pod, Zeroable}; +use bytemuck::{Pod, Zeroable}; #[cfg(feature = "serde")] use serde::{Deserialize, Serialize}; @@ -47,55 +49,59 @@ use serde::{Deserialize, Serialize}; use crate::storable::StorableVec; //---------------------------------------------------------------------------------------------------- Aliases -// TODO: document these, why they exist, and their purpose. -// -// Notes: -// - Keep this sorted A-Z +// These type aliases exist as many Monero-related types are the exact same. +// For clarity, they're given type aliases as to not confuse them. -/// TODO +/// An output's amount. pub type Amount = u64; -/// TODO +/// The index of an [`Amount`] in a list of duplicate `Amount`s. pub type AmountIndex = u64; -/// TODO +/// A list of [`AmountIndex`]s. pub type AmountIndices = StorableVec; -/// TODO +/// A serialized block. pub type BlockBlob = StorableVec; -/// TODO +/// A block's hash. pub type BlockHash = [u8; 32]; -/// TODO +/// A block's height. pub type BlockHeight = u64; -/// TODO +/// A key image. pub type KeyImage = [u8; 32]; -/// TODO +/// Pruned serialized bytes. pub type PrunedBlob = StorableVec; -/// TODO +/// A prunable serialized bytes. pub type PrunableBlob = StorableVec; -/// TODO +/// A prunable hash. pub type PrunableHash = [u8; 32]; -/// TODO +/// A serialized transaction. pub type TxBlob = StorableVec; -/// TODO +/// A transaction's global index, or ID. pub type TxId = u64; -/// TODO +/// A transaction's hash. pub type TxHash = [u8; 32]; -/// TODO +/// The unlock time value of an output. pub type UnlockTime = u64; //---------------------------------------------------------------------------------------------------- BlockInfoV1 -/// TODO +/// A identifier for a pre-RCT [`Output`]. +/// +/// This can also serve as an identifier for [`RctOutput`]'s +/// when [`PreRctOutputId::amount`] is set to `0`, although, +/// in that case, only [`AmountIndex`] needs to be known. +/// +/// This is the key to the [`Outputs`](crate::tables::Outputs) table. /// /// ```rust /// # use std::borrow::*; @@ -121,14 +127,24 @@ pub type UnlockTime = u64; #[derive(Copy, Clone, Debug, PartialEq, PartialOrd, Eq, Ord, Hash, Pod, Zeroable)] #[repr(C)] pub struct PreRctOutputId { - /// TODO + /// Amount of the output. + /// + /// This should be `0` if the output is an [`RctOutput`]. pub amount: Amount, - /// TODO + /// The index of the output with the same `amount`. + /// + /// In the case of [`Output`]'s, this is the index of the list + /// of outputs with the same clear amount. + /// + /// In the case of [`RctOutput`]'s, this is the + /// global index of _all_ `RctOutput`s pub amount_index: AmountIndex, } //---------------------------------------------------------------------------------------------------- BlockInfoV3 -/// TODO +/// Block information. +/// +/// This is the value in the [`BlockInfos`](crate::tables::BlockInfos) table. /// /// ```rust /// # use std::borrow::*; @@ -160,27 +176,34 @@ pub struct PreRctOutputId { #[derive(Copy, Clone, Debug, PartialEq, PartialOrd, Eq, Ord, Hash, Pod, Zeroable)] #[repr(C)] pub struct BlockInfo { - /// TODO + /// The UNIX time at which the block was mined. pub timestamp: u64, - /// TODO + /// The total amount of coins mined in all blocks so far, including this block's. pub cumulative_generated_coins: u64, - /// TODO + /// The adjusted block size, in bytes. + /// + /// See [`block_weight`](https://monero-book.cuprate.org/consensus_rules/blocks/weights.html#blocks-weight). pub weight: u64, /// Least-significant 64 bits of the 128-bit cumulative difficulty. pub cumulative_difficulty_low: u64, /// Most-significant 64 bits of the 128-bit cumulative difficulty. pub cumulative_difficulty_high: u64, - /// TODO + /// The block's hash. pub block_hash: [u8; 32], - /// TODO + /// The total amount of RCT outputs so far, including this block's. pub cumulative_rct_outs: u64, - /// TODO + /// The long term block weight, based on the median weight of the preceding `100_000` blocks. + /// + /// See [`long_term_weight`](https://monero-book.cuprate.org/consensus_rules/blocks/weights.html#long-term-block-weight). pub long_term_weight: u64, } //---------------------------------------------------------------------------------------------------- OutputFlags bitflags::bitflags! { - /// TODO + /// Bit flags for [`Output`]s and [`RctOutput`]s, + /// + /// Currently only the first bit is used and, if set, + /// it means this output has a non-zero unlock time. /// /// ```rust /// # use std::borrow::*; @@ -209,7 +232,7 @@ bitflags::bitflags! { } //---------------------------------------------------------------------------------------------------- Output -/// TODO +/// A pre-RCT (v1) output's data. /// /// ```rust /// # use std::borrow::*; @@ -237,18 +260,20 @@ bitflags::bitflags! { #[derive(Copy, Clone, Debug, PartialEq, PartialOrd, Eq, Ord, Hash, Pod, Zeroable)] #[repr(C)] pub struct Output { - /// TODO + /// The public key of the output. pub key: [u8; 32], - /// We could get this from the tx_idx with the Tx Heights table but that would require another look up per out. + /// The block height this output belongs to. + // PERF: We could get this from the tx_idx with the `TxHeights` + // table but that would require another look up per out. pub height: u32, - /// Bit flags for this output, currently only the first bit is used and, if set, it means this output has a non-zero unlock time. + /// Bit flags for this output. pub output_flags: OutputFlags, - /// TODO + /// The index of the transaction this output belongs to. pub tx_idx: u64, } //---------------------------------------------------------------------------------------------------- RctOutput -/// TODO +/// An RCT (v2+) output's data. /// /// ```rust /// # use std::borrow::*; @@ -277,13 +302,15 @@ pub struct Output { #[derive(Copy, Clone, Debug, PartialEq, PartialOrd, Eq, Ord, Hash, Pod, Zeroable)] #[repr(C)] pub struct RctOutput { - /// TODO + /// The public key of the output. pub key: [u8; 32], - /// We could get this from the tx_idx with the Tx Heights table but that would require another look up per out. + /// The block height this output belongs to. + // PERF: We could get this from the tx_idx with the `TxHeights` + // table but that would require another look up per out. pub height: u32, /// Bit flags for this output, currently only the first bit is used and, if set, it means this output has a non-zero unlock time. pub output_flags: OutputFlags, - /// TODO + /// The index of the transaction this output belongs to. pub tx_idx: u64, /// The amount commitment of this output. pub commitment: [u8; 32], diff --git a/database/src/unsafe_sendable.rs b/database/src/unsafe_sendable.rs index b11d82a8..94472933 100644 --- a/database/src/unsafe_sendable.rs +++ b/database/src/unsafe_sendable.rs @@ -8,8 +8,6 @@ use std::{ use bytemuck::TransparentWrapper; -use crate::storable::StorableVec; - //---------------------------------------------------------------------------------------------------- Aliases #[derive(Copy, Clone, Debug, PartialEq, PartialOrd, Eq, Ord, Hash, TransparentWrapper)] #[repr(transparent)] @@ -43,6 +41,7 @@ impl UnsafeSendable { } /// Extract the inner `T`. + #[allow(dead_code)] pub(crate) fn into_inner(self) -> T { self.0 } diff --git a/helper/src/fs.rs b/helper/src/fs.rs index 9a08b3c7..2e859229 100644 --- a/helper/src/fs.rs +++ b/helper/src/fs.rs @@ -1,11 +1,31 @@ //! Cuprate directories and filenames. //! -//! # TODO -//! Document how environment variables can change these. +//! # Environment variables on Linux +//! Note that this module's functions uses [`dirs`], +//! which adheres to the XDG standard on Linux. //! -//! # Reference -//! -//! +//! This means that the values returned by these functions +//! may change at runtime depending on environment variables, +//! for example: +//! +//! By default the config directory is `~/.config`, however +//! if `$XDG_CONFIG_HOME` is set to something, that will be +//! used instead. +//! +//! ```rust +//! # use cuprate_helper::fs::*; +//! # if cfg!(target_os = "linux") { +//! std::env::set_var("XDG_CONFIG_HOME", "/custom/path"); +//! assert_eq!( +//! cuprate_config_dir().to_string_lossy(), +//! "/custom/path/cuprate" +//! ); +//! # } +//! ``` +//! +//! Reference: +//! - +//! - //---------------------------------------------------------------------------------------------------- Use use std::{ diff --git a/types/README.md b/types/README.md index bbe03c17..6a2015af 100644 --- a/types/README.md +++ b/types/README.md @@ -1,21 +1,20 @@ # `cuprate-types` Various data types shared by Cuprate. - -1. [File Structure](#file-structure) - - [`src/`](#src) +- [1. File Structure](#1-file-structure) + - [1.1 `src/`](#11-src) --- -# File Structure +## 1. File Structure A quick reference of the structure of the folders & files in `cuprate-types`. Note that `lib.rs/mod.rs` files are purely for re-exporting/visibility/lints, and contain no code. Each sub-directory has a corresponding `mod.rs`. -## `src/` +### 1.1 `src/` The top-level `src/` files. | File | Purpose | |---------------------|---------| | `service.rs` | Types used in database requests; `enum {Request,Response}` -| `types.rs` | Various general types used by Cuprate +| `types.rs` | Various general types used by Cuprate \ No newline at end of file diff --git a/types/src/lib.rs b/types/src/lib.rs index ea96515c..8c077901 100644 --- a/types/src/lib.rs +++ b/types/src/lib.rs @@ -1,6 +1,10 @@ //! Cuprate shared data types. //! -//! TODO +//! This crate is a kitchen-sink for data types that are shared across `Cuprate`. +//! +//! # Features flags +//! The `service` module, containing `cuprate_database` request/response +//! types, must be enabled with the `service` feature (on by default). //---------------------------------------------------------------------------------------------------- Lints // Forbid lints. @@ -59,7 +63,6 @@ unused_comparisons, nonstandard_style )] -#![allow(unreachable_code, unused_variables, dead_code, unused_imports)] // TODO: remove #![allow( // FIXME: this lint affects crates outside of // `database/` for some reason, allow for now. @@ -70,9 +73,6 @@ // although it is sometimes nice. clippy::must_use_candidate, - // TODO: should be removed after all `todo!()`'s are gone. - clippy::diverging_sub_expression, - clippy::module_name_repetitions, clippy::module_inception, clippy::redundant_pub_crate, diff --git a/types/src/service.rs b/types/src/service.rs index 2917536b..078c8464 100644 --- a/types/src/service.rs +++ b/types/src/service.rs @@ -1,4 +1,10 @@ //! Database [`ReadRequest`]s, [`WriteRequest`]s, and [`Response`]s. +//! +//! See [`cuprate_database`](https://github.com/Cuprate/cuprate/blob/00c3692eac6b2669e74cfd8c9b41c7e704c779ad/database/src/service/mod.rs#L1-L59)'s +//! `service` module for more usage/documentation. +//! +//! Tests that assert particular requests lead to particular +//! responses are also tested in `cuprate_database`. //---------------------------------------------------------------------------------------------------- Import use std::{ @@ -6,8 +12,6 @@ use std::{ ops::Range, }; -use monero_serai::{block::Block, transaction::Transaction}; - #[cfg(feature = "borsh")] use borsh::{BorshDeserialize, BorshSerialize}; #[cfg(feature = "serde")] @@ -17,63 +21,151 @@ use crate::types::{ExtendedBlockHeader, OutputOnChain, VerifiedBlockInformation} //---------------------------------------------------------------------------------------------------- ReadRequest /// A read request to the database. +/// +/// This pairs with [`Response`], where each variant here +/// matches in name with a `Response` variant. For example, +/// the proper response for a [`ReadRequest::BlockHash`] +/// would be a [`Response::BlockHash`]. +/// +/// See `Response` for the expected responses per `Request`. #[derive(Debug, Clone, PartialEq, Eq)] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[cfg_attr(feature = "borsh", derive(BorshSerialize, BorshDeserialize))] pub enum ReadRequest { - /// TODO + /// Request a block's extended header. + /// + /// The input is the block's height. BlockExtendedHeader(u64), - /// TODO + + /// Request a block's hash. + /// + /// The input is the block's height. BlockHash(u64), - /// TODO + + /// Request a range of block extended headers. + /// + /// The input is a range of block heights. BlockExtendedHeaderInRange(Range), - /// TODO + + /// Request the current chain height. + /// + /// Note that this is not the top-block height. ChainHeight, - /// TODO + + /// Request the total amount of generated coins (atomic units) so far. GeneratedCoins, - /// TODO + + /// Request data for multiple outputs. + /// + /// The input is a `HashMap` where: + /// - Key = output amount + /// - Value = set of amount indices + /// + /// For pre-RCT outputs, the amount is non-zero, + /// and the amount indices represent the wanted + /// indices of duplicate amount outputs, i.e.: + /// + /// ```ignore + /// // list of outputs with amount 10 + /// [0, 1, 2, 3, 4, 5] + /// // ^ ^ + /// // we only want these two, so we would provide + /// // `amount: 10, amount_index: {1, 3}` + /// ``` + /// + /// For RCT outputs, the amounts would be `0` and + /// the amount indices would represent the global + /// RCT output indices. Outputs(HashMap>), - /// TODO + + /// Request the amount of outputs with a certain amount. + /// + /// The input is a list of output amounts. NumberOutputsWithAmount(Vec), - /// TODO + + /// Check that all key images within a set arer not spent. + /// + /// Input is a set of key images. CheckKIsNotSpent(HashSet<[u8; 32]>), } //---------------------------------------------------------------------------------------------------- WriteRequest /// A write request to the database. +/// +/// There is currently only 1 write request to the database, +/// as such, the only valid [`Response`] to this request is +/// the proper response for a [`Response::WriteBlockOk`]. #[derive(Debug, Clone, PartialEq, Eq)] // #[cfg_attr(feature = "borsh", derive(BorshSerialize, BorshDeserialize))] pub enum WriteRequest { - /// TODO + /// Request that a block be written to the database. + /// + /// Input is an already verified block. WriteBlock(VerifiedBlockInformation), } //---------------------------------------------------------------------------------------------------- Response /// A response from the database. +/// +/// These are the data types returned when using sending a `Request`. +/// +/// This pairs with [`ReadRequest`] and [`WriteRequest`], +/// see those two for more info. #[derive(Debug, Clone, PartialEq, Eq)] // #[cfg_attr(feature = "borsh", derive(BorshSerialize, BorshDeserialize))] pub enum Response { //------------------------------------------------------ Reads - /// TODO + /// Response to [`ReadRequest::BlockExtendedHeader`]. + /// + /// Inner value is the extended headed of the requested block. BlockExtendedHeader(ExtendedBlockHeader), - /// TODO + + /// Response to [`ReadRequest::BlockHash`]. + /// + /// Inner value is the hash of the requested block. BlockHash([u8; 32]), - /// TODO + + /// Response to [`ReadRequest::BlockExtendedHeaderInRange`]. + /// + /// Inner value is the list of extended header(s) of the requested block(s). BlockExtendedHeaderInRange(Vec), - /// TODO + + /// Response to [`ReadRequest::ChainHeight`]. + /// + /// Inner value is the chain height, and the top block's hash. ChainHeight(u64, [u8; 32]), - /// TODO + + /// Response to [`ReadRequest::GeneratedCoins`]. + /// + /// Inner value is the total amount of generated coins so far, in atomic units. GeneratedCoins(u64), - /// TODO + + /// Response to [`ReadRequest::Outputs`]. + /// + /// Inner value is all the outputs requested, + /// associated with their amount and amount index. Outputs(HashMap>), - /// TODO + + /// Response to [`ReadRequest::NumberOutputsWithAmount`]. + /// + /// Inner value is a `HashMap` of all the outputs requested where: + /// - Key = output amount + /// - Value = count of outputs with the same amount NumberOutputsWithAmount(HashMap), - /// TODO - /// returns true if key images are spent + + /// Response to [`ReadRequest::CheckKIsNotSpent`]. + /// + /// The inner value is `true` if _any_ of the key images + /// were spent (exited in the database already). + /// + /// The inner value is `false` if _none_ of the key images were spent. CheckKIsNotSpent(bool), //------------------------------------------------------ Writes - /// TODO + /// Response to [`WriteRequest::WriteBlock`]. + /// + /// This response indicates that the requested block has + /// successfully been written to the database without error. WriteBlockOk, } diff --git a/types/src/types.rs b/types/src/types.rs index 82652c9b..aeef4535 100644 --- a/types/src/types.rs +++ b/types/src/types.rs @@ -1,4 +1,4 @@ -//! TODO +//! Various shared data types in Cuprate. //---------------------------------------------------------------------------------------------------- Import use std::sync::Arc; @@ -15,88 +15,113 @@ use borsh::{BorshDeserialize, BorshSerialize}; use serde::{Deserialize, Serialize}; //---------------------------------------------------------------------------------------------------- ExtendedBlockHeader -/// TODO +/// Extended header data of a block. +/// +/// This contains various metadata of a block, but not the block blob itself. +/// +/// For more definitions, see also: . #[derive(Copy, Clone, Default, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[cfg_attr(feature = "borsh", derive(BorshSerialize, BorshDeserialize))] pub struct ExtendedBlockHeader { - /// TODO - /// This is a `cuprate_consensus::HardFork`. + /// The block's major version. + /// + /// This can also be represented with `cuprate_consensus::HardFork`. + /// + /// This is the same value as [`monero_serai::block::BlockHeader::major_version`]. pub version: u8, - /// TODO - /// This is a `cuprate_consensus::HardFork`. + /// The block's hard-fork vote. + /// + /// This can also be represented with `cuprate_consensus::HardFork`. + /// + /// This is the same value as [`monero_serai::block::BlockHeader::minor_version`]. pub vote: u8, - /// TODO + /// The UNIX time at which the block was mined. pub timestamp: u64, - /// TODO + /// The total amount of coins mined in all blocks so far, including this block's. pub cumulative_difficulty: u128, - /// TODO + /// The adjusted block size, in bytes. pub block_weight: usize, - /// TODO + /// The long term block weight, based on the median weight of the preceding `100_000` blocks. pub long_term_weight: usize, } //---------------------------------------------------------------------------------------------------- TransactionVerificationData -/// TODO +/// Data needed to verify a transaction. +/// +/// This represents data that allows verification of a transaction, +/// although it doesn't mean it _has_ been verified. #[derive(Clone, Debug, PartialEq, Eq)] // #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] // FIXME: monero_serai // #[cfg_attr(feature = "borsh", derive(BorshSerialize, BorshDeserialize))] pub struct TransactionVerificationData { - /// TODO + /// The transaction itself. pub tx: Transaction, - /// TODO + /// The serialized byte form of [`Self::tx`]. + /// + /// [`Transaction::serialize`]. pub tx_blob: Vec, - /// TODO + /// The transaction's weight. + /// + /// [`Transaction::weight`]. pub tx_weight: usize, - /// TODO + /// The transaction's total fees. pub fee: u64, - /// TODO + /// The transaction's hash. + /// + /// [`Transaction::hash`]. pub tx_hash: [u8; 32], } //---------------------------------------------------------------------------------------------------- VerifiedBlockInformation -/// TODO +/// Verified information of a block. +/// +/// This represents a block that has already been verified to be correct. +/// +/// For more definitions, see also: . #[derive(Clone, Debug, PartialEq, Eq)] // #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] // FIXME: monero_serai // #[cfg_attr(feature = "borsh", derive(BorshSerialize, BorshDeserialize))] pub struct VerifiedBlockInformation { - /// TODO + /// The block itself. pub block: Block, - /// TODO - pub txs: Vec>, - /// TODO - pub block_hash: [u8; 32], - /// TODO - pub pow_hash: [u8; 32], - /// TODO - pub height: u64, - /// TODO - pub generated_coins: u64, - /// TODO - pub weight: usize, - /// TODO - pub long_term_weight: usize, - /// TODO - pub cumulative_difficulty: u128, - /// TODO - /// - /// + /// The serialized byte form of [`Self::block`]. + /// + /// [`Block::serialize`]. pub block_blob: Vec, + /// All the transactions in the block, excluding the [`Block::miner_tx`]. + pub txs: Vec>, + /// The block's hash. + /// + /// [`Block::hash`]. + pub block_hash: [u8; 32], + /// The block's proof-of-work hash. + pub pow_hash: [u8; 32], + /// The block's height. + pub height: u64, + /// The amount of generated coins (atomic units) in this block. + pub generated_coins: u64, + /// The adjusted block size, in bytes. + pub weight: usize, + /// The long term block weight, which is the weight factored in with previous block weights. + pub long_term_weight: usize, + /// The cumulative difficulty of all blocks up until and including this block. + pub cumulative_difficulty: u128, } //---------------------------------------------------------------------------------------------------- OutputOnChain -/// An already approved previous transaction output. +/// An already existing transaction output. #[derive(Clone, Debug, PartialEq, Eq)] // #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] // FIXME: monero_serai // #[cfg_attr(feature = "borsh", derive(BorshSerialize, BorshDeserialize))] pub struct OutputOnChain { - /// TODO + /// The block height this output belongs to. pub height: u64, - /// TODO + /// The timelock of this output, if any. pub time_lock: Timelock, - /// TODO + /// The public key of this output, if any. pub key: Option, - /// TODO + /// The output's commitment. pub commitment: EdwardsPoint, }