From e88ac0140e0779027bbb48860f7016ee934e2d98 Mon Sep 17 00:00:00 2001 From: "hinto.janai" Date: Tue, 3 Sep 2024 20:46:57 -0400 Subject: [PATCH] serde --- books/architecture/src/SUMMARY.md | 13 +++--- .../storage/common/service/initialization.md | 1 + .../src/storage/common/service/intro.md | 1 + .../src/storage/common/service/requests.md | 1 + .../src/storage/common/service/resizing.md | 12 +++++ .../src/storage/common/service/responses.md | 1 + .../src/storage/common/service/shutdown.md | 1 + .../storage/common/service/thread-model.md | 1 + books/architecture/src/storage/db/resizing.md | 9 +++- books/architecture/src/storage/db/serde.md | 45 ++++++++++++++++++- .../src/storage/db/service/initialization.md | 1 - .../src/storage/db/service/intro.md | 1 - .../src/storage/db/service/requests.md | 1 - .../src/storage/db/service/responses.md | 1 - .../src/storage/db/service/shutdown.md | 1 - .../src/storage/db/service/thread-model.md | 1 - 16 files changed, 77 insertions(+), 14 deletions(-) create mode 100644 books/architecture/src/storage/common/service/initialization.md create mode 100644 books/architecture/src/storage/common/service/intro.md create mode 100644 books/architecture/src/storage/common/service/requests.md create mode 100644 books/architecture/src/storage/common/service/resizing.md create mode 100644 books/architecture/src/storage/common/service/responses.md create mode 100644 books/architecture/src/storage/common/service/shutdown.md create mode 100644 books/architecture/src/storage/common/service/thread-model.md delete mode 100644 books/architecture/src/storage/db/service/initialization.md delete mode 100644 books/architecture/src/storage/db/service/intro.md delete mode 100644 books/architecture/src/storage/db/service/requests.md delete mode 100644 books/architecture/src/storage/db/service/responses.md delete mode 100644 books/architecture/src/storage/db/service/shutdown.md delete mode 100644 books/architecture/src/storage/db/service/thread-model.md diff --git a/books/architecture/src/SUMMARY.md b/books/architecture/src/SUMMARY.md index 9dac10f..14a478e 100644 --- a/books/architecture/src/SUMMARY.md +++ b/books/architecture/src/SUMMARY.md @@ -45,12 +45,13 @@ - [🟢 Common behavior](storage/common/intro.md) - [🟢 Types](storage/common/types.md) - [🟢 `ops`](storage/common/ops.md) - - [🟢 `tower::Service`](storage/common/intro.md) - - [🟢 Initialization](storage/common/initialization.md) - - [🟢 Requests](storage/common/requests.md) - - [🟢 Responses](storage/common/responses.md) - - [🟢 Thread model](storage/common/thread-model.md) - - [🟢 Shutdown](storage/common/shutdown.md) + - [🟢 `tower::Service`](storage/common/service/intro.md) + - [🟢 Initialization](storage/common/service/initialization.md) + - [🟢 Requests](storage/common/service/requests.md) + - [🟢 Responses](storage/common/service/responses.md) + - [🟢 Resizing](storage/common/service/resizing.md) + - [🟢 Thread model](storage/common/service/thread-model.md) + - [🟢 Shutdown](storage/common/service/shutdown.md) - [⚪️ Blockchain](storage/blockchain/intro.md) - [🟢 Schema](storage/blockchain/schema/intro.md) - [🟢 Tables](storage/blockchain/schema/tables.md) diff --git a/books/architecture/src/storage/common/service/initialization.md b/books/architecture/src/storage/common/service/initialization.md new file mode 100644 index 0000000..59f994c --- /dev/null +++ b/books/architecture/src/storage/common/service/initialization.md @@ -0,0 +1 @@ +# 🟢 Initialization diff --git a/books/architecture/src/storage/common/service/intro.md b/books/architecture/src/storage/common/service/intro.md new file mode 100644 index 0000000..bca23ca --- /dev/null +++ b/books/architecture/src/storage/common/service/intro.md @@ -0,0 +1 @@ +# 🟢 tower::Service diff --git a/books/architecture/src/storage/common/service/requests.md b/books/architecture/src/storage/common/service/requests.md new file mode 100644 index 0000000..7391375 --- /dev/null +++ b/books/architecture/src/storage/common/service/requests.md @@ -0,0 +1 @@ +# 🟢 Requests diff --git a/books/architecture/src/storage/common/service/resizing.md b/books/architecture/src/storage/common/service/resizing.md new file mode 100644 index 0000000..214b1fb --- /dev/null +++ b/books/architecture/src/storage/common/service/resizing.md @@ -0,0 +1,12 @@ +# Resizing +Database backends that require manually resizing will, by default, use a similar algorithm as `monerod`'s. + +Note that this only relates to the [`Service`](../common/service/intro.md) section, where the database is handled by `cuprate_database_service` itself, not the user. In the case of a user directly using `cuprate_database`, it is up to them on how to resize. The database will return [`RuntimeError::ResizeNeeded`](https://doc.cuprate.org/cuprate_database/enum.RuntimeError.html#variant.ResizeNeeded) when it needs resizing. + +Within `service`, the resizing logic defined [here](https://github.com/Cuprate/cuprate/blob/2ac90420c658663564a71b7ecb52d74f3c2c9d0f/database/src/service/write.rs#L139-L201) does the following: + +- If there's not enough space to fit a write request's data, start a resize +- Each resize adds around [`1_073_745_920`](https://github.com/Cuprate/cuprate/blob/2ac90420c658663564a71b7ecb52d74f3c2c9d0f/database/src/resize.rs#L104-L160) bytes to the current map size +- A resize will be attempted `3` times before failing + +There are other [resizing algorithms](https://github.com/Cuprate/cuprate/blob/2ac90420c658663564a71b7ecb52d74f3c2c9d0f/database/src/resize.rs#L38-L47) that define how the database's memory map grows, although currently the behavior of [`monerod`](https://github.com/Cuprate/cuprate/blob/2ac90420c658663564a71b7ecb52d74f3c2c9d0f/database/src/resize.rs#L104-L160) is closely followed. \ No newline at end of file diff --git a/books/architecture/src/storage/common/service/responses.md b/books/architecture/src/storage/common/service/responses.md new file mode 100644 index 0000000..3ba68e0 --- /dev/null +++ b/books/architecture/src/storage/common/service/responses.md @@ -0,0 +1 @@ +# 🟢 Responses diff --git a/books/architecture/src/storage/common/service/shutdown.md b/books/architecture/src/storage/common/service/shutdown.md new file mode 100644 index 0000000..cfe83db --- /dev/null +++ b/books/architecture/src/storage/common/service/shutdown.md @@ -0,0 +1 @@ +# 🟢 Shutdown diff --git a/books/architecture/src/storage/common/service/thread-model.md b/books/architecture/src/storage/common/service/thread-model.md new file mode 100644 index 0000000..b2addfc --- /dev/null +++ b/books/architecture/src/storage/common/service/thread-model.md @@ -0,0 +1 @@ +# 🟢 Thread model diff --git a/books/architecture/src/storage/db/resizing.md b/books/architecture/src/storage/db/resizing.md index 777e2d4..ebf989e 100644 --- a/books/architecture/src/storage/db/resizing.md +++ b/books/architecture/src/storage/db/resizing.md @@ -1 +1,8 @@ -# ⚪️ Resizing +# Resizing +`cuprate_database` itself does not handle memory map resizes automatically +(for database backends that need resizing, i.e. heed/LMDB). + +When a user directly using `cuprate_database`, it is up to them on how to resize. The database will return [`RuntimeError::ResizeNeeded`](https://doc.cuprate.org/cuprate_database/enum.RuntimeError.html#variant.ResizeNeeded) when it needs resizing. + +However, `cuprate_database` exposes some [resizing algorithms](https://doc.cuprate.org/cuprate_database/resize/index.html) +that define how the database's memory map grows. \ No newline at end of file diff --git a/books/architecture/src/storage/db/serde.md b/books/architecture/src/storage/db/serde.md index 5c559e5..de17f30 100644 --- a/books/architecture/src/storage/db/serde.md +++ b/books/architecture/src/storage/db/serde.md @@ -1 +1,44 @@ -# ⚪️ (De)serialization +# (De)serialization +All types stored inside the database are either bytes already or are perfectly bitcast-able. + +As such, they do not incur heavy (de)serialization costs when storing/fetching them from the database. The main (de)serialization used is [`bytemuck`](https://docs.rs/bytemuck)'s traits and casting functions. + +## Size and layout +The size & layout of types is stable across compiler versions, as they are set and determined with [`#[repr(C)]`](https://doc.rust-lang.org/nomicon/other-reprs.html#reprc) and `bytemuck`'s derive macros such as [`bytemuck::Pod`](https://docs.rs/bytemuck/latest/bytemuck/derive.Pod.html). + +Note that the data stored in the tables are still type-safe; we still refer to the key and values within our tables by the type. + +## How +The main deserialization `trait` for database storage is [`Storable`](https://doc.cuprate.org/cuprate_database/trait.Storable.html). + +- Before storage, the type is [simply cast into bytes](https://github.com/Cuprate/cuprate/blob/2ac90420c658663564a71b7ecb52d74f3c2c9d0f/database/src/storable.rs#L125) +- When fetching, the bytes are [simply cast into the type](https://github.com/Cuprate/cuprate/blob/2ac90420c658663564a71b7ecb52d74f3c2c9d0f/database/src/storable.rs#L130) + +When a type is casted into bytes, [the reference is casted](https://docs.rs/bytemuck/latest/bytemuck/fn.bytes_of.html), i.e. this is zero-cost serialization. + +However, it is worth noting that when bytes are casted into the type, [it is copied](https://docs.rs/bytemuck/latest/bytemuck/fn.pod_read_unaligned.html). This is due to byte alignment guarantee issues with both backends, see: +- +- + +Without this, `bytemuck` will panic with [`TargetAlignmentGreaterAndInputNotAligned`](https://docs.rs/bytemuck/latest/bytemuck/enum.PodCastError.html#variant.TargetAlignmentGreaterAndInputNotAligned) when casting. + +Copying the bytes fixes this problem, although it is more costly than necessary. However, in the main use-case for `cuprate_database` (`tower::Service` API) the bytes would need to be owned regardless as the `Request/Response` API uses owned data types (`T`, `Vec`, `HashMap`, etc). + +Practically speaking, this means lower-level database functions that normally look like such: +```rust +fn get(key: &Key) -> &Value; +``` +end up looking like this in `cuprate_database`: +```rust +fn get(key: &Key) -> Value; +``` + +Since each backend has its own (de)serialization methods, our types are wrapped in compatibility types that map our `Storable` functions into whatever is required for the backend, e.g: +- [`StorableHeed`](https://github.com/Cuprate/cuprate/blob/2ac90420c658663564a71b7ecb52d74f3c2c9d0f/database/src/backend/heed/storable.rs#L11-L45) +- [`StorableRedb`](https://github.com/Cuprate/cuprate/blob/2ac90420c658663564a71b7ecb52d74f3c2c9d0f/database/src/backend/redb/storable.rs#L11-L30) + +Compatibility structs also exist for any `Storable` containers: +- [`StorableVec`](https://github.com/Cuprate/cuprate/blob/2ac90420c658663564a71b7ecb52d74f3c2c9d0f/database/src/storable.rs#L135-L191) +- [`StorableBytes`](https://github.com/Cuprate/cuprate/blob/2ac90420c658663564a71b7ecb52d74f3c2c9d0f/database/src/storable.rs#L208-L241) + +Again, it's unfortunate that these must be owned, although in the `tower::Service` use-case, they would have to be owned anyway. \ No newline at end of file diff --git a/books/architecture/src/storage/db/service/initialization.md b/books/architecture/src/storage/db/service/initialization.md deleted file mode 100644 index d96fb30..0000000 --- a/books/architecture/src/storage/db/service/initialization.md +++ /dev/null @@ -1 +0,0 @@ -# ⚪️ Initialization diff --git a/books/architecture/src/storage/db/service/intro.md b/books/architecture/src/storage/db/service/intro.md deleted file mode 100644 index c6b2c74..0000000 --- a/books/architecture/src/storage/db/service/intro.md +++ /dev/null @@ -1 +0,0 @@ -# ⚪️ The service diff --git a/books/architecture/src/storage/db/service/requests.md b/books/architecture/src/storage/db/service/requests.md deleted file mode 100644 index 3f6a1e9..0000000 --- a/books/architecture/src/storage/db/service/requests.md +++ /dev/null @@ -1 +0,0 @@ -# ⚪️ Requests diff --git a/books/architecture/src/storage/db/service/responses.md b/books/architecture/src/storage/db/service/responses.md deleted file mode 100644 index 404ae0c..0000000 --- a/books/architecture/src/storage/db/service/responses.md +++ /dev/null @@ -1 +0,0 @@ -# ⚪️ Responses diff --git a/books/architecture/src/storage/db/service/shutdown.md b/books/architecture/src/storage/db/service/shutdown.md deleted file mode 100644 index 1bdc92d..0000000 --- a/books/architecture/src/storage/db/service/shutdown.md +++ /dev/null @@ -1 +0,0 @@ -# ⚪️ Shutdown diff --git a/books/architecture/src/storage/db/service/thread-model.md b/books/architecture/src/storage/db/service/thread-model.md deleted file mode 100644 index 4bcfa01..0000000 --- a/books/architecture/src/storage/db/service/thread-model.md +++ /dev/null @@ -1 +0,0 @@ -# ⚪️ Thread model