mirror of
https://github.com/Cuprate/cuprate.git
synced 2024-11-16 15:58:17 +00:00
storage: split cuprate-blockchain <-> cuprate-database (#160)
* storage: port some code `cuprate-blockchain` -> `database` * database: remove `Tables` references * database: remove old `cuprate-blockchain` type references * find/replace `cuprate_blockchain` -> `database`, add `create_db()` * database: fix redb * database: use readme for docs, link in `lib.rs` * database: fix `open_db_ro`, `open_db_rw`, `create_db` behavior * database: add open table tests * database: fix tests, remove blockchain specific references * database: remove `ReaderThreads`, make `db_directory` mandatory * initial `cuprate-blockchain` split * fix doc links * rename, fix database config * blockchain: create `crate::open()`, `OpenTables::create_tables()` * more compat fixes * fix imports * fix conflicts * align cargo.toml * docs * fixes * add `unused_crate_dependencies` lint, fix * blockchain: add open table tests
This commit is contained in:
hinto-janai
GitHub
parent
e405786a73
commit
a438279aa8
61 changed files with 2743 additions and 1795 deletions
337
Cargo.lock
generated
337
Cargo.lock
generated
|
|
@ -493,27 +493,21 @@ version = "0.0.0"
|
|||
dependencies = [
|
||||
"bitflags 2.5.0",
|
||||
"bytemuck",
|
||||
"bytes",
|
||||
"cfg-if",
|
||||
"crossbeam",
|
||||
"cuprate-database",
|
||||
"cuprate-helper",
|
||||
"cuprate-pruning",
|
||||
"cuprate-test-utils",
|
||||
"cuprate-types",
|
||||
"curve25519-dalek",
|
||||
"futures",
|
||||
"heed",
|
||||
"hex",
|
||||
"hex-literal",
|
||||
"monero-serai",
|
||||
"page_size",
|
||||
"paste",
|
||||
"pretty_assertions",
|
||||
"rayon",
|
||||
"redb",
|
||||
"serde",
|
||||
"tempfile",
|
||||
"thiserror",
|
||||
"thread_local",
|
||||
"tokio",
|
||||
"tokio-util",
|
||||
|
|
@ -595,7 +589,18 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "cuprate-database"
|
||||
version = "0.0.0"
|
||||
version = "0.0.1"
|
||||
dependencies = [
|
||||
"bytemuck",
|
||||
"bytes",
|
||||
"cfg-if",
|
||||
"heed",
|
||||
"page_size",
|
||||
"redb",
|
||||
"serde",
|
||||
"tempfile",
|
||||
"thiserror",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cuprate-epee-encoding"
|
||||
|
|
@ -803,9 +808,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "curve25519-dalek"
|
||||
version = "4.1.3"
|
||||
version = "4.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be"
|
||||
checksum = "0a677b8922c94e01bdbb12126b0bc852f00447528dee1782229af9c720c3f348"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"cpufeatures",
|
||||
|
|
@ -813,6 +818,7 @@ dependencies = [
|
|||
"digest",
|
||||
"fiat-crypto",
|
||||
"group",
|
||||
"platforms",
|
||||
"rand_core",
|
||||
"rustc_version",
|
||||
"subtle",
|
||||
|
|
@ -910,6 +916,17 @@ dependencies = [
|
|||
"windows-sys 0.48.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "displaydoc"
|
||||
version = "0.2.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "487585f4d0c6655fe74905e2504d8ad6908e4db67f744eb140876906c2f3175d"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.66",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "doxygen-rs"
|
||||
version = "0.4.2"
|
||||
|
|
@ -1267,9 +1284,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "httparse"
|
||||
version = "1.9.4"
|
||||
version = "1.9.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0fcc0b4a115bf80b728eb8ea024ad5bd707b615bfed49e0665b6e0f86fd082d9"
|
||||
checksum = "d0e7a4dd27b9476dc40cb050d3632d3bba3a70ddbff012285f7f8559a1e7e545"
|
||||
|
||||
[[package]]
|
||||
name = "hyper"
|
||||
|
|
@ -1352,13 +1369,133 @@ dependencies = [
|
|||
]
|
||||
|
||||
[[package]]
|
||||
name = "idna"
|
||||
version = "0.5.0"
|
||||
name = "icu_collections"
|
||||
version = "1.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6"
|
||||
checksum = "db2fa452206ebee18c4b5c2274dbf1de17008e874b4dc4f0aea9d01ca79e4526"
|
||||
dependencies = [
|
||||
"unicode-bidi",
|
||||
"unicode-normalization",
|
||||
"displaydoc",
|
||||
"yoke",
|
||||
"zerofrom",
|
||||
"zerovec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "icu_locid"
|
||||
version = "1.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "13acbb8371917fc971be86fc8057c41a64b521c184808a698c02acc242dbf637"
|
||||
dependencies = [
|
||||
"displaydoc",
|
||||
"litemap",
|
||||
"tinystr",
|
||||
"writeable",
|
||||
"zerovec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "icu_locid_transform"
|
||||
version = "1.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "01d11ac35de8e40fdeda00d9e1e9d92525f3f9d887cdd7aa81d727596788b54e"
|
||||
dependencies = [
|
||||
"displaydoc",
|
||||
"icu_locid",
|
||||
"icu_locid_transform_data",
|
||||
"icu_provider",
|
||||
"tinystr",
|
||||
"zerovec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "icu_locid_transform_data"
|
||||
version = "1.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fdc8ff3388f852bede6b579ad4e978ab004f139284d7b28715f773507b946f6e"
|
||||
|
||||
[[package]]
|
||||
name = "icu_normalizer"
|
||||
version = "1.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "19ce3e0da2ec68599d193c93d088142efd7f9c5d6fc9b803774855747dc6a84f"
|
||||
dependencies = [
|
||||
"displaydoc",
|
||||
"icu_collections",
|
||||
"icu_normalizer_data",
|
||||
"icu_properties",
|
||||
"icu_provider",
|
||||
"smallvec",
|
||||
"utf16_iter",
|
||||
"utf8_iter",
|
||||
"write16",
|
||||
"zerovec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "icu_normalizer_data"
|
||||
version = "1.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f8cafbf7aa791e9b22bec55a167906f9e1215fd475cd22adfcf660e03e989516"
|
||||
|
||||
[[package]]
|
||||
name = "icu_properties"
|
||||
version = "1.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1f8ac670d7422d7f76b32e17a5db556510825b29ec9154f235977c9caba61036"
|
||||
dependencies = [
|
||||
"displaydoc",
|
||||
"icu_collections",
|
||||
"icu_locid_transform",
|
||||
"icu_properties_data",
|
||||
"icu_provider",
|
||||
"tinystr",
|
||||
"zerovec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "icu_properties_data"
|
||||
version = "1.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "67a8effbc3dd3e4ba1afa8ad918d5684b8868b3b26500753effea8d2eed19569"
|
||||
|
||||
[[package]]
|
||||
name = "icu_provider"
|
||||
version = "1.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6ed421c8a8ef78d3e2dbc98a973be2f3770cb42b606e3ab18d6237c4dfde68d9"
|
||||
dependencies = [
|
||||
"displaydoc",
|
||||
"icu_locid",
|
||||
"icu_provider_macros",
|
||||
"stable_deref_trait",
|
||||
"tinystr",
|
||||
"writeable",
|
||||
"yoke",
|
||||
"zerofrom",
|
||||
"zerovec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "icu_provider_macros"
|
||||
version = "1.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.66",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "idna"
|
||||
version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4716a3a0933a1d01c2f72450e89596eb51dd34ef3c211ccd875acdf1f8fe47ed"
|
||||
dependencies = [
|
||||
"icu_normalizer",
|
||||
"icu_properties",
|
||||
"smallvec",
|
||||
"utf8_iter",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -1439,6 +1576,12 @@ version = "0.4.14"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89"
|
||||
|
||||
[[package]]
|
||||
name = "litemap"
|
||||
version = "0.7.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "643cb0b8d4fcc284004d5fd0d67ccf61dfffadb7f75e1e71bc420f4688a3a704"
|
||||
|
||||
[[package]]
|
||||
name = "lmdb-master-sys"
|
||||
version = "0.2.1"
|
||||
|
|
@ -1478,9 +1621,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "memchr"
|
||||
version = "2.7.4"
|
||||
version = "2.7.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3"
|
||||
checksum = "6c8640c5d730cb13ebd907d8d04b52f55ac9a2eec55b440c8892f40d56c76c1d"
|
||||
|
||||
[[package]]
|
||||
name = "merlin"
|
||||
|
|
@ -1496,9 +1639,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "miniz_oxide"
|
||||
version = "0.7.4"
|
||||
version = "0.7.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b8a240ddb74feaf34a79a7add65a741f3167852fba007066dcac1ca548d89c08"
|
||||
checksum = "87dfd01fe195c66b572b37921ad8803d010623c0aca821bea2302239d155cdae"
|
||||
dependencies = [
|
||||
"adler",
|
||||
]
|
||||
|
|
@ -1768,6 +1911,12 @@ version = "0.1.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
|
||||
|
||||
[[package]]
|
||||
name = "platforms"
|
||||
version = "3.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "db23d408679286588f4d4644f965003d056e3dd5abcaaa938116871d7ce2fee7"
|
||||
|
||||
[[package]]
|
||||
name = "ppv-lite86"
|
||||
version = "0.2.17"
|
||||
|
|
@ -1967,9 +2116,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "redox_syscall"
|
||||
version = "0.5.2"
|
||||
version = "0.5.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c82cf8cff14456045f55ec4241383baeff27af886adb72ffb2162f99911de0fd"
|
||||
checksum = "469052894dcb553421e483e4209ee581a45100d31b4018de03e5a7ad86374a7e"
|
||||
dependencies = [
|
||||
"bitflags 2.5.0",
|
||||
]
|
||||
|
|
@ -2287,6 +2436,12 @@ version = "0.9.8"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67"
|
||||
|
||||
[[package]]
|
||||
name = "stable_deref_trait"
|
||||
version = "1.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3"
|
||||
|
||||
[[package]]
|
||||
name = "std-shims"
|
||||
version = "0.1.1"
|
||||
|
|
@ -2345,6 +2500,17 @@ dependencies = [
|
|||
"crossbeam-queue",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "synstructure"
|
||||
version = "0.13.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.66",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tap"
|
||||
version = "1.0.1"
|
||||
|
|
@ -2403,20 +2569,15 @@ dependencies = [
|
|||
]
|
||||
|
||||
[[package]]
|
||||
name = "tinyvec"
|
||||
version = "1.6.0"
|
||||
name = "tinystr"
|
||||
version = "0.7.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50"
|
||||
checksum = "9117f5d4db391c1cf6927e7bea3db74b9a1c1add8f7eda9ffd5364f40f57b82f"
|
||||
dependencies = [
|
||||
"tinyvec_macros",
|
||||
"displaydoc",
|
||||
"zerovec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tinyvec_macros"
|
||||
version = "0.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
|
||||
|
||||
[[package]]
|
||||
name = "tokio"
|
||||
version = "1.38.0"
|
||||
|
|
@ -2609,27 +2770,12 @@ version = "0.1.4"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-bidi"
|
||||
version = "0.3.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-ident"
|
||||
version = "1.0.12"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-normalization"
|
||||
version = "0.1.23"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a56d1686db2308d901306f92a263857ef59ea39678a5458e7cb17f01415101f5"
|
||||
dependencies = [
|
||||
"tinyvec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "untrusted"
|
||||
version = "0.9.0"
|
||||
|
|
@ -2638,15 +2784,27 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1"
|
|||
|
||||
[[package]]
|
||||
name = "url"
|
||||
version = "2.5.2"
|
||||
version = "2.5.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "22784dbdf76fdde8af1aeda5622b546b422b6fc585325248a2bf9f5e41e94d6c"
|
||||
checksum = "f7c25da092f0a868cdf09e8674cd3b7ef3a7d92a24253e663a2fb85e2496de56"
|
||||
dependencies = [
|
||||
"form_urlencoded",
|
||||
"idna",
|
||||
"percent-encoding",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "utf16_iter"
|
||||
version = "1.0.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246"
|
||||
|
||||
[[package]]
|
||||
name = "utf8_iter"
|
||||
version = "1.0.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be"
|
||||
|
||||
[[package]]
|
||||
name = "version_check"
|
||||
version = "0.9.4"
|
||||
|
|
@ -2963,6 +3121,18 @@ dependencies = [
|
|||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "write16"
|
||||
version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936"
|
||||
|
||||
[[package]]
|
||||
name = "writeable"
|
||||
version = "0.5.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51"
|
||||
|
||||
[[package]]
|
||||
name = "wyz"
|
||||
version = "0.5.1"
|
||||
|
|
@ -2978,6 +3148,30 @@ version = "0.5.1"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "09041cd90cf85f7f8b2df60c646f853b7f535ce68f85244eb6731cf89fa498ec"
|
||||
|
||||
[[package]]
|
||||
name = "yoke"
|
||||
version = "0.7.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6c5b1314b079b0930c31e3af543d8ee1757b1951ae1e1565ec704403a7240ca5"
|
||||
dependencies = [
|
||||
"serde",
|
||||
"stable_deref_trait",
|
||||
"yoke-derive",
|
||||
"zerofrom",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "yoke-derive"
|
||||
version = "0.7.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "28cc31741b18cb6f1d5ff12f5b7523e3d6eb0852bbbad19d73905511d9849b95"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.66",
|
||||
"synstructure",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zerocopy"
|
||||
version = "0.7.34"
|
||||
|
|
@ -2998,6 +3192,27 @@ dependencies = [
|
|||
"syn 2.0.66",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zerofrom"
|
||||
version = "0.1.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "91ec111ce797d0e0784a1116d0ddcdbea84322cd79e5d5ad173daeba4f93ab55"
|
||||
dependencies = [
|
||||
"zerofrom-derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zerofrom-derive"
|
||||
version = "0.1.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0ea7b4a3637ea8669cedf0f1fd5c286a17f3de97b8dd5a70a6c167a1730e63a5"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.66",
|
||||
"synstructure",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zeroize"
|
||||
version = "1.8.1"
|
||||
|
|
@ -3017,3 +3232,25 @@ dependencies = [
|
|||
"quote",
|
||||
"syn 2.0.66",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zerovec"
|
||||
version = "0.10.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bb2cc8827d6c0994478a15c53f374f46fbd41bea663d809b14744bc42e6b109c"
|
||||
dependencies = [
|
||||
"yoke",
|
||||
"zerofrom",
|
||||
"zerovec-derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zerovec-derive"
|
||||
version = "0.10.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "97cf56601ee5052b4417d90c8755c6683473c926039908196cf35d99f893ebe7"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.66",
|
||||
]
|
||||
|
|
|
|||
|
|
@ -3,7 +3,9 @@ use std::{fmt::Write, fs::write};
|
|||
use clap::Parser;
|
||||
use tower::{Service, ServiceExt};
|
||||
|
||||
use cuprate_blockchain::{config::ConfigBuilder, service::DatabaseReadHandle, RuntimeError};
|
||||
use cuprate_blockchain::{
|
||||
config::ConfigBuilder, cuprate_database::RuntimeError, service::DatabaseReadHandle,
|
||||
};
|
||||
use cuprate_types::blockchain::{BCReadRequest, BCResponse};
|
||||
|
||||
use cuprate_fast_sync::{hash_of_hashes, BlockId, HashOfHashes};
|
||||
|
|
|
|||
|
|
@ -9,30 +9,28 @@ repository = "https://github.com/Cuprate/cuprate/tree/main/storage/cuprate-bloc
|
|||
keywords = ["cuprate", "blockchain", "database"]
|
||||
|
||||
[features]
|
||||
default = ["heed", "redb", "service"]
|
||||
# default = ["redb", "service"]
|
||||
# default = ["redb-memory", "service"]
|
||||
heed = ["dep:heed"]
|
||||
redb = ["dep:redb"]
|
||||
redb-memory = ["redb"]
|
||||
default = ["heed", "service"]
|
||||
# default = ["redb", "service"]
|
||||
# default = ["redb-memory", "service"]
|
||||
heed = ["cuprate-database/heed"]
|
||||
redb = ["cuprate-database/redb"]
|
||||
redb-memory = ["cuprate-database/redb-memory"]
|
||||
service = ["dep:crossbeam", "dep:futures", "dep:tokio", "dep:tokio-util", "dep:tower", "dep:rayon"]
|
||||
|
||||
[dependencies]
|
||||
bitflags = { workspace = true, features = ["serde", "bytemuck"] }
|
||||
bytemuck = { version = "1.14.3", features = ["must_cast", "derive", "min_const_generics", "extern_crate_alloc"] }
|
||||
bytes = { workspace = true }
|
||||
cfg-if = { workspace = true }
|
||||
# FIXME:
|
||||
# We only need the `thread` feature if `service` is enabled.
|
||||
# Figure out how to enable features of an already pulled in dependency conditionally.
|
||||
cuprate-database = { path = "../database" }
|
||||
cuprate-helper = { path = "../../helper", features = ["fs", "thread", "map"] }
|
||||
cuprate-types = { path = "../../types", features = ["blockchain"] }
|
||||
|
||||
bitflags = { workspace = true, features = ["serde", "bytemuck"] }
|
||||
bytemuck = { version = "1.14.3", features = ["must_cast", "derive", "min_const_generics", "extern_crate_alloc"] }
|
||||
curve25519-dalek = { workspace = true }
|
||||
cuprate-pruning = { path = "../../pruning" }
|
||||
monero-serai = { workspace = true, features = ["std"] }
|
||||
paste = { workspace = true }
|
||||
page_size = { version = "0.6.0" } # Needed for database resizes, they must be a multiple of the OS page size.
|
||||
thiserror = { workspace = true }
|
||||
|
||||
# `service` feature.
|
||||
crossbeam = { workspace = true, features = ["std"], optional = true }
|
||||
|
|
@ -43,17 +41,12 @@ tower = { workspace = true, features = ["full"], optional = true }
|
|||
thread_local = { workspace = true }
|
||||
rayon = { workspace = true, optional = true }
|
||||
|
||||
# Optional features.
|
||||
heed = { version = "0.20.0", features = ["read-txn-no-tls"], optional = true }
|
||||
redb = { version = "2.1.0", optional = true }
|
||||
serde = { workspace = true, optional = true }
|
||||
|
||||
[dev-dependencies]
|
||||
bytemuck = { version = "1.14.3", features = ["must_cast", "derive", "min_const_generics", "extern_crate_alloc"] }
|
||||
cuprate-helper = { path = "../../helper", features = ["thread"] }
|
||||
cuprate-helper = { path = "../../helper", features = ["thread"] }
|
||||
cuprate-test-utils = { path = "../../test-utils" }
|
||||
page_size = { version = "0.6.0" }
|
||||
tempfile = { version = "3.10.0" }
|
||||
|
||||
bytemuck = { version = "1.14.3", features = ["must_cast", "derive", "min_const_generics", "extern_crate_alloc"] }
|
||||
tempfile = { version = "3.10.0" }
|
||||
pretty_assertions = { workspace = true }
|
||||
hex = { workspace = true }
|
||||
hex-literal = { workspace = true }
|
||||
hex = { workspace = true }
|
||||
hex-literal = { workspace = true }
|
||||
|
|
|
|||
600
storage/blockchain/DESIGN.md
Normal file
600
storage/blockchain/DESIGN.md
Normal file
|
|
@ -0,0 +1,600 @@
|
|||
# Database
|
||||
FIXME: This documentation must be updated and moved to the architecture book.
|
||||
|
||||
Cuprate's blockchain implementation.
|
||||
|
||||
- [1. Documentation](#1-documentation)
|
||||
- [2. File structure](#2-file-structure)
|
||||
- [2.1 `src/`](#21-src)
|
||||
- [2.2 `src/backend/`](#22-srcbackend)
|
||||
- [2.3 `src/config/`](#23-srcconfig)
|
||||
- [2.4 `src/ops/`](#24-srcops)
|
||||
- [2.5 `src/service/`](#25-srcservice)
|
||||
- [3. Backends](#3-backends)
|
||||
- [3.1 heed](#31-heed)
|
||||
- [3.2 redb](#32-redb)
|
||||
- [3.3 redb-memory](#33-redb-memory)
|
||||
- [3.4 sanakirja](#34-sanakirja)
|
||||
- [3.5 MDBX](#35-mdbx)
|
||||
- [4. Layers](#4-layers)
|
||||
- [4.1 Backend](#41-backend)
|
||||
- [4.2 Trait](#42-trait)
|
||||
- [4.3 ConcreteEnv](#43-concreteenv)
|
||||
- [4.4 ops](#44-ops)
|
||||
- [4.5 service](#45-service)
|
||||
- [5. The service](#5-the-service)
|
||||
- [5.1 Initialization](#51-initialization)
|
||||
- [5.2 Requests](#53-requests)
|
||||
- [5.3 Responses](#54-responses)
|
||||
- [5.4 Thread model](#52-thread-model)
|
||||
- [5.5 Shutdown](#55-shutdown)
|
||||
- [6. Syncing](#6-Syncing)
|
||||
- [7. Resizing](#7-resizing)
|
||||
- [8. (De)serialization](#8-deserialization)
|
||||
- [9. Schema](#9-schema)
|
||||
- [9.1 Tables](#91-tables)
|
||||
- [9.2 Multimap tables](#92-multimap-tables)
|
||||
- [10. Known issues and tradeoffs](#10-known-issues-and-tradeoffs)
|
||||
- [10.1 Traits abstracting backends](#101-traits-abstracting-backends)
|
||||
- [10.2 Hot-swappable backends](#102-hot-swappable-backends)
|
||||
- [10.3 Copying unaligned bytes](#103-copying-unaligned-bytes)
|
||||
- [10.4 Endianness](#104-endianness)
|
||||
- [10.5 Extra table data](#105-extra-table-data)
|
||||
|
||||
---
|
||||
|
||||
## 1. Documentation
|
||||
Documentation for `database/` is split into 3 locations:
|
||||
|
||||
| Documentation location | Purpose |
|
||||
|---------------------------|---------|
|
||||
| `database/README.md` | High level design of `cuprate-database`
|
||||
| `cuprate-database` | Practical usage documentation/warnings/notes/etc
|
||||
| Source file `// comments` | Implementation-specific details (e.g, how many reader threads to spawn?)
|
||||
|
||||
This README serves as the implementation design document.
|
||||
|
||||
For actual practical usage, `cuprate-database`'s types and general usage are documented via standard Rust tooling.
|
||||
|
||||
Run:
|
||||
```bash
|
||||
cargo doc --package cuprate-database --open
|
||||
```
|
||||
at the root of the repo to open/read the documentation.
|
||||
|
||||
If this documentation is too abstract, refer to any of the source files, they are heavily commented. There are many `// Regular comments` that explain more implementation specific details that aren't present here or in the docs. Use the file reference below to find what you're looking for.
|
||||
|
||||
The code within `src/` is also littered with some `grep`-able comments containing some keywords:
|
||||
|
||||
| Word | Meaning |
|
||||
|-------------|---------|
|
||||
| `INVARIANT` | This code makes an _assumption_ that must be upheld for correctness
|
||||
| `SAFETY` | This `unsafe` code is okay, for `x,y,z` reasons
|
||||
| `FIXME` | This code works but isn't ideal
|
||||
| `HACK` | This code is a brittle workaround
|
||||
| `PERF` | This code is weird for performance reasons
|
||||
| `TODO` | This must be implemented; There should be 0 of these in production code
|
||||
| `SOMEDAY` | This should be implemented... someday
|
||||
|
||||
## 2. File structure
|
||||
A quick reference of the structure of the folders & files in `cuprate-database`.
|
||||
|
||||
Note that `lib.rs/mod.rs` files are purely for re-exporting/visibility/lints, and contain no code. Each sub-directory has a corresponding `mod.rs`.
|
||||
|
||||
### 2.1 `src/`
|
||||
The top-level `src/` files.
|
||||
|
||||
| File | Purpose |
|
||||
|------------------------|---------|
|
||||
| `constants.rs` | General constants used throughout `cuprate-database`
|
||||
| `database.rs` | Abstracted database; `trait DatabaseR{o,w}`
|
||||
| `env.rs` | Abstracted database environment; `trait Env`
|
||||
| `error.rs` | Database error types
|
||||
| `free.rs` | General free functions (related to the database)
|
||||
| `key.rs` | Abstracted database keys; `trait Key`
|
||||
| `resize.rs` | Database resizing algorithms
|
||||
| `storable.rs` | Data (de)serialization; `trait Storable`
|
||||
| `table.rs` | Database table abstraction; `trait Table`
|
||||
| `tables.rs` | All the table definitions used by `cuprate-database`
|
||||
| `tests.rs` | Utilities for `cuprate_database` testing
|
||||
| `transaction.rs` | Database transaction abstraction; `trait TxR{o,w}`
|
||||
| `types.rs` | Database-specific types
|
||||
| `unsafe_unsendable.rs` | Marker type to impl `Send` for objects not `Send`
|
||||
|
||||
### 2.2 `src/backend/`
|
||||
This folder contains the implementation for actual databases used as the backend for `cuprate-database`.
|
||||
|
||||
Each backend has its own folder.
|
||||
|
||||
| Folder/File | Purpose |
|
||||
|-------------|---------|
|
||||
| `heed/` | Backend using using [`heed`](https://github.com/meilisearch/heed) (LMDB)
|
||||
| `redb/` | Backend using [`redb`](https://github.com/cberner/redb)
|
||||
| `tests.rs` | Backend-agnostic tests
|
||||
|
||||
All backends follow the same file structure:
|
||||
|
||||
| File | Purpose |
|
||||
|------------------|---------|
|
||||
| `database.rs` | Implementation of `trait DatabaseR{o,w}`
|
||||
| `env.rs` | Implementation of `trait Env`
|
||||
| `error.rs` | Implementation of backend's errors to `cuprate_database`'s error types
|
||||
| `storable.rs` | Compatibility layer between `cuprate_database::Storable` and backend-specific (de)serialization
|
||||
| `transaction.rs` | Implementation of `trait TxR{o,w}`
|
||||
| `types.rs` | Type aliases for long backend-specific types
|
||||
|
||||
### 2.3 `src/config/`
|
||||
This folder contains the `cupate_database::config` module; configuration options for the database.
|
||||
|
||||
| File | Purpose |
|
||||
|---------------------|---------|
|
||||
| `config.rs` | Main database `Config` struct
|
||||
| `reader_threads.rs` | Reader thread configuration for `service` thread-pool
|
||||
| `sync_mode.rs` | Disk sync configuration for backends
|
||||
|
||||
### 2.4 `src/ops/`
|
||||
This folder contains the `cupate_database::ops` module.
|
||||
|
||||
These are higher-level functions abstracted over the database, that are Monero-related.
|
||||
|
||||
| File | Purpose |
|
||||
|-----------------|---------|
|
||||
| `block.rs` | Block related (main functions)
|
||||
| `blockchain.rs` | Blockchain related (height, cumulative values, etc)
|
||||
| `key_image.rs` | Key image related
|
||||
| `macros.rs` | Macros specific to `ops/`
|
||||
| `output.rs` | Output related
|
||||
| `property.rs` | Database properties (pruned, version, etc)
|
||||
| `tx.rs` | Transaction related
|
||||
|
||||
### 2.5 `src/service/`
|
||||
This folder contains the `cupate_database::service` module.
|
||||
|
||||
The `async`hronous request/response API other Cuprate crates use instead of managing the database directly themselves.
|
||||
|
||||
| File | Purpose |
|
||||
|----------------|---------|
|
||||
| `free.rs` | General free functions used (related to `cuprate_database::service`)
|
||||
| `read.rs` | Read thread-pool definitions and logic
|
||||
| `tests.rs` | Thread-pool tests and test helper functions
|
||||
| `types.rs` | `cuprate_database::service`-related type aliases
|
||||
| `write.rs` | Writer thread definitions and logic
|
||||
|
||||
## 3. Backends
|
||||
`cuprate-database`'s `trait`s allow abstracting over the actual database, such that any backend in particular could be used.
|
||||
|
||||
Each database's implementation for those `trait`'s are located in its respective folder in `src/backend/${DATABASE_NAME}/`.
|
||||
|
||||
### 3.1 heed
|
||||
The default database used is [`heed`](https://github.com/meilisearch/heed) (LMDB). The upstream versions from [`crates.io`](https://crates.io/crates/heed) are used. `LMDB` should not need to be installed as `heed` has a build script that pulls it in automatically.
|
||||
|
||||
`heed`'s filenames inside Cuprate's database folder (`~/.local/share/cuprate/database/`) are:
|
||||
|
||||
| Filename | Purpose |
|
||||
|------------|---------|
|
||||
| `data.mdb` | Main data file
|
||||
| `lock.mdb` | Database lock file
|
||||
|
||||
`heed`-specific notes:
|
||||
- [There is a maximum reader limit](https://github.com/monero-project/monero/blob/059028a30a8ae9752338a7897329fe8012a310d5/src/blockchain_db/lmdb/db_lmdb.cpp#L1372). Other potential processes (e.g. `xmrblocks`) that are also reading the `data.mdb` file need to be accounted for
|
||||
- [LMDB does not work on remote filesystem](https://github.com/LMDB/lmdb/blob/b8e54b4c31378932b69f1298972de54a565185b1/libraries/liblmdb/lmdb.h#L129)
|
||||
|
||||
### 3.2 redb
|
||||
The 2nd database backend is the 100% Rust [`redb`](https://github.com/cberner/redb).
|
||||
|
||||
The upstream versions from [`crates.io`](https://crates.io/crates/redb) are used.
|
||||
|
||||
`redb`'s filenames inside Cuprate's database folder (`~/.local/share/cuprate/database/`) are:
|
||||
|
||||
| Filename | Purpose |
|
||||
|-------------|---------|
|
||||
| `data.redb` | Main data file
|
||||
|
||||
<!-- TODO: document DB on remote filesystem (does redb allow this?) -->
|
||||
|
||||
### 3.3 redb-memory
|
||||
This backend is 100% the same as `redb`, although, it uses `redb::backend::InMemoryBackend` which is a database that completely resides in memory instead of a file.
|
||||
|
||||
All other details about this should be the same as the normal `redb` backend.
|
||||
|
||||
### 3.4 sanakirja
|
||||
[`sanakirja`](https://docs.rs/sanakirja) was a candidate as a backend, however there were problems with maximum value sizes.
|
||||
|
||||
The default maximum value size is [1012 bytes](https://docs.rs/sanakirja/1.4.1/sanakirja/trait.Storable.html) which was too small for our requirements. Using [`sanakirja::Slice`](https://docs.rs/sanakirja/1.4.1/sanakirja/union.Slice.html) and [sanakirja::UnsizedStorage](https://docs.rs/sanakirja/1.4.1/sanakirja/trait.UnsizedStorable.html) was attempted, but there were bugs found when inserting a value in-between `512..=4096` bytes.
|
||||
|
||||
As such, it is not implemented.
|
||||
|
||||
### 3.5 MDBX
|
||||
[`MDBX`](https://erthink.github.io/libmdbx) was a candidate as a backend, however MDBX deprecated the custom key/value comparison functions, this makes it a bit trickier to implement [`9.2 Multimap tables`](#92-multimap-tables). It is also quite similar to the main backend LMDB (of which it was originally a fork of).
|
||||
|
||||
As such, it is not implemented (yet).
|
||||
|
||||
## 4. Layers
|
||||
`cuprate_database` is logically abstracted into 5 layers, with each layer being built upon the last.
|
||||
|
||||
Starting from the lowest:
|
||||
1. Backend
|
||||
2. Trait
|
||||
3. ConcreteEnv
|
||||
4. `ops`
|
||||
5. `service`
|
||||
|
||||
<!-- TODO: insert image here after database/ split -->
|
||||
|
||||
### 4.1 Backend
|
||||
This is the actual database backend implementation (or a Rust shim over one).
|
||||
|
||||
Examples:
|
||||
- `heed` (LMDB)
|
||||
- `redb`
|
||||
|
||||
`cuprate_database` itself just uses a backend, it does not implement one.
|
||||
|
||||
All backends have the following attributes:
|
||||
- [Embedded](https://en.wikipedia.org/wiki/Embedded_database)
|
||||
- [Multiversion concurrency control](https://en.wikipedia.org/wiki/Multiversion_concurrency_control)
|
||||
- [ACID](https://en.wikipedia.org/wiki/ACID)
|
||||
- Are `(key, value)` oriented and have the expected API (`get()`, `insert()`, `delete()`)
|
||||
- Are table oriented (`"table_name" -> (key, value)`)
|
||||
- Allows concurrent readers
|
||||
|
||||
### 4.2 Trait
|
||||
`cuprate_database` provides a set of `trait`s that abstract over the various database backends.
|
||||
|
||||
This allows the function signatures and behavior to stay the same but allows for swapping out databases in an easier fashion.
|
||||
|
||||
All common behavior of the backend's are encapsulated here and used instead of using the backend directly.
|
||||
|
||||
Examples:
|
||||
- [`trait Env`](https://github.com/Cuprate/cuprate/blob/2ac90420c658663564a71b7ecb52d74f3c2c9d0f/database/src/env.rs)
|
||||
- [`trait {TxRo, TxRw}`](https://github.com/Cuprate/cuprate/blob/2ac90420c658663564a71b7ecb52d74f3c2c9d0f/database/src/transaction.rs)
|
||||
- [`trait {DatabaseRo, DatabaseRw}`](https://github.com/Cuprate/cuprate/blob/2ac90420c658663564a71b7ecb52d74f3c2c9d0f/database/src/database.rs)
|
||||
|
||||
For example, instead of calling `LMDB` or `redb`'s `get()` function directly, `DatabaseRo::get()` is called.
|
||||
|
||||
### 4.3 ConcreteEnv
|
||||
This is the non-generic, concrete `struct` provided by `cuprate_database` that contains all the data necessary to operate the database. The actual database backend `ConcreteEnv` will use internally depends on which backend feature is used.
|
||||
|
||||
`ConcreteEnv` implements `trait Env`, which opens the door to all the other traits.
|
||||
|
||||
The equivalent objects in the backends themselves are:
|
||||
- [`heed::Env`](https://docs.rs/heed/0.20.0/heed/struct.Env.html)
|
||||
- [`redb::Database`](https://docs.rs/redb/2.1.0/redb/struct.Database.html)
|
||||
|
||||
This is the main object used when handling the database directly, although that is not strictly necessary as a user if the [`4.5 service`](#45-service) layer is used.
|
||||
|
||||
### 4.4 ops
|
||||
These are Monero-specific functions that use the abstracted `trait` forms of the database.
|
||||
|
||||
Instead of dealing with the database directly:
|
||||
- `get()`
|
||||
- `delete()`
|
||||
|
||||
the `ops` layer provides more abstract functions that deal with commonly used Monero operations:
|
||||
- `add_block()`
|
||||
- `pop_block()`
|
||||
|
||||
### 4.5 service
|
||||
The final layer abstracts the database completely into a [Monero-specific `async` request/response API](https://github.com/Cuprate/cuprate/blob/2ac90420c658663564a71b7ecb52d74f3c2c9d0f/types/src/service.rs#L18-L78) using [`tower::Service`](https://docs.rs/tower/latest/tower/trait.Service.html).
|
||||
|
||||
For more information on this layer, see the next section: [`5. The service`](#5-the-service).
|
||||
|
||||
## 5. The service
|
||||
The main API `cuprate_database` exposes for other crates to use is the `cuprate_database::service` module.
|
||||
|
||||
This module exposes an `async` request/response API with `tower::Service`, backed by a threadpool, that allows reading/writing Monero-related data from/to the database.
|
||||
|
||||
`cuprate_database::service` itself manages the database using a separate writer thread & reader thread-pool, and uses the previously mentioned [`4.4 ops`](#44-ops) functions when responding to requests.
|
||||
|
||||
### 5.1 Initialization
|
||||
The service is started simply by calling: [`cuprate_database::service::init()`](https://github.com/Cuprate/cuprate/blob/d0ac94a813e4cd8e0ed8da5e85a53b1d1ace2463/database/src/service/free.rs#L23).
|
||||
|
||||
This function initializes the database, spawns threads, and returns a:
|
||||
- Read handle to the database (cloneable)
|
||||
- Write handle to the database (not cloneable)
|
||||
|
||||
These "handles" implement the `tower::Service` trait, which allows sending requests and receiving responses `async`hronously.
|
||||
|
||||
### 5.2 Requests
|
||||
Along with the 2 handles, there are 2 types of requests:
|
||||
- [`ReadRequest`](https://github.com/Cuprate/cuprate/blob/d0ac94a813e4cd8e0ed8da5e85a53b1d1ace2463/types/src/service.rs#L23-L90)
|
||||
- [`WriteRequest`](https://github.com/Cuprate/cuprate/blob/d0ac94a813e4cd8e0ed8da5e85a53b1d1ace2463/types/src/service.rs#L93-L105)
|
||||
|
||||
`ReadRequest` is for retrieving various types of information from the database.
|
||||
|
||||
`WriteRequest` currently only has 1 variant: to write a block to the database.
|
||||
|
||||
### 5.3 Responses
|
||||
After sending one of the above requests using the read/write handle, the value returned is _not_ the response, yet an `async`hronous channel that will eventually return the response:
|
||||
```rust,ignore
|
||||
// Send a request.
|
||||
// tower::Service::call()
|
||||
// V
|
||||
let response_channel: Channel = read_handle.call(ReadResponse::ChainHeight)?;
|
||||
|
||||
// Await the response.
|
||||
let response: ReadResponse = response_channel.await?;
|
||||
|
||||
// Assert the response is what we expected.
|
||||
assert_eq!(matches!(response), Response::ChainHeight(_));
|
||||
```
|
||||
|
||||
After `await`ing the returned channel, a `Response` will eventually be returned when the `service` threadpool has fetched the value from the database and sent it off.
|
||||
|
||||
Both read/write requests variants match in name with `Response` variants, i.e.
|
||||
- `ReadRequest::ChainHeight` leads to `Response::ChainHeight`
|
||||
- `WriteRequest::WriteBlock` leads to `Response::WriteBlockOk`
|
||||
|
||||
### 5.4 Thread model
|
||||
As mentioned in the [`4. Layers`](#4-layers) section, the base database abstractions themselves are not concerned with parallelism, they are mostly functions to be called from a single-thread.
|
||||
|
||||
However, the `cuprate_database::service` API, _does_ have a thread model backing it.
|
||||
|
||||
When [`cuprate_database::service`'s initialization function](https://github.com/Cuprate/cuprate/blob/9c27ba5791377d639cb5d30d0f692c228568c122/database/src/service/free.rs#L33-L44) is called, threads will be spawned and maintained until the user drops (disconnects) the returned handles.
|
||||
|
||||
The current behavior for thread count is:
|
||||
- [1 writer thread](https://github.com/Cuprate/cuprate/blob/9c27ba5791377d639cb5d30d0f692c228568c122/database/src/service/write.rs#L52-L66)
|
||||
- [As many reader threads as there are system threads](https://github.com/Cuprate/cuprate/blob/9c27ba5791377d639cb5d30d0f692c228568c122/database/src/service/read.rs#L104-L126)
|
||||
|
||||
For example, on a system with 32-threads, `cuprate_database` will spawn:
|
||||
- 1 writer thread
|
||||
- 32 reader threads
|
||||
|
||||
whose sole responsibility is to listen for database requests, access the database (potentially in parallel), and return a response.
|
||||
|
||||
Note that the `1 system thread = 1 reader thread` model is only the default setting, the reader thread count can be configured by the user to be any number between `1 .. amount_of_system_threads`.
|
||||
|
||||
The reader threads are managed by [`rayon`](https://docs.rs/rayon).
|
||||
|
||||
For an example of where multiple reader threads are used: given a request that asks if any key-image within a set already exists, `cuprate_database` will [split that work between the threads with `rayon`](https://github.com/Cuprate/cuprate/blob/9c27ba5791377d639cb5d30d0f692c228568c122/database/src/service/read.rs#L490-L503).
|
||||
|
||||
### 5.5 Shutdown
|
||||
Once the read/write handles are `Drop`ed, the backing thread(pool) will gracefully exit, automatically.
|
||||
|
||||
Note the writer thread and reader threadpool aren't connected whatsoever; dropping the write handle will make the writer thread exit, however, the reader handle is free to be held onto and can be continued to be read from - and vice-versa for the write handle.
|
||||
|
||||
## 6. Syncing
|
||||
`cuprate_database`'s database has 5 disk syncing modes.
|
||||
|
||||
1. FastThenSafe
|
||||
1. Safe
|
||||
1. Async
|
||||
1. Threshold
|
||||
1. Fast
|
||||
|
||||
The default mode is `Safe`.
|
||||
|
||||
This means that upon each transaction commit, all the data that was written will be fully synced to disk. This is the slowest, but safest mode of operation.
|
||||
|
||||
Note that upon any database `Drop`, whether via `service` or dropping the database directly, the current implementation will sync to disk regardless of any configuration.
|
||||
|
||||
For more information on the other modes, read the documentation [here](https://github.com/Cuprate/cuprate/blob/2ac90420c658663564a71b7ecb52d74f3c2c9d0f/database/src/config/sync_mode.rs#L63-L144).
|
||||
|
||||
## 7. Resizing
|
||||
Database backends that require manually resizing will, by default, use a similar algorithm as `monerod`'s.
|
||||
|
||||
Note that this only relates to the `service` module, where the database is handled by `cuprate_database` itself, not the user. In the case of a user directly using `cuprate_database`, it is up to them on how to resize.
|
||||
|
||||
Within `service`, the resizing logic defined [here](https://github.com/Cuprate/cuprate/blob/2ac90420c658663564a71b7ecb52d74f3c2c9d0f/database/src/service/write.rs#L139-L201) does the following:
|
||||
|
||||
- If there's not enough space to fit a write request's data, start a resize
|
||||
- Each resize adds around [`1_073_745_920`](https://github.com/Cuprate/cuprate/blob/2ac90420c658663564a71b7ecb52d74f3c2c9d0f/database/src/resize.rs#L104-L160) bytes to the current map size
|
||||
- A resize will be attempted `3` times before failing
|
||||
|
||||
There are other [resizing algorithms](https://github.com/Cuprate/cuprate/blob/2ac90420c658663564a71b7ecb52d74f3c2c9d0f/database/src/resize.rs#L38-L47) that define how the database's memory map grows, although currently the behavior of [`monerod`](https://github.com/Cuprate/cuprate/blob/2ac90420c658663564a71b7ecb52d74f3c2c9d0f/database/src/resize.rs#L104-L160) is closely followed.
|
||||
|
||||
## 8. (De)serialization
|
||||
All types stored inside the database are either bytes already, or are perfectly bitcast-able.
|
||||
|
||||
As such, they do not incur heavy (de)serialization costs when storing/fetching them from the database. The main (de)serialization used is [`bytemuck`](https://docs.rs/bytemuck)'s traits and casting functions.
|
||||
|
||||
The size & layout of types is stable across compiler versions, as they are set and determined with [`#[repr(C)]`](https://doc.rust-lang.org/nomicon/other-reprs.html#reprc) and `bytemuck`'s derive macros such as [`bytemuck::Pod`](https://docs.rs/bytemuck/latest/bytemuck/derive.Pod.html).
|
||||
|
||||
Note that the data stored in the tables are still type-safe; we still refer to the key and values within our tables by the type.
|
||||
|
||||
The main deserialization `trait` for database storage is: [`cuprate_database::Storable`](https://github.com/Cuprate/cuprate/blob/2ac90420c658663564a71b7ecb52d74f3c2c9d0f/database/src/storable.rs#L16-L115).
|
||||
|
||||
- Before storage, the type is [simply cast into bytes](https://github.com/Cuprate/cuprate/blob/2ac90420c658663564a71b7ecb52d74f3c2c9d0f/database/src/storable.rs#L125)
|
||||
- When fetching, the bytes are [simply cast into the type](https://github.com/Cuprate/cuprate/blob/2ac90420c658663564a71b7ecb52d74f3c2c9d0f/database/src/storable.rs#L130)
|
||||
|
||||
When a type is casted into bytes, [the reference is casted](https://docs.rs/bytemuck/latest/bytemuck/fn.bytes_of.html), i.e. this is zero-cost serialization.
|
||||
|
||||
However, it is worth noting that when bytes are casted into the type, [it is copied](https://docs.rs/bytemuck/latest/bytemuck/fn.pod_read_unaligned.html). This is due to byte alignment guarantee issues with both backends, see:
|
||||
- https://github.com/AltSysrq/lmdb-zero/issues/8
|
||||
- https://github.com/cberner/redb/issues/360
|
||||
|
||||
Without this, `bytemuck` will panic with [`TargetAlignmentGreaterAndInputNotAligned`](https://docs.rs/bytemuck/latest/bytemuck/enum.PodCastError.html#variant.TargetAlignmentGreaterAndInputNotAligned) when casting.
|
||||
|
||||
Copying the bytes fixes this problem, although it is more costly than necessary. However, in the main use-case for `cuprate_database` (the `service` module) the bytes would need to be owned regardless as the `Request/Response` API uses owned data types (`T`, `Vec<T>`, `HashMap<K, V>`, etc).
|
||||
|
||||
Practically speaking, this means lower-level database functions that normally look like such:
|
||||
```rust
|
||||
fn get(key: &Key) -> &Value;
|
||||
```
|
||||
end up looking like this in `cuprate_database`:
|
||||
```rust
|
||||
fn get(key: &Key) -> Value;
|
||||
```
|
||||
|
||||
Since each backend has its own (de)serialization methods, our types are wrapped in compatibility types that map our `Storable` functions into whatever is required for the backend, e.g:
|
||||
- [`StorableHeed<T>`](https://github.com/Cuprate/cuprate/blob/2ac90420c658663564a71b7ecb52d74f3c2c9d0f/database/src/backend/heed/storable.rs#L11-L45)
|
||||
- [`StorableRedb<T>`](https://github.com/Cuprate/cuprate/blob/2ac90420c658663564a71b7ecb52d74f3c2c9d0f/database/src/backend/redb/storable.rs#L11-L30)
|
||||
|
||||
Compatibility structs also exist for any `Storable` containers:
|
||||
- [`StorableVec<T>`](https://github.com/Cuprate/cuprate/blob/2ac90420c658663564a71b7ecb52d74f3c2c9d0f/database/src/storable.rs#L135-L191)
|
||||
- [`StorableBytes`](https://github.com/Cuprate/cuprate/blob/2ac90420c658663564a71b7ecb52d74f3c2c9d0f/database/src/storable.rs#L208-L241)
|
||||
|
||||
Again, it's unfortunate that these must be owned, although in `service`'s use-case, they would have to be owned anyway.
|
||||
|
||||
## 9. Schema
|
||||
This following section contains Cuprate's database schema, it may change throughout the development of Cuprate, as such, nothing here is final.
|
||||
|
||||
### 9.1 Tables
|
||||
The `CamelCase` names of the table headers documented here (e.g. `TxIds`) are the actual type name of the table within `cuprate_database`.
|
||||
|
||||
Note that words written within `code blocks` mean that it is a real type defined and usable within `cuprate_database`. Other standard types like u64 and type aliases (TxId) are written normally.
|
||||
|
||||
Within `cuprate_database::tables`, the below table is essentially defined as-is with [a macro](https://github.com/Cuprate/cuprate/blob/31ce89412aa174fc33754f22c9a6d9ef5ddeda28/database/src/tables.rs#L369-L470).
|
||||
|
||||
Many of the data types stored are the same data types, although are different semantically, as such, a map of aliases used and their real data types is also provided below.
|
||||
|
||||
| Alias | Real Type |
|
||||
|----------------------------------------------------|-----------|
|
||||
| BlockHeight, Amount, AmountIndex, TxId, UnlockTime | u64
|
||||
| BlockHash, KeyImage, TxHash, PrunableHash | [u8; 32]
|
||||
|
||||
| Table | Key | Value | Description |
|
||||
|-------------------|----------------------|--------------------|-------------|
|
||||
| `BlockBlobs` | BlockHeight | `StorableVec<u8>` | Maps a block's height to a serialized byte form of a block
|
||||
| `BlockHeights` | BlockHash | BlockHeight | Maps a block's hash to its height
|
||||
| `BlockInfos` | BlockHeight | `BlockInfo` | Contains metadata of all blocks
|
||||
| `KeyImages` | KeyImage | () | This table is a set with no value, it stores transaction key images
|
||||
| `NumOutputs` | Amount | u64 | Maps an output's amount to the number of outputs with that amount
|
||||
| `Outputs` | `PreRctOutputId` | `Output` | This table contains legacy CryptoNote outputs which have clear amounts. This table will not contain an output with 0 amount.
|
||||
| `PrunedTxBlobs` | TxId | `StorableVec<u8>` | Contains pruned transaction blobs (even if the database is not pruned)
|
||||
| `PrunableTxBlobs` | TxId | `StorableVec<u8>` | Contains the prunable part of a transaction
|
||||
| `PrunableHashes` | TxId | PrunableHash | Contains the hash of the prunable part of a transaction
|
||||
| `RctOutputs` | AmountIndex | `RctOutput` | Contains RingCT outputs mapped from their global RCT index
|
||||
| `TxBlobs` | TxId | `StorableVec<u8>` | Serialized transaction blobs (bytes)
|
||||
| `TxIds` | TxHash | TxId | Maps a transaction's hash to its index/ID
|
||||
| `TxHeights` | TxId | BlockHeight | Maps a transaction's ID to the height of the block it comes from
|
||||
| `TxOutputs` | TxId | `StorableVec<u64>` | Gives the amount indices of a transaction's outputs
|
||||
| `TxUnlockTime` | TxId | UnlockTime | Stores the unlock time of a transaction (only if it has a non-zero lock time)
|
||||
|
||||
The definitions for aliases and types (e.g. `RctOutput`) are within the [`cuprate_database::types`](https://github.com/Cuprate/cuprate/blob/31ce89412aa174fc33754f22c9a6d9ef5ddeda28/database/src/types.rs#L51) module.
|
||||
|
||||
<!-- TODO(Boog900): We could split this table again into `RingCT (non-miner) Outputs` and `RingCT (miner) Outputs` as for miner outputs we can store the amount instead of commitment saving 24 bytes per miner output. -->
|
||||
|
||||
### 9.2 Multimap tables
|
||||
When referencing outputs, Monero will [use the amount and the amount index](https://github.com/monero-project/monero/blob/c8214782fb2a769c57382a999eaf099691c836e7/src/blockchain_db/lmdb/db_lmdb.cpp#L3447-L3449). This means 2 keys are needed to reach an output.
|
||||
|
||||
With LMDB you can set the `DUP_SORT` flag on a table and then set the key/value to:
|
||||
```rust
|
||||
Key = KEY_PART_1
|
||||
```
|
||||
```rust
|
||||
Value = {
|
||||
KEY_PART_2,
|
||||
VALUE // The actual value we are storing.
|
||||
}
|
||||
```
|
||||
|
||||
Then you can set a custom value sorting function that only takes `KEY_PART_2` into account; this is how `monerod` does it.
|
||||
|
||||
This requires that the underlying database supports:
|
||||
- multimap tables
|
||||
- custom sort functions on values
|
||||
- setting a cursor on a specific key/value
|
||||
|
||||
---
|
||||
|
||||
Another way to implement this is as follows:
|
||||
```rust
|
||||
Key = { KEY_PART_1, KEY_PART_2 }
|
||||
```
|
||||
```rust
|
||||
Value = VALUE
|
||||
```
|
||||
|
||||
Then the key type is simply used to look up the value; this is how `cuprate_database` does it.
|
||||
|
||||
For example, the key/value pair for outputs is:
|
||||
```rust
|
||||
PreRctOutputId => Output
|
||||
```
|
||||
where `PreRctOutputId` looks like this:
|
||||
```rust
|
||||
struct PreRctOutputId {
|
||||
amount: u64,
|
||||
amount_index: u64,
|
||||
}
|
||||
```
|
||||
|
||||
## 10. Known issues and tradeoffs
|
||||
`cuprate_database` takes many tradeoffs, whether due to:
|
||||
- Prioritizing certain values over others
|
||||
- Not having a better solution
|
||||
- Being "good enough"
|
||||
|
||||
This is a list of the larger ones, along with issues that don't have answers yet.
|
||||
|
||||
### 10.1 Traits abstracting backends
|
||||
Although all database backends used are very similar, they have some crucial differences in small implementation details that must be worked around when conforming them to `cuprate_database`'s traits.
|
||||
|
||||
Put simply: using `cuprate_database`'s traits is less efficient and more awkward than using the backend directly.
|
||||
|
||||
For example:
|
||||
- [Data types must be wrapped in compatibility layers when they otherwise wouldn't be](https://github.com/Cuprate/cuprate/blob/d0ac94a813e4cd8e0ed8da5e85a53b1d1ace2463/database/src/backend/heed/env.rs#L101-L116)
|
||||
- [There are types that only apply to a specific backend, but are visible to all](https://github.com/Cuprate/cuprate/blob/d0ac94a813e4cd8e0ed8da5e85a53b1d1ace2463/database/src/error.rs#L86-L89)
|
||||
- [There are extra layers of abstraction to smoothen the differences between all backends](https://github.com/Cuprate/cuprate/blob/d0ac94a813e4cd8e0ed8da5e85a53b1d1ace2463/database/src/env.rs#L62-L68)
|
||||
- [Existing functionality of backends must be taken away, as it isn't supported in the others](https://github.com/Cuprate/cuprate/blob/d0ac94a813e4cd8e0ed8da5e85a53b1d1ace2463/database/src/database.rs#L27-L34)
|
||||
|
||||
This is a _tradeoff_ that `cuprate_database` takes, as:
|
||||
- The backend itself is usually not the source of bottlenecks in the greater system, as such, small inefficiencies are OK
|
||||
- None of the lost functionality is crucial for operation
|
||||
- The ability to use, test, and swap between multiple database backends is [worth it](https://github.com/Cuprate/cuprate/pull/35#issuecomment-1952804393)
|
||||
|
||||
### 10.2 Hot-swappable backends
|
||||
Using a different backend is really as simple as re-building `cuprate_database` with a different feature flag:
|
||||
```bash
|
||||
# Use LMDB.
|
||||
cargo build --package cuprate-database --features heed
|
||||
|
||||
# Use redb.
|
||||
cargo build --package cuprate-database --features redb
|
||||
```
|
||||
|
||||
This is "good enough" for now, however ideally, this hot-swapping of backends would be able to be done at _runtime_.
|
||||
|
||||
As it is now, `cuprate_database` cannot compile both backends and swap based on user input at runtime; it must be compiled with a certain backend, which will produce a binary with only that backend.
|
||||
|
||||
This also means things like [CI testing multiple backends is awkward](https://github.com/Cuprate/cuprate/blob/main/.github/workflows/ci.yml#L132-L136), as we must re-compile with different feature flags instead.
|
||||
|
||||
### 10.3 Copying unaligned bytes
|
||||
As mentioned in [`8. (De)serialization`](#8-deserialization), bytes are _copied_ when they are turned into a type `T` due to unaligned bytes being returned from database backends.
|
||||
|
||||
Using a regular reference cast results in an improperly aligned type `T`; [such a type even existing causes undefined behavior](https://doc.rust-lang.org/reference/behavior-considered-undefined.html). In our case, `bytemuck` saves us by panicking before this occurs.
|
||||
|
||||
Thus, when using `cuprate_database`'s database traits, an _owned_ `T` is returned.
|
||||
|
||||
This is doubly unfortunately for `&[u8]` as this does not even need deserialization.
|
||||
|
||||
For example, `StorableVec` could have been this:
|
||||
```rust
|
||||
enum StorableBytes<'a, T: Storable> {
|
||||
Owned(T),
|
||||
Ref(&'a T),
|
||||
}
|
||||
```
|
||||
but this would require supporting types that must be copied regardless with the occasional `&[u8]` that can be returned without casting. This was hard to do so in a generic way, thus all `[u8]`'s are copied and returned as owned `StorableVec`s.
|
||||
|
||||
This is a _tradeoff_ `cuprate_database` takes as:
|
||||
- `bytemuck::pod_read_unaligned` is cheap enough
|
||||
- The main API, `service`, needs to return owned value anyway
|
||||
- Having no references removes a lot of lifetime complexity
|
||||
|
||||
The alternative is either:
|
||||
- Using proper (de)serialization instead of casting (which comes with its own costs)
|
||||
- Somehow fixing the alignment issues in the backends mentioned previously
|
||||
|
||||
### 10.4 Endianness
|
||||
`cuprate_database`'s (de)serialization and storage of bytes are native-endian, as in, byte storage order will depend on the machine it is running on.
|
||||
|
||||
As Cuprate's build-targets are all little-endian ([big-endian by default machines barely exist](https://en.wikipedia.org/wiki/Endianness#Hardware)), this doesn't matter much and the byte ordering can be seen as a constant.
|
||||
|
||||
Practically, this means `cuprated`'s database files can be transferred across computers, as can `monerod`'s.
|
||||
|
||||
### 10.5 Extra table data
|
||||
Some of `cuprate_database`'s tables differ from `monerod`'s tables, for example, the way [`9.2 Multimap tables`](#92-multimap-tables) tables are done requires that the primary key is stored _for all_ entries, compared to `monerod` only needing to store it once.
|
||||
|
||||
For example:
|
||||
```rust
|
||||
// `monerod` only stores `amount: 1` once,
|
||||
// `cuprated` stores it each time it appears.
|
||||
struct PreRctOutputId { amount: 1, amount_index: 0 }
|
||||
struct PreRctOutputId { amount: 1, amount_index: 1 }
|
||||
```
|
||||
|
||||
This means `cuprated`'s database will be slightly larger than `monerod`'s.
|
||||
|
||||
The current method `cuprate_database` uses will be "good enough" until usage shows that it must be optimized as multimap tables are tricky to implement across all backends.
|
||||
|
|
@ -1,600 +1,105 @@
|
|||
# Database
|
||||
FIXME: This documentation must be updated and moved to the architecture book.
|
||||
Cuprate's blockchain database.
|
||||
|
||||
Cuprate's blockchain implementation.
|
||||
This documentation is mostly for practical usage of `cuprate_blockchain`.
|
||||
|
||||
- [1. Documentation](#1-documentation)
|
||||
- [2. File structure](#2-file-structure)
|
||||
- [2.1 `src/`](#21-src)
|
||||
- [2.2 `src/backend/`](#22-srcbackend)
|
||||
- [2.3 `src/config/`](#23-srcconfig)
|
||||
- [2.4 `src/ops/`](#24-srcops)
|
||||
- [2.5 `src/service/`](#25-srcservice)
|
||||
- [3. Backends](#3-backends)
|
||||
- [3.1 heed](#31-heed)
|
||||
- [3.2 redb](#32-redb)
|
||||
- [3.3 redb-memory](#33-redb-memory)
|
||||
- [3.4 sanakirja](#34-sanakirja)
|
||||
- [3.5 MDBX](#35-mdbx)
|
||||
- [4. Layers](#4-layers)
|
||||
- [4.1 Backend](#41-backend)
|
||||
- [4.2 Trait](#42-trait)
|
||||
- [4.3 ConcreteEnv](#43-concreteenv)
|
||||
- [4.4 ops](#44-ops)
|
||||
- [4.5 service](#45-service)
|
||||
- [5. The service](#5-the-service)
|
||||
- [5.1 Initialization](#51-initialization)
|
||||
- [5.2 Requests](#53-requests)
|
||||
- [5.3 Responses](#54-responses)
|
||||
- [5.4 Thread model](#52-thread-model)
|
||||
- [5.5 Shutdown](#55-shutdown)
|
||||
- [6. Syncing](#6-Syncing)
|
||||
- [7. Resizing](#7-resizing)
|
||||
- [8. (De)serialization](#8-deserialization)
|
||||
- [9. Schema](#9-schema)
|
||||
- [9.1 Tables](#91-tables)
|
||||
- [9.2 Multimap tables](#92-multimap-tables)
|
||||
- [10. Known issues and tradeoffs](#10-known-issues-and-tradeoffs)
|
||||
- [10.1 Traits abstracting backends](#101-traits-abstracting-backends)
|
||||
- [10.2 Hot-swappable backends](#102-hot-swappable-backends)
|
||||
- [10.3 Copying unaligned bytes](#103-copying-unaligned-bytes)
|
||||
- [10.4 Endianness](#104-endianness)
|
||||
- [10.5 Extra table data](#105-extra-table-data)
|
||||
For a high-level overview, see the database section in
|
||||
[Cuprate's architecture book](https://architecture.cuprate.org).
|
||||
|
||||
---
|
||||
# Purpose
|
||||
This crate does 3 things:
|
||||
1. Uses [`cuprate_database`] as a base database layer
|
||||
1. Implements various `Monero` related [operations](ops), [tables], and [types]
|
||||
1. Exposes a [`tower::Service`] backed by a thread-pool
|
||||
|
||||
## 1. Documentation
|
||||
Documentation for `database/` is split into 3 locations:
|
||||
Each layer builds on-top of the previous.
|
||||
|
||||
| Documentation location | Purpose |
|
||||
|---------------------------|---------|
|
||||
| `database/README.md` | High level design of `cuprate-database`
|
||||
| `cuprate-database` | Practical usage documentation/warnings/notes/etc
|
||||
| Source file `// comments` | Implementation-specific details (e.g, how many reader threads to spawn?)
|
||||
As a user of `cuprate_blockchain`, consider using the higher-level [`service`] module,
|
||||
or at the very least the [`ops`] module instead of interacting with the `cuprate_database` traits directly.
|
||||
|
||||
This README serves as the implementation design document.
|
||||
# `cuprate_database`
|
||||
Consider reading `cuprate_database`'s crate documentation before this crate, as it is the first layer.
|
||||
|
||||
For actual practical usage, `cuprate-database`'s types and general usage are documented via standard Rust tooling.
|
||||
|
||||
Run:
|
||||
```bash
|
||||
cargo doc --package cuprate-database --open
|
||||
If/when this crate needs is used, be sure to use the version that this crate re-exports, e.g.:
|
||||
```rust
|
||||
use cuprate_blockchain::{
|
||||
cuprate_database::RuntimeError,
|
||||
};
|
||||
```
|
||||
at the root of the repo to open/read the documentation.
|
||||
This ensures the types/traits used from `cuprate_database` are the same ones used by `cuprate_blockchain` internally.
|
||||
|
||||
If this documentation is too abstract, refer to any of the source files, they are heavily commented. There are many `// Regular comments` that explain more implementation specific details that aren't present here or in the docs. Use the file reference below to find what you're looking for.
|
||||
# Feature flags
|
||||
The `service` module requires the `service` feature to be enabled.
|
||||
See the module for more documentation.
|
||||
|
||||
The code within `src/` is also littered with some `grep`-able comments containing some keywords:
|
||||
|
||||
| Word | Meaning |
|
||||
|-------------|---------|
|
||||
| `INVARIANT` | This code makes an _assumption_ that must be upheld for correctness
|
||||
| `SAFETY` | This `unsafe` code is okay, for `x,y,z` reasons
|
||||
| `FIXME` | This code works but isn't ideal
|
||||
| `HACK` | This code is a brittle workaround
|
||||
| `PERF` | This code is weird for performance reasons
|
||||
| `TODO` | This must be implemented; There should be 0 of these in production code
|
||||
| `SOMEDAY` | This should be implemented... someday
|
||||
|
||||
## 2. File structure
|
||||
A quick reference of the structure of the folders & files in `cuprate-database`.
|
||||
|
||||
Note that `lib.rs/mod.rs` files are purely for re-exporting/visibility/lints, and contain no code. Each sub-directory has a corresponding `mod.rs`.
|
||||
|
||||
### 2.1 `src/`
|
||||
The top-level `src/` files.
|
||||
|
||||
| File | Purpose |
|
||||
|------------------------|---------|
|
||||
| `constants.rs` | General constants used throughout `cuprate-database`
|
||||
| `database.rs` | Abstracted database; `trait DatabaseR{o,w}`
|
||||
| `env.rs` | Abstracted database environment; `trait Env`
|
||||
| `error.rs` | Database error types
|
||||
| `free.rs` | General free functions (related to the database)
|
||||
| `key.rs` | Abstracted database keys; `trait Key`
|
||||
| `resize.rs` | Database resizing algorithms
|
||||
| `storable.rs` | Data (de)serialization; `trait Storable`
|
||||
| `table.rs` | Database table abstraction; `trait Table`
|
||||
| `tables.rs` | All the table definitions used by `cuprate-database`
|
||||
| `tests.rs` | Utilities for `cuprate_database` testing
|
||||
| `transaction.rs` | Database transaction abstraction; `trait TxR{o,w}`
|
||||
| `types.rs` | Database-specific types
|
||||
| `unsafe_unsendable.rs` | Marker type to impl `Send` for objects not `Send`
|
||||
|
||||
### 2.2 `src/backend/`
|
||||
This folder contains the implementation for actual databases used as the backend for `cuprate-database`.
|
||||
|
||||
Each backend has its own folder.
|
||||
|
||||
| Folder/File | Purpose |
|
||||
|-------------|---------|
|
||||
| `heed/` | Backend using using [`heed`](https://github.com/meilisearch/heed) (LMDB)
|
||||
| `redb/` | Backend using [`redb`](https://github.com/cberner/redb)
|
||||
| `tests.rs` | Backend-agnostic tests
|
||||
|
||||
All backends follow the same file structure:
|
||||
|
||||
| File | Purpose |
|
||||
|------------------|---------|
|
||||
| `database.rs` | Implementation of `trait DatabaseR{o,w}`
|
||||
| `env.rs` | Implementation of `trait Env`
|
||||
| `error.rs` | Implementation of backend's errors to `cuprate_database`'s error types
|
||||
| `storable.rs` | Compatibility layer between `cuprate_database::Storable` and backend-specific (de)serialization
|
||||
| `transaction.rs` | Implementation of `trait TxR{o,w}`
|
||||
| `types.rs` | Type aliases for long backend-specific types
|
||||
|
||||
### 2.3 `src/config/`
|
||||
This folder contains the `cupate_database::config` module; configuration options for the database.
|
||||
|
||||
| File | Purpose |
|
||||
|---------------------|---------|
|
||||
| `config.rs` | Main database `Config` struct
|
||||
| `reader_threads.rs` | Reader thread configuration for `service` thread-pool
|
||||
| `sync_mode.rs` | Disk sync configuration for backends
|
||||
|
||||
### 2.4 `src/ops/`
|
||||
This folder contains the `cupate_database::ops` module.
|
||||
|
||||
These are higher-level functions abstracted over the database, that are Monero-related.
|
||||
|
||||
| File | Purpose |
|
||||
|-----------------|---------|
|
||||
| `block.rs` | Block related (main functions)
|
||||
| `blockchain.rs` | Blockchain related (height, cumulative values, etc)
|
||||
| `key_image.rs` | Key image related
|
||||
| `macros.rs` | Macros specific to `ops/`
|
||||
| `output.rs` | Output related
|
||||
| `property.rs` | Database properties (pruned, version, etc)
|
||||
| `tx.rs` | Transaction related
|
||||
|
||||
### 2.5 `src/service/`
|
||||
This folder contains the `cupate_database::service` module.
|
||||
|
||||
The `async`hronous request/response API other Cuprate crates use instead of managing the database directly themselves.
|
||||
|
||||
| File | Purpose |
|
||||
|----------------|---------|
|
||||
| `free.rs` | General free functions used (related to `cuprate_database::service`)
|
||||
| `read.rs` | Read thread-pool definitions and logic
|
||||
| `tests.rs` | Thread-pool tests and test helper functions
|
||||
| `types.rs` | `cuprate_database::service`-related type aliases
|
||||
| `write.rs` | Writer thread definitions and logic
|
||||
|
||||
## 3. Backends
|
||||
`cuprate-database`'s `trait`s allow abstracting over the actual database, such that any backend in particular could be used.
|
||||
|
||||
Each database's implementation for those `trait`'s are located in its respective folder in `src/backend/${DATABASE_NAME}/`.
|
||||
|
||||
### 3.1 heed
|
||||
The default database used is [`heed`](https://github.com/meilisearch/heed) (LMDB). The upstream versions from [`crates.io`](https://crates.io/crates/heed) are used. `LMDB` should not need to be installed as `heed` has a build script that pulls it in automatically.
|
||||
|
||||
`heed`'s filenames inside Cuprate's database folder (`~/.local/share/cuprate/database/`) are:
|
||||
|
||||
| Filename | Purpose |
|
||||
|------------|---------|
|
||||
| `data.mdb` | Main data file
|
||||
| `lock.mdb` | Database lock file
|
||||
|
||||
`heed`-specific notes:
|
||||
- [There is a maximum reader limit](https://github.com/monero-project/monero/blob/059028a30a8ae9752338a7897329fe8012a310d5/src/blockchain_db/lmdb/db_lmdb.cpp#L1372). Other potential processes (e.g. `xmrblocks`) that are also reading the `data.mdb` file need to be accounted for
|
||||
- [LMDB does not work on remote filesystem](https://github.com/LMDB/lmdb/blob/b8e54b4c31378932b69f1298972de54a565185b1/libraries/liblmdb/lmdb.h#L129)
|
||||
|
||||
### 3.2 redb
|
||||
The 2nd database backend is the 100% Rust [`redb`](https://github.com/cberner/redb).
|
||||
|
||||
The upstream versions from [`crates.io`](https://crates.io/crates/redb) are used.
|
||||
|
||||
`redb`'s filenames inside Cuprate's database folder (`~/.local/share/cuprate/database/`) are:
|
||||
|
||||
| Filename | Purpose |
|
||||
|-------------|---------|
|
||||
| `data.redb` | Main data file
|
||||
|
||||
<!-- TODO: document DB on remote filesystem (does redb allow this?) -->
|
||||
|
||||
### 3.3 redb-memory
|
||||
This backend is 100% the same as `redb`, although, it uses `redb::backend::InMemoryBackend` which is a database that completely resides in memory instead of a file.
|
||||
|
||||
All other details about this should be the same as the normal `redb` backend.
|
||||
|
||||
### 3.4 sanakirja
|
||||
[`sanakirja`](https://docs.rs/sanakirja) was a candidate as a backend, however there were problems with maximum value sizes.
|
||||
|
||||
The default maximum value size is [1012 bytes](https://docs.rs/sanakirja/1.4.1/sanakirja/trait.Storable.html) which was too small for our requirements. Using [`sanakirja::Slice`](https://docs.rs/sanakirja/1.4.1/sanakirja/union.Slice.html) and [sanakirja::UnsizedStorage](https://docs.rs/sanakirja/1.4.1/sanakirja/trait.UnsizedStorable.html) was attempted, but there were bugs found when inserting a value in-between `512..=4096` bytes.
|
||||
|
||||
As such, it is not implemented.
|
||||
|
||||
### 3.5 MDBX
|
||||
[`MDBX`](https://erthink.github.io/libmdbx) was a candidate as a backend, however MDBX deprecated the custom key/value comparison functions, this makes it a bit trickier to implement [`9.2 Multimap tables`](#92-multimap-tables). It is also quite similar to the main backend LMDB (of which it was originally a fork of).
|
||||
|
||||
As such, it is not implemented (yet).
|
||||
|
||||
## 4. Layers
|
||||
`cuprate_database` is logically abstracted into 5 layers, with each layer being built upon the last.
|
||||
|
||||
Starting from the lowest:
|
||||
1. Backend
|
||||
2. Trait
|
||||
3. ConcreteEnv
|
||||
4. `ops`
|
||||
5. `service`
|
||||
|
||||
<!-- TODO: insert image here after database/ split -->
|
||||
|
||||
### 4.1 Backend
|
||||
This is the actual database backend implementation (or a Rust shim over one).
|
||||
|
||||
Examples:
|
||||
Different database backends are enabled by the feature flags:
|
||||
- `heed` (LMDB)
|
||||
- `redb`
|
||||
|
||||
`cuprate_database` itself just uses a backend, it does not implement one.
|
||||
The default is `heed`.
|
||||
|
||||
All backends have the following attributes:
|
||||
- [Embedded](https://en.wikipedia.org/wiki/Embedded_database)
|
||||
- [Multiversion concurrency control](https://en.wikipedia.org/wiki/Multiversion_concurrency_control)
|
||||
- [ACID](https://en.wikipedia.org/wiki/ACID)
|
||||
- Are `(key, value)` oriented and have the expected API (`get()`, `insert()`, `delete()`)
|
||||
- Are table oriented (`"table_name" -> (key, value)`)
|
||||
- Allows concurrent readers
|
||||
`tracing` is always enabled and cannot be disabled via feature-flag.
|
||||
<!-- FIXME: tracing should be behind a feature flag -->
|
||||
|
||||
### 4.2 Trait
|
||||
`cuprate_database` provides a set of `trait`s that abstract over the various database backends.
|
||||
# Invariants when not using `service`
|
||||
`cuprate_blockchain` can be used without the `service` feature enabled but
|
||||
there are some things that must be kept in mind when doing so.
|
||||
|
||||
This allows the function signatures and behavior to stay the same but allows for swapping out databases in an easier fashion.
|
||||
Failing to uphold these invariants may cause panics.
|
||||
|
||||
All common behavior of the backend's are encapsulated here and used instead of using the backend directly.
|
||||
1. `LMDB` requires the user to resize the memory map resizing (see [`cuprate_database::RuntimeError::ResizeNeeded`]
|
||||
1. `LMDB` has a maximum reader transaction count, currently it is set to `128`
|
||||
1. `LMDB` has [maximum key/value byte size](http://www.lmdb.tech/doc/group__internal.html#gac929399f5d93cef85f874b9e9b1d09e0) which must not be exceeded
|
||||
|
||||
Examples:
|
||||
- [`trait Env`](https://github.com/Cuprate/cuprate/blob/2ac90420c658663564a71b7ecb52d74f3c2c9d0f/database/src/env.rs)
|
||||
- [`trait {TxRo, TxRw}`](https://github.com/Cuprate/cuprate/blob/2ac90420c658663564a71b7ecb52d74f3c2c9d0f/database/src/transaction.rs)
|
||||
- [`trait {DatabaseRo, DatabaseRw}`](https://github.com/Cuprate/cuprate/blob/2ac90420c658663564a71b7ecb52d74f3c2c9d0f/database/src/database.rs)
|
||||
# Examples
|
||||
The below is an example of using `cuprate_blockchain`
|
||||
lowest API, i.e. using a mix of this crate and `cuprate_database`'s traits directly -
|
||||
**this is NOT recommended.**
|
||||
|
||||
For example, instead of calling `LMDB` or `redb`'s `get()` function directly, `DatabaseRo::get()` is called.
|
||||
For examples of the higher-level APIs, see:
|
||||
- [`ops`]
|
||||
- [`service`]
|
||||
|
||||
### 4.3 ConcreteEnv
|
||||
This is the non-generic, concrete `struct` provided by `cuprate_database` that contains all the data necessary to operate the database. The actual database backend `ConcreteEnv` will use internally depends on which backend feature is used.
|
||||
|
||||
`ConcreteEnv` implements `trait Env`, which opens the door to all the other traits.
|
||||
|
||||
The equivalent objects in the backends themselves are:
|
||||
- [`heed::Env`](https://docs.rs/heed/0.20.0/heed/struct.Env.html)
|
||||
- [`redb::Database`](https://docs.rs/redb/2.1.0/redb/struct.Database.html)
|
||||
|
||||
This is the main object used when handling the database directly, although that is not strictly necessary as a user if the [`4.5 service`](#45-service) layer is used.
|
||||
|
||||
### 4.4 ops
|
||||
These are Monero-specific functions that use the abstracted `trait` forms of the database.
|
||||
|
||||
Instead of dealing with the database directly:
|
||||
- `get()`
|
||||
- `delete()`
|
||||
|
||||
the `ops` layer provides more abstract functions that deal with commonly used Monero operations:
|
||||
- `add_block()`
|
||||
- `pop_block()`
|
||||
|
||||
### 4.5 service
|
||||
The final layer abstracts the database completely into a [Monero-specific `async` request/response API](https://github.com/Cuprate/cuprate/blob/2ac90420c658663564a71b7ecb52d74f3c2c9d0f/types/src/service.rs#L18-L78) using [`tower::Service`](https://docs.rs/tower/latest/tower/trait.Service.html).
|
||||
|
||||
For more information on this layer, see the next section: [`5. The service`](#5-the-service).
|
||||
|
||||
## 5. The service
|
||||
The main API `cuprate_database` exposes for other crates to use is the `cuprate_database::service` module.
|
||||
|
||||
This module exposes an `async` request/response API with `tower::Service`, backed by a threadpool, that allows reading/writing Monero-related data from/to the database.
|
||||
|
||||
`cuprate_database::service` itself manages the database using a separate writer thread & reader thread-pool, and uses the previously mentioned [`4.4 ops`](#44-ops) functions when responding to requests.
|
||||
|
||||
### 5.1 Initialization
|
||||
The service is started simply by calling: [`cuprate_database::service::init()`](https://github.com/Cuprate/cuprate/blob/d0ac94a813e4cd8e0ed8da5e85a53b1d1ace2463/database/src/service/free.rs#L23).
|
||||
|
||||
This function initializes the database, spawns threads, and returns a:
|
||||
- Read handle to the database (cloneable)
|
||||
- Write handle to the database (not cloneable)
|
||||
|
||||
These "handles" implement the `tower::Service` trait, which allows sending requests and receiving responses `async`hronously.
|
||||
|
||||
### 5.2 Requests
|
||||
Along with the 2 handles, there are 2 types of requests:
|
||||
- [`ReadRequest`](https://github.com/Cuprate/cuprate/blob/d0ac94a813e4cd8e0ed8da5e85a53b1d1ace2463/types/src/service.rs#L23-L90)
|
||||
- [`WriteRequest`](https://github.com/Cuprate/cuprate/blob/d0ac94a813e4cd8e0ed8da5e85a53b1d1ace2463/types/src/service.rs#L93-L105)
|
||||
|
||||
`ReadRequest` is for retrieving various types of information from the database.
|
||||
|
||||
`WriteRequest` currently only has 1 variant: to write a block to the database.
|
||||
|
||||
### 5.3 Responses
|
||||
After sending one of the above requests using the read/write handle, the value returned is _not_ the response, yet an `async`hronous channel that will eventually return the response:
|
||||
```rust,ignore
|
||||
// Send a request.
|
||||
// tower::Service::call()
|
||||
// V
|
||||
let response_channel: Channel = read_handle.call(ReadResponse::ChainHeight)?;
|
||||
|
||||
// Await the response.
|
||||
let response: ReadResponse = response_channel.await?;
|
||||
|
||||
// Assert the response is what we expected.
|
||||
assert_eq!(matches!(response), Response::ChainHeight(_));
|
||||
```
|
||||
|
||||
After `await`ing the returned channel, a `Response` will eventually be returned when the `service` threadpool has fetched the value from the database and sent it off.
|
||||
|
||||
Both read/write requests variants match in name with `Response` variants, i.e.
|
||||
- `ReadRequest::ChainHeight` leads to `Response::ChainHeight`
|
||||
- `WriteRequest::WriteBlock` leads to `Response::WriteBlockOk`
|
||||
|
||||
### 5.4 Thread model
|
||||
As mentioned in the [`4. Layers`](#4-layers) section, the base database abstractions themselves are not concerned with parallelism, they are mostly functions to be called from a single-thread.
|
||||
|
||||
However, the `cuprate_database::service` API, _does_ have a thread model backing it.
|
||||
|
||||
When [`cuprate_database::service`'s initialization function](https://github.com/Cuprate/cuprate/blob/9c27ba5791377d639cb5d30d0f692c228568c122/database/src/service/free.rs#L33-L44) is called, threads will be spawned and maintained until the user drops (disconnects) the returned handles.
|
||||
|
||||
The current behavior for thread count is:
|
||||
- [1 writer thread](https://github.com/Cuprate/cuprate/blob/9c27ba5791377d639cb5d30d0f692c228568c122/database/src/service/write.rs#L52-L66)
|
||||
- [As many reader threads as there are system threads](https://github.com/Cuprate/cuprate/blob/9c27ba5791377d639cb5d30d0f692c228568c122/database/src/service/read.rs#L104-L126)
|
||||
|
||||
For example, on a system with 32-threads, `cuprate_database` will spawn:
|
||||
- 1 writer thread
|
||||
- 32 reader threads
|
||||
|
||||
whose sole responsibility is to listen for database requests, access the database (potentially in parallel), and return a response.
|
||||
|
||||
Note that the `1 system thread = 1 reader thread` model is only the default setting, the reader thread count can be configured by the user to be any number between `1 .. amount_of_system_threads`.
|
||||
|
||||
The reader threads are managed by [`rayon`](https://docs.rs/rayon).
|
||||
|
||||
For an example of where multiple reader threads are used: given a request that asks if any key-image within a set already exists, `cuprate_database` will [split that work between the threads with `rayon`](https://github.com/Cuprate/cuprate/blob/9c27ba5791377d639cb5d30d0f692c228568c122/database/src/service/read.rs#L490-L503).
|
||||
|
||||
### 5.5 Shutdown
|
||||
Once the read/write handles are `Drop`ed, the backing thread(pool) will gracefully exit, automatically.
|
||||
|
||||
Note the writer thread and reader threadpool aren't connected whatsoever; dropping the write handle will make the writer thread exit, however, the reader handle is free to be held onto and can be continued to be read from - and vice-versa for the write handle.
|
||||
|
||||
## 6. Syncing
|
||||
`cuprate_database`'s database has 5 disk syncing modes.
|
||||
|
||||
1. FastThenSafe
|
||||
1. Safe
|
||||
1. Async
|
||||
1. Threshold
|
||||
1. Fast
|
||||
|
||||
The default mode is `Safe`.
|
||||
|
||||
This means that upon each transaction commit, all the data that was written will be fully synced to disk. This is the slowest, but safest mode of operation.
|
||||
|
||||
Note that upon any database `Drop`, whether via `service` or dropping the database directly, the current implementation will sync to disk regardless of any configuration.
|
||||
|
||||
For more information on the other modes, read the documentation [here](https://github.com/Cuprate/cuprate/blob/2ac90420c658663564a71b7ecb52d74f3c2c9d0f/database/src/config/sync_mode.rs#L63-L144).
|
||||
|
||||
## 7. Resizing
|
||||
Database backends that require manually resizing will, by default, use a similar algorithm as `monerod`'s.
|
||||
|
||||
Note that this only relates to the `service` module, where the database is handled by `cuprate_database` itself, not the user. In the case of a user directly using `cuprate_database`, it is up to them on how to resize.
|
||||
|
||||
Within `service`, the resizing logic defined [here](https://github.com/Cuprate/cuprate/blob/2ac90420c658663564a71b7ecb52d74f3c2c9d0f/database/src/service/write.rs#L139-L201) does the following:
|
||||
|
||||
- If there's not enough space to fit a write request's data, start a resize
|
||||
- Each resize adds around [`1_073_745_920`](https://github.com/Cuprate/cuprate/blob/2ac90420c658663564a71b7ecb52d74f3c2c9d0f/database/src/resize.rs#L104-L160) bytes to the current map size
|
||||
- A resize will be attempted `3` times before failing
|
||||
|
||||
There are other [resizing algorithms](https://github.com/Cuprate/cuprate/blob/2ac90420c658663564a71b7ecb52d74f3c2c9d0f/database/src/resize.rs#L38-L47) that define how the database's memory map grows, although currently the behavior of [`monerod`](https://github.com/Cuprate/cuprate/blob/2ac90420c658663564a71b7ecb52d74f3c2c9d0f/database/src/resize.rs#L104-L160) is closely followed.
|
||||
|
||||
## 8. (De)serialization
|
||||
All types stored inside the database are either bytes already, or are perfectly bitcast-able.
|
||||
|
||||
As such, they do not incur heavy (de)serialization costs when storing/fetching them from the database. The main (de)serialization used is [`bytemuck`](https://docs.rs/bytemuck)'s traits and casting functions.
|
||||
|
||||
The size & layout of types is stable across compiler versions, as they are set and determined with [`#[repr(C)]`](https://doc.rust-lang.org/nomicon/other-reprs.html#reprc) and `bytemuck`'s derive macros such as [`bytemuck::Pod`](https://docs.rs/bytemuck/latest/bytemuck/derive.Pod.html).
|
||||
|
||||
Note that the data stored in the tables are still type-safe; we still refer to the key and values within our tables by the type.
|
||||
|
||||
The main deserialization `trait` for database storage is: [`cuprate_database::Storable`](https://github.com/Cuprate/cuprate/blob/2ac90420c658663564a71b7ecb52d74f3c2c9d0f/database/src/storable.rs#L16-L115).
|
||||
|
||||
- Before storage, the type is [simply cast into bytes](https://github.com/Cuprate/cuprate/blob/2ac90420c658663564a71b7ecb52d74f3c2c9d0f/database/src/storable.rs#L125)
|
||||
- When fetching, the bytes are [simply cast into the type](https://github.com/Cuprate/cuprate/blob/2ac90420c658663564a71b7ecb52d74f3c2c9d0f/database/src/storable.rs#L130)
|
||||
|
||||
When a type is casted into bytes, [the reference is casted](https://docs.rs/bytemuck/latest/bytemuck/fn.bytes_of.html), i.e. this is zero-cost serialization.
|
||||
|
||||
However, it is worth noting that when bytes are casted into the type, [it is copied](https://docs.rs/bytemuck/latest/bytemuck/fn.pod_read_unaligned.html). This is due to byte alignment guarantee issues with both backends, see:
|
||||
- https://github.com/AltSysrq/lmdb-zero/issues/8
|
||||
- https://github.com/cberner/redb/issues/360
|
||||
|
||||
Without this, `bytemuck` will panic with [`TargetAlignmentGreaterAndInputNotAligned`](https://docs.rs/bytemuck/latest/bytemuck/enum.PodCastError.html#variant.TargetAlignmentGreaterAndInputNotAligned) when casting.
|
||||
|
||||
Copying the bytes fixes this problem, although it is more costly than necessary. However, in the main use-case for `cuprate_database` (the `service` module) the bytes would need to be owned regardless as the `Request/Response` API uses owned data types (`T`, `Vec<T>`, `HashMap<K, V>`, etc).
|
||||
|
||||
Practically speaking, this means lower-level database functions that normally look like such:
|
||||
```rust
|
||||
fn get(key: &Key) -> &Value;
|
||||
use cuprate_blockchain::{
|
||||
cuprate_database::{
|
||||
ConcreteEnv,
|
||||
Env, EnvInner,
|
||||
DatabaseRo, DatabaseRw, TxRo, TxRw,
|
||||
},
|
||||
config::ConfigBuilder,
|
||||
tables::{Tables, TablesMut},
|
||||
OpenTables,
|
||||
};
|
||||
|
||||
# fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
// Create a configuration for the database environment.
|
||||
let tmp_dir = tempfile::tempdir()?;
|
||||
let db_dir = tmp_dir.path().to_owned();
|
||||
let config = ConfigBuilder::new()
|
||||
.db_directory(db_dir.into())
|
||||
.build();
|
||||
|
||||
// Initialize the database environment.
|
||||
let env = cuprate_blockchain::open(config)?;
|
||||
|
||||
// Open up a transaction + tables for writing.
|
||||
let env_inner = env.env_inner();
|
||||
let tx_rw = env_inner.tx_rw()?;
|
||||
let mut tables = env_inner.open_tables_mut(&tx_rw)?;
|
||||
|
||||
// ⚠️ Write data to the tables directly.
|
||||
// (not recommended, use `ops` or `service`).
|
||||
const KEY_IMAGE: [u8; 32] = [88; 32];
|
||||
tables.key_images_mut().put(&KEY_IMAGE, &())?;
|
||||
|
||||
// Commit the data written.
|
||||
drop(tables);
|
||||
TxRw::commit(tx_rw)?;
|
||||
|
||||
// Read the data, assert it is correct.
|
||||
let tx_ro = env_inner.tx_ro()?;
|
||||
let tables = env_inner.open_tables(&tx_ro)?;
|
||||
let (key_image, _) = tables.key_images().first()?;
|
||||
assert_eq!(key_image, KEY_IMAGE);
|
||||
# Ok(()) }
|
||||
```
|
||||
end up looking like this in `cuprate_database`:
|
||||
```rust
|
||||
fn get(key: &Key) -> Value;
|
||||
```
|
||||
|
||||
Since each backend has its own (de)serialization methods, our types are wrapped in compatibility types that map our `Storable` functions into whatever is required for the backend, e.g:
|
||||
- [`StorableHeed<T>`](https://github.com/Cuprate/cuprate/blob/2ac90420c658663564a71b7ecb52d74f3c2c9d0f/database/src/backend/heed/storable.rs#L11-L45)
|
||||
- [`StorableRedb<T>`](https://github.com/Cuprate/cuprate/blob/2ac90420c658663564a71b7ecb52d74f3c2c9d0f/database/src/backend/redb/storable.rs#L11-L30)
|
||||
|
||||
Compatibility structs also exist for any `Storable` containers:
|
||||
- [`StorableVec<T>`](https://github.com/Cuprate/cuprate/blob/2ac90420c658663564a71b7ecb52d74f3c2c9d0f/database/src/storable.rs#L135-L191)
|
||||
- [`StorableBytes`](https://github.com/Cuprate/cuprate/blob/2ac90420c658663564a71b7ecb52d74f3c2c9d0f/database/src/storable.rs#L208-L241)
|
||||
|
||||
Again, it's unfortunate that these must be owned, although in `service`'s use-case, they would have to be owned anyway.
|
||||
|
||||
## 9. Schema
|
||||
This following section contains Cuprate's database schema, it may change throughout the development of Cuprate, as such, nothing here is final.
|
||||
|
||||
### 9.1 Tables
|
||||
The `CamelCase` names of the table headers documented here (e.g. `TxIds`) are the actual type name of the table within `cuprate_database`.
|
||||
|
||||
Note that words written within `code blocks` mean that it is a real type defined and usable within `cuprate_database`. Other standard types like u64 and type aliases (TxId) are written normally.
|
||||
|
||||
Within `cuprate_database::tables`, the below table is essentially defined as-is with [a macro](https://github.com/Cuprate/cuprate/blob/31ce89412aa174fc33754f22c9a6d9ef5ddeda28/database/src/tables.rs#L369-L470).
|
||||
|
||||
Many of the data types stored are the same data types, although are different semantically, as such, a map of aliases used and their real data types is also provided below.
|
||||
|
||||
| Alias | Real Type |
|
||||
|----------------------------------------------------|-----------|
|
||||
| BlockHeight, Amount, AmountIndex, TxId, UnlockTime | u64
|
||||
| BlockHash, KeyImage, TxHash, PrunableHash | [u8; 32]
|
||||
|
||||
| Table | Key | Value | Description |
|
||||
|-------------------|----------------------|--------------------|-------------|
|
||||
| `BlockBlobs` | BlockHeight | `StorableVec<u8>` | Maps a block's height to a serialized byte form of a block
|
||||
| `BlockHeights` | BlockHash | BlockHeight | Maps a block's hash to its height
|
||||
| `BlockInfos` | BlockHeight | `BlockInfo` | Contains metadata of all blocks
|
||||
| `KeyImages` | KeyImage | () | This table is a set with no value, it stores transaction key images
|
||||
| `NumOutputs` | Amount | u64 | Maps an output's amount to the number of outputs with that amount
|
||||
| `Outputs` | `PreRctOutputId` | `Output` | This table contains legacy CryptoNote outputs which have clear amounts. This table will not contain an output with 0 amount.
|
||||
| `PrunedTxBlobs` | TxId | `StorableVec<u8>` | Contains pruned transaction blobs (even if the database is not pruned)
|
||||
| `PrunableTxBlobs` | TxId | `StorableVec<u8>` | Contains the prunable part of a transaction
|
||||
| `PrunableHashes` | TxId | PrunableHash | Contains the hash of the prunable part of a transaction
|
||||
| `RctOutputs` | AmountIndex | `RctOutput` | Contains RingCT outputs mapped from their global RCT index
|
||||
| `TxBlobs` | TxId | `StorableVec<u8>` | Serialized transaction blobs (bytes)
|
||||
| `TxIds` | TxHash | TxId | Maps a transaction's hash to its index/ID
|
||||
| `TxHeights` | TxId | BlockHeight | Maps a transaction's ID to the height of the block it comes from
|
||||
| `TxOutputs` | TxId | `StorableVec<u64>` | Gives the amount indices of a transaction's outputs
|
||||
| `TxUnlockTime` | TxId | UnlockTime | Stores the unlock time of a transaction (only if it has a non-zero lock time)
|
||||
|
||||
The definitions for aliases and types (e.g. `RctOutput`) are within the [`cuprate_database::types`](https://github.com/Cuprate/cuprate/blob/31ce89412aa174fc33754f22c9a6d9ef5ddeda28/database/src/types.rs#L51) module.
|
||||
|
||||
<!-- TODO(Boog900): We could split this table again into `RingCT (non-miner) Outputs` and `RingCT (miner) Outputs` as for miner outputs we can store the amount instead of commitment saving 24 bytes per miner output. -->
|
||||
|
||||
### 9.2 Multimap tables
|
||||
When referencing outputs, Monero will [use the amount and the amount index](https://github.com/monero-project/monero/blob/c8214782fb2a769c57382a999eaf099691c836e7/src/blockchain_db/lmdb/db_lmdb.cpp#L3447-L3449). This means 2 keys are needed to reach an output.
|
||||
|
||||
With LMDB you can set the `DUP_SORT` flag on a table and then set the key/value to:
|
||||
```rust
|
||||
Key = KEY_PART_1
|
||||
```
|
||||
```rust
|
||||
Value = {
|
||||
KEY_PART_2,
|
||||
VALUE // The actual value we are storing.
|
||||
}
|
||||
```
|
||||
|
||||
Then you can set a custom value sorting function that only takes `KEY_PART_2` into account; this is how `monerod` does it.
|
||||
|
||||
This requires that the underlying database supports:
|
||||
- multimap tables
|
||||
- custom sort functions on values
|
||||
- setting a cursor on a specific key/value
|
||||
|
||||
---
|
||||
|
||||
Another way to implement this is as follows:
|
||||
```rust
|
||||
Key = { KEY_PART_1, KEY_PART_2 }
|
||||
```
|
||||
```rust
|
||||
Value = VALUE
|
||||
```
|
||||
|
||||
Then the key type is simply used to look up the value; this is how `cuprate_database` does it.
|
||||
|
||||
For example, the key/value pair for outputs is:
|
||||
```rust
|
||||
PreRctOutputId => Output
|
||||
```
|
||||
where `PreRctOutputId` looks like this:
|
||||
```rust
|
||||
struct PreRctOutputId {
|
||||
amount: u64,
|
||||
amount_index: u64,
|
||||
}
|
||||
```
|
||||
|
||||
## 10. Known issues and tradeoffs
|
||||
`cuprate_database` takes many tradeoffs, whether due to:
|
||||
- Prioritizing certain values over others
|
||||
- Not having a better solution
|
||||
- Being "good enough"
|
||||
|
||||
This is a list of the larger ones, along with issues that don't have answers yet.
|
||||
|
||||
### 10.1 Traits abstracting backends
|
||||
Although all database backends used are very similar, they have some crucial differences in small implementation details that must be worked around when conforming them to `cuprate_database`'s traits.
|
||||
|
||||
Put simply: using `cuprate_database`'s traits is less efficient and more awkward than using the backend directly.
|
||||
|
||||
For example:
|
||||
- [Data types must be wrapped in compatibility layers when they otherwise wouldn't be](https://github.com/Cuprate/cuprate/blob/d0ac94a813e4cd8e0ed8da5e85a53b1d1ace2463/database/src/backend/heed/env.rs#L101-L116)
|
||||
- [There are types that only apply to a specific backend, but are visible to all](https://github.com/Cuprate/cuprate/blob/d0ac94a813e4cd8e0ed8da5e85a53b1d1ace2463/database/src/error.rs#L86-L89)
|
||||
- [There are extra layers of abstraction to smoothen the differences between all backends](https://github.com/Cuprate/cuprate/blob/d0ac94a813e4cd8e0ed8da5e85a53b1d1ace2463/database/src/env.rs#L62-L68)
|
||||
- [Existing functionality of backends must be taken away, as it isn't supported in the others](https://github.com/Cuprate/cuprate/blob/d0ac94a813e4cd8e0ed8da5e85a53b1d1ace2463/database/src/database.rs#L27-L34)
|
||||
|
||||
This is a _tradeoff_ that `cuprate_database` takes, as:
|
||||
- The backend itself is usually not the source of bottlenecks in the greater system, as such, small inefficiencies are OK
|
||||
- None of the lost functionality is crucial for operation
|
||||
- The ability to use, test, and swap between multiple database backends is [worth it](https://github.com/Cuprate/cuprate/pull/35#issuecomment-1952804393)
|
||||
|
||||
### 10.2 Hot-swappable backends
|
||||
Using a different backend is really as simple as re-building `cuprate_database` with a different feature flag:
|
||||
```bash
|
||||
# Use LMDB.
|
||||
cargo build --package cuprate-database --features heed
|
||||
|
||||
# Use redb.
|
||||
cargo build --package cuprate-database --features redb
|
||||
```
|
||||
|
||||
This is "good enough" for now, however ideally, this hot-swapping of backends would be able to be done at _runtime_.
|
||||
|
||||
As it is now, `cuprate_database` cannot compile both backends and swap based on user input at runtime; it must be compiled with a certain backend, which will produce a binary with only that backend.
|
||||
|
||||
This also means things like [CI testing multiple backends is awkward](https://github.com/Cuprate/cuprate/blob/main/.github/workflows/ci.yml#L132-L136), as we must re-compile with different feature flags instead.
|
||||
|
||||
### 10.3 Copying unaligned bytes
|
||||
As mentioned in [`8. (De)serialization`](#8-deserialization), bytes are _copied_ when they are turned into a type `T` due to unaligned bytes being returned from database backends.
|
||||
|
||||
Using a regular reference cast results in an improperly aligned type `T`; [such a type even existing causes undefined behavior](https://doc.rust-lang.org/reference/behavior-considered-undefined.html). In our case, `bytemuck` saves us by panicking before this occurs.
|
||||
|
||||
Thus, when using `cuprate_database`'s database traits, an _owned_ `T` is returned.
|
||||
|
||||
This is doubly unfortunately for `&[u8]` as this does not even need deserialization.
|
||||
|
||||
For example, `StorableVec` could have been this:
|
||||
```rust
|
||||
enum StorableBytes<'a, T: Storable> {
|
||||
Owned(T),
|
||||
Ref(&'a T),
|
||||
}
|
||||
```
|
||||
but this would require supporting types that must be copied regardless with the occasional `&[u8]` that can be returned without casting. This was hard to do so in a generic way, thus all `[u8]`'s are copied and returned as owned `StorableVec`s.
|
||||
|
||||
This is a _tradeoff_ `cuprate_database` takes as:
|
||||
- `bytemuck::pod_read_unaligned` is cheap enough
|
||||
- The main API, `service`, needs to return owned value anyway
|
||||
- Having no references removes a lot of lifetime complexity
|
||||
|
||||
The alternative is either:
|
||||
- Using proper (de)serialization instead of casting (which comes with its own costs)
|
||||
- Somehow fixing the alignment issues in the backends mentioned previously
|
||||
|
||||
### 10.4 Endianness
|
||||
`cuprate_database`'s (de)serialization and storage of bytes are native-endian, as in, byte storage order will depend on the machine it is running on.
|
||||
|
||||
As Cuprate's build-targets are all little-endian ([big-endian by default machines barely exist](https://en.wikipedia.org/wiki/Endianness#Hardware)), this doesn't matter much and the byte ordering can be seen as a constant.
|
||||
|
||||
Practically, this means `cuprated`'s database files can be transferred across computers, as can `monerod`'s.
|
||||
|
||||
### 10.5 Extra table data
|
||||
Some of `cuprate_database`'s tables differ from `monerod`'s tables, for example, the way [`9.2 Multimap tables`](#92-multimap-tables) tables are done requires that the primary key is stored _for all_ entries, compared to `monerod` only needing to store it once.
|
||||
|
||||
For example:
|
||||
```rust
|
||||
// `monerod` only stores `amount: 1` once,
|
||||
// `cuprated` stores it each time it appears.
|
||||
struct PreRctOutputId { amount: 1, amount_index: 0 }
|
||||
struct PreRctOutputId { amount: 1, amount_index: 1 }
|
||||
```
|
||||
|
||||
This means `cuprated`'s database will be slightly larger than `monerod`'s.
|
||||
|
||||
The current method `cuprate_database` uses will be "good enough" until usage shows that it must be optimized as multimap tables are tricky to implement across all backends.
|
||||
|
|
|
|||
|
|
@ -1,550 +0,0 @@
|
|||
//! Tests for `cuprate_blockchain`'s backends.
|
||||
//!
|
||||
//! These tests are fully trait-based, meaning there
|
||||
//! is no reference to `backend/`-specific types.
|
||||
//!
|
||||
//! As such, which backend is tested is
|
||||
//! dependant on the feature flags used.
|
||||
//!
|
||||
//! | Feature flag | Tested backend |
|
||||
//! |---------------|----------------|
|
||||
//! | Only `redb` | `redb`
|
||||
//! | Anything else | `heed`
|
||||
//!
|
||||
//! `redb`, and it only must be enabled for it to be tested.
|
||||
|
||||
//---------------------------------------------------------------------------------------------------- Import
|
||||
|
||||
use crate::{
|
||||
database::{DatabaseIter, DatabaseRo, DatabaseRw},
|
||||
env::{Env, EnvInner},
|
||||
error::RuntimeError,
|
||||
resize::ResizeAlgorithm,
|
||||
storable::StorableVec,
|
||||
tables::{
|
||||
BlockBlobs, BlockHeights, BlockInfos, KeyImages, NumOutputs, Outputs, PrunableHashes,
|
||||
PrunableTxBlobs, PrunedTxBlobs, RctOutputs, TxBlobs, TxHeights, TxIds, TxOutputs,
|
||||
TxUnlockTime,
|
||||
},
|
||||
tables::{TablesIter, TablesMut},
|
||||
tests::tmp_concrete_env,
|
||||
transaction::{TxRo, TxRw},
|
||||
types::{
|
||||
Amount, AmountIndex, AmountIndices, BlockBlob, BlockHash, BlockHeight, BlockInfo, KeyImage,
|
||||
Output, OutputFlags, PreRctOutputId, PrunableBlob, PrunableHash, PrunedBlob, RctOutput,
|
||||
TxBlob, TxHash, TxId, UnlockTime,
|
||||
},
|
||||
ConcreteEnv,
|
||||
};
|
||||
|
||||
//---------------------------------------------------------------------------------------------------- Tests
|
||||
/// Simply call [`Env::open`]. If this fails, something is really wrong.
|
||||
#[test]
|
||||
fn open() {
|
||||
tmp_concrete_env();
|
||||
}
|
||||
|
||||
/// Create database transactions, but don't write any data.
|
||||
#[test]
|
||||
fn tx() {
|
||||
let (env, _tempdir) = tmp_concrete_env();
|
||||
let env_inner = env.env_inner();
|
||||
|
||||
TxRo::commit(env_inner.tx_ro().unwrap()).unwrap();
|
||||
TxRw::commit(env_inner.tx_rw().unwrap()).unwrap();
|
||||
TxRw::abort(env_inner.tx_rw().unwrap()).unwrap();
|
||||
}
|
||||
|
||||
/// Open (and verify) that all database tables
|
||||
/// exist already after calling [`Env::open`].
|
||||
#[test]
|
||||
fn open_db() {
|
||||
let (env, _tempdir) = tmp_concrete_env();
|
||||
let env_inner = env.env_inner();
|
||||
let tx_ro = env_inner.tx_ro().unwrap();
|
||||
let tx_rw = env_inner.tx_rw().unwrap();
|
||||
|
||||
// Open all tables in read-only mode.
|
||||
// This should be updated when tables are modified.
|
||||
env_inner.open_db_ro::<BlockBlobs>(&tx_ro).unwrap();
|
||||
env_inner.open_db_ro::<BlockHeights>(&tx_ro).unwrap();
|
||||
env_inner.open_db_ro::<BlockInfos>(&tx_ro).unwrap();
|
||||
env_inner.open_db_ro::<KeyImages>(&tx_ro).unwrap();
|
||||
env_inner.open_db_ro::<NumOutputs>(&tx_ro).unwrap();
|
||||
env_inner.open_db_ro::<Outputs>(&tx_ro).unwrap();
|
||||
env_inner.open_db_ro::<PrunableHashes>(&tx_ro).unwrap();
|
||||
env_inner.open_db_ro::<PrunableTxBlobs>(&tx_ro).unwrap();
|
||||
env_inner.open_db_ro::<PrunedTxBlobs>(&tx_ro).unwrap();
|
||||
env_inner.open_db_ro::<RctOutputs>(&tx_ro).unwrap();
|
||||
env_inner.open_db_ro::<TxBlobs>(&tx_ro).unwrap();
|
||||
env_inner.open_db_ro::<TxHeights>(&tx_ro).unwrap();
|
||||
env_inner.open_db_ro::<TxIds>(&tx_ro).unwrap();
|
||||
env_inner.open_db_ro::<TxOutputs>(&tx_ro).unwrap();
|
||||
env_inner.open_db_ro::<TxUnlockTime>(&tx_ro).unwrap();
|
||||
TxRo::commit(tx_ro).unwrap();
|
||||
|
||||
// Open all tables in read/write mode.
|
||||
env_inner.open_db_rw::<BlockBlobs>(&tx_rw).unwrap();
|
||||
env_inner.open_db_rw::<BlockHeights>(&tx_rw).unwrap();
|
||||
env_inner.open_db_rw::<BlockInfos>(&tx_rw).unwrap();
|
||||
env_inner.open_db_rw::<KeyImages>(&tx_rw).unwrap();
|
||||
env_inner.open_db_rw::<NumOutputs>(&tx_rw).unwrap();
|
||||
env_inner.open_db_rw::<Outputs>(&tx_rw).unwrap();
|
||||
env_inner.open_db_rw::<PrunableHashes>(&tx_rw).unwrap();
|
||||
env_inner.open_db_rw::<PrunableTxBlobs>(&tx_rw).unwrap();
|
||||
env_inner.open_db_rw::<PrunedTxBlobs>(&tx_rw).unwrap();
|
||||
env_inner.open_db_rw::<RctOutputs>(&tx_rw).unwrap();
|
||||
env_inner.open_db_rw::<TxBlobs>(&tx_rw).unwrap();
|
||||
env_inner.open_db_rw::<TxHeights>(&tx_rw).unwrap();
|
||||
env_inner.open_db_rw::<TxIds>(&tx_rw).unwrap();
|
||||
env_inner.open_db_rw::<TxOutputs>(&tx_rw).unwrap();
|
||||
env_inner.open_db_rw::<TxUnlockTime>(&tx_rw).unwrap();
|
||||
TxRw::commit(tx_rw).unwrap();
|
||||
}
|
||||
|
||||
/// Test `Env` resizes.
|
||||
#[test]
|
||||
fn resize() {
|
||||
// This test is only valid for `Env`'s that need to resize manually.
|
||||
if !ConcreteEnv::MANUAL_RESIZE {
|
||||
return;
|
||||
}
|
||||
|
||||
let (env, _tempdir) = tmp_concrete_env();
|
||||
|
||||
// Resize by the OS page size.
|
||||
let page_size = crate::resize::page_size();
|
||||
let old_size = env.current_map_size();
|
||||
env.resize_map(Some(ResizeAlgorithm::FixedBytes(page_size)));
|
||||
|
||||
// Assert it resized exactly by the OS page size.
|
||||
let new_size = env.current_map_size();
|
||||
assert_eq!(new_size, old_size + page_size.get());
|
||||
}
|
||||
|
||||
/// Test that `Env`'s that don't manually resize.
|
||||
#[test]
|
||||
#[should_panic = "unreachable"]
|
||||
fn non_manual_resize_1() {
|
||||
if ConcreteEnv::MANUAL_RESIZE {
|
||||
unreachable!();
|
||||
} else {
|
||||
let (env, _tempdir) = tmp_concrete_env();
|
||||
env.resize_map(None);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[should_panic = "unreachable"]
|
||||
fn non_manual_resize_2() {
|
||||
if ConcreteEnv::MANUAL_RESIZE {
|
||||
unreachable!();
|
||||
} else {
|
||||
let (env, _tempdir) = tmp_concrete_env();
|
||||
env.current_map_size();
|
||||
}
|
||||
}
|
||||
|
||||
/// Test all `DatabaseR{o,w}` operations.
|
||||
#[test]
|
||||
fn db_read_write() {
|
||||
let (env, _tempdir) = tmp_concrete_env();
|
||||
let env_inner = env.env_inner();
|
||||
let tx_rw = env_inner.tx_rw().unwrap();
|
||||
let mut table = env_inner.open_db_rw::<Outputs>(&tx_rw).unwrap();
|
||||
|
||||
/// The (1st) key.
|
||||
const KEY: PreRctOutputId = PreRctOutputId {
|
||||
amount: 1,
|
||||
amount_index: 123,
|
||||
};
|
||||
/// The expected value.
|
||||
const VALUE: Output = Output {
|
||||
key: [35; 32],
|
||||
height: 45_761_798,
|
||||
output_flags: OutputFlags::empty(),
|
||||
tx_idx: 2_353_487,
|
||||
};
|
||||
/// How many `(key, value)` pairs will be inserted.
|
||||
const N: u64 = 100;
|
||||
|
||||
/// Assert 2 `Output`'s are equal, and that accessing
|
||||
/// their fields don't result in an unaligned panic.
|
||||
fn assert_same(output: Output) {
|
||||
assert_eq!(output, VALUE);
|
||||
assert_eq!(output.key, VALUE.key);
|
||||
assert_eq!(output.height, VALUE.height);
|
||||
assert_eq!(output.output_flags, VALUE.output_flags);
|
||||
assert_eq!(output.tx_idx, VALUE.tx_idx);
|
||||
}
|
||||
|
||||
assert!(table.is_empty().unwrap());
|
||||
|
||||
// Insert keys.
|
||||
let mut key = KEY;
|
||||
for _ in 0..N {
|
||||
table.put(&key, &VALUE).unwrap();
|
||||
key.amount += 1;
|
||||
}
|
||||
|
||||
assert_eq!(table.len().unwrap(), N);
|
||||
|
||||
// Assert the first/last `(key, value)`s are there.
|
||||
{
|
||||
assert!(table.contains(&KEY).unwrap());
|
||||
let get: Output = table.get(&KEY).unwrap();
|
||||
assert_same(get);
|
||||
|
||||
let first: Output = table.first().unwrap().1;
|
||||
assert_same(first);
|
||||
|
||||
let last: Output = table.last().unwrap().1;
|
||||
assert_same(last);
|
||||
}
|
||||
|
||||
// Commit transactions, create new ones.
|
||||
drop(table);
|
||||
TxRw::commit(tx_rw).unwrap();
|
||||
let tx_ro = env_inner.tx_ro().unwrap();
|
||||
let table_ro = env_inner.open_db_ro::<Outputs>(&tx_ro).unwrap();
|
||||
let tx_rw = env_inner.tx_rw().unwrap();
|
||||
let mut table = env_inner.open_db_rw::<Outputs>(&tx_rw).unwrap();
|
||||
|
||||
// Assert the whole range is there.
|
||||
{
|
||||
let range = table_ro.get_range(..).unwrap();
|
||||
let mut i = 0;
|
||||
for result in range {
|
||||
let value: Output = result.unwrap();
|
||||
assert_same(value);
|
||||
|
||||
i += 1;
|
||||
}
|
||||
assert_eq!(i, N);
|
||||
}
|
||||
|
||||
// `get_range()` tests.
|
||||
let mut key = KEY;
|
||||
key.amount += N;
|
||||
let range = KEY..key;
|
||||
|
||||
// Assert count is correct.
|
||||
assert_eq!(
|
||||
N as usize,
|
||||
table_ro.get_range(range.clone()).unwrap().count()
|
||||
);
|
||||
|
||||
// Assert each returned value from the iterator is owned.
|
||||
{
|
||||
let mut iter = table_ro.get_range(range.clone()).unwrap();
|
||||
let value: Output = iter.next().unwrap().unwrap(); // 1. take value out
|
||||
drop(iter); // 2. drop the `impl Iterator + 'a`
|
||||
assert_same(value); // 3. assert even without the iterator, the value is alive
|
||||
}
|
||||
|
||||
// Assert each value is the same.
|
||||
{
|
||||
let mut iter = table_ro.get_range(range).unwrap();
|
||||
for _ in 0..N {
|
||||
let value: Output = iter.next().unwrap().unwrap();
|
||||
assert_same(value);
|
||||
}
|
||||
}
|
||||
|
||||
// Assert `update()` works.
|
||||
{
|
||||
const HEIGHT: u32 = 999;
|
||||
|
||||
assert_ne!(table.get(&KEY).unwrap().height, HEIGHT);
|
||||
|
||||
table
|
||||
.update(&KEY, |mut value| {
|
||||
value.height = HEIGHT;
|
||||
Some(value)
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(table.get(&KEY).unwrap().height, HEIGHT);
|
||||
}
|
||||
|
||||
// Assert deleting works.
|
||||
{
|
||||
table.delete(&KEY).unwrap();
|
||||
let value = table.get(&KEY);
|
||||
assert!(!table.contains(&KEY).unwrap());
|
||||
assert!(matches!(value, Err(RuntimeError::KeyNotFound)));
|
||||
// Assert the other `(key, value)` pairs are still there.
|
||||
let mut key = KEY;
|
||||
key.amount += N - 1; // we used inclusive `0..N`
|
||||
let value = table.get(&key).unwrap();
|
||||
assert_same(value);
|
||||
}
|
||||
|
||||
// Assert `take()` works.
|
||||
{
|
||||
let mut key = KEY;
|
||||
key.amount += 1;
|
||||
let value = table.take(&key).unwrap();
|
||||
assert_eq!(value, VALUE);
|
||||
|
||||
let get = table.get(&KEY);
|
||||
assert!(!table.contains(&key).unwrap());
|
||||
assert!(matches!(get, Err(RuntimeError::KeyNotFound)));
|
||||
|
||||
// Assert the other `(key, value)` pairs are still there.
|
||||
key.amount += 1;
|
||||
let value = table.get(&key).unwrap();
|
||||
assert_same(value);
|
||||
}
|
||||
|
||||
drop(table);
|
||||
TxRw::commit(tx_rw).unwrap();
|
||||
|
||||
// Assert `clear_db()` works.
|
||||
{
|
||||
let mut tx_rw = env_inner.tx_rw().unwrap();
|
||||
env_inner.clear_db::<Outputs>(&mut tx_rw).unwrap();
|
||||
let table = env_inner.open_db_rw::<Outputs>(&tx_rw).unwrap();
|
||||
assert!(table.is_empty().unwrap());
|
||||
for n in 0..N {
|
||||
let mut key = KEY;
|
||||
key.amount += n;
|
||||
let value = table.get(&key);
|
||||
assert!(matches!(value, Err(RuntimeError::KeyNotFound)));
|
||||
assert!(!table.contains(&key).unwrap());
|
||||
}
|
||||
|
||||
// Reader still sees old value.
|
||||
assert!(!table_ro.is_empty().unwrap());
|
||||
|
||||
// Writer sees updated value (nothing).
|
||||
assert!(table.is_empty().unwrap());
|
||||
}
|
||||
}
|
||||
|
||||
/// Assert that `key`'s in database tables are sorted in
|
||||
/// an ordered B-Tree fashion, i.e. `min_value -> max_value`.
|
||||
#[test]
|
||||
fn tables_are_sorted() {
|
||||
let (env, _tmp) = tmp_concrete_env();
|
||||
let env_inner = env.env_inner();
|
||||
let tx_rw = env_inner.tx_rw().unwrap();
|
||||
let mut tables_mut = env_inner.open_tables_mut(&tx_rw).unwrap();
|
||||
|
||||
// Insert `{5, 4, 3, 2, 1, 0}`, assert each new
|
||||
// number inserted is the minimum `first()` value.
|
||||
for key in (0..6).rev() {
|
||||
tables_mut.num_outputs_mut().put(&key, &123).unwrap();
|
||||
let (first, _) = tables_mut.num_outputs_mut().first().unwrap();
|
||||
assert_eq!(first, key);
|
||||
}
|
||||
|
||||
drop(tables_mut);
|
||||
TxRw::commit(tx_rw).unwrap();
|
||||
let tx_rw = env_inner.tx_rw().unwrap();
|
||||
|
||||
// Assert iterators are ordered.
|
||||
{
|
||||
let tx_ro = env_inner.tx_ro().unwrap();
|
||||
let tables = env_inner.open_tables(&tx_ro).unwrap();
|
||||
let t = tables.num_outputs_iter();
|
||||
let iter = t.iter().unwrap();
|
||||
let keys = t.keys().unwrap();
|
||||
for ((i, iter), key) in (0..6).zip(iter).zip(keys) {
|
||||
let (iter, _) = iter.unwrap();
|
||||
let key = key.unwrap();
|
||||
assert_eq!(i, iter);
|
||||
assert_eq!(iter, key);
|
||||
}
|
||||
}
|
||||
|
||||
let mut tables_mut = env_inner.open_tables_mut(&tx_rw).unwrap();
|
||||
let t = tables_mut.num_outputs_mut();
|
||||
|
||||
// Assert the `first()` values are the minimum, i.e. `{0, 1, 2}`
|
||||
for key in 0..3 {
|
||||
let (first, _) = t.first().unwrap();
|
||||
assert_eq!(first, key);
|
||||
t.delete(&key).unwrap();
|
||||
}
|
||||
|
||||
// Assert the `last()` values are the maximum, i.e. `{5, 4, 3}`
|
||||
for key in (3..6).rev() {
|
||||
let (last, _) = tables_mut.num_outputs_mut().last().unwrap();
|
||||
assert_eq!(last, key);
|
||||
tables_mut.num_outputs_mut().delete(&key).unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
//---------------------------------------------------------------------------------------------------- Table Tests
|
||||
/// Test multiple tables and their key + values.
|
||||
///
|
||||
/// Each one of these tests:
|
||||
/// - Opens a specific table
|
||||
/// - Essentially does the `db_read_write` test
|
||||
macro_rules! test_tables {
|
||||
($(
|
||||
$table:ident, // Table type
|
||||
$key_type:ty => // Key (type)
|
||||
$value_type:ty, // Value (type)
|
||||
$key:expr => // Key (the value)
|
||||
$value:expr, // Value (the value)
|
||||
)* $(,)?) => { paste::paste! { $(
|
||||
// Test function's name is the table type in `snake_case`.
|
||||
#[test]
|
||||
fn [<$table:snake>]() {
|
||||
// Open the database env and table.
|
||||
let (env, _tempdir) = tmp_concrete_env();
|
||||
let env_inner = env.env_inner();
|
||||
let mut tx_rw = env_inner.tx_rw().unwrap();
|
||||
let mut table = env_inner.open_db_rw::<$table>(&mut tx_rw).unwrap();
|
||||
|
||||
/// The expected key.
|
||||
const KEY: $key_type = $key;
|
||||
// The expected value.
|
||||
let value: $value_type = $value;
|
||||
|
||||
// Assert a passed value is equal to the const value.
|
||||
let assert_eq = |v: &$value_type| {
|
||||
assert_eq!(v, &value);
|
||||
};
|
||||
|
||||
// Insert the key.
|
||||
table.put(&KEY, &value).unwrap();
|
||||
// Assert key is there.
|
||||
{
|
||||
let value: $value_type = table.get(&KEY).unwrap();
|
||||
assert_eq(&value);
|
||||
}
|
||||
|
||||
assert!(table.contains(&KEY).unwrap());
|
||||
assert_eq!(table.len().unwrap(), 1);
|
||||
|
||||
// Commit transactions, create new ones.
|
||||
drop(table);
|
||||
TxRw::commit(tx_rw).unwrap();
|
||||
let mut tx_rw = env_inner.tx_rw().unwrap();
|
||||
let tx_ro = env_inner.tx_ro().unwrap();
|
||||
let mut table = env_inner.open_db_rw::<$table>(&tx_rw).unwrap();
|
||||
let table_ro = env_inner.open_db_ro::<$table>(&tx_ro).unwrap();
|
||||
|
||||
// Assert `get_range()` works.
|
||||
{
|
||||
let range = KEY..;
|
||||
assert_eq!(1, table_ro.get_range(range.clone()).unwrap().count());
|
||||
let mut iter = table_ro.get_range(range).unwrap();
|
||||
let value = iter.next().unwrap().unwrap();
|
||||
assert_eq(&value);
|
||||
}
|
||||
|
||||
// Assert deleting works.
|
||||
{
|
||||
table.delete(&KEY).unwrap();
|
||||
let value = table.get(&KEY);
|
||||
assert!(matches!(value, Err(RuntimeError::KeyNotFound)));
|
||||
assert!(!table.contains(&KEY).unwrap());
|
||||
assert_eq!(table.len().unwrap(), 0);
|
||||
}
|
||||
|
||||
table.put(&KEY, &value).unwrap();
|
||||
|
||||
// Assert `clear_db()` works.
|
||||
{
|
||||
drop(table);
|
||||
env_inner.clear_db::<$table>(&mut tx_rw).unwrap();
|
||||
let table = env_inner.open_db_rw::<$table>(&mut tx_rw).unwrap();
|
||||
let value = table.get(&KEY);
|
||||
assert!(matches!(value, Err(RuntimeError::KeyNotFound)));
|
||||
assert!(!table.contains(&KEY).unwrap());
|
||||
assert_eq!(table.len().unwrap(), 0);
|
||||
}
|
||||
}
|
||||
)*}};
|
||||
}
|
||||
|
||||
// Notes:
|
||||
// - Keep this sorted A-Z (by table name)
|
||||
test_tables! {
|
||||
BlockBlobs, // Table type
|
||||
BlockHeight => BlockBlob, // Key type => Value type
|
||||
123 => StorableVec(vec![1,2,3,4,5,6,7,8]), // Actual key => Actual value
|
||||
|
||||
BlockHeights,
|
||||
BlockHash => BlockHeight,
|
||||
[32; 32] => 123,
|
||||
|
||||
BlockInfos,
|
||||
BlockHeight => BlockInfo,
|
||||
123 => BlockInfo {
|
||||
timestamp: 1,
|
||||
cumulative_generated_coins: 123,
|
||||
weight: 321,
|
||||
cumulative_difficulty_low: 111,
|
||||
cumulative_difficulty_high: 111,
|
||||
block_hash: [54; 32],
|
||||
cumulative_rct_outs: 2389,
|
||||
long_term_weight: 2389,
|
||||
},
|
||||
|
||||
KeyImages,
|
||||
KeyImage => (),
|
||||
[32; 32] => (),
|
||||
|
||||
NumOutputs,
|
||||
Amount => AmountIndex,
|
||||
123 => 123,
|
||||
|
||||
TxBlobs,
|
||||
TxId => TxBlob,
|
||||
123 => StorableVec(vec![1,2,3,4,5,6,7,8]),
|
||||
|
||||
TxIds,
|
||||
TxHash => TxId,
|
||||
[32; 32] => 123,
|
||||
|
||||
TxHeights,
|
||||
TxId => BlockHeight,
|
||||
123 => 123,
|
||||
|
||||
TxOutputs,
|
||||
TxId => AmountIndices,
|
||||
123 => StorableVec(vec![1,2,3,4,5,6,7,8]),
|
||||
|
||||
TxUnlockTime,
|
||||
TxId => UnlockTime,
|
||||
123 => 123,
|
||||
|
||||
Outputs,
|
||||
PreRctOutputId => Output,
|
||||
PreRctOutputId {
|
||||
amount: 1,
|
||||
amount_index: 2,
|
||||
} => Output {
|
||||
key: [1; 32],
|
||||
height: 1,
|
||||
output_flags: OutputFlags::empty(),
|
||||
tx_idx: 3,
|
||||
},
|
||||
|
||||
PrunedTxBlobs,
|
||||
TxId => PrunedBlob,
|
||||
123 => StorableVec(vec![1,2,3,4,5,6,7,8]),
|
||||
|
||||
PrunableTxBlobs,
|
||||
TxId => PrunableBlob,
|
||||
123 => StorableVec(vec![1,2,3,4,5,6,7,8]),
|
||||
|
||||
PrunableHashes,
|
||||
TxId => PrunableHash,
|
||||
123 => [32; 32],
|
||||
|
||||
RctOutputs,
|
||||
AmountIndex => RctOutput,
|
||||
123 => RctOutput {
|
||||
key: [1; 32],
|
||||
height: 1,
|
||||
output_flags: OutputFlags::empty(),
|
||||
tx_idx: 3,
|
||||
commitment: [3; 32],
|
||||
},
|
||||
}
|
||||
|
|
@ -1,21 +1,15 @@
|
|||
//! The main [`Config`] struct, holding all configurable values.
|
||||
|
||||
//---------------------------------------------------------------------------------------------------- Import
|
||||
use std::{
|
||||
borrow::Cow,
|
||||
path::{Path, PathBuf},
|
||||
};
|
||||
use std::{borrow::Cow, path::Path};
|
||||
|
||||
#[cfg(feature = "serde")]
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use cuprate_database::{config::SyncMode, resize::ResizeAlgorithm};
|
||||
use cuprate_helper::fs::cuprate_blockchain_dir;
|
||||
|
||||
use crate::{
|
||||
config::{ReaderThreads, SyncMode},
|
||||
constants::DATABASE_DATA_FILENAME,
|
||||
resize::ResizeAlgorithm,
|
||||
};
|
||||
use crate::config::ReaderThreads;
|
||||
|
||||
//---------------------------------------------------------------------------------------------------- ConfigBuilder
|
||||
/// Builder for [`Config`].
|
||||
|
|
@ -27,14 +21,11 @@ pub struct ConfigBuilder {
|
|||
/// [`Config::db_directory`].
|
||||
db_directory: Option<Cow<'static, Path>>,
|
||||
|
||||
/// [`Config::sync_mode`].
|
||||
sync_mode: Option<SyncMode>,
|
||||
/// [`Config::cuprate_database_config`].
|
||||
db_config: cuprate_database::config::ConfigBuilder,
|
||||
|
||||
/// [`Config::reader_threads`].
|
||||
reader_threads: Option<ReaderThreads>,
|
||||
|
||||
/// [`Config::resize_algorithm`].
|
||||
resize_algorithm: Option<ResizeAlgorithm>,
|
||||
}
|
||||
|
||||
impl ConfigBuilder {
|
||||
|
|
@ -42,12 +33,13 @@ impl ConfigBuilder {
|
|||
///
|
||||
/// [`ConfigBuilder::build`] can be called immediately
|
||||
/// after this function to use default values.
|
||||
pub const fn new() -> Self {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
db_directory: None,
|
||||
sync_mode: None,
|
||||
db_config: cuprate_database::config::ConfigBuilder::new(Cow::Borrowed(
|
||||
cuprate_blockchain_dir(),
|
||||
)),
|
||||
reader_threads: None,
|
||||
resize_algorithm: None,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -65,57 +57,37 @@ impl ConfigBuilder {
|
|||
.db_directory
|
||||
.unwrap_or_else(|| Cow::Borrowed(cuprate_blockchain_dir()));
|
||||
|
||||
// Add the database filename to the directory.
|
||||
let db_file = {
|
||||
let mut db_file = db_directory.to_path_buf();
|
||||
db_file.push(DATABASE_DATA_FILENAME);
|
||||
Cow::Owned(db_file)
|
||||
};
|
||||
let reader_threads = self.reader_threads.unwrap_or_default();
|
||||
let db_config = self
|
||||
.db_config
|
||||
.db_directory(db_directory)
|
||||
.reader_threads(reader_threads.as_threads())
|
||||
.build();
|
||||
|
||||
Config {
|
||||
db_directory,
|
||||
db_file,
|
||||
sync_mode: self.sync_mode.unwrap_or_default(),
|
||||
reader_threads: self.reader_threads.unwrap_or_default(),
|
||||
resize_algorithm: self.resize_algorithm.unwrap_or_default(),
|
||||
db_config,
|
||||
reader_threads,
|
||||
}
|
||||
}
|
||||
|
||||
/// Set a custom database directory (and file) [`Path`].
|
||||
#[must_use]
|
||||
pub fn db_directory(mut self, db_directory: PathBuf) -> Self {
|
||||
self.db_directory = Some(Cow::Owned(db_directory));
|
||||
pub fn db_directory(mut self, db_directory: Cow<'static, Path>) -> Self {
|
||||
self.db_directory = Some(db_directory);
|
||||
self
|
||||
}
|
||||
|
||||
/// Tune the [`ConfigBuilder`] for the highest performing,
|
||||
/// but also most resource-intensive & maybe risky settings.
|
||||
///
|
||||
/// Good default for testing, and resource-available machines.
|
||||
/// Calls [`cuprate_database::config::ConfigBuilder::sync_mode`].
|
||||
#[must_use]
|
||||
pub fn fast(mut self) -> Self {
|
||||
self.sync_mode = Some(SyncMode::Fast);
|
||||
self.reader_threads = Some(ReaderThreads::OnePerThread);
|
||||
self.resize_algorithm = Some(ResizeAlgorithm::default());
|
||||
pub fn sync_mode(mut self, sync_mode: SyncMode) -> Self {
|
||||
self.db_config = self.db_config.sync_mode(sync_mode);
|
||||
self
|
||||
}
|
||||
|
||||
/// Tune the [`ConfigBuilder`] for the lowest performing,
|
||||
/// but also least resource-intensive settings.
|
||||
///
|
||||
/// Good default for resource-limited machines, e.g. a cheap VPS.
|
||||
/// Calls [`cuprate_database::config::ConfigBuilder::resize_algorithm`].
|
||||
#[must_use]
|
||||
pub fn low_power(mut self) -> Self {
|
||||
self.sync_mode = Some(SyncMode::default());
|
||||
self.reader_threads = Some(ReaderThreads::One);
|
||||
self.resize_algorithm = Some(ResizeAlgorithm::default());
|
||||
self
|
||||
}
|
||||
|
||||
/// Set a custom [`SyncMode`].
|
||||
#[must_use]
|
||||
pub const fn sync_mode(mut self, sync_mode: SyncMode) -> Self {
|
||||
self.sync_mode = Some(sync_mode);
|
||||
pub fn resize_algorithm(mut self, resize_algorithm: ResizeAlgorithm) -> Self {
|
||||
self.db_config = self.db_config.resize_algorithm(resize_algorithm);
|
||||
self
|
||||
}
|
||||
|
||||
|
|
@ -126,102 +98,96 @@ impl ConfigBuilder {
|
|||
self
|
||||
}
|
||||
|
||||
/// Set a custom [`ResizeAlgorithm`].
|
||||
/// Tune the [`ConfigBuilder`] for the highest performing,
|
||||
/// but also most resource-intensive & maybe risky settings.
|
||||
///
|
||||
/// Good default for testing, and resource-available machines.
|
||||
#[must_use]
|
||||
pub const fn resize_algorithm(mut self, resize_algorithm: ResizeAlgorithm) -> Self {
|
||||
self.resize_algorithm = Some(resize_algorithm);
|
||||
pub fn fast(mut self) -> Self {
|
||||
self.db_config =
|
||||
cuprate_database::config::ConfigBuilder::new(Cow::Borrowed(cuprate_blockchain_dir()))
|
||||
.fast();
|
||||
|
||||
self.reader_threads = Some(ReaderThreads::OnePerThread);
|
||||
self
|
||||
}
|
||||
|
||||
/// Tune the [`ConfigBuilder`] for the lowest performing,
|
||||
/// but also least resource-intensive settings.
|
||||
///
|
||||
/// Good default for resource-limited machines, e.g. a cheap VPS.
|
||||
#[must_use]
|
||||
pub fn low_power(mut self) -> Self {
|
||||
self.db_config =
|
||||
cuprate_database::config::ConfigBuilder::new(Cow::Borrowed(cuprate_blockchain_dir()))
|
||||
.low_power();
|
||||
|
||||
self.reader_threads = Some(ReaderThreads::One);
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for ConfigBuilder {
|
||||
fn default() -> Self {
|
||||
let db_directory = Cow::Borrowed(cuprate_blockchain_dir());
|
||||
Self {
|
||||
db_directory: Some(Cow::Borrowed(cuprate_blockchain_dir())),
|
||||
sync_mode: Some(SyncMode::default()),
|
||||
db_directory: Some(db_directory.clone()),
|
||||
db_config: cuprate_database::config::ConfigBuilder::new(db_directory),
|
||||
reader_threads: Some(ReaderThreads::default()),
|
||||
resize_algorithm: Some(ResizeAlgorithm::default()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
//---------------------------------------------------------------------------------------------------- Config
|
||||
/// Database [`Env`](crate::Env) configuration.
|
||||
/// `cuprate_blockchain` configuration.
|
||||
///
|
||||
/// This is the struct passed to [`Env::open`](crate::Env::open) that
|
||||
/// allows the database to be configured in various ways.
|
||||
/// This is a configuration built on-top of [`cuprate_database::config::Config`].
|
||||
///
|
||||
/// It contains configuration specific to this crate, plus the database config.
|
||||
///
|
||||
/// For construction, either use [`ConfigBuilder`] or [`Config::default`].
|
||||
///
|
||||
// SOMEDAY: there's are many more options to add in the future.
|
||||
#[derive(Debug, Clone, PartialEq, PartialOrd)]
|
||||
#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
|
||||
pub struct Config {
|
||||
//------------------------ Database PATHs
|
||||
// These are private since we don't want
|
||||
// users messing with them after construction.
|
||||
/// The directory used to store all database files.
|
||||
///
|
||||
/// By default, if no value is provided in the [`Config`]
|
||||
/// constructor functions, this will be [`cuprate_blockchain_dir`].
|
||||
///
|
||||
// SOMEDAY: we should also support `/etc/cuprated.conf`.
|
||||
// This could be represented with an `enum DbPath { Default, Custom, Etc, }`
|
||||
pub(crate) db_directory: Cow<'static, Path>,
|
||||
/// The actual database data file.
|
||||
///
|
||||
/// This is private, and created from the above `db_directory`.
|
||||
pub(crate) db_file: Cow<'static, Path>,
|
||||
|
||||
/// Disk synchronization mode.
|
||||
pub sync_mode: SyncMode,
|
||||
/// The database configuration.
|
||||
pub db_config: cuprate_database::config::Config,
|
||||
|
||||
/// Database reader thread count.
|
||||
pub reader_threads: ReaderThreads,
|
||||
|
||||
/// Database memory map resizing algorithm.
|
||||
///
|
||||
/// This is used as the default fallback, but
|
||||
/// custom algorithms can be used as well with
|
||||
/// [`Env::resize_map`](crate::Env::resize_map).
|
||||
pub resize_algorithm: ResizeAlgorithm,
|
||||
}
|
||||
|
||||
impl Config {
|
||||
/// Create a new [`Config`] with sane default settings.
|
||||
///
|
||||
/// The [`Config::db_directory`] will be [`cuprate_blockchain_dir`].
|
||||
/// The [`cuprate_database::config::Config::db_directory`]
|
||||
/// will be set to [`cuprate_blockchain_dir`].
|
||||
///
|
||||
/// All other values will be [`Default::default`].
|
||||
///
|
||||
/// Same as [`Config::default`].
|
||||
///
|
||||
/// ```rust
|
||||
/// use cuprate_blockchain::{config::*, resize::*, DATABASE_DATA_FILENAME};
|
||||
/// use cuprate_database::{
|
||||
/// config::SyncMode,
|
||||
/// resize::ResizeAlgorithm,
|
||||
/// DATABASE_DATA_FILENAME,
|
||||
/// };
|
||||
/// use cuprate_helper::fs::*;
|
||||
///
|
||||
/// use cuprate_blockchain::config::*;
|
||||
///
|
||||
/// let config = Config::new();
|
||||
///
|
||||
/// assert_eq!(config.db_directory(), cuprate_blockchain_dir());
|
||||
/// assert!(config.db_file().starts_with(cuprate_blockchain_dir()));
|
||||
/// assert!(config.db_file().ends_with(DATABASE_DATA_FILENAME));
|
||||
/// assert_eq!(config.sync_mode, SyncMode::default());
|
||||
/// assert_eq!(config.db_config.db_directory(), cuprate_blockchain_dir());
|
||||
/// assert!(config.db_config.db_file().starts_with(cuprate_blockchain_dir()));
|
||||
/// assert!(config.db_config.db_file().ends_with(DATABASE_DATA_FILENAME));
|
||||
/// assert_eq!(config.db_config.sync_mode, SyncMode::default());
|
||||
/// assert_eq!(config.db_config.resize_algorithm, ResizeAlgorithm::default());
|
||||
/// assert_eq!(config.reader_threads, ReaderThreads::default());
|
||||
/// assert_eq!(config.resize_algorithm, ResizeAlgorithm::default());
|
||||
/// ```
|
||||
pub fn new() -> Self {
|
||||
ConfigBuilder::default().build()
|
||||
}
|
||||
|
||||
/// Return the absolute [`Path`] to the database directory.
|
||||
pub const fn db_directory(&self) -> &Cow<'_, Path> {
|
||||
&self.db_directory
|
||||
}
|
||||
|
||||
/// Return the absolute [`Path`] to the database data file.
|
||||
pub const fn db_file(&self) -> &Cow<'_, Path> {
|
||||
&self.db_file
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for Config {
|
||||
|
|
|
|||
|
|
@ -1,28 +1,31 @@
|
|||
//! Database [`Env`](crate::Env) configuration.
|
||||
//! Database configuration.
|
||||
//!
|
||||
//! This module contains the main [`Config`]uration struct
|
||||
//! for the database [`Env`](crate::Env)ironment, and types
|
||||
//! related to configuration settings.
|
||||
//! for the database [`Env`](cuprate_database::Env)ironment,
|
||||
//! and blockchain-specific configuration.
|
||||
//!
|
||||
//! It also contains types related to configuration settings.
|
||||
//!
|
||||
//! The main constructor is the [`ConfigBuilder`].
|
||||
//!
|
||||
//! These configurations are processed at runtime, meaning
|
||||
//! the `Env` can/will dynamically adjust its behavior
|
||||
//! based on these values.
|
||||
//! the `Env` can/will dynamically adjust its behavior based
|
||||
//! on these values.
|
||||
//!
|
||||
//! # Example
|
||||
//! ```rust
|
||||
//! use cuprate_blockchain::{
|
||||
//! Env,
|
||||
//! config::{ConfigBuilder, ReaderThreads, SyncMode}
|
||||
//! cuprate_database::{Env, config::SyncMode},
|
||||
//! config::{ConfigBuilder, ReaderThreads},
|
||||
//! };
|
||||
//!
|
||||
//! # fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
//! let db_dir = tempfile::tempdir()?;
|
||||
//! let tmp_dir = tempfile::tempdir()?;
|
||||
//! let db_dir = tmp_dir.path().to_owned();
|
||||
//!
|
||||
//! let config = ConfigBuilder::new()
|
||||
//! // Use a custom database directory.
|
||||
//! .db_directory(db_dir.path().to_path_buf())
|
||||
//! .db_directory(db_dir.into())
|
||||
//! // Use as many reader threads as possible (when using `service`).
|
||||
//! .reader_threads(ReaderThreads::OnePerThread)
|
||||
//! // Use the fastest sync mode.
|
||||
|
|
@ -33,7 +36,7 @@
|
|||
//! // Start a database `service` using this configuration.
|
||||
//! let (reader_handle, _) = cuprate_blockchain::service::init(config.clone())?;
|
||||
//! // It's using the config we provided.
|
||||
//! assert_eq!(reader_handle.env().config(), &config);
|
||||
//! assert_eq!(reader_handle.env().config(), &config.db_config);
|
||||
//! # Ok(()) }
|
||||
//! ```
|
||||
|
||||
|
|
@ -42,6 +45,3 @@ pub use config::{Config, ConfigBuilder};
|
|||
|
||||
mod reader_threads;
|
||||
pub use reader_threads::ReaderThreads;
|
||||
|
||||
mod sync_mode;
|
||||
pub use sync_mode::SyncMode;
|
||||
|
|
|
|||
|
|
@ -1,7 +1,6 @@
|
|||
//! General constants used throughout `cuprate-blockchain`.
|
||||
|
||||
//---------------------------------------------------------------------------------------------------- Import
|
||||
use cfg_if::cfg_if;
|
||||
|
||||
//---------------------------------------------------------------------------------------------------- Version
|
||||
/// Current major version of the database.
|
||||
|
|
@ -30,57 +29,6 @@ TODO: instructions on:
|
|||
3. General advice for preventing corruption
|
||||
4. etc";
|
||||
|
||||
//---------------------------------------------------------------------------------------------------- Misc
|
||||
/// Static string of the `crate` being used as the database backend.
|
||||
///
|
||||
/// | Backend | Value |
|
||||
/// |---------|-------|
|
||||
/// | `heed` | `"heed"`
|
||||
/// | `redb` | `"redb"`
|
||||
pub const DATABASE_BACKEND: &str = {
|
||||
cfg_if! {
|
||||
if #[cfg(all(feature = "redb", not(feature = "heed")))] {
|
||||
"redb"
|
||||
} else {
|
||||
"heed"
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
/// Cuprate's database filename.
|
||||
///
|
||||
/// Used in [`Config::db_file`](crate::config::Config::db_file).
|
||||
///
|
||||
/// | Backend | Value |
|
||||
/// |---------|-------|
|
||||
/// | `heed` | `"data.mdb"`
|
||||
/// | `redb` | `"data.redb"`
|
||||
pub const DATABASE_DATA_FILENAME: &str = {
|
||||
cfg_if! {
|
||||
if #[cfg(all(feature = "redb", not(feature = "heed")))] {
|
||||
"data.redb"
|
||||
} else {
|
||||
"data.mdb"
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
/// Cuprate's database lock filename.
|
||||
///
|
||||
/// | Backend | Value |
|
||||
/// |---------|-------|
|
||||
/// | `heed` | `Some("lock.mdb")`
|
||||
/// | `redb` | `None` (redb doesn't use a file lock)
|
||||
pub const DATABASE_LOCK_FILENAME: Option<&str> = {
|
||||
cfg_if! {
|
||||
if #[cfg(all(feature = "redb", not(feature = "heed")))] {
|
||||
None
|
||||
} else {
|
||||
Some("lock.mdb")
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
//---------------------------------------------------------------------------------------------------- Tests
|
||||
#[cfg(test)]
|
||||
mod test {}
|
||||
|
|
|
|||
|
|
@ -1,8 +1,73 @@
|
|||
//! General free functions (related to the database).
|
||||
|
||||
//---------------------------------------------------------------------------------------------------- Import
|
||||
use cuprate_database::{ConcreteEnv, Env, EnvInner, InitError, RuntimeError, TxRw};
|
||||
|
||||
use crate::{config::Config, open_tables::OpenTables};
|
||||
|
||||
//---------------------------------------------------------------------------------------------------- Free functions
|
||||
/// Open the blockchain database, using the passed [`Config`].
|
||||
///
|
||||
/// This calls [`cuprate_database::Env::open`] and prepares the
|
||||
/// database to be ready for blockchain-related usage, e.g.
|
||||
/// table creation, table sort order, etc.
|
||||
///
|
||||
/// All tables found in [`crate::tables`] will be
|
||||
/// ready for usage in the returned [`ConcreteEnv`].
|
||||
///
|
||||
/// # Errors
|
||||
/// This will error if:
|
||||
/// - The database file could not be opened
|
||||
/// - A write transaction could not be opened
|
||||
/// - A table could not be created/opened
|
||||
#[cold]
|
||||
#[inline(never)] // only called once
|
||||
pub fn open(config: Config) -> Result<ConcreteEnv, InitError> {
|
||||
// Attempt to open the database environment.
|
||||
let env = <ConcreteEnv as Env>::open(config.db_config)?;
|
||||
|
||||
/// Convert runtime errors to init errors.
|
||||
///
|
||||
/// INVARIANT:
|
||||
/// `cuprate_database`'s functions mostly return the former
|
||||
/// so we must convert them. We have knowledge of which errors
|
||||
/// makes sense in this functions context so we panic on
|
||||
/// unexpected ones.
|
||||
fn runtime_to_init_error(runtime: RuntimeError) -> InitError {
|
||||
match runtime {
|
||||
RuntimeError::Io(io_error) => io_error.into(),
|
||||
|
||||
// These errors shouldn't be happening here.
|
||||
RuntimeError::KeyExists
|
||||
| RuntimeError::KeyNotFound
|
||||
| RuntimeError::ResizeNeeded
|
||||
| RuntimeError::TableNotFound => unreachable!(),
|
||||
}
|
||||
}
|
||||
|
||||
// INVARIANT: We must ensure that all tables are created,
|
||||
// `cuprate_database` has no way of knowing _which_ tables
|
||||
// we want since it is agnostic, so we are responsible for this.
|
||||
{
|
||||
let env_inner = env.env_inner();
|
||||
let tx_rw = env_inner.tx_rw();
|
||||
let tx_rw = match tx_rw {
|
||||
Ok(tx_rw) => tx_rw,
|
||||
Err(e) => return Err(runtime_to_init_error(e)),
|
||||
};
|
||||
|
||||
// Create all tables.
|
||||
if let Err(e) = OpenTables::create_tables(&env_inner, &tx_rw) {
|
||||
return Err(runtime_to_init_error(e));
|
||||
};
|
||||
|
||||
if let Err(e) = tx_rw.commit() {
|
||||
return Err(runtime_to_init_error(e));
|
||||
}
|
||||
}
|
||||
|
||||
Ok(env)
|
||||
}
|
||||
|
||||
//---------------------------------------------------------------------------------------------------- Tests
|
||||
#[cfg(test)]
|
||||
|
|
|
|||
|
|
@ -1,145 +1,4 @@
|
|||
//! Cuprate's database abstraction.
|
||||
//!
|
||||
//! This documentation is mostly for practical usage of `cuprate_blockchain`.
|
||||
//!
|
||||
//! For a high-level overview,
|
||||
//! see [`database/README.md`](https://github.com/Cuprate/cuprate/blob/main/database/README.md).
|
||||
//!
|
||||
//! # Purpose
|
||||
//! This crate does 3 things:
|
||||
//! 1. Abstracts various database backends with traits
|
||||
//! 2. Implements various `Monero` related [operations](ops), [tables], and [types]
|
||||
//! 3. Exposes a [`tower::Service`] backed by a thread-pool
|
||||
//!
|
||||
//! Each layer builds on-top of the previous.
|
||||
//!
|
||||
//! As a user of `cuprate_blockchain`, consider using the higher-level [`service`] module,
|
||||
//! or at the very least the [`ops`] module instead of interacting with the database traits directly.
|
||||
//!
|
||||
//! With that said, many database traits and internals (like [`DatabaseRo::get`]) are exposed.
|
||||
//!
|
||||
//! # Terminology
|
||||
//! To be more clear on some terms used in this crate:
|
||||
//!
|
||||
//! | Term | Meaning |
|
||||
//! |------------------|--------------------------------------|
|
||||
//! | `Env` | The 1 database environment, the "whole" thing
|
||||
//! | `DatabaseR{o,w}` | A _actively open_ readable/writable `key/value` store
|
||||
//! | `Table` | Solely the metadata of a `Database` (the `key` and `value` types, and the name)
|
||||
//! | `TxR{o,w}` | A read/write transaction
|
||||
//! | `Storable` | A data that type can be stored in the database
|
||||
//!
|
||||
//! The dataflow is `Env` -> `Tx` -> `Database`
|
||||
//!
|
||||
//! Which reads as:
|
||||
//! 1. You have a database `Environment`
|
||||
//! 1. You open up a `Transaction`
|
||||
//! 1. You open a particular `Table` from that `Environment`, getting a `Database`
|
||||
//! 1. You can now read/write data from/to that `Database`
|
||||
//!
|
||||
//! # `ConcreteEnv`
|
||||
//! This crate exposes [`ConcreteEnv`], which is a non-generic/non-dynamic,
|
||||
//! concrete object representing a database [`Env`]ironment.
|
||||
//!
|
||||
//! The actual backend for this type is determined via feature flags.
|
||||
//!
|
||||
//! This object existing means `E: Env` doesn't need to be spread all through the codebase,
|
||||
//! however, it also means some small invariants should be kept in mind.
|
||||
//!
|
||||
//! As `ConcreteEnv` is just a re-exposed type which has varying inner types,
|
||||
//! it means some properties will change depending on the backend used.
|
||||
//!
|
||||
//! For example:
|
||||
//! - [`std::mem::size_of::<ConcreteEnv>`]
|
||||
//! - [`std::mem::align_of::<ConcreteEnv>`]
|
||||
//!
|
||||
//! Things like these functions are affected by the backend and inner data,
|
||||
//! and should not be relied upon. This extends to any `struct/enum` that contains `ConcreteEnv`.
|
||||
//!
|
||||
//! `ConcreteEnv` invariants you can rely on:
|
||||
//! - It implements [`Env`]
|
||||
//! - Upon [`Drop::drop`], all database data will sync to disk
|
||||
//!
|
||||
//! Note that `ConcreteEnv` itself is not a clonable type,
|
||||
//! it should be wrapped in [`std::sync::Arc`].
|
||||
//!
|
||||
//! <!-- SOMEDAY: replace `ConcreteEnv` with `fn Env::open() -> impl Env`/
|
||||
//! and use `<E: Env>` everywhere it is stored instead. This would allow
|
||||
//! generic-backed dynamic runtime selection of the database backend, i.e.
|
||||
//! the user can select which database backend they use. -->
|
||||
//!
|
||||
//! # Feature flags
|
||||
//! The `service` module requires the `service` feature to be enabled.
|
||||
//! See the module for more documentation.
|
||||
//!
|
||||
//! Different database backends are enabled by the feature flags:
|
||||
//! - `heed` (LMDB)
|
||||
//! - `redb`
|
||||
//!
|
||||
//! The default is `heed`.
|
||||
//!
|
||||
//! `tracing` is always enabled and cannot be disabled via feature-flag.
|
||||
//! <!-- FIXME: tracing should be behind a feature flag -->
|
||||
//!
|
||||
//! # Invariants when not using `service`
|
||||
//! `cuprate_blockchain` can be used without the `service` feature enabled but
|
||||
//! there are some things that must be kept in mind when doing so.
|
||||
//!
|
||||
//! Failing to uphold these invariants may cause panics.
|
||||
//!
|
||||
//! 1. `LMDB` requires the user to resize the memory map resizing (see [`RuntimeError::ResizeNeeded`]
|
||||
//! 1. `LMDB` has a maximum reader transaction count, currently it is set to `128`
|
||||
//! 1. `LMDB` has [maximum key/value byte size](http://www.lmdb.tech/doc/group__internal.html#gac929399f5d93cef85f874b9e9b1d09e0) which must not be exceeded
|
||||
//!
|
||||
//! # Examples
|
||||
//! The below is an example of using `cuprate_blockchain`'s
|
||||
//! lowest API, i.e. using the database directly.
|
||||
//!
|
||||
//! For examples of the higher-level APIs, see:
|
||||
//! - [`ops`]
|
||||
//! - [`service`]
|
||||
//!
|
||||
//! ```rust
|
||||
//! use cuprate_blockchain::{
|
||||
//! ConcreteEnv,
|
||||
//! config::ConfigBuilder,
|
||||
//! Env, EnvInner,
|
||||
//! tables::{Tables, TablesMut},
|
||||
//! DatabaseRo, DatabaseRw, TxRo, TxRw,
|
||||
//! };
|
||||
//!
|
||||
//! # fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
//! // Create a configuration for the database environment.
|
||||
//! let db_dir = tempfile::tempdir()?;
|
||||
//! let config = ConfigBuilder::new()
|
||||
//! .db_directory(db_dir.path().to_path_buf())
|
||||
//! .build();
|
||||
//!
|
||||
//! // Initialize the database environment.
|
||||
//! let env = ConcreteEnv::open(config)?;
|
||||
//!
|
||||
//! // Open up a transaction + tables for writing.
|
||||
//! let env_inner = env.env_inner();
|
||||
//! let tx_rw = env_inner.tx_rw()?;
|
||||
//! let mut tables = env_inner.open_tables_mut(&tx_rw)?;
|
||||
//!
|
||||
//! // ⚠️ Write data to the tables directly.
|
||||
//! // (not recommended, use `ops` or `service`).
|
||||
//! const KEY_IMAGE: [u8; 32] = [88; 32];
|
||||
//! tables.key_images_mut().put(&KEY_IMAGE, &())?;
|
||||
//!
|
||||
//! // Commit the data written.
|
||||
//! drop(tables);
|
||||
//! TxRw::commit(tx_rw)?;
|
||||
//!
|
||||
//! // Read the data, assert it is correct.
|
||||
//! let tx_ro = env_inner.tx_ro()?;
|
||||
//! let tables = env_inner.open_tables(&tx_ro)?;
|
||||
//! let (key_image, _) = tables.key_images().first()?;
|
||||
//! assert_eq!(key_image, KEY_IMAGE);
|
||||
//! # Ok(()) }
|
||||
//! ```
|
||||
|
||||
#![doc = include_str!("../README.md")]
|
||||
//---------------------------------------------------------------------------------------------------- Lints
|
||||
// Forbid lints.
|
||||
// Our code, and code generated (e.g macros) cannot overrule these.
|
||||
|
|
@ -190,6 +49,7 @@
|
|||
clippy::pedantic,
|
||||
clippy::nursery,
|
||||
clippy::cargo,
|
||||
unused_crate_dependencies,
|
||||
unused_doc_comments,
|
||||
unused_mut,
|
||||
missing_docs,
|
||||
|
|
@ -220,7 +80,14 @@
|
|||
clippy::option_if_let_else,
|
||||
)]
|
||||
// Allow some lints when running in debug mode.
|
||||
#![cfg_attr(debug_assertions, allow(clippy::todo, clippy::multiple_crate_versions))]
|
||||
#![cfg_attr(
|
||||
debug_assertions,
|
||||
allow(
|
||||
clippy::todo,
|
||||
clippy::multiple_crate_versions,
|
||||
// unused_crate_dependencies,
|
||||
)
|
||||
)]
|
||||
// Allow some lints in tests.
|
||||
#![cfg_attr(
|
||||
test,
|
||||
|
|
@ -247,47 +114,22 @@ compile_error!("Cuprate is only compatible with 64-bit CPUs");
|
|||
//
|
||||
// Documentation for each module is located in the respective file.
|
||||
|
||||
mod backend;
|
||||
pub use backend::ConcreteEnv;
|
||||
|
||||
pub mod config;
|
||||
|
||||
mod constants;
|
||||
pub use constants::{
|
||||
DATABASE_BACKEND, DATABASE_CORRUPT_MSG, DATABASE_DATA_FILENAME, DATABASE_LOCK_FILENAME,
|
||||
DATABASE_VERSION,
|
||||
};
|
||||
pub use constants::{DATABASE_CORRUPT_MSG, DATABASE_VERSION};
|
||||
|
||||
mod database;
|
||||
pub use database::{DatabaseIter, DatabaseRo, DatabaseRw};
|
||||
mod open_tables;
|
||||
pub use open_tables::OpenTables;
|
||||
|
||||
mod env;
|
||||
pub use env::{Env, EnvInner};
|
||||
|
||||
mod error;
|
||||
pub use error::{InitError, RuntimeError};
|
||||
|
||||
pub(crate) mod free;
|
||||
|
||||
pub mod resize;
|
||||
|
||||
mod key;
|
||||
pub use key::Key;
|
||||
|
||||
mod storable;
|
||||
pub use storable::{Storable, StorableBytes, StorableVec};
|
||||
mod free;
|
||||
pub use free::open;
|
||||
|
||||
pub mod ops;
|
||||
|
||||
mod table;
|
||||
pub use table::Table;
|
||||
|
||||
pub mod tables;
|
||||
|
||||
pub mod types;
|
||||
|
||||
mod transaction;
|
||||
pub use transaction::{TxRo, TxRw};
|
||||
pub use cuprate_database;
|
||||
|
||||
//---------------------------------------------------------------------------------------------------- Feature-gated
|
||||
#[cfg(feature = "service")]
|
||||
|
|
|
|||
188
storage/blockchain/src/open_tables.rs
Normal file
188
storage/blockchain/src/open_tables.rs
Normal file
|
|
@ -0,0 +1,188 @@
|
|||
//! TODO
|
||||
|
||||
//---------------------------------------------------------------------------------------------------- Import
|
||||
use cuprate_database::{EnvInner, RuntimeError, TxRo, TxRw};
|
||||
|
||||
use crate::tables::{TablesIter, TablesMut};
|
||||
|
||||
//---------------------------------------------------------------------------------------------------- Table function macro
|
||||
/// `crate`-private macro for callings functions on all tables.
|
||||
///
|
||||
/// This calls the function `$fn` with the optional
|
||||
/// arguments `$args` on all tables - returning early
|
||||
/// (within whatever scope this is called) if any
|
||||
/// of the function calls error.
|
||||
///
|
||||
/// Else, it evaluates to an `Ok((tuple, of, all, table, types, ...))`,
|
||||
/// i.e., an `impl Table[Mut]` wrapped in `Ok`.
|
||||
macro_rules! call_fn_on_all_tables_or_early_return {
|
||||
(
|
||||
$($fn:ident $(::)?)*
|
||||
(
|
||||
$($arg:ident),* $(,)?
|
||||
)
|
||||
) => {{
|
||||
Ok((
|
||||
$($fn ::)*<$crate::tables::BlockInfos>($($arg),*)?,
|
||||
$($fn ::)*<$crate::tables::BlockBlobs>($($arg),*)?,
|
||||
$($fn ::)*<$crate::tables::BlockHeights>($($arg),*)?,
|
||||
$($fn ::)*<$crate::tables::KeyImages>($($arg),*)?,
|
||||
$($fn ::)*<$crate::tables::NumOutputs>($($arg),*)?,
|
||||
$($fn ::)*<$crate::tables::PrunedTxBlobs>($($arg),*)?,
|
||||
$($fn ::)*<$crate::tables::PrunableHashes>($($arg),*)?,
|
||||
$($fn ::)*<$crate::tables::Outputs>($($arg),*)?,
|
||||
$($fn ::)*<$crate::tables::PrunableTxBlobs>($($arg),*)?,
|
||||
$($fn ::)*<$crate::tables::RctOutputs>($($arg),*)?,
|
||||
$($fn ::)*<$crate::tables::TxBlobs>($($arg),*)?,
|
||||
$($fn ::)*<$crate::tables::TxIds>($($arg),*)?,
|
||||
$($fn ::)*<$crate::tables::TxHeights>($($arg),*)?,
|
||||
$($fn ::)*<$crate::tables::TxOutputs>($($arg),*)?,
|
||||
$($fn ::)*<$crate::tables::TxUnlockTime>($($arg),*)?,
|
||||
))
|
||||
}};
|
||||
}
|
||||
pub(crate) use call_fn_on_all_tables_or_early_return;
|
||||
|
||||
//---------------------------------------------------------------------------------------------------- OpenTables
|
||||
/// Open all tables at once.
|
||||
///
|
||||
/// This trait encapsulates the functionality of opening all tables at once.
|
||||
/// It can be seen as the "constructor" for the [`Tables`](crate::tables::Tables) object.
|
||||
///
|
||||
/// Note that this is already implemented on [`cuprate_database::EnvInner`], thus:
|
||||
/// - You don't need to implement this
|
||||
/// - It can be called using `env_inner.open_tables()` notation
|
||||
///
|
||||
/// # Example
|
||||
/// ```rust
|
||||
/// use cuprate_blockchain::{
|
||||
/// cuprate_database::{Env, EnvInner},
|
||||
/// config::ConfigBuilder,
|
||||
/// tables::{Tables, TablesMut},
|
||||
/// OpenTables,
|
||||
/// };
|
||||
///
|
||||
/// # fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
/// // Create a configuration for the database environment.
|
||||
/// let tmp_dir = tempfile::tempdir()?;
|
||||
/// let db_dir = tmp_dir.path().to_owned();
|
||||
/// let config = ConfigBuilder::new()
|
||||
/// .db_directory(db_dir.into())
|
||||
/// .build();
|
||||
///
|
||||
/// // Initialize the database environment.
|
||||
/// let env = cuprate_blockchain::open(config)?;
|
||||
///
|
||||
/// // Open up a transaction.
|
||||
/// let env_inner = env.env_inner();
|
||||
/// let tx_rw = env_inner.tx_rw()?;
|
||||
///
|
||||
/// // Open _all_ tables in write mode using [`OpenTables::open_tables_mut`].
|
||||
/// // Note how this is being called on `env_inner`.
|
||||
/// // |
|
||||
/// // v
|
||||
/// let mut tables = env_inner.open_tables_mut(&tx_rw)?;
|
||||
/// # Ok(()) }
|
||||
/// ```
|
||||
pub trait OpenTables<'env, Ro, Rw>
|
||||
where
|
||||
Self: 'env,
|
||||
Ro: TxRo<'env>,
|
||||
Rw: TxRw<'env>,
|
||||
{
|
||||
/// Open all tables in read/iter mode.
|
||||
///
|
||||
/// This calls [`EnvInner::open_db_ro`] on all database tables
|
||||
/// and returns a structure that allows access to all tables.
|
||||
///
|
||||
/// # Errors
|
||||
/// This will only return [`RuntimeError::Io`] if it errors.
|
||||
///
|
||||
/// As all tables are created upon [`crate::open`],
|
||||
/// this function will never error because a table doesn't exist.
|
||||
fn open_tables(&'env self, tx_ro: &Ro) -> Result<impl TablesIter, RuntimeError>;
|
||||
|
||||
/// Open all tables in read-write mode.
|
||||
///
|
||||
/// This calls [`EnvInner::open_db_rw`] on all database tables
|
||||
/// and returns a structure that allows access to all tables.
|
||||
///
|
||||
/// # Errors
|
||||
/// This will only return [`RuntimeError::Io`] on errors.
|
||||
fn open_tables_mut(&'env self, tx_rw: &Rw) -> Result<impl TablesMut, RuntimeError>;
|
||||
|
||||
/// Create all database tables.
|
||||
///
|
||||
/// This will create all the [`Table`](cuprate_database::Table)s
|
||||
/// found in [`tables`](crate::tables).
|
||||
///
|
||||
/// # Errors
|
||||
/// This will only return [`RuntimeError::Io`] on errors.
|
||||
fn create_tables(&'env self, tx_rw: &Rw) -> Result<(), RuntimeError>;
|
||||
}
|
||||
|
||||
impl<'env, Ei, Ro, Rw> OpenTables<'env, Ro, Rw> for Ei
|
||||
where
|
||||
Ei: EnvInner<'env, Ro, Rw>,
|
||||
Ro: TxRo<'env>,
|
||||
Rw: TxRw<'env>,
|
||||
{
|
||||
fn open_tables(&'env self, tx_ro: &Ro) -> Result<impl TablesIter, RuntimeError> {
|
||||
call_fn_on_all_tables_or_early_return! {
|
||||
Self::open_db_ro(self, tx_ro)
|
||||
}
|
||||
}
|
||||
|
||||
fn open_tables_mut(&'env self, tx_rw: &Rw) -> Result<impl TablesMut, RuntimeError> {
|
||||
call_fn_on_all_tables_or_early_return! {
|
||||
Self::open_db_rw(self, tx_rw)
|
||||
}
|
||||
}
|
||||
|
||||
fn create_tables(&'env self, tx_rw: &Rw) -> Result<(), RuntimeError> {
|
||||
match call_fn_on_all_tables_or_early_return! {
|
||||
Self::create_db(self, tx_rw)
|
||||
} {
|
||||
Ok(_) => Ok(()),
|
||||
Err(e) => Err(e),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
//---------------------------------------------------------------------------------------------------- Tests
|
||||
#[cfg(test)]
|
||||
mod test {
|
||||
use std::borrow::Cow;
|
||||
|
||||
use cuprate_database::{Env, EnvInner};
|
||||
|
||||
use crate::{config::ConfigBuilder, tests::tmp_concrete_env};
|
||||
|
||||
use super::*;
|
||||
|
||||
/// Tests that [`crate::open`] creates all tables.
|
||||
#[test]
|
||||
fn test_all_tables_are_created() {
|
||||
let (env, _tmp) = tmp_concrete_env();
|
||||
let env_inner = env.env_inner();
|
||||
let tx_ro = env_inner.tx_ro().unwrap();
|
||||
env_inner.open_tables(&tx_ro).unwrap();
|
||||
}
|
||||
|
||||
/// Tests that directory [`cuprate_database::ConcreteEnv`]
|
||||
/// usage does NOT create all tables.
|
||||
#[test]
|
||||
#[should_panic(expected = "`Result::unwrap()` on an `Err` value: TableNotFound")]
|
||||
fn test_no_tables_are_created() {
|
||||
let tempdir = tempfile::tempdir().unwrap();
|
||||
let config = ConfigBuilder::new()
|
||||
.db_directory(Cow::Owned(tempdir.path().into()))
|
||||
.low_power()
|
||||
.build();
|
||||
let env = cuprate_database::ConcreteEnv::open(config.db_config).unwrap();
|
||||
|
||||
let env_inner = env.env_inner();
|
||||
let tx_ro = env_inner.tx_ro().unwrap();
|
||||
env_inner.open_tables(&tx_ro).unwrap();
|
||||
}
|
||||
}
|
||||
|
|
@ -4,12 +4,13 @@
|
|||
use bytemuck::TransparentWrapper;
|
||||
use monero_serai::block::Block;
|
||||
|
||||
use cuprate_database::{
|
||||
RuntimeError, StorableVec, {DatabaseRo, DatabaseRw},
|
||||
};
|
||||
use cuprate_helper::map::{combine_low_high_bits_to_u128, split_u128_into_low_high_bits};
|
||||
use cuprate_types::{ExtendedBlockHeader, VerifiedBlockInformation};
|
||||
|
||||
use crate::{
|
||||
database::{DatabaseRo, DatabaseRw},
|
||||
error::RuntimeError,
|
||||
ops::{
|
||||
blockchain::{chain_height, cumulative_generated_coins},
|
||||
macros::doc_error,
|
||||
|
|
@ -18,7 +19,6 @@ use crate::{
|
|||
},
|
||||
tables::{BlockHeights, BlockInfos, Tables, TablesMut},
|
||||
types::{BlockHash, BlockHeight, BlockInfo},
|
||||
StorableVec,
|
||||
};
|
||||
|
||||
//---------------------------------------------------------------------------------------------------- `add_block_*`
|
||||
|
|
@ -265,14 +265,15 @@ pub fn block_exists(
|
|||
mod test {
|
||||
use pretty_assertions::assert_eq;
|
||||
|
||||
use cuprate_database::{Env, EnvInner, TxRw};
|
||||
use cuprate_test_utils::data::{block_v16_tx0, block_v1_tx2, block_v9_tx3};
|
||||
|
||||
use super::*;
|
||||
|
||||
use crate::{
|
||||
open_tables::OpenTables,
|
||||
ops::tx::{get_tx, tx_exists},
|
||||
tests::{assert_all_tables_are_empty, tmp_concrete_env, AssertTableLen},
|
||||
transaction::TxRw,
|
||||
Env, EnvInner,
|
||||
};
|
||||
|
||||
/// Tests all above block functions.
|
||||
|
|
|
|||
|
|
@ -1,9 +1,9 @@
|
|||
//! Blockchain functions - chain height, generated coins, etc.
|
||||
|
||||
//---------------------------------------------------------------------------------------------------- Import
|
||||
use cuprate_database::{DatabaseRo, RuntimeError};
|
||||
|
||||
use crate::{
|
||||
database::DatabaseRo,
|
||||
error::RuntimeError,
|
||||
ops::macros::doc_error,
|
||||
tables::{BlockHeights, BlockInfos},
|
||||
types::BlockHeight,
|
||||
|
|
@ -81,15 +81,16 @@ pub fn cumulative_generated_coins(
|
|||
mod test {
|
||||
use pretty_assertions::assert_eq;
|
||||
|
||||
use cuprate_database::{Env, EnvInner, TxRw};
|
||||
use cuprate_test_utils::data::{block_v16_tx0, block_v1_tx2, block_v9_tx3};
|
||||
|
||||
use super::*;
|
||||
|
||||
use crate::{
|
||||
open_tables::OpenTables,
|
||||
ops::block::add_block,
|
||||
tables::Tables,
|
||||
tests::{assert_all_tables_are_empty, tmp_concrete_env, AssertTableLen},
|
||||
transaction::TxRw,
|
||||
Env, EnvInner,
|
||||
};
|
||||
|
||||
/// Tests all above functions.
|
||||
|
|
|
|||
|
|
@ -1,9 +1,9 @@
|
|||
//! Key image functions.
|
||||
|
||||
//---------------------------------------------------------------------------------------------------- Import
|
||||
use cuprate_database::{DatabaseRo, DatabaseRw, RuntimeError};
|
||||
|
||||
use crate::{
|
||||
database::{DatabaseRo, DatabaseRw},
|
||||
error::RuntimeError,
|
||||
ops::macros::{doc_add_block_inner_invariant, doc_error},
|
||||
tables::KeyImages,
|
||||
types::KeyImage,
|
||||
|
|
@ -47,12 +47,14 @@ pub fn key_image_exists(
|
|||
mod test {
|
||||
use hex_literal::hex;
|
||||
|
||||
use cuprate_database::{Env, EnvInner, TxRw};
|
||||
|
||||
use super::*;
|
||||
|
||||
use crate::{
|
||||
open_tables::OpenTables,
|
||||
tables::{Tables, TablesMut},
|
||||
tests::{assert_all_tables_are_empty, tmp_concrete_env, AssertTableLen},
|
||||
transaction::TxRw,
|
||||
Env, EnvInner,
|
||||
};
|
||||
|
||||
/// Tests all above key-image functions.
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@
|
|||
//! it is up to the caller to decide what happens if one them return
|
||||
//! an error.
|
||||
//!
|
||||
//! To maintain atomicity, transactions should be [`abort`](crate::transaction::TxRw::abort)ed
|
||||
//! To maintain atomicity, transactions should be [`abort`](cuprate_database::TxRw::abort)ed
|
||||
//! if one of the functions failed.
|
||||
//!
|
||||
//! For example, if [`add_block()`](block::add_block) is called and returns an [`Err`],
|
||||
|
|
@ -55,25 +55,28 @@
|
|||
//! use hex_literal::hex;
|
||||
//!
|
||||
//! use cuprate_test_utils::data::block_v16_tx0;
|
||||
//!
|
||||
//! use cuprate_blockchain::{
|
||||
//! ConcreteEnv,
|
||||
//! cuprate_database::{
|
||||
//! ConcreteEnv,
|
||||
//! Env, EnvInner,
|
||||
//! DatabaseRo, DatabaseRw, TxRo, TxRw,
|
||||
//! },
|
||||
//! OpenTables,
|
||||
//! config::ConfigBuilder,
|
||||
//! Env, EnvInner,
|
||||
//! tables::{Tables, TablesMut},
|
||||
//! DatabaseRo, DatabaseRw, TxRo, TxRw,
|
||||
//! ops::block::{add_block, pop_block},
|
||||
//! };
|
||||
//!
|
||||
//! # fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
//! // Create a configuration for the database environment.
|
||||
//! let db_dir = tempfile::tempdir()?;
|
||||
//! let tmp_dir = tempfile::tempdir()?;
|
||||
//! let db_dir = tmp_dir.path().to_owned();
|
||||
//! let config = ConfigBuilder::new()
|
||||
//! .db_directory(db_dir.path().to_path_buf())
|
||||
//! .db_directory(db_dir.into())
|
||||
//! .build();
|
||||
//!
|
||||
//! // Initialize the database environment.
|
||||
//! let env = ConcreteEnv::open(config)?;
|
||||
//! let env = cuprate_blockchain::open(config)?;
|
||||
//!
|
||||
//! // Open up a transaction + tables for writing.
|
||||
//! let env_inner = env.env_inner();
|
||||
|
|
|
|||
|
|
@ -4,12 +4,13 @@
|
|||
use curve25519_dalek::{constants::ED25519_BASEPOINT_POINT, edwards::CompressedEdwardsY, Scalar};
|
||||
use monero_serai::{transaction::Timelock, H};
|
||||
|
||||
use cuprate_database::{
|
||||
RuntimeError, {DatabaseRo, DatabaseRw},
|
||||
};
|
||||
use cuprate_helper::map::u64_to_timelock;
|
||||
use cuprate_types::OutputOnChain;
|
||||
|
||||
use crate::{
|
||||
database::{DatabaseRo, DatabaseRw},
|
||||
error::RuntimeError,
|
||||
ops::macros::{doc_add_block_inner_invariant, doc_error},
|
||||
tables::{Outputs, RctOutputs, Tables, TablesMut, TxUnlockTime},
|
||||
types::{Amount, AmountIndex, Output, OutputFlags, PreRctOutputId, RctOutput},
|
||||
|
|
@ -247,15 +248,18 @@ pub fn id_to_output_on_chain(
|
|||
#[cfg(test)]
|
||||
mod test {
|
||||
use super::*;
|
||||
|
||||
use pretty_assertions::assert_eq;
|
||||
|
||||
use cuprate_database::{Env, EnvInner};
|
||||
|
||||
use crate::{
|
||||
open_tables::OpenTables,
|
||||
tables::{Tables, TablesMut},
|
||||
tests::{assert_all_tables_are_empty, tmp_concrete_env, AssertTableLen},
|
||||
types::OutputFlags,
|
||||
Env, EnvInner,
|
||||
};
|
||||
|
||||
use pretty_assertions::assert_eq;
|
||||
|
||||
/// Dummy `Output`.
|
||||
const OUTPUT: Output = Output {
|
||||
key: [44; 32],
|
||||
|
|
|
|||
|
|
@ -5,7 +5,10 @@
|
|||
//---------------------------------------------------------------------------------------------------- Import
|
||||
use cuprate_pruning::PruningSeed;
|
||||
|
||||
use crate::{error::RuntimeError, ops::macros::doc_error};
|
||||
use cuprate_database::RuntimeError;
|
||||
|
||||
use crate::ops::macros::doc_error;
|
||||
|
||||
//---------------------------------------------------------------------------------------------------- Free Functions
|
||||
/// SOMEDAY
|
||||
///
|
||||
|
|
|
|||
|
|
@ -5,9 +5,9 @@ use bytemuck::TransparentWrapper;
|
|||
use curve25519_dalek::{constants::ED25519_BASEPOINT_POINT, Scalar};
|
||||
use monero_serai::transaction::{Input, Timelock, Transaction};
|
||||
|
||||
use cuprate_database::{DatabaseRo, DatabaseRw, RuntimeError, StorableVec};
|
||||
|
||||
use crate::{
|
||||
database::{DatabaseRo, DatabaseRw},
|
||||
error::RuntimeError,
|
||||
ops::{
|
||||
key_image::{add_key_image, remove_key_image},
|
||||
macros::{doc_add_block_inner_invariant, doc_error},
|
||||
|
|
@ -17,7 +17,6 @@ use crate::{
|
|||
},
|
||||
tables::{TablesMut, TxBlobs, TxIds},
|
||||
types::{BlockHeight, Output, OutputFlags, PreRctOutputId, RctOutput, TxHash, TxId},
|
||||
StorableVec,
|
||||
};
|
||||
|
||||
//---------------------------------------------------------------------------------------------------- Private
|
||||
|
|
@ -325,14 +324,17 @@ pub fn tx_exists(
|
|||
#[cfg(test)]
|
||||
mod test {
|
||||
use super::*;
|
||||
|
||||
use pretty_assertions::assert_eq;
|
||||
|
||||
use cuprate_database::{Env, EnvInner, TxRw};
|
||||
use cuprate_test_utils::data::{tx_v1_sig0, tx_v1_sig2, tx_v2_rct3};
|
||||
|
||||
use crate::{
|
||||
open_tables::OpenTables,
|
||||
tables::Tables,
|
||||
tests::{assert_all_tables_are_empty, tmp_concrete_env, AssertTableLen},
|
||||
transaction::TxRw,
|
||||
Env, EnvInner,
|
||||
};
|
||||
use cuprate_test_utils::data::{tx_v1_sig0, tx_v1_sig2, tx_v2_rct3};
|
||||
use pretty_assertions::assert_eq;
|
||||
|
||||
/// Tests all above tx functions when only inputting `Transaction` data (no Block).
|
||||
#[test]
|
||||
|
|
|
|||
|
|
@ -3,11 +3,11 @@
|
|||
//---------------------------------------------------------------------------------------------------- Import
|
||||
use std::sync::Arc;
|
||||
|
||||
use cuprate_database::InitError;
|
||||
|
||||
use crate::{
|
||||
config::Config,
|
||||
error::InitError,
|
||||
service::{DatabaseReadHandle, DatabaseWriteHandle},
|
||||
ConcreteEnv, Env,
|
||||
};
|
||||
|
||||
//---------------------------------------------------------------------------------------------------- Init
|
||||
|
|
@ -19,12 +19,12 @@ use crate::{
|
|||
/// thread-pool and writer thread will exit automatically.
|
||||
///
|
||||
/// # Errors
|
||||
/// This will forward the error if [`Env::open`] failed.
|
||||
/// This will forward the error if [`crate::open`] failed.
|
||||
pub fn init(config: Config) -> Result<(DatabaseReadHandle, DatabaseWriteHandle), InitError> {
|
||||
let reader_threads = config.reader_threads;
|
||||
|
||||
// Initialize the database itself.
|
||||
let db = Arc::new(ConcreteEnv::open(config)?);
|
||||
let db = Arc::new(crate::open(config)?);
|
||||
|
||||
// Spawn the Reader thread pool and Writer.
|
||||
let readers = DatabaseReadHandle::init(&db, reader_threads);
|
||||
|
|
|
|||
|
|
@ -36,9 +36,9 @@
|
|||
//! - The last [`DatabaseReadHandle`] is dropped => reader thread-pool exits
|
||||
//! - The last [`DatabaseWriteHandle`] is dropped => writer thread exits
|
||||
//!
|
||||
//! Upon dropping the [`crate::ConcreteEnv`]:
|
||||
//! Upon dropping the [`cuprate_database::ConcreteEnv`]:
|
||||
//! - All un-processed database transactions are completed
|
||||
//! - All data gets flushed to disk (caused by [`Drop::drop`] impl on [`crate::ConcreteEnv`])
|
||||
//! - All data gets flushed to disk (caused by [`Drop::drop`] impl on `ConcreteEnv`)
|
||||
//!
|
||||
//! ## Request and Response
|
||||
//! To interact with the database (whether reading or writing data),
|
||||
|
|
@ -66,14 +66,18 @@
|
|||
//! use cuprate_types::blockchain::{BCReadRequest, BCWriteRequest, BCResponse};
|
||||
//! use cuprate_test_utils::data::block_v16_tx0;
|
||||
//!
|
||||
//! use cuprate_blockchain::{ConcreteEnv, config::ConfigBuilder, Env};
|
||||
//! use cuprate_blockchain::{
|
||||
//! cuprate_database::Env,
|
||||
//! config::ConfigBuilder,
|
||||
//! };
|
||||
//!
|
||||
//! # #[tokio::main]
|
||||
//! # async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
//! // Create a configuration for the database environment.
|
||||
//! let db_dir = tempfile::tempdir()?;
|
||||
//! let tmp_dir = tempfile::tempdir()?;
|
||||
//! let db_dir = tmp_dir.path().to_owned();
|
||||
//! let config = ConfigBuilder::new()
|
||||
//! .db_directory(db_dir.path().to_path_buf())
|
||||
//! .db_directory(db_dir.into())
|
||||
//! .build();
|
||||
//!
|
||||
//! // Initialize the database thread-pool.
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ use thread_local::ThreadLocal;
|
|||
use tokio::sync::{OwnedSemaphorePermit, Semaphore};
|
||||
use tokio_util::sync::PollSemaphore;
|
||||
|
||||
use cuprate_database::{ConcreteEnv, DatabaseRo, Env, EnvInner, RuntimeError};
|
||||
use cuprate_helper::asynch::InfallibleOneshotReceiver;
|
||||
use cuprate_types::{
|
||||
blockchain::{BCReadRequest, BCResponse},
|
||||
|
|
@ -21,7 +22,7 @@ use cuprate_types::{
|
|||
|
||||
use crate::{
|
||||
config::ReaderThreads,
|
||||
error::RuntimeError,
|
||||
open_tables::OpenTables,
|
||||
ops::block::block_exists,
|
||||
ops::{
|
||||
block::{get_block_extended_header_from_height, get_block_info},
|
||||
|
|
@ -33,7 +34,6 @@ use crate::{
|
|||
tables::{BlockHeights, BlockInfos, Tables},
|
||||
types::BlockHash,
|
||||
types::{Amount, AmountIndex, BlockHeight, KeyImage, PreRctOutputId},
|
||||
ConcreteEnv, DatabaseRo, Env, EnvInner,
|
||||
};
|
||||
|
||||
//---------------------------------------------------------------------------------------------------- DatabaseReadHandle
|
||||
|
|
@ -233,7 +233,7 @@ fn map_request(
|
|||
/// <https://github.com/Cuprate/cuprate/pull/113#discussion_r1576762346>
|
||||
#[inline]
|
||||
fn thread_local<T: Send>(env: &impl Env) -> ThreadLocal<T> {
|
||||
ThreadLocal::with_capacity(env.config().reader_threads.as_threads().get())
|
||||
ThreadLocal::with_capacity(env.config().reader_threads.get())
|
||||
}
|
||||
|
||||
/// Take in a `ThreadLocal<impl Tables>` and return an `&impl Tables + Send`.
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@
|
|||
|
||||
//---------------------------------------------------------------------------------------------------- Use
|
||||
use std::{
|
||||
borrow::Cow,
|
||||
collections::{HashMap, HashSet},
|
||||
sync::Arc,
|
||||
};
|
||||
|
|
@ -14,6 +15,7 @@ use std::{
|
|||
use pretty_assertions::assert_eq;
|
||||
use tower::{Service, ServiceExt};
|
||||
|
||||
use cuprate_database::{ConcreteEnv, DatabaseIter, DatabaseRo, Env, EnvInner, RuntimeError};
|
||||
use cuprate_test_utils::data::{block_v16_tx0, block_v1_tx2, block_v9_tx3};
|
||||
use cuprate_types::{
|
||||
blockchain::{BCReadRequest, BCResponse, BCWriteRequest},
|
||||
|
|
@ -22,6 +24,7 @@ use cuprate_types::{
|
|||
|
||||
use crate::{
|
||||
config::ConfigBuilder,
|
||||
open_tables::OpenTables,
|
||||
ops::{
|
||||
block::{get_block_extended_header_from_height, get_block_info},
|
||||
blockchain::chain_height,
|
||||
|
|
@ -31,7 +34,6 @@ use crate::{
|
|||
tables::{Tables, TablesIter},
|
||||
tests::AssertTableLen,
|
||||
types::{Amount, AmountIndex, PreRctOutputId},
|
||||
ConcreteEnv, DatabaseIter, DatabaseRo, Env, EnvInner, RuntimeError,
|
||||
};
|
||||
|
||||
//---------------------------------------------------------------------------------------------------- Helper functions
|
||||
|
|
@ -44,7 +46,7 @@ fn init_service() -> (
|
|||
) {
|
||||
let tempdir = tempfile::tempdir().unwrap();
|
||||
let config = ConfigBuilder::new()
|
||||
.db_directory(tempdir.path().into())
|
||||
.db_directory(Cow::Owned(tempdir.path().into()))
|
||||
.low_power()
|
||||
.build();
|
||||
let (reader, writer) = init(config).unwrap();
|
||||
|
|
|
|||
|
|
@ -5,11 +5,10 @@
|
|||
//---------------------------------------------------------------------------------------------------- Use
|
||||
use futures::channel::oneshot::Sender;
|
||||
|
||||
use cuprate_database::RuntimeError;
|
||||
use cuprate_helper::asynch::InfallibleOneshotReceiver;
|
||||
use cuprate_types::blockchain::BCResponse;
|
||||
|
||||
use crate::error::RuntimeError;
|
||||
|
||||
//---------------------------------------------------------------------------------------------------- Types
|
||||
/// The actual type of the response.
|
||||
///
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ use std::{
|
|||
|
||||
use futures::channel::oneshot;
|
||||
|
||||
use cuprate_database::{ConcreteEnv, Env, EnvInner, RuntimeError, TxRw};
|
||||
use cuprate_helper::asynch::InfallibleOneshotReceiver;
|
||||
use cuprate_types::{
|
||||
blockchain::{BCResponse, BCWriteRequest},
|
||||
|
|
@ -15,11 +16,8 @@ use cuprate_types::{
|
|||
};
|
||||
|
||||
use crate::{
|
||||
env::{Env, EnvInner},
|
||||
error::RuntimeError,
|
||||
open_tables::OpenTables,
|
||||
service::types::{ResponseReceiver, ResponseResult, ResponseSender},
|
||||
transaction::TxRw,
|
||||
ConcreteEnv,
|
||||
};
|
||||
|
||||
//---------------------------------------------------------------------------------------------------- Constants
|
||||
|
|
|
|||
|
|
@ -15,17 +15,15 @@
|
|||
//! This module also contains a set of traits for
|
||||
//! accessing _all_ tables defined here at once.
|
||||
//!
|
||||
//! For example, this is the object returned by [`EnvInner::open_tables`](crate::EnvInner::open_tables).
|
||||
//! For example, this is the object returned by [`OpenTables::open_tables`](crate::OpenTables::open_tables).
|
||||
|
||||
//---------------------------------------------------------------------------------------------------- Import
|
||||
use crate::{
|
||||
database::{DatabaseIter, DatabaseRo, DatabaseRw},
|
||||
table::Table,
|
||||
types::{
|
||||
Amount, AmountIndex, AmountIndices, BlockBlob, BlockHash, BlockHeight, BlockInfo, KeyImage,
|
||||
Output, PreRctOutputId, PrunableBlob, PrunableHash, PrunedBlob, RctOutput, TxBlob, TxHash,
|
||||
TxId, UnlockTime,
|
||||
},
|
||||
use cuprate_database::{DatabaseIter, DatabaseRo, DatabaseRw, Table};
|
||||
|
||||
use crate::types::{
|
||||
Amount, AmountIndex, AmountIndices, BlockBlob, BlockHash, BlockHeight, BlockInfo, KeyImage,
|
||||
Output, PreRctOutputId, PrunableBlob, PrunableHash, PrunedBlob, RctOutput, TxBlob, TxHash,
|
||||
TxId, UnlockTime,
|
||||
};
|
||||
|
||||
//---------------------------------------------------------------------------------------------------- Sealed
|
||||
|
|
@ -61,7 +59,7 @@ macro_rules! define_trait_tables {
|
|||
/// `(tuple, containing, all, table, types, ...)`.
|
||||
///
|
||||
/// This is used to return a _single_ object from functions like
|
||||
/// [`EnvInner::open_tables`](crate::EnvInner::open_tables) rather
|
||||
/// [`OpenTables::open_tables`](crate::OpenTables::open_tables) rather
|
||||
/// than the tuple containing the tables itself.
|
||||
///
|
||||
/// To replace `tuple.0` style indexing, `field_accessor_functions()`
|
||||
|
|
@ -98,7 +96,7 @@ macro_rules! define_trait_tables {
|
|||
///
|
||||
/// # Errors
|
||||
/// This returns errors on regular database errors.
|
||||
fn all_tables_empty(&self) -> Result<bool, $crate::error::RuntimeError>;
|
||||
fn all_tables_empty(&self) -> Result<bool, cuprate_database::RuntimeError>;
|
||||
}
|
||||
|
||||
/// Object containing all opened [`Table`]s in read + iter mode.
|
||||
|
|
@ -183,7 +181,7 @@ macro_rules! define_trait_tables {
|
|||
}
|
||||
)*
|
||||
|
||||
fn all_tables_empty(&self) -> Result<bool, $crate::error::RuntimeError> {
|
||||
fn all_tables_empty(&self) -> Result<bool, cuprate_database::RuntimeError> {
|
||||
$(
|
||||
if !DatabaseRo::is_empty(&self.$index)? {
|
||||
return Ok(false);
|
||||
|
|
@ -265,44 +263,6 @@ define_trait_tables! {
|
|||
TxUnlockTime => 14,
|
||||
}
|
||||
|
||||
//---------------------------------------------------------------------------------------------------- Table function macro
|
||||
/// `crate`-private macro for callings functions on all tables.
|
||||
///
|
||||
/// This calls the function `$fn` with the optional
|
||||
/// arguments `$args` on all tables - returning early
|
||||
/// (within whatever scope this is called) if any
|
||||
/// of the function calls error.
|
||||
///
|
||||
/// Else, it evaluates to an `Ok((tuple, of, all, table, types, ...))`,
|
||||
/// i.e., an `impl Table[Mut]` wrapped in `Ok`.
|
||||
macro_rules! call_fn_on_all_tables_or_early_return {
|
||||
(
|
||||
$($fn:ident $(::)?)*
|
||||
(
|
||||
$($arg:ident),* $(,)?
|
||||
)
|
||||
) => {{
|
||||
Ok((
|
||||
$($fn ::)*<$crate::tables::BlockInfos>($($arg),*)?,
|
||||
$($fn ::)*<$crate::tables::BlockBlobs>($($arg),*)?,
|
||||
$($fn ::)*<$crate::tables::BlockHeights>($($arg),*)?,
|
||||
$($fn ::)*<$crate::tables::KeyImages>($($arg),*)?,
|
||||
$($fn ::)*<$crate::tables::NumOutputs>($($arg),*)?,
|
||||
$($fn ::)*<$crate::tables::PrunedTxBlobs>($($arg),*)?,
|
||||
$($fn ::)*<$crate::tables::PrunableHashes>($($arg),*)?,
|
||||
$($fn ::)*<$crate::tables::Outputs>($($arg),*)?,
|
||||
$($fn ::)*<$crate::tables::PrunableTxBlobs>($($arg),*)?,
|
||||
$($fn ::)*<$crate::tables::RctOutputs>($($arg),*)?,
|
||||
$($fn ::)*<$crate::tables::TxBlobs>($($arg),*)?,
|
||||
$($fn ::)*<$crate::tables::TxIds>($($arg),*)?,
|
||||
$($fn ::)*<$crate::tables::TxHeights>($($arg),*)?,
|
||||
$($fn ::)*<$crate::tables::TxOutputs>($($arg),*)?,
|
||||
$($fn ::)*<$crate::tables::TxUnlockTime>($($arg),*)?,
|
||||
))
|
||||
}};
|
||||
}
|
||||
pub(crate) use call_fn_on_all_tables_or_early_return;
|
||||
|
||||
//---------------------------------------------------------------------------------------------------- Table macro
|
||||
/// Create all tables, should be used _once_.
|
||||
///
|
||||
|
|
@ -332,6 +292,7 @@ macro_rules! tables {
|
|||
/// ## Table Name
|
||||
/// ```rust
|
||||
/// # use cuprate_blockchain::{*,tables::*};
|
||||
/// use cuprate_database::Table;
|
||||
#[doc = concat!(
|
||||
"assert_eq!(",
|
||||
stringify!([<$table:camel>]),
|
||||
|
|
@ -363,9 +324,8 @@ macro_rules! tables {
|
|||
// - Keep this sorted A-Z (by table name)
|
||||
// - Tables are defined in plural to avoid name conflicts with types
|
||||
// - If adding/changing a table also edit:
|
||||
// a) the tests in `src/backend/tests.rs`
|
||||
// b) `Env::open` to make sure it creates the table (for all backends)
|
||||
// c) `call_fn_on_all_tables_or_early_return!()` macro defined in this file
|
||||
// - the tests in `src/backend/tests.rs`
|
||||
// - `call_fn_on_all_tables_or_early_return!()` macro in `src/open_tables.rs`
|
||||
tables! {
|
||||
/// Serialized block blobs (bytes).
|
||||
///
|
||||
|
|
|
|||
|
|
@ -5,11 +5,13 @@
|
|||
//! - only used internally
|
||||
|
||||
//---------------------------------------------------------------------------------------------------- Import
|
||||
use std::fmt::Debug;
|
||||
use std::{borrow::Cow, fmt::Debug};
|
||||
|
||||
use pretty_assertions::assert_eq;
|
||||
|
||||
use crate::{config::ConfigBuilder, tables::Tables, ConcreteEnv, DatabaseRo, Env, EnvInner};
|
||||
use cuprate_database::{ConcreteEnv, DatabaseRo, Env, EnvInner};
|
||||
|
||||
use crate::{config::ConfigBuilder, open_tables::OpenTables, tables::Tables};
|
||||
|
||||
//---------------------------------------------------------------------------------------------------- Struct
|
||||
/// Named struct to assert the length of all tables.
|
||||
|
|
@ -67,10 +69,10 @@ impl AssertTableLen {
|
|||
pub(crate) fn tmp_concrete_env() -> (ConcreteEnv, tempfile::TempDir) {
|
||||
let tempdir = tempfile::tempdir().unwrap();
|
||||
let config = ConfigBuilder::new()
|
||||
.db_directory(tempdir.path().into())
|
||||
.db_directory(Cow::Owned(tempdir.path().into()))
|
||||
.low_power()
|
||||
.build();
|
||||
let env = ConcreteEnv::open(config).unwrap();
|
||||
let env = crate::open(config).unwrap();
|
||||
|
||||
(env, tempdir)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -46,7 +46,7 @@ use bytemuck::{Pod, Zeroable};
|
|||
#[cfg(feature = "serde")]
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::storable::StorableVec;
|
||||
use cuprate_database::StorableVec;
|
||||
|
||||
//---------------------------------------------------------------------------------------------------- Aliases
|
||||
// These type aliases exist as many Monero-related types are the exact same.
|
||||
|
|
@ -106,6 +106,8 @@ pub type UnlockTime = u64;
|
|||
/// ```rust
|
||||
/// # use std::borrow::*;
|
||||
/// # use cuprate_blockchain::{*, types::*};
|
||||
/// use cuprate_database::Storable;
|
||||
///
|
||||
/// // Assert Storable is correct.
|
||||
/// let a = PreRctOutputId {
|
||||
/// amount: 1,
|
||||
|
|
@ -149,6 +151,8 @@ pub struct PreRctOutputId {
|
|||
/// ```rust
|
||||
/// # use std::borrow::*;
|
||||
/// # use cuprate_blockchain::{*, types::*};
|
||||
/// use cuprate_database::Storable;
|
||||
///
|
||||
/// // Assert Storable is correct.
|
||||
/// let a = BlockInfo {
|
||||
/// timestamp: 1,
|
||||
|
|
@ -208,6 +212,8 @@ bitflags::bitflags! {
|
|||
/// ```rust
|
||||
/// # use std::borrow::*;
|
||||
/// # use cuprate_blockchain::{*, types::*};
|
||||
/// use cuprate_database::Storable;
|
||||
///
|
||||
/// // Assert Storable is correct.
|
||||
/// let a = OutputFlags::NON_ZERO_UNLOCK_TIME;
|
||||
/// let b = Storable::as_bytes(&a);
|
||||
|
|
@ -237,6 +243,8 @@ bitflags::bitflags! {
|
|||
/// ```rust
|
||||
/// # use std::borrow::*;
|
||||
/// # use cuprate_blockchain::{*, types::*};
|
||||
/// use cuprate_database::Storable;
|
||||
///
|
||||
/// // Assert Storable is correct.
|
||||
/// let a = Output {
|
||||
/// key: [1; 32],
|
||||
|
|
@ -278,6 +286,8 @@ pub struct Output {
|
|||
/// ```rust
|
||||
/// # use std::borrow::*;
|
||||
/// # use cuprate_blockchain::{*, types::*};
|
||||
/// use cuprate_database::Storable;
|
||||
///
|
||||
/// // Assert Storable is correct.
|
||||
/// let a = RctOutput {
|
||||
/// key: [1; 32],
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
[package]
|
||||
name = "cuprate-database"
|
||||
version = "0.0.0"
|
||||
version = "0.0.1"
|
||||
edition = "2021"
|
||||
description = "Cuprate's database abstraction"
|
||||
license = "MIT"
|
||||
|
|
@ -9,7 +9,26 @@ repository = "https://github.com/Cuprate/cuprate/tree/main/storage/database"
|
|||
keywords = ["cuprate", "database"]
|
||||
|
||||
[features]
|
||||
default = ["heed"]
|
||||
# default = ["redb"]
|
||||
# default = ["redb-memory"]
|
||||
heed = ["dep:heed"]
|
||||
redb = ["dep:redb"]
|
||||
redb-memory = ["redb"]
|
||||
|
||||
[dependencies]
|
||||
bytemuck = { version = "1.14.3", features = ["must_cast", "derive", "min_const_generics", "extern_crate_alloc"] }
|
||||
bytes = { workspace = true }
|
||||
cfg-if = { workspace = true }
|
||||
page_size = { version = "0.6.0" } # Needed for database resizes, they must be a multiple of the OS page size.
|
||||
thiserror = { workspace = true }
|
||||
|
||||
# Optional features.
|
||||
heed = { version = "0.20.0", features = ["read-txn-no-tls"], optional = true }
|
||||
redb = { version = "2.1.0", optional = true }
|
||||
serde = { workspace = true, optional = true }
|
||||
|
||||
[dev-dependencies]
|
||||
bytemuck = { version = "1.14.3", features = ["must_cast", "derive", "min_const_generics", "extern_crate_alloc"] }
|
||||
page_size = { version = "0.6.0" }
|
||||
tempfile = { version = "3.10.0" }
|
||||
143
storage/database/README.md
Normal file
143
storage/database/README.md
Normal file
|
|
@ -0,0 +1,143 @@
|
|||
Cuprate's database abstraction.
|
||||
|
||||
This documentation is mostly for practical usage of `cuprate-database`.
|
||||
|
||||
For a high-level overview, see the database section in
|
||||
[Cuprate's architecture book](https://architecture.cuprate.org).
|
||||
|
||||
If you need blockchain specific capabilities, consider using the higher-level
|
||||
`cuprate-blockchain` crate which builds upon this one.
|
||||
|
||||
# Purpose
|
||||
This crate abstracts various database backends with traits. The databases are:
|
||||
|
||||
All backends have the following attributes:
|
||||
- [Embedded](https://en.wikipedia.org/wiki/Embedded_database)
|
||||
- [Multiversion concurrency control](https://en.wikipedia.org/wiki/Multiversion_concurrency_control)
|
||||
- [ACID](https://en.wikipedia.org/wiki/ACID)
|
||||
- Are `(key, value)` oriented and have the expected API (`get()`, `insert()`, `delete()`)
|
||||
- Are table oriented (`"table_name" -> (key, value)`)
|
||||
- Allows concurrent readers
|
||||
|
||||
# Terminology
|
||||
To be more clear on some terms used in this crate:
|
||||
|
||||
| Term | Meaning |
|
||||
|------------------|--------------------------------------|
|
||||
| `Env` | The 1 database environment, the "whole" thing
|
||||
| `DatabaseR{o,w}` | A _actively open_ readable/writable `key/value` store
|
||||
| `Table` | Solely the metadata of a `cuprate_database` (the `key` and `value` types, and the name)
|
||||
| `TxR{o,w}` | A read/write transaction
|
||||
| `Storable` | A data that type can be stored in the database
|
||||
|
||||
The dataflow is `Env` -> `Tx` -> `cuprate_database`
|
||||
|
||||
Which reads as:
|
||||
1. You have a database `Environment`
|
||||
1. You open up a `Transaction`
|
||||
1. You open a particular `Table` from that `Environment`, getting a `cuprate_database`
|
||||
1. You can now read/write data from/to that `cuprate_database`
|
||||
|
||||
# Concrete types
|
||||
You should _not_ rely on the concrete type of any abstracted backend.
|
||||
|
||||
For example, when using the `heed` backend, [`Env`]'s associated [`TxRw`] type
|
||||
is `RefCell<heed::RwTxn<'_>>`. In order to ensure compatibility with other backends
|
||||
and to not create backend-specific code, you should _not_ refer to that concrete type.
|
||||
|
||||
Use generics and trait notation in these situations:
|
||||
- `impl<T: TxRw> Trait for Object`
|
||||
- `fn() -> impl TxRw`
|
||||
|
||||
# `ConcreteEnv`
|
||||
This crate exposes [`ConcreteEnv`], which is a non-generic/non-dynamic,
|
||||
concrete object representing a database [`Env`]ironment.
|
||||
|
||||
The actual backend for this type is determined via feature flags.
|
||||
|
||||
This object existing means `E: Env` doesn't need to be spread all through the codebase,
|
||||
however, it also means some small invariants should be kept in mind.
|
||||
|
||||
As `ConcreteEnv` is just a re-exposed type which has varying inner types,
|
||||
it means some properties will change depending on the backend used.
|
||||
|
||||
For example:
|
||||
- [`std::mem::size_of::<ConcreteEnv>`]
|
||||
- [`std::mem::align_of::<ConcreteEnv>`]
|
||||
|
||||
Things like these functions are affected by the backend and inner data,
|
||||
and should not be relied upon. This extends to any `struct/enum` that contains `ConcreteEnv`.
|
||||
|
||||
`ConcreteEnv` invariants you can rely on:
|
||||
- It implements [`Env`]
|
||||
- Upon [`Drop::drop`], all database data will sync to disk
|
||||
|
||||
Note that `ConcreteEnv` itself is not a clonable type,
|
||||
it should be wrapped in [`std::sync::Arc`].
|
||||
|
||||
<!-- SOMEDAY: replace `ConcreteEnv` with `fn Env::open() -> impl Env`/
|
||||
and use `<E: Env>` everywhere it is stored instead. This would allow
|
||||
generic-backed dynamic runtime selection of the database backend, i.e.
|
||||
the user can select which database backend they use. -->
|
||||
|
||||
# Feature flags
|
||||
Different database backends are enabled by the feature flags:
|
||||
- `heed` (LMDB)
|
||||
- `redb`
|
||||
|
||||
The default is `heed`.
|
||||
|
||||
`tracing` is always enabled and cannot be disabled via feature-flag.
|
||||
<!-- FIXME: tracing should be behind a feature flag -->
|
||||
|
||||
# Examples
|
||||
The below is an example of using `cuprate-database`.
|
||||
|
||||
```rust
|
||||
use cuprate_database::{
|
||||
ConcreteEnv,
|
||||
config::ConfigBuilder,
|
||||
Env, EnvInner,
|
||||
DatabaseRo, DatabaseRw, TxRo, TxRw,
|
||||
};
|
||||
|
||||
# fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
// Create a configuration for the database environment.
|
||||
let tmp_dir = tempfile::tempdir()?;
|
||||
let db_dir = tmp_dir.path().to_owned();
|
||||
let config = ConfigBuilder::new(db_dir.into()).build();
|
||||
|
||||
// Initialize the database environment.
|
||||
let env = ConcreteEnv::open(config)?;
|
||||
|
||||
// Define metadata for a table.
|
||||
struct Table;
|
||||
impl cuprate_database::Table for Table {
|
||||
// The name of the table is "table".
|
||||
const NAME: &'static str = "table";
|
||||
// The key type is a `u8`.
|
||||
type Key = u8;
|
||||
// The key type is a `u64`.
|
||||
type Value = u64;
|
||||
}
|
||||
|
||||
// Open up a transaction + tables for writing.
|
||||
let env_inner = env.env_inner();
|
||||
let tx_rw = env_inner.tx_rw()?;
|
||||
// We must create the table first or the next line will error.
|
||||
env_inner.create_db::<Table>(&tx_rw)?;
|
||||
let mut table = env_inner.open_db_rw::<Table>(&tx_rw)?;
|
||||
|
||||
// Write data to the table.
|
||||
table.put(&0, &1)?;
|
||||
|
||||
// Commit the data written.
|
||||
drop(table);
|
||||
TxRw::commit(tx_rw)?;
|
||||
|
||||
// Read the data, assert it is correct.
|
||||
let tx_ro = env_inner.tx_ro()?;
|
||||
let table = env_inner.open_db_ro::<Table>(&tx_ro)?;
|
||||
assert_eq!(table.first()?, (0, 1));
|
||||
# Ok(()) }
|
||||
```
|
||||
|
|
@ -7,12 +7,11 @@ use std::{
|
|||
sync::{RwLock, RwLockReadGuard},
|
||||
};
|
||||
|
||||
use heed::{DatabaseOpenOptions, EnvFlags, EnvOpenOptions};
|
||||
use heed::{EnvFlags, EnvOpenOptions};
|
||||
|
||||
use crate::{
|
||||
backend::heed::{
|
||||
database::{HeedTableRo, HeedTableRw},
|
||||
storable::StorableHeed,
|
||||
types::HeedDb,
|
||||
},
|
||||
config::{Config, SyncMode},
|
||||
|
|
@ -21,13 +20,12 @@ use crate::{
|
|||
error::{InitError, RuntimeError},
|
||||
resize::ResizeAlgorithm,
|
||||
table::Table,
|
||||
tables::call_fn_on_all_tables_or_early_return,
|
||||
};
|
||||
|
||||
//---------------------------------------------------------------------------------------------------- Consts
|
||||
/// Panic message when there's a table missing.
|
||||
const PANIC_MSG_MISSING_TABLE: &str =
|
||||
"cuprate_blockchain::Env should uphold the invariant that all tables are already created";
|
||||
"cuprate_database::Env should uphold the invariant that all tables are already created";
|
||||
|
||||
//---------------------------------------------------------------------------------------------------- ConcreteEnv
|
||||
/// A strongly typed, concrete database environment, backed by `heed`.
|
||||
|
|
@ -184,8 +182,7 @@ impl Env for ConcreteEnv {
|
|||
// For now:
|
||||
// - No other program using our DB exists
|
||||
// - Almost no-one has a 126+ thread CPU
|
||||
let reader_threads =
|
||||
u32::try_from(config.reader_threads.as_threads().get()).unwrap_or(u32::MAX);
|
||||
let reader_threads = u32::try_from(config.reader_threads.get()).unwrap_or(u32::MAX);
|
||||
env_open_options.max_readers(if reader_threads < 110 {
|
||||
126
|
||||
} else {
|
||||
|
|
@ -199,34 +196,6 @@ impl Env for ConcreteEnv {
|
|||
// <https://docs.rs/heed/0.20.0/heed/struct.EnvOpenOptions.html#method.open>
|
||||
let env = unsafe { env_open_options.open(config.db_directory())? };
|
||||
|
||||
/// Function that creates the tables based off the passed `T: Table`.
|
||||
fn create_table<T: Table>(
|
||||
env: &heed::Env,
|
||||
tx_rw: &mut heed::RwTxn<'_>,
|
||||
) -> Result<(), InitError> {
|
||||
DatabaseOpenOptions::new(env)
|
||||
.name(<T as Table>::NAME)
|
||||
.types::<StorableHeed<<T as Table>::Key>, StorableHeed<<T as Table>::Value>>()
|
||||
.create(tx_rw)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
let mut tx_rw = env.write_txn()?;
|
||||
// Create all tables.
|
||||
// FIXME: this macro is kinda awkward.
|
||||
{
|
||||
let env = &env;
|
||||
let tx_rw = &mut tx_rw;
|
||||
match call_fn_on_all_tables_or_early_return!(create_table(env, tx_rw)) {
|
||||
Ok(_) => (),
|
||||
Err(e) => return Err(e),
|
||||
}
|
||||
}
|
||||
|
||||
// INVARIANT: this should never return `ResizeNeeded` due to adding
|
||||
// some tables since we added some leeway to the memory map above.
|
||||
tx_rw.commit()?;
|
||||
|
||||
Ok(Self {
|
||||
env: RwLock::new(env),
|
||||
config,
|
||||
|
|
@ -302,7 +271,7 @@ where
|
|||
Ok(HeedTableRo {
|
||||
db: self
|
||||
.open_database(tx_ro, Some(T::NAME))?
|
||||
.expect(PANIC_MSG_MISSING_TABLE),
|
||||
.ok_or(RuntimeError::TableNotFound)?,
|
||||
tx_ro,
|
||||
})
|
||||
}
|
||||
|
|
@ -312,17 +281,19 @@ where
|
|||
&self,
|
||||
tx_rw: &RefCell<heed::RwTxn<'env>>,
|
||||
) -> Result<impl DatabaseRw<T>, RuntimeError> {
|
||||
let tx_ro = tx_rw.borrow();
|
||||
|
||||
// Open up a read/write database using our table's const metadata.
|
||||
Ok(HeedTableRw {
|
||||
db: self
|
||||
.open_database(&tx_ro, Some(T::NAME))?
|
||||
.expect(PANIC_MSG_MISSING_TABLE),
|
||||
db: self.create_database(&mut tx_rw.borrow_mut(), Some(T::NAME))?,
|
||||
tx_rw,
|
||||
})
|
||||
}
|
||||
|
||||
fn create_db<T: Table>(&self, tx_rw: &RefCell<heed::RwTxn<'env>>) -> Result<(), RuntimeError> {
|
||||
// INVARIANT: `heed` creates tables with `open_database` if they don't exist.
|
||||
self.open_db_rw::<T>(tx_rw)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn clear_db<T: Table>(
|
||||
&self,
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
//! Conversion from `heed::Error` -> `cuprate_blockchain`'s errors.
|
||||
//! Conversion from `heed::Error` -> `cuprate_database`'s errors.
|
||||
|
||||
//---------------------------------------------------------------------------------------------------- Use
|
||||
use crate::constants::DATABASE_CORRUPT_MSG;
|
||||
|
|
@ -85,7 +85,7 @@ impl From<heed::Error> for crate::RuntimeError {
|
|||
E2::Corrupted | E2::PageNotFound => panic!("{mdb_error:#?}\n{DATABASE_CORRUPT_MSG}"),
|
||||
|
||||
// These errors should not occur, and if they do,
|
||||
// the best thing `cuprate_blockchain` can do for
|
||||
// the best thing `cuprate_database` can do for
|
||||
// safety is to panic right here.
|
||||
E2::Panic
|
||||
| E2::PageFull
|
||||
|
|
@ -134,12 +134,12 @@ impl From<heed::Error> for crate::RuntimeError {
|
|||
// Don't use a key that is `>511` bytes.
|
||||
// <http://www.lmdb.tech/doc/group__mdb.html#gaaf0be004f33828bf2fb09d77eb3cef94>
|
||||
| E2::BadValSize
|
||||
=> panic!("fix the database code! {mdb_error:#?}"),
|
||||
=> panic!("E2: fix the database code! {mdb_error:#?}"),
|
||||
},
|
||||
|
||||
// Only if we write incorrect code.
|
||||
E1::DatabaseClosing | E1::BadOpenOptions { .. } | E1::Encoding(_) | E1::Decoding(_) => {
|
||||
panic!("fix the database code! {error:#?}")
|
||||
panic!("E1: fix the database code! {error:#?}")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
//! `cuprate_blockchain::Storable` <-> `heed` serde trait compatibility layer.
|
||||
//! `cuprate_database::Storable` <-> `heed` serde trait compatibility layer.
|
||||
|
||||
//---------------------------------------------------------------------------------------------------- Use
|
||||
use std::{borrow::Cow, marker::PhantomData};
|
||||
|
|
@ -9,7 +9,7 @@ use crate::storable::Storable;
|
|||
|
||||
//---------------------------------------------------------------------------------------------------- StorableHeed
|
||||
/// The glue struct that implements `heed`'s (de)serialization
|
||||
/// traits on any type that implements `cuprate_blockchain::Storable`.
|
||||
/// traits on any type that implements `cuprate_database::Storable`.
|
||||
///
|
||||
/// Never actually gets constructed, just used for trait bound translations.
|
||||
pub(super) struct StorableHeed<T>(PhantomData<T>)
|
||||
|
|
@ -8,7 +8,6 @@ use crate::{
|
|||
env::{Env, EnvInner},
|
||||
error::{InitError, RuntimeError},
|
||||
table::Table,
|
||||
tables::call_fn_on_all_tables_or_early_return,
|
||||
TxRw,
|
||||
};
|
||||
|
||||
|
|
@ -22,7 +21,7 @@ pub struct ConcreteEnv {
|
|||
/// (and in current use).
|
||||
config: Config,
|
||||
|
||||
/// A cached, redb version of `cuprate_blockchain::config::SyncMode`.
|
||||
/// A cached, redb version of `cuprate_database::config::SyncMode`.
|
||||
/// `redb` needs the sync mode to be set _per_ TX, so we
|
||||
/// will continue to use this value every `Env::tx_rw`.
|
||||
durability: redb::Durability,
|
||||
|
|
@ -90,31 +89,6 @@ impl Env for ConcreteEnv {
|
|||
// `redb` creates tables if they don't exist.
|
||||
// <https://docs.rs/redb/latest/redb/struct.WriteTransaction.html#method.open_table>
|
||||
|
||||
/// Function that creates the tables based off the passed `T: Table`.
|
||||
fn create_table<T: Table>(tx_rw: &redb::WriteTransaction) -> Result<(), InitError> {
|
||||
let table: redb::TableDefinition<
|
||||
'static,
|
||||
StorableRedb<<T as Table>::Key>,
|
||||
StorableRedb<<T as Table>::Value>,
|
||||
> = redb::TableDefinition::new(<T as Table>::NAME);
|
||||
|
||||
// `redb` creates tables on open if not already created.
|
||||
tx_rw.open_table(table)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// Create all tables.
|
||||
// FIXME: this macro is kinda awkward.
|
||||
let mut tx_rw = env.begin_write()?;
|
||||
{
|
||||
let tx_rw = &mut tx_rw;
|
||||
match call_fn_on_all_tables_or_early_return!(create_table(tx_rw)) {
|
||||
Ok(_) => (),
|
||||
Err(e) => return Err(e),
|
||||
}
|
||||
}
|
||||
tx_rw.commit()?;
|
||||
|
||||
// Check for file integrity.
|
||||
// FIXME: should we do this? is it slow?
|
||||
env.check_integrity()?;
|
||||
|
|
@ -174,7 +148,6 @@ where
|
|||
let table: redb::TableDefinition<'static, StorableRedb<T::Key>, StorableRedb<T::Value>> =
|
||||
redb::TableDefinition::new(T::NAME);
|
||||
|
||||
// INVARIANT: Our `?` error conversion will panic if the table does not exist.
|
||||
Ok(tx_ro.open_table(table)?)
|
||||
}
|
||||
|
||||
|
|
@ -187,11 +160,17 @@ where
|
|||
let table: redb::TableDefinition<'static, StorableRedb<T::Key>, StorableRedb<T::Value>> =
|
||||
redb::TableDefinition::new(T::NAME);
|
||||
|
||||
// `redb` creates tables if they don't exist, so this should never panic.
|
||||
// `redb` creates tables if they don't exist, so this shouldn't return `RuntimeError::TableNotFound`.
|
||||
// <https://docs.rs/redb/latest/redb/struct.WriteTransaction.html#method.open_table>
|
||||
Ok(tx_rw.open_table(table)?)
|
||||
}
|
||||
|
||||
fn create_db<T: Table>(&self, tx_rw: &redb::WriteTransaction) -> Result<(), RuntimeError> {
|
||||
// INVARIANT: `redb` creates tables if they don't exist.
|
||||
self.open_db_rw::<T>(tx_rw)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn clear_db<T: Table>(&self, tx_rw: &mut redb::WriteTransaction) -> Result<(), RuntimeError> {
|
||||
let table: redb::TableDefinition<
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
//! Conversion from `redb`'s errors -> `cuprate_blockchain`'s errors.
|
||||
//! Conversion from `redb`'s errors -> `cuprate_database`'s errors.
|
||||
//!
|
||||
//! HACK: There's a lot of `_ =>` usage here because
|
||||
//! `redb`'s errors are `#[non_exhaustive]`...
|
||||
|
|
@ -131,12 +131,13 @@ impl From<redb::TableError> for RuntimeError {
|
|||
match error {
|
||||
E::Storage(error) => error.into(),
|
||||
|
||||
E::TableDoesNotExist(_) => Self::TableNotFound,
|
||||
|
||||
// Only if we write incorrect code.
|
||||
E::TableTypeMismatch { .. }
|
||||
| E::TableIsMultimap(_)
|
||||
| E::TableIsNotMultimap(_)
|
||||
| E::TypeDefinitionChanged { .. }
|
||||
| E::TableDoesNotExist(_)
|
||||
| E::TableAlreadyOpen(..) => panic!("fix the database code! {error:#?}"),
|
||||
|
||||
// HACK: Handle new errors as `redb` adds them.
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
//! `cuprate_blockchain::Storable` <-> `redb` serde trait compatibility layer.
|
||||
//! `cuprate_database::Storable` <-> `redb` serde trait compatibility layer.
|
||||
|
||||
//---------------------------------------------------------------------------------------------------- Use
|
||||
use std::{cmp::Ordering, fmt::Debug, marker::PhantomData};
|
||||
|
|
@ -9,7 +9,7 @@ use crate::{key::Key, storable::Storable};
|
|||
|
||||
//---------------------------------------------------------------------------------------------------- StorableRedb
|
||||
/// The glue structs that implements `redb`'s (de)serialization
|
||||
/// traits on any type that implements `cuprate_blockchain::Key`.
|
||||
/// traits on any type that implements `cuprate_database::Key`.
|
||||
///
|
||||
/// Never actually get constructed, just used for trait bound translations.
|
||||
#[derive(Debug)]
|
||||
374
storage/database/src/backend/tests.rs
Normal file
374
storage/database/src/backend/tests.rs
Normal file
|
|
@ -0,0 +1,374 @@
|
|||
//! Tests for `cuprate_database`'s backends.
|
||||
//!
|
||||
//! These tests are fully trait-based, meaning there
|
||||
//! is no reference to `backend/`-specific types.
|
||||
//!
|
||||
//! As such, which backend is tested is
|
||||
//! dependant on the feature flags used.
|
||||
//!
|
||||
//! | Feature flag | Tested backend |
|
||||
//! |---------------|----------------|
|
||||
//! | Only `redb` | `redb`
|
||||
//! | Anything else | `heed`
|
||||
//!
|
||||
//! `redb`, and it only must be enabled for it to be tested.
|
||||
|
||||
//---------------------------------------------------------------------------------------------------- Import
|
||||
use crate::{
|
||||
database::{DatabaseIter, DatabaseRo, DatabaseRw},
|
||||
env::{Env, EnvInner},
|
||||
error::RuntimeError,
|
||||
resize::ResizeAlgorithm,
|
||||
tests::{tmp_concrete_env, TestTable},
|
||||
transaction::{TxRo, TxRw},
|
||||
ConcreteEnv,
|
||||
};
|
||||
|
||||
//---------------------------------------------------------------------------------------------------- Tests
|
||||
/// Simply call [`Env::open`]. If this fails, something is really wrong.
|
||||
#[test]
|
||||
fn open() {
|
||||
tmp_concrete_env();
|
||||
}
|
||||
|
||||
/// Create database transactions, but don't write any data.
|
||||
#[test]
|
||||
fn tx() {
|
||||
let (env, _tempdir) = tmp_concrete_env();
|
||||
let env_inner = env.env_inner();
|
||||
|
||||
TxRo::commit(env_inner.tx_ro().unwrap()).unwrap();
|
||||
TxRw::commit(env_inner.tx_rw().unwrap()).unwrap();
|
||||
TxRw::abort(env_inner.tx_rw().unwrap()).unwrap();
|
||||
}
|
||||
|
||||
/// Test [`Env::open`] and creating/opening tables.
|
||||
#[test]
|
||||
fn open_db() {
|
||||
let (env, _tempdir) = tmp_concrete_env();
|
||||
let env_inner = env.env_inner();
|
||||
|
||||
// Create table.
|
||||
{
|
||||
let tx_rw = env_inner.tx_rw().unwrap();
|
||||
env_inner.create_db::<TestTable>(&tx_rw).unwrap();
|
||||
TxRw::commit(tx_rw).unwrap();
|
||||
}
|
||||
|
||||
let tx_ro = env_inner.tx_ro().unwrap();
|
||||
let tx_rw = env_inner.tx_rw().unwrap();
|
||||
|
||||
// Open table in read-only mode.
|
||||
env_inner.open_db_ro::<TestTable>(&tx_ro).unwrap();
|
||||
TxRo::commit(tx_ro).unwrap();
|
||||
|
||||
// Open table in read/write mode.
|
||||
env_inner.open_db_rw::<TestTable>(&tx_rw).unwrap();
|
||||
TxRw::commit(tx_rw).unwrap();
|
||||
}
|
||||
|
||||
/// Assert that opening a read-only table before creating errors.
|
||||
#[test]
|
||||
fn open_ro_uncreated_table() {
|
||||
let (env, _tempdir) = tmp_concrete_env();
|
||||
let env_inner = env.env_inner();
|
||||
let tx_ro = env_inner.tx_ro().unwrap();
|
||||
|
||||
// Open uncreated table.
|
||||
let error = env_inner.open_db_ro::<TestTable>(&tx_ro);
|
||||
assert!(matches!(error, Err(RuntimeError::TableNotFound)));
|
||||
}
|
||||
|
||||
/// Assert that opening a read/write table before creating is OK.
|
||||
#[test]
|
||||
fn open_rw_uncreated_table() {
|
||||
let (env, _tempdir) = tmp_concrete_env();
|
||||
let env_inner = env.env_inner();
|
||||
let tx_rw = env_inner.tx_rw().unwrap();
|
||||
|
||||
// Open uncreated table.
|
||||
let _table = env_inner.open_db_rw::<TestTable>(&tx_rw).unwrap();
|
||||
}
|
||||
|
||||
/// Assert that opening a read-only table after creating is OK.
|
||||
#[test]
|
||||
fn open_ro_created_table() {
|
||||
let (env, _tempdir) = tmp_concrete_env();
|
||||
let env_inner = env.env_inner();
|
||||
|
||||
// Assert uncreated table errors.
|
||||
{
|
||||
let tx_ro = env_inner.tx_ro().unwrap();
|
||||
let error = env_inner.open_db_ro::<TestTable>(&tx_ro);
|
||||
assert!(matches!(error, Err(RuntimeError::TableNotFound)));
|
||||
}
|
||||
|
||||
// Create table.
|
||||
{
|
||||
let tx_rw = env_inner.tx_rw().unwrap();
|
||||
env_inner.create_db::<TestTable>(&tx_rw).unwrap();
|
||||
TxRw::commit(tx_rw).unwrap();
|
||||
}
|
||||
|
||||
// Assert created table is now OK.
|
||||
let tx_ro = env_inner.tx_ro().unwrap();
|
||||
let _table = env_inner.open_db_ro::<TestTable>(&tx_ro).unwrap();
|
||||
}
|
||||
|
||||
/// Test `Env` resizes.
|
||||
#[test]
|
||||
fn resize() {
|
||||
// This test is only valid for `Env`'s that need to resize manually.
|
||||
if !ConcreteEnv::MANUAL_RESIZE {
|
||||
return;
|
||||
}
|
||||
|
||||
let (env, _tempdir) = tmp_concrete_env();
|
||||
|
||||
// Resize by the OS page size.
|
||||
let page_size = crate::resize::page_size();
|
||||
let old_size = env.current_map_size();
|
||||
env.resize_map(Some(ResizeAlgorithm::FixedBytes(page_size)));
|
||||
|
||||
// Assert it resized exactly by the OS page size.
|
||||
let new_size = env.current_map_size();
|
||||
assert_eq!(new_size, old_size + page_size.get());
|
||||
}
|
||||
|
||||
/// Test that `Env`'s that don't manually resize.
|
||||
#[test]
|
||||
#[should_panic = "unreachable"]
|
||||
fn non_manual_resize_1() {
|
||||
if ConcreteEnv::MANUAL_RESIZE {
|
||||
unreachable!();
|
||||
}
|
||||
let (env, _tempdir) = tmp_concrete_env();
|
||||
env.resize_map(None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[should_panic = "unreachable"]
|
||||
fn non_manual_resize_2() {
|
||||
if ConcreteEnv::MANUAL_RESIZE {
|
||||
unreachable!();
|
||||
}
|
||||
let (env, _tempdir) = tmp_concrete_env();
|
||||
env.current_map_size();
|
||||
}
|
||||
|
||||
/// Test all `DatabaseR{o,w}` operations.
|
||||
#[test]
|
||||
fn db_read_write() {
|
||||
let (env, _tempdir) = tmp_concrete_env();
|
||||
let env_inner = env.env_inner();
|
||||
let tx_rw = env_inner.tx_rw().unwrap();
|
||||
let mut table = env_inner.open_db_rw::<TestTable>(&tx_rw).unwrap();
|
||||
|
||||
/// The (1st) key.
|
||||
const KEY: u8 = 0;
|
||||
/// The expected value.
|
||||
const VALUE: u64 = 0;
|
||||
/// How many `(key, value)` pairs will be inserted.
|
||||
const N: u8 = 100;
|
||||
|
||||
/// Assert a u64 is the same as `VALUE`.
|
||||
fn assert_value(value: u64) {
|
||||
assert_eq!(value, VALUE);
|
||||
}
|
||||
|
||||
assert!(table.is_empty().unwrap());
|
||||
|
||||
// Insert keys.
|
||||
let mut key = KEY;
|
||||
#[allow(clippy::explicit_counter_loop)] // we need the +1 side effect
|
||||
for _ in 0..N {
|
||||
table.put(&key, &VALUE).unwrap();
|
||||
key += 1;
|
||||
}
|
||||
|
||||
assert_eq!(table.len().unwrap(), u64::from(N));
|
||||
|
||||
// Assert the first/last `(key, value)`s are there.
|
||||
{
|
||||
assert!(table.contains(&KEY).unwrap());
|
||||
let get = table.get(&KEY).unwrap();
|
||||
assert_value(get);
|
||||
|
||||
let first = table.first().unwrap().1;
|
||||
assert_value(first);
|
||||
|
||||
let last = table.last().unwrap().1;
|
||||
assert_value(last);
|
||||
}
|
||||
|
||||
// Commit transactions, create new ones.
|
||||
drop(table);
|
||||
TxRw::commit(tx_rw).unwrap();
|
||||
let tx_ro = env_inner.tx_ro().unwrap();
|
||||
let table_ro = env_inner.open_db_ro::<TestTable>(&tx_ro).unwrap();
|
||||
let tx_rw = env_inner.tx_rw().unwrap();
|
||||
let mut table = env_inner.open_db_rw::<TestTable>(&tx_rw).unwrap();
|
||||
|
||||
// Assert the whole range is there.
|
||||
{
|
||||
let range = table_ro.get_range(..).unwrap();
|
||||
let mut i = 0;
|
||||
for result in range {
|
||||
let value = result.unwrap();
|
||||
assert_value(value);
|
||||
i += 1;
|
||||
}
|
||||
assert_eq!(i, N);
|
||||
}
|
||||
|
||||
// `get_range()` tests.
|
||||
let mut key = KEY;
|
||||
key += N;
|
||||
let range = KEY..key;
|
||||
|
||||
// Assert count is correct.
|
||||
assert_eq!(
|
||||
N as usize,
|
||||
table_ro.get_range(range.clone()).unwrap().count()
|
||||
);
|
||||
|
||||
// Assert each returned value from the iterator is owned.
|
||||
{
|
||||
let mut iter = table_ro.get_range(range.clone()).unwrap();
|
||||
let value = iter.next().unwrap().unwrap(); // 1. take value out
|
||||
drop(iter); // 2. drop the `impl Iterator + 'a`
|
||||
assert_value(value); // 3. assert even without the iterator, the value is alive
|
||||
}
|
||||
|
||||
// Assert each value is the same.
|
||||
{
|
||||
let mut iter = table_ro.get_range(range).unwrap();
|
||||
for _ in 0..N {
|
||||
let value = iter.next().unwrap().unwrap();
|
||||
assert_value(value);
|
||||
}
|
||||
}
|
||||
|
||||
// Assert `update()` works.
|
||||
{
|
||||
const NEW_VALUE: u64 = 999;
|
||||
|
||||
assert_ne!(table.get(&KEY).unwrap(), NEW_VALUE);
|
||||
|
||||
#[allow(unused_assignments)]
|
||||
table
|
||||
.update(&KEY, |mut value| {
|
||||
value = NEW_VALUE;
|
||||
Some(value)
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(table.get(&KEY).unwrap(), NEW_VALUE);
|
||||
}
|
||||
|
||||
// Assert deleting works.
|
||||
{
|
||||
table.delete(&KEY).unwrap();
|
||||
let value = table.get(&KEY);
|
||||
assert!(!table.contains(&KEY).unwrap());
|
||||
assert!(matches!(value, Err(RuntimeError::KeyNotFound)));
|
||||
// Assert the other `(key, value)` pairs are still there.
|
||||
let mut key = KEY;
|
||||
key += N - 1; // we used inclusive `0..N`
|
||||
let value = table.get(&key).unwrap();
|
||||
assert_value(value);
|
||||
}
|
||||
|
||||
// Assert `take()` works.
|
||||
{
|
||||
let mut key = KEY;
|
||||
key += 1;
|
||||
let value = table.take(&key).unwrap();
|
||||
assert_eq!(value, VALUE);
|
||||
|
||||
let get = table.get(&KEY);
|
||||
assert!(!table.contains(&key).unwrap());
|
||||
assert!(matches!(get, Err(RuntimeError::KeyNotFound)));
|
||||
|
||||
// Assert the other `(key, value)` pairs are still there.
|
||||
key += 1;
|
||||
let value = table.get(&key).unwrap();
|
||||
assert_value(value);
|
||||
}
|
||||
|
||||
drop(table);
|
||||
TxRw::commit(tx_rw).unwrap();
|
||||
|
||||
// Assert `clear_db()` works.
|
||||
{
|
||||
let mut tx_rw = env_inner.tx_rw().unwrap();
|
||||
env_inner.clear_db::<TestTable>(&mut tx_rw).unwrap();
|
||||
let table = env_inner.open_db_rw::<TestTable>(&tx_rw).unwrap();
|
||||
assert!(table.is_empty().unwrap());
|
||||
for n in 0..N {
|
||||
let mut key = KEY;
|
||||
key += n;
|
||||
let value = table.get(&key);
|
||||
assert!(matches!(value, Err(RuntimeError::KeyNotFound)));
|
||||
assert!(!table.contains(&key).unwrap());
|
||||
}
|
||||
|
||||
// Reader still sees old value.
|
||||
assert!(!table_ro.is_empty().unwrap());
|
||||
|
||||
// Writer sees updated value (nothing).
|
||||
assert!(table.is_empty().unwrap());
|
||||
}
|
||||
}
|
||||
|
||||
/// Assert that `key`'s in database tables are sorted in
|
||||
/// an ordered B-Tree fashion, i.e. `min_value -> max_value`.
|
||||
#[test]
|
||||
fn tables_are_sorted() {
|
||||
let (env, _tmp) = tmp_concrete_env();
|
||||
let env_inner = env.env_inner();
|
||||
let tx_rw = env_inner.tx_rw().unwrap();
|
||||
let mut table = env_inner.open_db_rw::<TestTable>(&tx_rw).unwrap();
|
||||
|
||||
// Insert `{5, 4, 3, 2, 1, 0}`, assert each new
|
||||
// number inserted is the minimum `first()` value.
|
||||
for key in (0..6).rev() {
|
||||
table.put(&key, &123).unwrap();
|
||||
let (first, _) = table.first().unwrap();
|
||||
assert_eq!(first, key);
|
||||
}
|
||||
|
||||
drop(table);
|
||||
TxRw::commit(tx_rw).unwrap();
|
||||
let tx_rw = env_inner.tx_rw().unwrap();
|
||||
|
||||
// Assert iterators are ordered.
|
||||
{
|
||||
let tx_ro = env_inner.tx_ro().unwrap();
|
||||
let table = env_inner.open_db_ro::<TestTable>(&tx_ro).unwrap();
|
||||
let iter = table.iter().unwrap();
|
||||
let keys = table.keys().unwrap();
|
||||
for ((i, iter), key) in (0..6).zip(iter).zip(keys) {
|
||||
let (iter, _) = iter.unwrap();
|
||||
let key = key.unwrap();
|
||||
assert_eq!(i, iter);
|
||||
assert_eq!(iter, key);
|
||||
}
|
||||
}
|
||||
|
||||
let mut table = env_inner.open_db_rw::<TestTable>(&tx_rw).unwrap();
|
||||
|
||||
// Assert the `first()` values are the minimum, i.e. `{0, 1, 2}`
|
||||
for key in 0..3 {
|
||||
let (first, _) = table.first().unwrap();
|
||||
assert_eq!(first, key);
|
||||
table.delete(&key).unwrap();
|
||||
}
|
||||
|
||||
// Assert the `last()` values are the maximum, i.e. `{5, 4, 3}`
|
||||
for key in (3..6).rev() {
|
||||
let (last, _) = table.last().unwrap();
|
||||
assert_eq!(last, key);
|
||||
table.delete(&key).unwrap();
|
||||
}
|
||||
}
|
||||
31
storage/database/src/config/backend.rs
Normal file
31
storage/database/src/config/backend.rs
Normal file
|
|
@ -0,0 +1,31 @@
|
|||
//! SOMEDAY
|
||||
|
||||
//---------------------------------------------------------------------------------------------------- Import
|
||||
use std::{
|
||||
borrow::Cow,
|
||||
num::NonZeroUsize,
|
||||
path::{Path, PathBuf},
|
||||
};
|
||||
|
||||
#[cfg(feature = "serde")]
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use cuprate_helper::fs::database_dir;
|
||||
|
||||
use crate::{
|
||||
config::{ReaderThreads, SyncMode},
|
||||
constants::DATABASE_DATA_FILENAME,
|
||||
resize::ResizeAlgorithm,
|
||||
};
|
||||
|
||||
//---------------------------------------------------------------------------------------------------- Backend
|
||||
/// SOMEDAY: allow runtime hot-swappable backends.
|
||||
#[derive(Copy, Clone, Debug, Default, PartialEq, PartialOrd, Eq, Ord, Hash)]
|
||||
#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
|
||||
pub enum Backend {
|
||||
#[default]
|
||||
/// SOMEDAY
|
||||
Heed,
|
||||
/// SOMEDAY
|
||||
Redb,
|
||||
}
|
||||
210
storage/database/src/config/config.rs
Normal file
210
storage/database/src/config/config.rs
Normal file
|
|
@ -0,0 +1,210 @@
|
|||
//! The main [`Config`] struct, holding all configurable values.
|
||||
|
||||
//---------------------------------------------------------------------------------------------------- Import
|
||||
use std::{borrow::Cow, num::NonZeroUsize, path::Path};
|
||||
|
||||
#[cfg(feature = "serde")]
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::{config::SyncMode, constants::DATABASE_DATA_FILENAME, resize::ResizeAlgorithm};
|
||||
|
||||
//---------------------------------------------------------------------------------------------------- Constants
|
||||
/// Default value for [`Config::reader_threads`].
|
||||
///
|
||||
/// ```rust
|
||||
/// use cuprate_database::config::*;
|
||||
/// assert_eq!(READER_THREADS_DEFAULT.get(), 126);
|
||||
/// ```
|
||||
pub const READER_THREADS_DEFAULT: NonZeroUsize = match NonZeroUsize::new(126) {
|
||||
Some(n) => n,
|
||||
None => unreachable!(),
|
||||
};
|
||||
|
||||
//---------------------------------------------------------------------------------------------------- ConfigBuilder
|
||||
/// Builder for [`Config`].
|
||||
///
|
||||
// SOMEDAY: there's are many more options to add in the future.
|
||||
#[derive(Debug, Clone, PartialEq, PartialOrd)]
|
||||
#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
|
||||
pub struct ConfigBuilder {
|
||||
/// [`Config::db_directory`].
|
||||
db_directory: Cow<'static, Path>,
|
||||
|
||||
/// [`Config::sync_mode`].
|
||||
sync_mode: Option<SyncMode>,
|
||||
|
||||
/// [`Config::reader_threads`].
|
||||
reader_threads: Option<NonZeroUsize>,
|
||||
|
||||
/// [`Config::resize_algorithm`].
|
||||
resize_algorithm: Option<ResizeAlgorithm>,
|
||||
}
|
||||
|
||||
impl ConfigBuilder {
|
||||
/// Create a new [`ConfigBuilder`].
|
||||
///
|
||||
/// [`ConfigBuilder::build`] can be called immediately
|
||||
/// after this function to use default values.
|
||||
pub const fn new(db_directory: Cow<'static, Path>) -> Self {
|
||||
Self {
|
||||
db_directory,
|
||||
sync_mode: None,
|
||||
reader_threads: Some(READER_THREADS_DEFAULT),
|
||||
resize_algorithm: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Build into a [`Config`].
|
||||
///
|
||||
/// # Default values
|
||||
/// - [`READER_THREADS_DEFAULT`] is used for [`Config::reader_threads`]
|
||||
/// - [`Default::default`] is used for all other values (except the `db_directory`)
|
||||
pub fn build(self) -> Config {
|
||||
// Add the database filename to the directory.
|
||||
let db_file = {
|
||||
let mut db_file = self.db_directory.to_path_buf();
|
||||
db_file.push(DATABASE_DATA_FILENAME);
|
||||
Cow::Owned(db_file)
|
||||
};
|
||||
|
||||
Config {
|
||||
db_directory: self.db_directory,
|
||||
db_file,
|
||||
sync_mode: self.sync_mode.unwrap_or_default(),
|
||||
reader_threads: self.reader_threads.unwrap_or(READER_THREADS_DEFAULT),
|
||||
resize_algorithm: self.resize_algorithm.unwrap_or_default(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Set a custom database directory (and file) [`Path`].
|
||||
#[must_use]
|
||||
pub fn db_directory(mut self, db_directory: Cow<'static, Path>) -> Self {
|
||||
self.db_directory = db_directory;
|
||||
self
|
||||
}
|
||||
|
||||
/// Tune the [`ConfigBuilder`] for the highest performing,
|
||||
/// but also most resource-intensive & maybe risky settings.
|
||||
///
|
||||
/// Good default for testing, and resource-available machines.
|
||||
#[must_use]
|
||||
pub fn fast(mut self) -> Self {
|
||||
self.sync_mode = Some(SyncMode::Fast);
|
||||
self.resize_algorithm = Some(ResizeAlgorithm::default());
|
||||
self
|
||||
}
|
||||
|
||||
/// Tune the [`ConfigBuilder`] for the lowest performing,
|
||||
/// but also least resource-intensive settings.
|
||||
///
|
||||
/// Good default for resource-limited machines, e.g. a cheap VPS.
|
||||
#[must_use]
|
||||
pub fn low_power(mut self) -> Self {
|
||||
self.sync_mode = Some(SyncMode::default());
|
||||
self.resize_algorithm = Some(ResizeAlgorithm::default());
|
||||
self
|
||||
}
|
||||
|
||||
/// Set a custom [`SyncMode`].
|
||||
#[must_use]
|
||||
pub const fn sync_mode(mut self, sync_mode: SyncMode) -> Self {
|
||||
self.sync_mode = Some(sync_mode);
|
||||
self
|
||||
}
|
||||
|
||||
/// Set a custom [`Config::reader_threads`].
|
||||
#[must_use]
|
||||
pub const fn reader_threads(mut self, reader_threads: NonZeroUsize) -> Self {
|
||||
self.reader_threads = Some(reader_threads);
|
||||
self
|
||||
}
|
||||
|
||||
/// Set a custom [`ResizeAlgorithm`].
|
||||
#[must_use]
|
||||
pub const fn resize_algorithm(mut self, resize_algorithm: ResizeAlgorithm) -> Self {
|
||||
self.resize_algorithm = Some(resize_algorithm);
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
//---------------------------------------------------------------------------------------------------- Config
|
||||
/// Database [`Env`](crate::Env) configuration.
|
||||
///
|
||||
/// This is the struct passed to [`Env::open`](crate::Env::open) that
|
||||
/// allows the database to be configured in various ways.
|
||||
///
|
||||
/// For construction, use [`ConfigBuilder`].
|
||||
///
|
||||
// SOMEDAY: there's are many more options to add in the future.
|
||||
#[derive(Debug, Clone, PartialEq, PartialOrd)]
|
||||
#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
|
||||
pub struct Config {
|
||||
//------------------------ Database PATHs
|
||||
// These are private since we don't want
|
||||
// users messing with them after construction.
|
||||
/// The directory used to store all database files.
|
||||
///
|
||||
// SOMEDAY: we should also support `/etc/cuprated.conf`.
|
||||
// This could be represented with an `enum DbPath { Default, Custom, Etc, }`
|
||||
pub(crate) db_directory: Cow<'static, Path>,
|
||||
/// The actual database data file.
|
||||
///
|
||||
/// This is private, and created from the above `db_directory`.
|
||||
pub(crate) db_file: Cow<'static, Path>,
|
||||
|
||||
/// Disk synchronization mode.
|
||||
pub sync_mode: SyncMode,
|
||||
|
||||
/// Database reader thread count.
|
||||
///
|
||||
/// Set the number of slots in the reader table.
|
||||
///
|
||||
/// This is only used in LMDB, see
|
||||
/// <https://github.com/LMDB/lmdb/blob/b8e54b4c31378932b69f1298972de54a565185b1/libraries/liblmdb/mdb.c#L794-L799>.
|
||||
///
|
||||
/// By default, this value is [`READER_THREADS_DEFAULT`].
|
||||
pub reader_threads: NonZeroUsize,
|
||||
|
||||
/// Database memory map resizing algorithm.
|
||||
///
|
||||
/// This is used as the default fallback, but
|
||||
/// custom algorithms can be used as well with
|
||||
/// [`Env::resize_map`](crate::Env::resize_map).
|
||||
pub resize_algorithm: ResizeAlgorithm,
|
||||
}
|
||||
|
||||
impl Config {
|
||||
/// Create a new [`Config`] with sane default settings.
|
||||
///
|
||||
/// The [`Config::db_directory`] must be passed.
|
||||
///
|
||||
/// All other values will be [`Default::default`].
|
||||
///
|
||||
/// ```rust
|
||||
/// use cuprate_database::{config::*, resize::*, DATABASE_DATA_FILENAME};
|
||||
///
|
||||
/// let tmp_dir = tempfile::tempdir().unwrap();
|
||||
/// let db_directory = tmp_dir.path().to_owned();
|
||||
/// let config = Config::new(db_directory.clone().into());
|
||||
///
|
||||
/// assert_eq!(*config.db_directory(), db_directory);
|
||||
/// assert!(config.db_file().starts_with(db_directory));
|
||||
/// assert!(config.db_file().ends_with(DATABASE_DATA_FILENAME));
|
||||
/// assert_eq!(config.sync_mode, SyncMode::default());
|
||||
/// assert_eq!(config.reader_threads, READER_THREADS_DEFAULT);
|
||||
/// assert_eq!(config.resize_algorithm, ResizeAlgorithm::default());
|
||||
/// ```
|
||||
pub fn new(db_directory: Cow<'static, Path>) -> Self {
|
||||
ConfigBuilder::new(db_directory).build()
|
||||
}
|
||||
|
||||
/// Return the absolute [`Path`] to the database directory.
|
||||
pub const fn db_directory(&self) -> &Cow<'_, Path> {
|
||||
&self.db_directory
|
||||
}
|
||||
|
||||
/// Return the absolute [`Path`] to the database data file.
|
||||
pub const fn db_file(&self) -> &Cow<'_, Path> {
|
||||
&self.db_file
|
||||
}
|
||||
}
|
||||
40
storage/database/src/config/mod.rs
Normal file
40
storage/database/src/config/mod.rs
Normal file
|
|
@ -0,0 +1,40 @@
|
|||
//! Database [`Env`](crate::Env) configuration.
|
||||
//!
|
||||
//! This module contains the main [`Config`]uration struct
|
||||
//! for the database [`Env`](crate::Env)ironment, and types
|
||||
//! related to configuration settings.
|
||||
//!
|
||||
//! The main constructor is the [`ConfigBuilder`].
|
||||
//!
|
||||
//! These configurations are processed at runtime, meaning
|
||||
//! the `Env` can/will dynamically adjust its behavior
|
||||
//! based on these values.
|
||||
//!
|
||||
//! # Example
|
||||
//! ```rust
|
||||
//! use cuprate_database::{
|
||||
//! ConcreteEnv, Env,
|
||||
//! config::{ConfigBuilder, SyncMode}
|
||||
//! };
|
||||
//!
|
||||
//! # fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
//! let db_dir = tempfile::tempdir()?;
|
||||
//!
|
||||
//! let config = ConfigBuilder::new(db_dir.path().to_path_buf().into())
|
||||
//! // Use the fastest sync mode.
|
||||
//! .sync_mode(SyncMode::Fast)
|
||||
//! // Build into `Config`
|
||||
//! .build();
|
||||
//!
|
||||
//! // Open the database using this configuration.
|
||||
//! let env = ConcreteEnv::open(config.clone())?;
|
||||
//! // It's using the config we provided.
|
||||
//! assert_eq!(env.config(), &config);
|
||||
//! # Ok(()) }
|
||||
//! ```
|
||||
|
||||
mod config;
|
||||
pub use config::{Config, ConfigBuilder, READER_THREADS_DEFAULT};
|
||||
|
||||
mod sync_mode;
|
||||
pub use sync_mode::SyncMode;
|
||||
135
storage/database/src/config/sync_mode.rs
Normal file
135
storage/database/src/config/sync_mode.rs
Normal file
|
|
@ -0,0 +1,135 @@
|
|||
//! Database [`Env`](crate::Env) configuration.
|
||||
//!
|
||||
//! This module contains the main [`Config`]uration struct
|
||||
//! for the database [`Env`](crate::Env)ironment, and data
|
||||
//! structures related to any configuration setting.
|
||||
//!
|
||||
//! These configurations are processed at runtime, meaning
|
||||
//! the `Env` can/will dynamically adjust its behavior
|
||||
//! based on these values.
|
||||
|
||||
//---------------------------------------------------------------------------------------------------- Import
|
||||
|
||||
#[cfg(feature = "serde")]
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
//---------------------------------------------------------------------------------------------------- SyncMode
|
||||
/// Disk synchronization mode.
|
||||
///
|
||||
/// This controls how/when the database syncs its data to disk.
|
||||
///
|
||||
/// Regardless of the variant chosen, dropping [`Env`](crate::Env)
|
||||
/// will always cause it to fully sync to disk.
|
||||
///
|
||||
/// # Sync vs Async
|
||||
/// All invariants except [`SyncMode::Async`] & [`SyncMode::Fast`]
|
||||
/// are `synchronous`, as in the database will wait until the OS has
|
||||
/// finished syncing all the data to disk before continuing.
|
||||
///
|
||||
/// `SyncMode::Async` & `SyncMode::Fast` are `asynchronous`, meaning
|
||||
/// the database will _NOT_ wait until the data is fully synced to disk
|
||||
/// before continuing. Note that this doesn't mean the database itself
|
||||
/// won't be synchronized between readers/writers, but rather that the
|
||||
/// data _on disk_ may not be immediately synchronized after a write.
|
||||
///
|
||||
/// Something like:
|
||||
/// ```rust,ignore
|
||||
/// db.put("key", value);
|
||||
/// db.get("key");
|
||||
/// ```
|
||||
/// will be fine, most likely pulling from memory instead of disk.
|
||||
///
|
||||
/// # SOMEDAY
|
||||
/// Dynamic sync's are not yet supported.
|
||||
///
|
||||
/// Only:
|
||||
///
|
||||
/// - [`SyncMode::Safe`]
|
||||
/// - [`SyncMode::Async`]
|
||||
/// - [`SyncMode::Fast`]
|
||||
///
|
||||
/// are supported, all other variants will panic on [`crate::Env::open`].
|
||||
#[derive(Copy, Clone, Debug, Default, PartialEq, PartialOrd, Eq, Ord, Hash)]
|
||||
#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
|
||||
pub enum SyncMode {
|
||||
/// Use [`SyncMode::Fast`] until fully synced,
|
||||
/// then use [`SyncMode::Safe`].
|
||||
///
|
||||
// # SOMEDAY: how to implement this?
|
||||
// ref: <https://github.com/monero-project/monero/issues/1463>
|
||||
// monerod-solution: <https://github.com/monero-project/monero/pull/1506>
|
||||
// cuprate-issue: <https://github.com/Cuprate/cuprate/issues/78>
|
||||
//
|
||||
// We could:
|
||||
// ```rust,ignore
|
||||
// if current_db_block <= top_block.saturating_sub(N) {
|
||||
// // don't sync()
|
||||
// } else {
|
||||
// // sync()
|
||||
// }
|
||||
// ```
|
||||
// where N is some threshold we pick that is _close_ enough
|
||||
// to being synced where we want to start being safer.
|
||||
//
|
||||
// Essentially, when we are in a certain % range of being finished,
|
||||
// switch to safe mode, until then, go fast.
|
||||
FastThenSafe,
|
||||
|
||||
#[default]
|
||||
/// Fully sync to disk per transaction.
|
||||
///
|
||||
/// Every database transaction commit will
|
||||
/// fully sync all data to disk, _synchronously_,
|
||||
/// so the database (writer) halts until synced.
|
||||
///
|
||||
/// This is expected to be very slow.
|
||||
///
|
||||
/// This matches:
|
||||
/// - LMDB without any special sync flags
|
||||
/// - [`redb::Durability::Immediate`](https://docs.rs/redb/1.5.0/redb/enum.Durability.html#variant.Immediate)
|
||||
Safe,
|
||||
|
||||
/// Asynchrously sync to disk per transaction.
|
||||
///
|
||||
/// This is the same as [`SyncMode::Safe`],
|
||||
/// but the syncs will be asynchronous, i.e.
|
||||
/// each transaction commit will sync to disk,
|
||||
/// but only eventually, not necessarily immediately.
|
||||
///
|
||||
/// This matches:
|
||||
/// - [`MDB_MAPASYNC`](http://www.lmdb.tech/doc/group__mdb__env.html#gab034ed0d8e5938090aef5ee0997f7e94)
|
||||
/// - [`redb::Durability::Eventual`](https://docs.rs/redb/1.5.0/redb/enum.Durability.html#variant.Eventual)
|
||||
Async,
|
||||
|
||||
/// Fully sync to disk after we cross this transaction threshold.
|
||||
///
|
||||
/// After committing [`usize`] amount of database
|
||||
/// transactions, it will be sync to disk.
|
||||
///
|
||||
/// `0` behaves the same as [`SyncMode::Safe`], and a ridiculously large
|
||||
/// number like `usize::MAX` is practically the same as [`SyncMode::Fast`].
|
||||
Threshold(usize),
|
||||
|
||||
/// Only flush at database shutdown.
|
||||
///
|
||||
/// This is the fastest, yet unsafest option.
|
||||
///
|
||||
/// It will cause the database to never _actively_ sync,
|
||||
/// letting the OS decide when to flush data to disk.
|
||||
///
|
||||
/// This matches:
|
||||
/// - [`MDB_NOSYNC`](http://www.lmdb.tech/doc/group__mdb__env.html#ga5791dd1adb09123f82dd1f331209e12e) + [`MDB_MAPASYNC`](http://www.lmdb.tech/doc/group__mdb__env.html#gab034ed0d8e5938090aef5ee0997f7e94)
|
||||
/// - [`redb::Durability::None`](https://docs.rs/redb/1.5.0/redb/enum.Durability.html#variant.None)
|
||||
///
|
||||
/// `monerod` reference: <https://github.com/monero-project/monero/blob/7b7958bbd9d76375c47dc418b4adabba0f0b1785/src/blockchain_db/lmdb/db_lmdb.cpp#L1380-L1381>
|
||||
///
|
||||
/// # Corruption
|
||||
/// In the case of a system crash, the database
|
||||
/// may become corrupted when using this option.
|
||||
//
|
||||
// FIXME: we could call this `unsafe`
|
||||
// and use that terminology in the config file
|
||||
// so users know exactly what they are getting
|
||||
// themselves into.
|
||||
Fast,
|
||||
}
|
||||
74
storage/database/src/constants.rs
Normal file
74
storage/database/src/constants.rs
Normal file
|
|
@ -0,0 +1,74 @@
|
|||
//! General constants used throughout `cuprate-blockchain`.
|
||||
|
||||
//---------------------------------------------------------------------------------------------------- Import
|
||||
use cfg_if::cfg_if;
|
||||
|
||||
//---------------------------------------------------------------------------------------------------- Error Messages
|
||||
/// Corrupt database error message.
|
||||
///
|
||||
/// The error message shown to end-users in panic
|
||||
/// messages if we think the database is corrupted.
|
||||
///
|
||||
/// This is meant to be user-friendly.
|
||||
pub const DATABASE_CORRUPT_MSG: &str = r"Cuprate has encountered a fatal error. The database may be corrupted.
|
||||
|
||||
TODO: instructions on:
|
||||
1. What to do
|
||||
2. How to fix (re-sync, recover, etc)
|
||||
3. General advice for preventing corruption
|
||||
4. etc";
|
||||
|
||||
//---------------------------------------------------------------------------------------------------- Misc
|
||||
/// Static string of the `crate` being used as the database backend.
|
||||
///
|
||||
/// | Backend | Value |
|
||||
/// |---------|-------|
|
||||
/// | `heed` | `"heed"`
|
||||
/// | `redb` | `"redb"`
|
||||
pub const DATABASE_BACKEND: &str = {
|
||||
cfg_if! {
|
||||
if #[cfg(all(feature = "redb", not(feature = "heed")))] {
|
||||
"redb"
|
||||
} else {
|
||||
"heed"
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
/// Cuprate's database filename.
|
||||
///
|
||||
/// Used in [`Config::db_file`](crate::config::Config::db_file).
|
||||
///
|
||||
/// | Backend | Value |
|
||||
/// |---------|-------|
|
||||
/// | `heed` | `"data.mdb"`
|
||||
/// | `redb` | `"data.redb"`
|
||||
pub const DATABASE_DATA_FILENAME: &str = {
|
||||
cfg_if! {
|
||||
if #[cfg(all(feature = "redb", not(feature = "heed")))] {
|
||||
"data.redb"
|
||||
} else {
|
||||
"data.mdb"
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
/// Cuprate's database lock filename.
|
||||
///
|
||||
/// | Backend | Value |
|
||||
/// |---------|-------|
|
||||
/// | `heed` | `Some("lock.mdb")`
|
||||
/// | `redb` | `None` (redb doesn't use a file lock)
|
||||
pub const DATABASE_LOCK_FILENAME: Option<&str> = {
|
||||
cfg_if! {
|
||||
if #[cfg(all(feature = "redb", not(feature = "heed")))] {
|
||||
None
|
||||
} else {
|
||||
Some("lock.mdb")
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
//---------------------------------------------------------------------------------------------------- Tests
|
||||
#[cfg(test)]
|
||||
mod test {}
|
||||
|
|
@ -9,7 +9,6 @@ use crate::{
|
|||
error::{InitError, RuntimeError},
|
||||
resize::ResizeAlgorithm,
|
||||
table::Table,
|
||||
tables::{call_fn_on_all_tables_or_early_return, TablesIter, TablesMut},
|
||||
transaction::{TxRo, TxRw},
|
||||
};
|
||||
|
||||
|
|
@ -81,13 +80,13 @@ pub trait Env: Sized {
|
|||
/// Open the database environment, using the passed [`Config`].
|
||||
///
|
||||
/// # Invariants
|
||||
/// This function **must** create all tables listed in [`crate::tables`].
|
||||
/// This function does not create any tables.
|
||||
///
|
||||
/// The rest of the functions depend on the fact
|
||||
/// they already exist, or else they will panic.
|
||||
/// You must create all possible tables with [`EnvInner::create_db`]
|
||||
/// before attempting to open any.
|
||||
///
|
||||
/// # Errors
|
||||
/// This will error if the database could not be opened.
|
||||
/// This will error if the database file could not be opened.
|
||||
///
|
||||
/// This is the only [`Env`] function that will return
|
||||
/// an [`InitError`] instead of a [`RuntimeError`].
|
||||
|
|
@ -180,10 +179,14 @@ pub trait Env: Sized {
|
|||
macro_rules! doc_table_error {
|
||||
() => {
|
||||
r"# Errors
|
||||
This will only return [`RuntimeError::Io`] if it errors.
|
||||
This will only return [`RuntimeError::Io`] on normal errors.
|
||||
|
||||
As all tables are created upon [`Env::open`],
|
||||
this function will never error because a table doesn't exist."
|
||||
If the specified table is not created upon before this function is called,
|
||||
this will return an error.
|
||||
|
||||
Implementation detail you should NOT rely on:
|
||||
- This only panics on `heed`
|
||||
- `redb` will create the table if it does not exist"
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -196,6 +199,12 @@ this function will never error because a table doesn't exist."
|
|||
/// As noted in `Env::env_inner`, this is a `RwLockReadGuard`
|
||||
/// when using the `heed` backend, be aware of this and do
|
||||
/// not hold onto an `EnvInner` for a long time.
|
||||
///
|
||||
/// # Tables
|
||||
/// Note that when opening tables with [`EnvInner::open_db_ro`],
|
||||
/// they must be created first or else it will return error.
|
||||
///
|
||||
/// See [`EnvInner::open_db_rw`] and [`EnvInner::create_db`] for creating tables.
|
||||
pub trait EnvInner<'env, Ro, Rw>
|
||||
where
|
||||
Self: 'env,
|
||||
|
|
@ -229,7 +238,11 @@ where
|
|||
/// // (name, key/value type)
|
||||
/// ```
|
||||
///
|
||||
#[doc = doc_table_error!()]
|
||||
/// # Errors
|
||||
/// This will only return [`RuntimeError::Io`] on normal errors.
|
||||
///
|
||||
/// If the specified table is not created upon before this function is called,
|
||||
/// this will return [`RuntimeError::TableNotFound`].
|
||||
fn open_db_ro<T: Table>(
|
||||
&self,
|
||||
tx_ro: &Ro,
|
||||
|
|
@ -246,32 +259,22 @@ where
|
|||
/// This will open the database [`Table`]
|
||||
/// passed as a generic to this function.
|
||||
///
|
||||
#[doc = doc_table_error!()]
|
||||
/// # Errors
|
||||
/// This will only return [`RuntimeError::Io`] on errors.
|
||||
///
|
||||
/// Implementation details: Both `heed` & `redb` backends create
|
||||
/// the table with this function if it does not already exist. For safety and
|
||||
/// clear intent, you should still consider using [`EnvInner::create_db`] instead.
|
||||
fn open_db_rw<T: Table>(&self, tx_rw: &Rw) -> Result<impl DatabaseRw<T>, RuntimeError>;
|
||||
|
||||
/// Open all tables in read/iter mode.
|
||||
/// Create a database table.
|
||||
///
|
||||
/// This calls [`EnvInner::open_db_ro`] on all database tables
|
||||
/// and returns a structure that allows access to all tables.
|
||||
/// This will create the database [`Table`]
|
||||
/// passed as a generic to this function.
|
||||
///
|
||||
#[doc = doc_table_error!()]
|
||||
fn open_tables(&self, tx_ro: &Ro) -> Result<impl TablesIter, RuntimeError> {
|
||||
call_fn_on_all_tables_or_early_return! {
|
||||
Self::open_db_ro(self, tx_ro)
|
||||
}
|
||||
}
|
||||
|
||||
/// Open all tables in read-write mode.
|
||||
///
|
||||
/// This calls [`EnvInner::open_db_rw`] on all database tables
|
||||
/// and returns a structure that allows access to all tables.
|
||||
///
|
||||
#[doc = doc_table_error!()]
|
||||
fn open_tables_mut(&self, tx_rw: &Rw) -> Result<impl TablesMut, RuntimeError> {
|
||||
call_fn_on_all_tables_or_early_return! {
|
||||
Self::open_db_rw(self, tx_rw)
|
||||
}
|
||||
}
|
||||
/// # Errors
|
||||
/// This will only return [`RuntimeError::Io`] on errors.
|
||||
fn create_db<T: Table>(&self, tx_rw: &Rw) -> Result<(), RuntimeError>;
|
||||
|
||||
/// Clear all `(key, value)`'s from a database table.
|
||||
///
|
||||
|
|
@ -66,7 +66,7 @@ pub enum InitError {
|
|||
/// 2. (De)serialization
|
||||
/// 3. Shutdown errors
|
||||
///
|
||||
/// as `cuprate_blockchain` upholds the invariant that:
|
||||
/// as `cuprate_database` upholds the invariant that:
|
||||
///
|
||||
/// 1. All tables exist
|
||||
/// 2. (De)serialization never fails
|
||||
|
|
@ -88,6 +88,10 @@ pub enum RuntimeError {
|
|||
#[error("database memory map must be resized")]
|
||||
ResizeNeeded,
|
||||
|
||||
/// The given table did not exist in the database.
|
||||
#[error("database table did not exist")]
|
||||
TableNotFound,
|
||||
|
||||
/// A [`std::io::Error`].
|
||||
#[error("I/O error: {0}")]
|
||||
Io(#[from] std::io::Error),
|
||||
|
|
@ -23,7 +23,7 @@ pub trait Key: Storable + Sized {
|
|||
/// not a comparison of the key's value.
|
||||
///
|
||||
/// ```rust
|
||||
/// # use cuprate_blockchain::*;
|
||||
/// # use cuprate_database::*;
|
||||
/// assert_eq!(
|
||||
/// <u64 as Key>::compare([0].as_slice(), [1].as_slice()),
|
||||
/// std::cmp::Ordering::Less,
|
||||
|
|
@ -1 +1,152 @@
|
|||
#![doc = include_str!("../README.md")]
|
||||
//---------------------------------------------------------------------------------------------------- Lints
|
||||
// Forbid lints.
|
||||
// Our code, and code generated (e.g macros) cannot overrule these.
|
||||
#![forbid(
|
||||
// `unsafe` is allowed but it _must_ be
|
||||
// commented with `SAFETY: reason`.
|
||||
clippy::undocumented_unsafe_blocks,
|
||||
|
||||
// Never.
|
||||
unused_unsafe,
|
||||
redundant_semicolons,
|
||||
unused_allocation,
|
||||
coherence_leak_check,
|
||||
while_true,
|
||||
clippy::missing_docs_in_private_items,
|
||||
|
||||
// Maybe can be put into `#[deny]`.
|
||||
unconditional_recursion,
|
||||
for_loops_over_fallibles,
|
||||
unused_braces,
|
||||
unused_labels,
|
||||
keyword_idents,
|
||||
non_ascii_idents,
|
||||
variant_size_differences,
|
||||
single_use_lifetimes,
|
||||
|
||||
// Probably can be put into `#[deny]`.
|
||||
future_incompatible,
|
||||
let_underscore,
|
||||
break_with_label_and_loop,
|
||||
duplicate_macro_attributes,
|
||||
exported_private_dependencies,
|
||||
large_assignments,
|
||||
overlapping_range_endpoints,
|
||||
semicolon_in_expressions_from_macros,
|
||||
noop_method_call,
|
||||
unreachable_pub,
|
||||
)]
|
||||
// Deny lints.
|
||||
// Some of these are `#[allow]`'ed on a per-case basis.
|
||||
#![deny(
|
||||
clippy::all,
|
||||
clippy::correctness,
|
||||
clippy::suspicious,
|
||||
clippy::style,
|
||||
clippy::complexity,
|
||||
clippy::perf,
|
||||
clippy::pedantic,
|
||||
clippy::nursery,
|
||||
clippy::cargo,
|
||||
unused_crate_dependencies,
|
||||
unused_doc_comments,
|
||||
unused_mut,
|
||||
missing_docs,
|
||||
deprecated,
|
||||
unused_comparisons,
|
||||
nonstandard_style
|
||||
)]
|
||||
#![allow(
|
||||
// FIXME: this lint affects crates outside of
|
||||
// `database/` for some reason, allow for now.
|
||||
clippy::cargo_common_metadata,
|
||||
|
||||
// FIXME: adding `#[must_use]` onto everything
|
||||
// might just be more annoying than useful...
|
||||
// although it is sometimes nice.
|
||||
clippy::must_use_candidate,
|
||||
|
||||
// FIXME: good lint but too many false positives
|
||||
// with our `Env` + `RwLock` setup.
|
||||
clippy::significant_drop_tightening,
|
||||
|
||||
// FIXME: good lint but is less clear in most cases.
|
||||
clippy::items_after_statements,
|
||||
|
||||
clippy::module_name_repetitions,
|
||||
clippy::module_inception,
|
||||
clippy::redundant_pub_crate,
|
||||
clippy::option_if_let_else,
|
||||
)]
|
||||
// Allow some lints when running in debug mode.
|
||||
#![cfg_attr(
|
||||
debug_assertions,
|
||||
allow(
|
||||
clippy::todo,
|
||||
clippy::multiple_crate_versions,
|
||||
// unused_crate_dependencies,
|
||||
)
|
||||
)]
|
||||
// Allow some lints in tests.
|
||||
#![cfg_attr(
|
||||
test,
|
||||
allow(
|
||||
clippy::cognitive_complexity,
|
||||
clippy::needless_pass_by_value,
|
||||
clippy::cast_possible_truncation,
|
||||
clippy::too_many_lines
|
||||
)
|
||||
)]
|
||||
|
||||
//---------------------------------------------------------------------------------------------------- Public API
|
||||
// Import private modules, export public types.
|
||||
//
|
||||
// Documentation for each module is located in the respective file.
|
||||
|
||||
mod backend;
|
||||
pub use backend::ConcreteEnv;
|
||||
|
||||
pub mod config;
|
||||
|
||||
mod constants;
|
||||
pub use constants::{
|
||||
DATABASE_BACKEND, DATABASE_CORRUPT_MSG, DATABASE_DATA_FILENAME, DATABASE_LOCK_FILENAME,
|
||||
};
|
||||
|
||||
mod database;
|
||||
pub use database::{DatabaseIter, DatabaseRo, DatabaseRw};
|
||||
|
||||
mod env;
|
||||
pub use env::{Env, EnvInner};
|
||||
|
||||
mod error;
|
||||
pub use error::{InitError, RuntimeError};
|
||||
|
||||
pub mod resize;
|
||||
|
||||
mod key;
|
||||
pub use key::Key;
|
||||
|
||||
mod storable;
|
||||
pub use storable::{Storable, StorableBytes, StorableVec};
|
||||
|
||||
mod table;
|
||||
pub use table::Table;
|
||||
|
||||
mod transaction;
|
||||
pub use transaction::{TxRo, TxRw};
|
||||
|
||||
//---------------------------------------------------------------------------------------------------- Private
|
||||
#[cfg(test)]
|
||||
pub(crate) mod tests;
|
||||
|
||||
//----------------------------------------------------------------------------------------------------
|
||||
// HACK: needed to satisfy the `unused_crate_dependencies` lint.
|
||||
cfg_if::cfg_if! {
|
||||
if #[cfg(feature = "redb")] {
|
||||
use redb as _;
|
||||
} else {
|
||||
use heed as _;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -50,7 +50,7 @@ impl ResizeAlgorithm {
|
|||
/// Returns [`Self::Monero`].
|
||||
///
|
||||
/// ```rust
|
||||
/// # use cuprate_blockchain::resize::*;
|
||||
/// # use cuprate_database::resize::*;
|
||||
/// assert!(matches!(ResizeAlgorithm::new(), ResizeAlgorithm::Monero));
|
||||
/// ```
|
||||
#[inline]
|
||||
|
|
@ -75,7 +75,7 @@ impl Default for ResizeAlgorithm {
|
|||
/// Calls [`Self::new`].
|
||||
///
|
||||
/// ```rust
|
||||
/// # use cuprate_blockchain::resize::*;
|
||||
/// # use cuprate_database::resize::*;
|
||||
/// assert_eq!(ResizeAlgorithm::new(), ResizeAlgorithm::default());
|
||||
/// ```
|
||||
#[inline]
|
||||
|
|
@ -113,7 +113,7 @@ pub fn page_size() -> NonZeroUsize {
|
|||
/// [^2]: `1_073_745_920`
|
||||
///
|
||||
/// ```rust
|
||||
/// # use cuprate_blockchain::resize::*;
|
||||
/// # use cuprate_database::resize::*;
|
||||
/// // The value this function will increment by
|
||||
/// // (assuming page multiple of 4096).
|
||||
/// const N: usize = 1_073_741_824;
|
||||
|
|
@ -129,7 +129,7 @@ pub fn page_size() -> NonZeroUsize {
|
|||
/// This function will panic if adding onto `current_size_bytes` overflows [`usize::MAX`].
|
||||
///
|
||||
/// ```rust,should_panic
|
||||
/// # use cuprate_blockchain::resize::*;
|
||||
/// # use cuprate_database::resize::*;
|
||||
/// // Ridiculous large numbers panic.
|
||||
/// monero(usize::MAX);
|
||||
/// ```
|
||||
|
|
@ -166,7 +166,7 @@ pub fn monero(current_size_bytes: usize) -> NonZeroUsize {
|
|||
/// and then round up to nearest OS page size.
|
||||
///
|
||||
/// ```rust
|
||||
/// # use cuprate_blockchain::resize::*;
|
||||
/// # use cuprate_database::resize::*;
|
||||
/// let page_size: usize = page_size().get();
|
||||
///
|
||||
/// // Anything below the page size will round up to the page size.
|
||||
|
|
@ -185,7 +185,7 @@ pub fn monero(current_size_bytes: usize) -> NonZeroUsize {
|
|||
/// This function will panic if adding onto `current_size_bytes` overflows [`usize::MAX`].
|
||||
///
|
||||
/// ```rust,should_panic
|
||||
/// # use cuprate_blockchain::resize::*;
|
||||
/// # use cuprate_database::resize::*;
|
||||
/// // Ridiculous large numbers panic.
|
||||
/// fixed_bytes(1, usize::MAX);
|
||||
/// ```
|
||||
|
|
@ -221,7 +221,7 @@ pub fn fixed_bytes(current_size_bytes: usize, add_bytes: usize) -> NonZeroUsize
|
|||
/// (rounded up to the OS page size).
|
||||
///
|
||||
/// ```rust
|
||||
/// # use cuprate_blockchain::resize::*;
|
||||
/// # use cuprate_database::resize::*;
|
||||
/// let page_size: usize = page_size().get();
|
||||
///
|
||||
/// // Anything below the page size will round up to the page size.
|
||||
|
|
@ -247,7 +247,7 @@ pub fn fixed_bytes(current_size_bytes: usize, add_bytes: usize) -> NonZeroUsize
|
|||
/// is closer to [`usize::MAX`] than the OS page size.
|
||||
///
|
||||
/// ```rust,should_panic
|
||||
/// # use cuprate_blockchain::resize::*;
|
||||
/// # use cuprate_database::resize::*;
|
||||
/// // Ridiculous large numbers panic.
|
||||
/// percent(usize::MAX, 1.001);
|
||||
/// ```
|
||||
|
|
@ -22,14 +22,10 @@ use bytes::Bytes;
|
|||
///
|
||||
/// will automatically implement [`Storable`].
|
||||
///
|
||||
/// This includes:
|
||||
/// - Most primitive types
|
||||
/// - All types in [`tables`](crate::tables)
|
||||
///
|
||||
/// See [`StorableVec`] & [`StorableBytes`] for storing slices of `T: Storable`.
|
||||
///
|
||||
/// ```rust
|
||||
/// # use cuprate_blockchain::*;
|
||||
/// # use cuprate_database::*;
|
||||
/// # use std::borrow::*;
|
||||
/// let number: u64 = 0;
|
||||
///
|
||||
|
|
@ -77,7 +73,7 @@ pub trait Storable: Debug {
|
|||
///
|
||||
/// # Examples
|
||||
/// ```rust
|
||||
/// # use cuprate_blockchain::*;
|
||||
/// # use cuprate_database::*;
|
||||
/// assert_eq!(<()>::BYTE_LENGTH, Some(0));
|
||||
/// assert_eq!(u8::BYTE_LENGTH, Some(1));
|
||||
/// assert_eq!(u16::BYTE_LENGTH, Some(2));
|
||||
|
|
@ -99,7 +95,7 @@ pub trait Storable: Debug {
|
|||
///
|
||||
/// # Blanket implementation
|
||||
/// The blanket implementation that covers all types used
|
||||
/// by `cuprate_blockchain` will simply bitwise copy `bytes`
|
||||
/// by `cuprate_database` will simply bitwise copy `bytes`
|
||||
/// into `Self`.
|
||||
///
|
||||
/// The bytes do not have be correctly aligned.
|
||||
|
|
@ -136,7 +132,7 @@ where
|
|||
///
|
||||
/// # Example
|
||||
/// ```rust
|
||||
/// # use cuprate_blockchain::*;
|
||||
/// # use cuprate_database::*;
|
||||
/// //---------------------------------------------------- u8
|
||||
/// let vec: StorableVec<u8> = StorableVec(vec![0,1]);
|
||||
///
|
||||
|
|
@ -202,7 +198,7 @@ impl<T> Borrow<[T]> for StorableVec<T> {
|
|||
/// A [`Storable`] version of [`Bytes`].
|
||||
///
|
||||
/// ```rust
|
||||
/// # use cuprate_blockchain::*;
|
||||
/// # use cuprate_database::*;
|
||||
/// # use bytes::Bytes;
|
||||
/// let bytes: StorableBytes = StorableBytes(Bytes::from_static(&[0,1]));
|
||||
///
|
||||
|
|
@ -8,12 +8,7 @@ use crate::{key::Key, storable::Storable};
|
|||
/// Database table metadata.
|
||||
///
|
||||
/// Purely compile time information for database tables.
|
||||
///
|
||||
/// ## Sealed
|
||||
/// This trait is [`Sealed`](https://rust-lang.github.io/api-guidelines/future-proofing.html#sealed-traits-protect-against-downstream-implementations-c-sealed).
|
||||
///
|
||||
/// It is only implemented on the types inside [`tables`][crate::tables].
|
||||
pub trait Table: crate::tables::private::Sealed + 'static {
|
||||
pub trait Table: 'static {
|
||||
/// Name of the database table.
|
||||
const NAME: &'static str;
|
||||
|
||||
35
storage/database/src/tests.rs
Normal file
35
storage/database/src/tests.rs
Normal file
|
|
@ -0,0 +1,35 @@
|
|||
//! Utilities for `cuprate_database` testing.
|
||||
//!
|
||||
//! These types/fn's are only:
|
||||
//! - enabled on #[cfg(test)]
|
||||
//! - only used internally
|
||||
|
||||
//---------------------------------------------------------------------------------------------------- Import
|
||||
use std::borrow::Cow;
|
||||
|
||||
use crate::{config::ConfigBuilder, table::Table, ConcreteEnv, Env};
|
||||
|
||||
//---------------------------------------------------------------------------------------------------- struct
|
||||
/// A test table.
|
||||
pub(crate) struct TestTable;
|
||||
|
||||
impl Table for TestTable {
|
||||
const NAME: &'static str = "test_table";
|
||||
type Key = u8;
|
||||
type Value = u64;
|
||||
}
|
||||
|
||||
//---------------------------------------------------------------------------------------------------- fn
|
||||
/// Create an `Env` in a temporarily directory.
|
||||
/// The directory is automatically removed after the `TempDir` is dropped.
|
||||
///
|
||||
/// FIXME: changing this to `-> impl Env` causes lifetime errors...
|
||||
pub(crate) fn tmp_concrete_env() -> (ConcreteEnv, tempfile::TempDir) {
|
||||
let tempdir = tempfile::tempdir().unwrap();
|
||||
let config = ConfigBuilder::new(Cow::Owned(tempdir.path().into()))
|
||||
.low_power()
|
||||
.build();
|
||||
let env = ConcreteEnv::open(config).unwrap();
|
||||
|
||||
(env, tempdir)
|
||||
}
|
||||
Loading…
Reference in a new issue