From 97f433c6947c890af7bd9f9a944cb913439f135f Mon Sep 17 00:00:00 2001 From: Luke Parker Date: Sat, 9 Mar 2024 01:58:58 -0500 Subject: [PATCH] Redo how WAL/logs are limited by the DB Adds a patch to the latest rocksdb. --- Cargo.lock | 37 +++++++++++++++++++------------------ Cargo.toml | 3 +++ common/db/Cargo.toml | 2 +- common/db/src/rocks.rs | 21 +++++++++++++-------- patches/rocksdb/Cargo.toml | 26 ++++++++++++++++++++++++++ patches/rocksdb/src/lib.rs | 1 + 6 files changed, 63 insertions(+), 27 deletions(-) create mode 100644 patches/rocksdb/Cargo.toml create mode 100644 patches/rocksdb/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index dd2cc6f7..64516b0b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -445,17 +445,16 @@ dependencies = [ [[package]] name = "bindgen" -version = "0.65.1" +version = "0.69.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfdf7b466f9a4903edc73f95d6d2bcd5baf8ae620638762244d3f60143643cc5" +checksum = "a00dc851838a2120612785d195287475a3ac45514741da670b735818822129a0" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.4.2", "cexpr", "clang-sys", + "itertools", "lazy_static", "lazycell", - "peeking_take_while", - "prettyplease 0.2.16", "proc-macro2", "quote", "regex", @@ -2954,7 +2953,7 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite 0.2.13", - "socket2 0.4.10", + "socket2 0.5.5", "tokio", "tower-service", "tracing", @@ -3423,7 +3422,7 @@ dependencies = [ "num_cpus", "parking_lot 0.12.1", "regex", - "rocksdb", + "rocksdb 0.21.0", "smallvec", ] @@ -3938,9 +3937,9 @@ dependencies = [ [[package]] name = "librocksdb-sys" -version = "0.11.0+8.1.1" +version = "0.16.0+8.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3386f101bcb4bd252d8e9d2fb41ec3b0862a15a62b478c355b2982efa469e3e" +checksum = "ce3d60bc059831dc1c83903fb45c103f75db65c5a7bf22272764d9cc683e348c" dependencies = [ "bindgen", "bzip2-sys", @@ -3950,6 +3949,7 @@ dependencies = [ "libz-sys", "lz4-sys", "tikv-jemalloc-sys", + "zstd-sys", ] [[package]] @@ -4759,7 +4759,7 @@ version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "681030a937600a36906c185595136d26abfebb4aa9c65701cefcaf8578bb982b" dependencies = [ - "proc-macro-crate 1.3.1", + "proc-macro-crate 3.1.0", "proc-macro2", "quote", "syn 2.0.52", @@ -5179,12 +5179,6 @@ dependencies = [ "sha2", ] -[[package]] -name = "peeking_take_while" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" - [[package]] name = "pem" version = "1.1.1" @@ -5986,8 +5980,15 @@ dependencies = [ [[package]] name = "rocksdb" version = "0.21.0" +dependencies = [ + "rocksdb 0.22.0", +] + +[[package]] +name = "rocksdb" +version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb6f170a4041d50a0ce04b0d2e14916d6ca863ea2e422689a5b694395d299ffe" +checksum = "6bd13e55d6d7b8cd0ea569161127567cd587676c99f4472f779a0279aa60a7a7" dependencies = [ "libc", "librocksdb-sys", @@ -7468,7 +7469,7 @@ name = "serai-db" version = "0.1.0" dependencies = [ "parity-db", - "rocksdb", + "rocksdb 0.21.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 25601c46..bcc344ed 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,6 +3,7 @@ resolver = "2" members = [ # Version patches "patches/zstd", + "patches/rocksdb", "patches/proc-macro-crate", # std patches @@ -112,6 +113,8 @@ dockertest = { git = "https://github.com/kayabaNerve/dockertest-rs", branch = "a # wasmtime pulls in an old version for this zstd = { path = "patches/zstd" } +# Needed for WAL compression +rocksdb = { path = "patches/rocksdb" } # proc-macro-crate 2 binds to an old version of toml for msrv so we patch to 3 proc-macro-crate = { path = "patches/proc-macro-crate" } diff --git a/common/db/Cargo.toml b/common/db/Cargo.toml index 78d486a1..e422b346 100644 --- a/common/db/Cargo.toml +++ b/common/db/Cargo.toml @@ -18,7 +18,7 @@ workspace = true [dependencies] parity-db = { version = "0.4", default-features = false, optional = true } -rocksdb = { version = "0.21", default-features = false, features = ["lz4"], optional = true } +rocksdb = { version = "0.21", default-features = false, features = ["zstd"], optional = true } [features] parity-db = ["dep:parity-db"] diff --git a/common/db/src/rocks.rs b/common/db/src/rocks.rs index 01762151..046bbfd3 100644 --- a/common/db/src/rocks.rs +++ b/common/db/src/rocks.rs @@ -1,6 +1,8 @@ use std::sync::Arc; -use rocksdb::{DBCompressionType, ThreadMode, SingleThreaded, Options, Transaction, TransactionDB}; +use rocksdb::{ + DBCompressionType, ThreadMode, SingleThreaded, LogLevel, Options, Transaction, TransactionDB, +}; use crate::*; @@ -37,13 +39,16 @@ pub type RocksDB = Arc>; pub fn new_rocksdb(path: &str) -> RocksDB { let mut options = Options::default(); options.create_if_missing(true); - options.set_compression_type(DBCompressionType::Lz4); - options.set_wal_size_limit_mb(128); - // 1 GB - options.set_max_total_wal_size(1 << 30); + options.set_compression_type(DBCompressionType::Zstd); + // 128 MB - options.set_max_log_file_size(1 << 27); - options.set_recycle_log_file_num(5); - options.set_keep_log_file_num(5); + options.set_wal_compression_type(DBCompressionType::Zstd); + options.set_max_total_wal_size(128 * 1024 * 1024); + + // 1 MB + options.set_log_level(LogLevel::Warn); + options.set_max_log_file_size(1024 * 1024); + options.set_recycle_log_file_num(1); + Arc::new(TransactionDB::open(&options, &Default::default(), path).unwrap()) } diff --git a/patches/rocksdb/Cargo.toml b/patches/rocksdb/Cargo.toml new file mode 100644 index 00000000..d9967deb --- /dev/null +++ b/patches/rocksdb/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "rocksdb" +version = "0.21.0" +description = "rocksdb which patches to the latest update" +license = "MIT" +repository = "https://github.com/serai-dex/serai/tree/develop/patches/rocksdb" +authors = ["Luke Parker "] +keywords = [] +edition = "2021" +rust-version = "1.70" + +[package.metadata.docs.rs] +all-features = true +rustdoc-args = ["--cfg", "docsrs"] + +[dependencies] +rocksdb = { version = "0.22", default-features = false } + +[features] +jemalloc = ["rocksdb/jemalloc"] +snappy = ["rocksdb/snappy"] +lz4 = ["rocksdb/lz4"] +zstd = ["rocksdb/zstd"] +zlib = ["rocksdb/zlib"] +bzip2 = ["rocksdb/bzip2"] +default = ["snappy", "lz4", "zstd", "zlib", "bzip2"] diff --git a/patches/rocksdb/src/lib.rs b/patches/rocksdb/src/lib.rs new file mode 100644 index 00000000..bd209ce8 --- /dev/null +++ b/patches/rocksdb/src/lib.rs @@ -0,0 +1 @@ +pub use rocksdb::*;