Ecosystem/serai
Ecosystem/serai
1
0
Fork 0
mirror of https://github.com/serai-dex/serai.git synced 2024-11-16 17:07:35 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Redo how WAL/logs are limited by the DB

Browse source 
Adds a patch to the latest rocksdb.
This commit is contained in:
Luke Parker 2024-03-09 01:58:58 -05:00
parent 10f5ec51ca
commit 97f433c694
No known key found for this signature in database
6 changed files with 63 additions and 27 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

37
Cargo.lock generated
View file

@ -445,17 +445,16 @@ dependencies = [
[[package]]
name = "bindgen"
version = "0.65.1"
version = "0.69.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cfdf7b466f9a4903edc73f95d6d2bcd5baf8ae620638762244d3f60143643cc5"
checksum = "a00dc851838a2120612785d195287475a3ac45514741da670b735818822129a0"
dependencies = [
"bitflags 1.3.2",
"bitflags 2.4.2",
"cexpr",
"clang-sys",
"itertools",
"lazy_static",
"lazycell",
"peeking_take_while",
"prettyplease 0.2.16",
"proc-macro2",
"quote",
"regex",
@ -2954,7 +2953,7 @@ dependencies = [
"httpdate",
"itoa",
"pin-project-lite 0.2.13",
"socket2 0.4.10",
"socket2 0.5.5",
"tokio",
"tower-service",
"tracing",
@ -3423,7 +3422,7 @@ dependencies = [
"num_cpus",
"parking_lot 0.12.1",
"regex",
"rocksdb",
"rocksdb 0.21.0",
"smallvec",
]
@ -3938,9 +3937,9 @@ dependencies = [
[[package]]
name = "librocksdb-sys"
version = "0.11.0+8.1.1"
version = "0.16.0+8.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d3386f101bcb4bd252d8e9d2fb41ec3b0862a15a62b478c355b2982efa469e3e"
checksum = "ce3d60bc059831dc1c83903fb45c103f75db65c5a7bf22272764d9cc683e348c"
dependencies = [
"bindgen",
"bzip2-sys",
@ -3950,6 +3949,7 @@ dependencies = [
"libz-sys",
"lz4-sys",
"tikv-jemalloc-sys",
"zstd-sys",
]
[[package]]
@ -4759,7 +4759,7 @@ version = "0.7.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "681030a937600a36906c185595136d26abfebb4aa9c65701cefcaf8578bb982b"
dependencies = [
"proc-macro-crate 1.3.1",
"proc-macro-crate 3.1.0",
"proc-macro2",
"quote",
"syn 2.0.52",
@ -5179,12 +5179,6 @@ dependencies = [
"sha2",
]
[[package]]
name = "peeking_take_while"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099"
[[package]]
name = "pem"
version = "1.1.1"
@ -5986,8 +5980,15 @@ dependencies = [
[[package]]
name = "rocksdb"
version = "0.21.0"
dependencies = [
"rocksdb 0.22.0",
]
[[package]]
name = "rocksdb"
version = "0.22.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bb6f170a4041d50a0ce04b0d2e14916d6ca863ea2e422689a5b694395d299ffe"
checksum = "6bd13e55d6d7b8cd0ea569161127567cd587676c99f4472f779a0279aa60a7a7"
dependencies = [
"libc",
"librocksdb-sys",
@ -7468,7 +7469,7 @@ name = "serai-db"
version = "0.1.0"
dependencies = [
"parity-db",
"rocksdb",
"rocksdb 0.21.0",
]
[[package]]

3
Cargo.toml
View file

@ -3,6 +3,7 @@ resolver = "2"
members = [
# Version patches
"patches/zstd",
"patches/rocksdb",
"patches/proc-macro-crate",
# std patches
@ -112,6 +113,8 @@ dockertest = { git = "https://github.com/kayabaNerve/dockertest-rs", branch = "a
# wasmtime pulls in an old version for this
zstd = { path = "patches/zstd" }
# Needed for WAL compression
rocksdb = { path = "patches/rocksdb" }
# proc-macro-crate 2 binds to an old version of toml for msrv so we patch to 3
proc-macro-crate = { path = "patches/proc-macro-crate" }

2
common/db/Cargo.toml
View file

@ -18,7 +18,7 @@ workspace = true
[dependencies]
parity-db = { version = "0.4", default-features = false, optional = true }
rocksdb = { version = "0.21", default-features = false, features = ["lz4"], optional = true }
rocksdb = { version = "0.21", default-features = false, features = ["zstd"], optional = true }
[features]
parity-db = ["dep:parity-db"]

21
common/db/src/rocks.rs
View file

@ -1,6 +1,8 @@
use std::sync::Arc;
use rocksdb::{DBCompressionType, ThreadMode, SingleThreaded, Options, Transaction, TransactionDB};
use rocksdb::{
DBCompressionType, ThreadMode, SingleThreaded, LogLevel, Options, Transaction, TransactionDB,
};
use crate::*;
@ -37,13 +39,16 @@ pub type RocksDB = Arc<TransactionDB<SingleThreaded>>;
pub fn new_rocksdb(path: &str) -> RocksDB {
let mut options = Options::default();
options.create_if_missing(true);
options.set_compression_type(DBCompressionType::Lz4);
options.set_wal_size_limit_mb(128);
// 1 GB
options.set_max_total_wal_size(1 << 30);
options.set_compression_type(DBCompressionType::Zstd);
// 128 MB
options.set_max_log_file_size(1 << 27);
options.set_recycle_log_file_num(5);
options.set_keep_log_file_num(5);
options.set_wal_compression_type(DBCompressionType::Zstd);
options.set_max_total_wal_size(128 * 1024 * 1024);
// 1 MB
options.set_log_level(LogLevel::Warn);
options.set_max_log_file_size(1024 * 1024);
options.set_recycle_log_file_num(1);
Arc::new(TransactionDB::open(&options, &Default::default(), path).unwrap())
}

26
patches/rocksdb/Cargo.toml Normal file
View file

@ -0,0 +1,26 @@
[package]
name = "rocksdb"
version = "0.21.0"
description = "rocksdb which patches to the latest update"
license = "MIT"
repository = "https://github.com/serai-dex/serai/tree/develop/patches/rocksdb"
authors = ["Luke Parker <lukeparker5132@gmail.com>"]
keywords = []
edition = "2021"
rust-version = "1.70"
[package.metadata.docs.rs]
all-features = true
rustdoc-args = ["--cfg", "docsrs"]
[dependencies]
rocksdb = { version = "0.22", default-features = false }
[features]
jemalloc = ["rocksdb/jemalloc"]
snappy = ["rocksdb/snappy"]
lz4 = ["rocksdb/lz4"]
zstd = ["rocksdb/zstd"]
zlib = ["rocksdb/zlib"]
bzip2 = ["rocksdb/bzip2"]
default = ["snappy", "lz4", "zstd", "zlib", "bzip2"]

1
patches/rocksdb/src/lib.rs Normal file
View file

@ -0,0 +1 @@
pub use rocksdb::*;