types: fix pruned BlockCompleteEntry (#338 )

fix pruned `BlockCompleteEntry`
Fix ci and loosen version requirements (#335 )
2024-12-22 19:49:33 +00:00 · 2024-11-11 23:16:08 +00:00 · 2024-11-04 15:22:43 +00:00 · 2024-11-02 00:45:56 +00:00
9 changed files with 426 additions and 267 deletions
--- a/.github/workflows/audit.yml
+++ b/.github/workflows/audit.yml
@ -1,34 +0,0 @@
-# This runs `cargo audit` on all dependencies (only if Cargo deps changed)
-
-name: Audit
-
-on:
-  push:
-    paths:
-      - '**/Cargo.toml'
-      - '**/Cargo.lock'
-  workflow_dispatch:
-
-env:
-  CARGO_TERM_COLOR: always
-
-jobs:
-  audit:
-
-    runs-on: ubuntu-latest
-
-    steps:
-    - name: Cache
-      uses: actions/cache@v4
-      with:
-        path: |
-          ~/.cargo
-          target
-        key: audit
-    - uses: actions/checkout@v4
-      with:
-        submodules: recursive
-    - name: Install dependencies
-      run: cargo install cargo-audit --locked
-    - name: Audit
-      run: cargo audit
--- a/Cargo.lock
+++ b/Cargo.lock
--- a/Cargo.toml
+++ b/Cargo.toml
@ -16,6 +16,7 @@ members = [
 	"net/wire",
 	"p2p/p2p",
 	"p2p/p2p-core",
+	"p2p/bucket",
 	"p2p/dandelion-tower",
 	"p2p/async-buffer",
 	"p2p/address-book",
@ -64,6 +65,7 @@ cuprate-levin               = { path = "net/levin"              ,default-feature
 cuprate-wire                = { path = "net/wire"               ,default-features = false}
 cuprate-p2p                 = { path = "p2p/p2p"                ,default-features = false}
 cuprate-p2p-core            = { path = "p2p/p2p-core"           ,default-features = false}
+cuprate-p2p-bucket          = { path = "p2p/p2p-bucket"         ,default-features = false}
 cuprate-dandelion-tower     = { path = "p2p/dandelion-tower"    ,default-features = false}
 cuprate-async-buffer        = { path = "p2p/async-buffer"       ,default-features = false}
 cuprate-address-book        = { path = "p2p/address-book"       ,default-features = false}
@ -79,54 +81,55 @@ cuprate-rpc-types           = { path = "rpc/types"              ,default-feature
 cuprate-rpc-interface       = { path = "rpc/interface"          ,default-features = false}

 # External dependencies
-anyhow                = { version = "1.0.89", default-features = false }
-async-trait           = { version = "0.1.82", default-features = false }
-bitflags              = { version = "2.6.0", default-features = false }
+anyhow                = { version = "1", default-features = false }
+arrayvec              = { version = "0.7", default-features = false }
+async-trait           = { version = "0.1", default-features = false }
+bitflags              = { version = "2", default-features = false }
 blake3                = { version = "1", default-features = false }
-borsh                 = { version = "1.5.1", default-features = false }
-bytemuck              = { version = "1.18.0", default-features = false }
-bytes                 = { version = "1.7.2", default-features = false }
-cfg-if                = { version = "1.0.0", default-features = false }
-clap                  = { version = "4.5.17", default-features = false }
-chrono                = { version = "0.4.38", default-features = false }
-crypto-bigint         = { version = "0.5.5", default-features = false }
-crossbeam             = { version = "0.8.4", default-features = false }
-const_format          = { version = "0.2.33", default-features = false }
-curve25519-dalek      = { version = "4.1.3", default-features = false }
-dashmap               = { version = "5.5.3", default-features = false }
-dirs                  = { version = "5.0.1", default-features = false }
-futures               = { version = "0.3.30", default-features = false }
-hex                   = { version = "0.4.3", default-features = false }
+borsh                 = { version = "1", default-features = false }
+bytemuck              = { version = "1", default-features = false }
+bytes                 = { version = "1", default-features = false }
+cfg-if                = { version = "1", default-features = false }
+clap                  = { version = "4", default-features = false }
+chrono                = { version = "0.4", default-features = false }
+crypto-bigint         = { version = "0.5", default-features = false }
+crossbeam             = { version = "0.8", default-features = false }
+const_format          = { version = "0.2", default-features = false }
+curve25519-dalek      = { version = "4", default-features = false }
+dashmap               = { version = "6", default-features = false }
+dirs                  = { version = "5", default-features = false }
+futures               = { version = "0.3", default-features = false }
+hex                   = { version = "0.4", default-features = false }
 hex-literal           = { version = "0.4", default-features = false }
-indexmap              = { version = "2.5.0", default-features = false }
+indexmap              = { version = "2", default-features = false }
 monero-serai          = { git = "https://github.com/Cuprate/serai.git", rev = "d5205ce", default-features = false }
-paste                 = { version = "1.0.15", default-features = false }
-pin-project           = { version = "1.1.5", default-features = false }
+paste                 = { version = "1", default-features = false }
+pin-project           = { version = "1", default-features = false }
 randomx-rs            = { git = "https://github.com/Cuprate/randomx-rs.git", rev = "0028464", default-features = false }
-rand                  = { version = "0.8.5", default-features = false }
-rand_distr            = { version = "0.4.3", default-features = false }
-rayon                 = { version = "1.10.0", default-features = false }
-serde_bytes           = { version = "0.11.15", default-features = false }
-serde_json            = { version = "1.0.128", default-features = false }
-serde                 = { version = "1.0.210", default-features = false }
-strum                 = { version = "0.26.3", default-features = false }
-thiserror             = { version = "1.0.63", default-features = false }
-thread_local          = { version = "1.1.8", default-features = false }
-tokio-util            = { version = "0.7.12", default-features = false }
-tokio-stream          = { version = "0.1.16", default-features = false }
-tokio                 = { version = "1.40.0", default-features = false }
+rand                  = { version = "0.8", default-features = false }
+rand_distr            = { version = "0.4", default-features = false }
+rayon                 = { version = "1", default-features = false }
+serde_bytes           = { version = "0.11", default-features = false }
+serde_json            = { version = "1", default-features = false }
+serde                 = { version = "1", default-features = false }
+strum                 = { version = "0.26", default-features = false }
+thiserror             = { version = "1", default-features = false }
+thread_local          = { version = "1", default-features = false }
+tokio-util            = { version = "0.7", default-features = false }
+tokio-stream          = { version = "0.1", default-features = false }
+tokio                 = { version = "1", default-features = false }
 tower                 = { git = "https://github.com/Cuprate/tower.git", rev = "6c7faf0", default-features = false } # <https://github.com/tower-rs/tower/pull/796>
-tracing-subscriber    = { version = "0.3.18", default-features = false }
-tracing               = { version = "0.1.40", default-features = false }
+tracing-subscriber    = { version = "0.3", default-features = false }
+tracing               = { version = "0.1", default-features = false }

 ## workspace.dev-dependencies
 monero-rpc                = { git = "https://github.com/Cuprate/serai.git", rev = "d5205ce" }
 monero-simple-request-rpc = { git = "https://github.com/Cuprate/serai.git", rev = "d5205ce" }
 tempfile                  = { version = "3" }
-pretty_assertions         = { version = "1.4.1" }
+pretty_assertions         = { version = "1" }
 proptest                  = { version = "1" }
-proptest-derive           = { version = "0.4.0" }
-tokio-test                = { version = "0.4.4" }
+proptest-derive           = { version = "0.5" }
+tokio-test                = { version = "0.4" }

 ## TODO:
 ## Potential dependencies.
--- a/books/architecture/src/appendix/crates.md
+++ b/books/architecture/src/appendix/crates.md
@ -35,6 +35,7 @@ cargo doc --open --package cuprate-blockchain
 | [`cuprate-async-buffer`](https://doc.cuprate.org/cuprate_async_buffer) | [`p2p/async-buffer/`](https://github.com/Cuprate/cuprate/tree/main/p2p/async-buffer) | A bounded SPSC, FIFO, asynchronous buffer that supports arbitrary weights for values
 | [`cuprate-dandelion-tower`](https://doc.cuprate.org/cuprate_dandelion_tower) | [`p2p/dandelion-tower/`](https://github.com/Cuprate/cuprate/tree/main/p2p/dandelion-tower) | TODO
 | [`cuprate-p2p`](https://doc.cuprate.org/cuprate_p2p) | [`p2p/p2p/`](https://github.com/Cuprate/cuprate/tree/main/p2p/p2p) | TODO
+| [`cuprate-p2p-bucket`](https://doc.cuprate.org/cuprate_p2p_bucket) | [`p2p/bucket/`](https://github.com/Cuprate/cuprate/tree/main/p2p/bucket) | A collection data structure discriminating its items into "buckets" of limited size.
 | [`cuprate-p2p-core`](https://doc.cuprate.org/cuprate_p2p_core) | [`p2p/p2p-core/`](https://github.com/Cuprate/cuprate/tree/main/p2p/p2p-core) | TODO

 ## Storage
--- a/consensus/src/tests/mock_db.rs
+++ b/consensus/src/tests/mock_db.rs
@ -1,5 +1,3 @@
-#![expect(non_local_definitions, reason = "proptest macro")]
-
 use std::{
    future::Future,
    pin::Pin,
--- a/deny.toml
+++ b/deny.toml
@ -81,6 +81,9 @@ ignore = [
    #{ id = "RUSTSEC-0000-0000", reason = "you can specify a reason the advisory is ignored" },
    #"a-crate-that-is-yanked@0.1.1", # you can also ignore yanked crate versions if you wish
    #{ crate = "a-crate-that-is-yanked@0.1.1", reason = "you can specify why you are ignoring the yanked crate" },
+
+    # TODO: check this is sorted before a beta release.
+    { id = "RUSTSEC-2024-0370", reason = "unmaintained crate, not necessarily vulnerable yet." }
 ]
 # If this is true, then cargo deny will use the git executable to fetch advisory database.
 # If this is false, then it uses a built-in git library.
@ -110,6 +113,7 @@ allow = [
    "Apache-2.0",                     # https://tldrlegal.com/license/apache-license-2.0-(apache-2.0)
    "MPL-2.0",                        # https://www.mozilla.org/en-US/MPL/2.0/FAQ/
    "BSL-1.0",                        # https://tldrlegal.com/license/boost-software-license-1.0-explained
+    "Zlib",                           # https://spdx.org/licenses/Zlib.html

    # OpenSSL 3.0+ uses Apache-2.0
    # OpenSSL 1.x.x uses https://www.openssl.org/source/license-openssl-ssleay.txt
--- a/p2p/bucket/Cargo.toml
+++ b/p2p/bucket/Cargo.toml
@ -0,0 +1,13 @@
+[package]
+name = "cuprate-p2p-bucket"
+version = "0.1.0"
+edition = "2021"
+license = "MIT"
+authors = ["SyntheticBird"]
+
+[dependencies]
+arrayvec = { workspace = true }
+rand     = { workspace = true, features = ["std", "std_rng"]}
+
+[lints]
+workspace = true
--- a/p2p/bucket/src/lib.rs
+++ b/p2p/bucket/src/lib.rs
@ -0,0 +1,172 @@
+//! Bucket data structure
+//!
+//! A collection data structure that discriminates its unique items and place them into "buckets".
+//!
+//! The item must implement the [`Bucketable`] trait that defines how to create the discriminant
+//! from the item type. The data structure will internally contain any item into "buckets" or vectors
+//! of sized capacity `N` that regroup all the stored items with this specific discriminant.
+//!
+//! A practical example of this data structure is for storing `N` amount of IP discriminated by their subnets.
+//! You can store in each "buckets" corresponding to a `/16` subnet up to `N` IPs of that subnet.
+//!
+//! # Example
+//!
+//! ```
+//! use cuprate_p2p_bucket::Bucket;
+//! use std::net::Ipv4Addr;
+//!
+//! // Create a new bucket that can store at most 2 IPs in a particular `/16` subnet.
+//! let mut bucket = Bucket::<2,Ipv4Addr>::new();
+//!
+//! // Fulfill the `96.96.0.0/16` bucket.
+//! bucket.push("96.96.0.1".parse().unwrap());
+//! bucket.push("96.96.0.2".parse().unwrap());
+//! assert_eq!(2, bucket.len());
+//! assert_eq!(2, bucket.len_bucket(&[96_u8,96_u8]).unwrap());
+//!
+//! // Push a new IP from another subnet
+//! bucket.push("127.0.0.1".parse().unwrap());
+//! assert_eq!(3, bucket.len());
+//! assert_eq!(2, bucket.len_bucket(&[96_u8,96_u8]).unwrap());
+//! assert_eq!(1, bucket.len_bucket(&[127_u8,0_u8]).unwrap());
+//!
+//! // Attempting to push a new IP within `96.96.0.0/16` bucket will return the IP back
+//! // as this subnet is already full.
+//! let pushed = bucket.push("96.96.0.3".parse().unwrap());
+//! assert!(pushed.is_some());
+//! assert_eq!(2, bucket.len_bucket(&[96_u8,96_u8]).unwrap());
+//!
+//! ```
+
+use arrayvec::{ArrayVec, CapacityError};
+use rand::random;
+
+use std::{collections::BTreeMap, net::Ipv4Addr};
+
+/// A discriminant that can be computed from the type.
+pub trait Bucketable: Sized + Eq + Clone {
+    /// The type of the discriminant being used in the Binary tree.
+    type Discriminant: Ord + AsRef<[u8]>;
+
+    /// Method that can compute the discriminant from the item.
+    fn discriminant(&self) -> Self::Discriminant;
+}
+
+/// A collection data structure discriminating its unique items
+/// with a specified method. Limiting the amount of items stored
+/// with that discriminant to the const `N`.
+pub struct Bucket<const N: usize, I: Bucketable> {
+    /// The storage of the bucket
+    storage: BTreeMap<I::Discriminant, ArrayVec<I, N>>,
+}
+
+impl<const N: usize, I: Bucketable> Bucket<N, I> {
+    /// Create a new Bucket
+    pub const fn new() -> Self {
+        Self {
+            storage: BTreeMap::new(),
+        }
+    }
+
+    /// Push a new element into the Bucket
+    ///
+    /// Will internally create a new vector for each new discriminant being
+    /// generated from an item.
+    ///
+    /// This function WILL NOT push the element if it already exists.
+    ///
+    /// Return `None` if the item has been pushed or ignored. `Some(I)` if
+    /// the vector is full.
+    ///
+    /// # Example
+    ///
+    /// ```
+    /// use cuprate_p2p_bucket::Bucket;
+    /// use std::net::Ipv4Addr;
+    ///
+    /// let mut bucket = Bucket::<8,Ipv4Addr>::new();
+    ///
+    /// // Push a first IP address.
+    /// bucket.push("127.0.0.1".parse().unwrap());
+    /// assert_eq!(1, bucket.len());
+    ///
+    /// // Push the same IP address a second time.
+    /// bucket.push("127.0.0.1".parse().unwrap());
+    /// assert_eq!(1, bucket.len());
+    /// ```
+    pub fn push(&mut self, item: I) -> Option<I> {
+        let discriminant = item.discriminant();
+
+        if let Some(vec) = self.storage.get_mut(&discriminant) {
+            // Push the item if it doesn't exist.
+            if !vec.contains(&item) {
+                return vec.try_push(item).err().map(CapacityError::element);
+            }
+        } else {
+            // Initialize the vector if not found.
+            let mut vec = ArrayVec::<I, N>::new();
+            vec.push(item);
+            self.storage.insert(discriminant, vec);
+        }
+
+        None
+    }
+
+    /// Will attempt to remove an item from the bucket.
+    pub fn remove(&mut self, item: &I) -> Option<I> {
+        self.storage.get_mut(&item.discriminant()).and_then(|vec| {
+            vec.iter()
+                .enumerate()
+                .find_map(|(i, v)| (item == v).then_some(i))
+                .map(|index| vec.swap_remove(index))
+        })
+    }
+
+    /// Return the number of item stored within the storage
+    pub fn len(&self) -> usize {
+        self.storage.values().map(ArrayVec::len).sum()
+    }
+
+    /// Return the number of item stored with a specific discriminant.
+    ///
+    /// This method returns None if the bucket with this discriminant
+    /// doesn't exist.
+    pub fn len_bucket(&self, discriminant: &I::Discriminant) -> Option<usize> {
+        self.storage.get(discriminant).map(ArrayVec::len)
+    }
+
+    /// Return `true` if the storage contains no items
+    pub fn is_empty(&self) -> bool {
+        self.len() == 0
+    }
+
+    /// Return a reference to an item chosen at random.
+    ///
+    /// Repeated use of this function will provide a normal distribution of
+    /// items based on their discriminants.
+    pub fn get_random(&mut self) -> Option<&I> {
+        // Get the total amount of discriminants to explore.
+        let len = self.storage.len();
+
+        // Get a random bucket.
+        let (_, vec) = self.storage.iter().nth(random::<usize>() / len).unwrap();
+
+        // Return a reference chose at random.
+        vec.get(random::<usize>() / vec.len())
+    }
+}
+
+impl<const N: usize, I: Bucketable> Default for Bucket<N, I> {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl Bucketable for Ipv4Addr {
+    /// We are discriminating by `/16` subnets.
+    type Discriminant = [u8; 2];
+
+    fn discriminant(&self) -> Self::Discriminant {
+        [self.octets()[0], self.octets()[1]]
+    }
+}
--- a/types/src/block_complete_entry.rs
+++ b/types/src/block_complete_entry.rs
@ -136,7 +136,7 @@ impl TransactionBlobs {
 #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
 pub struct PrunedTxBlobEntry {
    /// The transaction.
-    pub tx: Bytes,
+    pub blob: Bytes,
    /// The prunable transaction hash.
    pub prunable_hash: ByteArray<32>,
 }
@ -144,7 +144,7 @@ pub struct PrunedTxBlobEntry {
 #[cfg(feature = "epee")]
 epee_object!(
    PrunedTxBlobEntry,
-    tx: Bytes,
+    blob: Bytes,
    prunable_hash: ByteArray<32>,
 );
Author	SHA1	Message	Date
Boog900	5a5f88cb13	types: fix pruned `BlockCompleteEntry` (#338 ) Some checks failed Architecture mdBook / build (push) Has been cancelled Details CI / fmt (push) Has been cancelled Details CI / typo (push) Has been cancelled Details CI / ci (macos-latest, stable, bash) (push) Has been cancelled Details CI / ci (ubuntu-latest, stable, bash) (push) Has been cancelled Details CI / ci (windows-latest, stable-x86_64-pc-windows-gnu, msys2 {0}) (push) Has been cancelled Details Deny / audit (push) Has been cancelled Details Doc / build (push) Has been cancelled Details Doc / deploy (push) Has been cancelled Details fix pruned `BlockCompleteEntry`	2024-11-11 23:16:08 +00:00
Boog900	525e20e841	Fix ci and loosen version requirements (#335 ) * add deny exception + loosen version requirements * add a comment * remove `expect`	2024-11-04 15:22:43 +00:00
SyntheticBird	b6c4adc83a	p2p: Implement P2P Bucket data structure (#329 ) Implements P2P Bucket data structure This commit implements a "Bucket" data structure that is a collection of data that discriminates its items into "buckets" (vector of size N) following a defined function. - Implements Bucket data structure and Bucketable trait - Implements Bucketable for Ipv4Addr - Added the crate to the workspace dependencies - Added arrayvec as a dependency	2024-11-02 00:45:56 +00:00