Full stack test framework

.github/workflows/full-stack-tests.yml

@@ -0,0 +1,24 @@
+name: Full Stack Tests
+
+on:
+  push:
+    branches:
+      - develop
+
+  pull_request:
+
+  workflow_dispatch:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Install Build Dependencies
+        uses: ./.github/actions/build-dependencies
+        with:
+          github-token: ${{ inputs.github-token }}
+
+      - name: Run Full Stack Docker tests
+        run: cd tests/full-stack && GITHUB_CI=true RUST_BACKTRACE=1 cargo test

Cargo.lock

@@ -8174,6 +8174,24 @@ dependencies = [
 name = "serai-env"
 version = "0.1.0"
 
+[[package]]
+name = "serai-full-stack-tests"
+version = "0.1.0"
+dependencies = [
+ "bitcoin-serai",
+ "dockertest",
+ "hex",
+ "monero-serai",
+ "rand_core 0.6.4",
+ "serai-client",
+ "serai-coordinator-tests",
+ "serai-docker-tests",
+ "serai-message-queue-tests",
+ "serai-processor-tests",
+ "tokio",
+ "zeroize",
+]
+
 [[package]]
 name = "serai-in-instructions-pallet"
 version = "0.1.0"

Cargo.toml

@@ -57,6 +57,7 @@ members = [
   "tests/message-queue",
   "tests/processor",
   "tests/coordinator",
+  "tests/full-stack",
   "tests/reproducible-runtime",
 ]
 

deny.toml

@@ -69,6 +69,7 @@ exceptions = [
   { allow = ["AGPL-3.0"], name = "serai-message-queue-tests" },
   { allow = ["AGPL-3.0"], name = "serai-processor-tests" },
   { allow = ["AGPL-3.0"], name = "serai-coordinator-tests" },
+  { allow = ["AGPL-3.0"], name = "serai-full-stack-tests" },
   { allow = ["AGPL-3.0"], name = "serai-reproducible-runtime-tests" },
 ]
 

tests/full-stack/Cargo.toml

@@ -0,0 +1,33 @@
+[package]
+name = "serai-full-stack-tests"
+version = "0.1.0"
+description = "Tests for Serai's Full Stack"
+license = "AGPL-3.0-only"
+repository = "https://github.com/serai-dex/serai/tree/develop/tests/full-stack"
+authors = ["Luke Parker <lukeparker5132@gmail.com>"]
+keywords = []
+edition = "2021"
+publish = false
+
+[package.metadata.docs.rs]
+all-features = true
+rustdoc-args = ["--cfg", "docsrs"]
+
+[dependencies]
+hex = "0.4"
+
+zeroize = { version = "1", default-features = false }
+rand_core = { version = "0.6", default-features = false }
+
+bitcoin-serai = { path = "../../coins/bitcoin" }
+monero-serai = { path = "../../coins/monero" }
+
+serai-client = { path = "../../substrate/client", features = ["serai"] }
+
+tokio = { version = "1", features = ["time"] }
+
+dockertest = "0.3"
+serai-docker-tests = { path = "../docker" }
+serai-message-queue-tests = { path = "../message-queue" }
+serai-processor-tests = { path = "../processor" }
+serai-coordinator-tests = { path = "../coordinator" }

tests/full-stack/LICENSE

@@ -0,0 +1,15 @@
+AGPL-3.0-only license
+
+Copyright (c) 2023 Luke Parker
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License Version 3 as
+published by the Free Software Foundation.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.

tests/full-stack/src/lib.rs

@@ -0,0 +1,131 @@
+use std::{
+  sync::{OnceLock, Mutex},
+  fs,
+};
+
+use serai_client::primitives::NetworkId;
+
+use dockertest::{
+  LogAction, LogPolicy, LogSource, LogOptions, StartPolicy, Composition, DockerOperations,
+};
+
+#[cfg(test)]
+mod tests;
+
+static UNIQUE_ID: OnceLock<Mutex<u16>> = OnceLock::new();
+
+use serai_processor_tests::{network_instance, processor_instance};
+use serai_message_queue_tests::instance as message_queue_instance;
+use serai_coordinator_tests::{coordinator_instance, serai_composition};
+
+#[allow(unused)]
+pub struct Handles {
+  bitcoin: (String, u32),
+  bitcoin_processor: String,
+  monero: (String, u32),
+  monero_processor: String,
+  message_queue: String,
+  coordinator: String,
+  serai: String,
+}
+
+pub fn full_stack(name: &str) -> (Handles, Vec<Composition>) {
+  let (coord_key, message_queue_keys, message_queue_composition) = message_queue_instance();
+
+  let (bitcoin_composition, bitcoin_port) = network_instance(NetworkId::Bitcoin);
+  let bitcoin_processor_composition =
+    processor_instance(NetworkId::Bitcoin, bitcoin_port, message_queue_keys[&NetworkId::Bitcoin]);
+
+  let (monero_composition, monero_port) = network_instance(NetworkId::Monero);
+  let monero_processor_composition =
+    processor_instance(NetworkId::Monero, monero_port, message_queue_keys[&NetworkId::Monero]);
+
+  let coordinator_composition = coordinator_instance(name, coord_key);
+  let serai_composition = serai_composition(name);
+
+  // Give every item in this stack a unique ID
+  // Uses a Mutex as we can't generate a 8-byte random ID without hitting hostname length limits
+  let (first, unique_id) = {
+    let unique_id_mutex = UNIQUE_ID.get_or_init(|| Mutex::new(0));
+    let mut unique_id_lock = unique_id_mutex.lock().unwrap();
+    let first = *unique_id_lock == 0;
+    let unique_id = hex::encode(unique_id_lock.to_be_bytes());
+    *unique_id_lock += 1;
+    (first, unique_id)
+  };
+
+  let logs_path = [std::env::current_dir().unwrap().to_str().unwrap(), ".test-logs", "full-stack"]
+    .iter()
+    .collect::<std::path::PathBuf>();
+  if first {
+    let _ = fs::remove_dir_all(&logs_path);
+    fs::create_dir_all(&logs_path).expect("couldn't create logs directory");
+    assert!(
+      fs::read_dir(&logs_path).expect("couldn't read the logs folder").next().is_none(),
+      "logs folder wasn't empty, despite removing it at the start of the run",
+    );
+  }
+  let logs_path = logs_path.to_str().unwrap().to_string();
+
+  let mut compositions = vec![];
+  let mut handles = vec![];
+  for composition in [
+    message_queue_composition,
+    bitcoin_composition,
+    bitcoin_processor_composition,
+    monero_composition,
+    monero_processor_composition,
+    coordinator_composition,
+    serai_composition,
+  ] {
+    let name = format!("{}-{}", composition.handle(), &unique_id);
+
+    compositions.push(
+      composition
+        .with_start_policy(StartPolicy::Strict)
+        .with_container_name(name.clone())
+        .with_log_options(Some(LogOptions {
+          action: if std::env::var("GITHUB_CI") == Ok("true".to_string()) {
+            LogAction::Forward
+          } else {
+            LogAction::ForwardToFile { path: logs_path.clone() }
+          },
+          policy: LogPolicy::Always,
+          source: LogSource::Both,
+        })),
+    );
+
+    handles.push(compositions.last().unwrap().handle());
+  }
+  let handles = Handles {
+    message_queue: handles.remove(0),
+    bitcoin: (handles.remove(0), bitcoin_port),
+    bitcoin_processor: handles.remove(0),
+    monero: (handles.remove(0), monero_port),
+    monero_processor: handles.remove(0),
+    coordinator: handles.remove(0),
+    serai: handles.remove(0),
+  };
+
+  {
+    let bitcoin_processor_composition = compositions.get_mut(2).unwrap();
+    bitcoin_processor_composition
+      .inject_container_name(handles.message_queue.clone(), "MESSAGE_QUEUE_RPC");
+    bitcoin_processor_composition
+      .inject_container_name(handles.bitcoin.0.clone(), "NETWORK_RPC_HOSTNAME");
+  }
+
+  {
+    let monero_processor_composition = compositions.get_mut(4).unwrap();
+    monero_processor_composition
+      .inject_container_name(handles.message_queue.clone(), "MESSAGE_QUEUE_RPC");
+    monero_processor_composition
+      .inject_container_name(handles.monero.0.clone(), "NETWORK_RPC_HOSTNAME");
+  }
+
+  let coordinator_composition = compositions.get_mut(5).unwrap();
+  coordinator_composition.inject_container_name(handles.message_queue.clone(), "MESSAGE_QUEUE_RPC");
+  coordinator_composition.inject_container_name(handles.serai.clone(), "SERAI_HOSTNAME");
+
+  (handles, compositions)
+}

tests/full-stack/src/tests/mint_and_burn.rs

@@ -0,0 +1,15 @@
+use std::{sync::Mutex, time::Duration};
+
+use crate::tests::*;
+
+#[tokio::test]
+async fn mint_and_burn_test() {
+  let _one_at_a_time = ONE_AT_A_TIME.get_or_init(|| Mutex::new(())).lock();
+  let (handles, test) = new_test();
+
+  test
+    .run_async(|_ops| async move {
+      tokio::time::sleep(Duration::from_secs(300)).await;
+    })
+    .await;
+}

tests/full-stack/src/tests/mod.rs

@@ -0,0 +1,33 @@
+use std::sync::OnceLock;
+
+use dockertest::DockerTest;
+
+use crate::*;
+
+mod mint_and_burn;
+
+pub(crate) const VALIDATORS: usize = 4;
+// pub(crate) const THRESHOLD: usize = ((VALIDATORS * 2) / 3) + 1;
+
+pub(crate) static ONE_AT_A_TIME: OnceLock<Mutex<()>> = OnceLock::new();
+
+pub(crate) fn new_test() -> (Vec<Handles>, DockerTest) {
+  let mut validators = vec![];
+  let mut test = DockerTest::new();
+  for i in 0 .. VALIDATORS {
+    let (handles, compositions) = full_stack(match i {
+      0 => "Alice",
+      1 => "Bob",
+      2 => "Charlie",
+      3 => "Dave",
+      4 => "Eve",
+      5 => "Ferdie",
+      _ => panic!("needed a 7th name for a serai node"),
+    });
+    validators.push(handles);
+    for composition in compositions {
+      test.add_composition(composition);
+    }
+  }
+  (validators, test)
+}