Resolve #327

Also runs `cargo update` and moves where we install the wasm toolchain in the
Dockerfile for better caching properties.

.github/workflows/tests.yml

@@ -62,10 +62,11 @@ jobs:
             -p serai-primitives \
             -p serai-coins-primitives \
             -p serai-coins-pallet \
-            -p serai-in-instructions-primitives \
-            -p serai-in-instructions-pallet \
             -p serai-validator-sets-primitives \
             -p serai-validator-sets-pallet \
+            -p serai-in-instructions-primitives \
+            -p serai-in-instructions-pallet \
+            -p serai-signals-pallet \
             -p serai-runtime \
             -p serai-node
 
@@ -74,15 +75,10 @@ jobs:
     steps:
       - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac
 
-      - name: Test Dependencies
+      - name: Build Dependencies
-        uses: ./.github/actions/test-dependencies
+        uses: ./.github/actions/build-dependencies
         with:
           github-token: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Build node
-        run: |
-          cd substrate/node
-          cargo build
-
       - name: Run Tests
         run: GITHUB_CI=true RUST_BACKTRACE=1 cargo test --all-features -p serai-client

orchestration/serai/Dockerfile

@@ -4,6 +4,9 @@ LABEL description="STAGE 1: Build"
 # Upgrade and add dev dependencies
 RUN apt update && apt upgrade -y && apt install -y git pkg-config make clang libssl-dev protobuf-compiler && apt autoremove -y && apt clean
 
+# Add the wasm toolchain
+RUN rustup target add wasm32-unknown-unknown
+
 # Add files for build
 ADD common /serai/common
 ADD crypto /serai/crypto
@@ -20,9 +23,6 @@ ADD AGPL-3.0 /serai
 
 WORKDIR /serai
 
-# Add the wasm toolchain
-RUN rustup target add wasm32-unknown-unknown
-
 # Mount the caches and build
 RUN --mount=type=cache,target=/root/.cargo \
   --mount=type=cache,target=/usr/local/cargo/registry \

substrate/client/Cargo.toml

@@ -32,9 +32,8 @@ ciphersuite = { path = "../../crypto/ciphersuite", version = "0.4", optional = t
 monero-serai = { path = "../../coins/monero", version = "0.1.4-alpha", optional = true }
 
 [dev-dependencies]
-lazy_static = "1"
-
 rand_core = "0.6"
+hex = "0.4"
 
 blake2 = "0.10"
 
@@ -44,6 +43,9 @@ schnorrkel = { path = "../../crypto/schnorrkel", package = "frost-schnorrkel" }
 
 tokio = "1"
 
+dockertest = "0.3"
+serai-docker-tests = { path = "../../tests/docker" }
+
 [features]
 serai = ["thiserror", "scale-info", "subxt"]
 

substrate/client/tests/batch.rs

@@ -18,10 +18,10 @@ use serai_client::{
 };
 
 mod common;
-use common::{serai, in_instructions::provide_batch};
+use common::in_instructions::provide_batch;
 
 serai_test!(
-  async fn publish_batch() {
+  publish_batch: (|serai: Serai| async move {
     let network = NetworkId::Bitcoin;
     let id = 0;
 
@@ -45,9 +45,8 @@ serai_test!(
       }],
     };
 
-    let block = provide_batch(batch.clone()).await;
+    let block = provide_batch(&serai, batch.clone()).await;
 
-    let serai = serai().await;
     let serai = serai.as_of(block);
     {
       let serai = serai.in_instructions();
@@ -71,5 +70,5 @@ serai_test!(
     );
     assert_eq!(serai.coin_supply(coin).await.unwrap(), amount);
     assert_eq!(serai.coin_balance(coin, address).await.unwrap(), amount);
-  }
+  })
 );

substrate/client/tests/burn.rs

@@ -25,10 +25,10 @@ use serai_client::{
 };
 
 mod common;
-use common::{serai, tx::publish_tx, in_instructions::provide_batch};
+use common::{tx::publish_tx, in_instructions::provide_batch};
 
 serai_test!(
-  async fn burn() {
+  burn: (|serai: Serai| async move {
     let network = NetworkId::Bitcoin;
     let id = 0;
 
@@ -53,9 +53,8 @@ serai_test!(
       }],
     };
 
-    let block = provide_batch(batch.clone()).await;
+    let block = provide_batch(&serai, batch.clone()).await;
 
-    let serai = serai().await;
     let serai = serai.as_of(block);
     let batches = serai.in_instructions().batch_events().await.unwrap();
     assert_eq!(
@@ -90,6 +89,7 @@ serai_test!(
     };
     let serai = serai.into_inner();
     let block = publish_tx(
+      serai,
       &serai
         .sign(
           &PairSigner::new(pair),
@@ -106,5 +106,5 @@ serai_test!(
     assert_eq!(events, vec![CoinsEvent::Burn { from: address.into(), instruction }]);
     assert_eq!(serai.coin_supply(coin).await.unwrap(), Amount(0));
     assert_eq!(serai.coin_balance(coin, address).await.unwrap(), Amount(0));
-  }
+  })
 );

substrate/client/tests/common/in_instructions.rs

@@ -14,15 +14,13 @@ use serai_client::{
     primitives::{Batch, SignedBatch, batch_message},
     InInstructionsEvent,
   },
-  SeraiInInstructions,
+  SeraiInInstructions, Serai,
 };
 
-use crate::common::{serai, tx::publish_tx, validator_sets::set_keys};
+use crate::common::{tx::publish_tx, validator_sets::set_keys};
 
 #[allow(dead_code)]
-pub async fn provide_batch(batch: Batch) -> [u8; 32] {
+pub async fn provide_batch(serai: &Serai, batch: Batch) -> [u8; 32] {
-  let serai = serai().await;
-
   // TODO: Get the latest session
   let set = ValidatorSet { session: Session(0), network: batch.network };
   let pair = insecure_pair_from_name(&format!("ValidatorSet {:?}", set));
@@ -32,15 +30,18 @@ pub async fn provide_batch(batch: Batch) -> [u8; 32] {
     keys
   } else {
     let keys = (pair.public(), vec![].try_into().unwrap());
-    set_keys(set, keys.clone()).await;
+    set_keys(serai, set, keys.clone()).await;
     keys
   };
   assert_eq!(keys.0, pair.public());
 
-  let block = publish_tx(&SeraiInInstructions::execute_batch(SignedBatch {
+  let block = publish_tx(
-    batch: batch.clone(),
+    serai,
-    signature: pair.sign(&batch_message(&batch)),
+    &SeraiInInstructions::execute_batch(SignedBatch {
-  }))
+      batch: batch.clone(),
+      signature: pair.sign(&batch_message(&batch)),
+    }),
+  )
   .await;
 
   let batches = serai.as_of(block).in_instructions().batch_events().await.unwrap();

substrate/client/tests/common/mod.rs

@@ -1,89 +1,57 @@
-use lazy_static::lazy_static;
-
-use tokio::sync::Mutex;
-
-use serai_client::Serai;
-
 pub mod tx;
 pub mod validator_sets;
 pub mod in_instructions;
 
-pub const URL: &str = "ws://127.0.0.1:9944";
-
-pub async fn serai() -> Serai {
-  Serai::new(URL).await.unwrap()
-}
-
-lazy_static! {
-  pub static ref SEQUENTIAL: Mutex<()> = Mutex::new(());
-}
-
 #[macro_export]
 macro_rules! serai_test {
-  ($(async fn $name: ident() $body: block)*) => {
+  ($($name: ident: $test: expr)*) => {
     $(
       #[tokio::test]
       async fn $name() {
-        use std::process::Command;
+        use dockertest::{
+          PullPolicy, StartPolicy, LogOptions, LogAction, LogPolicy, LogSource, Image, Composition,
+          DockerTest,
+        };
 
-        let guard = common::SEQUENTIAL.lock().await;
+        serai_docker_tests::build("serai".to_string());
 
-        let is_running = || {
+        let mut composition = Composition::with_image(
-          !(
+          Image::with_repository("serai-dev-serai").pull_policy(PullPolicy::Never),
-            if let Ok(res) = Command::new("pidof").arg("serai-node").output() {
+        )
-              res
+        .with_cmd(vec![
-            } else {
+          "serai-node".to_string(),
-              Command::new("pgrep")
+          "--dev".to_string(),
-                .arg("serai-node")
+          "--unsafe-rpc-external".to_string(),
-                .output()
+          "--rpc-cors".to_string(),
-                .expect("neither pidof nor pgrep were available")
+          "all".to_string(),
+        ])
+        .with_start_policy(StartPolicy::Strict)
+        .with_log_options(Some(LogOptions {
+          action: LogAction::Forward,
+          policy: LogPolicy::Always,
+          source: LogSource::Both,
+        }));
+        composition.publish_all_ports();
+
+        let handle = composition.handle();
+
+        let mut test = DockerTest::new();
+        test.add_composition(composition);
+        test.run_async(|ops| async move {
+          // Sleep until the Substrate RPC starts
+          let serai_rpc = ops.handle(&handle).host_port(9944).unwrap();
+          let serai_rpc = format!("ws://{}:{}", serai_rpc.0, serai_rpc.1);
+          // Bound execution to 60 seconds
+          for _ in 0 .. 60 {
+            tokio::time::sleep(core::time::Duration::from_secs(1)).await;
+            let Ok(client) = Serai::new(&serai_rpc).await else { continue };
+            if client.latest_block_hash().await.is_err() {
+              continue;
             }
-          ).stdout.is_empty()
+            break;
-        };
-
-        // Spawn a fresh Serai node
-        let mut command = {
-          use core::time::Duration;
-          use std::path::Path;
-
-          // Make sure a node isn't already running
-          assert!(!is_running());
-
-          let node = {
-            let this_crate = Path::new(env!("CARGO_MANIFEST_DIR"));
-            let top_level = this_crate.join("../../");
-            top_level.join("target/debug/serai-node")
-          };
-
-          let command = Command::new(node).arg("--dev").spawn().unwrap();
-          while Serai::new(common::URL).await.is_err() {
-            tokio::time::sleep(Duration::from_secs(1)).await;
           }
-          let serai = serai().await;
+          #[allow(clippy::redundant_closure_call)]
-          while serai.latest_block_hash().await.is_err() {
+          $test(Serai::new(&serai_rpc).await.unwrap()).await;
-            tokio::time::sleep(Duration::from_secs(1)).await;
-          }
-          // TODO: https://github.com/serai-dex/serai/247
-          if std::env::var("GITHUB_CI") == Ok("true".to_string()) {
-            tokio::time::sleep(Duration::from_secs(60)).await;
-          }
-
-          // Sanity check the pidof command is well-formed
-          assert!(is_running());
-
-          command
-        };
-
-        let local = tokio::task::LocalSet::new();
-        local.run_until(async move {
-          if let Err(err) = tokio::task::spawn_local(async move { $body }).await {
-            drop(guard);
-            let _ = command.kill();
-            Err(err).unwrap()
-          } else {
-            command.kill().unwrap();
-          }
-          assert!(!is_running());
         }).await;
       }
     )*

substrate/client/tests/common/tx.rs

@@ -2,14 +2,10 @@ use core::time::Duration;
 
 use tokio::time::sleep;
 
-use serai_client::subxt::utils::Encoded;
+use serai_client::{subxt::utils::Encoded, Serai};
-
-use crate::common::serai;
 
 #[allow(dead_code)]
-pub async fn publish_tx(tx: &Encoded) -> [u8; 32] {
+pub async fn publish_tx(serai: &Serai, tx: &Encoded) -> [u8; 32] {
-  let serai = serai().await;
-
   let mut latest =
     serai.block(serai.latest_block_hash().await.unwrap()).await.unwrap().unwrap().number();
 

substrate/client/tests/common/validator_sets.rs

@@ -15,17 +15,16 @@ use serai_client::{
     primitives::{ValidatorSet, KeyPair, musig_context, musig_key, set_keys_message},
     ValidatorSetsEvent,
   },
-  SeraiValidatorSets,
+  SeraiValidatorSets, Serai,
 };
 
-use crate::common::{serai, tx::publish_tx};
+use crate::common::tx::publish_tx;
 
 #[allow(dead_code)]
-pub async fn set_keys(set: ValidatorSet, key_pair: KeyPair) -> [u8; 32] {
+pub async fn set_keys(serai: &Serai, set: ValidatorSet, key_pair: KeyPair) -> [u8; 32] {
   let pair = insecure_pair_from_name("Alice");
   let public = pair.public();
 
-  let serai = serai().await;
   let public_key = <Ristretto as Ciphersuite>::read_G::<&[u8]>(&mut public.0.as_ref()).unwrap();
   assert_eq!(
     serai
@@ -71,11 +70,10 @@ pub async fn set_keys(set: ValidatorSet, key_pair: KeyPair) -> [u8; 32] {
   );
 
   // Vote in a key pair
-  let block = publish_tx(&SeraiValidatorSets::set_keys(
+  let block = publish_tx(
-    set.network,
+    serai,
-    key_pair.clone(),
+    &SeraiValidatorSets::set_keys(set.network, key_pair.clone(), Signature(sig.to_bytes())),
-    Signature(sig.to_bytes()),
+  )
-  ))
   .await;
 
   assert_eq!(

substrate/client/tests/time.rs

@@ -5,12 +5,9 @@ use tokio::time::sleep;
 use serai_client::Serai;
 
 mod common;
-use common::serai;
 
 serai_test!(
-  async fn time() {
+  time: (|serai: Serai| async move {
-    let serai = serai().await;
-
     let mut number = serai.latest_block().await.unwrap().number();
     let mut done = 0;
     while done < 3 {
@@ -27,5 +24,5 @@ serai_test!(
       assert!(now.saturating_sub(block.time().unwrap()) < 5);
       done += 1;
     }
-  }
+  })
 );

substrate/client/tests/validator_sets.rs

@@ -12,10 +12,10 @@ use serai_client::{
 };
 
 mod common;
-use common::{serai, validator_sets::set_keys};
+use common::validator_sets::set_keys;
 
 serai_test!(
-  async fn set_keys_test() {
+  set_keys_test: (|serai: Serai| async move {
     let network = NetworkId::Bitcoin;
     let set = ValidatorSet { session: Session(0), network };
 
@@ -30,8 +30,6 @@ serai_test!(
     OsRng.fill_bytes(&mut external_key);
     let key_pair = (Public(ristretto_key), external_key.try_into().unwrap());
 
-    let serai = serai().await;
-
     // Make sure the genesis is as expected
     assert_eq!(
       serai
@@ -57,7 +55,7 @@ serai_test!(
       assert_eq!(vs_serai.musig_key(set).await.unwrap().unwrap(), musig_key(set, &[public]).0);
     }
 
-    let block = set_keys(set, key_pair.clone()).await;
+    let block = set_keys(&serai, set, key_pair.clone()).await;
 
     // While the set_keys function should handle this, it's beneficial to
     // independently test it
@@ -67,5 +65,5 @@ serai_test!(
       vec![ValidatorSetsEvent::KeyGen { set, key_pair: key_pair.clone() }]
     );
     assert_eq!(serai.keys(set).await.unwrap(), Some(key_pair));
-  }
+  })
 );