Add coordinator rotation test (#535)

* add node side unit test

* complete rotation test for all networks

* set up the fast-epoch docker file

* fix pr comments

* add coordinator side rotation test

* bug fixes

* Remove EPOCH_INTERVAL

* Minor nits

* Add note on origin of publish_tx function in tests/coordinator

* Correct ThresholdParams assert_eq

* fmt

* Correct detection of handover completion

* Restore key gen message match from develop

It was modified in response to the handover completion bug, which has now been
resolved.

* bug fixes

* Correct invalid constant

* Typo fixes

* remove selecting participant to remove at random

---------

Co-authored-by: Luke Parker <lukeparker5132@gmail.com>

substrate/abi/src/grandpa.rs

@@ -1,4 +1,3 @@
-use sp_core::{ConstU32, bounded::BoundedVec};
 use sp_consensus_grandpa::EquivocationProof;
 
 use serai_primitives::{BlockNumber, SeraiAddress};
@@ -19,7 +18,7 @@ pub enum Call {
 #[cfg_attr(feature = "serde", derive(serde::Serialize))]
 #[cfg_attr(all(feature = "std", feature = "serde"), derive(serde::Deserialize))]
 pub enum Event {
-  NewAuthorities { authority_set: BoundedVec<(SeraiAddress, u64), ConstU32<0>> },
+  NewAuthorities { authority_set: alloc::vec::Vec<(SeraiAddress, u64)> },
   // TODO: Remove these
   Paused,
   Resumed,

substrate/client/src/serai/mod.rs

@@ -195,10 +195,10 @@ impl Serai {
   }
 
   async fn active_network_validators(&self, network: NetworkId) -> Result<Vec<Public>, SeraiError> {
-    let hash: String = self
+    let validators: String = self
       .call("state_call", ["SeraiRuntimeApi_validators".to_string(), hex::encode(network.encode())])
       .await?;
-    let bytes = Self::hex_decode(hash)?;
+    let bytes = Self::hex_decode(validators)?;
     let r = Vec::<Public>::decode(&mut bytes.as_slice())
       .map_err(|e| SeraiError::ErrorInResponse(e.to_string()))?;
     Ok(r)

substrate/client/tests/common/in_instructions.rs

@@ -31,7 +31,7 @@ pub async fn provide_batch(serai: &Serai, batch: Batch) -> [u8; 32] {
     keys
   } else {
     let keys = KeyPair(pair.public(), vec![].try_into().unwrap());
-    set_keys(serai, set, keys.clone()).await;
+    set_keys(serai, set, keys.clone(), &[insecure_pair_from_name("Alice")]).await;
     keys
   };
   assert_eq!(keys.0, pair.public());

substrate/client/tests/common/validator_sets.rs

@@ -14,7 +14,6 @@ use frost::dkg::musig::musig;
 use schnorrkel::Schnorrkel;
 
 use serai_client::{
-  primitives::insecure_pair_from_name,
   validator_sets::{
     primitives::{ValidatorSet, KeyPair, musig_context, set_keys_message},
     ValidatorSetsEvent,
@@ -25,26 +24,40 @@ use serai_client::{
 use crate::common::tx::publish_tx;
 
 #[allow(dead_code)]
-pub async fn set_keys(serai: &Serai, set: ValidatorSet, key_pair: KeyPair) -> [u8; 32] {
-  let pair = insecure_pair_from_name("Alice");
-  let public = pair.public();
+pub async fn set_keys(
+  serai: &Serai,
+  set: ValidatorSet,
+  key_pair: KeyPair,
+  pairs: &[Pair],
+) -> [u8; 32] {
+  let mut pub_keys = vec![];
+  for pair in pairs {
+    let public_key =
+      <Ristretto as Ciphersuite>::read_G::<&[u8]>(&mut pair.public().0.as_ref()).unwrap();
+    pub_keys.push(public_key);
+  }
 
-  let public_key = <Ristretto as Ciphersuite>::read_G::<&[u8]>(&mut public.0.as_ref()).unwrap();
-  let secret_key = <Ristretto as Ciphersuite>::read_F::<&[u8]>(
-    &mut pair.as_ref().secret.to_bytes()[.. 32].as_ref(),
-  )
-  .unwrap();
-  assert_eq!(Ristretto::generator() * secret_key, public_key);
-  let threshold_keys =
-    musig::<Ristretto>(&musig_context(set), &Zeroizing::new(secret_key), &[public_key]).unwrap();
+  let mut threshold_keys = vec![];
+  for i in 0 .. pairs.len() {
+    let secret_key = <Ristretto as Ciphersuite>::read_F::<&[u8]>(
+      &mut pairs[i].as_ref().secret.to_bytes()[.. 32].as_ref(),
+    )
+    .unwrap();
+    assert_eq!(Ristretto::generator() * secret_key, pub_keys[i]);
+
+    threshold_keys.push(
+      musig::<Ristretto>(&musig_context(set), &Zeroizing::new(secret_key), &pub_keys).unwrap(),
+    );
+  }
+
+  let mut musig_keys = HashMap::new();
+  for tk in threshold_keys {
+    musig_keys.insert(tk.params().i(), tk.into());
+  }
 
   let sig = frost::tests::sign_without_caching(
     &mut OsRng,
-    frost::tests::algorithm_machines(
-      &mut OsRng,
-      &Schnorrkel::new(b"substrate"),
-      &HashMap::from([(threshold_keys.params().i(), threshold_keys.into())]),
-    ),
+    frost::tests::algorithm_machines(&mut OsRng, &Schnorrkel::new(b"substrate"), &musig_keys),
     &set_keys_message(&set, &[], &key_pair),
   );
 

substrate/client/tests/validator_sets.rs

@@ -1,36 +1,71 @@
 use rand_core::{RngCore, OsRng};
 
-use sp_core::{sr25519::Public, Pair};
+use sp_core::{
+  sr25519::{Public, Pair},
+  Pair as PairTrait,
+};
 
 use serai_client::{
-  primitives::{NETWORKS, NetworkId, insecure_pair_from_name},
+  primitives::{NETWORKS, NetworkId, BlockHash, insecure_pair_from_name},
   validator_sets::{
     primitives::{Session, ValidatorSet, KeyPair},
     ValidatorSetsEvent,
   },
+  in_instructions::{
+    primitives::{Batch, SignedBatch, batch_message},
+    SeraiInInstructions,
+  },
   Amount, Serai,
 };
 
 mod common;
-use common::validator_sets::{set_keys, allocate_stake, deallocate_stake};
+use common::{
+  tx::publish_tx,
+  validator_sets::{allocate_stake, deallocate_stake, set_keys},
+};
 
-const EPOCH_INTERVAL: u64 = 5;
+fn get_random_key_pair() -> KeyPair {
+  let mut ristretto_key = [0; 32];
+  OsRng.fill_bytes(&mut ristretto_key);
+  let mut external_key = vec![0; 33];
+  OsRng.fill_bytes(&mut external_key);
+  KeyPair(Public(ristretto_key), external_key.try_into().unwrap())
+}
+
+async fn get_ordered_keys(serai: &Serai, network: NetworkId, accounts: &[Pair]) -> Vec<Pair> {
+  // retrieve the current session validators so that we know the order of the keys
+  // that is necessary for the correct musig signature.
+  let validators = serai
+    .as_of_latest_finalized_block()
+    .await
+    .unwrap()
+    .validator_sets()
+    .active_network_validators(network)
+    .await
+    .unwrap();
+
+  // collect the pairs of the validators
+  let mut pairs = vec![];
+  for v in validators {
+    let p = accounts.iter().find(|pair| pair.public() == v).unwrap().clone();
+    pairs.push(p);
+  }
+
+  pairs
+}
 
 serai_test!(
   set_keys_test: (|serai: Serai| async move {
     let network = NetworkId::Bitcoin;
     let set = ValidatorSet { session: Session(0), network };
 
-    let public = insecure_pair_from_name("Alice").public();
+    let pair = insecure_pair_from_name("Alice");
+    let public = pair.public();
 
     // Neither of these keys are validated
     // The external key is infeasible to validate on-chain, the Ristretto key is feasible
     // TODO: Should the Ristretto key be validated?
-    let mut ristretto_key = [0; 32];
-    OsRng.fill_bytes(&mut ristretto_key);
-    let mut external_key = vec![0; 33];
-    OsRng.fill_bytes(&mut external_key);
-    let key_pair = KeyPair(Public(ristretto_key), external_key.try_into().unwrap());
+    let key_pair = get_random_key_pair();
 
     // Make sure the genesis is as expected
     assert_eq!(
@@ -62,7 +97,7 @@ serai_test!(
       assert_eq!(participants_ref, [public].as_ref());
     }
 
-    let block = set_keys(&serai, set, key_pair.clone()).await;
+    let block = set_keys(&serai, set, key_pair.clone(), &[pair]).await;
 
     // While the set_keys function should handle this, it's beneficial to
     // independently test it
@@ -149,11 +184,13 @@ async fn validator_set_rotation() {
       );
 
       // genesis accounts
-      let pair1 = insecure_pair_from_name("Alice");
-      let pair2 = insecure_pair_from_name("Bob");
-      let pair3 = insecure_pair_from_name("Charlie");
-      let pair4 = insecure_pair_from_name("Dave");
-      let pair5 = insecure_pair_from_name("Eve");
+      let accounts = vec![
+        insecure_pair_from_name("Alice"),
+        insecure_pair_from_name("Bob"),
+        insecure_pair_from_name("Charlie"),
+        insecure_pair_from_name("Dave"),
+        insecure_pair_from_name("Eve"),
+      ];
 
       // amounts for single key share per network
       let key_shares = HashMap::from([
@@ -164,8 +201,9 @@ async fn validator_set_rotation() {
       ]);
 
       // genesis participants per network
+      #[allow(clippy::redundant_closure_for_method_calls)]
       let default_participants =
-        vec![pair1.public(), pair2.public(), pair3.public(), pair4.public()];
+        accounts[.. 4].to_vec().iter().map(|pair| pair.public()).collect::<Vec<_>>();
       let mut participants = HashMap::from([
         (NetworkId::Serai, default_participants.clone()),
         (NetworkId::Bitcoin, default_participants.clone()),
@@ -181,28 +219,83 @@ async fn validator_set_rotation() {
         participants.sort();
         verify_session_and_active_validators(&serai, network, 0, participants).await;
 
-        // add 1 participant & verify
-        let hash =
-          allocate_stake(&serai, network, key_shares[&network], &pair5, i.try_into().unwrap())
-            .await;
-        participants.push(pair5.public());
-        participants.sort();
-        verify_session_and_active_validators(
+        // add 1 participant
+        let last_participant = accounts[4].clone();
+        let hash = allocate_stake(
           &serai,
           network,
-          get_active_session(&serai, network, hash).await,
-          participants,
+          key_shares[&network],
+          &last_participant,
+          i.try_into().unwrap(),
         )
         .await;
+        participants.push(last_participant.public());
+        // the session at which set changes becomes active
+        let activation_session = get_session_at_which_changes_activate(&serai, network, hash).await;
 
-        // remove 1 participant & verify
-        let hash =
-          deallocate_stake(&serai, network, key_shares[&network], &pair2, i.try_into().unwrap())
-            .await;
-        participants.swap_remove(participants.iter().position(|k| *k == pair2.public()).unwrap());
-        let active_session = get_active_session(&serai, network, hash).await;
+        // set the keys if it is an external set
+        if network != NetworkId::Serai {
+          let set = ValidatorSet { session: Session(0), network };
+          let key_pair = get_random_key_pair();
+          let pairs = get_ordered_keys(&serai, network, &accounts).await;
+          set_keys(&serai, set, key_pair, &pairs).await;
+        }
+
+        // verify
         participants.sort();
-        verify_session_and_active_validators(&serai, network, active_session, participants).await;
+        verify_session_and_active_validators(&serai, network, activation_session, participants)
+          .await;
+
+        // remove 1 participant
+        let participant_to_remove = accounts[1].clone();
+        let hash = deallocate_stake(
+          &serai,
+          network,
+          key_shares[&network],
+          &participant_to_remove,
+          i.try_into().unwrap(),
+        )
+        .await;
+        participants.swap_remove(
+          participants.iter().position(|k| *k == participant_to_remove.public()).unwrap(),
+        );
+        let activation_session = get_session_at_which_changes_activate(&serai, network, hash).await;
+
+        if network != NetworkId::Serai {
+          // set the keys if it is an external set
+          let set = ValidatorSet { session: Session(1), network };
+
+          // we need the whole substrate key pair to sign the batch
+          let (substrate_pair, key_pair) = {
+            let pair = insecure_pair_from_name("session-1-key-pair");
+            let public = pair.public();
+
+            let mut external_key = vec![0; 33];
+            OsRng.fill_bytes(&mut external_key);
+
+            (pair, KeyPair(public, external_key.try_into().unwrap()))
+          };
+          let pairs = get_ordered_keys(&serai, network, &accounts).await;
+          set_keys(&serai, set, key_pair, &pairs).await;
+
+          // provide a batch to complete the handover and retire the previous set
+          let mut block_hash = BlockHash([0; 32]);
+          OsRng.fill_bytes(&mut block_hash.0);
+          let batch = Batch { network, id: 0, block: block_hash, instructions: vec![] };
+          publish_tx(
+            &serai,
+            &SeraiInInstructions::execute_batch(SignedBatch {
+              batch: batch.clone(),
+              signature: substrate_pair.sign(&batch_message(&batch)),
+            }),
+          )
+          .await;
+        }
+
+        // verify
+        participants.sort();
+        verify_session_and_active_validators(&serai, network, activation_session, participants)
+          .await;
 
         // check pending deallocations
         let pending = serai
@@ -212,8 +305,8 @@ async fn validator_set_rotation() {
           .validator_sets()
           .pending_deallocations(
             network,
-            pair2.public(),
-            Session(u32::try_from(active_session + 1).unwrap()),
+            participant_to_remove.public(),
+            Session(activation_session + 1),
           )
           .await
           .unwrap();
@@ -223,24 +316,39 @@ async fn validator_set_rotation() {
     .await;
 }
 
+async fn session_for_block(serai: &Serai, block: [u8; 32], network: NetworkId) -> u32 {
+  serai.as_of(block).validator_sets().session(network).await.unwrap().unwrap().0
+}
+
 async fn verify_session_and_active_validators(
   serai: &Serai,
   network: NetworkId,
-  session: u64,
+  session: u32,
   participants: &[Public],
 ) {
-  // wait untill the epoch block finalized
-  let epoch_block = (session * EPOCH_INTERVAL) + 1;
-  while serai.finalized_block_by_number(epoch_block).await.unwrap().is_none() {
-    // sleep 1 block
-    tokio::time::sleep(tokio::time::Duration::from_secs(6)).await;
-  }
-  let serai_for_block =
-    serai.as_of(serai.finalized_block_by_number(epoch_block).await.unwrap().unwrap().hash());
+  // wait until the active session. This wait should be max 30 secs since the epoch time.
+  let block = tokio::time::timeout(core::time::Duration::from_secs(2 * 60), async move {
+    loop {
+      let mut block = serai.latest_finalized_block_hash().await.unwrap();
+      if session_for_block(serai, block, network).await < session {
+        // Sleep a block
+        tokio::time::sleep(core::time::Duration::from_secs(6)).await;
+        continue;
+      }
+      while session_for_block(serai, block, network).await > session {
+        block = serai.block(block).await.unwrap().unwrap().header.parent_hash.0;
+      }
+      assert_eq!(session_for_block(serai, block, network).await, session);
+      break block;
+    }
+  })
+  .await
+  .unwrap();
+  let serai_for_block = serai.as_of(block);
 
   // verify session
   let s = serai_for_block.validator_sets().session(network).await.unwrap().unwrap();
-  assert_eq!(u64::from(s.0), session);
+  assert_eq!(s.0, session);
 
   // verify participants
   let mut validators =
@@ -249,10 +357,11 @@ async fn verify_session_and_active_validators(
   assert_eq!(validators, participants);
 
   // make sure finalization continues as usual after the changes
-  tokio::time::timeout(tokio::time::Duration::from_secs(60), async move {
+  let current_finalized_block = serai.latest_finalized_block().await.unwrap().header.number;
+  tokio::time::timeout(core::time::Duration::from_secs(60), async move {
     let mut finalized_block = serai.latest_finalized_block().await.unwrap().header.number;
-    while finalized_block <= epoch_block + 2 {
-      tokio::time::sleep(tokio::time::Duration::from_secs(6)).await;
+    while finalized_block <= current_finalized_block + 2 {
+      tokio::time::sleep(core::time::Duration::from_secs(6)).await;
       finalized_block = serai.latest_finalized_block().await.unwrap().header.number;
     }
   })
@@ -262,15 +371,18 @@ async fn verify_session_and_active_validators(
   // TODO: verify key shares as well?
 }
 
-async fn get_active_session(serai: &Serai, network: NetworkId, hash: [u8; 32]) -> u64 {
-  let block_number = serai.block(hash).await.unwrap().unwrap().header.number;
-  let epoch = block_number / EPOCH_INTERVAL;
+async fn get_session_at_which_changes_activate(
+  serai: &Serai,
+  network: NetworkId,
+  hash: [u8; 32],
+) -> u32 {
+  let session = session_for_block(serai, hash, network).await;
 
   // changes should be active in the next session
   if network == NetworkId::Serai {
     // it takes 1 extra session for serai net to make the changes active.
-    epoch + 2
+    session + 2
   } else {
-    epoch + 1
+    session + 1
   }
 }

substrate/validator-sets/pallet/src/lib.rs

@@ -643,8 +643,9 @@ pub mod pallet {
     // Checks if this session has completed the handover from the prior session.
     fn handover_completed(network: NetworkId, session: Session) -> bool {
       let Some(current_session) = Self::session(network) else { return false };
-      // No handover occurs on genesis
-      if current_session.0 == 0 {
+
+      // If the session we've been queried about is old, it must have completed its handover
+      if current_session.0 > session.0 {
         return true;
       }
       // If the session we've been queried about has yet to start, it can't have completed its
@@ -652,19 +653,21 @@ pub mod pallet {
       if current_session.0 < session.0 {
         return false;
       }
-      if current_session.0 == session.0 {
-        // Handover is automatically complete for Serai as it doesn't have a handover protocol
-        // If not Serai, check the prior session had its keys cleared, which happens once its
-        // retired
-        return (network == NetworkId::Serai) ||
-          (!Keys::<T>::contains_key(ValidatorSet {
-            network,
-            session: Session(current_session.0 - 1),
-          }));
+
+      // Handover is automatically complete for Serai as it doesn't have a handover protocol
+      if network == NetworkId::Serai {
+        return true;
       }
-      // We're currently in a future session, meaning this session definitely performed itself
-      // handover
-      true
+
+      // The current session must have set keys for its handover to be completed
+      if !Keys::<T>::contains_key(ValidatorSet { network, session }) {
+        return false;
+      }
+
+      // This must be the first session (which has set keys) OR the prior session must have been
+      // retired (signified by its keys no longer being present)
+      (session.0 == 0) ||
+        (!Keys::<T>::contains_key(ValidatorSet { network, session: Session(session.0 - 1) }))
     }
 
     fn new_session() {
@@ -682,6 +685,8 @@ pub mod pallet {
       }
     }
 
+    // TODO: This is called retire_set, yet just starts retiring the set
+    // Update the nomenclature within this function
     pub fn retire_set(set: ValidatorSet) {
       // If the prior prior set didn't report, emit they're retired now
       if PendingSlashReport::<T>::get(set.network).is_some() {

tests/coordinator/src/lib.rs

@@ -60,12 +60,18 @@ pub fn coordinator_instance(
   )
 }
 
-pub fn serai_composition(name: &str) -> TestBodySpecification {
-  serai_docker_tests::build("serai".to_string());
-
-  TestBodySpecification::with_image(
-    Image::with_repository("serai-dev-serai").pull_policy(PullPolicy::Never),
-  )
+pub fn serai_composition(name: &str, fast_epoch: bool) -> TestBodySpecification {
+  (if fast_epoch {
+    serai_docker_tests::build("serai-fast-epoch".to_string());
+    TestBodySpecification::with_image(
+      Image::with_repository("serai-dev-serai-fast-epoch").pull_policy(PullPolicy::Never),
+    )
+  } else {
+    serai_docker_tests::build("serai".to_string());
+    TestBodySpecification::with_image(
+      Image::with_repository("serai-dev-serai").pull_policy(PullPolicy::Never),
+    )
+  })
   .replace_env(
     [("SERAI_NAME".to_string(), name.to_lowercase()), ("KEY".to_string(), " ".to_string())].into(),
   )

tests/coordinator/src/tests/batch.rs

@@ -260,21 +260,29 @@ pub async fn batch(
 
 #[tokio::test]
 async fn batch_test() {
-  new_test(|mut processors: Vec<Processor>| async move {
-    let (processor_is, substrate_key, _) = key_gen::<Secp256k1>(&mut processors).await;
-    batch(
-      &mut processors,
-      &processor_is,
-      Session(0),
-      &substrate_key,
-      Batch {
-        network: NetworkId::Bitcoin,
-        id: 0,
-        block: BlockHash([0x22; 32]),
-        instructions: vec![],
-      },
-    )
-    .await;
-  })
+  new_test(
+    |mut processors: Vec<Processor>| async move {
+      // pop the last participant since genesis keygen has only 4 participants
+      processors.pop().unwrap();
+      assert_eq!(processors.len(), COORDINATORS);
+
+      let (processor_is, substrate_key, _) =
+        key_gen::<Secp256k1>(&mut processors, Session(0)).await;
+      batch(
+        &mut processors,
+        &processor_is,
+        Session(0),
+        &substrate_key,
+        Batch {
+          network: NetworkId::Bitcoin,
+          id: 0,
+          block: BlockHash([0x22; 32]),
+          instructions: vec![],
+        },
+      )
+      .await;
+    },
+    false,
+  )
   .await;
 }

tests/coordinator/src/tests/key_gen.rs

@@ -23,10 +23,12 @@ use crate::tests::*;
 
 pub async fn key_gen<C: Ciphersuite>(
   processors: &mut [Processor],
+  session: Session,
 ) -> (Vec<u8>, Zeroizing<<Ristretto as Ciphersuite>::F>, Zeroizing<C::F>) {
+  let coordinators = processors.len();
   let mut participant_is = vec![];
 
-  let set = ValidatorSet { session: Session(0), network: NetworkId::Bitcoin };
+  let set = ValidatorSet { session, network: NetworkId::Bitcoin };
   let id = KeyGenId { session: set.session, attempt: 0 };
 
   for (i, processor) in processors.iter_mut().enumerate() {
@@ -46,8 +48,8 @@ pub async fn key_gen<C: Ciphersuite>(
       CoordinatorMessage::KeyGen(messages::key_gen::CoordinatorMessage::GenerateKey {
         id,
         params: ThresholdParams::new(
-          u16::try_from(((COORDINATORS * 2) / 3) + 1).unwrap(),
-          u16::try_from(COORDINATORS).unwrap(),
+          u16::try_from(((coordinators * 2) / 3) + 1).unwrap(),
+          u16::try_from(coordinators).unwrap(),
           participant_is[i],
         )
         .unwrap(),
@@ -65,7 +67,7 @@ pub async fn key_gen<C: Ciphersuite>(
 
   wait_for_tributary().await;
   for (i, processor) in processors.iter_mut().enumerate() {
-    let mut commitments = (0 .. u8::try_from(COORDINATORS).unwrap())
+    let mut commitments = (0 .. u8::try_from(coordinators).unwrap())
       .map(|l| {
         (
           participant_is[usize::from(l)],
@@ -83,7 +85,7 @@ pub async fn key_gen<C: Ciphersuite>(
     );
 
     // Recipient it's for -> (Sender i, Recipient i)
-    let mut shares = (0 .. u8::try_from(COORDINATORS).unwrap())
+    let mut shares = (0 .. u8::try_from(coordinators).unwrap())
       .map(|l| {
         (
           participant_is[usize::from(l)],
@@ -118,7 +120,7 @@ pub async fn key_gen<C: Ciphersuite>(
       CoordinatorMessage::KeyGen(messages::key_gen::CoordinatorMessage::Shares {
         id,
         shares: {
-          let mut shares = (0 .. u8::try_from(COORDINATORS).unwrap())
+          let mut shares = (0 .. u8::try_from(coordinators).unwrap())
             .map(|l| {
               (
                 participant_is[usize::from(l)],
@@ -182,14 +184,14 @@ pub async fn key_gen<C: Ciphersuite>(
               .unwrap()
               .as_secs()
               .abs_diff(context.serai_time) <
-              70
+              (60 * 60 * 3) // 3 hours, which should exceed the length of any test we run
           );
           assert_eq!(context.network_latest_finalized_block.0, [0; 32]);
           assert_eq!(set.session, session);
           assert_eq!(key_pair.0 .0, substrate_key);
           assert_eq!(&key_pair.1, &network_key);
         }
-        _ => panic!("coordinator didn't respond with ConfirmKeyPair"),
+        _ => panic!("coordinator didn't respond with ConfirmKeyPair. msg: {msg:?}"),
       }
       message = Some(msg);
     } else {
@@ -220,8 +222,15 @@ pub async fn key_gen<C: Ciphersuite>(
 
 #[tokio::test]
 async fn key_gen_test() {
-  new_test(|mut processors: Vec<Processor>| async move {
-    key_gen::<Secp256k1>(&mut processors).await;
-  })
+  new_test(
+    |mut processors: Vec<Processor>| async move {
+      // pop the last participant since genesis keygen has only 4 participants
+      processors.pop().unwrap();
+      assert_eq!(processors.len(), COORDINATORS);
+
+      key_gen::<Secp256k1>(&mut processors, Session(0)).await;
+    },
+    false,
+  )
   .await;
 }

tests/coordinator/src/tests/mod.rs

@@ -22,6 +22,8 @@ mod sign;
 #[allow(unused_imports)]
 pub use sign::sign;
 
+mod rotation;
+
 pub(crate) const COORDINATORS: usize = 4;
 pub(crate) const THRESHOLD: usize = ((COORDINATORS * 2) / 3) + 1;
 
@@ -39,13 +41,15 @@ impl<F: Send + Future, TB: 'static + Send + Sync + Fn(Vec<Processor>) -> F> Test
   }
 }
 
-pub(crate) async fn new_test(test_body: impl TestBody) {
+pub(crate) async fn new_test(test_body: impl TestBody, fast_epoch: bool) {
   let mut unique_id_lock = UNIQUE_ID.get_or_init(|| Mutex::new(0)).lock().await;
 
   let mut coordinators = vec![];
   let mut test = DockerTest::new().with_network(dockertest::Network::Isolated);
   let mut coordinator_compositions = vec![];
-  for i in 0 .. COORDINATORS {
+  // Spawn one extra coordinator which isn't in-set
+  #[allow(clippy::range_plus_one)]
+  for i in 0 .. (COORDINATORS + 1) {
     let name = match i {
       0 => "Alice",
       1 => "Bob",
@@ -55,7 +59,7 @@ pub(crate) async fn new_test(test_body: impl TestBody) {
       5 => "Ferdie",
       _ => panic!("needed a 7th name for a serai node"),
     };
-    let serai_composition = serai_composition(name);
+    let serai_composition = serai_composition(name, fast_epoch);
 
     let (processor_key, message_queue_keys, message_queue_composition) =
       serai_message_queue_tests::instance();

tests/coordinator/src/tests/rotation.rs

@@ -0,0 +1,169 @@
+use tokio::time::{sleep, Duration};
+
+use ciphersuite::Secp256k1;
+
+use serai_client::{
+  primitives::{insecure_pair_from_name, NetworkId},
+  validator_sets::{
+    self,
+    primitives::{Session, ValidatorSet},
+    ValidatorSetsEvent,
+  },
+  Amount, Pair, Transaction,
+};
+
+use crate::{*, tests::*};
+
+// TODO: This is duplicated with serai-client's tests
+async fn publish_tx(serai: &Serai, tx: &Transaction) -> [u8; 32] {
+  let mut latest = serai
+    .block(serai.latest_finalized_block_hash().await.unwrap())
+    .await
+    .unwrap()
+    .unwrap()
+    .number();
+
+  serai.publish(tx).await.unwrap();
+
+  // Get the block it was included in
+  // TODO: Add an RPC method for this/check the guarantee on the subscription
+  let mut ticks = 0;
+  loop {
+    latest += 1;
+
+    let block = {
+      let mut block;
+      while {
+        block = serai.finalized_block_by_number(latest).await.unwrap();
+        block.is_none()
+      } {
+        sleep(Duration::from_secs(1)).await;
+        ticks += 1;
+
+        if ticks > 60 {
+          panic!("60 seconds without inclusion in a finalized block");
+        }
+      }
+      block.unwrap()
+    };
+
+    for transaction in &block.transactions {
+      if transaction == tx {
+        return block.hash();
+      }
+    }
+  }
+}
+
+#[allow(dead_code)]
+async fn allocate_stake(
+  serai: &Serai,
+  network: NetworkId,
+  amount: Amount,
+  pair: &Pair,
+  nonce: u32,
+) -> [u8; 32] {
+  // get the call
+  let tx =
+    serai.sign(pair, validator_sets::SeraiValidatorSets::allocate(network, amount), nonce, 0);
+  publish_tx(serai, &tx).await
+}
+
+#[allow(dead_code)]
+async fn deallocate_stake(
+  serai: &Serai,
+  network: NetworkId,
+  amount: Amount,
+  pair: &Pair,
+  nonce: u32,
+) -> [u8; 32] {
+  // get the call
+  let tx =
+    serai.sign(pair, validator_sets::SeraiValidatorSets::deallocate(network, amount), nonce, 0);
+  publish_tx(serai, &tx).await
+}
+
+async fn get_session(serai: &Serai, network: NetworkId) -> Session {
+  serai
+    .as_of_latest_finalized_block()
+    .await
+    .unwrap()
+    .validator_sets()
+    .session(network)
+    .await
+    .unwrap()
+    .unwrap()
+}
+
+async fn wait_till_session_1(serai: &Serai, network: NetworkId) {
+  let mut current_session = get_session(serai, network).await;
+
+  while current_session.0 < 1 {
+    sleep(Duration::from_secs(6)).await;
+    current_session = get_session(serai, network).await;
+  }
+}
+
+async fn most_recent_new_set_event(serai: &Serai, network: NetworkId) -> ValidatorSetsEvent {
+  let mut current_block = serai.latest_finalized_block().await.unwrap();
+  loop {
+    let events = serai.as_of(current_block.hash()).validator_sets().new_set_events().await.unwrap();
+    for event in events {
+      match event {
+        ValidatorSetsEvent::NewSet { set } => {
+          if set.network == network {
+            return event;
+          }
+        }
+        _ => panic!("new_set_events gave non-NewSet event: {event:?}"),
+      }
+    }
+    current_block = serai.block(current_block.header.parent_hash.0).await.unwrap().unwrap();
+  }
+}
+
+#[tokio::test]
+async fn set_rotation_test() {
+  new_test(
+    |mut processors: Vec<Processor>| async move {
+      // exclude the last processor from keygen since we will add him later
+      let mut excluded = processors.pop().unwrap();
+      assert_eq!(processors.len(), COORDINATORS);
+
+      // excluded participant
+      let pair5 = insecure_pair_from_name("Eve");
+      let network = NetworkId::Bitcoin;
+      let amount = Amount(1_000_000 * 10_u64.pow(8));
+      let serai = processors[0].serai().await;
+
+      // allocate now for the last participant so that it is guaranteed to be included into session
+      // 1 set. This doesn't affect the genesis set at all since that is a predetermined set.
+      allocate_stake(&serai, network, amount, &pair5, 0).await;
+
+      // genesis keygen
+      let _ = key_gen::<Secp256k1>(&mut processors, Session(0)).await;
+      // Even the excluded processor should receive the key pair confirmation
+      match excluded.recv_message().await {
+        CoordinatorMessage::Substrate(
+          messages::substrate::CoordinatorMessage::ConfirmKeyPair { session, .. },
+        ) => assert_eq!(session, Session(0)),
+        _ => panic!("excluded got message other than ConfirmKeyPair"),
+      }
+
+      // wait until next session to see the effect on coordinator
+      wait_till_session_1(&serai, network).await;
+
+      // Ensure the new validator was included in the new set
+      assert_eq!(
+        most_recent_new_set_event(&serai, network).await,
+        ValidatorSetsEvent::NewSet { set: ValidatorSet { session: Session(1), network } },
+      );
+
+      // add the last participant & do the keygen
+      processors.push(excluded);
+      let _ = key_gen::<Secp256k1>(&mut processors, Session(1)).await;
+    },
+    true,
+  )
+  .await;
+}

tests/coordinator/src/tests/sign.rs

@@ -168,161 +168,172 @@ pub async fn sign(
 
 #[tokio::test]
 async fn sign_test() {
-  new_test(|mut processors: Vec<Processor>| async move {
-    let (participant_is, substrate_key, _) = key_gen::<Secp256k1>(&mut processors).await;
+  new_test(
+    |mut processors: Vec<Processor>| async move {
+      // pop the last participant since genesis keygen has only 4 participant.
+      processors.pop().unwrap();
+      assert_eq!(processors.len(), COORDINATORS);
 
-    // 'Send' external coins into Serai
-    let serai = processors[0].serai().await;
-    let (serai_pair, serai_addr) = {
-      let mut name = [0; 4];
-      OsRng.fill_bytes(&mut name);
-      let pair = insecure_pair_from_name(&hex::encode(name));
-      let address = SeraiAddress::from(pair.public());
+      let (participant_is, substrate_key, _) =
+        key_gen::<Secp256k1>(&mut processors, Session(0)).await;
 
-      // Fund the new account to pay for fees
-      let balance = Balance { coin: Coin::Serai, amount: Amount(1_000_000_000) };
+      // 'Send' external coins into Serai
+      let serai = processors[0].serai().await;
+      let (serai_pair, serai_addr) = {
+        let mut name = [0; 4];
+        OsRng.fill_bytes(&mut name);
+        let pair = insecure_pair_from_name(&hex::encode(name));
+        let address = SeraiAddress::from(pair.public());
+
+        // Fund the new account to pay for fees
+        let balance = Balance { coin: Coin::Serai, amount: Amount(1_000_000_000) };
+        serai
+          .publish(&serai.sign(
+            &insecure_pair_from_name("Ferdie"),
+            SeraiCoins::transfer(address, balance),
+            0,
+            Default::default(),
+          ))
+          .await
+          .unwrap();
+
+        (pair, address)
+      };
+
+      #[allow(clippy::inconsistent_digit_grouping)]
+      let amount = Amount(1_000_000_00);
+      let balance = Balance { coin: Coin::Bitcoin, amount };
+
+      let coin_block = BlockHash([0x33; 32]);
+      let block_included_in = batch(
+        &mut processors,
+        &participant_is,
+        Session(0),
+        &substrate_key,
+        Batch {
+          network: NetworkId::Bitcoin,
+          id: 0,
+          block: coin_block,
+          instructions: vec![InInstructionWithBalance {
+            instruction: InInstruction::Transfer(serai_addr),
+            balance,
+          }],
+        },
+      )
+      .await;
+
+      {
+        let block_included_in_hash =
+          serai.finalized_block_by_number(block_included_in).await.unwrap().unwrap().hash();
+
+        let serai = serai.as_of(block_included_in_hash);
+        let serai = serai.coins();
+        assert_eq!(
+          serai.coin_balance(Coin::Serai, serai_addr).await.unwrap(),
+          Amount(1_000_000_000)
+        );
+
+        // Verify the mint occurred as expected
+        assert_eq!(
+          serai.mint_events().await.unwrap(),
+          vec![CoinsEvent::Mint { to: serai_addr, balance }]
+        );
+        assert_eq!(serai.coin_supply(Coin::Bitcoin).await.unwrap(), amount);
+        assert_eq!(serai.coin_balance(Coin::Bitcoin, serai_addr).await.unwrap(), amount);
+      }
+
+      // Trigger a burn
+      let out_instruction = OutInstructionWithBalance {
+        balance,
+        instruction: OutInstruction {
+          address: ExternalAddress::new(b"external".to_vec()).unwrap(),
+          data: None,
+        },
+      };
       serai
         .publish(&serai.sign(
-          &insecure_pair_from_name("Ferdie"),
-          SeraiCoins::transfer(address, balance),
+          &serai_pair,
+          SeraiCoins::burn_with_instruction(out_instruction.clone()),
           0,
           Default::default(),
         ))
         .await
         .unwrap();
 
-      (pair, address)
-    };
-
-    #[allow(clippy::inconsistent_digit_grouping)]
-    let amount = Amount(1_000_000_00);
-    let balance = Balance { coin: Coin::Bitcoin, amount };
-
-    let coin_block = BlockHash([0x33; 32]);
-    let block_included_in = batch(
-      &mut processors,
-      &participant_is,
-      Session(0),
-      &substrate_key,
-      Batch {
-        network: NetworkId::Bitcoin,
-        id: 0,
-        block: coin_block,
-        instructions: vec![InInstructionWithBalance {
-          instruction: InInstruction::Transfer(serai_addr),
-          balance,
-        }],
-      },
-    )
-    .await;
-
-    {
-      let block_included_in_hash =
-        serai.finalized_block_by_number(block_included_in).await.unwrap().unwrap().hash();
-
-      let serai = serai.as_of(block_included_in_hash);
-      let serai = serai.coins();
-      assert_eq!(serai.coin_balance(Coin::Serai, serai_addr).await.unwrap(), Amount(1_000_000_000));
-
-      // Verify the mint occurred as expected
-      assert_eq!(
-        serai.mint_events().await.unwrap(),
-        vec![CoinsEvent::Mint { to: serai_addr, balance }]
-      );
-      assert_eq!(serai.coin_supply(Coin::Bitcoin).await.unwrap(), amount);
-      assert_eq!(serai.coin_balance(Coin::Bitcoin, serai_addr).await.unwrap(), amount);
-    }
-
-    // Trigger a burn
-    let out_instruction = OutInstructionWithBalance {
-      balance,
-      instruction: OutInstruction {
-        address: ExternalAddress::new(b"external".to_vec()).unwrap(),
-        data: None,
-      },
-    };
-    serai
-      .publish(&serai.sign(
-        &serai_pair,
-        SeraiCoins::burn_with_instruction(out_instruction.clone()),
-        0,
-        Default::default(),
-      ))
-      .await
-      .unwrap();
-
-    // TODO: We *really* need a helper for this pattern
-    let mut last_serai_block = block_included_in;
-    'outer: for _ in 0 .. 20 {
-      tokio::time::sleep(Duration::from_secs(6)).await;
-      if std::env::var("GITHUB_CI") == Ok("true".to_string()) {
+      // TODO: We *really* need a helper for this pattern
+      let mut last_serai_block = block_included_in;
+      'outer: for _ in 0 .. 20 {
         tokio::time::sleep(Duration::from_secs(6)).await;
-      }
-
-      while last_serai_block <= serai.latest_finalized_block().await.unwrap().number() {
-        let burn_events = serai
-          .as_of(serai.finalized_block_by_number(last_serai_block).await.unwrap().unwrap().hash())
-          .coins()
-          .burn_with_instruction_events()
-          .await
-          .unwrap();
-
-        if !burn_events.is_empty() {
-          assert_eq!(burn_events.len(), 1);
-          assert_eq!(
-            burn_events[0],
-            CoinsEvent::BurnWithInstruction {
-              from: serai_addr,
-              instruction: out_instruction.clone()
-            }
-          );
-          break 'outer;
+        if std::env::var("GITHUB_CI") == Ok("true".to_string()) {
+          tokio::time::sleep(Duration::from_secs(6)).await;
         }
-        last_serai_block += 1;
-      }
-    }
 
-    let last_serai_block =
-      serai.finalized_block_by_number(last_serai_block).await.unwrap().unwrap();
-    let last_serai_block_hash = last_serai_block.hash();
-    let serai = serai.as_of(last_serai_block_hash);
-    let serai = serai.coins();
-    assert_eq!(serai.coin_supply(Coin::Bitcoin).await.unwrap(), Amount(0));
-    assert_eq!(serai.coin_balance(Coin::Bitcoin, serai_addr).await.unwrap(), Amount(0));
+        while last_serai_block <= serai.latest_finalized_block().await.unwrap().number() {
+          let burn_events = serai
+            .as_of(serai.finalized_block_by_number(last_serai_block).await.unwrap().unwrap().hash())
+            .coins()
+            .burn_with_instruction_events()
+            .await
+            .unwrap();
 
-    let mut plan_id = [0; 32];
-    OsRng.fill_bytes(&mut plan_id);
-    let plan_id = plan_id;
-
-    // We should now get a SubstrateBlock
-    for processor in &mut processors {
-      assert_eq!(
-        processor.recv_message().await,
-        messages::CoordinatorMessage::Substrate(
-          messages::substrate::CoordinatorMessage::SubstrateBlock {
-            context: SubstrateContext {
-              serai_time: last_serai_block.time().unwrap() / 1000,
-              network_latest_finalized_block: coin_block,
-            },
-            block: last_serai_block.number(),
-            burns: vec![out_instruction.clone()],
-            batches: vec![],
+          if !burn_events.is_empty() {
+            assert_eq!(burn_events.len(), 1);
+            assert_eq!(
+              burn_events[0],
+              CoinsEvent::BurnWithInstruction {
+                from: serai_addr,
+                instruction: out_instruction.clone()
+              }
+            );
+            break 'outer;
           }
-        )
-      );
+          last_serai_block += 1;
+        }
+      }
 
-      // Send the ACK, claiming there's a plan to sign
-      processor
-        .send_message(messages::ProcessorMessage::Coordinator(
-          messages::coordinator::ProcessorMessage::SubstrateBlockAck {
-            block: last_serai_block.number(),
-            plans: vec![PlanMeta { session: Session(0), id: plan_id }],
-          },
-        ))
-        .await;
-    }
+      let last_serai_block =
+        serai.finalized_block_by_number(last_serai_block).await.unwrap().unwrap();
+      let last_serai_block_hash = last_serai_block.hash();
+      let serai = serai.as_of(last_serai_block_hash);
+      let serai = serai.coins();
+      assert_eq!(serai.coin_supply(Coin::Bitcoin).await.unwrap(), Amount(0));
+      assert_eq!(serai.coin_balance(Coin::Bitcoin, serai_addr).await.unwrap(), Amount(0));
 
-    sign(&mut processors, &participant_is, Session(0), plan_id).await;
-  })
+      let mut plan_id = [0; 32];
+      OsRng.fill_bytes(&mut plan_id);
+      let plan_id = plan_id;
+
+      // We should now get a SubstrateBlock
+      for processor in &mut processors {
+        assert_eq!(
+          processor.recv_message().await,
+          messages::CoordinatorMessage::Substrate(
+            messages::substrate::CoordinatorMessage::SubstrateBlock {
+              context: SubstrateContext {
+                serai_time: last_serai_block.time().unwrap() / 1000,
+                network_latest_finalized_block: coin_block,
+              },
+              block: last_serai_block.number(),
+              burns: vec![out_instruction.clone()],
+              batches: vec![],
+            }
+          )
+        );
+
+        // Send the ACK, claiming there's a plan to sign
+        processor
+          .send_message(messages::ProcessorMessage::Coordinator(
+            messages::coordinator::ProcessorMessage::SubstrateBlockAck {
+              block: last_serai_block.number(),
+              plans: vec![PlanMeta { session: Session(0), id: plan_id }],
+            },
+          ))
+          .await;
+      }
+
+      sign(&mut processors, &participant_is, Session(0), plan_id).await;
+    },
+    false,
+  )
   .await;
 }

tests/full-stack/src/tests/mod.rs

@@ -69,7 +69,7 @@ pub(crate) async fn new_test(test_body: impl TestBody) {
     let monero_processor_composition = monero_processor_composition.swap_remove(0);
 
     let coordinator_composition = coordinator_instance(name, coord_key);
-    let serai_composition = serai_composition(name);
+    let serai_composition = serai_composition(name, false);
 
     // Give every item in this stack a unique ID
     // Uses a Mutex as we can't generate a 8-byte random ID without hitting hostname length limits