Ecosystem/serai
Ecosystem/serai
1
0
Fork 0
mirror of https://github.com/serai-dex/serai.git synced 2025-01-10 21:04:40 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

P2P task per Tributary, not per message

Browse source
This commit is contained in:
Luke Parker 2023-09-25 22:58:40 -04:00
parent e1801b57c9
commit 7312428a44
No known key found for this signature in database
1 changed files with 119 additions and 114 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

233
coordinator/src/main.rs
View file

@ -346,134 +346,139 @@ pub async fn handle_p2p<D: Db, P: P2p>(
p2p: P, p2p: P,
mut new_tributary: broadcast::Receiver<ActiveTributary<D, P>>, mut new_tributary: broadcast::Receiver<ActiveTributary<D, P>>,
) { ) {
let tributaries = Arc::new(RwLock::new(HashMap::new())); let channels = Arc::new(RwLock::new(HashMap::new()));
loop { tokio::spawn({
while let Ok(tributary) = { let p2p = p2p.clone();
match new_tributary.try_recv() { let channels = channels.clone();
Ok(tributary) => Ok(tributary), async move {
Err(broadcast::error::TryRecvError::Empty) => Err(()), loop {
Err(broadcast::error::TryRecvError::Lagged(_)) => { let tributary = new_tributary.recv().await.unwrap();
panic!("handle_p2p lagged to handle new_tributary") let genesis = tributary.spec.genesis();
}
Err(broadcast::error::TryRecvError::Closed) => panic!("new_tributary sender closed"),
}
} {
// TODO: Because the below maintains a read lock, this will never process until all prior P2P
// messages were handled. That's a notable latency spike
tributaries.write().await.insert(tributary.spec.genesis(), tributary);
}
let mut msg = p2p.receive().await; let (send, mut recv) = mpsc::unbounded_channel();
// Spawn a dedicated task to handle this message, ensuring any singularly latent message channels.write().await.insert(genesis, send);
// doesn't hold everything up
// TODO: Move to one task per tributary (or two. One for Tendermint, one for Tributary)
tokio::spawn({
let p2p = p2p.clone();
let tributaries = tributaries.clone();
async move {
match msg.kind {
P2pMessageKind::KeepAlive => {}
P2pMessageKind::Tributary(genesis) => { tokio::spawn({
let tributaries = tributaries.read().await; let p2p = p2p.clone();
let Some(tributary) = tributaries.get(&genesis) else { async move {
log::debug!("received p2p message for unknown network"); let mut msg: Message<P> = recv.recv().await.unwrap();
return; match msg.kind {
}; P2pMessageKind::KeepAlive => {}
log::trace!("handling message for tributary {:?}", tributary.spec.set()); P2pMessageKind::Tributary(msg_genesis) => {
if tributary.tributary.handle_message(&msg.msg).await { assert_eq!(msg_genesis, genesis);
P2p::broadcast(&p2p, msg.kind, msg.msg).await; log::trace!("handling message for tributary {:?}", tributary.spec.set());
} if tributary.tributary.handle_message(&msg.msg).await {
} P2p::broadcast(&p2p, msg.kind, msg.msg).await;
}
}
// TODO2: Rate limit this per timestamp // TODO2: Rate limit this per timestamp
// And/or slash on Heartbeat which justifies a response, since the node obviously was // And/or slash on Heartbeat which justifies a response, since the node obviously was
// offline and we must now use our bandwidth to compensate for them? // offline and we must now use our bandwidth to compensate for them?
P2pMessageKind::Heartbeat(genesis) => { // TODO: Dedicated task for heartbeats
if msg.msg.len() != 40 { P2pMessageKind::Heartbeat(msg_genesis) => {
log::error!("validator sent invalid heartbeat"); assert_eq!(msg_genesis, genesis);
return; if msg.msg.len() != 40 {
} log::error!("validator sent invalid heartbeat");
return;
}
let tributaries = tributaries.read().await; let tributary_read = &tributary.tributary;
let Some(tributary) = tributaries.get(&genesis) else {
log::debug!("received heartbeat message for unknown network");
return;
};
let tributary_read = &tributary.tributary;
/* /*
// Have sqrt(n) nodes reply with the blocks // Have sqrt(n) nodes reply with the blocks
let mut responders = (tributary.spec.n() as f32).sqrt().floor() as u64; let mut responders = (tributary.spec.n() as f32).sqrt().floor() as u64;
// Try to have at least 3 responders // Try to have at least 3 responders
if responders < 3 { if responders < 3 {
responders = tributary.spec.n().min(3).into(); responders = tributary.spec.n().min(3).into();
} }
*/ */
// Have up to three nodes respond // Have up to three nodes respond
let responders = u64::from(tributary.spec.n().min(3)); let responders = u64::from(tributary.spec.n().min(3));
// Decide which nodes will respond by using the latest block's hash as a mutually // Decide which nodes will respond by using the latest block's hash as a mutually
// agreed upon entropy source // agreed upon entropy source
// This isn't a secure source of entropy, yet it's fine for this // This isn't a secure source of entropy, yet it's fine for this
let entropy = u64::from_le_bytes(tributary_read.tip().await[.. 8].try_into().unwrap()); let entropy =
// If n = 10, responders = 3, we want start to be 0 ..= 7 (so the highest is 7, 8, 9) u64::from_le_bytes(tributary_read.tip().await[.. 8].try_into().unwrap());
// entropy % (10 + 1) - 3 = entropy % 8 = 0 ..= 7 // If n = 10, responders = 3, we want `start` to be 0 ..= 7
let start = // (so the highest is 7, 8, 9)
usize::try_from(entropy % (u64::from(tributary.spec.n() + 1) - responders)).unwrap(); // entropy % (10 + 1) - 3 = entropy % 8 = 0 ..= 7
let mut selected = false; let start =
for validator in usize::try_from(entropy % (u64::from(tributary.spec.n() + 1) - responders))
&tributary.spec.validators()[start .. (start + usize::try_from(responders).unwrap())] .unwrap();
{ let mut selected = false;
if our_key == validator.0 { for validator in &tributary.spec.validators()
selected = true; [start .. (start + usize::try_from(responders).unwrap())]
break; {
if our_key == validator.0 {
selected = true;
break;
}
}
if !selected {
log::debug!("received heartbeat and not selected to respond");
return;
}
log::debug!("received heartbeat and selected to respond");
let reader = tributary_read.reader();
let mut latest = msg.msg[.. 32].try_into().unwrap();
while let Some(next) = reader.block_after(&latest) {
let mut res = reader.block(&next).unwrap().serialize();
res.extend(reader.commit(&next).unwrap());
// Also include the timestamp used within the Heartbeat
res.extend(&msg.msg[32 .. 40]);
p2p.send(msg.sender, P2pMessageKind::Block(tributary.spec.genesis()), res).await;
latest = next;
}
}
P2pMessageKind::Block(msg_genesis) => {
assert_eq!(msg_genesis, genesis);
let mut msg_ref: &[u8] = msg.msg.as_ref();
let Ok(block) = Block::<Transaction>::read(&mut msg_ref) else {
log::error!("received block message with an invalidly serialized block");
return;
};
// Get just the commit
msg.msg.drain(.. (msg.msg.len() - msg_ref.len()));
msg.msg.drain((msg.msg.len() - 8) ..);
let res = tributary.tributary.sync_block(block, msg.msg).await;
log::debug!("received block from {:?}, sync_block returned {}", msg.sender, res);
} }
} }
if !selected {
log::debug!("received heartbeat and not selected to respond");
return;
}
log::debug!("received heartbeat and selected to respond");
let reader = tributary_read.reader();
let mut latest = msg.msg[.. 32].try_into().unwrap();
while let Some(next) = reader.block_after(&latest) {
let mut res = reader.block(&next).unwrap().serialize();
res.extend(reader.commit(&next).unwrap());
// Also include the timestamp used within the Heartbeat
res.extend(&msg.msg[32 .. 40]);
p2p.send(msg.sender, P2pMessageKind::Block(tributary.spec.genesis()), res).await;
latest = next;
}
} }
});
}
}
});
P2pMessageKind::Block(genesis) => { loop {
let mut msg_ref: &[u8] = msg.msg.as_ref(); let msg = p2p.receive().await;
let Ok(block) = Block::<Transaction>::read(&mut msg_ref) else { match msg.kind {
log::error!("received block message with an invalidly serialized block"); P2pMessageKind::KeepAlive => {}
return; P2pMessageKind::Tributary(genesis) => {
}; if let Some(channel) = channels.read().await.get(&genesis) {
// Get just the commit channel.send(msg).unwrap();
msg.msg.drain(.. (msg.msg.len() - msg_ref.len()));
msg.msg.drain((msg.msg.len() - 8) ..);
let tributaries = tributaries.read().await;
let Some(tributary) = tributaries.get(&genesis) else {
log::debug!("received block message for unknown network");
return;
};
let res = tributary.tributary.sync_block(block, msg.msg).await;
log::debug!("received block from {:?}, sync_block returned {}", msg.sender, res);
}
} }
} }
}); P2pMessageKind::Heartbeat(genesis) => {
if let Some(channel) = channels.read().await.get(&genesis) {
channel.send(msg).unwrap();
}
}
P2pMessageKind::Block(genesis) => {
if let Some(channel) = channels.read().await.get(&genesis) {
channel.send(msg).unwrap();
}
}
}
} }
} }