Add post-grace peer recovery coordinator

This commit is contained in:
2026-06-20 15:37:33 -04:00
parent 5564af02f9
commit 2c93c1c24f
7 changed files with 578 additions and 28 deletions
+137
View File
@@ -78,6 +78,31 @@ async fn await_joined(rx: &mut mpsc::Receiver<RoomEvent>, peer_id: EndpointId) -
}
}
async fn await_joined_all(rx: &mut mpsc::Receiver<RoomEvent>, peer_ids: &[EndpointId]) {
let deadline = tokio::time::Instant::now() + EVENT_TIMEOUT;
let mut remaining = peer_ids.to_vec();
while !remaining.is_empty() {
match tokio::time::timeout_at(deadline, rx.recv()).await {
Ok(Some(RoomEvent::PeerJoined(id, _))) => remaining.retain(|wanted| *wanted != id),
Ok(Some(_)) => {}
Ok(None) => panic!("room event channel closed while waiting for PeerJoined set"),
Err(_) => panic!("timed out waiting for PeerJoined set: {remaining:?}"),
}
}
}
async fn await_left(rx: &mut mpsc::Receiver<RoomEvent>, peer_id: EndpointId) {
let deadline = tokio::time::Instant::now() + EVENT_TIMEOUT;
loop {
match tokio::time::timeout_at(deadline, rx.recv()).await {
Ok(Some(RoomEvent::PeerLeft(id))) if id == peer_id => return,
Ok(Some(_)) => continue,
Ok(None) => panic!("room event channel closed while waiting for PeerLeft"),
Err(_) => panic!("timed out waiting for PeerLeft({peer_id:?})"),
}
}
}
async fn await_absent(room: &IrohGossipState, peer_id: EndpointId) {
let deadline = tokio::time::Instant::now() + EVENT_TIMEOUT;
loop {
@@ -215,3 +240,115 @@ async fn rebootstrap_uses_retained_full_address_with_empty_lookup() {
let recovered = await_joined(&mut events_a, b_id).await;
assert_eq!(recovered.name, "Bob rebound");
}
#[tokio::test]
async fn demoted_peer_requires_a_fresh_signed_announce_to_rejoin() {
let a = spawn_node(SecretKey::generate()).await;
let b = spawn_node(SecretKey::generate()).await;
let ticket = ticket(b.endpoint.addr());
let mut events_a = a.room.subscribe_events().await.expect("subscribe A events");
establish_room(&a, &b, &ticket, &mut events_a).await;
a.room.mark_peer_disconnected(b.endpoint.id());
assert!(
!a.room
.active_peers()
.iter()
.any(|(id, _)| *id == b.endpoint.id()),
"demotion must revoke live roster membership"
);
b.room
.update_self_state(state("Bob authenticated again", b.endpoint.addr()))
.await
.expect("broadcast fresh signed announce");
let recovered = await_joined(&mut events_a, b.endpoint.id()).await;
assert_eq!(recovered.name, "Bob authenticated again");
}
#[tokio::test]
async fn signed_leave_after_demotion_still_emits_peer_left() {
let a = spawn_node(SecretKey::generate()).await;
let b = spawn_node(SecretKey::generate()).await;
let ticket = ticket(b.endpoint.addr());
let mut events_a = a.room.subscribe_events().await.expect("subscribe A events");
establish_room(&a, &b, &ticket, &mut events_a).await;
a.room.mark_peer_disconnected(b.endpoint.id());
b.room
.leave()
.await
.expect("broadcast signed Leave after demotion");
await_left(&mut events_a, b.endpoint.id()).await;
}
#[tokio::test]
async fn targeted_rebootstrap_preserves_healthy_peer_in_three_peer_room() {
let a = spawn_node(SecretKey::generate()).await;
let b = spawn_node(SecretKey::generate()).await;
let c_secret = SecretKey::generate();
let c = spawn_node(c_secret.clone()).await;
let c_id = c.endpoint.id();
let ticket = ticket(c.endpoint.addr());
let mut events_a = a.room.subscribe_events().await.expect("subscribe A events");
let mut events_b = b.room.subscribe_events().await.expect("subscribe B events");
c.room
.join(&ticket, state("Carol", c.endpoint.addr()), vec![])
.await
.expect("C hosts topic");
b.room
.join(&ticket, state("Bob", b.endpoint.addr()), vec![])
.await
.expect("B joins C");
await_joined(&mut events_b, c_id).await;
a.room
.join(&ticket, state("Alice", a.endpoint.addr()), vec![])
.await
.expect("A joins C");
await_joined_all(&mut events_a, &[b.endpoint.id(), c_id]).await;
await_joined(&mut events_b, a.endpoint.id()).await;
// Remove only C. A and B keep their existing topic subscriptions and remain
// mutually present while C is rebound to a fresh address.
c.endpoint.close().await;
drop(c);
a.room.mark_peer_disconnected(c_id);
b.room.mark_peer_disconnected(c_id);
let c_rebound = spawn_node(c_secret).await;
let rebound_addr = c_rebound.endpoint.addr();
c_rebound
.room
.join(
&ticket,
state("Carol recovered", rebound_addr.clone()),
vec![],
)
.await
.expect("rebound C rejoins as host without bootstrap");
a.room
.rebootstrap_peers(vec![rebound_addr])
.await
.expect("A targets only C for recovery");
let recovered = await_joined(&mut events_a, c_id).await;
assert_eq!(recovered.name, "Carol recovered");
await_joined(&mut events_b, c_id).await;
assert!(
a.room
.active_peers()
.iter()
.any(|(id, _)| *id == b.endpoint.id()),
"healthy B must remain present at A throughout C recovery"
);
assert!(
b.room
.active_peers()
.iter()
.any(|(id, _)| *id == a.endpoint.id()),
"healthy A must remain present at B throughout C recovery"
);
}