Add post-grace peer recovery coordinator
This commit is contained in:
@@ -78,6 +78,31 @@ async fn await_joined(rx: &mut mpsc::Receiver<RoomEvent>, peer_id: EndpointId) -
|
||||
}
|
||||
}
|
||||
|
||||
async fn await_joined_all(rx: &mut mpsc::Receiver<RoomEvent>, peer_ids: &[EndpointId]) {
|
||||
let deadline = tokio::time::Instant::now() + EVENT_TIMEOUT;
|
||||
let mut remaining = peer_ids.to_vec();
|
||||
while !remaining.is_empty() {
|
||||
match tokio::time::timeout_at(deadline, rx.recv()).await {
|
||||
Ok(Some(RoomEvent::PeerJoined(id, _))) => remaining.retain(|wanted| *wanted != id),
|
||||
Ok(Some(_)) => {}
|
||||
Ok(None) => panic!("room event channel closed while waiting for PeerJoined set"),
|
||||
Err(_) => panic!("timed out waiting for PeerJoined set: {remaining:?}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn await_left(rx: &mut mpsc::Receiver<RoomEvent>, peer_id: EndpointId) {
|
||||
let deadline = tokio::time::Instant::now() + EVENT_TIMEOUT;
|
||||
loop {
|
||||
match tokio::time::timeout_at(deadline, rx.recv()).await {
|
||||
Ok(Some(RoomEvent::PeerLeft(id))) if id == peer_id => return,
|
||||
Ok(Some(_)) => continue,
|
||||
Ok(None) => panic!("room event channel closed while waiting for PeerLeft"),
|
||||
Err(_) => panic!("timed out waiting for PeerLeft({peer_id:?})"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn await_absent(room: &IrohGossipState, peer_id: EndpointId) {
|
||||
let deadline = tokio::time::Instant::now() + EVENT_TIMEOUT;
|
||||
loop {
|
||||
@@ -215,3 +240,115 @@ async fn rebootstrap_uses_retained_full_address_with_empty_lookup() {
|
||||
let recovered = await_joined(&mut events_a, b_id).await;
|
||||
assert_eq!(recovered.name, "Bob rebound");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn demoted_peer_requires_a_fresh_signed_announce_to_rejoin() {
|
||||
let a = spawn_node(SecretKey::generate()).await;
|
||||
let b = spawn_node(SecretKey::generate()).await;
|
||||
let ticket = ticket(b.endpoint.addr());
|
||||
let mut events_a = a.room.subscribe_events().await.expect("subscribe A events");
|
||||
|
||||
establish_room(&a, &b, &ticket, &mut events_a).await;
|
||||
a.room.mark_peer_disconnected(b.endpoint.id());
|
||||
assert!(
|
||||
!a.room
|
||||
.active_peers()
|
||||
.iter()
|
||||
.any(|(id, _)| *id == b.endpoint.id()),
|
||||
"demotion must revoke live roster membership"
|
||||
);
|
||||
|
||||
b.room
|
||||
.update_self_state(state("Bob authenticated again", b.endpoint.addr()))
|
||||
.await
|
||||
.expect("broadcast fresh signed announce");
|
||||
|
||||
let recovered = await_joined(&mut events_a, b.endpoint.id()).await;
|
||||
assert_eq!(recovered.name, "Bob authenticated again");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn signed_leave_after_demotion_still_emits_peer_left() {
|
||||
let a = spawn_node(SecretKey::generate()).await;
|
||||
let b = spawn_node(SecretKey::generate()).await;
|
||||
let ticket = ticket(b.endpoint.addr());
|
||||
let mut events_a = a.room.subscribe_events().await.expect("subscribe A events");
|
||||
|
||||
establish_room(&a, &b, &ticket, &mut events_a).await;
|
||||
a.room.mark_peer_disconnected(b.endpoint.id());
|
||||
|
||||
b.room
|
||||
.leave()
|
||||
.await
|
||||
.expect("broadcast signed Leave after demotion");
|
||||
await_left(&mut events_a, b.endpoint.id()).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn targeted_rebootstrap_preserves_healthy_peer_in_three_peer_room() {
|
||||
let a = spawn_node(SecretKey::generate()).await;
|
||||
let b = spawn_node(SecretKey::generate()).await;
|
||||
let c_secret = SecretKey::generate();
|
||||
let c = spawn_node(c_secret.clone()).await;
|
||||
let c_id = c.endpoint.id();
|
||||
let ticket = ticket(c.endpoint.addr());
|
||||
let mut events_a = a.room.subscribe_events().await.expect("subscribe A events");
|
||||
let mut events_b = b.room.subscribe_events().await.expect("subscribe B events");
|
||||
|
||||
c.room
|
||||
.join(&ticket, state("Carol", c.endpoint.addr()), vec![])
|
||||
.await
|
||||
.expect("C hosts topic");
|
||||
b.room
|
||||
.join(&ticket, state("Bob", b.endpoint.addr()), vec![])
|
||||
.await
|
||||
.expect("B joins C");
|
||||
await_joined(&mut events_b, c_id).await;
|
||||
a.room
|
||||
.join(&ticket, state("Alice", a.endpoint.addr()), vec![])
|
||||
.await
|
||||
.expect("A joins C");
|
||||
await_joined_all(&mut events_a, &[b.endpoint.id(), c_id]).await;
|
||||
await_joined(&mut events_b, a.endpoint.id()).await;
|
||||
|
||||
// Remove only C. A and B keep their existing topic subscriptions and remain
|
||||
// mutually present while C is rebound to a fresh address.
|
||||
c.endpoint.close().await;
|
||||
drop(c);
|
||||
a.room.mark_peer_disconnected(c_id);
|
||||
b.room.mark_peer_disconnected(c_id);
|
||||
let c_rebound = spawn_node(c_secret).await;
|
||||
let rebound_addr = c_rebound.endpoint.addr();
|
||||
c_rebound
|
||||
.room
|
||||
.join(
|
||||
&ticket,
|
||||
state("Carol recovered", rebound_addr.clone()),
|
||||
vec![],
|
||||
)
|
||||
.await
|
||||
.expect("rebound C rejoins as host without bootstrap");
|
||||
|
||||
a.room
|
||||
.rebootstrap_peers(vec![rebound_addr])
|
||||
.await
|
||||
.expect("A targets only C for recovery");
|
||||
let recovered = await_joined(&mut events_a, c_id).await;
|
||||
assert_eq!(recovered.name, "Carol recovered");
|
||||
await_joined(&mut events_b, c_id).await;
|
||||
|
||||
assert!(
|
||||
a.room
|
||||
.active_peers()
|
||||
.iter()
|
||||
.any(|(id, _)| *id == b.endpoint.id()),
|
||||
"healthy B must remain present at A throughout C recovery"
|
||||
);
|
||||
assert!(
|
||||
b.room
|
||||
.active_peers()
|
||||
.iter()
|
||||
.any(|(id, _)| *id == a.endpoint.id()),
|
||||
"healthy A must remain present at B throughout C recovery"
|
||||
);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user