test: regression coverage for transport-driven reconnect eviction
Extract the conn-event handling into a testable ConnEventHandler (grace window as a field) and add tests/reconnect_eviction.rs covering: - second outage after a reconnect still evicts (the bug fixed in bc1a0a2; proven to fail when the transport-arming is disabled) - a reconnect within grace is not evicted - a first-ever dial is not given an eviction timer - a graceful leave scrubs seen_connected so a later rejoin dials cleanly Behavior-preserving refactor: the conn-event task now builds a ConnEventHandler and forwards each event to it; arm_grace_timer takes the grace Duration as a param (production passes RECONNECT_GRACE). All existing transport/reconnect integration tests still pass; clippy clean. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
+92
-46
@@ -83,6 +83,7 @@ fn arm_grace_timer(
|
||||
transport: &Arc<IrohTransport>,
|
||||
jitter: &Arc<Mutex<HashMap<EndpointId, JitterBuffer>>>,
|
||||
ui_tx: &mpsc::Sender<UiEvent>,
|
||||
grace: Duration,
|
||||
peer_id: EndpointId,
|
||||
) {
|
||||
let mut timers_guard = timers.lock().unwrap();
|
||||
@@ -95,7 +96,7 @@ fn arm_grace_timer(
|
||||
let timers_evict = timers.clone();
|
||||
let seen_evict = seen_connected.clone();
|
||||
let handle = tokio::spawn(async move {
|
||||
tokio::time::sleep(RECONNECT_GRACE).await;
|
||||
tokio::time::sleep(grace).await;
|
||||
crate::log_msg(&format!("Reconnect grace expired; evicting peer {:?}", peer_id));
|
||||
transport_evict.disconnect_peer(peer_id).await;
|
||||
jitter_evict.lock().await.remove(&peer_id);
|
||||
@@ -106,6 +107,87 @@ fn arm_grace_timer(
|
||||
timers_guard.insert(peer_id, handle);
|
||||
}
|
||||
|
||||
/// Handles the transport's per-peer link-state stream (`ConnEvent`): arms/cancels
|
||||
/// reconnect grace timers, tracks which peers we've linked with, and forwards
|
||||
/// link state to the UI. Pulled out of the conn-event task as a unit so the
|
||||
/// reconnect-eviction behavior can be tested without standing up a full session.
|
||||
/// Exposed (with a `grace` override) for that purpose; not part of the public API.
|
||||
pub struct ConnEventHandler {
|
||||
ui_tx: mpsc::Sender<UiEvent>,
|
||||
grace_timers: GraceTimers,
|
||||
seen_connected: SeenConnected,
|
||||
transport: Arc<IrohTransport>,
|
||||
jitter: Arc<Mutex<HashMap<EndpointId, JitterBuffer>>>,
|
||||
grace: Duration,
|
||||
}
|
||||
|
||||
impl ConnEventHandler {
|
||||
pub fn new(
|
||||
ui_tx: mpsc::Sender<UiEvent>,
|
||||
grace_timers: GraceTimers,
|
||||
seen_connected: SeenConnected,
|
||||
transport: Arc<IrohTransport>,
|
||||
jitter: Arc<Mutex<HashMap<EndpointId, JitterBuffer>>>,
|
||||
) -> Self {
|
||||
Self {
|
||||
ui_tx,
|
||||
grace_timers,
|
||||
seen_connected,
|
||||
transport,
|
||||
jitter,
|
||||
grace: RECONNECT_GRACE,
|
||||
}
|
||||
}
|
||||
|
||||
/// Override the eviction grace window. For tests that can't wait 45s.
|
||||
pub fn with_grace(mut self, grace: Duration) -> Self {
|
||||
self.grace = grace;
|
||||
self
|
||||
}
|
||||
|
||||
pub async fn handle(&self, event: ConnEvent) {
|
||||
match event {
|
||||
ConnEvent::Connecting(id) => {
|
||||
// A reconnect (we've linked with this peer before): arm an eviction
|
||||
// timer so a peer that never comes back is cleared even when gossip
|
||||
// doesn't re-report the drop — the transport reliably re-emits this
|
||||
// on every outage, gossip's NeighborDown does not. A first-ever dial
|
||||
// (not yet in seen_connected) gets no timer; Connected cancels it on
|
||||
// recovery.
|
||||
if self.seen_connected.lock().unwrap().contains(&id) {
|
||||
arm_grace_timer(
|
||||
&self.grace_timers,
|
||||
&self.seen_connected,
|
||||
&self.transport,
|
||||
&self.jitter,
|
||||
&self.ui_tx,
|
||||
self.grace,
|
||||
id,
|
||||
);
|
||||
}
|
||||
let _ = self.ui_tx.send(UiEvent::PeerConnecting { id }).await;
|
||||
}
|
||||
ConnEvent::Connected(id) => {
|
||||
// The audio link came back — the peer recovered within the grace
|
||||
// window, so cancel its eviction.
|
||||
cancel_grace_timer(&self.grace_timers, &id);
|
||||
self.seen_connected.lock().unwrap().insert(id);
|
||||
let _ = self.ui_tx.send(UiEvent::PeerConnected { id }).await;
|
||||
}
|
||||
ConnEvent::Left(id) => {
|
||||
// The peer closed its link gracefully (intentional leave) — evict
|
||||
// immediately, like a PeerLeft, instead of leaving it "reconnecting"
|
||||
// until the grace timer or the slow gossip Leave.
|
||||
cancel_grace_timer(&self.grace_timers, &id);
|
||||
self.seen_connected.lock().unwrap().remove(&id);
|
||||
self.transport.disconnect_peer(id).await;
|
||||
self.jitter.lock().await.remove(&id);
|
||||
let _ = self.ui_tx.send(UiEvent::PeerLeft { id }).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct ActiveSession {
|
||||
endpoint: Endpoint,
|
||||
router: Router,
|
||||
@@ -549,6 +631,7 @@ async fn run_core_loop(
|
||||
&transport_events,
|
||||
&jitter_events,
|
||||
&ui_tx_events,
|
||||
RECONNECT_GRACE,
|
||||
peer_id,
|
||||
);
|
||||
}
|
||||
@@ -565,53 +648,16 @@ async fn run_core_loop(
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let ui_tx_conn = ui_tx.clone();
|
||||
let grace_timers_conn = grace_timers.clone();
|
||||
let seen_connected_conn = seen_connected.clone();
|
||||
let transport_conn = transport.clone();
|
||||
let jitter_conn = jitter.clone();
|
||||
let conn_handler = ConnEventHandler::new(
|
||||
ui_tx.clone(),
|
||||
grace_timers.clone(),
|
||||
seen_connected.clone(),
|
||||
transport.clone(),
|
||||
jitter.clone(),
|
||||
);
|
||||
let conn_event_task = tokio::spawn(async move {
|
||||
while let Some(event) = conn_events.recv().await {
|
||||
match event {
|
||||
ConnEvent::Connecting(id) => {
|
||||
// A reconnect (we've linked with this peer before):
|
||||
// arm an eviction timer so a peer that never comes
|
||||
// back is cleared even when gossip doesn't re-report
|
||||
// the drop — the transport reliably re-emits this on
|
||||
// every outage, gossip's NeighborDown does not. A
|
||||
// first-ever dial (not yet in seen_connected) gets no
|
||||
// timer; ConnEvent::Connected cancels it on recovery.
|
||||
if seen_connected_conn.lock().unwrap().contains(&id) {
|
||||
arm_grace_timer(
|
||||
&grace_timers_conn,
|
||||
&seen_connected_conn,
|
||||
&transport_conn,
|
||||
&jitter_conn,
|
||||
&ui_tx_conn,
|
||||
id,
|
||||
);
|
||||
}
|
||||
let _ = ui_tx_conn.send(UiEvent::PeerConnecting { id }).await;
|
||||
}
|
||||
ConnEvent::Connected(id) => {
|
||||
// The audio link came back — the peer recovered
|
||||
// within the grace window, so cancel its eviction.
|
||||
cancel_grace_timer(&grace_timers_conn, &id);
|
||||
seen_connected_conn.lock().unwrap().insert(id);
|
||||
let _ = ui_tx_conn.send(UiEvent::PeerConnected { id }).await;
|
||||
}
|
||||
ConnEvent::Left(id) => {
|
||||
// The peer closed its link gracefully (intentional
|
||||
// leave) — evict immediately, like a PeerLeft,
|
||||
// instead of leaving it "reconnecting" until the
|
||||
// grace timer or the slow gossip Leave.
|
||||
cancel_grace_timer(&grace_timers_conn, &id);
|
||||
seen_connected_conn.lock().unwrap().remove(&id);
|
||||
transport_conn.disconnect_peer(id).await;
|
||||
jitter_conn.lock().await.remove(&id);
|
||||
let _ = ui_tx_conn.send(UiEvent::PeerLeft { id }).await;
|
||||
}
|
||||
}
|
||||
conn_handler.handle(event).await;
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
Reference in New Issue
Block a user