feat: reconnect on transient link loss with per-peer supervisors

Audio links now survive a dropped QUIC connection instead of silently
dying until the peer leaves and rejoins the room.

Transport: each peer is owned by a single supervisor task running a
connect → run → reconnect loop. The deterministic-initiator rule (lower
id dials, higher accepts) holds on every reconnect, so one shared
connection re-forms each time; the dialer redials with capped backoff
and the acceptor awaits the inbound link, switching to a replacement
immediately if one arrives before its own close fires. Inbound
connections are routed to the supervisor via a per-peer channel.

Fixes a latent bug from the prior design: aborting a peer's tasks left
the detached send/read loops running, holding Connection clones so the
link never actually closed. The loops now live in abort-on-drop guards
scoped to the supervisor, so cancelling it releases the connection.

UX: a new ConnEvent stream surfaces per-peer link state to the UI, which
shows "Connecting…"/"Reconnecting…" with a yellow indicator and border
while a peer's audio link is down, returning to normal when it recovers.

Tests: the loopback test now runs through the supervisor path, plus a
new test drives a real drop (explicit close of a controlled peer
endpoint) and asserts the dialer re-dials the stable address and audio
resumes over the rebuilt connection.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
2026-05-29 17:01:21 -04:00
co-authored by Claude Opus 4.8
parent 26c3758d0d
commit ccf2bff87c
6 changed files with 477 additions and 81 deletions
+179 -2
View File
@@ -8,6 +8,8 @@
use std::sync::Arc;
use std::time::Duration;
use tokio::sync::mpsc;
use bytes::Bytes;
use iroh::address_lookup::memory::MemoryLookup;
use iroh::endpoint::presets;
@@ -15,10 +17,13 @@ use iroh::protocol::Router;
use iroh::{Endpoint, RelayMode};
use opus::{Application, Channels};
use iroh::endpoint::Connection;
use iroh::protocol::{AcceptError, ProtocolHandler};
use peerspeak::codec::AudioEncoder;
use peerspeak::codec::opus_impl::OpusEncoder;
use peerspeak::core::jitter::{FRAME_SAMPLES, JitterBuffer};
use peerspeak::network::NetworkTransport;
use peerspeak::network::{ConnEvent, NetworkTransport};
use peerspeak::network::iroh_impl::IrohTransport;
const AUDIO_ALPN: &[u8] = b"peerspeak-audio";
@@ -31,9 +36,72 @@ struct Node {
}
async fn spawn_node() -> Node {
spawn_node_with_key(iroh::SecretKey::generate()).await
}
/// A minimal protocol handler for the reconnect test's "peer" side: it hands
/// every accepted audio connection out through a channel so the test can hold
/// it, read datagrams from it, and explicitly `close()` it to drive a drop —
/// all on a single, stable endpoint address that the dialer can re-reach.
#[derive(Clone)]
struct CaptureProtocol {
conns_tx: mpsc::UnboundedSender<Connection>,
}
impl std::fmt::Debug for CaptureProtocol {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("CaptureProtocol").finish_non_exhaustive()
}
}
impl ProtocolHandler for CaptureProtocol {
fn accept(
&self,
connection: Connection,
) -> impl std::future::Future<Output = Result<(), AcceptError>> + Send {
let conns_tx = self.conns_tx.clone();
async move {
// The channel retains the connection, which keeps it open after this
// future resolves (iroh closes accepted connections with no retained
// handle).
let _ = conns_tx.send(connection);
Ok(())
}
}
}
struct CapturePeer {
endpoint: Endpoint,
_router: Router,
lookup: MemoryLookup,
conns_rx: mpsc::UnboundedReceiver<Connection>,
}
async fn spawn_capture_peer(secret: iroh::SecretKey) -> CapturePeer {
let lookup = MemoryLookup::new();
let endpoint = Endpoint::builder(presets::Minimal)
.secret_key(iroh::SecretKey::generate())
.secret_key(secret)
.relay_mode(RelayMode::Disabled)
.address_lookup(lookup.clone())
.bind()
.await
.expect("bind capture peer");
let (conns_tx, conns_rx) = mpsc::unbounded_channel();
let router = Router::builder(endpoint.clone())
.accept(AUDIO_ALPN, CaptureProtocol { conns_tx })
.spawn();
CapturePeer { endpoint, _router: router, lookup, conns_rx }
}
/// Spawn a node with a specific secret key. Reusing a key gives the respawned
/// node the same `EndpointId`, which is how we simulate a peer dropping off the
/// network and coming back (new address, same identity) for the reconnect test.
async fn spawn_node_with_key(secret: iroh::SecretKey) -> Node {
let lookup = MemoryLookup::new();
let endpoint = Endpoint::builder(presets::Minimal)
.secret_key(secret)
// Direct-only: two endpoints on the same host reach each other via the
// addresses we seed into each other's lookup, with no external relay.
.relay_mode(RelayMode::Disabled)
@@ -68,6 +136,24 @@ fn packet(enc: &mut OpusEncoder, seq: u32) -> Bytes {
Bytes::from(buf)
}
/// Drain link-state events until a `Connected` arrives, returning whether a
/// `Connecting` was seen first (the down→up transition). Panics on timeout so a
/// stuck reconnect fails loudly rather than hanging the suite.
async fn await_reconnect(
rx: &mut mpsc::Receiver<ConnEvent>,
deadline: tokio::time::Instant,
) -> bool {
let mut saw_connecting = false;
loop {
match tokio::time::timeout_at(deadline, rx.recv()).await {
Ok(Some(ConnEvent::Connecting(_))) => saw_connecting = true,
Ok(Some(ConnEvent::Connected(_))) => return saw_connecting,
Ok(None) => panic!("conn events channel closed unexpectedly"),
Err(_) => panic!("timed out waiting for Connected"),
}
}
}
#[tokio::test]
async fn loopback_sequenced_audio_reaches_peer_and_decodes() {
let a = spawn_node().await;
@@ -142,3 +228,94 @@ async fn loopback_sequenced_audio_reaches_peer_and_decodes() {
"expected to decode ~{N} frames, got {decoded_frames} (received {received})"
);
}
/// Read datagrams off a raw connection until `target` arrive or the deadline
/// passes, asserting each carries the 4-byte sequence header.
async fn count_audio(conn: &Connection, target: u32, deadline: tokio::time::Instant) -> u32 {
let mut received = 0;
while let Ok(Ok(bytes)) = tokio::time::timeout_at(deadline, conn.read_datagram()).await {
assert!(bytes.len() >= 4, "packet carries a sequence header");
received += 1;
if received >= target {
break;
}
}
received
}
#[tokio::test]
async fn dialer_reconnects_after_link_drops() {
// The unit under test is the IrohTransport dialer. The peer is a hand-rolled
// endpoint we fully control, so we can explicitly close the live connection
// (which the dialer observes promptly, unlike a silent handle drop) and then
// accept the dialer's re-dial — all on one stable address it can re-reach.
//
// The dialer must hold the lower id so the transport's deterministic rule
// makes it dial, so order the two identities up front.
let s1 = iroh::SecretKey::generate();
let s2 = iroh::SecretKey::generate();
let (dialer_secret, peer_secret) = if s1.public().to_string() < s2.public().to_string() {
(s1, s2)
} else {
(s2, s1)
};
let dialer = spawn_node_with_key(dialer_secret).await;
let mut peer = spawn_capture_peer(peer_secret).await;
let dialer_id = dialer.endpoint.id();
let peer_id = peer.endpoint.id();
assert!(dialer_id.to_string() < peer_id.to_string());
// Seed addresses both ways and form the initial link.
dialer.lookup.add_endpoint_info(peer.endpoint.addr());
peer.lookup.add_endpoint_info(dialer.endpoint.addr());
let mut conn_events = dialer
.transport
.subscribe_conn_events()
.await
.expect("subscribe dialer conn events");
dialer.transport.connect_peer(peer_id).await;
// The peer accepts the dialer's initial connection.
let conn1 = tokio::time::timeout(Duration::from_secs(10), peer.conns_rx.recv())
.await
.expect("timed out awaiting initial connection")
.expect("connection channel closed");
assert!(
await_reconnect(&mut conn_events, tokio::time::Instant::now() + Duration::from_secs(10)).await,
"initial link should report Connecting then Connected"
);
// Drop the live link by closing it from the peer side. An explicit close
// sends a CONNECTION_CLOSE the dialer sees right away.
conn1.close(0u32.into(), b"transient drop");
drop(conn1);
// The dialer's supervisor should re-dial the (unchanged) peer address, and
// the peer accepts the fresh connection.
let conn2 = tokio::time::timeout(Duration::from_secs(15), peer.conns_rx.recv())
.await
.expect("timed out awaiting reconnect")
.expect("connection channel closed");
assert!(
await_reconnect(&mut conn_events, tokio::time::Instant::now() + Duration::from_secs(15)).await,
"dropped link should report Connecting (down) then Connected (recovered)"
);
// Audio must flow over the rebuilt link. Give the freshly-registered send
// path a beat, then stream frames and confirm they reach the peer on conn2.
tokio::time::sleep(Duration::from_millis(200)).await;
let mut enc = OpusEncoder::new(48000, Channels::Mono, Application::Voip).unwrap();
for seq in 0..30u32 {
dialer.transport.broadcast(packet(&mut enc, seq));
tokio::time::sleep(Duration::from_millis(5)).await;
}
let received = count_audio(&conn2, 25, tokio::time::Instant::now() + Duration::from_secs(3)).await;
assert!(
received >= 20,
"audio should resume after reconnect; got {received} frames"
);
}