feat: reconnect on transient link loss with per-peer supervisors
Audio links now survive a dropped QUIC connection instead of silently dying until the peer leaves and rejoins the room. Transport: each peer is owned by a single supervisor task running a connect → run → reconnect loop. The deterministic-initiator rule (lower id dials, higher accepts) holds on every reconnect, so one shared connection re-forms each time; the dialer redials with capped backoff and the acceptor awaits the inbound link, switching to a replacement immediately if one arrives before its own close fires. Inbound connections are routed to the supervisor via a per-peer channel. Fixes a latent bug from the prior design: aborting a peer's tasks left the detached send/read loops running, holding Connection clones so the link never actually closed. The loops now live in abort-on-drop guards scoped to the supervisor, so cancelling it releases the connection. UX: a new ConnEvent stream surfaces per-peer link state to the UI, which shows "Connecting…"/"Reconnecting…" with a yellow indicator and border while a peer's audio link is down, returning to normal when it recovers. Tests: the loopback test now runs through the supervisor path, plus a new test drives a real drop (explicit close of a controlled peer endpoint) and asserts the dialer re-dials the stable address and audio resumes over the rebuilt connection. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
+179
-2
@@ -8,6 +8,8 @@
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use tokio::sync::mpsc;
|
||||
|
||||
use bytes::Bytes;
|
||||
use iroh::address_lookup::memory::MemoryLookup;
|
||||
use iroh::endpoint::presets;
|
||||
@@ -15,10 +17,13 @@ use iroh::protocol::Router;
|
||||
use iroh::{Endpoint, RelayMode};
|
||||
use opus::{Application, Channels};
|
||||
|
||||
use iroh::endpoint::Connection;
|
||||
use iroh::protocol::{AcceptError, ProtocolHandler};
|
||||
|
||||
use peerspeak::codec::AudioEncoder;
|
||||
use peerspeak::codec::opus_impl::OpusEncoder;
|
||||
use peerspeak::core::jitter::{FRAME_SAMPLES, JitterBuffer};
|
||||
use peerspeak::network::NetworkTransport;
|
||||
use peerspeak::network::{ConnEvent, NetworkTransport};
|
||||
use peerspeak::network::iroh_impl::IrohTransport;
|
||||
|
||||
const AUDIO_ALPN: &[u8] = b"peerspeak-audio";
|
||||
@@ -31,9 +36,72 @@ struct Node {
|
||||
}
|
||||
|
||||
async fn spawn_node() -> Node {
|
||||
spawn_node_with_key(iroh::SecretKey::generate()).await
|
||||
}
|
||||
|
||||
/// A minimal protocol handler for the reconnect test's "peer" side: it hands
|
||||
/// every accepted audio connection out through a channel so the test can hold
|
||||
/// it, read datagrams from it, and explicitly `close()` it to drive a drop —
|
||||
/// all on a single, stable endpoint address that the dialer can re-reach.
|
||||
#[derive(Clone)]
|
||||
struct CaptureProtocol {
|
||||
conns_tx: mpsc::UnboundedSender<Connection>,
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for CaptureProtocol {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("CaptureProtocol").finish_non_exhaustive()
|
||||
}
|
||||
}
|
||||
|
||||
impl ProtocolHandler for CaptureProtocol {
|
||||
fn accept(
|
||||
&self,
|
||||
connection: Connection,
|
||||
) -> impl std::future::Future<Output = Result<(), AcceptError>> + Send {
|
||||
let conns_tx = self.conns_tx.clone();
|
||||
async move {
|
||||
// The channel retains the connection, which keeps it open after this
|
||||
// future resolves (iroh closes accepted connections with no retained
|
||||
// handle).
|
||||
let _ = conns_tx.send(connection);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct CapturePeer {
|
||||
endpoint: Endpoint,
|
||||
_router: Router,
|
||||
lookup: MemoryLookup,
|
||||
conns_rx: mpsc::UnboundedReceiver<Connection>,
|
||||
}
|
||||
|
||||
async fn spawn_capture_peer(secret: iroh::SecretKey) -> CapturePeer {
|
||||
let lookup = MemoryLookup::new();
|
||||
let endpoint = Endpoint::builder(presets::Minimal)
|
||||
.secret_key(iroh::SecretKey::generate())
|
||||
.secret_key(secret)
|
||||
.relay_mode(RelayMode::Disabled)
|
||||
.address_lookup(lookup.clone())
|
||||
.bind()
|
||||
.await
|
||||
.expect("bind capture peer");
|
||||
|
||||
let (conns_tx, conns_rx) = mpsc::unbounded_channel();
|
||||
let router = Router::builder(endpoint.clone())
|
||||
.accept(AUDIO_ALPN, CaptureProtocol { conns_tx })
|
||||
.spawn();
|
||||
|
||||
CapturePeer { endpoint, _router: router, lookup, conns_rx }
|
||||
}
|
||||
|
||||
/// Spawn a node with a specific secret key. Reusing a key gives the respawned
|
||||
/// node the same `EndpointId`, which is how we simulate a peer dropping off the
|
||||
/// network and coming back (new address, same identity) for the reconnect test.
|
||||
async fn spawn_node_with_key(secret: iroh::SecretKey) -> Node {
|
||||
let lookup = MemoryLookup::new();
|
||||
let endpoint = Endpoint::builder(presets::Minimal)
|
||||
.secret_key(secret)
|
||||
// Direct-only: two endpoints on the same host reach each other via the
|
||||
// addresses we seed into each other's lookup, with no external relay.
|
||||
.relay_mode(RelayMode::Disabled)
|
||||
@@ -68,6 +136,24 @@ fn packet(enc: &mut OpusEncoder, seq: u32) -> Bytes {
|
||||
Bytes::from(buf)
|
||||
}
|
||||
|
||||
/// Drain link-state events until a `Connected` arrives, returning whether a
|
||||
/// `Connecting` was seen first (the down→up transition). Panics on timeout so a
|
||||
/// stuck reconnect fails loudly rather than hanging the suite.
|
||||
async fn await_reconnect(
|
||||
rx: &mut mpsc::Receiver<ConnEvent>,
|
||||
deadline: tokio::time::Instant,
|
||||
) -> bool {
|
||||
let mut saw_connecting = false;
|
||||
loop {
|
||||
match tokio::time::timeout_at(deadline, rx.recv()).await {
|
||||
Ok(Some(ConnEvent::Connecting(_))) => saw_connecting = true,
|
||||
Ok(Some(ConnEvent::Connected(_))) => return saw_connecting,
|
||||
Ok(None) => panic!("conn events channel closed unexpectedly"),
|
||||
Err(_) => panic!("timed out waiting for Connected"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn loopback_sequenced_audio_reaches_peer_and_decodes() {
|
||||
let a = spawn_node().await;
|
||||
@@ -142,3 +228,94 @@ async fn loopback_sequenced_audio_reaches_peer_and_decodes() {
|
||||
"expected to decode ~{N} frames, got {decoded_frames} (received {received})"
|
||||
);
|
||||
}
|
||||
|
||||
/// Read datagrams off a raw connection until `target` arrive or the deadline
|
||||
/// passes, asserting each carries the 4-byte sequence header.
|
||||
async fn count_audio(conn: &Connection, target: u32, deadline: tokio::time::Instant) -> u32 {
|
||||
let mut received = 0;
|
||||
while let Ok(Ok(bytes)) = tokio::time::timeout_at(deadline, conn.read_datagram()).await {
|
||||
assert!(bytes.len() >= 4, "packet carries a sequence header");
|
||||
received += 1;
|
||||
if received >= target {
|
||||
break;
|
||||
}
|
||||
}
|
||||
received
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn dialer_reconnects_after_link_drops() {
|
||||
// The unit under test is the IrohTransport dialer. The peer is a hand-rolled
|
||||
// endpoint we fully control, so we can explicitly close the live connection
|
||||
// (which the dialer observes promptly, unlike a silent handle drop) and then
|
||||
// accept the dialer's re-dial — all on one stable address it can re-reach.
|
||||
//
|
||||
// The dialer must hold the lower id so the transport's deterministic rule
|
||||
// makes it dial, so order the two identities up front.
|
||||
let s1 = iroh::SecretKey::generate();
|
||||
let s2 = iroh::SecretKey::generate();
|
||||
let (dialer_secret, peer_secret) = if s1.public().to_string() < s2.public().to_string() {
|
||||
(s1, s2)
|
||||
} else {
|
||||
(s2, s1)
|
||||
};
|
||||
|
||||
let dialer = spawn_node_with_key(dialer_secret).await;
|
||||
let mut peer = spawn_capture_peer(peer_secret).await;
|
||||
|
||||
let dialer_id = dialer.endpoint.id();
|
||||
let peer_id = peer.endpoint.id();
|
||||
assert!(dialer_id.to_string() < peer_id.to_string());
|
||||
|
||||
// Seed addresses both ways and form the initial link.
|
||||
dialer.lookup.add_endpoint_info(peer.endpoint.addr());
|
||||
peer.lookup.add_endpoint_info(dialer.endpoint.addr());
|
||||
|
||||
let mut conn_events = dialer
|
||||
.transport
|
||||
.subscribe_conn_events()
|
||||
.await
|
||||
.expect("subscribe dialer conn events");
|
||||
dialer.transport.connect_peer(peer_id).await;
|
||||
|
||||
// The peer accepts the dialer's initial connection.
|
||||
let conn1 = tokio::time::timeout(Duration::from_secs(10), peer.conns_rx.recv())
|
||||
.await
|
||||
.expect("timed out awaiting initial connection")
|
||||
.expect("connection channel closed");
|
||||
assert!(
|
||||
await_reconnect(&mut conn_events, tokio::time::Instant::now() + Duration::from_secs(10)).await,
|
||||
"initial link should report Connecting then Connected"
|
||||
);
|
||||
|
||||
// Drop the live link by closing it from the peer side. An explicit close
|
||||
// sends a CONNECTION_CLOSE the dialer sees right away.
|
||||
conn1.close(0u32.into(), b"transient drop");
|
||||
drop(conn1);
|
||||
|
||||
// The dialer's supervisor should re-dial the (unchanged) peer address, and
|
||||
// the peer accepts the fresh connection.
|
||||
let conn2 = tokio::time::timeout(Duration::from_secs(15), peer.conns_rx.recv())
|
||||
.await
|
||||
.expect("timed out awaiting reconnect")
|
||||
.expect("connection channel closed");
|
||||
assert!(
|
||||
await_reconnect(&mut conn_events, tokio::time::Instant::now() + Duration::from_secs(15)).await,
|
||||
"dropped link should report Connecting (down) then Connected (recovered)"
|
||||
);
|
||||
|
||||
// Audio must flow over the rebuilt link. Give the freshly-registered send
|
||||
// path a beat, then stream frames and confirm they reach the peer on conn2.
|
||||
tokio::time::sleep(Duration::from_millis(200)).await;
|
||||
let mut enc = OpusEncoder::new(48000, Channels::Mono, Application::Voip).unwrap();
|
||||
for seq in 0..30u32 {
|
||||
dialer.transport.broadcast(packet(&mut enc, seq));
|
||||
tokio::time::sleep(Duration::from_millis(5)).await;
|
||||
}
|
||||
|
||||
let received = count_audio(&conn2, 25, tokio::time::Instant::now() + Duration::from_secs(3)).await;
|
||||
assert!(
|
||||
received >= 20,
|
||||
"audio should resume after reconnect; got {received} frames"
|
||||
);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user