feat: reconnect on transient link loss with per-peer supervisors
Audio links now survive a dropped QUIC connection instead of silently dying until the peer leaves and rejoins the room. Transport: each peer is owned by a single supervisor task running a connect → run → reconnect loop. The deterministic-initiator rule (lower id dials, higher accepts) holds on every reconnect, so one shared connection re-forms each time; the dialer redials with capped backoff and the acceptor awaits the inbound link, switching to a replacement immediately if one arrives before its own close fires. Inbound connections are routed to the supervisor via a per-peer channel. Fixes a latent bug from the prior design: aborting a peer's tasks left the detached send/read loops running, holding Connection clones so the link never actually closed. The loops now live in abort-on-drop guards scoped to the supervisor, so cancelling it releases the connection. UX: a new ConnEvent stream surfaces per-peer link state to the UI, which shows "Connecting…"/"Reconnecting…" with a yellow indicator and border while a peer's audio link is down, returning to normal when it recovers. Tests: the loopback test now runs through the supervisor path, plus a new test drives a real drop (explicit close of a controlled peer endpoint) and asserts the dialer re-dials the stable address and audio resumes over the rebuilt connection. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 4.8
+222
-74
@@ -1,4 +1,4 @@
|
||||
use crate::network::{NetworkTransport, NetError};
|
||||
use crate::network::{NetworkTransport, NetError, ConnEvent};
|
||||
use iroh::{Endpoint, EndpointId};
|
||||
use iroh::endpoint::Connection;
|
||||
use bytes::Bytes;
|
||||
@@ -6,6 +6,7 @@ use tokio::sync::mpsc;
|
||||
use tokio::sync::mpsc::Receiver;
|
||||
use std::sync::{Arc, Mutex as StdMutex};
|
||||
use std::collections::HashMap;
|
||||
use std::time::Duration;
|
||||
use async_trait::async_trait;
|
||||
|
||||
const AUDIO_ALPN: &[u8] = b"peerspeak-audio";
|
||||
@@ -14,77 +15,215 @@ const AUDIO_ALPN: &[u8] = b"peerspeak-audio";
|
||||
/// useless latency — keep it shallow and drop the oldest frame when full.
|
||||
const SEND_QUEUE_DEPTH: usize = 8;
|
||||
|
||||
/// State shared between the transport and its protocol handler so both inbound
|
||||
/// (accepted) and outbound (dialed) connections register the same way.
|
||||
/// How many inbound connections we'll buffer for a single peer's supervisor.
|
||||
/// During a healthy link the supervisor drains this promptly; a small buffer
|
||||
/// just covers the brief window between detecting a drop and re-awaiting.
|
||||
const INBOUND_QUEUE_DEPTH: usize = 4;
|
||||
|
||||
/// Reconnect backoff bounds for the dialing side. Starts gentle so a momentary
|
||||
/// blip recovers almost instantly, and caps so a long outage doesn't hammer.
|
||||
const INITIAL_BACKOFF: Duration = Duration::from_millis(250);
|
||||
const MAX_BACKOFF: Duration = Duration::from_secs(5);
|
||||
|
||||
/// State shared between the transport, its protocol handler, and every per-peer
|
||||
/// supervisor task. One supervisor owns a peer's whole connection lifecycle.
|
||||
struct Shared {
|
||||
/// Used by dialer-side supervisors to (re)establish connections.
|
||||
endpoint: Endpoint,
|
||||
self_id: EndpointId,
|
||||
/// Sync-lockable send handles, so `broadcast` can fan out from the (non-async)
|
||||
/// capture/encode thread without touching the Tokio runtime.
|
||||
/// capture/encode thread without touching the Tokio runtime. A supervisor
|
||||
/// inserts its handle when a link comes up and removes it when the link dies,
|
||||
/// so `broadcast` only ever targets peers that are currently reachable.
|
||||
senders: StdMutex<HashMap<EndpointId, mpsc::Sender<Bytes>>>,
|
||||
/// Per-peer task handles + a retained connection clone. Keeping the clone
|
||||
/// alive is what stops iroh from closing an accepted connection once the
|
||||
/// `accept()` future returns.
|
||||
peers: tokio::sync::Mutex<HashMap<EndpointId, PeerTasks>>,
|
||||
/// One supervisor task per known peer.
|
||||
peers: tokio::sync::Mutex<HashMap<EndpointId, PeerHandle>>,
|
||||
incoming_tx: mpsc::Sender<(EndpointId, Bytes)>,
|
||||
/// Best-effort link-state notifications for the UI (connecting / connected).
|
||||
conn_events_tx: mpsc::Sender<ConnEvent>,
|
||||
}
|
||||
|
||||
struct PeerTasks {
|
||||
send_task: tokio::task::JoinHandle<()>,
|
||||
read_task: tokio::task::JoinHandle<()>,
|
||||
_conn: Connection,
|
||||
struct PeerHandle {
|
||||
/// The supervisor owning this peer's connect → run → reconnect loop.
|
||||
supervisor: tokio::task::JoinHandle<()>,
|
||||
/// Inbound connections accepted for this peer are routed here so the
|
||||
/// acceptor-side supervisor can pick them up (initial and reconnect).
|
||||
inbound_tx: mpsc::Sender<Connection>,
|
||||
}
|
||||
|
||||
/// Aborts a spawned task when dropped. The supervisor holds its send/read loops
|
||||
/// in these so that cancelling the supervisor (via `remove`'s `abort`) tears the
|
||||
/// loops down too — otherwise they'd keep their `Connection` clones alive and
|
||||
/// the link would never actually close.
|
||||
struct AbortOnDrop(tokio::task::JoinHandle<()>);
|
||||
|
||||
impl Drop for AbortOnDrop {
|
||||
fn drop(&mut self) {
|
||||
self.0.abort();
|
||||
}
|
||||
}
|
||||
|
||||
impl Shared {
|
||||
/// Register a live connection: spin up its send loop (datagrams out) and
|
||||
/// read loop (datagrams in). Idempotent — a second registration for an
|
||||
/// already-known peer is ignored so we never run duplicate loops.
|
||||
async fn register(self: &Arc<Self>, peer_id: EndpointId, conn: Connection) {
|
||||
{
|
||||
let peers = self.peers.lock().await;
|
||||
if peers.contains_key(&peer_id) {
|
||||
return;
|
||||
}
|
||||
/// Ensure a supervisor task exists for `peer_id`, returning its inbound
|
||||
/// connection sender. Idempotent: a second call for a known peer just hands
|
||||
/// back the existing sender. Both the reactive `connect_peer` path and the
|
||||
/// inbound `accept()` path funnel through here, so whichever fires first
|
||||
/// spins the peer up and the roles stay deterministic by id.
|
||||
async fn ensure_supervisor(self: &Arc<Self>, peer_id: EndpointId) -> mpsc::Sender<Connection> {
|
||||
let mut peers = self.peers.lock().await;
|
||||
if let Some(handle) = peers.get(&peer_id) {
|
||||
return handle.inbound_tx.clone();
|
||||
}
|
||||
|
||||
let (send_tx, mut send_rx) = mpsc::channel::<Bytes>(SEND_QUEUE_DEPTH);
|
||||
|
||||
let conn_send = conn.clone();
|
||||
let send_task = tokio::spawn(async move {
|
||||
while let Some(data) = send_rx.recv().await {
|
||||
if conn_send.send_datagram(data).is_err() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
let conn_read = conn.clone();
|
||||
let incoming_tx = self.incoming_tx.clone();
|
||||
let read_task = tokio::spawn(async move {
|
||||
while let Ok(bytes) = conn_read.read_datagram().await {
|
||||
if incoming_tx.send((peer_id, bytes)).await.is_err() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
self.senders.lock().unwrap().insert(peer_id, send_tx);
|
||||
self.peers.lock().await.insert(
|
||||
peer_id,
|
||||
PeerTasks { send_task, read_task, _conn: conn },
|
||||
);
|
||||
crate::log_msg(&format!("Transport: registered peer {:?}", peer_id));
|
||||
let (inbound_tx, inbound_rx) = mpsc::channel::<Connection>(INBOUND_QUEUE_DEPTH);
|
||||
let shared = self.clone();
|
||||
let supervisor = tokio::spawn(supervise(shared, peer_id, inbound_rx));
|
||||
let inbound_tx_ret = inbound_tx.clone();
|
||||
peers.insert(peer_id, PeerHandle { supervisor, inbound_tx });
|
||||
crate::log_msg(&format!("Transport: supervising peer {:?}", peer_id));
|
||||
inbound_tx_ret
|
||||
}
|
||||
|
||||
/// Permanently retire a peer (it left the room). Stops its supervisor — which
|
||||
/// cancels any in-flight dial or live link — and drops its send handle.
|
||||
async fn remove(&self, peer_id: EndpointId) {
|
||||
self.senders.lock().unwrap().remove(&peer_id);
|
||||
if let Some(tasks) = self.peers.lock().await.remove(&peer_id) {
|
||||
tasks.send_task.abort();
|
||||
tasks.read_task.abort();
|
||||
// Dropping `_conn` (the last retained clone) closes the connection.
|
||||
crate::log_msg(&format!("Transport: removed peer {:?}", peer_id));
|
||||
if let Some(handle) = self.peers.lock().await.remove(&peer_id) {
|
||||
handle.supervisor.abort();
|
||||
crate::log_msg(&format!("Transport: stopped supervising peer {:?}", peer_id));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Why a peer's live-link wait woke up.
|
||||
enum Wake {
|
||||
/// The QUIC connection closed (idle timeout or error) — need to reconnect.
|
||||
Closed,
|
||||
/// A fresh inbound connection arrived (acceptor side) — switch to it.
|
||||
Replacement(Connection),
|
||||
/// The peer was retired; tear down.
|
||||
Shutdown,
|
||||
}
|
||||
|
||||
/// Owns a single peer's connection lifecycle for as long as the peer is in the
|
||||
/// room: obtain a link, run the send/read loops, and on loss obtain a new one —
|
||||
/// with capped backoff on the dialing side. The deterministic-initiator rule
|
||||
/// (lower id dials, higher id accepts) is the same on first connect and every
|
||||
/// reconnect, so exactly one shared connection forms each time.
|
||||
async fn supervise(
|
||||
shared: Arc<Shared>,
|
||||
peer_id: EndpointId,
|
||||
mut inbound_rx: mpsc::Receiver<Connection>,
|
||||
) {
|
||||
let is_dialer = shared.self_id.to_string() < peer_id.to_string();
|
||||
let mut backoff = INITIAL_BACKOFF;
|
||||
|
||||
// Show "connecting" until the first link is actually up.
|
||||
let _ = shared.conn_events_tx.try_send(ConnEvent::Connecting(peer_id));
|
||||
|
||||
let mut conn = match obtain_conn(&shared, peer_id, is_dialer, &mut inbound_rx, &mut backoff).await {
|
||||
Some(conn) => conn,
|
||||
None => return, // retired before we ever connected
|
||||
};
|
||||
|
||||
loop {
|
||||
// A healthy link resets the dialer's backoff for the next outage.
|
||||
backoff = INITIAL_BACKOFF;
|
||||
|
||||
let (send_tx, mut send_rx) = mpsc::channel::<Bytes>(SEND_QUEUE_DEPTH);
|
||||
shared.senders.lock().unwrap().insert(peer_id, send_tx);
|
||||
let _ = shared.conn_events_tx.try_send(ConnEvent::Connected(peer_id));
|
||||
crate::log_msg(&format!("Transport: peer {:?} link up", peer_id));
|
||||
|
||||
// Run until the link dies, a replacement arrives, or we're retired. The
|
||||
// send/read loops are held in abort-on-drop guards scoped to this block,
|
||||
// so however we leave it — normal exit or the supervisor being aborted
|
||||
// mid-`select` — both loops stop and release their `Connection` clones.
|
||||
let wake = {
|
||||
let conn_send = conn.clone();
|
||||
let _send_guard = AbortOnDrop(tokio::spawn(async move {
|
||||
while let Some(data) = send_rx.recv().await {
|
||||
if conn_send.send_datagram(data).is_err() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}));
|
||||
|
||||
let conn_read = conn.clone();
|
||||
let incoming_tx = shared.incoming_tx.clone();
|
||||
let _read_guard = AbortOnDrop(tokio::spawn(async move {
|
||||
while let Ok(bytes) = conn_read.read_datagram().await {
|
||||
if incoming_tx.send((peer_id, bytes)).await.is_err() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}));
|
||||
|
||||
// Only the acceptor watches for replacements: if the dialer notices
|
||||
// the drop and re-dials before our own `closed()` fires, we switch to
|
||||
// the new link immediately instead of waiting out the idle timeout.
|
||||
tokio::select! {
|
||||
_ = conn.closed() => Wake::Closed,
|
||||
inbound = inbound_rx.recv(), if !is_dialer => match inbound {
|
||||
Some(new_conn) => Wake::Replacement(new_conn),
|
||||
None => Wake::Shutdown,
|
||||
},
|
||||
}
|
||||
};
|
||||
|
||||
// Dropping the send handle makes `broadcast` skip this peer until the
|
||||
// link is back, instead of piling frames into a dead connection.
|
||||
shared.senders.lock().unwrap().remove(&peer_id);
|
||||
|
||||
match wake {
|
||||
Wake::Shutdown => return,
|
||||
Wake::Replacement(new_conn) => {
|
||||
crate::log_msg(&format!("Transport: peer {:?} replaced with new inbound link", peer_id));
|
||||
let _ = shared.conn_events_tx.try_send(ConnEvent::Connecting(peer_id));
|
||||
conn = new_conn;
|
||||
}
|
||||
Wake::Closed => {
|
||||
crate::log_msg(&format!("Transport: peer {:?} link dropped; reconnecting", peer_id));
|
||||
let _ = shared.conn_events_tx.try_send(ConnEvent::Connecting(peer_id));
|
||||
conn = match obtain_conn(&shared, peer_id, is_dialer, &mut inbound_rx, &mut backoff).await {
|
||||
Some(conn) => conn,
|
||||
None => return, // retired while reconnecting
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Block until we hold a live connection to the peer, or `None` if the peer was
|
||||
/// retired first. The dialer redials with capped exponential backoff; the
|
||||
/// acceptor waits for the dialer to come to it (a closed `inbound_rx` means the
|
||||
/// supervisor was retired).
|
||||
async fn obtain_conn(
|
||||
shared: &Arc<Shared>,
|
||||
peer_id: EndpointId,
|
||||
is_dialer: bool,
|
||||
inbound_rx: &mut mpsc::Receiver<Connection>,
|
||||
backoff: &mut Duration,
|
||||
) -> Option<Connection> {
|
||||
if is_dialer {
|
||||
loop {
|
||||
match shared.endpoint.connect(peer_id, AUDIO_ALPN).await {
|
||||
Ok(conn) => return Some(conn),
|
||||
Err(e) => {
|
||||
crate::log_msg(&format!(
|
||||
"Transport: dial to {:?} failed: {:?}; retrying in {:?}",
|
||||
peer_id, e, backoff
|
||||
));
|
||||
tokio::time::sleep(*backoff).await;
|
||||
*backoff = (*backoff * 2).min(MAX_BACKOFF);
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
inbound_rx.recv().await
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct AudioProtocol {
|
||||
shared: Arc<Shared>,
|
||||
@@ -104,38 +243,51 @@ impl iroh::protocol::ProtocolHandler for AudioProtocol {
|
||||
let peer_id = connection.remote_id();
|
||||
let shared = self.shared.clone();
|
||||
async move {
|
||||
// Register and return: the retained connection clone in `PeerTasks`
|
||||
// keeps the connection open after this future resolves.
|
||||
shared.register(peer_id, connection).await;
|
||||
// Deterministic roles: the lower id dials, the higher id accepts.
|
||||
// If we're the dialer for this peer we never consume inbound links
|
||||
// (we make our own), so just let this one drop.
|
||||
if shared.self_id.to_string() < peer_id.to_string() {
|
||||
return Ok(());
|
||||
}
|
||||
// Route the connection to this peer's supervisor (creating it if the
|
||||
// inbound link beat the gossip join event). try_send keeps the
|
||||
// protocol handler from ever blocking; a full queue only happens if
|
||||
// links are churning, and the supervisor will get the next one.
|
||||
let inbound_tx = shared.ensure_supervisor(peer_id).await;
|
||||
if inbound_tx.try_send(connection).is_err() {
|
||||
crate::log_msg(&format!("Transport: dropped inbound link from {:?} (queue full)", peer_id));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct IrohTransport {
|
||||
endpoint: Endpoint,
|
||||
self_id: EndpointId,
|
||||
shared: Arc<Shared>,
|
||||
incoming_rx: tokio::sync::Mutex<Option<mpsc::Receiver<(EndpointId, Bytes)>>>,
|
||||
conn_events_rx: tokio::sync::Mutex<Option<mpsc::Receiver<ConnEvent>>>,
|
||||
}
|
||||
|
||||
impl IrohTransport {
|
||||
pub fn new(endpoint: Endpoint) -> (Self, AudioProtocol) {
|
||||
let (incoming_tx, incoming_rx) = mpsc::channel(1000);
|
||||
let (conn_events_tx, conn_events_rx) = mpsc::channel(100);
|
||||
let self_id = endpoint.id();
|
||||
|
||||
let shared = Arc::new(Shared {
|
||||
endpoint,
|
||||
self_id,
|
||||
senders: StdMutex::new(HashMap::new()),
|
||||
peers: tokio::sync::Mutex::new(HashMap::new()),
|
||||
incoming_tx,
|
||||
conn_events_tx,
|
||||
});
|
||||
|
||||
let protocol = AudioProtocol { shared: shared.clone() };
|
||||
let transport = Self {
|
||||
endpoint,
|
||||
self_id,
|
||||
shared,
|
||||
incoming_rx: tokio::sync::Mutex::new(Some(incoming_rx)),
|
||||
conn_events_rx: tokio::sync::Mutex::new(Some(conn_events_rx)),
|
||||
};
|
||||
|
||||
(transport, protocol)
|
||||
@@ -145,20 +297,9 @@ impl IrohTransport {
|
||||
#[async_trait]
|
||||
impl NetworkTransport for IrohTransport {
|
||||
async fn connect_peer(&self, peer_id: EndpointId) {
|
||||
// Deterministic initiator: only the lexicographically-lower id dials, so
|
||||
// a full-mesh pair forms exactly one shared connection instead of two
|
||||
// racing ones. The higher id waits for the inbound `accept()`.
|
||||
if self.self_id.to_string() >= peer_id.to_string() {
|
||||
return;
|
||||
}
|
||||
if self.shared.peers.lock().await.contains_key(&peer_id) {
|
||||
return;
|
||||
}
|
||||
|
||||
match self.endpoint.connect(peer_id, AUDIO_ALPN).await {
|
||||
Ok(conn) => self.shared.register(peer_id, conn).await,
|
||||
Err(e) => crate::log_msg(&format!("Transport: dial to {:?} failed: {:?}", peer_id, e)),
|
||||
}
|
||||
// Start (or confirm) the peer's supervisor. It owns dialing/accepting and
|
||||
// reconnection from here on; this call is just the reactive trigger.
|
||||
self.shared.ensure_supervisor(peer_id).await;
|
||||
}
|
||||
|
||||
async fn disconnect_peer(&self, peer_id: EndpointId) {
|
||||
@@ -180,4 +321,11 @@ impl NetworkTransport for IrohTransport {
|
||||
.take()
|
||||
.ok_or_else(|| NetError::Other("Datagram receiver already subscribed".to_string()))
|
||||
}
|
||||
|
||||
async fn subscribe_conn_events(&self) -> Result<Receiver<ConnEvent>, NetError> {
|
||||
let mut rx_guard = self.conn_events_rx.lock().await;
|
||||
rx_guard
|
||||
.take()
|
||||
.ok_or_else(|| NetError::Other("Connection events already subscribed".to_string()))
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user