feat: reconnect on transient link loss with per-peer supervisors

Audio links now survive a dropped QUIC connection instead of silently
dying until the peer leaves and rejoins the room.

Transport: each peer is owned by a single supervisor task running a
connect → run → reconnect loop. The deterministic-initiator rule (lower
id dials, higher accepts) holds on every reconnect, so one shared
connection re-forms each time; the dialer redials with capped backoff
and the acceptor awaits the inbound link, switching to a replacement
immediately if one arrives before its own close fires. Inbound
connections are routed to the supervisor via a per-peer channel.

Fixes a latent bug from the prior design: aborting a peer's tasks left
the detached send/read loops running, holding Connection clones so the
link never actually closed. The loops now live in abort-on-drop guards
scoped to the supervisor, so cancelling it releases the connection.

UX: a new ConnEvent stream surfaces per-peer link state to the UI, which
shows "Connecting…"/"Reconnecting…" with a yellow indicator and border
while a peer's audio link is down, returning to normal when it recovers.

Tests: the loopback test now runs through the supervisor path, plus a
new test drives a real drop (explicit close of a controlled peer
endpoint) and asserts the dialer re-dials the stable address and audio
resumes over the rebuilt connection.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
2026-05-29 17:01:21 -04:00
co-authored by Claude Opus 4.8
parent 26c3758d0d
commit ccf2bff87c
6 changed files with 477 additions and 81 deletions
+31 -4
View File
@@ -10,7 +10,7 @@ use iced::{
Color, Background, Border, Element, Subscription, Task, Theme, Event, keyboard,
};
use iroh::EndpointId;
use std::collections::HashMap;
use std::collections::{HashMap, HashSet};
use std::sync::{Arc, OnceLock};
use tokio::sync::Mutex;
@@ -80,6 +80,11 @@ pub struct AppState {
peers: HashMap<EndpointId, PeerState>,
peer_volumes: HashMap<EndpointId, f32>,
audio_levels: HashMap<EndpointId, f32>,
/// Peers whose audio link is currently down (initial connect or reconnect).
connecting: HashSet<EndpointId>,
/// Peers we've had a live link to at least once — used to say "Reconnecting"
/// rather than "Connecting" the second time around.
ever_connected: HashSet<EndpointId>,
controller: Arc<CoreController>,
current_screen: Screen,
}
@@ -121,6 +126,8 @@ impl Default for AppState {
peers: HashMap::new(),
peer_volumes: HashMap::new(),
audio_levels: HashMap::new(),
connecting: HashSet::new(),
ever_connected: HashSet::new(),
controller,
current_screen: Screen::Home,
}
@@ -206,6 +213,8 @@ fn update(state: &mut AppState, message: AppMessage) -> Task<AppMessage> {
state.ticket = "".to_string();
state.peers.clear();
state.audio_levels.clear();
state.connecting.clear();
state.ever_connected.clear();
state.status_message = "Ready to connect".to_string();
state.current_screen = Screen::Home;
}
@@ -215,10 +224,19 @@ fn update(state: &mut AppState, message: AppMessage) -> Task<AppMessage> {
UiEvent::PeerLeft { id } => {
state.peers.remove(&id);
state.audio_levels.remove(&id);
state.connecting.remove(&id);
state.ever_connected.remove(&id);
}
UiEvent::PeerUpdated { id, state: peer_state } => {
state.peers.insert(id, peer_state);
}
UiEvent::PeerConnecting { id } => {
state.connecting.insert(id);
}
UiEvent::PeerConnected { id } => {
state.connecting.remove(&id);
state.ever_connected.insert(id);
}
UiEvent::AudioLevels(levels) => {
for (id, val) in levels {
state.audio_levels.insert(id, val);
@@ -326,6 +344,7 @@ fn view(state: &AppState) -> Element<'_, AppMessage> {
let color_red = Color::from_rgb8(243, 139, 168);
let color_maroon = Color::from_rgb8(233, 146, 160);
let color_green = Color::from_rgb8(166, 227, 161);
let color_yellow = Color::from_rgb8(249, 226, 175);
// Style Helpers
let c_style = move |bg: Color, b_color: Color, radius: f32| {
@@ -585,9 +604,17 @@ fn view(state: &AppState) -> Element<'_, AppMessage> {
for (peer_id, peer) in &state.peers {
let level = state.audio_levels.get(peer_id).copied().unwrap_or(0.0);
let is_speaking = level > 0.01;
let is_connecting = state.connecting.contains(peer_id);
let is_speaking = !is_connecting && level > 0.01;
let indicator = if peer.is_muted {
let indicator = if is_connecting {
let label = if state.ever_connected.contains(peer_id) {
"[Reconnecting…]"
} else {
"[Connecting…]"
};
text(label).size(14).color(color_yellow)
} else if peer.is_muted {
text("[Muted]").size(14).color(color_red)
} else if is_speaking {
text("[Speaking]").size(14).color(color_green)
@@ -620,7 +647,7 @@ fn view(state: &AppState) -> Element<'_, AppMessage> {
let card = container(card_content)
.style(c_style(
if is_speaking { color_base } else { color_mantle },
if is_speaking { color_green } else { color_surface },
if is_connecting { color_yellow } else if is_speaking { color_green } else { color_surface },
6.0
))
.padding(12);
+4
View File
@@ -24,6 +24,10 @@ pub enum UiEvent {
PeerJoined { id: EndpointId, state: PeerState },
PeerLeft { id: EndpointId },
PeerUpdated { id: EndpointId, state: PeerState },
/// Audio link to a peer is being (re)established — show a connecting state.
PeerConnecting { id: EndpointId },
/// Audio link to a peer is up and carrying audio.
PeerConnected { id: EndpointId },
AudioLevels(Vec<(EndpointId, f32)>),
Error(String),
}
+26 -1
View File
@@ -5,7 +5,7 @@ use crate::audio::{AudioBackend, pipewire_impl::PipeWireBackend};
use crate::codec::{AudioEncoder, opus_impl::OpusEncoder};
use crate::core::jitter::{JitterBuffer, FRAME_SAMPLES};
use crate::network::{
NetworkTransport, RoomState, PeerState, RoomEvent, PeerSpeakTicket,
NetworkTransport, RoomState, PeerState, RoomEvent, ConnEvent, PeerSpeakTicket,
iroh_impl::IrohTransport,
gossip::IrohGossipState,
};
@@ -54,6 +54,7 @@ struct ActiveSession {
datagram_task: tokio::task::JoinHandle<()>,
mixer_task: tokio::task::JoinHandle<()>,
event_task: tokio::task::JoinHandle<()>,
conn_event_task: tokio::task::JoinHandle<()>,
}
impl ActiveSession {
@@ -62,6 +63,7 @@ impl ActiveSession {
self.datagram_task.abort();
self.mixer_task.abort();
self.event_task.abort();
self.conn_event_task.abort();
crate::log_msg("Aborted tasks");
let audio_backend_clone = audio_backend.clone();
@@ -413,6 +415,28 @@ async fn run_core_loop(
}
});
// 5. Connection-state forwarder: turns transport link state into
// per-peer UI indicators (connecting / reconnecting vs. live).
let mut conn_events = match transport.subscribe_conn_events().await {
Ok(rx) => rx,
Err(e) => {
let _ = ui_tx.send(UiEvent::Error(format!("Failed to subscribe conn events: {}", e))).await;
continue;
}
};
let ui_tx_conn = ui_tx.clone();
let conn_event_task = tokio::spawn(async move {
while let Some(event) = conn_events.recv().await {
let msg = match event {
ConnEvent::Connecting(id) => UiEvent::PeerConnecting { id },
ConnEvent::Connected(id) => UiEvent::PeerConnected { id },
};
if ui_tx_conn.send(msg).await.is_err() {
break;
}
}
});
let session = ActiveSession {
endpoint: endpoint.clone(),
router,
@@ -421,6 +445,7 @@ async fn run_core_loop(
datagram_task,
mixer_task,
event_task,
conn_event_task,
};
let self_id = endpoint.id().to_string();
+222 -74
View File
@@ -1,4 +1,4 @@
use crate::network::{NetworkTransport, NetError};
use crate::network::{NetworkTransport, NetError, ConnEvent};
use iroh::{Endpoint, EndpointId};
use iroh::endpoint::Connection;
use bytes::Bytes;
@@ -6,6 +6,7 @@ use tokio::sync::mpsc;
use tokio::sync::mpsc::Receiver;
use std::sync::{Arc, Mutex as StdMutex};
use std::collections::HashMap;
use std::time::Duration;
use async_trait::async_trait;
const AUDIO_ALPN: &[u8] = b"peerspeak-audio";
@@ -14,77 +15,215 @@ const AUDIO_ALPN: &[u8] = b"peerspeak-audio";
/// useless latency — keep it shallow and drop the oldest frame when full.
const SEND_QUEUE_DEPTH: usize = 8;
/// State shared between the transport and its protocol handler so both inbound
/// (accepted) and outbound (dialed) connections register the same way.
/// How many inbound connections we'll buffer for a single peer's supervisor.
/// During a healthy link the supervisor drains this promptly; a small buffer
/// just covers the brief window between detecting a drop and re-awaiting.
const INBOUND_QUEUE_DEPTH: usize = 4;
/// Reconnect backoff bounds for the dialing side. Starts gentle so a momentary
/// blip recovers almost instantly, and caps so a long outage doesn't hammer.
const INITIAL_BACKOFF: Duration = Duration::from_millis(250);
const MAX_BACKOFF: Duration = Duration::from_secs(5);
/// State shared between the transport, its protocol handler, and every per-peer
/// supervisor task. One supervisor owns a peer's whole connection lifecycle.
struct Shared {
/// Used by dialer-side supervisors to (re)establish connections.
endpoint: Endpoint,
self_id: EndpointId,
/// Sync-lockable send handles, so `broadcast` can fan out from the (non-async)
/// capture/encode thread without touching the Tokio runtime.
/// capture/encode thread without touching the Tokio runtime. A supervisor
/// inserts its handle when a link comes up and removes it when the link dies,
/// so `broadcast` only ever targets peers that are currently reachable.
senders: StdMutex<HashMap<EndpointId, mpsc::Sender<Bytes>>>,
/// Per-peer task handles + a retained connection clone. Keeping the clone
/// alive is what stops iroh from closing an accepted connection once the
/// `accept()` future returns.
peers: tokio::sync::Mutex<HashMap<EndpointId, PeerTasks>>,
/// One supervisor task per known peer.
peers: tokio::sync::Mutex<HashMap<EndpointId, PeerHandle>>,
incoming_tx: mpsc::Sender<(EndpointId, Bytes)>,
/// Best-effort link-state notifications for the UI (connecting / connected).
conn_events_tx: mpsc::Sender<ConnEvent>,
}
struct PeerTasks {
send_task: tokio::task::JoinHandle<()>,
read_task: tokio::task::JoinHandle<()>,
_conn: Connection,
struct PeerHandle {
/// The supervisor owning this peer's connect → run → reconnect loop.
supervisor: tokio::task::JoinHandle<()>,
/// Inbound connections accepted for this peer are routed here so the
/// acceptor-side supervisor can pick them up (initial and reconnect).
inbound_tx: mpsc::Sender<Connection>,
}
/// Aborts a spawned task when dropped. The supervisor holds its send/read loops
/// in these so that cancelling the supervisor (via `remove`'s `abort`) tears the
/// loops down too — otherwise they'd keep their `Connection` clones alive and
/// the link would never actually close.
struct AbortOnDrop(tokio::task::JoinHandle<()>);
impl Drop for AbortOnDrop {
fn drop(&mut self) {
self.0.abort();
}
}
impl Shared {
/// Register a live connection: spin up its send loop (datagrams out) and
/// read loop (datagrams in). Idempotent a second registration for an
/// already-known peer is ignored so we never run duplicate loops.
async fn register(self: &Arc<Self>, peer_id: EndpointId, conn: Connection) {
{
let peers = self.peers.lock().await;
if peers.contains_key(&peer_id) {
return;
}
/// Ensure a supervisor task exists for `peer_id`, returning its inbound
/// connection sender. Idempotent: a second call for a known peer just hands
/// back the existing sender. Both the reactive `connect_peer` path and the
/// inbound `accept()` path funnel through here, so whichever fires first
/// spins the peer up and the roles stay deterministic by id.
async fn ensure_supervisor(self: &Arc<Self>, peer_id: EndpointId) -> mpsc::Sender<Connection> {
let mut peers = self.peers.lock().await;
if let Some(handle) = peers.get(&peer_id) {
return handle.inbound_tx.clone();
}
let (send_tx, mut send_rx) = mpsc::channel::<Bytes>(SEND_QUEUE_DEPTH);
let conn_send = conn.clone();
let send_task = tokio::spawn(async move {
while let Some(data) = send_rx.recv().await {
if conn_send.send_datagram(data).is_err() {
break;
}
}
});
let conn_read = conn.clone();
let incoming_tx = self.incoming_tx.clone();
let read_task = tokio::spawn(async move {
while let Ok(bytes) = conn_read.read_datagram().await {
if incoming_tx.send((peer_id, bytes)).await.is_err() {
break;
}
}
});
self.senders.lock().unwrap().insert(peer_id, send_tx);
self.peers.lock().await.insert(
peer_id,
PeerTasks { send_task, read_task, _conn: conn },
);
crate::log_msg(&format!("Transport: registered peer {:?}", peer_id));
let (inbound_tx, inbound_rx) = mpsc::channel::<Connection>(INBOUND_QUEUE_DEPTH);
let shared = self.clone();
let supervisor = tokio::spawn(supervise(shared, peer_id, inbound_rx));
let inbound_tx_ret = inbound_tx.clone();
peers.insert(peer_id, PeerHandle { supervisor, inbound_tx });
crate::log_msg(&format!("Transport: supervising peer {:?}", peer_id));
inbound_tx_ret
}
/// Permanently retire a peer (it left the room). Stops its supervisor — which
/// cancels any in-flight dial or live link — and drops its send handle.
async fn remove(&self, peer_id: EndpointId) {
self.senders.lock().unwrap().remove(&peer_id);
if let Some(tasks) = self.peers.lock().await.remove(&peer_id) {
tasks.send_task.abort();
tasks.read_task.abort();
// Dropping `_conn` (the last retained clone) closes the connection.
crate::log_msg(&format!("Transport: removed peer {:?}", peer_id));
if let Some(handle) = self.peers.lock().await.remove(&peer_id) {
handle.supervisor.abort();
crate::log_msg(&format!("Transport: stopped supervising peer {:?}", peer_id));
}
}
}
/// Why a peer's live-link wait woke up.
enum Wake {
/// The QUIC connection closed (idle timeout or error) — need to reconnect.
Closed,
/// A fresh inbound connection arrived (acceptor side) — switch to it.
Replacement(Connection),
/// The peer was retired; tear down.
Shutdown,
}
/// Owns a single peer's connection lifecycle for as long as the peer is in the
/// room: obtain a link, run the send/read loops, and on loss obtain a new one —
/// with capped backoff on the dialing side. The deterministic-initiator rule
/// (lower id dials, higher id accepts) is the same on first connect and every
/// reconnect, so exactly one shared connection forms each time.
async fn supervise(
shared: Arc<Shared>,
peer_id: EndpointId,
mut inbound_rx: mpsc::Receiver<Connection>,
) {
let is_dialer = shared.self_id.to_string() < peer_id.to_string();
let mut backoff = INITIAL_BACKOFF;
// Show "connecting" until the first link is actually up.
let _ = shared.conn_events_tx.try_send(ConnEvent::Connecting(peer_id));
let mut conn = match obtain_conn(&shared, peer_id, is_dialer, &mut inbound_rx, &mut backoff).await {
Some(conn) => conn,
None => return, // retired before we ever connected
};
loop {
// A healthy link resets the dialer's backoff for the next outage.
backoff = INITIAL_BACKOFF;
let (send_tx, mut send_rx) = mpsc::channel::<Bytes>(SEND_QUEUE_DEPTH);
shared.senders.lock().unwrap().insert(peer_id, send_tx);
let _ = shared.conn_events_tx.try_send(ConnEvent::Connected(peer_id));
crate::log_msg(&format!("Transport: peer {:?} link up", peer_id));
// Run until the link dies, a replacement arrives, or we're retired. The
// send/read loops are held in abort-on-drop guards scoped to this block,
// so however we leave it — normal exit or the supervisor being aborted
// mid-`select` — both loops stop and release their `Connection` clones.
let wake = {
let conn_send = conn.clone();
let _send_guard = AbortOnDrop(tokio::spawn(async move {
while let Some(data) = send_rx.recv().await {
if conn_send.send_datagram(data).is_err() {
break;
}
}
}));
let conn_read = conn.clone();
let incoming_tx = shared.incoming_tx.clone();
let _read_guard = AbortOnDrop(tokio::spawn(async move {
while let Ok(bytes) = conn_read.read_datagram().await {
if incoming_tx.send((peer_id, bytes)).await.is_err() {
break;
}
}
}));
// Only the acceptor watches for replacements: if the dialer notices
// the drop and re-dials before our own `closed()` fires, we switch to
// the new link immediately instead of waiting out the idle timeout.
tokio::select! {
_ = conn.closed() => Wake::Closed,
inbound = inbound_rx.recv(), if !is_dialer => match inbound {
Some(new_conn) => Wake::Replacement(new_conn),
None => Wake::Shutdown,
},
}
};
// Dropping the send handle makes `broadcast` skip this peer until the
// link is back, instead of piling frames into a dead connection.
shared.senders.lock().unwrap().remove(&peer_id);
match wake {
Wake::Shutdown => return,
Wake::Replacement(new_conn) => {
crate::log_msg(&format!("Transport: peer {:?} replaced with new inbound link", peer_id));
let _ = shared.conn_events_tx.try_send(ConnEvent::Connecting(peer_id));
conn = new_conn;
}
Wake::Closed => {
crate::log_msg(&format!("Transport: peer {:?} link dropped; reconnecting", peer_id));
let _ = shared.conn_events_tx.try_send(ConnEvent::Connecting(peer_id));
conn = match obtain_conn(&shared, peer_id, is_dialer, &mut inbound_rx, &mut backoff).await {
Some(conn) => conn,
None => return, // retired while reconnecting
};
}
}
}
}
/// Block until we hold a live connection to the peer, or `None` if the peer was
/// retired first. The dialer redials with capped exponential backoff; the
/// acceptor waits for the dialer to come to it (a closed `inbound_rx` means the
/// supervisor was retired).
async fn obtain_conn(
shared: &Arc<Shared>,
peer_id: EndpointId,
is_dialer: bool,
inbound_rx: &mut mpsc::Receiver<Connection>,
backoff: &mut Duration,
) -> Option<Connection> {
if is_dialer {
loop {
match shared.endpoint.connect(peer_id, AUDIO_ALPN).await {
Ok(conn) => return Some(conn),
Err(e) => {
crate::log_msg(&format!(
"Transport: dial to {:?} failed: {:?}; retrying in {:?}",
peer_id, e, backoff
));
tokio::time::sleep(*backoff).await;
*backoff = (*backoff * 2).min(MAX_BACKOFF);
}
}
}
} else {
inbound_rx.recv().await
}
}
#[derive(Clone)]
pub struct AudioProtocol {
shared: Arc<Shared>,
@@ -104,38 +243,51 @@ impl iroh::protocol::ProtocolHandler for AudioProtocol {
let peer_id = connection.remote_id();
let shared = self.shared.clone();
async move {
// Register and return: the retained connection clone in `PeerTasks`
// keeps the connection open after this future resolves.
shared.register(peer_id, connection).await;
// Deterministic roles: the lower id dials, the higher id accepts.
// If we're the dialer for this peer we never consume inbound links
// (we make our own), so just let this one drop.
if shared.self_id.to_string() < peer_id.to_string() {
return Ok(());
}
// Route the connection to this peer's supervisor (creating it if the
// inbound link beat the gossip join event). try_send keeps the
// protocol handler from ever blocking; a full queue only happens if
// links are churning, and the supervisor will get the next one.
let inbound_tx = shared.ensure_supervisor(peer_id).await;
if inbound_tx.try_send(connection).is_err() {
crate::log_msg(&format!("Transport: dropped inbound link from {:?} (queue full)", peer_id));
}
Ok(())
}
}
}
pub struct IrohTransport {
endpoint: Endpoint,
self_id: EndpointId,
shared: Arc<Shared>,
incoming_rx: tokio::sync::Mutex<Option<mpsc::Receiver<(EndpointId, Bytes)>>>,
conn_events_rx: tokio::sync::Mutex<Option<mpsc::Receiver<ConnEvent>>>,
}
impl IrohTransport {
pub fn new(endpoint: Endpoint) -> (Self, AudioProtocol) {
let (incoming_tx, incoming_rx) = mpsc::channel(1000);
let (conn_events_tx, conn_events_rx) = mpsc::channel(100);
let self_id = endpoint.id();
let shared = Arc::new(Shared {
endpoint,
self_id,
senders: StdMutex::new(HashMap::new()),
peers: tokio::sync::Mutex::new(HashMap::new()),
incoming_tx,
conn_events_tx,
});
let protocol = AudioProtocol { shared: shared.clone() };
let transport = Self {
endpoint,
self_id,
shared,
incoming_rx: tokio::sync::Mutex::new(Some(incoming_rx)),
conn_events_rx: tokio::sync::Mutex::new(Some(conn_events_rx)),
};
(transport, protocol)
@@ -145,20 +297,9 @@ impl IrohTransport {
#[async_trait]
impl NetworkTransport for IrohTransport {
async fn connect_peer(&self, peer_id: EndpointId) {
// Deterministic initiator: only the lexicographically-lower id dials, so
// a full-mesh pair forms exactly one shared connection instead of two
// racing ones. The higher id waits for the inbound `accept()`.
if self.self_id.to_string() >= peer_id.to_string() {
return;
}
if self.shared.peers.lock().await.contains_key(&peer_id) {
return;
}
match self.endpoint.connect(peer_id, AUDIO_ALPN).await {
Ok(conn) => self.shared.register(peer_id, conn).await,
Err(e) => crate::log_msg(&format!("Transport: dial to {:?} failed: {:?}", peer_id, e)),
}
// Start (or confirm) the peer's supervisor. It owns dialing/accepting and
// reconnection from here on; this call is just the reactive trigger.
self.shared.ensure_supervisor(peer_id).await;
}
async fn disconnect_peer(&self, peer_id: EndpointId) {
@@ -180,4 +321,11 @@ impl NetworkTransport for IrohTransport {
.take()
.ok_or_else(|| NetError::Other("Datagram receiver already subscribed".to_string()))
}
async fn subscribe_conn_events(&self) -> Result<Receiver<ConnEvent>, NetError> {
let mut rx_guard = self.conn_events_rx.lock().await;
rx_guard
.take()
.ok_or_else(|| NetError::Other("Connection events already subscribed".to_string()))
}
}
+15
View File
@@ -36,6 +36,18 @@ pub enum RoomEvent {
PeerUpdated(EndpointId, PeerState),
}
/// Transport-level link state for a peer, surfaced so the UI can show when a
/// peer's audio connection is being (re)established versus actually carrying
/// audio. Distinct from `RoomEvent`: a peer can be present in the gossip room
/// while its audio link is momentarily down and reconnecting.
#[derive(Debug, Clone)]
pub enum ConnEvent {
/// No live audio link yet — initial connect or reconnecting after a drop.
Connecting(EndpointId),
/// A live audio link is established and carrying datagrams.
Connected(EndpointId),
}
#[derive(Serialize, Deserialize, Clone, Debug)]
pub struct PeerSpeakTicket {
pub host_addr: iroh::EndpointAddr,
@@ -83,6 +95,9 @@ pub trait NetworkTransport: Send + Sync {
/// Subscribes to incoming datagrams from any peer.
async fn receive_datagrams(&self) -> Result<Receiver<(EndpointId, Bytes)>, NetError>;
/// Subscribes to per-peer link-state changes (connecting / connected).
async fn subscribe_conn_events(&self) -> Result<Receiver<ConnEvent>, NetError>;
}
#[async_trait]