feat: redesign audio networking for real-network resilience

Replaces the fire-and-forget datagram path with sequenced packets, a
per-peer jitter buffer, and persistent per-peer send tasks. Together these
fix three intertwined weaknesses that only showed up off localhost.

Packet format: every audio frame now carries a 4-byte little-endian
sequence number header ([seq][opus payload]), the basis for reordering and
loss detection.

Jitter buffer (core/jitter.rs): incoming packets are reordered by sequence
behind a fixed ~60ms playout delay. Missing sequences with later packets
already buffered are concealed via Opus PLC (decode(None)) -- a path the
decoder supported but nothing ever invoked. Underruns go idle and re-buffer
rather than concealing indefinitely. Covered by unit tests using real
encoded frames (reorder, gap-conceal, prime, late-drop).

Transport (network/iroh_impl.rs): each peer gets one long-lived send task
fed by a shallow bounded channel (drop-oldest on backpressure), instead of
spawning a throwaway task per peer per 20ms frame. Connections are now
established reactively on peer-join and torn down on peer-leave; the
lexicographically-lower EndpointId dials so a full-mesh pair forms exactly
one shared bidirectional connection instead of two racing ones. This also
removes the previous lock-held-across-connect().await serialization.

Opus decoder: PLC output is now sized to one 20ms frame, so concealment
synthesizes 20ms instead of a 120ms burst from the oversized max buffer.

Known follow-up (Tier 2): no reconnect on transient connection loss; a
send error currently retires the peer until they rejoin.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
2026-05-29 15:48:43 -04:00
co-authored by Claude Opus 4.8
parent 7af0235736
commit 875e6e124c
5 changed files with 437 additions and 176 deletions
+192
View File
@@ -0,0 +1,192 @@
//! Per-peer jitter buffer with Opus packet-loss concealment.
//!
//! Incoming audio arrives as unreliable QUIC datagrams that can be reordered,
//! duplicated, or dropped on real networks. Each packet carries a monotonic
//! sequence number (assigned by the sender). This buffer reorders packets by
//! sequence, holds a small fixed playout delay to absorb jitter, and — when a
//! sequence is missing but later packets have already arrived — synthesizes a
//! concealment frame via Opus PLC instead of emitting a click of silence.
use crate::codec::{AudioDecoder, CodecError, opus_impl::OpusDecoder};
use opus::Channels;
use std::collections::BTreeMap;
/// Samples per channel in one transmitted frame (20ms @ 48kHz mono).
pub const FRAME_SAMPLES: usize = 960;
/// How many frames to buffer before playout begins (~60ms). This is the
/// tolerance window for reordering and jitter; larger = more resilient but
/// more latency.
const TARGET_DELAY_FRAMES: usize = 3;
/// Hard cap on buffered frames (~640ms). If we ever exceed this we've fallen
/// badly behind, so we drop the oldest and resync rather than grow unbounded.
const MAX_BUFFERED_FRAMES: usize = 32;
pub struct JitterBuffer {
decoder: OpusDecoder,
/// Reorder window: sequence number -> encoded Opus payload.
packets: BTreeMap<u32, Vec<u8>>,
/// Next sequence we expect to play. `None` means idle/buffering: we are
/// waiting to accumulate `TARGET_DELAY_FRAMES` before (re)starting playout.
next_seq: Option<u32>,
}
/// Wrapping-aware "is `a` strictly before `b`" for sequence numbers.
fn seq_before(a: u32, b: u32) -> bool {
a != b && b.wrapping_sub(a) < (1 << 31)
}
impl JitterBuffer {
pub fn new() -> Result<Self, CodecError> {
Ok(Self {
decoder: OpusDecoder::new(48000, Channels::Mono, FRAME_SAMPLES)?,
packets: BTreeMap::new(),
next_seq: None,
})
}
/// Store a received packet, dropping ones we've already played past and
/// bounding total depth.
pub fn insert(&mut self, seq: u32, payload: Vec<u8>) {
// Too late: this sequence has already been played (or concealed).
if let Some(next) = self.next_seq
&& seq_before(seq, next)
{
return;
}
self.packets.insert(seq, payload);
while self.packets.len() > MAX_BUFFERED_FRAMES {
let oldest = *self.packets.keys().next().expect("non-empty");
self.packets.remove(&oldest);
// We've discarded backlog; resync the playout head to the new front.
self.next_seq = self.packets.keys().next().copied();
}
}
/// Produce the next 20ms PCM frame for playout, or `None` when idle or
/// still buffering (the caller should treat `None` as silence).
pub fn pop_frame(&mut self) -> Option<Vec<i16>> {
match self.next_seq {
None => {
// Buffering: start playout once we have enough to absorb jitter.
if self.packets.len() >= TARGET_DELAY_FRAMES {
self.next_seq = self.packets.keys().next().copied();
self.pop_frame()
} else {
None
}
}
Some(next) => {
if let Some(payload) = self.packets.remove(&next) {
self.next_seq = Some(next.wrapping_add(1));
self.decoder.decode(Some(&payload)).ok()
} else if self.packets.is_empty() {
// Underrun: the talker has gone quiet (or stopped). Go idle
// and re-buffer before resuming, rather than concealing forever.
self.next_seq = None;
None
} else {
// Gap with later packets already buffered: a packet was lost
// or reordered out of window. Conceal this frame via Opus PLC.
self.next_seq = Some(next.wrapping_add(1));
self.decoder.decode(None).ok()
}
}
}
}
/// True when nothing is buffered and playout is idle (talker silent).
pub fn is_idle(&self) -> bool {
self.next_seq.is_none() && self.packets.is_empty()
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::codec::AudioEncoder;
use crate::codec::opus_impl::OpusEncoder;
use opus::{Application, Channels};
/// A real, decodable Opus packet for one 20ms mono frame at amplitude `amp`.
fn frame(enc: &mut OpusEncoder, amp: i16) -> Vec<u8> {
let pcm: Vec<i16> = (0..FRAME_SAMPLES)
.map(|i| if i % 2 == 0 { amp } else { -amp })
.collect();
enc.encode(&pcm).unwrap()
}
#[test]
fn buffers_then_plays_in_order() {
let mut enc = OpusEncoder::new(48000, Channels::Mono, Application::Voip).unwrap();
let mut jb = JitterBuffer::new().unwrap();
// Below the target delay, playout hasn't primed yet.
jb.insert(0, frame(&mut enc, 1000));
assert!(jb.pop_frame().is_none());
// Reaching the target delay primes playout and yields the first frame.
jb.insert(1, frame(&mut enc, 1000));
jb.insert(2, frame(&mut enc, 1000));
assert_eq!(jb.pop_frame().map(|f| f.len()), Some(FRAME_SAMPLES));
assert_eq!(jb.pop_frame().map(|f| f.len()), Some(FRAME_SAMPLES));
assert_eq!(jb.pop_frame().map(|f| f.len()), Some(FRAME_SAMPLES));
// Drained: idle again.
assert!(jb.pop_frame().is_none());
assert!(jb.is_idle());
}
#[test]
fn reorders_out_of_order_arrivals() {
let mut enc = OpusEncoder::new(48000, Channels::Mono, Application::Voip).unwrap();
let mut jb = JitterBuffer::new().unwrap();
// Arrive scrambled but within the buffering window.
jb.insert(2, frame(&mut enc, 800));
jb.insert(0, frame(&mut enc, 800));
jb.insert(1, frame(&mut enc, 800));
// Three real frames come out (in sequence order), then idle.
assert!(jb.pop_frame().is_some());
assert!(jb.pop_frame().is_some());
assert!(jb.pop_frame().is_some());
assert!(jb.pop_frame().is_none());
}
#[test]
fn conceals_gap_when_later_packets_present() {
let mut enc = OpusEncoder::new(48000, Channels::Mono, Application::Voip).unwrap();
let mut jb = JitterBuffer::new().unwrap();
// Seq 2 is missing, but 0,1,3 arrive — enough to prime.
jb.insert(0, frame(&mut enc, 1200));
jb.insert(1, frame(&mut enc, 1200));
jb.insert(3, frame(&mut enc, 1200));
assert!(jb.pop_frame().is_some()); // seq 0
assert!(jb.pop_frame().is_some()); // seq 1
// seq 2 missing but seq 3 buffered -> Opus PLC produces a concealment frame.
let concealed = jb.pop_frame();
assert_eq!(concealed.map(|f| f.len()), Some(FRAME_SAMPLES));
assert!(jb.pop_frame().is_some()); // seq 3
assert!(jb.pop_frame().is_none());
}
#[test]
fn drops_packets_already_played() {
let mut enc = OpusEncoder::new(48000, Channels::Mono, Application::Voip).unwrap();
let mut jb = JitterBuffer::new().unwrap();
jb.insert(5, frame(&mut enc, 600));
jb.insert(6, frame(&mut enc, 600));
jb.insert(7, frame(&mut enc, 600));
assert!(jb.pop_frame().is_some()); // primes at seq 5, plays 5
assert!(jb.pop_frame().is_some()); // 6
// A straggler for an already-played sequence must be discarded.
jb.insert(5, frame(&mut enc, 600));
assert_eq!(jb.packets.len(), 1); // only seq 7 remains buffered
}
}