feat: redesign audio networking for real-network resilience
Replaces the fire-and-forget datagram path with sequenced packets, a per-peer jitter buffer, and persistent per-peer send tasks. Together these fix three intertwined weaknesses that only showed up off localhost. Packet format: every audio frame now carries a 4-byte little-endian sequence number header ([seq][opus payload]), the basis for reordering and loss detection. Jitter buffer (core/jitter.rs): incoming packets are reordered by sequence behind a fixed ~60ms playout delay. Missing sequences with later packets already buffered are concealed via Opus PLC (decode(None)) -- a path the decoder supported but nothing ever invoked. Underruns go idle and re-buffer rather than concealing indefinitely. Covered by unit tests using real encoded frames (reorder, gap-conceal, prime, late-drop). Transport (network/iroh_impl.rs): each peer gets one long-lived send task fed by a shallow bounded channel (drop-oldest on backpressure), instead of spawning a throwaway task per peer per 20ms frame. Connections are now established reactively on peer-join and torn down on peer-leave; the lexicographically-lower EndpointId dials so a full-mesh pair forms exactly one shared bidirectional connection instead of two racing ones. This also removes the previous lock-held-across-connect().await serialization. Opus decoder: PLC output is now sized to one 20ms frame, so concealment synthesizes 20ms instead of a 120ms burst from the oversized max buffer. Known follow-up (Tier 2): no reconnect on transient connection loss; a send error currently retires the peer until they rejoin. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,192 @@
|
||||
//! Per-peer jitter buffer with Opus packet-loss concealment.
|
||||
//!
|
||||
//! Incoming audio arrives as unreliable QUIC datagrams that can be reordered,
|
||||
//! duplicated, or dropped on real networks. Each packet carries a monotonic
|
||||
//! sequence number (assigned by the sender). This buffer reorders packets by
|
||||
//! sequence, holds a small fixed playout delay to absorb jitter, and — when a
|
||||
//! sequence is missing but later packets have already arrived — synthesizes a
|
||||
//! concealment frame via Opus PLC instead of emitting a click of silence.
|
||||
|
||||
use crate::codec::{AudioDecoder, CodecError, opus_impl::OpusDecoder};
|
||||
use opus::Channels;
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
/// Samples per channel in one transmitted frame (20ms @ 48kHz mono).
|
||||
pub const FRAME_SAMPLES: usize = 960;
|
||||
|
||||
/// How many frames to buffer before playout begins (~60ms). This is the
|
||||
/// tolerance window for reordering and jitter; larger = more resilient but
|
||||
/// more latency.
|
||||
const TARGET_DELAY_FRAMES: usize = 3;
|
||||
|
||||
/// Hard cap on buffered frames (~640ms). If we ever exceed this we've fallen
|
||||
/// badly behind, so we drop the oldest and resync rather than grow unbounded.
|
||||
const MAX_BUFFERED_FRAMES: usize = 32;
|
||||
|
||||
pub struct JitterBuffer {
|
||||
decoder: OpusDecoder,
|
||||
/// Reorder window: sequence number -> encoded Opus payload.
|
||||
packets: BTreeMap<u32, Vec<u8>>,
|
||||
/// Next sequence we expect to play. `None` means idle/buffering: we are
|
||||
/// waiting to accumulate `TARGET_DELAY_FRAMES` before (re)starting playout.
|
||||
next_seq: Option<u32>,
|
||||
}
|
||||
|
||||
/// Wrapping-aware "is `a` strictly before `b`" for sequence numbers.
|
||||
fn seq_before(a: u32, b: u32) -> bool {
|
||||
a != b && b.wrapping_sub(a) < (1 << 31)
|
||||
}
|
||||
|
||||
impl JitterBuffer {
|
||||
pub fn new() -> Result<Self, CodecError> {
|
||||
Ok(Self {
|
||||
decoder: OpusDecoder::new(48000, Channels::Mono, FRAME_SAMPLES)?,
|
||||
packets: BTreeMap::new(),
|
||||
next_seq: None,
|
||||
})
|
||||
}
|
||||
|
||||
/// Store a received packet, dropping ones we've already played past and
|
||||
/// bounding total depth.
|
||||
pub fn insert(&mut self, seq: u32, payload: Vec<u8>) {
|
||||
// Too late: this sequence has already been played (or concealed).
|
||||
if let Some(next) = self.next_seq
|
||||
&& seq_before(seq, next)
|
||||
{
|
||||
return;
|
||||
}
|
||||
self.packets.insert(seq, payload);
|
||||
|
||||
while self.packets.len() > MAX_BUFFERED_FRAMES {
|
||||
let oldest = *self.packets.keys().next().expect("non-empty");
|
||||
self.packets.remove(&oldest);
|
||||
// We've discarded backlog; resync the playout head to the new front.
|
||||
self.next_seq = self.packets.keys().next().copied();
|
||||
}
|
||||
}
|
||||
|
||||
/// Produce the next 20ms PCM frame for playout, or `None` when idle or
|
||||
/// still buffering (the caller should treat `None` as silence).
|
||||
pub fn pop_frame(&mut self) -> Option<Vec<i16>> {
|
||||
match self.next_seq {
|
||||
None => {
|
||||
// Buffering: start playout once we have enough to absorb jitter.
|
||||
if self.packets.len() >= TARGET_DELAY_FRAMES {
|
||||
self.next_seq = self.packets.keys().next().copied();
|
||||
self.pop_frame()
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
Some(next) => {
|
||||
if let Some(payload) = self.packets.remove(&next) {
|
||||
self.next_seq = Some(next.wrapping_add(1));
|
||||
self.decoder.decode(Some(&payload)).ok()
|
||||
} else if self.packets.is_empty() {
|
||||
// Underrun: the talker has gone quiet (or stopped). Go idle
|
||||
// and re-buffer before resuming, rather than concealing forever.
|
||||
self.next_seq = None;
|
||||
None
|
||||
} else {
|
||||
// Gap with later packets already buffered: a packet was lost
|
||||
// or reordered out of window. Conceal this frame via Opus PLC.
|
||||
self.next_seq = Some(next.wrapping_add(1));
|
||||
self.decoder.decode(None).ok()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// True when nothing is buffered and playout is idle (talker silent).
|
||||
pub fn is_idle(&self) -> bool {
|
||||
self.next_seq.is_none() && self.packets.is_empty()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::codec::AudioEncoder;
|
||||
use crate::codec::opus_impl::OpusEncoder;
|
||||
use opus::{Application, Channels};
|
||||
|
||||
/// A real, decodable Opus packet for one 20ms mono frame at amplitude `amp`.
|
||||
fn frame(enc: &mut OpusEncoder, amp: i16) -> Vec<u8> {
|
||||
let pcm: Vec<i16> = (0..FRAME_SAMPLES)
|
||||
.map(|i| if i % 2 == 0 { amp } else { -amp })
|
||||
.collect();
|
||||
enc.encode(&pcm).unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn buffers_then_plays_in_order() {
|
||||
let mut enc = OpusEncoder::new(48000, Channels::Mono, Application::Voip).unwrap();
|
||||
let mut jb = JitterBuffer::new().unwrap();
|
||||
|
||||
// Below the target delay, playout hasn't primed yet.
|
||||
jb.insert(0, frame(&mut enc, 1000));
|
||||
assert!(jb.pop_frame().is_none());
|
||||
|
||||
// Reaching the target delay primes playout and yields the first frame.
|
||||
jb.insert(1, frame(&mut enc, 1000));
|
||||
jb.insert(2, frame(&mut enc, 1000));
|
||||
assert_eq!(jb.pop_frame().map(|f| f.len()), Some(FRAME_SAMPLES));
|
||||
assert_eq!(jb.pop_frame().map(|f| f.len()), Some(FRAME_SAMPLES));
|
||||
assert_eq!(jb.pop_frame().map(|f| f.len()), Some(FRAME_SAMPLES));
|
||||
// Drained: idle again.
|
||||
assert!(jb.pop_frame().is_none());
|
||||
assert!(jb.is_idle());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reorders_out_of_order_arrivals() {
|
||||
let mut enc = OpusEncoder::new(48000, Channels::Mono, Application::Voip).unwrap();
|
||||
let mut jb = JitterBuffer::new().unwrap();
|
||||
|
||||
// Arrive scrambled but within the buffering window.
|
||||
jb.insert(2, frame(&mut enc, 800));
|
||||
jb.insert(0, frame(&mut enc, 800));
|
||||
jb.insert(1, frame(&mut enc, 800));
|
||||
|
||||
// Three real frames come out (in sequence order), then idle.
|
||||
assert!(jb.pop_frame().is_some());
|
||||
assert!(jb.pop_frame().is_some());
|
||||
assert!(jb.pop_frame().is_some());
|
||||
assert!(jb.pop_frame().is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn conceals_gap_when_later_packets_present() {
|
||||
let mut enc = OpusEncoder::new(48000, Channels::Mono, Application::Voip).unwrap();
|
||||
let mut jb = JitterBuffer::new().unwrap();
|
||||
|
||||
// Seq 2 is missing, but 0,1,3 arrive — enough to prime.
|
||||
jb.insert(0, frame(&mut enc, 1200));
|
||||
jb.insert(1, frame(&mut enc, 1200));
|
||||
jb.insert(3, frame(&mut enc, 1200));
|
||||
|
||||
assert!(jb.pop_frame().is_some()); // seq 0
|
||||
assert!(jb.pop_frame().is_some()); // seq 1
|
||||
// seq 2 missing but seq 3 buffered -> Opus PLC produces a concealment frame.
|
||||
let concealed = jb.pop_frame();
|
||||
assert_eq!(concealed.map(|f| f.len()), Some(FRAME_SAMPLES));
|
||||
assert!(jb.pop_frame().is_some()); // seq 3
|
||||
assert!(jb.pop_frame().is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn drops_packets_already_played() {
|
||||
let mut enc = OpusEncoder::new(48000, Channels::Mono, Application::Voip).unwrap();
|
||||
let mut jb = JitterBuffer::new().unwrap();
|
||||
|
||||
jb.insert(5, frame(&mut enc, 600));
|
||||
jb.insert(6, frame(&mut enc, 600));
|
||||
jb.insert(7, frame(&mut enc, 600));
|
||||
assert!(jb.pop_frame().is_some()); // primes at seq 5, plays 5
|
||||
assert!(jb.pop_frame().is_some()); // 6
|
||||
|
||||
// A straggler for an already-played sequence must be discarded.
|
||||
jb.insert(5, frame(&mut enc, 600));
|
||||
assert_eq!(jb.packets.len(), 1); // only seq 7 remains buffered
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user