From 185d47aa8de7f94fc9ebcb813bc08848a2ab5a65 Mon Sep 17 00:00:00 2001 From: Mollusk Date: Fri, 19 Jun 2026 04:42:51 -0400 Subject: [PATCH] W4 (WIP): dep-free resampler + capture/config wiring (playback pending) - src/audio/resample.rs: pure linear PushResampler (capture) + StereoPullResampler (playback pull), 6 unit tests green on Linux. - choose_config: prefer native 48kHz, else fall back to device default config and convert at the boundary instead of hard-erroring. - run_capture: resample device-rate mono -> 48kHz on the drain thread. - i16<->f32 helpers. Playback build_output remap still TODO (Codex). Co-Authored-By: Claude Opus 4.8 --- src/audio/cpal_impl.rs | 133 ++++++++++++++------ src/audio/mod.rs | 4 + src/audio/resample.rs | 274 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 374 insertions(+), 37 deletions(-) create mode 100644 src/audio/resample.rs diff --git a/src/audio/cpal_impl.rs b/src/audio/cpal_impl.rs index 144e4e0..dec29b4 100644 --- a/src/audio/cpal_impl.rs +++ b/src/audio/cpal_impl.rs @@ -30,12 +30,26 @@ //! device/format/WASAPI failure surfaces as a real `Err` to the caller instead of //! leaving the UI in a joined-but-silent room. //! -//! ## Sample rate +//! ## Sample rate and channel layout (W4) //! -//! The whole pipeline assumes 48 kHz (Opus + the 960-sample frame). Phase 1 only -//! selects a native-48 kHz device config; if the device can't do 48 kHz we return -//! a clear error rather than silently producing pitch-shifted audio. Arbitrary -//! sample-rate support (resampling) is a Phase 1.1 follow-up. +//! The whole pipeline runs internally at 48 kHz (Opus + the 960-sample frame) and +//! mono capture / stereo playback. We prefer a native-48 kHz device config so the +//! common case is conversion-free and bit-exact. When the device can't do 48 kHz +//! (commonly a 44.1 kHz-only endpoint) or can't do stereo output, we fall back to +//! the device's default config and convert at the boundary with the dep-free +//! [`super::resample`] linear resamplers instead of hard-erroring: +//! +//! - **Capture**: the device-rate mono stream is resampled to 48 kHz on the +//! capture drain thread (off the RT callback) before framing. +//! - **Playback**: the internal 48 kHz stereo bus is resampled to the device rate +//! and remapped to the device channel count inside the output RT callback, which +//! pulls internal frames from the ring on demand (allocation-free, so RT-safe). +//! The ring, prefill, and `ring_fill` pacing stay in internal 48 kHz-stereo +//! units, so the mixer is unchanged. +//! +//! Linear interpolation has no anti-aliasing filter (see [`super::resample`] docs); +//! it is adequate for speech and keeps the matching-rate path bit-exact, with the +//! seam ready for a higher-quality resampler later. use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}; use std::sync::mpsc::{channel, Receiver, RecvTimeoutError, Sender}; @@ -50,6 +64,7 @@ use ringbuf::{ HeapRb, }; +use super::resample::{PushResampler, StereoPullResampler}; use super::{AudioBackend, AudioDevice, AudioError, PLAYBACK_CHANNELS, PLAYBACK_TARGET_SAMPLES}; /// The one sample rate the pipeline supports (Opus + the 20 ms frame). @@ -234,12 +249,9 @@ pub fn enumerate_audio_devices() -> Vec { // --------------------------------------------------------------------------- /// Resolve a device (by `target` name, else the system default) and a stream -/// config running natively at [`SAMPLE_RATE`]. -/// -/// For output we require [`PLAYBACK_CHANNELS`] (stereo) so the interleaved ring -/// maps 1:1 to the device buffer; for input we prefer mono but accept any channel -/// count and downmix. A device with no 48 kHz config is a hard error (no -/// resampling yet — see module docs). +/// config. We prefer a config running natively at [`SAMPLE_RATE`] (conversion-free); +/// if the device has none, we fall back to its default config and resample/remap at +/// the boundary (W4 — see module docs and [`choose_config`]). fn resolve( output: bool, target: Option, @@ -287,8 +299,14 @@ fn find_device_by_name(host: &cpal::Host, output: bool, name: &str) -> Option".to_string()), - if output { "stereo output" } else { "input" }, - )) - }) + // Case 3: no native 48 kHz — fall back to the device default and convert. + let def = if output { + device.default_output_config() + } else { + device.default_input_config() + } + .map_err(|e| AudioError::Device(e.to_string()))?; + crate::log_msg(&format!( + "cpal: device '{}' has no native {SAMPLE_RATE} Hz {} config; using {} Hz / {} ch with linear resampling (W4)", + device.name().unwrap_or_else(|_| "".to_string()), + if output { "output" } else { "input" }, + def.sample_rate().0, + def.channels(), + )); + Ok(def) } // --------------------------------------------------------------------------- @@ -351,9 +380,10 @@ fn run_capture( // Fallible device/stream setup. We report the real error to `start_capture` // before doing any work, so a join never lands in a silent room. - let setup = || -> Result<(Stream, String, SampleFormat, usize), AudioError> { + let setup = || -> Result<(Stream, String, SampleFormat, usize, u32), AudioError> { let (device, config, sample_format) = resolve(false, target)?; let channels = config.channels as usize; + let device_rate = config.sample_rate.0; let stream = match sample_format { SampleFormat::F32 => { build_input::(&device, &config, producer, channels, overrun.clone()) @@ -370,10 +400,10 @@ fn run_capture( }?; stream.play().map_err(|e| AudioError::Stream(e.to_string()))?; let name = device.name().unwrap_or_else(|_| "".to_string()); - Ok((stream, name, sample_format, channels)) + Ok((stream, name, sample_format, channels, device_rate)) }; - let (stream, dev_name, sample_format, channels) = match setup() { + let (stream, dev_name, sample_format, channels, device_rate) = match setup() { Ok(v) => { let _ = ready.send(Ok(())); v @@ -384,24 +414,41 @@ fn run_capture( } }; crate::log_msg(&format!( - "cpal capture started: device='{dev_name}' format={sample_format:?} channels={channels} rate={SAMPLE_RATE} Hz" + "cpal capture started: device='{dev_name}' format={sample_format:?} channels={channels} device_rate={device_rate} Hz -> {SAMPLE_RATE} Hz" )); - // Drain the RT ring on this thread: pop mono samples, frame them (the `Vec` - // allocation lives here, off the RT path), and send completed frames. Keep - // `stream` alive until `stop()` flips the flag. + // If the device isn't at 48 kHz, resample its mono stream up/down to 48 kHz on + // this (non-RT) thread before framing (W4). At 48 kHz this stays None and the + // samples pass straight through, bit-exact. + let mut resampler = (device_rate != SAMPLE_RATE).then(|| PushResampler::new(device_rate, SAMPLE_RATE)); + // Reused scratch for a sample's resampled output (off-RT alloc; tiny — at most + // a couple of samples per input). Avoids a nested-closure borrow over `acc`/`tx`. + let mut resampled: Vec = Vec::new(); + + // Drain the RT ring on this thread: pop mono samples, (resample,) frame them + // (the `Vec` allocation lives here, off the RT path), and send completed + // frames. Keep `stream` alive until `stop()` flips the flag. let mut acc = FrameAccumulator::new(CAPTURE_FRAME); let mut last_overrun = 0u64; while running.load(Ordering::Relaxed) { let mut drained = false; while let Some(sample) = consumer.try_pop() { drained = true; - if let Some(frame) = acc.push(sample) { - // Consumer gone (call ended) → stop feeding; the stream is - // dropped below on the way out. - if tx.send(frame).is_err() { - drop(stream); - return; + resampled.clear(); + match resampler { + Some(ref mut rs) => { + rs.push(i16_to_f32(sample), |out| resampled.push(f32_to_i16(out))); + } + None => resampled.push(sample), + } + for s in resampled.drain(..) { + if let Some(frame) = acc.push(s) { + // Consumer gone (call ended) → stop feeding; the stream is + // dropped below on the way out. + if tx.send(frame).is_err() { + drop(stream); + return; + } } } } @@ -466,6 +513,18 @@ where (sum / frame.len() as i32) as i16 } +/// Scale an i16 PCM sample to f32 in roughly `[-1, 1]` for interpolation. +#[inline] +fn i16_to_f32(s: i16) -> f32 { + s as f32 / 32768.0 +} + +/// Convert an interpolated f32 sample back to i16, clamping to range. +#[inline] +fn f32_to_i16(x: f32) -> i16 { + (x * 32768.0).clamp(i16::MIN as f32, i16::MAX as f32) as i16 +} + /// Accumulates mono samples into fixed-size [`CAPTURE_FRAME`] frames. Pulled out /// of the RT callback so the framing is unit-testable. struct FrameAccumulator { diff --git a/src/audio/mod.rs b/src/audio/mod.rs index 2699c90..1d068e0 100644 --- a/src/audio/mod.rs +++ b/src/audio/mod.rs @@ -61,6 +61,10 @@ pub mod gate; pub mod limiter; pub mod multitrack; pub mod pan; +// Linear resamplers used by the Windows/cpal backend (W4). Platform-neutral and +// pure, so it builds (and its tests run) everywhere even though only the cpal +// backend wires it in. +pub mod resample; #[cfg(target_os = "linux")] pub mod echo_cancel; #[cfg(target_os = "linux")] diff --git a/src/audio/resample.rs b/src/audio/resample.rs new file mode 100644 index 0000000..4560a67 --- /dev/null +++ b/src/audio/resample.rs @@ -0,0 +1,274 @@ +//! Dep-free linear-interpolation resamplers for the Windows/cpal backend (W4). +//! +//! The pipeline runs internally at 48 kHz (Opus + the 20 ms frame), but a WASAPI +//! endpoint may run at a different rate (commonly 44.1 kHz) and/or a non-stereo +//! channel layout. These convert at the device boundary so such a device plays and +//! captures instead of hard-erroring (the W4 limitation in the Windows port). +//! +//! ## Where each is used +//! - [`PushResampler`] (single channel) converts **capture** from the device rate +//! to 48 kHz on the capture drain thread — off the RT callback. +//! - [`StereoPullResampler`] converts **playback** from the internal 48 kHz stereo +//! bus to the device rate inside the output RT callback, pulling internal frames +//! from the ring on demand. It allocates nothing in `next`, so it is RT-safe. +//! +//! ## Quality +//! This is plain linear interpolation with no anti-aliasing filter: correct, +//! allocation-free, and adequate for speech, but it adds some aliasing when +//! downsampling. The seam is intentionally tiny so a higher-quality polyphase/FIR +//! resampler (e.g. the `rubato` crate, pending a supply-chain decision) can later +//! replace the internals without touching the cpal backend. The matching-rate / +//! matching-layout path in the backend bypasses these entirely and stays bit-exact. + +/// Linear interpolation between `a` and `b` at fractional position `frac` in `[0, 1)`. +#[inline] +fn lerp(a: f32, b: f32, frac: f32) -> f32 { + a + (b - a) * frac +} + +/// Stateful single-channel **push** resampler: feed input samples at `in_rate`, +/// receive output samples at `out_rate` through an `emit` callback. It carries the +/// fractional read position and the previous input sample across calls, so feeding +/// the stream block-by-block joins seamlessly. Neither [`push`](Self::push) nor +/// [`process`](Self::process) allocates. +pub struct PushResampler { + /// Input samples consumed per output sample (`in_rate / out_rate`). + step: f64, + /// Position of the next output sample, in input-sample units, measured from the + /// index of `prev` (the most recent input). Always advanced to stay `< 1.0` + /// after each input is consumed. + next: f64, + /// The previous input sample (left edge of the current interpolation segment). + prev: f32, + /// Whether any input has been seen yet (anchors the first output at input[0]). + started: bool, +} + +impl PushResampler { + /// Build a resampler from `in_rate` to `out_rate` (both in Hz, must be > 0). + pub fn new(in_rate: u32, out_rate: u32) -> Self { + debug_assert!(in_rate > 0 && out_rate > 0); + Self { + step: in_rate as f64 / out_rate as f64, + next: 0.0, + prev: 0.0, + started: false, + } + } + + /// Feed one input sample; `emit` is called for each output sample produced + /// (zero or more, depending on the rate ratio). + pub fn push(&mut self, cur: f32, mut emit: impl FnMut(f32)) { + if !self.started { + // First sample: just establish the left edge. Linear interpolation + // needs the next input as the right edge, so the first output is + // produced on the next push. This gives exact alignment + // (`output[k] == input[k]` at equal rates) with one input-sample of + // latency — negligible (~20 µs at 48 kHz). + self.started = true; + self.prev = cur; + self.next = 0.0; + return; + } + // `prev` sits at position 0 of this segment and `cur` at position 1; emit + // every output whose position falls in [0, 1). + while self.next < 1.0 { + emit(lerp(self.prev, cur, self.next as f32)); + self.next += self.step; + } + self.next -= 1.0; + self.prev = cur; + } + + /// Convenience for tests / batch callers: push a whole slice. + pub fn process(&mut self, input: &[f32], mut emit: impl FnMut(f32)) { + for &s in input { + self.push(s, &mut emit); + } + } +} + +/// Stateful stereo **pull** resampler: produce output frames at `out_rate` by +/// pulling input frames at `in_rate` from a closure on demand. Call +/// [`next`](Self::next) once per output frame; it pulls as many input frames as the +/// ratio requires and returns the interpolated `(left, right)`, or `None` when the +/// puller runs dry (an underrun). Allocates nothing, so it is safe in an RT output +/// callback. +pub struct StereoPullResampler { + /// Input frames consumed per output frame (`in_rate / out_rate`). + step: f64, + /// Position of the next output frame within `[prev, cur)`, in `[0, 1)`. + frac: f64, + /// Left edge of the current interpolation segment. + prev: (f32, f32), + /// Right edge of the current interpolation segment. + cur: (f32, f32), + /// Whether `prev`/`cur` have been primed from the puller yet. + primed: bool, +} + +impl StereoPullResampler { + /// Build a resampler from `in_rate` to `out_rate` (both in Hz, must be > 0). + pub fn new(in_rate: u32, out_rate: u32) -> Self { + debug_assert!(in_rate > 0 && out_rate > 0); + Self { + step: in_rate as f64 / out_rate as f64, + frac: 0.0, + prev: (0.0, 0.0), + cur: (0.0, 0.0), + primed: false, + } + } + + /// Produce the next output frame, pulling input frames via `pull` as needed. + /// Returns `None` if `pull` returns `None` before the frame can be formed + /// (underrun); the caller should substitute silence for that frame. + pub fn next(&mut self, mut pull: impl FnMut() -> Option<(f32, f32)>) -> Option<(f32, f32)> { + if !self.primed { + // Prime both edges from two pulls so the first output frame aligns + // exactly with the first input frame (`out[0] == in[0]` at equal + // rates). Needs two frames available to start, which the prefilled + // playback ring always has. + self.prev = pull()?; + self.cur = pull()?; + self.primed = true; + self.frac = 0.0; + } + // Advance the segment until the read position lands inside [prev, cur). + while self.frac >= 1.0 { + self.prev = self.cur; + self.cur = pull()?; + self.frac -= 1.0; + } + let f = self.frac as f32; + let out = (lerp(self.prev.0, self.cur.0, f), lerp(self.prev.1, self.cur.1, f)); + self.frac += self.step; + Some(out) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Equal rates align exactly: `output[k] == input[k]`. The final input lands on + /// the next push (one-sample streaming latency), so we get `n - 1` outputs. + #[test] + fn push_identity_when_rates_match() { + let mut r = PushResampler::new(48_000, 48_000); + let input = [0.0, 0.1, 0.2, 0.3, 0.4]; + let mut out = Vec::new(); + r.process(&input, |s| out.push(s)); + assert_eq!(out.len(), input.len() - 1); + for (a, b) in out.iter().zip(input.iter()) { + assert!((a - b).abs() < 1e-6, "{a} vs {b}"); + } + } + + /// Upsampling 2x roughly doubles the output count and the midpoints interpolate. + #[test] + fn push_upsample_2x_interpolates_midpoints() { + let mut r = PushResampler::new(24_000, 48_000); // step = 0.5 + let input = [0.0, 1.0, 2.0, 3.0]; + let mut out = Vec::new(); + r.process(&input, |s| out.push(s)); + // (n - 1) segments at 2 outputs each = 6. + assert_eq!(out.len(), 6, "out {out:?}"); + // A half-step between 1.0 and 2.0 must appear near 1.5. + assert!( + out.iter().any(|&s| (s - 1.5).abs() < 1e-3), + "expected a ~1.5 midpoint in {out:?}" + ); + } + + /// Downsampling drops the rate: fewer outputs than inputs, monotonic ramp preserved. + #[test] + fn push_downsample_reduces_count() { + let mut r = PushResampler::new(48_000, 44_100); // step ~1.088 + let input: Vec = (0..441).map(|i| i as f32).collect(); + let mut out = Vec::new(); + r.process(&input, |s| out.push(s)); + // 441 in @ 48k -> ~405 out @ 44.1k. + assert!( + (390..=410).contains(&out.len()), + "expected ~405 outputs, got {}", + out.len() + ); + // Output stays within the input's value range and is non-decreasing. + for w in out.windows(2) { + assert!(w[1] >= w[0] - 1e-3, "ramp should not reverse: {w:?}"); + } + assert!(*out.last().unwrap() <= 440.0 + 1e-3); + } + + /// Pull resampler at equal rates returns each input frame in order, aligned. + /// Two-pull priming uses one frame of lookahead, so `n` inputs yield `n - 1` + /// outputs (the last frame emits once a successor arrives). + #[test] + fn pull_identity_when_rates_match() { + let mut r = StereoPullResampler::new(48_000, 48_000); + let frames = [(0.0, 9.0), (1.0, 8.0), (2.0, 7.0), (3.0, 6.0)]; + let mut idx = 0; + let mut out = Vec::new(); + while let Some(f) = r.next(|| { + let v = frames.get(idx).copied(); + idx += 1; + v + }) { + out.push(f); + } + assert_eq!(out.len(), frames.len() - 1, "out {out:?}"); + for (got, want) in out.iter().zip(frames.iter()) { + assert!((got.0 - want.0).abs() < 1e-6 && (got.1 - want.1).abs() < 1e-6); + } + } + + /// Pull resampler reports underrun (`None`) once the source is exhausted. + #[test] + fn pull_returns_none_on_underrun() { + let mut r = StereoPullResampler::new(48_000, 44_100); // step ~1.088 -> pulls >1 per out + let frames = [(0.0, 0.0), (1.0, -1.0)]; + let mut idx = 0; + let mut pull = || { + let v = frames.get(idx).copied(); + idx += 1; + v + }; + // First frame primes + emits; subsequent calls eventually exhaust the source. + let mut produced = 0; + let mut hit_none = false; + for _ in 0..10 { + if r.next(&mut pull).is_some() { + produced += 1; + } else { + hit_none = true; + break; + } + } + assert!(produced >= 1, "should produce at least the primed frame"); + assert!(hit_none, "should report underrun once the puller is dry"); + } + + /// Downsampling via pull consumes more input frames than it emits output frames. + #[test] + fn pull_downsample_consumes_more_than_it_emits() { + let mut r = StereoPullResampler::new(48_000, 24_000); // step = 2.0 + let input: Vec<(f32, f32)> = (0..100).map(|i| (i as f32, -(i as f32))).collect(); + let mut idx = 0; + let mut emitted = 0; + for _ in 0..40 { + let f = r.next(|| { + let v = input.get(idx).copied(); + idx += 1; + v + }); + if f.is_some() { + emitted += 1; + } else { + break; + } + } + // At step 2.0 we consume ~2 input frames per output frame. + assert!(idx > emitted, "consumed {idx} input, emitted {emitted} output"); + } +}