From fde6f8a6802020e545a5eb8e114f013fbe075f39 Mon Sep 17 00:00:00 2001 From: Mollusk Date: Sat, 13 Jun 2026 23:48:07 -0400 Subject: [PATCH] =?UTF-8?q?feat(audio):=20multitrack=20stem=20recording=20?= =?UTF-8?q?=E2=80=94=20Stage=201=20(pure=20core=20+=20plan)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Scopes the differentiating "record every peer to their own synced track" feature and lands its pure, isolated core (no live-audio wiring yet). - docs/multitrack-recording-plan.md: scope contract + locked decisions (raw stems pre-volume/mute, stems + a mixed track, silence-pad late joiners). - src/audio/multitrack.rs: MultitrackRecorder over the existing WavWriter. One master clock = the mixer cycle; every end_cycle() appends exactly FRAME_SAMPLES to every track (silence where idle) so all stems stay sample-aligned. add_peer back-pads a late joiner to cycle 0; track_filename gives fs-safe `-.wav` (reuses sanitize_name). Optional mix track for "Both" mode. - +5 unit tests: equal length across tracks, late-joiner leading silence, stems-only omits mix, fit() pad/truncate, filename slugging/disambiguation. Stage 2 (wire into the mixer) is next, behind a checkpoint. 168 lib tests, clippy --all-targets clean. Co-Authored-By: Claude Opus 4.8 --- docs/multitrack-recording-plan.md | 41 ++++ src/audio/mod.rs | 1 + src/audio/multitrack.rs | 316 ++++++++++++++++++++++++++++++ 3 files changed, 358 insertions(+) create mode 100644 docs/multitrack-recording-plan.md create mode 100644 src/audio/multitrack.rs diff --git a/docs/multitrack-recording-plan.md b/docs/multitrack-recording-plan.md new file mode 100644 index 0000000..27a95d2 --- /dev/null +++ b/docs/multitrack-recording-plan.md @@ -0,0 +1,41 @@ +# Multitrack (stem) recording — plan / scope contract + +**Status:** Stage 1 in progress (2026-06-13). This doc is the scope contract; update it as stages land. + +## Goal & differentiation + +Record each call participant to their **own synced WAV track** (stems) — podcast/streamer-grade source for post-production — natively, offline, with no server or bot. This is possible because PeerSpeak is **full-mesh P2P**: every client already decodes each peer into its own buffer and sums them locally (the playout mixer in `src/core/mod.rs`). Server-mixed tools (Discord/Zoom) only get one pre-blended stream, so they can't do this without a bot (Craig) or paid tier. Holding the unmixed stems on every machine is the moat. + +## Locked design decisions (user, 2026-06-13) + +1. **Raw stems** — record each peer's decoded audio **before** local volume/mute/limiter, so the stems are pristine source (your local listening tweaks don't bake in). +2. **Output = stems + a mixed track** ("Both"): per-peer stems + your mic stem + a convenience mixed WAV, all in one session folder. +3. **Silence-pad late joiners from session start** — a peer who joins mid-recording gets leading silence so every track lines up at sample 0 (drop them all on a timeline → synced). + +## Architecture / tap point + +The mixer loop (`src/core/mod.rs` ~L761–788) already pops a decoded `frame` per peer with `peer_id` in scope, then discards the association into `peer_frames` for summing. We tap the **raw** frame there (before `apply_volume`/mute/limiter) for stems. Mic arrives separately via the capture task (`push_mic`, ~L663). Alignment uses the existing per-mixer-cycle clock: **every cycle, write exactly `FRAME_SAMPLES` (960) to every track** — real audio for peers who produced a frame this cycle, silence for those idle — so all tracks stay sample-aligned by construction (same principle as today's mic FIFO in `recorder.rs`). + +## Stages + +### Stage 1 — Pure `MultitrackRecorder` core (no wiring) — *in progress* +`src/audio/multitrack.rs`. Owns a per-peer `HashMap` + a mic track + an optional mix track (Both mode), each a reused `recorder::WavWriter`, plus a `cycles` counter (the master clock). API: +- `add_peer(id, name)` — create the track, pre-pad with `cycles * FRAME_SAMPLES` silence (late-join alignment); idempotent. +- `write_peer(id, &[i16])` / `write_mic(&[i16])` / `write_mix(&[i16])` — append one cycle's frame (fit to `FRAME_SAMPLES`: zero-pad if short), mark the track written-this-cycle. +- `end_cycle()` — pad every track NOT written this cycle with `FRAME_SAMPLES` silence, reset flags, `cycles += 1`. Guarantees equal lengths. +- `finalize(self)` — finalize all writers. +Pure/testable: tests assert "after N cycles every track is exactly N×FRAME_SAMPLES" and "a peer added at cycle k carries k×FRAME_SAMPLES leading silence." Plus a filesystem-safe track-naming helper (`me.wav`, `-.wav`, `mix.wav`) reusing `sanitize::sanitize_name`. + +### Stage 2 — Wire into the mixer + mic tasks +Retain `peer_id` alongside each decoded frame in the mixer; tap the raw frame for stems; per cycle call `write_peer` for present peers, `write_mix` with the finished mix, `write_mic` from the mic FIFO, then `end_cycle`. `add_peer` on join (`RoomEvent::PeerJoined`/`PeerUpdated` carries the name), keep the track on leave (it just goes silent). Keep the existing single-file mixed `Recorder` path for `Mixed` mode. Gate by `is_recording` like today. + +### Stage 3 — Config + UI +`AppConfig.recording_mode: Mixed | Multitrack | Both` (serde-default `Mixed`, back-compat). New **"Recording"** category in Settings (fits the category-header layout) with the mode picker + an output-dir note. Output to a per-session dir `~/peerspeak-recordings//` (Multitrack/Both); `Mixed` keeps today's single-file behaviour. + +### Stage 4 — Field-test (dopedart) +Solo first (mic + silence stems, inspect WAVs), then 2-machine desktop↔dopedart: each voice isolated on its own track, tracks sample-aligned + in sync, late-joiner leading-silence correct, peer-leave handled. + +## Notes / out of scope +- **Disk:** mono 48kHz/16-bit ≈ 5.5 MB/min/track. A 5-person 1-hr call ≈ ~1.6 GB. Acceptable; a FLAC option is a future wishlist item. +- Stems are mono (matches the whole pipeline). Spatial/stereo is a separate wishlist item (W1). +- No editing/mixing UI — files only; users post-produce in their DAW/editor. diff --git a/src/audio/mod.rs b/src/audio/mod.rs index 4e2a47d..9e513cf 100644 --- a/src/audio/mod.rs +++ b/src/audio/mod.rs @@ -54,6 +54,7 @@ pub trait AudioBackend: Send + Sync { pub mod echo_cancel; pub mod gate; pub mod limiter; +pub mod multitrack; pub mod pipewire_impl; pub mod pw_cli; pub mod recorder; diff --git a/src/audio/multitrack.rs b/src/audio/multitrack.rs new file mode 100644 index 0000000..ed228c2 --- /dev/null +++ b/src/audio/multitrack.rs @@ -0,0 +1,316 @@ +//! Multitrack (stem) recording: one synced WAV per peer + your mic, plus an +//! optional convenience mixed track — podcast/streamer-grade source for +//! post-production. See `docs/multitrack-recording-plan.md`. +//! +//! All tracks share one master clock: the playout-mixer cycle. Every cycle, +//! **exactly `FRAME_SAMPLES` samples are appended to every track** — real audio +//! for peers who produced a frame that cycle, silence for those idle — so every +//! track stays sample-aligned by construction. A peer that joins mid-recording +//! has its track pre-padded with silence back to cycle 0, so all stems line up +//! at sample 0 on a timeline. +//! +//! This module is pure plumbing over [`WavWriter`]: no audio decode, no +//! networking, no realtime work. The mixer (a non-RT task) drives it. + +use std::collections::HashMap; +use std::io; +use std::path::{Path, PathBuf}; + +use iroh::EndpointId; + +use crate::audio::recorder::WavWriter; +use crate::core::jitter::FRAME_SAMPLES; + +/// Cap on the silence chunk written at once when pre-padding a late joiner, so a +/// long-running call can't trigger a single multi-hundred-MB allocation. +const SILENCE_CHUNK: usize = FRAME_SAMPLES * 256; + +/// One output track: its WAV writer plus whether it has been written *this* +/// cycle (so `end_cycle` knows which tracks to pad with silence). +struct Track { + writer: WavWriter, + written_this_cycle: bool, +} + +impl Track { + fn create(path: &Path) -> io::Result { + Ok(Self { + writer: WavWriter::new(path)?, + written_this_cycle: false, + }) + } + + /// Append `frame` fitted to exactly `frame_samples` (zero-padded if short), + /// and mark the track as written for this cycle. + fn write_frame(&mut self, frame: &[i16], frame_samples: usize) -> io::Result<()> { + self.writer.write_samples(&fit(frame, frame_samples))?; + self.written_this_cycle = true; + Ok(()) + } + + /// Append `samples` of silence (no cycle-marking — used for padding). + fn write_silence(&mut self, samples: usize) -> io::Result<()> { + let mut remaining = samples; + while remaining > 0 { + let n = remaining.min(SILENCE_CHUNK); + self.writer.write_samples(&vec![0i16; n])?; + remaining -= n; + } + Ok(()) + } +} + +/// Return `frame` resized to exactly `n` samples: truncated if longer (shouldn't +/// happen — Opus frames are uniform), zero-padded if shorter. +fn fit(frame: &[i16], n: usize) -> Vec { + let mut v = Vec::with_capacity(n); + let take = frame.len().min(n); + v.extend_from_slice(&frame[..take]); + v.resize(n, 0); + v +} + +/// A filesystem-safe stem filename: a slug of the (already untrusted-sanitized) +/// display name plus a short id suffix to disambiguate same-named peers, e.g. +/// `alice-3b1f9c2a.wav`. Falls back to `peer` when the name slugs to nothing. +pub fn track_filename(name: &str, id: &EndpointId) -> String { + let clean = crate::sanitize::sanitize_name(name); + let mut slug: String = clean + .chars() + .map(|c| if c.is_ascii_alphanumeric() { c.to_ascii_lowercase() } else { '-' }) + .collect(); + // Collapse runs of '-' and trim them off the ends. + while slug.contains("--") { + slug = slug.replace("--", "-"); + } + let slug = slug.trim_matches('-'); + let slug = if slug.is_empty() { "peer" } else { slug }; + let short: String = id.to_string().chars().take(8).collect(); + format!("{slug}-{short}.wav") +} + +/// A live multitrack recording: per-peer stems + your mic, plus an optional +/// mixed track, all under one session directory and clocked together. +pub struct MultitrackRecorder { + dir: PathBuf, + frame_samples: usize, + /// Cycles recorded so far = the shared length (in frames) of every track. + cycles: u64, + peers: HashMap, + mic: Track, + /// Present in "Both" mode (stems + mixed), absent in "stems only". + mix: Option, +} + +impl MultitrackRecorder { + /// Create a recording in `dir` (which must already exist). `with_mix` adds + /// the convenience mixed track (`mix.wav`). Your mic is always `me.wav`. + pub fn create(dir: &Path, frame_samples: usize, with_mix: bool) -> io::Result { + let mic = Track::create(&dir.join("me.wav"))?; + let mix = if with_mix { + Some(Track::create(&dir.join("mix.wav"))?) + } else { + None + }; + Ok(Self { + dir: dir.to_path_buf(), + frame_samples, + cycles: 0, + peers: HashMap::new(), + mic, + mix, + }) + } + + /// Register a peer's stem track, pre-padding it with silence back to cycle 0 + /// so it aligns with the others. Idempotent: a peer already tracked is left + /// as-is (re-announce / name change doesn't restart their file). + pub fn add_peer(&mut self, id: EndpointId, name: &str) -> io::Result<()> { + if self.peers.contains_key(&id) { + return Ok(()); + } + let mut track = Track::create(&self.dir.join(track_filename(name, &id)))?; + track.write_silence(self.cycles as usize * self.frame_samples)?; + self.peers.insert(id, track); + Ok(()) + } + + /// Record one peer's decoded frame for the current cycle. If the peer wasn't + /// registered yet (write raced ahead of the join event), auto-register it + /// with an id-only name so no audio is dropped. + pub fn write_peer(&mut self, id: EndpointId, frame: &[i16]) -> io::Result<()> { + if !self.peers.contains_key(&id) { + self.add_peer(id, "")?; + } + let fs = self.frame_samples; + self.peers.get_mut(&id).unwrap().write_frame(frame, fs) + } + + /// Record your mic frame for the current cycle. + pub fn write_mic(&mut self, frame: &[i16]) -> io::Result<()> { + let fs = self.frame_samples; + self.mic.write_frame(frame, fs) + } + + /// Record the finished mixed-bus frame for the current cycle (no-op in + /// stems-only mode). + pub fn write_mix(&mut self, frame: &[i16]) -> io::Result<()> { + let fs = self.frame_samples; + if let Some(mix) = self.mix.as_mut() { + mix.write_frame(frame, fs)?; + } + Ok(()) + } + + /// Close out the current cycle: every track that wasn't written this cycle + /// gets one frame of silence, so all tracks advance in lockstep. Call once + /// per mixer cycle, after the per-track writes. + pub fn end_cycle(&mut self) -> io::Result<()> { + let fs = self.frame_samples; + for track in self + .peers + .values_mut() + .chain(std::iter::once(&mut self.mic)) + .chain(self.mix.as_mut()) + { + if !track.written_this_cycle { + track.write_silence(fs)?; + } + track.written_this_cycle = false; + } + self.cycles += 1; + Ok(()) + } + + /// Finalize every track's WAV header. Consumes the recorder. + pub fn finalize(self) -> io::Result<()> { + self.mic.writer.finalize()?; + if let Some(mix) = self.mix { + mix.writer.finalize()?; + } + for (_, track) in self.peers { + track.writer.finalize()?; + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use iroh::SecretKey; + + fn an_id() -> EndpointId { + SecretKey::generate().public() + } + + /// Samples of PCM data in a finished WAV file: (len - 44-byte header) / 2. + fn wav_samples(path: &Path) -> usize { + let len = std::fs::metadata(path).unwrap().len() as usize; + (len - 44) / 2 + } + + fn tmpdir(tag: &str) -> PathBuf { + let d = std::env::temp_dir().join(format!("ps-mt-{}-{}", tag, std::process::id())); + let _ = std::fs::remove_dir_all(&d); + std::fs::create_dir_all(&d).unwrap(); + d + } + + #[test] + fn fit_pads_and_truncates() { + assert_eq!(fit(&[1, 2], 4), vec![1, 2, 0, 0]); + assert_eq!(fit(&[1, 2, 3, 4], 2), vec![1, 2]); + assert_eq!(fit(&[], 3), vec![0, 0, 0]); + } + + #[test] + fn track_filename_is_fs_safe_and_disambiguated() { + let id = an_id(); + let short: String = id.to_string().chars().take(8).collect(); + assert_eq!(track_filename("Alice", &id), format!("alice-{short}.wav")); + // Spaces / punctuation collapse to single dashes, trimmed. + assert_eq!(track_filename(" Bob the Builder! ", &id), format!("bob-the-builder-{short}.wav")); + // A name that sanitizes/slugs to nothing falls back to "peer". + assert_eq!(track_filename("!!!", &id), format!("peer-{short}.wav")); + } + + #[test] + fn all_tracks_equal_length_after_n_cycles() { + let dir = tmpdir("equal"); + let frame = 4; // tiny frame for the test + let p1 = an_id(); + let p2 = an_id(); + let mut rec = MultitrackRecorder::create(&dir, frame, true).unwrap(); + rec.add_peer(p1, "p1").unwrap(); + rec.add_peer(p2, "p2").unwrap(); + + // 3 cycles; p1 talks every cycle, p2 only on cycle 2, mic talks twice. + for c in 0..3 { + rec.write_peer(p1, &[1, 1, 1, 1]).unwrap(); + if c == 2 { + rec.write_peer(p2, &[2, 2, 2, 2]).unwrap(); + } + if c < 2 { + rec.write_mic(&[9, 9, 9, 9]).unwrap(); + } + rec.write_mix(&[5, 5, 5, 5]).unwrap(); + rec.end_cycle().unwrap(); + } + rec.finalize().unwrap(); + + let expected = 3 * frame; + assert_eq!(wav_samples(&dir.join("me.wav")), expected, "mic padded to full length"); + assert_eq!(wav_samples(&dir.join("mix.wav")), expected); + assert_eq!(wav_samples(&dir.join(track_filename("p1", &p1))), expected); + assert_eq!(wav_samples(&dir.join(track_filename("p2", &p2))), expected, "silent peer still full length"); + } + + #[test] + fn late_joiner_is_silence_padded_to_start() { + let dir = tmpdir("late"); + let frame = 4; + let early = an_id(); + let late = an_id(); + let mut rec = MultitrackRecorder::create(&dir, frame, false).unwrap(); + rec.add_peer(early, "early").unwrap(); + + // 2 cycles before the late peer joins. + for _ in 0..2 { + rec.write_peer(early, &[1, 1, 1, 1]).unwrap(); + rec.end_cycle().unwrap(); + } + // Late peer joins at cycle 2. + rec.add_peer(late, "late").unwrap(); + for _ in 0..3 { + rec.write_peer(early, &[1, 1, 1, 1]).unwrap(); + rec.write_peer(late, &[2, 2, 2, 2]).unwrap(); + rec.end_cycle().unwrap(); + } + rec.finalize().unwrap(); + + // Both tracks are the full 5 cycles long (late one was back-padded). + assert_eq!(wav_samples(&dir.join(track_filename("early", &early))), 5 * frame); + assert_eq!(wav_samples(&dir.join(track_filename("late", &late))), 5 * frame); + + // The late track's first 2 cycles are silence, then the real audio. + let bytes = std::fs::read(dir.join(track_filename("late", &late))).unwrap(); + let data = &bytes[44..]; + let read_sample = |i: usize| i16::from_le_bytes([data[i * 2], data[i * 2 + 1]]); + for i in 0..(2 * frame) { + assert_eq!(read_sample(i), 0, "leading silence at sample {i}"); + } + assert_eq!(read_sample(2 * frame), 2, "real audio starts at cycle 2"); + } + + #[test] + fn stems_only_writes_no_mix_file() { + let dir = tmpdir("nomix"); + let mut rec = MultitrackRecorder::create(&dir, 4, false).unwrap(); + rec.write_mix(&[1, 2, 3, 4]).unwrap(); // no-op + rec.end_cycle().unwrap(); + rec.finalize().unwrap(); + assert!(dir.join("me.wav").exists()); + assert!(!dir.join("mix.wav").exists(), "no mix track in stems-only mode"); + } +}