From f6520b79f75c53ba5438ef78c3f90911a7a64196 Mon Sep 17 00:00:00 2001 From: Mollusk Date: Sun, 14 Jun 2026 00:04:49 -0400 Subject: [PATCH] =?UTF-8?q?feat(audio):=20multitrack=20stem=20recording=20?= =?UTF-8?q?=E2=80=94=20Stage=202=20(wire=20into=20the=20mixer)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wires MultitrackRecorder into the live audio path, behind a recording_mode. - config: RecordingMode { Mixed, Multitrack, Both } + AppConfig.recording_mode (serde-default Mixed, back-compat); CoreCommand::SetRecordingMode, sent at app startup from config. - multitrack.rs: mic now arrives async via push_mic into an internal FIFO, drained one frame per end_cycle (mirrors recorder.rs) so the mic track tracks the cycle clock; added dir() accessor. mic is a plain WavWriter now. - core: parallel `multitrack` slot + `is_multitrack` fast-path gate (exactly one of the mixed/multitrack recorders is active). SetRecording start branches on mode: Mixed → single-file Recorder (unchanged); Multitrack/Both → a per-session dir, MultitrackRecorder, and registers everyone already in the room (named, silence-aligned from t=0). The mixer taps each peer's RAW frame (pre-volume/ mute/limiter) into stems and writes peer stems + mix (Both) + end_cycle per cycle; the capture thread pushes mic to whichever recorder; PeerJoined adds a late joiner's stem track. stop_recording finalizes both. No UI yet to pick the mode (Stage 3) — defaults to Mixed, so behaviour is unchanged until then; set recording_mode in config.json to exercise stems. 168 lib tests, clippy --all-targets clean. Co-Authored-By: Claude Opus 4.8 --- src/app/mod.rs | 1 + src/audio/multitrack.rs | 60 +++++++++++----- src/config.rs | 39 ++++++++++ src/core/messages.rs | 5 +- src/core/mod.rs | 153 ++++++++++++++++++++++++++++++++++------ 5 files changed, 218 insertions(+), 40 deletions(-) diff --git a/src/app/mod.rs b/src/app/mod.rs index 42c8e7c..58dec01 100644 --- a/src/app/mod.rs +++ b/src/app/mod.rs @@ -267,6 +267,7 @@ impl Default for AppState { let _ = controller.send(CoreCommand::SetInputVolume(config.input_volume)); let _ = controller.send(CoreCommand::SetOutputVolume(config.output_volume)); let _ = controller.send(CoreCommand::SetNetworkMode(config.network_mode)); + let _ = controller.send(CoreCommand::SetRecordingMode(config.recording_mode)); let _ = controller.send(CoreCommand::SetPixelpassPath(config.pixelpass_path.clone())); let pixelpass_available = crate::screenshare::is_available(config.pixelpass_path.as_deref()); diff --git a/src/audio/multitrack.rs b/src/audio/multitrack.rs index ed228c2..5c5228f 100644 --- a/src/audio/multitrack.rs +++ b/src/audio/multitrack.rs @@ -12,7 +12,7 @@ //! This module is pure plumbing over [`WavWriter`]: no audio decode, no //! networking, no realtime work. The mixer (a non-RT task) drives it. -use std::collections::HashMap; +use std::collections::{HashMap, VecDeque}; use std::io; use std::path::{Path, PathBuf}; @@ -25,6 +25,11 @@ use crate::core::jitter::FRAME_SAMPLES; /// long-running call can't trigger a single multi-hundred-MB allocation. const SILENCE_CHUNK: usize = FRAME_SAMPLES * 256; +/// Cap on buffered mic samples (~200ms @ 48kHz). Bounds how far the mic track +/// can drift if the capture clock runs ahead of the mixer cycle; past it the +/// oldest mic audio is dropped. Mirrors `recorder::MAX_MIC_FIFO`. +const MAX_MIC_FIFO: usize = 48_000 / 5; + /// One output track: its WAV writer plus whether it has been written *this* /// cycle (so `end_cycle` knows which tracks to pad with silence). struct Track { @@ -97,7 +102,11 @@ pub struct MultitrackRecorder { /// Cycles recorded so far = the shared length (in frames) of every track. cycles: u64, peers: HashMap, - mic: Track, + /// Your mic track. Fed asynchronously from the capture thread via + /// [`push_mic`](MultitrackRecorder::push_mic) into `mic_fifo`, then drained + /// one frame per `end_cycle` so it aligns with the cycle clock. + mic: WavWriter, + mic_fifo: VecDeque, /// Present in "Both" mode (stems + mixed), absent in "stems only". mix: Option, } @@ -106,7 +115,7 @@ impl MultitrackRecorder { /// Create a recording in `dir` (which must already exist). `with_mix` adds /// the convenience mixed track (`mix.wav`). Your mic is always `me.wav`. pub fn create(dir: &Path, frame_samples: usize, with_mix: bool) -> io::Result { - let mic = Track::create(&dir.join("me.wav"))?; + let mic = WavWriter::new(&dir.join("me.wav"))?; let mix = if with_mix { Some(Track::create(&dir.join("mix.wav"))?) } else { @@ -118,10 +127,16 @@ impl MultitrackRecorder { cycles: 0, peers: HashMap::new(), mic, + mic_fifo: VecDeque::new(), mix, }) } + /// The session directory holding all the track files. + pub fn dir(&self) -> &Path { + &self.dir + } + /// Register a peer's stem track, pre-padding it with silence back to cycle 0 /// so it aligns with the others. Idempotent: a peer already tracked is left /// as-is (re-announce / name change doesn't restart their file). @@ -146,10 +161,23 @@ impl MultitrackRecorder { self.peers.get_mut(&id).unwrap().write_frame(frame, fs) } - /// Record your mic frame for the current cycle. - pub fn write_mic(&mut self, frame: &[i16]) -> io::Result<()> { - let fs = self.frame_samples; - self.mic.write_frame(frame, fs) + /// Buffer a frame of your transmitted mic audio (called from the capture + /// thread, asynchronously to the mixer cycle). Bounded: oldest samples drop + /// past [`MAX_MIC_FIFO`] so the mic track's offset can't grow without limit. + pub fn push_mic(&mut self, frame: &[i16]) { + self.mic_fifo.extend(frame.iter().copied()); + let overflow = self.mic_fifo.len().saturating_sub(MAX_MIC_FIFO); + if overflow > 0 { + self.mic_fifo.drain(..overflow); + } + } + + /// Pull exactly `n` mic samples from the FIFO, silence-padded on underrun. + fn drain_mic(&mut self, n: usize) -> Vec { + let take = n.min(self.mic_fifo.len()); + let mut v: Vec = self.mic_fifo.drain(..take).collect(); + v.resize(n, 0); + v } /// Record the finished mixed-bus frame for the current cycle (no-op in @@ -167,12 +195,12 @@ impl MultitrackRecorder { /// per mixer cycle, after the per-track writes. pub fn end_cycle(&mut self) -> io::Result<()> { let fs = self.frame_samples; - for track in self - .peers - .values_mut() - .chain(std::iter::once(&mut self.mic)) - .chain(self.mix.as_mut()) - { + // Mic: always one frame per cycle, drained from the FIFO (silence on + // underrun), so it tracks the cycle clock like the peer stems. + let mic_frame = self.drain_mic(fs); + self.mic.write_samples(&mic_frame)?; + // Peers + the optional mix track: pad any not written this cycle. + for track in self.peers.values_mut().chain(self.mix.as_mut()) { if !track.written_this_cycle { track.write_silence(fs)?; } @@ -184,7 +212,7 @@ impl MultitrackRecorder { /// Finalize every track's WAV header. Consumes the recorder. pub fn finalize(self) -> io::Result<()> { - self.mic.writer.finalize()?; + self.mic.finalize()?; if let Some(mix) = self.mix { mix.writer.finalize()?; } @@ -245,14 +273,14 @@ mod tests { rec.add_peer(p1, "p1").unwrap(); rec.add_peer(p2, "p2").unwrap(); - // 3 cycles; p1 talks every cycle, p2 only on cycle 2, mic talks twice. + // 3 cycles; p1 talks every cycle, p2 only on cycle 2, mic pushed twice. for c in 0..3 { rec.write_peer(p1, &[1, 1, 1, 1]).unwrap(); if c == 2 { rec.write_peer(p2, &[2, 2, 2, 2]).unwrap(); } if c < 2 { - rec.write_mic(&[9, 9, 9, 9]).unwrap(); + rec.push_mic(&[9, 9, 9, 9]); } rec.write_mix(&[5, 5, 5, 5]).unwrap(); rec.end_cycle().unwrap(); diff --git a/src/config.rs b/src/config.rs index b8fe617..3105e19 100644 --- a/src/config.rs +++ b/src/config.rs @@ -44,6 +44,41 @@ impl RoomLayout { [RoomLayout::ThreeColumn, RoomLayout::BottomDock, RoomLayout::Drawer]; } +/// What a call recording captures. `Mixed` is the original single-file behaviour; +/// `Multitrack`/`Both` write per-peer stems for post-production (see +/// `docs/multitrack-recording-plan.md`). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] +pub enum RecordingMode { + /// One mixed WAV (your mic + the incoming mix). Smallest; the default. + #[default] + Mixed, + /// One WAV per peer + your mic, all sample-aligned. Rebuild the mix yourself. + Multitrack, + /// Per-peer stems + your mic + a convenience mixed track. + Both, +} + +impl RecordingMode { + /// All variants, in picker display order. + pub const ALL: [RecordingMode; 3] = + [RecordingMode::Mixed, RecordingMode::Multitrack, RecordingMode::Both]; + + /// True when this mode writes per-peer stem tracks (Multitrack or Both). + pub fn is_multitrack(self) -> bool { + matches!(self, RecordingMode::Multitrack | RecordingMode::Both) + } +} + +impl std::fmt::Display for RecordingMode { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(match self { + RecordingMode::Mixed => "Mixed (single file)", + RecordingMode::Multitrack => "Multitrack (per-peer stems)", + RecordingMode::Both => "Both (stems + mixed)", + }) + } +} + impl std::fmt::Display for RoomLayout { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.write_str(match self { @@ -141,6 +176,9 @@ pub struct AppConfig { /// Chosen UI colour theme. #[serde(default)] pub theme: AppTheme, + /// What a call recording captures (mixed / per-peer stems / both). + #[serde(default)] + pub recording_mode: RecordingMode, #[serde(default)] pub custom_sound_self_join: Option, #[serde(default)] @@ -195,6 +233,7 @@ impl Default for AppConfig { chat_drawer_width: default_chat_drawer_width(), room_layout: RoomLayout::default(), theme: AppTheme::default(), + recording_mode: RecordingMode::default(), custom_sound_self_join: None, custom_sound_peer_join: None, custom_sound_peer_leave: None, diff --git a/src/core/messages.rs b/src/core/messages.rs index 3f9c1d9..8037895 100644 --- a/src/core/messages.rs +++ b/src/core/messages.rs @@ -1,4 +1,4 @@ -use crate::config::NetworkMode; +use crate::config::{NetworkMode, RecordingMode}; use crate::network::PeerState; use iroh::EndpointId; @@ -29,6 +29,9 @@ pub enum CoreCommand { /// Start/stop recording the call to a local WAV (your mic + the incoming /// mix). No-op start if already recording / not in a call. SetRecording(bool), + /// Set what a recording captures (mixed / per-peer stems / both). Takes + /// effect on the next recording start. Sent at startup from config. + SetRecordingMode(RecordingMode), /// Broadcast a room text-chat message. No-op when not in a call. SendChat(String), /// Set the pixelpass binary location (config override, empty = use `$PATH`). diff --git a/src/core/mod.rs b/src/core/mod.rs index 678fa96..d27f26e 100644 --- a/src/core/mod.rs +++ b/src/core/mod.rs @@ -11,7 +11,8 @@ use crate::network::{ }; use crate::core::messages::{CoreCommand, UiEvent}; -use crate::config::NetworkMode; +use crate::config::{NetworkMode, RecordingMode}; +use crate::audio::multitrack::MultitrackRecorder; use iroh::{Endpoint, EndpointId, RelayMode, endpoint::presets, protocol::Router}; use iroh_gossip::net::Gossip; use tokio::sync::{mpsc, Mutex}; @@ -379,9 +380,15 @@ impl ActiveSession { async fn stop_recording( recorder: &Arc>>, is_recording: &Arc, + multitrack: &Arc>>, + is_multitrack: &Arc, ui_tx: &mpsc::Sender, ) { is_recording.store(false, Ordering::Relaxed); + is_multitrack.store(false, Ordering::Relaxed); + // Exactly one slot is ever active for a given recording, but finalize both + // defensively. The reported path is the file (mixed) or the session dir + // (multitrack). let rec = recorder.lock().unwrap().take(); if let Some(rec) = rec { let path = rec.path().to_string_lossy().to_string(); @@ -391,6 +398,15 @@ async fn stop_recording( crate::log_msg(&format!("Recording saved: {path}")); let _ = ui_tx.send(UiEvent::RecordingStopped { path }).await; } + let mt = multitrack.lock().unwrap().take(); + if let Some(mt) = mt { + let path = mt.dir().to_string_lossy().to_string(); + if let Err(e) = mt.finalize() { + crate::log_msg(&format!("Failed to finalize multitrack recording: {e}")); + } + crate::log_msg(&format!("Multitrack recording saved: {path}/")); + let _ = ui_tx.send(UiEvent::RecordingStopped { path }).await; + } } async fn run_core_loop( @@ -417,6 +433,14 @@ async fn run_core_loop( let recorder: Arc>> = Arc::new(std::sync::Mutex::new(None)); let is_recording = Arc::new(AtomicBool::new(false)); + // Multitrack (stem) recording: a parallel recorder used instead of the mixed + // `recorder` when `recording_mode` is Multitrack/Both. Exactly one of the two + // slots is ever active. `is_multitrack` is the fast-path gate the audio loops + // read (cheap) to decide whether to tap raw per-peer stems this cycle. + let multitrack: Arc>> = + Arc::new(std::sync::Mutex::new(None)); + let is_multitrack = Arc::new(AtomicBool::new(false)); + let mut recording_mode = RecordingMode::default(); let peer_volumes = Arc::new(Mutex::new(HashMap::::new())); // Peers locally muted by us: decoded for level metering but not mixed. let locally_muted = Arc::new(Mutex::new(HashSet::::new())); @@ -438,7 +462,7 @@ async fn run_core_loop( // Finalize any recording before tearing down the old session — its // capture/mixer feeders are about to stop. - stop_recording(&recorder, &is_recording, &ui_tx).await; + stop_recording(&recorder, &is_recording, &multitrack, &is_multitrack, &ui_tx).await; // Clean up any existing session if let Some(session) = active_session.take() { @@ -608,6 +632,8 @@ async fn run_core_loop( let ui_tx_capture = ui_tx.clone(); let recorder_capture = recorder.clone(); let is_recording_capture = is_recording.clone(); + let multitrack_capture = multitrack.clone(); + let is_multitrack_capture = is_multitrack.clone(); let capture_thread = std::thread::spawn(move || { use opus::{Channels, Application}; @@ -657,10 +683,14 @@ async fn run_core_loop( // Record what we transmit (post-gain, post-gate, post-mute): // this is exactly the mic audio peers receive from us. The // mixer task pairs it with the incoming mix. - if is_recording_capture.load(Ordering::Relaxed) - && let Some(rec) = recorder_capture.lock().unwrap().as_mut() - { - rec.push_mic(&pcm); + if is_recording_capture.load(Ordering::Relaxed) { + if is_multitrack_capture.load(Ordering::Relaxed) { + if let Some(mt) = multitrack_capture.lock().unwrap().as_mut() { + mt.push_mic(&pcm); + } + } else if let Some(rec) = recorder_capture.lock().unwrap().as_mut() { + rec.push_mic(&pcm); + } } if let Ok(encoded) = encoder.encode(&pcm) { @@ -729,6 +759,8 @@ async fn run_core_loop( let ring_fill_mixer = ring_fill.clone(); let recorder_mixer = recorder.clone(); let is_recording_mixer = is_recording.clone(); + let multitrack_mixer = multitrack.clone(); + let is_multitrack_mixer = is_multitrack.clone(); let mixer_task = tokio::spawn(async move { // Mix-bus soft limiter: rides loud multi-peer moments down to // the ceiling instead of hard-clipping. State carries across @@ -760,6 +792,13 @@ async fn run_core_loop( let muted_peers = locally_muted_mixer.lock().await.clone(); let mut peer_frames = Vec::new(); + // Multitrack stem capture: tap each peer's RAW decoded frame + // (pre-volume, pre-mute, pre-limiter) so the stems are clean + // source. Only collected while a multitrack recording is live. + let mt_active = is_recording_mixer.load(Ordering::Relaxed) + && is_multitrack_mixer.load(Ordering::Relaxed); + let mut stems: Vec<(EndpointId, Vec)> = Vec::new(); + { let mut guard = jitter_mixer.lock().await; for (&peer_id, buffer) in guard.iter_mut() { @@ -770,6 +809,10 @@ async fn run_core_loop( continue; }; + if mt_active { + stems.push((peer_id, frame.clone())); + } + let vol = current_volumes.get(&peer_id).copied().unwrap_or(1.0); apply_volume(&mut frame, vol); @@ -795,10 +838,25 @@ async fn run_core_loop( let out_gain = f32::from_bits(output_gain_mixer.load(Ordering::Relaxed)); let mixed = limiter.process(&mixed_sum, out_gain); - // Record the true call audio (incoming mix + our mic), - // independent of local deafen — deafen only silences our - // own monitor, not what the call actually carried. - if is_recording_mixer.load(Ordering::Relaxed) + // Record the true call audio, independent of local deafen — + // deafen only silences our own monitor, not what the call + // carried. Multitrack writes raw per-peer stems (+ the mixed + // track in Both mode) one aligned frame per cycle; Mixed mode + // writes the single blended file as before. + if mt_active { + if let Some(mt) = multitrack_mixer.lock().unwrap().as_mut() { + let res = (|| -> std::io::Result<()> { + for (id, f) in &stems { + mt.write_peer(*id, f)?; + } + mt.write_mix(&mixed)?; + mt.end_cycle() + })(); + if let Err(e) = res { + crate::log_msg(&format!("Multitrack write failed: {e}")); + } + } + } else if is_recording_mixer.load(Ordering::Relaxed) && let Some(rec) = recorder_mixer.lock().unwrap().as_mut() && let Err(e) = rec.write_frame(&mixed) { @@ -840,6 +898,8 @@ async fn run_core_loop( let grace_timers_events = grace_timers.clone(); let seen_connected: SeenConnected = Arc::new(std::sync::Mutex::new(HashSet::new())); let seen_connected_events = seen_connected.clone(); + let multitrack_events = multitrack.clone(); + let is_multitrack_events = is_multitrack.clone(); let event_task = tokio::spawn(async move { while let Some(event) = room_events.recv().await { match event { @@ -852,6 +912,14 @@ async fn run_core_loop( // Hand over the full address so reconnects can dial // it directly rather than via the gossip lookup. transport_events.connect_peer(state.addr.clone()).await; + // If a multitrack recording is live, give this peer + // its own stem track (silence-padded back to t=0). + if is_multitrack_events.load(Ordering::Relaxed) + && let Some(mt) = multitrack_events.lock().unwrap().as_mut() + && let Err(e) = mt.add_peer(peer_id, &state.name) + { + crate::log_msg(&format!("multitrack add_peer (join) failed: {e}")); + } let _ = ui_tx_events.send(UiEvent::PeerJoined { id: peer_id, state }).await; } RoomEvent::PeerLeft(peer_id) => { @@ -944,7 +1012,7 @@ async fn run_core_loop( CoreCommand::Leave => { // Finalize any recording first, while the audio feeders are alive. - stop_recording(&recorder, &is_recording, &ui_tx).await; + stop_recording(&recorder, &is_recording, &multitrack, &is_multitrack, &ui_tx).await; current_sharing = None; if let Some(session) = active_session.take() { session.shutdown(audio_backend.clone()).await; @@ -1039,6 +1107,10 @@ async fn run_core_loop( network_mode = mode; } + CoreCommand::SetRecordingMode(mode) => { + recording_mode = mode; + } + CoreCommand::SetRecording(enabled) => { if enabled { // Only record while in a call, and not already recording. @@ -1049,23 +1121,58 @@ async fn run_core_loop( } else if !is_recording.load(Ordering::Relaxed) { match dirs::home_dir() { Some(home) => { - let dir = home.join("peerspeak-recordings"); + let base = home.join("peerspeak-recordings"); let now = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .map(|d| d.as_secs()) .unwrap_or(0); - let started = std::fs::create_dir_all(&dir) - .map_err(|e| e.to_string()) - .and_then(|_| { - crate::audio::recorder::Recorder::create(&dir, now) + let result: Result = if recording_mode.is_multitrack() { + // Multitrack/Both: a per-session directory of stems. + let stamp = crate::audio::recorder::timestamp_filename(now); + let session_dir = base.join(stamp.trim_end_matches(".wav")); + std::fs::create_dir_all(&session_dir) + .map_err(|e| e.to_string()) + .and_then(|_| { + MultitrackRecorder::create( + &session_dir, + FRAME_SAMPLES, + matches!(recording_mode, RecordingMode::Both), + ) .map_err(|e| e.to_string()) - }); - match started { - Ok(rec) => { - let path = rec.path().to_string_lossy().to_string(); - *recorder.lock().unwrap() = Some(rec); + }) + .map(|mut mt| { + // Register everyone already in the room so their + // stems are silence-aligned from t=0. + if let Some(session) = &active_session { + for (id, st) in session.room_state.active_peers() { + if let Err(e) = mt.add_peer(id, &st.name) { + crate::log_msg(&format!("multitrack add_peer failed: {e}")); + } + } + } + let path = mt.dir().to_string_lossy().to_string(); + *multitrack.lock().unwrap() = Some(mt); + is_multitrack.store(true, Ordering::Relaxed); + path + }) + } else { + // Mixed: one file (original behaviour). + std::fs::create_dir_all(&base) + .map_err(|e| e.to_string()) + .and_then(|_| { + crate::audio::recorder::Recorder::create(&base, now) + .map_err(|e| e.to_string()) + }) + .map(|rec| { + let path = rec.path().to_string_lossy().to_string(); + *recorder.lock().unwrap() = Some(rec); + path + }) + }; + match result { + Ok(path) => { is_recording.store(true, Ordering::Relaxed); - crate::log_msg(&format!("Recording started: {path}")); + crate::log_msg(&format!("Recording started ({recording_mode:?}): {path}")); let _ = ui_tx .send(UiEvent::RecordingStarted { path }) .await; @@ -1085,7 +1192,7 @@ async fn run_core_loop( } } } else { - stop_recording(&recorder, &is_recording, &ui_tx).await; + stop_recording(&recorder, &is_recording, &multitrack, &is_multitrack, &ui_tx).await; } }