mollusk/peerspeak
Public Access
1
0
Fork 0
Code Issues Pull Requests 3 Actions Packages Projects Releases 10 Wiki Activity

Add inline chat audio player

Browse Source
This commit is contained in:
mollusk
2026-06-21 01:05:24 -04:00
parent 79b24fd567
commit bcb597a0ea
6 changed files with 535 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.lock
Generated
+1
View File
@@ -2628,6 +2628,7 @@ dependencies = [
"iced_core",
"log",
"rustc-hash 2.1.2",
"tokio",
"wasm-bindgen-futures",
"wasmtimer",
]
Cargo.toml
+1 -1
View File
@@ -28,7 +28,7 @@ async-trait = "0.1.89"
base64 = "0.22.1"
bytes = "1.11.1"
dirs = "6.0.0"
iced = { version = "0.14.0", features = ["canvas", "image"] }
iced = { version = "0.14.0", features = ["canvas", "image", "tokio"] }
# W4 custom avatars: decode/resize an arbitrary user image (png/jpeg only to keep
# the codec surface small). The matching native file picker (`rfd`) is platform-
# gated below — its backend differs per OS (xdg-portal on Linux, Win32 on Windows).
src/app/mod.rs
+179 -2
View File
@@ -2,6 +2,10 @@ use crate::core::{CoreController, messages::{CoreCommand, UiEvent}};
use crate::network::PeerState;
use crate::notify::{self, Sound};
use crate::audio::eq::{EqSettings, EQ_GAIN_DB_MAX, EQ_GAIN_DB_MIN};
use crate::audio::clip_player::{
ClipPlayer, SharedClipStatus, format_time as format_clip_time, progress as clip_progress,
seek_target, status_snapshot,
};
use crate::audio::{AudioDevice, enumerate_audio_devices};
use crate::config::{AppConfig, NetworkMode, RecordingMode, RoomLayout};
use crate::hotkeys::{format_binding, HotkeyAction, HotkeyContext, KeyBinding};
@@ -283,6 +287,13 @@ pub enum AppMessage {
AttachmentFilePicked(Option<(String, Vec<u8>)>),
/// Save (downloading first if needed) a received attachment to disk.
SaveAttachment(crate::files::AttachmentId),
/// Fetch (if needed) and start an inline audio attachment.
PlayAudio(crate::files::AttachmentId),
PauseAudio,
ResumeAudio,
SeekAudio(crate::files::AttachmentId, f32),
/// Redraw cadence while an inline clip is active.
AudioTick,
/// Send the current chat input line (Enter or the Send button).
ChatSubmit,
/// Open a clicked chat link in the system browser (A13).
@@ -390,6 +401,15 @@ pub struct AppState {
/// Attachment ids the user asked to save before the bytes arrived; when the
/// fetch completes a save dialog is opened for them.
pending_saves: std::collections::HashSet<crate::files::AttachmentId>,
/// Clip ids waiting for the existing attachment fetch path to return bytes.
pending_plays: HashSet<crate::files::AttachmentId>,
/// Filename-hinted audio whose fetched bytes or decoder validation failed;
/// these entries fall back to the normal file chip.
invalid_audio: HashSet<crate::files::AttachmentId>,
/// Independent system-default-device player for chat clips. It never enters
/// the call capture/mixer path.
clip_player: ClipPlayer,
clip_status: SharedClipStatus,
/// Last known window size, tracked so divider clamps stay valid on resize.
/// (The divider positions themselves are persisted in `config`.)
window_size: Size,
@@ -522,6 +542,7 @@ impl Default for AppState {
let selected_output = output_devices.iter().find(|d| d.name == config.output_device).cloned();
let background_image = load_background_bytes(&config);
let (clip_player, clip_status) = ClipPlayer::new();
Self {
// Pre-fill the nickname with the last one used (or "Peer" by default).
@@ -552,6 +573,10 @@ impl Default for AppState {
attachment_data: HashMap::new(),
image_handle_cache: HashMap::new(),
pending_saves: std::collections::HashSet::new(),
pending_plays: HashSet::new(),
invalid_audio: HashSet::new(),
clip_player,
clip_status,
chat_input: String::new(),
window_size: Size::new(ww, wh),
layout_picker_open: false,
@@ -676,10 +701,15 @@ fn initial_window_position(
}
}
fn subscription(_state: &AppState) -> Subscription<AppMessage> {
fn subscription(state: &AppState) -> Subscription<AppMessage> {
let core_sub = Subscription::run(core_subscription).map(AppMessage::UiEventReceived);
let event_sub = iced::event::listen().map(AppMessage::EventOccurred);
Subscription::batch(vec![core_sub, event_sub])
let audio_sub = if status_snapshot(&state.clip_status).playing_id.is_some() {
iced::time::every(std::time::Duration::from_millis(250)).map(|_| AppMessage::AudioTick)
} else {
Subscription::none()
};
Subscription::batch(vec![core_sub, event_sub, audio_sub])
}
fn shutdown_timeout_task() -> Task<AppMessage> {
@@ -951,6 +981,7 @@ fn update(state: &mut AppState, message: AppMessage) -> Task<AppMessage> {
notify::play(Sound::SelfJoin, state.config.custom_sound_self_join.as_deref());
}
UiEvent::RoomLeft => {
state.clip_player.stop();
state.ticket = "".to_string();
state.peers.clear();
state.audio_levels.clear();
@@ -960,6 +991,11 @@ fn update(state: &mut AppState, message: AppMessage) -> Task<AppMessage> {
state.recording_started = None;
state.chat_messages.clear();
state.chat_input.clear();
state.attachment_data.clear();
state.image_handle_cache.clear();
state.pending_saves.clear();
state.pending_plays.clear();
state.invalid_audio.clear();
state.connecting.clear();
state.ever_connected.clear();
state.status_message = "Ready to connect".to_string();
@@ -1056,13 +1092,18 @@ fn update(state: &mut AppState, message: AppMessage) -> Task<AppMessage> {
);
}
let needs_save = state.pending_saves.remove(&id);
let needs_play = state.pending_plays.remove(&id);
state.attachment_data.insert(id, AttachmentState::Ready(data));
if needs_save {
save_attachment_to_disk(state, id);
}
if needs_play {
play_ready_audio(state, id);
}
}
UiEvent::AttachmentFailed { id, error } => {
state.pending_saves.remove(&id);
state.pending_plays.remove(&id);
state.attachment_data.insert(id, AttachmentState::Failed(error.clone()));
state.status_message = format!("Attachment failed: {error}");
}
@@ -1635,6 +1676,45 @@ fn update(state: &mut AppState, message: AppMessage) -> Task<AppMessage> {
}
}
}
AppMessage::PlayAudio(id) => {
if matches!(state.attachment_data.get(&id), Some(AttachmentState::Ready(_))) {
play_ready_audio(state, id);
} else if let Some((from, att)) = find_attachment_source(state, id) {
if let Ok(eid) = from.parse::<EndpointId>() {
// Repeated clicks while the transfer is pending must not
// launch duplicate fetches.
if state.pending_plays.insert(id) {
state.status_message = format!("Loading {}…", att.name);
let _ = state
.controller
.send(CoreCommand::FetchAttachment { from: eid, attachment: att });
}
} else {
state.status_message = "Can't play: unknown sender.".to_string();
}
}
}
AppMessage::PauseAudio => state.clip_player.pause(),
AppMessage::ResumeAudio => state.clip_player.resume(),
AppMessage::SeekAudio(id, fraction) => {
let clip = status_snapshot(&state.clip_status);
if clip.playing_id == Some(id)
&& let Some(total) = clip.total
{
state.clip_player.seek(seek_target(fraction, total));
}
}
AppMessage::AudioTick => {
let clip = status_snapshot(&state.clip_status);
if let Some(failure) = clip.failure {
if failure.invalid_audio {
state.invalid_audio.insert(failure.id);
}
state.pending_plays.remove(&failure.id);
state.status_message = format!("Audio playback failed: {}", failure.error);
state.clip_player.stop();
}
}
AppMessage::OpenUrl(url) => {
// Defence in depth: only ever hand http(s) URLs to the opener. The
// link span's href came from `linkify`, which only emits http/https,
@@ -1885,6 +1965,21 @@ fn find_attachment_source(
})
}
/// Validate cached bytes and hand them to the independent clip player. A false
/// filename hint falls back to the generic file chip without reaching rodio.
fn play_ready_audio(state: &mut AppState, id: crate::files::AttachmentId) {
let Some(AttachmentState::Ready(data)) = state.attachment_data.get(&id) else {
return;
};
if crate::files::is_probably_audio(data) {
state.invalid_audio.remove(&id);
state.clip_player.play(id, data.clone());
} else {
state.invalid_audio.insert(id);
state.status_message = "This attachment is not valid supported audio.".to_string();
}
}
/// Write a ready attachment's bytes to a user-chosen location via a native save
/// dialog. The default filename comes from the (already-sanitized) descriptor.
/// No-op if the bytes aren't ready. Sync dialog: the brief block is acceptable
@@ -3833,6 +3928,7 @@ fn view(state: &AppState) -> Element<'_, AppMessage> {
.color(color_subtext),
);
} else {
let clip_status = status_snapshot(&state.clip_status);
for m in &state.chat_messages {
let name_color = if m.mine { color_green } else { color_lavender };
// Split the (already-sanitized) message into text + URL spans so
@@ -3899,6 +3995,87 @@ fn view(state: &AppState) -> Element<'_, AppMessage> {
.color(color_subtext)
.into(),
}
} else if crate::files::looks_like_audio_name(&att.name)
&& !state.invalid_audio.contains(&att.id)
{
let active = clip_status.playing_id == Some(att.id);
let loading = state.pending_plays.contains(&att.id)
&& !matches!(data, Some(AttachmentState::Ready(_)));
let position = if active {
clip_status.position
} else {
std::time::Duration::ZERO
};
let total = active.then_some(clip_status.total).flatten();
let play_button = if loading {
button(text("Loading…").size(12))
} else if active && clip_status.paused {
button(text("Play").size(12)).on_press(AppMessage::ResumeAudio)
} else if active {
button(text("Pause").size(12)).on_press(AppMessage::PauseAudio)
} else {
button(text("Play").size(12))
.on_press(AppMessage::PlayAudio(att.id))
}
.style(b_style(
color_blue,
color_lavender,
color_crust,
6.0,
))
.padding(6);
let elapsed = format_clip_time(position);
let duration = total
.map(format_clip_time)
.unwrap_or_else(|| "--:--".to_string());
column![
row![
text(format!(
"{} ({})",
att.name,
crate::files::human_size(att.size)
))
.size(12)
.color(color_text),
button(text(if matches!(data, Some(AttachmentState::Ready(_))) {
"Save"
} else {
"Download"
})
.size(12))
.on_press(AppMessage::SaveAttachment(att.id))
.style(b_style(
color_surface,
color_overlay,
color_text,
6.0,
))
.padding(6),
]
.spacing(8)
.align_y(iced::alignment::Vertical::Center),
row![
play_button,
slider(
0.0..=1.0,
if active {
clip_progress(position, total)
} else {
0.0
},
move |fraction| AppMessage::SeekAudio(att.id, fraction),
)
.step(0.001)
.width(iced::Length::Fixed(180.0)),
text(format!("{elapsed} / {duration}"))
.size(11)
.color(color_subtext),
]
.spacing(8)
.align_y(iced::alignment::Vertical::Center),
]
.spacing(4)
.into()
} else {
let ready =
matches!(data, Some(AttachmentState::Ready(_)));
src/audio/clip_player.rs
+280
View File
@@ -0,0 +1,280 @@
//! Independent playback engine for inline chat audio attachments.
//!
//! The rodio device sink stays on a dedicated OS thread and never enters iced
//! state or the call-audio pipeline. The GUI sends small commands and reads a
//! shared status snapshot at its redraw cadence.
use crate::files::AttachmentId;
use rodio::{Decoder, DeviceSinkBuilder, MixerDeviceSink, Player, Source};
use std::io::Cursor;
use std::sync::{Arc, Mutex, mpsc};
use std::time::Duration;
/// State published by the playback thread for the GUI.
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct ClipStatus {
pub playing_id: Option<AttachmentId>,
pub position: Duration,
pub total: Option<Duration>,
pub paused: bool,
/// Set when output initialization or decoding rejects the requested clip.
/// The app consumes this as a signal to fall back to the normal file chip.
pub failure: Option<ClipFailure>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ClipFailure {
pub id: AttachmentId,
pub error: String,
/// Decoder rejection means the filename hint should fall back to a file
/// chip. Output-device failures remain retryable as audio.
pub invalid_audio: bool,
}
pub type SharedClipStatus = Arc<Mutex<ClipStatus>>;
#[derive(Debug)]
enum ClipCommand {
Play(AttachmentId, Vec<u8>),
Pause,
Resume,
Seek(Duration),
Stop,
}
/// Cheap, `Send` command handle for the dedicated playback thread.
pub struct ClipPlayer {
command_tx: mpsc::Sender<ClipCommand>,
status: SharedClipStatus,
}
impl ClipPlayer {
/// Start the playback worker. The system output device is opened lazily on
/// first Play, so merely launching PeerSpeak never claims another stream.
pub fn new() -> (Self, SharedClipStatus) {
let (command_tx, command_rx) = mpsc::channel();
let status = Arc::new(Mutex::new(ClipStatus::default()));
let worker_status = Arc::clone(&status);
std::thread::Builder::new()
.name("peerspeak-clip-player".to_string())
.spawn(move || playback_worker(command_rx, worker_status))
.expect("failed to spawn clip playback thread");
(
Self {
command_tx,
status: Arc::clone(&status),
},
status,
)
}
pub fn play(&self, id: AttachmentId, bytes: Vec<u8>) {
update_status(&self.status, |status| {
status.playing_id = Some(id);
status.position = Duration::ZERO;
status.total = None;
status.paused = false;
status.failure = None;
});
let _ = self.command_tx.send(ClipCommand::Play(id, bytes));
}
pub fn pause(&self) {
let _ = self.command_tx.send(ClipCommand::Pause);
}
pub fn resume(&self) {
let _ = self.command_tx.send(ClipCommand::Resume);
}
pub fn seek(&self, position: Duration) {
let _ = self.command_tx.send(ClipCommand::Seek(position));
}
pub fn stop(&self) {
let _ = self.command_tx.send(ClipCommand::Stop);
}
}
fn playback_worker(command_rx: mpsc::Receiver<ClipCommand>, status: SharedClipStatus) {
let mut output: Option<MixerDeviceSink> = None;
let mut player: Option<Player> = None;
loop {
match command_rx.recv_timeout(Duration::from_millis(100)) {
Ok(ClipCommand::Play(id, bytes)) => {
let source = match Decoder::new(Cursor::new(bytes)) {
Ok(source) => source,
Err(error) => {
fail(
&status,
id,
format!("unsupported or invalid audio: {error}"),
true,
);
continue;
}
};
let total = source.total_duration();
if output.is_none() {
match DeviceSinkBuilder::open_default_sink() {
Ok(sink) => {
player = Some(Player::connect_new(sink.mixer()));
output = Some(sink);
}
Err(error) => {
fail(
&status,
id,
format!("audio output unavailable: {error}"),
false,
);
continue;
}
}
}
if let Some(player) = player.as_ref() {
player.clear();
player.append(source);
player.play();
update_status(&status, |s| {
s.playing_id = Some(id);
s.position = Duration::ZERO;
s.total = total;
s.paused = false;
s.failure = None;
});
}
}
Ok(ClipCommand::Pause) => {
if let Some(player) = player.as_ref() {
player.pause();
update_status(&status, |s| s.paused = true);
}
}
Ok(ClipCommand::Resume) => {
if let Some(player) = player.as_ref() {
player.play();
update_status(&status, |s| s.paused = false);
}
}
Ok(ClipCommand::Seek(position)) => {
if let Some(player) = player.as_ref()
&& player.try_seek(position).is_ok()
{
update_status(&status, |s| s.position = position);
}
}
Ok(ClipCommand::Stop) => {
if let Some(player) = player.as_ref() {
player.clear();
}
reset(&status);
}
Err(mpsc::RecvTimeoutError::Disconnected) => break,
Err(mpsc::RecvTimeoutError::Timeout) => {}
}
if let Some(player) = player.as_ref() {
let (active, failed) = status
.lock()
.map(|s| (s.playing_id.is_some(), s.failure.is_some()))
.unwrap_or_default();
if active && !failed && player.empty() {
reset(&status);
} else if active && !failed {
update_status(&status, |s| {
s.position = player.get_pos();
s.paused = player.is_paused();
});
}
}
}
}
fn fail(status: &SharedClipStatus, id: AttachmentId, error: String, invalid_audio: bool) {
crate::log_msg(&format!("Inline audio playback failed: {error}"));
update_status(status, |s| {
// Keep the id active until the GUI observes the failure on its next
// tick. This guarantees the active-only timer cannot disappear in the
// small window between sending Play and decoder/output failure.
s.playing_id = Some(id);
s.position = Duration::ZERO;
s.total = None;
s.paused = false;
s.failure = Some(ClipFailure {
id,
error,
invalid_audio,
});
});
}
fn reset(status: &SharedClipStatus) {
update_status(status, |s| *s = ClipStatus::default());
}
fn update_status(status: &SharedClipStatus, update: impl FnOnce(&mut ClipStatus)) {
if let Ok(mut status) = status.lock() {
update(&mut status);
}
}
pub fn status_snapshot(status: &SharedClipStatus) -> ClipStatus {
status.lock().map(|s| s.clone()).unwrap_or_default()
}
/// Format clip time as `mm:ss` (hours are folded into minutes).
pub fn format_time(duration: Duration) -> String {
let seconds = duration.as_secs();
format!("{}:{:02}", seconds / 60, seconds % 60)
}
/// Playback progress in `0.0..=1.0`; unknown and zero durations report zero.
pub fn progress(position: Duration, total: Option<Duration>) -> f32 {
let Some(total) = total.filter(|duration| !duration.is_zero()) else {
return 0.0;
};
(position.as_secs_f64() / total.as_secs_f64()).clamp(0.0, 1.0) as f32
}
/// Convert a slider fraction into a clamped position within a clip.
pub fn seek_target(fraction: f32, total: Duration) -> Duration {
total.mul_f64(f64::from(fraction.clamp(0.0, 1.0)))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn formats_clip_time() {
assert_eq!(format_time(Duration::ZERO), "0:00");
assert_eq!(format_time(Duration::from_secs(65)), "1:05");
assert_eq!(format_time(Duration::from_secs(3_661)), "61:01");
}
#[test]
fn progress_handles_unknown_zero_and_clamps() {
assert_eq!(progress(Duration::from_secs(1), None), 0.0);
assert_eq!(progress(Duration::from_secs(1), Some(Duration::ZERO)), 0.0);
assert_eq!(
progress(Duration::from_secs(5), Some(Duration::from_secs(10))),
0.5
);
assert_eq!(
progress(Duration::from_secs(20), Some(Duration::from_secs(10))),
1.0
);
}
#[test]
fn seek_target_clamps_fraction() {
let total = Duration::from_secs(100);
assert_eq!(seek_target(0.25, total), Duration::from_secs(25));
assert_eq!(seek_target(-1.0, total), Duration::ZERO);
assert_eq!(seek_target(2.0, total), total);
}
}
src/audio/mod.rs
+1
View File
@@ -56,6 +56,7 @@ pub trait AudioBackend: Send + Sync {
fn stop(&self) -> Result<(), AudioError>;
}
pub mod clip_player;
pub mod eq;
pub mod gate;
pub mod limiter;
src/files.rs
+73
View File
@@ -125,6 +125,29 @@ pub fn is_probably_image(bytes: &[u8]) -> bool {
png || jpeg || gif || bmp || webp
}
/// Sniff the leading bytes for an audio container supported by the inline clip
/// player. Audio remains [`AttachmentKind::File`] on the wire; this receiver-side
/// check confirms that a filename-based player hint actually contains WAV, MP3,
/// Ogg Vorbis, or FLAC data before playback is attempted.
pub fn is_probably_audio(bytes: &[u8]) -> bool {
let flac = bytes.starts_with(b"fLaC");
let ogg = bytes.starts_with(b"OggS");
let wav = bytes.len() >= 12 && bytes.starts_with(b"RIFF") && &bytes[8..12] == b"WAVE";
let mp3_id3 = bytes.starts_with(b"ID3");
let mp3_frame = bytes.len() >= 2 && bytes[0] == 0xFF && bytes[1] & 0xE0 == 0xE0;
flac || ogg || wav || mp3_id3 || mp3_frame
}
/// Whether a sanitized attachment name has an extension supported by the
/// inline audio player. This is only a pre-fetch presentation hint; fetched
/// bytes are confirmed with [`is_probably_audio`] before being decoded.
pub fn looks_like_audio_name(name: &str) -> bool {
let Some((_, extension)) = name.rsplit_once('.') else {
return false;
};
matches!(extension.to_ascii_lowercase().as_str(), "wav" | "mp3" | "ogg" | "oga" | "flac")
}
/// The attachment kind for some file bytes: [`AttachmentKind::Image`] if it
/// sniffs as an image container, else [`AttachmentKind::File`].
pub fn classify(bytes: &[u8]) -> AttachmentKind {
@@ -242,9 +265,59 @@ mod tests {
assert!(!is_probably_image(b""));
}
#[test]
fn audio_sniffing_recognizes_supported_containers() {
assert!(is_probably_audio(b"fLaC\0\0\0\x22"));
assert!(is_probably_audio(b"OggS\0\x02"));
let mut wav = b"RIFF".to_vec();
wav.extend_from_slice(&[0, 0, 0, 0]);
wav.extend_from_slice(b"WAVE");
assert!(is_probably_audio(&wav));
assert!(is_probably_audio(b"ID3\x04\0\0"));
assert!(is_probably_audio(&[0xFF, 0xFB, 0x90, 0x64]));
}
#[test]
fn audio_sniffing_disambiguates_wav_from_webp() {
let mut wav = b"RIFF".to_vec();
wav.extend_from_slice(&[0, 0, 0, 0]);
wav.extend_from_slice(b"WAVE");
assert!(is_probably_audio(&wav));
assert!(!is_probably_image(&wav));
let mut webp = b"RIFF".to_vec();
webp.extend_from_slice(&[0, 0, 0, 0]);
webp.extend_from_slice(b"WEBP");
assert!(is_probably_image(&webp));
assert!(!is_probably_audio(&webp));
}
#[test]
fn audio_sniffing_rejects_non_audio() {
assert!(!is_probably_audio(b"%PDF-1.7"));
assert!(!is_probably_audio(&[0x89, b'P', b'N', b'G']));
assert!(!is_probably_audio(&[]));
assert!(!is_probably_audio(&[0xFF]));
}
#[test]
fn audio_name_detection_is_case_insensitive() {
for name in ["clip.wav", "clip.mp3", "clip.ogg", "clip.oga", "clip.flac"] {
assert!(looks_like_audio_name(name), "{name}");
}
assert!(looks_like_audio_name("VOICE.MP3"));
assert!(looks_like_audio_name("mix.FlAc"));
assert!(!looks_like_audio_name("recording"));
assert!(!looks_like_audio_name("notes.pdf"));
assert!(!looks_like_audio_name("photo.webp"));
}
#[test]
fn classify_maps_sniff_to_kind() {
assert_eq!(classify(&[0xFF, 0xD8, 0xFF]), AttachmentKind::Image);
assert_eq!(classify(b"fLaC\0\0\0\x22"), AttachmentKind::File);
assert_eq!(classify(b"plain text"), AttachmentKind::File);
}