fix(host): contain capture owners and fail closed
Bound the libpipewire router shutdown without detaching its OS handle, contain GStreamer and pactl children in parent-bound process groups, and make ownership failures terminal through the capture supervisor.\n\nAdd focused lifecycle tests plus a serialized live router teardown gate.
This commit is contained in:
@@ -0,0 +1,188 @@
|
|||||||
|
//! Linux child-process containment for capture-side helpers.
|
||||||
|
//!
|
||||||
|
//! PixelPass owns `gst-launch-1.0` and the short-lived `pactl load-module`
|
||||||
|
//! workers. They must not outlive a host that is killed or fail-stops: GStreamer
|
||||||
|
//! can retain a portal/PipeWire capture resource, and a late pactl mutation can
|
||||||
|
//! race teardown. Each child therefore gets both:
|
||||||
|
//!
|
||||||
|
//! - `PR_SET_PDEATHSIG(SIGKILL)`, with the standard parent-race check; and
|
||||||
|
//! - its own process group, so explicit teardown reaches descendants as well as
|
||||||
|
//! the direct child.
|
||||||
|
//!
|
||||||
|
//! This is intentionally the inverse of [`super::process`], whose viewer
|
||||||
|
//! players are user-facing detached processes and are meant to survive their
|
||||||
|
//! launcher.
|
||||||
|
|
||||||
|
use nix::libc;
|
||||||
|
use nix::sys::signal::{Signal, killpg};
|
||||||
|
use nix::unistd::Pid;
|
||||||
|
use std::io;
|
||||||
|
use std::os::unix::process::CommandExt;
|
||||||
|
use std::process::{Child, Command};
|
||||||
|
|
||||||
|
/// Install parent-death and process-group containment on `command`.
|
||||||
|
///
|
||||||
|
/// The closure runs between `fork` and `exec`, so it contains only direct
|
||||||
|
/// async-signal-safe syscalls. A failure aborts the spawn rather than launching
|
||||||
|
/// an uncontained graph-mutating child.
|
||||||
|
fn configure(command: &mut Command) {
|
||||||
|
let expected_parent = std::process::id() as libc::pid_t;
|
||||||
|
// SAFETY: `setpgid`, `prctl`, `getppid`, and `_exit` are direct syscalls and
|
||||||
|
// are async-signal-safe in the post-fork child. No allocation, logging, or
|
||||||
|
// locking occurs in the closure.
|
||||||
|
unsafe {
|
||||||
|
command.pre_exec(move || {
|
||||||
|
if libc::setpgid(0, 0) == -1 {
|
||||||
|
return Err(io::Error::last_os_error());
|
||||||
|
}
|
||||||
|
if libc::prctl(libc::PR_SET_PDEATHSIG, libc::SIGKILL) == -1 {
|
||||||
|
return Err(io::Error::last_os_error());
|
||||||
|
}
|
||||||
|
|
||||||
|
// The parent may have died after fork but before PR_SET_PDEATHSIG
|
||||||
|
// was installed. Checking after the prctl closes that window: a
|
||||||
|
// later death delivers SIGKILL, an earlier one is handled here.
|
||||||
|
if libc::getppid() != expected_parent {
|
||||||
|
libc::_exit(127);
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Spawn a contained blocking child.
|
||||||
|
pub fn spawn(command: &mut Command) -> io::Result<Child> {
|
||||||
|
configure(command);
|
||||||
|
command.spawn()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Spawn a contained Tokio child.
|
||||||
|
pub fn spawn_tokio(command: &mut tokio::process::Command) -> io::Result<tokio::process::Child> {
|
||||||
|
configure(command.as_std_mut());
|
||||||
|
command.spawn()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Signal the process group created by [`configure`].
|
||||||
|
pub fn signal_group(leader_pid: u32, signal: Signal) -> nix::Result<()> {
|
||||||
|
killpg(Pid::from_raw(leader_pid as i32), signal)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
use std::io::Write;
|
||||||
|
use std::process::Stdio;
|
||||||
|
use std::time::{Duration, Instant};
|
||||||
|
|
||||||
|
const PDEATH_HELPER: &str = "PIXELPASS_TEST_PDEATH_HELPER";
|
||||||
|
|
||||||
|
fn process_is_running(pid: u32) -> bool {
|
||||||
|
let Ok(stat) = std::fs::read_to_string(format!("/proc/{pid}/stat")) else {
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
// comm is parenthesized and may contain spaces; the state byte follows
|
||||||
|
// the final `) `. A zombie holds no resources and only awaits init's
|
||||||
|
// reap, so it is not a surviving capture child for this gate.
|
||||||
|
stat.rsplit_once(") ")
|
||||||
|
.and_then(|(_, rest)| rest.as_bytes().first().copied())
|
||||||
|
.is_some_and(|state| state != b'Z' && state != b'X')
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn contained_child_has_its_own_process_group() {
|
||||||
|
let mut command = Command::new("sleep");
|
||||||
|
command.arg("30");
|
||||||
|
let mut child = spawn(&mut command).expect("spawn contained child");
|
||||||
|
let pid = child.id();
|
||||||
|
|
||||||
|
// SAFETY: getpgid is a read-only syscall for the live child we own.
|
||||||
|
let pgid = unsafe { libc::getpgid(pid as libc::pid_t) };
|
||||||
|
assert_eq!(pgid, pid as libc::pid_t);
|
||||||
|
|
||||||
|
signal_group(pid, Signal::SIGKILL).expect("kill contained group");
|
||||||
|
child.wait().expect("reap contained child");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn process_group_signal_reaches_descendants() {
|
||||||
|
let mut command = Command::new("sh");
|
||||||
|
command
|
||||||
|
.args(["-c", "sleep 30 & child=$!; echo $child; wait"])
|
||||||
|
.stdout(Stdio::piped());
|
||||||
|
let mut leader = spawn(&mut command).expect("spawn group leader");
|
||||||
|
let mut line = String::new();
|
||||||
|
use std::io::BufRead;
|
||||||
|
std::io::BufReader::new(leader.stdout.take().expect("piped stdout"))
|
||||||
|
.read_line(&mut line)
|
||||||
|
.expect("read descendant pid");
|
||||||
|
let descendant: u32 = line.trim().parse().expect("numeric descendant pid");
|
||||||
|
assert!(process_is_running(descendant));
|
||||||
|
|
||||||
|
signal_group(leader.id(), Signal::SIGKILL).expect("kill whole group");
|
||||||
|
leader.wait().expect("reap group leader");
|
||||||
|
let deadline = Instant::now() + Duration::from_secs(2);
|
||||||
|
while process_is_running(descendant) && Instant::now() < deadline {
|
||||||
|
std::thread::sleep(Duration::from_millis(10));
|
||||||
|
}
|
||||||
|
assert!(
|
||||||
|
!process_is_running(descendant),
|
||||||
|
"a descendant must not survive explicit group teardown"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Subprocess-only half of the parent-death gate. The outer test launches
|
||||||
|
/// this exact test in a disposable harness process; when that process exits,
|
||||||
|
/// the contained sleep must receive SIGKILL.
|
||||||
|
#[test]
|
||||||
|
fn pdeathsig_helper() {
|
||||||
|
if std::env::var_os(PDEATH_HELPER).is_none() {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let mut command = Command::new("sleep");
|
||||||
|
command
|
||||||
|
.arg("30")
|
||||||
|
.stdout(Stdio::null())
|
||||||
|
.stderr(Stdio::null());
|
||||||
|
let child = spawn(&mut command).expect("spawn pdeath child");
|
||||||
|
println!("PIXELPASS_CONTAINED_PID={}", child.id());
|
||||||
|
std::io::stdout().flush().expect("flush child pid");
|
||||||
|
// std::process::Child has no kill-on-drop behavior. Returning lets the
|
||||||
|
// helper harness exit while the contained process is still live.
|
||||||
|
drop(child);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn parent_death_kills_the_contained_child() {
|
||||||
|
let helper = std::env::current_exe().expect("test executable path");
|
||||||
|
let output = Command::new(helper)
|
||||||
|
.args([
|
||||||
|
"--exact",
|
||||||
|
"common::contained::tests::pdeathsig_helper",
|
||||||
|
"--nocapture",
|
||||||
|
])
|
||||||
|
.env(PDEATH_HELPER, "1")
|
||||||
|
.output()
|
||||||
|
.expect("run pdeath helper harness");
|
||||||
|
assert!(
|
||||||
|
output.status.success(),
|
||||||
|
"helper failed: {}",
|
||||||
|
String::from_utf8_lossy(&output.stderr)
|
||||||
|
);
|
||||||
|
let stdout = String::from_utf8_lossy(&output.stdout);
|
||||||
|
let pid: u32 = stdout
|
||||||
|
.lines()
|
||||||
|
.find_map(|line| line.strip_prefix("PIXELPASS_CONTAINED_PID="))
|
||||||
|
.expect("helper printed contained pid")
|
||||||
|
.parse()
|
||||||
|
.expect("numeric contained pid");
|
||||||
|
|
||||||
|
let deadline = Instant::now() + Duration::from_secs(2);
|
||||||
|
while process_is_running(pid) && Instant::now() < deadline {
|
||||||
|
std::thread::sleep(Duration::from_millis(10));
|
||||||
|
}
|
||||||
|
assert!(
|
||||||
|
!process_is_running(pid),
|
||||||
|
"PR_SET_PDEATHSIG must stop the child when its PixelPass parent exits"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,6 +1,7 @@
|
|||||||
pub mod alpn;
|
pub mod alpn;
|
||||||
pub mod bandwidth;
|
pub mod bandwidth;
|
||||||
pub mod config;
|
pub mod config;
|
||||||
|
pub mod contained;
|
||||||
// The friends stack (persistent identity + control plane) is GUI-only — a
|
// The friends stack (persistent identity + control plane) is GUI-only — a
|
||||||
// headless CLI host runs no presence service — so it's gated with the feature
|
// headless CLI host runs no presence service — so it's gated with the feature
|
||||||
// that pulls the rest of the GUI, keeping the headless build lean.
|
// that pulls the rest of the GUI, keeping the headless build lean.
|
||||||
|
|||||||
+249
-35
@@ -37,11 +37,14 @@ use std::io::{self, Read};
|
|||||||
use std::process::{Child, Command, ExitStatus, Stdio};
|
use std::process::{Child, Command, ExitStatus, Stdio};
|
||||||
use std::rc::Rc;
|
use std::rc::Rc;
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
use std::thread::JoinHandle;
|
use std::sync::atomic::{AtomicBool, AtomicU8, Ordering};
|
||||||
use std::time::{Duration, Instant};
|
use std::time::{Duration, Instant};
|
||||||
|
|
||||||
use crate::cli::HostOpts;
|
use crate::cli::HostOpts;
|
||||||
|
use crate::common::contained;
|
||||||
|
use crate::host::health;
|
||||||
use crate::host::ledger::{self, LedgerError, ModuleLedger, UnloadOutcome};
|
use crate::host::ledger::{self, LedgerError, ModuleLedger, UnloadOutcome};
|
||||||
|
use crate::host::owned_thread::OwnedThread;
|
||||||
use crate::repair::plan::{self as repair_plan, Fingerprint, Shape};
|
use crate::repair::plan::{self as repair_plan, Fingerprint, Shape};
|
||||||
|
|
||||||
/// How long a `pactl load-module` worker may run before it is killed and reaped.
|
/// How long a `pactl load-module` worker may run before it is killed and reaped.
|
||||||
@@ -54,6 +57,8 @@ use crate::repair::plan::{self as repair_plan, Fingerprint, Shape};
|
|||||||
/// connect/list/unload requests instead of a second pactl connection.
|
/// connect/list/unload requests instead of a second pactl connection.
|
||||||
const PACTL_BUDGET: Duration = Duration::from_secs(5);
|
const PACTL_BUDGET: Duration = Duration::from_secs(5);
|
||||||
const PACTL_REAP_BUDGET: Duration = Duration::from_secs(1);
|
const PACTL_REAP_BUDGET: Duration = Duration::from_secs(1);
|
||||||
|
const ROUTER_RUNNING_STOP_BUDGET: Duration = Duration::from_secs(2);
|
||||||
|
const ROUTER_STARTING_STOP_BUDGET: Duration = Duration::from_secs(5);
|
||||||
|
|
||||||
/// Owns the pactl-loaded modules plus, when filtering is active, the
|
/// Owns the pactl-loaded modules plus, when filtering is active, the
|
||||||
/// libpipewire stream-router thread. Drop unloads modules as a backstop;
|
/// libpipewire stream-router thread. Drop unloads modules as a backstop;
|
||||||
@@ -71,12 +76,13 @@ pub struct Routing {
|
|||||||
sink_name: String,
|
sink_name: String,
|
||||||
stream_router: Option<StreamRouter>,
|
stream_router: Option<StreamRouter>,
|
||||||
event_task: Option<tokio::task::JoinHandle<()>>,
|
event_task: Option<tokio::task::JoinHandle<()>>,
|
||||||
|
health: health::Reporter,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Routing {
|
impl Routing {
|
||||||
/// Create the per-PID null-sink + loopback. If `opts.app` is set,
|
/// Create the per-PID null-sink + loopback. If `opts.app` is set,
|
||||||
/// also spawn the libpipewire thread that reroutes matching streams.
|
/// also spawn the libpipewire thread that reroutes matching streams.
|
||||||
pub async fn start(opts: &HostOpts) -> Result<Self> {
|
pub(super) async fn start(opts: &HostOpts, health: health::Reporter) -> Result<Self> {
|
||||||
let pid = std::process::id();
|
let pid = std::process::id();
|
||||||
let sink_name = repair_plan::sink_name_for(pid);
|
let sink_name = repair_plan::sink_name_for(pid);
|
||||||
let ledger = ModuleLedger::new();
|
let ledger = ModuleLedger::new();
|
||||||
@@ -90,6 +96,7 @@ impl Routing {
|
|||||||
sink_name: sink_name.clone(),
|
sink_name: sink_name.clone(),
|
||||||
stream_router: None,
|
stream_router: None,
|
||||||
event_task: None,
|
event_task: None,
|
||||||
|
health: health.clone(),
|
||||||
};
|
};
|
||||||
|
|
||||||
// Every module this host loads carries an ownership token, minted per
|
// Every module this host loads carries an ownership token, minted per
|
||||||
@@ -125,7 +132,8 @@ impl Routing {
|
|||||||
);
|
);
|
||||||
|
|
||||||
if let Some(app) = &opts.app {
|
if let Some(app) = &opts.app {
|
||||||
let (router, mut event_rx) = StreamRouter::spawn(app.clone(), sink_name.clone())?;
|
let (router, mut event_rx) =
|
||||||
|
StreamRouter::spawn(app.clone(), sink_name.clone(), health.clone())?;
|
||||||
let ledger_for_task = Arc::clone(&ledger);
|
let ledger_for_task = Arc::clone(&ledger);
|
||||||
let strict = opts.strict_audio;
|
let strict = opts.strict_audio;
|
||||||
let event_task = tokio::spawn(async move {
|
let event_task = tokio::spawn(async move {
|
||||||
@@ -235,27 +243,44 @@ impl Routing {
|
|||||||
// task cannot register another mutation even if it receives one last
|
// task cannot register another mutation even if it receives one last
|
||||||
// router event while shutdown is in progress.
|
// router event while shutdown is in progress.
|
||||||
self.ledger.close();
|
self.ledger.close();
|
||||||
if let Some(router) = self.stream_router.take() {
|
let router_stopped = if let Some(router) = self.stream_router.take() {
|
||||||
// ⚠️ Still an unbounded join: a wedged PipeWire thread parks this
|
router.shutdown().await
|
||||||
// task indefinitely. That is the pre-existing defect S3b exists for.
|
} else {
|
||||||
// Nothing here makes it worse, and the ledger is what will make
|
true
|
||||||
// bounding it safe when it lands.
|
};
|
||||||
router.shutdown();
|
|
||||||
}
|
|
||||||
if let Some(mut task) = self.event_task.take() {
|
if let Some(mut task) = self.event_task.take() {
|
||||||
// The router's exit drops the event senders, so the task normally
|
if !router_stopped {
|
||||||
// ends by itself. Abort is the fallback, and it is awaited through
|
// A quarantined router still owns its event sender, so this task
|
||||||
// `&mut JoinHandle` so the future is genuinely dropped — and with it
|
// cannot finish naturally. Cancel and await it before ledger
|
||||||
// any in-flight permit — before reconciliation reads the ledger.
|
// reconciliation; the router timeout already poisoned the host.
|
||||||
// Dropping the handle instead would *detach* the task, which is how a
|
|
||||||
// load could still land after teardown believed it was finished.
|
|
||||||
if tokio::time::timeout(PACTL_BUDGET, &mut task).await.is_err() {
|
|
||||||
tracing::warn!(
|
|
||||||
"audio routing: the event task did not finish within {PACTL_BUDGET:?}; \
|
|
||||||
cancelling it"
|
|
||||||
);
|
|
||||||
task.abort();
|
task.abort();
|
||||||
let _ = task.await;
|
let _ = task.await;
|
||||||
|
} else {
|
||||||
|
// The router's exit drops the event senders, so the task normally
|
||||||
|
// ends by itself. Abort is the fallback, and it is awaited through
|
||||||
|
// `&mut JoinHandle` so the future is genuinely dropped — and with
|
||||||
|
// it any in-flight permit — before reconciliation reads the
|
||||||
|
// ledger. Dropping the handle instead would *detach* the task,
|
||||||
|
// which is how a load could still land after teardown believed it
|
||||||
|
// was finished.
|
||||||
|
match tokio::time::timeout(PACTL_BUDGET, &mut task).await {
|
||||||
|
Ok(Ok(())) => {}
|
||||||
|
Ok(Err(e)) => {
|
||||||
|
self.health
|
||||||
|
.poison(format!("audio routing event task failed: {e}"));
|
||||||
|
}
|
||||||
|
Err(_) => {
|
||||||
|
tracing::warn!(
|
||||||
|
"audio routing: the event task did not finish within {PACTL_BUDGET:?}; \
|
||||||
|
cancelling it"
|
||||||
|
);
|
||||||
|
self.health.poison(format!(
|
||||||
|
"audio routing event task did not stop within {PACTL_BUDGET:?}"
|
||||||
|
));
|
||||||
|
task.abort();
|
||||||
|
let _ = task.await;
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -269,6 +294,8 @@ impl Routing {
|
|||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
tracing::warn!("audio routing: module-operation wait task failed: {e}");
|
tracing::warn!("audio routing: module-operation wait task failed: {e}");
|
||||||
|
self.health
|
||||||
|
.poison(format!("audio module-operation wait task failed: {e}"));
|
||||||
}
|
}
|
||||||
cleanup_modules(&self.ledger).await;
|
cleanup_modules(&self.ledger).await;
|
||||||
|
|
||||||
@@ -278,6 +305,9 @@ impl Routing {
|
|||||||
"audio routing: some audio modules could not be removed safely; \
|
"audio routing: some audio modules could not be removed safely; \
|
||||||
`pixelpass --repair` will clean up anything left behind"
|
`pixelpass --repair` will clean up anything left behind"
|
||||||
);
|
);
|
||||||
|
self.health.poison(
|
||||||
|
"audio routing teardown left module ownership unresolved; repair is required",
|
||||||
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -292,7 +322,7 @@ impl Drop for Routing {
|
|||||||
fn drop(&mut self) {
|
fn drop(&mut self) {
|
||||||
self.ledger.close();
|
self.ledger.close();
|
||||||
if let Some(router) = self.stream_router.take() {
|
if let Some(router) = self.stream_router.take() {
|
||||||
router.shutdown();
|
drop(router);
|
||||||
}
|
}
|
||||||
if let Some(task) = self.event_task.take() {
|
if let Some(task) = self.event_task.take() {
|
||||||
task.abort();
|
task.abort();
|
||||||
@@ -305,6 +335,8 @@ impl Drop for Routing {
|
|||||||
"audio routing: torn down with modules that could not be removed safely; \
|
"audio routing: torn down with modules that could not be removed safely; \
|
||||||
run `pixelpass --repair` to clean up anything left behind"
|
run `pixelpass --repair` to clean up anything left behind"
|
||||||
);
|
);
|
||||||
|
self.health
|
||||||
|
.poison("audio routing Drop left module ownership unresolved; repair is required");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -635,7 +667,7 @@ impl Drop for ReapedChild {
|
|||||||
if self.reaped {
|
if self.reaped {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
let _ = self.child.kill();
|
self.kill_group();
|
||||||
match self.reap_within(PACTL_REAP_BUDGET) {
|
match self.reap_within(PACTL_REAP_BUDGET) {
|
||||||
Ok(Some(_)) => {}
|
Ok(Some(_)) => {}
|
||||||
Ok(None) => tracing::warn!(
|
Ok(None) => tracing::warn!(
|
||||||
@@ -647,6 +679,12 @@ impl Drop for ReapedChild {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl ReapedChild {
|
impl ReapedChild {
|
||||||
|
fn kill_group(&mut self) {
|
||||||
|
let _ = contained::signal_group(self.child.id(), nix::sys::signal::Signal::SIGKILL);
|
||||||
|
// Backstop in case the group disappeared between lookup and signal.
|
||||||
|
let _ = self.child.kill();
|
||||||
|
}
|
||||||
|
|
||||||
fn reap_within(&mut self, budget: Duration) -> io::Result<Option<ExitStatus>> {
|
fn reap_within(&mut self, budget: Duration) -> io::Result<Option<ExitStatus>> {
|
||||||
let deadline = Instant::now() + budget;
|
let deadline = Instant::now() + budget;
|
||||||
loop {
|
loop {
|
||||||
@@ -665,7 +703,7 @@ impl ReapedChild {
|
|||||||
fn bounded_output(command: &mut Command, budget: Duration) -> io::Result<BoundedOutput> {
|
fn bounded_output(command: &mut Command, budget: Duration) -> io::Result<BoundedOutput> {
|
||||||
command.stdout(Stdio::piped()).stderr(Stdio::piped());
|
command.stdout(Stdio::piped()).stderr(Stdio::piped());
|
||||||
let mut child = ReapedChild {
|
let mut child = ReapedChild {
|
||||||
child: command.spawn()?,
|
child: contained::spawn(command)?,
|
||||||
reaped: false,
|
reaped: false,
|
||||||
};
|
};
|
||||||
let stdout = child
|
let stdout = child
|
||||||
@@ -700,7 +738,7 @@ fn bounded_output(command: &mut Command, budget: Duration) -> io::Result<Bounded
|
|||||||
break (status, false);
|
break (status, false);
|
||||||
}
|
}
|
||||||
if Instant::now() >= deadline {
|
if Instant::now() >= deadline {
|
||||||
let _ = child.child.kill();
|
child.kill_group();
|
||||||
let Some(status) = child.reap_within(PACTL_REAP_BUDGET)? else {
|
let Some(status) = child.reap_within(PACTL_REAP_BUDGET)? else {
|
||||||
return Err(io::Error::new(
|
return Err(io::Error::new(
|
||||||
io::ErrorKind::TimedOut,
|
io::ErrorKind::TimedOut,
|
||||||
@@ -807,24 +845,53 @@ enum Event {
|
|||||||
/// Handle to the libpipewire stream-router thread.
|
/// Handle to the libpipewire stream-router thread.
|
||||||
pub struct StreamRouter {
|
pub struct StreamRouter {
|
||||||
cmd_tx: pipewire::channel::Sender<Cmd>,
|
cmd_tx: pipewire::channel::Sender<Cmd>,
|
||||||
thread: Option<JoinHandle<()>>,
|
thread: OwnedThread,
|
||||||
|
phase: Arc<AtomicU8>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const ROUTER_STARTING: u8 = 0;
|
||||||
|
const ROUTER_RUNNING: u8 = 1;
|
||||||
|
const ROUTER_EXITED: u8 = 2;
|
||||||
|
|
||||||
impl StreamRouter {
|
impl StreamRouter {
|
||||||
/// Spawn the libpipewire thread. Returns the router handle and the
|
/// Spawn the libpipewire thread. Returns the router handle and the
|
||||||
/// event receiver tokio side polls.
|
/// event receiver tokio side polls.
|
||||||
fn spawn(
|
fn spawn(
|
||||||
filter_name: String,
|
filter_name: String,
|
||||||
sink_name: String,
|
sink_name: String,
|
||||||
|
health: health::Reporter,
|
||||||
) -> Result<(Self, tokio::sync::mpsc::UnboundedReceiver<Event>)> {
|
) -> Result<(Self, tokio::sync::mpsc::UnboundedReceiver<Event>)> {
|
||||||
let (cmd_tx, cmd_rx) = pipewire::channel::channel::<Cmd>();
|
let (cmd_tx, cmd_rx) = pipewire::channel::channel::<Cmd>();
|
||||||
let (event_tx, event_rx) = tokio::sync::mpsc::unbounded_channel::<Event>();
|
let (event_tx, event_rx) = tokio::sync::mpsc::unbounded_channel::<Event>();
|
||||||
|
let phase = Arc::new(AtomicU8::new(ROUTER_STARTING));
|
||||||
|
let phase_for_thread = Arc::clone(&phase);
|
||||||
|
let shutdown_observed = Arc::new(AtomicBool::new(false));
|
||||||
|
let shutdown_for_thread = Arc::clone(&shutdown_observed);
|
||||||
|
let health_for_thread = health.clone();
|
||||||
|
|
||||||
let thread = std::thread::Builder::new()
|
let thread = std::thread::Builder::new()
|
||||||
.name("pixelpass-pw-router".to_string())
|
.name("pixelpass-pw-router".to_string())
|
||||||
.spawn(move || {
|
.spawn(move || {
|
||||||
if let Err(e) = run_router(filter_name, sink_name, cmd_rx, event_tx) {
|
let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
|
||||||
tracing::warn!("audio routing: libpipewire thread exited with error: {e:#}");
|
run_router(
|
||||||
|
filter_name,
|
||||||
|
sink_name,
|
||||||
|
cmd_rx,
|
||||||
|
event_tx,
|
||||||
|
Arc::clone(&phase_for_thread),
|
||||||
|
Arc::clone(&shutdown_for_thread),
|
||||||
|
)
|
||||||
|
}));
|
||||||
|
phase_for_thread.store(ROUTER_EXITED, Ordering::Release);
|
||||||
|
match result {
|
||||||
|
Ok(result) => report_router_exit(
|
||||||
|
&health_for_thread,
|
||||||
|
shutdown_for_thread.load(Ordering::Acquire),
|
||||||
|
result,
|
||||||
|
),
|
||||||
|
Err(_) => {
|
||||||
|
health_for_thread.poison("libpipewire router thread panicked");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
.context("failed to spawn libpipewire router thread")?;
|
.context("failed to spawn libpipewire router thread")?;
|
||||||
@@ -832,19 +899,46 @@ impl StreamRouter {
|
|||||||
Ok((
|
Ok((
|
||||||
Self {
|
Self {
|
||||||
cmd_tx,
|
cmd_tx,
|
||||||
thread: Some(thread),
|
thread: OwnedThread::new("libpipewire router thread", thread, health),
|
||||||
|
phase,
|
||||||
},
|
},
|
||||||
event_rx,
|
event_rx,
|
||||||
))
|
))
|
||||||
}
|
}
|
||||||
|
|
||||||
fn shutdown(mut self) {
|
async fn shutdown(mut self) -> bool {
|
||||||
// Best-effort: if the send fails the thread is already gone.
|
// Best-effort: if the send fails the thread is already gone.
|
||||||
let _ = self.cmd_tx.send(Cmd::Shutdown);
|
let _ = self.cmd_tx.send(Cmd::Shutdown);
|
||||||
if let Some(t) = self.thread.take()
|
let budget = router_shutdown_budget(self.phase.load(Ordering::Acquire));
|
||||||
&& let Err(e) = t.join()
|
self.thread.join_within(budget).await
|
||||||
{
|
}
|
||||||
tracing::warn!("audio routing: pw thread join failed: {e:?}");
|
}
|
||||||
|
|
||||||
|
impl Drop for StreamRouter {
|
||||||
|
fn drop(&mut self) {
|
||||||
|
// If async shutdown is cancelled, wake the MainLoop before OwnedThread's
|
||||||
|
// Drop poisons/quarantines the still-owned handle.
|
||||||
|
let _ = self.cmd_tx.send(Cmd::Shutdown);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn router_shutdown_budget(phase: u8) -> Duration {
|
||||||
|
if phase == ROUTER_STARTING {
|
||||||
|
ROUTER_STARTING_STOP_BUDGET
|
||||||
|
} else {
|
||||||
|
ROUTER_RUNNING_STOP_BUDGET
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn report_router_exit(health: &health::Reporter, shutdown_observed: bool, result: Result<()>) {
|
||||||
|
match result {
|
||||||
|
Ok(()) if shutdown_observed => {}
|
||||||
|
Ok(()) => {
|
||||||
|
health.poison("libpipewire router thread exited without a shutdown command");
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
tracing::warn!("audio routing: libpipewire thread exited with error: {e:#}");
|
||||||
|
health.poison(format!("libpipewire router thread failed: {e:#}"));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -856,6 +950,8 @@ fn run_router(
|
|||||||
sink_name: String,
|
sink_name: String,
|
||||||
cmd_rx: pipewire::channel::Receiver<Cmd>,
|
cmd_rx: pipewire::channel::Receiver<Cmd>,
|
||||||
event_tx: tokio::sync::mpsc::UnboundedSender<Event>,
|
event_tx: tokio::sync::mpsc::UnboundedSender<Event>,
|
||||||
|
phase: Arc<AtomicU8>,
|
||||||
|
shutdown_observed: Arc<AtomicBool>,
|
||||||
) -> Result<()> {
|
) -> Result<()> {
|
||||||
use pipewire::{self as pw, types::ObjectType};
|
use pipewire::{self as pw, types::ObjectType};
|
||||||
|
|
||||||
@@ -878,8 +974,10 @@ fn run_router(
|
|||||||
// Cmd handler: clear metadata for routed streams, then quit.
|
// Cmd handler: clear metadata for routed streams, then quit.
|
||||||
let main_loop_for_cmd = main_loop.clone();
|
let main_loop_for_cmd = main_loop.clone();
|
||||||
let state_for_cmd = Rc::clone(&state);
|
let state_for_cmd = Rc::clone(&state);
|
||||||
|
let shutdown_for_cmd = Arc::clone(&shutdown_observed);
|
||||||
let _cmd_recv = cmd_rx.attach(main_loop.loop_(), move |cmd| match cmd {
|
let _cmd_recv = cmd_rx.attach(main_loop.loop_(), move |cmd| match cmd {
|
||||||
Cmd::Shutdown => {
|
Cmd::Shutdown => {
|
||||||
|
shutdown_for_cmd.store(true, Ordering::Release);
|
||||||
let s = state_for_cmd.borrow();
|
let s = state_for_cmd.borrow();
|
||||||
if let Some(meta) = &s.default_metadata {
|
if let Some(meta) = &s.default_metadata {
|
||||||
for &nid in &s.routed_node_ids {
|
for &nid in &s.routed_node_ids {
|
||||||
@@ -978,6 +1076,7 @@ fn run_router(
|
|||||||
.register();
|
.register();
|
||||||
|
|
||||||
tracing::info!(filter = %filter_name, "audio routing: pw thread running");
|
tracing::info!(filter = %filter_name, "audio routing: pw thread running");
|
||||||
|
phase.store(ROUTER_RUNNING, Ordering::Release);
|
||||||
main_loop.run();
|
main_loop.run();
|
||||||
tracing::info!("audio routing: pw thread exiting");
|
tracing::info!("audio routing: pw thread exiting");
|
||||||
Ok(())
|
Ok(())
|
||||||
@@ -1074,6 +1173,7 @@ mod tests {
|
|||||||
use super::*;
|
use super::*;
|
||||||
use crate::host::ledger::SlotState;
|
use crate::host::ledger::SlotState;
|
||||||
use crate::repair::plan::{ModuleObservation, classify};
|
use crate::repair::plan::{ModuleObservation, classify};
|
||||||
|
use std::sync::mpsc;
|
||||||
|
|
||||||
/// Whole-desktop routing: no app filter, so no PipeWire thread and no event
|
/// Whole-desktop routing: no app filter, so no PipeWire thread and no event
|
||||||
/// task — just the null-sink and its default-sink loopback.
|
/// task — just the null-sink and its default-sink loopback.
|
||||||
@@ -1094,6 +1194,86 @@ mod tests {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn router_exit_is_healthy_only_after_the_thread_observed_shutdown() {
|
||||||
|
let (clean, _) = health::channel();
|
||||||
|
report_router_exit(&clean, true, Ok(()));
|
||||||
|
assert!(clean.fault().is_none());
|
||||||
|
|
||||||
|
let (unexpected, _) = health::channel();
|
||||||
|
report_router_exit(&unexpected, false, Ok(()));
|
||||||
|
assert_eq!(
|
||||||
|
unexpected.fault().as_deref(),
|
||||||
|
Some("libpipewire router thread exited without a shutdown command")
|
||||||
|
);
|
||||||
|
|
||||||
|
let (failed, _) = health::channel();
|
||||||
|
report_router_exit(&failed, false, Err(anyhow::anyhow!("fixture failure")));
|
||||||
|
assert!(
|
||||||
|
failed
|
||||||
|
.fault()
|
||||||
|
.as_deref()
|
||||||
|
.is_some_and(|reason| reason.contains("fixture failure"))
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn router_shutdown_has_distinct_startup_and_running_budgets() {
|
||||||
|
assert_eq!(
|
||||||
|
router_shutdown_budget(ROUTER_STARTING),
|
||||||
|
ROUTER_STARTING_STOP_BUDGET
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
router_shutdown_budget(ROUTER_RUNNING),
|
||||||
|
ROUTER_RUNNING_STOP_BUDGET
|
||||||
|
);
|
||||||
|
assert!(ROUTER_STARTING_STOP_BUDGET > ROUTER_RUNNING_STOP_BUDGET);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn cancelling_router_shutdown_keeps_the_thread_owned_and_poisons() {
|
||||||
|
let (cmd_tx, _cmd_rx) = pipewire::channel::channel::<Cmd>();
|
||||||
|
let (release_tx, release_rx) = mpsc::channel();
|
||||||
|
let (health, _) = health::channel();
|
||||||
|
let thread = std::thread::spawn(move || {
|
||||||
|
let _ = release_rx.recv();
|
||||||
|
});
|
||||||
|
let router = StreamRouter {
|
||||||
|
cmd_tx,
|
||||||
|
thread: OwnedThread::new("cancellation fixture", thread, health.clone()),
|
||||||
|
phase: Arc::new(AtomicU8::new(ROUTER_STARTING)),
|
||||||
|
};
|
||||||
|
|
||||||
|
assert!(
|
||||||
|
tokio::time::timeout(Duration::from_millis(20), router.shutdown())
|
||||||
|
.await
|
||||||
|
.is_err(),
|
||||||
|
"the outer timeout must cancel shutdown before its policy deadline"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
health.fault().is_some(),
|
||||||
|
"cancellation must poison instead of detaching the OS handle"
|
||||||
|
);
|
||||||
|
release_tx.send(()).expect("release quarantined fixture");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn bounded_module_worker_uses_the_contained_spawn_path() {
|
||||||
|
let mut command = Command::new("sh");
|
||||||
|
command.args([
|
||||||
|
"-c",
|
||||||
|
"read pid comm state ppid pgrp rest < /proc/self/stat; printf '%s %s' \"$pid\" \"$pgrp\"",
|
||||||
|
]);
|
||||||
|
let output = bounded_output(&mut command, Duration::from_secs(1))
|
||||||
|
.expect("run contained module-worker fixture");
|
||||||
|
assert!(output.status.success());
|
||||||
|
let ids = String::from_utf8(output.stdout).expect("ascii pid/pgid");
|
||||||
|
let mut ids = ids.split_whitespace();
|
||||||
|
let pid = ids.next().expect("child pid");
|
||||||
|
let pgid = ids.next().expect("child process group");
|
||||||
|
assert_eq!(pid, pgid, "module worker must lead its own process group");
|
||||||
|
}
|
||||||
|
|
||||||
/// The module table exactly as `--repair` observes it.
|
/// The module table exactly as `--repair` observes it.
|
||||||
fn module_snapshot() -> Vec<(u32, String, String)> {
|
fn module_snapshot() -> Vec<(u32, String, String)> {
|
||||||
let mut session =
|
let mut session =
|
||||||
@@ -1148,7 +1328,8 @@ mod tests {
|
|||||||
#[ignore = "loads real Pulse modules; run with --ignored --test-threads=1"]
|
#[ignore = "loads real Pulse modules; run with --ignored --test-threads=1"]
|
||||||
async fn live_teardown_leaves_the_module_table_as_it_found_it() {
|
async fn live_teardown_leaves_the_module_table_as_it_found_it() {
|
||||||
let before = module_snapshot();
|
let before = module_snapshot();
|
||||||
let routing = Routing::start(&whole_desktop_opts())
|
let (health, _) = health::channel();
|
||||||
|
let routing = Routing::start(&whole_desktop_opts(), health)
|
||||||
.await
|
.await
|
||||||
.expect("routing starts");
|
.expect("routing starts");
|
||||||
|
|
||||||
@@ -1180,6 +1361,39 @@ mod tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Exercise the real libpipewire mainloop command path, not only the
|
||||||
|
/// whole-desktop module path above. The deliberately unmatched app filter
|
||||||
|
/// is enough to start the router without moving an unrelated live stream.
|
||||||
|
#[tokio::test]
|
||||||
|
#[ignore = "uses the real Pulse/PipeWire graph; run with --ignored --test-threads=1"]
|
||||||
|
async fn live_stream_router_stops_within_its_policy_budget() {
|
||||||
|
let before = module_snapshot();
|
||||||
|
let mut opts = whole_desktop_opts();
|
||||||
|
opts.app = Some("__pixelpass_s3b_no_matching_application__".to_string());
|
||||||
|
opts.strict_audio = true;
|
||||||
|
let (health, _) = health::channel();
|
||||||
|
let routing = Routing::start(&opts, health.clone())
|
||||||
|
.await
|
||||||
|
.expect("per-app routing starts");
|
||||||
|
|
||||||
|
// Let the OS thread reach its normal running phase so this covers the
|
||||||
|
// tighter steady-state budget rather than only startup containment.
|
||||||
|
tokio::time::sleep(Duration::from_millis(100)).await;
|
||||||
|
tokio::time::timeout(Duration::from_secs(10), routing.shutdown())
|
||||||
|
.await
|
||||||
|
.expect("router and graph teardown stay globally bounded");
|
||||||
|
|
||||||
|
assert!(
|
||||||
|
health.fault().is_none(),
|
||||||
|
"an observed shutdown and successful join must remain healthy"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
module_snapshot(),
|
||||||
|
before,
|
||||||
|
"per-app teardown must leave the module table byte-identical"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
/// The orphan race, staged against a real server: a load cancelled while
|
/// The orphan race, staged against a real server: a load cancelled while
|
||||||
/// `pactl` is in flight must still be findable and removable.
|
/// `pactl` is in flight must still be findable and removable.
|
||||||
///
|
///
|
||||||
|
|||||||
+5
-3
@@ -8,18 +8,20 @@ use anyhow::Result;
|
|||||||
|
|
||||||
use crate::cli::HostOpts;
|
use crate::cli::HostOpts;
|
||||||
use crate::common::display::DisplayServer;
|
use crate::common::display::DisplayServer;
|
||||||
|
use crate::host::health;
|
||||||
use crate::host::pipeline::CaptureHandle;
|
use crate::host::pipeline::CaptureHandle;
|
||||||
use crate::host::quality::EffectiveQuality;
|
use crate::host::quality::EffectiveQuality;
|
||||||
use crate::host::{wayland, x11};
|
use crate::host::{wayland, x11};
|
||||||
|
|
||||||
pub async fn spawn(
|
pub(super) async fn spawn(
|
||||||
display: DisplayServer,
|
display: DisplayServer,
|
||||||
opts: &HostOpts,
|
opts: &HostOpts,
|
||||||
quality: &EffectiveQuality,
|
quality: &EffectiveQuality,
|
||||||
|
health: health::Reporter,
|
||||||
) -> Result<CaptureHandle> {
|
) -> Result<CaptureHandle> {
|
||||||
match display {
|
match display {
|
||||||
DisplayServer::Wayland => wayland::start(opts, quality).await,
|
DisplayServer::Wayland => wayland::start(opts, quality, health).await,
|
||||||
DisplayServer::X11 => x11::start(opts, quality).await,
|
DisplayServer::X11 => x11::start(opts, quality, health).await,
|
||||||
DisplayServer::Unknown => unreachable!("caller guarantees display != Unknown"),
|
DisplayServer::Unknown => unreachable!("caller guarantees display != Unknown"),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,83 @@
|
|||||||
|
//! Terminal health shared by capture components and the host supervisor.
|
||||||
|
//!
|
||||||
|
//! A capture-side ownership failure is not recoverable inside the same host
|
||||||
|
//! process: a wedged PipeWire owner or an unaccounted Pulse module can collide
|
||||||
|
//! with the next capture. Health is therefore one-way (`Healthy -> Poisoned`)
|
||||||
|
//! and first-fault-wins so a later, less precise teardown symptom cannot erase
|
||||||
|
//! the original cause.
|
||||||
|
|
||||||
|
use std::sync::Arc;
|
||||||
|
use tokio::sync::watch;
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||||
|
pub(super) enum State {
|
||||||
|
Healthy,
|
||||||
|
Poisoned(Arc<str>),
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone)]
|
||||||
|
pub(super) struct Reporter {
|
||||||
|
tx: watch::Sender<State>,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) struct Monitor {
|
||||||
|
rx: watch::Receiver<State>,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn channel() -> (Reporter, Monitor) {
|
||||||
|
let (tx, rx) = watch::channel(State::Healthy);
|
||||||
|
(Reporter { tx }, Monitor { rx })
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Reporter {
|
||||||
|
/// Poison the host exactly once. Returns true for the first fault.
|
||||||
|
pub(super) fn poison(&self, reason: impl Into<Arc<str>>) -> bool {
|
||||||
|
let reason = reason.into();
|
||||||
|
self.tx.send_if_modified(move |state| {
|
||||||
|
if matches!(state, State::Healthy) {
|
||||||
|
*state = State::Poisoned(reason);
|
||||||
|
true
|
||||||
|
} else {
|
||||||
|
false
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
pub(super) fn fault(&self) -> Option<Arc<str>> {
|
||||||
|
match &*self.tx.borrow() {
|
||||||
|
State::Healthy => None,
|
||||||
|
State::Poisoned(reason) => Some(Arc::clone(reason)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Monitor {
|
||||||
|
pub(super) fn fault(&self) -> Option<Arc<str>> {
|
||||||
|
match &*self.rx.borrow() {
|
||||||
|
State::Healthy => None,
|
||||||
|
State::Poisoned(reason) => Some(Arc::clone(reason)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) async fn changed(&mut self) {
|
||||||
|
// The supervisor owns a Reporter for the whole run, so closure is not a
|
||||||
|
// normal state. Treat it like a wake and let `fault()` decide.
|
||||||
|
let _ = self.rx.changed().await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn poison_is_terminal_and_first_fault_wins() {
|
||||||
|
let (reporter, mut monitor) = channel();
|
||||||
|
assert!(reporter.poison("first"));
|
||||||
|
monitor.changed().await;
|
||||||
|
assert_eq!(monitor.fault().as_deref(), Some("first"));
|
||||||
|
assert!(!reporter.poison("second"));
|
||||||
|
assert_eq!(reporter.fault().as_deref(), Some("first"));
|
||||||
|
}
|
||||||
|
}
|
||||||
+98
-2
@@ -2,8 +2,10 @@ pub mod aec;
|
|||||||
pub mod audio;
|
pub mod audio;
|
||||||
pub mod audit;
|
pub mod audit;
|
||||||
mod capture;
|
mod capture;
|
||||||
|
mod health;
|
||||||
pub mod ledger;
|
pub mod ledger;
|
||||||
mod observer;
|
mod observer;
|
||||||
|
mod owned_thread;
|
||||||
mod pipeline;
|
mod pipeline;
|
||||||
mod quality;
|
mod quality;
|
||||||
mod serve;
|
mod serve;
|
||||||
@@ -127,12 +129,15 @@ pub async fn run(opts: HostOpts) -> Result<()> {
|
|||||||
});
|
});
|
||||||
|
|
||||||
let (sup_tx, sup_rx) = mpsc::channel::<SupervisorMsg>(16);
|
let (sup_tx, sup_rx) = mpsc::channel::<SupervisorMsg>(16);
|
||||||
|
let (capture_health, capture_health_monitor) = health::channel();
|
||||||
let supervisor = tokio::spawn(supervise(
|
let supervisor = tokio::spawn(supervise(
|
||||||
opts.clone(),
|
opts.clone(),
|
||||||
quality,
|
quality,
|
||||||
display,
|
display,
|
||||||
resolution.value,
|
resolution.value,
|
||||||
sup_rx,
|
sup_rx,
|
||||||
|
(capture_health, capture_health_monitor),
|
||||||
|
cancel.clone(),
|
||||||
));
|
));
|
||||||
|
|
||||||
// Command channel for the GUI front-end: read `kick <endpoint-id>` lines
|
// Command channel for the GUI front-end: read `kick <endpoint-id>` lines
|
||||||
@@ -289,14 +294,46 @@ async fn supervise(
|
|||||||
display: DisplayServer,
|
display: DisplayServer,
|
||||||
max_viewers: u32,
|
max_viewers: u32,
|
||||||
mut rx: mpsc::Receiver<SupervisorMsg>,
|
mut rx: mpsc::Receiver<SupervisorMsg>,
|
||||||
|
capture_health: (health::Reporter, health::Monitor),
|
||||||
|
host_cancel: CancellationToken,
|
||||||
) {
|
) {
|
||||||
|
let (capture_health, mut capture_health_monitor) = capture_health;
|
||||||
let mut handle: Option<CaptureHandle> = None;
|
let mut handle: Option<CaptureHandle> = None;
|
||||||
// Active viewers, keyed by endpoint id, holding each one's kill switch.
|
// Active viewers, keyed by endpoint id, holding each one's kill switch.
|
||||||
// The count is just `viewers.len()`. (A given endpoint connecting twice is
|
// The count is just `viewers.len()`. (A given endpoint connecting twice is
|
||||||
// a non-case here: each viewer process uses a fresh ephemeral identity.)
|
// a non-case here: each viewer process uses a fresh ephemeral identity.)
|
||||||
let mut viewers: HashMap<String, CancellationToken> = HashMap::new();
|
let mut viewers: HashMap<String, CancellationToken> = HashMap::new();
|
||||||
|
|
||||||
while let Some(msg) = rx.recv().await {
|
loop {
|
||||||
|
// Poison is terminal for this host process. A surviving wedged owner or
|
||||||
|
// an unaccounted graph mutation can collide with a later capture, so do
|
||||||
|
// not detach it and keep advertising the ticket. Cancelling the host
|
||||||
|
// closes the endpoint; PeerSpeak's S2 EOF path then clears presence.
|
||||||
|
if let Some(reason) = capture_health_monitor.fault() {
|
||||||
|
tracing::error!(%reason, "capture supervisor poisoned — stopping host");
|
||||||
|
host_cancel.cancel();
|
||||||
|
for cancel in viewers.values() {
|
||||||
|
cancel.cancel();
|
||||||
|
}
|
||||||
|
if let Some(h) = handle.take() {
|
||||||
|
h.shutdown().await;
|
||||||
|
output::emit(output::Event::Capture {
|
||||||
|
state: output::CaptureState::Stopped,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
let msg = tokio::select! {
|
||||||
|
// Health wins a simultaneous race with another viewer request: a
|
||||||
|
// poisoned host must not begin one more capture.
|
||||||
|
biased;
|
||||||
|
_ = capture_health_monitor.changed() => continue,
|
||||||
|
msg = rx.recv() => msg,
|
||||||
|
};
|
||||||
|
let Some(msg) = msg else {
|
||||||
|
break;
|
||||||
|
};
|
||||||
match msg {
|
match msg {
|
||||||
SupervisorMsg::AddViewer { id, cancel, reply } => {
|
SupervisorMsg::AddViewer { id, cancel, reply } => {
|
||||||
let count = viewers.len() as u32;
|
let count = viewers.len() as u32;
|
||||||
@@ -310,8 +347,37 @@ async fn supervise(
|
|||||||
|
|
||||||
if handle.is_none() {
|
if handle.is_none() {
|
||||||
tracing::info!("first viewer arriving — spawning capture");
|
tracing::info!("first viewer arriving — spawning capture");
|
||||||
match capture::spawn(display, &opts, &quality).await {
|
let spawned = tokio::select! {
|
||||||
|
// A subsystem can fail while the portal/GStreamer setup
|
||||||
|
// future is still running. Do not wait for setup to
|
||||||
|
// return and then admit one viewer to an already-poisoned
|
||||||
|
// capture.
|
||||||
|
biased;
|
||||||
|
_ = capture_health_monitor.changed() => {
|
||||||
|
let reason = capture_health_monitor
|
||||||
|
.fault()
|
||||||
|
.unwrap_or_else(|| "capture health channel changed unexpectedly".into());
|
||||||
|
let _ = reply.send(Err(format!(
|
||||||
|
"capture ownership failed during startup: {reason}"
|
||||||
|
)));
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
result = capture::spawn(
|
||||||
|
display,
|
||||||
|
&opts,
|
||||||
|
&quality,
|
||||||
|
capture_health.clone(),
|
||||||
|
) => result,
|
||||||
|
};
|
||||||
|
match spawned {
|
||||||
Ok(h) => {
|
Ok(h) => {
|
||||||
|
if let Some(reason) = capture_health_monitor.fault() {
|
||||||
|
h.shutdown().await;
|
||||||
|
let _ = reply.send(Err(format!(
|
||||||
|
"capture ownership failed during startup: {reason}"
|
||||||
|
)));
|
||||||
|
continue;
|
||||||
|
}
|
||||||
handle = Some(h);
|
handle = Some(h);
|
||||||
output::emit(output::Event::Capture {
|
output::emit(output::Event::Capture {
|
||||||
state: output::CaptureState::Started,
|
state: output::CaptureState::Started,
|
||||||
@@ -568,4 +634,34 @@ mod tests {
|
|||||||
assert_eq!(initial_app_audio_state(&opts(None, true)), None);
|
assert_eq!(initial_app_audio_state(&opts(None, true)), None);
|
||||||
assert_eq!(initial_app_audio_state(&opts(None, false)), None);
|
assert_eq!(initial_app_audio_state(&opts(None, false)), None);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn supervisor_health_poison_cancels_the_host_with_command_channel_open() {
|
||||||
|
let opts = opts(None, false);
|
||||||
|
let quality = quality::resolve(&opts, 1);
|
||||||
|
let (tx, rx) = mpsc::channel(1);
|
||||||
|
let (health, monitor) = health::channel();
|
||||||
|
let cancel = CancellationToken::new();
|
||||||
|
|
||||||
|
let supervisor = tokio::spawn(supervise(
|
||||||
|
opts,
|
||||||
|
quality,
|
||||||
|
DisplayServer::Unknown,
|
||||||
|
1,
|
||||||
|
rx,
|
||||||
|
(health.clone(), monitor),
|
||||||
|
cancel.clone(),
|
||||||
|
));
|
||||||
|
assert!(health.poison("test ownership fault"));
|
||||||
|
tokio::time::timeout(Duration::from_secs(1), supervisor)
|
||||||
|
.await
|
||||||
|
.expect("poisoned supervisor must stop promptly")
|
||||||
|
.expect("supervisor task must not panic");
|
||||||
|
assert!(cancel.is_cancelled());
|
||||||
|
|
||||||
|
// The sender deliberately stayed open until the supervisor exited. If
|
||||||
|
// the health arm were removed, the task above would still be blocked on
|
||||||
|
// `rx.recv()` and the timeout would fail.
|
||||||
|
drop(tx);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,170 @@
|
|||||||
|
//! Cancellation-safe ownership for blocking subsystem threads.
|
||||||
|
//!
|
||||||
|
//! `std::thread::JoinHandle` has no timed join. Moving it into
|
||||||
|
//! `spawn_blocking` only moves the problem: cancelling the async waiter detaches
|
||||||
|
//! the blocking task and loses the only handle. Instead this owner polls
|
||||||
|
//! `is_finished()` while retaining the handle, joins only after completion, and
|
||||||
|
//! quarantines an unfinished handle on timeout or Drop. Quarantine is
|
||||||
|
//! process-lifetime ownership, not recovery; the shared health channel makes
|
||||||
|
//! the supervisor terminate the poisoned host.
|
||||||
|
|
||||||
|
use super::health::Reporter;
|
||||||
|
use std::sync::{Mutex, OnceLock};
|
||||||
|
use std::thread::JoinHandle;
|
||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
|
static QUARANTINED: OnceLock<Mutex<Vec<JoinHandle<()>>>> = OnceLock::new();
|
||||||
|
|
||||||
|
fn quarantine(handle: JoinHandle<()>) {
|
||||||
|
let mut handles = QUARANTINED
|
||||||
|
.get_or_init(|| Mutex::new(Vec::new()))
|
||||||
|
.lock()
|
||||||
|
.unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||||
|
|
||||||
|
// Reap anything that happened to finish since the previous quarantine;
|
||||||
|
// every still-running handle remains owned until process exit.
|
||||||
|
let old = std::mem::take(&mut *handles);
|
||||||
|
for old_handle in old {
|
||||||
|
if old_handle.is_finished() {
|
||||||
|
let _ = old_handle.join();
|
||||||
|
} else {
|
||||||
|
handles.push(old_handle);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
handles.push(handle);
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) struct OwnedThread {
|
||||||
|
name: &'static str,
|
||||||
|
handle: Option<JoinHandle<()>>,
|
||||||
|
health: Reporter,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl OwnedThread {
|
||||||
|
pub(super) fn new(name: &'static str, handle: JoinHandle<()>, health: Reporter) -> Self {
|
||||||
|
Self {
|
||||||
|
name,
|
||||||
|
handle: Some(handle),
|
||||||
|
health,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Wait up to `budget`, retaining the OS handle across every await.
|
||||||
|
///
|
||||||
|
/// Returns true only when the thread was joined successfully. Timeout and
|
||||||
|
/// panic poison the process; a timeout also moves the still-running handle
|
||||||
|
/// into process-lifetime quarantine.
|
||||||
|
pub(super) async fn join_within(&mut self, budget: Duration) -> bool {
|
||||||
|
let deadline = tokio::time::Instant::now() + budget;
|
||||||
|
loop {
|
||||||
|
let Some(handle) = self.handle.as_ref() else {
|
||||||
|
return true;
|
||||||
|
};
|
||||||
|
if handle.is_finished() {
|
||||||
|
let handle = self.handle.take().expect("checked as present");
|
||||||
|
return match handle.join() {
|
||||||
|
Ok(()) => true,
|
||||||
|
Err(_) => {
|
||||||
|
self.health
|
||||||
|
.poison(format!("{} panicked during shutdown", self.name));
|
||||||
|
false
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
if tokio::time::Instant::now() >= deadline {
|
||||||
|
self.health.poison(format!(
|
||||||
|
"{} did not stop within {:?}; its thread handle is quarantined",
|
||||||
|
self.name, budget
|
||||||
|
));
|
||||||
|
quarantine(self.handle.take().expect("checked as present"));
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Drop for OwnedThread {
|
||||||
|
fn drop(&mut self) {
|
||||||
|
let Some(handle) = self.handle.take() else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
if handle.is_finished() {
|
||||||
|
if handle.join().is_err() {
|
||||||
|
self.health
|
||||||
|
.poison(format!("{} panicked before it was joined", self.name));
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
self.health.poison(format!(
|
||||||
|
"{} was dropped before it stopped; its thread handle is quarantined",
|
||||||
|
self.name
|
||||||
|
));
|
||||||
|
quarantine(handle);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
use crate::host::health;
|
||||||
|
use std::sync::mpsc;
|
||||||
|
|
||||||
|
fn reap_finished_quarantine() {
|
||||||
|
let Some(handles) = QUARANTINED.get() else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
let mut handles = handles
|
||||||
|
.lock()
|
||||||
|
.unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||||
|
let old = std::mem::take(&mut *handles);
|
||||||
|
for handle in old {
|
||||||
|
if handle.is_finished() {
|
||||||
|
let _ = handle.join();
|
||||||
|
} else {
|
||||||
|
handles.push(handle);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn completed_thread_is_joined_without_poison() {
|
||||||
|
let (health, _) = health::channel();
|
||||||
|
let handle = std::thread::spawn(|| {});
|
||||||
|
let mut owner = OwnedThread::new("test worker", handle, health.clone());
|
||||||
|
assert!(owner.join_within(Duration::from_secs(1)).await);
|
||||||
|
assert!(health.fault().is_none());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn timeout_poisons_and_quarantines_instead_of_detaching() {
|
||||||
|
let (release_tx, release_rx) = mpsc::channel();
|
||||||
|
let (health, _) = health::channel();
|
||||||
|
let handle = std::thread::spawn(move || {
|
||||||
|
let _ = release_rx.recv();
|
||||||
|
});
|
||||||
|
let mut owner = OwnedThread::new("wedged worker", handle, health.clone());
|
||||||
|
|
||||||
|
assert!(!owner.join_within(Duration::from_millis(20)).await);
|
||||||
|
assert!(health.fault().is_some());
|
||||||
|
drop(owner);
|
||||||
|
release_tx.send(()).expect("release quarantined worker");
|
||||||
|
std::thread::sleep(Duration::from_millis(20));
|
||||||
|
reap_finished_quarantine();
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn dropping_an_unfinished_owner_poisons_and_quarantines() {
|
||||||
|
let (release_tx, release_rx) = mpsc::channel();
|
||||||
|
let (health, _) = health::channel();
|
||||||
|
let handle = std::thread::spawn(move || {
|
||||||
|
let _ = release_rx.recv();
|
||||||
|
});
|
||||||
|
drop(OwnedThread::new("cancelled worker", handle, health.clone()));
|
||||||
|
assert!(health.fault().is_some());
|
||||||
|
release_tx.send(()).expect("release quarantined worker");
|
||||||
|
std::thread::sleep(Duration::from_millis(20));
|
||||||
|
reap_finished_quarantine();
|
||||||
|
}
|
||||||
|
}
|
||||||
+197
-26
@@ -6,22 +6,122 @@
|
|||||||
//! and lives here. Backends call [`spawn`] with just their source-element args.
|
//! and lives here. Backends call [`spawn`] with just their source-element args.
|
||||||
|
|
||||||
use anyhow::{Context, Result, bail};
|
use anyhow::{Context, Result, bail};
|
||||||
use nix::sys::signal::{Signal, kill};
|
use nix::sys::signal::Signal;
|
||||||
use nix::unistd::Pid;
|
|
||||||
use std::process::Stdio;
|
use std::process::Stdio;
|
||||||
|
use std::sync::Arc;
|
||||||
|
use std::sync::atomic::{AtomicBool, Ordering};
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
use tokio::process::{Child, Command};
|
use tokio::process::{Child, Command};
|
||||||
use tokio::time::timeout;
|
use tokio::time::timeout;
|
||||||
|
|
||||||
use super::audio::Routing;
|
use super::audio::Routing;
|
||||||
|
use super::health;
|
||||||
use super::quality::EffectiveQuality;
|
use super::quality::EffectiveQuality;
|
||||||
use super::serve::Serve;
|
use super::serve::Serve;
|
||||||
use crate::cli::HostOpts;
|
use crate::cli::HostOpts;
|
||||||
|
use crate::common::contained;
|
||||||
|
|
||||||
pub struct CaptureHandle {
|
const GST_TERM_BUDGET: Duration = Duration::from_secs(1);
|
||||||
gst: Option<Child>,
|
const GST_KILL_BUDGET: Duration = Duration::from_secs(1);
|
||||||
|
|
||||||
|
/// Owns the contained GStreamer process from spawn through confirmed reap.
|
||||||
|
///
|
||||||
|
/// Keeping this guard alive during `Serve::bind` closes the old constructor
|
||||||
|
/// leak: any error after spawn kills the whole process group, not merely the
|
||||||
|
/// direct child. An unconfirmed Drop poisons the host so the supervisor cannot
|
||||||
|
/// start another capture on top of a possibly-live portal/PipeWire owner.
|
||||||
|
struct CaptureProcess {
|
||||||
|
child: Child,
|
||||||
|
leader_pid: u32,
|
||||||
|
reaped: bool,
|
||||||
|
health: health::Reporter,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl CaptureProcess {
|
||||||
|
fn new(child: Child, health: health::Reporter) -> Result<Self> {
|
||||||
|
let leader_pid = child
|
||||||
|
.id()
|
||||||
|
.context("gst-launch-1.0 exited before its process id was recorded")?;
|
||||||
|
Ok(Self {
|
||||||
|
child,
|
||||||
|
leader_pid,
|
||||||
|
reaped: false,
|
||||||
|
health,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn take_stdout(&mut self) -> Option<tokio::process::ChildStdout> {
|
||||||
|
self.child.stdout.take()
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn shutdown(&mut self) {
|
||||||
|
// Reap an already-dead child before addressing its process group. A
|
||||||
|
// zombie still reserves its pid, but after `try_wait` succeeds that pid
|
||||||
|
// may be reused; returning here avoids ever signalling a new group that
|
||||||
|
// inherited the old numeric id.
|
||||||
|
match self.child.try_wait() {
|
||||||
|
Ok(Some(_)) => {
|
||||||
|
self.reaped = true;
|
||||||
|
self.health
|
||||||
|
.poison("gst-launch-1.0 exited before PixelPass began capture shutdown");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
Ok(None) => {}
|
||||||
|
Err(e) => tracing::warn!(
|
||||||
|
"capture: could not inspect gst before SIGTERM ({e}); continuing teardown"
|
||||||
|
),
|
||||||
|
}
|
||||||
|
|
||||||
|
let _ = contained::signal_group(self.leader_pid, Signal::SIGTERM);
|
||||||
|
match timeout(GST_TERM_BUDGET, self.child.wait()).await {
|
||||||
|
Ok(Ok(_)) => {
|
||||||
|
self.reaped = true;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
Ok(Err(e)) => tracing::warn!(
|
||||||
|
"capture: gst wait after SIGTERM failed ({e}); escalating to SIGKILL"
|
||||||
|
),
|
||||||
|
Err(_) => tracing::warn!(
|
||||||
|
"capture: gst did not exit within {GST_TERM_BUDGET:?}; escalating to SIGKILL"
|
||||||
|
),
|
||||||
|
}
|
||||||
|
|
||||||
|
let _ = contained::signal_group(self.leader_pid, Signal::SIGKILL);
|
||||||
|
let _ = self.child.start_kill();
|
||||||
|
match timeout(GST_KILL_BUDGET, self.child.wait()).await {
|
||||||
|
Ok(Ok(_)) => self.reaped = true,
|
||||||
|
Ok(Err(e)) => {
|
||||||
|
self.health.poison(format!(
|
||||||
|
"gst-launch-1.0 could not be reaped after SIGKILL: {e}"
|
||||||
|
));
|
||||||
|
}
|
||||||
|
Err(_) => {
|
||||||
|
self.health.poison(format!(
|
||||||
|
"gst-launch-1.0 did not exit within {GST_KILL_BUDGET:?} after SIGKILL"
|
||||||
|
));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Drop for CaptureProcess {
|
||||||
|
fn drop(&mut self) {
|
||||||
|
if self.reaped {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let _ = contained::signal_group(self.leader_pid, Signal::SIGKILL);
|
||||||
|
let _ = self.child.start_kill();
|
||||||
|
self.health.poison(
|
||||||
|
"gst-launch-1.0 was dropped before a confirmed reap; its process group was killed",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) struct CaptureHandle {
|
||||||
|
gst: Option<CaptureProcess>,
|
||||||
audio: Option<Routing>,
|
audio: Option<Routing>,
|
||||||
serve: Option<Serve>,
|
serve: Option<Serve>,
|
||||||
|
stopping: Arc<AtomicBool>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl CaptureHandle {
|
impl CaptureHandle {
|
||||||
@@ -32,19 +132,15 @@ impl CaptureHandle {
|
|||||||
.local_port()
|
.local_port()
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Graceful teardown: SIGTERM gst, give it ~1s to exit, then SIGKILL,
|
/// Graceful teardown: SIGTERM the gst process group, give it ~1s to exit,
|
||||||
|
/// then SIGKILL and a second bounded reap,
|
||||||
/// unload audio routing (if any), then tear down the serve layer.
|
/// unload audio routing (if any), then tear down the serve layer.
|
||||||
/// The serve reader will see EOF on gst stdout and exit on its own;
|
/// The serve reader will see EOF on gst stdout and exit on its own;
|
||||||
/// serve.shutdown() is the backstop.
|
/// serve.shutdown() is the backstop.
|
||||||
pub async fn shutdown(mut self) {
|
pub async fn shutdown(mut self) {
|
||||||
if let Some(child) = self.gst.as_mut()
|
self.stopping.store(true, Ordering::Release);
|
||||||
&& let Some(pid) = child.id()
|
if let Some(mut gst) = self.gst.take() {
|
||||||
{
|
gst.shutdown().await;
|
||||||
let _ = kill(Pid::from_raw(pid as i32), Signal::SIGTERM);
|
|
||||||
}
|
|
||||||
if let Some(child) = self.gst.as_mut() {
|
|
||||||
let _ = timeout(Duration::from_millis(1000), child.wait()).await;
|
|
||||||
let _ = child.start_kill();
|
|
||||||
}
|
}
|
||||||
if let Some(audio) = self.audio.take() {
|
if let Some(audio) = self.audio.take() {
|
||||||
audio.shutdown().await;
|
audio.shutdown().await;
|
||||||
@@ -57,10 +153,9 @@ impl CaptureHandle {
|
|||||||
|
|
||||||
impl Drop for CaptureHandle {
|
impl Drop for CaptureHandle {
|
||||||
fn drop(&mut self) {
|
fn drop(&mut self) {
|
||||||
if let Some(child) = self.gst.as_mut() {
|
self.stopping.store(true, Ordering::Release);
|
||||||
let _ = child.start_kill();
|
// CaptureProcess kills the whole process group and poisons the host;
|
||||||
}
|
// Routing's and Serve's own Drop impls handle their respective layers.
|
||||||
// Routing's and Serve's own Drop impls handle the rest.
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -73,14 +168,15 @@ impl Drop for CaptureHandle {
|
|||||||
/// `after_spawn` runs once, immediately after the gst child is launched —
|
/// `after_spawn` runs once, immediately after the gst child is launched —
|
||||||
/// Wayland uses it to `close` the pipewire fd it leaked into the child; X11
|
/// Wayland uses it to `close` the pipewire fd it leaked into the child; X11
|
||||||
/// passes a no-op.
|
/// passes a no-op.
|
||||||
pub async fn spawn(
|
pub(super) async fn spawn(
|
||||||
opts: &HostOpts,
|
opts: &HostOpts,
|
||||||
quality: &EffectiveQuality,
|
quality: &EffectiveQuality,
|
||||||
source_dims: Option<(u32, u32)>,
|
source_dims: Option<(u32, u32)>,
|
||||||
source_args: Vec<String>,
|
source_args: Vec<String>,
|
||||||
|
health: health::Reporter,
|
||||||
after_spawn: impl FnOnce(),
|
after_spawn: impl FnOnce(),
|
||||||
) -> Result<CaptureHandle> {
|
) -> Result<CaptureHandle> {
|
||||||
let (audio_routing, audio_device) = setup_audio(opts).await?;
|
let (audio_routing, audio_device) = setup_audio(opts, health.clone()).await?;
|
||||||
let args = build_args(&source_args, &audio_device, opts, quality, source_dims);
|
let args = build_args(&source_args, &audio_device, opts, quality, source_dims);
|
||||||
|
|
||||||
let mut gst_cmd = Command::new("gst-launch-1.0");
|
let mut gst_cmd = Command::new("gst-launch-1.0");
|
||||||
@@ -88,29 +184,32 @@ pub async fn spawn(
|
|||||||
.args(&args)
|
.args(&args)
|
||||||
.stdin(Stdio::null())
|
.stdin(Stdio::null())
|
||||||
.stdout(Stdio::piped())
|
.stdout(Stdio::piped())
|
||||||
.stderr(Stdio::inherit());
|
.stderr(Stdio::inherit())
|
||||||
|
.kill_on_drop(true);
|
||||||
if std::env::var_os("PIXELPASS_GST_DEBUG").is_some() {
|
if std::env::var_os("PIXELPASS_GST_DEBUG").is_some() {
|
||||||
gst_cmd.env("GST_DEBUG", "3");
|
gst_cmd.env("GST_DEBUG", "3");
|
||||||
}
|
}
|
||||||
let mut gst = gst_cmd.spawn().context("failed to spawn gst-launch-1.0")?;
|
let gst = contained::spawn_tokio(&mut gst_cmd).context("failed to spawn gst-launch-1.0")?;
|
||||||
|
let mut gst = CaptureProcess::new(gst, health.clone())?;
|
||||||
|
|
||||||
// Backend-specific post-spawn cleanup (Wayland closes its leaked pw fd here,
|
// Backend-specific post-spawn cleanup (Wayland closes its leaked pw fd here,
|
||||||
// once gst has inherited its own copy).
|
// once gst has inherited its own copy).
|
||||||
after_spawn();
|
after_spawn();
|
||||||
|
|
||||||
let gst_stdout = gst
|
let gst_stdout = gst
|
||||||
.stdout
|
.take_stdout()
|
||||||
.take()
|
|
||||||
.context("gst-launch-1.0 stdout pipe unavailable")?;
|
.context("gst-launch-1.0 stdout pipe unavailable")?;
|
||||||
|
|
||||||
// Hand stdout to the serve layer, which binds the localhost HTTP listener
|
// Hand stdout to the serve layer, which binds the localhost HTTP listener
|
||||||
// and runs the broadcast fanout. No demux/remux, no codec assumptions.
|
// and runs the broadcast fanout. No demux/remux, no codec assumptions.
|
||||||
let serve = Serve::bind(gst_stdout).await?;
|
let stopping = Arc::new(AtomicBool::new(false));
|
||||||
|
let serve = Serve::bind(gst_stdout, health.clone(), Arc::clone(&stopping)).await?;
|
||||||
|
|
||||||
Ok(CaptureHandle {
|
Ok(CaptureHandle {
|
||||||
gst: Some(gst),
|
gst: Some(gst),
|
||||||
audio: audio_routing,
|
audio: audio_routing,
|
||||||
serve: Some(serve),
|
serve: Some(serve),
|
||||||
|
stopping,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -120,12 +219,15 @@ pub async fn spawn(
|
|||||||
/// is set (no app filter — captures everything via the null-sink, used for
|
/// is set (no app filter — captures everything via the null-sink, used for
|
||||||
/// dogfooding the loopback path). Otherwise we capture the default sink's
|
/// dogfooding the loopback path). Otherwise we capture the default sink's
|
||||||
/// monitor (system audio out), not the default source (the mic).
|
/// monitor (system audio out), not the default source (the mic).
|
||||||
async fn setup_audio(opts: &HostOpts) -> Result<(Option<Routing>, String)> {
|
async fn setup_audio(
|
||||||
|
opts: &HostOpts,
|
||||||
|
health: health::Reporter,
|
||||||
|
) -> Result<(Option<Routing>, String)> {
|
||||||
let routing_requested =
|
let routing_requested =
|
||||||
opts.app.is_some() || std::env::var_os("PIXELPASS_AUDIO_VIA_NULL_SINK").is_some();
|
opts.app.is_some() || std::env::var_os("PIXELPASS_AUDIO_VIA_NULL_SINK").is_some();
|
||||||
let audio_routing = if routing_requested {
|
let audio_routing = if routing_requested {
|
||||||
Some(
|
Some(
|
||||||
Routing::start(opts)
|
Routing::start(opts, health)
|
||||||
.await
|
.await
|
||||||
.context("audio routing setup failed")?,
|
.context("audio routing setup failed")?,
|
||||||
)
|
)
|
||||||
@@ -349,3 +451,72 @@ async fn default_audio_monitor() -> Result<String> {
|
|||||||
}
|
}
|
||||||
Ok(format!("{sink}.monitor"))
|
Ok(format!("{sink}.monitor"))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
use std::time::{Duration, Instant};
|
||||||
|
|
||||||
|
fn process_is_running(pid: u32) -> bool {
|
||||||
|
let Ok(stat) = std::fs::read_to_string(format!("/proc/{pid}/stat")) else {
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
stat.rsplit_once(") ")
|
||||||
|
.and_then(|(_, rest)| rest.as_bytes().first().copied())
|
||||||
|
.is_some_and(|state| state != b'Z' && state != b'X')
|
||||||
|
}
|
||||||
|
|
||||||
|
fn sleeping_capture(health: health::Reporter) -> CaptureProcess {
|
||||||
|
let mut command = Command::new("sleep");
|
||||||
|
command.arg("30").kill_on_drop(true);
|
||||||
|
let child = contained::spawn_tokio(&mut command).expect("spawn contained fixture");
|
||||||
|
CaptureProcess::new(child, health).expect("capture process guard")
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn graceful_capture_shutdown_confirms_reap_without_poison() {
|
||||||
|
let (health, _) = health::channel();
|
||||||
|
let mut capture = sleeping_capture(health.clone());
|
||||||
|
capture.shutdown().await;
|
||||||
|
assert!(health.fault().is_none());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn an_already_exited_capture_is_not_misreported_as_a_clean_shutdown() {
|
||||||
|
let (health, _) = health::channel();
|
||||||
|
let mut command = Command::new("true");
|
||||||
|
command.kill_on_drop(true);
|
||||||
|
let child = contained::spawn_tokio(&mut command).expect("spawn short contained fixture");
|
||||||
|
let mut capture = CaptureProcess::new(child, health.clone()).expect("capture guard");
|
||||||
|
|
||||||
|
let deadline = Instant::now() + Duration::from_secs(1);
|
||||||
|
while process_is_running(capture.leader_pid) && Instant::now() < deadline {
|
||||||
|
tokio::task::yield_now().await;
|
||||||
|
}
|
||||||
|
assert!(
|
||||||
|
!process_is_running(capture.leader_pid),
|
||||||
|
"short fixture must exit before shutdown begins"
|
||||||
|
);
|
||||||
|
|
||||||
|
capture.shutdown().await;
|
||||||
|
assert_eq!(
|
||||||
|
health.fault().as_deref(),
|
||||||
|
Some("gst-launch-1.0 exited before PixelPass began capture shutdown")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn dropping_live_capture_kills_its_group_and_poisons() {
|
||||||
|
let (health, _) = health::channel();
|
||||||
|
let capture = sleeping_capture(health.clone());
|
||||||
|
let pid = capture.leader_pid;
|
||||||
|
drop(capture);
|
||||||
|
assert!(health.fault().is_some());
|
||||||
|
|
||||||
|
let deadline = Instant::now() + Duration::from_secs(2);
|
||||||
|
while process_is_running(pid) && Instant::now() < deadline {
|
||||||
|
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||||
|
}
|
||||||
|
assert!(!process_is_running(pid));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
+55
-3
@@ -10,6 +10,7 @@
|
|||||||
|
|
||||||
use anyhow::{Context, Result, bail};
|
use anyhow::{Context, Result, bail};
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
use std::sync::atomic::{AtomicBool, Ordering};
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||||
use tokio::net::{TcpListener, TcpStream};
|
use tokio::net::{TcpListener, TcpStream};
|
||||||
@@ -18,6 +19,8 @@ use tokio::sync::broadcast;
|
|||||||
use tokio::task::JoinHandle;
|
use tokio::task::JoinHandle;
|
||||||
use tokio::time::{Instant, sleep};
|
use tokio::time::{Instant, sleep};
|
||||||
|
|
||||||
|
use super::health;
|
||||||
|
|
||||||
/// Broadcast-channel capacity in chunks. Each chunk is up to 64 KiB from
|
/// Broadcast-channel capacity in chunks. Each chunk is up to 64 KiB from
|
||||||
/// the capture child's stdout, so 16 chunks ≈ 1 MiB ≈ ~2 s of buffered
|
/// the capture child's stdout, so 16 chunks ≈ 1 MiB ≈ ~2 s of buffered
|
||||||
/// jitter at typical bitrates. A viewer that falls behind by more than
|
/// jitter at typical bitrates. A viewer that falls behind by more than
|
||||||
@@ -40,14 +43,18 @@ impl Serve {
|
|||||||
/// Bind a localhost listener on a random port, set up the broadcast
|
/// Bind a localhost listener on a random port, set up the broadcast
|
||||||
/// fanout, and spawn the reader + accept-loop tasks. The provided
|
/// fanout, and spawn the reader + accept-loop tasks. The provided
|
||||||
/// `stdout` is assumed to produce MPEG-TS bytes.
|
/// `stdout` is assumed to produce MPEG-TS bytes.
|
||||||
pub async fn bind(stdout: ChildStdout) -> Result<Self> {
|
pub(super) async fn bind(
|
||||||
|
stdout: ChildStdout,
|
||||||
|
health: health::Reporter,
|
||||||
|
stopping: Arc<AtomicBool>,
|
||||||
|
) -> Result<Self> {
|
||||||
let listener = TcpListener::bind("127.0.0.1:0")
|
let listener = TcpListener::bind("127.0.0.1:0")
|
||||||
.await
|
.await
|
||||||
.context("could not bind local capture HTTP listener")?;
|
.context("could not bind local capture HTTP listener")?;
|
||||||
let port = listener.local_addr()?.port();
|
let port = listener.local_addr()?.port();
|
||||||
|
|
||||||
let (tx, _) = broadcast::channel::<Arc<Vec<u8>>>(FANOUT_CAPACITY);
|
let (tx, _) = broadcast::channel::<Arc<Vec<u8>>>(FANOUT_CAPACITY);
|
||||||
let reader = tokio::spawn(pump_to_broadcast(stdout, tx.clone()));
|
let reader = tokio::spawn(pump_to_broadcast(stdout, tx.clone(), health, stopping));
|
||||||
let server = tokio::spawn(run_accept_loop(listener, tx));
|
let server = tokio::spawn(run_accept_loop(listener, tx));
|
||||||
|
|
||||||
Ok(Self {
|
Ok(Self {
|
||||||
@@ -105,12 +112,20 @@ pub async fn connect_to_capture(port: u16, max_wait: Duration) -> Result<TcpStre
|
|||||||
/// current subscribers. `broadcast::send` returns Err when there are no
|
/// current subscribers. `broadcast::send` returns Err when there are no
|
||||||
/// receivers; we ignore it so the capture child isn't backpressured
|
/// receivers; we ignore it so the capture child isn't backpressured
|
||||||
/// waiting for a viewer.
|
/// waiting for a viewer.
|
||||||
async fn pump_to_broadcast(mut stdout: ChildStdout, tx: broadcast::Sender<Arc<Vec<u8>>>) {
|
async fn pump_to_broadcast(
|
||||||
|
mut stdout: impl tokio::io::AsyncRead + Unpin,
|
||||||
|
tx: broadcast::Sender<Arc<Vec<u8>>>,
|
||||||
|
health: health::Reporter,
|
||||||
|
stopping: Arc<AtomicBool>,
|
||||||
|
) {
|
||||||
let mut buf = vec![0u8; READ_CHUNK];
|
let mut buf = vec![0u8; READ_CHUNK];
|
||||||
loop {
|
loop {
|
||||||
match stdout.read(&mut buf).await {
|
match stdout.read(&mut buf).await {
|
||||||
Ok(0) => {
|
Ok(0) => {
|
||||||
tracing::info!("capture stdout EOF — fanout reader exiting");
|
tracing::info!("capture stdout EOF — fanout reader exiting");
|
||||||
|
if !stopping.load(Ordering::Acquire) {
|
||||||
|
health.poison("GStreamer capture stdout closed unexpectedly");
|
||||||
|
}
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
Ok(n) => {
|
Ok(n) => {
|
||||||
@@ -119,6 +134,9 @@ async fn pump_to_broadcast(mut stdout: ChildStdout, tx: broadcast::Sender<Arc<Ve
|
|||||||
}
|
}
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
tracing::warn!("capture stdout read error: {e}");
|
tracing::warn!("capture stdout read error: {e}");
|
||||||
|
if !stopping.load(Ordering::Acquire) {
|
||||||
|
health.poison(format!("GStreamer capture stdout failed: {e}"));
|
||||||
|
}
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -192,3 +210,37 @@ async fn drain_http_request(sock: &mut TcpStream) -> bool {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn unexpected_capture_eof_poisons_the_host() {
|
||||||
|
let (reader, writer) = tokio::io::duplex(16);
|
||||||
|
let (tx, _) = broadcast::channel(FANOUT_CAPACITY);
|
||||||
|
let (health, _) = health::channel();
|
||||||
|
let stopping = Arc::new(AtomicBool::new(false));
|
||||||
|
|
||||||
|
drop(writer);
|
||||||
|
pump_to_broadcast(reader, tx, health.clone(), stopping).await;
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
health.fault().as_deref(),
|
||||||
|
Some("GStreamer capture stdout closed unexpectedly")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn expected_capture_eof_during_shutdown_stays_healthy() {
|
||||||
|
let (reader, writer) = tokio::io::duplex(16);
|
||||||
|
let (tx, _) = broadcast::channel(FANOUT_CAPACITY);
|
||||||
|
let (health, _) = health::channel();
|
||||||
|
let stopping = Arc::new(AtomicBool::new(true));
|
||||||
|
|
||||||
|
drop(writer);
|
||||||
|
pump_to_broadcast(reader, tx, health.clone(), stopping).await;
|
||||||
|
|
||||||
|
assert!(health.fault().is_none());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
+7
-1
@@ -14,11 +14,16 @@ use ashpd::{
|
|||||||
use nix::fcntl::{FcntlArg, FdFlag, fcntl};
|
use nix::fcntl::{FcntlArg, FdFlag, fcntl};
|
||||||
use std::os::fd::{AsFd, AsRawFd, OwnedFd, RawFd};
|
use std::os::fd::{AsFd, AsRawFd, OwnedFd, RawFd};
|
||||||
|
|
||||||
|
use super::health;
|
||||||
use super::pipeline::{self, CaptureHandle};
|
use super::pipeline::{self, CaptureHandle};
|
||||||
use super::quality::EffectiveQuality;
|
use super::quality::EffectiveQuality;
|
||||||
use crate::cli::HostOpts;
|
use crate::cli::HostOpts;
|
||||||
|
|
||||||
pub async fn start(opts: &HostOpts, quality: &EffectiveQuality) -> Result<CaptureHandle> {
|
pub(super) async fn start(
|
||||||
|
opts: &HostOpts,
|
||||||
|
quality: &EffectiveQuality,
|
||||||
|
health: health::Reporter,
|
||||||
|
) -> Result<CaptureHandle> {
|
||||||
// 1. Negotiate the screencast session with the portal.
|
// 1. Negotiate the screencast session with the portal.
|
||||||
let proxy = Screencast::new()
|
let proxy = Screencast::new()
|
||||||
.await
|
.await
|
||||||
@@ -81,6 +86,7 @@ pub async fn start(opts: &HostOpts, quality: &EffectiveQuality) -> Result<Captur
|
|||||||
quality,
|
quality,
|
||||||
Some((w as u32, h as u32)),
|
Some((w as u32, h as u32)),
|
||||||
source_args,
|
source_args,
|
||||||
|
health,
|
||||||
move || {
|
move || {
|
||||||
// Parent no longer needs the pipewire fd — gst inherited its own copy.
|
// Parent no longer needs the pipewire fd — gst inherited its own copy.
|
||||||
drop(pw_fd);
|
drop(pw_fd);
|
||||||
|
|||||||
+7
-2
@@ -10,11 +10,16 @@ use tokio::process::Command;
|
|||||||
use x11rb::connection::Connection;
|
use x11rb::connection::Connection;
|
||||||
use x11rb::protocol::xproto::ConnectionExt;
|
use x11rb::protocol::xproto::ConnectionExt;
|
||||||
|
|
||||||
|
use super::health;
|
||||||
use super::pipeline::{self, CaptureHandle};
|
use super::pipeline::{self, CaptureHandle};
|
||||||
use super::quality::EffectiveQuality;
|
use super::quality::EffectiveQuality;
|
||||||
use crate::cli::HostOpts;
|
use crate::cli::HostOpts;
|
||||||
|
|
||||||
pub async fn start(opts: &HostOpts, quality: &EffectiveQuality) -> Result<CaptureHandle> {
|
pub(super) async fn start(
|
||||||
|
opts: &HostOpts,
|
||||||
|
quality: &EffectiveQuality,
|
||||||
|
health: health::Reporter,
|
||||||
|
) -> Result<CaptureHandle> {
|
||||||
let xid = if opts.window {
|
let xid = if opts.window {
|
||||||
Some(pick_window().await?)
|
Some(pick_window().await?)
|
||||||
} else {
|
} else {
|
||||||
@@ -60,7 +65,7 @@ pub async fn start(opts: &HostOpts, quality: &EffectiveQuality) -> Result<Captur
|
|||||||
}
|
}
|
||||||
|
|
||||||
// X11 has no leaked fd to clean up, so the post-spawn hook is a no-op.
|
// X11 has no leaked fd to clean up, so the post-spawn hook is a no-op.
|
||||||
pipeline::spawn(opts, quality, source_dims, source_args, || {}).await
|
pipeline::spawn(opts, quality, source_dims, source_args, health, || {}).await
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Run `xwininfo` and let the user click the window they want to share, then
|
/// Run `xwininfo` and let the user click the window they want to share, then
|
||||||
|
|||||||
Reference in New Issue
Block a user