feat(host): recover desktop audio fanout after sink replacement

src/host/graph.rs

@@ -12,7 +12,7 @@ use super::audio::parse_object_serial;
 use super::fanout::FanoutController;
 use super::health;
 use super::observer::Readiness;
-use super::observer::adapter::RegistryObserverHandle;
+use super::observer::adapter::{FanoutControl, RegistryObserverHandle};
 use super::owned_thread::OwnedThread;
 use super::taint::snapshot::{GlobalId, Serial};
 use crate::common::output::AudioExclusionEvent;
@@ -123,6 +123,9 @@ enum Cmd {
     /// the main loop. Dropping the connection removes the native non-lingering
     /// sink only after those restoration writes have reached the server.
     Stop,
+    /// Recreate an unexpectedly removed connection-owned sink on this same
+    /// PipeWire connection. Only the actor's own callbacks enqueue this.
+    RecreateSink,
 }
 
 /// Tokio-side owner of the PipeWire actor thread.
@@ -143,9 +146,11 @@ pub(super) struct AudioGraphOwner {
 /// connection that retains non-lingering native PipeWire links; legacy
 /// `Routing` remains a separate type in `host::audio`.
 pub(super) struct BareCaptureSink {
-    // Drop order is load-bearing: close the fan-out observer connection (and
-    // therefore every non-lingering link), then detach the status task, before
-    // the sink-owning connection.
+    // Shutdown order is load-bearing: stop identity forwarding, close the
+    // fan-out observer connection (and therefore every non-lingering link),
+    // then drain status before closing the sink-owning connection.
+    replacement_stop: Option<oneshot::Sender<()>>,
+    replacement_forwarder: Option<tokio::task::JoinHandle<()>>,
     fanout: Option<RegistryObserverHandle>,
     status_forwarder: Option<tokio::task::JoinHandle<()>>,
     graph_owner: Option<AudioGraphOwner>,
@@ -155,9 +160,10 @@ pub(super) struct BareCaptureSink {
 impl BareCaptureSink {
     pub(super) async fn start(health: health::Reporter) -> Result<Self> {
         let spec = CaptureSinkSpec::for_pid(std::process::id());
-        let (graph_owner, _event_rx) = AudioGraphOwner::start(None, spec, health.clone())
-            .await
-            .context("failed to start the bare connection-owned capture sink")?;
+        let (graph_owner, _event_rx, identity_rx) =
+            AudioGraphOwner::start(None, spec, health.clone())
+                .await
+                .context("failed to start the bare connection-owned capture sink")?;
         let identity = graph_owner.identity().clone();
         let (status_tx, status_rx) = mpsc::unbounded_channel();
         let status_forwarder = tokio::spawn(forward_audio_exclusion_status(status_rx));
@@ -167,7 +173,7 @@ impl BareCaptureSink {
                 AecConfig::Off,
                 status_tx,
             )),
-            health,
+            health.clone(),
         )
         .context("failed to start the desktop-excluding fan-out observer")
         {
@@ -184,7 +190,17 @@ impl BareCaptureSink {
             graph_owner.shutdown().await;
             return Err(error).context("desktop-excluding fan-out did not become ready");
         }
+        let (replacement_stop, replacement_stop_rx) = oneshot::channel();
+        let replacement_forwarder = tokio::spawn(forward_capture_sink_replacements(
+            identity_rx,
+            fanout.fanout_control(),
+            identity.serial,
+            replacement_stop_rx,
+            health,
+        ));
         Ok(Self {
+            replacement_stop: Some(replacement_stop),
+            replacement_forwarder: Some(replacement_forwarder),
             fanout: Some(fanout),
             status_forwarder: Some(status_forwarder),
             graph_owner: Some(graph_owner),
@@ -204,6 +220,14 @@ impl BareCaptureSink {
     }
 
     pub(super) async fn shutdown(mut self) {
+        if let Some(stop) = self.replacement_stop.take() {
+            let _ = stop.send(());
+        }
+        if let Some(replacement_forwarder) = self.replacement_forwarder.take()
+            && let Err(error) = replacement_forwarder.await
+        {
+            tracing::warn!(%error, "audio fan-out: replacement forwarder task failed");
+        }
         // Joining the observer closes its PipeWire connection and drops every
         // retained non-lingering link before the capture sink can disappear.
         if let Some(fanout) = self.fanout.take() {
@@ -222,6 +246,39 @@ impl BareCaptureSink {
     }
 }
 
+async fn forward_capture_sink_replacements(
+    mut identity_rx: mpsc::UnboundedReceiver<SinkIdentity>,
+    fanout: FanoutControl,
+    mut current_serial: u64,
+    mut stop_rx: oneshot::Receiver<()>,
+    health: health::Reporter,
+) {
+    loop {
+        tokio::select! {
+            biased;
+            _ = &mut stop_rx => break,
+            identity = identity_rx.recv() => {
+                let Some(identity) = identity else {
+                    break;
+                };
+                if identity.serial == current_serial {
+                    continue;
+                }
+                current_serial = identity.serial;
+                tracing::info!(
+                    global_id = identity.global_id,
+                    serial = identity.serial,
+                    "audio fan-out: retargeting to recreated capture sink"
+                );
+                if !fanout.replace_capture_sink(Serial(identity.serial)) {
+                    health.poison("audio fan-out observer rejected capture-sink replacement");
+                    break;
+                }
+            }
+        }
+    }
+}
+
 async fn forward_audio_exclusion_status(
     mut status_rx: mpsc::UnboundedReceiver<AudioExclusionEvent>,
 ) {
@@ -263,9 +320,14 @@ impl AudioGraphOwner {
         filter_name: Option<String>,
         spec: CaptureSinkSpec,
         health: health::Reporter,
-    ) -> Result<(Self, mpsc::UnboundedReceiver<GraphEvent>)> {
+    ) -> Result<(
+        Self,
+        mpsc::UnboundedReceiver<GraphEvent>,
+        mpsc::UnboundedReceiver<SinkIdentity>,
+    )> {
         let (cmd_tx, cmd_rx) = pw::channel::channel::<Cmd>();
         let (event_tx, event_rx) = mpsc::unbounded_channel::<GraphEvent>();
+        let (identity_tx, identity_rx) = mpsc::unbounded_channel::<SinkIdentity>();
         let (ready_tx, ready_rx) = oneshot::channel::<SinkIdentity>();
         let phase = Arc::new(AtomicU8::new(GRAPH_STARTING));
         let phase_for_thread = Arc::clone(&phase);
@@ -273,6 +335,7 @@ impl AudioGraphOwner {
         let shutdown_for_thread = Arc::clone(&shutdown_observed);
         let health_for_thread = health.clone();
         let spec_for_thread = spec.clone();
+        let cmd_tx_for_thread = cmd_tx.clone();
 
         let thread = std::thread::Builder::new()
             .name("pixelpass-audio-graph".to_string())
@@ -282,7 +345,9 @@ impl AudioGraphOwner {
                         filter_name,
                         spec_for_thread,
                         cmd_rx,
+                        cmd_tx_for_thread,
                         event_tx,
+                        identity_tx,
                         ready_tx,
                         Arc::clone(&phase_for_thread),
                         Arc::clone(&shutdown_for_thread),
@@ -350,7 +415,7 @@ impl AudioGraphOwner {
             monitor = %identity.monitor_name,
             "audio graph: connection-owned capture sink ready"
         );
-        Ok((owner, event_rx))
+        Ok((owner, event_rx, identity_rx))
     }
 
     pub(super) fn identity(&self) -> &SinkIdentity {
@@ -503,7 +568,10 @@ struct ActorState {
     filter_lower: Option<String>,
     readiness: NativeReadiness,
     ready_tx: Option<oneshot::Sender<SinkIdentity>>,
+    identity_tx: Option<mpsc::UnboundedSender<SinkIdentity>>,
     identity: Option<SinkIdentity>,
+    replacement_pending: bool,
+    replacement_prior_serial: Option<u64>,
     epoch: u64,
     pending: BTreeMap<u32, ObservedNode>,
     routed: BTreeMap<u32, RouteRecord>,
@@ -525,6 +593,10 @@ impl ActorState {
             return;
         };
         self.identity = Some(identity.clone());
+        self.replacement_pending = false;
+        if let Some(tx) = &self.identity_tx {
+            let _ = tx.send(identity.clone());
+        }
         if let Some(tx) = self.ready_tx.take() {
             let _ = tx.send(identity);
         }
@@ -573,7 +645,21 @@ impl ActorState {
         {
             let _ = tx.send(GraphEvent::LastRoutedStreamGone);
         }
-        self.readiness.bound_global_id == Some(id)
+        self.identity
+            .as_ref()
+            .is_some_and(|identity| identity.global_id == id)
+            || self.readiness.bound_global_id == Some(id)
+    }
+
+    fn begin_sink_replacement(&mut self) -> bool {
+        if self.closing || self.replacement_pending {
+            return false;
+        }
+        self.replacement_prior_serial = self.identity.take().map(|identity| identity.serial);
+        self.readiness.bound_global_id = None;
+        self.readiness.core_synced = false;
+        self.replacement_pending = true;
+        true
     }
 
     fn update_metadata(
@@ -669,6 +755,69 @@ fn try_flush(state: &Rc<RefCell<ActorState>>) {
     }
 }
 
+/// Move routes that still point at the removed owned serial directly to the
+/// replacement serial. A route changed by another actor during the gap is left
+/// alone, preserving the same compare-before-restore ownership rule used at
+/// shutdown. Bare Phase-6 capture has no routed records, but keeping the graph
+/// actor internally coherent avoids making sink recovery mode-dependent.
+fn retarget_routed_after_replacement(state: &Rc<RefCell<ActorState>>) {
+    let (metadata, actions, replacement_target) = {
+        let mut state = state.borrow_mut();
+        let Some(prior_serial) = state.replacement_prior_serial else {
+            return;
+        };
+        let Some(identity) = state.identity.as_ref() else {
+            return;
+        };
+        let replacement_target = identity.serial.to_string();
+        let prior_target = prior_serial.to_string();
+        let Some(metadata) = state.default_metadata.clone() else {
+            if state.routed.is_empty() {
+                state.replacement_prior_serial = None;
+            }
+            return;
+        };
+        let actions: Vec<u32> = state
+            .routed
+            .values()
+            .filter(|record| {
+                observation_is_current(&state.readiness.globals, &record.observed)
+                    && state
+                        .current_targets
+                        .get(&record.observed.global_id)
+                        .and_then(|target| target.value.as_deref())
+                        == Some(prior_target.as_str())
+            })
+            .map(|record| record.observed.global_id)
+            .collect();
+        for id in &actions {
+            state.current_targets.insert(
+                *id,
+                MetadataValue {
+                    type_: Some("Spa:Id".to_string()),
+                    value: Some(replacement_target.clone()),
+                },
+            );
+        }
+        state.replacement_prior_serial = None;
+        (metadata, actions, replacement_target)
+    };
+
+    for id in actions {
+        metadata.metadata.set_property(
+            id,
+            "target.object",
+            Some("Spa:Id"),
+            Some(&replacement_target),
+        );
+        tracing::info!(
+            node_id = id,
+            sink_serial = %replacement_target,
+            "audio graph: stream retargeted to replacement sink"
+        );
+    }
+}
+
 fn restoration_for(
     record: &RouteRecord,
     current: Option<&MetadataValue>,
@@ -707,12 +856,83 @@ fn prepare_quiesce(
     (metadata, actions)
 }
 
+fn request_sink_replacement(
+    state: &Rc<RefCell<ActorState>>,
+    cmd_tx: &pw::channel::Sender<Cmd>,
+    health: &health::Reporter,
+    main_loop: &pw::main_loop::MainLoopRc,
+) {
+    if !state.borrow_mut().begin_sink_replacement() {
+        return;
+    }
+    if cmd_tx.send(Cmd::RecreateSink).is_err() {
+        health.poison("audio graph could not enqueue capture-sink recreation");
+        main_loop.quit();
+    }
+}
+
+#[allow(clippy::too_many_arguments)]
+fn create_owned_sink(
+    core: &pw::core::CoreRc,
+    spec: &CaptureSinkSpec,
+    state: &Rc<RefCell<ActorState>>,
+    cmd_tx: &pw::channel::Sender<Cmd>,
+    health: &health::Reporter,
+    main_loop: &pw::main_loop::MainLoopRc,
+    shutdown_observed: &Arc<AtomicBool>,
+) -> Result<OwnedSink> {
+    let sink = core
+        .create_object::<pw::node::Node>("adapter", &spec.properties())
+        .context("could not create the connection-owned capture sink")?;
+    let state_for_bound = Rc::clone(state);
+    let state_for_proxy_remove = Rc::clone(state);
+    let cmd_for_proxy_remove = cmd_tx.clone();
+    let main_loop_for_proxy_remove = main_loop.clone();
+    let health_for_proxy_remove = health.clone();
+    let shutdown_for_proxy_remove = Arc::clone(shutdown_observed);
+    let main_loop_for_proxy_error = main_loop.clone();
+    let health_for_proxy_error = health.clone();
+    let shutdown_for_proxy_error = Arc::clone(shutdown_observed);
+    let listener = sink
+        .upcast_ref()
+        .add_listener_local()
+        .bound(move |global_id| {
+            state_for_bound.borrow_mut().observe_bound(global_id);
+        })
+        .removed(move || {
+            if !shutdown_for_proxy_remove.load(Ordering::Acquire) {
+                request_sink_replacement(
+                    &state_for_proxy_remove,
+                    &cmd_for_proxy_remove,
+                    &health_for_proxy_remove,
+                    &main_loop_for_proxy_remove,
+                );
+            }
+        })
+        .error(move |seq, res, message| {
+            tracing::warn!(seq, result = res, %message, "audio graph: capture-sink proxy error");
+            if !shutdown_for_proxy_error.load(Ordering::Acquire) {
+                health_for_proxy_error.poison(format!(
+                    "connection-owned capture-sink proxy error {res}: {message}"
+                ));
+                main_loop_for_proxy_error.quit();
+            }
+        })
+        .register();
+    Ok(OwnedSink {
+        _listener: listener,
+        _proxy: sink,
+    })
+}
+
 #[allow(clippy::too_many_arguments)]
 fn run_graph(
     filter_name: Option<String>,
     spec: CaptureSinkSpec,
     cmd_rx: pw::channel::Receiver<Cmd>,
+    cmd_tx: pw::channel::Sender<Cmd>,
     event_tx: mpsc::UnboundedSender<GraphEvent>,
+    identity_tx: mpsc::UnboundedSender<SinkIdentity>,
     ready_tx: oneshot::Sender<SinkIdentity>,
     phase: Arc<AtomicU8>,
     shutdown_observed: Arc<AtomicBool>,
@@ -732,7 +952,10 @@ fn run_graph(
         filter_lower: filter_name.map(|name| name.to_ascii_lowercase()),
         readiness: NativeReadiness::default(),
         ready_tx: Some(ready_tx),
+        identity_tx: Some(identity_tx),
         identity: None,
+        replacement_pending: false,
+        replacement_prior_serial: None,
         epoch: 0,
         pending: BTreeMap::new(),
         routed: BTreeMap::new(),
@@ -744,10 +967,15 @@ fn run_graph(
         stop_sync: None,
         closing: false,
     }));
+    let owned_sink = Rc::new(RefCell::new(None::<OwnedSink>));
 
     let main_loop_for_cmd = main_loop.clone();
     let core_for_cmd = core.clone();
     let state_for_cmd = Rc::clone(&state);
+    let sink_for_cmd = Rc::clone(&owned_sink);
+    let spec_for_cmd = spec.clone();
+    let cmd_tx_for_cmd = cmd_tx.clone();
+    let health_for_cmd = health.clone();
     let shutdown_for_cmd = Arc::clone(&shutdown_observed);
     let _cmd_receiver = cmd_rx.attach(main_loop.loop_(), move |cmd| match cmd {
         Cmd::Quiesce { ack } => {
@@ -796,6 +1024,41 @@ fn run_graph(
                 }
             }
         }
+        Cmd::RecreateSink => {
+            if shutdown_for_cmd.load(Ordering::Acquire) || state_for_cmd.borrow().closing {
+                return;
+            }
+            // The old proxy is already server-removed. Drop its callbacks
+            // before installing the next generation so no stale listener can
+            // affect the replacement's identity handshake.
+            drop(sink_for_cmd.borrow_mut().take());
+            match create_owned_sink(
+                &core_for_cmd,
+                &spec_for_cmd,
+                &state_for_cmd,
+                &cmd_tx_for_cmd,
+                &health_for_cmd,
+                &main_loop_for_cmd,
+                &shutdown_for_cmd,
+            ) {
+                Ok(sink) => {
+                    *sink_for_cmd.borrow_mut() = Some(sink);
+                    match core_for_cmd.sync(0) {
+                        Ok(seq) => state_for_cmd.borrow_mut().initial_sync = Some(seq),
+                        Err(error) => {
+                            health_for_cmd.poison(format!(
+                                "capture-sink replacement core.sync failed: {error}"
+                            ));
+                            main_loop_for_cmd.quit();
+                        }
+                    }
+                }
+                Err(error) => {
+                    health_for_cmd.poison(format!("capture-sink recreation failed: {error:#}"));
+                    main_loop_for_cmd.quit();
+                }
+            }
+        }
     });
 
     let state_for_done = Rc::clone(&state);
@@ -834,6 +1097,7 @@ fn run_graph(
                 // Existing matching streams may have been observed before the
                 // initial round-trip. Readiness becoming true is itself the
                 // event that makes those observations routable.
+                retarget_routed_after_replacement(&state_for_done);
                 try_flush(&state_for_done);
             }
             if stop_completed {
@@ -855,6 +1119,7 @@ fn run_graph(
     let state_for_remove = Rc::clone(&state);
     let main_loop_for_remove = main_loop.clone();
     let health_for_remove = health.clone();
+    let cmd_tx_for_remove = cmd_tx.clone();
     let shutdown_for_remove = Arc::clone(&shutdown_observed);
     let _registry_listener = registry
         .add_listener_local()
@@ -927,48 +1192,25 @@ fn run_graph(
         .global_remove(move |id| {
             let own_sink_removed = state_for_remove.borrow_mut().remove_global(id);
             if own_sink_removed && !shutdown_for_remove.load(Ordering::Acquire) {
-                health_for_remove.poison("connection-owned capture sink disappeared unexpectedly");
-                main_loop_for_remove.quit();
+                request_sink_replacement(
+                    &state_for_remove,
+                    &cmd_tx_for_remove,
+                    &health_for_remove,
+                    &main_loop_for_remove,
+                );
             }
         })
         .register();
 
-    let sink = core
-        .create_object::<pw::node::Node>("adapter", &spec.properties())
-        .context("could not create the connection-owned capture sink")?;
-    let state_for_bound = Rc::clone(&state);
-    let main_loop_for_proxy_remove = main_loop.clone();
-    let health_for_proxy_remove = health.clone();
-    let shutdown_for_proxy_remove = Arc::clone(&shutdown_observed);
-    let main_loop_for_proxy_error = main_loop.clone();
-    let health_for_proxy_error = health.clone();
-    let shutdown_for_proxy_error = Arc::clone(&shutdown_observed);
-    let listener = sink
-        .upcast_ref()
-        .add_listener_local()
-        .bound(move |global_id| {
-            state_for_bound.borrow_mut().observe_bound(global_id);
-        })
-        .removed(move || {
-            if !shutdown_for_proxy_remove.load(Ordering::Acquire) {
-                health_for_proxy_remove.poison("connection-owned capture-sink proxy was removed");
-                main_loop_for_proxy_remove.quit();
-            }
-        })
-        .error(move |seq, res, message| {
-            tracing::warn!(seq, result = res, %message, "audio graph: capture-sink proxy error");
-            if !shutdown_for_proxy_error.load(Ordering::Acquire) {
-                health_for_proxy_error.poison(format!(
-                    "connection-owned capture-sink proxy error {res}: {message}"
-                ));
-                main_loop_for_proxy_error.quit();
-            }
-        })
-        .register();
-    let _owned_sink = OwnedSink {
-        _listener: listener,
-        _proxy: sink,
-    };
+    *owned_sink.borrow_mut() = Some(create_owned_sink(
+        &core,
+        &spec,
+        &state,
+        &cmd_tx,
+        &health,
+        &main_loop,
+        &shutdown_observed,
+    )?);
 
     let initial_sync = core
         .sync(0)
@@ -1508,9 +1750,10 @@ mod tests {
             "live fixture name is already present"
         );
         let (health, _) = health::channel();
-        let (mut owner, _events) = AudioGraphOwner::start(None, spec.clone(), health.clone())
-            .await
-            .expect("connection-owned sink starts");
+        let (mut owner, _events, _identities) =
+            AudioGraphOwner::start(None, spec.clone(), health.clone())
+                .await
+                .expect("connection-owned sink starts");
         assert_ne!(owner.identity().global_id, u32::MAX);
         assert!(owner.identity().serial > 0);
         assert!(pulse_source_exists(spec.monitor_name()).await);
@@ -1532,4 +1775,45 @@ mod tests {
         );
         assert!(health.fault().is_none());
     }
+
+    #[tokio::test]
+    #[ignore = "live: destroys and recreates a real connection-owned PipeWire sink; serialize live audio tests"]
+    async fn live_removed_sink_is_recreated_with_a_fresh_identity() {
+        let spec = CaptureSinkSpec::for_pid(std::process::id());
+        assert!(!pulse_source_exists(spec.monitor_name()).await);
+        let (health, _) = health::channel();
+        let (owner, _events, mut identities) =
+            AudioGraphOwner::start(None, spec.clone(), health.clone())
+                .await
+                .expect("connection-owned sink starts");
+        let initial = owner.identity().clone();
+        let announced_initial = tokio::time::timeout(Duration::from_secs(1), identities.recv())
+            .await
+            .expect("initial identity announcement timed out")
+            .expect("identity channel closed");
+        assert_eq!(announced_initial, initial);
+
+        let destroy = tokio::process::Command::new("pw-cli")
+            .args(["destroy", &initial.global_id.to_string()])
+            .status()
+            .await
+            .expect("run pw-cli destroy");
+        assert!(destroy.success());
+        let replacement = tokio::time::timeout(Duration::from_secs(5), identities.recv())
+            .await
+            .expect("replacement identity announcement timed out")
+            .expect("identity channel closed during replacement");
+        assert_ne!(replacement.serial, initial.serial);
+        assert_eq!(replacement.name, initial.name);
+        assert_eq!(replacement.monitor_name, initial.monitor_name);
+        assert!(pulse_source_exists(spec.monitor_name()).await);
+        assert!(health.fault().is_none());
+
+        assert!(owner.shutdown().await);
+        let deadline = Instant::now() + Duration::from_secs(2);
+        while pulse_source_exists(spec.monitor_name()).await && Instant::now() < deadline {
+            tokio::time::sleep(Duration::from_millis(20)).await;
+        }
+        assert!(!pulse_source_exists(spec.monitor_name()).await);
+    }
 }