fix(audio): make module teardown cancellation-safe
This commit is contained in:
+553
-94
@@ -33,12 +33,13 @@
|
||||
//! # Why the permit is affine
|
||||
//!
|
||||
//! [`LoadPermit`] is not `Clone`, is consumed by value to settle, and its [`Drop`]
|
||||
//! marks the slot [`Reconcile::Load`] when it was never settled. That is what makes
|
||||
//! the guarantee structural rather than a discipline: a cancelled task drops its
|
||||
//! locals, so an aborted load *cannot* silently forget a module the server may
|
||||
//! already have created. Two permitted loads for one slot cannot both commit,
|
||||
//! because [`ModuleLedger::begin_load`] issues a permit only for a `Vacant` slot
|
||||
//! and every other state — including the ambiguous one — refuses.
|
||||
//! marks the slot [`Reconcile::Load`] when it was never settled. The permit moves
|
||||
//! into a registered blocking worker before any await can detach that work; either
|
||||
//! the worker settles it, or dropping the worker marks the question ambiguous.
|
||||
//! Teardown waits for all such permits before reconciling. Two permitted loads for
|
||||
//! one slot cannot both commit, because [`ModuleLedger::begin_load`] issues a permit
|
||||
//! only for a `Vacant` slot and every other state — including the ambiguous one —
|
||||
//! refuses.
|
||||
//!
|
||||
//! # Why an ambiguous slot blocks the next load
|
||||
//!
|
||||
@@ -50,8 +51,8 @@
|
||||
//! proceed until the question is answered is the whole point.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
|
||||
use std::sync::{Arc, Condvar, Mutex};
|
||||
|
||||
use anyhow::{Context as _, Result};
|
||||
|
||||
@@ -99,7 +100,7 @@ pub enum SlotState {
|
||||
Loaded { fp: Fingerprint },
|
||||
/// An unload is in flight. The fingerprint is retained deliberately: an unload
|
||||
/// that times out must not leave the module unrecorded.
|
||||
Unloading { fp: Fingerprint },
|
||||
Unloading { fp: Fingerprint, permit: u64 },
|
||||
/// The slot's real state is unknown and must be resolved against the server.
|
||||
Ambiguous(Reconcile),
|
||||
/// Resolution found something we refuse to act on. Terminal.
|
||||
@@ -129,6 +130,20 @@ pub enum LedgerError {
|
||||
Poisoned { shape: Shape, reason: String },
|
||||
/// `pactl` reported `PA_INVALID_INDEX` where an index was expected.
|
||||
InvalidIndex { shape: Shape },
|
||||
/// Teardown has begun, so event-driven work may no longer start.
|
||||
Closed { shape: Shape },
|
||||
/// The two inverse loopbacks must never coexist.
|
||||
Incompatible {
|
||||
shape: Shape,
|
||||
occupied: Shape,
|
||||
state: &'static str,
|
||||
},
|
||||
/// The capture sink cannot be removed while a loopback may still reference it.
|
||||
Referenced {
|
||||
shape: Shape,
|
||||
dependency: Shape,
|
||||
state: &'static str,
|
||||
},
|
||||
}
|
||||
|
||||
impl std::fmt::Display for LedgerError {
|
||||
@@ -145,6 +160,31 @@ impl std::fmt::Display for LedgerError {
|
||||
"pactl reported PA_INVALID_INDEX for the {} module",
|
||||
shape.label()
|
||||
),
|
||||
LedgerError::Closed { shape } => write!(
|
||||
f,
|
||||
"the module ledger is closing; refusing new work on the {} slot",
|
||||
shape.label()
|
||||
),
|
||||
LedgerError::Incompatible {
|
||||
shape,
|
||||
occupied,
|
||||
state,
|
||||
} => write!(
|
||||
f,
|
||||
"cannot load the {} while the incompatible {} slot is {state}",
|
||||
shape.label(),
|
||||
occupied.label()
|
||||
),
|
||||
LedgerError::Referenced {
|
||||
shape,
|
||||
dependency,
|
||||
state,
|
||||
} => write!(
|
||||
f,
|
||||
"cannot unload the {} while the dependent {} slot is {state}",
|
||||
shape.label(),
|
||||
dependency.label()
|
||||
),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -180,6 +220,59 @@ pub enum Resolution {
|
||||
pub struct ModuleLedger {
|
||||
slots: Mutex<BTreeMap<Shape, SlotState>>,
|
||||
next_permit: AtomicU64,
|
||||
closed: AtomicBool,
|
||||
operations: OperationCoordinator,
|
||||
}
|
||||
|
||||
/// Registers module mutations before they can be detached from their caller.
|
||||
///
|
||||
/// `spawn_blocking` work continues when the awaiting future is cancelled. The
|
||||
/// count is therefore registered synchronously, while the slot lock is held, and
|
||||
/// is released only by the affine permit's `Drop`. Teardown closes the ledger and
|
||||
/// waits on this condition variable before it asks the server what exists.
|
||||
struct OperationCoordinator {
|
||||
active: Mutex<usize>,
|
||||
idle: Condvar,
|
||||
server: Mutex<()>,
|
||||
}
|
||||
|
||||
impl OperationCoordinator {
|
||||
fn new() -> Self {
|
||||
Self {
|
||||
active: Mutex::new(0),
|
||||
idle: Condvar::new(),
|
||||
server: Mutex::new(()),
|
||||
}
|
||||
}
|
||||
|
||||
fn register(&self) {
|
||||
let mut active = self.active.lock().unwrap_or_else(|e| e.into_inner());
|
||||
*active = active
|
||||
.checked_add(1)
|
||||
.expect("module operation count overflow");
|
||||
}
|
||||
|
||||
fn finish(&self) {
|
||||
let mut active = self.active.lock().unwrap_or_else(|e| e.into_inner());
|
||||
*active = active
|
||||
.checked_sub(1)
|
||||
.expect("module operation count underflow");
|
||||
if *active == 0 {
|
||||
self.idle.notify_all();
|
||||
}
|
||||
}
|
||||
|
||||
fn wait_idle(&self) {
|
||||
let mut active = self.active.lock().unwrap_or_else(|e| e.into_inner());
|
||||
while *active != 0 {
|
||||
active = self.idle.wait(active).unwrap_or_else(|e| e.into_inner());
|
||||
}
|
||||
}
|
||||
|
||||
fn run<T>(&self, operation: impl FnOnce() -> T) -> T {
|
||||
let _serial = self.server.lock().unwrap_or_else(|e| e.into_inner());
|
||||
operation()
|
||||
}
|
||||
}
|
||||
|
||||
impl ModuleLedger {
|
||||
@@ -187,9 +280,36 @@ impl ModuleLedger {
|
||||
Arc::new(Self {
|
||||
slots: Mutex::new(BTreeMap::new()),
|
||||
next_permit: AtomicU64::new(1),
|
||||
closed: AtomicBool::new(false),
|
||||
operations: OperationCoordinator::new(),
|
||||
})
|
||||
}
|
||||
|
||||
/// Stop event-driven mutations and wait until every already-registered load
|
||||
/// or unload has settled its affine permit.
|
||||
///
|
||||
/// The slots-lock hand-off closes the only race that matters: an operation
|
||||
/// that saw `closed == false` has registered itself before this method can
|
||||
/// pass the hand-off and begin waiting.
|
||||
pub(super) fn close(&self) {
|
||||
self.closed.store(true, Ordering::SeqCst);
|
||||
drop(self.slots.lock().unwrap_or_else(|e| e.into_inner()));
|
||||
}
|
||||
|
||||
pub(super) fn wait_for_operations(&self) {
|
||||
self.operations.wait_idle();
|
||||
}
|
||||
|
||||
pub(super) fn close_and_wait(&self) {
|
||||
self.close();
|
||||
self.wait_for_operations();
|
||||
}
|
||||
|
||||
/// Serialize one server mutation or observation with every other such action.
|
||||
pub(super) fn with_server_operation<T>(&self, operation: impl FnOnce() -> T) -> T {
|
||||
self.operations.run(operation)
|
||||
}
|
||||
|
||||
/// The current state of one slot. Absent keys read as [`SlotState::Vacant`].
|
||||
///
|
||||
/// Test-only: production code never needs to look a slot up, because every
|
||||
@@ -219,6 +339,9 @@ impl ModuleLedger {
|
||||
token: OwnerToken,
|
||||
) -> Result<LoadPermit, LedgerError> {
|
||||
let mut slots = self.slots.lock().unwrap();
|
||||
if self.closed.load(Ordering::SeqCst) {
|
||||
return Err(LedgerError::Closed { shape });
|
||||
}
|
||||
let current = slots.get(&shape).cloned().unwrap_or(SlotState::Vacant);
|
||||
match current {
|
||||
SlotState::Vacant => {}
|
||||
@@ -232,7 +355,18 @@ impl ModuleLedger {
|
||||
});
|
||||
}
|
||||
}
|
||||
if let Some(occupied) = inverse_loopback(shape) {
|
||||
let state = slots.get(&occupied).cloned().unwrap_or(SlotState::Vacant);
|
||||
if !matches!(state, SlotState::Vacant) {
|
||||
return Err(LedgerError::Incompatible {
|
||||
shape,
|
||||
occupied,
|
||||
state: state.label(),
|
||||
});
|
||||
}
|
||||
}
|
||||
let permit = self.next_permit.fetch_add(1, Ordering::Relaxed);
|
||||
self.operations.register();
|
||||
slots.insert(
|
||||
shape,
|
||||
SlotState::Loading {
|
||||
@@ -253,36 +387,76 @@ impl ModuleLedger {
|
||||
|
||||
/// Move a `Loaded` slot to `Unloading` and hand back what to unload.
|
||||
///
|
||||
/// `None` for any other state: there is nothing to unload, or the slot is not
|
||||
/// in a condition to be acted on.
|
||||
pub fn begin_unload(&self, shape: Shape) -> Option<Fingerprint> {
|
||||
let mut slots = self.slots.lock().unwrap();
|
||||
let SlotState::Loaded { fp } = slots.get(&shape).cloned()? else {
|
||||
return None;
|
||||
};
|
||||
slots.insert(shape, SlotState::Unloading { fp: fp.clone() });
|
||||
Some(fp)
|
||||
/// `Ok(None)` means confirmed vacancy. Busy, poisoned, closed, and still-
|
||||
/// referenced states are errors so callers cannot mistake uncertainty for
|
||||
/// absence and load an inverse loopback or destroy the capture sink.
|
||||
pub fn begin_unload(
|
||||
self: &Arc<Self>,
|
||||
shape: Shape,
|
||||
) -> Result<Option<UnloadPermit>, LedgerError> {
|
||||
self.begin_unload_inner(shape, false)
|
||||
}
|
||||
|
||||
/// Record how an unload turned out. An uncertain one becomes ambiguous rather
|
||||
/// than being assumed done — the module is not forgotten either way.
|
||||
pub fn finish_unload(&self, shape: Shape, outcome: UnloadOutcome) {
|
||||
/// Teardown-only form of [`ModuleLedger::begin_unload`]. New event work is
|
||||
/// closed by then, but cleanup must still be able to remove tracked modules.
|
||||
pub(super) fn begin_cleanup_unload(
|
||||
self: &Arc<Self>,
|
||||
shape: Shape,
|
||||
) -> Result<Option<UnloadPermit>, LedgerError> {
|
||||
self.begin_unload_inner(shape, true)
|
||||
}
|
||||
|
||||
fn begin_unload_inner(
|
||||
self: &Arc<Self>,
|
||||
shape: Shape,
|
||||
cleanup: bool,
|
||||
) -> Result<Option<UnloadPermit>, LedgerError> {
|
||||
let mut slots = self.slots.lock().unwrap();
|
||||
let Some(SlotState::Unloading { fp }) = slots.get(&shape).cloned() else {
|
||||
return;
|
||||
};
|
||||
let next = match outcome {
|
||||
UnloadOutcome::Confirmed => SlotState::Vacant,
|
||||
UnloadOutcome::Uncertain(why) => {
|
||||
tracing::warn!(
|
||||
shape = fp.shape.label(),
|
||||
module = fp.id,
|
||||
"audio ledger: unload outcome uncertain ({why}); slot needs reconciling"
|
||||
);
|
||||
SlotState::Ambiguous(Reconcile::Unload { fp })
|
||||
if !cleanup && self.closed.load(Ordering::SeqCst) {
|
||||
return Err(LedgerError::Closed { shape });
|
||||
}
|
||||
let current = slots.get(&shape).cloned().unwrap_or(SlotState::Vacant);
|
||||
let fp = match current {
|
||||
SlotState::Vacant => return Ok(None),
|
||||
SlotState::Loaded { fp } => fp,
|
||||
SlotState::Poisoned { reason } => {
|
||||
return Err(LedgerError::Poisoned { shape, reason });
|
||||
}
|
||||
other => {
|
||||
return Err(LedgerError::Busy {
|
||||
shape,
|
||||
state: other.label(),
|
||||
});
|
||||
}
|
||||
};
|
||||
slots.insert(shape, next);
|
||||
if shape == Shape::LegacyCaptureSink {
|
||||
for dependency in [Shape::LoopbackIntoCapture, Shape::LoopbackOutOfCapture] {
|
||||
let state = slots.get(&dependency).cloned().unwrap_or(SlotState::Vacant);
|
||||
if !matches!(state, SlotState::Vacant) {
|
||||
return Err(LedgerError::Referenced {
|
||||
shape,
|
||||
dependency,
|
||||
state: state.label(),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
let permit = self.next_permit.fetch_add(1, Ordering::Relaxed);
|
||||
self.operations.register();
|
||||
slots.insert(
|
||||
shape,
|
||||
SlotState::Unloading {
|
||||
fp: fp.clone(),
|
||||
permit,
|
||||
},
|
||||
);
|
||||
Ok(Some(UnloadPermit {
|
||||
ledger: Arc::clone(self),
|
||||
shape,
|
||||
fp,
|
||||
permit,
|
||||
settled: false,
|
||||
}))
|
||||
}
|
||||
|
||||
/// Every slot currently holding a module, in [`Shape`] declaration order —
|
||||
@@ -313,14 +487,28 @@ impl ModuleLedger {
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Is every slot in a state we can explain? False while anything is ambiguous
|
||||
/// or poisoned.
|
||||
/// Is every slot in a state we can explain? False while an operation is in
|
||||
/// flight, its outcome is ambiguous, or a conflict poisoned the slot.
|
||||
pub fn is_settled(&self) -> bool {
|
||||
self.slots
|
||||
.lock()
|
||||
.unwrap()
|
||||
.values()
|
||||
.all(|state| !matches!(state, SlotState::Ambiguous(_) | SlotState::Poisoned { .. }))
|
||||
.all(|state| matches!(state, SlotState::Vacant | SlotState::Loaded { .. }))
|
||||
}
|
||||
|
||||
/// Has teardown removed every module rather than merely accounted for it?
|
||||
pub fn is_clean(&self) -> bool {
|
||||
self.slots
|
||||
.lock()
|
||||
.unwrap()
|
||||
.values()
|
||||
.all(|state| matches!(state, SlotState::Vacant))
|
||||
}
|
||||
|
||||
/// A stable snapshot used to stop cleanup when another pass made no progress.
|
||||
pub(super) fn snapshot(&self) -> BTreeMap<Shape, SlotState> {
|
||||
self.slots.lock().unwrap().clone()
|
||||
}
|
||||
|
||||
/// Apply a reconciliation result to an ambiguous slot.
|
||||
@@ -359,6 +547,14 @@ impl ModuleLedger {
|
||||
}
|
||||
}
|
||||
|
||||
fn inverse_loopback(shape: Shape) -> Option<Shape> {
|
||||
match shape {
|
||||
Shape::LoopbackIntoCapture => Some(Shape::LoopbackOutOfCapture),
|
||||
Shape::LoopbackOutOfCapture => Some(Shape::LoopbackIntoCapture),
|
||||
Shape::LegacyCaptureSink => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Permission to perform exactly one load, which must be settled by value.
|
||||
///
|
||||
/// Dropping it unsettled is not an error — it is the *reporting* path for a
|
||||
@@ -375,6 +571,12 @@ pub struct LoadPermit {
|
||||
}
|
||||
|
||||
impl LoadPermit {
|
||||
/// Run the server-facing part of this load in the same serialized domain as
|
||||
/// unload verification and reconciliation.
|
||||
pub fn with_server_operation<T>(&self, operation: impl FnOnce() -> T) -> T {
|
||||
self.ledger.with_server_operation(operation)
|
||||
}
|
||||
|
||||
/// Record that the server created the module at `index`.
|
||||
///
|
||||
/// Fails on [`PA_INVALID_INDEX`], leaving the permit unsettled so that
|
||||
@@ -427,29 +629,106 @@ impl LoadPermit {
|
||||
|
||||
impl Drop for LoadPermit {
|
||||
fn drop(&mut self) {
|
||||
if self.settled {
|
||||
if !self.settled {
|
||||
let mut slots = self.ledger.slots.lock().unwrap();
|
||||
// Only claim the slot if it is still *our* load. Anything else already
|
||||
// moved past this permit.
|
||||
if let Some(SlotState::Loading { permit, .. }) = slots.get(&self.shape)
|
||||
&& *permit == self.permit
|
||||
{
|
||||
tracing::warn!(
|
||||
shape = self.shape.label(),
|
||||
"audio ledger: a load was cancelled before its outcome was known; \
|
||||
the slot needs reconciling"
|
||||
);
|
||||
slots.insert(
|
||||
self.shape,
|
||||
SlotState::Ambiguous(Reconcile::Load {
|
||||
shape: self.shape,
|
||||
pid: self.pid,
|
||||
token: self.token.clone(),
|
||||
}),
|
||||
);
|
||||
}
|
||||
}
|
||||
self.ledger.operations.finish();
|
||||
}
|
||||
}
|
||||
|
||||
/// Permission to perform exactly one unload, which must be settled by value.
|
||||
///
|
||||
/// Like [`LoadPermit`], this is affine. Cancellation drops it unsettled, retaining
|
||||
/// the full fingerprint as a reconciliation question instead of leaving the slot
|
||||
/// stuck in an invisible `Unloading` state.
|
||||
#[must_use = "an unsettled permit marks the unload ambiguous when dropped"]
|
||||
pub struct UnloadPermit {
|
||||
ledger: Arc<ModuleLedger>,
|
||||
shape: Shape,
|
||||
fp: Fingerprint,
|
||||
permit: u64,
|
||||
settled: bool,
|
||||
}
|
||||
|
||||
impl UnloadPermit {
|
||||
pub fn fingerprint(&self) -> &Fingerprint {
|
||||
&self.fp
|
||||
}
|
||||
|
||||
pub fn with_server_operation<T>(&self, operation: impl FnOnce() -> T) -> T {
|
||||
self.ledger.with_server_operation(operation)
|
||||
}
|
||||
|
||||
/// Record how the server operation ended. An uncertain answer retains the
|
||||
/// fingerprint and makes the next action reconcile it before doing anything.
|
||||
pub fn finish(mut self, outcome: UnloadOutcome) {
|
||||
let mut slots = self.ledger.slots.lock().unwrap();
|
||||
let Some(SlotState::Unloading { fp, permit }) = slots.get(&self.shape).cloned() else {
|
||||
self.settled = true;
|
||||
return;
|
||||
};
|
||||
if permit != self.permit {
|
||||
self.settled = true;
|
||||
return;
|
||||
}
|
||||
let mut slots = self.ledger.slots.lock().unwrap();
|
||||
// Only claim the slot if it is still *our* load. Anything else already
|
||||
// moved past this permit.
|
||||
if let Some(SlotState::Loading { permit, .. }) = slots.get(&self.shape)
|
||||
&& *permit == self.permit
|
||||
{
|
||||
tracing::warn!(
|
||||
shape = self.shape.label(),
|
||||
"audio ledger: a load was cancelled before its outcome was known; \
|
||||
the slot needs reconciling"
|
||||
);
|
||||
slots.insert(
|
||||
self.shape,
|
||||
SlotState::Ambiguous(Reconcile::Load {
|
||||
shape: self.shape,
|
||||
pid: self.pid,
|
||||
token: self.token.clone(),
|
||||
}),
|
||||
);
|
||||
let next = match outcome {
|
||||
UnloadOutcome::Confirmed => SlotState::Vacant,
|
||||
UnloadOutcome::Uncertain(why) => {
|
||||
tracing::warn!(
|
||||
shape = fp.shape.label(),
|
||||
module = fp.id,
|
||||
"audio ledger: unload outcome uncertain ({why}); slot needs reconciling"
|
||||
);
|
||||
SlotState::Ambiguous(Reconcile::Unload { fp })
|
||||
}
|
||||
};
|
||||
slots.insert(self.shape, next);
|
||||
drop(slots);
|
||||
self.settled = true;
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for UnloadPermit {
|
||||
fn drop(&mut self) {
|
||||
if !self.settled {
|
||||
let mut slots = self.ledger.slots.lock().unwrap();
|
||||
if let Some(SlotState::Unloading { permit, .. }) = slots.get(&self.shape)
|
||||
&& *permit == self.permit
|
||||
{
|
||||
tracing::warn!(
|
||||
shape = self.shape.label(),
|
||||
module = self.fp.id,
|
||||
"audio ledger: an unload was cancelled before its outcome was known; \
|
||||
the slot needs reconciling"
|
||||
);
|
||||
slots.insert(
|
||||
self.shape,
|
||||
SlotState::Ambiguous(Reconcile::Unload {
|
||||
fp: self.fp.clone(),
|
||||
}),
|
||||
);
|
||||
}
|
||||
}
|
||||
self.ledger.operations.finish();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -509,26 +788,35 @@ pub fn resolve(reconcile: &Reconcile, observations: &[ModuleObservation]) -> Res
|
||||
/// the life of a share. Reconciliation is rare and off the hot path, so paying a
|
||||
/// connection for it costs nothing that matters.
|
||||
pub async fn reconcile_pending(ledger: &Arc<ModuleLedger>) -> Result<()> {
|
||||
let pending = ledger.pending();
|
||||
if pending.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
tracing::info!(
|
||||
n = pending.len(),
|
||||
"audio ledger: reconciling unresolved module slots against the server"
|
||||
);
|
||||
let observations = tokio::task::spawn_blocking(|| -> Result<Vec<ModuleObservation>> {
|
||||
let mut session = PulseSession::connect()?;
|
||||
session.list_modules()
|
||||
})
|
||||
.await
|
||||
.context("the Pulse listing task failed to run")?
|
||||
.context("could not list Pulse modules to reconcile the audio ledger")?;
|
||||
let ledger = Arc::clone(ledger);
|
||||
tokio::task::spawn_blocking(move || reconcile_pending_blocking(&ledger))
|
||||
.await
|
||||
.context("the Pulse reconciliation task failed to run")?
|
||||
}
|
||||
|
||||
for (shape, reconcile) in pending {
|
||||
ledger.apply(shape, resolve(&reconcile, &observations));
|
||||
}
|
||||
Ok(())
|
||||
/// Synchronous reconciliation for [`Routing::drop`](crate::host::audio::Routing).
|
||||
/// The caller closes and quiesces the ledger first; serialization here also makes
|
||||
/// ordinary async reconciliation wait behind any server operation already running.
|
||||
pub(super) fn reconcile_pending_blocking(ledger: &Arc<ModuleLedger>) -> Result<()> {
|
||||
ledger.with_server_operation(|| {
|
||||
let pending = ledger.pending();
|
||||
if pending.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
tracing::info!(
|
||||
n = pending.len(),
|
||||
"audio ledger: reconciling unresolved module slots against the server"
|
||||
);
|
||||
let mut session = PulseSession::connect()?;
|
||||
let observations = session
|
||||
.list_modules()
|
||||
.context("could not list Pulse modules to reconcile the audio ledger")?;
|
||||
|
||||
for (shape, reconcile) in pending {
|
||||
ledger.apply(shape, resolve(&reconcile, &observations));
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -684,14 +972,12 @@ mod tests {
|
||||
.expect("a vacant slot issues a permit")
|
||||
.commit(3)
|
||||
.expect("3 is a real index");
|
||||
assert_eq!(
|
||||
ledger.begin_unload(Shape::LoopbackIntoCapture),
|
||||
Some(fp.clone())
|
||||
);
|
||||
ledger.finish_unload(
|
||||
Shape::LoopbackIntoCapture,
|
||||
UnloadOutcome::Uncertain("pactl was killed".to_string()),
|
||||
);
|
||||
let permit = ledger
|
||||
.begin_unload(Shape::LoopbackIntoCapture)
|
||||
.expect("the ledger accepts the unload")
|
||||
.expect("the loaded slot issues a permit");
|
||||
assert_eq!(permit.fingerprint(), &fp);
|
||||
permit.finish(UnloadOutcome::Uncertain("pactl was killed".to_string()));
|
||||
assert_eq!(
|
||||
ledger.state(Shape::LoopbackIntoCapture),
|
||||
SlotState::Ambiguous(Reconcile::Unload { fp }),
|
||||
@@ -707,23 +993,194 @@ mod tests {
|
||||
.expect("a vacant slot issues a permit")
|
||||
.commit(3)
|
||||
.expect("3 is a real index");
|
||||
ledger.begin_unload(Shape::LoopbackIntoCapture);
|
||||
ledger.finish_unload(Shape::LoopbackIntoCapture, UnloadOutcome::Confirmed);
|
||||
ledger
|
||||
.begin_unload(Shape::LoopbackIntoCapture)
|
||||
.expect("the ledger accepts the unload")
|
||||
.expect("the loaded slot issues a permit")
|
||||
.finish(UnloadOutcome::Confirmed);
|
||||
assert_eq!(ledger.state(Shape::LoopbackIntoCapture), SlotState::Vacant);
|
||||
assert!(ledger.is_settled());
|
||||
assert!(ledger.is_clean());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn begin_unload_only_acts_on_a_loaded_slot() {
|
||||
fn dropping_an_unload_permit_marks_the_slot_ambiguous() {
|
||||
let ledger = ModuleLedger::new();
|
||||
assert_eq!(ledger.begin_unload(Shape::LegacyCaptureSink), None);
|
||||
let fp = ledger
|
||||
.begin_load(Shape::LoopbackIntoCapture, 42, token(7))
|
||||
.expect("a vacant slot issues a permit")
|
||||
.commit(3)
|
||||
.expect("3 is a real index");
|
||||
let permit = ledger
|
||||
.begin_unload(Shape::LoopbackIntoCapture)
|
||||
.expect("the ledger accepts the unload")
|
||||
.expect("the loaded slot issues a permit");
|
||||
assert!(matches!(
|
||||
ledger.state(Shape::LoopbackIntoCapture),
|
||||
SlotState::Unloading { .. }
|
||||
));
|
||||
assert!(!ledger.is_settled(), "in-flight unloads are not settled");
|
||||
drop(permit);
|
||||
assert_eq!(
|
||||
ledger.state(Shape::LoopbackIntoCapture),
|
||||
SlotState::Ambiguous(Reconcile::Unload { fp })
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn inverse_loopbacks_cannot_coexist_or_cross_an_unknown_boundary() {
|
||||
let ledger = ModuleLedger::new();
|
||||
ledger
|
||||
.begin_load(Shape::LoopbackIntoCapture, 42, token(7))
|
||||
.expect("the first loopback may load")
|
||||
.commit(3)
|
||||
.expect("3 is a real index");
|
||||
assert_eq!(
|
||||
ledger
|
||||
.begin_load(Shape::LoopbackOutOfCapture, 42, token(8))
|
||||
.err(),
|
||||
Some(LedgerError::Incompatible {
|
||||
shape: Shape::LoopbackOutOfCapture,
|
||||
occupied: Shape::LoopbackIntoCapture,
|
||||
state: "loaded"
|
||||
})
|
||||
);
|
||||
|
||||
let unload = ledger
|
||||
.begin_unload(Shape::LoopbackIntoCapture)
|
||||
.expect("the ledger accepts the unload")
|
||||
.expect("the loaded slot issues a permit");
|
||||
drop(unload);
|
||||
assert_eq!(
|
||||
ledger
|
||||
.begin_load(Shape::LoopbackOutOfCapture, 42, token(9))
|
||||
.err(),
|
||||
Some(LedgerError::Incompatible {
|
||||
shape: Shape::LoopbackOutOfCapture,
|
||||
occupied: Shape::LoopbackIntoCapture,
|
||||
state: "ambiguous"
|
||||
}),
|
||||
"an unconfirmed absence must block the inverse loopback"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn capture_sink_unload_waits_for_every_dependent_slot_to_be_vacant() {
|
||||
let ledger = ModuleLedger::new();
|
||||
ledger
|
||||
.begin_load(Shape::LegacyCaptureSink, 42, token(1))
|
||||
.expect("the sink may load")
|
||||
.commit(1)
|
||||
.expect("1 is a real index");
|
||||
ledger
|
||||
.begin_load(Shape::LoopbackIntoCapture, 42, token(2))
|
||||
.expect("the mirror may load")
|
||||
.commit(2)
|
||||
.expect("2 is a real index");
|
||||
assert_eq!(
|
||||
ledger.begin_unload(Shape::LegacyCaptureSink).err(),
|
||||
Some(LedgerError::Referenced {
|
||||
shape: Shape::LegacyCaptureSink,
|
||||
dependency: Shape::LoopbackIntoCapture,
|
||||
state: "loaded"
|
||||
})
|
||||
);
|
||||
ledger
|
||||
.begin_unload(Shape::LoopbackIntoCapture)
|
||||
.expect("the ledger accepts the unload")
|
||||
.expect("the mirror issues an unload permit")
|
||||
.finish(UnloadOutcome::Confirmed);
|
||||
assert!(
|
||||
ledger
|
||||
.begin_unload(Shape::LegacyCaptureSink)
|
||||
.expect("the sink is no longer referenced")
|
||||
.is_some()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn closing_waits_for_a_previously_registered_operation() {
|
||||
use std::sync::mpsc;
|
||||
use std::time::Duration;
|
||||
|
||||
let ledger = ModuleLedger::new();
|
||||
let permit = ledger
|
||||
.begin_load(Shape::LegacyCaptureSink, 42, token(7))
|
||||
.expect("the operation registers before it can be detached");
|
||||
let (entered_tx, entered_rx) = mpsc::channel();
|
||||
let (done_tx, done_rx) = mpsc::channel();
|
||||
let waiter_ledger = Arc::clone(&ledger);
|
||||
let waiter = std::thread::spawn(move || {
|
||||
entered_tx.send(()).unwrap();
|
||||
waiter_ledger.close_and_wait();
|
||||
done_tx.send(()).unwrap();
|
||||
});
|
||||
entered_rx.recv().unwrap();
|
||||
assert!(
|
||||
matches!(
|
||||
done_rx.recv_timeout(Duration::from_millis(30)),
|
||||
Err(mpsc::RecvTimeoutError::Timeout)
|
||||
),
|
||||
"the close barrier must not pass a live affine permit"
|
||||
);
|
||||
permit.abandon();
|
||||
done_rx
|
||||
.recv_timeout(Duration::from_secs(1))
|
||||
.expect("settling the operation releases teardown");
|
||||
waiter.join().unwrap();
|
||||
assert!(ledger.is_clean());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_closed_ledger_refuses_event_work_but_allows_cleanup() {
|
||||
let ledger = ModuleLedger::new();
|
||||
ledger
|
||||
.begin_load(Shape::LegacyCaptureSink, 42, token(7))
|
||||
.expect("the sink may load before close")
|
||||
.commit(3)
|
||||
.expect("3 is a real index");
|
||||
ledger.close_and_wait();
|
||||
assert_eq!(
|
||||
ledger
|
||||
.begin_load(Shape::LoopbackIntoCapture, 42, token(8))
|
||||
.err(),
|
||||
Some(LedgerError::Closed {
|
||||
shape: Shape::LoopbackIntoCapture
|
||||
})
|
||||
);
|
||||
assert_eq!(
|
||||
ledger.begin_unload(Shape::LegacyCaptureSink).err(),
|
||||
Some(LedgerError::Closed {
|
||||
shape: Shape::LegacyCaptureSink
|
||||
})
|
||||
);
|
||||
assert!(
|
||||
ledger
|
||||
.begin_cleanup_unload(Shape::LegacyCaptureSink)
|
||||
.expect("teardown retains its cleanup authority")
|
||||
.is_some()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn begin_unload_distinguishes_vacant_from_busy() {
|
||||
let ledger = ModuleLedger::new();
|
||||
assert!(
|
||||
ledger
|
||||
.begin_unload(Shape::LegacyCaptureSink)
|
||||
.expect("vacancy is not an error")
|
||||
.is_none()
|
||||
);
|
||||
let _permit = ledger
|
||||
.begin_load(Shape::LegacyCaptureSink, 42, token(7))
|
||||
.expect("a vacant slot issues a permit");
|
||||
assert_eq!(
|
||||
ledger.begin_unload(Shape::LegacyCaptureSink),
|
||||
None,
|
||||
"a load in flight has no id to unload yet"
|
||||
ledger.begin_unload(Shape::LegacyCaptureSink).err(),
|
||||
Some(LedgerError::Busy {
|
||||
shape: Shape::LegacyCaptureSink,
|
||||
state: "loading"
|
||||
}),
|
||||
"an in-flight load must not look like a confirmed vacancy"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -864,13 +1321,12 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn loaded_lists_modules_in_shape_declaration_order() {
|
||||
fn loaded_lists_a_loopback_before_the_capture_sink() {
|
||||
// Declaration order is unload order: the loopbacks that reference the
|
||||
// capture sink must come before the sink itself.
|
||||
let ledger = ModuleLedger::new();
|
||||
for (shape, id) in [
|
||||
(Shape::LegacyCaptureSink, 1),
|
||||
(Shape::LoopbackIntoCapture, 2),
|
||||
(Shape::LoopbackOutOfCapture, 3),
|
||||
] {
|
||||
ledger
|
||||
@@ -880,7 +1336,10 @@ mod tests {
|
||||
.expect("a real index");
|
||||
}
|
||||
let order: Vec<Shape> = ledger.loaded().into_iter().map(|fp| fp.shape).collect();
|
||||
assert_eq!(order, crate::repair::plan::ALL_SHAPES.to_vec());
|
||||
assert_eq!(
|
||||
order,
|
||||
vec![Shape::LoopbackOutOfCapture, Shape::LegacyCaptureSink]
|
||||
);
|
||||
assert_eq!(
|
||||
order.last(),
|
||||
Some(&Shape::LegacyCaptureSink),
|
||||
|
||||
Reference in New Issue
Block a user