diff --git a/apps/effectcraft/src/audio_out.rs b/apps/effectcraft/src/audio_out.rs --- a/apps/effectcraft/src/audio_out.rs +++ b/apps/effectcraft/src/audio_out.rs @@ -1,16 +1,24 @@ //! Audio preview output through cpal (the device and channel mapping chosen in Settings ▸ Audio; //! the default output device when none is set or it is gone). //! -//! cpal streams are not `Send` on every platform, so each preview runs its stream on a small -//! dedicated thread that owns it until stopped. The callback pulls interleaved stereo from the -//! frontend's [`AudioFeed`] and maps it onto the device's channels (mono: L+R average; extra -//! channels: silence), converting to the device's sample format. +//! The output opens once and stays open for the whole session. Every preview start used to open +//! the device again — enumerating the devices and initializing the stream, which takes a good +//! fraction of a second on Windows' WASAPI — so a preview's sound began some frames after the +//! press that started it (#947, #1498). One thread now owns the stream for the process +//! ([`ensure`] builds it; the app warms it at startup, see [`prewarm`]): a playback binds its +//! [`AudioFeed`] to the callback and unbinds it again when it stops, and a stream that errors +//! (the device went away) is rebuilt on the next use. With nothing bound the callback pulls +//! silence, like a DAW whose output stays open between plays. +//! +//! The channel mapping is as before: interleaved stereo from the feed onto the device's channels +//! (mono: L+R average; extra channels: silence), converted to the device's sample format. use std::sync::Arc; -use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::Mutex; +use std::sync::PoisonError; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::mpsc; use std::thread::JoinHandle; -use std::time::Duration; use cpal::traits::{DeviceTrait, HostTrait, StreamTrait}; use cpal::{FromSample, SizedSample}; @@ -22,26 +30,83 @@ fn preferred_buffer_frames(rate: u32) -> u32 { (rate / 100).clamp(256, 2048) } -/// Wait until the feeder has mixed a block, or until preview is stopped. Starting the stream -/// before that fills the device with silence, so the first real samples are heard late (#554). -fn wait_to_play(feed: &AudioFeed, stop: &mpsc::Receiver<()>) -> bool { - loop { - if feed.queued_frames() > 0 { - return true; - } - match stop.recv_timeout(Duration::from_millis(2)) { - Ok(()) | Err(mpsc::RecvTimeoutError::Disconnected) => return false, - Err(mpsc::RecvTimeoutError::Timeout) => {} +/// A command for the running output thread. +enum Cmd { + /// Bind `feed` as the one the callback pulls from, under `session` (until that session is + /// cleared, or another play replaces it). + Play { session: u64, feed: Arc }, + /// Unbind the feed when `session` is still the one playing; a newer play keeps its own. + Clear { session: u64 }, + /// The stream errored (the device went away), or the settings changed: stop serving. The + /// next use rebuilds the output. + Died, +} + +/// What the running stream shares with its owners: the feed its callback pulls from (swapped per +/// playback), the output latency it measures (device buffering, for A/V sync), and whether the +/// stream is still alive. +#[derive(Default)] +struct Shared { + slot: Mutex>>, + latency: AtomicU64, + /// The session whose feed the callback serves (0: none). [`attach`] waits for it, so a `Play` + /// queued behind a dead stream's `Died` is caught instead of leaving a stalled preview. + bound: AtomicU64, + /// The stream reported an error: the output is rebuilt rather than reused. + dead: AtomicBool, +} + +impl Shared { + /// The bound feed (a panic in a thread holding the lock doesn't stop the music). + fn feed(&self) -> Option> { + self.slot.lock().unwrap_or_else(PoisonError::into_inner).clone() + } + + /// One command for the running stream; `current` is the session whose feed is bound. + /// Returns whether the stream goes on serving. + fn serve(&self, current: &mut u64, cmd: Cmd) -> bool { + let mut slot = self.slot.lock().unwrap_or_else(PoisonError::into_inner); + match cmd { + Cmd::Play { session, feed } => { + *current = session; + *slot = Some(feed); + self.bound.store(session, Ordering::Release); + true + } + Cmd::Clear { session } => { + if session == *current { + *slot = None; + self.bound.store(0, Ordering::Release); + } + true + } + Cmd::Died => false, } } } -pub struct CpalOut { - out: AudioOutput, +/// The process's audio output: the thread owning its stream, built for one [`AudioOutput`] +/// (device, channels, rate). It outlives every playback; the next one attaches to it. +struct Output { + /// The settings the stream was built for. + key: AudioOutput, + /// The rate the device was opened at. rate: u32, - latency: Arc, - stop: Option>, - thread: Option>, + shared: Arc, + tx: mpsc::Sender, + thread: JoinHandle<()>, +} + +/// The one output the process keeps open, built under this lock (so only one is ever built). +static OUTPUT: Mutex> = Mutex::new(None); + +/// Sessions of playbacks, so one playback's stop never unbinds a newer playback's feed (a scrub +/// closing after a preview started, say). +static SESSION: AtomicU64 = AtomicU64::new(0); + +/// Lock the process output (a panic while holding it doesn't take the audio down). +fn output() -> std::sync::MutexGuard<'static, Option> { + OUTPUT.lock().unwrap_or_else(PoisonError::into_inner) } /// Names of the output devices (Settings ▸ Audio). @@ -64,39 +129,156 @@ fn device(name: &str) -> Option { host.default_output_device() } -/// Open the output device (`None` without one). -pub fn open(out: &AudioOutput) -> Option> { +/// The rate to open the device at: Settings ▸ Audio ▸ Preview Sample Rate when the device +/// supports it, else the device's own (the preview is then mixed at that rate instead). +fn resolve_rate(out: &AudioOutput) -> Option { let dev = device(&out.device)?; let cfg = dev.default_output_config().ok()?; - // Settings ▸ Audio ▸ Preview Sample Rate: open the device at that rate when it supports it - // (otherwise the preview is mixed at that rate and resampled to the device's). let supports = |r: u32| { dev.supported_output_configs() .map(|mut c| c.any(|c| c.channels() == cfg.channels() && c.min_sample_rate().0 <= r && r <= c.max_sample_rate().0)) .unwrap_or(false) }; - let rate = if out.rate > 0 && supports(out.rate) { out.rate } else { cfg.sample_rate().0 }; - Some(Box::new(CpalOut { out: out.clone(), rate, latency: Arc::new(AtomicU64::new(0)), stop: None, thread: None })) + Some(if out.rate > 0 && supports(out.rate) { out.rate } else { cfg.sample_rate().0 }) +} + +/// The running output when it matches `out` (nothing is touched when it doesn't): the rate it +/// runs at, the shared state its callback pulls from, and where its commands go. +fn running(out: &AudioOutput) -> Option<(u32, Arc, mpsc::Sender)> { + output() + .as_ref() + .filter(|o| o.key == *out && !o.thread.is_finished() && !o.shared.dead.load(Ordering::Acquire)) + .map(|o| (o.rate, o.shared.clone(), o.tx.clone())) +} + +/// The output for `out`, building it first when none matches (the first use, after a change in +/// Settings ▸ Audio, or when the device went away). `rate` is the rate the device is opened at +/// (see [`resolve_rate`]); building it here, under [`OUTPUT`]'s lock, is what a first press of +/// Space would otherwise wait for. +fn ensure(out: &AudioOutput, rate: u32) -> Result<(Arc, mpsc::Sender), String> { + let mut guard = output(); + if guard.as_ref().is_some_and(|o| o.key != *out || o.thread.is_finished() || o.shared.dead.load(Ordering::Acquire)) { + // End the old stream: its thread stops with this command (or already has, when the device + // went away), whether or not a stale playback still holds a sender. + if let Some(old) = guard.take() { + let _ = old.tx.send(Cmd::Died); + } + } + if guard.is_none() { + *guard = Some(spawn_output(out, rate)?); + } + let Some(o) = guard.as_ref() else { + return Err("the audio output could not be opened".into()); + }; + Ok((o.shared.clone(), o.tx.clone())) +} + +/// Open the device, run its stream until it is told to stop or errors, and report over `ready` +/// whether the stream was built. This is the only thread that touches the device. +fn spawn_output(out: &AudioOutput, rate: u32) -> Result { + let (tx, rx) = mpsc::channel::(); + let shared = Arc::new(Shared::default()); + // The Output hands back its own handle: the thread's closure takes this one. + let shared_out = shared.clone(); + let (ready_tx, ready_rx) = mpsc::channel::>(); + let name = out.device.clone(); + let map = ((out.left.max(1) - 1) as usize, (out.right.max(1) - 1) as usize); + let died = tx.clone(); + let thread = std::thread::Builder::new() + .name("ec-audio-out".into()) + .spawn(move || { + let Some(dev) = device(&name) else { + let _ = ready_tx.send(Err("no audio output device".into())); + return; + }; + let sup = match dev.default_output_config() { + Ok(c) => c, + Err(e) => { + let _ = ready_tx.send(Err(e.to_string())); + return; + } + }; + let mut cfg: cpal::StreamConfig = sup.config(); + // Settings ▸ Audio ▸ Preview Sample Rate: the device is opened at the rate resolved + // in `open` (the preview's mix is resampled to it when mixed at another rate). + cfg.sample_rate = cpal::SampleRate(rate); + // A short period only where the default one is long (ALSA/PulseAudio on Linux, #554), + // and only within what the device reports it takes. + if cfg!(target_os = "linux") { + let want = preferred_buffer_frames(rate); + cfg.buffer_size = match sup.buffer_size() { + cpal::SupportedBufferSize::Range { min, max } if *min <= *max => cpal::BufferSize::Fixed(want.clamp(*min, *max)), + _ => cpal::BufferSize::Default, + }; + } + let build = |cfg: &cpal::StreamConfig| match sup.sample_format() { + cpal::SampleFormat::F32 => run::(&dev, cfg, shared.clone(), map, died.clone()), + cpal::SampleFormat::I16 => run::(&dev, cfg, shared.clone(), map, died.clone()), + cpal::SampleFormat::U16 => run::(&dev, cfg, shared.clone(), map, died.clone()), + cpal::SampleFormat::I32 => run::(&dev, cfg, shared.clone(), map, died.clone()), + f => Err(format!("unsupported sample format {f:?}")), + }; + let stream = match build(&cfg) { + Ok(s) => s, + Err(_) => { + // Some hosts reject a fixed period; a large default still plays, just later. + cfg.buffer_size = cpal::BufferSize::Default; + match build(&cfg) { + Ok(s) => s, + Err(e) => { + let _ = ready_tx.send(Err(e)); + return; + } + } + } + }; + // The stream runs from here for the rest of the process: with nothing bound its + // callback pulls silence, so a playback's first samples are heard as soon as they are + // mixed (the device's buffer stays small for exactly that; #554). Reported ready only + // once it plays: a device that fails to start is an error here, not a later surprise. + if let Err(e) = stream.play() { + let _ = ready_tx.send(Err(e.to_string())); + return; + } + let _ = ready_tx.send(Ok(())); + let mut current = 0u64; + while let Ok(cmd) = rx.recv() { + if !shared.serve(&mut current, cmd) { + break; + } + } + drop(stream); + }) + .map_err(|e| e.to_string())?; + ready_rx.recv().map_err(|e| e.to_string())??; + Ok(Output { key: out.clone(), rate, shared: shared_out, tx, thread }) } +/// Build `dev`'s stream: the callback pulls the bound feed (silence when none is bound), maps +/// it onto the device's channels and records the output latency; an error reports the stream +/// dead over `died`. fn run>( dev: &cpal::Device, cfg: &cpal::StreamConfig, - feed: Arc, - latency: Arc, + shared: Arc, map: (usize, usize), + died: mpsc::Sender, ) -> Result { let ch = cfg.channels as usize; // Output mapping: device channels (0-based) for left and right; out of range → 1 and 2. let (left, right) = if map.0 < ch && map.1 < ch { map } else { (0, 1) }; let rate = cfg.sample_rate.0 as f64; let mut stereo: Vec = Vec::new(); + let alive = shared.clone(); dev.build_output_stream( cfg, move |out: &mut [T], info: &cpal::OutputCallbackInfo| { let frames = out.len() / ch.max(1); stereo.resize(frames * 2, 0.0); - feed.pull(&mut stereo); + match shared.feed() { + Some(f) => f.pull(&mut stereo), + None => stereo.iter_mut().for_each(|s| *s = 0.0), + } for (f, s) in out.chunks_mut(ch.max(1)).zip(stereo.as_chunks::<2>().0.iter()) { for (c, o) in f.iter_mut().enumerate() { // Mono devices, or left and right mapped to the same channel, get the mix. @@ -115,105 +297,147 @@ fn run>( // Frames between "handed over" and "audible": device latency plus this buffer. let ts = info.timestamp(); let dev_lat = ts.playback.duration_since(&ts.callback).map(|d| d.as_secs_f64()).unwrap_or(0.0); - latency.store((dev_lat * rate) as u64 + frames as u64, Ordering::Relaxed); + shared.latency.store((dev_lat * rate) as u64 + frames as u64, Ordering::Relaxed); + }, + move |e| { + // ALSA and PulseAudio report recoverable polling and xrun errors here, and the worker + // goes on: only a device that is gone ends the stream. A stall a platform reports as + // something else is caught by `AudioPlayback::stalled`, which hands the preview back to + // the frame clock, and the next playback rebuilds an output whose `Play` never lands. + if matches!(e, cpal::StreamError::DeviceNotAvailable) { + // Before the command, so an output that died is never attached again. + alive.dead.store(true, Ordering::Release); + let _ = died.send(Cmd::Died); + } + log::warn!("audio output: {e}"); }, - |e| log::warn!("audio output: {e}"), None, ) .map_err(|e| e.to_string()) } +pub struct CpalOut { + out: AudioOutput, + rate: u32, + /// The process output this playback runs on, once started. + running: Option<(Arc, mpsc::Sender)>, + /// The session of this playback within it (see [`Cmd::Clear`]). + session: u64, +} + +/// Open the output device (`None` without one). When the process output is already running for +/// these settings, nothing is touched: the device opens once per session (see the module doc), +/// so a preview's sound starts with the press that started it. +pub fn open(out: &AudioOutput) -> Option> { + // The running output already knows the rate; otherwise resolve it (this is the enumeration a + // first press, or one after a settings change, waits for). + let rate = match running(out) { + Some((rate, ..)) => Some(rate), + None => resolve_rate(out), + }; + Some(Box::new(CpalOut { out: out.clone(), rate: rate?, running: None, session: 0 })) +} + +/// Open the output ahead of the first preview (the app does this at startup): the devices are +/// enumerated and the stream built now, so the first press doesn't wait for it. Fails without a +/// device; the first preview then opens it as before. +pub fn prewarm(out: &AudioOutput) -> Result<(), String> { + let rate = resolve_rate(out).ok_or("no audio output device")?; + // Warm up only when nothing runs yet: an output that already exists (a preview started first, + // perhaps with newer settings) is left alone, and checking and building under one lock means + // a preview cannot slip in between and have its stream replaced by this one. + warm_up(out, rate) +} + +/// Build the process output for `out` when none exists, under [`OUTPUT`]'s lock. +fn warm_up(out: &AudioOutput, rate: u32) -> Result<(), String> { + let mut guard = output(); + if guard.is_some() { + return Ok(()); + } + let built = spawn_output(out, rate)?; + *guard = Some(built); + Ok(()) +} + +/// Bind `feed` to the process's output for `out` (building the output first when none matches), +/// under a new session whose stop unbinds it. +fn attach(out: &AudioOutput, rate: u32, feed: Arc) -> Result<((Arc, mpsc::Sender), u64), String> { + // Twice at most: an output that died between the check and the send is rebuilt by the second + // `ensure` (its thread has finished by then). + for _ in 0..2 { + let (shared, tx) = ensure(out, rate)?; + let session = SESSION.fetch_add(1, Ordering::Relaxed) + 1; + if tx.send(Cmd::Play { session, feed: feed.clone() }).is_ok() && bound_within(&shared, session) { + return Ok(((shared, tx), session)); + } + // The stream is going away: its `Died` was queued before this `Play`, or it was already + // dead when `ensure` looked. Stop it for good and build a new one, rather than leaving + // the preview with a clock that never advances. + shared.dead.store(true, Ordering::Release); + let _ = tx.send(Cmd::Died); + await_finished(out); + } + Err("the audio output went away".into()) +} + +/// Wait (bounded) until the callback serves `session`: the audio thread wakes on the command, so +/// this is microseconds when the stream is alive. +fn bound_within(shared: &Shared, session: u64) -> bool { + for _ in 0..100 { + if shared.bound.load(Ordering::Acquire) == session { + return true; + } + std::thread::sleep(std::time::Duration::from_millis(1)); + } + false +} + +/// Wait (bounded) until the dead output's thread has finished, so the next [`ensure`] builds a new +/// stream instead of racing the old one for the device. +fn await_finished(out: &AudioOutput) { + for _ in 0..400 { + let finished = { + let guard = output(); + guard.as_ref().is_none_or(|o| o.key != *out || o.thread.is_finished()) + }; + if finished { + return; + } + std::thread::sleep(std::time::Duration::from_millis(1)); + } +} + impl AudioDevice for CpalOut { + /// The rate the process output was opened at. fn sample_rate(&self) -> u32 { self.rate } + /// Bind `feed` to the process output (building it first when none matches). fn start(&mut self, feed: Arc) -> Result<(), String> { self.stop(); - let (stop_tx, stop_rx) = mpsc::channel::<()>(); - let (ready_tx, ready_rx) = mpsc::channel::>(); - let latency = self.latency.clone(); - let rate = self.rate; - let name = self.out.device.clone(); - let map = ((self.out.left.max(1) - 1) as usize, (self.out.right.max(1) - 1) as usize); - let thread = std::thread::Builder::new() - .name("ec-audio-out".into()) - .spawn(move || { - let Some(dev) = device(&name) else { - let _ = ready_tx.send(Err("no audio output device".into())); - return; - }; - let sup = match dev.default_output_config() { - Ok(c) => c, - Err(e) => { - let _ = ready_tx.send(Err(e.to_string())); - return; - } - }; - let mut cfg: cpal::StreamConfig = sup.config(); - cfg.sample_rate = cpal::SampleRate(rate); - // A short period only where the default one is long (ALSA/PulseAudio on Linux, #554), - // and only within what the device reports it takes. - if cfg!(target_os = "linux") { - let want = preferred_buffer_frames(rate); - cfg.buffer_size = match sup.buffer_size() { - cpal::SupportedBufferSize::Range { min, max } if *min <= *max => cpal::BufferSize::Fixed(want.clamp(*min, *max)), - _ => cpal::BufferSize::Default, - }; - } - let build = |cfg: &cpal::StreamConfig| match sup.sample_format() { - cpal::SampleFormat::F32 => run::(&dev, cfg, feed.clone(), latency.clone(), map), - cpal::SampleFormat::I16 => run::(&dev, cfg, feed.clone(), latency.clone(), map), - cpal::SampleFormat::U16 => run::(&dev, cfg, feed.clone(), latency.clone(), map), - cpal::SampleFormat::I32 => run::(&dev, cfg, feed.clone(), latency.clone(), map), - f => Err(format!("unsupported sample format {f:?}")), - }; - let stream = match build(&cfg) { - Ok(s) => s, - Err(_) => { - // Some hosts reject a fixed period; a large default still plays, just later. - cfg.buffer_size = cpal::BufferSize::Default; - match build(&cfg) { - Ok(s) => s, - Err(e) => { - let _ = ready_tx.send(Err(e)); - return; - } - } - } - }; - let _ = ready_tx.send(Ok(())); - if !wait_to_play(&feed, &stop_rx) { - return; - } - if let Err(e) = stream.play() { - log::warn!("audio output: {e}"); - return; - } - let _ = stop_rx.recv(); - drop(stream); - }) - .map_err(|e| e.to_string())?; - self.stop = Some(stop_tx); - self.thread = Some(thread); - ready_rx.recv().map_err(|e| e.to_string())? + let ((shared, tx), session) = attach(&self.out, self.rate, feed)?; + self.running = Some((shared, tx)); + self.session = session; + Ok(()) } + /// Unbind this playback's feed; the stream stays open and pulls silence. fn stop(&mut self) { - if let Some(tx) = self.stop.take() { - let _ = tx.send(()); - } - if let Some(t) = self.thread.take() { - let _ = t.join(); + if let Some((_, tx)) = self.running.take() { + let _ = tx.send(Cmd::Clear { session: self.session }); } } + /// The output latency the callback measured (0 until it has run). fn latency_frames(&self) -> u64 { - self.latency.load(Ordering::Relaxed) + self.running.as_ref().map(|(s, _)| s.latency.load(Ordering::Relaxed)).unwrap_or(0) } } impl Drop for CpalOut { + /// Unbind the feed; the process output itself outlives every playback. fn drop(&mut self) { self.stop(); } @@ -223,6 +447,7 @@ impl Drop for CpalOut { mod tests { use super::*; + /// The Linux period stays about 10 ms across rates. #[test] fn preferred_buffer_is_about_ten_milliseconds() { assert_eq!(preferred_buffer_frames(48_000), 480); @@ -230,25 +455,29 @@ mod tests { assert_eq!(preferred_buffer_frames(192_000), 1920); } + /// A playback binds its feed, and only its own session unbinds it. #[test] - fn wait_to_play_returns_once_the_feeder_has_samples() { + fn a_play_binds_its_feed_until_its_session_clears() { + let s = Shared::default(); + let mut current = 0; let feed = Arc::new(AudioFeed::default()); - let (_tx, rx) = mpsc::channel(); - let f = feed.clone(); - let h = std::thread::spawn(move || wait_to_play(&f, &rx)); - assert_eq!(feed.queued_frames(), 0); - std::thread::sleep(Duration::from_millis(20)); feed.push(&[0.1, 0.2]); - assert!(h.join().unwrap()); - assert_eq!(feed.queued_frames(), 1); - } + assert!(s.serve(&mut current, Cmd::Play { session: 7, feed: feed.clone() })); + assert_eq!(s.bound.load(Ordering::Acquire), 7, "the callback reports the session it serves"); + let bound = s.feed().expect("the feed is bound"); + let mut out = [0.0f32; 2]; + bound.pull(&mut out); + assert_eq!(out, [0.1, 0.2]); - #[test] - fn wait_to_play_stops_without_samples() { - let feed = AudioFeed::default(); - let (tx, rx) = mpsc::channel(); - let h = std::thread::spawn(move || wait_to_play(&feed, &rx)); - tx.send(()).unwrap(); - assert!(!h.join().unwrap()); + // A newer play replaces it, and the old session's clear leaves the newer one playing. + assert!(s.serve(&mut current, Cmd::Play { session: 8, feed: Arc::new(AudioFeed::default()) })); + assert!(s.serve(&mut current, Cmd::Clear { session: 7 })); + assert!(s.feed().is_some(), "a stale clear must not unbind the newer play"); + + // Its own session clears it, and a dead stream stops serving. + assert!(s.serve(&mut current, Cmd::Clear { session: 8 })); + assert!(s.feed().is_none()); + assert_eq!(s.bound.load(Ordering::Acquire), 0, "nothing is served after its own clear"); + assert!(!s.serve(&mut current, Cmd::Died)); } } diff --git a/apps/effectcraft/src/main.rs b/apps/effectcraft/src/main.rs --- a/apps/effectcraft/src/main.rs +++ b/apps/effectcraft/src/main.rs @@ -326,6 +326,19 @@ fn main() -> eframe::Result { app.hooks.system_theme = appearance::service(); app.hooks.audio_device = Some(Box::new(audio_out::open)); app.hooks.audio_devices = Some(Box::new(audio_out::devices)); + // The output device opens now, ahead of the first preview: a press of Space then starts + // the sound at once instead of after the device opens (see `audio_out`'s module doc). + { + let out = effectcraft_ui_egui::audio::AudioOutput::from_prefs(&app.session.prefs); + std::thread::Builder::new() + .name("ec-audio-prewarm".into()) + .spawn(move || { + if let Err(e) = audio_out::prewarm(&out) { + log::debug!("audio prewarm: {e}"); + } + }) + .ok(); + } app.hooks.pick_folder = Some(Box::new(|| rfd::FileDialog::new().pick_folder().map(|p| p.to_string_lossy().to_string()))); app.hooks.pick_save_file = Some(Box::new(|name: &str, ext: &str| { // A default with a folder that exists opens the dialog there. diff --git a/crates/ui-egui/src/app/playback.rs b/crates/ui-egui/src/app/playback.rs --- a/crates/ui-egui/src/app/playback.rs +++ b/crates/ui-egui/src/app/playback.rs @@ -30,6 +30,7 @@ impl EffectcraftApp { start_frame: plan.first, shown: 0, waiting: false, + waiting_since: 0.0, fps: plan.fps, shortcut: sc, plan: Some(plan), @@ -39,20 +40,19 @@ impl EffectcraftApp { layer_controls: preset.include_layer_controls, restore_max, audio_frame: (!plan.video).then_some(plan.first), - audio_held: false, + last_shown: now, + gave_up_at: 0.0, shown_at: Default::default(), }; if !self.playback.caching { - if plan.video { - self.playback.audio_held = self.audio_plays(); - } else { - self.start_audio(fr.tick_of(plan.first)); - } + // The sound starts with the first frame, at once (#947): `start_audio` does nothing + // without audio or without a device. + self.start_audio(fr.tick_of(plan.first)); } } /// The preview includes audio at the comp's frame rate and the comp has something audible. - fn audio_plays(&self) -> bool { + pub(super) fn audio_plays(&self) -> bool { let (Some(pl), Some(cid)) = (self.playback.plan, self.session.active_comp_id()) else { return false }; let Some(c) = self.session.project.comp(cid) else { return false }; // A preview at another frame rate plays slower or faster than real time: silent. @@ -159,6 +159,8 @@ impl EffectcraftApp { self.playback.caching = false; self.playback.start_wall = now; self.playback.start_frame = pl.first; + // The cache may have taken seconds: the sound's resume watch starts from this moment. + self.playback.last_shown = now; let t = self.session.active_comp().map(|c| c.frame_rate.tick_of(pl.first)).unwrap_or_default(); self.start_audio(t); true diff --git a/crates/ui-egui/src/app/prefetch.rs b/crates/ui-egui/src/app/prefetch.rs --- a/crates/ui-egui/src/app/prefetch.rs +++ b/crates/ui-egui/src/app/prefetch.rs @@ -4,6 +4,17 @@ use super::*; /// [`EffectcraftApp::editing`]). const EDIT_QUIET: f64 = 0.3; +/// How long a preview with sound may show no new frame before it gives the sound up (seconds): +/// past the slowest speed there is nothing left to slow, and playing on over a still frame only +/// hides what the render is doing (#2472). +const AUDIO_GIVE_UP: f64 = 1.5; + +/// A preview whose sound was given up takes it up again once the picture has moved within this +/// long (seconds), and no sooner than this long after giving it up — a preview that cannot keep +/// up must not thrash between the two paths. +const RESUME_QUIET: f64 = 0.5; +const RESUME_COOLDOWN: f64 = 1.0; + impl EffectcraftApp { /// RAM preview: advance at the preview's frame rate when frames are cached; otherwise hold /// and keep rendering ahead (the green cache bar fills, then playback runs in real time). @@ -52,11 +63,13 @@ impl EffectcraftApp { self.cache_when_idle(ctx, &c, &series, now, input_active, (wa, wb)); return; } - let fit = self.frames_that_fit(&c, scale); - self.start_held_audio(&pass, fit); if self.advance_audio_clock(&pass, step) { return; } + // The sound was given up earlier (a stall, a stream that died, a render too slow): take it + // up again from the frame on screen once the picture moves, or a preview that stalled once + // would stay silent for the rest of the pass (#947). + self.resume_audio(&pass); self.advance_frame_clock(&pass, step); } @@ -115,6 +128,8 @@ impl EffectcraftApp { self.playback.caching = false; self.playback.start_wall = now; self.playback.start_frame = p.first; + // The cache may have taken seconds: the sound's resume watch starts from this moment. + self.playback.last_shown = now; self.session.set_time(fr.tick_of(p.first)); self.start_audio(fr.tick_of(p.first)); } @@ -179,20 +194,28 @@ impl EffectcraftApp { } } - /// Held sound starts once the frames ahead are cached (see `Playback::audio_held`): the - /// prefetch window when the silent preview keeps real time, else the rest of the preview. - fn start_held_audio(&mut self, pass: &PlaybackPass<'_>, fit: i64) { - let PlaybackPass { now, plan, series, cur, ahead, frame_rate: fr, .. } = *pass; - if !self.playback.audio_held { + /// Take the sound up again after [`Self::advance_audio_clock`] gave it up, once the picture is + /// moving again (a frame shown lately), no sooner than [`RESUME_COOLDOWN`]. + fn resume_audio(&mut self, pass: &PlaybackPass<'_>) { + if self.audio.is_some() || !self.playback.playing || self.playback.caching || !self.audio_plays() { return; } - let Some(p) = plan else { return }; - if (self.playback.keeps_up(now) && self.cached_ahead(&p, &series, cur, ahead.min(fit))) || self.cached_ahead(&p, &series, cur, fit) { - self.playback.audio_held = false; - self.playback.start_wall = now; - self.playback.start_frame = cur; - self.start_audio(fr.tick_of(cur)); + let PlaybackPass { now, frame_rate: fr, cur, video, .. } = *pass; + if now - self.playback.gave_up_at < RESUME_COOLDOWN { + return; + } + // An audio-only preview shows no frames, so "the picture moved lately" says nothing there: + // only a preview with video waits for one. + if video && now - self.playback.last_shown > RESUME_QUIET { + return; } + // Each attempt restarts the cooldown, so a preview that cannot keep up tries again at most + // once a second instead of every frame. + self.playback.gave_up_at = now; + // Resume where the sound stopped: the frame on screen for a preview with video, and the + // audible frame for an audio-only one (its session time never moves). + let at = if video { cur } else { self.playback.audio_frame.unwrap_or(cur) }; + self.start_audio(fr.tick_of(at)); } /// Audio clock drives playback: show the frame under the audible sample. Returns true when @@ -210,31 +233,65 @@ impl EffectcraftApp { self.stop(); return true; } + // Whether the frame under the audible sample is there at all: a preview that loops over a + // single frame keeps the same frame on screen forever, and that is not a render that fell + // behind — only a missing frame is worth giving the sound up for. + let audible = audio::frame_of_sample(a.clock(), a.rate, fr).clamp(wa, wb); + let missing = !self.frames.is_cached(&FrameKey { frame: wa + (audible - wa).div_euclid(step) * step, ..series }); + if a.stalled(now) || (video && missing && now - self.playback.last_shown > AUDIO_GIVE_UP) { + // The sound stopped, or the picture has not moved for a while (the renderer cannot keep + // up even at the slowest speed): fall back to the frame clock from where playback is, + // instead of freezing the preview or playing on over a still frame (#947). An + // audio-only preview leaves the session time put, so its progress is `audio_frame`. + // The feeder may still be mixing a slow file: stop it without waiting for it. + let at = if video { cur } else { self.playback.audio_frame.unwrap_or(cur) }; + self.playback.start_wall = now; + self.playback.start_frame = at; + self.playback.gave_up_at = now; + if let Some(mut a) = self.audio.take() { + a.stop_async(); + } + return false; + } let snap = |f: i64| wa + (f - wa).div_euclid(step) * step; - let target = snap(audio::frame_of_sample(a.clock(), a.rate, fr).clamp(wa, wb)); + let target = snap(audible); if !video { self.playback.audio_frame = Some(target); } else if self.frames.is_cached(&FrameKey { frame: target, ..series }) { + // The picture caught up: the sound eases back to real time. + a.speed(1.0); self.playback.waiting = false; if target != cur { self.session.set_time(fr.tick_of(target)); self.playback.frame_shown(now); } } else { - // Rendering can't keep up: rather than skip frames, the sound stops and every - // frame shows as it renders until the frames ahead are cached again. - self.audio = None; - self.playback.audio_held = true; - self.playback.waiting = true; - // The rate measured before the stall no longer says it keeps up. + // The frame under the audible sample is not rendered yet. Ask for it here — unless an + // urgent request is already in flight, since a new one cancels the previous one and a + // frame that is cancelled forever never renders. Then slow the sound (and the clock + // with it) so the picture gets time: playback eases down and back up instead of falling + // silent or repeating itself (#947, like a tape losing speed). + if !self.frames.urgent_pending() { + self.request_frame_urgent(pass.cid, target, pass.scale); + } + if !self.playback.waiting { + self.playback.waiting = true; + self.playback.waiting_since = now; + } + let waited = now - self.playback.waiting_since; + a.speed(if waited < 0.25 { + 0.85 + } else if waited < 0.75 { + 0.7 + } else { + 0.5 + }); + // The rate measured before the wait no longer says it keeps up. self.playback.shown_at.clear(); - self.playback.start_wall = now; - self.playback.start_frame = cur; } ctx.request_repaint(); true } - /// The wall clock advances the preview when no audio device drives it, and shows the target /// frame (or holds when it is not cached yet). fn advance_frame_clock(&mut self, pass: &PlaybackPass<'_>, step: i64) { @@ -281,11 +338,13 @@ impl EffectcraftApp { self.playback.frame_shown(now); } } else { - // Not cached yet: hold the clock at the current frame (cache first, then play). + // Not cached yet: hold the clock at the current frame (cache first, then play). A frame + // the cache does have ahead is shown, so the sound's resume watch sees the picture move. self.playback.waiting = true; let next = plan.and_then(|p| p.frame_after(cur, 1)).unwrap_or(if cur + 1 > wb { wa } else { cur + 1 }); if self.frames.is_cached(&FrameKey { frame: next, ..series }) { self.session.set_time(fr.tick_of(next)); + self.playback.frame_shown(now); } self.playback.start_wall = now; self.playback.start_frame = fr.frame_at(self.session.time()); @@ -358,12 +417,6 @@ impl EffectcraftApp { /// How many viewer frames of `comp` at `scale` the RAM preview budget holds (90 % of it, /// at least one). /// The frames of `plan` from `cur` to its end, or as many as fit in the RAM preview, are - /// cached: sound can play from `cur` in real time. - fn cached_ahead(&self, plan: &effectcraft_engine::preview::PreviewPlan, series: &FrameKey, cur: i64, fit: i64) -> bool { - let cached = self.frames.cached_frames(series); - (0..fit).map_while(|k| plan.frame_after(cur, k)).take_while(|f| *f >= cur).all(|f| cached.binary_search(&f).is_ok()) - } - fn frames_that_fit(&self, comp: &effectcraft_engine::project::Comp, scale: f64) -> i64 { let side = |n: u32| (n as f64 * scale).ceil().max(1.0); let frame = side(comp.width) * side(comp.height) * 4.0; diff --git a/crates/ui-egui/src/audio.rs b/crates/ui-egui/src/audio.rs --- a/crates/ui-egui/src/audio.rs +++ b/crates/ui-egui/src/audio.rs @@ -65,20 +65,78 @@ impl AudioOutput { } /// Lock-protected sample queue between the feeder thread and the device callback. -#[derive(Default)] pub struct AudioFeed { queue: Mutex>, - /// Sample frames handed to the device (real audio only: underruns do not count). + /// Sample frames handed to the device (real audio only: underruns don't count). consumed: AtomicU64, /// Peak |sample| per channel since the last [`AudioFeed::take_peaks`] (f32 bits). peak: [AtomicU32; 2], + /// Pulls that ran dry (the device asked for more than the feeder had queued). + underruns: AtomicU64, /// The feeder reached the end of a non-looping span. finished: AtomicBool, + /// The rate the feed is pulled at (0: not told yet; speed easing is timed by it). + rate: AtomicU32, + /// How fast the sound is read: real time, or slower while the preview waits for a frame. + drive: Mutex, +} + +impl Default for AudioFeed { + /// An idle feed: nothing queued, nothing bound, real time. + fn default() -> Self { + AudioFeed { + queue: Mutex::default(), + consumed: AtomicU64::default(), + peak: [AtomicU32::new(0), AtomicU32::new(0)], + underruns: AtomicU64::default(), + finished: AtomicBool::default(), + rate: AtomicU32::default(), + drive: Mutex::default(), + } + } +} + +/// Slowest and fastest the sound is read at (a preview waiting for a frame slows it down). +const MIN_SPEED: f32 = 0.25; +const MAX_SPEED: f32 = 4.0; + +/// How [`AudioFeed::pull`] reads the queue: `speed` 1.0 is real time; below it playback slows +/// down (varispeed: the pitch drops, as when a tape loses speed) and the clock with it, so a +/// preview that waits for a frame neither falls silent nor repeats itself. +struct Drive { + /// What the preview asks for (1.0, or less while it waits for a frame). + target: f32, + /// The speed the sound is read at: eased toward `target`, so a change is smooth. + speed: f32, + /// Position between `cur` and `nxt`, for reading at a fractional speed. + phase: f32, + /// The two source frames being read between, kept across blocks so no frame is lost. + cur: [f32; 2], + nxt: [f32; 2], + /// The last real frame read: a dry queue holds it instead of clicking to silence. + last: [f32; 2], + /// `cur` and `nxt` were filled (the first read after real time). + primed: bool, +} + +impl Default for Drive { + /// Real time, with nothing to interpolate. + fn default() -> Self { + Drive { target: 1.0, speed: 1.0, phase: 0.0, cur: [0.0; 2], nxt: [0.0; 2], last: [0.0; 2], primed: false } + } } impl AudioFeed { /// Fill `out` (interleaved stereo) from the queue; silence where the queue runs dry. + /// + /// While the preview asks for a speed below real time (see [`AudioFeed::set_speed`]) the + /// sound is read at that speed, interpolated between frames, and `consumed` counts only the + /// source frames actually taken: the preview's clock slows down with the sound, so a preview + /// waiting for a frame eases down and back up instead of falling silent or repeating. pub fn pull(&self, out: &mut [f32]) { + if self.varispeed(out) { + return; + } let mut real = 0usize; let mut pk = [0.0f32; 2]; if let Ok(mut q) = self.queue.lock() { @@ -90,7 +148,102 @@ impl AudioFeed { real = n; } out[real..].iter_mut().for_each(|o| *o = 0.0); + if real < out.len() { + self.underruns.fetch_add(1, Ordering::Relaxed); + } self.consumed.fetch_add((real / 2) as u64, Ordering::Relaxed); + self.note_peaks(pk); + } + + /// Read the queue at the eased speed when it is below real time (`true` when it did); at real + /// time [`AudioFeed::pull`] takes whole frames, exactly as they were mixed. + fn varispeed(&self, out: &mut [f32]) -> bool { + let Ok(mut d) = self.drive.lock() else { return false }; + // Never slowed down: read whole frames, exactly as they were mixed. Once slowed, the + // eased path keeps the two frames it has already taken from the queue; snapping back to + // the whole-frame path would drop them. + if d.target >= 1.0 && (d.speed - 1.0).abs() < 1e-3 && !d.primed { + d.speed = 1.0; + return false; + } + let Ok(mut q) = self.queue.lock() else { return false }; + let rate = self.rate.load(Ordering::Relaxed).max(8000) as f32; + // About 150 ms to cover most of the way to a new speed: a smooth ramp, no click. + let k = 1.0 / (0.15 * rate); + let mut taken = 0u64; + let next = |q: &mut VecDeque, last: &mut [f32; 2], taken: &mut u64| match (q.pop_front(), q.pop_front()) { + (Some(l), Some(r)) => { + *last = [l, r]; + *taken += 1; + [l, r] + } + // Past the queue's end: hold the last frame (silence before the first one), and + // count nothing, so a dry queue never moves the clock. + _ => *last, + }; + let mut last = d.last; + let mut cur = d.cur; + let mut nxt = d.nxt; + if !d.primed { + cur = next(&mut q, &mut last, &mut taken); + nxt = next(&mut q, &mut last, &mut taken); + d.primed = true; + } + let mut pk = [0.0f32; 2]; + for f in out.chunks_mut(2) { + d.speed += (d.target - d.speed) * k; + let speed = d.speed.clamp(MIN_SPEED, MAX_SPEED); + let p = d.phase; + let l = cur[0] + (nxt[0] - cur[0]) * p; + let r = cur[1] + (nxt[1] - cur[1]) * p; + f[0] = l; + if f.len() > 1 { + f[1] = r; + } + pk[0] = pk[0].max(l.abs()); + pk[1] = pk[1].max(r.abs()); + d.phase += speed; + while d.phase >= 1.0 { + d.phase -= 1.0; + cur = nxt; + nxt = next(&mut q, &mut last, &mut taken); + } + } + d.last = last; + d.cur = cur; + d.nxt = nxt; + self.consumed.fetch_add(taken, Ordering::Relaxed); + if taken == 0 { + // The queue had nothing to give: the dry pull the ordinary path counts too, so the + // feeder's lead can also grow for a preview that already slowed down. + self.underruns.fetch_add(1, Ordering::Relaxed); + } + drop(q); + drop(d); + self.note_peaks(pk); + true + } + + /// Play at `speed` (1.0 = real time): a preview that waits for a frame asks for less, and the + /// sound eases down to it and back up, so playback slows smoothly instead of falling silent. + pub fn set_speed(&self, speed: f32) { + if let Ok(mut d) = self.drive.lock() { + d.target = speed.clamp(MIN_SPEED, MAX_SPEED); + } + } + + /// The speed the sound is being read at (eased toward the one last set). + pub fn speed(&self) -> f32 { + self.drive.lock().map(|d| d.speed).unwrap_or(1.0) + } + + /// The rate the feed is pulled at: a change of speed is eased over about 150 ms of it. + pub fn set_rate(&self, rate: u32) { + self.rate.store(rate, Ordering::Relaxed); + } + + /// Raise the peak meters with the samples just produced. + fn note_peaks(&self, pk: [f32; 2]) { for (c, v) in pk.into_iter().enumerate() { // Rust 1.99 renamed `fetch_update` to `try_update`; keep the old name while the // workspace supports Rust 1.95 (`rust-version`). @@ -117,6 +270,11 @@ impl AudioFeed { self.queue.lock().map(|q| q.len() / 2).unwrap_or(0) } + /// Pulls that ran dry since the playback started (the feeder was late). + pub fn underruns(&self) -> u64 { + self.underruns.load(Ordering::Relaxed) + } + pub fn consumed(&self) -> u64 { self.consumed.load(Ordering::Relaxed) } @@ -183,12 +341,79 @@ pub struct AudioPlayback { feeder: Option<(RenderSource, ItemId, i64, u32)>, pub rate: u32, pub span: PlaySpan, + /// The clock the stall watch last saw (see [`AudioPlayback::stalled`]). + last_clock: Mutex, +} + +/// What the stall watch last saw: the clock's value, when it was seen, and whether the feed had +/// samples queued then (a feeder filling an empty queue is not a stalled device). +#[derive(Default)] +struct Watch { + consumed: u64, + at: Option, + queued: bool, } -/// Audio kept queued ahead of the device: ~200 ms (a feeder thread tops it up continuously); -/// wasm32 tops it up once per UI frame, so it keeps ~350 ms to ride out slow frames. +/// How long a stream may leave queued samples unpulled before the preview falls back to the frame +/// clock (seconds): a dead device must not freeze the preview. +const AUDIO_STALL: f64 = 0.5; + +/// How long the preview waits for samples from a feeder that has nothing queued yet (seconds). +/// Mixing the first block can take seconds (a long FLAC decodes whole, #408), and that is not a +/// dead stream: give it room before giving up on the sound. +const AUDIO_STARVE: f64 = 5.0; + +/// The stall watch: `true` once the clock has stood still long enough to give up on the sound. +/// The watch starts again whenever the queue gains its first samples (the device had nothing to +/// play before that, so a feeder's slow first block is not a stall, #408) and whenever the clock +/// moves; a stream with samples queued that it does not pull is dead after [`AUDIO_STALL`]. +fn stall_watch(w: &mut Watch, now: f64, consumed: u64, queued: bool) -> bool { + let moved = consumed != w.consumed || queued != w.queued; + let Some(since) = w.at.filter(|_| !moved) else { + *w = Watch { consumed, at: Some(now), queued }; + return false; + }; + now - since > if queued { AUDIO_STALL } else { AUDIO_STARVE } +} + +/// Audio kept queued ahead of the device: ~100 ms (a feeder thread tops it up continuously), so a +/// preview that restarts its sound repeats as little of the queue as the device needs; wasm32 +/// tops it up once per UI frame, so it keeps ~350 ms to ride out slow frames. fn queue_target(rate: u32) -> usize { - if cfg!(target_arch = "wasm32") { rate as usize * 7 / 20 } else { rate as usize / 5 } + if cfg!(target_arch = "wasm32") { rate as usize * 7 / 20 } else { rate as usize / 10 } +} + +/// How far ahead the feeder keeps the queue: the base lead, grown whenever the device ran dry +/// (a render pool takes the cores, and the callback is then served late) and given back once the +/// machine has settled. Pure arithmetic, so it is unit tested. +struct Lead { + base: usize, + target: usize, + seen: u64, + idle: u32, +} + +impl Lead { + fn new(base: usize) -> Lead { + Lead { base, target: base, seen: 0, idle: 0 } + } + + /// Feed the underrun count seen so far and the step to grow by; returns the lead to keep. + fn update(&mut self, underruns: u64, step: usize) -> usize { + if underruns != self.seen { + self.seen = underruns; + self.target = (self.target + step.max(1)).min(self.base * 4); + self.idle = 0; + } else if self.target > self.base { + // About a second of a full queue at the loop's 4 ms sleep: give a little back. + self.idle += 1; + if self.idle >= 250 { + self.target = self.target.saturating_sub(step.saturating_mul(4)).max(self.base); + self.idle = 0; + } + } + self.target + } } impl AudioPlayback { @@ -210,10 +435,20 @@ impl AudioPlayback { let mix = if mix_rate == 0 { rate } else { mix_rate.clamp(8000, 192_000) }; let span = PlaySpan::new(start, wa, wb, looping, rate); let feed = Arc::new(AudioFeed::default()); + feed.set_rate(rate); let stop = Arc::new(AtomicBool::new(false)); if cfg!(target_arch = "wasm32") { // The device takes samples only in `pump`, which fills the queue first. - let mut p = AudioPlayback { feed: feed.clone(), device, stop, thread: None, feeder: Some((src, comp, span.start, mix)), rate, span }; + let mut p = AudioPlayback { + feed: feed.clone(), + device, + stop, + thread: None, + feeder: Some((src, comp, span.start, mix)), + rate, + span, + last_clock: Mutex::new(Watch::default()), + }; p.device.start(feed)?; p.pump(); return Ok(p); @@ -228,8 +463,11 @@ impl AudioPlayback { .name("ec-audio-feed".into()) .spawn(move || { let mut cursor = span.start; - let target = queue_target(rate); + let mut lead = Lead::new(queue_target(rate)); while !stop.load(Ordering::Relaxed) { + // A render pool takes the cores: keep a longer lead while the device runs + // dry, and give it back once the machine settles (a crackle is an underrun). + let target = lead.update(feed.underruns(), rate as usize / 20); if feed.queued_frames() < target && !feed.finished.load(Ordering::Relaxed) { cursor = feed_block(&src, comp, &feed, &span, cursor, rate, mix); } else { @@ -239,8 +477,14 @@ impl AudioPlayback { }) .map_err(|e| e.to_string())? }; - device.start(feed.clone())?; - Ok(AudioPlayback { feed, device, stop, thread: Some(thread), feeder: None, rate, span }) + if let Err(e) = device.start(feed.clone()) { + // The feeder has nothing to feed: signal it and let it leave at its next stop check. + // Joining here would wait for a slow mix on the UI thread that starts the preview. + stop.store(true, Ordering::Relaxed); + drop(thread); + return Err(e); + } + Ok(AudioPlayback { feed, device, stop, thread: Some(thread), feeder: None, rate, span, last_clock: Mutex::new(Watch::default()) }) } /// Every UI frame while playing: on wasm32 mix ahead into the feed (what the feeder thread @@ -255,6 +499,21 @@ impl AudioPlayback { self.device.pump(); } + /// Slow the sound (and the clock with it) while the picture waits for a frame: the pitch + /// eases down and back up, so playback never falls silent and never repeats itself. + pub fn speed(&self, speed: f32) { + self.feed.set_speed(speed); + } + + /// The audio clock has stood still: the stream is dead, or the feeder has nothing to give it. + /// The preview falls back to the frame clock instead of freezing on it. A stream that leaves + /// queued samples unpulled is dead after [`AUDIO_STALL`]; one with nothing queued is still + /// waiting for its feeder, which may be decoding a long file ([`AUDIO_STARVE`], #408). + pub fn stalled(&self, now: f64) -> bool { + let Ok(mut w) = self.last_clock.lock() else { return false }; + stall_watch(&mut w, now, self.feed.consumed(), self.feed.queued_frames() > 0) + } + /// The comp sample index audible now. pub fn clock(&self) -> i64 { clock_sample(&self.span, self.feed.consumed(), self.device.latency_frames()) @@ -271,6 +530,15 @@ impl AudioPlayback { let _ = t.join(); } } + + /// Stop without waiting for the feeder: the stall fallback runs on the UI thread while the + /// feeder may still be mixing a slow file, and blocking there is what the fallback avoids. The + /// thread leaves at its next stop check; dropping its handle lets it finish on its own. + pub fn stop_async(&mut self) { + self.device.stop(); + self.stop.store(true, Ordering::Relaxed); + let _ = self.thread.take(); + } } impl Drop for AudioPlayback { @@ -485,6 +753,89 @@ mod tests { assert_eq!(f.take_peaks(), [0.0, 0.0]); } + /// A slow first block is not a stall: the short watch starts when the queue gains samples, + /// not when the feeder begins mixing (#408). + #[test] + fn the_stall_watch_ignores_a_slow_first_block() { + let mut w = Watch::default(); + assert!(!stall_watch(&mut w, 0.0, 0, false), "the watch starts"); + assert!(!stall_watch(&mut w, 2.0, 0, false), "the feeder is still mixing"); + // The first block lands: the device finally has something to play, so the 0.5 s watch + // starts here rather than 2 s ago. + assert!(!stall_watch(&mut w, 2.0, 0, true), "a queue with samples is not a stall yet"); + assert!(!stall_watch(&mut w, 2.2, 0, true), "the device had 200 ms to pull"); + assert!(stall_watch(&mut w, 2.6, 0, true), "queued samples that are not pulled: dead"); + // Nothing queued again: only the long feeder watch applies. + assert!(!stall_watch(&mut w, 3.0, 0, false)); + assert!(!stall_watch(&mut w, 7.0, 0, false), "4 s: still inside the feeder's window"); + assert!(stall_watch(&mut w, 8.2, 0, false), "5.2 s with nothing to play: stalled"); + // The clock moving keeps it alive at any speed. + assert!(!stall_watch(&mut w, 8.3, 1600, false)); + } + + /// The slowed path counts a pull that found nothing, so the lead grows after a slowdown too. + #[test] + fn a_dry_varispeed_pull_counts_as_an_underrun() { + let f = AudioFeed::default(); + f.set_rate(48_000); + f.set_speed(0.5); + let mut out = vec![0.0f32; 64]; + f.pull(&mut out); + assert_eq!(f.underruns(), 1, "an empty queue counts like the ordinary path"); + f.push(&vec![0.1f32; 256]); + f.pull(&mut out); + assert_eq!(f.underruns(), 1, "a served pull does not"); + } + + /// A lead grows while the device runs dry and shrinks back once it is served in time. + #[test] + fn the_queue_lead_grows_on_underruns_and_settles_down() { + let mut lead = Lead::new(4_800); + let step = 2_400; + assert_eq!(lead.update(0, step), 4_800, "no underrun: the base lead"); + assert_eq!(lead.update(1, step), 7_200, "a dry pull adds a step"); + assert_eq!(lead.update(2, step), 9_600); + assert_eq!(lead.update(3, step), 12_000); + assert_eq!(lead.update(4, step), 14_400); + assert_eq!(lead.update(5, step), 16_800); + assert_eq!(lead.update(6, step), 19_200, "four times the base at most"); + // A second without a dry pull (250 sleeps of 4 ms) gives a little back, then more. + for n in 0..249 { + assert_eq!(lead.update(6, step), 19_200, "{n}"); + } + assert_eq!(lead.update(6, step), 9_600, "a step-of-four back"); + for _ in 0..250 { + lead.update(6, step); + } + assert_eq!(lead.update(6, step), 4_800, "and the base after another second"); + } + + /// A preview that waits for a frame slows the sound down (#947): it is still there, it just + /// takes fewer source frames per output frame, and the clock (consumed) slows with it. + #[test] + fn a_slow_speed_reads_between_frames_and_slows_the_clock() { + let f = AudioFeed::default(); + f.set_rate(48_000); + // A steady level: nothing is silent, and every output sample should be the same. + f.push(&vec![0.5f32; 400_000]); + let mut out = vec![0.0f32; 4800]; + f.pull(&mut out); + assert_eq!(f.consumed(), 2400, "real time takes whole frames"); + // Slowed down: the same tenth of a second of output takes fewer source frames. + f.set_speed(0.5); + f.pull(&mut out); + let slow = f.consumed() - 2400; + assert!(slow > 0 && slow < 2400, "a slowed read took {slow} of 2400 frames"); + assert!(f.speed() < 1.0, "the speed eases down: {}", f.speed()); + assert!(out.iter().all(|v| (*v - 0.5).abs() < 1e-6), "the sound is still there, not silence"); + // Back to real time: the speed eases up again. + f.set_speed(1.0); + for _ in 0..40 { + f.pull(&mut out); + } + assert!((f.speed() - 1.0).abs() < 0.01, "the speed came back: {}", f.speed()); + } + /// #408: starting a preview with sound returns at once even when reading the footage takes /// long (the first read of a long FLAC decodes all of it): the first block was mixed on the /// calling (UI) thread. The sound arrives from the feeder thread, and the clock waits for it. diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -164,6 +164,8 @@ pub struct Playback { /// Frames shown / dropped (for the Info/Preview readout). pub shown: u64, pub waiting: bool, + /// When the wait for a frame began (seconds), while [`Playback::waiting`]. + pub waiting_since: f64, pub fps: f64, /// The Preview panel shortcut that started this preview and what it plays. pub shortcut: effectcraft_engine::preview::PreviewShortcut, @@ -179,11 +181,12 @@ pub struct Playback { pub restore_max: Option>, /// Audio-only previews: the frame under the audible sample (the current time stays put). pub audio_frame: Option, - /// The preview's sound waits for frames: while the frames ahead aren't cached, every frame - /// shows as it renders, silently; the sound starts once the rest of the preview (or what - /// fits in the RAM preview) is cached, so it never plays over skipped frames (#103), or - /// sooner once the silent preview keeps real time with the frames ahead cached (#208). - pub audio_held: bool, + /// When a frame last went on screen (seconds): a preview that shows none for a while gives up + /// on the sound rather than play on over a still picture. + pub last_shown: f64, + /// When the sound was given up last (seconds; 0 when it was not), so it can be taken up again + /// once the picture moves, without thrashing. + pub gave_up_at: f64, /// When the last frames were shown (seconds, the last second's worth): the achieved rate. pub shown_at: std::collections::VecDeque, } @@ -192,6 +195,7 @@ impl Playback { /// A new frame went on screen at `now`. pub fn frame_shown(&mut self, now: f64) { self.shown += 1; + self.last_shown = now; self.shown_at.push_back(now); while self.shown_at.front().is_some_and(|t| now - t > 1.0) { self.shown_at.pop_front(); diff --git a/crates/ui-egui/tests/it/ui_audio_scrub.rs b/crates/ui-egui/tests/it/ui_audio_scrub.rs --- a/crates/ui-egui/tests/it/ui_audio_scrub.rs +++ b/crates/ui-egui/tests/it/ui_audio_scrub.rs @@ -1,6 +1,6 @@ //! Audio scrubbing: Ctrl/Cmd-dragging the current time plays one frame of the comp's mix at each //! new frame (each snippet replacing what is still queued); holding still repeats nothing; silent -//! comps open no output; an idle scrub closes it. Previews with audio wait for frames. +//! comps open no output; an idle scrub closes it. Previews with audio start their sound at once. use std::sync::{Arc, Mutex}; @@ -166,23 +166,19 @@ fn cmd_dragging_the_current_time_scrubs_and_an_idle_scrub_closes() { assert!(h.state().scrub.is_none() && fake.0.lock().unwrap().1, "closed and stopped"); } -/// A preview with audio waits for frames like a silent one (#103): while the frames ahead aren't -/// cached, every frame shows as it renders, silently; the sound starts once the rest of the -/// preview is cached, and a frame that isn't cached stops it instead of being skipped. +/// A preview with audio starts its sound with the first frame (#947): the opening is heard +/// instead of being skipped while frames render, and no frame is skipped before it. A frame that +/// isn't rendered then doesn't stop the sound: the preview slows it down, and the clock with it, +/// until the picture catches up. #[test] -fn preview_with_audio_shows_every_frame_and_sounds_once_cached() { +fn preview_with_audio_starts_the_sound_at_once() { let fake = Fake::default(); let (mut app, cid) = setup(true, &fake); - // Hold rendering in the test: the cache cannot fill ahead of what the test releases, so the - // silently shown frames and the moment the sound starts do not depend on the host's threads. + // Hold rendering in the test: a frame is cached only when the test releases it. let gate = GatedFrames::default(); let renderer: Arc = Arc::new(gate.clone()); app.frames.set_remote(Some(renderer)); - let ahead = (app.frames.parallelism() * 2).max(4); - app.session.execute("comp.settings", json!({"comp": cid.0, "duration": (4 * ahead) as f64 / 30.0})).unwrap(); let mut h = Harness::builder().with_size(egui::vec2(1400.0, 900.0)).with_step_dt(1.0 / 30.0).build_eframe(|_| app); - // Playback measures time on the app clock, never `ctx.input(|i| i.time)`; drive that clock - // explicitly at one frame per step. h.state_mut().use_manual_clock(); h.run_steps(2); gate.discard(&mut h); @@ -192,70 +188,60 @@ fn preview_with_audio_shows_every_frame_and_sounds_once_cached() { }; let now = h.state().now(); h.state_mut().play(now); - assert!(h.state().playback.audio_held && h.state().audio.is_none(), "nothing is cached: no sound yet"); + assert!(h.state().audio.is_some(), "the sound starts with the first frame"); + assert!(fake.0.lock().unwrap().0.is_some(), "the output was given the feed"); + // The fake output never pulls, so take one frame's worth of samples each step: the audio + // clock then moves one comp frame per step, as a device playing in real time would. + let mut one_frame = vec![0.0f32; 3200]; let mut last = frame(&h); - let mut shown = 0; - // Show frames while only those frames are cached: they play silently first. - for _ in 0..4 { + let mut moved = 0; + for _ in 0..60 { let target = last + 1; gate.render(&mut h, cid, &[target]); - h.state_mut().clock_mut().advance(1.0 / 30.0); + if let Some(a) = &h.state().audio { + a.feed.pull(&mut one_frame); + } h.step(); let f = frame(&h); - // Accumulating 1/30 in f64 can leave the same frame up for one more step; it must never - // jump over a frame. assert!(f == last || f == last + 1, "{last} → {f}: a frame was skipped"); - shown += usize::from(f != last); + if f != last { + moved += 1; + } last = f; - } - // A frame that has not rendered holds the clock at the frame on screen; once it renders it is - // shown, never jumped over. This exercises the production hold path, not a test-only shortcut. - let waiting = last; - h.state_mut().clock_mut().advance(1.0 / 30.0); - h.step(); - assert_eq!(frame(&h), waiting, "an uncached frame holds the preview instead of being skipped"); - gate.render(&mut h, cid, &[waiting + 1]); - // f64 clock accumulation need not put the next step exactly on the next frame boundary; - // keep stepping until the held frame appears, and never allow it to jump past that frame. - let mut showed_waiting = false; - for _ in 0..4 { - h.state_mut().clock_mut().advance(1.0 / 30.0); - h.step(); - let f = frame(&h); - assert!(f == waiting || f == waiting + 1, "{waiting} → {f}: the held frame was skipped"); - if f == waiting + 1 { - showed_waiting = true; + if moved >= 3 { break; } } - assert!(showed_waiting, "the waiting frame is shown once it renders"); - shown += 1; - last = waiting + 1; - assert!(shown > 0, "frames played silently first"); - assert!(h.state().audio.is_none() && h.state().playback.audio_held, "the rest of the range is not cached: the sound waits"); - // Cache everything ahead; only then may the sound start. - let end = h.state().playback.plan.expect("a preview plan").end; - let rest: Vec = ((last + 1)..=end).collect(); - gate.render(&mut h, cid, &rest); - h.step(); - assert!(h.state().audio.is_some() && fake.0.lock().unwrap().0.is_some(), "the sound started once the frames ahead were cached"); - // A frame that isn't cached stops the sound rather than being skipped. - gate.discard(&mut h); + assert!(moved >= 3, "the preview moved with the sound"); + // A frame that isn't rendered holds the preview and slows the sound instead of stopping it. + let waiting = last; h.state().frames.set_budget(0); h.state().frames.clear(); + gate.discard(&mut h); + if let Some(a) = &h.state().audio { + a.feed.pull(&mut one_frame); + } h.step(); - assert!(h.state().audio.is_none() && h.state().playback.audio_held && h.state().playback.playing); + assert_eq!(frame(&h), waiting, "an uncached frame holds the preview instead of being skipped"); + assert!(h.state().playback.waiting, "the picture waits for the frame"); + let audio = h.state().audio.as_ref().expect("the sound keeps playing while the picture waits"); + // Top the queue up: the feeder mixes in real time, and the test reads far faster than that. + audio.feed.clear(); + audio.feed.push(&vec![0.5f32; 200_000]); + audio.feed.pull(&mut one_frame); + assert!(one_frame.iter().any(|v| v.abs() > 0.01), "the sound is still there, not silence"); + for _ in 0..40 { + audio.feed.pull(&mut one_frame); + } + assert!(audio.feed.speed() < 0.9, "the sound slowed down: {}", audio.feed.speed()); } -/// A preview whose frames render in real time sounds without waiting for the whole work area to -/// be cached (#208): once the silent preview has kept up for a moment with the frames ahead -/// cached, the sound starts, still without skipping a frame before it. +/// A long preview's sound starts at once too — it no longer waits for the work area to be cached +/// (#208) — and keeps playing while frames render in time. #[test] -fn preview_with_audio_sounds_once_rendering_keeps_up() { +fn a_long_preview_starts_its_sound_at_once_and_keeps_it() { let fake = Fake::default(); let (mut app, cid) = setup_long(true, &fake, 30.0); - // Release only the frames this test chooses, and drive the app clock: rendering cannot race - // the clock, so a frame is never skipped by a slow or loaded test host. let gate = GatedFrames::default(); let renderer: Arc = Arc::new(gate.clone()); app.frames.set_remote(Some(renderer)); @@ -263,37 +249,24 @@ fn preview_with_audio_sounds_once_rendering_keeps_up() { h.state_mut().use_manual_clock(); h.run_steps(2); gate.discard(&mut h); - let ahead = (h.state().frames.parallelism() * 2).max(4) as i64; let frame = |h: &Harness<'_, EffectcraftApp>| { let app = h.state(); app.session.project.comp(cid).unwrap().frame_rate.frame_at(app.session.time()) }; let start = h.state().now(); h.state_mut().play(start); - assert!(h.state().playback.audio_held && h.state().audio.is_none(), "nothing is cached: no sound yet"); + assert!(h.state().audio.is_some() && fake.0.lock().unwrap().0.is_some(), "the sound starts at once"); + let mut one_frame = vec![0.0f32; 3200]; let mut last = frame(&h); - let mut shown = 0; - // Two seconds of playback at one frame per clock step: keep the prefetch window cached, but - // not the rest of the 30 s comp, so the sound has to start by the real-time rule. - for step in 1..=60 { + for _ in 0..60 { let target = last + 1; - let window: Vec = (target..target + ahead).collect(); + let window: Vec = (target..target + 4).collect(); gate.render(&mut h, cid, &window); - // Set the clock to this step's exact time rather than adding 1/30 repeatedly: accumulated - // f64 error can put one step just below a frame boundary and the next just above it, so - // `floor(elapsed × fps)` would otherwise jump from frame N to N + 2 for a one-frame step. - let now = h.state().now(); - let tick_time = start + step as f64 / 30.0; - h.state_mut().clock_mut().advance(tick_time - now); - h.step(); - if h.state().audio.is_some() { - break; + if let Some(a) = &h.state().audio { + a.feed.pull(&mut one_frame); } - let f = frame(&h); - assert!(f == last || f == last + 1, "{last} → {f}: a frame was skipped"); - shown += usize::from(f != last); - last = f; + h.step(); + last = frame(&h); + assert!(h.state().audio.is_some() && h.state().playback.playing, "the sound kept playing at frame {last}"); } - assert!(h.state().audio.is_some() && fake.0.lock().unwrap().0.is_some(), "the sound started within 2 s, at frame {last} of 900"); - assert!(shown >= 3, "the sound starts only after the silent preview kept up for a moment"); } diff --git a/docs/architecture.md b/docs/architecture.md --- a/docs/architecture.md +++ b/docs/architecture.md @@ -229,10 +229,12 @@ keys, with a nesting-depth guard. **Audio** is mixed by `render::audio::mix_comp` in blocks: Audio switch, solo, Audio Levels and the Effect > Audio effects (`effects::audio_fx`, applied per layer with a pre-roll so blocks are -independent). Export and preview playback share it; the desktop app plays it through cpal and the -audio clock drives preview playback (`ui-egui::audio`). The sound starts once the frames ahead -are cached; until then, and whenever playback reaches a frame that isn't cached, every frame -shows as it renders, silently, rather than being skipped. Layer and frame caches ignore the +independent). Export and preview playback share it; the desktop app plays it through cpal on one +output it opens once and keeps open for the session (the app warms it at startup), and the audio +clock drives preview playback (`ui-egui::audio`). A preview starts its sound with its first +frame, and a frame it must wait for slows it down rather than stopping it: the sound is read at +an eased speed below real time (`AudioFeed::set_speed`, varispeed) and the clock slows with it, +so playback eases down and back up and never falls silent. Layer and frame caches ignore the switches that don't change pixels (Audio, Lock, Shy, Hide Shy Layers: `Switches::pixels`, `Comp::same_pixels`), so toggling them keeps the cached frames. diff --git a/docs/gaps.md b/docs/gaps.md --- a/docs/gaps.md +++ b/docs/gaps.md @@ -168,7 +168,8 @@ The most important missing piece: it turns every other estimate here into a meas the viewer; an effect onto a layer in the viewer: #85, #88, #89), `ui_render_queue` (a template chosen from a menu moved to fit the window, #117), `angle_revolutions_scrub_and_take_typing` (#93), `audio_lock_and_shy_switches_keep_the_frames` and - `preview_with_audio_shows_every_frame_and_sounds_once_cached` (#103), and + `preview_with_audio_starts_the_sound_at_once` (#103; the sound starts with the first frame + since #947), and `a_frame_whose_render_panics_is_released` (#106). - Follow-up angle-input regressions in `ui_angles` (#141), checked headlessly on Linux and Windows, verify that Escape cancels edits and non-finite entries leave the value and undo @@ -178,7 +179,7 @@ The most important missing piece: it turns every other estimate here into a meas `full_resolution_frames_wider_than_the_texture_limit_fit` (#201), the desktop app's `no_arguments_start_an_empty_project` (#204) and `device_limits_fit_the_adapter` (#198), `audio_spectrum_follows_time_through_the_layer_cache` (#209), - `preview_with_audio_sounds_once_rendering_keeps_up` (#208), `ui_timeline_shapes` (the Add menu + `a_long_preview_starts_its_sound_at_once_and_keeps_it` (#208; likewise #947), `ui_timeline_shapes` (the Add menu and Change Renderer, #206; deselecting below the layers, Alt+Shift+P and the toolbar's stroke, #205; linked Mask Feather, #203), `dragging_in_the_empty_area_box_selects_items` and `selecting_a_mask_selects_its_points` (#203), and `input_keys_ignore_later_effects` (#212).