diff --git a/crates/ui-egui/examples/bench/playback.rs b/crates/ui-egui/examples/bench/playback.rs --- a/crates/ui-egui/examples/bench/playback.rs +++ b/crates/ui-egui/examples/bench/playback.rs @@ -23,7 +23,7 @@ use std::time::{Duration, Instant}; use filmcraft_engine::Session; use filmcraft_project::{ItemId, ParamValue, Project, SequenceSettings, TrackKind, find_effect, resolve_auto_points}; use filmcraft_time::{FrameRate, Tick, TimeRange}; -use filmcraft_ui_egui::frames::{FrameKey, FrameServer, JobRecord, PREROLL_TIMEOUT_S, PlaybackMeter, Target, process_cpu_time, thread_cpu_time}; +use filmcraft_ui_egui::frames::{FrameKey, FrameServer, JobRecord, Lookback, PREROLL_TIMEOUT_S, PlaybackMeter, Target, process_cpu_time, thread_cpu_time}; use serde_json::{Value, json}; use crate::fixtures::fixture; @@ -147,7 +147,7 @@ impl Display { if let Some((comp, dev)) = self.gpu.as_mut() { let plan = server.get_plan(&key).map(|p| (key, p)); exact = plan.is_some(); - if let Some((k, plan)) = plan.or_else(|| server.nearest_plan(key, 6)) + if let Some((k, plan)) = plan.or_else(|| server.nearest_plan(key, Lookback::backward(6))) && self.last != Some(k) { let before = comp.uploaded_bytes; diff --git a/crates/ui-egui/src/frames.rs b/crates/ui-egui/src/frames.rs --- a/crates/ui-egui/src/frames.rs +++ b/crates/ui-egui/src/frames.rs @@ -191,6 +191,28 @@ pub struct FrameStats { pub job_ms: f64, /// The most recent jobs: (source ms, render ms). pub recent: VecDeque<(f32, f32)>, + /// The same jobs' targets and slowest source items, for an estimate of the source being + /// played ([`FrameServer::plan_lookback`]). + pub recent_origins: VecDeque, +} + +/// Where a frame job's timing sample comes from: its target, and the source item whose fetch took +/// longest (the one that set the job's pace; None when it fetched none, a render preview). +/// Different clips of a sequence share its target, so the target alone does not say which source +/// a sample measured. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct SampleOrigin { + pub target: Target, + pub source: Option, +} + +/// The samples of `recent` (one `origins` entry each) that measure `target`'s current source: its +/// samples whose slowest source is that of its newest sample. When playback moves from a fast clip +/// to a slow one, the first sample of the slow clip replaces the whole estimate instead of being +/// outvoted by the fast clip's samples until they age out of the window. +pub fn current_source_samples(recent: &[(f32, f32)], origins: &[SampleOrigin], target: Target) -> Vec<(f32, f32)> { + let Some(source) = origins.iter().rev().find(|o| o.target == target).map(|o| o.source) else { return Vec::new() }; + recent.iter().zip(origins).filter(|(_, o)| o.target == target && o.source == source).map(|(r, _)| *r).collect() } impl FrameStats { @@ -275,12 +297,20 @@ pub fn process_cpu_time() -> Option { thread_local! { /// (wall, cpu) spent in source `video_frame` calls by this worker during the current job. static SOURCE_TIME: Cell<(Duration, Duration)> = const { Cell::new((Duration::ZERO, Duration::ZERO)) }; + /// The source item whose single fetch took longest during the current job, and that time. + static SLOWEST_SOURCE: Cell> = const { Cell::new(None) }; /// This worker is profiling the current job (thread CPU of source calls is measured). static PROFILING: Cell = const { Cell::new(false) }; } /// Run `f`, adding its wall/CPU time to this thread's source-time accumulator. fn timed_source(f: impl FnOnce() -> R) -> R { + timed_source_of(None, f) +} + +/// [`timed_source`] for a fetch from `item`, which becomes the job's slowest source if no fetch +/// took longer (see [`SampleOrigin`]). +fn timed_source_of(item: Option, f: impl FnOnce() -> R) -> R { let cpu = || if PROFILING.with(Cell::get) { thread_cpu_time().unwrap_or_default() } else { Duration::ZERO }; let (w0, c0) = (Instant::now(), cpu()); let r = f(); @@ -289,18 +319,25 @@ fn timed_source(f: impl FnOnce() -> R) -> R { let (w, c) = s.get(); s.set((w + dw, c + dc)); }); + if let Some(item) = item { + SLOWEST_SOURCE.with(|s| { + if s.get().is_none_or(|(_, t)| dw > t) { + s.set(Some((item, dw))); + } + }); + } r } -/// A media source that times its `video_frame` calls (see [`JobRecord::source_wall`]). -struct TimedSource(filmcraft_media::SharedSource); +/// A media source of `item` that times its `video_frame` calls (see [`JobRecord::source_wall`]). +struct TimedSource(filmcraft_media::SharedSource, ItemId); impl filmcraft_media::MediaSource for TimedSource { fn info(&self) -> &filmcraft_media::MediaInfo { self.0.info() } fn video_frame(&self, req: filmcraft_media::FrameRequest) -> filmcraft_media::Result> { - timed_source(|| self.0.video_frame(req)) + timed_source_of(Some(self.1), || self.0.video_frame(req)) } fn audio(&self, start: i64, frames: usize, sample_rate: u32) -> filmcraft_media::Result { self.0.audio(start, frames, sample_rate) @@ -319,7 +356,7 @@ struct JobProvider { impl filmcraft_render::SourceProvider for JobProvider { fn source(&self, item: ItemId) -> Option { let s = self.inner.pool.monitor_source(item, self.inner.source(item)?); - Some(Arc::new(TimedSource(s))) + Some(Arc::new(TimedSource(s, item))) } } @@ -437,6 +474,9 @@ pub const PREROLL_TIMEOUT_S: f64 = 0.5; /// first frame ahead of the playhead worth starting (nearer ones would be late however soon they /// start) and `stride` the spacing the workers can finish in real time: the frames in between /// are dropped on purpose and evenly, instead of every frame arriving late. +/// +/// This is the *render* cost, which spreads over the workers. A source that decodes one picture at +/// a time is a different matter: see [`source_cannot_keep_up`] and [`SOURCE_BOUND_AHEAD`]. pub fn playback_plan(render_cost: f64, fps: f64, workers: usize) -> (i64, i64) { let frames_per_job = render_cost * fps; let load = frames_per_job * 1.15 / workers.max(1) as f64; @@ -446,6 +486,84 @@ pub fn playback_plan(render_cost: f64, fps: f64, workers: usize) -> (i64, i64) { (frames_per_job.ceil() as i64, load.ceil() as i64) } +/// Frames queued ahead of the playhead while the source cannot make the frame rate: a short queue. +/// The decoder is the serialized stage, so the queue *is* the latency of every picture — and of a +/// user's scrub, which waits behind it. +pub const SOURCE_BOUND_AHEAD: i64 = 3; + +/// How many of the most recent frame jobs the source estimate reads (`source_estimate`): old +/// samples (a burst of scrubs, a previous playback) age out instead of pinning it. +pub const SOURCE_WINDOW: usize = 32; + +/// Whether one decoder cannot produce the sequence's frame rate (`source_cost` is one picture's +/// decode plus readback, in seconds; the workers cannot share it). +/// +/// Spacing requests out does not help such a source: a long-GOP stream decodes the frames between +/// two requests anyway, so asking further ahead only makes each job costlier to reach, and the +/// picture the user is waiting for (or their own scrub) queues behind those multi-second jobs. What +/// helps is asking for the frames just ahead ([`SOURCE_BOUND_AHEAD`]) and letting the presenter show +/// the newest picture the source has ([`FrameServer::plan_lookback`]) — dropping frames instead of +/// freezing (#2696). +pub fn source_cannot_keep_up(source_cost: f64, fps: f64) -> bool { + source_cost * fps > 1.0 +} + +/// The most frames [`FrameServer::plan_lookback`] reaches back: a second at 240 fps. The monitors +/// probe each frame in that range, so a hostile frame rate (zero frame duration: infinite fps) +/// must not turn it into an endless loop. +pub const MAX_LOOKBACK: i64 = 240; + +/// Where a monitor may look for a picture while the exact frame is not ready: up to `frames` +/// frames *behind* the playhead in the play direction `dir` (frames already passed: earlier ones +/// forward, later ones in reverse playback, where a picture still finishing is above the playhead). +/// Paused, `dir` is 1: a scrub shows an earlier picture, such as the keyframe a seek decodes from. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Lookback { + pub frames: i64, + pub dir: i64, +} + +impl Lookback { + /// Looking back towards earlier frames (paused, forward playback, the reference frame). + pub fn backward(frames: i64) -> Self { + Lookback { frames, dir: 1 } + } + + /// The frames to try instead of `frame`, nearest first. + pub fn frames_before(self, frame: i64) -> impl Iterator { + let dir = if self.dir < 0 { -1 } else { 1 }; + (1..=self.frames.clamp(0, MAX_LOOKBACK)).map(move |d| frame.saturating_sub(d * dir)) + } +} + +/// [`FrameServer::plan_lookback`] for a measured picture `latency` (s; None before any job ran) +/// at `fps`: the latency in frames, at least 2, at most a second of frames and [`MAX_LOOKBACK`]. +pub fn lookback_frames(latency: Option, fps: f64) -> i64 { + let Some(latency) = latency else { return 6 }; + let fps = if fps.is_finite() { fps.clamp(2.0, MAX_LOOKBACK as f64) } else { MAX_LOOKBACK as f64 }; + let frames = latency * fps; + if frames.is_finite() { frames.ceil().clamp(2.0, fps) as i64 } else { 2 } +} + +/// How far from the playhead (in frames, either way) playback keeps jobs for its view: the frames +/// it asks for (`lead + ahead * stride`, times `step`), plus slack for frames asked a refresh ago +/// while it renders. A source-bound playback gets no slack: frames still queued from the deeper +/// render-bound prefetch would otherwise run on the serialized decoder before the short queue. +pub fn playback_span(lead: i64, ahead: i64, stride: i64, step: i64, source_bound: bool) -> i64 { + let slack = if source_bound { 0 } else { 26 }; + lead.saturating_add(ahead.saturating_mul(stride)).saturating_mul(step).saturating_add(slack) +} + +/// Whether playback keeps a job for the frame `off` frames from the playhead (`dir`: the play +/// direction): ahead of the playhead and within `span`, or at most `behind` frames late. Queued +/// jobs get `behind` 0; a source-bound playback lets *running* ones finish while they are within +/// the monitor's look-back ([`FrameServer::plan_lookback`]): a cancelled job's result is dropped, +/// so cancelling the picture a decoder is finishing would leave the monitor nothing newer to show. +pub fn keeps_playback_job(off: i64, dir: i64, span: i64, behind: i64) -> bool { + let ahead = off.saturating_mul(dir); + if ahead >= 0 { ahead < span } else { ahead >= behind.saturating_neg() } +} + impl FrameServer { /// Settings ▸ Memory ▸ frame cache budget (bytes); evicts down to it right away. pub fn set_cache_budget(&self, bytes: usize) { @@ -559,10 +677,11 @@ impl FrameServer { } /// Nearest cached plan at or before `frame`. - pub fn nearest_plan(&self, key: FrameKey, max_back: i64) -> Option<(FrameKey, Arc)> { + /// The plan of `key`'s frame, else of the nearest frame within `back` (see [`Lookback`]). + pub fn nearest_plan(&self, key: FrameKey, back: Lookback) -> Option<(FrameKey, Arc)> { let mut g = self.shared.plans.lock().unwrap_or_else(PoisonError::into_inner); - (0..=max_back).find_map(|d| { - let k = FrameKey { frame: key.frame - d, ..key }; + std::iter::once(key.frame).chain(back.frames_before(key.frame)).find_map(|frame| { + let k = FrameKey { frame, ..key }; g.get(&k).map(|p| (k, p)) }) } @@ -643,7 +762,16 @@ impl FrameServer { } // During preroll the clock has not started: only the frames that gate start, not a // prefetch run that walks the rest of a LongGOP and evicts the playhead pictures. - let ahead = if preroll { (PREROLL_FRAMES - 1).max(0) } else { prefetch_depth(speed) }; + // the frames the monitor shows per second: below 1x that is fewer than the frame rate + let shown_fps = rate.as_f64() * speed.abs() / step as f64; + let source_bound = !preroll && source_cannot_keep_up(self.source_cost(key.target), shown_fps); + let ahead = if preroll { + (PREROLL_FRAMES - 1).max(0) + } else if source_bound { + SOURCE_BOUND_AHEAD + } else { + prefetch_depth(speed) + }; for i in 0..ahead { let f = key.frame + (lead + i * stride) * dir * step; if f >= 0 { @@ -653,8 +781,10 @@ impl FrameServer { } } let (target, rev, cur) = (key.target, key.revision, key.frame); - let span = (lead + ahead * stride) * step + 26; - self.retain_queue(|k| k.target != target || (k.revision == rev && (k.frame - cur) * dir >= 0 && (k.frame - cur).abs() < span)); + let span = playback_span(lead, ahead, stride, step, source_bound); + let behind = if source_bound { self.plan_lookback(key.target, rate.as_f64()) } else { 0 }; + let keep = |k: &FrameKey, behind| k.target != target || (k.revision == rev && keeps_playback_job(k.frame - cur, dir, span, behind)); + self.retain_jobs(|k| keep(k, 0), |k| keep(k, behind)); } /// Whether playback can start at `key`: the next [`PREROLL_FRAMES`] frames (up to `last`) @@ -664,6 +794,56 @@ impl FrameServer { (0..PREROLL_FRAMES).map(|i| key.frame + i * step).filter(|f| *f >= 0 && *f <= last).all(|f| self.is_ready(&FrameKey { frame: f, ..key })) } + /// The estimate playback paces on, from the recent frame jobs' source phase (ms, oldest + /// first): `(cost, latency)`, both in seconds. + /// + /// `cost` is the decoder's own time for one picture, taken as a low quantile: the job that + /// waited least for its turn is the one not queued behind others, so waiting does not feed back + /// into the plan (a deep queue would otherwise keep raising the estimate). `latency` is the + /// median, which *is* what a picture waits when the source is behind — what the monitor's + /// lookback needs. + /// + /// Only the most recent [`SOURCE_WINDOW`] jobs count, so a burst of scrubs (each seeking from a + /// keyframe, seconds long) ages out instead of pinning the estimate at "cannot keep up". + pub fn source_estimate(recent: &[(f32, f32)]) -> Option<(f64, f64)> { + let tail = recent.len().saturating_sub(SOURCE_WINDOW); + let mut v: Vec = recent.get(tail..)?.iter().map(|r| r.0).collect(); + if v.is_empty() { + return None; + } + v.sort_by(|a, b| a.total_cmp(b)); + // `.get` per AGENTS §0.4: a quantile of a sorted non-empty vector, never an index panic + let at = |q: f64| f64::from(v.get(((v.len() - 1) as f64 * q).round() as usize).copied().unwrap_or(0.0)) / 1e3; + Some((at(0.1), at(0.5))) + } + + /// [`source_estimate`](Self::source_estimate) of this server's recent jobs for `target`'s + /// current source ([`current_source_samples`]): thumbnails, another monitor's frames or another + /// clip of the sequence say nothing about the source being played. + fn source_estimate_now(&self, target: Target) -> Option<(f64, f64)> { + let st = self.shared.stats.lock().unwrap_or_else(PoisonError::into_inner); + let (recent, origins): (Vec<(f32, f32)>, Vec) = (st.recent.iter().copied().collect(), st.recent_origins.iter().copied().collect()); + drop(st); + Self::source_estimate(¤t_source_samples(&recent, &origins, target)) + } + + /// The current per-frame source cost estimate (s) of `target`, see [`playback_plan`]. + fn source_cost(&self, target: Target) -> f64 { + self.source_estimate_now(target).map_or(0.0, |(cost, _)| cost) + } + + /// How far behind the playhead a plan may be and still be worth showing, in frames: the measured + /// latency of a picture, bounded to a second of frames. + /// + /// A second is deliberate: seeking in a long-GOP stream decodes from the previous keyframe + /// (0.96 s in the reporter's clip), so showing that keyframe while the exact frame decodes is + /// what makes a scrub feel immediate — while anything older is a different shot. It also covers + /// a source that cannot make the frame rate: the monitor shows the newest picture it has rather + /// than freezing (#2696). + pub fn plan_lookback(&self, target: Target, fps: f64) -> i64 { + lookback_frames(self.source_estimate_now(target).map(|(_, latency)| latency), fps) + } + /// The current per-frame render cost estimate (s), see [`playback_plan`]. pub fn render_cost(&self) -> f64 { *self.shared.render_cost.lock().unwrap_or_else(PoisonError::into_inner) @@ -673,9 +853,15 @@ impl FrameServer { /// Jobs already running for such frames are cancelled: a worker blocked behind a decoder /// would otherwise go on to decode a frame playback has passed, possibly from its keyframe. pub fn retain_queue(&self, keep: impl Fn(&FrameKey) -> bool) { - self.shared.queue.lock().unwrap_or_else(PoisonError::into_inner).retain(|j| keep(&j.key)); + self.retain_jobs(&keep, &keep); + } + + /// [`retain_queue`](Self::retain_queue) with one rule for queued jobs and another for running + /// ones (cancelled when they fail `keep_running`). + pub fn retain_jobs(&self, keep_queued: impl Fn(&FrameKey) -> bool, keep_running: impl Fn(&FrameKey) -> bool) { + self.shared.queue.lock().unwrap_or_else(PoisonError::into_inner).retain(|j| keep_queued(&j.key)); for (k, c, _) in self.shared.in_flight.lock().unwrap_or_else(PoisonError::into_inner).iter() { - if !keep(k) { + if !keep_running(k) { c.store(true, Ordering::Relaxed); } } @@ -900,6 +1086,7 @@ fn run_job(sh: &Shared, job: &Job, pool: &Arc, services: &Arc, services: &Arc= FrameStats::RECENT { + st.recent_origins.pop_front(); + } + st.recent_origins.push_back(SampleOrigin { target: job.key.target, source: SLOWEST_SOURCE.with(|s| s.get()).map(|(item, _)| item) }); } if profiling { let (source_wall, source_cpu) = SOURCE_TIME.with(|s| s.get()); @@ -1059,7 +1250,10 @@ fn render_job(job: &Job, pool: &Arc, services: &Arc, pr #[cfg(test)] mod tests { - use super::{PlaybackMeter, Target, background_work, playback_plan}; + use super::{ + Lookback, PlaybackMeter, SOURCE_BOUND_AHEAD, SampleOrigin, Target, background_work, current_source_samples, keeps_playback_job, lookback_frames, + playback_plan, playback_span, prefetch_depth, source_cannot_keep_up, + }; use filmcraft_project::ItemId; #[test] @@ -1106,16 +1300,120 @@ mod tests { assert!(!filmcraft_engine::autosave::Preferences::default().playback.draft_decode, "off by default"); } + #[test] + fn source_estimate_ignores_the_queue_and_ages_out_scrubs() { + use super::{FrameServer, SOURCE_WINDOW}; + let source_estimate = FrameServer::source_estimate; + // The cost is a low quantile: a queue that stretches most jobs must not stretch the cost + // (that would feed back into the plan), while the latency is the median and does see it. + let queued: Vec<(f32, f32)> = (0..SOURCE_WINDOW).map(|i| if i % 8 == 0 { (30.0, 1.0) } else { (900.0, 1.0) }).collect(); + let (cost, latency) = source_estimate(&queued).expect("samples"); + assert!((cost - 0.030).abs() < 1e-6, "cost {cost}"); + assert!(latency > 0.5, "latency {latency}"); + // A burst of scrubs (each seeking from a keyframe, seconds long) ages out: only the last + // SOURCE_WINDOW jobs count, so playback afterwards is not pinned at "cannot keep up". + let mut burst: Vec<(f32, f32)> = vec![(4000.0, 1.0); 200]; + burst.extend((0..SOURCE_WINDOW).map(|_| (28.0f32, 1.0f32))); + let (cost, latency) = source_estimate(&burst).expect("samples"); + assert!((cost - 0.028).abs() < 1e-6 && (latency - 0.028).abs() < 1e-6, "cost {cost}, latency {latency}"); + // Nothing ran yet. + assert!(source_estimate(&[]).is_none()); + } + #[test] fn playback_plan_skips_frames_only_when_workers_cannot_keep_up() { // decode-only / light frames: every frame from the playhead on assert_eq!(playback_plan(0.0, 24.0, 6), (1, 1)); assert_eq!(playback_plan(0.15, 24.0, 6), (1, 1)); + // a serialized source that cannot make the frame rate gets a short queue, never a stride: + // asking further ahead would only decode more frames before the one that is waited for + assert!(source_cannot_keep_up(0.030, 50.0)); + assert!(!source_cannot_keep_up(0.012, 50.0)); + assert!(SOURCE_BOUND_AHEAD < prefetch_depth(1.0), "shorter than the render-bound lookahead"); // 0.5 s of effects per frame at 24 fps on 6 workers: 12 frames pass while one renders, // so start 12 ahead, and every 3rd frame is what 6 workers can finish assert_eq!(playback_plan(0.5, 24.0, 6), (12, 3)); } + #[test] + fn lookback_is_bounded_for_any_frame_rate() { + assert_eq!(lookback_frames(None, 50.0), 6); + // 0.8 s of picture latency at 50 fps: 40 frames; at most a second + assert_eq!(lookback_frames(Some(0.8), 50.0), 40); + assert_eq!(lookback_frames(Some(5.0), 50.0), 50); + assert_eq!(lookback_frames(Some(0.001), 50.0), 2); + // a zero frame duration (infinite fps), NaN or a negative latency stay bounded + for (latency, fps) in [(0.8, f64::INFINITY), (0.8, f64::NAN), (f64::NAN, 50.0), (f64::INFINITY, 50.0), (-1.0, 50.0), (0.8, 0.0)] { + let n = lookback_frames(Some(latency), fps); + assert!((2..=super::MAX_LOOKBACK).contains(&n), "{latency} s at {fps} fps: {n}"); + } + } + + #[test] + fn source_bound_playback_keeps_a_short_queue_and_the_pictures_being_finished() { + // switching to source-bound: frames queued under the render-bound depth fall outside + let span = playback_span(1, SOURCE_BOUND_AHEAD, 1, 1, true); + assert!((1..=SOURCE_BOUND_AHEAD).all(|off| keeps_playback_job(off, 1, span, 0)), "the frames it asks for"); + assert!(!keeps_playback_job(prefetch_depth(1.0), 1, span, 0), "a deep prefetch frame is dropped"); + assert!(keeps_playback_job(prefetch_depth(1.0), 1, playback_span(1, prefetch_depth(1.0), 1, 1, false), 0)); + // behind the playhead: queued jobs go, a running one within the look-back may finish + assert!(!keeps_playback_job(-2, 1, span, 0)); + assert!(keeps_playback_job(-2, 1, span, 5)); + assert!(!keeps_playback_job(-6, 1, span, 5), "older than the look-back"); + // the look-back, not the (short) forward span, bounds how late a running job may be + assert!(keeps_playback_job(-10, 1, span, 40) && span < 10); + // reverse play: "behind" is the other way + assert!(keeps_playback_job(-1, -1, span, 0) && !keeps_playback_job(1, -1, span, 0)); + } + + #[test] + fn source_estimate_follows_the_source_being_played() { + use super::FrameServer; + let seq = Target::SequencePlan(ItemId(1)); + let (fast, slow) = (Some(ItemId(2)), Some(ItemId(3))); + // a fast clip played for 31 jobs, then the playhead reached a slow clip of the same + // sequence (the same target): one slow sample + let mut recent: Vec<(f32, f32)> = vec![(5.0, 1.0); 31]; + let mut origins: Vec = vec![SampleOrigin { target: seq, source: fast }; 31]; + recent.push((80.0, 1.0)); + origins.push(SampleOrigin { target: seq, source: slow }); + // a thumbnail of another target finished last: it is not the playback source + recent.push((1.0, 1.0)); + origins.push(SampleOrigin { target: Target::Item(ItemId(9)), source: Some(ItemId(9)) }); + let samples = current_source_samples(&recent, &origins, seq); + assert_eq!(samples, vec![(80.0, 1.0)], "only the slow clip's samples"); + let (cost, _) = FrameServer::source_estimate(&samples).expect("a sample"); + assert!(source_cannot_keep_up(cost, 50.0), "the slow clip makes playback source-bound at once ({cost} s)"); + // by target alone the fast clip's samples kept the estimate low + let by_target: Vec<(f32, f32)> = recent.iter().zip(&origins).filter(|(_, o)| o.target == seq).map(|(r, _)| *r).collect(); + let (cost, _) = FrameServer::source_estimate(&by_target).expect("samples"); + assert!(!source_cannot_keep_up(cost, 50.0), "{cost} s"); + // and back on the fast clip, the slow samples stop counting + recent.push((5.0, 1.0)); + origins.push(SampleOrigin { target: seq, source: fast }); + assert!(current_source_samples(&recent, &origins, seq).iter().all(|r| r.0 == 5.0)); + // nothing for a target that ran no job + assert!(current_source_samples(&recent, &origins, Target::Item(ItemId(4))).is_empty()); + } + + #[test] + fn lookback_follows_the_play_direction() { + // paused or playing forward: earlier frames; playing in reverse: later ones + assert_eq!(Lookback::backward(3).frames_before(10).collect::>(), vec![9, 8, 7]); + assert_eq!(Lookback { frames: 3, dir: -1 }.frames_before(10).collect::>(), vec![11, 12, 13]); + assert_eq!(Lookback::backward(0).frames_before(10).count(), 0); + // a reverse source-bound playback keeps a running job 3 frames above the playhead, and the + // monitor's look-back reaches it (it used to search below the playhead only) + let span = playback_span(1, SOURCE_BOUND_AHEAD, 1, 1, true); + assert!(keeps_playback_job(3, -1, span, 5)); + assert!(Lookback { frames: 5, dir: -1 }.frames_before(100).any(|f| f == 103)); + // hostile sizes stay bounded and never overflow + assert_eq!(Lookback::backward(i64::MAX).frames_before(0).count() as i64, super::MAX_LOOKBACK); + assert_eq!(Lookback { frames: -5, dir: 1 }.frames_before(0).count(), 0); + assert!(Lookback { frames: 3, dir: -1 }.frames_before(i64::MAX).all(|f| f == i64::MAX)); + assert!(Lookback::backward(3).frames_before(i64::MIN).all(|f| f == i64::MIN)); + } + #[test] fn meter_resync_does_not_count_hidden_time() { let mut m = PlaybackMeter::default(); diff --git a/crates/ui-egui/src/panels/monitor.rs b/crates/ui-egui/src/panels/monitor.rs --- a/crates/ui-egui/src/panels/monitor.rs +++ b/crates/ui-egui/src/panels/monitor.rs @@ -206,9 +206,9 @@ pub fn show(app: &mut FilmcraftApp, ui: &mut egui::Ui, rect: Rect, which: Which) let draft = crate::frames::draft_playback(playing, scale, app.session.prefs.playback.draft_decode); let key = FrameKey { target, frame, size: size_key, revision: rev, draft }; let project = app.session.project.clone(); + let speed = if which == Which::Program { app.playback.speed } else { app.source_playback.clock.speed }; if playing { let preroll = which == Which::Program && app.playback.preroll.is_some(); - let speed = if which == Which::Program { app.playback.speed } else { app.source_playback.clock.speed }; app.frames.schedule_playback(key, rate, scale, &project, speed, preroll); if preroll { app.playback.preroll_ready = app.frames.preroll_ready(key, app.playback.speed, rate.frame_at(duration) - 1); @@ -224,21 +224,26 @@ pub fn show(app: &mut FilmcraftApp, ui: &mut egui::Ui, rect: Rect, which: Which) } else { format!("monitor-{prefix}") }; + // behind the playhead in the play direction: in reverse playback the pictures still + // finishing (kept running by `schedule_playback`) are above it + let back = crate::frames::Lookback { frames: app.frames.plan_lookback(key.target, rate.as_f64()), dir: if playing && speed < 0.0 { -1 } else { 1 } }; let (shown, on_screen) = if use_gpu { let found = app .frames .get_plan(&key) .map(|p| (key, p)) - .or_else(|| app.frames.nearest_plan(key, 6)) + .or_else(|| app.frames.nearest_plan(key, back)) .or_else(|| asked.iter().find_map(|k| app.frames.get_plan(k).map(|p| (*k, p)))); match found { Some((k, plan)) => (app.gpu_present(k, &plan).map(|(id, _)| id), Some(k)), None => (app.gpu.as_ref().and_then(|g| g.texture), app.gpu.as_ref().and_then(|g| g.last_key)), } - } else if grid { - cpu_texture_as(app, &ctx, &tex_name, key, &asked, display, true) } else { - cpu_texture(app, &ctx, &tex_name, key, &asked, display) + if grid { + cpu_texture_as(app, &ctx, &tex_name, key, &asked, display, true, back) + } else { + cpu_texture(app, &ctx, &tex_name, key, &asked, display, back) + } }; let exact = on_screen == Some(key); if which == Which::Program { @@ -274,9 +279,9 @@ pub fn show(app: &mut FilmcraftApp, ui: &mut egui::Ui, rect: Rect, which: Which) ui.painter().rect_filled(rpic, 0.0, t.monitor_bg); } let (reference, _) = if mv.transparency_grid { - cpu_texture_as(app, &ctx, &format!("monitor-{prefix}-ref-grid"), rkey, &[], DisplayMode::Composite, true) + cpu_texture_as(app, &ctx, &format!("monitor-{prefix}-ref-grid"), rkey, &[], DisplayMode::Composite, true, crate::frames::Lookback::backward(6)) } else { - cpu_texture(app, &ctx, &format!("monitor-{prefix}-ref"), rkey, &[], DisplayMode::Composite) + cpu_texture(app, &ctx, &format!("monitor-{prefix}-ref"), rkey, &[], DisplayMode::Composite, crate::frames::Lookback::backward(6)) }; if let Some(tex) = reference { ui.painter().image(tex, rpic, Rect::from_min_max(pos2(0.0, 0.0), pos2(1.0, 1.0)), Color32::WHITE); @@ -506,9 +511,10 @@ fn asked_before(app: &mut FilmcraftApp, prefix: &str, key: FrameKey, playing: bo before } -/// A monitor texture from the CPU frame path: the exact frame, else the nearest cached one before -/// it, else the newest ready one of `asked` (see [`asked_before`]), with the display mode's -/// channel mapping. Returns the texture and the frame it shows. +/// A monitor texture from the CPU frame path: the exact frame, else the nearest cached one within +/// `back` (the GPU path's look-back while playing, in the play direction), else the newest ready +/// one of `asked` (see [`asked_before`]), with the display mode's channel mapping. Returns the +/// texture and the frame it shows. fn cpu_texture( app: &mut FilmcraftApp, ctx: &egui::Context, @@ -516,11 +522,13 @@ fn cpu_texture( key: FrameKey, asked: &[FrameKey], mode: DisplayMode, + back: crate::frames::Lookback, ) -> (Option, Option) { - cpu_texture_as(app, ctx, name, key, asked, mode, false) + cpu_texture_as(app, ctx, name, key, asked, mode, false, back) } /// [`cpu_texture`], keeping the frame's alpha when `alpha` is set (the Transparency Grid). +#[allow(clippy::too_many_arguments)] fn cpu_texture_as( app: &mut FilmcraftApp, ctx: &egui::Context, @@ -529,12 +537,13 @@ fn cpu_texture_as( asked: &[FrameKey], mode: DisplayMode, alpha: bool, + back: crate::frames::Lookback, ) -> (Option, Option) { let found = app .frames .get(&key) .map(|img| (key, img)) - .or_else(|| (1..=6).map(|d| FrameKey { frame: key.frame.saturating_sub(d), ..key }).find_map(|k| app.frames.get(&k).map(|img| (k, img)))) + .or_else(|| back.frames_before(key.frame).map(|frame| FrameKey { frame, ..key }).find_map(|k| app.frames.get(&k).map(|img| (k, img)))) .or_else(|| asked.iter().find_map(|k| app.frames.get(k).map(|img| (*k, img)))); match found { Some((k, img)) => { diff --git a/docs/architecture.md b/docs/architecture.md --- a/docs/architecture.md +++ b/docs/architecture.md @@ -343,7 +343,14 @@ file ──► codecs (MP4/MOV, MKV, audio) demux + decode, GOP-aware see `filmcraft_media::cancel`) jobs for frames the playhead has passed; a new on-screen frame replaces the one asked for before (scrubbing); Stop cancels the prefetch. When frames cost more than the workers can render in real time (CPU effects), it starts only frames that can still be - on time and spaces them evenly (`playback_plan`). When decoding is what falls behind, each job + on time and spaces them evenly (`playback_plan`). When one decoder cannot produce the frame rate (a + slow 4K source; `source_cannot_keep_up`, estimated from the source time of the recent jobs that + fetched from the source being played), playback + asks for only the next `SOURCE_BOUND_AHEAD` (3) frames, because the decoder is the serialized + stage and a longer queue only adds latency, and the monitor shows the newest picture the source + has (`plan_lookback`: behind the playhead in the play direction by the measured picture latency, + at most one second) instead of + freezing on one. When decoding is what falls behind, each job tells its sources which frames are already late (`filmcraft_media::cancel::with_catch_up`: the playhead's distance, two seconds for a scrubbed-to frame), and the GOP cache skips non-reference pictures of late frames on the way (`VideoDecoder::is_disposable`, H.264 diff --git a/docs/gaps.md b/docs/gaps.md --- a/docs/gaps.md +++ b/docs/gaps.md @@ -1,6 +1,6 @@ # Where FilmCraft falls short of Premiere Pro -> **Last reviewed:** 2026-10-10 · **Last updated:** 2026-10-10 · **Change:** major (new file: ROADMAP's "Where we are lacking" re-measured against Premiere Pro 26.5.2 and the open issue list) · **Target:** Adobe Premiere Pro 2026 (26.5.2) +> **Last reviewed:** 2026-10-10 · **Last updated:** 2026-10-24 · **Change:** minor (G16: Program playback on a source slower than real time drops frames instead of freezing (#2696)) · **Target:** Adobe Premiere Pro 2026 (26.5.2) Every known shortfall, one entry each, ranked by what most stops an editor using FilmCraft for real work. This is the work list: agents should prefer it over adding checklist items. Numbers are in @@ -272,7 +272,10 @@ platforms** G4, G9, G17; **performance** G16; **ecosystem** G11; **measurement** CDEF and loop restoration in parallel 64-row bands, bit-exact (#847; 1080p GOP 1.16–1.30×, low delay 1.28×, 2160p 1.14× at 16 threads). VP9 software decode loop-filters a frame as a wavefront over superblock rows on the decoder's pool instead of on one thread, bit-identical, - 1.2–1.5× faster with 16 threads (#980). + 1.2–1.5× faster with 16 threads (#980). A source that decodes slower than real time (4K50 HEVC at + 26–28 ms a picture) no longer freezes the Program monitor: playback keeps a three-frame queue + while one decoder cannot make the frame rate and shows the newest picture the source has, so it + drops frames instead and a click on the timeline is not stuck behind queued jobs (#2696). - **Estimate:** 30–60 h. Doc: [performance.md](performance.md). ## G17. Audio device and playback reliability @@ -323,3 +326,4 @@ platforms** G4, G9, G17; **performance** G16; **ecosystem** G11; **measurement** | 2026-10-12 | minor | G10: MP1 (`A_MPEG/L1`) in Matroska imports (#1402) | | 2026-10-12 | minor | G10: OpenEXR PXR24 import | | 2026-10-24 | minor | G8: dragging the cut on a transition's timeline in Effect Controls moves the cut, and the transition moves with it (#577) | +| 2026-10-24 | minor | G16: Program playback on a source slower than real time drops frames instead of freezing (#2696) | diff --git a/docs/target-app-parity.md b/docs/target-app-parity.md --- a/docs/target-app-parity.md +++ b/docs/target-app-parity.md @@ -1,6 +1,6 @@ # Parity with Adobe Premiere Pro -> **Last reviewed:** 2026-10-10 · **Last updated:** 2026-10-10 · **Change:** major (full re-measure against Premiere Pro 2026 26.5.2; replaces the scorecard and estimate that lived in ROADMAP.md) · **Target:** Adobe Premiere Pro 2026 (26.5.2, macOS) +> **Last reviewed:** 2026-10-10 · **Last updated:** 2026-10-24 · **Change:** minor (Performance: Program playback on a source slower than real time drops frames instead of freezing (#2696)) · **Target:** Adobe Premiere Pro 2026 (26.5.2, macOS) The authoritative assessment of how close FilmCraft is to Premiere Pro, and how much work is left. [ROADMAP.md](../ROADMAP.md) summarizes it; [gaps.md](gaps.md) lists every shortfall one at a time. @@ -218,7 +218,7 @@ full-area depth (e.g. the ~20 common effects rather than all 93 and their approx | Titles and captions | 7% | 75% | Text, shapes, templates, SRT / SCC / VTT; `.mogrt` not read by design | | Export with presets | 9% | 75% | H.264, HEVC (hardware), ProRes, DNxHR, presets, queue; bitrate miss (#371) | | Save, reopen, auto-save, relink, proxies | 4% | 80% | Atomic saves, migrations, recovery, Link Media | -| Playback speed on their machine | 9% | 65% | 4K H.264 / HEVC real time with hardware decode on macOS; lag / CPU reports (#523, #525) | +| Playback speed on their machine | 9% | 65% | 4K H.264 / HEVC real time with hardware decode on macOS; lag / CPU reports (#523, #525); a source slower than real time drops frames instead of freezing the Program monitor (#2696) | | **Weighted depth** | 100% | **74.8%** | | Discounts for what still stops real work: @@ -354,6 +354,7 @@ control surfaces), and validation in Avid Media Composer / Pro Tools. | Date | Change | Summary | |---|---|---| +| 2026-10-24 | minor | Performance: Program playback on a source slower than real time drops frames instead of freezing (#2696) | | 2026-10-10 | minor | Readiness table: hours to ~95% for each audience (full ~1,450–2,600, mainstream ~500–850, essentials ~180–350); full number confirmed as an additive weighted sum (no change) | | 2026-10-10 | minor | Added the three readiness numbers: full ready-for-real-work formula written out (~55% -> ~56%, same inputs, now computed rather than rounded down), mainstream practitioner ~55% and essentials user ~60% with written weights and discounts, and counted user evidence (31 people praising, 0 switch reports, ~64 core-path vs ~29 niche open bugs) | | 2026-10-10 | minor | Stage checked against the new core-workflow gate (docs/roadmap.md#alpha-gate): passes, stays alpha |