diff --git a/artcraft-native/src/backend/audio.rs b/artcraft-native/src/backend/audio.rs --- a/artcraft-native/src/backend/audio.rs +++ b/artcraft-native/src/backend/audio.rs @@ -7,13 +7,16 @@ use std::collections::HashMap; use std::io::Cursor; use std::sync::Arc; use std::sync::mpsc::{Receiver, Sender, channel}; -use std::time::Duration; +use std::time::{Duration, Instant}; use log::warn; use rodio::{Decoder, OutputStream, OutputStreamHandle, Sink, Source}; use tokio::sync::Semaphore; +/// Delay before retrying failed remote previews. +const RETRY_AFTER: Duration = Duration::from_secs(5); /// The webapp's reference preview volume. + const PREVIEW_VOLUME: f32 = 0.2; /// Peaks kept per waveform (resampled to however many bars are drawn). const PEAK_COUNT: usize = 256; @@ -32,7 +35,8 @@ pub struct AudioPlayer { current: Option, tx: Sender, rx: Receiver, - waveforms: HashMap, + waveforms: HashMap)>, + next_request: u64, limiter: Arc, /// The tracks' volume slider, 0–1. track_volume: f32, @@ -65,6 +69,7 @@ pub struct Playback { /// The clip that's playing, by the caller's key (a track's key is its URL). struct Clip { + request: u64, key: String, track: bool, /// Where to start once it has downloaded (a seek before it loaded). @@ -72,14 +77,14 @@ struct Clip { } enum Loaded { - Clip { key: String, bytes: Result, String> }, + Clip { key: String, request: u64, bytes: Result, String> }, Waveform { url: String, waveform: Option }, } impl AudioPlayer { pub fn new(rt: tokio::runtime::Handle, http: reqwest::Client) -> Self { let (tx, rx) = channel(); - Self { rt, http, output: None, sink: None, current: None, tx, rx, waveforms: HashMap::new(), limiter: Arc::new(Semaphore::new(MAX_CONCURRENT_WAVEFORMS)), track_volume: 1.0 } + Self { rt, http, output: None, sink: None, current: None, tx, rx, waveforms: HashMap::new(), next_request: 0, limiter: Arc::new(Semaphore::new(MAX_CONCURRENT_WAVEFORMS)), track_volume: 1.0 } } /// Starts a reference clip under `key`, or stops it if it's what's playing. @@ -160,10 +165,17 @@ impl AudioPlayer { /// The waveform of the track at `url`, downloading and decoding it the first time it's asked /// for. pub fn waveform(&mut self, ctx: &egui::Context, url: &str) -> WaveformLookup { - if let Some(lookup) = self.waveforms.get(url) { - return lookup.clone(); + if let Some((lookup, retry_at)) = self.waveforms.get(url) { + match retry_at { + Some(deadline) if Instant::now() >= *deadline => {}, + Some(deadline) => { + ctx.request_repaint_after(deadline.saturating_duration_since(Instant::now())); + return lookup.clone(); + }, + None => return lookup.clone(), + } } - self.waveforms.insert(url.to_owned(), WaveformLookup::Loading); + self.waveforms.insert(url.to_owned(), (WaveformLookup::Loading, None)); let (http, limiter, tx, ctx, url) = (self.http.clone(), self.limiter.clone(), self.tx.clone(), ctx.clone(), url.to_owned()); self.rt.spawn(async move { let waveform = match limiter.acquire_owned().await { @@ -189,11 +201,11 @@ impl AudioPlayer { while let Ok(loaded) = self.rx.try_recv() { match loaded { Loaded::Waveform { url, waveform } => { - self.waveforms.insert(url, waveform.map_or(WaveformLookup::Failed, |w| WaveformLookup::Ready(Arc::new(w)))); + self.waveforms.insert(url, waveform.map_or_else(|| (WaveformLookup::Failed, Some(Instant::now() + RETRY_AFTER)), |w| (WaveformLookup::Ready(Arc::new(w)), None))); }, - Loaded::Clip { key, bytes } => { + Loaded::Clip { key, request, bytes } => { // Superseded by another clip (or stopped) while downloading. - let Some(clip) = self.current.as_ref().filter(|c| c.key == key) else { + let Some(clip) = self.current.as_ref().filter(|c| c.key == key && c.request == request) else { continue; }; let (volume, start_at) = (if clip.track { self.track_volume } else { PREVIEW_VOLUME }, clip.start_at); @@ -214,11 +226,13 @@ impl AudioPlayer { fn start(&mut self, ctx: &egui::Context, key: &str, url: &str, track: bool, start_at: f32) { self.stop(); - self.current = Some(Clip { key: key.to_owned(), track, start_at }); + self.next_request += 1; + let request = self.next_request; + self.current = Some(Clip { key: key.to_owned(), request, track, start_at }); let (http, tx, ctx, key, url) = (self.http.clone(), self.tx.clone(), ctx.clone(), key.to_owned(), url.to_owned()); self.rt.spawn(async move { let bytes = download(&http, &url).await; - let _ = tx.send(Loaded::Clip { key, bytes }); + let _ = tx.send(Loaded::Clip { key, request, bytes }); ctx.request_repaint(); }); } @@ -341,3 +355,37 @@ mod tests { assert_eq!(*waveform.peaks.last().unwrap(), 1.0, "ends loud"); } } + +#[cfg(test)] +mod recovery_tests { + use super::*; + + #[tokio::test] + async fn stale_same_key_success_and_failure_cannot_replace_a_new_clip() { + let mut player = AudioPlayer::new(tokio::runtime::Handle::current(), reqwest::Client::new()); + player.current = Some(Clip { key: "same".into(), request: 2, track: true, start_at: 8.0 }); + for bytes in [Err("old failure".into()), Ok(Arc::from(&b"old audio"[..]))] { + player.tx.send(Loaded::Clip { key: "same".into(), request: 1, bytes }).unwrap(); + } + assert!(player.poll().is_none()); + assert_eq!(player.track("same").unwrap().position, 8.0); + assert!(player.output.is_none()); + player.tx.send(Loaded::Clip { key: "same".into(), request: 2, bytes: Err("current failure".into()) }).unwrap(); + assert!(player.poll().unwrap().contains("current failure")); + assert!(player.current.is_none()); + } + + #[tokio::test] + async fn failed_waveform_retries_after_cooldown_only_once() { + let mut player = AudioPlayer::new(tokio::runtime::Handle::current(), reqwest::Client::new()); + let ctx = egui::Context::default(); + let url = "http://127.0.0.1:1/waveform"; + player.tx.send(Loaded::Waveform { url: url.into(), waveform: None }).unwrap(); + player.poll(); + assert!(matches!(player.waveform(&ctx, url), WaveformLookup::Failed)); + player.waveforms.get_mut(url).unwrap().1 = Some(Instant::now() - RETRY_AFTER); + assert!(matches!(player.waveform(&ctx, url), WaveformLookup::Loading)); + assert!(matches!(player.waveform(&ctx, url), WaveformLookup::Loading)); + assert!(player.waveforms[url].1.is_none()); + } +} diff --git a/artcraft-native/src/backend/media_cache.rs b/artcraft-native/src/backend/media_cache.rs --- a/artcraft-native/src/backend/media_cache.rs +++ b/artcraft-native/src/backend/media_cache.rs @@ -7,9 +7,9 @@ use std::collections::HashMap; use std::io::Cursor; -use std::sync::Arc; +use std::sync::{Arc, Weak}; use std::sync::mpsc::{Receiver, Sender, channel}; -use std::time::Duration; +use std::time::{Duration, Instant}; use egui::{ColorImage, TextureHandle, TextureOptions}; use image::AnimationDecoder; @@ -17,6 +17,8 @@ use log::warn; use tokio::runtime::Handle; use tokio::sync::Semaphore; +/// Delay before retrying failed remote previews. +const RETRY_AFTER: Duration = Duration::from_secs(5); /// How many images may download at once. const MAX_CONCURRENT_DOWNLOADS: usize = 6; /// Longest side of a decoded image; larger ones are downscaled before upload. @@ -38,6 +40,7 @@ pub struct MediaCache { rx: Receiver, entries: HashMap, frame: u64, + local_sources: HashMap>, } /// What [`MediaCache::get`] knows about an image. @@ -61,7 +64,7 @@ enum Entry { total: f32, last_used: u64, }, - Failed, + Failed { retry_at: Instant }, } struct Loaded { @@ -77,7 +80,7 @@ enum Decoded { impl MediaCache { pub fn new(rt: Handle, http: reqwest::Client) -> Self { let (tx, rx) = channel(); - Self { rt, http, limiter: Arc::new(Semaphore::new(MAX_CONCURRENT_DOWNLOADS)), tx, rx, entries: HashMap::new(), frame: 0 } + Self { rt, http, limiter: Arc::new(Semaphore::new(MAX_CONCURRENT_DOWNLOADS)), tx, rx, entries: HashMap::new(), frame: 0, local_sources: HashMap::new() } } /// The texture for `url`, starting its download the first time it's asked for. @@ -92,7 +95,9 @@ impl MediaCache { } /// Decodes `bytes` off the UI thread and registers the result under `key`. + /// Keep a clone of `bytes` while attached so evicted local textures can be decoded again. pub fn insert_bytes(&mut self, ctx: &egui::Context, key: &str, bytes: Arc<[u8]>) { + self.local_sources.insert(key.to_owned(), Arc::downgrade(&bytes)); if self.entries.contains_key(key) { return; } @@ -120,20 +125,29 @@ impl MediaCache { let frames = frames.into_iter().enumerate().map(|(i, (img, d))| (ctx.load_texture(format!("{key}#{i}"), img, TextureOptions::LINEAR), d)).collect(); Entry::Animated { frames, total, last_used: self.frame } }, - None => Entry::Failed, + None => Entry::Failed { retry_at: Instant::now() + RETRY_AFTER }, }; } let frame = self.frame; + self.local_sources.retain(|_, bytes| bytes.strong_count() > 0); let crowded = self.entries.len() > SOFT_CAPACITY; self.entries.retain(|_, e| match e { Entry::Animated { last_used, .. } => frame.saturating_sub(*last_used) < EVICT_ANIMATED_AFTER_FRAMES, Entry::Ready { last_used, .. } | Entry::Loading { last_used } => !crowded || frame.saturating_sub(*last_used) < EVICT_AFTER_FRAMES, - Entry::Failed => true, + Entry::Failed { retry_at } => Instant::now() < *retry_at, }); } fn lookup(&mut self, ctx: &egui::Context, key: &str, url: &str, animated: bool) -> Lookup { let frame = self.frame; + if matches!(self.entries.get(key), Some(Entry::Failed { retry_at }) if Instant::now() >= *retry_at) { + self.entries.remove(key); + } + if !self.entries.contains_key(key) { + if let Some(bytes) = self.local_sources.get(key).and_then(Weak::upgrade) { + self.insert_bytes(ctx, key, bytes); + } + } match self.entries.get_mut(key) { Some(Entry::Ready { texture, last_used }) => { *last_used = frame; @@ -157,7 +171,10 @@ impl MediaCache { *last_used = frame; Lookup::Loading }, - Some(Entry::Failed) => Lookup::Failed, + Some(Entry::Failed { retry_at }) => { + ctx.request_repaint_after(retry_at.saturating_duration_since(Instant::now())); + Lookup::Failed + }, None => { self.entries.insert(key.to_owned(), Entry::Loading { last_used: frame }); self.spawn_download(ctx, key.to_owned(), url.to_owned(), animated); @@ -242,3 +259,57 @@ fn to_color_image(rgba: image::RgbaImage) -> ColorImage { let size = [rgba.width() as usize, rgba.height() as usize]; ColorImage::from_rgba_unmultiplied(size, rgba.as_raw()) } + +#[cfg(test)] +mod recovery_tests { + use super::*; + + #[tokio::test] + async fn failed_preview_retries_after_cooldown() { + let mut cache = MediaCache::new(Handle::current(), reqwest::Client::new()); + let ctx = egui::Context::default(); + let url = "http://127.0.0.1:1/image"; + cache.entries.insert(url.into(), Entry::Loading { last_used: 0 }); + cache.tx.send(Loaded { key: url.into(), decoded: None }).unwrap(); + cache.poll(&ctx); + assert!(matches!(cache.get(&ctx, url), Lookup::Failed)); + cache.entries.insert(url.into(), Entry::Failed { retry_at: Instant::now() - RETRY_AFTER }); + assert!(matches!(cache.get(&ctx, url), Lookup::Loading)); + assert!(matches!(cache.entries[url], Entry::Loading { .. })); + } + + #[tokio::test] + async fn attached_local_source_survives_texture_eviction_and_releases_on_detach() { + let mut cache = MediaCache::new(Handle::current(), reqwest::Client::new()); + let ctx = egui::Context::default(); + let mut png = Cursor::new(Vec::new()); + image::DynamicImage::new_rgba8(1, 1).write_to(&mut png, image::ImageFormat::Png).unwrap(); + let bytes: Arc<[u8]> = Arc::from(png.into_inner()); + cache.insert_bytes(&ctx, "local://test", bytes.clone()); + wait_ready(&mut cache, &ctx).await; + cache.frame += EVICT_AFTER_FRAMES; + for n in 0..SOFT_CAPACITY { + cache.entries.insert(format!("other{n}"), Entry::Loading { last_used: cache.frame }); + } + cache.poll(&ctx); + assert!(!cache.entries.contains_key("local://test")); + assert!(matches!(cache.get(&ctx, "local://test"), Lookup::Loading)); + wait_ready(&mut cache, &ctx).await; + drop(bytes); + tokio::time::timeout(Duration::from_secs(2), async { + while cache.local_sources["local://test"].strong_count() > 0 { tokio::task::yield_now().await; } + }).await.unwrap(); + cache.poll(&ctx); + assert!(!cache.local_sources.contains_key("local://test")); + } + + async fn wait_ready(cache: &mut MediaCache, ctx: &egui::Context) { + tokio::time::timeout(Duration::from_secs(2), async { + loop { + cache.poll(ctx); + if matches!(cache.get(ctx, "local://test"), Lookup::Ready(_)) { break; } + tokio::time::sleep(Duration::from_millis(5)).await; + } + }).await.unwrap(); + } +} diff --git a/artcraft-native/src/overlays/characters.rs b/artcraft-native/src/overlays/characters.rs --- a/artcraft-native/src/overlays/characters.rs +++ b/artcraft-native/src/overlays/characters.rs @@ -265,7 +265,9 @@ impl CharactersModal { Ok(bytes) => { let mut img = RefMedia::uploading(RefKind::Image, None); let key = format!("local://{}", img.id); - cache.insert_bytes(ui.ctx(), &key, Arc::from(bytes)); + let bytes: Arc<[u8]> = Arc::from(bytes); + img.local_preview = Some(bytes.clone()); + cache.insert_bytes(ui.ctx(), &key, bytes); img.preview = Some(key); actions.push(CharactersAction::Upload { ref_id: img.id, path }); self.image = Some(img); diff --git a/artcraft-native/src/pages/common.rs b/artcraft-native/src/pages/common.rs --- a/artcraft-native/src/pages/common.rs +++ b/artcraft-native/src/pages/common.rs @@ -266,7 +266,9 @@ pub fn upload_path(env: &mut Env<'_>, refs: &mut References, path: &Path, kind: let key = format!("local://{}", item.id); match std::fs::read(path) { Ok(bytes) => { - env.cache.insert_bytes(&env.ctx, &key, Arc::from(bytes)); + let bytes: Arc<[u8]> = Arc::from(bytes); + item.local_preview = Some(bytes.clone()); + env.cache.insert_bytes(&env.ctx, &key, bytes); item.preview = Some(key); }, Err(err) => { @@ -283,7 +285,9 @@ pub fn upload_path(env: &mut Env<'_>, refs: &mut References, path: &Path, kind: pub fn upload_png(env: &mut Env<'_>, refs: &mut References, png: Vec, slot: ImageSlot) { let mut item = RefMedia::uploading(RefKind::Image, None); let key = format!("local://{}", item.id); - env.cache.insert_bytes(&env.ctx, &key, Arc::from(png.clone())); + let bytes: Arc<[u8]> = Arc::from(png.clone()); + item.local_preview = Some(bytes.clone()); + env.cache.insert_bytes(&env.ctx, &key, bytes); item.preview = Some(key); env.backend.upload_png(item.id, png); place(refs, item, slot); diff --git a/artcraft-native/src/pages/create_video.rs b/artcraft-native/src/pages/create_video.rs --- a/artcraft-native/src/pages/create_video.rs +++ b/artcraft-native/src/pages/create_video.rs @@ -477,6 +477,7 @@ impl CreateVideoPage { s.generate_audio = audio; } self.refs.clear(); + s.input_mode = if prompt.maybe_context_images.iter().flatten().any(|c| matches!(c.semantic.as_str(), "vid_start_frame" | "vid_end_frame")) { InputMode::Keyframe } else { InputMode::Reference }; for c in prompt.maybe_context_images.iter().flatten() { let kind = match c.semantic.as_str() { "vid_ref" => RefKind::Video, @@ -485,6 +486,7 @@ impl CreateVideoPage { }; let item = RefMedia::from_library(kind, c.media_token.clone(), c.media_links.thumbnail(256).filter(|_| kind != RefKind::Audio), Some(c.media_links.cdn_url.clone()), 0.0); match (c.semantic.as_str(), kind) { + ("vid_start_frame", _) => self.refs.first_frame = Some(item), ("vid_end_frame", _) => self.refs.last_frame = Some(item), (_, RefKind::Image) if s.input_mode == InputMode::Keyframe && self.refs.first_frame.is_none() => self.refs.first_frame = Some(item), (_, kind) => self.refs.list_mut(kind).push(item), @@ -638,3 +640,35 @@ mod tests { assert!(mentioned_characters("no mentions", &characters, &none).is_empty()); } } + +#[cfg(test)] +mod recreate_tests { + use super::*; + use crate::backend::wire::ContextImage; + + #[test] + fn restores_keyframes_even_when_end_frame_is_listed_first() { + let mut page = CreateVideoPage::default(); + page.apply_prompt(&prompt(&[("end", "vid_end_frame"), ("start", "vid_start_frame")])); + assert_eq!(page.settings.input_mode, InputMode::Keyframe); + let model = ModelInfo { start_frame: true, end_frame: true, ..Default::default() }; + let fields = page.fields(&model, false); + assert_eq!(fields["start_frame_image_media_token"], "start"); + assert_eq!(fields["end_frame_image_media_token"], "end"); + assert!(!fields.contains_key("reference_image_media_tokens")); + } + + #[test] + fn reference_prompt_resets_a_previous_keyframe_mode() { + let mut page = CreateVideoPage::default(); + page.settings.input_mode = InputMode::Keyframe; + page.apply_prompt(&prompt(&[("ref", "imgref")])); + assert_eq!(page.settings.input_mode, InputMode::Reference); + assert!(page.refs.first_frame.is_none()); + assert_eq!(page.refs.images[0].token.as_deref(), Some("ref")); + } + + fn prompt(items: &[(&str, &str)]) -> Prompt { + Prompt { maybe_context_images: Some(items.iter().map(|(token, semantic)| ContextImage { media_token: token.to_string(), semantic: semantic.to_string(), ..Default::default() }).collect()), ..Default::default() } + } +} diff --git a/artcraft-native/src/prompt_box/refs.rs b/artcraft-native/src/prompt_box/refs.rs --- a/artcraft-native/src/prompt_box/refs.rs +++ b/artcraft-native/src/prompt_box/refs.rs @@ -1,6 +1,7 @@ //! Reference media attached to a prompt: images, videos and audio clips, either uploaded from //! disk or picked from the library. +use std::sync::Arc; use std::sync::atomic::{AtomicU64, Ordering}; static NEXT_ID: AtomicU64 = AtomicU64::new(1); @@ -49,6 +50,8 @@ pub struct RefMedia { pub token: Option, /// Media cache key of the thumbnail (a CDN URL or a `local://` key). pub preview: Option, + /// Local preview source, retained only while this reference remains attached. + pub local_preview: Option>, /// The full-size URL, for the preview modal. pub full_url: Option, /// Seconds, for videos and audio (counts toward the model's total-duration limit). @@ -57,11 +60,11 @@ pub struct RefMedia { impl RefMedia { pub fn uploading(kind: RefKind, preview: Option) -> Self { - Self { id: NEXT_ID.fetch_add(1, Ordering::Relaxed), kind, status: RefStatus::Uploading, token: None, preview, full_url: None, duration_secs: 0.0 } + Self { id: NEXT_ID.fetch_add(1, Ordering::Relaxed), kind, status: RefStatus::Uploading, token: None, preview, local_preview: None, full_url: None, duration_secs: 0.0 } } pub fn from_library(kind: RefKind, token: String, preview: Option, full_url: Option, duration_secs: f32) -> Self { - Self { id: NEXT_ID.fetch_add(1, Ordering::Relaxed), kind, status: RefStatus::Ready, token: Some(token), preview, full_url, duration_secs } + Self { id: NEXT_ID.fetch_add(1, Ordering::Relaxed), kind, status: RefStatus::Ready, token: Some(token), preview, local_preview: None, full_url, duration_secs } } } diff --git a/netlify.toml b/netlify.toml new file mode 100644 --- /dev/null +++ b/netlify.toml @@ -0,0 +1,7 @@ +# The `storyteller-docs` Netlify site still builds this repository, but its +# build command (./script/netlify_build_docs.sh) and the docs themselves now +# live in storytold/artcraft-services, so every deploy preview here fails. +# Skip those builds until the Netlify site is repointed or disconnected, then +# delete this file. +[build] + ignore = "exit 0"