diff --git a/artcraft-native/src/app.rs b/artcraft-native/src/app.rs --- a/artcraft-native/src/app.rs +++ b/artcraft-native/src/app.rs @@ -24,6 +24,7 @@ use crate::overlays::lightbox::{Lightbox, LightboxAction, fit_inside}; use crate::overlays::providers::{AccountAction, AccountsView, LoginEvent, MidjourneyLogin, SetupAction, setup_dialog}; use crate::pages::common::{self, AppRequest, Env, FeedKey}; use crate::pages::create_audio::{AudioSettings, CreateAudioPage}; +use crate::pages::frame_extractor::FrameExtractorPage; use crate::pages::create_image::{CreateImagePage, ImageSettings}; use crate::pages::create_video::{CreateVideoPage, VideoSettings}; use crate::pages::other; @@ -86,6 +87,7 @@ pub struct ArtcraftApp { image: CreateImagePage, video: CreateVideoPage, audio: CreateAudioPage, + frames: FrameExtractorPage, library: FeedStore, ratios: RatioCache, /// Prompt texts by prompt token (list view, lightbox). @@ -132,7 +134,7 @@ impl ArtcraftApp { backend.load_models(modality); } backend.refresh_session(); - let mut app = Self { cache, player, toasts: Toasts::default(), catalog: Catalog::default(), user: None, credits: None, page: saved.page, enter_to_generate: saved.enter_to_generate, view_mode: saved.view_mode, autoplay: saved.autoplay, sidebar_open: saved.sidebar_open, image: CreateImagePage::new(saved.image), video: CreateVideoPage::new(saved.video), audio: CreateAudioPage::new(saved.audio), library: FeedStore::default(), ratios: RatioCache::default(), prompts: HashMap::new(), prompt_records: HashMap::new(), requested_prompts: HashSet::new(), pending_recreate: None, batch_kinds: HashMap::new(), lightbox: None, picker: None, login: None, characters_modal: None, characters: Vec::new(), pending_characters: Vec::new(), settings_open: false, provider_setup: None, midjourney_login: None, fal_key_input: String::new(), preview: None, requests: Vec::new(), screenshot: ScreenshotRequest::from_env(), next_poll: 0.0, next_credits: CREDITS_INTERVAL, models_retry_at: None, backend }; + let mut app = Self { cache, player, toasts: Toasts::default(), catalog: Catalog::default(), user: None, credits: None, page: saved.page, enter_to_generate: saved.enter_to_generate, view_mode: saved.view_mode, autoplay: saved.autoplay, sidebar_open: saved.sidebar_open, image: CreateImagePage::new(saved.image), video: CreateVideoPage::new(saved.video), audio: CreateAudioPage::new(saved.audio), frames: FrameExtractorPage::default(), library: FeedStore::default(), ratios: RatioCache::default(), prompts: HashMap::new(), prompt_records: HashMap::new(), requested_prompts: HashSet::new(), pending_recreate: None, batch_kinds: HashMap::new(), lightbox: None, picker: None, login: None, characters_modal: None, characters: Vec::new(), pending_characters: Vec::new(), settings_open: false, provider_setup: None, midjourney_login: None, fal_key_input: String::new(), preview: None, requests: Vec::new(), screenshot: ScreenshotRequest::from_env(), next_poll: 0.0, next_credits: CREDITS_INTERVAL, models_retry_at: None, backend }; app.restore_provider_jobs(); // Diagnostics: `ARTCRAFT_OPEN=settings` or `midjourney-login` opens that dialog at launch. match std::env::var("ARTCRAFT_OPEN").as_deref() { @@ -634,6 +636,7 @@ impl ArtcraftApp { Page::CreateImage => self.image.ui(ui, &mut env), Page::CreateVideo => self.video.ui(ui, &mut env), Page::CreateAudio => self.audio.ui(ui, &mut env), + Page::FrameExtractor => self.frames.ui(ui, &mut env), Page::Library => other::library(ui, &mut env, &mut self.library), page => other::coming_soon(ui, page), } diff --git a/artcraft-native/src/backend/video.rs b/artcraft-native/src/backend/video.rs --- a/artcraft-native/src/backend/video.rs +++ b/artcraft-native/src/backend/video.rs @@ -31,11 +31,11 @@ pub fn ffmpeg_available() -> bool { } #[derive(Clone, Copy, Debug)] -struct VideoInfo { - width: u32, - height: u32, - fps: f64, - duration: f64, +pub struct VideoInfo { + pub width: u32, + pub height: u32, + pub fps: f64, + pub duration: f64, } pub struct VideoPlayer { @@ -330,7 +330,7 @@ impl Source for PcmStream { /// Reads the clip's size, rate and length. A small probe keeps this ~2 s on CDN files instead of /// ~14 s (ffprobe otherwise scans much of the file over HTTP). -fn probe(url: &str) -> Result { +pub fn probe(url: &str) -> Result { #[derive(serde::Deserialize, Default)] #[serde(default)] struct Probe { @@ -359,6 +359,16 @@ fn probe(url: &str) -> Result { Ok(VideoInfo { width: stream.width.max(2), height: stream.height.max(2), fps: if fps.is_finite() && fps > 0.0 { fps } else { 24.0 }, duration: probe.format.duration.parse().unwrap_or(0.0) }) } +/// One frame of `path` at `secs`, at the clip's full size, as PNG bytes. +pub fn grab_frame(path: &str, secs: f64) -> Result, String> { + let out = command("ffmpeg").args(["-nostdin", "-loglevel", "error", "-ss", &format!("{:.3}", secs.max(0.0)), "-i", path, "-frames:v", "1", "-f", "image2pipe", "-vcodec", "png", "-"]).output().map_err(|e| format!("ffmpeg: {e}"))?; + if out.stdout.is_empty() { + let detail = String::from_utf8_lossy(&out.stderr); + return Err(if detail.trim().is_empty() { "no frame at this time".to_owned() } else { detail.trim().to_owned() }); + } + Ok(out.stdout) +} + /// The decoded size: at most `MAX_WIDTH` wide, both sides even (as `scale` requires). fn output_size(info: VideoInfo) -> (u32, u32) { let w = info.width.min(MAX_WIDTH) & !1; @@ -418,5 +428,9 @@ mod tests { assert!((pts - 1.0).abs() < 1e-9, "frames are stamped from the start position"); let count = 1 + stream.frames.iter().count(); assert!((20..=28).contains(&count), "about a second of frames at 24 fps, got {count}"); + + let png = grab_frame(&url, 1.0).unwrap(); + assert_eq!(image::load_from_memory(&png).unwrap().to_rgba8().dimensions(), (640, 360)); + assert!(grab_frame(&url, 60.0).is_err(), "no frame past the end of the clip"); } } diff --git a/artcraft-native/src/pages/frame_extractor.rs b/artcraft-native/src/pages/frame_extractor.rs new file mode 100644 --- /dev/null +++ b/artcraft-native/src/pages/frame_extractor.rs @@ -0,0 +1,249 @@ +//! Frame Extract: pick a local video, scrub to a moment and save that frame as a PNG. Frames come +//! from FFmpeg (`backend::video`), so the page asks for it when it isn't installed. + +use std::path::PathBuf; +use std::sync::mpsc::{channel, Receiver}; + +use egui::{vec2, Align2, ColorImage, FontId, Rect, TextureHandle, TextureOptions, Ui}; + +use crate::backend::video::{self, VideoInfo}; +use crate::overlays::lightbox::fit_inside; +use crate::pages::common::Env; +use crate::theme; +use crate::ui::icons::Icon; +use crate::ui::widgets; + +const VIDEO_EXTENSIONS: [&str; 8] = ["mp4", "mov", "mkv", "webm", "avi", "m4v", "wmv", "flv"]; +const CONTROLS_HEIGHT: f32 = 132.0; + +#[derive(Default)] +pub struct FrameExtractorPage { + clip: Option, +} + +impl FrameExtractorPage { + pub fn ui(&mut self, ui: &mut Ui, env: &mut Env) { + let area = ui.max_rect(); + if !video::ffmpeg_available() { + ui.painter().text(area.center(), Align2::CENTER_CENTER, "Frame Extract needs FFmpeg. Install it and make sure ffmpeg and ffprobe are on your PATH.", FontId::new(15.0, egui::FontFamily::Proportional), theme::MUTED); + return; + } + if let Some(clip) = &mut self.clip { + clip.poll(&env.ctx); + } + let mut picked = None; + let mut save = None; + let stage = Rect::from_min_max(area.min, egui::pos2(area.right(), area.bottom() - CONTROLS_HEIGHT)); + self.stage(ui, stage); + let controls = Rect::from_min_max(egui::pos2(area.left() + 24.0, stage.bottom()), area.max - vec2(24.0, 16.0)); + let mut child = ui.new_child(egui::UiBuilder::new().max_rect(controls)); + child.horizontal(|ui| { + if widgets::button(ui, Some(Icon::Folder), if self.clip.is_some() { "Choose another video" } else { "Choose a video" }, widgets::Kind::Primary, 0.0, theme::CONTROL_H).clicked() { + picked = rfd::FileDialog::new().add_filter("Video", &VIDEO_EXTENSIONS).pick_file(); + } + if let Some(clip) = &self.clip { + ui.label(egui::RichText::new(clip.path.file_name().map(|n| n.to_string_lossy().into_owned()).unwrap_or_default()).color(theme::MUTED)); + } + }); + if let Some(clip) = &mut self.clip { + if let Some(info) = clip.info { + child.add_space(8.0); + child.horizontal(|ui| { + if widgets::button(ui, Some(Icon::ChevronLeft), "Frame", widgets::Kind::Secondary, 0.0, theme::CONTROL_H).clicked() { + clip.position = step(clip.position, -1.0, info); + } + if widgets::button(ui, Some(Icon::ChevronRight), "Frame", widgets::Kind::Secondary, 0.0, theme::CONTROL_H).clicked() { + clip.position = step(clip.position, 1.0, info); + } + ui.style_mut().spacing.slider_width = (ui.available_width() - 330.0).max(120.0); + ui.add(egui::Slider::new(&mut clip.position, 0.0..=(info.duration - 1.0 / info.fps).max(0.0)).show_value(false)); + ui.label(egui::RichText::new(format!("{} / {}", timestamp(clip.position), timestamp(info.duration))).monospace().color(theme::MUTED)); + }); + child.add_space(8.0); + child.horizontal(|ui| { + ui.add_enabled_ui(clip.frame.is_some(), |ui| { + if widgets::button(ui, Some(Icon::Download), "Save frame", widgets::Kind::Primary, 0.0, theme::CONTROL_H).clicked() { + if let Some(frame) = &clip.frame { + save = Some((frame_file_name(&clip.path.to_string_lossy(), frame.time), frame.png.clone())); + } + } + }); + }); + } + } + if let Some(path) = picked { + self.clip = Some(Clip::open(path)); + } + if let Some((name, png)) = save { + if let Some(path) = rfd::FileDialog::new().set_file_name(&name).add_filter("PNG", &["png"]).save_file() { + match std::fs::write(&path, png) { + Ok(()) => env.toasts.success(format!("Saved {}", path.display())), + Err(e) => env.toasts.error(format!("Couldn't save the frame: {e}")), + } + } + } + } + + fn stage(&self, ui: &mut Ui, stage: Rect) { + let p = ui.painter(); + p.rect_filled(stage.shrink2(vec2(24.0, 16.0)), 0.0, egui::Color32::BLACK); + let inner = stage.shrink2(vec2(28.0, 20.0)); + let message = match &self.clip { + None => Some(("Choose a video to pull frames out of it.", theme::MUTED)), + Some(clip) => match (&clip.error, &clip.frame) { + (Some(e), _) => { + ui.painter().text(inner.center(), Align2::CENTER_CENTER, format!("Couldn't read this video: {e}"), FontId::new(13.0, egui::FontFamily::Proportional), theme::BAD); + None + }, + (None, Some(frame)) => { + let fit = fit_inside(frame.texture.size_vec2(), inner.size()); + egui::Image::new(&frame.texture).paint_at(ui, Rect::from_center_size(inner.center(), fit)); + None + }, + (None, None) => Some(("Reading the video…", theme::MUTED)), + }, + }; + if let Some((text, color)) = message { + ui.painter().text(inner.center(), Align2::CENTER_CENTER, text, FontId::new(15.0, egui::FontFamily::Proportional), color); + } + } +} + +/// One decoded frame: the PNG bytes saved on request and the texture shown on screen. +struct Frame { + time: f64, + png: Vec, + texture: TextureHandle, +} + +struct Clip { + path: PathBuf, + probe: Option>>, + info: Option, + /// The moment the slider points at. + position: f64, + /// A decode in flight, so a fast drag asks for one frame at a time. + grab: Option<(f64, Receiver, String>>)>, + frame: Option, + error: Option, + /// The moment whose decode failed; it isn't retried until the slider moves elsewhere. + failed_at: Option, +} + +impl Clip { + fn open(path: PathBuf) -> Self { + let (tx, rx) = channel(); + let probe_path = path.to_string_lossy().into_owned(); + std::thread::spawn(move || { + let _ = tx.send(video::probe(&probe_path)); + }); + Self { path, probe: Some(rx), info: None, position: 0.0, grab: None, frame: None, error: None, failed_at: None } + } + + /// Whether the slider is on a moment that has no frame yet and no decode running or failed. + fn wants_frame(&self) -> bool { + self.info.is_some() && self.grab.is_none() && self.frame.as_ref().map(|f| f.time) != Some(self.position) && self.failed_at != Some(self.position) + } + + fn poll(&mut self, ctx: &egui::Context) { + if let Some(rx) = &self.probe { + if let Ok(result) = rx.try_recv() { + self.probe = None; + match result { + Ok(info) => self.info = Some(info), + Err(e) => self.error = Some(e), + } + } + } + if let Some((time, rx)) = &self.grab { + if let Ok(result) = rx.try_recv() { + let time = *time; + self.grab = None; + match result.and_then(|png| decode(&png).map(|image| (png, image))) { + Ok((png, image)) => { + let texture = ctx.load_texture("frame-extractor", image, TextureOptions::LINEAR); + self.frame = Some(Frame { time, png, texture }); + self.error = None; + }, + Err(e) => { + self.error = Some(e); + self.failed_at = Some(time); + }, + } + } + } + // Decode the moment the slider is on once the previous frame is back. + if self.wants_frame() { + self.error = None; + let (tx, rx) = channel(); + let path = self.path.to_string_lossy().into_owned(); + let time = self.position; + std::thread::spawn(move || { + let _ = tx.send(video::grab_frame(&path, time)); + }); + self.grab = Some((time, rx)); + } + if self.probe.is_some() || self.grab.is_some() { + ctx.request_repaint_after(std::time::Duration::from_millis(30)); + } + } +} + +fn decode(png: &[u8]) -> Result { + let image = image::load_from_memory(png).map_err(|e| format!("couldn't read the frame: {e}"))?.to_rgba8(); + Ok(ColorImage::from_rgba_unmultiplied([image.width() as usize, image.height() as usize], image.as_raw())) +} + +/// `1:05.250`, the moment shown next to the slider. +fn timestamp(secs: f64) -> String { + let millis = (secs.max(0.0) * 1000.0).round() as u64; + format!("{}:{:02}.{:03}", millis / 60_000, millis / 1000 % 60, millis % 1000) +} + +/// The suggested file name for a saved frame: `clip_0-01-05.250.png`. +fn frame_file_name(clip: &str, secs: f64) -> String { + let stem = std::path::Path::new(clip).file_stem().map(|s| s.to_string_lossy().into_owned()).unwrap_or_else(|| "frame".to_owned()); + format!("{stem}_{}.png", timestamp(secs).replace(':', "-")) +} + +/// One frame earlier or later, kept inside the clip. +fn step(position: f64, frames: f64, info: VideoInfo) -> f64 { + (position + frames / info.fps).clamp(0.0, (info.duration - 1.0 / info.fps).max(0.0)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn timestamps_show_minutes_seconds_and_millis() { + assert_eq!(timestamp(0.0), "0:00.000"); + assert_eq!(timestamp(65.25), "1:05.250"); + assert_eq!(timestamp(-3.0), "0:00.000"); + } + + #[test] + fn frame_names_carry_the_clip_name_and_moment() { + assert_eq!(frame_file_name("/videos/holiday.mp4", 65.25), "holiday_1-05.250.png"); + assert_eq!(frame_file_name("", 0.0), "frame_0-00.000.png"); + } + + #[test] + fn a_failed_moment_is_retried_only_after_the_slider_moves() { + let mut clip = Clip { path: PathBuf::from("clip.mp4"), probe: None, info: Some(info()), position: 2.0, grab: None, frame: None, error: Some("no frame at this time".to_owned()), failed_at: Some(2.0) }; + assert!(!clip.wants_frame()); + clip.position = 3.0; + assert!(clip.wants_frame()); + } + + #[test] + fn stepping_stays_inside_the_clip() { + assert_eq!(step(0.0, -1.0, info()), 0.0); + assert!((step(1.0, 1.0, info()) - 1.04).abs() < 1e-9); + assert!((step(9.99, 5.0, info()) - 9.96).abs() < 1e-9); + } + + fn info() -> VideoInfo { + VideoInfo { width: 1280, height: 720, fps: 25.0, duration: 10.0 } + } +} diff --git a/artcraft-native/src/pages/mod.rs b/artcraft-native/src/pages/mod.rs --- a/artcraft-native/src/pages/mod.rs +++ b/artcraft-native/src/pages/mod.rs @@ -2,5 +2,6 @@ pub mod common; pub mod create_audio; pub mod create_image; pub mod create_video; +pub mod frame_extractor; pub mod other; pub mod toolbar; diff --git a/artcraft-native/src/shell.rs b/artcraft-native/src/shell.rs --- a/artcraft-native/src/shell.rs +++ b/artcraft-native/src/shell.rs @@ -88,6 +88,7 @@ impl Page { "create-image" | "image" => Page::CreateImage, "create-video" | "video" => Page::CreateVideo, "create-audio" | "audio" => Page::CreateAudio, + "frame-extractor" | "frames" => Page::FrameExtractor, "library" => Page::Library, _ => return None, }) @@ -95,7 +96,7 @@ impl Page { /// Pages the native app has; the rest open a "coming soon" page. pub fn is_ready(self) -> bool { - matches!(self, Page::Home | Page::CreateImage | Page::CreateVideo | Page::CreateAudio | Page::Library) + matches!(self, Page::Home | Page::CreateImage | Page::CreateVideo | Page::CreateAudio | Page::FrameExtractor | Page::Library) } /// The same tool in the web app.