collective
Browse / github.com/storytold/filmcraft / patch
gpu: 10-bit video uploads as its codes, with no half-float conversion
gpu: 10-bit video uploads as its codes, with no half-float conversion ## What Every 10- and 12-bit Y'CbCr plane was converted to half floats per sample (a table lookup each) before upload: half of the CPU time of 4K HDR playback, on the frame workers. - Where the adapter has 16-bit normalized textures (Vulkan, Metal, DX12), those planes upload as they are (R16Unorm, sampled as code / 65535). The app, the export renderer and the tests request the feature where the adapter has it (filmcraft_gpu::optional_features); elsewhere (WebGPU) planes are converted as before. - The frame workers leave such planes unconverted (prepare_with). A plane converted anyway uploads
- Status
- open
- Score
- 0.00
- Adopters
- 0 other operators
- Author
- zsoli69 (imported from pull request #3215, open)
- Base
10d626ccc4888e0f020f9ca314c2bc9148ded4f4- Size
- +162 −35 in 11 files
- Review
- pending
- Shared
- 2026-10-28
- Id
c50675638a90ef490c09684b
How it went
No other operator has reported using it yet.
Diff
raw · Cargo.lock apps/filmcraft/src/graphics.rs crates/gpu/Cargo.toml crates/gpu/src/blend_tests.rs crates/gpu/src/export_renderer.rs crates/gpu/src/lib.rs crates/gpu/src/tests.rs crates/ui-egui/src/app.rs crates/ui-egui/src/app/gpu.rs crates/ui-egui/src/frames.rs docs/architecture.md
@@ -2187,6 +2187,7 @@ dependencies = [ name = "filmcraft-gpu" version = "0.13.0" dependencies = [+ "bytemuck", "filmcraft-color", "filmcraft-export", "filmcraft-frame",@@ -439,6 +439,14 @@ fn apply_for(options: &mut NativeOptions, mode: Mode, backend_override: Option<B // Pro: the device is lost and the window never opens. `WGPU_VALIDATION_INDIRECT_CALL=1` turns // the check back on. create.instance_descriptor.flags = create.instance_descriptor.flags.difference(wgpu::InstanceFlags::VALIDATION_INDIRECT_CALL).with_env();+ // the compositor's optional features (16-bit normalized textures: 10-bit video uploads as+ // its codes) wherever the adapter has them+ let device_descriptor = create.device_descriptor.clone();+ create.device_descriptor = Arc::new(move |adapter| {+ let mut d = device_descriptor(adapter);+ d.required_features |= filmcraft_gpu::optional_features(adapter);+ d+ }); match mode { // eframe's default (`WGPU_BACKEND`, else PRIMARY | GL), except on Windows: there it // includes GL. Creating that backend can crash inside AMD's atio6axx.dll before adapter@@ -17,6 +17,7 @@ filmcraft-project = { workspace = true } filmcraft-render = { workspace = true } filmcraft-time = { workspace = true } wgpu = { version = "30", default-features = false, features = ["wgsl"] }+bytemuck = "1" log = { workspace = true } pollster = "1" rayon = { workspace = true }@@ -13,7 +13,7 @@ use filmcraft_render::plan::execute_cpu; pub(crate) fn device() -> Option<(wgpu::Device, wgpu::Queue)> { let instance = wgpu::Instance::default(); let adapter = pollster::block_on(instance.request_adapter(&wgpu::RequestAdapterOptions::default())).ok()?;- pollster::block_on(adapter.request_device(&wgpu::DeviceDescriptor::default())).ok()+ pollster::block_on(adapter.request_device(&wgpu::DeviceDescriptor { required_features: optional_features(&adapter), ..Default::default() })).ok() } pub(crate) fn yuv_frame(w: u32, h: u32) -> Arc<VideoFrame> {@@ -11,7 +11,7 @@ //! the CPU renderer; a `FramePlan::Image` plan (HDR / wide gamut / anything the plan sends to the //! CPU) is returned unchanged — it already is the CPU image. -use crate::{GpuCompositor, prepare};+use crate::{GpuCompositor, prepare_with}; use filmcraft_project::{ItemId, Project}; use filmcraft_render::plan::{FramePlan, plan_frame}; use filmcraft_render::{Image, RenderOptions, SourceProvider};@@ -113,7 +113,8 @@ impl ExportRenderer { } let instance = wgpu::Instance::default(); let adapter = pollster::block_on(instance.request_adapter(&wgpu::RequestAdapterOptions::default())).ok()?;- let (device, queue) = pollster::block_on(adapter.request_device(&wgpu::DeviceDescriptor::default())).ok()?;+ let desc = wgpu::DeviceDescriptor { required_features: crate::optional_features(&adapter), ..Default::default() };+ let (device, queue) = pollster::block_on(adapter.request_device(&desc)).ok()?; // wgpu's default handler panics on a validation or out-of-memory error; log it and // send the remaining frames to the CPU instead. device.on_uncaptured_error(std::sync::Arc::new(|e: wgpu::Error| mark_device_failed(&format!("GPU error: {e}"))));@@ -140,7 +141,8 @@ impl ExportRenderer { if device_failed() { return None; }- let prep = prepare(&plan);+ // 10-bit planes as their codes where the device takes 16-bit normalized textures+ let prep = prepare_with(&plan, self.compositor.uploads_raw_16bit()); let (w, h, px) = self.compositor.render_export_prepared(&plan, Some(&prep))?; // an error reported while this frame was drawn leaves its pixels untrusted if device_failed() {@@ -94,6 +94,19 @@ pub struct GpuCompositor { pub cpu_transitions: u64, /// Total bytes uploaded (stats). pub uploaded_bytes: u64,+ /// 16-bit Y'CbCr planes upload as they are (`R16Unorm`, [`OPTIONAL_FEATURES`]) instead of+ /// being converted to half floats first.+ raw16: bool,+}++/// Device features the compositor uses when the adapter has them; request them with+/// [`optional_features`]. `TEXTURE_FORMAT_16BIT_NORM`: 10- and 12-bit Y'CbCr planes upload as+/// their 16-bit codes, with no per-sample conversion on the CPU.+pub const OPTIONAL_FEATURES: wgpu::Features = wgpu::Features::TEXTURE_FORMAT_16BIT_NORM;++/// The [`OPTIONAL_FEATURES`] `adapter` has, for its `DeviceDescriptor::required_features`.+pub fn optional_features(adapter: &wgpu::Adapter) -> wgpu::Features {+ adapter.features() & OPTIONAL_FEATURES } fn tex_entry(binding: u32) -> wgpu::BindGroupLayoutEntry {@@ -255,6 +268,15 @@ pub fn prepare_frame(f: &VideoFrame) -> Option<Prepared> { /// Convert every layer of a plan for upload. Thread-safe and GPU-free: run it on a worker. pub fn prepare(plan: &FramePlan) -> PreparedPlan {+ prepare_with(plan, false)+}++/// [`prepare`] for a compositor that uploads 16-bit planes as they are+/// ([`GpuCompositor::uploads_raw_16bit`]): those are left unconverted. A mismatch is harmless:+/// a converted plane uploads as half floats on any device, an unconverted one is converted at+/// upload where the device has no 16-bit normalized textures.+pub fn prepare_with(plan: &FramePlan, raw16: bool) -> PreparedPlan {+ let prepare_layers = |layers: &[PlanLayer]| prepare_layers(layers, raw16); match plan { FramePlan::Layers { layers, .. } => prepare_layers(layers), FramePlan::Image(img) => PreparedPlan { image: Some(Prepared { planes: vec![f32_to_f16_bytes(&img.px)] }), ..Default::default() },@@ -271,8 +293,9 @@ pub fn prepare(plan: &FramePlan) -> PreparedPlan { } } -fn prepare_layers(layers: &[PlanLayer]) -> PreparedPlan {- PreparedPlan { layers: layers.iter().map(|l| prepare_frame(&l.frame)).collect(), ..Default::default() }+fn prepare_layers(layers: &[PlanLayer], raw16: bool) -> PreparedPlan {+ let prepare = |f: &VideoFrame| if raw16 && matches!(f.data, PixelData::Yuv16 { .. }) { None } else { prepare_frame(f) };+ PreparedPlan { layers: layers.iter().map(|l| prepare(&l.frame)).collect(), ..Default::default() } } impl GpuCompositor {@@ -427,9 +450,16 @@ impl GpuCompositor { gpu_transitions: 0, cpu_transitions: 0, uploaded_bytes: 0,+ raw16: device.features().contains(wgpu::Features::TEXTURE_FORMAT_16BIT_NORM), } } + /// Whether 16-bit Y'CbCr planes upload as they are (the device has 16-bit normalized+ /// textures): pass it to [`prepare_with`].+ pub fn uploads_raw_16bit(&self) -> bool {+ self.raw16+ }+ fn target( device: &wgpu::Device, slot: &mut Option<(wgpu::Texture, wgpu::TextureView, (u32, u32))>,@@ -549,25 +579,36 @@ impl GpuCompositor { let (sx, sy) = chroma.shifts(); let (cw, ch) = (w.div_ceil(1 << sx), h.div_ceil(1 << sy)); let scale = (1u32 << bits) as f32;- let owned;- let p = match prep {- Some(p) => p,- None => {- owned = Prepared { planes: planes.iter().chain(alpha.as_ref()).map(|p| codes_to_f16_bytes(p, *bits)).collect() };- &owned- }+ // (full-resolution alpha plane: its codes span 0..2^bits - 1, hence its scale)+ let (y, u, v, a, code_scale, alpha_scale) = if prep.is_none() && self.raw16 {+ // the codes as they are: R16Unorm samples are code / 65535+ let plane = |s: &mut Self, w, h, p: &[u16]| s.plane_texture(w, h, wgpu::TextureFormat::R16Unorm, bytemuck::cast_slice(p), 2);+ let y = plane(self, w, h, &planes[0]);+ let u = plane(self, cw, ch, &planes[1]);+ let v = plane(self, cw, ch, &planes[2]);+ let a = alpha.as_ref().map(|a| plane(self, w, h, a));+ (y, u, v, a, 65535.0, 65535.0 / (scale - 1.0))+ } else {+ let owned;+ let p = match prep {+ Some(p) => p,+ None => {+ owned = Prepared { planes: planes.iter().chain(alpha.as_ref()).map(|p| codes_to_f16_bytes(p, *bits)).collect() };+ &owned+ }+ };+ let y = self.plane_texture(w, h, wgpu::TextureFormat::R16Float, &p.planes[0], 2);+ let u = self.plane_texture(cw, ch, wgpu::TextureFormat::R16Float, &p.planes[1], 2);+ let v = self.plane_texture(cw, ch, wgpu::TextureFormat::R16Float, &p.planes[2], 2);+ let a = alpha.as_ref().and_then(|_| p.planes.get(3)).map(|b| self.plane_texture(w, h, wgpu::TextureFormat::R16Float, b, 2));+ (y, u, v, a, scale, scale / (scale - 1.0)) };- let y = self.plane_texture(w, h, wgpu::TextureFormat::R16Float, &p.planes[0], 2);- let u = self.plane_texture(cw, ch, wgpu::TextureFormat::R16Float, &p.planes[1], 2);- let v = self.plane_texture(cw, ch, wgpu::TextureFormat::R16Float, &p.planes[2], 2);- // (full-resolution alpha plane: its codes span 0..2^bits - 1, hence the scale)- let a = alpha.as_ref().and_then(|_| p.planes.get(3)).map(|b| self.plane_texture(w, h, wgpu::TextureFormat::R16Float, b, 2));- let alpha_scale = if a.is_some() { scale / (scale - 1.0) } else { 0.0 };+ let alpha_scale = if a.is_some() { alpha_scale } else { 0.0 }; let _ = Chroma::C420; Uploaded { views: [y, u, v, a.unwrap_or_else(dummy)], kind: 2,- code_scale: scale,+ code_scale, alpha_scale, chroma: (cw, ch), last_used: self.clock,@@ -576,7 +617,12 @@ impl GpuCompositor { } }; self.uploads.insert(key, up);- // keep the most recent uploads (enough for several stacked layers + playback lookahead)+ self.evict_uploads();+ key+ }++ /// Keep the most recent uploads (enough for several stacked layers + playback lookahead).+ fn evict_uploads(&mut self) { if self.uploads.len() > 24 { let mut v: Vec<(u64, (usize, u32, u32))> = self.uploads.iter().map(|(k, u)| (u.last_used, *k)).collect(); v.sort_unstable();@@ -584,7 +630,6 @@ impl GpuCompositor { self.uploads.remove(&k); } }- key } /// The source description of an uploaded frame.@@ -1159,7 +1204,7 @@ fn wait_mapped(device: &wgpu::Device) -> Option<()> { /// Shader source: kind (0 RGBA8 sRGB, 1 linear premultiplied RGBA, 2 YUV), sampling scales, dimensions, color. struct SrcInfo { kind: u32,- /// Texture sample -> code units: 255 for R8Unorm, 2^bits for R16Float.+ /// Texture sample -> code units: 255 for R8Unorm, 2^bits for R16Float, 65535 for R16Unorm. code_scale: f32, /// YUV quantization levels: 2^bits; R8Unorm's decoding scale is one less. code_levels: f32,@@ -6,7 +6,7 @@ use filmcraft_render::plan::execute_cpu; fn device() -> Option<(wgpu::Device, wgpu::Queue)> { let instance = wgpu::Instance::default(); let adapter = pollster::block_on(instance.request_adapter(&wgpu::RequestAdapterOptions::default())).ok()?;- pollster::block_on(adapter.request_device(&wgpu::DeviceDescriptor::default())).ok()+ pollster::block_on(adapter.request_device(&wgpu::DeviceDescriptor { required_features: optional_features(&adapter), ..Default::default() })).ok() } #[test]@@ -39,6 +39,50 @@ fn yuv_code_range_uniforms() { assert!(mismatches.is_empty(), "{}", mismatches.join("\n")); } +/// 10-bit 4:2:0 with an alpha plane uploads as its codes (`R16Unorm`) where the device has 16-bit+/// normalized textures, with the same picture as the half-float upload and the CPU; plans+/// prepared for such a compositor leave those planes unconverted.+#[test]+fn raw_16bit_planes_match_half_floats() {+ let Some((dev, q)) = device() else {+ eprintln!("no GPU adapter; skipping");+ return;+ };+ let mut raw = GpuCompositor::new(&dev, &q);+ if !raw.uploads_raw_16bit() {+ eprintln!("no 16-bit normalized textures on this adapter; skipping");+ return;+ }+ let (w, h) = (64u32, 16u32);+ let (cw, ch) = (w / 2, h / 2);+ let y: Vec<u16> = (0..w * h).map(|i| (64 + (i % w) * 876 / w) as u16).collect();+ // (gentle chroma ramps: the shader's bilinear chroma vs the CPU's differ on steep ones)+ let u: Vec<u16> = (0..cw * ch).map(|i| (448 + (i / cw) * 64 / ch) as u16).collect();+ let v: Vec<u16> = (0..cw * ch).map(|i| (600 - (i % cw) * 64 / cw) as u16).collect();+ let a: Vec<u16> = (0..w * h).map(|i| ((i / w) * 1023 / (h - 1)) as u16).collect();+ let frame = Arc::new(VideoFrame {+ width: w,+ height: h,+ data: PixelData::Yuv16 { planes: [Arc::new(y), Arc::new(u), Arc::new(v)], chroma: Chroma::C420, bits: 10, alpha: Some(Arc::new(a)) },+ color: filmcraft_color::ColorInfo::REC709,+ par: (1, 1),+ pts: Default::default(),+ });+ let plan = FramePlan::Layers { width: w as usize, height: h as usize, layers: vec![PlanLayer::new(frame, Affine::IDENTITY, 1.0, Blend::Normal)] };+ let skipped = prepare_with(&plan, true);+ assert!(skipped.layers.iter().all(Option::is_none), "raw 16-bit planes are not converted");+ raw.composite_prepared(&plan, Some(&skipped));+ let (_, _, raw_px) = raw.read_output().expect("raw readback");+ let mut half = GpuCompositor::new(&dev, &q);+ half.composite_prepared(&plan, Some(&prepare(&plan)));+ let (_, _, half_px) = half.read_output().expect("half-float readback");+ let cpu = execute_cpu(&plan).over_black_rgba8();+ let max = |a: &[u8], b: &[u8]| a.iter().zip(b).map(|(x, y)| x.abs_diff(*y)).max().unwrap_or(0);+ assert!(max(&raw_px, &half_px) <= 1, "raw vs half floats: {}", max(&raw_px, &half_px));+ let (p99, mean, _) = super::blend_tests::stats8(&cpu, &raw_px, &vec![true; (w * h) as usize]);+ assert!(p99 <= 6 && mean < 1.5, "raw vs CPU: p99 {p99}, mean {mean}");+}+ #[test] fn yuv_code_range_gpu_matches_cpu() { let Some((dev, q)) = device() else {@@ -295,16 +339,26 @@ fn prepared_upload_matches_inline_conversion() { ], }; let image = FramePlan::Image(filmcraft_render::Image { w: w as usize, h: h as usize, px: f32_layer });+ // Both upload modes: half floats (forced, so devices with 16-bit normalized textures cover it+ // too) and the codes as they are where the device has them. A plan is prepared for the mode+ // of the compositor that draws it, as the frame workers do (`prepare_with` with+ // `uploads_raw_16bit`); mixing the modes is a different picture by up to a level, which+ // `raw_16bit_planes_match_half_floats` bounds (it is exact on some GPUs, not on lavapipe).+ let modes: &[bool] = if GpuCompositor::new(&dev, &q).uploads_raw_16bit() { &[false, true] } else { &[false] }; for plan in [layers, image] {- let mut a = GpuCompositor::new(&dev, &q);- a.composite(&plan);- let inline = a.read_output().expect("readback");- let mut b = GpuCompositor::new(&dev, &q);- let prep = prepare(&plan);- assert!(prep.bytes() > 0);- b.composite_prepared(&plan, Some(&prep));- let prepared = b.read_output().expect("readback");- assert!(inline == prepared, "prepared upload differs");+ for &raw16 in modes {+ let mut a = GpuCompositor::new(&dev, &q);+ a.raw16 = raw16;+ a.composite(&plan);+ let inline = a.read_output().expect("readback");+ let mut b = GpuCompositor::new(&dev, &q);+ b.raw16 = raw16;+ let prep = prepare_with(&plan, raw16);+ assert!(prep.bytes() > 0);+ b.composite_prepared(&plan, Some(&prep));+ let prepared = b.read_output().expect("readback");+ assert!(inline == prepared, "prepared upload differs (raw 16-bit uploads: {raw16})");+ } } } @@ -177,6 +177,8 @@ impl FilmcraftApp { self.session.previews.clone(), FrameServer::default_workers(), ));+ // a new server starts with raw 16-bit planes off: keep what the live compositor takes+ self.frames.set_raw_16bit(self.gpu.as_ref().is_some_and(|g| g.compositor.uploads_raw_16bit())); self.textures.clear(); self.thumb_used.clear(); self.tl.reset_media_caches();@@ -23,6 +23,7 @@ impl FilmcraftApp { match made { Ok(compositor) if !broken.load(std::sync::atomic::Ordering::Relaxed) => { let max_texture = rs.device.limits().max_texture_dimension_2d;+ self.frames.set_raw_16bit(compositor.uploads_raw_16bit()); self.gpu = Some(GpuState { render_state: rs, compositor, texture: None, last_key: None, size: (0, 0), last_ms: 0.0, broken, max_texture }); } _ => log::error!("GPU compositor failed to initialise; compositing on the CPU"),@@ -32,6 +33,7 @@ impl FilmcraftApp { /// Drop the GPU compositor after a GPU error; the monitors composite on the CPU from then on. fn disable_gpu(&mut self, why: &str) { log::error!("GPU compositor disabled: {why}");+ self.frames.set_raw_16bit(false); if let Some(g) = self.gpu.take() && let Some(id) = g.texture {@@ -165,6 +165,9 @@ struct Shared { workers: usize, /// Always-on counters for `perf.stats`. stats: Mutex<FrameStats>,+ /// The GPU compositor uploads 16-bit planes as they are: plans leave them unconverted+ /// ([`filmcraft_gpu::prepare_with`]).+ raw16: AtomicBool, /// Set (under the queue lock) when the [`FrameServer`] is dropped: the workers return. stop: AtomicBool, }@@ -477,6 +480,7 @@ impl FrameServer { render_cost: Mutex::new(0.0), workers: workers.max(1), stats: Mutex::new(FrameStats::default()),+ raw16: AtomicBool::new(false), stop: AtomicBool::new(false), }); let repaint: Arc<Mutex<Option<Box<dyn Fn() + Send + Sync>>>> = Arc::new(Mutex::new(None));@@ -507,6 +511,12 @@ impl FrameServer { } } + /// Whether the GPU compositor showing the plans uploads 16-bit planes as they are+ /// ([`filmcraft_gpu::GpuCompositor::uploads_raw_16bit`]).+ pub fn set_raw_16bit(&self, on: bool) {+ self.shared.raw16.store(on, Ordering::Relaxed);+ }+ /// Collect a [`JobRecord`] per finished job (benchmarks); `take_records` drains them. pub fn set_profiling(&self, on: bool) { self.shared.profiling.store(on, Ordering::Relaxed);@@ -915,7 +925,7 @@ fn run_job(sh: &Shared, job: &Job, pool: &Arc<MediaPool>, services: &Arc<dyn Ser loading = filmcraft_media::pending::take(); if !job.cancel.load(Ordering::Relaxed) && !loading { // Convert texels for upload here, not on the UI thread when the frame is shown.- let prepared = filmcraft_gpu::prepare(&plan);+ let prepared = filmcraft_gpu::prepare_with(&plan, sh.raw16.load(Ordering::Relaxed)); sh.plans.lock().unwrap_or_else(PoisonError::into_inner).insert(job.key, GpuPlan { plan, prepared }); } pv@@ -355,7 +355,9 @@ file ──► codecs (MP4/MOV, MKV, audio) demux + decode, GOP-aware see requests only (a paused frame, render or export decodes them again exactly), and draft plans hand the GPU box-decimated Y'CbCr planes at the drawn size. `perf.stats` reports the counters. GPU plans carry their texels already converted for upload (`filmcraft_gpu::prepare`), so the UI- thread only copies them.+ thread only copies them. Where the adapter has 16-bit normalized textures (Vulkan, Metal,+ DX12; `filmcraft_gpu::optional_features`), 10- and 12-bit planes are left as their codes+ (`prepare_with`) and upload as `R16Unorm`; elsewhere (WebGPU) they are converted to half floats. - **Audio clock.** The desktop app passes a cpal output (`apps/filmcraft/src/audio.rs`) to the UI as `AudioOut`. While playing, the samples played by the sound card drive the playhead and video follows. Without an audio device, playback falls back to the wall clock. `PlaybackMeter` counts timeline