collective
Browse / github.com/storytold/photocraft / patch
perf(cms): tabulate the first-stage curves for 8-bit exact pipelines
cms: 8-bit conversion through an analytic (Lab) pipeline re-evaluates the input curves per pixel ## Proposal `crates/cms/src/transform.rs` `run_u8`: when the transform has no lookup-table acceleration (`Core::Exact`, for example any RGB -> Lab conversion, whose head is only curves, a matrix and Lab maths), every 8-bit pixel evaluates the first stage's curves with `powf` on all three channels. An 8-bit channel has only 256 possible values, so tabulate the first stage's curves once per transform (256 entries per channel, computed with the same `Curve::eval` on `code / 255.0`) and run the remaining stages from there. ## Measured (prototype) 1 MP sRGB -> Lab D50 through `convert_u8`, single
- Status
- open
- Score
- 0.00
- Adopters
- 0 other operators
- Author
- vangel76 (imported from pull request #9747, open)
- Base
a0ce166acd2a64ff1200f793d4006db61c09f4d6- Size
- +173 −1 in 3 files
- Review
- pending
- Shared
- 2026-10-28
- Id
d14a39a7c0774f6297ed4f46
How it went
No other operator has reported using it yet.
Diff
raw · crates/cms/src/pipeline.rs crates/cms/src/transform.rs crates/cms/tests/it/cms.rs
@@ -183,6 +183,17 @@ impl Pipeline { out[..self.outputs].copy_from_slice(&v[..self.outputs]); } + /// Evaluates the stages after the first one on values already passed through it.+ #[inline]+ pub(crate) fn eval_after_first(&self, v: &mut [f32; MAX], out: &mut [f32]) {+ let mut n = self.inputs;+ for s in self.stages.iter().skip(1) {+ n = s.run(v, n);+ }+ let _ = n;+ out[..self.outputs].copy_from_slice(&v[..self.outputs]);+ }+ /// Removes no-op stages and cancels adjacent inverse pairs (Lab↔XYZ, encode/decode). pub fn optimize(&mut self) { let mut out: Vec<Stage> = Vec::with_capacity(self.stages.len());@@ -42,6 +42,8 @@ pub const GRID_4D: usize = 17; const TABLE_1D: usize = GpuMatrixShaper::INPUT_INTERVALS; /// Resolution of output shaper tables (indexed by `x^(1/4)`). const OUT_TABLE: usize = GpuMatrixShaper::OUTPUT_ENTRIES;+// The blocked matrix/TRC path wraps table indices with a mask instead of a bounds check.+const _: () = assert!(OUT_TABLE.is_power_of_two()); /// ICC v4 perceptual reference medium black (ICC.1:2010 §6.3.4.4), XYZ. const PERCEPTUAL_BLACK: [f64; 3] = [0.00336, 0.0034731, 0.00287];@@ -91,6 +93,8 @@ pub struct Transform { post: Option<Post>, /// Pre-located grid coordinates per 8-bit code: (offset contribution, fraction) per input. grid8: Vec<[(u32, f32); 256]>,+ /// For an exact core: the first stage's curves at every 8-bit code.+ exact8: Vec<[f32; 256]>, } impl Transform {@@ -178,7 +182,13 @@ impl Transform { .collect(), _ => Vec::new(), };- Transform { inputs, outputs, opts, pipeline, core, post, grid8 }+ let exact8 = match (&core, pipeline.stages.first()) {+ (Core::Exact, Some(Stage::Curves(cs))) if cs.len() == inputs && inputs <= 4 => {+ cs.iter().map(|c| std::array::from_fn(|i| c.eval(i as f32 / 255.0))).collect()+ }+ _ => Vec::new(),+ };+ Transform { inputs, outputs, opts, pipeline, core, post, grid8, exact8 } } /// White preservation: the device-link node at the source's device white gets exactly the@@ -386,6 +396,10 @@ impl Transform { } } }+ Core::Shaper { curves8, matrix: Some((3, 3, m, off)), .. } if self.shaper3_ok(curves8.len(), m.len(), off.len()) => {+ let load = |sp: &[u8]| [curves8[0][sp[0] as usize], curves8[1][sp[1] as usize], curves8[2][sp[2] as usize]];+ self.shaper3_blocks(s, ss, d, ds, extra, m, off, load, q);+ } Core::Shaper { curves8, matrix, .. } => { let mut v = [0.0f32; 16]; for (sp, dp) in s.chunks_exact(ss).zip(d.chunks_exact_mut(ds)) {@@ -402,6 +416,21 @@ impl Transform { } } }+ Core::Exact if !self.exact8.is_empty() => {+ for (sp, dp) in s.chunks_exact(ss).zip(d.chunks_exact_mut(ds)) {+ let mut v = [0.0f32; 16];+ for k in 0..self.inputs {+ v[k] = self.exact8[k][sp[k] as usize];+ }+ self.pipeline.eval_after_first(&mut v, &mut out);+ for k in 0..self.outputs {+ dp[k] = q(out[k]);+ }+ for e in 0..extra {+ dp[self.outputs + e] = sp[self.inputs + e];+ }+ }+ } _ => { let mut inp = [0.0f32; 16]; for (sp, dp) in s.chunks_exact(ss).zip(d.chunks_exact_mut(ds)) {@@ -420,6 +449,74 @@ impl Transform { } } + fn shaper3_ok(&self, curves: usize, m: usize, off: usize) -> bool {+ self.inputs == 3+ && self.outputs == 3+ && curves == 3+ && m == 9+ && off == 3+ && self.post.as_ref().is_some_and(|p| p.tables.len() == 3 && p.tables.iter().all(|t| t.len() == OUT_TABLE))+ }++ /// Matrix/TRC transform of whole pixels in blocks: the matrix and the output-table index+ /// maths run over a block of pixels at a time (same operations, same order, per pixel).+ #[allow(clippy::too_many_arguments)]+ fn shaper3_blocks<S: Copy>(+ &self,+ s: &[S],+ ss: usize,+ d: &mut [S],+ ds: usize,+ extra: usize,+ m: &[f32],+ off: &[f32],+ load: impl Fn(&[S]) -> [f32; 3],+ store: impl Fn(f32) -> S,+ ) {+ const B: usize = 64;+ let Some(post) = self.post.as_ref() else { return };+ let tabs = [&post.tables[0][..OUT_TABLE], &post.tables[1][..OUT_TABLE], &post.tables[2][..OUT_TABLE]];+ let m: [f32; 9] = std::array::from_fn(|i| m[i]);+ let off: [f32; 3] = std::array::from_fn(|i| off[i]);+ let n_px = (s.len() / ss).min(d.len() / ds);+ let mut x = [[0.0f32; B]; 3];+ let mut start = 0;+ while start < n_px {+ let n = B.min(n_px - start);+ let sb = &s[start * ss..(start + n) * ss];+ let db = &mut d[start * ds..(start + n) * ds];+ for (j, sp) in sb.chunks_exact(ss).enumerate() {+ let [a, b, c] = load(sp);+ for r in 0..3 {+ let mut acc = off[r];+ acc += m[r * 3] * a;+ acc += m[r * 3 + 1] * b;+ acc += m[r * 3 + 2] * c;+ x[r][j] = acc;+ }+ }+ for (ch, t) in tabs.iter().enumerate() {+ let mut u = [0.0f32; B];+ let mut ix = [0i32; B];+ for j in 0..B {+ let v = x[ch][j].clamp(0.0, 1.0).sqrt().sqrt() * (OUT_TABLE - 1) as f32;+ let i = (v as i32).min(OUT_TABLE as i32 - 2);+ ix[j] = i;+ u[j] = v - i as f32;+ }+ for (j, dp) in db.chunks_exact_mut(ds).enumerate() {+ let i = ix[j] as usize & (OUT_TABLE - 1);+ let (v0, v1) = (t[i], t[(i + 1) & (OUT_TABLE - 1)]);+ dp[ch] = store(v0 + (v1 - v0) * u[j]);+ }+ }+ for (sp, dp) in sb.chunks_exact(ss).zip(db.chunks_exact_mut(ds)) {+ dp[3..3 + extra].copy_from_slice(&sp[3..3 + extra]);+ }+ start += n;+ }+ }+ /// Converts 16-bit pixels (see [`Transform::convert_f32`] for the layout). pub fn convert_u16(&self, src: &[u16], src_stride: usize, dst: &mut [u16], dst_stride: usize, copy_extra: bool) { self.check(src_stride, dst_stride);@@ -428,6 +525,13 @@ impl Transform { } fn run_u16(&self, s: &[u16], ss: usize, d: &mut [u16], ds: usize, extra: usize) {+ if let Core::Shaper { curves, matrix: Some((3, 3, m, off)), .. } = &self.core+ && self.shaper3_ok(curves.len(), m.len(), off.len())+ {+ let load =+ |sp: &[u16]| [table(&curves[0], sp[0] as f32 / 65535.0), table(&curves[1], sp[1] as f32 / 65535.0), table(&curves[2], sp[2] as f32 / 65535.0)];+ return self.shaper3_blocks(s, ss, d, ds, extra, m, off, load, |v: f32| (v.clamp(0.0, 1.0) * 65535.0 + 0.5) as u16);+ } let mut inp = [0.0f32; 16]; let mut out = [0.0f32; 16]; for (sp, dp) in s.chunks_exact(ss).zip(d.chunks_exact_mut(ds)) {@@ -819,3 +819,60 @@ fn device_links_parse_and_are_reported_unsupported() { // The output space still bounds the tag: a CMYK→CMYK link with a 4→3 table is malformed. assert!(matches!(Profile::parse(&device_link(b"CMYK", b"CMYK", 4, 3)), Err(photocraft_cms::CmsError::Invalid(_)))); }++/// The blocked matrix/TRC path (64 pixels at a time) must match the per-pixel evaluation for+/// every pixel count around the block size, with and without extra channels.+#[test]+fn matrix_trc_blocks_match_per_pixel_eval() {+ let pairs = [(Builtin::Srgb, Builtin::DisplayP3), (Builtin::ProPhotoCompat, Builtin::Srgb), (Builtin::AdobeRgbCompat, Builtin::Rec2020)];+ for (a, b) in pairs {+ let t = Transform::new(a.profile(), b.profile(), Intent::RelativeColorimetric, true).unwrap();+ for &(ss, ds, copy) in &[(3usize, 3usize, false), (4, 4, true), (5, 4, true)] {+ let max = 20011;+ let seed = |i: usize| (i.wrapping_mul(2654435761) >> 7) as u32;+ let src16: Vec<u16> = (0..max * ss).map(|i| (seed(i) >> 3) as u16).collect();+ let src8: Vec<u8> = (0..max * ss).map(|i| (seed(i) >> 11) as u8).collect();+ let mut full8 = vec![0u8; max * ds];+ t.convert_u8(&src8, ss, &mut full8, ds, copy);+ for px in [1usize, 2, 63, 64, 65, 127, 128, 129, 1000] {+ let mut o16 = vec![0u16; px * ds];+ t.convert_u16(&src16[..px * ss], ss, &mut o16, ds, copy);+ let mut o8 = vec![0u8; px * ds];+ t.convert_u8(&src8[..px * ss], ss, &mut o8, ds, copy);+ assert_eq!(o8[..], full8[..px * ds], "u8 prefix of a longer buffer, {px} px");+ for p in 0..px {+ let inp: Vec<f32> = (0..3).map(|k| src16[p * ss + k] as f32 / 65535.0).collect();+ let mut want = [0.0f32; 3];+ t.eval_fast(&inp, &mut want);+ for k in 0..3 {+ assert_eq!(o16[p * ds + k], (want[k].clamp(0.0, 1.0) * 65535.0 + 0.5) as u16, "u16 px {p} ch {k} of {px}");+ }+ if copy && ds > 3 {+ assert_eq!(o16[p * ds + 3], src16[p * ss + 3]);+ assert_eq!(o8[p * ds + 3], src8[p * ss + 3]);+ }+ }+ }+ }+ }+}++/// 8-bit conversion through an analytic pipeline (no lookup table, here RGB to Lab) tabulates+/// the first stage's curves per code; the result must equal the exact per-pixel evaluation.+#[test]+fn exact_core_u8_matches_exact_eval() {+ let t = Transform::new(srgb(), lab(), Intent::RelativeColorimetric, true).unwrap();+ let px = 256usize * 256;+ let src: Vec<u8> = (0..px as u32).flat_map(|i| [(i & 255) as u8, (i >> 8) as u8, (i.wrapping_mul(2654435761) >> 13) as u8, 200]).collect();+ let mut out = vec![0u8; px * 4];+ t.convert_u8(&src, 4, &mut out, 4, true);+ for (sp, dp) in src.as_chunks::<4>().0.iter().zip(out.as_chunks::<4>().0) {+ let inp = [sp[0] as f32 / 255.0, sp[1] as f32 / 255.0, sp[2] as f32 / 255.0];+ let mut want = [0.0f32; 3];+ t.eval(&inp, &mut want);+ for k in 0..3 {+ assert_eq!(dp[k], (want[k].clamp(0.0, 1.0) * 255.0 + 0.5) as u8, "{sp:?} ch {k}");+ }+ assert_eq!(dp[3], 200);+ }+}