diff --git a/crates/cms/src/pipeline.rs b/crates/cms/src/pipeline.rs --- a/crates/cms/src/pipeline.rs +++ b/crates/cms/src/pipeline.rs @@ -183,6 +183,17 @@ impl Pipeline { out[..self.outputs].copy_from_slice(&v[..self.outputs]); } + /// Evaluates the stages after the first one on values already passed through it. + #[inline] + pub(crate) fn eval_after_first(&self, v: &mut [f32; MAX], out: &mut [f32]) { + let mut n = self.inputs; + for s in self.stages.iter().skip(1) { + n = s.run(v, n); + } + let _ = n; + out[..self.outputs].copy_from_slice(&v[..self.outputs]); + } + /// Removes no-op stages and cancels adjacent inverse pairs (Lab↔XYZ, encode/decode). pub fn optimize(&mut self) { let mut out: Vec = Vec::with_capacity(self.stages.len()); diff --git a/crates/cms/src/transform.rs b/crates/cms/src/transform.rs --- a/crates/cms/src/transform.rs +++ b/crates/cms/src/transform.rs @@ -42,6 +42,8 @@ pub const GRID_4D: usize = 17; const TABLE_1D: usize = GpuMatrixShaper::INPUT_INTERVALS; /// Resolution of output shaper tables (indexed by `x^(1/4)`). const OUT_TABLE: usize = GpuMatrixShaper::OUTPUT_ENTRIES; +// The blocked matrix/TRC path wraps table indices with a mask instead of a bounds check. +const _: () = assert!(OUT_TABLE.is_power_of_two()); /// ICC v4 perceptual reference medium black (ICC.1:2010 §6.3.4.4), XYZ. const PERCEPTUAL_BLACK: [f64; 3] = [0.00336, 0.0034731, 0.00287]; @@ -91,6 +93,8 @@ pub struct Transform { post: Option, /// Pre-located grid coordinates per 8-bit code: (offset contribution, fraction) per input. grid8: Vec<[(u32, f32); 256]>, + /// For an exact core: the first stage's curves at every 8-bit code. + exact8: Vec<[f32; 256]>, } impl Transform { @@ -178,7 +182,13 @@ impl Transform { .collect(), _ => Vec::new(), }; - Transform { inputs, outputs, opts, pipeline, core, post, grid8 } + let exact8 = match (&core, pipeline.stages.first()) { + (Core::Exact, Some(Stage::Curves(cs))) if cs.len() == inputs && inputs <= 4 => { + cs.iter().map(|c| std::array::from_fn(|i| c.eval(i as f32 / 255.0))).collect() + } + _ => Vec::new(), + }; + Transform { inputs, outputs, opts, pipeline, core, post, grid8, exact8 } } /// White preservation: the device-link node at the source's device white gets exactly the @@ -386,6 +396,10 @@ impl Transform { } } } + Core::Shaper { curves8, matrix: Some((3, 3, m, off)), .. } if self.shaper3_ok(curves8.len(), m.len(), off.len()) => { + let load = |sp: &[u8]| [curves8[0][sp[0] as usize], curves8[1][sp[1] as usize], curves8[2][sp[2] as usize]]; + self.shaper3_blocks(s, ss, d, ds, extra, m, off, load, q); + } Core::Shaper { curves8, matrix, .. } => { let mut v = [0.0f32; 16]; for (sp, dp) in s.chunks_exact(ss).zip(d.chunks_exact_mut(ds)) { @@ -402,6 +416,21 @@ impl Transform { } } } + Core::Exact if !self.exact8.is_empty() => { + for (sp, dp) in s.chunks_exact(ss).zip(d.chunks_exact_mut(ds)) { + let mut v = [0.0f32; 16]; + for k in 0..self.inputs { + v[k] = self.exact8[k][sp[k] as usize]; + } + self.pipeline.eval_after_first(&mut v, &mut out); + for k in 0..self.outputs { + dp[k] = q(out[k]); + } + for e in 0..extra { + dp[self.outputs + e] = sp[self.inputs + e]; + } + } + } _ => { let mut inp = [0.0f32; 16]; for (sp, dp) in s.chunks_exact(ss).zip(d.chunks_exact_mut(ds)) { @@ -420,6 +449,74 @@ impl Transform { } } + fn shaper3_ok(&self, curves: usize, m: usize, off: usize) -> bool { + self.inputs == 3 + && self.outputs == 3 + && curves == 3 + && m == 9 + && off == 3 + && self.post.as_ref().is_some_and(|p| p.tables.len() == 3 && p.tables.iter().all(|t| t.len() == OUT_TABLE)) + } + + /// Matrix/TRC transform of whole pixels in blocks: the matrix and the output-table index + /// maths run over a block of pixels at a time (same operations, same order, per pixel). + #[allow(clippy::too_many_arguments)] + fn shaper3_blocks( + &self, + s: &[S], + ss: usize, + d: &mut [S], + ds: usize, + extra: usize, + m: &[f32], + off: &[f32], + load: impl Fn(&[S]) -> [f32; 3], + store: impl Fn(f32) -> S, + ) { + const B: usize = 64; + let Some(post) = self.post.as_ref() else { return }; + let tabs = [&post.tables[0][..OUT_TABLE], &post.tables[1][..OUT_TABLE], &post.tables[2][..OUT_TABLE]]; + let m: [f32; 9] = std::array::from_fn(|i| m[i]); + let off: [f32; 3] = std::array::from_fn(|i| off[i]); + let n_px = (s.len() / ss).min(d.len() / ds); + let mut x = [[0.0f32; B]; 3]; + let mut start = 0; + while start < n_px { + let n = B.min(n_px - start); + let sb = &s[start * ss..(start + n) * ss]; + let db = &mut d[start * ds..(start + n) * ds]; + for (j, sp) in sb.chunks_exact(ss).enumerate() { + let [a, b, c] = load(sp); + for r in 0..3 { + let mut acc = off[r]; + acc += m[r * 3] * a; + acc += m[r * 3 + 1] * b; + acc += m[r * 3 + 2] * c; + x[r][j] = acc; + } + } + for (ch, t) in tabs.iter().enumerate() { + let mut u = [0.0f32; B]; + let mut ix = [0i32; B]; + for j in 0..B { + let v = x[ch][j].clamp(0.0, 1.0).sqrt().sqrt() * (OUT_TABLE - 1) as f32; + let i = (v as i32).min(OUT_TABLE as i32 - 2); + ix[j] = i; + u[j] = v - i as f32; + } + for (j, dp) in db.chunks_exact_mut(ds).enumerate() { + let i = ix[j] as usize & (OUT_TABLE - 1); + let (v0, v1) = (t[i], t[(i + 1) & (OUT_TABLE - 1)]); + dp[ch] = store(v0 + (v1 - v0) * u[j]); + } + } + for (sp, dp) in sb.chunks_exact(ss).zip(db.chunks_exact_mut(ds)) { + dp[3..3 + extra].copy_from_slice(&sp[3..3 + extra]); + } + start += n; + } + } + /// Converts 16-bit pixels (see [`Transform::convert_f32`] for the layout). pub fn convert_u16(&self, src: &[u16], src_stride: usize, dst: &mut [u16], dst_stride: usize, copy_extra: bool) { self.check(src_stride, dst_stride); @@ -428,6 +525,13 @@ impl Transform { } fn run_u16(&self, s: &[u16], ss: usize, d: &mut [u16], ds: usize, extra: usize) { + if let Core::Shaper { curves, matrix: Some((3, 3, m, off)), .. } = &self.core + && self.shaper3_ok(curves.len(), m.len(), off.len()) + { + let load = + |sp: &[u16]| [table(&curves[0], sp[0] as f32 / 65535.0), table(&curves[1], sp[1] as f32 / 65535.0), table(&curves[2], sp[2] as f32 / 65535.0)]; + return self.shaper3_blocks(s, ss, d, ds, extra, m, off, load, |v: f32| (v.clamp(0.0, 1.0) * 65535.0 + 0.5) as u16); + } let mut inp = [0.0f32; 16]; let mut out = [0.0f32; 16]; for (sp, dp) in s.chunks_exact(ss).zip(d.chunks_exact_mut(ds)) { diff --git a/crates/cms/tests/it/cms.rs b/crates/cms/tests/it/cms.rs --- a/crates/cms/tests/it/cms.rs +++ b/crates/cms/tests/it/cms.rs @@ -819,3 +819,60 @@ fn device_links_parse_and_are_reported_unsupported() { // The output space still bounds the tag: a CMYK→CMYK link with a 4→3 table is malformed. assert!(matches!(Profile::parse(&device_link(b"CMYK", b"CMYK", 4, 3)), Err(photocraft_cms::CmsError::Invalid(_)))); } + +/// The blocked matrix/TRC path (64 pixels at a time) must match the per-pixel evaluation for +/// every pixel count around the block size, with and without extra channels. +#[test] +fn matrix_trc_blocks_match_per_pixel_eval() { + let pairs = [(Builtin::Srgb, Builtin::DisplayP3), (Builtin::ProPhotoCompat, Builtin::Srgb), (Builtin::AdobeRgbCompat, Builtin::Rec2020)]; + for (a, b) in pairs { + let t = Transform::new(a.profile(), b.profile(), Intent::RelativeColorimetric, true).unwrap(); + for &(ss, ds, copy) in &[(3usize, 3usize, false), (4, 4, true), (5, 4, true)] { + let max = 20011; + let seed = |i: usize| (i.wrapping_mul(2654435761) >> 7) as u32; + let src16: Vec = (0..max * ss).map(|i| (seed(i) >> 3) as u16).collect(); + let src8: Vec = (0..max * ss).map(|i| (seed(i) >> 11) as u8).collect(); + let mut full8 = vec![0u8; max * ds]; + t.convert_u8(&src8, ss, &mut full8, ds, copy); + for px in [1usize, 2, 63, 64, 65, 127, 128, 129, 1000] { + let mut o16 = vec![0u16; px * ds]; + t.convert_u16(&src16[..px * ss], ss, &mut o16, ds, copy); + let mut o8 = vec![0u8; px * ds]; + t.convert_u8(&src8[..px * ss], ss, &mut o8, ds, copy); + assert_eq!(o8[..], full8[..px * ds], "u8 prefix of a longer buffer, {px} px"); + for p in 0..px { + let inp: Vec = (0..3).map(|k| src16[p * ss + k] as f32 / 65535.0).collect(); + let mut want = [0.0f32; 3]; + t.eval_fast(&inp, &mut want); + for k in 0..3 { + assert_eq!(o16[p * ds + k], (want[k].clamp(0.0, 1.0) * 65535.0 + 0.5) as u16, "u16 px {p} ch {k} of {px}"); + } + if copy && ds > 3 { + assert_eq!(o16[p * ds + 3], src16[p * ss + 3]); + assert_eq!(o8[p * ds + 3], src8[p * ss + 3]); + } + } + } + } + } +} + +/// 8-bit conversion through an analytic pipeline (no lookup table, here RGB to Lab) tabulates +/// the first stage's curves per code; the result must equal the exact per-pixel evaluation. +#[test] +fn exact_core_u8_matches_exact_eval() { + let t = Transform::new(srgb(), lab(), Intent::RelativeColorimetric, true).unwrap(); + let px = 256usize * 256; + let src: Vec = (0..px as u32).flat_map(|i| [(i & 255) as u8, (i >> 8) as u8, (i.wrapping_mul(2654435761) >> 13) as u8, 200]).collect(); + let mut out = vec![0u8; px * 4]; + t.convert_u8(&src, 4, &mut out, 4, true); + for (sp, dp) in src.as_chunks::<4>().0.iter().zip(out.as_chunks::<4>().0) { + let inp = [sp[0] as f32 / 255.0, sp[1] as f32 / 255.0, sp[2] as f32 / 255.0]; + let mut want = [0.0f32; 3]; + t.eval(&inp, &mut want); + for k in 0..3 { + assert_eq!(dp[k], (want[k].clamp(0.0, 1.0) * 255.0 + 0.5) as u8, "{sp:?} ch {k}"); + } + assert_eq!(dp[3], 200); + } +}