collective
Browse / github.com/storytold/photocraft / patch
perf(cms): block the 8/16-bit matrix/TRC conversion over 64 pixels
cms: 8/16-bit matrix/TRC conversion does 2.3x too many instructions per pixel (blocked path, bit-identical) ## Proposal `crates/cms/src/transform.rs`: the 8-bit and 16-bit matrix/TRC paths (`run_u8`, `run_u16`) spend most of their instructions on per-pixel overhead. Run the 3x3 matrix and the output-table index maths over blocks of 64 pixels with fixed-size arrays instead of one pixel at a time through `Vec` and `Option` indirection (`apply_matrix`, `Post::eval`, `eval_fast`). This is the path of every RGB-to-RGB conversion between matrix/TRC profiles: the colour-managed canvas display for a non-sRGB document or monitor (`convert_u8` via `display_color::apply_u8`), Convert to Profile and Assign on
- Status
- open
- Score
- 0.00
- Adopters
- 0 other operators
- Author
- vangel76 (imported from pull request #9746, open)
- Base
a0ce166acd2a64ff1200f793d4006db61c09f4d6- Size
- +118 −0 in 2 files
- Review
- pending
- Shared
- 2026-10-28
- Id
0e2c237e8c822bb4e864cdb8
How it went
No other operator has reported using it yet.
Diff
raw · crates/cms/src/transform.rs crates/cms/tests/it/cms.rs
@@ -42,6 +42,8 @@ pub const GRID_4D: usize = 17; const TABLE_1D: usize = GpuMatrixShaper::INPUT_INTERVALS; /// Resolution of output shaper tables (indexed by `x^(1/4)`). const OUT_TABLE: usize = GpuMatrixShaper::OUTPUT_ENTRIES;+// The blocked matrix/TRC path wraps table indices with a mask instead of a bounds check.+const _: () = assert!(OUT_TABLE.is_power_of_two()); /// ICC v4 perceptual reference medium black (ICC.1:2010 §6.3.4.4), XYZ. const PERCEPTUAL_BLACK: [f64; 3] = [0.00336, 0.0034731, 0.00287];@@ -386,6 +388,10 @@ impl Transform { } } }+ Core::Shaper { curves8, matrix: Some((3, 3, m, off)), .. } if self.shaper3_ok(curves8.len(), m.len(), off.len()) => {+ let load = |sp: &[u8]| [curves8[0][sp[0] as usize], curves8[1][sp[1] as usize], curves8[2][sp[2] as usize]];+ self.shaper3_blocks(s, ss, d, ds, extra, m, off, load, q);+ } Core::Shaper { curves8, matrix, .. } => { let mut v = [0.0f32; 16]; for (sp, dp) in s.chunks_exact(ss).zip(d.chunks_exact_mut(ds)) {@@ -420,6 +426,74 @@ impl Transform { } } + fn shaper3_ok(&self, curves: usize, m: usize, off: usize) -> bool {+ self.inputs == 3+ && self.outputs == 3+ && curves == 3+ && m == 9+ && off == 3+ && self.post.as_ref().is_some_and(|p| p.tables.len() == 3 && p.tables.iter().all(|t| t.len() == OUT_TABLE))+ }++ /// Matrix/TRC transform of whole pixels in blocks: the matrix and the output-table index+ /// maths run over a block of pixels at a time (same operations, same order, per pixel).+ #[allow(clippy::too_many_arguments)]+ fn shaper3_blocks<S: Copy>(+ &self,+ s: &[S],+ ss: usize,+ d: &mut [S],+ ds: usize,+ extra: usize,+ m: &[f32],+ off: &[f32],+ load: impl Fn(&[S]) -> [f32; 3],+ store: impl Fn(f32) -> S,+ ) {+ const B: usize = 64;+ let Some(post) = self.post.as_ref() else { return };+ let tabs = [&post.tables[0][..OUT_TABLE], &post.tables[1][..OUT_TABLE], &post.tables[2][..OUT_TABLE]];+ let m: [f32; 9] = std::array::from_fn(|i| m[i]);+ let off: [f32; 3] = std::array::from_fn(|i| off[i]);+ let n_px = (s.len() / ss).min(d.len() / ds);+ let mut x = [[0.0f32; B]; 3];+ let mut start = 0;+ while start < n_px {+ let n = B.min(n_px - start);+ let sb = &s[start * ss..(start + n) * ss];+ let db = &mut d[start * ds..(start + n) * ds];+ for (j, sp) in sb.chunks_exact(ss).enumerate() {+ let [a, b, c] = load(sp);+ for r in 0..3 {+ let mut acc = off[r];+ acc += m[r * 3] * a;+ acc += m[r * 3 + 1] * b;+ acc += m[r * 3 + 2] * c;+ x[r][j] = acc;+ }+ }+ for (ch, t) in tabs.iter().enumerate() {+ let mut u = [0.0f32; B];+ let mut ix = [0i32; B];+ for j in 0..B {+ let v = x[ch][j].clamp(0.0, 1.0).sqrt().sqrt() * (OUT_TABLE - 1) as f32;+ let i = (v as i32).min(OUT_TABLE as i32 - 2);+ ix[j] = i;+ u[j] = v - i as f32;+ }+ for (j, dp) in db.chunks_exact_mut(ds).enumerate() {+ let i = ix[j] as usize & (OUT_TABLE - 1);+ let (v0, v1) = (t[i], t[(i + 1) & (OUT_TABLE - 1)]);+ dp[ch] = store(v0 + (v1 - v0) * u[j]);+ }+ }+ for (sp, dp) in sb.chunks_exact(ss).zip(db.chunks_exact_mut(ds)) {+ dp[3..3 + extra].copy_from_slice(&sp[3..3 + extra]);+ }+ start += n;+ }+ }+ /// Converts 16-bit pixels (see [`Transform::convert_f32`] for the layout). pub fn convert_u16(&self, src: &[u16], src_stride: usize, dst: &mut [u16], dst_stride: usize, copy_extra: bool) { self.check(src_stride, dst_stride);@@ -428,6 +502,13 @@ impl Transform { } fn run_u16(&self, s: &[u16], ss: usize, d: &mut [u16], ds: usize, extra: usize) {+ if let Core::Shaper { curves, matrix: Some((3, 3, m, off)), .. } = &self.core+ && self.shaper3_ok(curves.len(), m.len(), off.len())+ {+ let load =+ |sp: &[u16]| [table(&curves[0], sp[0] as f32 / 65535.0), table(&curves[1], sp[1] as f32 / 65535.0), table(&curves[2], sp[2] as f32 / 65535.0)];+ return self.shaper3_blocks(s, ss, d, ds, extra, m, off, load, |v: f32| (v.clamp(0.0, 1.0) * 65535.0 + 0.5) as u16);+ } let mut inp = [0.0f32; 16]; let mut out = [0.0f32; 16]; for (sp, dp) in s.chunks_exact(ss).zip(d.chunks_exact_mut(ds)) {@@ -819,3 +819,40 @@ fn device_links_parse_and_are_reported_unsupported() { // The output space still bounds the tag: a CMYK→CMYK link with a 4→3 table is malformed. assert!(matches!(Profile::parse(&device_link(b"CMYK", b"CMYK", 4, 3)), Err(photocraft_cms::CmsError::Invalid(_)))); }++/// The blocked matrix/TRC path (64 pixels at a time) must match the per-pixel evaluation for+/// every pixel count around the block size, with and without extra channels.+#[test]+fn matrix_trc_blocks_match_per_pixel_eval() {+ let pairs = [(Builtin::Srgb, Builtin::DisplayP3), (Builtin::ProPhotoCompat, Builtin::Srgb), (Builtin::AdobeRgbCompat, Builtin::Rec2020)];+ for (a, b) in pairs {+ let t = Transform::new(a.profile(), b.profile(), Intent::RelativeColorimetric, true).unwrap();+ for &(ss, ds, copy) in &[(3usize, 3usize, false), (4, 4, true), (5, 4, true)] {+ let max = 20011;+ let seed = |i: usize| (i.wrapping_mul(2654435761) >> 7) as u32;+ let src16: Vec<u16> = (0..max * ss).map(|i| (seed(i) >> 3) as u16).collect();+ let src8: Vec<u8> = (0..max * ss).map(|i| (seed(i) >> 11) as u8).collect();+ let mut full8 = vec![0u8; max * ds];+ t.convert_u8(&src8, ss, &mut full8, ds, copy);+ for px in [1usize, 2, 63, 64, 65, 127, 128, 129, 1000] {+ let mut o16 = vec![0u16; px * ds];+ t.convert_u16(&src16[..px * ss], ss, &mut o16, ds, copy);+ let mut o8 = vec![0u8; px * ds];+ t.convert_u8(&src8[..px * ss], ss, &mut o8, ds, copy);+ assert_eq!(o8[..], full8[..px * ds], "u8 prefix of a longer buffer, {px} px");+ for p in 0..px {+ let inp: Vec<f32> = (0..3).map(|k| src16[p * ss + k] as f32 / 65535.0).collect();+ let mut want = [0.0f32; 3];+ t.eval_fast(&inp, &mut want);+ for k in 0..3 {+ assert_eq!(o16[p * ds + k], (want[k].clamp(0.0, 1.0) * 65535.0 + 0.5) as u16, "u16 px {p} ch {k} of {px}");+ }+ if copy && ds > 3 {+ assert_eq!(o16[p * ds + 3], src16[p * ss + 3]);+ assert_eq!(o8[p * ds + 3], src8[p * ss + 3]);+ }+ }+ }+ }+ }+}