collective
Browse / github.com/storytold/filmcraft / patch
h264: const-generic MC kernels, i16 first pass, inline window row copy
h264: MC filters with compile-time block width and i16 first pass Found while auditing `crates/h264` at upstream/main bfc23840 (Ryzen 7800X3D, single thread, `perf stat -e instructions:u,cycles:u` and a flat `perf record` profile; fixtures: x264 1080p CRF 20 long-GOP B, CAVLC, intra-only, and a 3840x2160 high-bitrate stream, each with an ffmpeg md5). Gains marked "estimated" were not measured. Today: 1080p long-GOP ~152 M instructions/frame (~100 fps single thread), 1080p intra 342 M (39 fps), 2160p high-bitrate 2340 M (5 fps). **MC filters use runtime block widths and i32 intermediates** (`inter.rs:116-229` `mc_luma_win`, `248-264` `mc_chroma_win`). The wi
- Status
- open
- Score
- 0.00
- Adopters
- 0 other operators
- Author
- vangel76 (imported from pull request #3183, open)
- Base
bfc23840b603e69c782302c6ed04d4a9f6ec26a8- Size
- +133 −74 in 2 files
- Review
- pending
- Shared
- 2026-10-28
- Id
a444d323d73a0d19efae4c1b
How it went
No other operator has reported using it yet.
Diff
raw · crates/h264/src/inter.rs crates/h264/src/picture.rs
@@ -72,21 +72,41 @@ fn avg(a: u8, b: u8) -> u8 { ((a as u16 + b as u16 + 1) >> 1) as u8 } -/// Horizontal 6-tap (unclipped) at every position of `row` for `n` outputs: out[c] uses row[c..c + 6].+/// 6-tap filter in i16: the unclipped result of a first pass lies in -2550..=10710. #[inline(always)]-fn htap_row(row: &[u8], out: &mut [i32], n: usize) {- let row = &row[..n + 5];- for (o, w) in out[..n].iter_mut().zip(row.windows(6)) {- *o = tap6(w[0] as i32, w[1] as i32, w[2] as i32, w[3] as i32, w[4] as i32, w[5] as i32);+fn tap6_16(a: u8, b: u8, c: u8, d: u8, e: u8, f: u8) -> i16 {+ let [a, b, c, d, e, f] = [a, b, c, d, e, f].map(i16::from);+ a - 5 * b + 20 * c + 20 * d - 5 * e + f+}++/// Round and clip a first-pass value: Clip1((v + 16) >> 5).+#[inline(always)]+fn round5(v: i16) -> u8 {+ ((v + 16) >> 5).clamp(0, 255) as u8+}++/// Horizontal 6-tap (unclipped) for `BW` outputs: out[c] uses row[c..c + 6].+#[inline(always)]+fn htap_row<const BW: usize>(row: &[u8], out: &mut [i16; 16]) {+ let row = &row[..BW + 5];+ for c in 0..BW {+ out[c] = tap6_16(row[c], row[c + 1], row[c + 2], row[c + 3], row[c + 4], row[c + 5]);+ }+}++/// Vertical 6-tap (unclipped) over six rows at columns 0..BW.+#[inline(always)]+fn vtap_row<const BW: usize>(rows: [&[u8]; 6], out: &mut [i16; 16]) {+ let [a, b, c, d, e, f] = rows.map(|r| &r[..BW]);+ for i in 0..BW {+ out[i] = tap6_16(a[i], b[i], c[i], d[i], e[i], f[i]); } } -/// Vertical 6-tap (unclipped) over six rows at columns 0..n. #[inline(always)]-fn vtap_row(rows: [&[u8]; 6], out: &mut [i32], n: usize) {- let [a, b, c, d, e, f] = rows.map(|r| &r[..n]);- for i in 0..n {- out[i] = tap6(a[i] as i32, b[i] as i32, c[i] as i32, d[i] as i32, e[i] as i32, f[i] as i32);+fn copy_fixed<const BW: usize>(dst: &mut [u8], src: &[u8]) {+ if let (Some(d), Some(s)) = (dst.first_chunk_mut::<BW>(), src.first_chunk::<BW>()) {+ *d = *s; } } @@ -108,90 +128,90 @@ pub fn mc_luma(p: PlaneRef, x: i32, y: i32, fx: u32, fy: u32, bw: usize, bh: usi } /// Luma interpolation from a prepared window: `src[0]` is the sample at (x - 2, y - 2) and `ss` the-/// window stride; the window covers (bw + 5) x (bh + 5) samples.+/// window stride; the window covers (bw + 5) x (bh + 5) samples. `bw` is 4, 8 or 16. #[allow(clippy::too_many_arguments)] pub fn mc_luma_win(src: &[u8], ss: usize, fx: u32, fy: u32, bw: usize, bh: usize, out: &mut [u8], os: usize) {+ match bw {+ 16 => luma_win::<16>(src, ss, fx, fy, bh, out, os),+ 8 => luma_win::<8>(src, ss, fx, fy, bh, out, os),+ _ => luma_win::<4>(src, ss, fx, fy, bh, out, os),+ }+}++#[inline(always)]+fn luma_win<const BW: usize>(src: &[u8], ss: usize, fx: u32, fy: u32, bh: usize, out: &mut [u8], os: usize) { // window row r (0..bh+5) starts at src[r * ss]; sample G of block (r, c) is at window (r + 2, c + 2)- let wrow = |r: usize| &src[r * ss..r * ss + bw + 5];- let mut t = [0i32; 16];+ let wrow = |r: usize| &src[r * ss..r * ss + BW + 5];+ let rows6 = |r: usize, off: usize| [wrow(r), wrow(r + 1), wrow(r + 2), wrow(r + 3), wrow(r + 4), wrow(r + 5)].map(|x| &x[off..]);+ let mut t = [0i16; 16]; match (fx, fy) { (0, 0) => { for r in 0..bh {- copy_n(&mut out[r * os..], &wrow(r + 2)[2..], bw);+ copy_fixed::<BW>(&mut out[r * os..], &wrow(r + 2)[2..]); } } (_, 0) => {+ let g = if fx == 1 { 2 } else { 3 }; for r in 0..bh { let row = wrow(r + 2);- htap_row(row, &mut t, bw);- let o = &mut out[r * os..r * os + bw];- match fx {- 1 => {- for c in 0..bw {- o[c] = avg(row[c + 2], clip1((t[c] + 16) >> 5));- }- }- 2 => {- for c in 0..bw {- o[c] = clip1((t[c] + 16) >> 5);- }+ htap_row::<BW>(row, &mut t);+ let o = &mut out[r * os..r * os + BW];+ if fx == 2 {+ for c in 0..BW {+ o[c] = round5(t[c]); }- _ => {- for c in 0..bw {- o[c] = avg(row[c + 3], clip1((t[c] + 16) >> 5));- }+ } else {+ let g = &row[g..g + BW];+ for c in 0..BW {+ o[c] = avg(g[c], round5(t[c])); } } } } (0, _) => { for r in 0..bh {- let rows = [wrow(r), wrow(r + 1), wrow(r + 2), wrow(r + 3), wrow(r + 4), wrow(r + 5)].map(|x| &x[2..]);- vtap_row(rows, &mut t, bw);- let g = if fy == 1 { rows[2] } else { rows[3] };- let o = &mut out[r * os..r * os + bw];+ let rows = rows6(r, 2);+ vtap_row::<BW>(rows, &mut t);+ let o = &mut out[r * os..r * os + BW]; if fy == 2 {- for c in 0..bw {- o[c] = clip1((t[c] + 16) >> 5);+ for c in 0..BW {+ o[c] = round5(t[c]); } } else {- for c in 0..bw {- o[c] = avg(g[c], clip1((t[c] + 16) >> 5));+ let g = &(if fy == 1 { rows[2] } else { rows[3] })[..BW];+ for c in 0..BW {+ o[c] = avg(g[c], round5(t[c])); } } } } (2, _) | (_, 2) => { // unclipped horizontal half samples b1 for window rows 0..bh+5- let mut b1 = [[0i32; 16]; WIN];+ let mut b1 = [[0i16; 16]; WIN]; for (r, bt) in b1.iter_mut().enumerate().take(bh + 5) {- htap_row(wrow(r), bt, bw);+ htap_row::<BW>(wrow(r), bt); }- let mut v = [0i32; 16];+ let mut j = [0u8; 16];+ let mut v = [0i16; 16]; for r in 0..bh {- let o = &mut out[r * os..r * os + bw];- let (b0, b1r, b2, b3, b4, b5) = (&b1[r], &b1[r + 1], &b1[r + 2], &b1[r + 3], &b1[r + 4], &b1[r + 5]);- for c in 0..bw {- t[c] = tap6(b0[c], b1r[c], b2[c], b3[c], b4[c], b5[c]);+ let o = &mut out[r * os..r * os + BW];+ let [b0, b1r, b2, b3, b4, b5] = [&b1[r], &b1[r + 1], &b1[r + 2], &b1[r + 3], &b1[r + 4], &b1[r + 5]];+ for c in 0..BW {+ let [a, b, cc, d, e, f] = [b0[c], b1r[c], b2[c], b3[c], b4[c], b5[c]].map(i32::from);+ j[c] = clip1((tap6(a, b, cc, d, e, f) + 512) >> 10); } match (fx, fy) {- (2, 2) => {- for c in 0..bw {- o[c] = clip1((t[c] + 512) >> 10);- }- }+ (2, 2) => o.copy_from_slice(&j[..BW]), (2, 1) | (2, 3) => { let bh_ = if fy == 1 { b2 } else { b3 };- for c in 0..bw {- o[c] = avg(clip1((bh_[c] + 16) >> 5), clip1((t[c] + 512) >> 10));+ for c in 0..BW {+ o[c] = avg(round5(bh_[c]), j[c]); } } _ => {- let off = if fx == 1 { 2 } else { 3 };- let rows = [wrow(r), wrow(r + 1), wrow(r + 2), wrow(r + 3), wrow(r + 4), wrow(r + 5)].map(|x| &x[off..]);- vtap_row(rows, &mut v, bw);- for c in 0..bw {- o[c] = avg(clip1((v[c] + 16) >> 5), clip1((t[c] + 512) >> 10));+ vtap_row::<BW>(rows6(r, if fx == 1 { 2 } else { 3 }), &mut v);+ for c in 0..BW {+ o[c] = avg(round5(v[c]), j[c]); } } }@@ -201,14 +221,13 @@ pub fn mc_luma_win(src: &[u8], ss: usize, fx: u32, fy: u32, bw: usize, bh: usize // e, g, p, r: average of a horizontal half sample (row r or r + 1) and a vertical one (col c or c + 1) let hoff = if fy == 1 { 2 } else { 3 }; let voff = if fx == 1 { 2 } else { 3 };- let mut v = [0i32; 16];+ let mut v = [0i16; 16]; for r in 0..bh {- htap_row(wrow(r + hoff), &mut t, bw);- let rows = [wrow(r), wrow(r + 1), wrow(r + 2), wrow(r + 3), wrow(r + 4), wrow(r + 5)].map(|x| &x[voff..]);- vtap_row(rows, &mut v, bw);- let o = &mut out[r * os..r * os + bw];- for c in 0..bw {- o[c] = avg(clip1((t[c] + 16) >> 5), clip1((v[c] + 16) >> 5));+ htap_row::<BW>(wrow(r + hoff), &mut t);+ vtap_row::<BW>(rows6(r, voff), &mut v);+ let o = &mut out[r * os..r * os + BW];+ for c in 0..BW {+ o[c] = avg(round5(t[c]), round5(v[c])); } } }@@ -240,24 +259,33 @@ pub fn mc_chroma(p: PlaneRef, x: i32, y: i32, fx: u32, fy: u32, bw: usize, bh: u } /// Chroma interpolation from a prepared (bw + 1) x (bh + 1) window whose first sample is the integer-/// position of the block.+/// position of the block. `bw` is 2, 4 or 8. #[allow(clippy::too_many_arguments)] pub fn mc_chroma_win(src: &[u8], ss: usize, fx: u32, fy: u32, bw: usize, bh: usize, out: &mut [u8], os: usize) {- let (fx, fy) = (fx as u16, fy as u16);- let (w00, w10, w01, w11) = ((8 - fx) * (8 - fy), fx * (8 - fy), (8 - fx) * fy, fx * fy);+ match bw {+ 8 => chroma_win::<8>(src, ss, fx, fy, bh, out, os),+ 4 => chroma_win::<4>(src, ss, fx, fy, bh, out, os),+ _ => chroma_win::<2>(src, ss, fx, fy, bh, out, os),+ }+}++#[inline(always)]+fn chroma_win<const BW: usize>(src: &[u8], ss: usize, fx: u32, fy: u32, bh: usize, out: &mut [u8], os: usize) { if fx == 0 && fy == 0 { for r in 0..bh {- copy_n(&mut out[r * os..], &src[r * ss..], bw);+ copy_fixed::<BW>(&mut out[r * os..], &src[r * ss..]); } return; }+ let (fx, fy) = (fx as u16, fy as u16);+ let (w00, w10, w01, w11) = ((8 - fx) * (8 - fy), fx * (8 - fy), (8 - fx) * fy, fx * fy); for r in 0..bh {- let r0 = &src[r * ss..r * ss + bw + 1];- let r1 = &src[(r + 1) * ss..(r + 1) * ss + bw + 1];- let o = &mut out[r * os..r * os + bw];- for (c, o) in o.iter_mut().enumerate() {+ let r0 = &src[r * ss..r * ss + BW + 1];+ let r1 = &src[(r + 1) * ss..(r + 1) * ss + BW + 1];+ let o = &mut out[r * os..r * os + BW];+ for c in 0..BW { let v = w00 * r0[c] as u16 + w10 * r0[c + 1] as u16 + w01 * r1[c] as u16 + w11 * r1[c + 1] as u16;- *o = ((v + 32) >> 6) as u8;+ o[c] = ((v + 32) >> 6) as u8; } } }@@ -500,6 +528,34 @@ mod tests { } } + #[test]+ fn chroma_matches_formula() {+ let mut seed = 11u32;+ let src: Vec<u8> = (0..9 * 9)+ .map(|_| {+ seed = seed.wrapping_mul(1664525).wrapping_add(1013904223);+ (seed >> 24) as u8+ })+ .collect();+ for bw in [2, 4, 8] {+ for bh in [2, 4, 8] {+ for fx in 0..8 {+ for fy in 0..8 {+ let mut out = [0u8; 64];+ mc_chroma_win(&src, 9, fx, fy, bw, bh, &mut out, 8);+ for r in 0..bh {+ for c in 0..bw {+ let s = |dr: usize, dc: usize| src[(r + dr) * 9 + c + dc] as u32;+ let want = ((8 - fx) * (8 - fy) * s(0, 0) + fx * (8 - fy) * s(0, 1) + (8 - fx) * fy * s(1, 0) + fx * fy * s(1, 1) + 32) >> 6;+ assert_eq!(out[r * 8 + c] as u32, want, "{bw}x{bh} frac {fx},{fy} at {r},{c}");+ }+ }+ }+ }+ }+ }+ }+ #[test] fn weighting() { let a = [100u8; 4];@@ -302,7 +302,10 @@ fn copy_line<const N: usize>(line: &[u8], x0: i32, out: &mut [u8], w: usize) { } /// Copy `out.len()` samples of `line` starting at x0, replicating edge samples outside the line.-#[inline(always)]+/// Kept out of line so the fixed-size path of `copy_line` stays an inline move rather than being+/// merged with this variable-length copy into a libc memcpy call.+#[cold]+#[inline(never)] fn copy_clamped(line: &[u8], x0: i32, out: &mut [u8]) { let w = out.len(); if x0 >= 0 && x0 as usize + w <= line.len() {