diff --git a/Cargo.lock b/Cargo.lock --- a/Cargo.lock +++ b/Cargo.lock @@ -7486,8 +7486,11 @@ version = "0.14.0" dependencies = [ "brotli", "flate2", + "lightcraft-faces", "serde_json", "sha2", + "zune-core", + "zune-jpeg", ] [[package]] diff --git a/docs/faces.md b/docs/faces.md --- a/docs/faces.md +++ b/docs/faces.md @@ -235,6 +235,29 @@ The detector's output matches OpenCV's own YuNet on a 45-photo public-domain tes mean box overlap 0.97), including marble busts and paintings, with no false boxes on the landscape and architecture photos in the set. +### And how well does it find faces? + +`cargo xtask ai-bench` measures the detector on the **WIDER FACE validation set** (3,226 photos, 39,708 faces) and prints the +average precision (AP) for the benchmark's three settings: easy, medium and hard (hard includes the smallest faces). The photos are the publisher's, not ours: they are licensed CC BY-NC-ND 4.0, for non-commercial research +use only, so the command downloads them to your machine only when you pass `--accept-terms`, and they are never part of +this repository. + +```text +cargo xtask ai-bench fetch --accept-terms # WIDER_val.zip, the evaluation toolkit's ground truth and YuNet, + # into corpus/ai-bench (SHA-256 checked; needs curl and tar or unzip) +cargo xtask ai-bench run face-detection --set wider-face-val +``` + +`--dir DIR` puts the files elsewhere (the set goes in `DIR/wider-face-val`), `--model FILE` measures another +YuNet-architecture `.onnx`, and `--limit N` scores N photos spread evenly, for a quick look. Detection uses the same +code as **Photo ▸ Detect Faces** (a 640-pixel input), keeps every box scoring at least 0.02 as the protocol expects, and +scores by the benchmark's published procedure: intersection over union of at least 0.5, scores normalised over the whole +set, 1,000 thresholds. The scoring is our own implementation of that description, not the toolkit's MATLAB code. + +Measured with the pinned YuNet 2023mar model on an Intel Core i9-13950HX (32 threads, CPU only): **87.8 easy, 83.7 medium, +59.8 hard** AP in 41 s for the 3,226 photos. The published figures for this model are 88.44 / 86.56 / 75.03. The gap is widest on hard, which includes the smallest +faces, and those are what a 640-pixel input resolves worst (the limit noted above). + ## Suggesting who is in a photo With **Recognise faces** switched on in Settings ▸ Faces and a recognition model chosen (**Use**; the switch cannot be diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml --- a/xtask/Cargo.toml +++ b/xtask/Cargo.toml @@ -10,6 +10,10 @@ workspace = true [dependencies] serde_json = { workspace = true } +# `cargo xtask ai-bench` (pure Rust): LightCraft's own face detector and JPEG decoder, both already in the lockfile +lightcraft-faces = { workspace = true } +zune-jpeg = { workspace = true } +zune-core = { workspace = true } # corpus download checksums sha2 = { workspace = true } # precompressed web bundle (`cargo xtask web`); pure Rust diff --git a/xtask/src/ai_bench/fetch.rs b/xtask/src/ai_bench/fetch.rs new file mode 100644 --- /dev/null +++ b/xtask/src/ai_bench/fetch.rs @@ -0,0 +1,137 @@ +//! `cargo xtask ai-bench fetch`: download the WIDER FACE validation set and the YuNet model into the benchmark folder, +//! check each file against its pinned SHA-256 and unpack the archives. Needs `curl` and `tar` (or `unzip`), as `cargo xtask corpus` needs `curl`. + +use std::path::Path; +use std::process::Command; + +use lightcraft_faces::known; + +/// The terms the user accepts by passing `--accept-terms`. +const TERMS: &str = "WIDER FACE (http://shuoyang1213.me/WIDERFACE/) is licensed CC BY-NC-ND 4.0 (as its publisher's Hugging Face card states): \ +non-commercial research use only, no derivatives. The photos are downloaded to your machine and used only to measure detection accuracy; they \ +are never committed or redistributed. Pass --accept-terms to confirm you accept these terms."; + +/// A file to download: name, URL and SHA-256. +struct Archive { + name: &'static str, + url: &'static str, + sha256: &'static str, +} + +const ARCHIVES: [Archive; 2] = [ + Archive { + name: "WIDER_val.zip", + url: "https://huggingface.co/datasets/CUHK-CSE/wider_face/resolve/main/data/WIDER_val.zip", + sha256: "f9efbd09f28c5d2d884be8c0eaef3967158c866a593fc36ab0413e4b2a58a17a", + }, + Archive { + name: "eval_tools.zip", + url: "http://shuoyang1213.me/WIDERFACE/support/eval_script/eval_tools.zip", + sha256: "1cf49c8243fa8a1632efe7f6aa09fd7a3994ac838c9008bdbc9dd1d547bb234a", + }, +]; + +/// `curl` `url` to `dest`, unless `dest` already has `sha256`; a download with another hash is deleted and an error. +fn download(url: &str, dest: &Path, sha256: &str) -> Result<(), String> { + if crate::corpus::file_sha256(dest).is_ok_and(|h| h == sha256) { + return Ok(()); + } + let mut curl = Command::new("curl"); + curl.args(["-fsSL", "--create-dirs", "-o"]).arg(dest).arg(url); + crate::run(curl, &format!("curl {url}"))?; + let actual = crate::corpus::file_sha256(dest)?; + if actual != sha256 { + std::fs::remove_file(dest).map_err(|e| format!("remove {}: {e}", dest.display()))?; + return Err(format!("{}: sha256 is {actual}, expected {sha256} (file deleted; source {url})", dest.display())); + } + Ok(()) +} + +/// Unpack `zip` into `into` with the system's bsdtar (Windows, macOS), else `unzip`. +fn unpack(zip: &Path, into: &Path) -> Result<(), String> { + std::fs::create_dir_all(into).map_err(|e| format!("create {}: {e}", into.display()))?; + let mut tar = Command::new("tar"); + tar.arg("-xf").arg(zip).arg("-C").arg(into); + if crate::run(tar, &format!("tar -xf {}", zip.display())).is_ok() { + return Ok(()); + } + let mut unzip = Command::new("unzip"); + unzip.args(["-q", "-o"]).arg(zip).arg("-d").arg(into); + crate::run(unzip, &format!("unzip {}", zip.display())) +} + +/// Unpack `archive` (already checked against `sha256`) with `unpack` unless a completion marker for exactly that archive +/// exists. The marker is written, atomically, only after `unpack` succeeds: the unpacked folders can exist half-filled after an +/// interrupted extraction, so they never count as done and the next fetch extracts again over them. +fn unpack_once(set: &Path, archive: &str, sha256: &str, unpack: impl FnOnce() -> Result<(), String>) -> Result<(), String> { + let marker = set.join(format!(".{archive}.unpacked")); + if std::fs::read_to_string(&marker).is_ok_and(|done| done.trim() == sha256) { + return Ok(()); + } + unpack()?; + let pending = set.join(format!(".{archive}.unpacked.tmp")); + std::fs::write(&pending, sha256).map_err(|e| format!("write {}: {e}", pending.display()))?; + std::fs::rename(&pending, &marker).map_err(|e| format!("rename {}: {e}", pending.display())) +} + +/// Fetch everything `run` needs under `dir`: `wider-face-val/` and `models/`. +pub fn run(dir: &Path, accept_terms: bool) -> Result<(), String> { + if !accept_terms { + return Err(TERMS.to_string()); + } + let set = dir.join(super::SET); + for archive in ARCHIVES { + download(archive.url, &set.join(archive.name), archive.sha256)?; + unpack_once(&set, archive.name, archive.sha256, || unpack(&set.join(archive.name), &set))?; + } + let model = known::download(known::YUNET_ID).ok_or("LightCraft offers no download of YuNet")?; + download(&model.url, &dir.join("models").join(&model.file_name), &model.sha256)?; + println!("fetched into {}", dir.display()); + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn an_interrupted_extraction_is_redone_and_a_finished_one_is_not() { + let set = std::env::temp_dir().join(format!("ai-bench-fetch-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&set); + assert!(std::fs::create_dir_all(&set).is_ok()); + // the extraction dies after creating the folder that used to count as "done" + let interrupted = unpack_once(&set, "a.zip", "abc", || { + std::fs::create_dir_all(set.join("WIDER_val").join("images")).map_err(|e| e.to_string())?; + Err("interrupted".to_string()) + }); + assert!(interrupted.is_err()); + assert!(set.join("WIDER_val").join("images").is_dir()); + let mut runs = 0; + assert!( + unpack_once(&set, "a.zip", "abc", || { + runs += 1; + Ok(()) + }) + .is_ok() + ); + assert_eq!(runs, 1, "the next fetch must extract again"); + assert!( + unpack_once(&set, "a.zip", "abc", || { + runs += 1; + Ok(()) + }) + .is_ok() + ); + assert_eq!(runs, 1, "a finished extraction is not repeated"); + // a different archive (hash) under the same name is extracted again + assert!( + unpack_once(&set, "a.zip", "def", || { + runs += 1; + Ok(()) + }) + .is_ok() + ); + assert_eq!(runs, 2); + let _ = std::fs::remove_dir_all(&set); + } +} diff --git a/xtask/src/ai_bench/mat.rs b/xtask/src/ai_bench/mat.rs new file mode 100644 --- /dev/null +++ b/xtask/src/ai_bench/mat.rs @@ -0,0 +1,315 @@ +//! A reader for MATLAB level-5 MAT-files (the format of the WIDER FACE ground truth), written from MathWorks' published +//! file-format description: numeric arrays, char arrays and cell arrays, compressed or not, little-endian. Sizes are +//! checked against the data, nesting is bounded, and anything unexpected is an error. + +use std::io::Read; + +const MAX_DEPTH: usize = 16; +/// Largest decompressed variable. +const MAX_INFLATED: u64 = 1 << 30; +/// Largest single dimension of a matrix (every real variable is far smaller; a hostile one must not size an allocation). +const MAX_DIM: usize = 1 << 30; + +#[derive(Clone, Debug, PartialEq)] +pub enum Value { + /// A numeric matrix, column-major, as f64. + Num { + dims: Vec, + data: Vec, + }, + Char(String), + Cell(Vec), + /// Any other class (struct, sparse, objects): present but not read. + Other, +} + +impl Value { + pub fn cells(&self) -> Option<&[Value]> { + match self { + Value::Cell(items) => Some(items), + _ => None, + } + } + + pub fn text(&self) -> Option<&str> { + match self { + Value::Char(text) => Some(text), + _ => None, + } + } + + /// Rows of a numeric matrix (stored column-major); an empty matrix is no rows. + pub fn rows(&self) -> Option>> { + let Value::Num { dims, data } = self else { return None }; + let (r, c) = (dims.first().copied().unwrap_or(0), dims.get(1).copied().unwrap_or(1)); + if r.checked_mul(c) != Some(data.len()) { + return None; + } + if data.is_empty() { + return Some(Vec::new()); + } + Some((0..r).map(|i| (0..c).filter_map(|j| data.get(i + j * r).copied()).collect()).collect()) + } + + /// All numbers of a numeric matrix. + pub fn numbers(&self) -> Option<&[f64]> { + match self { + Value::Num { data, .. } => Some(data), + _ => None, + } + } +} + +fn bad(what: &str) -> String { + format!("malformed MAT-file: {what}") +} + +struct Reader<'a> { + b: &'a [u8], + at: usize, +} + +impl<'a> Reader<'a> { + fn bytes(&self, len: usize) -> Option<&'a [u8]> { + self.b.get(self.at..self.at.checked_add(len)?) + } + + fn u32(&mut self) -> Result { + let word = self.bytes(4).and_then(|s| <[u8; 4]>::try_from(s).ok()).ok_or_else(|| bad("truncated tag"))?; + self.at += 4; + Ok(u32::from_le_bytes(word)) + } + + fn done(&self) -> bool { + self.at >= self.b.len() + } + + /// The next data element: its type and bytes. + fn element(&mut self) -> Result<(u32, &'a [u8]), String> { + let first = self.u32()?; + if first >> 16 != 0 { + // small element: type and size in one word, data in the next four bytes + let (ty, size) = (first & 0xffff, (first >> 16) as usize); + let data = self.bytes(size.min(4)).ok_or_else(|| bad("truncated small element"))?; + self.at += 4; + return Ok((ty, data)); + } + let size = self.u32()? as usize; + let data = self.bytes(size).ok_or_else(|| bad("an element runs past the end of the file"))?; + let end = self.at + size; + // compressed elements are not padded; everything else is padded to 8 bytes + self.at = if first == 15 { end } else { end.checked_next_multiple_of(8).ok_or_else(|| bad("size overflow"))? }; + Ok((first, data)) + } +} + +/// The values of a numeric data element of MAT type `ty`. +fn numbers(ty: u32, d: &[u8]) -> Result, String> { + fn le(d: &[u8]) -> impl Iterator + '_ { + d.as_chunks::().0.iter().copied() + } + Ok(match ty { + 1 => d.iter().map(|b| f64::from(*b as i8)).collect(), + 2 => d.iter().map(|b| f64::from(*b)).collect(), + 3 => le::<2>(d).map(|c| f64::from(i16::from_le_bytes(c))).collect(), + 4 => le::<2>(d).map(|c| f64::from(u16::from_le_bytes(c))).collect(), + 5 => le::<4>(d).map(|c| f64::from(i32::from_le_bytes(c))).collect(), + 6 => le::<4>(d).map(|c| f64::from(u32::from_le_bytes(c))).collect(), + 7 => le::<4>(d).map(|c| f64::from(f32::from_le_bytes(c))).collect(), + 9 => le::<8>(d).map(f64::from_le_bytes).collect(), + 12 => le::<8>(d).map(|c| i64::from_le_bytes(c) as f64).collect(), + 13 => le::<8>(d).map(|c| u64::from_le_bytes(c) as f64).collect(), + _ => return Err(bad(&format!("numeric data of type {ty}"))), + }) +} + +/// A miMATRIX element's body. +fn matrix(body: &[u8], depth: usize) -> Result<(String, Value), String> { + if depth > MAX_DEPTH { + return Err(bad("nested too deeply")); + } + if body.is_empty() { + return Ok((String::new(), Value::Num { dims: vec![0, 0], data: Vec::new() })); + } + let mut r = Reader { b: body, at: 0 }; + let (_, flags) = r.element()?; + let class = flags.first().copied().ok_or_else(|| bad("no array flags"))?; + let complex = flags.get(1).is_some_and(|f| f & 0x08 != 0); + let (dty, dims) = r.element()?; + let dims: Vec = numbers(dty, dims)?.into_iter().map(|d| if d.is_finite() && d >= 0.0 { d as usize } else { 0 }).collect(); + if dims.iter().any(|d| *d > MAX_DIM) { + return Err(bad("a dimension larger than any real variable")); + } + let count = dims.iter().try_fold(1usize, |n, d| n.checked_mul(*d)).ok_or_else(|| bad("dimensions overflow"))?; + let (_, name) = r.element()?; + let name = String::from_utf8_lossy(name).into_owned(); + let value = match class { + 1 => { + // every item takes at least a tag (8 bytes), which bounds a hostile count by the data + if count > body.len() / 8 { + return Err(bad("a cell array larger than its data")); + } + let mut items = Vec::with_capacity(count); + for _ in 0..count { + let (ty, b) = r.element()?; + if ty != 14 { + return Err(bad("a cell that is not a matrix")); + } + items.push(matrix(b, depth + 1)?.1); + } + Value::Cell(items) + } + 4 => { + let (ty, d) = r.element()?; + let text = match ty { + 16 => String::from_utf8_lossy(d).into_owned(), + _ => numbers(ty, d)?.into_iter().map(|c| char::from_u32(c as u32).unwrap_or('\u{fffd}')).collect(), + }; + Value::Char(text) + } + 6..=15 => { + let (ty, d) = r.element()?; + let data = numbers(ty, d)?; + if data.len() != count { + return Err(bad("a matrix whose data does not match its dimensions")); + } + if complex && !r.done() { + r.element()?; + } + Value::Num { dims, data } + } + _ => Value::Other, + }; + Ok((name, value)) +} + +/// The variables in a MAT-file's bytes, by name. +pub fn parse(bytes: &[u8]) -> Result, String> { + if bytes.len() < 128 { + return Err(bad("shorter than its header")); + } + if bytes.get(126..128) != Some(b"IM") { + return Err(bad("not a little-endian level-5 MAT-file (v7.3 HDF5 files are not supported)")); + } + let mut r = Reader { b: bytes, at: 128 }; + let mut out = Vec::new(); + while !r.done() { + let (ty, body) = r.element()?; + match ty { + 14 => out.push(matrix(body, 0)?), + 15 => { + let mut inflated = Vec::new(); + flate2::read::ZlibDecoder::new(body) + .take(MAX_INFLATED) + .read_to_end(&mut inflated) + .map_err(|e| bad(&format!("compressed variable: {e}")))?; + let (ity, ibody) = Reader { b: &inflated, at: 0 }.element()?; + if ity == 14 { + out.push(matrix(ibody, 0)?); + } + } + _ => {} + } + } + Ok(out) +} + +/// The variable `name` in `vars`. +pub fn var<'a>(vars: &'a [(String, Value)], name: &str) -> Result<&'a Value, String> { + vars.iter().find(|(n, _)| n == name).map(|(_, v)| v).ok_or_else(|| format!("the MAT-file has no variable `{name}`")) +} + +/// Builders for synthetic MAT-files, shared with the ground-truth tests. +#[cfg(test)] +pub(crate) mod tests { + use super::*; + + fn element(ty: u32, data: &[u8]) -> Vec { + let mut out = ty.to_le_bytes().to_vec(); + out.extend_from_slice(&(data.len() as u32).to_le_bytes()); + out.extend_from_slice(data); + out.resize(out.len().div_ceil(8) * 8, 0); + out + } + + fn header(class: u8, dims: &[i32], name: &str) -> Vec { + let mut out = element(6, &[class, 0, 0, 0, 0, 0, 0, 0]); + out.extend(element(5, &dims.iter().flat_map(|d| d.to_le_bytes()).collect::>())); + out.extend(element(1, name.as_bytes())); + out + } + + /// A double matrix as a miMATRIX element (column-major `data`). + pub(crate) fn double(name: &str, rows: i32, cols: i32, data: &[f64]) -> Vec { + let mut body = header(6, &[rows, cols], name); + body.extend(element(9, &data.iter().flat_map(|d| d.to_le_bytes()).collect::>())); + element(14, &body) + } + + pub(crate) fn chars(name: &str, text: &str) -> Vec { + let mut body = header(4, &[1, text.len() as i32], name); + body.extend(element(16, text.as_bytes())); + element(14, &body) + } + + pub(crate) fn cell(name: &str, items: &[Vec]) -> Vec { + let mut body = header(1, &[items.len() as i32, 1], name); + for item in items { + body.extend_from_slice(item); + } + element(14, &body) + } + + /// A MAT-file: the 128-byte header, then `variables`. + pub(crate) fn file(variables: &[Vec]) -> Vec { + let mut out = vec![0u8; 128]; + out[126] = b'I'; + out[127] = b'M'; + for v in variables { + out.extend_from_slice(v); + } + out + } + + #[test] + fn reads_numbers_text_and_nested_cells() { + let bytes = file(&[double("m", 2, 2, &[1.0, 2.0, 3.0, 4.0]), cell("c", &[chars("", "ab"), cell("", &[double("", 1, 1, &[7.0])])])]); + let vars = parse(&bytes).unwrap(); + // column-major [1 3; 2 4] has the rows [1, 3] and [2, 4] + assert_eq!(var(&vars, "m").unwrap().rows(), Some(vec![vec![1.0, 3.0], vec![2.0, 4.0]])); + let c = var(&vars, "c").unwrap().cells().unwrap(); + assert_eq!(c[0].text(), Some("ab")); + assert_eq!(c[1].cells().unwrap()[0].numbers(), Some(&[7.0][..])); + assert!(var(&vars, "absent").is_err()); + } + + #[test] + fn a_matrix_without_data_has_no_rows_whatever_its_dimensions_claim() { + // dims [N, 0] hold no data; building N empty rows from a hostile N would exhaust memory + let empty = Value::Num { dims: vec![1 << 20, 0], data: Vec::new() }; + assert_eq!(empty.rows(), Some(Vec::new())); + // a dimension no real variable has is refused at parse time + let hostile = file(&[double("m", i32::MAX, 0, &[])]); + assert!(parse(&hostile).is_err()); + // an ordinary empty matrix still parses and has no rows + let vars = parse(&file(&[double("m", 0, 0, &[])])).unwrap(); + assert_eq!(var(&vars, "m").unwrap().rows(), Some(Vec::new())); + } + + #[test] + fn malformed_files_are_errors() { + assert!(parse(b"too short").is_err()); + let mut wrong_endian = file(&[]); + wrong_endian[126] = b'M'; + wrong_endian[127] = b'I'; + assert!(parse(&wrong_endian).is_err()); + let good = file(&[double("m", 1, 2, &[1.0, 2.0])]); + for cut in [130, 140, good.len() - 1] { + assert!(parse(&good[..cut]).is_err(), "cut at {cut}"); + } + // a cell array that claims far more items than it has bytes + let mut hostile = header(1, &[i32::MAX, 1], "c"); + hostile.extend(element(14, &[])); + assert!(parse(&file(&[element(14, &hostile)])).is_err()); + } +} diff --git a/xtask/src/ai_bench/metrics.rs b/xtask/src/ai_bench/metrics.rs new file mode 100644 --- /dev/null +++ b/xtask/src/ai_bench/metrics.rs @@ -0,0 +1,192 @@ +//! WIDER FACE average precision, implemented from the published description of the evaluation (Yang et al., "WIDER +//! FACE", CVPR 2016, and the evaluation toolkit's documented procedure; no evaluation code was copied): detection +//! scores are normalised over the whole set, a detection hits a face when their intersection over union is at least 0.5 +//! (with the toolkit's inclusive-pixel `+1` convention), faces outside the setting's keep list are neither counted nor +//! punished, precision and recall are taken at 1,000 score thresholds and the area under the curve is the VOC +//! all-point average precision. + +/// How many score thresholds the precision/recall curve has. +const THRESHOLDS: usize = 1000; +/// Overlap at which a detection hits a face. +const MIN_IOU: f64 = 0.5; + +/// One image: ground-truth boxes `[x, y, w, h]`, the 1-based indices of the faces this setting counts, and detections +/// `[x, y, w, h, score]`, all in pixels. +pub struct Image<'a> { + pub gt: &'a [[f64; 4]], + pub keep: &'a [usize], + pub preds: &'a [[f64; 5]], +} + +/// IoU of two `[x1, y1, x2, y2]` boxes, counting pixels inclusively (a box from 0 to 9 is ten pixels wide). +fn iou(a: &[f64; 4], b: &[f64; 4]) -> f64 { + let iw = a[2].min(b[2]) - a[0].max(b[0]) + 1.0; + let ih = a[3].min(b[3]) - a[1].max(b[1]) + 1.0; + if iw <= 0.0 || ih <= 0.0 { + return 0.0; + } + let area = |r: &[f64; 4]| (r[2] - r[0] + 1.0) * (r[3] - r[1] + 1.0); + let inter = iw * ih; + let union = area(a) + area(b) - inter; + if union > 0.0 { inter / union } else { 0.0 } +} + +/// VOC all-point average precision of a precision/recall curve (recall ascending). +fn voc_ap(recall: &[f64], precision: &[f64]) -> f64 { + let mut mrec = vec![0.0]; + mrec.extend_from_slice(recall); + mrec.push(1.0); + let mut mpre = vec![0.0]; + mpre.extend_from_slice(precision); + mpre.push(0.0); + for i in (1..mpre.len()).rev() { + mpre[i - 1] = mpre[i - 1].max(mpre[i]); + } + mrec.windows(2).zip(mpre.iter().skip(1)).filter(|(r, _)| r[1] != r[0]).map(|(r, p)| (r[1] - r[0]) * p).sum() +} + +/// The WIDER FACE average precision (0 to 1) of one setting (easy, medium or hard: the `keep` lists) over `images`. +pub fn wider_ap(images: &[Image]) -> f64 { + let (mut lo, mut hi) = (f64::INFINITY, f64::NEG_INFINITY); + for p in images.iter().flat_map(|i| i.preds).filter(|p| p[4].is_finite()) { + lo = lo.min(p[4]); + hi = hi.max(p[4]); + } + let span = if hi > lo { hi - lo } else { 1.0 }; + // per threshold: detections kept that are not on an uncounted face, and counted faces found + let mut curve = vec![(0.0f64, 0.0f64); THRESHOLDS]; + let mut faces = 0usize; + for img in images { + faces += img.keep.len(); + if img.gt.is_empty() || img.preds.is_empty() { + continue; + } + let gt: Vec<[f64; 4]> = img.gt.iter().map(|g| [g[0], g[1], g[0] + g[2], g[1] + g[3]]).collect(); + let mut preds: Vec<([f64; 4], f64)> = img + .preds + .iter() + .filter(|p| p[4].is_finite()) + .map(|p| ([p[0], p[1], p[0] + p[2], p[1] + p[3]], ((p[4] - lo) / span).clamp(0.0, 1.0))) + .collect(); + preds.sort_by(|a, b| b.1.total_cmp(&a.1)); + // per face: whether this setting counts it, and 0 unseen / 1 found / -1 matched by a detection that is then ignored + let mut state: Vec<(bool, i8)> = vec![(false, 0); gt.len()]; + for k in img.keep { + if let Some(s) = k.checked_sub(1).and_then(|i| state.get_mut(i)) { + s.0 = true; + } + } + // after each detection (best first): the proposals so far and the counted faces found so far + let (mut proposals, mut recalled) = (0usize, 0usize); + let mut after: Vec<(usize, usize)> = Vec::with_capacity(preds.len()); + for (b, _) in &preds { + let mut best = (0.0f64, None); + for (j, g) in gt.iter().enumerate() { + let o = iou(b, g); + if o > best.0 { + best = (o, Some(j)); + } + } + let mut ignored = false; + if let (o, Some(j)) = best + && o >= MIN_IOU + && let Some((counted, found)) = state.get_mut(j) + { + if !*counted { + if *found == 1 { + recalled = recalled.saturating_sub(1); + } + *found = -1; + ignored = true; + } else if *found == 0 { + *found = 1; + recalled += 1; + } + } + if !ignored { + proposals += 1; + } + after.push((proposals, recalled)); + } + for (t, point) in curve.iter_mut().enumerate() { + let threshold = 1.0 - (t + 1) as f64 / THRESHOLDS as f64; + if let Some((p, r)) = preds.iter().rposition(|p| p.1 >= threshold).and_then(|last| after.get(last)) { + point.0 += *p as f64; + point.1 += *r as f64; + } + } + } + if faces == 0 { + return 0.0; + } + let precision: Vec = curve.iter().map(|c| if c.0 > 0.0 { c.1 / c.0 } else { 0.0 }).collect(); + let recall: Vec = curve.iter().map(|c| c.1 / faces as f64).collect(); + voc_ap(&recall, &precision) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn close(a: f64, b: f64) { + assert!((a - b).abs() < 1e-9, "{a} != {b}"); + } + + #[test] + fn iou_counts_pixels_inclusively() { + let a = [0.0, 0.0, 9.0, 9.0]; // 10 x 10 pixels + close(iou(&a, &a), 1.0); + close(iou(&a, &[20.0, 20.0, 29.0, 29.0]), 0.0); + // 5 x 10 shared of 10 x 10 and 10 x 10: 50 / 150 + close(iou(&a, &[5.0, 0.0, 14.0, 9.0]), 1.0 / 3.0); + // touching at one pixel column still shares a pixel under the inclusive convention + close(iou(&a, &[9.0, 0.0, 18.0, 9.0]), 10.0 / 190.0); + // a degenerate pair has no overlap rather than a division by zero + close(iou(&[5.0, 5.0, 4.0, 4.0], &[5.0, 5.0, 4.0, 4.0]), 0.0); + } + + #[test] + fn a_detection_hits_from_half_overlap() { + let gt = [[0.0, 0.0, 10.0, 10.0]]; // pixels 0..=10 under the +1 convention: 11 x 11 + // 11 x 11 box against an 11 x 6 detection sharing its top rows: 66 / 121 > 0.5 hits + let hit = [[0.0, 0.0, 10.0, 5.0, 1.0]]; + close(wider_ap(&[Image { gt: >, keep: &[1], preds: &hit }]), 1.0); + // 11 x 5: 55 / 121 < 0.5 misses + let miss = [[0.0, 0.0, 10.0, 4.0, 1.0]]; + close(wider_ap(&[Image { gt: >, keep: &[1], preds: &miss }]), 0.0); + } + + /// Two faces, three detections: a hit (score .75), a false positive (.5) and a hit (.25). Scores normalise to 1, .5 + /// and 0. At thresholds .999 to .501 only the first detection counts (precision 1, recall 1/2, 499 times); at .500 to + /// .001 the first two (precision 1/2, recall 1/2, 500 times); at 0 all three (precision 2/3, recall 1). The + /// precision envelope is 1 up to recall 1/2 and 2/3 beyond it: AP = 1/2 * 1 + 1/2 * 2/3 = 5/6. + #[test] + fn average_precision_by_hand() { + let gt = [[0.0, 0.0, 10.0, 10.0], [100.0, 100.0, 10.0, 10.0]]; + let preds = [[0.0, 0.0, 10.0, 10.0, 0.75], [300.0, 300.0, 10.0, 10.0, 0.5], [100.0, 100.0, 10.0, 10.0, 0.25]]; + close(wider_ap(&[Image { gt: >, keep: &[1, 2], preds: &preds }]), 5.0 / 6.0); + // the same detections without the false positive: a perfect curve + let clean = [preds[0], preds[2]]; + close(wider_ap(&[Image { gt: >, keep: &[1, 2], preds: &clean }]), 1.0); + } + + #[test] + fn faces_outside_the_keep_list_are_neither_missed_nor_punished() { + let gt = [[0.0, 0.0, 10.0, 10.0], [100.0, 100.0, 10.0, 10.0]]; + // only the first face counts; a detection on the second must not count as a false positive + let both = [[0.0, 0.0, 10.0, 10.0, 0.9], [100.0, 100.0, 10.0, 10.0, 0.8]]; + close(wider_ap(&[Image { gt: >, keep: &[1], preds: &both }]), 1.0); + // and not finding it costs no recall + close(wider_ap(&[Image { gt: >, keep: &[1], preds: &both[..1] }]), 1.0); + } + + #[test] + fn nothing_to_find_or_nothing_found_scores_zero() { + let gt = [[0.0, 0.0, 10.0, 10.0]]; + close(wider_ap(&[]), 0.0); + close(wider_ap(&[Image { gt: >, keep: &[1], preds: &[] }]), 0.0); + close(wider_ap(&[Image { gt: >, keep: &[], preds: &[[0.0, 0.0, 10.0, 10.0, 0.9]] }]), 0.0); + // a non-finite score is dropped, not propagated + close(wider_ap(&[Image { gt: >, keep: &[1], preds: &[[0.0, 0.0, 10.0, 10.0, f64::NAN]] }]), 0.0); + } +} diff --git a/xtask/src/ai_bench/mod.rs b/xtask/src/ai_bench/mod.rs new file mode 100644 --- /dev/null +++ b/xtask/src/ai_bench/mod.rs @@ -0,0 +1,111 @@ +//! `cargo xtask ai-bench`: measure how well LightCraft's own models do on public benchmarks, so the accuracy figures in the +//! docs can be reproduced. For now one benchmark: face detection (YuNet) on the WIDER FACE validation set. +//! +//! ```text +//! cargo xtask ai-bench fetch --accept-terms [--dir DIR] +//! cargo xtask ai-bench run face-detection --set wider-face-val [--dir DIR] [--model FILE] [--limit N] [--threads N] +//! ``` + +mod fetch; +mod mat; +mod metrics; +mod wider; + +use std::path::{Path, PathBuf}; + +use lightcraft_faces::{known, yunet}; + +/// The only benchmark set so far. +const SET: &str = "wider-face-val"; + +pub const USAGE: &str = "\ + ai-bench fetch --accept-terms [--dir DIR] + download the WIDER FACE validation photos (CC BY-NC-ND 4.0, research use only) and its evaluation + toolkit's ground truth, and the YuNet model, into DIR (default corpus/ai-bench), checking pinned + SHA-256 sums; nothing is committed + ai-bench run face-detection --set wider-face-val [--dir DIR] [--model FILE] [--limit N] [--threads N] + run YuNet on every validation photo and print the WIDER FACE average precision for the easy, + medium and hard settings and the wall time (--limit N: N photos spread evenly, for a quick look; + --model: another YuNet-architecture .onnx, default DIR/models/face_detection_yunet_2023mar.onnx) +"; + +struct Args { + dir: PathBuf, + model: Option, + limit: Option, + threads: usize, + accept_terms: bool, + set: Option, +} + +fn parse(root: &Path, args: &[&str]) -> Result { + let mut a = Args { + dir: root.join("corpus").join("ai-bench"), + model: None, + limit: None, + threads: std::thread::available_parallelism().map_or(4, |n| n.get()), + accept_terms: false, + set: None, + }; + let mut it = args.iter(); + while let Some(&flag) = it.next() { + if flag == "--accept-terms" { + a.accept_terms = true; + continue; + } + let value = it.next().ok_or_else(|| format!("{flag} needs a value"))?; + let number = |what: &str| value.parse::().map_err(|_| format!("{what} must be a whole number, not {value}")); + match flag { + "--dir" => a.dir = PathBuf::from(value), + "--model" => a.model = Some(PathBuf::from(value)), + "--set" => a.set = Some((*value).to_string()), + "--limit" => a.limit = Some(number("--limit")?), + "--threads" => a.threads = number("--threads")?, + _ => return Err(format!("unknown option `{flag}`\n\n{USAGE}")), + } + } + Ok(a) +} + +pub fn run(root: &Path, args: &[&str]) -> Result<(), String> { + match args { + ["fetch", rest @ ..] => { + let a = parse(root, rest)?; + fetch::run(&a.dir, a.accept_terms) + } + ["run", "face-detection", rest @ ..] => { + let a = parse(root, rest)?; + match a.set.as_deref() { + Some(SET) => face_detection(&a), + other => Err(format!("face-detection needs --set {SET} (the only set so far), not {}", other.unwrap_or("nothing"))), + } + } + _ => Err(format!("usage:\n{USAGE}")), + } +} + +fn face_detection(a: &Args) -> Result<(), String> { + let file = known::download(known::YUNET_ID).map(|d| d.file_name).ok_or("no YuNet file name")?; + let model = a.model.clone().unwrap_or_else(|| a.dir.join("models").join(file)); + let bytes = std::fs::read(&model).map_err(|e| format!("{}: {e} (run `cargo xtask ai-bench fetch` or pass --model)", model.display()))?; + let detector = yunet::Detector::new(&bytes).map_err(|e| format!("{}: {e}", model.display()))?; + let pinned = crate::corpus::file_sha256(&model).is_ok_and(|h| h == known::YUNET_SHA256); + let side = detector.input_side(); + println!( + "YuNet {} ({}), input {side} px, {} thread(s)", + model.display(), + if pinned { "the pinned 2023mar model" } else { "NOT the pinned 2023mar model" }, + a.threads + ); + let r = wider::run(&a.dir.join(SET), &detector, a.limit, a.threads)?; + println!( + "WIDER FACE validation, {} photos{}", + r.photos, + if r.failed > 0 { format!(" ({} failed, scored as no detections)", r.failed) } else { String::new() } + ); + for (setting, ap) in wider::SETTINGS.iter().zip(r.ap) { + println!(" AP {setting:<6} {ap:.1}"); + } + println!(" wall time {:.1} s ({:.0} ms per photo)", r.seconds, r.seconds * 1000.0 / r.photos.max(1) as f64); + Ok(()) +} diff --git a/xtask/src/ai_bench/wider.rs b/xtask/src/ai_bench/wider.rs new file mode 100644 --- /dev/null +++ b/xtask/src/ai_bench/wider.rs @@ -0,0 +1,343 @@ +//! WIDER FACE validation: read the ground truth, run a detector on every photo, score the three settings. + +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::time::Instant; + +use lightcraft_faces::yunet::{Detector, Options}; + +use super::mat; +use super::metrics::{self, Image}; + +/// Lowest detection score kept: the protocol wants nearly every detection, the curve then sweeps the threshold. +const MIN_SCORE: f32 = 0.02; +/// Most detections per photo (the densest WIDER crowds hold well over a thousand faces). +const MAX_FACES: usize = 2000; +pub const SETTINGS: [&str; 3] = ["easy", "medium", "hard"]; + +/// The validation ground truth: per photo, its path and face boxes `[x, y, w, h]`; per setting, the 1-based faces it counts. +pub struct GroundTruth { + pub images: Vec, + pub boxes: Vec>, + pub keep: [Vec>; 3], +} + +fn read_mat(path: &Path) -> Result, String> { + let bytes = std::fs::read(path).map_err(|e| format!("{}: {e} (run `cargo xtask ai-bench fetch`)", path.display()))?; + mat::parse(&bytes).map_err(|e| format!("{}: {e}", path.display())) +} + +/// Read the ground truth under `dir`: `eval_tools/ground_truth/*.mat` (the evaluation toolkit's) and the photos in +/// `WIDER_val/images//.jpg`. +pub fn load(dir: &Path) -> Result { + let gt = dir.join("eval_tools").join("ground_truth"); + let main = read_mat(>.join("wider_face_val.mat"))?; + let events = mat::var(&main, "event_list")?.cells().ok_or("event_list is not a cell array")?; + let files = mat::var(&main, "file_list")?.cells().ok_or("file_list is not a cell array")?; + let boxes = mat::var(&main, "face_bbx_list")?.cells().ok_or("face_bbx_list is not a cell array")?; + if files.len() != events.len() || boxes.len() != events.len() { + return Err(format!("event_list has {} events, file_list {} and face_bbx_list {}", events.len(), files.len(), boxes.len())); + } + // per event: its name, photo names and box lists, the photo counts checked before anything is paired up + let mut per_event = Vec::with_capacity(events.len()); + for (e, ((event, files), boxes)) in events.iter().zip(files).zip(boxes).enumerate() { + let event = event.text().ok_or_else(|| format!("event {e}: event_list entry is not text"))?; + let files = files.cells().ok_or_else(|| format!("event {e}: file_list entry is not a cell of cells"))?; + let boxes = boxes.cells().ok_or_else(|| format!("event {e}: face_bbx_list entry is not a cell of cells"))?; + if files.len() != boxes.len() { + return Err(format!("event {e}: file_list has {} photos, face_bbx_list {}", files.len(), boxes.len())); + } + per_event.push((event, files, boxes)); + } + let mut keeps: Vec>> = Vec::new(); + for setting in SETTINGS { + let vars = read_mat(>.join(format!("wider_{setting}_val.mat")))?; + let lists = mat::var(&vars, "gt_list")?.cells().ok_or("gt_list is not a cell array")?; + if lists.len() != per_event.len() { + return Err(format!("the {setting} setting lists {} events, the ground truth {}", lists.len(), per_event.len())); + } + let mut flat = Vec::new(); + for (e, (event, (_, files, _))) in lists.iter().zip(&per_event).enumerate() { + let imgs = event.cells().ok_or_else(|| format!("event {e}: the {setting} setting is not a cell of cells"))?; + if imgs.len() != files.len() { + return Err(format!("event {e}: the {setting} setting lists {} photos, the ground truth {}", imgs.len(), files.len())); + } + for (i, img) in imgs.iter().enumerate() { + flat.push(face_indices(img, setting, e, i)?); + } + } + keeps.push(flat); + } + let root = dir.join("WIDER_val").join("images"); + let mut out_images = Vec::new(); + let mut out_boxes = Vec::new(); + for (e, (event, files, boxes)) in per_event.iter().enumerate() { + for (file, face) in files.iter().zip(*boxes) { + let name = file.text().ok_or_else(|| format!("event {e}: file_list entry is not text"))?; + out_images.push(root.join(event).join(format!("{name}.jpg"))); + let rows = face.rows().ok_or_else(|| format!("event {e}: face_bbx_list entry is not a matrix"))?; + out_boxes.push(rows.iter().filter_map(|r| <[f64; 4]>::try_from(r.get(..4)?).ok()).collect()); + } + } + let [easy, medium, hard]: [Vec>; 3] = keeps.try_into().map_err(|_| "three settings expected")?; + Ok(GroundTruth { images: out_images, boxes: out_boxes, keep: [easy, medium, hard] }) +} + +/// The 1-based face numbers one photo's `gt_list` entry counts; a fractional, non-positive or absurd number is an error. +fn face_indices(entry: &mat::Value, setting: &str, event: usize, photo: usize) -> Result, String> { + entry + .numbers() + .unwrap_or(&[]) + .iter() + .map(|&v| { + if v.is_finite() && v.fract() == 0.0 && (1.0..=1e9).contains(&v) { + Ok(v as usize) + } else { + Err(format!("event {event}, photo {photo}: the {setting} setting has the face index {v}, expected a whole number from 1")) + } + }) + .collect() +} + +/// A photo as 8-bit RGB: `(pixels, width, height)`. +fn load_rgb(path: &Path) -> Result<(Vec, usize, usize), String> { + use zune_core::bytestream::ZCursor; + use zune_core::colorspace::ColorSpace; + use zune_core::options::DecoderOptions; + let bytes = std::fs::read(path).map_err(|e| format!("{}: {e}", path.display()))?; + let options = DecoderOptions::default().jpeg_set_out_colorspace(ColorSpace::RGB); + let mut decoder = zune_jpeg::JpegDecoder::new_with_options(ZCursor::new(&bytes), options); + let pixels = decoder.decode().map_err(|e| format!("{}: {e:?}", path.display()))?; + let info = decoder.info().ok_or_else(|| format!("{}: no image information", path.display()))?; + let (w, h) = (usize::from(info.width), usize::from(info.height)); + match pixels.len().checked_div(w.saturating_mul(h)) { + Some(3) => Ok((pixels, w, h)), + Some(1) => Ok((pixels.iter().flat_map(|g| [*g; 3]).collect(), w, h)), + _ => Err(format!("{}: unexpected pixel layout", path.display())), + } +} + +/// The detections of one photo as `[x, y, w, h, score]` in pixels. +fn detect(detector: &Detector, path: &Path) -> Result, String> { + let (rgb, w, h) = load_rgb(path)?; + // the benchmark's own score, overlap and count limits; any other option keeps the detector's default + #[allow(clippy::needless_update, reason = "options the detector gains later keep their defaults here")] + let options = Options { score: MIN_SCORE, nms_iou: nms_iou(), max_faces: MAX_FACES, ..Options::default() }; + let faces = detector.detect(&rgb, w, h, &options).map_err(|e| format!("{}: {e}", path.display()))?; + let (w, h) = (w as f64, h as f64); + Ok(faces + .iter() + .map(|f| [f64::from(f.x0) * w, f64::from(f.y0) * h, f64::from(f.x1 - f.x0) * w, f64::from(f.y1 - f.y0) * h, f64::from(f.score)]) + .collect()) +} + +/// The suppression overlap LightCraft ships for YuNet. +fn nms_iou() -> f32 { + lightcraft_faces::known::yunet().thresholds.nms_iou.unwrap_or(0.4) +} + +pub struct Run { + pub photos: usize, + /// Photos that could not be read or searched (scored as having no detections). + pub failed: usize, + pub seconds: f64, + /// Average precision in percent for easy, medium and hard. + pub ap: [f64; 3], +} + +/// Detect on every photo (or `limit` of them, spread evenly) on `threads` threads and score the three settings. +pub fn run(dir: &Path, detector: &Detector, limit: Option, threads: usize) -> Result { + let truth = load(dir)?; + let n = truth.images.len(); + let chosen: Vec = match limit { + Some(k) if k > 0 && k < n => (0..k).map(|i| i * n / k).collect(), + _ => (0..n).collect(), + }; + let next = AtomicUsize::new(0); + let finished = AtomicUsize::new(0); + let started = Instant::now(); + let step = (chosen.len() / 20).max(1); + let joined: Vec, String>)>, String>> = std::thread::scope(|scope| { + let workers: Vec<_> = (0..threads.clamp(1, 64)) + .map(|_| { + scope.spawn(|| { + let mut done = Vec::new(); + loop { + let at = next.fetch_add(1, Ordering::Relaxed); + let Some(path) = chosen.get(at).and_then(|&i| truth.images.get(i)) else { break }; + done.push((at, detect(detector, path))); + let count = finished.fetch_add(1, Ordering::Relaxed) + 1; + if count.is_multiple_of(step) { + eprintln!(" {count}/{} ({:.1}/s)", chosen.len(), count as f64 / started.elapsed().as_secs_f64().max(1e-9)); + } + } + done + }) + }) + .collect(); + workers.into_iter().map(|w| w.join().map_err(|_| "a detection worker stopped unexpectedly".to_string())).collect() + }); + let seconds = started.elapsed().as_secs_f64(); + let mut preds: Vec> = vec![Vec::new(); chosen.len()]; + let mut failed = 0; + for worker in joined { + for (at, result) in worker? { + match (result, preds.get_mut(at)) { + (Ok(found), Some(slot)) => *slot = found, + (Err(e), _) => { + if failed < 5 { + eprintln!(" warning: {e}"); + } + failed += 1; + } + (Ok(_), None) => {} + } + } + } + let mut ap = [0.0; 3]; + for (s, score) in ap.iter_mut().enumerate() { + let images: Vec = chosen + .iter() + .zip(&preds) + .filter_map(|(&i, p)| Some(Image { gt: truth.boxes.get(i)?, keep: truth.keep.get(s)?.get(i)?, preds: p })) + .collect(); + *score = metrics::wider_ap(&images) * 100.0; + } + Ok(Run { photos: chosen.len(), failed, seconds, ap }) +} + +#[cfg(test)] +mod tests { + use super::super::mat::tests::{cell, chars, double, file}; + use super::*; + + /// A scratch directory that is removed afterwards. + struct Scratch(PathBuf); + + impl Scratch { + fn new(name: &str) -> Scratch { + let dir = std::env::temp_dir().join(format!("ai-bench-{name}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + assert!(std::fs::create_dir_all(dir.join("eval_tools").join("ground_truth")).is_ok()); + Scratch(dir) + } + + fn gt(&self, name: &str, bytes: &[u8]) { + assert!(std::fs::write(self.0.join("eval_tools").join("ground_truth").join(name), bytes).is_ok()); + } + } + + impl Drop for Scratch { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } + } + + /// One event with one photo "a" holding two faces; the settings count the faces in `keeps`. + fn write_truth(s: &Scratch, keeps: [&[f64]; 3]) { + // the boxes [1 2 3 4] and [10 20 30 40], column-major + let boxes = double("", 2, 4, &[1.0, 10.0, 2.0, 20.0, 3.0, 30.0, 4.0, 40.0]); + let main = file(&[ + cell("event_list", &[chars("", "0--Parade")]), + cell("file_list", &[cell("", &[chars("", "a")])]), + cell("face_bbx_list", &[cell("", &[boxes])]), + ]); + s.gt("wider_face_val.mat", &main); + for (setting, keep) in SETTINGS.iter().zip(keeps) { + let list = file(&[cell("gt_list", &[cell("", &[double("", keep.len() as i32, 1, keep)])])]); + s.gt(&format!("wider_{setting}_val.mat"), &list); + } + } + + #[test] + fn ground_truth_reads_boxes_photos_and_the_faces_each_setting_counts() { + let s = Scratch::new("reads"); + write_truth(&s, [&[1.0], &[1.0, 2.0], &[2.0]]); + let truth = load(&s.0).unwrap(); + assert_eq!(truth.images, vec![s.0.join("WIDER_val").join("images").join("0--Parade").join("a.jpg")]); + assert_eq!(truth.boxes, vec![vec![[1.0, 2.0, 3.0, 4.0], [10.0, 20.0, 30.0, 40.0]]]); + assert_eq!(truth.keep, [vec![vec![1]], vec![vec![1, 2]], vec![vec![2]]]); + } + + #[test] + fn a_missing_annotation_file_is_an_error_that_says_how_to_fetch_it() { + let s = Scratch::new("missing"); + let e = load(&s.0).err().unwrap(); + assert!(e.contains("wider_face_val.mat") && e.contains("ai-bench fetch"), "{e}"); + // the main file alone is not enough: the settings' files are named too + write_truth(&s, [&[1.0], &[1.0], &[1.0]]); + assert!(std::fs::remove_file(s.0.join("eval_tools").join("ground_truth").join("wider_hard_val.mat")).is_ok()); + assert!(load(&s.0).err().unwrap().contains("wider_hard_val.mat")); + } + + #[test] + fn malformed_annotation_files_are_errors_not_panics() { + let s = Scratch::new("malformed"); + write_truth(&s, [&[1.0], &[1.0], &[1.0]]); + // garbage, a truncated file and a valid file without the variable the loader needs + s.gt("wider_face_val.mat", b"this is not a MAT-file"); + assert!(load(&s.0).is_err()); + write_truth(&s, [&[1.0], &[1.0], &[1.0]]); + let good = std::fs::read(s.0.join("eval_tools").join("ground_truth").join("wider_easy_val.mat")).unwrap(); + s.gt("wider_easy_val.mat", &good[..good.len() - 5]); + assert!(load(&s.0).is_err()); + s.gt("wider_easy_val.mat", &file(&[double("other", 1, 1, &[1.0])])); + assert!(load(&s.0).err().unwrap().contains("gt_list")); + } + + #[test] + fn fractional_zero_and_negative_face_indices_are_errors_not_truncated() { + for bad in [1.5, 0.0, -1.0, f64::NAN] { + let s = Scratch::new("index"); + write_truth(&s, [&[1.0], &[1.0], &[1.0]]); + let list = file(&[cell("gt_list", &[cell("", &[double("", 2, 1, &[1.0, bad])])])]); + s.gt("wider_medium_val.mat", &list); + let e = load(&s.0).err().unwrap_or_default(); + assert!(e.contains("medium") && e.contains("face index"), "{bad}: {e:?}"); + } + } + + #[test] + fn event_and_photo_counts_are_checked_and_errors_name_the_event() { + // top level: two events, one file list + let s = Scratch::new("top"); + write_truth(&s, [&[1.0], &[1.0], &[1.0]]); + let boxes = double("", 1, 4, &[1.0, 2.0, 3.0, 4.0]); + let main = file(&[ + cell("event_list", &[chars("", "0--A"), chars("", "1--B")]), + cell("file_list", &[cell("", &[chars("", "a")])]), + cell("face_bbx_list", &[cell("", std::slice::from_ref(&boxes)), cell("", std::slice::from_ref(&boxes))]), + ]); + s.gt("wider_face_val.mat", &main); + let e = load(&s.0).err().unwrap_or_default(); + assert!(e.contains("event_list") && e.contains("file_list"), "{e:?}"); + // event 1 has two files but one box list + let main = file(&[ + cell("event_list", &[chars("", "0--A"), chars("", "1--B")]), + cell("file_list", &[cell("", &[chars("", "a")]), cell("", &[chars("", "b"), chars("", "c")])]), + cell("face_bbx_list", &[cell("", std::slice::from_ref(&boxes)), cell("", &[boxes])]), + ]); + s.gt("wider_face_val.mat", &main); + let e = load(&s.0).err().unwrap_or_default(); + assert!(e.contains("event 1"), "{e:?}"); + } + + #[test] + fn a_setting_with_the_wrong_event_count_is_an_error() { + let s = Scratch::new("events"); + write_truth(&s, [&[1.0], &[1.0], &[1.0]]); + let item = cell("", &[double("", 1, 1, &[1.0])]); + s.gt("wider_hard_val.mat", &file(&[cell("gt_list", &[item.clone(), item])])); + let e = load(&s.0).err().unwrap_or_default(); + assert!(e.contains("hard") && e.contains("2 events"), "{e:?}"); + } + + #[test] + fn settings_that_disagree_with_the_ground_truth_on_the_photo_count_are_an_error() { + let s = Scratch::new("count"); + write_truth(&s, [&[1.0], &[1.0], &[1.0]]); + let two = file(&[cell("gt_list", &[cell("", &[double("", 1, 1, &[1.0]), double("", 1, 1, &[1.0])])])]); + s.gt("wider_medium_val.mat", &two); + assert!(load(&s.0).err().unwrap().contains("event 0")); + } +} diff --git a/xtask/src/corpus.rs b/xtask/src/corpus.rs --- a/xtask/src/corpus.rs +++ b/xtask/src/corpus.rs @@ -46,7 +46,8 @@ Tests that use a corpus skip cleanly when it is absent. )) } -fn file_sha256(path: &Path) -> Result { +/// SHA-256 of a file as lowercase hex; `ai_bench` reuses it to verify pinned downloads. +pub(crate) fn file_sha256(path: &Path) -> Result { use sha2::{Digest, Sha256}; let mut file = std::fs::File::open(path).map_err(|e| format!("open {}: {e}", path.display()))?; let mut hash = Sha256::new(); diff --git a/xtask/src/main.rs b/xtask/src/main.rs --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -3,6 +3,7 @@ //! Pure Rust (std + serde_json; flate2/brotli for the web bundle). External tools (`cargo`, `curl`, `tar`) are //! invoked through `std::process::Command`. +mod ai_bench; mod assets; mod bench; mod ci_lock; @@ -33,6 +34,8 @@ usage: cargo xtask commands: assets every image/icon/font/media file is attributed in assets/ATTRIBUTION.md; no Adobe assets + ai-bench fetch|run + reproduce the face-detection accuracy on WIDER FACE validation; see docs/faces.md bench [FILE] [--strict] [--threshold PCT] run the render benchmark, append to target/bench/history.jsonl, compare CPU time with the previous run (default input: corpus/raw/arw-sony-a7m3-compressed.arw) @@ -75,6 +78,7 @@ fn main() -> ExitCode { Some("ico") => ico::run(&rest), Some("version") => version::run(&root(), &rest), Some("layers") => cmd_layers(), + Some("ai-bench") => ai_bench::run(&root(), &rest), Some("assets") => assets::run(&root()), Some("bench") => bench::run(&root(), &rest), Some("parity") => parity::run(&root(), rest.contains(&"--write")),