diff --git a/.cargo/config.toml b/.cargo/config.toml --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -1,2 +1,12 @@ [alias] xtask = "run --package xtask --" + +# Plain `cargo test` runs at most 4 tests at once in each test binary, as `cargo xtask ci` does: +# libtest's default of one per CPU ran the UI tests out of memory on a 16-core machine (#3059). +# A RUST_TEST_THREADS already in the environment wins (`force = false`). +# `cargo xtask` runs under this config too: the marker beside it holds the value set here, so xtask +# can tell this default from a caller's setting and size `ci`'s test threads by available RAM. Keep +# the two values equal (`xtask` tests it). +[env] +RUST_TEST_THREADS = { value = "4", force = false } +LIGHTCRAFT_TEST_THREADS_DEFAULT = { value = "4", force = true } diff --git a/.config/nextest.toml b/.config/nextest.toml new file mode 100644 --- /dev/null +++ b/.config/nextest.toml @@ -0,0 +1,5 @@ +# Plain `cargo nextest run` runs at most 4 tests at once, as plain `cargo test` does +# (`.cargo/config.toml`, #3059). NEXTEST_TEST_THREADS or `--test-threads` wins; `cargo xtask ci` +# and `cargo xtask test` set it from available RAM. +[profile.default] +test-threads = 4 diff --git a/README.md b/README.md --- a/README.md +++ b/README.md @@ -406,6 +406,7 @@ cargo xtask web --serve # the same app in the br cargo run --release -p lightcraft-cli -- render photo.jpg -o out.jpg --set light.exposure=0.5 cargo xtask ci # fmt, clippy, HEIF, tests, parity, layers, assets, WASM cargo xtask ci --quick # only what your branch's changes can break (minutes) +cargo xtask test -p lightcraft-raw --keep-going # tests only, with ci's caps; every failing test binary ``` `cargo xtask ci --quick` compares with the merge base of `origin/main` (`--base REF` for another) @@ -422,10 +423,25 @@ available when it starts (at most one per CPU; 4 if memory can't be read), so an builds with fewer. Test threads follow, capped at 4. Full-debuginfo linkers, one per CPU, otherwise exhaust RAM before any test runs. GPU coverage is unchanged. Explicit `CARGO_PROFILE_DEV_DEBUG`, `CARGO_BUILD_JOBS`, `RUST_TEST_THREADS` and -`NEXTEST_TEST_THREADS` settings override these defaults. The tests run under +`NEXTEST_TEST_THREADS` settings override these defaults. Outside `ci`, `.cargo/config.toml` limits +plain `cargo test` to 4 tests at once in each test binary (`RUST_TEST_THREADS=4` unless the +environment sets it; libtest's own default is one per CPU), and `.config/nextest.toml` does the same +for `cargo nextest run`. `cargo xtask` inherits that default but doesn't count it as an explicit +setting (a marker beside it, `LIGHTCRAFT_TEST_THREADS_DEFAULT`, tells it apart), so `ci` still +lowers the threads on a machine short of RAM. An explicit `RUST_TEST_THREADS` other than 4 wins +as-is; an explicit 4 can't be told apart from the default, so `ci` and `cargo xtask test` may lower +it to their RAM-aware value (never raise it). The tests run under `cargo test`; `LIGHTCRAFT_NEXTEST=1` runs them under [cargo-nextest](https://nexte.st) instead (one process per test, slower here because each regenerates the demo scenes). Ordinary development commands keep their own settings. +To run tests without the rest of `ci`, use `cargo xtask test` rather than `cargo test --workspace`, +which gets none of `ci`'s RAM-aware caps (only the 4-thread default above) and takes no CI slot. +It has `ci`'s caps and waits for its slot, builds every test target +once (the same build as `ci`'s, so neither rebuilds the other) and runs each test binary, or those +of the `-p PKG` packages, with whatever follows `--` (`cargo xtask test -p lightcraft-ui-egui -- +i18n::`). It stops at the first failing binary, as `ci` does; `--keep-going` runs them all and +names each that failed. `--doc` adds the doctests; `LIGHTCRAFT_NEXTEST=1` applies here too. + **Release builds on memory-constrained machines:** [#1196](https://github.com/storytold/lightcraft/issues/1196) reports Arch/AUR release-link OOM kills on a 16 GB machine when the desktop and CLI build concurrently with full debug information, thin LTO and one codegen unit. The CI defaults above do not apply to diff --git a/docs/architecture.md b/docs/architecture.md --- a/docs/architecture.md +++ b/docs/architecture.md @@ -1,6 +1,6 @@ # Architecture -> **Last reviewed:** 2026-10-10 · **Last updated:** 2026-10-23 · **Change:** minor (`cargo xtask ci`: one integration-test binary per crate, doctests apart, one WASM check) · **Target:** Adobe Lightroom Classic 15.5 / Lightroom 9.6 +> **Last reviewed:** 2026-10-10 · **Last updated:** 2026-10-26 · **Change:** minor (plain `cargo test` runs at most 4 tests at once; `ci` still sizes its threads by RAM) · **Target:** Adobe Lightroom Classic 15.5 / Lightroom 9.6 How LightCraft is built today. The longer design document (`plan/architecture.md`) is local-only (`plan/` is gitignored); this page is the committed, current summary. Sizes are Rust lines counted @@ -73,6 +73,23 @@ module per file (`cargo test -p --test it ::`); a test that chan whole process (a global allocator, closing the GPU) keeps a binary of its own in `tests/`. The WASM build is one `cargo check --target wasm32-unknown-unknown --keep-going` over the wasm-safe crates. +`ci`'s resource caps (build jobs by available RAM, at most 4 test threads, line-table debuginfo) and +its one-run-per-machine slot (`xtask/src/ci_lock.rs`) also cover `cargo xtask test [-p PKG]… +[--keep-going] [--doc] [-- ARGS]` (`xtask/src/test.rs`), the way to run the tests alone: subcommands +opt into the caps with `use_ci_caps`. It builds every workspace test target once, as `ci` does, and +runs each test binary, or the `-p` packages' binaries, itself (`test::run_binary`, which `ci --quick` +uses too); `--keep-going` runs every binary and names each that failed instead of stopping at the +first. + +Outside `ci`, `.cargo/config.toml`'s `[env]` limits plain `cargo test` to 4 tests at once per test +binary (`RUST_TEST_THREADS = { value = "4", force = false }`, so a value already in the environment +wins), and `.config/nextest.toml` sets `test-threads = 4` for `cargo nextest run`. `cargo xtask` runs +under that config too, so `[env]` also sets a marker, `LIGHTCRAFT_TEST_THREADS_DEFAULT`, to the same +value: a `RUST_TEST_THREADS` equal to it counts as unset (`caller_sets` in `xtask/src/main.rs`) and the +caps replace it with the RAM-aware value, while any other value is the caller's and wins as-is. An +explicit `RUST_TEST_THREADS=4` can't be told apart from the default, so `ci` and `xtask test` may +lower it to their RAM-aware value (never raise it). + `ci` also probes the machine: OS, CPUs, RAM, the GPU adapters of each backend (from the ignored `gpu_probe` test of `lightcraft-gpu`, run after the workspace tests so it reuses their build; xtask has no wgpu), the corpus and `CRAFT_FONTS_DIR`. Tests @@ -87,6 +104,8 @@ the `lightcraft-gpu` tests once per backend with a hardware adapter (`LIGHTCRAFT | Date | Change | Summary | |---|---|---| +| 2026-10-26 | minor | Quality gates: plain `cargo test` (and `cargo nextest run`) runs at most 4 tests at once; xtask tells that default from a caller's `RUST_TEST_THREADS` | +| 2026-10-26 | minor | Quality gates: `cargo xtask test` runs the tests with `ci`'s caps and slot, and can keep going past a failing binary | | 2026-10-23 | minor | Quality gates: one integration-test binary per crate, doctests apart, nextest on request, one WASM check | | 2026-10-11 | minor | Quality gates: the CI machine probe, visible skips (`testkit`) and `target/ci/report.json` | | 2026-10-10 | major | Created from the code (crate sizes, layers, data model, rendering, control, gates) | diff --git a/xtask/src/ci.rs b/xtask/src/ci.rs --- a/xtask/src/ci.rs +++ b/xtask/src/ci.rs @@ -63,7 +63,7 @@ pub(crate) fn total_ram_gb() -> Option { const TEST_TARGETS: [&str; 3] = ["--lib", "--bins", "--tests"]; /// `args` followed by [`TEST_TARGETS`]. -fn with_test_targets<'a>(args: &[&'a str]) -> Vec<&'a str> { +pub(crate) fn with_test_targets<'a>(args: &[&'a str]) -> Vec<&'a str> { args.iter().copied().chain(TEST_TARGETS).collect() } @@ -72,7 +72,7 @@ fn with_test_targets<'a>(args: &[&'a str]) -> Vec<&'a str> { /// are cached per process: libtest runs a binary's tests in one process and generates each scene /// once, while nextest's process per test regenerates them (measured: the ui-egui tests take about /// 30 min under nextest, under 5 min under cargo test). -fn nextest() -> bool { +pub(crate) fn nextest() -> bool { static FOUND: std::sync::OnceLock = std::sync::OnceLock::new(); *FOUND.get_or_init(|| { if std::env::var_os("LIGHTCRAFT_NEXTEST").is_none_or(|v| v != "1") { @@ -107,60 +107,6 @@ fn doctests() -> (Command, String) { (c, "cargo test --workspace --doc".to_owned()) } -/// `ci --quick`'s tests: build every workspace test target, then run only the test binaries of -/// `packages`. Building with `-p` instead would resolve features per selection: different -/// artifacts from `ci`'s workspace build, so it would recompile ~130 crates. Doc tests are left to -/// the full `ci`. -fn test_selected(root: &Path, packages: &std::collections::BTreeSet, skip_log: &Path) -> Result<(), String> { - let manifests: std::collections::BTreeMap = metadata()?["packages"] - .as_array() - .into_iter() - .flatten() - .filter_map(|p| Some((p["manifest_path"].as_str()?.to_owned(), p["name"].as_str()?.to_owned()))) - .filter(|(_, name)| packages.contains(name)) - .collect(); - let what = "cargo test --workspace --no-run --lib --bins --tests"; - eprintln!("$ {what} (then the test binaries of {})", packages.iter().map(String::as_str).collect::>().join(", ")); - let out = cargo() - .args(with_test_targets(&["test", "--workspace", "--no-run", "--message-format=json-render-diagnostics"])) - .stderr(std::process::Stdio::inherit()) - .output() - .map_err(|e| format!("{what}: failed to spawn: {e}"))?; - if !out.status.success() { - return Err(format!("{what}: exited with {}", out.status)); - } - let mut binaries = Vec::new(); - for line in String::from_utf8_lossy(&out.stdout).lines() { - let Ok(msg) = serde_json::from_str::(line) else { continue }; - if msg["reason"] != "compiler-artifact" || msg["profile"]["test"] != true { - continue; - } - let (Some(exe), Some(manifest)) = (msg["executable"].as_str(), msg["manifest_path"].as_str()) else { continue }; - if let Some(name) = manifests.get(manifest) { - binaries.push((name.clone(), manifest.to_owned(), exe.to_owned())); - } - } - if binaries.is_empty() { - println!("no test binaries in {}", packages.iter().map(String::as_str).collect::>().join(", ")); - return Ok(()); - } - let mut failed = Vec::new(); - for (name, manifest, exe) in &binaries { - let dir = Path::new(manifest).parent().unwrap_or(root); - let mut c = Command::new(exe); - c.current_dir(dir).env("CARGO_MANIFEST_DIR", dir).env("CARGO_PKG_NAME", name).env(ci_report::SKIP_LOG_ENV, skip_log); - if std::env::var_os("RUST_TEST_THREADS").is_none() { - // as the cargo runs of ci (`cargo()`): more threads make the frame-budget tests flaky - c.env("RUST_TEST_THREADS", std::thread::available_parallelism().map_or(4, |n| n.get()).min(4).to_string()); - } - let file = Path::new(exe).file_name().map(|f| f.to_string_lossy().into_owned()).unwrap_or_default(); - if run(c, &format!("{name}: {file}")).is_err() { - failed.push(format!("{name} ({file})")); - } - } - if failed.is_empty() { Ok(()) } else { Err(format!("tests failed in {}", failed.join(", "))) } -} - /// The test that prints the messages each language lacks (shown in English); it always passes. const TRANSLATION_GAPS_TEST: &str = "i18n::tests::translation_gaps_report"; @@ -210,6 +156,7 @@ pub(crate) fn cmd_ci(args: &[&str]) -> Result<(), String> { if base.is_some() && !quick { return Err("ci: --base only applies to --quick".into()); } + use_ci_caps(); let root = root(); let command = std::iter::once("cargo xtask ci").chain(args.iter().copied()).collect::>().join(" "); // held until ci returns: one run at a time per machine (ci_lock.rs) @@ -296,7 +243,7 @@ pub(crate) fn cmd_ci(args: &[&str]) -> Result<(), String> { ( "test", Box::new(|| match &plan { - Some(p) if !p.workspace_wide => test_selected(&root, &p.packages, &ci_report::skip_log(&root, "test")), + Some(p) if !p.workspace_wide => test::selected(&root, &p.packages, &ci_report::skip_log(&root, "test")), _ => test(tests(&with_test_targets(&["--workspace"])), "test").and_then(|()| test(doctests(), "test")), }), ), @@ -459,4 +406,87 @@ mod ci_jobs_tests { assert_eq!(ci_jobs(None, 32), 4, "unknown RAM keeps the old default"); assert_eq!(ci_jobs(None, 2), 2); } + + #[test] + fn caps_cover_build_and_test_threads_and_yield_to_explicit_settings() { + let caps = super::ci_caps(); + let keys: Vec<&str> = caps.iter().map(|(k, _)| *k).collect(); + assert_eq!(keys, ["CARGO_PROFILE_DEV_DEBUG", "CARGO_BUILD_JOBS", "RUST_TEST_THREADS", "NEXTEST_TEST_THREADS"]); + for (key, value) in caps.iter().filter(|(k, _)| k.ends_with("TEST_THREADS")) { + let n: usize = value.parse().unwrap(); + assert!((1..=4).contains(&n), "{key}={value}"); + } + let set = super::caps_to_set(caps, |key| key == "RUST_TEST_THREADS"); + assert_eq!(set.iter().map(|(k, _)| *k).collect::>(), ["CARGO_PROFILE_DEV_DEBUG", "CARGO_BUILD_JOBS", "NEXTEST_TEST_THREADS"]); + assert!(super::caps_to_set(caps, |_| true).is_empty()); + } + + /// An environment of `vars` for [`super::set_by_caller`]. + fn env(vars: &[(&str, &str)]) -> impl Fn(&str) -> Option { + let vars: Vec<(String, String)> = vars.iter().map(|(k, v)| ((*k).to_owned(), (*v).to_owned())).collect(); + move |key| vars.iter().find(|(k, _)| k == key).map(|(_, v)| v.into()) + } + + #[test] + fn the_config_test_threads_default_is_not_a_callers_setting() { + use super::set_by_caller; + const MARKER: &str = super::TEST_THREADS_DEFAULT_MARKER; + let threads = "RUST_TEST_THREADS"; + assert!(!set_by_caller(threads, env(&[])), "unset, outside cargo"); + assert!(!set_by_caller(threads, env(&[(MARKER, "4")])), "unset (no RUST_TEST_THREADS at all)"); + assert!(!set_by_caller(threads, env(&[(threads, "4"), (MARKER, "4")])), "the config default: ci sizes it by RAM"); + assert!(set_by_caller(threads, env(&[(threads, "9"), (MARKER, "4")])), "RUST_TEST_THREADS=9 cargo xtask ci"); + assert!(set_by_caller(threads, env(&[(threads, "2"), (MARKER, "4")])), "a lower explicit value wins too"); + assert!(set_by_caller(threads, env(&[(threads, "4")])), "no marker: xtask run outside cargo, the value is the caller's"); + assert!(set_by_caller(threads, env(&[(threads, "")])), "set, even if empty"); + // the marker only speaks for RUST_TEST_THREADS + assert!(set_by_caller("NEXTEST_TEST_THREADS", env(&[("NEXTEST_TEST_THREADS", "4"), (MARKER, "4")]))); + assert!(set_by_caller("CARGO_BUILD_JOBS", env(&[("CARGO_BUILD_JOBS", "4"), (MARKER, "4")]))); + assert!(!set_by_caller("CARGO_BUILD_JOBS", env(&[(MARKER, "4")]))); + } + + /// The `RUST_TEST_THREADS` a capped run's test binaries get in environment `vars` when `ci`'s + /// RAM-aware value is `cap`: the cap where the caller didn't set it, else the environment's. + fn test_threads(vars: &[(&str, &str)], cap: &str) -> String { + let caps = [("RUST_TEST_THREADS", cap.to_owned())]; + let get = env(vars); + let set = super::caps_to_set(&caps, |key| super::set_by_caller(key, &get)); + match set.first() { + Some((_, value)) => (*value).to_owned(), + None => get("RUST_TEST_THREADS").unwrap().into_string().unwrap(), + } + } + + #[test] + fn an_explicit_test_thread_count_wins_except_one_equal_to_the_config_default() { + const MARKER: &str = super::TEST_THREADS_DEFAULT_MARKER; + let threads = "RUST_TEST_THREADS"; + assert_eq!(test_threads(&[(threads, "9"), (MARKER, "4")], "2"), "9", "explicit 9 is kept"); + assert_eq!(test_threads(&[(threads, "2"), (MARKER, "4")], "3"), "2", "explicit 2 is kept"); + assert_eq!(test_threads(&[(threads, "4"), (MARKER, "4")], "2"), "2", "the config default yields to the RAM-aware cap"); + // An explicit RUST_TEST_THREADS=4 looks the same as the config default (documented): it + // may be lowered, never raised. + assert_eq!(test_threads(&[(threads, "4"), (MARKER, "4")], "2"), "2", "explicit 4 under cargo is lowered like the default"); + assert_eq!(test_threads(&[(threads, "4")], "2"), "4", "no marker (xtask run outside cargo): 4 is the caller's"); + assert_eq!(test_threads(&[(MARKER, "4")], "3"), "3", "unset: the cap"); + } + + /// The quoted `value` of the `key = { value = "…", … }` line in `.cargo/config.toml`'s `[env]`. + fn config_env(config: &str, key: &str) -> Option<(String, String)> { + let line = config.lines().find(|l| l.split('=').next().is_some_and(|k| k.trim() == key))?; + let value = line.split("value = \"").nth(1)?.split('"').next()?.to_owned(); + Some((value, line.to_owned())) + } + + #[test] + fn the_cargo_config_sets_the_test_threads_default_and_its_marker_to_one_value() { + let config = std::fs::read_to_string(super::root().join(".cargo/config.toml")).unwrap(); + let (threads, line) = config_env(&config, "RUST_TEST_THREADS").expect("RUST_TEST_THREADS in [env]"); + assert!(line.contains("force = false"), "an explicit RUST_TEST_THREADS must win: {line}"); + let (marker, _) = config_env(&config, super::TEST_THREADS_DEFAULT_MARKER).expect("the marker in [env]"); + assert_eq!(threads, marker, "xtask tells the default apart by this marker"); + let n: usize = threads.parse().unwrap(); + assert!((1..=4).contains(&n), "{n}"); + assert!(config.contains("[env]")); + } } diff --git a/xtask/src/main.rs b/xtask/src/main.rs --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -12,6 +12,7 @@ mod ico; mod layers; mod parity; mod stats; +mod test; mod version; mod web; @@ -58,6 +59,13 @@ commands: break: clippy, the changed packages' tests, and heif / parity / wasm when they apply. One ci runs at a time per machine; others wait (LIGHTCRAFT_CI_SLOTS=N for N at once, LIGHTCRAFT_CI_LOCK=off to skip the lock) + test [-p PKG]... [--keep-going] [--doc] [-- ARGS] + the workspace's tests (or PKG's) with ci's caps (build jobs by free RAM, at most 4 test + threads, line-table debuginfo) in ci's machine-wide slot, instead of `cargo test`: + builds every test target once, then runs each test binary with ARGS (a filter, + --exact, --skip, --nocapture); stops at the first failing binary unless --keep-going + (alias --no-fail-fast), which runs them all and names each that failed; --doc runs the + doctests after them. LIGHTCRAFT_NEXTEST=1 runs the tests under cargo nextest corpus [--download] show where test corpora live; --download fetches PngSuite and CC0 raw samples (raw.pixls.us) into corpus/ and checks their sha256 stats [--exact] count tests and lines per crate (--exact: ask the test harness via `-- --list`) @@ -81,6 +89,7 @@ fn main() -> ExitCode { Some("wasm") => cmd_wasm(), Some("web") => web::run(&rest), Some("ci") => cmd_ci(&rest), + Some("test") => test::run(&rest), Some("corpus") => cmd_corpus(rest.contains(&"--download")), Some("stats") => stats::run(&root(), rest.contains(&"--exact")), Some("raw-frame") => cmd_raw_frame(&rest), @@ -125,32 +134,81 @@ pub fn root() -> PathBuf { pub fn cargo() -> Command { let mut c = Command::new(std::env::var("CARGO").unwrap_or_else(|_| "cargo".into())); c.current_dir(root()); - // Full Windows debuginfo plus one linker per CPU can exhaust RAM before any - // tests run. Scope these overridable defaults to CI, including parity/WASM - // subprocesses, without changing ordinary developer builds or GPU coverage. - if std::env::args().nth(1).as_deref() == Some("ci") { - static JOBS: std::sync::OnceLock<(String, String)> = std::sync::OnceLock::new(); - let (build, threads) = JOBS.get_or_init(|| { - let ram_mb = available_ram_mb().or_else(|| total_ram_gb().map(|gb| gb.saturating_mul(1024) / 2)); - let jobs = ci_jobs(ram_mb, std::thread::available_parallelism().map_or(4, |n| n.get())); - // More test threads barely shorten CI (a few heavy tests dominate) but make the - // wall-clock frame-budget tests flaky on a loaded machine. - (jobs.to_string(), jobs.min(4).to_string()) - }); - for (key, value) in [ - ("CARGO_PROFILE_DEV_DEBUG", "line-tables-only"), - ("CARGO_BUILD_JOBS", build.as_str()), - ("RUST_TEST_THREADS", threads.as_str()), - ("NEXTEST_TEST_THREADS", threads.as_str()), - ] { - if std::env::var_os(key).is_none() { - c.env(key, value); - } - } - } + apply_ci_caps(&mut c); c } +/// Whether this run gives the cargo and test processes it starts CI's resource caps +/// ([`apply_ci_caps`]). Off until a subcommand opts in with [`use_ci_caps`]. +static CI_CAPS: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false); + +/// Opt this run into CI's resource caps: the subcommands that build and run the test suite (`ci`, +/// `test`) call it first, so every cargo and test binary they start, including parity and WASM +/// subprocesses, gets them. Ordinary developer builds keep their own settings. +fn use_ci_caps() { + CI_CAPS.store(true, std::sync::atomic::Ordering::Relaxed); +} + +/// The environment defaults of a capped run, worked out once: line-table debuginfo, build jobs by +/// available RAM ([`ci_jobs`]) and test threads at most 4. +fn ci_caps() -> &'static [(&'static str, String); 4] { + static CAPS: std::sync::OnceLock<[(&'static str, String); 4]> = std::sync::OnceLock::new(); + CAPS.get_or_init(|| { + let ram_mb = available_ram_mb().or_else(|| total_ram_gb().map(|gb| gb.saturating_mul(1024) / 2)); + let jobs = ci_jobs(ram_mb, std::thread::available_parallelism().map_or(4, |n| n.get())); + // More test threads barely shorten CI (a few heavy tests dominate) but make the + // wall-clock frame-budget tests flaky on a loaded machine. + let threads = jobs.min(4).to_string(); + [ + ("CARGO_PROFILE_DEV_DEBUG", "line-tables-only".to_owned()), + ("CARGO_BUILD_JOBS", jobs.to_string()), + ("RUST_TEST_THREADS", threads.clone()), + ("NEXTEST_TEST_THREADS", threads), + ] + }) +} + +/// The caps `c` gets: each of `caps` the environment doesn't set itself (`is_set`, normally +/// [`caller_sets`]), so a caller's setting wins; see [`set_by_caller`] for an explicit +/// `RUST_TEST_THREADS=4`. +fn caps_to_set<'a>(caps: &'a [(&'static str, String)], is_set: impl Fn(&str) -> bool) -> Vec<(&'static str, &'a str)> { + caps.iter().filter(|(key, _)| !is_set(key)).map(|(key, value)| (*key, value.as_str())).collect() +} + +/// The variable `.cargo/config.toml` sets beside its `RUST_TEST_THREADS` default, to the same +/// value (#3059). `cargo xtask` runs under that config, so it inherits the default like any cargo +/// run; the marker tells it apart from a caller's own setting. +const TEST_THREADS_DEFAULT_MARKER: &str = "LIGHTCRAFT_TEST_THREADS_DEFAULT"; + +/// Whether the caller set `key` in the environment (`get` reads it), rather than leaving it unset +/// or to the `RUST_TEST_THREADS` default of `.cargo/config.toml`: that one counts as unset when it +/// equals the marker [`TEST_THREADS_DEFAULT_MARKER`] holds. So an explicit value other than the +/// default (4) wins as-is, but an explicit `RUST_TEST_THREADS=4` can't be told apart from the +/// default: `ci` and `xtask test` may lower it to their RAM-aware value (never raise it, as that is +/// at most 4). Without the marker (xtask run outside cargo) every value is the caller's. +fn set_by_caller(key: &str, get: impl Fn(&str) -> Option) -> bool { + let Some(value) = get(key) else { + return false; + }; + !(key == "RUST_TEST_THREADS" && get(TEST_THREADS_DEFAULT_MARKER).is_some_and(|default| default == value)) +} + +/// [`set_by_caller`] for this process's environment (an explicit `RUST_TEST_THREADS=4` counts as +/// the config default). +pub fn caller_sets(key: &str) -> bool { + set_by_caller(key, |k| std::env::var_os(k)) +} + +/// Give `c` CI's resource caps when this run opted in ([`use_ci_caps`]). Full Windows debuginfo +/// plus one linker per CPU can exhaust RAM before any tests run, and one test thread per CPU +/// can exhaust it while they run (#3057). A caller's setting wins ([`caller_sets`]), except that an +/// explicit `RUST_TEST_THREADS=4` looks like the config default and may be lowered. +fn apply_ci_caps(c: &mut Command) { + if CI_CAPS.load(std::sync::atomic::Ordering::Relaxed) { + c.envs(caps_to_set(ci_caps(), caller_sets)); + } +} + pub fn run(mut cmd: Command, what: &str) -> Result<(), String> { eprintln!("$ {what}"); let status = cmd.status().map_err(|e| format!("{what}: failed to spawn: {e}"))?; diff --git a/xtask/src/test.rs b/xtask/src/test.rs new file mode 100644 --- /dev/null +++ b/xtask/src/test.rs @@ -0,0 +1,510 @@ +//! `cargo xtask test [-p PKG]… [--keep-going] [--doc] [-- ARGS]`: the workspace tests, or some +//! packages' tests, under the same caps as `ci` and in its machine-wide slot. +//! +//! Plain `cargo test --workspace` takes no CI slot and gets only `.cargo/config.toml`'s 4 test +//! threads per binary (libtest's own default, one per CPU, ran the ui-egui tests alone out of memory +//! on a 16-core machine, #3057, #3059), not `ci`'s RAM-aware caps. This runs them the +//! way `ci` does: build jobs by available RAM, at most 4 test threads, line-table debuginfo +//! ([`super::use_ci_caps`]), one run per machine ([`ci_lock`]). +//! +//! It builds every workspace test target once (`cargo test --workspace --no-run`, the same +//! artifacts as `ci`'s test step, so nothing rebuilds between them) and then runs each test binary +//! itself ([`run_binary`]), passing it `ARGS` (a filter, `--exact`, `--skip`, `--nocapture`, …). +//! It stops at the first failing binary, as `ci` does; `--keep-going` runs every binary and names +//! each that failed. `--doc` runs the doctests after them. With `LIGHTCRAFT_NEXTEST=1` (and +//! cargo-nextest installed) the binaries run under `cargo nextest run` instead, as in `ci`. +//! +//! `ci --quick` runs the changed packages' tests through the same functions ([`selected`]). + +use std::collections::{BTreeMap, BTreeSet}; +use std::path::Path; +use std::process::Command; + +use crate::ci_lock; + +/// What `cargo xtask test` was asked to run. +#[derive(Debug, Default, PartialEq, Eq)] +pub struct Args<'a> { + /// `-p` packages; empty for the whole workspace. + pub packages: BTreeSet, + /// Run every test binary and report all that failed, rather than stop at the first. + pub keep_going: bool, + /// Run the doctests too, after the test binaries. + pub doc: bool, + /// What follows `--`, passed to every test binary (and the doctests). + pub test_args: Vec<&'a str>, +} + +/// Parse the arguments after `cargo xtask test`. +pub fn parse<'a>(args: &[&'a str]) -> Result, String> { + let mut parsed = Args::default(); + let mut it = args.iter().copied(); + while let Some(arg) = it.next() { + match arg { + "--" => { + parsed.test_args = it.by_ref().collect(); + } + // `--no-fail-fast` is cargo's name for it + "--keep-going" | "--no-fail-fast" => parsed.keep_going = true, + "--doc" => parsed.doc = true, + "-p" | "--package" => { + let Some(pkg) = it.next().filter(|p| !p.is_empty() && !p.starts_with('-')) else { + return Err(format!("test: {arg} needs a package name")); + }; + parsed.packages.insert(pkg.to_owned()); + } + _ => match arg.strip_prefix("--package=").or_else(|| arg.strip_prefix("-p")).map(|p| p.trim_start_matches('=')).filter(|p| !p.is_empty()) + { + Some(pkg) => { + parsed.packages.insert(pkg.to_owned()); + } + None => return Err(format!("test: unknown option `{arg}` (test arguments go after `--`)")), + }, + } + } + Ok(parsed) +} + +/// A workspace package: its name and whether it has a library (so doctests). +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Package { + pub name: String, + pub lib: bool, +} + +/// The workspace packages in `metadata` (`cargo metadata --no-deps`) by manifest path, or only +/// those in `only` (an error names any that aren't workspace packages). +pub fn packages(metadata: &serde_json::Value, only: &BTreeSet) -> Result, String> { + let all: BTreeMap = metadata["packages"] + .as_array() + .into_iter() + .flatten() + .filter_map(|p| { + let lib = p["targets"] + .as_array() + .into_iter() + .flatten() + .any(|t| t["kind"].as_array().into_iter().flatten().any(|k| matches!(k.as_str(), Some("lib" | "rlib" | "proc-macro")))); + Some((p["manifest_path"].as_str()?.to_owned(), Package { name: p["name"].as_str()?.to_owned(), lib })) + }) + .collect(); + let names: BTreeSet<&str> = all.values().map(|p| p.name.as_str()).collect(); + let unknown: Vec<&str> = only.iter().map(String::as_str).filter(|p| !names.contains(p)).collect(); + if !unknown.is_empty() { + return Err(format!( + "test: not a workspace package: {} (packages: {})", + unknown.join(", "), + names.into_iter().collect::>().join(", ") + )); + } + Ok(all.into_iter().filter(|(_, p)| only.is_empty() || only.contains(&p.name)).collect()) +} + +/// A test binary cargo built: its package, the package's manifest and the executable. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Binary { + pub package: String, + pub manifest: String, + pub exe: String, +} + +impl Binary { + /// `package (file)`, how the log and the summary name it. + pub fn label(&self) -> String { + let file = Path::new(&self.exe).file_name().map(|f| f.to_string_lossy().into_owned()).unwrap_or_default(); + format!("{} ({file})", self.package) + } +} + +/// The test binaries of `packages` among the artifacts in cargo's JSON messages (`stdout` of a +/// `--message-format=json` build), by package and file name. +pub fn binaries(stdout: &str, packages: &BTreeMap) -> Vec { + let mut found: Vec = stdout + .lines() + .filter_map(|line| serde_json::from_str::(line).ok()) + .filter(|msg| msg["reason"] == "compiler-artifact" && msg["profile"]["test"] == true) + .filter_map(|msg| { + let (exe, manifest) = (msg["executable"].as_str()?, msg["manifest_path"].as_str()?); + let package = packages.get(manifest)?; + Some(Binary { package: package.name.clone(), manifest: manifest.to_owned(), exe: exe.to_owned() }) + }) + .collect(); + found.sort_by(|a, b| (&a.package, &a.exe).cmp(&(&b.package, &b.exe))); + found.dedup(); + found +} + +/// Build every workspace test target once and return the test binaries of `packages`. Building +/// with `-p` instead would resolve features per selection: different artifacts from `ci`'s +/// workspace build, so it would recompile ~130 crates. +pub fn build(packages: &BTreeMap, whole_workspace: bool) -> Result, String> { + let what = "cargo test --workspace --no-run --lib --bins --tests"; + let then = if whole_workspace { "every test binary".to_owned() } else { format!("the test binaries of {}", names(packages.values())) }; + eprintln!("$ {what} (then {then})"); + let out = super::cargo() + .args(crate::ci::with_test_targets(&["test", "--workspace", "--no-run", "--message-format=json-render-diagnostics"])) + .stderr(std::process::Stdio::inherit()) + .output() + .map_err(|e| format!("{what}: failed to spawn: {e}"))?; + if !out.status.success() { + return Err(format!("{what}: exited with {}", out.status)); + } + Ok(binaries(&String::from_utf8_lossy(&out.stdout), packages)) +} + +fn names<'a>(packages: impl Iterator) -> String { + packages.map(|p| p.name.as_str()).collect::>().join(", ") +} + +/// The resource limits a test binary run directly gets, where the caller doesn't set them +/// ([`super::caller_sets`]: `.cargo/config.toml`'s `RUST_TEST_THREADS` default doesn't count). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Caps { + /// `cargo xtask test`: all of CI's caps ([`super::apply_ci_caps`]), so `RUST_TEST_THREADS` is + /// the RAM-aware `min(build jobs by available RAM, 4)` that `ci`'s cargo runs get. + Ci, + /// `ci --quick`: only `RUST_TEST_THREADS = min(CPUs, 4)`, its default before `test` existed. + QuickCi, +} + +/// Run one test binary as `cargo test` would (from its package's directory, with its +/// `CARGO_MANIFEST_DIR` and `CARGO_PKG_NAME`) under `caps`, with `args`; skips the tests record +/// go to `skip_log`. The one place `test` and `ci --quick` start a test binary and wait for it. +/// The error says how it ended: its exit status, the signal that killed it, or why it didn't start. +pub fn run_binary(root: &Path, binary: &Binary, args: &[&str], skip_log: Option<&Path>, caps: Caps) -> Result<(), String> { + let dir = Path::new(&binary.manifest).parent().unwrap_or(root); + let mut c = Command::new(&binary.exe); + c.current_dir(dir).env("CARGO_MANIFEST_DIR", dir).env("CARGO_PKG_NAME", &binary.package).args(args); + if let Some(log) = skip_log { + c.env(crate::ci_report::SKIP_LOG_ENV, log); + } + match caps { + Caps::Ci => super::apply_ci_caps(&mut c), + Caps::QuickCi => { + if !super::caller_sets("RUST_TEST_THREADS") { + // as the cargo runs of ci (`cargo()`): more threads make the frame-budget tests flaky + c.env("RUST_TEST_THREADS", std::thread::available_parallelism().map_or(4, |n| n.get()).min(4).to_string()); + } + } + } + let shown = std::iter::once(binary.label()).chain(args.iter().map(|a| (*a).to_owned())).collect::>().join(" "); + eprintln!("$ {shown}"); + let status = c.status().map_err(|e| format!("failed to start: {e}"))?; + if status.success() { Ok(()) } else { Err(describe(status)) } +} + +/// How a process that failed ended: `exit status: 101`, or on Unix the signal that killed it +/// (`signal: 9 (SIGKILL)`), with a hint for SIGKILL, which is what the out-of-memory killer sends. +pub fn describe(status: std::process::ExitStatus) -> String { + #[cfg(unix)] + { + use std::os::unix::process::ExitStatusExt; + if status.signal() == Some(9) { + return format!("{status}: killed, maybe for running out of memory"); + } + } + status.to_string() +} + +/// The outcome of running a list of test binaries. +#[derive(Debug, Default, PartialEq, Eq)] +pub struct Outcome { + pub passed: usize, + /// The labels of the binaries that failed, each with how it ended. + pub failed: Vec<(String, String)>, + /// Binaries not run: the run stopped at the first failure. + pub not_run: usize, +} + +/// Run `binaries` in order with `run_one`; stop at the first failure unless `keep_going`. +pub fn run_all(binaries: &[Binary], keep_going: bool, mut run_one: impl FnMut(&Binary) -> Result<(), String>) -> Outcome { + let mut outcome = Outcome::default(); + for (i, binary) in binaries.iter().enumerate() { + let Err(error) = run_one(binary) else { + outcome.passed += 1; + continue; + }; + outcome.failed.push((binary.label(), error)); + if !keep_going { + outcome.not_run = binaries.len().saturating_sub(i + 1); + break; + } + } + outcome +} + +/// `ci --quick`'s tests: the test binaries of `packages`, every one of them (each failure named). +/// Doctests are left to the full `ci`. +pub fn selected(root: &Path, packages: &BTreeSet, skip_log: &Path) -> Result<(), String> { + let packages = self::packages(&super::metadata()?, packages)?; + let binaries = build(&packages, false)?; + if binaries.is_empty() { + println!("no test binaries in {}", names(packages.values())); + return Ok(()); + } + let outcome = run_all(&binaries, true, |b| run_binary(root, b, &[], Some(skip_log), Caps::QuickCi)); + if outcome.failed.is_empty() { + Ok(()) + } else { + Err(format!("tests failed in {}", outcome.failed.iter().map(|(label, _)| label.as_str()).collect::>().join(", "))) + } +} + +/// The `cargo nextest run` arguments for `args`: the workspace build, the `-p` packages as a +/// filter expression (so the build is still the workspace's), `--no-fail-fast` for `--keep-going`. +pub fn nextest_args<'a>(args: &Args<'a>) -> Vec { + let mut v: Vec = crate::ci::with_test_targets(&["--workspace"]).into_iter().map(str::to_owned).collect(); + if !args.packages.is_empty() { + let filter = args.packages.iter().map(|p| format!("package(={p})")).collect::>().join(" | "); + v.extend(["-E".to_owned(), filter]); + } + if args.keep_going { + v.push("--no-fail-fast".to_owned()); + } + if !args.test_args.is_empty() { + v.push("--".to_owned()); + v.extend(args.test_args.iter().map(|a| (*a).to_owned())); + } + v +} + +/// The `cargo test --doc` arguments for `args`: the workspace's doctests, or those of the `-p` +/// packages that have a library (`None` when none has); `--no-fail-fast` for `--keep-going`, so a +/// package's failing doctests don't stop the next package's. +pub fn doc_args<'a>(args: &Args<'a>, packages: &BTreeMap) -> Option> { + let mut v = vec!["test".to_owned(), "--doc".to_owned()]; + if args.packages.is_empty() { + v.push("--workspace".to_owned()); + } else { + let libs: Vec<&Package> = packages.values().filter(|p| p.lib).collect(); + if libs.is_empty() { + return None; + } + for p in libs { + v.extend(["-p".to_owned(), p.name.clone()]); + } + } + if args.keep_going { + v.push("--no-fail-fast".to_owned()); + } + if !args.test_args.is_empty() { + v.push("--".to_owned()); + v.extend(args.test_args.iter().map(|a| (*a).to_owned())); + } + Some(v) +} + +/// `cargo xtask test`. +pub fn run(raw: &[&str]) -> Result<(), String> { + let args = parse(raw)?; + super::use_ci_caps(); + let root = super::root(); + let command = std::iter::once("cargo xtask test").chain(raw.iter().copied()).collect::>().join(" "); + // held until the run ends: one ci or test run at a time per machine (ci_lock.rs) + let _slot = ci_lock::acquire(&root, &command)?; + let packages = packages(&super::metadata()?, &args.packages)?; + let mut failed = Vec::new(); + let mut summary = Vec::new(); + if crate::ci::nextest() { + let nextest = nextest_args(&args); + let mut c = super::cargo(); + c.args(["nextest", "run"]).args(&nextest); + if let Err(e) = super::run(c, &format!("cargo nextest run {}", nextest.join(" "))) { + failed.push(("cargo nextest run".to_owned(), e)); + } + } else { + let binaries = build(&packages, args.packages.is_empty())?; + if binaries.is_empty() { + println!("no test binaries in {}", names(packages.values())); + } + let outcome = run_all(&binaries, args.keep_going, |b| run_binary(&root, b, &args.test_args, None, Caps::Ci)); + summary.push(format!("{} of {} test binaries passed", outcome.passed, binaries.len())); + if outcome.not_run > 0 { + summary.push(format!("{} not run after the first failure (--keep-going runs them)", outcome.not_run)); + } + failed.extend(outcome.failed); + } + if args.doc && !failed.is_empty() && !args.keep_going { + summary.push("doctests not run after the failure".to_owned()); + } else if args.doc { + match doc_args(&args, &packages) { + Some(doc) => { + let mut c = super::cargo(); + c.args(&doc); + if let Err(e) = super::run(c, &format!("cargo {}", doc.join(" "))) { + failed.push(("doctests".to_owned(), e)); + } + } + None => println!("no doctests: none of {} has a library", names(packages.values())), + } + } + println!("\ntest summary: {}", if summary.is_empty() { "done".to_owned() } else { summary.join("; ") }); + if failed.is_empty() { + println!(" all passed"); + return Ok(()); + } + for (label, error) in &failed { + println!(" FAIL {label}: {error}"); + } + Err(format!("tests failed in {}", failed.iter().map(|(label, _)| label.as_str()).collect::>().join(", "))) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_packages_keep_going_doc_and_test_args() { + let a = parse(&[ + "-p", + "lightcraft-geom", + "--package", + "xtask", + "--package=lightcraft-raw", + "-plightcraft-color", + "--keep-going", + "--doc", + "--", + "foo::", + "--exact", + ]) + .unwrap(); + assert_eq!(a.packages.iter().map(String::as_str).collect::>(), ["lightcraft-color", "lightcraft-geom", "lightcraft-raw", "xtask"]); + assert!(a.keep_going && a.doc); + assert_eq!(a.test_args, ["foo::", "--exact"]); + assert_eq!(parse(&[]).unwrap(), Args::default(), "no arguments: every package, fail fast, no doctests"); + assert!(parse(&["--no-fail-fast"]).unwrap().keep_going, "cargo's name for it"); + assert_eq!(parse(&["--", "--keep-going", "-p"]).unwrap().test_args, ["--keep-going", "-p"], "after `--` everything is the binaries'"); + } + + #[test] + fn rejects_unknown_options_and_a_missing_package() { + assert!(parse(&["--workspace"]).unwrap_err().contains("unknown option `--workspace`")); + assert!(parse(&["foo"]).unwrap_err().contains("after `--`"), "a bare filter must follow `--`"); + assert!(parse(&["-p"]).unwrap_err().contains("needs a package name")); + assert!(parse(&["-p", "--doc"]).unwrap_err().contains("needs a package name")); + assert!(parse(&["--package="]).is_err()); + assert!(parse(&["-p="]).is_err()); + } + + fn metadata() -> serde_json::Value { + serde_json::json!({"packages": [ + {"name": "a", "manifest_path": "/w/a/Cargo.toml", "targets": [{"kind": ["lib"]}, {"kind": ["test"]}]}, + {"name": "b", "manifest_path": "/w/b/Cargo.toml", "targets": [{"kind": ["bin"]}]}, + {"name": "c", "manifest_path": "/w/c/Cargo.toml", "targets": [{"kind": ["proc-macro"]}]}, + ]}) + } + + #[test] + fn selects_workspace_packages_and_names_unknown_ones() { + let all = packages(&metadata(), &BTreeSet::new()).unwrap(); + assert_eq!(all.len(), 3); + assert_eq!(all["/w/b/Cargo.toml"], Package { name: "b".into(), lib: false }); + assert!(all["/w/c/Cargo.toml"].lib, "a proc-macro crate has doctests"); + let only = packages(&metadata(), &["b".to_owned()].into()).unwrap(); + assert_eq!(only.keys().collect::>(), ["/w/b/Cargo.toml"]); + let err = packages(&metadata(), &["a".to_owned(), "nope".to_owned()].into()).unwrap_err(); + assert!(err.contains("not a workspace package: nope") && err.contains("packages: a, b, c"), "{err}"); + } + + #[test] + fn finds_the_test_binaries_of_the_selected_packages() { + let sel = packages(&metadata(), &["a".to_owned()].into()).unwrap(); + let stdout = [ + r#"{"reason":"compiler-artifact","profile":{"test":true},"manifest_path":"/w/a/Cargo.toml","executable":"/t/debug/deps/it-2"}"#, + r#"{"reason":"compiler-artifact","profile":{"test":false},"manifest_path":"/w/a/Cargo.toml","executable":null}"#, + r#"{"reason":"compiler-artifact","profile":{"test":true},"manifest_path":"/w/b/Cargo.toml","executable":"/t/debug/deps/b-1"}"#, + r#"{"reason":"build-finished","success":true}"#, + "not json", + r#"{"reason":"compiler-artifact","profile":{"test":true},"manifest_path":"/w/a/Cargo.toml","executable":"/t/debug/deps/a-1"}"#, + ] + .join("\n"); + let found = binaries(&stdout, &sel); + assert_eq!(found.iter().map(Binary::label).collect::>(), ["a (a-1)", "a (it-2)"]); + assert_eq!(found.first().map(|b| b.manifest.as_str()), Some("/w/a/Cargo.toml")); + } + + fn bins(names: &[&str]) -> Vec { + names.iter().map(|n| Binary { package: (*n).to_owned(), manifest: format!("/w/{n}/Cargo.toml"), exe: format!("/t/{n}-0") }).collect() + } + + #[test] + fn keep_going_runs_every_binary_and_names_each_failure() { + let binaries = bins(&["a", "b", "c", "d"]); + let fail = |b: &Binary| if b.package == "b" || b.package == "d" { Err("exit 101".to_owned()) } else { Ok(()) }; + let mut ran = Vec::new(); + let outcome = run_all(&binaries, true, |b| { + ran.push(b.package.clone()); + fail(b) + }); + assert_eq!(ran, ["a", "b", "c", "d"]); + let failed = vec![("b (b-0)".into(), "exit 101".into()), ("d (d-0)".into(), "exit 101".into())]; + assert_eq!(outcome, Outcome { passed: 2, failed, not_run: 0 }); + } + + #[test] + fn without_keep_going_the_first_failure_stops_the_run() { + let binaries = bins(&["a", "b", "c", "d"]); + let mut ran = Vec::new(); + let outcome = run_all(&binaries, false, |b| { + ran.push(b.package.clone()); + if b.package == "b" { Err("exit 101".to_owned()) } else { Ok(()) } + }); + assert_eq!(ran, ["a", "b"]); + assert_eq!(outcome, Outcome { passed: 1, failed: vec![("b (b-0)".into(), "exit 101".into())], not_run: 2 }); + assert_eq!(run_all(&binaries, false, |_| Ok(())), Outcome { passed: 4, failed: vec![], not_run: 0 }); + } + + #[test] + fn nextest_keeps_the_workspace_build_and_filters_packages() { + let all = parse(&[]).unwrap(); + assert_eq!(nextest_args(&all), ["--workspace", "--lib", "--bins", "--tests"]); + let some = parse(&["-p", "b", "-p", "a", "--keep-going", "--", "foo"]).unwrap(); + assert_eq!( + nextest_args(&some), + ["--workspace", "--lib", "--bins", "--tests", "-E", "package(=a) | package(=b)", "--no-fail-fast", "--", "foo"] + ); + } + + #[test] + fn doctests_cover_the_workspace_or_the_selected_libraries() { + let all = packages(&metadata(), &BTreeSet::new()).unwrap(); + assert_eq!(doc_args(&parse(&["--doc"]).unwrap(), &all).unwrap(), ["test", "--doc", "--workspace"]); + let args = parse(&["-p", "a", "-p", "b", "--doc", "--", "x"]).unwrap(); + let sel = packages(&metadata(), &args.packages).unwrap(); + assert_eq!(doc_args(&args, &sel).unwrap(), ["test", "--doc", "-p", "a", "--", "x"], "b has no library"); + let keep_going = parse(&["-p", "a", "--doc", "--keep-going", "--", "x"]).unwrap(); + assert_eq!(doc_args(&keep_going, &sel).unwrap(), ["test", "--doc", "-p", "a", "--no-fail-fast", "--", "x"], "cargo's flag, before `--`"); + assert_eq!(doc_args(&parse(&["--keep-going"]).unwrap(), &all).unwrap(), ["test", "--doc", "--workspace", "--no-fail-fast"]); + let bin_only = parse(&["-p", "b"]).unwrap(); + assert_eq!(doc_args(&bin_only, &packages(&metadata(), &bin_only.packages).unwrap()), None); + } + + #[cfg(unix)] + #[test] + fn a_failure_says_how_the_binary_ended() { + use std::os::unix::process::ExitStatusExt; + // wait(2) statuses: an exit code in the high byte, a terminating signal in the low bits + assert_eq!(describe(std::process::ExitStatus::from_raw(101 << 8)), "exit status: 101"); + let killed = describe(std::process::ExitStatus::from_raw(9)); + assert!(killed.contains("SIGKILL") && killed.contains("out of memory"), "{killed}"); + assert!(describe(std::process::ExitStatus::from_raw(11)).contains("SIGSEGV")); + } + + #[test] + fn a_binary_that_cannot_start_reports_why() { + let missing = Binary { package: "a".into(), manifest: "/nonexistent/a/Cargo.toml".into(), exe: "/nonexistent/a/target/a-0".into() }; + let err = run_binary(Path::new("/nonexistent"), &missing, &[], None, Caps::QuickCi).unwrap_err(); + assert!(err.starts_with("failed to start:"), "{err}"); + } + + #[cfg(unix)] + #[test] + fn a_failing_binary_reports_its_exit_status() { + let binary = Binary { package: "sh".into(), manifest: "/Cargo.toml".into(), exe: "/bin/sh".into() }; + let err = run_binary(Path::new("/"), &binary, &["-c", "exit 3"], None, Caps::QuickCi).unwrap_err(); + assert_eq!(err, "exit status: 3"); + let err = run_binary(Path::new("/"), &binary, &["-c", "kill -9 $$"], None, Caps::QuickCi).unwrap_err(); + assert!(err.contains("SIGKILL"), "{err}"); + } +}