diff --git a/xtask/field-oracle/scout-1.txt b/xtask/field-oracle/scout-1.txt new file mode 100644 --- /dev/null +++ b/xtask/field-oracle/scout-1.txt @@ -0,0 +1,59 @@ +# Field codes behind the Word-versus-WordCraft differences in the first scout report (2026-10-26). +# One code per line (the text between the braces, without them); blank lines and # lines are skipped. + +# Last format switch wins +NUMPAGES \* ArabicDash \* roman +NUMPAGES \* roman \* ArabicDash +PAGE \* Hex \* CardText +PAGE \* roman \* Arabic + +# Basics that already match += 790 \* CardText += 790 \* CardText \* Caps += 21 \* OrdText \* FirstCap += 31 \* ArabicDash += 458 \* Hex += 458 \* Hex \* Lower + +# Caps lowercases the rest of each word; FirstCap leaves it +TITLE "WEEKLY REPORT" \* Caps +TITLE "wEEKLY rEPORT" \* Caps +TITLE "wEEKLY rEPORT" \* FirstCap +TITLE "WEEKLY REPORT" \* Lower \* Caps +TITLE "WEEKLY REPORT" \* Caps \* Lower += 255 \* Hex \* Caps + +# DollarText truncates the whole part and rounds cents modulo 100 += 0.999 \* DollarText += 1.995 \* DollarText += 999999.995 \* DollarText += 12.345 \* DollarText += -0.004 \* DollarText += 1.4 \* DollarText += -1 \* DollarText + +# Numbers outside a text format's range += 1000140 \* CardText += 999999 \* OrdText += 1000000 \* OrdText += 32767 \* roman += 32768 \* roman += 1000000 \* Hex += -5 \* CardText += -5 \* ArabicDash += 0 \* roman += 0 \* alphabetic += 0 \* OrdText += 27 \* alphabetic += 2.5 \* CardText + +# Backslash is an escape inside a quoted argument +TITLE "x \* Upper" +SUBJECT "Draft two" \* Upper + +# Formula fields outside a table += 1+1 +=1+1 += SUM(1,2) += 2*3 += SUM(2,3) \* CardText diff --git a/xtask/src/field_oracle.ps1 b/xtask/src/field_oracle.ps1 new file mode 100644 --- /dev/null +++ b/xtask/src/field_oracle.ps1 @@ -0,0 +1,36 @@ +# Runs field codes in real Word through COM. Driven by `cargo xtask field-oracle`, which defines +# $codeHex (hex of the UTF-8 codes, one per line, newline-joined) before this text and reads one +# `indexhex-of-result` line per code from stdout. Only temporary documents are used: alerts +# are off, documents close without saving, and only a Word this script started is killed. +$ErrorActionPreference = 'Stop' +function FromHex([string]$h) { + if ($h.Length -eq 0) { return '' } + $b = New-Object byte[] ($h.Length / 2) + for ($i = 0; $i -lt $b.Length; $i++) { $b[$i] = [Convert]::ToByte($h.Substring(2 * $i, 2), 16) } + [Text.Encoding]::UTF8.GetString($b) +} +function ToHex([string]$s) { -join ([Text.Encoding]::UTF8.GetBytes($s) | ForEach-Object { $_.ToString('x2') }) } + +$before = @(Get-Process WINWORD -ErrorAction SilentlyContinue | ForEach-Object { $_.Id }) +$word = $null +try { + $word = New-Object -ComObject Word.Application + $word.Visible = $false + $word.DisplayAlerts = 0 + $doc = $word.Documents.Add() + $codes = (FromHex $codeHex) -split "`n" + for ($i = 0; $i -lt $codes.Count; $i++) { + try { + $end = $doc.Content.End - 1 + $f = $doc.Fields.Add($doc.Range($end, $end), -1, $codes[$i], $false) + [void]$f.Update() + $res = $f.Result.Text + } catch { $res = 'COM error: ' + $_.Exception.Message } + $doc.Content.InsertParagraphAfter() + '{0}{1}{2}' -f $i, "`t", (ToHex (($res -replace "[\r\n\a]+", ' ').TrimEnd())) + } + $doc.Close(0) +} finally { + if ($word) { try { $word.Quit(0) } catch {} } + Get-Process WINWORD -ErrorAction SilentlyContinue | Where-Object { $before -notcontains $_.Id } | Stop-Process -Force +} diff --git a/xtask/src/field_oracle.rs b/xtask/src/field_oracle.rs new file mode 100644 --- /dev/null +++ b/xtask/src/field_oracle.rs @@ -0,0 +1,290 @@ +//! `cargo xtask field-oracle`: field codes in, a Word-versus-WordCraft table out. +//! +//! Each code is inserted as a field in a temporary document of real Word (COM through +//! `powershell.exe`; from WSL it needs `WSL_INTEROP`) and of `wordcraft-cli`, both are updated, and +//! the two result texts are compared. Word's answers are never stored: Word is asked each time. + +use std::io::Write; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::time::{Duration, Instant}; + +const PS_SCRIPT: &str = include_str!("field_oracle.ps1"); +const WORD_TIMEOUT: Duration = Duration::from_secs(120); + +pub const USAGE: &str = "\ +usage: cargo xtask field-oracle ... [--out table.md] [--cli wordcraft-cli] + codes.txt one field code per line (the text between the braces); blank and # lines are skipped + --code add one code from the command line (repeatable) + --out also write the table to this file + --cli a built wordcraft-cli (default: cargo run -p wordcraft-cli) + --no-word skip Word and print WordCraft's results only +"; + +/// The codes in `text`: one per line, blank and `#` lines skipped. +pub fn parse_codes(text: &str) -> Vec { + text.lines().map(str::trim).filter(|l| !l.is_empty() && !l.starts_with('#')).map(str::to_string).collect() +} + +fn to_hex(s: &str) -> String { + s.bytes().map(|b| format!("{b:02x}")).collect() +} + +fn from_hex(h: &str) -> Option { + let bytes: Option> = (0..h.len()).step_by(2).map(|i| u8::from_str_radix(h.get(i..i + 2)?, 16).ok()).collect(); + String::from_utf8(bytes?).ok() +} + +/// Word's results by code index, from the script's `indexhex` lines. +pub fn parse_word_output(out: &str, n: usize) -> Result, String> { + let mut res = vec![None; n]; + for line in out.lines().map(str::trim).filter(|l| !l.is_empty()) { + let (i, hex) = line.split_once('\t').unwrap_or((line, "")); + let i: usize = i.parse().map_err(|_| format!("unexpected output from Word: {line}"))?; + let text = from_hex(hex).ok_or_else(|| format!("bad result from Word: {line}"))?; + *res.get_mut(i).ok_or_else(|| format!("Word returned index {i} for {n} codes"))? = Some(text); + } + res.into_iter().enumerate().map(|(i, r)| r.ok_or_else(|| format!("Word returned no result for code {i}"))).collect() +} + +fn powershell() -> Command { + let mut c = if cfg!(windows) { + Command::new("powershell.exe") + } else { + let wsl = Path::new("/mnt/c/Windows/System32/WindowsPowerShell/v1.0/powershell.exe"); + let mut c = Command::new(if wsl.exists() { wsl.to_str().unwrap_or("powershell.exe") } else { "powershell.exe" }); + if std::env::var_os("WSL_INTEROP").is_none() { + c.env("WSL_INTEROP", "/run/WSL/2_interop"); + } + c + }; + c.args(["-NoProfile", "-NonInteractive", "-ExecutionPolicy", "Bypass", "-File"]); + c +} + +/// A path PowerShell understands: `wslpath -w` under WSL, the path itself on Windows. +fn windows_path(p: &Path) -> Result { + if cfg!(windows) { + return Ok(p.to_string_lossy().into_owned()); + } + let o = Command::new("wslpath").arg("-w").arg(p).output().map_err(|e| format!("wslpath: {e}"))?; + if !o.status.success() { + return Err(format!("wslpath -w {}: {}", p.display(), String::from_utf8_lossy(&o.stderr).trim())); + } + Ok(String::from_utf8_lossy(&o.stdout).trim().to_string()) +} + +fn word_results(codes: &[String]) -> Result, String> { + if codes.iter().any(|c| c.contains('\n')) { + return Err("a field code cannot contain a newline".into()); + } + // A script file, not `-Command -`: PowerShell reads a piped multi-line script line by line and runs nothing. + let dir = std::env::temp_dir().join(format!("field-oracle-word-{}", std::process::id())); + std::fs::create_dir_all(&dir).map_err(|e| format!("{}: {e}", dir.display()))?; + let ps1 = dir.join("field_oracle.ps1"); + let run = std::fs::write(&ps1, format!("$codeHex = '{}'\n{PS_SCRIPT}", to_hex(&codes.join("\n")))) + .map_err(|e| format!("{}: {e}", ps1.display())) + .and_then(|()| run_script(&ps1, codes.len())); + let _ = std::fs::remove_dir_all(&dir); + run +} + +fn run_script(ps1: &Path, n: usize) -> Result, String> { + let mut child = powershell() + .arg(windows_path(ps1)?) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .map_err(|e| format!("powershell.exe: {e}"))?; + // Read both pipes off-thread so a full pipe never stalls Word's script while we poll the clock. + let (mut out, mut err) = (child.stdout.take(), child.stderr.take()); + let out = std::thread::spawn(move || read_all(out.as_mut())); + let err = std::thread::spawn(move || read_all(err.as_mut())); + let start = Instant::now(); + let status = loop { + if let Some(s) = child.try_wait().map_err(|e| format!("powershell.exe: {e}"))? { + break s; + } + if start.elapsed() > WORD_TIMEOUT { + // The script's `finally` can't run now; Word is left to the caller, never killed by name. + let _ = child.kill(); + return Err(format!("Word did not finish within {}s (a Word it started may still be running)", WORD_TIMEOUT.as_secs())); + } + std::thread::sleep(Duration::from_millis(100)); + }; + let (out, err) = (out.join().unwrap_or_default(), err.join().unwrap_or_default()); + if !status.success() { + return Err(format!("Word run failed ({status}): {}", err.trim())); + } + parse_word_output(&out, n) +} + +fn read_all(r: Option<&mut impl std::io::Read>) -> String { + let mut s = String::new(); + if let Some(r) = r { + let mut b = Vec::new(); + let _ = r.read_to_end(&mut b); + s = String::from_utf8_lossy(&b).into_owned(); + } + s +} + +fn cli_command(cli: Option<&Path>) -> Command { + match cli { + Some(p) => Command::new(p), + None => { + let mut c = crate::cargo(); + c.args(["run", "-q", "-p", "wordcraft-cli", "--"]); + c + } + } +} + +/// WordCraft's results: one paragraph per code, every field updated, saved as text. +fn wordcraft_results(codes: &[String], cli: Option<&Path>) -> Result, String> { + let dir = std::env::temp_dir().join(format!("field-oracle-{}", std::process::id())); + std::fs::create_dir_all(&dir).map_err(|e| format!("{}: {e}", dir.display()))?; + let out = dir.join("out.txt"); + let mut commands = Vec::new(); + for c in codes { + commands.push(serde_json::json!({"command": "insert.field", "params": {"instr": c}})); + commands.push(serde_json::json!("text.newParagraph")); + } + commands.push(serde_json::json!({"command": "references.updateFields", "params": {"all": true}})); + let script = serde_json::json!({"template": "blank", "commands": commands, "save": [out.to_string_lossy()]}); + let run = (|| { + let mut child = cli_command(cli) + .args(["run", "--script", "-"]) + .stdin(Stdio::piped()) + .stdout(Stdio::null()) + .stderr(Stdio::piped()) + .spawn() + .map_err(|e| format!("wordcraft-cli: {e}"))?; + child.stdin.take().ok_or("wordcraft-cli: no stdin")?.write_all(script.to_string().as_bytes()).map_err(|e| format!("wordcraft-cli: {e}"))?; + let o = child.wait_with_output().map_err(|e| format!("wordcraft-cli: {e}"))?; + if !o.status.success() { + return Err(format!("wordcraft-cli failed ({}): {}", o.status, String::from_utf8_lossy(&o.stderr).trim())); + } + std::fs::read_to_string(&out).map_err(|e| format!("{}: {e}", out.display())) + })(); + let _ = std::fs::remove_dir_all(&dir); + split_paragraphs(&run?, codes.len()) +} + +/// The first `n` lines of the saved text (a trailing line break and the empty last paragraph are not results). +pub fn split_paragraphs(text: &str, n: usize) -> Result, String> { + let lines: Vec<&str> = text.lines().collect(); + if lines.len() < n { + return Err(format!("WordCraft saved {} paragraphs for {n} codes (a result with a line break?)", lines.len())); + } + Ok(lines.iter().take(n).map(|l| l.trim_end().to_string()).collect()) +} + +fn cell(s: &str) -> String { + format!("`{}`", s.replace('|', "\\|").replace('`', "'")) +} + +/// Fields WordCraft fills in at layout time: the text export has no result for them (render to see them). +fn is_layout_field(code: &str) -> bool { + let name = code.split_whitespace().next().unwrap_or("").to_ascii_uppercase(); + ["PAGE", "NUMPAGES", "SECTIONPAGES", "SECTION"].contains(&name.as_str()) +} + +/// The comparison table. `word` is `None` when Word was skipped. +pub fn table(codes: &[String], word: Option<&[String]>, wc: &[String]) -> String { + let mut s = String::from("| Code | Word | WordCraft | |\n|---|---|---|---|\n"); + let (mut same, mut diff) = (0, 0); + for (i, code) in codes.iter().enumerate() { + let w = word.and_then(|w| w.get(i)); + let verdict = match w { + None => "", + Some(_) if is_layout_field(code) && wc.get(i).is_none_or(String::is_empty) => "n/a (layout field)", + Some(w) if Some(w) == wc.get(i) => { + same += 1; + "match" + } + Some(_) => { + diff += 1; + "**DIFF**" + } + }; + s += &format!("| {} | {} | {} | {verdict} |\n", cell(code), w.map_or(String::new(), |w| cell(w)), cell(wc.get(i).map_or("", String::as_str))); + } + if word.is_some() { + s += &format!("\n{} codes: {same} match, {diff} differ.\n", codes.len()); + } + s +} + +pub fn run(root: &Path, args: &[&str]) -> Result<(), String> { + let (mut codes, mut out, mut cli, mut no_word) = (Vec::new(), None::, None::, false); + let mut it = args.iter(); + while let Some(a) = it.next() { + match *a { + "--code" => codes.push(it.next().ok_or("--code needs a value")?.to_string()), + "--out" => out = Some(it.next().ok_or("--out needs a value")?.into()), + "--cli" => cli = Some(it.next().ok_or("--cli needs a value")?.into()), + "--no-word" => no_word = true, + f if f.starts_with("--") => return Err(format!("unknown option `{f}`\n\n{USAGE}")), + f => { + let p = Path::new(f); + let p = if p.exists() { p.to_path_buf() } else { root.join(f) }; + codes.extend(parse_codes(&std::fs::read_to_string(&p).map_err(|e| format!("{}: {e}", p.display()))?)); + } + } + } + if codes.is_empty() { + return Err(format!("no field codes given\n\n{USAGE}")); + } + let wc = wordcraft_results(&codes, cli.as_deref())?; + let word = if no_word { None } else { Some(word_results(&codes)?) }; + let md = table(&codes, word.as_deref(), &wc); + print!("{md}"); + if let Some(o) = out { + std::fs::write(&o, &md).map_err(|e| format!("{}: {e}", o.display()))?; + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn codes_skip_comments_and_blanks() { + assert_eq!(parse_codes("# c\n\n= 1+1 \n PAGE \\* roman\n"), ["= 1+1", "PAGE \\* roman"]); + } + + #[test] + fn hex_round_trips_unicode() { + assert_eq!(from_hex(&to_hex("a→b \\* Upper")).as_deref(), Some("a→b \\* Upper")); + assert_eq!(from_hex("zz"), None); + assert_eq!(from_hex("abc"), None); + } + + #[test] + fn word_output_maps_by_index_and_rejects_gaps() { + let out = format!("1\t{}\n0\t\n", to_hex("six")); + assert_eq!(parse_word_output(&out, 2).unwrap(), ["", "six"]); + assert!(parse_word_output("0\t\n", 2).is_err()); + assert!(parse_word_output("5\t\n", 2).is_err()); + } + + #[test] + fn table_marks_differences_and_escapes_pipes() { + let codes = vec!["a".to_string(), "b|c".to_string()]; + let t = table(&codes, Some(&["1".into(), "2".into()]), &["1".into(), "3".into()]); + let layout = table(&["PAGE \\* roman".to_string()], Some(&["i".into()]), &["".into()]); + assert!(layout.contains("n/a (layout field)") && layout.contains("0 match, 0 differ")); + assert!(t.contains("| `a` | `1` | `1` | match |")); + assert!(t.contains("| `b\\|c` | `2` | `3` | **DIFF** |")); + assert!(t.contains("2 codes: 1 match, 1 differ.")); + } + + #[test] + fn paragraphs_drop_the_trailing_empty_one() { + assert_eq!(split_paragraphs("x\ny \n\n", 2).unwrap(), ["x", "y"]); + assert!(split_paragraphs("x\n", 2).is_err()); + } +} diff --git a/xtask/src/main.rs b/xtask/src/main.rs --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -4,6 +4,7 @@ //! invoked through `std::process::Command`. mod assets; +mod field_oracle; mod ico; mod layers; mod parity; @@ -34,6 +35,9 @@ commands: visual-diff [--private] [--dpi N] compare two PDFs of a document page by page (SSIM, line breaks, offsets); needs poppler (pdftoppm, pdftotext) + field-oracle ... [--out table.md] [--cli wordcraft-cli] [--no-word] + run field codes in real Word (COM) and in wordcraft-cli, print a comparison table + (xtask/field-oracle/scout-1.txt is the first corpus; Word and PowerShell needed) ico ... pack square PNGs (<= 256 px) into a Windows .ico (see packaging/icons.sh) "; @@ -51,6 +55,7 @@ fn main() -> ExitCode { Some("corpus") => cmd_corpus(rest.contains(&"--download")), Some("stats") => stats::run(&root(), rest.contains(&"--exact")), Some("parity") => parity::run(&root()), + Some("field-oracle") => field_oracle::run(&root(), &rest), Some("ico") => ico::run(&rest), Some("visual-diff") => visual_diff::run(&rest), Some("version") => version::run(&root(), &rest),