collective
Browse / github.com/storytold/effectcraft / patch
CLI: run --commands reads {"command", "params"} steps from a JSON or JSON Lines file, or standard input (#2251)
effectcraft-cli run: read the commands from a file or stdin (argv limits on Windows and Linux) ## Summary `effectcraft-cli run <id> <json> [<id> <json> ...]` takes every command as a pair of arguments on the command line. That hits two OS limits: - **Windows:** `CreateProcess` caps the whole command line at 32,767 characters. - **Linux:** a single argument is capped at 128 KiB (`MAX_ARG_STRLEN`), and all arguments plus the environment share `ARG_MAX` (2 MiB). A project built from generated commands runs into these quickly. For example, a text comp with per-character keyframes, a shape layer from `edit.pasteSvg` with a large SVG, or a few dozen comps built in one run. In one of our bu
- Status
- open
- Score
- 0.00
- Adopters
- 0 other operators
- Author
- marcoarnulfo (imported from pull request #2311, open)
- Base
eb5108786b90bf994a6abe4e6c0b2614c6aeaa29- Size
- +197 −8 in 3 files
- Review
- pending
- Shared
- 2026-10-28
- Id
2c1ce193389cf735cf5cec4e
How it went
No other operator has reported using it yet.
Diff
raw · apps/effectcraft-cli/src/main.rs apps/effectcraft-cli/tests/it/cli.rs docs/agents.md
@@ -7,6 +7,7 @@ //! effectcraft-cli exec --list [--filter TEXT] [--schemas] list engine commands (as `commands`) //! effectcraft-cli run <id> <json> [<id> <json> …] run several commands in order //! effectcraft-cli run --args-file F the same, one argument per line of F+//! effectcraft-cli run --commands FILE|- steps {"command", "params"} from FILE or stdin //! effectcraft-cli props <comp> <layer> [--flat] [--time S] a layer's property tree (with paths) //! effectcraft-cli get <comp> <layer> <path> [--time S] read a property //! effectcraft-cli set <comp> <layer> <path> <value> [--time S] [--expression E]@@ -98,6 +99,10 @@ const USAGE: &str = "usage: effectcraft-cli <info|commands|exec|run|props|get|se exec <command-id> [--params JSON] run one engine command (exec --list: list them, as `commands`) run <id> <json> [<id> <json> ...] run several commands in order run --args-file F the same, one argument per line of F (no command-line length limit)+ run --commands FILE|- steps from FILE, or standard input for -: a JSON array of+ {\"command\": ID, \"params\": {...}} (or \"id\" for \"command\"), or one+ such object per line (JSON Lines); at most 64 MiB and 100000 steps.+ Repeatable; run after the <id> <json> pairs and --args-file props <comp> <layer> [--flat] [--time S] a layer's property tree with paths get <comp> <layer> <path> [--time S] read a property set <comp> <layer> <path> <value> [--time S] [--expression E]@@ -151,6 +156,7 @@ const VALUED: &[&str] = &[ "--depth", "--eval", "--args-file",+ "--commands", // render "--template", "--values",@@ -334,6 +340,87 @@ impl From<String> for Failure { } } +/// Most bytes `run --commands` reads from a file or standard input: the steps are untrusted input.+const MAX_COMMANDS_BYTES: u64 = 64 * 1024 * 1024;+/// Most steps one `run --commands` takes.+const MAX_COMMANDS_STEPS: usize = 100_000;++/// The text of `--commands source`: the file `source`, or standard input for `-`, refused past+/// [`MAX_COMMANDS_BYTES`]. This reader, [`parse_commands`] and [`command_step`] are ported from+/// vectorcraft-cli's (storytold/vectorcraft#3630), with a tuple in place of its `Step` enum.+fn read_commands(source: &str) -> Result<String, String> {+ use std::io::Read;+ let mut bytes = vec![];+ let read = if source == "-" {+ std::io::stdin().lock().take(MAX_COMMANDS_BYTES + 1).read_to_end(&mut bytes)+ } else {+ std::fs::File::open(source).and_then(|f| f.take(MAX_COMMANDS_BYTES + 1).read_to_end(&mut bytes))+ };+ read.map_err(|e| format!("--commands {source}: {e}"))?;+ if bytes.len() as u64 > MAX_COMMANDS_BYTES {+ return Err(format!("--commands {source}: more than {} MiB of steps", MAX_COMMANDS_BYTES / (1024 * 1024)));+ }+ String::from_utf8(bytes).map_err(|_| format!("--commands {source}: not UTF-8 text"))+}++/// The steps of `--commands source` from its `text`: a JSON array of `{"command": ID, "params":+/// {…}}` objects (`"id"` may stand for `"command"`), or one such object per line (JSON Lines, blank+/// lines skipped). `first` is the run's index of the first of them, which an error names with the+/// entry's place in the text.+fn parse_commands(source: &str, text: &str, first: usize) -> Result<Vec<(String, Value)>, String> {+ // Notepad on Windows starts UTF-8 files with a byte order mark.+ let text = text.strip_prefix('\u{feff}').unwrap_or(text);+ let too_many = || format!("--commands {source}: more than {MAX_COMMANDS_STEPS} steps");+ // Each entry with where it is in the text: its line for JSON Lines.+ let mut entries: Vec<(Option<usize>, Value)> = vec![];+ if text.trim_start().starts_with('[') {+ let v: Value = serde_json::from_str(text).map_err(|e| format!("--commands {source}: {e}"))?;+ let Value::Array(items) = v else { return Err(format!("--commands {source}: not a JSON array")) };+ if items.len() > MAX_COMMANDS_STEPS {+ return Err(too_many());+ }+ entries.extend(items.into_iter().map(|v| (None, v)));+ } else {+ for (n, line) in text.lines().enumerate().filter(|(_, l)| !l.trim().is_empty()) {+ if entries.len() == MAX_COMMANDS_STEPS {+ return Err(too_many());+ }+ let v = serde_json::from_str(line).map_err(|e| format!("--commands {source}: step {} (line {}): {e}", first + entries.len(), n + 1))?;+ entries.push((Some(n + 1), v));+ }+ }+ entries+ .into_iter()+ .enumerate()+ .map(|(k, (line, v))| {+ command_step(v).map_err(|e| {+ let place = line.map_or_else(|| format!("entry {}", k + 1), |l| format!("line {l}"));+ format!("--commands {source}: step {} ({place}): {e}", first + k)+ })+ })+ .collect()+}++/// The command id and params of one `--commands` entry, `{"command": ID, "params": {…}}`.+fn command_step(v: Value) -> Result<(String, Value), String> {+ let Value::Object(mut o) = v else { return Err("a step must be an object such as {\"command\": \"comp.new\", \"params\": {}}".into()) };+ let id = match (o.remove("command"), o.remove("id")) {+ (Some(Value::String(id)), None) | (None, Some(Value::String(id))) => id,+ (Some(_), Some(_)) => return Err("give `command` or `id`, not both".into()),+ (None, None) => return Err("no `command`".into()),+ _ => return Err("`command` must be a string, a command id".into()),+ };+ let params = match o.remove("params") {+ None | Some(Value::Null) => json!({}),+ Some(p @ Value::Object(_)) => p,+ Some(_) => return Err(format!("`params` of {id} must be a JSON object")),+ };+ if let Some(key) = o.keys().next() {+ return Err(format!("unknown key `{key}`: a step has `command` (or `id`) and `params`"));+ }+ Ok((id, params))+}+ fn usage_err<T>(m: &str) -> Result<T, Failure> { Err(Failure::Usage(m.to_string())) }@@ -524,14 +611,14 @@ fn run(cmd: &str, args: &Args, json_out: bool) -> Result<(), Failure> { let text = std::fs::read_to_string(f).map_err(|e| Failure::Usage(format!("--args-file {f}: {e}")))?; pairs.extend(text.lines().map(str::trim).filter(|l| !l.is_empty()).map(String::from)); }- if pairs.is_empty() {- return usage_err("run needs <command-id> [<json>] pairs");+ let sources = args.opts("--commands");+ if pairs.is_empty() && sources.is_empty() {+ return usage_err("run needs <command-id> [<json>] pairs or --commands FILE|-"); }- let mut b = backend(args, true)?;- let mut results = vec![];+ let mut steps = vec![]; let mut i = 0; while i < pairs.len() {- let id = &pairs[i];+ let id = pairs[i].clone(); let params = match pairs.get(i + 1).filter(|p| p.trim_start().starts_with('{')) { Some(p) => { i += 1;@@ -540,9 +627,26 @@ fn run(cmd: &str, args: &Args, json_out: bool) -> Result<(), Failure> { None => json!({}), }; i += 1;- let r = b.exec(id, params).map_err(|e| Failure::Error(format!("{id}: {e}")))?;- if let Some(e) = failed_render(id, &r) {- return Err(Failure::Error(format!("{id}: {e}")));+ steps.push((id, params));+ }+ if sources.iter().filter(|s| **s == "-").count() > 1 {+ return usage_err("--commands - is given twice: standard input is read once");+ }+ for &source in &sources {+ let text = read_commands(source).map_err(Failure::Usage)?;+ steps.extend(parse_commands(source, &text, steps.len()).map_err(Failure::Usage)?);+ }+ if steps.is_empty() {+ return Err(Failure::Usage(format!("--commands {}: no commands", sources.join(", "))));+ }+ let mut b = backend(args, true)?;+ let mut results = vec![];+ // The step's index closes the message: a generated list repeats ids, and callers match+ // the leading `<id>: `.+ for (n, (id, params)) in steps.into_iter().enumerate() {+ let r = b.exec(&id, params).map_err(|e| Failure::Error(format!("{id}: {e} (step {n})")))?;+ if let Some(e) = failed_render(&id, &r) {+ return Err(Failure::Error(format!("{id}: {e} (step {n})"))); } if !json_out { emit(&r, false);@@ -133,6 +133,78 @@ fn run_reads_arguments_from_a_file() { assert!(String::from_utf8_lossy(&out.stderr).contains("--args-file no/such/file.txt")); } +/// Run with `stdin` piped in; returns (exit code, stdout, stderr).+fn run_with_stdin(args: &[&str], stdin: &str) -> (i32, String, String) {+ let mut child = bin().args(args).stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::piped()).spawn().unwrap();+ child.stdin.take().unwrap().write_all(stdin.as_bytes()).unwrap();+ let out = child.wait_with_output().unwrap();+ (out.status.code().unwrap_or(-1), String::from_utf8_lossy(&out.stdout).into_owned(), String::from_utf8_lossy(&out.stderr).into_owned())+}++/// `run --commands FILE|-` (#2251) reads `{"command", "params"}` steps from a JSON array or JSON+/// Lines, from a file or standard input, and runs them as the same positional pairs would: after+/// the pairs, so a generated list can follow a hand-written prefix. One command whose params are+/// over Linux's 128 KiB per-argument limit goes through.+#[test]+fn run_reads_commands_from_a_file_or_stdin() {+ let steps = [+ ("comp.new", json!({"name": "A", "width": 64, "height": 64})),+ ("layer.newSolid", json!({"width": 8, "height": 8, "name": "One"})),+ ("layer.newSolid", json!({"width": 8, "height": 8, "name": "Two"})),+ ("project.summary", json!({})),+ ];+ let pairs: Vec<String> = steps.iter().flat_map(|(id, p)| [id.to_string(), p.to_string()]).collect();+ let pairs: Vec<&str> = pairs.iter().map(String::as_str).collect();+ let expected = ok_json(&[&["run", "--empty"][..], &pairs[..]].concat());+ let layers = |v: &Value| v[3]["result"]["items"][0]["layers"].as_u64();+ assert_eq!(layers(&expected), Some(2), "{expected}");++ let objects: Vec<Value> = steps.iter().map(|(id, p)| json!({"command": id, "params": p})).collect();+ let array = tmp("commands.json");+ std::fs::write(&array, serde_json::to_string_pretty(&objects).unwrap()).unwrap();+ assert_eq!(ok_json(&["run", "--commands", array.to_str().unwrap(), "--empty"]), expected);++ let lines: String = objects.iter().map(|o| format!("{o}\n\n")).collect();+ let (code, out, err) = run_with_stdin(&["run", "--commands", "-", "--empty", "--json"], &lines);+ assert_eq!(code, 0, "{err}");+ assert_eq!(serde_json::from_str::<Value>(&out).unwrap(), expected);++ // Positional pairs first, then the file.+ let rest = tmp("commands-rest.jsonl");+ std::fs::write(&rest, objects[1..].iter().map(|o| format!("{o}\n")).collect::<String>()).unwrap();+ assert_eq!(ok_json(&[&["run", "--empty", "--commands", rest.to_str().unwrap()][..], &pairs[..2]].concat()), expected);++ let name = "n".repeat(140 * 1024);+ let big = tmp("commands-big.jsonl");+ std::fs::write(&big, format!("{}\n{}\n", json!({"command": "comp.new", "params": {"name": name}}), json!({"command": "project.summary"}))).unwrap();+ let v = ok_json(&["run", "--commands", big.to_str().unwrap(), "--empty"]);+ assert_eq!(v[1]["result"]["items"][0]["name"].as_str().map(str::len), Some(name.len()));++ for (input, args, error) in [+ ("", &["run", "--commands", "-", "--empty"][..], "--commands -: no commands"),+ (+ "{\"command\":\"project.summary\"}\n{\"command\":\"x\",\"param\":{}}\n",+ &["run", "project.summary", "{}", "--commands", "-", "--empty"],+ "step 2 (line 2): unknown key `param`",+ ),+ ("[{\"command\":\"project.summary\"}, {\"params\":{}}]", &["run", "--commands", "-", "--empty"], "step 1 (entry 2): no `command`"),+ ("{}", &["run", "--commands", "-", "--commands", "-", "--empty"], "standard input is read once"),+ ] {+ let (code, out, err) = run_with_stdin(args, input);+ assert_eq!(code, 2, "{input:?}: {err}");+ assert!(err.contains(error), "{input:?}: {err}");+ assert!(out.is_empty(), "{input:?}: nothing ran");+ }+ // An empty list after positional pairs is no error: there is still something to run.+ let (code, _, err) = run_with_stdin(&["run", "project.summary", "--commands", "-", "--empty"], "");+ assert_eq!(code, 0, "{err}");+ // A step that fails when it runs is named by its index too.+ let (code, out, err) = run_with_stdin(&["run", "--commands", "-", "--empty", "--json"], "{\"command\":\"project.summary\"}\n{\"command\":\"no.such\"}\n");+ assert_eq!(code, 1, "{err}");+ let e = serde_json::from_str::<Value>(&out).unwrap()["error"].as_str().map(String::from).unwrap_or_default();+ assert!(e.starts_with("no.such: ") && e.ends_with(" (step 1)"), "{e}");+}+ #[test] fn render_frame_writes_png() { let out = tmp("f.png");@@ -677,6 +677,19 @@ effectcraft-cli exec file.importLottie '{"path":"anim.json"}' main.ecproj --save effectcraft-cli render main.ecproj --comp Main --out main.mov --format prores --progress # progress lines on stderr for a program that reads them ``` +A long `run` doesn't fit on a command line (Windows caps it at 32 767 characters, Linux one+argument at 128 KiB). `run --args-file F` reads the same `<id> <json>` arguments one per line;+`run --commands FILE` reads steps as a JSON array of `{"command": ID, "params": {…}}` objects, or one+such object per line (JSON Lines), and `--commands -` reads them from standard input. `--commands`+may be repeated; its steps run after the positional pairs and `--args-file`. A malformed step is a+usage error naming its index (from 0, as in the results) before anything runs, and the error of a step+that fails ends with `(step N)`:++```sh+effectcraft-cli run --commands steps.json --empty --save-as main.ecproj+generate-steps | effectcraft-cli run main.ecproj --commands - --save --json+```+ Scripts written against After Effects' documented scripting API run with `script`: ```sh