diff --git a/crates/formats/src/latex/tests.rs b/crates/formats/src/latex/tests.rs --- a/crates/formats/src/latex/tests.rs +++ b/crates/formats/src/latex/tests.rs @@ -284,9 +284,9 @@ fn equations_survive_a_round_trip() { "{:?}", bp.objects ); - // Formats without equations keep the text. + // Markdown writes the same LaTeX between dollars. let md = crate::markdown::export(&d); - assert!(md.contains("π r^2"), "{md}"); + assert!(md.contains("Area $\\pi r^2$"), "{md}"); } #[test] diff --git a/crates/formats/src/markdown.rs b/crates/formats/src/markdown.rs --- a/crates/formats/src/markdown.rs +++ b/crates/formats/src/markdown.rs @@ -6,8 +6,11 @@ //! and inlines: emphasis/strong (the CommonMark delimiter algorithm), `~~strike~~`, code spans, //! inline links, autolinks, images (`data:` URIs become pictures, others their alt text), //! backslash escapes, entities, HTML comments (``), `
` and `` anchors. +//! A `$$…$$` block (one line or several) is a display equation, read through the LaTeX math reader. //! -//! Export writes the same constructs; pictures become `![alt](media-name)` placeholders. +//! Export writes the same constructs; pictures become `![alt](media-name)` placeholders, and +//! equations LaTeX: a `$$…$$` line for a display equation alone in its paragraph, `$…$` for any +//! other. use std::collections::HashMap; use std::sync::Arc; diff --git a/crates/formats/src/markdown/blocks.rs b/crates/formats/src/markdown/blocks.rs --- a/crates/formats/src/markdown/blocks.rs +++ b/crates/formats/src/markdown/blocks.rs @@ -385,6 +385,38 @@ fn code_para(line: &str) -> FBlock { FBlock::Para(p) } +/// A display equation opening at `lines[i]` (`$$x$$`, or `$$` lines around LaTeX), as linear +/// text, and the index after its last line. The first later line holding `$$` must close it; +/// a blank line first (TeX math has no paragraph breaks), a stray `$$` or nothing to convert +/// leaves the lines as text. Each search stops at the next opener, so a run of unclosed +/// openers is read once. +fn display_math(lines: &[String], i: usize) -> Option<(String, usize)> { + let first = lines.get(i)?.trim_start_matches([' ', '\t']).strip_prefix("$$")?.trim_end(); + let mut src = first.to_string(); + let mut next = i + 1; + match first.strip_suffix("$$") { + Some(one) => src = one.to_string(), + None => loop { + let l = lines.get(next).filter(|l| !is_blank(l))?; + next += 1; + src.push('\n'); + match l.trim().strip_suffix("$$") { + Some(last) => { + src.push_str(last); + break; + } + None if l.contains("$$") => return None, + None => src.push_str(l), + } + }, + } + if src.contains("$$") { + return None; + } + let linear = crate::latex::latex_to_linear(&src); + (!linear.trim().is_empty()).then_some((linear, next)) +} + /// Parse block structure. `level` is the list nesting level for lists found here. fn parse_blocks(lines: &[String], depth: usize, level: u8, refs: &Refs) -> Vec { let mut out = Vec::new(); @@ -463,6 +495,13 @@ fn parse_blocks(lines: &[String], depth: usize, level: u8, refs: &Refs) -> Vec String { let mut out = String::with_capacity(s.len() + 8); - for c in s.chars() { + let mut chars = s.chars().peekable(); + while let Some(c) = chars.next() { match c { '\\' | '`' | '*' | '_' | '[' | ']' | '<' | '>' | '~' | '|' | '&' => { out.push('\\'); out.push(c); } + // `$$` opens a display equation; a single `$` is left as typed. + '$' if chars.peek() == Some(&'$') => out.push_str("\\$"), '\u{000C}' | '\u{000E}' => {} _ => out.push(c), } @@ -89,19 +92,20 @@ enum Mark { /// Inline spans → Markdown with properly nested markers. pub(super) fn inlines_md(inlines: &[Inline]) -> String { - inlines_md_impl(inlines, false) + inlines_md_impl(inlines, false, true) } fn inlines_md_table(inlines: &[Inline]) -> String { - inlines_md_impl(inlines, true) + inlines_md_impl(inlines, true, false) } -fn inlines_md_impl(inlines: &[Inline], in_table: bool) -> String { +/// `own_line`: the inlines fill a line that a `$$` display block can stand on. +fn inlines_md_impl(inlines: &[Inline], in_table: bool, own_line: bool) -> String { // Split into (text, fmt) pieces with whitespace moved outside emphasis. let mut pieces: Vec<(String, Fmt, Option)> = Vec::new(); - let eq = crate::model::equations_as_text(inlines); // Markdown keeps no ruby: its base text goes through the text path below. - let inlines = crate::model::ruby_as_text(&eq); + let inlines = crate::model::ruby_as_text(inlines); + let alone = own_line && inlines.len() == 1; for i in inlines.iter() { match i { Inline::Text(t, f) => { @@ -133,7 +137,19 @@ fn inlines_md_impl(inlines: &[Inline], in_table: bool) -> String { pieces.push((String::new(), Fmt::default(), Some(format!("")))); } Inline::Figure(alt) => pieces.push((String::new(), Fmt::default(), Some(format!("![{}]()", escape(alt))))), - Inline::Equation { .. } => {} + Inline::Equation { linear, display } => { + let latex = crate::latex::linear_to_latex(linear).replace(['\r', '\n'], " "); + let latex = latex.trim(); + if latex.is_empty() { + pieces.push((linear.clone(), Fmt::default(), None)); + } else if *display && alone { + pieces.push((String::new(), Fmt::default(), Some(format!("$${latex}$$")))); + } else { + // Only a body line of its own can be a `$$` block; a bare `|` would end the cell. + let latex = if in_table { latex.replace('|', "\\|") } else { latex.to_string() }; + pieces.push((String::new(), Fmt::default(), Some(format!("${latex}$")))); + } + } // Read as its base text by `ruby_as_text` above. Inline::Ruby { .. } => {} } @@ -146,6 +162,8 @@ fn inlines_md_impl(inlines: &[Inline], in_table: bool) -> String { out.push_str(marker(m)); } }; + // Whether `out` ends with raw output, such as an equation's closing `$`. + let mut after_raw = false; for (text, f, raw) in pieces { // Links are the outermost construct; a script or other non-web target is plain text. let f_link = f.link.clone().filter(|l| wordcraft_doc::link::is_safe_href(l)); @@ -156,7 +174,13 @@ fn inlines_md_impl(inlines: &[Inline], in_table: bool) -> String { } link = None; if let Some(r) = raw { + // A text `$` before an opening `$` would make `$$`. + if r.starts_with('$') && !after_raw && out.ends_with('$') { + out.pop(); + out.push_str("\\$"); + } out.push_str(&r); + after_raw = true; continue; } if let Some(l) = f_link { @@ -188,8 +212,14 @@ fn inlines_md_impl(inlines: &[Inline], in_table: bool) -> String { out.push_str(&code_span(&text)); } } else { - out.push_str(&escape(&text).replace('\n', "\\\n").replace('\t', " ")); + let t = escape(&text).replace('\n', "\\\n").replace('\t', " "); + // After an equation or another span's `$`, which a span split by underline can leave. + if out.ends_with('$') && t.starts_with('$') { + out.push('\\'); + } + out.push_str(&t); } + after_raw = false; } close_all(&mut out, &mut stack); if let Some(l) = &link { @@ -334,7 +364,11 @@ pub(super) fn write_blocks(blocks: &[FBlock], out: &mut Vec) { continue; } FBlock::Para(p) => { - let body = inlines_md(&p.inlines); + // A `#` line is read before `$$` blocks, so a heading writes its equations `$…$`. + let body = match p.kind { + Kind::Heading(_) | Kind::Title => inlines_md_impl(&p.inlines, false, false), + _ => inlines_md(&p.inlines), + }; match p.list { Some(li) => { let lv = li.level.min(8) as usize; diff --git a/crates/formats/src/markdown/tests.rs b/crates/formats/src/markdown/tests.rs --- a/crates/formats/src/markdown/tests.rs +++ b/crates/formats/src/markdown/tests.rs @@ -585,3 +585,130 @@ fn exported_link_destinations_read_back_unchanged() { assert_eq!(md("https://example.com/a b"), "[x]()"); assert_eq!(md("https://e.com/x\ny"), "[x](https://e.com/xy)"); } + +/// The equations in a flow's paragraphs: linear text and whether each is displayed. +fn equations(flow: &model::Flow) -> Vec<(String, bool)> { + flow.blocks + .iter() + .filter_map(|b| if let FBlock::Para(p) = b { Some(p) } else { None }) + .flat_map(|p| &p.inlines) + .filter_map(|i| if let Inline::Equation { linear, display } = i { Some((linear.clone(), *display)) } else { None }) + .collect() +} + +fn para_texts(flow: &model::Flow) -> Vec { + flow.blocks.iter().filter_map(|b| if let FBlock::Para(p) = b { Some(p.text()) } else { None }).collect() +} + +/// `$$…$$` blocks are display equations, on one line or several, and a `$$` line ends the +/// paragraph before it, as in "Display equation:" followed by the formula (#4130). +#[test] +fn display_math_blocks_import_as_display_equations() { + let flow = parse("$$\\alpha \\rightarrow \\beta$$"); + assert_eq!(equations(&flow), [("α → β".to_string(), true)]); + assert_eq!(flow.blocks.len(), 1); + for src in [" $$\n\\frac{a}{b}\n$$", "$$\\frac{a}\n{b}$$", "$$\n\\frac{a}{b} $$ "] { + assert_eq!(equations(&parse(src)), [("a/b".to_string(), true)], "{src:?}"); + } + let oint = "\\oint_C \\mathbf{B} \\cdot d\\mathbf{l} = \\mu_0 \\left( \\mathbf{I}_{\\text{enc}} + \\varepsilon_0 \\frac{d\\Phi_E}{dt} \\right)"; + let linear = crate::latex::latex_to_linear(oint); + let flow = parse(&format!("Display equation:\n$${oint}$$\nafter")); + assert_eq!(para_texts(&flow), ["Display equation:".to_string(), linear.clone(), "after".to_string()]); + assert_eq!(equations(&flow), [(linear, true)]); + for src in ["> $$x^2$$", "- $$x^2$$"] { + assert_eq!(equations(&parse(src)), [("x^2".to_string(), true)], "{src:?}"); + } +} + +/// What can't be a display block stays text: a blank line before the closing `$$`, a stray +/// `$$` on a line, text after the closing one, nothing to convert, or a code block. +#[test] +fn display_math_that_cannot_be_read_stays_text() { + for (src, texts) in [ + ("$$x\n\ny$$", vec!["$$x", "y$$"]), + ("$$$$", vec!["$$$$"]), + ("$$ $$", vec!["$$ $$"]), + ("$$a $$ b $$", vec!["$$a $$ b $$"]), + ("$$x$$ (1)", vec!["$$x$$ (1)"]), + ("$$\\text{}$$", vec!["$$\\text{}$$"]), + ("$$a\nb $$ c\nd$$", vec!["$$a b $$ c d$$"]), + ("```\n$$x$$\n```", vec!["$$x$$"]), + ] { + let flow = parse(src); + assert!(equations(&flow).is_empty(), "{src:?}"); + assert_eq!(para_texts(&flow), texts, "{src:?}"); + } +} + +#[test] +fn a_run_of_unclosed_display_openers_parses_in_linear_time() { + // Each search for the closing `$$` stops at the next line holding one; searching on to + // the end of the paragraph from every opener was quadratic. + let src = "$$a\n".repeat(200_000); + let t = std::time::Instant::now(); + let flow = parse(&src); + assert!(t.elapsed() < std::time::Duration::from_secs(10), "{:?}", t.elapsed()); + assert!(equations(&flow).is_empty()); + assert_eq!(para_texts(&flow).len(), 1); +} + +/// Equations are written as LaTeX between dollars: a `$$…$$` line for a display equation +/// alone in its paragraph, `$…$` for any other (a heading's too), with escaped bars in a +/// table cell. Before, Markdown wrote their linear text and reopening the file lost them (#4130). +#[test] +fn equations_export_as_latex_and_display_lines_read_back() { + let eq = |linear: &str, display: bool| Inline::Equation { linear: linear.into(), display }; + let para = |inlines: Vec| FBlock::Para(Para { inlines, ..Default::default() }); + let text = |t: &str| Inline::Text(t.into(), Fmt::default()); + let cell = Cell { blocks: vec![para(vec![eq("|x|", true)])], ..Default::default() }; + let flow = model::Flow { + blocks: vec![ + para(vec![text("Energy "), eq("E = mc^2", false), text(".")]), + para(vec![eq("α → β", true)]), + para(vec![text("so "), eq("x", true)]), + FBlock::Para(Para { kind: Kind::Heading(2), inlines: vec![eq("y", true)], ..Default::default() }), + FBlock::Table(FTable { rows: vec![vec![cell], vec![Cell::default()]], ..Default::default() }), + ], + ..Default::default() + }; + let md = export(&model::to_doc(&flow)); + let lines: Vec<&str> = md.lines().collect(); + for line in ["Energy $E = mc^2$.", "$$\\alpha \\to \\beta$$", "so $x$", "## $y$", "| $\\|x\\|$ |"] { + assert!(lines.contains(&line), "{line:?} in {md:?}"); + } + let back = model::from_doc(&import(&md)); + assert!(equations(&back).contains(&("α → β".to_string(), true)), "{md:?}"); + let Some(FBlock::Table(t)) = back.blocks.iter().find(|b| matches!(b, FBlock::Table(_))) else { panic!("{md:?}") }; + assert!(t.rows.iter().all(|r| r.len() == 1), "{md:?}"); + // Nothing to write as LaTeX: the linear text stays. + assert_eq!(inlines_md(&[eq(" ", false)]), " "); +} + +/// Text never writes `$$`, which would read back as a display equation: its first `$` is +/// escaped. A single `$` stays as typed, so inline math a file still holds as text is saved +/// as it came. +#[test] +fn text_dollars_never_read_back_as_display_math() { + for (text, written) in [("Pay $19.50 and $21.00", "Pay $19.50 and $21.00"), ("$$x$$", "\\$$x\\$$"), ("$$$ and $", "\\$\\$$ and $")] { + let p = Para { inlines: vec![Inline::Text(text.into(), Fmt::default())], ..Default::default() }; + let md = export(&model::to_doc(&model::Flow { blocks: vec![FBlock::Para(p)], ..Default::default() })); + assert_eq!(md.trim_end(), written); + let back = model::from_doc(&import(&md)); + assert!(equations(&back).is_empty(), "{md:?}"); + assert_eq!(para_texts(&back), [text], "{md:?}"); + } + // Nor does a text `$` next to an equation's delimiter or another span's `$`. + let text = |t: &str| Inline::Text(t.into(), Fmt::default()); + let eq = |display: bool| Inline::Equation { linear: "x".into(), display }; + for (inlines, written) in [ + (vec![text("$"), eq(false), text("$")], "\\$$x$\\$"), + (vec![text("a $"), eq(true)], "a \\$$x$"), + (vec![text("$"), text("$x$"), text("$")], "$\\$x$\\$"), + ] { + let md = inlines_md(&inlines); + assert_eq!(md, written); + assert!(equations(&model::from_doc(&import(&md))).is_empty(), "{md:?}"); + } + let src = "The relation $E = mc^2$ holds, at $5 a go."; + assert_eq!(export(&import(src)), format!("{src}\n")); +} diff --git a/docs/file-format-parity.md b/docs/file-format-parity.md --- a/docs/file-format-parity.md +++ b/docs/file-format-parity.md @@ -63,7 +63,7 @@ Weights are the share of a Word user's file traffic (estimate). Read/write: ✅ | `.xlsx`/`.xlsm`/`.xls`/`.ods` as a mail-merge recipient list | — | R (data source) | ✅ | — | ~90% | 1–3 | `crates/formats/src/sheet.rs`, `cmd/mailings.rs` | Read-only, one sheet's cell values (#334): shared/inline/rich strings, numbers, booleans, cached formula results, dates and times as ISO 8601 (`yyyy-mm-dd`) from the cell's number format, sparse cells; ODS value types, repeated rows/columns, covered cells. Select Table picks the sheet. Capped (64 MiB per part, 100,000 rows, 1,000 columns, 2 M cells, 32,767 characters a cell, 64 MiB of cell text a sheet so a shared string repeated in every cell can't exhaust memory, #1287). Legacy `.xls` (Excel 97–2003 BIFF8, #778): SST/LABELSST, LABEL, NUMBER, RK/MULRK, BOOLERR and cached FORMULA results with the same date handling; Excel 5.0/95 and password-protected `.xls` are refused with a message; a CSV/TSV named `.xls` reads as text; an HTML page or XML spreadsheet named `.xls` gets a message saying which it is (#1287). Text lists (.csv/.tsv/.txt) decode UTF-8/UTF-16 by BOM or shape, else the detected legacy encoding (GBK/GB18030, Big5, Shift_JIS, EUC-KR, Windows code pages; `encoding` param overrides) (#778). Not shown: number formats other than dates (currency, percent, thousands separators); no Access or Outlook sources | | Encrypted/password-protected `.docx` | 1% | R/W | ✅ | ✅ | ~60% | 1–2 | `crates/docx/src/crypt.rs` (9 unit tests), `tests/malformed.rs`, `crates/engine/src/tests.rs::password_protected_documents_round_trip`, `aes_encrypted_packages_open_under_any_word_extension` | ECMA-376 / [MS-OFFCRYPTO] agile encryption (AES-256-CBC, SHA-512): a protected `.docx` opens with its password and saves encrypted (#55, #281). Standard encryption (Word 2007: AES-128/192/256-ECB, SHA-1, 50,000 spins) opens too, and either kind opens when named `.doc`/`.dot` (gap 9); saving stays agile. Synthetic files only; never checked against Word-made protected files | | Encrypted/password-protected `.doc` (XOR/RC4) | <1% | R | 🟡 | — | ~60% | 1–2 | `crates/docbin/src/tests/encrypted.rs` | XOR obfuscation, RC4 and RC4 CryptoAPI open with a password (`docbin` `crypt.rs`, #1079). An AES header (`fAES`) is refused with a clear message: [MS-OFFCRYPTO] defines AES only for encrypted packages, which open through the `.docx` reader (gap 9) | -| `.md` Markdown | — | (no) | ✅ | ✅ | — | — | `crates/formats/src/tests.rs` | Beyond Word | +| `.md` Markdown | — | (no) | ✅ | ✅ | — | — | `crates/formats/src/tests.rs`, `crates/formats/src/markdown.rs` unit tests | Beyond Word. A `$$…$$` block opens as a display equation; equations save as LaTeX, a `$$…$$` line for a display equation alone in its paragraph and `$…$` for any other, and a `$$` in text is written `\$$` (#4130). Inline `$…$` doesn't open as an equation yet | | `.tex` LaTeX | — | — | ✅ | ✅ | — | — | `crates/formats/src/tests.rs` | Beyond Word (#11), incl. math | | Pictures for Insert › Picture, drag-and-drop and paste | — | R | ✅ | — | — | — | `insert_picture_takes_a_windows_metafile` (engine), `paste_picture.rs` | PNG, JPEG, GIF, WebP, BMP, and Windows Metafile `.wmf` / Enhanced Metafile `.emf` at their own frame size (#802), drawn by `crates/metafile`. Not yet: SVG, TIFF, ICO, HEIC | | `.png` page images | — | (Save as picture, Mac) | — | ✅ | — | — | `io_ext.rs` | |