collective
Browse / github.com/storytold/pdfcraft / patch
fix(xfa): draw Romanian and Polish letters in XFA-to-static output instead of '?'
fix(xfa): draw Romanian and Polish letters in XFA-to-static output instead of '?' Refs #3833 (Romanian character support). ## What `xfa_to_static` lays XFA text out in the standard-14 fonts with `/Encoding /WinAnsiEncoding`, and `win_ansi()` turns every other character into `?`. Romanian (ă Ă ș Ș ț Ț ş Ş ţ Ţ) and Polish (ą Ą ę Ę ł Ł ś Ś ż Ż ź Ź ć Ć ń Ń) forms therefore lose those letters in labels, field values and the extracted text. Romanian ANAF tax declarations (dynamic LiveCycle XFA) come out as "DECLARA?IE PRIVIND OBLIGA?IILE DE PLAT?", while their layout and values are otherwise excellent. ## How - `pdfcraft_fonts::LATIN_EXTRA`: 26 letters at control codes WinAns
- Status
- open
- Score
- 0.00
- Adopters
- 0 other operators
- Author
- GigiG1 (imported from pull request #4890, open)
- Base
1dd2737b47a9c1c9562f0af563053d1bf510e4bf- Size
- +230 −10 in 7 files
- Review
- pending
- Shared
- 2026-10-28
- Id
e06bcf8cc01c7c4bd7233fd4
How it went
No other operator has reported using it yet.
Diff
raw · crates/fonts/src/lib.rs crates/fonts/src/pdf.rs crates/fonts/src/pdf/tests.rs crates/fonts/src/std14.rs crates/xfa/src/pdf.rs crates/xfa/src/pdf/common.rs crates/xfa/src/text.rs
@@ -159,6 +159,67 @@ pub fn win_ansi(s: &str) -> Vec<u8> { s.chars().map(|c| if c == '\t' { b' ' } else { win_ansi_byte(c).unwrap_or(b'?') }).collect() } +/// Latin letters outside WinAnsiEncoding that Romanian and Polish forms use, as (character, code,+/// glyph name, base letter). ț Ț are named `uni021B`/`uni021A`: the glyph list maps+/// `tcommaaccent`/`Tcommaaccent` to the cedilla letters U+0163/U+0162, so text extracted from+/// those names would read ţ Ţ. XFA layout draws text in the standard-14 fonts; their+/// `/Encoding` adds these through `/Differences` at control codes WinAnsiEncoding leaves+/// undefined (0x01-0x08, 0x0B, 0x0E-0x1F), so no WinAnsi character is displaced. 0x09, 0x0A,+/// 0x0C and 0x0D are skipped: viewers treat tab and line feed as white space when drawing, a+/// literal string turns an unescaped carriage return into a line feed, and form feed separates+/// pages in extracted text. In the standard-14 metrics each glyph has its base letter's advance width.+pub const LATIN_EXTRA: &[(char, u8, &str, char)] = &[+ ('ă', 0x01, "abreve", 'a'),+ ('Ă', 0x02, "Abreve", 'A'),+ ('ș', 0x03, "scommaaccent", 's'),+ ('Ș', 0x04, "Scommaaccent", 'S'),+ ('ț', 0x05, "uni021B", 't'),+ ('Ț', 0x06, "uni021A", 'T'),+ ('ş', 0x07, "scedilla", 's'),+ ('Ş', 0x08, "Scedilla", 'S'),+ ('ţ', 0x0b, "tcedilla", 't'),+ ('Ţ', 0x0e, "Tcedilla", 'T'),+ ('ą', 0x0f, "aogonek", 'a'),+ ('Ą', 0x10, "Aogonek", 'A'),+ ('ę', 0x11, "eogonek", 'e'),+ ('Ę', 0x12, "Eogonek", 'E'),+ ('ł', 0x13, "lslash", 'l'),+ ('Ł', 0x14, "Lslash", 'L'),+ ('ś', 0x15, "sacute", 's'),+ ('Ś', 0x16, "Sacute", 'S'),+ ('ż', 0x17, "zdotaccent", 'z'),+ ('Ż', 0x18, "Zdotaccent", 'Z'),+ ('ź', 0x19, "zacute", 'z'),+ ('Ź', 0x1a, "Zacute", 'Z'),+ ('ć', 0x1b, "cacute", 'c'),+ ('Ć', 0x1c, "Cacute", 'C'),+ ('ń', 0x1d, "nacute", 'n'),+ ('Ń', 0x1e, "Nacute", 'N'),+];++/// Encode text in WinAnsiEncoding plus [`LATIN_EXTRA`]: those letters become their codes, the+/// rest as [`win_ansi`] encodes them (tabs become spaces, unmappable characters `?`).+pub fn win_ansi_extra(s: &str) -> Vec<u8> {+ s.chars()+ .map(|c| match LATIN_EXTRA.iter().find(|e| e.0 == c) {+ Some(e) => e.1,+ None if c == '\t' => b' ',+ None => win_ansi_byte(c).unwrap_or(b'?'),+ })+ .collect()+}++/// The base letter a [`LATIN_EXTRA`] character is measured as, if `c` is one.+pub fn extra_base_letter(c: char) -> Option<char> {+ LATIN_EXTRA.iter().find(|e| e.0 == c).map(|e| e.3)+}++/// [`first_undrawable`] for text written with [`win_ansi_extra`]: a [`LATIN_EXTRA`] letter is+/// drawn, so only what that encoding still writes as `?` is reported.+pub fn first_undrawable_extra(s: &str) -> Option<char> {+ s.chars().find(|c| !matches!(c, '\n' | '\r') && !win_ansi_encodable(*c) && extra_base_letter(*c).is_none())+}+ /// The character encoding of a predefined Unicode CMap (`UniJIS-UTF16-H`, `UniGB-UCS2-H`, …; /// ISO 32000-2 §9.7.5.2), which maps Unicode text straight to codes. #[derive(Clone, Copy, Debug, PartialEq, Eq)]@@ -293,6 +354,45 @@ pub fn literal(bytes: &[u8]) -> Vec<u8> { mod tests { use super::*; + #[test]+ fn win_ansi_extra_encodes_romanian_and_polish_letters_without_question_marks() {+ let bytes = win_ansi_extra("Declarație ș Łódź");+ assert_eq!(&bytes[..7], win_ansi("Declara").as_slice());+ assert_eq!(bytes[7], 0x05, "ț");+ assert_eq!(bytes[11], 0x03, "ș");+ assert!(!bytes.contains(&b'?'));+ // ó is WinAnsi (0xF3) and stays so; Ł and ź come from the extra table+ assert_eq!(&bytes[13..], &[0x14, 0xf3, b'd', 0x19]);+ }++ #[test]+ fn win_ansi_extra_leaves_winansi_text_as_win_ansi_writes_it() {+ let text = "AaBb 0123456789 !\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~ âîé€ Š\t";+ assert_eq!(win_ansi_extra(text), win_ansi(text));+ // still `?` for what neither table holds+ assert_eq!(win_ansi_extra("Ω"), b"?".to_vec());+ }++ #[test]+ fn latin_extra_codes_are_distinct_control_codes_and_never_white_space() {+ let mut seen = std::collections::HashSet::new();+ for e in LATIN_EXTRA {+ assert!((0x01..0x20).contains(&e.1), "{:?} code {:#x}", e.0, e.1);+ assert!(![0x09, 0x0a, 0x0c, 0x0d].contains(&e.1), "{:?} on a white-space code {:#x}", e.0, e.1);+ assert!(seen.insert(e.1), "code {:#x} used twice", e.1);+ assert!(win_ansi_byte(e.0).is_none(), "{:?} is already WinAnsi", e.0);+ assert_eq!(extra_base_letter(e.0), Some(e.3));+ }+ }++ #[test]+ fn first_undrawable_extra_skips_latin_extra_letters_only() {+ assert_eq!(first_undrawable_extra("Declarație ș Łódź\r\n"), None);+ assert_eq!(first_undrawable_extra("Obligații Ω"), Some('Ω'), "the first truly undrawable, not 'ț'");+ assert_eq!(first_undrawable("Obligații Ω"), Some('ț'));+ assert_eq!(first_undrawable_extra("签名"), Some('签'));+ }+ #[test] fn wrap_keeps_leading_spaces() { assert_eq!(wrap(" indented", 11.0, 500.0), [" indented"]);@@ -833,13 +833,20 @@ impl Metrics { /// Where the standard-14 table, indexed by WinAnsi, holds the glyph `code` draws in this font: /// its encoding (base and `/Differences`) names the glyph, so a remapped code is measured as- /// the glyph it now shows (#2894). Symbol and ZapfDingbats are indexed by their own encodings,- /// and a glyph WinAnsi has no code for keeps the code itself.+ /// the glyph it now shows (#2894). A [`crate::LATIN_EXTRA`] letter is measured as its base+ /// letter, as [`crate::Std14::text_width_extra`] lays it out. Symbol and ZapfDingbats are+ /// indexed by their own encodings, and a glyph WinAnsi has no code for keeps the code itself. fn std14_code(&self, face: crate::Std14, code: u32) -> u32 { if matches!(face, crate::Std14::Symbol | crate::Std14::ZapfDingbats) { return code; }- self.unicode.encoding.get(&code).and_then(|t| only_char(t)).and_then(crate::win_ansi_byte).map_or(code, u32::from)+ self.unicode+ .encoding+ .get(&code)+ .and_then(|t| only_char(t))+ .map(|c| crate::extra_base_letter(c).unwrap_or(c))+ .and_then(crate::win_ansi_byte)+ .map_or(code, u32::from) } /// Word spacing applies to the single-byte code 32 (§9.3.3).@@ -1207,6 +1214,11 @@ fn glyph_component(part: &str) -> Option<String> { if let Ok(i) = crate::encodings::NAMES.binary_search_by(|(n, _)| (*n).cmp(part)) { return char::from_u32(crate::encodings::NAMES[i].1).map(String::from); }+ // The Romanian and Polish letters PdfCraft's XFA output names in its `/Differences`+ // (`abreve`, `scommaaccent`, …), which no table above holds.+ if let Some(e) = crate::LATIN_EXTRA.iter().find(|e| e.2 == part) {+ return Some(e.0.to_string());+ } // `from_str_radix` accepts a leading `+`, which is not a glyph name, so the digits are checked. let hexish = |h: &str| !h.is_empty() && h.bytes().all(|b| b.is_ascii_hexdigit()); if let Some(hex) = part.strip_prefix("uni").filter(|h| h.len() >= 4 && h.len() % 4 == 0 && hexish(h)) {@@ -289,6 +289,27 @@ fn standard_fonts_without_widths_measure_the_glyph_their_differences_name() { assert_eq!(Metrics::from_dict(&doc, &f).width(87), 0.5, "/Widths go by code"); } +/// The encoding PdfCraft's XFA output gives the standard fonts (`LATIN_EXTRA` in `/Differences`)+/// reads back: a field appearance in such a `/DR` font encodes ă, ș, ț instead of falling back to+/// Helvetica and `?`, and measures each as its base letter.+#[test]+fn latin_extra_differences_encode_decode_and_measure_as_the_base_letter() {+ let mut doc = Document::new_empty();+ let mut enc = Dict::new();+ enc.set(b"BaseEncoding".to_vec(), Object::name("WinAnsiEncoding"));+ let diffs = crate::LATIN_EXTRA.iter().flat_map(|e| [Object::Int(i64::from(e.1)), Object::name(e.2)]).collect();+ enc.set(b"Differences".to_vec(), Object::Array(diffs));+ let f = font(&mut doc, vec![("Subtype", Object::name("Type1")), ("BaseFont", Object::name("Helvetica")), ("Encoding", Object::Dict(enc))]);+ let m = Metrics::from_dict(&doc, &f);+ for e in crate::LATIN_EXTRA {+ assert_eq!(glyph_unicode(e.2), Some(e.0), "{}", e.2);+ assert_eq!(m.encode(&e.0.to_string()), Some(vec![e.1]), "{:?}", e.0);+ assert_eq!(m.width(u32::from(e.1)), m.width(u32::from(e.3 as u8)), "{:?} as {:?}", e.0, e.3);+ }+ let text = "Declarație ŞI Łódź";+ assert_eq!(m.decode(&m.encode(text).expect("all encodable")), text);+}+ #[test] fn subset_fonts_only_encode_the_glyphs_they_have() { let mut doc = Document::new_empty();@@ -110,6 +110,14 @@ impl Std14 { crate::win_ansi(text).iter().map(|&c| self.width(c)).sum::<f64>() * size / 1000.0 } + /// [`Std14::text_width`] for text written with [`crate::win_ansi_extra`]: a+ /// [`crate::LATIN_EXTRA`] letter is measured as its base letter, whose advance it shares in+ /// the standard-14 metrics (abreve = a, scommaaccent = s, lslash = l, …).+ pub fn text_width_extra(self, text: &str, size: f64) -> f64 {+ let base: String = text.chars().map(|c| crate::extra_base_letter(c).unwrap_or(c)).collect();+ self.text_width(&base, size)+ }+ /// Courier is the only fixed-pitch family of the fourteen. pub fn is_fixed_pitch(self) -> bool { matches!(self, Self::Courier | Self::CourierBold | Self::CourierOblique | Self::CourierBoldOblique)@@ -125,6 +133,15 @@ impl Std14 { mod tests { use super::*; + #[test]+ fn extra_letters_measure_as_their_base_letters() {+ for font in [Std14::Helvetica, Std14::TimesBold, Std14::Courier] {+ assert_eq!(font.text_width_extra("ș", 12.0), font.text_width("s", 12.0));+ assert_eq!(font.text_width_extra("Declarație", 10.0), font.text_width("Declaratie", 10.0));+ assert_eq!(font.text_width_extra("plain", 10.0), font.text_width("plain", 10.0));+ }+ }+ #[test] fn sans_serif_names_are_not_mapped_to_times() { assert!(!matches!(Std14::recognise("SansSerif"), Some((Std14::TimesRoman, _))));@@ -5,7 +5,7 @@ use std::collections::{BTreeMap, HashMap, HashSet}; use pdfcraft_cos::{Dict, Document, ObjRef, Object, PdfString, Stream};-use pdfcraft_fonts::{literal, win_ansi};+use pdfcraft_fonts::{literal, win_ansi_extra}; use crate::XfaError; use crate::layout::{Action, BorderShape, Form, Item, Page, Rounded, Widget, WidgetKind};@@ -17,7 +17,10 @@ mod strip; pub use common::jpeg_size; pub(crate) use common::rounded;-use common::{all_faces, content_hash, empty_form, hex32, jpeg_info, js_action, n, parse_hex32, rgb, rgb_array, standard_font, state_name};+use common::{+ all_faces, content_hash, empty_form, hex32, is_plain_standard_font, jpeg_info, js_action, n, parse_hex32, rgb, rgb_array, standard_font,+ state_name,+}; pub(crate) use strip::USAGE_RIGHTS_REMOVED; use strip::drop_usage_rights; pub use strip::{Stripped, XfaShape, existing_layout, strip, xfa_shape};@@ -623,7 +626,7 @@ impl Emitter<'_> { Item::Text(s) => { // The form's own text, set in the standard fonts: what they can't draw is // shown as `?` and reported once (the layout goes on), never silently.- if let Some(c) = pdfcraft_fonts::first_undrawable(&s.text) {+ if let Some(c) = pdfcraft_fonts::first_undrawable_extra(&s.text) { const PREFIX: &str = "some of the form's text has characters the standard fonts can't draw"; if !self.warnings.iter().any(|w| w.starts_with(PREFIX)) { self.warnings.push(format!(@@ -644,7 +647,7 @@ impl Emitter<'_> { format!("{open} /{name} {} Tf {} rg{tz} 1 0 0 1 {} {} Tm ", n(s.face.size), rgb(s.face.color), n(s.x), n(h - s.baseline)) .bytes(), );- content.extend(literal(&win_ansi(&s.text)));+ content.extend(literal(&win_ansi_extra(&s.text))); content.extend_from_slice(format!(" Tj{close}\n").as_bytes()); if let Some(r) = self.text_link(s, h, page_ref, &mut last_link)? { annots.push(Object::Ref(r));@@ -897,7 +900,10 @@ pub fn write_form(doc: &mut Document, form: &Form) -> Result<Written, XfaError> let mut dr = acro.get(b"DR").map(|d| doc.resolve(d)).and_then(|d| d.as_dict().cloned()).unwrap_or_default(); let mut dr_fonts = dr.get(b"Font").map(|f| doc.resolve(f)).and_then(|f| f.as_dict().cloned()).unwrap_or_default(); for (name, base) in all_faces() {- if !dr_fonts.contains(name.as_bytes()) {+ // The form's own font under one of these names is kept unless it is the plain WinAnsi+ // standard font, whose field values would otherwise draw ă or ș as `?`.+ let plain = dr_fonts.get(name.as_bytes()).and_then(|f| doc.resolve(f).as_dict().map(|f| is_plain_standard_font(f, base)));+ if plain.unwrap_or(true) { dr_fonts.set(name.as_bytes().to_vec(), standard_font(base)); } }@@ -949,6 +955,48 @@ pub fn pdf_rect(r: Rect, page_height: f64) -> [f64; 4] { mod tests { use super::parse_hex32; + #[test]+ fn standard_fonts_name_the_extra_latin_letters_in_their_encoding() {+ let font = super::standard_font("Helvetica");+ let enc = font.as_dict().and_then(|f| f.get(b"Encoding")).and_then(|e| e.as_dict()).expect("an /Encoding dictionary");+ assert_eq!(enc.get(b"BaseEncoding").and_then(|b| b.as_name()), Some(&b"WinAnsiEncoding"[..]));+ let diffs = enc.get(b"Differences").and_then(|d| d.as_array()).expect("/Differences");+ assert_eq!(diffs.len(), pdfcraft_fonts::LATIN_EXTRA.len() * 2);+ assert_eq!((diffs[0].as_int(), diffs[1].as_name()), (Some(1), Some(&b"abreve"[..])));+ assert_eq!((diffs[4].as_int(), diffs[5].as_name()), (Some(3), Some(&b"scommaaccent"[..])));+ // ț by its code point: `tcommaaccent` means U+0163 ţ in the glyph list+ assert_eq!((diffs[8].as_int(), diffs[9].as_name()), (Some(5), Some(&b"uni021B"[..])));+ // ţ (cedilla) must not sit on tab or line feed, where viewers draw nothing+ let tcedilla = diffs.chunks(2).find(|p| p[1].as_name() == Some(&b"tcedilla"[..])).and_then(|p| p[0].as_int());+ assert!(tcedilla.is_some_and(|c| ![9, 10, 12, 13].contains(&c)));+ }++ #[test]+ fn only_plain_standard_dr_fonts_take_the_extra_latin_encoding() {+ use pdfcraft_cos::{Dict, Object};+ let plain = |base: &str| {+ let mut f = Dict::new();+ f.set(b"Type".to_vec(), Object::name("Font"));+ f.set(b"Subtype".to_vec(), Object::name("Type1"));+ f.set(b"BaseFont".to_vec(), Object::name(base));+ f.set(b"Encoding".to_vec(), Object::name("WinAnsiEncoding"));+ f+ };+ assert!(super::is_plain_standard_font(&plain("Helvetica"), "Helvetica"));+ assert!(!super::is_plain_standard_font(&plain("ArialMT"), "Helvetica"), "another font under our name");+ let mut embedded = plain("Helvetica");+ embedded.set(b"FontDescriptor".to_vec(), Object::Dict(Dict::new()));+ assert!(!super::is_plain_standard_font(&embedded, "Helvetica"));+ let mut measured = plain("Helvetica");+ measured.set(b"Widths".to_vec(), Object::Array(Vec::new()));+ assert!(!super::is_plain_standard_font(&measured, "Helvetica"));+ let mut mapped = plain("Helvetica");+ mapped.set(b"ToUnicode".to_vec(), Object::Ref(pdfcraft_cos::ObjRef::new(99, 0)));+ assert!(!super::is_plain_standard_font(&mapped, "Helvetica"), "its /ToUnicode would be lost");+ let ours = super::standard_font("Helvetica");+ assert!(!super::is_plain_standard_font(ours.as_dict().unwrap(), "Helvetica"), "already carries the extra letters");+ }+ #[test] fn parse_hex32_rejects_non_hex_digits() { for prefix in ["+f", "-f", " f"] {@@ -92,10 +92,32 @@ pub(super) fn standard_font(base: &str) -> Object { f.set(b"Type".to_vec(), Object::name("Font")); f.set(b"Subtype".to_vec(), Object::name("Type1")); f.set(b"BaseFont".to_vec(), Object::name(base));- f.set(b"Encoding".to_vec(), Object::name("WinAnsiEncoding"));+ // WinAnsi plus the Romanian and Polish letters it lacks (`LATIN_EXTRA`, codes 1..), which+ // `win_ansi_extra` writes: a plain WinAnsiEncoding drew them as `?`.+ let mut enc = Dict::new();+ enc.set(b"Type".to_vec(), Object::name("Encoding"));+ enc.set(b"BaseEncoding".to_vec(), Object::name("WinAnsiEncoding"));+ // One `code /name` pair per letter: the codes skip the white-space controls, so they are not one run.+ let diffs: Vec<Object> = pdfcraft_fonts::LATIN_EXTRA.iter().flat_map(|e| [Object::Int(i64::from(e.1)), Object::name(e.2)]).collect();+ enc.set(b"Differences".to_vec(), Object::Array(diffs));+ f.set(b"Encoding".to_vec(), Object::Dict(enc)); Object::Dict(f) } +/// Whether `font` is a non-embedded standard-14 `base` in plain WinAnsiEncoding, as forms bring+/// in their `/DR`: [`standard_font`] replaces it without changing how any code already drawn in+/// it looks (the `LATIN_EXTRA` codes are undefined in WinAnsi), and adds the letters it lacks.+/// Embedded, measured (`/Widths`) or differently encoded fonts are left as they are, and so is+/// one with a `/ToUnicode` map: extraction reads codes through it, which the replacement would drop.+pub(super) fn is_plain_standard_font(font: &Dict, base: &str) -> bool {+ font.name(b"Subtype") == Some(&b"Type1"[..])+ && font.name(b"BaseFont") == Some(base.as_bytes())+ && font.name(b"Encoding") == Some(&b"WinAnsiEncoding"[..])+ && !font.contains(b"FontDescriptor")+ && !font.contains(b"Widths")+ && !font.contains(b"ToUnicode")+}+ /// Every face the content streams or `/DA` strings may name. pub(super) fn all_faces() -> Vec<(&'static str, &'static str)> { let mut out = Vec::new();@@ -132,7 +132,7 @@ impl Face { pub fn width(&self, s: &str) -> f64 { #[cfg(test)] MEASURED.with(|m| m.set(m.get().saturating_add(s.len() as u64)));- self.family.std14(self.bold, self.italic).text_width(s, self.size) * self.h_scale+ self.family.std14(self.bold, self.italic).text_width_extra(s, self.size) * self.h_scale } }