diff --git a/crates/proof/src/affix.rs b/crates/proof/src/affix.rs --- a/crates/proof/src/affix.rs +++ b/crates/proof/src/affix.rs @@ -11,10 +11,14 @@ //! also be stored in WordCraft's compact format ([`AffixDict::to_bytes`]): the rules as text //! lines and the stems front-coded, raw-deflated together (the Polish dictionary is under 1 MB). //! -//! Supported: one-character flags (at most 128 distinct, any Unicode character — `FLAG UTF-8`), -//! `PFX`/`SFX` with strip, append and condition (`.`, letters, `[abc]`, `[^abc]`), cross -//! products, continuation classes on suffixes (at most two suffixes in a row, as in Hunspell), -//! `FORBIDDENWORD`, `NOSUGGEST`, and the suggestion tables `TRY`, `KEY`, `MAP` and `REP`. Rejected: compounding, +//! Supported: one-character flags (at most 256 distinct, any Unicode character — `FLAG UTF-8`), +//! flag aliases (`AF`: a number standing for a set of flags), `PFX`/`SFX` with strip, append and +//! condition (`.`, letters, `[abc]`, `[^abc]`), cross products, continuation classes on suffixes +//! (at most two suffixes in a row, as in Hunspell), `FORBIDDENWORD`, `NOSUGGEST`, `KEEPCASE` (the +//! stem is a word only in its own letter case), input conversion (`ICONV`: ligatures such as `fi` +//! read as `fi`) and ignored characters (`IGNORE`), and the suggestion tables `TRY`, `KEY`, `MAP` +//! and `REP`. +//! Morphological data (`AM` aliases, the fields after a stem) is skipped. Rejected: compounding, //! long or numeric flags, `NEEDAFFIX` and the other directives that change which words are correct. //! //! Lookups never panic: stems sit in one string, found through an open-addressing hash table, and @@ -33,30 +37,91 @@ const EMPTY: u32 = u32::MAX; /// Directives that change which words are correct in ways this engine doesn't implement. const UNSUPPORTED: &[&str] = &[ - "AF", - "AM", "CHECKSHARPS", "CIRCUMFIX", "COMPLEXPREFIXES", "COMPOUNDBEGIN", "COMPOUNDEND", + "COMPOUNDFIRST", "COMPOUNDFLAG", + "COMPOUNDLAST", "COMPOUNDMIDDLE", "COMPOUNDMIN", "COMPOUNDPERMITFLAG", "COMPOUNDRULE", "FULLSTRIP", - "ICONV", - "IGNORE", - "KEEPCASE", - "LEMMA_PRESENT", "NEEDAFFIX", "OCONV", "ONLYINCOMPOUND", "PSEUDOROOT", - "SUBSTANDARD", ]; +/// Most distinct flags a dictionary may use: every single-byte flag (the Hungarian dictionary +/// uses 158). +const MAX_FLAGS: usize = 256; + +/// A set of flags, one bit per distinct flag character of the dictionary. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] +struct Mask([u64; MAX_FLAGS / 64]); + +impl Mask { + const EMPTY: Mask = Mask([0; MAX_FLAGS / 64]); + + /// The set holding flag number `i` alone (empty when `i` is out of range). + fn bit(i: usize) -> Mask { + let mut m = Mask::EMPTY; + if let Some(w) = m.0.get_mut(i / 64) { + *w = 1 << (i % 64); + } + m + } + + /// The number of the lowest flag in the set, if any. + fn first(self) -> Option { + self.0.iter().enumerate().find(|(_, w)| **w != 0).map(|(i, w)| i * 64 + w.trailing_zeros() as usize) + } + + fn is_empty(self) -> bool { + self.0.iter().all(|&w| w == 0) + } + + /// Do the two sets share a flag? + fn meets(self, other: Mask) -> bool { + !(self & other).is_empty() + } + + /// Does this set hold every flag of `other`? + fn covers(self, other: Mask) -> bool { + (self & other) == other + } +} + +impl std::ops::BitAnd for Mask { + type Output = Mask; + fn bitand(mut self, o: Mask) -> Mask { + for (a, b) in self.0.iter_mut().zip(o.0) { + *a &= b; + } + self + } +} + +impl std::ops::BitOr for Mask { + type Output = Mask; + fn bitor(mut self, o: Mask) -> Mask { + self |= o; + self + } +} + +impl std::ops::BitOrAssign for Mask { + fn bitor_assign(&mut self, o: Mask) { + for (a, b) in self.0.iter_mut().zip(o.0) { + *a |= b; + } + } +} + /// One character class of a rule's condition. #[derive(Clone, Debug, PartialEq, Eq)] enum Cond { @@ -130,14 +195,14 @@ const MAX_FORMS_WORK: usize = 1 << 18; #[derive(Clone, Debug)] struct Rule { /// Bit of the rule's flag in the stems' flag masks. - bit: u128, + bit: Mask, flag: char, cross: bool, strip: Box, add: Box, /// Continuation classes (`SFX A 0 able/B`): flags the derived form carries, licensing /// further affixes. Chained application is bounded by [`MAX_CHAIN`]. - cont: u128, + cont: Mask, cond: Box<[Cond]>, cond_text: Box, } @@ -174,11 +239,13 @@ pub struct AffixDict { /// Per stem: index into `sets`. flags: Vec, /// Distinct flag masks. - sets: Vec, + sets: Vec, /// The `FORBIDDENWORD` flag's bit; stems carrying it are not words (0: none declared). - forbidden: u128, + forbidden: Mask, /// Stems with NOSUGGEST are correct, but none of their forms are suggested. - no_suggest: u128, + no_suggest: Mask, + /// Stems with SUBSTANDARD are correct too, and likewise never suggested. + substandard: Mask, /// Open-addressing hash table of stem indices (`EMPTY` = free). table: Vec, /// Flag char of each bit. @@ -195,6 +262,12 @@ pub struct AffixDict { key: String, map: Vec>, rep: Vec<(String, String)>, + /// The `KEEPCASE` flag's bit: such stems are words only in their own letter case (0: none). + keepcase: Mask, + /// `ICONV`: input conversions applied before a lookup, `(from, to)`. + iconv: Vec<(String, String)>, + /// `IGNORE`: characters removed from input words (and from the stems and appended texts). + ignore: Box<[char]>, /// Provenance and licence lines, kept through [`AffixDict::to_bytes`]. comments: Vec, } @@ -263,6 +336,57 @@ struct Rules { /// The `FORBIDDENWORD` flag: stems carrying it are listed but never correct. forbidden: Option, no_suggest: Option, + substandard: Option, + /// The `KEEPCASE` flag. + keepcase: Option, + iconv: Vec<(String, String)>, + ignore: Vec, + /// `AF` flag aliases: alias `n` (1-based) stands for `aliases[n - 1]`. + aliases: Vec, + /// The `AF` table opened with its count line, so flag fields are alias numbers. A table + /// without one is rejected, as in Hunspell: its lines are ignored. + aliases_on: bool, +} + +impl Rules { + /// Flags as written in a `.dic` entry or a continuation class: the flag characters + /// themselves, or with an `AF` table the alias its leading digits number, as Hunspell reads + /// them (`2x` is alias 2). A field without digits or an undefined alias reads as no flags: + /// the stem is still a word, the rule still applies, without the flags (Hunspell warns). + fn flags<'a>(&'a self, written: &'a str) -> &'a str { + if !self.aliases_on { + return written; + } + let digits = written.bytes().take_while(u8::is_ascii_digit).count(); + written + .get(..digits) + .and_then(|d| d.parse::().ok()) + .and_then(|n| n.checked_sub(1)) + .and_then(|i| self.aliases.get(i)) + .map_or("", String::as_str) + } + + /// `w` without the `IGNORE` characters. + fn strip_ignored(&self, w: &str) -> String { + if self.ignore.is_empty() { w.to_string() } else { w.chars().filter(|c| !self.ignore.contains(c)).collect() } + } +} + +/// The fields of an `.aff`/`.dic` line: Hunspell separates them with spaces and tabs only, so a +/// flag byte read as U+0085 or U+00A0 (see [`crate::hunspell::decode`]) stays a flag. +fn split_fields(line: &str) -> impl Iterator { + line.split([' ', '\t']).filter(|f| !f.is_empty()) +} + +/// The one flag character a directive (`FORBIDDENWORD w`) names. +fn one_flag(arg: &str, dir: &str, n: usize) -> Result { + let mut it = arg.chars().filter(|&c| c != EMOJI_SELECTOR); + match (it.next(), it.next()) { + // Only the field separators of `split_fields` can't be a flag: a flag byte read as + // U+0085 or U+00A0 is one. + (Some(c), None) if !matches!(c, ' ' | '\t') => Ok(c), + _ => Err(format!("line {}: {dir} needs one flag character", n + 1)), + } } /// A rule as parsed from text, before flags become bits: `(flag, cross, strip, add, cont, cond)`. @@ -304,8 +428,10 @@ fn parse_aff(aff: &str) -> Result { let mut cross: HashMap<(bool, char), bool> = HashMap::new(); let mut map_header = false; let mut rep_header = false; + let mut af_header = false; + let mut iconv_header = false; for (n, line) in aff.lines().enumerate() { - let f: Vec<&str> = line.split_whitespace().collect(); + let f: Vec<&str> = split_fields(line).collect(); let Some(&dir) = f.first() else { continue }; if dir.starts_with('#') { continue; @@ -314,25 +440,38 @@ fn parse_aff(aff: &str) -> Result { match dir { "SET" | "LANG" | "NAME" | "VERSION" | "HOME" | "WORDCHARS" | "MAXNGRAMSUGS" | "MAXDIFF" | "ONLYMAXDIFF" | "NOSPLITSUGS" | "SUGSWITHDOTS" | "WARN" | "FORBIDWARN" | "MAXCPDSUGS" => {} + // Morphological aliases, and a flag that only matters for morphological generation: + // the words it marks are correct. + "AM" | "LEMMA_PRESENT" => {} + "SUBSTANDARD" => r.substandard = Some(one_flag(at(1), dir, n)?), + // The first `AF` or `ICONV` line is the table's count, as in Hunspell, even with a + // trailing comment (`AF 1559 # aliases`). + "AF" => { + if !af_header { + af_header = true; + // A count below 1 rejects the table too: flags stay literal (`AF 0`). + r.aliases_on = at(1).parse::().is_ok_and(|n| n > 0); + } else if r.aliases_on { + r.aliases.push(at(1).to_string()); + } + } + "ICONV" => { + if !iconv_header && at(1).parse::().is_ok() { + iconv_header = true; + } else if f.len() >= 3 { + r.iconv.push((at(1).to_string(), at(2).to_string())); + } + } + "IGNORE" => r.ignore = at(1).chars().collect(), + "KEEPCASE" => r.keepcase = Some(one_flag(at(1), dir, n)?), // Per-character flag modes (`FLAG UTF-8`, as VERO's pt_BR dictionary uses) work // with the one-character parsing here; only the multi-character encodings differ. "FLAG" if !matches!(at(1), "UTF-8" | "DEFAULT" | "utf-8" | "default" | "UTF8" | "") => { return Err(format!("line {}: FLAG {} is not supported (one-character flags only)", n + 1, at(1))); } "FLAG" => {} - directive @ ("FORBIDDENWORD" | "NOSUGGEST") => { - let mut it = at(1).chars().filter(|&c| c != EMOJI_SELECTOR); - match (it.next(), it.next()) { - (Some(c), None) if !c.is_whitespace() => { - if directive == "FORBIDDENWORD" { - r.forbidden = Some(c); - } else { - r.no_suggest = Some(c); - } - } - _ => return Err(format!("line {}: {directive} needs one flag character", n + 1)), - } - } + "FORBIDDENWORD" => r.forbidden = Some(one_flag(at(1), dir, n)?), + "NOSUGGEST" => r.no_suggest = Some(one_flag(at(1), dir, n)?), "TRY" => r.try_chars = cap_try(at(1)), "KEY" => r.key = at(1).to_string(), "MAP" => { @@ -343,7 +482,7 @@ fn parse_aff(aff: &str) -> Result { } } "REP" => { - if !rep_header && f.len() == 2 && at(1).parse::().is_ok() { + if !rep_header && at(1).parse::().is_ok() { rep_header = true; } else if f.len() >= 3 { r.rep.push((at(1).replace('_', " "), at(2).replace('_', " "))); @@ -383,6 +522,19 @@ fn parse_aff(aff: &str) -> Result { _ => {} } } + // Aliases may be declared after the rules that use them; resolve once all are read. + // Hunspell removes `IGNORE` characters from the appended text only: a stripped text that + // holds one never matches a stem (stems lose them too), so the rule never applies. + let resolve = |rules: &mut Vec, r: &Rules| { + for rule in rules.iter_mut() { + rule.4 = r.flags(&rule.4).to_string(); + rule.3 = r.strip_ignored(&rule.3); + } + }; + let (mut pfx, mut sfx) = (std::mem::take(&mut r.prefixes), std::mem::take(&mut r.suffixes)); + resolve(&mut pfx, &r); + resolve(&mut sfx, &r); + (r.prefixes, r.suffixes) = (pfx, sfx); Ok(r) } @@ -397,7 +549,12 @@ impl AffixDict { let rules = parse_aff(aff)?; let mut stems: Vec<(String, String)> = Vec::new(); for (n, line) in dic.lines().enumerate() { - let Some(tok) = line.split_whitespace().next() else { continue }; + // The entry may hold spaces (`ad hoc/B\tpo:adv`); it ends where Hunspell ends it. + // Leading blanks belong to it, as in Hunspell (` lab` is no `lab`). + let tok = line.get(..crate::hunspell::dic_entry_end(line.as_bytes())).unwrap_or_default(); + if tok.is_empty() { + continue; + } if n == 0 && tok.parse::().is_ok() { continue; } @@ -420,9 +577,11 @@ impl AffixDict { word.push(c); } } + let word = rules.strip_ignored(&word); if word.is_empty() || word.len() > MAX_WORD_BYTES || word.contains(['\n', '/']) { continue; } + let flags = rules.flags(&flags).to_string(); stems.push((word, flags)); } AffixDict::build(rules, stems, comments.iter().map(ToString::to_string).collect()) @@ -434,11 +593,12 @@ impl AffixDict { flag_chars.extend(stems.iter().flat_map(|s| s.1.chars())); flag_chars.sort_unstable(); flag_chars.dedup(); - if flag_chars.len() > 128 { - return Err(format!("{} distinct flags; at most 128 are supported", flag_chars.len())); + if flag_chars.len() > MAX_FLAGS { + return Err(format!("{} distinct flags; at most {MAX_FLAGS} are supported", flag_chars.len())); } - let bit_of = |c: char| flag_chars.iter().position(|&x| x == c).map_or(0, |i| 1u128 << i); - let forbidden = rules.forbidden.map_or(0, bit_of); + let bit_of = |c: char| flag_chars.iter().position(|&x| x == c).map_or(Mask::EMPTY, Mask::bit); + let forbidden = rules.forbidden.map_or(Mask::EMPTY, bit_of); + let keepcase = rules.keepcase.map_or(Mask::EMPTY, bit_of); // Stems sorted by bytes; homonyms (the same word twice) are merged. stems.sort_by(|a, b| a.0.as_bytes().cmp(b.0.as_bytes())); let mut d = AffixDict { @@ -446,7 +606,11 @@ impl AffixDict { offs: Vec::with_capacity(stems.len() + 1), flag_chars: flag_chars.clone(), forbidden, - no_suggest: rules.no_suggest.map_or(0, bit_of), + no_suggest: rules.no_suggest.map_or(Mask::EMPTY, bit_of), + substandard: rules.substandard.map_or(Mask::EMPTY, bit_of), + keepcase, + iconv: rules.iconv, + ignore: rules.ignore.into_boxed_slice(), try_chars: rules.try_chars, key: rules.key, map: rules.map, @@ -455,16 +619,16 @@ impl AffixDict { ..AffixDict::default() }; d.offs.push(0); - let mut set_ids: HashMap = HashMap::new(); + let mut set_ids: HashMap = HashMap::new(); let mut i = 0; while i < stems.len() { let Some((word, _)) = stems.get(i) else { break }; - let mut mask = 0u128; + let mut mask = Mask::EMPTY; let mut j = i; while let Some((w, f)) = stems.get(j) && w == word { - mask |= f.chars().map(bit_of).fold(0, |a, b| a | b); + mask |= f.chars().map(bit_of).fold(Mask::EMPTY, |a, b| a | b); j += 1; } let next_id = u16::try_from(d.sets.len()).map_err(|_| "too many distinct flag sets".to_string())?; @@ -478,7 +642,7 @@ impl AffixDict { i = j; } let mk = |(flag, cross, strip, add, cont, cond): RawRule| -> Result { - let cont = cont.chars().fold(0u128, |m, c| m | bit_of(c)); + let cont = cont.chars().fold(Mask::EMPTY, |m, c| m | bit_of(c)); Ok(Rule { bit: bit_of(flag), flag, cross, strip: strip.into(), add: add.into(), cont, cond: parse_cond(&cond)?, cond_text: cond.into() }) }; d.prefixes = rules.prefixes.into_iter().map(mk).collect::>()?; @@ -493,7 +657,7 @@ impl AffixDict { // Each (appended, stripped) pair's group, found by hash rather than by scanning the // appended string's groups: a dictionary with many strips per suffix stays linear. let mut group_of: FxMap<(&str, &str), usize> = FxMap::default(); - let mut by_flag: Vec> = vec![Vec::new(); 128]; + let mut by_flag: Vec> = vec![Vec::new(); MAX_FLAGS]; for (i, r) in self.suffixes.iter().enumerate() { let groups = idx.entry(r.add.clone()).or_default(); let Ok(i) = u32::try_from(i) else { break }; @@ -504,9 +668,7 @@ impl AffixDict { groups.push(SfxGroup { strip: r.strip.clone(), rules: vec![i] }); } } - if r.bit != 0 - && let Some(list) = by_flag.get_mut(r.bit.trailing_zeros() as usize) - { + if let Some(list) = r.bit.first().and_then(|b| by_flag.get_mut(b)) { list.push(i); } } @@ -551,12 +713,12 @@ impl AffixDict { } /// The flag characters of `mask`, in flag order (how the compact format stores flags). - fn flag_chars_for(&self, mask: u128) -> String { - self.flag_chars.iter().enumerate().filter(|&(i, _)| mask & (1u128 << i) != 0).map(|(_, c)| c).collect() + fn flag_chars_for(&self, mask: Mask) -> String { + self.flag_chars.iter().enumerate().filter(|&(i, _)| mask.meets(Mask::bit(i))).map(|(_, c)| c).collect() } /// The character of `mask` when it holds exactly one flag. - fn flag_char(&self, mask: u128) -> Option { + fn flag_char(&self, mask: Mask) -> Option { let s = self.flag_chars_for(mask); let mut it = s.chars(); match (it.next(), it.next()) { @@ -589,11 +751,12 @@ impl AffixDict { } /// Flag mask of a stem, if `w` is one. - fn stem_flags(&self, w: &str) -> Option { - self.stem_flags_excluding(w, self.forbidden) + fn stem_flags(&self, w: &str) -> Option { + self.stem_flags_except(w, Mask::EMPTY) } - fn stem_flags_excluding(&self, w: &str, excluded: u128) -> Option { + /// Flag mask of a stem, if `w` is one and carries none of the flags in `deny`. + fn stem_flags_except(&self, w: &str, deny: Mask) -> Option { if self.table.is_empty() { return None; } @@ -607,8 +770,9 @@ impl AffixDict { if self.word(t as usize) == Some(w) { let id = *self.flags.get(t as usize)?; let mask = self.sets.get(usize::from(id)).copied(); - // All derivations use this lookup: exclusions apply to stems and their forms. - return mask.filter(|m| m & excluded == 0); + // A stem carrying FORBIDDENWORD is listed but is not a word: no direct hit, + // and (this is the only stem lookup) no affixed form is built on it either. + return mask.filter(|m| !m.meets(self.forbidden | deny)); } slot = (slot + 1) & mask; } @@ -621,8 +785,8 @@ impl AffixDict { /// The rules of a group that could lead to `w` are merged into one mask of their flags and /// followed once, not once per rule: the answer is the same, but thousands of rules of one /// flag cost one walk, not one each (#3613). - fn form_has_flag(&self, w: &str, flags: u128, depth: u8, excluded: u128) -> bool { - if self.stem_flags_excluding(w, excluded).is_some_and(|f| f & flags != 0) { + fn form_has_flag(&self, w: &str, flags: Mask, depth: u8, deny: Mask) -> bool { + if self.stem_flags_except(w, deny).is_some_and(|f| f.meets(flags)) { return true; } if depth == 0 { @@ -641,9 +805,9 @@ impl AffixDict { .rules .iter() .filter_map(|&i| self.suffixes.get(i as usize)) - .filter(|r| r.cont & flags != 0 && r.cond_matches(&root, true)) - .fold(0u128, |m, r| m | r.bit); - if inner != 0 && self.form_has_flag(&root, inner, depth - 1, excluded) { + .filter(|r| r.cont.meets(flags) && r.cond_matches(&root, true)) + .fold(Mask::EMPTY, |m, r| m | r.bit); + if !inner.is_empty() && self.form_has_flag(&root, inner, depth - 1, deny) { return true; } } @@ -651,21 +815,55 @@ impl AffixDict { false } - /// Is `w` exactly (case and all) a stem or an affixed form of one? + /// Is `w` exactly (case and all) a stem or an affixed form of one? `ICONV` and `IGNORE` + /// apply first. pub fn contains(&self, w: &str) -> bool { - self.contains_excluding(w, self.forbidden) + self.contains_except(&self.normalize(w), Mask::EMPTY) } - /// Exactly spelled dictionary words eligible for suggestions (NOSUGGEST stems excluded). + /// Exactly spelled dictionary words eligible for suggestions (NOSUGGEST and SUBSTANDARD + /// stems excluded). pub fn contains_suggestible(&self, w: &str) -> bool { - self.contains_excluding(w, self.forbidden | self.no_suggest) + self.contains_except(&self.normalize(w), self.no_suggest | self.substandard) + } + + /// `w` after the `ICONV` conversions (at each position the longest match wins) and without + /// the `IGNORE` characters. + fn normalize<'a>(&self, w: &'a str) -> std::borrow::Cow<'a, str> { + if self.iconv.is_empty() && self.ignore.is_empty() { + return std::borrow::Cow::Borrowed(w); + } + // One pass: the input is copied only from the first change on. + let mut out: Option = None; + let mut rest = w; + while let Some(c) = rest.chars().next() { + let done = w.get(..w.len() - rest.len()).unwrap_or_default(); + let hit = self.iconv.iter().filter(|(from, _)| !from.is_empty() && rest.starts_with(from.as_str())).max_by_key(|(from, _)| from.len()); + match hit { + Some((from, to)) => { + out.get_or_insert_with(|| done.to_string()).extend(to.chars().filter(|c| !self.ignore.contains(c))); + rest = rest.get(from.len()..).unwrap_or_default(); + } + None => { + if self.ignore.contains(&c) { + out.get_or_insert_with(|| done.to_string()); + } else if let Some(o) = out.as_mut() { + o.push(c); + } + rest = rest.get(c.len_utf8()..).unwrap_or_default(); + } + } + } + out.map_or(std::borrow::Cow::Borrowed(w), std::borrow::Cow::Owned) } - fn contains_excluding(&self, w: &str, excluded: u128) -> bool { + /// [`AffixDict::contains`] on normalized input, where stems carrying a flag in `deny` don't + /// count (case variants of `KEEPCASE` stems). + fn contains_except(&self, w: &str, deny: Mask) -> bool { if w.is_empty() || w.len() > MAX_WORD_BYTES { return false; } - if self.stem_flags_excluding(w, excluded).is_some() || self.suffixed(w, 0, excluded) { + if self.stem_flags_except(w, deny).is_some() || self.suffixed(w, Mask::EMPTY, deny) { return true; } let mut root = String::with_capacity(w.len() + 8); @@ -680,13 +878,13 @@ impl AffixDict { if !r.cond_matches(&root, false) { continue; } - if self.stem_flags_excluding(&root, excluded).is_some_and(|f| f & r.bit != 0) { + if self.stem_flags_except(&root, deny).is_some_and(|f| f.meets(r.bit)) { return true; } - if self.form_has_flag(&root, r.bit, INNER_CHAIN, excluded) { + if self.form_has_flag(&root, r.bit, INNER_CHAIN, deny) { return true; } - if r.cross && self.suffixed(&root, r.bit, excluded) { + if r.cross && self.suffixed(&root, r.bit, deny) { return true; } } @@ -695,7 +893,7 @@ impl AffixDict { /// Is `w` a stem with one suffix? With `need` set (a prefix was taken off), the stem must /// carry those flags too and the suffix must allow cross products. - fn suffixed(&self, w: &str, need: u128, excluded: u128) -> bool { + fn suffixed(&self, w: &str, need: Mask, deny: Mask) -> bool { let mut root = String::with_capacity(w.len() + 8); // Each split point from the end: the appended string is `w[b..]`, at most the longest // suffix, and something must stay before it. @@ -710,18 +908,18 @@ impl AffixDict { let rules = || g.rules.iter().filter_map(|&i| self.suffixes.get(i as usize)); // The stem path: `root` is a stem carrying the rule's flag (and the prefix's, // when a cross product is being checked). - if let Some(f) = self.stem_flags_excluding(&root, excluded) - && f & need == need - && rules().any(|r| f & r.bit != 0 && (need == 0 || r.cross) && r.cond_matches(&root, true)) + if let Some(f) = self.stem_flags_except(&root, deny) + && f.covers(need) + && rules().any(|r| f.meets(r.bit) && (need.is_empty() || r.cross) && r.cond_matches(&root, true)) { return true; } // Continuation classes: `root` need not be a stem carrying this rule's flag — // it may be a form that gained the flag from another rule's continuation. The // matching rules' flags are looked for together, in one walk. - if need == 0 { - let flags = rules().filter(|r| r.cond_matches(&root, true)).fold(0u128, |m, r| m | r.bit); - if flags != 0 && self.form_has_flag(&root, flags, INNER_CHAIN, excluded) { + if need.is_empty() { + let flags = rules().filter(|r| r.cond_matches(&root, true)).fold(Mask::EMPTY, |m, r| m | r.bit); + if !flags.is_empty() && self.form_has_flag(&root, flags, INNER_CHAIN, deny) { return true; } } @@ -732,26 +930,29 @@ impl AffixDict { /// Is `w` correct, Hunspell style: as written, or — for a capitalized word or one in capitals /// — in lower case (and capitalized), since a sentence may start with any word and headings - /// may be in capitals. Lower-case input never matches a capitalized stem (`warszawa`). + /// may be in capitals. Lower-case input never matches a capitalized stem (`warszawa`), and a + /// `KEEPCASE` stem matches only as written. pub fn check(&self, w: &str) -> bool { - self.check_excluding(w, self.forbidden) + self.check_except(w, Mask::EMPTY) } /// Whether a suggestion is correct in its displayed case and has an eligible stem. pub fn can_suggest(&self, w: &str) -> bool { - self.check_excluding(w, self.forbidden | self.no_suggest) + self.check_except(w, self.no_suggest | self.substandard) } - fn check_excluding(&self, w: &str, excluded: u128) -> bool { - if self.contains_excluding(w, excluded) { + fn check_except(&self, w: &str, deny: Mask) -> bool { + let w = self.normalize(w); + if self.contains_except(&w, deny) { return true; } - match casing(w) { + let deny = deny | self.keepcase; + match casing(&w) { Casing::Lower | Casing::Mixed => false, - Casing::Initial => self.contains_excluding(&w.to_lowercase(), excluded), + Casing::Initial => self.contains_except(&w.to_lowercase(), deny), Casing::Upper => { let lower = w.to_lowercase(); - self.contains_excluding(&capitalize(&lower), excluded) || self.contains_excluding(&lower, excluded) + self.contains_except(&capitalize(&lower), deny) || self.contains_except(&lower, deny) } } } @@ -766,11 +967,11 @@ impl AffixDict { let mut cross_sfx = Vec::new(); // Suffix forms queued with the continuation classes they carry, for chained expansion; // each (form, classes) pair is expanded once. - let mut chain: Vec<(String, u128)> = Vec::new(); - let mut queued: std::collections::HashSet<(String, u128)> = std::collections::HashSet::new(); + let mut chain: Vec<(String, Mask)> = Vec::new(); + let mut queued: std::collections::HashSet<(String, Mask)> = std::collections::HashSet::new(); let mut work = 0usize; for r in &self.suffixes { - if f & r.bit == 0 || !r.cond_matches(stem, true) { + if !f.meets(r.bit) || !r.cond_matches(stem, true) { continue; } let Some(base) = stem.strip_suffix(&*r.strip) else { continue }; @@ -778,7 +979,7 @@ impl AffixDict { if r.cross { cross_sfx.push(form.clone()); } - if r.cont != 0 && queued.insert((form.clone(), r.cont)) { + if !r.cont.is_empty() && queued.insert((form.clone(), r.cont)) { chain.push((form.clone(), r.cont)); } out.push(form); @@ -804,7 +1005,7 @@ impl AffixDict { } let Some(base) = form.strip_suffix(&*r.strip) else { continue }; let next = format!("{base}{}", r.add); - if r.cont != 0 && queued.insert((next.clone(), r.cont)) { + if !r.cont.is_empty() && queued.insert((next.clone(), r.cont)) { chain.push((next.clone(), r.cont)); } out.push(next); @@ -815,7 +1016,7 @@ impl AffixDict { } } for r in &self.prefixes { - if f & r.bit == 0 || !r.cond_matches(stem, false) { + if !f.meets(r.bit) || !r.cond_matches(stem, false) { continue; } let Some(rest) = stem.strip_prefix(&*r.strip) else { continue }; @@ -839,11 +1040,11 @@ impl AffixDict { } /// The suffix rules carrying any flag of `mask`, in rule order. - fn suffixes_with(&self, mask: u128) -> Vec { + fn suffixes_with(&self, mask: Mask) -> Vec { let mut rules: Vec = Vec::new(); let mut lists = 0; for (bit, list) in self.sfx_by_flag.iter().enumerate() { - if bit < 128 && mask & (1u128 << bit) != 0 && !list.is_empty() { + if mask.meets(Mask::bit(bit)) && !list.is_empty() { rules.extend_from_slice(list); lists += 1; } @@ -904,13 +1105,21 @@ impl AffixDict { for (a, b) in &self.rep { out.push_str(&format!("REP {} {}\n", a.replace(' ', "_"), b.replace(' ', "_"))); } - for (directive, mask) in [("FORBIDDENWORD", self.forbidden), ("NOSUGGEST", self.no_suggest)] { - if mask != 0 + for (directive, mask) in + [("FORBIDDENWORD", self.forbidden), ("NOSUGGEST", self.no_suggest), ("SUBSTANDARD", self.substandard), ("KEEPCASE", self.keepcase)] + { + if !mask.is_empty() && let Some(c) = self.flag_char(mask) { out.push_str(&format!("{directive} {c}\n")); } } + for (a, b) in &self.iconv { + out.push_str(&format!("ICONV {a} {b}\n")); + } + if !self.ignore.is_empty() { + out.push_str(&format!("IGNORE {}\n", self.ignore.iter().collect::())); + } let z = |s: &str| if s.is_empty() { "0".to_string() } else { s.to_string() }; for (kind, rules) in [("PFX", &self.prefixes), ("SFX", &self.suffixes)] { for r in rules { @@ -934,8 +1143,8 @@ impl AffixDict { let p = prev.iter().zip(b).take_while(|(x, y)| x == y).count().min(255); raw.push(p as u8); raw.extend_from_slice(b.get(p..).unwrap_or_default()); - let mask = self.flags.get(i).and_then(|&id| self.sets.get(usize::from(id))).copied().unwrap_or(0); - if mask != 0 { + let mask = self.flags.get(i).and_then(|&id| self.sets.get(usize::from(id))).copied().unwrap_or_default(); + if !mask.is_empty() { raw.push(b'/'); raw.extend_from_slice(self.flag_chars_for(mask).as_bytes()); } @@ -962,26 +1171,24 @@ impl AffixDict { comments.push(c.to_string()); continue; } - let f: Vec<&str> = line.split_whitespace().collect(); + let f: Vec<&str> = split_fields(line).collect(); let at = |k: usize| f.get(k).copied().unwrap_or(""); match at(0) { "TRY" => rules.try_chars = cap_try(at(1)), "KEY" => rules.key = at(1).to_string(), - "MAP" => rules.map.push(at(1).chars().collect()), - "REP" => rules.rep.push((at(1).replace('_', " "), at(2).replace('_', " "))), - directive @ ("FORBIDDENWORD" | "NOSUGGEST") => { - let mut it = at(1).chars(); - match (it.next(), it.next()) { - (Some(c), None) if !c.is_whitespace() => { - if directive == "FORBIDDENWORD" { - rules.forbidden = Some(c); - } else { - rules.no_suggest = Some(c); - } - } - _ => return Err(format!("affix dictionary: bad {directive} in `{line}`")), + directive @ ("FORBIDDENWORD" | "NOSUGGEST" | "SUBSTANDARD" | "KEEPCASE") => { + let c = one_flag(at(1), directive, 0).map_err(|_| format!("affix dictionary: bad {directive} in `{line}`"))?; + match directive { + "FORBIDDENWORD" => rules.forbidden = Some(c), + "NOSUGGEST" => rules.no_suggest = Some(c), + "SUBSTANDARD" => rules.substandard = Some(c), + _ => rules.keepcase = Some(c), } } + "ICONV" if f.len() == 3 => rules.iconv.push((at(1).to_string(), at(2).to_string())), + "IGNORE" => rules.ignore = at(1).chars().collect(), + "MAP" => rules.map.push(at(1).chars().collect()), + "REP" => rules.rep.push((at(1).replace('_', " "), at(2).replace('_', " "))), kind @ ("PFX" | "SFX") => { let mut fc = at(1).chars(); let (Some(flag), None) = (fc.next(), fc.next()) else { return Err(format!("affix dictionary: bad flag in `{line}`")) }; @@ -1223,7 +1430,7 @@ pub(crate) mod tests { #[test] fn more_than_64_distinct_flags_load() { // VERO pt_BR uses 106 distinct flags (FLAG UTF-8). Build one with every basic Latin - // letter, the digits and eight accented letters: 70 flags, 64 < n <= 128. + // letter, the digits and eight accented letters: 70 flags, more than 64. let mut flags = String::new(); let mut aff = String::from("SET UTF-8\nFLAG UTF-8\n"); let chars: Vec = ('a'..='z').chain('A'..='Z').chain('0'..='9').chain(['á', 'é', 'í', 'ó', 'ú', 'ç', 'ã', 'õ']).collect(); @@ -1239,6 +1446,24 @@ pub(crate) mod tests { assert!(!d.contains("verboß"), "ß is not a flag and not a stem"); } + #[test] + fn more_than_128_distinct_flags_load() { + // The Hungarian dictionary uses 158 single-byte flags: 200 flags here, U+0080 up. + let flags: String = (0x80u32..0x80 + 200).filter_map(char::from_u32).collect(); + let mut aff = String::from("SET UTF-8\n"); + for (i, c) in flags.chars().enumerate() { + aff.push_str(&format!("SFX {c} N 1\nSFX {c} 0 x{i} .\n")); + } + let half: String = flags.chars().step_by(2).collect(); + let d = AffixDict::from_hunspell(&aff, &format!("1\nszó/{half}\n"), &[]).unwrap(); + assert!(d.contains("szóx0") && d.contains("szóx198") && d.contains("szóx130")); + assert!(!d.contains("szóx1") && !d.contains("szóx199") && !d.contains("szóx131")); + let back = AffixDict::from_bytes(&d.to_bytes()).unwrap(); + assert!(back.contains("szóx198") && !back.contains("szóx199")); + let too_many: String = (0x100u32..0x100 + 257).filter_map(char::from_u32).collect(); + assert!(AffixDict::from_hunspell("SET UTF-8\n", &format!("1\nszó/{too_many}\n"), &[]).is_err()); + } + #[test] fn forbidden_words_are_rejected() { // VERO pt_BR marks spellings it must reject (flag ý under FLAG UTF-8) as stems carrying @@ -1364,6 +1589,167 @@ pub(crate) mod tests { assert!(d.contains("kot") && d.contains("koty")); } + #[test] + fn flag_aliases_stand_for_flag_sets() { + // `AF`: `/1` in the `.dic` and in a continuation class means the first alias's flags. + // Morphological aliases (`AM`) and the fields after a stem are skipped. + let aff = "SET UTF-8\nAF 2\nAF ab # 1\nAF c # 2\nAM 1\nAM po:noun\nSFX a Y 1\nSFX a 0 ok/2 . 1\nSFX b Y 1\nSFX b 0 at . 1\nSFX c Y 1\nSFX c 0 ban . 1\n"; + let d = AffixDict::from_hunspell(aff, "1\nház/1\t1\n", &[]).unwrap(); + assert!(d.contains("ház") && d.contains("házok") && d.contains("házat") && d.contains("házokban")); + assert!(!d.contains("házban"), "the stem has a and b, not c"); + // An undefined alias reads as no flags, as in Hunspell: the stem loads, without affixes. + let undefined = AffixDict::from_hunspell(aff, "2\nház/3\nkert/1\n", &[]).unwrap(); + assert!(undefined.check("ház") && !undefined.check("házok"), "alias 3 is not defined"); + assert!(undefined.check("kertok"), "the other entries keep their aliases"); + let back = AffixDict::from_bytes(&d.to_bytes()).unwrap(); + assert!(back.contains("házokban") && !back.contains("házban"), "aliases are expanded in the compact format"); + } + + #[test] + fn iconv_and_ignore_normalize_input() { + // Ligatures read as their letters; `IGNORE` characters vanish from input and stems. + let aff = "SET UTF-8\nICONV 2\nICONV fi fi\nICONV ffl ffl\nIGNORE ()\nSFX a Y 1\nSFX a 0 s .\n"; + let d = AffixDict::from_hunspell(aff, "2\nfine/a\nwaf(fle)/a\n", &[]).unwrap(); + assert!(d.check("fine") && d.check("fines") && d.check("Fine") && d.contains("fine")); + assert!(d.check("waffle") && d.check("waffles") && d.check("wa(ffle)")); + assert!(!d.check("finer")); + let back = AffixDict::from_bytes(&d.to_bytes()).unwrap(); + assert!(back.check("fines") && back.check("waffle") && back.check("waf(fle)")); + } + + #[test] + fn alias_numbers_are_read_as_hunspell_reads_them() { + // Checked with the Hunspell 1.7 command line. With an `AF` table a flag field is the alias + // its leading digits number: `fal/2x` is alias 2, `lab/b` has no flags. + let aff = "SET UTF-8\nAF 2\nAF B\nAF C\nSFX B Y 1\nSFX B 0 s .\nSFX C Y 1\nSFX C 0 x .\nSFX b Y 1\nSFX b 0 q .\n"; + let d = AffixDict::from_hunspell(aff, "3\nlab/b\nfal/2x\nkut/1\n", &[]).unwrap(); + assert!(d.check("lab") && !d.check("labq") && !d.check("labs")); + assert!(d.check("falx") && !d.check("fals") && d.check("kuts")); + // A table without its count line is rejected: no alias applies. + let broken = AffixDict::from_hunspell("SET UTF-8\nAF B\nAF C\nSFX B Y 1\nSFX B 0 s .\n", "2\nlab/1\nfal/2\n", &[]).unwrap(); + assert!(broken.check("lab") && !broken.check("labs") && !broken.check("fals")); + // `AF 0` is rejected the same way: `lab/B` keeps its literal flag. + let zero = AffixDict::from_hunspell("SET UTF-8\nAF 0\nSFX B Y 1\nSFX B 0 s .\n", "1\nlab/B\n", &[]).unwrap(); + assert!(zero.check("lab") && zero.check("labs")); + // An undefined alias in a continuation class leaves the rule in force, without it. + let cont = AffixDict::from_hunspell("SET UTF-8\nAF 1\nAF a\nSFX a Y 1\nSFX a 0 ok/7 .\n", "1\nhaz/1\n", &[]).unwrap(); + assert!(cont.check("haz") && cont.check("hazok")); + } + + #[test] + fn a_rep_count_line_may_carry_a_comment() { + let d = AffixDict::from_hunspell("SET UTF-8\nREP 1 # confusions\nREP x y\n", "1\nay\n", &[]).unwrap(); + assert_eq!(d.rep(), [("x".to_string(), "y".to_string())]); + } + + #[test] + fn count_lines_may_carry_a_comment() { + // Hunspell reads the first `AF` and `ICONV` line as the count, comment and all: `AF 2 #` + // is no alias, and `ICONV 1 # …` doesn't turn `1` into `#`. + let aff = "SET UTF-8\nAF 2 # aliases\nAF a # 1\nAF b # 2\nICONV 1 # ligatures\nICONV q y\nSFX a Y 1\nSFX a 0 ok .\nSFX b Y 1\nSFX b 0 at .\n"; + let d = AffixDict::from_hunspell(aff, "3\nhaz/1\nkert/2\na1\n", &[]).unwrap(); + assert!(d.check("hazok") && !d.check("hazat") && d.check("kertat") && !d.check("kertok"), "alias 1 is `a`"); + assert!(d.check("a1") && !d.check("a#")); + // The UTF-8 reader keeps such a line as it is, so both readers agree on it. + let (a, dic) = crate::hunspell::decode(aff.as_bytes(), "1\nhaz/1\n".as_bytes()).unwrap(); + assert!(a.contains("AF 2 # aliases") && a.contains("ICONV 1 # ligatures"), "{a}"); + assert!(AffixDict::from_hunspell(&a, &dic, &[]).unwrap().check("hazok")); + } + + #[test] + fn dic_entries_may_hold_spaces() { + // `ad hoc/B` is one entry, as in Hunspell; `ad` alone is no word. + let aff = "SET UTF-8\nSFX B Y 1\nSFX B 0 s .\n"; + // Checked with the Hunspell 1.7 command line: without a tab or a morphological field the + // whole line is the entry (`kot/B 12` has the flag `B`), and leading blanks belong to it + // (` lab` is no `lab`). + let d = AffixDict::from_hunspell(aff, "5\nad hoc/B\tpo:adv\nword/B po:noun\nkot/B 12\n lab/B\nés így/B\n", &[]).unwrap(); + assert!(d.contains("ad hoc") && d.contains("ad hocs") && d.contains("és így") && d.contains("és ígys")); + assert!(!d.contains("ad") && !d.contains("hoc") && !d.contains("ads") && !d.contains("így")); + assert!(d.contains("words") && d.contains("kots") && !d.contains("kot/B")); + assert!(!d.contains("lab") && !d.contains("labs")); + } + + #[test] + fn ignored_characters_leave_the_stripped_text_alone() { + // Hunspell removes `IGNORE` characters from the appended text, not from the stripped + // one: `SFX a bx c` never matches the stem `abx` (stored as `ab`), so `ac` is wrong, + // while `SFX a b cx` appends `c` (checked with the Hunspell 1.7 command line). + let strip = AffixDict::from_hunspell("SET UTF-8\nIGNORE x\nSFX a Y 1\nSFX a bx c .\n", "1\nabx/a\n", &[]).unwrap(); + assert!(strip.check("ab") && strip.check("abx") && !strip.check("ac")); + let add = AffixDict::from_hunspell("SET UTF-8\nIGNORE x\nSFX a Y 1\nSFX a b cx .\n", "1\nab/a\n", &[]).unwrap(); + assert!(add.check("ac") && add.check("acx")); + for d in [&strip, &add] { + let back = AffixDict::from_bytes(&d.to_bytes()).unwrap(); + assert_eq!(back.check("ac"), d.check("ac")); + } + } + + #[test] + fn normalizing_borrows_when_nothing_changes() { + let d = AffixDict::from_hunspell("SET UTF-8\nICONV 1\nICONV fi fi\nIGNORE ()\n", "1\nfine\n", &[]).unwrap(); + assert!(matches!(d.normalize("fine"), std::borrow::Cow::Borrowed("fine"))); + assert_eq!(d.normalize("(fi)ne"), "fine"); + assert!(matches!(dict().normalize("kot"), std::borrow::Cow::Borrowed("kot"))); + } + + #[test] + fn keepcase_stems_match_only_as_written() { + let aff = "SET UTF-8\nKEEPCASE k\nSFX a Y 1\nSFX a 0 s .\n"; + let d = AffixDict::from_hunspell(aff, "3\npH/k\nkm/ka\nkot/a\n", &[]).unwrap(); + assert!(d.check("pH") && d.check("km") && d.check("kms")); + assert!(!d.check("PH") && !d.check("Km") && !d.check("KM") && !d.check("Kms")); + assert!(d.check("Kot") && d.check("KOTS"), "other stems still follow the sentence and heading case"); + let back = AffixDict::from_bytes(&d.to_bytes()).unwrap(); + assert!(back.check("km") && !back.check("Km")); + } + + #[test] + fn substandard_and_lemma_present_words_are_correct() { + // `SUBSTANDARD` and `LEMMA_PRESENT` steer morphological generation, and a SUBSTANDARD + // stem is never suggested; the stems and rules they mark are still correct. + let aff = "SET UTF-8\nSUBSTANDARD x\nLEMMA_PRESENT l\nSFX a Y 2\nSFX a 0 y .\nSFX a 0 em/x .\n"; + let d = AffixDict::from_hunspell(aff, "2\nkot/a\npies/axl\n", &[]).unwrap(); + for dict in [&d, &AffixDict::from_bytes(&d.to_bytes()).unwrap()] { + assert!(dict.check("kot") && dict.check("koty") && dict.check("kotem")); + assert!(dict.check("pies") && dict.check("piesy") && dict.check("Piesem")); + assert!(dict.can_suggest("koty") && dict.can_suggest("kotem") && dict.contains_suggestible("kot")); + assert!(!dict.can_suggest("piesy") && !dict.can_suggest("Pies") && !dict.contains_suggestible("pies")); + } + } + + #[test] + fn unsupported_directives_are_refused_and_the_rest_load() { + // A dictionary that needs a directive outside this engine fails to load rather than + // accepting the wrong words. + for dir in UNSUPPORTED { + assert!(AffixDict::from_hunspell(&format!("SET UTF-8\n{dir} X\n"), DIC, &[]).is_err(), "{dir} should be refused"); + } + // The directives this engine reads, with no other change to the dictionary (`AF` would + // turn the flags of `DIC` into alias numbers; it has tests of its own). + for line in ["AM 1\nAM po:noun", "ICONV 1\nICONV fi fi", "IGNORE ()", "KEEPCASE k", "SUBSTANDARD x", "LEMMA_PRESENT l"] { + let d = AffixDict::from_hunspell(&format!("{AFF}{line}\n"), DIC, &[]).unwrap_or_else(|e| panic!("{line}: {e}")); + assert!(d.check("koty") && d.check("niekoty") && !d.check("kotx"), "{line}"); + } + } + + #[test] + fn fields_split_on_spaces_and_tabs_only() { + // A flag byte read as U+00A0 (no-break space) or U+0085 is a flag, not a separator. + let aff = "SET UTF-8\nSFX \u{a0} Y 1\nSFX \u{a0} 0 y .\nSFX \u{85} Y 1\nSFX \u{85} 0 em .\n"; + let d = AffixDict::from_hunspell(aff, "1\nkot/\u{a0}\u{85}\n", &[]).unwrap(); + assert!(d.contains("koty") && d.contains("kotem")); + let back = AffixDict::from_bytes(&d.to_bytes()).unwrap(); + assert!(back.contains("koty") && back.contains("kotem")); + // The same bytes as one-flag directives, as Hunspell reads them. + let aff = "SET UTF-8\nFORBIDDENWORD \u{a0}\nKEEPCASE \u{85}\nSFX a Y 1\nSFX a 0 s .\n"; + let d = AffixDict::from_hunspell(aff, "3\nkot/a\nrossz/\u{a0}\npH/\u{85}\n", &[]).unwrap(); + for dict in [&d, &AffixDict::from_bytes(&d.to_bytes()).unwrap()] { + assert!(dict.check("kots") && !dict.check("rossz")); + assert!(dict.check("pH") && !dict.check("PH")); + } + } + #[test] fn unsupported_and_broken_input_is_an_error() { assert!(AffixDict::from_hunspell("FLAG long\n", DIC, &[]).is_err()); diff --git a/crates/proof/src/hunspell.rs b/crates/proof/src/hunspell.rs --- a/crates/proof/src/hunspell.rs +++ b/crates/proof/src/hunspell.rs @@ -218,13 +218,23 @@ fn decode_high(bytes: &[u8], high: &str) -> String { /// (UTF-8 when there is none; ISO 8859-2 is the other one Polish dictionaries use, ISO 8859-1 the /// one the original Portuguese VERO uses, ISO 8859-9 the one older Turkish dictionaries use, KOI8-R /// and Windows-1251 the ones older Russian dictionaries use). +/// +/// Under a UTF-8 `SET` with single-byte flags (no `FLAG` line, or `FLAG DEFAULT`), flags are bytes, +/// as in Hunspell: the Hungarian dictionary writes its affix flags and flag aliases as bytes above +/// 0x7F that aren't UTF-8. Those fields are read byte by byte (each byte becomes U+0000..U+00FF, +/// one flag per byte); the words, affixes and conditions around them are UTF-8. With any other +/// `FLAG` (`UTF-8`, `num`, `long`) flags are text and the whole file is UTF-8: `SFX 1234` keeps +/// its number (the Turkish dictionary's `FLAG num`). pub fn decode(aff: &[u8], dic: &[u8]) -> Result<(String, String), String> { - let head = String::from_utf8_lossy(aff.get(..aff.len().min(8192)).unwrap_or_default()).to_string(); - let set = head.lines().find_map(|l| l.trim_start_matches('\u{feff}').strip_prefix("SET ")).map(str::trim).unwrap_or("UTF-8").to_ascii_uppercase(); + let (set, flag) = directives(aff); + let set = set.unwrap_or_else(|| "UTF-8".into()); match set.as_str() { "UTF-8" | "UTF8" => { - let text = |b: &[u8], what: &str| String::from_utf8(b.to_vec()).map_err(|e| format!("{what}: not UTF-8: {e}")); - Ok((text(aff, ".aff")?, text(dic, ".dic")?)) + if flag.as_deref().is_some_and(|f| f != "DEFAULT") { + let text = |b: &[u8], what: &str| String::from_utf8(b.to_vec()).map_err(|e| format!("{what}: not UTF-8: {e}")); + return Ok((text(aff, ".aff")?, text(dic, ".dic")?)); + } + Ok((decode_utf8_aff(aff)?, decode_utf8_dic(dic)?)) } "ISO8859-1" | "ISO-8859-1" => Ok((decode_latin1(aff), decode_latin1(dic))), "ISO8859-2" | "ISO-8859-2" => Ok((decode_latin2(aff), decode_latin2(dic))), @@ -235,6 +245,225 @@ pub fn decode(aff: &[u8], dic: &[u8]) -> Result<(String, String), String> { } } +/// The first arguments of the `.aff`'s first `SET` and `FLAG` lines, in capitals, found in one pass +/// over the whole file: a `FLAG UTF-8` after the rules still decides how the `.dic` flags are read, +/// as in Hunspell. Only the first argument counts: `FLAG UTF-8 # Unicode flags` is UTF-8. +fn directives(aff: &[u8]) -> (Option, Option) { + let (mut set, mut flag) = (None, None); + for l in aff.split(|&b| b == b'\n') { + if set.is_some() && flag.is_some() { + break; + } + let l = l.strip_prefix("\u{feff}".as_bytes()).unwrap_or(l); + let slot = match l.first() { + Some(b'S') if set.is_none() => (&mut set, &b"SET"[..]), + Some(b'F') if flag.is_none() => (&mut flag, &b"FLAG"[..]), + _ => continue, + }; + let arg = l + .strip_prefix(slot.1) + .filter(|rest| rest.first().is_some_and(|b| matches!(b, b' ' | b'\t'))) + .and_then(|rest| rest.split(|b| matches!(b, b' ' | b'\t' | b'\r')).find(|f| !f.is_empty())); + if let Some(a) = arg { + *slot.0 = Some(String::from_utf8_lossy(a).to_ascii_uppercase()); + } + } + (set, flag) +} + +/// Directives whose one argument is a flag. +const FLAG_DIRECTIVES: &[&str] = &[ + "CIRCUMFIX", + "COMPOUNDBEGIN", + "COMPOUNDEND", + "COMPOUNDFLAG", + "COMPOUNDFORBIDFLAG", + "COMPOUNDLAST", + "COMPOUNDFIRST", + "COMPOUNDMIDDLE", + "COMPOUNDPERMITFLAG", + "COMPOUNDROOT", + "FORBIDDENWORD", + "FORCEUCASE", + "GENERATE", + "HU_KOTOHANGZO", + "KEEPCASE", + "LEMMA_PRESENT", + "NEEDAFFIX", + "NOSUGGEST", + "ONLYINCOMPOUND", + "ONLYROOT", + "PSEUDOROOT", + "SUBSTANDARD", + "WARN", +]; + +/// Directives that only describe the dictionary; a non-UTF-8 byte in them (the Hungarian +/// dictionary's `NAME` is ISO 8859-2) is replaced instead of failing the load. +const METADATA_DIRECTIVES: &[&str] = &["NAME", "HOME", "VERSION", "LANG", "AM"]; + +/// `text/flags`: the text as UTF-8 and the flags byte by byte (`\/` is a literal slash). `what` +/// names the line in an error. +fn text_and_flags(field: &[u8], what: impl Fn() -> String) -> Result { + let mut slash = None; + let mut i = 0; + while let Some(&b) = field.get(i) { + if b == b'\\' && field.get(i + 1) == Some(&b'/') { + i += 2; + continue; + } + if b == b'/' { + slash = Some(i); + break; + } + i += 1; + } + let (text, flags) = match slash { + Some(k) => (field.get(..k).unwrap_or_default(), field.get(k..).unwrap_or_default()), + None => (field, &[][..]), + }; + let mut out = std::str::from_utf8(text).map_err(|e| format!("{}: not UTF-8: {e}", what()))?.to_string(); + out.push_str(&decode_latin1(flags)); + Ok(out) +} + +/// Split a line into fields on spaces and tabs, keeping the separators (so a line rebuilt from +/// its parts is the original). +fn fields(line: &[u8]) -> Vec<&[u8]> { + let mut out = Vec::new(); + let mut start = 0; + let mut in_sep = line.first().is_some_and(|b| matches!(b, b' ' | b'\t')); + for (i, b) in line.iter().enumerate() { + let sep = matches!(b, b' ' | b'\t'); + if sep != in_sep { + out.push(line.get(start..i).unwrap_or_default()); + start = i; + in_sep = sep; + } + } + out.push(line.get(start..).unwrap_or_default()); + out +} + +/// A UTF-8 `.aff` with single-byte flags (see [`decode`]). +fn decode_utf8_aff(aff: &[u8]) -> Result { + let mut out = String::with_capacity(aff.len()); + for (n, line) in aff.split(|&b| b == b'\n').enumerate() { + if n > 0 { + out.push('\n'); + } + if let Ok(s) = std::str::from_utf8(line) + && s.is_ascii() + { + out.push_str(s); + continue; + } + let parts = fields(line); + // Field index among the non-separator parts. + let words: Vec = + parts.iter().enumerate().filter(|(_, p)| !p.first().is_some_and(|b| matches!(b, b' ' | b'\t'))).map(|(i, _)| i).collect(); + let dir = words + .first() + .and_then(|&i| parts.get(i)) + .map(|p| String::from_utf8_lossy(p).trim_start_matches('\u{feff}').to_string()) + .unwrap_or_default(); + let what = || format!(".aff line {}", n + 1); + if dir.starts_with('#') || METADATA_DIRECTIVES.contains(&dir.as_str()) { + out.push_str(&String::from_utf8_lossy(line)); + continue; + } + // A trailing `# comment` (the Hungarian `AF` lines number themselves) is never a word or + // a flag; from there on the line is kept as text. It can only start after the directive's + // own fields, since a field such as an affix condition may itself begin with `#`. + let field_text = |k: usize| words.get(k).and_then(|&w| parts.get(w)).map(|p| String::from_utf8_lossy(p).into_owned()); + let first_free = comment_field(&dir, field_text); + let comment_at = words.iter().skip(first_free).position(|&w| parts.get(w).is_some_and(|f| f.first() == Some(&b'#'))).map(|k| k + first_free); + // The field index of the next non-separator part. + let mut next_field = 0; + for (i, p) in parts.iter().enumerate() { + let field = (!p.first().is_some_and(|b| matches!(b, b' ' | b'\t'))).then(|| { + next_field += 1; + next_field - 1 + }); + let in_comment = comment_at.is_some_and(|c| words.get(c).is_some_and(|&w| i >= w)); + let decoded = match (dir.as_str(), field) { + _ if in_comment => String::from_utf8_lossy(p).into_owned(), + ("AF" | "COMPOUNDRULE" | "SYLLABLENUM", Some(1)) => decode_latin1(p), + ("PFX" | "SFX", Some(3)) | ("CHECKCOMPOUNDPATTERN", Some(1 | 2)) => text_and_flags(p, what)?, + // A field that names one flag keeps its first byte, as Hunspell's `decode_flag` + // does: `SFX é` is the flag 0xC3, the first of the two flags `kert/é` carries. + ("PFX" | "SFX", Some(1)) => decode_latin1(p.get(..1).unwrap_or_default()), + (d, Some(1)) if FLAG_DIRECTIVES.contains(&d) => decode_latin1(p.get(..1).unwrap_or_default()), + _ => std::str::from_utf8(p).map_err(|e| format!("{}: not UTF-8: {e}", what()))?.to_string(), + }; + out.push_str(&decoded); + } + } + Ok(out) +} + +/// The first field of an `.aff` line where a trailing `# comment` may start: after the +/// directive's own fields. A table's count line (`REP 12`, `PFX A Y 3`) has fewer fields than its +/// rows (`REP a b`, `PFX A 0 x .`). `field(k)` is the line's `k`-th field. +fn comment_field(dir: &str, field: impl Fn(usize) -> Option) -> usize { + let number = |k: usize| field(k).is_some_and(|f| f.parse::().is_ok()); + match dir { + "PFX" | "SFX" if field(2).is_some_and(|f| f == "Y" || f == "N") && number(3) => 4, + "PFX" | "SFX" => 5, + "REP" | "ICONV" | "OCONV" | "PHONE" | "MAP" | "BREAK" | "AF" | "AM" | "COMPOUNDRULE" | "CHECKCOMPOUNDPATTERN" + if number(1) && field(2).is_none_or(|f| f.starts_with('#')) => + { + 2 + } + "REP" | "ICONV" | "OCONV" | "PHONE" => 3, + "CHECKCOMPOUNDPATTERN" => 4, + _ => 2, + } +} + +/// Where the `word/flags` entry of a `.dic` line ends, as Hunspell reads it: at a tab or at the +/// first morphological field (a space or tab, two characters and `:`, as in ` po:noun`), whichever +/// comes first, so an entry may hold spaces (`ad hoc`); with neither, at the end of the line +/// (`kot/B 12` has the flags `B 12`, as in Hunspell). +pub(crate) fn dic_entry_end(line: &[u8]) -> usize { + let blank = |b: &u8| matches!(b, b' ' | b'\t'); + let morph = line.windows(4).position(|w| matches!(w, [a, b, c, b':'] if blank(a) && !blank(b) && !blank(c))); + let tab = line.iter().position(|&b| b == b'\t'); + let end = match (morph, tab) { + (Some(m), Some(t)) => m.min(t), + (m, t) => match m.or(t) { + Some(e) => e, + None => line.len(), + }, + }; + // Blanks before the morphological field or the line end aren't part of the entry. + line.get(..end).map_or(end, |e| e.iter().rposition(|b| !blank(b)).map_or(0, |k| k + 1)) +} + +/// A UTF-8 `.dic` with single-byte flags: the word is UTF-8, the flags after `/` are bytes. +fn decode_utf8_dic(dic: &[u8]) -> Result { + let mut out = String::with_capacity(dic.len()); + for (n, line) in dic.split(|&b| b == b'\n').enumerate() { + if n > 0 { + out.push('\n'); + } + if let Ok(s) = std::str::from_utf8(line) + && s.split_once('/').is_none_or(|(_, flags)| flags.is_ascii()) + { + out.push_str(s); + continue; + } + let what = || format!(".dic line {}", n + 1); + // The first field is `word/flags` (it may hold spaces, see [`dic_entry_end`]); + // morphological fields after it are UTF-8 text. + let end = dic_entry_end(line); + out.push_str(&text_and_flags(line.get(..end).unwrap_or_default(), what)?); + let rest = line.get(end..).unwrap_or_default(); + out.push_str(std::str::from_utf8(rest).map_err(|e| format!("{}: not UTF-8: {e}", what()))?); + } + Ok(out) +} + #[cfg(test)] mod tests { use super::*; @@ -280,6 +509,78 @@ mod tests { assert!(decode(b"SET ISO-8859-1\n", b"\xe7").is_ok()); } + #[test] + fn utf8_dictionaries_keep_single_byte_flags() { + // The Hungarian dictionary is `SET UTF-8` but writes flags as single bytes above 0x7F, + // which aren't UTF-8; each byte is one flag. Its `NAME` line is ISO 8859-2. + let aff = b"SET UTF-8\nNAME Magyar helyes\xedr\xe1si\nAF 2\nAF \xd5a # 1\nAF \xe9 # 2\nSFX \xd5 Y 1\nSFX \xd5 0 ok/2 . 7\nSFX \xe9 Y 1\nSFX \xe9 0 ban .\nSFX a Y 1\nSFX a 0 at .\n"; + let dic = "2\nház/1\t1\nkert/\u{e9}\n".as_bytes(); + let (a, d) = decode(aff, dic).unwrap(); + assert!(a.contains("AF \u{d5}a # 1") && a.contains("AF \u{e9} # 2") && a.contains("SFX \u{d5} 0 ok/2 . 7")); + assert!(d.contains("kert/\u{c3}\u{a9}"), "a UTF-8 é in the flags is two one-byte flags"); + let dict = crate::affix::AffixDict::from_hunspell(&a, &d, &[]).unwrap(); + assert!(dict.check("ház") && dict.check("házok") && dict.check("házokban") && dict.check("házat")); + assert!(!dict.check("házban")); + // `FLAG UTF-8` keeps the whole file UTF-8, and words must still be UTF-8. + let (_, d) = decode(b"SET UTF-8\nFLAG UTF-8\n", "kert/\u{e9}\n".as_bytes()).unwrap(); + assert_eq!(d, "kert/\u{e9}\n"); + assert!(decode(b"SET UTF-8\nSFX a Y 1\nSFX a 0 \xe9 .\n", b"").is_err(), "an affix must be UTF-8"); + assert!(decode(b"SET UTF-8\n", b"1\nh\xe1z/a\n").is_err(), "a stem must be UTF-8"); + } + + #[test] + fn single_flag_fields_keep_their_first_byte() { + // Without `FLAG UTF-8`, a UTF-8 `é` (0xC3 0xA9) in a field that names one flag is the + // flag 0xC3; in a `.dic` flag list it is the two flags 0xC3 and 0xA9. + let aff = "SET UTF-8\nKEEPCASE é\nSFX é Y 1\nSFX é 0 ok .\nSYLLABLENUM é\n".as_bytes(); + let (a, d) = decode(aff, "2\nkert/é\nNATO/é\n".as_bytes()).unwrap(); + assert!(a.contains("KEEPCASE \u{c3}\n") && a.contains("SFX \u{c3} Y 1") && a.contains("SFX \u{c3} 0 ok ."), "{a}"); + assert!(a.contains("SYLLABLENUM \u{c3}\u{a9}"), "SYLLABLENUM lists flags: {a}"); + assert!(d.contains("kert/\u{c3}\u{a9}")); + let dict = crate::affix::AffixDict::from_hunspell(&a, &d, &[]).unwrap(); + assert!(dict.check("kert") && dict.check("kertok")); + assert!(dict.check("NATO") && !dict.check("Nato"), "KEEPCASE é reaches NATO/é"); + } + + #[test] + fn numeric_and_long_flags_stay_text() { + // `FLAG num` (the Turkish dictionary) and `FLAG long`: no byte-wise flag reading, so a + // number with a non-ASCII rule or comment keeps every digit. + let aff = "SET UTF-8\nFLAG num\nFORBIDDENWORD 6000 # tiltott ő\nSFX 1234 Y 1\nSFX 1234 0 ları .\n"; + let (a, d) = decode(aff.as_bytes(), "1\nkitap/1234\n".as_bytes()).unwrap(); + assert_eq!(a, aff); + assert_eq!(d, "1\nkitap/1234\n"); + let tr = crate::tr::dict::TrDict::from_hunspell(&a, &d).unwrap(); + assert!(tr.contains("kitapları") && tr.contains("kitap")); + let long = "SET UTF-8\nFLAG long\nSFX Aa Y 1\nSFX Aa 0 ős .\n"; + assert_eq!(decode(long.as_bytes(), "1\nkő/Aa\n".as_bytes()).unwrap().0, long); + assert!(decode(b"SET UTF-8\nFLAG num\n", b"1\nh\xe1z/1\n").is_err(), "the .dic must be UTF-8"); + // `FLAG DEFAULT` is the single-byte default. + let (_, d) = decode(b"SET UTF-8\nFLAG default\n", b"1\nkert/\xe9\n").unwrap(); + assert_eq!(d, "1\nkert/\u{e9}\n"); + } + + #[test] + fn flag_utf8_counts_anywhere_in_the_aff() { + // A `FLAG UTF-8` after a long rule table still makes the `.dic` flags Unicode. + let mut aff = b"SET UTF-8\nSFX a Y 400\n".to_vec(); + for _ in 0..400 { + aff.extend_from_slice(b"SFX a 0 aaaaaaaaaaaaaaaaaaaaaaaa .\n"); + } + aff.extend_from_slice(b"FLAG UTF-8\r\n"); + assert!(aff.len() > 8192); + let (_, d) = decode(&aff, "kert/\u{e9}\n".as_bytes()).unwrap(); + assert_eq!(d, "kert/\u{e9}\n"); + } + + #[test] + fn a_comment_after_flag_utf8_keeps_unicode_flags() { + let (_, d) = decode(b"SET UTF-8\nFLAG UTF-8 # Unicode flags\n", "kert/\u{e9}\n".as_bytes()).unwrap(); + assert_eq!(d, "kert/\u{e9}\n"); + let (_, d) = decode(b"SET\tUTF-8 # comment\nFLAG\t\tUTF-8\n", "kert/\u{e9}\n".as_bytes()).unwrap(); + assert_eq!(d, "kert/\u{e9}\n", "repeated spaces and tabs separate the argument"); + } + #[test] fn decodes_russian_single_byte_encodings() { // `ёлка` in KOI8-R (0xA3 0xCC 0xCB 0xC1) and in Windows-1251 (0xB8 0xEB 0xEA 0xE0). @@ -291,6 +592,41 @@ mod tests { assert_eq!(decode_high(&[0x98], CP1251_HIGH), "\u{fffd}"); } + #[test] + fn comments_start_after_the_directive_fields() { + let f = |line: &'static str| move |k: usize| line.split_whitespace().nth(k).map(str::to_string); + assert_eq!(comment_field("PFX", f("PFX A Y 3 # count")), 4); + assert_eq!(comment_field("SFX", f("SFX A 0 # .")), 5, "a rule's condition may be `#`"); + assert_eq!(comment_field("AF", f("AF 1234 # aliases")), 2); + assert_eq!(comment_field("AF", f("AF Aa # 1")), 2); + assert_eq!(comment_field("REP", f("REP 12")), 2); + assert_eq!(comment_field("REP", f("REP # x")), 3); + assert_eq!(comment_field("CHECKCOMPOUNDPATTERN", f("CHECKCOMPOUNDPATTERN a b c")), 4); + assert_eq!(comment_field("KEEPCASE", f("KEEPCASE k")), 2); + // A byte flag after an append text that begins with `#` stays a flag, not a comment. + let (a, _) = decode(b"SET UTF-8\nSFX \xd5 Y 1 # header\nSFX \xd5 0 #/\xe9 . # rule\n", b"").unwrap(); + assert!(a.contains("SFX \u{d5} 0 #/\u{e9} . # rule"), "{a}"); + } + + #[test] + fn dic_entries_end_where_hunspell_ends_them() { + let end = |l: &str| l.get(..dic_entry_end(l.as_bytes())).unwrap().to_string(); + assert_eq!(end("word/A po:noun\tts:x"), "word/A", "the morphological field comes before the tab"); + assert_eq!(end("ad hoc/B\tpo:adv"), "ad hoc/B", "a tab ends a phrase"); + assert_eq!(end("ad hoc/B po:adv"), "ad hoc/B"); + assert_eq!(end("word/A 123"), "word/A 123", "neither: the whole line, as in Hunspell"); + assert_eq!(end("ad hoc/B "), "ad hoc/B"); + assert_eq!(end("word"), "word"); + assert_eq!(end("a:b/A"), "a:b/A", "a colon inside the word isn't a field"); + // Single-byte flags of a phrase entry are read byte by byte, then the UTF-8 morphology. + let (_, d) = decode(b"SET UTF-8\n", b"1\nad hoc/\xe9\tpo:adv\n").unwrap(); + assert_eq!(d, "1\nad hoc/\u{e9}\tpo:adv\n"); + // Without a tab or a morphological field the whole line is the entry: its byte flags + // don't fail the load (the Hungarian dictionary has such lines: `tök jó/396`). + let (_, d) = decode(b"SET UTF-8\n", b"1\nad hoc/\xe9\n").unwrap(); + assert_eq!(d, "1\nad hoc/\u{e9}\n"); + } + #[test] fn libreoffice_extension_dirs_follow_the_language() { // LibreOffice's IDs are per dictionary: Polish is `dict-pl`, Brazilian Portuguese