collective
Browse / github.com/storytold/wordcraft / patch
proof: flag aliases, single-byte flags, ICONV, IGNORE and KEEPCASE in affix dictionaries (#735)
proof: flag aliases, single-byte flags, ICONV, IGNORE and KEEPCASE in affix dictionaries (#735) Supersedes #1859 (same commits, rebased on current `main`; the review history is there). First of three steps for Hungarian spell checking (#735); compounding (#2070) and the Hungarian module (#2385) follow once this lands. ## What changes The Hungarian Hunspell dictionary (`hu_HU`, Magyar Ispell) uses affix-file features the engine didn't read. This PR adds them; compounding and `NEEDAFFIX` stay refused, so `hu_HU` doesn't load yet. **Reading the files** (`hunspell.rs`) - Flags are read byte by byte when the `.aff` has no `FLAG` line or `FLAG DEFAULT`, as Hunspell does: `hu_HU` says `SET U
- Status
- open
- Score
- 0.00
- Adopters
- 0 other operators
- Author
- Skaywalker (imported from pull request #4575, open)
- Base
2cf1d6c7ae94723a45bd6e7e8c884fe375148750- Size
- +835 −113 in 2 files
- Review
- pending
- Shared
- 2026-10-28
- Id
aa49536a3e0868c8a85201f0
How it went
No other operator has reported using it yet.
Diff
raw · crates/proof/src/affix.rs crates/proof/src/hunspell.rs
@@ -11,10 +11,14 @@ //! also be stored in WordCraft's compact format ([`AffixDict::to_bytes`]): the rules as text //! lines and the stems front-coded, raw-deflated together (the Polish dictionary is under 1 MB). //!-//! Supported: one-character flags (at most 128 distinct, any Unicode character — `FLAG UTF-8`),-//! `PFX`/`SFX` with strip, append and condition (`.`, letters, `[abc]`, `[^abc]`), cross-//! products, continuation classes on suffixes (at most two suffixes in a row, as in Hunspell),-//! `FORBIDDENWORD`, `NOSUGGEST`, and the suggestion tables `TRY`, `KEY`, `MAP` and `REP`. Rejected: compounding,+//! Supported: one-character flags (at most 256 distinct, any Unicode character — `FLAG UTF-8`),+//! flag aliases (`AF`: a number standing for a set of flags), `PFX`/`SFX` with strip, append and+//! condition (`.`, letters, `[abc]`, `[^abc]`), cross products, continuation classes on suffixes+//! (at most two suffixes in a row, as in Hunspell), `FORBIDDENWORD`, `NOSUGGEST`, `KEEPCASE` (the+//! stem is a word only in its own letter case), input conversion (`ICONV`: ligatures such as `fi`+//! read as `fi`) and ignored characters (`IGNORE`), and the suggestion tables `TRY`, `KEY`, `MAP`+//! and `REP`.+//! Morphological data (`AM` aliases, the fields after a stem) is skipped. Rejected: compounding, //! long or numeric flags, `NEEDAFFIX` and the other directives that change which words are correct. //! //! Lookups never panic: stems sit in one string, found through an open-addressing hash table, and@@ -33,30 +37,91 @@ const EMPTY: u32 = u32::MAX; /// Directives that change which words are correct in ways this engine doesn't implement. const UNSUPPORTED: &[&str] = &[- "AF",- "AM", "CHECKSHARPS", "CIRCUMFIX", "COMPLEXPREFIXES", "COMPOUNDBEGIN", "COMPOUNDEND",+ "COMPOUNDFIRST", "COMPOUNDFLAG",+ "COMPOUNDLAST", "COMPOUNDMIDDLE", "COMPOUNDMIN", "COMPOUNDPERMITFLAG", "COMPOUNDRULE", "FULLSTRIP",- "ICONV",- "IGNORE",- "KEEPCASE",- "LEMMA_PRESENT", "NEEDAFFIX", "OCONV", "ONLYINCOMPOUND", "PSEUDOROOT",- "SUBSTANDARD", ]; +/// Most distinct flags a dictionary may use: every single-byte flag (the Hungarian dictionary+/// uses 158).+const MAX_FLAGS: usize = 256;++/// A set of flags, one bit per distinct flag character of the dictionary.+#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)]+struct Mask([u64; MAX_FLAGS / 64]);++impl Mask {+ const EMPTY: Mask = Mask([0; MAX_FLAGS / 64]);++ /// The set holding flag number `i` alone (empty when `i` is out of range).+ fn bit(i: usize) -> Mask {+ let mut m = Mask::EMPTY;+ if let Some(w) = m.0.get_mut(i / 64) {+ *w = 1 << (i % 64);+ }+ m+ }++ /// The number of the lowest flag in the set, if any.+ fn first(self) -> Option<usize> {+ self.0.iter().enumerate().find(|(_, w)| **w != 0).map(|(i, w)| i * 64 + w.trailing_zeros() as usize)+ }++ fn is_empty(self) -> bool {+ self.0.iter().all(|&w| w == 0)+ }++ /// Do the two sets share a flag?+ fn meets(self, other: Mask) -> bool {+ !(self & other).is_empty()+ }++ /// Does this set hold every flag of `other`?+ fn covers(self, other: Mask) -> bool {+ (self & other) == other+ }+}++impl std::ops::BitAnd for Mask {+ type Output = Mask;+ fn bitand(mut self, o: Mask) -> Mask {+ for (a, b) in self.0.iter_mut().zip(o.0) {+ *a &= b;+ }+ self+ }+}++impl std::ops::BitOr for Mask {+ type Output = Mask;+ fn bitor(mut self, o: Mask) -> Mask {+ self |= o;+ self+ }+}++impl std::ops::BitOrAssign for Mask {+ fn bitor_assign(&mut self, o: Mask) {+ for (a, b) in self.0.iter_mut().zip(o.0) {+ *a |= b;+ }+ }+}+ /// One character class of a rule's condition. #[derive(Clone, Debug, PartialEq, Eq)] enum Cond {@@ -130,14 +195,14 @@ const MAX_FORMS_WORK: usize = 1 << 18; #[derive(Clone, Debug)] struct Rule { /// Bit of the rule's flag in the stems' flag masks.- bit: u128,+ bit: Mask, flag: char, cross: bool, strip: Box<str>, add: Box<str>, /// Continuation classes (`SFX A 0 able/B`): flags the derived form carries, licensing /// further affixes. Chained application is bounded by [`MAX_CHAIN`].- cont: u128,+ cont: Mask, cond: Box<[Cond]>, cond_text: Box<str>, }@@ -174,11 +239,13 @@ pub struct AffixDict { /// Per stem: index into `sets`. flags: Vec<u16>, /// Distinct flag masks.- sets: Vec<u128>,+ sets: Vec<Mask>, /// The `FORBIDDENWORD` flag's bit; stems carrying it are not words (0: none declared).- forbidden: u128,+ forbidden: Mask, /// Stems with NOSUGGEST are correct, but none of their forms are suggested.- no_suggest: u128,+ no_suggest: Mask,+ /// Stems with SUBSTANDARD are correct too, and likewise never suggested.+ substandard: Mask, /// Open-addressing hash table of stem indices (`EMPTY` = free). table: Vec<u32>, /// Flag char of each bit.@@ -195,6 +262,12 @@ pub struct AffixDict { key: String, map: Vec<Vec<char>>, rep: Vec<(String, String)>,+ /// The `KEEPCASE` flag's bit: such stems are words only in their own letter case (0: none).+ keepcase: Mask,+ /// `ICONV`: input conversions applied before a lookup, `(from, to)`.+ iconv: Vec<(String, String)>,+ /// `IGNORE`: characters removed from input words (and from the stems and appended texts).+ ignore: Box<[char]>, /// Provenance and licence lines, kept through [`AffixDict::to_bytes`]. comments: Vec<String>, }@@ -263,6 +336,57 @@ struct Rules { /// The `FORBIDDENWORD` flag: stems carrying it are listed but never correct. forbidden: Option<char>, no_suggest: Option<char>,+ substandard: Option<char>,+ /// The `KEEPCASE` flag.+ keepcase: Option<char>,+ iconv: Vec<(String, String)>,+ ignore: Vec<char>,+ /// `AF` flag aliases: alias `n` (1-based) stands for `aliases[n - 1]`.+ aliases: Vec<String>,+ /// The `AF` table opened with its count line, so flag fields are alias numbers. A table+ /// without one is rejected, as in Hunspell: its lines are ignored.+ aliases_on: bool,+}++impl Rules {+ /// Flags as written in a `.dic` entry or a continuation class: the flag characters+ /// themselves, or with an `AF` table the alias its leading digits number, as Hunspell reads+ /// them (`2x` is alias 2). A field without digits or an undefined alias reads as no flags:+ /// the stem is still a word, the rule still applies, without the flags (Hunspell warns).+ fn flags<'a>(&'a self, written: &'a str) -> &'a str {+ if !self.aliases_on {+ return written;+ }+ let digits = written.bytes().take_while(u8::is_ascii_digit).count();+ written+ .get(..digits)+ .and_then(|d| d.parse::<usize>().ok())+ .and_then(|n| n.checked_sub(1))+ .and_then(|i| self.aliases.get(i))+ .map_or("", String::as_str)+ }++ /// `w` without the `IGNORE` characters.+ fn strip_ignored(&self, w: &str) -> String {+ if self.ignore.is_empty() { w.to_string() } else { w.chars().filter(|c| !self.ignore.contains(c)).collect() }+ }+}++/// The fields of an `.aff`/`.dic` line: Hunspell separates them with spaces and tabs only, so a+/// flag byte read as U+0085 or U+00A0 (see [`crate::hunspell::decode`]) stays a flag.+fn split_fields(line: &str) -> impl Iterator<Item = &str> {+ line.split([' ', '\t']).filter(|f| !f.is_empty())+}++/// The one flag character a directive (`FORBIDDENWORD w`) names.+fn one_flag(arg: &str, dir: &str, n: usize) -> Result<char, String> {+ let mut it = arg.chars().filter(|&c| c != EMOJI_SELECTOR);+ match (it.next(), it.next()) {+ // Only the field separators of `split_fields` can't be a flag: a flag byte read as+ // U+0085 or U+00A0 is one.+ (Some(c), None) if !matches!(c, ' ' | '\t') => Ok(c),+ _ => Err(format!("line {}: {dir} needs one flag character", n + 1)),+ } } /// A rule as parsed from text, before flags become bits: `(flag, cross, strip, add, cont, cond)`.@@ -304,8 +428,10 @@ fn parse_aff(aff: &str) -> Result<Rules, String> { let mut cross: HashMap<(bool, char), bool> = HashMap::new(); let mut map_header = false; let mut rep_header = false;+ let mut af_header = false;+ let mut iconv_header = false; for (n, line) in aff.lines().enumerate() {- let f: Vec<&str> = line.split_whitespace().collect();+ let f: Vec<&str> = split_fields(line).collect(); let Some(&dir) = f.first() else { continue }; if dir.starts_with('#') { continue;@@ -314,25 +440,38 @@ fn parse_aff(aff: &str) -> Result<Rules, String> { match dir { "SET" | "LANG" | "NAME" | "VERSION" | "HOME" | "WORDCHARS" | "MAXNGRAMSUGS" | "MAXDIFF" | "ONLYMAXDIFF" | "NOSPLITSUGS" | "SUGSWITHDOTS" | "WARN" | "FORBIDWARN" | "MAXCPDSUGS" => {}+ // Morphological aliases, and a flag that only matters for morphological generation:+ // the words it marks are correct.+ "AM" | "LEMMA_PRESENT" => {}+ "SUBSTANDARD" => r.substandard = Some(one_flag(at(1), dir, n)?),+ // The first `AF` or `ICONV` line is the table's count, as in Hunspell, even with a+ // trailing comment (`AF 1559 # aliases`).+ "AF" => {+ if !af_header {+ af_header = true;+ // A count below 1 rejects the table too: flags stay literal (`AF 0`).+ r.aliases_on = at(1).parse::<usize>().is_ok_and(|n| n > 0);+ } else if r.aliases_on {+ r.aliases.push(at(1).to_string());+ }+ }+ "ICONV" => {+ if !iconv_header && at(1).parse::<usize>().is_ok() {+ iconv_header = true;+ } else if f.len() >= 3 {+ r.iconv.push((at(1).to_string(), at(2).to_string()));+ }+ }+ "IGNORE" => r.ignore = at(1).chars().collect(),+ "KEEPCASE" => r.keepcase = Some(one_flag(at(1), dir, n)?), // Per-character flag modes (`FLAG UTF-8`, as VERO's pt_BR dictionary uses) work // with the one-character parsing here; only the multi-character encodings differ. "FLAG" if !matches!(at(1), "UTF-8" | "DEFAULT" | "utf-8" | "default" | "UTF8" | "") => { return Err(format!("line {}: FLAG {} is not supported (one-character flags only)", n + 1, at(1))); } "FLAG" => {}- directive @ ("FORBIDDENWORD" | "NOSUGGEST") => {- let mut it = at(1).chars().filter(|&c| c != EMOJI_SELECTOR);- match (it.next(), it.next()) {- (Some(c), None) if !c.is_whitespace() => {- if directive == "FORBIDDENWORD" {- r.forbidden = Some(c);- } else {- r.no_suggest = Some(c);- }- }- _ => return Err(format!("line {}: {directive} needs one flag character", n + 1)),- }- }+ "FORBIDDENWORD" => r.forbidden = Some(one_flag(at(1), dir, n)?),+ "NOSUGGEST" => r.no_suggest = Some(one_flag(at(1), dir, n)?), "TRY" => r.try_chars = cap_try(at(1)), "KEY" => r.key = at(1).to_string(), "MAP" => {@@ -343,7 +482,7 @@ fn parse_aff(aff: &str) -> Result<Rules, String> { } } "REP" => {- if !rep_header && f.len() == 2 && at(1).parse::<usize>().is_ok() {+ if !rep_header && at(1).parse::<usize>().is_ok() { rep_header = true; } else if f.len() >= 3 { r.rep.push((at(1).replace('_', " "), at(2).replace('_', " ")));@@ -383,6 +522,19 @@ fn parse_aff(aff: &str) -> Result<Rules, String> { _ => {} } }+ // Aliases may be declared after the rules that use them; resolve once all are read.+ // Hunspell removes `IGNORE` characters from the appended text only: a stripped text that+ // holds one never matches a stem (stems lose them too), so the rule never applies.+ let resolve = |rules: &mut Vec<RawRule>, r: &Rules| {+ for rule in rules.iter_mut() {+ rule.4 = r.flags(&rule.4).to_string();+ rule.3 = r.strip_ignored(&rule.3);+ }+ };+ let (mut pfx, mut sfx) = (std::mem::take(&mut r.prefixes), std::mem::take(&mut r.suffixes));+ resolve(&mut pfx, &r);+ resolve(&mut sfx, &r);+ (r.prefixes, r.suffixes) = (pfx, sfx); Ok(r) } @@ -397,7 +549,12 @@ impl AffixDict { let rules = parse_aff(aff)?; let mut stems: Vec<(String, String)> = Vec::new(); for (n, line) in dic.lines().enumerate() {- let Some(tok) = line.split_whitespace().next() else { continue };+ // The entry may hold spaces (`ad hoc/B\tpo:adv`); it ends where Hunspell ends it.+ // Leading blanks belong to it, as in Hunspell (` lab` is no `lab`).+ let tok = line.get(..crate::hunspell::dic_entry_end(line.as_bytes())).unwrap_or_default();+ if tok.is_empty() {+ continue;+ } if n == 0 && tok.parse::<usize>().is_ok() { continue; }@@ -420,9 +577,11 @@ impl AffixDict { word.push(c); } }+ let word = rules.strip_ignored(&word); if word.is_empty() || word.len() > MAX_WORD_BYTES || word.contains(['\n', '/']) { continue; }+ let flags = rules.flags(&flags).to_string(); stems.push((word, flags)); } AffixDict::build(rules, stems, comments.iter().map(ToString::to_string).collect())@@ -434,11 +593,12 @@ impl AffixDict { flag_chars.extend(stems.iter().flat_map(|s| s.1.chars())); flag_chars.sort_unstable(); flag_chars.dedup();- if flag_chars.len() > 128 {- return Err(format!("{} distinct flags; at most 128 are supported", flag_chars.len()));+ if flag_chars.len() > MAX_FLAGS {+ return Err(format!("{} distinct flags; at most {MAX_FLAGS} are supported", flag_chars.len())); }- let bit_of = |c: char| flag_chars.iter().position(|&x| x == c).map_or(0, |i| 1u128 << i);- let forbidden = rules.forbidden.map_or(0, bit_of);+ let bit_of = |c: char| flag_chars.iter().position(|&x| x == c).map_or(Mask::EMPTY, Mask::bit);+ let forbidden = rules.forbidden.map_or(Mask::EMPTY, bit_of);+ let keepcase = rules.keepcase.map_or(Mask::EMPTY, bit_of); // Stems sorted by bytes; homonyms (the same word twice) are merged. stems.sort_by(|a, b| a.0.as_bytes().cmp(b.0.as_bytes())); let mut d = AffixDict {@@ -446,7 +606,11 @@ impl AffixDict { offs: Vec::with_capacity(stems.len() + 1), flag_chars: flag_chars.clone(), forbidden,- no_suggest: rules.no_suggest.map_or(0, bit_of),+ no_suggest: rules.no_suggest.map_or(Mask::EMPTY, bit_of),+ substandard: rules.substandard.map_or(Mask::EMPTY, bit_of),+ keepcase,+ iconv: rules.iconv,+ ignore: rules.ignore.into_boxed_slice(), try_chars: rules.try_chars, key: rules.key, map: rules.map,@@ -455,16 +619,16 @@ impl AffixDict { ..AffixDict::default() }; d.offs.push(0);- let mut set_ids: HashMap<u128, u16> = HashMap::new();+ let mut set_ids: HashMap<Mask, u16> = HashMap::new(); let mut i = 0; while i < stems.len() { let Some((word, _)) = stems.get(i) else { break };- let mut mask = 0u128;+ let mut mask = Mask::EMPTY; let mut j = i; while let Some((w, f)) = stems.get(j) && w == word {- mask |= f.chars().map(bit_of).fold(0, |a, b| a | b);+ mask |= f.chars().map(bit_of).fold(Mask::EMPTY, |a, b| a | b); j += 1; } let next_id = u16::try_from(d.sets.len()).map_err(|_| "too many distinct flag sets".to_string())?;@@ -478,7 +642,7 @@ impl AffixDict { i = j; } let mk = |(flag, cross, strip, add, cont, cond): RawRule| -> Result<Rule, String> {- let cont = cont.chars().fold(0u128, |m, c| m | bit_of(c));+ let cont = cont.chars().fold(Mask::EMPTY, |m, c| m | bit_of(c)); Ok(Rule { bit: bit_of(flag), flag, cross, strip: strip.into(), add: add.into(), cont, cond: parse_cond(&cond)?, cond_text: cond.into() }) }; d.prefixes = rules.prefixes.into_iter().map(mk).collect::<Result<_, _>>()?;@@ -493,7 +657,7 @@ impl AffixDict { // Each (appended, stripped) pair's group, found by hash rather than by scanning the // appended string's groups: a dictionary with many strips per suffix stays linear. let mut group_of: FxMap<(&str, &str), usize> = FxMap::default();- let mut by_flag: Vec<Vec<u32>> = vec![Vec::new(); 128];+ let mut by_flag: Vec<Vec<u32>> = vec![Vec::new(); MAX_FLAGS]; for (i, r) in self.suffixes.iter().enumerate() { let groups = idx.entry(r.add.clone()).or_default(); let Ok(i) = u32::try_from(i) else { break };@@ -504,9 +668,7 @@ impl AffixDict { groups.push(SfxGroup { strip: r.strip.clone(), rules: vec![i] }); } }- if r.bit != 0- && let Some(list) = by_flag.get_mut(r.bit.trailing_zeros() as usize)- {+ if let Some(list) = r.bit.first().and_then(|b| by_flag.get_mut(b)) { list.push(i); } }@@ -551,12 +713,12 @@ impl AffixDict { } /// The flag characters of `mask`, in flag order (how the compact format stores flags).- fn flag_chars_for(&self, mask: u128) -> String {- self.flag_chars.iter().enumerate().filter(|&(i, _)| mask & (1u128 << i) != 0).map(|(_, c)| c).collect()+ fn flag_chars_for(&self, mask: Mask) -> String {+ self.flag_chars.iter().enumerate().filter(|&(i, _)| mask.meets(Mask::bit(i))).map(|(_, c)| c).collect() } /// The character of `mask` when it holds exactly one flag.- fn flag_char(&self, mask: u128) -> Option<char> {+ fn flag_char(&self, mask: Mask) -> Option<char> { let s = self.flag_chars_for(mask); let mut it = s.chars(); match (it.next(), it.next()) {@@ -589,11 +751,12 @@ impl AffixDict { } /// Flag mask of a stem, if `w` is one.- fn stem_flags(&self, w: &str) -> Option<u128> {- self.stem_flags_excluding(w, self.forbidden)+ fn stem_flags(&self, w: &str) -> Option<Mask> {+ self.stem_flags_except(w, Mask::EMPTY) } - fn stem_flags_excluding(&self, w: &str, excluded: u128) -> Option<u128> {+ /// Flag mask of a stem, if `w` is one and carries none of the flags in `deny`.+ fn stem_flags_except(&self, w: &str, deny: Mask) -> Option<Mask> { if self.table.is_empty() { return None; }@@ -607,8 +770,9 @@ impl AffixDict { if self.word(t as usize) == Some(w) { let id = *self.flags.get(t as usize)?; let mask = self.sets.get(usize::from(id)).copied();- // All derivations use this lookup: exclusions apply to stems and their forms.- return mask.filter(|m| m & excluded == 0);+ // A stem carrying FORBIDDENWORD is listed but is not a word: no direct hit,+ // and (this is the only stem lookup) no affixed form is built on it either.+ return mask.filter(|m| !m.meets(self.forbidden | deny)); } slot = (slot + 1) & mask; }@@ -621,8 +785,8 @@ impl AffixDict { /// The rules of a group that could lead to `w` are merged into one mask of their flags and /// followed once, not once per rule: the answer is the same, but thousands of rules of one /// flag cost one walk, not one each (#3613).- fn form_has_flag(&self, w: &str, flags: u128, depth: u8, excluded: u128) -> bool {- if self.stem_flags_excluding(w, excluded).is_some_and(|f| f & flags != 0) {+ fn form_has_flag(&self, w: &str, flags: Mask, depth: u8, deny: Mask) -> bool {+ if self.stem_flags_except(w, deny).is_some_and(|f| f.meets(flags)) { return true; } if depth == 0 {@@ -641,9 +805,9 @@ impl AffixDict { .rules .iter() .filter_map(|&i| self.suffixes.get(i as usize))- .filter(|r| r.cont & flags != 0 && r.cond_matches(&root, true))- .fold(0u128, |m, r| m | r.bit);- if inner != 0 && self.form_has_flag(&root, inner, depth - 1, excluded) {+ .filter(|r| r.cont.meets(flags) && r.cond_matches(&root, true))+ .fold(Mask::EMPTY, |m, r| m | r.bit);+ if !inner.is_empty() && self.form_has_flag(&root, inner, depth - 1, deny) { return true; } }@@ -651,21 +815,55 @@ impl AffixDict { false } - /// Is `w` exactly (case and all) a stem or an affixed form of one?+ /// Is `w` exactly (case and all) a stem or an affixed form of one? `ICONV` and `IGNORE`+ /// apply first. pub fn contains(&self, w: &str) -> bool {- self.contains_excluding(w, self.forbidden)+ self.contains_except(&self.normalize(w), Mask::EMPTY) } - /// Exactly spelled dictionary words eligible for suggestions (NOSUGGEST stems excluded).+ /// Exactly spelled dictionary words eligible for suggestions (NOSUGGEST and SUBSTANDARD+ /// stems excluded). pub fn contains_suggestible(&self, w: &str) -> bool {- self.contains_excluding(w, self.forbidden | self.no_suggest)+ self.contains_except(&self.normalize(w), self.no_suggest | self.substandard)+ }++ /// `w` after the `ICONV` conversions (at each position the longest match wins) and without+ /// the `IGNORE` characters.+ fn normalize<'a>(&self, w: &'a str) -> std::borrow::Cow<'a, str> {+ if self.iconv.is_empty() && self.ignore.is_empty() {+ return std::borrow::Cow::Borrowed(w);+ }+ // One pass: the input is copied only from the first change on.+ let mut out: Option<String> = None;+ let mut rest = w;+ while let Some(c) = rest.chars().next() {+ let done = w.get(..w.len() - rest.len()).unwrap_or_default();+ let hit = self.iconv.iter().filter(|(from, _)| !from.is_empty() && rest.starts_with(from.as_str())).max_by_key(|(from, _)| from.len());+ match hit {+ Some((from, to)) => {+ out.get_or_insert_with(|| done.to_string()).extend(to.chars().filter(|c| !self.ignore.contains(c)));+ rest = rest.get(from.len()..).unwrap_or_default();+ }+ None => {+ if self.ignore.contains(&c) {+ out.get_or_insert_with(|| done.to_string());+ } else if let Some(o) = out.as_mut() {+ o.push(c);+ }+ rest = rest.get(c.len_utf8()..).unwrap_or_default();+ }+ }+ }+ out.map_or(std::borrow::Cow::Borrowed(w), std::borrow::Cow::Owned) } - fn contains_excluding(&self, w: &str, excluded: u128) -> bool {+ /// [`AffixDict::contains`] on normalized input, where stems carrying a flag in `deny` don't+ /// count (case variants of `KEEPCASE` stems).+ fn contains_except(&self, w: &str, deny: Mask) -> bool { if w.is_empty() || w.len() > MAX_WORD_BYTES { return false; }- if self.stem_flags_excluding(w, excluded).is_some() || self.suffixed(w, 0, excluded) {+ if self.stem_flags_except(w, deny).is_some() || self.suffixed(w, Mask::EMPTY, deny) { return true; } let mut root = String::with_capacity(w.len() + 8);@@ -680,13 +878,13 @@ impl AffixDict { if !r.cond_matches(&root, false) { continue; }- if self.stem_flags_excluding(&root, excluded).is_some_and(|f| f & r.bit != 0) {+ if self.stem_flags_except(&root, deny).is_some_and(|f| f.meets(r.bit)) { return true; }- if self.form_has_flag(&root, r.bit, INNER_CHAIN, excluded) {+ if self.form_has_flag(&root, r.bit, INNER_CHAIN, deny) { return true; }- if r.cross && self.suffixed(&root, r.bit, excluded) {+ if r.cross && self.suffixed(&root, r.bit, deny) { return true; } }@@ -695,7 +893,7 @@ impl AffixDict { /// Is `w` a stem with one suffix? With `need` set (a prefix was taken off), the stem must /// carry those flags too and the suffix must allow cross products.- fn suffixed(&self, w: &str, need: u128, excluded: u128) -> bool {+ fn suffixed(&self, w: &str, need: Mask, deny: Mask) -> bool { let mut root = String::with_capacity(w.len() + 8); // Each split point from the end: the appended string is `w[b..]`, at most the longest // suffix, and something must stay before it.@@ -710,18 +908,18 @@ impl AffixDict { let rules = || g.rules.iter().filter_map(|&i| self.suffixes.get(i as usize)); // The stem path: `root` is a stem carrying the rule's flag (and the prefix's, // when a cross product is being checked).- if let Some(f) = self.stem_flags_excluding(&root, excluded)- && f & need == need- && rules().any(|r| f & r.bit != 0 && (need == 0 || r.cross) && r.cond_matches(&root, true))+ if let Some(f) = self.stem_flags_except(&root, deny)+ && f.covers(need)+ && rules().any(|r| f.meets(r.bit) && (need.is_empty() || r.cross) && r.cond_matches(&root, true)) { return true; } // Continuation classes: `root` need not be a stem carrying this rule's flag — // it may be a form that gained the flag from another rule's continuation. The // matching rules' flags are looked for together, in one walk.- if need == 0 {- let flags = rules().filter(|r| r.cond_matches(&root, true)).fold(0u128, |m, r| m | r.bit);- if flags != 0 && self.form_has_flag(&root, flags, INNER_CHAIN, excluded) {+ if need.is_empty() {+ let flags = rules().filter(|r| r.cond_matches(&root, true)).fold(Mask::EMPTY, |m, r| m | r.bit);+ if !flags.is_empty() && self.form_has_flag(&root, flags, INNER_CHAIN, deny) { return true; } }@@ -732,26 +930,29 @@ impl AffixDict { /// Is `w` correct, Hunspell style: as written, or — for a capitalized word or one in capitals /// — in lower case (and capitalized), since a sentence may start with any word and headings- /// may be in capitals. Lower-case input never matches a capitalized stem (`warszawa`).+ /// may be in capitals. Lower-case input never matches a capitalized stem (`warszawa`), and a+ /// `KEEPCASE` stem matches only as written. pub fn check(&self, w: &str) -> bool {- self.check_excluding(w, self.forbidden)+ self.check_except(w, Mask::EMPTY) } /// Whether a suggestion is correct in its displayed case and has an eligible stem. pub fn can_suggest(&self, w: &str) -> bool {- self.check_excluding(w, self.forbidden | self.no_suggest)+ self.check_except(w, self.no_suggest | self.substandard) } - fn check_excluding(&self, w: &str, excluded: u128) -> bool {- if self.contains_excluding(w, excluded) {+ fn check_except(&self, w: &str, deny: Mask) -> bool {+ let w = self.normalize(w);+ if self.contains_except(&w, deny) { return true; }- match casing(w) {+ let deny = deny | self.keepcase;+ match casing(&w) { Casing::Lower | Casing::Mixed => false,- Casing::Initial => self.contains_excluding(&w.to_lowercase(), excluded),+ Casing::Initial => self.contains_except(&w.to_lowercase(), deny), Casing::Upper => { let lower = w.to_lowercase();- self.contains_excluding(&capitalize(&lower), excluded) || self.contains_excluding(&lower, excluded)+ self.contains_except(&capitalize(&lower), deny) || self.contains_except(&lower, deny) } } }@@ -766,11 +967,11 @@ impl AffixDict { let mut cross_sfx = Vec::new(); // Suffix forms queued with the continuation classes they carry, for chained expansion; // each (form, classes) pair is expanded once.- let mut chain: Vec<(String, u128)> = Vec::new();- let mut queued: std::collections::HashSet<(String, u128)> = std::collections::HashSet::new();+ let mut chain: Vec<(String, Mask)> = Vec::new();+ let mut queued: std::collections::HashSet<(String, Mask)> = std::collections::HashSet::new(); let mut work = 0usize; for r in &self.suffixes {- if f & r.bit == 0 || !r.cond_matches(stem, true) {+ if !f.meets(r.bit) || !r.cond_matches(stem, true) { continue; } let Some(base) = stem.strip_suffix(&*r.strip) else { continue };@@ -778,7 +979,7 @@ impl AffixDict { if r.cross { cross_sfx.push(form.clone()); }- if r.cont != 0 && queued.insert((form.clone(), r.cont)) {+ if !r.cont.is_empty() && queued.insert((form.clone(), r.cont)) { chain.push((form.clone(), r.cont)); } out.push(form);@@ -804,7 +1005,7 @@ impl AffixDict { } let Some(base) = form.strip_suffix(&*r.strip) else { continue }; let next = format!("{base}{}", r.add);- if r.cont != 0 && queued.insert((next.clone(), r.cont)) {+ if !r.cont.is_empty() && queued.insert((next.clone(), r.cont)) { chain.push((next.clone(), r.cont)); } out.push(next);@@ -815,7 +1016,7 @@ impl AffixDict { } } for r in &self.prefixes {- if f & r.bit == 0 || !r.cond_matches(stem, false) {+ if !f.meets(r.bit) || !r.cond_matches(stem, false) { continue; } let Some(rest) = stem.strip_prefix(&*r.strip) else { continue };@@ -839,11 +1040,11 @@ impl AffixDict { } /// The suffix rules carrying any flag of `mask`, in rule order.- fn suffixes_with(&self, mask: u128) -> Vec<u32> {+ fn suffixes_with(&self, mask: Mask) -> Vec<u32> { let mut rules: Vec<u32> = Vec::new(); let mut lists = 0; for (bit, list) in self.sfx_by_flag.iter().enumerate() {- if bit < 128 && mask & (1u128 << bit) != 0 && !list.is_empty() {+ if mask.meets(Mask::bit(bit)) && !list.is_empty() { rules.extend_from_slice(list); lists += 1; }@@ -904,13 +1105,21 @@ impl AffixDict { for (a, b) in &self.rep { out.push_str(&format!("REP {} {}\n", a.replace(' ', "_"), b.replace(' ', "_"))); }- for (directive, mask) in [("FORBIDDENWORD", self.forbidden), ("NOSUGGEST", self.no_suggest)] {- if mask != 0+ for (directive, mask) in+ [("FORBIDDENWORD", self.forbidden), ("NOSUGGEST", self.no_suggest), ("SUBSTANDARD", self.substandard), ("KEEPCASE", self.keepcase)]+ {+ if !mask.is_empty() && let Some(c) = self.flag_char(mask) { out.push_str(&format!("{directive} {c}\n")); } }+ for (a, b) in &self.iconv {+ out.push_str(&format!("ICONV {a} {b}\n"));+ }+ if !self.ignore.is_empty() {+ out.push_str(&format!("IGNORE {}\n", self.ignore.iter().collect::<String>()));+ } let z = |s: &str| if s.is_empty() { "0".to_string() } else { s.to_string() }; for (kind, rules) in [("PFX", &self.prefixes), ("SFX", &self.suffixes)] { for r in rules {@@ -934,8 +1143,8 @@ impl AffixDict { let p = prev.iter().zip(b).take_while(|(x, y)| x == y).count().min(255); raw.push(p as u8); raw.extend_from_slice(b.get(p..).unwrap_or_default());- let mask = self.flags.get(i).and_then(|&id| self.sets.get(usize::from(id))).copied().unwrap_or(0);- if mask != 0 {+ let mask = self.flags.get(i).and_then(|&id| self.sets.get(usize::from(id))).copied().unwrap_or_default();+ if !mask.is_empty() { raw.push(b'/'); raw.extend_from_slice(self.flag_chars_for(mask).as_bytes()); }@@ -962,26 +1171,24 @@ impl AffixDict { comments.push(c.to_string()); continue; }- let f: Vec<&str> = line.split_whitespace().collect();+ let f: Vec<&str> = split_fields(line).collect(); let at = |k: usize| f.get(k).copied().unwrap_or(""); match at(0) { "TRY" => rules.try_chars = cap_try(at(1)), "KEY" => rules.key = at(1).to_string(),- "MAP" => rules.map.push(at(1).chars().collect()),- "REP" => rules.rep.push((at(1).replace('_', " "), at(2).replace('_', " "))),- directive @ ("FORBIDDENWORD" | "NOSUGGEST") => {- let mut it = at(1).chars();- match (it.next(), it.next()) {- (Some(c), None) if !c.is_whitespace() => {- if directive == "FORBIDDENWORD" {- rules.forbidden = Some(c);- } else {- rules.no_suggest = Some(c);- }- }- _ => return Err(format!("affix dictionary: bad {directive} in `{line}`")),+ directive @ ("FORBIDDENWORD" | "NOSUGGEST" | "SUBSTANDARD" | "KEEPCASE") => {+ let c = one_flag(at(1), directive, 0).map_err(|_| format!("affix dictionary: bad {directive} in `{line}`"))?;+ match directive {+ "FORBIDDENWORD" => rules.forbidden = Some(c),+ "NOSUGGEST" => rules.no_suggest = Some(c),+ "SUBSTANDARD" => rules.substandard = Some(c),+ _ => rules.keepcase = Some(c), } }+ "ICONV" if f.len() == 3 => rules.iconv.push((at(1).to_string(), at(2).to_string())),+ "IGNORE" => rules.ignore = at(1).chars().collect(),+ "MAP" => rules.map.push(at(1).chars().collect()),+ "REP" => rules.rep.push((at(1).replace('_', " "), at(2).replace('_', " "))), kind @ ("PFX" | "SFX") => { let mut fc = at(1).chars(); let (Some(flag), None) = (fc.next(), fc.next()) else { return Err(format!("affix dictionary: bad flag in `{line}`")) };@@ -1223,7 +1430,7 @@ pub(crate) mod tests { #[test] fn more_than_64_distinct_flags_load() { // VERO pt_BR uses 106 distinct flags (FLAG UTF-8). Build one with every basic Latin- // letter, the digits and eight accented letters: 70 flags, 64 < n <= 128.+ // letter, the digits and eight accented letters: 70 flags, more than 64. let mut flags = String::new(); let mut aff = String::from("SET UTF-8\nFLAG UTF-8\n"); let chars: Vec<char> = ('a'..='z').chain('A'..='Z').chain('0'..='9').chain(['á', 'é', 'í', 'ó', 'ú', 'ç', 'ã', 'õ']).collect();@@ -1239,6 +1446,24 @@ pub(crate) mod tests { assert!(!d.contains("verboß"), "ß is not a flag and not a stem"); } + #[test]+ fn more_than_128_distinct_flags_load() {+ // The Hungarian dictionary uses 158 single-byte flags: 200 flags here, U+0080 up.+ let flags: String = (0x80u32..0x80 + 200).filter_map(char::from_u32).collect();+ let mut aff = String::from("SET UTF-8\n");+ for (i, c) in flags.chars().enumerate() {+ aff.push_str(&format!("SFX {c} N 1\nSFX {c} 0 x{i} .\n"));+ }+ let half: String = flags.chars().step_by(2).collect();+ let d = AffixDict::from_hunspell(&aff, &format!("1\nszó/{half}\n"), &[]).unwrap();+ assert!(d.contains("szóx0") && d.contains("szóx198") && d.contains("szóx130"));+ assert!(!d.contains("szóx1") && !d.contains("szóx199") && !d.contains("szóx131"));+ let back = AffixDict::from_bytes(&d.to_bytes()).unwrap();+ assert!(back.contains("szóx198") && !back.contains("szóx199"));+ let too_many: String = (0x100u32..0x100 + 257).filter_map(char::from_u32).collect();+ assert!(AffixDict::from_hunspell("SET UTF-8\n", &format!("1\nszó/{too_many}\n"), &[]).is_err());+ }+ #[test] fn forbidden_words_are_rejected() { // VERO pt_BR marks spellings it must reject (flag ý under FLAG UTF-8) as stems carrying@@ -1364,6 +1589,167 @@ pub(crate) mod tests { assert!(d.contains("kot") && d.contains("koty")); } + #[test]+ fn flag_aliases_stand_for_flag_sets() {+ // `AF`: `/1` in the `.dic` and in a continuation class means the first alias's flags.+ // Morphological aliases (`AM`) and the fields after a stem are skipped.+ let aff = "SET UTF-8\nAF 2\nAF ab # 1\nAF c # 2\nAM 1\nAM po:noun\nSFX a Y 1\nSFX a 0 ok/2 . 1\nSFX b Y 1\nSFX b 0 at . 1\nSFX c Y 1\nSFX c 0 ban . 1\n";+ let d = AffixDict::from_hunspell(aff, "1\nház/1\t1\n", &[]).unwrap();+ assert!(d.contains("ház") && d.contains("házok") && d.contains("házat") && d.contains("házokban"));+ assert!(!d.contains("házban"), "the stem has a and b, not c");+ // An undefined alias reads as no flags, as in Hunspell: the stem loads, without affixes.+ let undefined = AffixDict::from_hunspell(aff, "2\nház/3\nkert/1\n", &[]).unwrap();+ assert!(undefined.check("ház") && !undefined.check("házok"), "alias 3 is not defined");+ assert!(undefined.check("kertok"), "the other entries keep their aliases");+ let back = AffixDict::from_bytes(&d.to_bytes()).unwrap();+ assert!(back.contains("házokban") && !back.contains("házban"), "aliases are expanded in the compact format");+ }++ #[test]+ fn iconv_and_ignore_normalize_input() {+ // Ligatures read as their letters; `IGNORE` characters vanish from input and stems.+ let aff = "SET UTF-8\nICONV 2\nICONV fi fi\nICONV ffl ffl\nIGNORE ()\nSFX a Y 1\nSFX a 0 s .\n";+ let d = AffixDict::from_hunspell(aff, "2\nfine/a\nwaf(fle)/a\n", &[]).unwrap();+ assert!(d.check("fine") && d.check("fines") && d.check("Fine") && d.contains("fine"));+ assert!(d.check("waffle") && d.check("waffles") && d.check("wa(ffle)"));+ assert!(!d.check("finer"));+ let back = AffixDict::from_bytes(&d.to_bytes()).unwrap();+ assert!(back.check("fines") && back.check("waffle") && back.check("waf(fle)"));+ }++ #[test]+ fn alias_numbers_are_read_as_hunspell_reads_them() {+ // Checked with the Hunspell 1.7 command line. With an `AF` table a flag field is the alias+ // its leading digits number: `fal/2x` is alias 2, `lab/b` has no flags.+ let aff = "SET UTF-8\nAF 2\nAF B\nAF C\nSFX B Y 1\nSFX B 0 s .\nSFX C Y 1\nSFX C 0 x .\nSFX b Y 1\nSFX b 0 q .\n";+ let d = AffixDict::from_hunspell(aff, "3\nlab/b\nfal/2x\nkut/1\n", &[]).unwrap();+ assert!(d.check("lab") && !d.check("labq") && !d.check("labs"));+ assert!(d.check("falx") && !d.check("fals") && d.check("kuts"));+ // A table without its count line is rejected: no alias applies.+ let broken = AffixDict::from_hunspell("SET UTF-8\nAF B\nAF C\nSFX B Y 1\nSFX B 0 s .\n", "2\nlab/1\nfal/2\n", &[]).unwrap();+ assert!(broken.check("lab") && !broken.check("labs") && !broken.check("fals"));+ // `AF 0` is rejected the same way: `lab/B` keeps its literal flag.+ let zero = AffixDict::from_hunspell("SET UTF-8\nAF 0\nSFX B Y 1\nSFX B 0 s .\n", "1\nlab/B\n", &[]).unwrap();+ assert!(zero.check("lab") && zero.check("labs"));+ // An undefined alias in a continuation class leaves the rule in force, without it.+ let cont = AffixDict::from_hunspell("SET UTF-8\nAF 1\nAF a\nSFX a Y 1\nSFX a 0 ok/7 .\n", "1\nhaz/1\n", &[]).unwrap();+ assert!(cont.check("haz") && cont.check("hazok"));+ }++ #[test]+ fn a_rep_count_line_may_carry_a_comment() {+ let d = AffixDict::from_hunspell("SET UTF-8\nREP 1 # confusions\nREP x y\n", "1\nay\n", &[]).unwrap();+ assert_eq!(d.rep(), [("x".to_string(), "y".to_string())]);+ }++ #[test]+ fn count_lines_may_carry_a_comment() {+ // Hunspell reads the first `AF` and `ICONV` line as the count, comment and all: `AF 2 #`+ // is no alias, and `ICONV 1 # …` doesn't turn `1` into `#`.+ let aff = "SET UTF-8\nAF 2 # aliases\nAF a # 1\nAF b # 2\nICONV 1 # ligatures\nICONV q y\nSFX a Y 1\nSFX a 0 ok .\nSFX b Y 1\nSFX b 0 at .\n";+ let d = AffixDict::from_hunspell(aff, "3\nhaz/1\nkert/2\na1\n", &[]).unwrap();+ assert!(d.check("hazok") && !d.check("hazat") && d.check("kertat") && !d.check("kertok"), "alias 1 is `a`");+ assert!(d.check("a1") && !d.check("a#"));+ // The UTF-8 reader keeps such a line as it is, so both readers agree on it.+ let (a, dic) = crate::hunspell::decode(aff.as_bytes(), "1\nhaz/1\n".as_bytes()).unwrap();+ assert!(a.contains("AF 2 # aliases") && a.contains("ICONV 1 # ligatures"), "{a}");+ assert!(AffixDict::from_hunspell(&a, &dic, &[]).unwrap().check("hazok"));+ }++ #[test]+ fn dic_entries_may_hold_spaces() {+ // `ad hoc/B` is one entry, as in Hunspell; `ad` alone is no word.+ let aff = "SET UTF-8\nSFX B Y 1\nSFX B 0 s .\n";+ // Checked with the Hunspell 1.7 command line: without a tab or a morphological field the+ // whole line is the entry (`kot/B 12` has the flag `B`), and leading blanks belong to it+ // (` lab` is no `lab`).+ let d = AffixDict::from_hunspell(aff, "5\nad hoc/B\tpo:adv\nword/B po:noun\nkot/B 12\n lab/B\nés így/B\n", &[]).unwrap();+ assert!(d.contains("ad hoc") && d.contains("ad hocs") && d.contains("és így") && d.contains("és ígys"));+ assert!(!d.contains("ad") && !d.contains("hoc") && !d.contains("ads") && !d.contains("így"));+ assert!(d.contains("words") && d.contains("kots") && !d.contains("kot/B"));+ assert!(!d.contains("lab") && !d.contains("labs"));+ }++ #[test]+ fn ignored_characters_leave_the_stripped_text_alone() {+ // Hunspell removes `IGNORE` characters from the appended text, not from the stripped+ // one: `SFX a bx c` never matches the stem `abx` (stored as `ab`), so `ac` is wrong,+ // while `SFX a b cx` appends `c` (checked with the Hunspell 1.7 command line).+ let strip = AffixDict::from_hunspell("SET UTF-8\nIGNORE x\nSFX a Y 1\nSFX a bx c .\n", "1\nabx/a\n", &[]).unwrap();+ assert!(strip.check("ab") && strip.check("abx") && !strip.check("ac"));+ let add = AffixDict::from_hunspell("SET UTF-8\nIGNORE x\nSFX a Y 1\nSFX a b cx .\n", "1\nab/a\n", &[]).unwrap();+ assert!(add.check("ac") && add.check("acx"));+ for d in [&strip, &add] {+ let back = AffixDict::from_bytes(&d.to_bytes()).unwrap();+ assert_eq!(back.check("ac"), d.check("ac"));+ }+ }++ #[test]+ fn normalizing_borrows_when_nothing_changes() {+ let d = AffixDict::from_hunspell("SET UTF-8\nICONV 1\nICONV fi fi\nIGNORE ()\n", "1\nfine\n", &[]).unwrap();+ assert!(matches!(d.normalize("fine"), std::borrow::Cow::Borrowed("fine")));+ assert_eq!(d.normalize("(fi)ne"), "fine");+ assert!(matches!(dict().normalize("kot"), std::borrow::Cow::Borrowed("kot")));+ }++ #[test]+ fn keepcase_stems_match_only_as_written() {+ let aff = "SET UTF-8\nKEEPCASE k\nSFX a Y 1\nSFX a 0 s .\n";+ let d = AffixDict::from_hunspell(aff, "3\npH/k\nkm/ka\nkot/a\n", &[]).unwrap();+ assert!(d.check("pH") && d.check("km") && d.check("kms"));+ assert!(!d.check("PH") && !d.check("Km") && !d.check("KM") && !d.check("Kms"));+ assert!(d.check("Kot") && d.check("KOTS"), "other stems still follow the sentence and heading case");+ let back = AffixDict::from_bytes(&d.to_bytes()).unwrap();+ assert!(back.check("km") && !back.check("Km"));+ }++ #[test]+ fn substandard_and_lemma_present_words_are_correct() {+ // `SUBSTANDARD` and `LEMMA_PRESENT` steer morphological generation, and a SUBSTANDARD+ // stem is never suggested; the stems and rules they mark are still correct.+ let aff = "SET UTF-8\nSUBSTANDARD x\nLEMMA_PRESENT l\nSFX a Y 2\nSFX a 0 y .\nSFX a 0 em/x .\n";+ let d = AffixDict::from_hunspell(aff, "2\nkot/a\npies/axl\n", &[]).unwrap();+ for dict in [&d, &AffixDict::from_bytes(&d.to_bytes()).unwrap()] {+ assert!(dict.check("kot") && dict.check("koty") && dict.check("kotem"));+ assert!(dict.check("pies") && dict.check("piesy") && dict.check("Piesem"));+ assert!(dict.can_suggest("koty") && dict.can_suggest("kotem") && dict.contains_suggestible("kot"));+ assert!(!dict.can_suggest("piesy") && !dict.can_suggest("Pies") && !dict.contains_suggestible("pies"));+ }+ }++ #[test]+ fn unsupported_directives_are_refused_and_the_rest_load() {+ // A dictionary that needs a directive outside this engine fails to load rather than+ // accepting the wrong words.+ for dir in UNSUPPORTED {+ assert!(AffixDict::from_hunspell(&format!("SET UTF-8\n{dir} X\n"), DIC, &[]).is_err(), "{dir} should be refused");+ }+ // The directives this engine reads, with no other change to the dictionary (`AF` would+ // turn the flags of `DIC` into alias numbers; it has tests of its own).+ for line in ["AM 1\nAM po:noun", "ICONV 1\nICONV fi fi", "IGNORE ()", "KEEPCASE k", "SUBSTANDARD x", "LEMMA_PRESENT l"] {+ let d = AffixDict::from_hunspell(&format!("{AFF}{line}\n"), DIC, &[]).unwrap_or_else(|e| panic!("{line}: {e}"));+ assert!(d.check("koty") && d.check("niekoty") && !d.check("kotx"), "{line}");+ }+ }++ #[test]+ fn fields_split_on_spaces_and_tabs_only() {+ // A flag byte read as U+00A0 (no-break space) or U+0085 is a flag, not a separator.+ let aff = "SET UTF-8\nSFX \u{a0} Y 1\nSFX \u{a0} 0 y .\nSFX \u{85} Y 1\nSFX \u{85} 0 em .\n";+ let d = AffixDict::from_hunspell(aff, "1\nkot/\u{a0}\u{85}\n", &[]).unwrap();+ assert!(d.contains("koty") && d.contains("kotem"));+ let back = AffixDict::from_bytes(&d.to_bytes()).unwrap();+ assert!(back.contains("koty") && back.contains("kotem"));+ // The same bytes as one-flag directives, as Hunspell reads them.+ let aff = "SET UTF-8\nFORBIDDENWORD \u{a0}\nKEEPCASE \u{85}\nSFX a Y 1\nSFX a 0 s .\n";+ let d = AffixDict::from_hunspell(aff, "3\nkot/a\nrossz/\u{a0}\npH/\u{85}\n", &[]).unwrap();+ for dict in [&d, &AffixDict::from_bytes(&d.to_bytes()).unwrap()] {+ assert!(dict.check("kots") && !dict.check("rossz"));+ assert!(dict.check("pH") && !dict.check("PH"));+ }+ }+ #[test] fn unsupported_and_broken_input_is_an_error() { assert!(AffixDict::from_hunspell("FLAG long\n", DIC, &[]).is_err());@@ -218,13 +218,23 @@ fn decode_high(bytes: &[u8], high: &str) -> String { /// (UTF-8 when there is none; ISO 8859-2 is the other one Polish dictionaries use, ISO 8859-1 the /// one the original Portuguese VERO uses, ISO 8859-9 the one older Turkish dictionaries use, KOI8-R /// and Windows-1251 the ones older Russian dictionaries use).+///+/// Under a UTF-8 `SET` with single-byte flags (no `FLAG` line, or `FLAG DEFAULT`), flags are bytes,+/// as in Hunspell: the Hungarian dictionary writes its affix flags and flag aliases as bytes above+/// 0x7F that aren't UTF-8. Those fields are read byte by byte (each byte becomes U+0000..U+00FF,+/// one flag per byte); the words, affixes and conditions around them are UTF-8. With any other+/// `FLAG` (`UTF-8`, `num`, `long`) flags are text and the whole file is UTF-8: `SFX 1234` keeps+/// its number (the Turkish dictionary's `FLAG num`). pub fn decode(aff: &[u8], dic: &[u8]) -> Result<(String, String), String> {- let head = String::from_utf8_lossy(aff.get(..aff.len().min(8192)).unwrap_or_default()).to_string();- let set = head.lines().find_map(|l| l.trim_start_matches('\u{feff}').strip_prefix("SET ")).map(str::trim).unwrap_or("UTF-8").to_ascii_uppercase();+ let (set, flag) = directives(aff);+ let set = set.unwrap_or_else(|| "UTF-8".into()); match set.as_str() { "UTF-8" | "UTF8" => {- let text = |b: &[u8], what: &str| String::from_utf8(b.to_vec()).map_err(|e| format!("{what}: not UTF-8: {e}"));- Ok((text(aff, ".aff")?, text(dic, ".dic")?))+ if flag.as_deref().is_some_and(|f| f != "DEFAULT") {+ let text = |b: &[u8], what: &str| String::from_utf8(b.to_vec()).map_err(|e| format!("{what}: not UTF-8: {e}"));+ return Ok((text(aff, ".aff")?, text(dic, ".dic")?));+ }+ Ok((decode_utf8_aff(aff)?, decode_utf8_dic(dic)?)) } "ISO8859-1" | "ISO-8859-1" => Ok((decode_latin1(aff), decode_latin1(dic))), "ISO8859-2" | "ISO-8859-2" => Ok((decode_latin2(aff), decode_latin2(dic))),@@ -235,6 +245,225 @@ pub fn decode(aff: &[u8], dic: &[u8]) -> Result<(String, String), String> { } } +/// The first arguments of the `.aff`'s first `SET` and `FLAG` lines, in capitals, found in one pass+/// over the whole file: a `FLAG UTF-8` after the rules still decides how the `.dic` flags are read,+/// as in Hunspell. Only the first argument counts: `FLAG UTF-8 # Unicode flags` is UTF-8.+fn directives(aff: &[u8]) -> (Option<String>, Option<String>) {+ let (mut set, mut flag) = (None, None);+ for l in aff.split(|&b| b == b'\n') {+ if set.is_some() && flag.is_some() {+ break;+ }+ let l = l.strip_prefix("\u{feff}".as_bytes()).unwrap_or(l);+ let slot = match l.first() {+ Some(b'S') if set.is_none() => (&mut set, &b"SET"[..]),+ Some(b'F') if flag.is_none() => (&mut flag, &b"FLAG"[..]),+ _ => continue,+ };+ let arg = l+ .strip_prefix(slot.1)+ .filter(|rest| rest.first().is_some_and(|b| matches!(b, b' ' | b'\t')))+ .and_then(|rest| rest.split(|b| matches!(b, b' ' | b'\t' | b'\r')).find(|f| !f.is_empty()));+ if let Some(a) = arg {+ *slot.0 = Some(String::from_utf8_lossy(a).to_ascii_uppercase());+ }+ }+ (set, flag)+}++/// Directives whose one argument is a flag.+const FLAG_DIRECTIVES: &[&str] = &[+ "CIRCUMFIX",+ "COMPOUNDBEGIN",+ "COMPOUNDEND",+ "COMPOUNDFLAG",+ "COMPOUNDFORBIDFLAG",+ "COMPOUNDLAST",+ "COMPOUNDFIRST",+ "COMPOUNDMIDDLE",+ "COMPOUNDPERMITFLAG",+ "COMPOUNDROOT",+ "FORBIDDENWORD",+ "FORCEUCASE",+ "GENERATE",+ "HU_KOTOHANGZO",+ "KEEPCASE",+ "LEMMA_PRESENT",+ "NEEDAFFIX",+ "NOSUGGEST",+ "ONLYINCOMPOUND",+ "ONLYROOT",+ "PSEUDOROOT",+ "SUBSTANDARD",+ "WARN",+];++/// Directives that only describe the dictionary; a non-UTF-8 byte in them (the Hungarian+/// dictionary's `NAME` is ISO 8859-2) is replaced instead of failing the load.+const METADATA_DIRECTIVES: &[&str] = &["NAME", "HOME", "VERSION", "LANG", "AM"];++/// `text/flags`: the text as UTF-8 and the flags byte by byte (`\/` is a literal slash). `what`+/// names the line in an error.+fn text_and_flags(field: &[u8], what: impl Fn() -> String) -> Result<String, String> {+ let mut slash = None;+ let mut i = 0;+ while let Some(&b) = field.get(i) {+ if b == b'\\' && field.get(i + 1) == Some(&b'/') {+ i += 2;+ continue;+ }+ if b == b'/' {+ slash = Some(i);+ break;+ }+ i += 1;+ }+ let (text, flags) = match slash {+ Some(k) => (field.get(..k).unwrap_or_default(), field.get(k..).unwrap_or_default()),+ None => (field, &[][..]),+ };+ let mut out = std::str::from_utf8(text).map_err(|e| format!("{}: not UTF-8: {e}", what()))?.to_string();+ out.push_str(&decode_latin1(flags));+ Ok(out)+}++/// Split a line into fields on spaces and tabs, keeping the separators (so a line rebuilt from+/// its parts is the original).+fn fields(line: &[u8]) -> Vec<&[u8]> {+ let mut out = Vec::new();+ let mut start = 0;+ let mut in_sep = line.first().is_some_and(|b| matches!(b, b' ' | b'\t'));+ for (i, b) in line.iter().enumerate() {+ let sep = matches!(b, b' ' | b'\t');+ if sep != in_sep {+ out.push(line.get(start..i).unwrap_or_default());+ start = i;+ in_sep = sep;+ }+ }+ out.push(line.get(start..).unwrap_or_default());+ out+}++/// A UTF-8 `.aff` with single-byte flags (see [`decode`]).+fn decode_utf8_aff(aff: &[u8]) -> Result<String, String> {+ let mut out = String::with_capacity(aff.len());+ for (n, line) in aff.split(|&b| b == b'\n').enumerate() {+ if n > 0 {+ out.push('\n');+ }+ if let Ok(s) = std::str::from_utf8(line)+ && s.is_ascii()+ {+ out.push_str(s);+ continue;+ }+ let parts = fields(line);+ // Field index among the non-separator parts.+ let words: Vec<usize> =+ parts.iter().enumerate().filter(|(_, p)| !p.first().is_some_and(|b| matches!(b, b' ' | b'\t'))).map(|(i, _)| i).collect();+ let dir = words+ .first()+ .and_then(|&i| parts.get(i))+ .map(|p| String::from_utf8_lossy(p).trim_start_matches('\u{feff}').to_string())+ .unwrap_or_default();+ let what = || format!(".aff line {}", n + 1);+ if dir.starts_with('#') || METADATA_DIRECTIVES.contains(&dir.as_str()) {+ out.push_str(&String::from_utf8_lossy(line));+ continue;+ }+ // A trailing `# comment` (the Hungarian `AF` lines number themselves) is never a word or+ // a flag; from there on the line is kept as text. It can only start after the directive's+ // own fields, since a field such as an affix condition may itself begin with `#`.+ let field_text = |k: usize| words.get(k).and_then(|&w| parts.get(w)).map(|p| String::from_utf8_lossy(p).into_owned());+ let first_free = comment_field(&dir, field_text);+ let comment_at = words.iter().skip(first_free).position(|&w| parts.get(w).is_some_and(|f| f.first() == Some(&b'#'))).map(|k| k + first_free);+ // The field index of the next non-separator part.+ let mut next_field = 0;+ for (i, p) in parts.iter().enumerate() {+ let field = (!p.first().is_some_and(|b| matches!(b, b' ' | b'\t'))).then(|| {+ next_field += 1;+ next_field - 1+ });+ let in_comment = comment_at.is_some_and(|c| words.get(c).is_some_and(|&w| i >= w));+ let decoded = match (dir.as_str(), field) {+ _ if in_comment => String::from_utf8_lossy(p).into_owned(),+ ("AF" | "COMPOUNDRULE" | "SYLLABLENUM", Some(1)) => decode_latin1(p),+ ("PFX" | "SFX", Some(3)) | ("CHECKCOMPOUNDPATTERN", Some(1 | 2)) => text_and_flags(p, what)?,+ // A field that names one flag keeps its first byte, as Hunspell's `decode_flag`+ // does: `SFX é` is the flag 0xC3, the first of the two flags `kert/é` carries.+ ("PFX" | "SFX", Some(1)) => decode_latin1(p.get(..1).unwrap_or_default()),+ (d, Some(1)) if FLAG_DIRECTIVES.contains(&d) => decode_latin1(p.get(..1).unwrap_or_default()),+ _ => std::str::from_utf8(p).map_err(|e| format!("{}: not UTF-8: {e}", what()))?.to_string(),+ };+ out.push_str(&decoded);+ }+ }+ Ok(out)+}++/// The first field of an `.aff` line where a trailing `# comment` may start: after the+/// directive's own fields. A table's count line (`REP 12`, `PFX A Y 3`) has fewer fields than its+/// rows (`REP a b`, `PFX A 0 x .`). `field(k)` is the line's `k`-th field.+fn comment_field(dir: &str, field: impl Fn(usize) -> Option<String>) -> usize {+ let number = |k: usize| field(k).is_some_and(|f| f.parse::<usize>().is_ok());+ match dir {+ "PFX" | "SFX" if field(2).is_some_and(|f| f == "Y" || f == "N") && number(3) => 4,+ "PFX" | "SFX" => 5,+ "REP" | "ICONV" | "OCONV" | "PHONE" | "MAP" | "BREAK" | "AF" | "AM" | "COMPOUNDRULE" | "CHECKCOMPOUNDPATTERN"+ if number(1) && field(2).is_none_or(|f| f.starts_with('#')) =>+ {+ 2+ }+ "REP" | "ICONV" | "OCONV" | "PHONE" => 3,+ "CHECKCOMPOUNDPATTERN" => 4,+ _ => 2,+ }+}++/// Where the `word/flags` entry of a `.dic` line ends, as Hunspell reads it: at a tab or at the+/// first morphological field (a space or tab, two characters and `:`, as in ` po:noun`), whichever+/// comes first, so an entry may hold spaces (`ad hoc`); with neither, at the end of the line+/// (`kot/B 12` has the flags `B 12`, as in Hunspell).+pub(crate) fn dic_entry_end(line: &[u8]) -> usize {+ let blank = |b: &u8| matches!(b, b' ' | b'\t');+ let morph = line.windows(4).position(|w| matches!(w, [a, b, c, b':'] if blank(a) && !blank(b) && !blank(c)));+ let tab = line.iter().position(|&b| b == b'\t');+ let end = match (morph, tab) {+ (Some(m), Some(t)) => m.min(t),+ (m, t) => match m.or(t) {+ Some(e) => e,+ None => line.len(),+ },+ };+ // Blanks before the morphological field or the line end aren't part of the entry.+ line.get(..end).map_or(end, |e| e.iter().rposition(|b| !blank(b)).map_or(0, |k| k + 1))+}++/// A UTF-8 `.dic` with single-byte flags: the word is UTF-8, the flags after `/` are bytes.+fn decode_utf8_dic(dic: &[u8]) -> Result<String, String> {+ let mut out = String::with_capacity(dic.len());+ for (n, line) in dic.split(|&b| b == b'\n').enumerate() {+ if n > 0 {+ out.push('\n');+ }+ if let Ok(s) = std::str::from_utf8(line)+ && s.split_once('/').is_none_or(|(_, flags)| flags.is_ascii())+ {+ out.push_str(s);+ continue;+ }+ let what = || format!(".dic line {}", n + 1);+ // The first field is `word/flags` (it may hold spaces, see [`dic_entry_end`]);+ // morphological fields after it are UTF-8 text.+ let end = dic_entry_end(line);+ out.push_str(&text_and_flags(line.get(..end).unwrap_or_default(), what)?);+ let rest = line.get(end..).unwrap_or_default();+ out.push_str(std::str::from_utf8(rest).map_err(|e| format!("{}: not UTF-8: {e}", what()))?);+ }+ Ok(out)+}+ #[cfg(test)] mod tests { use super::*;@@ -280,6 +509,78 @@ mod tests { assert!(decode(b"SET ISO-8859-1\n", b"\xe7").is_ok()); } + #[test]+ fn utf8_dictionaries_keep_single_byte_flags() {+ // The Hungarian dictionary is `SET UTF-8` but writes flags as single bytes above 0x7F,+ // which aren't UTF-8; each byte is one flag. Its `NAME` line is ISO 8859-2.+ let aff = b"SET UTF-8\nNAME Magyar helyes\xedr\xe1si\nAF 2\nAF \xd5a # 1\nAF \xe9 # 2\nSFX \xd5 Y 1\nSFX \xd5 0 ok/2 . 7\nSFX \xe9 Y 1\nSFX \xe9 0 ban .\nSFX a Y 1\nSFX a 0 at .\n";+ let dic = "2\nház/1\t1\nkert/\u{e9}\n".as_bytes();+ let (a, d) = decode(aff, dic).unwrap();+ assert!(a.contains("AF \u{d5}a # 1") && a.contains("AF \u{e9} # 2") && a.contains("SFX \u{d5} 0 ok/2 . 7"));+ assert!(d.contains("kert/\u{c3}\u{a9}"), "a UTF-8 é in the flags is two one-byte flags");+ let dict = crate::affix::AffixDict::from_hunspell(&a, &d, &[]).unwrap();+ assert!(dict.check("ház") && dict.check("házok") && dict.check("házokban") && dict.check("házat"));+ assert!(!dict.check("házban"));+ // `FLAG UTF-8` keeps the whole file UTF-8, and words must still be UTF-8.+ let (_, d) = decode(b"SET UTF-8\nFLAG UTF-8\n", "kert/\u{e9}\n".as_bytes()).unwrap();+ assert_eq!(d, "kert/\u{e9}\n");+ assert!(decode(b"SET UTF-8\nSFX a Y 1\nSFX a 0 \xe9 .\n", b"").is_err(), "an affix must be UTF-8");+ assert!(decode(b"SET UTF-8\n", b"1\nh\xe1z/a\n").is_err(), "a stem must be UTF-8");+ }++ #[test]+ fn single_flag_fields_keep_their_first_byte() {+ // Without `FLAG UTF-8`, a UTF-8 `é` (0xC3 0xA9) in a field that names one flag is the+ // flag 0xC3; in a `.dic` flag list it is the two flags 0xC3 and 0xA9.+ let aff = "SET UTF-8\nKEEPCASE é\nSFX é Y 1\nSFX é 0 ok .\nSYLLABLENUM é\n".as_bytes();+ let (a, d) = decode(aff, "2\nkert/é\nNATO/é\n".as_bytes()).unwrap();+ assert!(a.contains("KEEPCASE \u{c3}\n") && a.contains("SFX \u{c3} Y 1") && a.contains("SFX \u{c3} 0 ok ."), "{a}");+ assert!(a.contains("SYLLABLENUM \u{c3}\u{a9}"), "SYLLABLENUM lists flags: {a}");+ assert!(d.contains("kert/\u{c3}\u{a9}"));+ let dict = crate::affix::AffixDict::from_hunspell(&a, &d, &[]).unwrap();+ assert!(dict.check("kert") && dict.check("kertok"));+ assert!(dict.check("NATO") && !dict.check("Nato"), "KEEPCASE é reaches NATO/é");+ }++ #[test]+ fn numeric_and_long_flags_stay_text() {+ // `FLAG num` (the Turkish dictionary) and `FLAG long`: no byte-wise flag reading, so a+ // number with a non-ASCII rule or comment keeps every digit.+ let aff = "SET UTF-8\nFLAG num\nFORBIDDENWORD 6000 # tiltott ő\nSFX 1234 Y 1\nSFX 1234 0 ları .\n";+ let (a, d) = decode(aff.as_bytes(), "1\nkitap/1234\n".as_bytes()).unwrap();+ assert_eq!(a, aff);+ assert_eq!(d, "1\nkitap/1234\n");+ let tr = crate::tr::dict::TrDict::from_hunspell(&a, &d).unwrap();+ assert!(tr.contains("kitapları") && tr.contains("kitap"));+ let long = "SET UTF-8\nFLAG long\nSFX Aa Y 1\nSFX Aa 0 ős .\n";+ assert_eq!(decode(long.as_bytes(), "1\nkő/Aa\n".as_bytes()).unwrap().0, long);+ assert!(decode(b"SET UTF-8\nFLAG num\n", b"1\nh\xe1z/1\n").is_err(), "the .dic must be UTF-8");+ // `FLAG DEFAULT` is the single-byte default.+ let (_, d) = decode(b"SET UTF-8\nFLAG default\n", b"1\nkert/\xe9\n").unwrap();+ assert_eq!(d, "1\nkert/\u{e9}\n");+ }++ #[test]+ fn flag_utf8_counts_anywhere_in_the_aff() {+ // A `FLAG UTF-8` after a long rule table still makes the `.dic` flags Unicode.+ let mut aff = b"SET UTF-8\nSFX a Y 400\n".to_vec();+ for _ in 0..400 {+ aff.extend_from_slice(b"SFX a 0 aaaaaaaaaaaaaaaaaaaaaaaa .\n");+ }+ aff.extend_from_slice(b"FLAG UTF-8\r\n");+ assert!(aff.len() > 8192);+ let (_, d) = decode(&aff, "kert/\u{e9}\n".as_bytes()).unwrap();+ assert_eq!(d, "kert/\u{e9}\n");+ }++ #[test]+ fn a_comment_after_flag_utf8_keeps_unicode_flags() {+ let (_, d) = decode(b"SET UTF-8\nFLAG UTF-8 # Unicode flags\n", "kert/\u{e9}\n".as_bytes()).unwrap();+ assert_eq!(d, "kert/\u{e9}\n");+ let (_, d) = decode(b"SET\tUTF-8 # comment\nFLAG\t\tUTF-8\n", "kert/\u{e9}\n".as_bytes()).unwrap();+ assert_eq!(d, "kert/\u{e9}\n", "repeated spaces and tabs separate the argument");+ }+ #[test] fn decodes_russian_single_byte_encodings() { // `ёлка` in KOI8-R (0xA3 0xCC 0xCB 0xC1) and in Windows-1251 (0xB8 0xEB 0xEA 0xE0).@@ -291,6 +592,41 @@ mod tests { assert_eq!(decode_high(&[0x98], CP1251_HIGH), "\u{fffd}"); } + #[test]+ fn comments_start_after_the_directive_fields() {+ let f = |line: &'static str| move |k: usize| line.split_whitespace().nth(k).map(str::to_string);+ assert_eq!(comment_field("PFX", f("PFX A Y 3 # count")), 4);+ assert_eq!(comment_field("SFX", f("SFX A 0 # .")), 5, "a rule's condition may be `#`");+ assert_eq!(comment_field("AF", f("AF 1234 # aliases")), 2);+ assert_eq!(comment_field("AF", f("AF Aa # 1")), 2);+ assert_eq!(comment_field("REP", f("REP 12")), 2);+ assert_eq!(comment_field("REP", f("REP # x")), 3);+ assert_eq!(comment_field("CHECKCOMPOUNDPATTERN", f("CHECKCOMPOUNDPATTERN a b c")), 4);+ assert_eq!(comment_field("KEEPCASE", f("KEEPCASE k")), 2);+ // A byte flag after an append text that begins with `#` stays a flag, not a comment.+ let (a, _) = decode(b"SET UTF-8\nSFX \xd5 Y 1 # header\nSFX \xd5 0 #/\xe9 . # rule\n", b"").unwrap();+ assert!(a.contains("SFX \u{d5} 0 #/\u{e9} . # rule"), "{a}");+ }++ #[test]+ fn dic_entries_end_where_hunspell_ends_them() {+ let end = |l: &str| l.get(..dic_entry_end(l.as_bytes())).unwrap().to_string();+ assert_eq!(end("word/A po:noun\tts:x"), "word/A", "the morphological field comes before the tab");+ assert_eq!(end("ad hoc/B\tpo:adv"), "ad hoc/B", "a tab ends a phrase");+ assert_eq!(end("ad hoc/B po:adv"), "ad hoc/B");+ assert_eq!(end("word/A 123"), "word/A 123", "neither: the whole line, as in Hunspell");+ assert_eq!(end("ad hoc/B "), "ad hoc/B");+ assert_eq!(end("word"), "word");+ assert_eq!(end("a:b/A"), "a:b/A", "a colon inside the word isn't a field");+ // Single-byte flags of a phrase entry are read byte by byte, then the UTF-8 morphology.+ let (_, d) = decode(b"SET UTF-8\n", b"1\nad hoc/\xe9\tpo:adv\n").unwrap();+ assert_eq!(d, "1\nad hoc/\u{e9}\tpo:adv\n");+ // Without a tab or a morphological field the whole line is the entry: its byte flags+ // don't fail the load (the Hungarian dictionary has such lines: `tök jó/396`).+ let (_, d) = decode(b"SET UTF-8\n", b"1\nad hoc/\xe9\n").unwrap();+ assert_eq!(d, "1\nad hoc/\u{e9}\n");+ }+ #[test] fn libreoffice_extension_dirs_follow_the_language() { // LibreOffice's IDs are per dictionary: Polish is `dict-pl`, Brazilian Portuguese