collective
Browse / github.com/storytold/designcraft / patch
Hyphenation: a Korean word or an address is not broken by English rules (#1688)
Korean body text: English hyphenation splits Hangul-attached Latin words ("New-/s와") and URLs Ordinary Korean documents (notices, newsletters) leave the paragraph language at the default English, and DesignCraft then hyphenates Latin words attached to Hangul and addresses. **Setup.** Windows 11 (Korean locale), DesignCraft `main` at `e74208dd`, `designcraft-cli` export to PDF. Measurements are from the PDF's glyph positions. Test documents: synthetic Korean paragraphs in Malgun Gothic, HCR Batang and Noto Sans KR, 9.5 pt, justified, columns 60 mm, 80 mm and full width, default spacing 80/100/133 %. On the current `main` (`348a8a4d`) the functions involved are unchanged (`compose/src
- Status
- open
- Score
- 0.00
- Adopters
- 0 other operators
- Author
- Hamza0101 (imported from pull request #1754, open)
- Base
50650f74d7328745cfcdd16098313a9fc31b23d5- Size
- +157 −0 in 3 files
- Review
- pending
- Shared
- 2026-10-28
- Id
a9d7dd3cca9ae968acd711a6
How it went
No other operator has reported using it yet.
Diff
raw · crates/compose/src/hyphen.rs crates/compose/src/lib.rs crates/compose/src/tests.rs
@@ -198,6 +198,82 @@ pub fn word_breaks_in(word: &str, lang: Lang) -> (Vec<usize>, Source) { /// Allowed hyphenation points in `word` as char indices (a hyphen goes before that char), with /// the paragraph limits applied to each letter run.+/// Whether a whitespace-delimited token is one no language's hyphenation should break.+///+/// Korean attaches particles straight onto Latin words ("News와", "PDF로"), and notices are full+/// of addresses. With the paragraph language left at the default English -- which is what an+/// ordinary Korean document has -- a word run inside such a token was hyphenated as English,+/// giving `New-` / `s와` and `library.exam-` / `ple.kr` (#1688).+///+/// A token is kept whole when it carries Hangul, kana or Han, or when it reads as an address: a+/// URL with a scheme, an e-mail address, a dotted domain, a `/`-rooted path or a Windows drive+/// path. Breaking at a hyphen the author typed is still allowed; this only stops hyphens being+/// invented inside the token.+pub fn keep_whole(token: &str) -> bool {+ let cjk = |c: char| {+ matches!(c as u32,+ 0xAC00..=0xD7A3 // Hangul syllables+ | 0x1100..=0x11FF // Hangul Jamo+ | 0x3130..=0x318F // Hangul compatibility Jamo+ | 0x3040..=0x30FF // kana+ | 0x4E00..=0x9FFF // CJK unified ideographs+ | 0x3400..=0x4DBF+ | 0xF900..=0xFAFF)+ };+ if token.chars().any(cjk) {+ return true;+ }+ let lower = token.to_ascii_lowercase();+ if lower.starts_with("http://") || lower.starts_with("https://") || lower.starts_with("ftp://") || lower.starts_with("www.") {+ return true;+ }+ // An e-mail address: one `@` with something either side, no space (tokens have none).+ if let Some((user, host)) = token.split_once('@') {+ return !user.is_empty() && host.contains('.');+ }+ // A path, rooted or with a drive letter.+ if token.starts_with('/') || token.starts_with('\\') {+ return true;+ }+ let mut cs = token.chars();+ if let (Some(a), Some(b), Some(c)) = (cs.next(), cs.next(), cs.next())+ && a.is_ascii_alphabetic()+ && b == ':'+ && (c == '/' || c == '\\')+ {+ return true;+ }+ // A domain: dot-separated labels up to the first `/`, whose last label is a short run of+ // letters -- "example.kr", "library.example.kr". Ordinary prose ("end. Next") never reaches+ // here, because a token holds no space.+ let host = lower.split('/').next().unwrap_or("");+ let labels: Vec<&str> = host.trim_end_matches('.').split('.').collect();+ if labels.len() >= 2+ && labels.iter().all(|l| !l.is_empty() && l.chars().all(|c| c.is_ascii_alphanumeric() || c == '-'))+ && labels.last().is_some_and(|tld| tld.len() <= 6 && tld.chars().all(|c| c.is_ascii_lowercase()))+ {+ return true;+ }+ false+}++/// Whether a paragraph is set in Korean, so automatic hyphenation does not apply to it.+///+/// Korean is set word by word and is not hyphenated (KLREQ). A paragraph with at least as many+/// Hangul syllables as Latin letters is treated as Korean however its language is set, since an+/// ordinary Korean document leaves the paragraph language at the default English (#1688).+pub fn mostly_hangul(text: &str) -> bool {+ let (mut hangul, mut latin) = (0usize, 0usize);+ for c in text.chars() {+ match c as u32 {+ 0xAC00..=0xD7A3 | 0x1100..=0x11FF | 0x3130..=0x318F => hangul += 1,+ _ if c.is_ascii_alphabetic() => latin += 1,+ _ => {}+ }+ }+ hangul > 0 && hangul >= latin+}+ pub fn hyphen_points(word: &str, lim: &Limits) -> Vec<usize> { hyphen_points_in(word, lim, Lang::English) }@@ -308,6 +384,40 @@ fn heuristic_points(c: &[char]) -> Vec<usize> { #[cfg(test)] mod tests {+ /// A token no language's hyphenation should break: Korean particles on Latin words, and the+ /// addresses a notice is full of (#1688).+ #[test]+ fn a_token_with_hangul_or_an_address_is_kept_whole() {+ for t in [+ "News\u{C640}", // News와+ "PDF\u{B85C}", // PDF로+ "\u{C77C}\u{BCF8}\u{C5B4}", // 일본어+ "https://example.kr/page",+ "www.example.kr",+ "library.example.kr",+ "mail@example.kr",+ "/usr/share/fonts",+ "C:\\Users\\Hong",+ "example.kr",+ ] {+ assert!(keep_whole(t), "{t} should be kept whole");+ }+ // Ordinary words, including ones with a hyphen or a full stop, are still hyphenated.+ for t in ["hyphenation", "well-known", "sentence.", "Mr.", "re-examine", "3.5"] {+ assert!(!keep_whole(t), "{t} should not be protected");+ }+ }++ /// A Korean paragraph gets no automatic hyphens whatever its language is set to (#1688).+ #[test]+ fn a_paragraph_that_is_mostly_hangul_is_not_hyphenated() {+ assert!(mostly_hangul("\u{C774}\u{BC88} \u{D1B5}\u{C9C0}\u{BB38}\u{C740} News\u{C640} \u{AD00}\u{B828}"));+ assert!(!mostly_hangul("An English paragraph about hyphenation and composition"));+ // A Latin paragraph with a Korean word in it is still hyphenated as Latin.+ assert!(!mostly_hangul("The Korean word \u{D55C}\u{AD6D} appears once in this English sentence"));+ assert!(!mostly_hangul(""));+ }+ use super::*; fn h(w: &str) -> String {@@ -4192,6 +4192,12 @@ fn hyphenation_points( if !pp.hyphenate { return out; }+ // Korean is set word by word and is not hyphenated (KLREQ). An ordinary Korean document+ // leaves the paragraph language at the default English, so the paragraph's own text decides+ // rather than its language setting (#1688).+ if hyphen::mostly_hangul(text) {+ return out;+ } let lim = hyphen::Limits { min_word: pp.hyph_min_word as usize, after_first: pp.hyph_after_first as usize,@@ -4236,6 +4242,14 @@ fn hyphenation_points( let (a, b) = (glyphs[i].byte, glyphs[j - 1].byte + glyphs[j - 1].len); if !discretionary && b > a && b <= text.len() && text.is_char_boundary(a) && text.is_char_boundary(b) { let word = &text[a..b];+ // The whitespace-delimited token the word sits in: a Latin run with a Korean particle+ // on it ("News와") or inside an address is not a word to hyphenate (#1688).+ let token_start = text[..a].rfind(char::is_whitespace).map_or(0, |i| i + text[i..].chars().next().map_or(1, char::len_utf8));+ let token_end = text[b..].find(char::is_whitespace).map_or(text.len(), |i| b + i);+ if hyphen::keep_whole(&text[token_start..token_end]) {+ i = j.max(i + 1);+ continue;+ } // Do not hyphenate the paragraph's last word unless allowed. let is_last_word = !pp.hyph_last_word && glyphs[j..].iter().all(|g| !g.is_letter()); let lang = foreign.iter().find(|(r, _)| r.contains(&a)).map_or(Some(hyphen::Lang::English), |(_, l)| *l);@@ -994,6 +994,39 @@ fn hyphen_limit_is_a_hard_constraint() { } } +/// A Korean notice leaves its paragraphs at the default English language, so English+/// hyphenation broke the Latin words Korean particles attach to and the addresses such a document+/// is full of: `New-` / `s와`, `library.exam-` / `ple.kr` (#1688). The lines may now be looser --+/// that is the cost of not hyphenating, and what #1691 is about -- but no hyphen is invented+/// inside those tokens.+#[test]+fn korean_text_and_addresses_are_not_hyphenated() {+ // Narrow enough that every line has to break somewhere.+ let rect = Rect::new(0.0, 0.0, 90.0, 100_000.0);+ let para = ParaAttrs { align: Some(Align::LeftJustified), hyphenate: Some(true), ..Default::default() };+ let hyphens_in = |text: &str| {+ let (d, sid, _) = doc_with(text, rect, para.clone());+ let cs = compose_story(&d, sid, &ComposeOptions::default());+ all_lines(&cs).iter().filter(|l| l.hyphenated).count()+ };++ // 이번 통지문은 News와 관련된 library.example.kr 주소를 포함합니다+ let korean = "\u{C774}\u{BC88} \u{D1B5}\u{C9C0}\u{BB38}\u{C740} News\u{C640} \u{AD00}\u{B828}\u{B41C} library.example.kr \u{C8FC}\u{C18C}\u{B97C} \u{D3EC}\u{D568}\u{D569}\u{B2C8}\u{B2E4}";+ assert_eq!(hyphens_in(korean), 0, "a Korean paragraph invents no hyphens");++ // An English paragraph holding the same address: the prose still hyphenates, the address does+ // not, so the protection is about the token and not about turning hyphenation off.+ let english = "This notice about hyphenation and composition mentions library.example.kr somewhere inside a long sentence";+ assert!(hyphens_in(english) > 0, "English prose is still hyphenated");+ let (d, sid, _) = doc_with(english, rect, para.clone());+ let cs = compose_story(&d, sid, &ComposeOptions::default());+ for l in all_lines(&cs).iter().filter(|l| l.hyphenated) {+ let upto = &english[..l.range.end.min(english.len())];+ let token = upto.rsplit(char::is_whitespace).next().unwrap_or("");+ assert!(!"library.example.kr".starts_with(token) || token.is_empty(), "a hyphen was invented inside the address: {token:?}");+ }+}+ #[test] fn hyphenation_zone_limits_ragged_hyphens() { let text = CORPUS.join(" ");