diff --git a/crates/text/src/fonts.rs b/crates/text/src/fonts.rs --- a/crates/text/src/fonts.rs +++ b/crates/text/src/fonts.rs @@ -158,6 +158,11 @@ fn info_from(n: FaceNames, data: FaceData, origin: &'static str) -> FaceInfo { } } +/// Whether the font at `index` of `data` maps `c` to a glyph (false when it can't be parsed). +pub fn maps_char(data: &[u8], index: u32, c: char) -> bool { + FontRef::from_index(data, index).ok().and_then(|f| f.charmap().map(c)).is_some_and(|g| g.to_u32() != 0) +} + /// A loaded (or loadable) face. pub struct Face { pub id: FaceId, @@ -220,6 +225,28 @@ impl Face { pub fn glyph(&self, c: char) -> Option { self.font()?.charmap().map(c).map(|g| g.to_u32()).filter(|g| *g != 0) } + /// The code points the face maps to a glyph, as sorted, merged inclusive ranges. A face whose + /// file is not loaded yet is read into a temporary buffer that is dropped afterwards (as + /// [`file_covers`] does). + pub fn coverage(&self) -> Vec<(u32, u32)> { + fn ranges(font: &FontRef<'_>) -> Vec<(u32, u32)> { + let mut cps: Vec = font.charmap().mappings().filter(|(_, g)| g.to_u32() != 0).map(|(cp, _)| cp).collect(); + cps.sort_unstable(); + let mut out: Vec<(u32, u32)> = Vec::new(); + for cp in cps { + match out.last_mut() { + Some(last) if cp <= last.1.saturating_add(1) => last.1 = last.1.max(cp), + _ => out.push((cp, cp)), + } + } + out + } + if let (FaceData::File(p), None) = (&self.info.data, self.bytes.get()) { + let Ok(data) = std::fs::read(p) else { return Vec::new() }; + return FontRef::from_index(&data, self.info.index).map(|f| ranges(&f)).unwrap_or_default(); + } + self.font().map(|f| ranges(&f)).unwrap_or_default() + } /// Whether the face can draw `c`: it maps `c` to a glyph that has an outline, a colour glyph /// or a bitmap. Some fonts map whole scripts to empty glyphs (#285); those don't count, so /// fallback moves on to a face that draws the character. Characters that are blank by nature @@ -932,6 +959,27 @@ pub fn fallback_for(c: char, prefer: FaceId) -> FaceId { fallback_with_locale(c, prefer, || CjkLocale::from_tag(&active_cjk_locale())) } +/// The installed faces [`fallback_for_locale`] tries first for `c`: the platform's usual families +/// for its script (CJK, Arabic, Hebrew, Indic, Thai, Georgian), in the style of `prefer`, for the +/// language `tag`. Empty for a character of no such script. Nothing is read: a face's file is read +/// when it is first asked for a glyph. +pub fn usual_faces(c: char, prefer: FaceId, tag: &str) -> Vec { + let script = script_of(c); + if script == FallbackScript::Other { + return Vec::new(); + } + if cfg!(not(target_arch = "wasm32")) && !system_scanned() { + scan_system(); + } + let tag = tag.trim(); + let locale = if tag.is_empty() || tag.eq_ignore_ascii_case("c") || tag.eq_ignore_ascii_case("posix") { + CjkLocale::from_tag(&environment_cjk_locale()) + } else { + CjkLocale::from_tag(tag) + }; + family_faces(script, &face(prefer), locale).to_vec() +} + /// Choose a fallback for an explicit interface language, independent of concurrent global /// language changes. Empty, `c` and `posix` use the OS environment rather than the global request. pub fn fallback_for_locale(c: char, prefer: FaceId, tag: &str) -> FaceId { @@ -1033,6 +1081,36 @@ fn system_fallback(c: char, style: &str) -> Option { mod tests { use super::*; + /// `coverage` reads a face's character map from its file without keeping the bytes, and + /// `usual_faces` names the faces to read it from. + #[test] + fn coverage_reads_the_file_without_keeping_it() { + let path = std::env::temp_dir().join(format!("effectcraft-coverage-{}.ttf", std::process::id())); + std::fs::write(&path, INTER_REGULAR).expect("write temp font"); + let info = FaceInfo { + family: "Inter Coverage".into(), + style: "Regular".into(), + weight: 400, + italic: false, + index: 0, + data: FaceData::File(path.clone()), + origin: "system", + native_family: None, + full_name: "Inter Coverage Regular".into(), + postscript: None, + variable: false, + instances: Vec::new(), + }; + let f = Face { id: 0, info, bytes: OnceLock::new(), shaper: OnceLock::new() }; + let ranges = f.coverage(); + let _ = std::fs::remove_file(&path); + let has = |c: char| ranges.iter().any(|&(a, b)| (a..=b).contains(&u32::from(c))); + assert!(has('A') && has('\u{416}') && !has('\u{5d0}'), "{ranges:?}"); + assert!(ranges.windows(2).all(|w| w[0].1.saturating_add(1) < w[1].0), "sorted and merged: {ranges:?}"); + assert!(f.bytes.get().is_none(), "reading the coverage must not keep the file"); + assert!(usual_faces('A', 0, "").is_empty(), "Latin has no usual fallback families"); + } + /// Concurrent language requests must neither choose another context's font nor reuse its /// cached family order. Synthetic fonts in a child process keep this independent of the host /// font installation and of other tests that exercise the process-global database/locale. diff --git a/crates/ui-egui/src/fallback_fonts.rs b/crates/ui-egui/src/fallback_fonts.rs new file mode 100644 --- /dev/null +++ b/crates/ui-egui/src/fallback_fonts.rs @@ -0,0 +1,450 @@ +//! Installed fallback faces, read when the interface first paints text that needs them. +//! +//! The interface draws Japanese, Chinese, Arabic, Hebrew, Indic, Thai, Georgian and Hangul text +//! (layer, item and comment names, the Georgian and CJK catalogs) with installed faces +//! (`theme::install_script_fallbacks`). Those files are large (Apple SD Gothic Neo alone is 53 MB) +//! and most sessions never draw a character from most of them, so only the faces the interface +//! language needs are read when the fonts are installed. [`Watch`], an egui plugin, looks at the +//! text each pass painted: when a character is in the script of a face that hasn't been read, and +//! the fonts installed so far can't draw it (or draw it only with a face that comes later in the +//! fallback order), the face is read, the fonts are installed again and egui redoes the pass +//! before anything is shown. The first frame that shows the text already has its glyphs. +//! +//! A character in no fallback's script that nothing can draw (an arrow or a math sign only an +//! installed face has) is looked up in the faces not read yet, whose character maps are read once +//! without keeping the files, and the face that has it is read: so nothing that drew before draws +//! as a box now, and a character no face has (a newer emoji) reads nothing. + +use std::collections::{HashMap, HashSet}; + +use egui::epaint::{ClippedShape, Shape}; +use egui::{FontDefinitions, FontFamily, FontId}; + +/// What [`plan`] needs to know of a fallback face. +pub(crate) trait Candidate { + /// Whether `c` is in the script the face is the fallback for. + fn in_script(&self, c: char) -> bool; + fn is_read(&self) -> bool; + /// Whether the face, once read, draws `c`. + fn covers(&self, c: char) -> bool; + /// Read the face; false when there is nothing to read. + fn read(&self) -> bool; + /// Whether the face, read or not, draws `c`. A face not read yet is looked up in its character + /// map, read once without keeping the file. + fn might_cover(&self, c: char) -> bool; +} + +/// How the fonts a pass was laid out with draw a character. +#[derive(Clone, Copy, Debug, PartialEq)] +struct Drawn { + /// A face ahead of the fallbacks draws it, or egui draws nothing for it. + early: bool, + /// Some face draws it. + any: bool, +} + +/// Read the fallbacks (in fallback order) the painted character `c` needs, given how the fonts +/// the pass was laid out with draw it, so that the same face draws `c` as when every fallback was +/// read at start-up. Returns whether a face was read. +/// +/// - Nothing is read when a face ahead of the fallbacks draws `c`. +/// - Otherwise the first fallback that has `c` draws it: one already read, or one whose script has +/// `c`, which is read (in order, until one has `c`). +/// - A character no face draws reads the first fallback not read yet whose character map has it +/// (a symbol only an installed face has), and nothing when none has (a newer emoji). One that a +/// face after the fallbacks draws (a Chinese catalog's extra face) is left to it. +fn plan(c: char, drawn: Drawn, fallbacks: &[&dyn Candidate]) -> bool { + if drawn.early { + return false; + } + let mut read_any = false; + for f in fallbacks { + if f.is_read() { + if f.covers(c) { + return read_any; + } + } else if f.in_script(c) && f.read() { + read_any = true; + if f.covers(c) { + return true; + } + } + } + if !drawn.any + && let Some(f) = fallbacks.iter().find(|f| !f.is_read() && !f.in_script(c) && f.might_cover(c)) + { + read_any |= f.read(); + } + read_any +} + +/// How `fonts` draws `c` in `family`, `has_glyph` being egui's answer. `late` names the faces that +/// come after the fallbacks' place in a family (the fallbacks themselves, and faces added after +/// them). egui's `has_glyph` answers no for the characters of the face that supplies its +/// replacement glyph (the ◻ of Noto Emoji, or the `?` of the first face in a family without one), +/// so the faces' character maps are asked too. +fn classify(fonts: &FontDefinitions, family: &FontFamily, c: char, has_glyph: bool, late: &[&str]) -> Drawn { + let Some(stack) = fonts.families.get(family) else { return Drawn { early: true, any: true } }; + let maps = |late_faces: bool| { + stack + .iter() + .filter(|name| late.contains(&name.as_str()) == late_faces) + .filter_map(|name| fonts.font_data.get(name)) + .any(|data| effectcraft_text::fonts::maps_char(&data.font, data.index, c)) + }; + let (early, later) = (maps(false), maps(true)); + Drawn { early: early || (has_glyph && !later), any: has_glyph || early || later } +} + +/// Characters that never need a fallback: ASCII, Latin-1 and Latin Extended-A (which every +/// interface font has), spacing and controls (egui draws nothing for them). +fn plain(c: char) -> bool { + u32::from(c) < 0x180 || c.is_whitespace() || c.is_control() +} + +/// The characters looked at so far, per font family (families differ in their faces): nothing more +/// to read for them. A bit per character of the Basic Multilingual Plane, a set for the others. +#[derive(Default)] +struct Settled(HashMap, HashSet)>); + +impl Settled { + fn insert(&mut self, family: FontFamily, c: char) { + let (bits, rest) = self.0.entry(family).or_insert_with(|| (vec![0; 1024], HashSet::new())); + let cp = u32::from(c); + match bits.get_mut((cp / 64) as usize) { + Some(word) => *word |= 1 << (cp % 64), + None => { + rest.insert(c); + } + } + } + + #[cfg(test)] + fn contains(&self, family: &FontFamily, c: char) -> bool { + self.0.get(family).is_some_and(|entry| settled_in(entry, c)) + } +} + +fn settled_in((bits, rest): &(Vec, HashSet), c: char) -> bool { + let cp = u32::from(c); + match bits.get((cp / 64) as usize) { + Some(word) => word & (1 << (cp % 64)) != 0, + None => rest.contains(&c), + } +} + +/// The characters (with the font family drawing them) in `shapes` that are not `settled`, each +/// once. Bounded: a pass with more new characters leaves the rest to the next pass. +fn new_characters(shapes: &[ClippedShape], settled: &Settled) -> Vec<(char, FontFamily)> { + const MAX: usize = 4096; + let mut seen = HashSet::new(); + let mut out = Vec::new(); + let mut stack: Vec<&Shape> = shapes.iter().map(|s| &s.shape).collect(); + while let Some(shape) = stack.pop() { + match shape { + Shape::Vec(v) => stack.extend(v.iter()), + Shape::Text(t) => { + let job = &t.galley.job; + if job.text.is_ascii() { + continue; + } + for section in &job.sections { + let Some(text) = job.text.get(section.byte_range.start.0..section.byte_range.end.0) else { continue }; + let family = §ion.format.font_id.family; + let done = settled.0.get(family); + for c in text.chars().filter(|c| !plain(*c) && !done.is_some_and(|d| settled_in(d, *c))) { + if out.len() >= MAX { + return out; + } + if seen.insert((c, family.clone())) { + out.push((c, family.clone())); + } + } + } + } + _ => {} + } + } + out +} + +/// The egui plugin that reads the installed fallbacks painted text needs (see the module docs). +/// `theme::install` adds it and keeps its interface language current. +pub(crate) struct Watch { + /// The interface language the fonts were installed for (it orders the CJK faces). + pub(crate) lang: String, + /// The faces read when painted text needs them, in fallback order, for the language named. + fallbacks: (String, Vec), + settled: Settled, + /// When the last read started, to log how long the redone pass took. + #[cfg(not(target_arch = "wasm32"))] + started: Option, +} + +impl Watch { + pub(crate) fn new(lang: &str) -> Self { + Self { + lang: lang.to_owned(), + fallbacks: (lang.to_owned(), crate::theme::lazy_fallbacks(lang)), + settled: Settled::default(), + #[cfg(not(target_arch = "wasm32"))] + started: None, + } + } +} + +impl egui::Plugin for Watch { + fn debug_name(&self) -> &'static str { + "effectcraft fallback fonts" + } + + /// A pass a `Grid` or an area's sizing already had redone can still be redone once more for a + /// face read in it, so its text doesn't show boxes for a frame. + fn setup(&mut self, ctx: &egui::Context) { + if let Some(three) = std::num::NonZeroUsize::new(3) { + ctx.options_mut(|o| o.max_passes = o.max_passes.max(three)); + } + } + + fn output_hook(&mut self, ctx: &egui::Context, output: &mut egui::FullOutput) { + #[cfg(not(target_arch = "wasm32"))] + if let Some(started) = self.started.take() { + log::info!("fonts: the pass redone with the new fallback face took {:.1} ms in all", started.elapsed().as_secs_f64() * 1e3); + } + if self.fallbacks.0 != self.lang { + // another interface language: other faces, in another order + self.fallbacks = (self.lang.clone(), crate::theme::lazy_fallbacks(&self.lang)); + self.settled = Settled::default(); + } + let fallbacks = &self.fallbacks.1; + if fallbacks.iter().all(|f| f.is_read()) { + return; + } + let chars = new_characters(&output.shapes, &self.settled); + if chars.is_empty() { + return; + } + #[cfg(not(target_arch = "wasm32"))] + let started = std::time::Instant::now(); + let candidates: Vec<&dyn Candidate> = fallbacks.iter().map(|f| f as &dyn Candidate).collect(); + let mut read = false; + let late: Vec<&str> = fallbacks.iter().map(|f| f.name()).collect(); + for (c, family) in chars { + // the fonts this pass was laid out with (new ones apply from the next pass) + let drawn = ctx.fonts_mut(|f| { + if !f.definitions().families.contains_key(&family) { + return Drawn { early: true, any: true }; // (painted text always has a family) + } + let has_glyph = f.has_glyph(&FontId::new(12.0, family.clone()), c); + classify(f.definitions(), &family, c, has_glyph, &late) + }); + read |= plan(c, drawn, &candidates); + self.settled.insert(family, c); + } + if !read { + return; + } + crate::theme::install_fonts(ctx, &self.lang); + #[cfg(not(target_arch = "wasm32"))] + { + let names: Vec<&str> = fallbacks.iter().filter(|f| f.is_read()).map(|f| f.name()).collect(); + log::info!("fonts: read installed fallback faces for {names:?} in {:.1} ms", started.elapsed().as_secs_f64() * 1e3); + self.started = Some(started); + } + // Redo the pass with the new face before anything is shown. egui allows a limited number of + // passes per frame: when this was the last, the next frame shows the glyphs. + if output.platform_output.num_completed_passes < ctx.options(|o| o.max_passes.get()) { + output.platform_output.request_discard_reasons.push(egui::RepaintCause::new_reason("read an installed fallback face")); + } else { + ctx.request_repaint(); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::cell::Cell; + use std::ops::RangeInclusive; + + /// A fallback for [`plan`]: its script, what it covers, whether it is installed, and how often + /// it was read. + struct Fake { + script: RangeInclusive, + covers: &'static str, + installed: bool, + read: Cell, + reads: Cell, + } + + impl Fake { + fn new(script: RangeInclusive, covers: &'static str) -> Self { + Self { script, covers, installed: true, read: Cell::new(false), reads: Cell::new(0) } + } + fn read_already(self) -> Self { + self.read.set(true); + self + } + } + + impl Candidate for Fake { + fn in_script(&self, c: char) -> bool { + self.script.contains(&c) + } + fn is_read(&self) -> bool { + self.read.get() + } + fn covers(&self, c: char) -> bool { + self.read.get() && self.covers.contains(c) + } + fn read(&self) -> bool { + if !self.installed { + return false; + } + if !self.read.get() { + self.reads.set(self.reads.get() + 1); + } + self.read.set(true); + true + } + fn might_cover(&self, c: char) -> bool { + self.installed && self.covers.contains(c) + } + } + + const HANGUL: RangeInclusive = '\u{ac00}'..='\u{d7a3}'; + const HAN: RangeInclusive = '\u{4e00}'..='\u{9fff}'; + const TAMIL: RangeInclusive = '\u{b80}'..='\u{bff}'; + const DEVANAGARI: RangeInclusive = '\u{900}'..='\u{97f}'; + + /// Drawn by a face ahead of the fallbacks; by a fallback (or a face after them) only; by none. + const EARLY: Drawn = Drawn { early: true, any: true }; + const LATE: Drawn = Drawn { early: false, any: true }; + const NONE: Drawn = Drawn { early: false, any: false }; + + #[test] + fn text_the_fonts_draw_reads_nothing() { + let korean = Fake::new(HANGUL, "한국"); + assert!(!plan('Ж', EARLY, &[&korean])); + // Hangul the craft-fonts' Chinese face already draws, ahead of the Korean fallback + assert!(!plan('한', EARLY, &[&korean])); + assert_eq!(korean.reads.get(), 0); + } + + #[test] + fn text_nothing_draws_reads_its_scripts_face() { + let hant = Fake::new(HAN, "中文"); + let korean = Fake::new(HANGUL, "한국"); + assert!(plan('한', NONE, &[&hant, &korean])); + assert_eq!((hant.reads.get(), korean.reads.get()), (0, 1), "only the Korean face"); + assert!(!plan('국', NONE, &[&hant, &korean]), "the face read draws the rest of its script"); + } + + /// One face for several scripts (Nirmala UI): read for Tamil, it draws Devanagari too, so the + /// Devanagari face isn't read. + #[test] + fn a_face_read_for_one_script_serves_the_next() { + let tamil = Fake::new(TAMIL, "தக"); + let devanagari = Fake::new(DEVANAGARI, "क"); + assert!(plan('த', NONE, &[&tamil, &devanagari])); + let tamil = Fake { covers: "தகक", ..tamil }; + assert!(!plan('क', LATE, &[&tamil, &devanagari])); + assert_eq!(devanagari.reads.get(), 0); + } + + #[test] + fn faces_for_a_script_are_read_in_order_until_one_covers_the_character() { + let sparse = Fake::new(HAN, "中"); + let full = Fake::new(HAN, "中文"); + let after = Fake::new(HAN, "中文"); + assert!(plan('文', NONE, &[&sparse, &full, &after])); + assert_eq!((sparse.reads.get(), full.reads.get(), after.reads.get()), (1, 1, 0)); + } + + /// The fallback order decides which face draws a character: when only a face that comes later + /// draws it, the earlier one is read so it draws it, as it did when every face was read at + /// start-up. + #[test] + fn an_earlier_face_is_read_when_a_later_one_draws_its_script() { + let hant = Fake::new(HAN, "中文"); + let korean = Fake::new(HANGUL, "한국中").read_already(); + assert!(plan('中', LATE, &[&hant, &korean])); + assert_eq!(hant.reads.get(), 1); + let hant = Fake::new(HAN, "中文").read_already(); + let korean = Fake::new(HANGUL, "한국中"); + assert!(!plan('中', LATE, &[&hant, &korean])); + assert_eq!(korean.reads.get(), 0); + } + + /// A character in no fallback's script that nothing draws reads the face whose character map + /// has it (a symbol only an installed face has), and nothing when no face has it (a newer + /// emoji, Ethiopic, a math sign nobody has). + #[test] + fn a_character_in_no_script_reads_only_the_face_that_has_it() { + let hant = Fake::new(HAN, "中"); + let korean = Fake::new(HANGUL, "한⇢"); + let tamil = Fake::new(TAMIL, "த"); + assert!(plan('⇢', NONE, &[&hant, &korean, &tamil])); + assert_eq!((hant.reads.get(), korean.reads.get(), tamil.reads.get()), (0, 1, 0), "only the face that has it"); + for c in ['\u{1fae0}', '\u{1200}', '\u{2a00}', '\u{f8ff}'] { + assert!(!plan(c, NONE, &[&hant, &korean, &tamil]), "{c:?}: no face has it"); + } + assert_eq!((hant.reads.get(), korean.reads.get(), tamil.reads.get()), (0, 1, 0)); + } + + #[test] + fn a_face_that_is_not_installed_is_skipped() { + let mut missing = Fake::new(HANGUL, "한"); + missing.installed = false; + let other = Fake::new(HAN, "中"); + assert!(!plan('한', NONE, &[&missing])); + assert!(plan('中', NONE, &[&missing, &other])); + assert_eq!(other.reads.get(), 1); + } + + /// Characters are settled per font family: one drawn in a family whose faces have it may still + /// need a fallback in another (a ★ Noto Emoji draws in the proportional family, but not in a + /// named family without Noto Emoji). + #[test] + fn characters_are_settled_per_family() { + let semibold = FontFamily::Name("semibold".into()); + let mut settled = Settled::default(); + settled.insert(FontFamily::Proportional, '★'); + settled.insert(FontFamily::Proportional, '\u{1fae0}'); + assert!(settled.contains(&FontFamily::Proportional, '★')); + assert!(settled.contains(&FontFamily::Proportional, '\u{1fae0}')); + assert!(!settled.contains(&semibold, '★')); + assert!(!settled.contains(&FontFamily::Proportional, '☆')); + } + + #[test] + fn only_new_characters_are_collected() { + let ctx = egui::Context::default(); + ctx.run_ui(egui::RawInput::default(), |_| {}).textures_delta.clear(); + let font = FontId::proportional(12.0); + let galley = ctx.fonts_mut(|f| f.layout_no_wrap("Hello DŽ ɕ 한국 한 ñ".into(), font.clone(), egui::Color32::WHITE)); + let text = Shape::galley(egui::Pos2::ZERO, galley, egui::Color32::WHITE); + let shapes = vec![ClippedShape { clip_rect: egui::Rect::EVERYTHING, shape: Shape::Vec(vec![text]) }]; + let chars: Vec = new_characters(&shapes, &Settled::default()).into_iter().map(|(c, _)| c).collect(); + assert_eq!(chars, ['DŽ', 'ɕ', '한', '국'], "each new character once; ASCII, Latin-1 and spaces skipped"); + let mut settled = Settled::default(); + settled.insert(FontFamily::Proportional, '한'); + let chars: Vec = new_characters(&shapes, &settled).into_iter().map(|(c, _)| c).collect(); + assert_eq!(chars, ['DŽ', 'ɕ', '국']); + } + + /// egui's `has_glyph` answers no for what Noto Emoji (the face of its replacement glyph) + /// draws: an emoji counts as drawn anyway, by a face ahead of the fallbacks unless only + /// fallbacks draw it. + #[test] + fn classify_finds_the_face_that_draws_a_character() { + let fonts = FontDefinitions::default(); + let p = FontFamily::Proportional; + assert_eq!(classify(&fonts, &p, '\u{1f3ac}', false, &[]), Drawn { early: true, any: true }, "🎬 is in Noto Emoji"); + let all_late = ["Ubuntu-Light", "NotoEmoji-Regular", "emoji-icon-font"]; + assert_eq!(classify(&fonts, &p, '\u{1f3ac}', false, &all_late), Drawn { early: false, any: true }, "drawn, but only by late faces"); + assert_eq!(classify(&fonts, &p, '한', false, &[]), Drawn { early: false, any: false }); + assert_eq!(classify(&fonts, &p, '\u{200d}', true, &[]), Drawn { early: true, any: true }, "egui draws nothing for a joiner"); + assert_eq!(classify(&fonts, &FontFamily::Name("none".into()), 'A', false, &[]), Drawn { early: true, any: true }); + } +} diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -17,6 +17,7 @@ pub mod dock; mod dock_shared; pub mod dock_ui; mod expression_diagnostics; +mod fallback_fonts; pub mod frames; pub mod gpu_failure; pub mod header; diff --git a/crates/ui-egui/src/theme.rs b/crates/ui-egui/src/theme.rs --- a/crates/ui-egui/src/theme.rs +++ b/crates/ui-egui/src/theme.rs @@ -329,8 +329,17 @@ static JETBRAINS_MONO: &[u8] = include_bytes!("../../../assets/fonts/JetBrainsMo /// system draws Chinese with the default setting. The CJK fallback that draws the interface's own /// script is registered first, so a Chinese interface is drawn by a Chinese face and a Japanese one /// by a Japanese face. `install` is called again when the language changes (see the `EffectcraftApp` -/// frame loop), so the fonts do not need a restart. +/// frame loop), so the fonts do not need a restart. Installed faces the language doesn't need are +/// read the first time the interface paints text that needs them ([`crate::fallback_fonts`]). pub fn install(ctx: &egui::Context, t: &Tokens, language: &str) { + install_fonts(ctx, language); + ctx.add_plugin(crate::fallback_fonts::Watch::new(language)); + ctx.with_plugin(|w: &mut crate::fallback_fonts::Watch| language.clone_into(&mut w.lang)); + apply_visuals(ctx, t); +} + +/// The fonts [`install`] sets for `language`, with the installed fallbacks read so far. +pub(crate) fn install_fonts(ctx: &egui::Context, language: &str) { let mut fonts = FontDefinitions::default(); fonts.font_data.insert("inter".into(), Arc::new(FontData::from_static(INTER_REGULAR))); fonts.font_data.insert("inter-medium".into(), Arc::new(FontData::from_static(INTER_MEDIUM))); @@ -349,7 +358,6 @@ pub fn install(ctx: &egui::Context, t: &Tokens, language: &str) { } install_script_fallbacks(&mut fonts, language); ctx.set_fonts(fonts); - apply_visuals(ctx, t); } /// Scripts the bundled fonts lack beyond CJK, each probed with one of its letters: names typed in @@ -386,6 +394,9 @@ const SCRIPT_PROBES: &[(char, &str)] = &[ /// other one still render; then [`SCRIPT_PROBES`], then Hangul, since the Japanese and Chinese faces /// may lack it (Hiragino Sans on macOS has none). The web build has no system fonts /// and installs nothing. +/// +/// Only the faces the interface language needs (its CJK faces and the catalog's extra faces) are +/// read here; the others are registered once painted text needed them ([`LazyFallback`]). fn install_script_fallbacks(family_fonts: &mut FontDefinitions, language: &str) -> Vec { #[cfg(target_arch = "wasm32")] { @@ -395,16 +406,8 @@ fn install_script_fallbacks(family_fonts: &mut FontDefinitions, language: &str) #[cfg(not(target_arch = "wasm32"))] { use effectcraft_text::fonts; - // Kana is unambiguously Japanese; Han ideographs are shared, so the order decides which - // language's face draws them. - const KANA: (char, &str) = ('あ', "japanese-system"); - // Hangul is probed last: with no Korean font installed the lookup can land on a catch-all - // face (Unifont) that also covers Arabic or Indic text but can't shape it, so the - // script-specific faces must be registered ahead of it. A face already seen is skipped. - const HANGUL: (char, &str) = ('한', "korean-system"); // Han text follows the interface language, not the operating system's locale. fonts::set_cjk_locale(cjk_locale(language)); - let base = fonts::resolve("Inter", "Regular").face; let han = (han_probe(language), "chinese-system"); let cjk = if is_chinese(language) { [han, KANA] } else { [KANA, han] }; let mut names = Vec::new(); @@ -421,17 +424,11 @@ fn install_script_fallbacks(family_fonts: &mut FontDefinitions, language: &str) if catalog && (uncovered_blocks.contains(&(probe as u32 >> 8)) || seen.iter().any(|&id| fonts::face(id).has_char(probe))) { continue; } - // The interface asks for the platform's UI face first; the order a composed frame - // renders with is `fonts::fallback_for`'s and is not touched here. Another context may - // install a different language concurrently, so the shared fallback is asked with this - // context's own locale rather than the global request. - let id = interface_fallback_candidates(name, language) - .iter() - .find_map(|family| { - let r = fonts::resolve(family, "Regular"); - (!r.missing && fonts::face(r.face).has_char(probe)).then_some(r.face) - }) - .unwrap_or_else(|| fonts::fallback_for_locale(probe, base, cjk_locale(language))); + // installed faces the language doesn't need wait for painted text to need them + if !catalog && !needed(name, language) && !is_read(name) { + continue; + } + let id = interface_face(probe, name, language); let face = fonts::face(id); if !face.has_char(probe) { if catalog { @@ -442,11 +439,9 @@ fn install_script_fallbacks(family_fonts: &mut FontDefinitions, language: &str) if seen.contains(&id) { continue; } - let Some(font) = face.font() else { continue }; + // egui borrows the bytes the font database already read: no copy of the file + let Some(data) = borrowed(face) else { continue }; seen.push(id); - // Use the already-read, parsed bytes rather than reading a font file twice. - let mut data = FontData::from_owned(font.data().as_bytes().to_vec()); - data.index = face.info.index; let name = if name == "catalog-system" { format!("{name}-{:x}", probe as u32) } else { name.to_string() }; family_fonts.font_data.insert(name.clone(), Arc::new(data)); for family in family_fonts.families.values_mut() { @@ -458,6 +453,248 @@ fn install_script_fallbacks(family_fonts: &mut FontDefinitions, language: &str) } } +// Kana is unambiguously Japanese; Han ideographs are shared, so the order decides which language's +// face draws them. +#[cfg(not(target_arch = "wasm32"))] +const KANA: (char, &str) = ('あ', "japanese-system"); +// Hangul is probed last: with no Korean font installed the lookup can land on a catch-all face +// (Unifont) that also covers Arabic or Indic text but can't shape it, so the script-specific faces +// must be registered ahead of it. A face already seen is skipped. +#[cfg(not(target_arch = "wasm32"))] +const HANGUL: (char, &str) = ('한', "korean-system"); + +/// The installed face the interface draws `probe` (of fallback `name`) with for `language`. The +/// interface asks for the platform's UI face first; the order a composed frame renders with is +/// `fonts::fallback_for`'s and is not touched here. Another context may install a different +/// language concurrently, so the shared fallback is asked with this context's own locale rather +/// than the global request. +#[cfg(not(target_arch = "wasm32"))] +fn interface_face(probe: char, name: &str, language: &str) -> effectcraft_text::fonts::FaceId { + use effectcraft_text::fonts; + let base = fonts::resolve("Inter", "Regular").face; + interface_fallback_candidates(name, language) + .iter() + .find_map(|family| { + let r = fonts::resolve(family, "Regular"); + (!r.missing && fonts::face(r.face).has_char(probe)).then_some(r.face) + }) + .unwrap_or_else(|| fonts::fallback_for_locale(probe, base, cjk_locale(language))) +} + +/// Whether the interface language needs fallback `name` whatever it paints: a Japanese or Chinese +/// interface draws its own labels with the CJK faces, in the language's order. +#[cfg(not(target_arch = "wasm32"))] +fn needed(name: &str, language: &str) -> bool { + matches!(name, "japanese-system" | "chinese-system") && !cjk_locale(language).is_empty() +} + +/// The installed fallbacks read for painted text so far ([`crate::fallback_fonts`]), by name. +static READ: std::sync::RwLock> = std::sync::RwLock::new(Vec::new()); + +fn is_read(name: &str) -> bool { + READ.read().unwrap_or_else(std::sync::PoisonError::into_inner).contains(&name) +} + +/// The code points of each installed fallback's script: the text its face is read for (Han, kana +/// and bopomofo with the CJK symbols, punctuation and full-width forms; Hangul with its enclosed and +/// half-width forms; each other script with its extensions and presentation forms). +#[allow(clippy::single_range_in_vec_init)] // one range for some scripts +const SCRIPT_RANGES: &[(&str, &[std::ops::RangeInclusive])] = &[ + ( + "japanese-system", + &[ + '\u{2e80}'..='\u{2fdf}', + '\u{2ff0}'..='\u{303f}', + '\u{3040}'..='\u{30ff}', + '\u{3190}'..='\u{319f}', + '\u{31c0}'..='\u{33ff}', + '\u{3400}'..='\u{4dbf}', + '\u{4e00}'..='\u{9fff}', + '\u{f900}'..='\u{faff}', + '\u{fe30}'..='\u{fe4f}', + '\u{ff00}'..='\u{ffef}', + '\u{1b000}'..='\u{1b16f}', + '\u{20000}'..='\u{3ffff}', + ], + ), + ( + "chinese-system", + &[ + '\u{2e80}'..='\u{2fdf}', + '\u{2ff0}'..='\u{303f}', + '\u{3100}'..='\u{312f}', + '\u{3190}'..='\u{33ff}', + '\u{3400}'..='\u{4dbf}', + '\u{4e00}'..='\u{9fff}', + '\u{f900}'..='\u{faff}', + '\u{fe30}'..='\u{fe4f}', + '\u{ff00}'..='\u{ffef}', + '\u{20000}'..='\u{3ffff}', + ], + ), + ( + "arabic-system", + &['\u{600}'..='\u{6ff}', '\u{750}'..='\u{77f}', '\u{870}'..='\u{8ff}', '\u{fb50}'..='\u{fdff}', '\u{fe70}'..='\u{fefc}', '\u{1ee00}'..='\u{1eeff}'], + ), + ("hebrew-system", &['\u{590}'..='\u{5ff}', '\u{fb1d}'..='\u{fb4f}']), + ("devanagari-system", &['\u{900}'..='\u{97f}', '\u{a8e0}'..='\u{a8ff}', '\u{1cd0}'..='\u{1cff}']), + ("bengali-system", &['\u{980}'..='\u{9ff}']), + ("gurmukhi-system", &['\u{a00}'..='\u{a7f}']), + ("gujarati-system", &['\u{a80}'..='\u{aff}']), + ("tamil-system", &['\u{b80}'..='\u{bff}', '\u{11fc0}'..='\u{11fff}']), + ("telugu-system", &['\u{c00}'..='\u{c7f}']), + ("kannada-system", &['\u{c80}'..='\u{cff}']), + ("malayalam-system", &['\u{d00}'..='\u{d7f}']), + ("thai-system", &['\u{e00}'..='\u{e7f}']), + ("georgian-system", &['\u{10a0}'..='\u{10ff}', '\u{1c90}'..='\u{1cbf}', '\u{2d00}'..='\u{2d2f}']), + ( + "korean-system", + &[ + '\u{1100}'..='\u{11ff}', + '\u{3130}'..='\u{318f}', + '\u{3200}'..='\u{321e}', + '\u{3260}'..='\u{327e}', + '\u{a960}'..='\u{a97f}', + '\u{ac00}'..='\u{d7ff}', + '\u{ffa0}'..='\u{ffdc}', + ], + ), +]; + +/// An installed fallback [`crate::fallback_fonts::Watch`] reads when painted text needs it. +pub(crate) struct LazyFallback { + probe: char, + name: &'static str, + language: String, +} + +impl LazyFallback { + pub(crate) fn name(&self) -> &'static str { + self.name + } + + /// The code points of the face this fallback would read, without reading it: the face the + /// interface asks for first (its UI family, else the first of the platform's usual families for + /// the script). Looked up once per fallback and language. + fn coverage(&self) -> Arc> { + type Cache = Vec<((&'static str, String), Arc>)>; + static CACHE: std::sync::Mutex = std::sync::Mutex::new(Vec::new()); + let key = (self.name, self.language.clone()); + if let Some((_, ranges)) = CACHE.lock().unwrap_or_else(std::sync::PoisonError::into_inner).iter().find(|(k, _)| *k == key) { + return ranges.clone(); + } + #[cfg(target_arch = "wasm32")] + let ranges: Arc> = Arc::default(); + #[cfg(not(target_arch = "wasm32"))] + let ranges = { + use effectcraft_text::fonts; + let base = fonts::resolve("Inter", "Regular").face; + let ui = interface_fallback_candidates(self.name, &self.language).iter().map(|family| fonts::resolve(family, "Regular")).find(|r| !r.missing); + let id = ui.map(|r| r.face).or_else(|| fonts::usual_faces(self.probe, base, cjk_locale(&self.language)).first().copied()); + Arc::new(id.map(|id| fonts::face(id).coverage()).unwrap_or_default()) + }; + CACHE.lock().unwrap_or_else(std::sync::PoisonError::into_inner).push((key, ranges.clone())); + ranges + } + + /// The installed face for the interface language, when one covers the probe. + fn face(&self) -> Option> { + #[cfg(target_arch = "wasm32")] + { + let _ = (self.probe, &self.language); + None + } + #[cfg(not(target_arch = "wasm32"))] + { + let face = effectcraft_text::fonts::face(interface_face(self.probe, self.name, &self.language)); + face.has_char(self.probe).then_some(face) + } + } +} + +impl crate::fallback_fonts::Candidate for LazyFallback { + fn in_script(&self, c: char) -> bool { + SCRIPT_RANGES.iter().find(|(name, _)| *name == self.name).is_some_and(|(_, ranges)| ranges.iter().any(|r| r.contains(&c))) + } + fn is_read(&self) -> bool { + is_read(self.name) + } + fn covers(&self, c: char) -> bool { + self.is_read() && self.face().is_some_and(|f| f.has_char(c)) + } + fn might_cover(&self, c: char) -> bool { + if self.is_read() { + return self.covers(c); + } + let cp = u32::from(c); + let within = |&(a, b): &(u32, u32)| { + if b < cp { + std::cmp::Ordering::Less + } else if a > cp { + std::cmp::Ordering::Greater + } else { + std::cmp::Ordering::Equal + } + }; + self.coverage().binary_search_by(within).is_ok() + } + fn read(&self) -> bool { + if self.face().is_none() { + return false; + } + let mut read = READ.write().unwrap_or_else(std::sync::PoisonError::into_inner); + if !read.contains(&self.name) { + read.push(self.name); + } + true + } +} + +/// The installed fallbacks for `language` that are read when painted text needs them, in the order +/// [`install_script_fallbacks`] registers them. +pub(crate) fn lazy_fallbacks(language: &str) -> Vec { + #[cfg(target_arch = "wasm32")] + { + let _ = language; + Vec::new() + } + #[cfg(not(target_arch = "wasm32"))] + { + let han = (han_probe(language), "chinese-system"); + let cjk = if is_chinese(language) { [han, KANA] } else { [KANA, han] }; + cjk.iter() + .chain(SCRIPT_PROBES) + .chain(std::iter::once(&HANGUL)) + .filter(|(_, name)| !needed(name, language)) + .map(|&(probe, name)| LazyFallback { probe, name, language: language.to_owned() }) + .collect() + } +} + +/// `face` for egui, borrowing the bytes the font database keeps for it. egui 0.36 copies owned font +/// data each time it builds its fonts, so handing it a copy of the file meant three copies of every +/// fallback face: the database's, ours and egui's. `None` when the face's file can't be read. +#[cfg(not(target_arch = "wasm32"))] +fn borrowed(face: Arc) -> Option { + let face = kept(face); + let font = face.font()?; + Some(FontData { index: face.info.index, ..FontData::from_static(font.data().as_bytes()) }) +} + +/// `face` for the rest of the process, so egui can borrow its bytes. The font database never +/// drops a face (ids are stable); one `Arc` per face the interface falls back to is kept here. +#[cfg(not(target_arch = "wasm32"))] +fn kept(face: Arc) -> &'static effectcraft_text::fonts::Face { + static KEPT: std::sync::Mutex> = std::sync::Mutex::new(Vec::new()); + let mut kept = KEPT.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + if let Some(f) = kept.iter().find(|f| f.id == face.id) { + return f; + } + let f: &'static effectcraft_text::fonts::Face = Box::leak(Box::new(face)); + kept.push(f); + f +} + /// The faces the interface asks for before it settles on the one a composed frame renders with. /// /// Windows ships a UI variant of its Japanese and Chinese families - `Meiryo`/`Meiryo UI`, `Yu @@ -591,34 +828,45 @@ mod script_font_tests { } /// Every character of `text` draws a real glyph in every UI font family with the `language` - /// interface's fonts installed. + /// interface's fonts installed, on the first frame that paints it: installed faces are read + /// during that frame ([`crate::fallback_fonts`]). fn assert_drawn(language: &str, text: &str) { let ctx = egui::Context::default(); super::install(&ctx, &super::Tokens::for_kind(super::ThemeKind::Dark), language); - let mut out = ctx.run_ui(egui::RawInput::default(), |_| {}); - // No renderer here: drop the frame's texture uploads (egui asserts on unhandled ones in debug). - out.textures_delta.clear(); - ctx.fonts_mut(|fonts| { - for family in [ - egui::FontFamily::Proportional, - egui::FontFamily::Monospace, - egui::FontFamily::Name("medium".into()), - egui::FontFamily::Name("semibold".into()), - ] { - let font = egui::FontId::new(13.0, family); - // epaint 0.36's has_glyph compares face keys, so it reports a false - // negative when a real glyph shares the replacement face. - // Check the rendered atlas glyph instead of that face-level predicate. - let missing = fonts.layout_no_wrap("\u{10ffff}".into(), font.clone(), egui::Color32::WHITE); - let missing_uv = missing.rows[0].glyphs[0].uv_rect; + let families = + [egui::FontFamily::Proportional, egui::FontFamily::Monospace, egui::FontFamily::Name("medium".into()), egui::FontFamily::Name("semibold".into())]; + // one label per character and family: each glyph is the first of its own galley + let mut out = ctx.run_ui(egui::RawInput::default(), |ui| { + for family in &families { for ch in text.chars() { - let rendered = fonts.layout_no_wrap(ch.to_string(), font.clone(), egui::Color32::WHITE); - let uv = rendered.rows[0].glyphs[0].uv_rect; - assert!(uv.size.x > 0.0 && uv.size.y > 0.0, "empty {ch} in {font:?}"); - assert_ne!((uv.min, uv.max), (missing_uv.min, missing_uv.max), "replacement glyph for {ch} in {font:?}"); + ui.label(egui::RichText::new(ch.to_string()).font(egui::FontId::new(13.0, family.clone()))); } } }); + // No renderer here: drop the frame's texture uploads (egui asserts on unhandled ones in debug). + out.textures_delta.clear(); + let galleys: Vec<_> = out + .shapes + .iter() + .filter_map(|s| match &s.shape { + egui::Shape::Text(t) => Some(t.galley.clone()), + _ => None, + }) + .filter(|g| g.job.text.chars().count() == 1 && text.contains(g.job.text.as_str())) + .collect(); + assert_eq!(galleys.len(), families.len() * text.chars().count(), "every label is painted"); + for galley in galleys { + let font = galley.job.sections[0].format.font_id.clone(); + let ch = galley.job.text.clone(); + // epaint 0.36's has_glyph compares face keys, so it reports a false + // negative when a real glyph shares the replacement face. + // Check the rendered atlas glyph instead of that face-level predicate. + let missing_uv = ctx.fonts_mut(|f| f.layout_no_wrap("\u{10ffff}".into(), font.clone(), egui::Color32::WHITE).rows[0].glyphs[0].uv_rect); + let uv = galley.rows[0].glyphs[0].uv_rect; + assert!(uv.size.x > 0.0 && uv.size.y > 0.0, "empty {ch} in {font:?}"); + assert_ne!((uv.min, uv.max), (missing_uv.min, missing_uv.max), "replacement glyph for {ch} in {font:?} on the first frame"); + } + ctx.run_ui(egui::RawInput::default(), |_| {}).textures_delta.clear(); } #[test] @@ -711,7 +959,10 @@ mod script_font_tests { } let ctx = egui::Context::default(); super::install(&ctx, &super::Tokens::for_kind(super::ThemeKind::Dark), "en"); - let mut out = ctx.run_ui(egui::RawInput::default(), |_| {}); + // the frame that first paints Arabic reads the installed Arabic face + let mut out = ctx.run_ui(egui::RawInput::default(), |ui| { + ui.label("مرحبا"); + }); out.textures_delta.clear(); ctx.fonts_mut(|fonts| { let font = egui::FontId::new(13.0, egui::FontFamily::Proportional); @@ -795,7 +1046,10 @@ mod tests { #[test] fn hangul_gets_a_face_when_one_is_installed() { + use crate::fallback_fonts::Candidate; use effectcraft_text::fonts; + // as the interface does the first time it paints Hangul + let _ = lazy_fallbacks("en").iter().find(|f| f.name() == "korean-system").map(|f| f.read()); let mut fonts_def = FontDefinitions::default(); install_script_fallbacks(&mut fonts_def, "en"); let base = fonts::resolve("Inter", "Regular").face; @@ -811,6 +1065,41 @@ mod tests { assert!(covered, "a face covering Hangul must be registered with egui"); } + /// egui borrows the font database's bytes for every installed fallback: installing the fonts + /// again (another language, another window) hands egui the same bytes, never a copy of the file. + #[test] + fn fallback_faces_are_borrowed_not_copied() { + let mut first = FontDefinitions::default(); + let installed = install_script_fallbacks(&mut first, "ja"); + let mut again = FontDefinitions::default(); + install_script_fallbacks(&mut again, "ja"); + if installed.is_empty() { + eprintln!("SKIPPED: no Japanese or Chinese system font installed"); + } + for name in &installed { + let (Some(a), Some(b)) = (first.font_data.get(name), again.font_data.get(name)) else { panic!("{name} is missing") }; + assert_eq!(a.font.as_ptr(), b.font.as_ptr(), "{name}: egui borrows the same bytes each install, never a copy"); + } + install_script_fallbacks(&mut first, "system"); + } + + /// The faces read when painted text needs them come in the order the fonts register them, and + /// a Japanese or Chinese interface reads its own CJK faces with the fonts instead. + #[cfg(not(target_arch = "wasm32"))] + #[test] + fn lazy_fallbacks_follow_the_registration_order() { + let names = |language: &str| lazy_fallbacks(language).iter().map(LazyFallback::name).collect::>(); + let en = names("en"); + assert_eq!(en.first(), Some(&"japanese-system"), "{en:?}"); + assert_eq!(en.get(1), Some(&"chinese-system"), "{en:?}"); + assert_eq!(en.last(), Some(&"korean-system"), "{en:?}"); + for language in ["ja", "zh-hans", "zh-hant"] { + let list = names(language); + assert!(!list.contains(&"japanese-system") && !list.contains(&"chinese-system"), "{language}: {list:?}"); + assert_eq!(list.first(), Some(&"arabic-system"), "{language}: {list:?}"); + } + } + #[test] fn a_fallback_face_is_registered_at_most_once() { let mut fonts = FontDefinitions::default(); diff --git a/docs/localization-parity.md b/docs/localization-parity.md --- a/docs/localization-parity.md +++ b/docs/localization-parity.md @@ -53,6 +53,10 @@ interface; Arabic and Hebrew text *in layers* is supported through its Middle Ea - **Fonts:** no bundled CJK font; the UI installs system fallbacks probed by script (Kana, Han, Arabic, Hebrew, Devanagari, Bengali, Gurmukhi, Gujarati, Tamil, Telugu, Kannada, Malayalam, Thai and Hangul: `crates/ui-egui/src/theme.rs`; the Hangul probe was added for #599, closed). The web build has no system fonts. + A Japanese or Chinese interface reads its CJK faces when the fonts are installed; every other + fallback is read the first time the interface paints text in its script, and that frame is + redone with it before it is shown (`crates/ui-egui/src/fallback_fonts.rs`). egui borrows the + font database's bytes, so each face is in memory once. - **Script support in the interface:** egui draws text without complex shaping or right-to-left layout, so an Arabic or Hindi interface would render unshaped and left to right. **In text layers** (`crates/text`) shaping (harfrust), bidi (UAX #9), vertical text and Tate-Chu-Yoko