diff --git a/CHANGELOG.md b/CHANGELOG.md index 4a4eb8b8..312a6e2e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,34 @@ All notable changes to PolterType are recorded here. The format is loosely based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and the project follows [Semantic Versioning](https://semver.org/). +## [Unreleased] + +### Added + +- **Arabic.** A bundled `ar` layout (Arabic 101, read off the OS rather + than transcribed), a 2.5 M-form dictionary from Ayaspell's stems + with the article forms added, Arabic's long vowels for the + plausibility detector, and a short + stop-word list checked so that none of its words is what an English + stop word types as. macOS maps `Arabic`, `Arabic - PC`, + `Arabic - QWERTY`, `Arabic - AZERTY` and `Arabic - North Africa` to + it; Linux maps `ara`. +- **macOS reads your keyboards from the OS.** The layout backend now + answers `describe_keymaps` with `UCKeyTranslate`, as the Windows one + does with `ToUnicodeEx`, so a variant whose keys differ from the + bundled table — Apple's own Arabic against Arabic 101, Russian + Phonetic against Russian — is corrected on the keys you actually + have. + +### Fixed + +- **Arabic presentation forms count as Arabic script.** `ﻻ` and the + rest of U+FB50–FDFF and U+FE70–FEFF read as "other script" and + dragged a word's script fit down. +- **`cargo xtask wordlists fetch` finds a `SET` anywhere in the `.aff`, + tab-separated or not,** and no longer parses rules it will not use + for a stems-only dictionary. + ## [0.38.2] — hands off first ### Fixed diff --git a/README.md b/README.md index b98ab22e..eb28a96d 100644 --- a/README.md +++ b/README.md @@ -386,22 +386,22 @@ them on the **Commands** pane in Settings. ## Languages -Fifteen layouts ship with the app, each with a full dictionary: +Sixteen layouts ship with the app, each with a full dictionary: **English (US) · Ukrainian · Russian · German · Spanish · French · Polish · Czech · Greek · Hebrew · Turkish · Bulgarian · Italian · -Portuguese (PT) · Portuguese (BR)** +Portuguese (PT) · Portuguese (BR) · Arabic** PolterType only loads the ones your OS actually has enabled, so -bundling fifteen costs a two-keyboard user nothing at runtime. +bundling sixteen costs a two-keyboard user nothing at runtime. **On Windows the bundled table is only a starting point.** A Windows layout is named after a *language*, but a language is not a keyboard — Bulgarian alone ships in three genuinely different arrangements. Since v0.14.3 PolterType asks Windows what each of your installed keyboards actually produces and uses that answer, so variants and even custom -layouts we have never heard of come out right. Linux and macOS read -the bundled tables, and a TOML of your own in +layouts we have never heard of come out right; macOS does the same +through `UCKeyTranslate`. Linux reads the bundled tables, and a TOML of your own in `/poltertype/layouts/` still outranks everything, everywhere. @@ -418,6 +418,11 @@ Two are worth a footnote rather than a surprise: because expanding its clitic prefixes yields 60 million of them. Hebrew shares its script with nothing else bundled, so detection leans on that and the dictionary refines it. +- **Arabic** follows Hebrew for the same reason — its article and + clitic particles are prefixes — and adds the article forms (`ال`, + `بال`, `لل`, …) back to the list. The lam-alef keys (`لا` on `B`) type + two code points, and a key here is one character, so they carry the + single-code-point form `ﻻ`, which looks the same. **Yours isn't here?** Adding a language is one TOML file and one wordlist — no Rust required. The walkthrough is diff --git a/crates/poltertype-core/build.rs b/crates/poltertype-core/build.rs index 0615c07d..5449aa5b 100644 --- a/crates/poltertype-core/build.rs +++ b/crates/poltertype-core/build.rs @@ -42,6 +42,7 @@ const LAYOUTS: &[(&str, &str)] = &[ ("it_it", "it-IT"), ("pt_pt", "pt-PT"), ("pt_br", "pt-BR"), + ("ar", "ar"), ]; fn main() { diff --git a/crates/poltertype-core/src/engine/tests.rs b/crates/poltertype-core/src/engine/tests.rs index 0a35db97..f66d58ae 100644 --- a/crates/poltertype-core/src/engine/tests.rs +++ b/crates/poltertype-core/src/engine/tests.rs @@ -754,6 +754,84 @@ mod engine_integration_tests { ); } + /// Both directions of the Arabic ↔ English mix-up through the real + /// pipeline, and neither word disturbed in its own layout. Renders + /// each word through both tables the way the engine does, then asks + /// the detectors. + #[test] + fn arabic_and_english_typed_in_the_wrong_layout_are_switched() { + use crate::layouts::LayoutDb; + let db = LayoutDb::load_embedded(); + let (en, ar) = (LayoutId::from("en-US"), LayoutId::from("ar")); + let en_map = db.get(&en).expect("en-US"); + let ar_map = db.get(&ar).expect("ar"); + let render = |typed: &str| -> String { + typed + .chars() + .map(|ch| { + let (sc, shift) = en_us_key(en_map, ch); + let (plain, shifted) = ar_map.keys[&sc]; + if shift { + shifted.unwrap_or(plain) + } else { + plain + } + }) + .collect() + }; + let judge = |current: &LayoutId, typed: &str| -> Option { + let candidates = vec![(en.clone(), typed.to_owned()), (ar.clone(), render(typed))]; + let ctx = poltertype_detect::DetectionContext { + current_layout: current, + candidates: &candidates, + recent_context: "", + }; + for d in real_detectors() { + match d.judge(&ctx) { + Verdict::NoOpinion => continue, + Verdict::Keep { .. } => return None, + Verdict::Switch(v) => return Some(v.best_layout), + } + } + None + }; + // السلام عليكم, كتاب, مدرسة on Arabic 101 — and السلام again + // with its lam-alef typed on `B`, which renders as `ﻻ`. + for typed in ["hgsghl", "ugd;l", ";jhf", "l]vsm", "hgsbl"] { + assert_eq!( + judge(&en, typed), + Some(ar.clone()), + "{typed:?} -> {}", + render(typed) + ); + assert_eq!( + judge(&ar, typed), + None, + "{} is Arabic typed in Arabic", + render(typed) + ); + } + // بسرعه and كتب carry no long vowel: one consonant cluster to the + // shape score, while their en-US renderings look like words. + // Typed in Arabic they must stay Arabic. + for typed in ["fsvui", ";jf"] { + assert_eq!( + judge(&ar, typed), + None, + "{} is Arabic typed in Arabic", + render(typed) + ); + } + for typed in ["hello", "world", "table", "school"] { + assert_eq!(judge(&en, typed), None, "{typed:?} is English"); + assert_eq!( + judge(&ar, typed), + Some(en.clone()), + "{typed:?} typed in Arabic" + ); + } + } + /// Issue #33, the residue on 0.21.0: `auto-switch ` typed in /// en-US came back as `ФГЕЩ-ЫЦШЕСР `. Both halves of that need /// explaining, and this pins the half that needs no Caps Lock — diff --git a/crates/poltertype-core/src/layouts/consts.rs b/crates/poltertype-core/src/layouts/consts.rs index ce3f8190..fff9862b 100644 --- a/crates/poltertype-core/src/layouts/consts.rs +++ b/crates/poltertype-core/src/layouts/consts.rs @@ -11,7 +11,7 @@ /// "missing TOML" warning at startup, never silently. pub const BUNDLED_LAYOUT_STEMS: &[&str] = &[ "en_us", "uk_ua", "ru_ru", "de_de", "es_es", "fr_fr", "pl_pl", "cs_cz", "el_gr", "he_il", - "tr_tr", "bg_bg", "it_it", "pt_pt", "pt_br", + "tr_tr", "bg_bg", "it_it", "pt_pt", "pt_br", "ar", ]; /// Fewest keys an [`poltertype_types::OsKeymap`] must carry before it diff --git a/crates/poltertype-core/src/layouts/helpers.rs b/crates/poltertype-core/src/layouts/helpers.rs index 7028a863..63fb59b5 100644 --- a/crates/poltertype-core/src/layouts/helpers.rs +++ b/crates/poltertype-core/src/layouts/helpers.rs @@ -46,6 +46,11 @@ pub fn derive_vowels(id: &LayoutId, script: Script) -> Vec { "bg-BG" => "аеиоуъюя".chars().collect(), "it-IT" => "aeiouàèéìíîòóùú".chars().collect(), "pt-PT" | "pt-BR" => "aeiouáàâãéêíóôõúü".chars().collect(), + // Arabic writes only its long vowels: alef in every hamza and + // madda form, waw, yeh, alef maksura, and the lam-alef keys. + // An empty set scored every Arabic word as one long consonant + // cluster — 0.0 — so plausibility could never pick Arabic. + "ar" => "اأإآويىﻻﻵﻷﻹ".chars().collect(), _ => match script { Script::Latin => "aeiouy".chars().collect(), Script::Cyrillic => "аеиіоуюяєїыэё".chars().collect(), diff --git a/crates/poltertype-core/src/layouts/tests.rs b/crates/poltertype-core/src/layouts/tests.rs index 46cc47fb..4240e70e 100644 --- a/crates/poltertype-core/src/layouts/tests.rs +++ b/crates/poltertype-core/src/layouts/tests.rs @@ -12,7 +12,7 @@ fn embedded_layouts_load() { let db = LayoutDb::load_embedded(); for id in [ "en-US", "uk-UA", "ru-RU", "de-DE", "es-ES", "fr-FR", "pl-PL", "cs-CZ", "el-GR", "he-IL", - "tr-TR", "bg-BG", "it-IT", "pt-PT", "pt-BR", + "tr-TR", "bg-BG", "it-IT", "pt-PT", "pt-BR", "ar", ] { assert!( db.get(&LayoutId::from(id)).is_some(), @@ -112,6 +112,9 @@ fn new_languages_translate_distinctive_keys() { // Hebrew is unicase — the letter positions are RTL glyphs. ("he-IL", 0x1E, false, 'ש'), ("he-IL", 0x14, false, 'א'), + // Arabic is unicase too; `B` types lam-alef as one code point. + ("ar", 0x1E, false, 'ش'), + ("ar", 0x30, false, '\u{FEFB}'), // Turkish's dotless/dotted i pair, the one that bites. ("tr-TR", 0x17, false, 'ı'), ("tr-TR", 0x17, true, 'I'), @@ -233,6 +236,7 @@ fn every_bundled_dictionary_knows_its_own_words() { ("cs-CZ", &["ahoj", "slovo", "dobrý", "protože", "děkuji"]), ("el-GR", &["λέξη", "είναι", "καλημέρα", "ευχαριστώ"]), ("he-IL", &["שלום", "מילה", "ספר", "תודה"]), + ("ar", &["كتاب", "مدرسة", "شكرا", "جميل"]), ( "tr-TR", &["kitap", "merhaba", "kelime", "güzel", "teşekkür"], diff --git a/crates/poltertype-detect/src/enums.rs b/crates/poltertype-detect/src/enums.rs index 2d5da86f..3ddd407e 100644 --- a/crates/poltertype-detect/src/enums.rs +++ b/crates/poltertype-detect/src/enums.rs @@ -47,7 +47,9 @@ impl Script { 0x0370..=0x03FF => Self::Greek, 0x0530..=0x058F => Self::Armenian, 0x0590..=0x05FF => Self::Hebrew, - 0x0600..=0x06FF => Self::Arabic, + // The presentation forms count too: the lam-alef keys of an + // Arabic keyboard type U+FEFB and its siblings. + 0x0600..=0x06FF | 0x0750..=0x077F | 0xFB50..=0xFDFF | 0xFE70..=0xFEFF => Self::Arabic, _ => Self::Other, } } diff --git a/crates/poltertype-detect/src/plausibility.rs b/crates/poltertype-detect/src/plausibility.rs index b87d9281..787910f9 100644 --- a/crates/poltertype-detect/src/plausibility.rs +++ b/crates/poltertype-detect/src/plausibility.rs @@ -98,6 +98,22 @@ impl WordPlausibilityDetector { None } + /// Whether `text` is wholly letters of `layout`'s own abjad. Arabic + /// and Hebrew leave short vowels unwritten, so a correctly typed word + /// reads as one consonant cluster (`بسرعه` scores 0.0) while its + /// Latin rendering lands vowels (`fsnui` scores 0.75) and wins. + /// Shape cannot tell those apart; only the dictionary can. + fn abjad_in_own_script(&self, layout: &LayoutId, text: &str) -> bool { + let Some(prof) = self.profiles.get(layout) else { + return false; + }; + let mut letters = text.chars().filter(|c| c.is_alphabetic()).peekable(); + matches!(prof.script, Script::Arabic | Script::Hebrew) + && non_word_char_count(text) == 0 + && letters.peek().is_some() + && letters.all(|c| Script::of(c) == prof.script) + } + /// The plain word-shape score, with no compound handling. fn score(prof: &LayoutProfile, text: &str) -> f32 { let letters: Vec = text @@ -216,6 +232,19 @@ impl Detector for WordPlausibilityDetector { }; } + // Leaving an abjad is the dictionary's call: an English word typed + // on the Arabic keys is a dictionary hit there, and anything less + // is safer kept as typed than replaced on shape alone. + if self.abjad_in_own_script(ctx.current_layout, current_text) { + return Verdict::Keep { + reason: format!( + "current {} is in {}'s own abjad; left to the dictionary", + logsafe::redact_word(current_text), + ctx.current_layout + ), + }; + } + if current_fit >= self.keep_threshold { return Verdict::Keep { reason: format!( diff --git a/crates/poltertype-layout/src/linux/shared.rs b/crates/poltertype-layout/src/linux/shared.rs index b92744a0..e28cf1af 100644 --- a/crates/poltertype-layout/src/linux/shared.rs +++ b/crates/poltertype-layout/src/linux/shared.rs @@ -100,6 +100,7 @@ pub fn bcp47_to_xkb(bcp: &str) -> Option<&'static str> { "hy-AM" => "am", "ka-GE" => "ge", "he-IL" => "il", + "ar" => "ara", "ja-JP" => "jp", "ko-KR" => "kr", _ => return None, diff --git a/crates/poltertype-layout/src/macos.rs b/crates/poltertype-layout/src/macos.rs index 964baace..b7fd2c03 100644 --- a/crates/poltertype-layout/src/macos.rs +++ b/crates/poltertype-layout/src/macos.rs @@ -16,9 +16,11 @@ use core_foundation::base::{CFRelease, CFTypeRef, OSStatus, TCFType}; use core_foundation::dictionary::{CFDictionary, CFDictionaryRef}; use core_foundation::string::{CFString, CFStringRef}; -use crate::{LayoutError, LayoutId, LayoutSwitcher}; +use crate::{LayoutError, LayoutId, LayoutSwitcher, OsKeymap}; +mod consts; mod ids; +mod keymap; use ids::{bcp47_to_tis_id, source_id_string, source_id_to_layout_id, tis_id_to_bcp47}; // ─── Carbon FFI ────────────────────────────────────────────────────── @@ -152,6 +154,10 @@ impl LayoutSwitcher for MacosLayoutSwitcher { fn backend_name(&self) -> &'static str { "macos-tis" } + + fn describe_keymaps(&self) -> Result, LayoutError> { + run_on_main(keymap::describe_all) + } } fn current_impl() -> Result { @@ -170,8 +176,9 @@ fn current_impl() -> Result { } } -fn list_active_impl() -> Result, LayoutError> { - // Filter for keyboard sources that the user can select. +/// The keyboard sources the user can select — what `list_active` names +/// and `describe_keymaps` describes. `None` when TIS returns no list. +fn selectable_keyboards() -> Option> { let filter = unsafe { let category_key = CFString::wrap_under_get_rule(kTISPropertyInputSourceCategory); let category_val = CFString::wrap_under_get_rule(kTISCategoryKeyboardInputSource); @@ -189,17 +196,24 @@ fn list_active_impl() -> Result, LayoutError> { unsafe { let arr_ref = TISCreateInputSourceList(filter.as_concrete_TypeRef(), false); if arr_ref.is_null() { - return Ok(Vec::new()); + return None; } - let arr: CFArray = CFArray::wrap_under_create_rule(arr_ref); - let mut out = Vec::with_capacity(arr.len() as usize); - for src in arr.iter() { - if let Some(id) = source_id_to_layout_id(*src) { - out.push(id); - } + Some(CFArray::wrap_under_create_rule(arr_ref)) + } +} + +fn list_active_impl() -> Result, LayoutError> { + let Some(arr) = selectable_keyboards() else { + return Ok(Vec::new()); + }; + let mut out = Vec::with_capacity(arr.len() as usize); + for src in arr.iter() { + // Safety: `src` lives as long as `arr`. + if let Some(id) = unsafe { source_id_to_layout_id(*src) } { + out.push(id); } - Ok(out) } + Ok(out) } fn switch_to_impl(id: &LayoutId) -> Result<(), LayoutError> { @@ -236,3 +250,6 @@ fn switch_to_impl(id: &LayoutId) -> Result<(), LayoutError> { Err(LayoutError::NotActive(id.clone())) } + +#[cfg(test)] +mod tests; diff --git a/crates/poltertype-layout/src/macos/consts.rs b/crates/poltertype-layout/src/macos/consts.rs new file mode 100644 index 00000000..858930d2 --- /dev/null +++ b/crates/poltertype-layout/src/macos/consts.rs @@ -0,0 +1,55 @@ +//! Constants the macOS layout backend shares between its files. + +/// Win SC Set-1 scancode → Apple virtual keycode for the character block, +/// the same keys the bundled TOMLs list. Kept here rather than borrowed +/// from `poltertype-input`, which this crate does not depend on. +pub(super) const CHARACTER_KEYS: &[(u32, u16)] = &[ + (0x29, 0x32), // backtick + (0x02, 0x12), + (0x03, 0x13), + (0x04, 0x14), + (0x05, 0x15), + (0x06, 0x17), + (0x07, 0x16), + (0x08, 0x1A), + (0x09, 0x1C), + (0x0A, 0x19), + (0x0B, 0x1D), + (0x0C, 0x1B), + (0x0D, 0x18), + (0x10, 0x0C), // Q + (0x11, 0x0D), + (0x12, 0x0E), + (0x13, 0x0F), + (0x14, 0x11), + (0x15, 0x10), + (0x16, 0x20), + (0x17, 0x22), + (0x18, 0x1F), + (0x19, 0x23), + (0x1A, 0x21), + (0x1B, 0x1E), + (0x1E, 0x00), // A + (0x1F, 0x01), + (0x20, 0x02), + (0x21, 0x03), + (0x22, 0x05), + (0x23, 0x04), + (0x24, 0x26), + (0x25, 0x28), + (0x26, 0x25), + (0x27, 0x29), + (0x28, 0x27), + (0x2B, 0x2A), // backslash + (0x2C, 0x06), // Z + (0x2D, 0x07), + (0x2E, 0x08), + (0x2F, 0x09), + (0x30, 0x0B), + (0x31, 0x2D), + (0x32, 0x2E), + (0x33, 0x2B), + (0x34, 0x2F), + (0x35, 0x2C), + (0x56, 0x0A), // ISO key left of Z +]; diff --git a/crates/poltertype-layout/src/macos/ids.rs b/crates/poltertype-layout/src/macos/ids.rs index bf15131a..4a121dc2 100644 --- a/crates/poltertype-layout/src/macos/ids.rs +++ b/crates/poltertype-layout/src/macos/ids.rs @@ -52,6 +52,11 @@ pub(super) fn tis_id_to_bcp47(id: &str) -> Option { "com.apple.keylayout.Hebrew" => "he-IL", "com.apple.keylayout.Hebrew-PC" => "he-IL", "com.apple.keylayout.Hebrew-QWERTY" => "he-IL", + "com.apple.keylayout.Arabic" => "ar", + "com.apple.keylayout.ArabicPC" => "ar", + "com.apple.keylayout.Arabic-QWERTY" => "ar", + "com.apple.keylayout.Arabic-AZERTY" => "ar", + "com.apple.keylayout.Arabic-NorthAfrica" => "ar", "com.apple.keylayout.Turkish" => "tr-TR", "com.apple.keylayout.Turkish-Standard" => "tr-TR", "com.apple.keylayout.Turkish-QWERTY-PC" => "tr-TR", @@ -89,6 +94,7 @@ pub(super) fn bcp47_to_tis_id(id: &str) -> Option { "el-GR" => "com.apple.keylayout.Greek", "cs-CZ" => "com.apple.keylayout.Czech", "he-IL" => "com.apple.keylayout.Hebrew", + "ar" => "com.apple.keylayout.Arabic", "tr-TR" => "com.apple.keylayout.Turkish", "bg-BG" => "com.apple.keylayout.Bulgarian", "it-IT" => "com.apple.keylayout.Italian", diff --git a/crates/poltertype-layout/src/macos/keymap.rs b/crates/poltertype-layout/src/macos/keymap.rs new file mode 100644 index 00000000..a9e64264 --- /dev/null +++ b/crates/poltertype-layout/src/macos/keymap.rs @@ -0,0 +1,171 @@ +//! Describing keyboards: what each key of the character block produces +//! under one TIS source, read with `UCKeyTranslate` against the source's +//! `kTISPropertyUnicodeKeyLayoutData` — the macOS counterpart of the +//! Windows backend's `ToUnicodeEx` walk. +//! +//! A language is not a keyboard: Apple ships `Arabic`, `Arabic - PC` and +//! `Arabic - North Africa`, which disagree on most of the bottom row, and +//! every one of them reports the same `LayoutId`. Asking the OS is the +//! only way the DB learns which one the user actually has. + +use core_foundation::base::{CFRelease, TCFType}; +use core_foundation::data::{CFData, CFDataRef}; +use core_foundation::string::CFStringRef; +use poltertype_types::OsKeymap; + +use super::consts::CHARACTER_KEYS; +use super::ids::{source_id_string, source_id_to_layout_id}; +use super::{ + TISCopyCurrentKeyboardInputSource, TISGetInputSourceProperty, TISInputSourceRef, + selectable_keyboards, +}; +use crate::LayoutError; + +#[link(name = "Carbon", kind = "framework")] +unsafe extern "C" { + static kTISPropertyUnicodeKeyLayoutData: CFStringRef; + fn UCKeyTranslate( + keyLayoutPtr: *const u8, + virtualKeyCode: u16, + keyAction: u16, + modifierKeyState: u32, + keyboardType: u32, + keyTranslateOptions: u32, + deadKeyState: *mut u32, + maxStringLength: usize, + actualStringLength: *mut usize, + unicodeString: *mut u16, + ) -> i32; + fn LMGetKbdType() -> u8; +} + +/// `kUCKeyActionDown`. +const KEY_ACTION_DOWN: u16 = 0; +/// `shiftKey >> 8`, the form `UCKeyTranslate` takes modifiers in. +const SHIFT_STATE: u32 = 0x02; +/// `kUCKeyTranslateNoDeadKeysMask`: a dead key reports its own character +/// instead of arming a compose state, which is what a key table wants. +const NO_DEAD_KEYS: u32 = 1; +/// Arabic tatweel, the kashida stretching mark. +const TATWEEL: char = '\u{0640}'; + +/// Describe one input source, or `None` when it carries no Unicode key +/// layout (input methods, palettes) and so is not a keyboard to describe. +/// +/// # Safety +/// `src` must be a live `TISInputSourceRef`. +pub(super) unsafe fn describe_source(src: TISInputSourceRef) -> Option { + let id = unsafe { source_id_to_layout_id(src) }?; + let variant = unsafe { source_id_string(src) }.unwrap_or_default(); + // Get-rule: the property is owned by the source, not by us. + let data_ref = + unsafe { TISGetInputSourceProperty(src, kTISPropertyUnicodeKeyLayoutData) } as CFDataRef; + if data_ref.is_null() { + return None; + } + let data = unsafe { CFData::wrap_under_get_rule(data_ref) }; + let layout = data.bytes().as_ptr(); + let kbd_type = u32::from(unsafe { LMGetKbdType() }); + + let mut keys = Vec::with_capacity(CHARACTER_KEYS.len()); + for &(sc, kvk) in CHARACTER_KEYS { + let Some(plain) = (unsafe { char_at(layout, kvk, 0, kbd_type) }) else { + continue; + }; + // `None` when Shift changes nothing — same convention the bundled + // TOMLs use for their optional `shift` field. + let shift = unsafe { char_at(layout, kvk, SHIFT_STATE, kbd_type) }.filter(|c| *c != plain); + keys.push((sc, plain, shift)); + } + tracing::debug!(%id, %variant, keys = keys.len(), "described keyboard"); + Some(OsKeymap { id, variant, keys }) +} + +/// The single printable character a key produces, or `None` for nothing, +/// a control character, or any other multi-character output — a key +/// table entry is one `char`. Arabic's lam-alef keys type two code points +/// and are folded to their presentation form, as `ar.toml` does. +/// +/// Tatweel (U+0640) is `None` too, as it is in `ar.toml`: it is +/// alphabetic but stretches a word rather than spelling one, and Apple's +/// Arabic keyboards put it on Shift+`-`, where it would make `_` read as +/// a letter and switch off the snake_case heuristic. +/// +/// # Safety +/// `layout` must point at a live `UCKeyboardLayout`. +unsafe fn char_at(layout: *const u8, kvk: u16, modifiers: u32, kbd_type: u32) -> Option { + let mut dead_state = 0u32; + let mut len = 0usize; + let mut buf = [0u16; 4]; + let status = unsafe { + UCKeyTranslate( + layout, + kvk, + KEY_ACTION_DOWN, + modifiers, + kbd_type, + NO_DEAD_KEYS, + &mut dead_state, + buf.len(), + &mut len, + buf.as_mut_ptr(), + ) + }; + if status != 0 || len == 0 { + return None; + } + if let Some(ligature) = lam_alef(&buf[..len]) { + return Some(ligature); + } + let mut chars = char::decode_utf16(buf[..len].iter().copied()); + let c = chars.next()?.ok()?; + if chars.next().is_some() || c.is_control() || c == TATWEEL { + return None; + } + Some(c) +} + +/// Every selectable keyboard, described; the one in effect comes first. +pub(super) fn describe_all() -> Result, LayoutError> { + let Some(arr) = selectable_keyboards() else { + return Ok(Vec::new()); + }; + // Safety: a copied source is retained and released right after its + // id is read; the sources in `arr` live as long as `arr`. + let current = unsafe { + let src = TISCopyCurrentKeyboardInputSource(); + if src.is_null() { + None + } else { + let id = source_id_string(src); + CFRelease(src); + id + } + }; + let mut out: Vec = arr + .iter() + .filter_map(|src| unsafe { describe_source(*src) }) + .collect(); + + // The keyboard in effect right now goes first: two keyboards of one + // language collapse to one `LayoutId`, and the DB keeps the first. + if let Some(pos) = out + .iter() + .position(|k| Some(&k.variant) == current.as_ref()) + { + out.swap(0, pos); + } + Ok(out) +} + +/// The single-code-point presentation form of a lam-alef pair, so the +/// `B` key of an Arabic keyboard keeps its entry instead of being dropped. +fn lam_alef(utf16: &[u16]) -> Option { + match utf16 { + [0x0644, 0x0627] => Some('\u{FEFB}'), + [0x0644, 0x0623] => Some('\u{FEF7}'), + [0x0644, 0x0625] => Some('\u{FEF9}'), + [0x0644, 0x0622] => Some('\u{FEF5}'), + _ => None, + } +} diff --git a/crates/poltertype-layout/src/macos/tests.rs b/crates/poltertype-layout/src/macos/tests.rs new file mode 100644 index 00000000..7496e931 --- /dev/null +++ b/crates/poltertype-layout/src/macos/tests.rs @@ -0,0 +1,42 @@ +//! The TIS calls themselves must run on the main dispatch queue, which a +//! test thread cannot reach, so these pin the key table the describer +//! walks rather than the OS answer. + +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + +use super::consts::CHARACTER_KEYS; + +/// Every scancode a bundled layout lists must be one the describer asks +/// about — a key it never asks about is a key an OS keymap silently +/// drops when it replaces the bundled table. +#[test] +fn character_keys_cover_every_bundled_scancode() { + let path = concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../data/layout-mappings/en_us.toml" + ); + let toml = std::fs::read_to_string(path).expect("read en_us.toml"); + for line in toml.lines().filter(|l| l.starts_with("0x")) { + let hex = line.split_whitespace().next().unwrap_or_default(); + let sc = u32::from_str_radix(hex.trim_start_matches("0x"), 16).expect("hex scancode"); + assert!( + CHARACTER_KEYS.iter().any(|&(k, _)| k == sc), + "0x{sc:02X} is bundled but never described" + ); + } +} + +/// One row per scancode and per Apple keycode — a duplicate would +/// describe one physical key twice under two names. +#[test] +fn character_keys_are_one_to_one() { + let mut scancodes = std::collections::HashSet::new(); + let mut keycodes = std::collections::HashSet::new(); + for &(sc, kvk) in CHARACTER_KEYS { + assert!(scancodes.insert(sc), "SC-1 0x{sc:02X} listed twice"); + assert!( + keycodes.insert(kvk), + "Apple keycode 0x{kvk:02X} listed twice" + ); + } +} diff --git a/data/layout-mappings/ar.toml b/data/layout-mappings/ar.toml new file mode 100644 index 00000000..bf9f1a04 --- /dev/null +++ b/data/layout-mappings/ar.toml @@ -0,0 +1,96 @@ +# Layout mapping: العربية (Arabic, standard "Arabic (101)"). +# +# Keyed by Win SC Set-1 scancode (hex). Read off the OS rather than +# transcribed: `UCKeyTranslate` against macOS `com.apple.keylayout.ArabicPC`, +# which is the Arabic (101) arrangement Windows ships and xkeyboard-config's +# `ara` follows. Arabic is unicase; the shifted level carries harakat, +# hamza forms and punctuation. +# +# The lam-alef keys produce a two-code-point ligature (`لا` on 0x30 `B`, +# `لإ` / `لأ` on shifted 0x14 / 0x22, `لآ` on shifted `B`), and a key +# here is one character. They carry the single-code-point presentation +# forms instead (U+FEFB, U+FEF9, U+FEF7, U+FEF5), which render the same. +# Leaving them out is worse: an unmapped key is dropped from a +# translation, so `gdb` would read as `لي`, a real word, and `B` would +# vanish from every corrected word. The wordlist carries each `لا` word +# in this spelling too, so `hgsbl` still finds `السلام`. +# +# And one character is kept off the table on purpose: tatweel `ـ` +# (U+0640) is Unicode-alphabetic but a stretching mark, not a letter. +# Listed on Shift+`-` it would make `_` read as a letter in some layout +# and switch off the snake_case heuristic for everyone, so 0x0C keeps +# Arabic 101's `_` and 0x24 carries no shifted level. +# +# Apple's own `Arabic` / `Arabic - North Africa` layouts differ from this +# one on several letter keys (B, N, M, comma, period, slash and the +# brackets); all Mac variants map to this id, and the macOS backend +# reads the keyboard actually installed from the OS, which outranks +# this table. +# +# The plausibility detector counts Arabic's written long vowels (alef, +# waw, yeh, alef maksura, and the lam-alef keys) — see `derive_vowels` — +# and the wordlist makes ar ↔ en-US sharp. No other bundled layout uses +# the Arabic script. + +id = "ar" +name = "العربية" +script = "Arabic" + +[keys] + +# Backtick +0x29 = { plain = "ذ", shift = "ّ" } + +# Number row +0x02 = { plain = "١", shift = "!" } +0x03 = { plain = "٢", shift = "@" } +0x04 = { plain = "٣", shift = "#" } +0x05 = { plain = "٤", shift = "$" } +0x06 = { plain = "٥", shift = "٪" } +0x07 = { plain = "٦", shift = "^" } +0x08 = { plain = "٧", shift = "&" } +0x09 = { plain = "٨", shift = "*" } +0x0A = { plain = "٩", shift = ")" } +0x0B = { plain = "٠", shift = "(" } +0x0C = { plain = "-", shift = "_" } +0x0D = { plain = "=", shift = "+" } + +# Top row +0x10 = { plain = "ض", shift = "َ" } +0x11 = { plain = "ص", shift = "ً" } +0x12 = { plain = "ث", shift = "ُ" } +0x13 = { plain = "ق", shift = "ٌ" } +0x14 = { plain = "ف", shift = "ﻹ" } +0x15 = { plain = "غ", shift = "إ" } +0x16 = { plain = "ع", shift = "‘" } +0x17 = { plain = "ه", shift = "÷" } +0x18 = { plain = "خ", shift = "×" } +0x19 = { plain = "ح", shift = "؛" } +0x1A = { plain = "ج", shift = "<" } +0x1B = { plain = "د", shift = ">" } + +# Home row +0x1E = { plain = "ش", shift = "ِ" } +0x1F = { plain = "س", shift = "ٍ" } +0x20 = { plain = "ي", shift = "]" } +0x21 = { plain = "ب", shift = "[" } +0x22 = { plain = "ل", shift = "ﻷ" } +0x23 = { plain = "ا", shift = "أ" } +0x24 = { plain = "ت" } +0x25 = { plain = "ن", shift = "،" } +0x26 = { plain = "م", shift = "/" } +0x27 = { plain = "ك", shift = ":" } +0x28 = { plain = "ط", shift = "\"" } + +# Bottom row +0x2B = { plain = "\\", shift = "|" } +0x2C = { plain = "ئ", shift = "~" } +0x2D = { plain = "ء", shift = "ْ" } +0x2E = { plain = "ؤ", shift = "}" } +0x2F = { plain = "ر", shift = "{" } +0x30 = { plain = "ﻻ", shift = "ﻵ" } +0x31 = { plain = "ى", shift = "آ" } +0x32 = { plain = "ة", shift = "’" } +0x33 = { plain = "و", shift = "," } +0x34 = { plain = "ز", shift = "." } +0x35 = { plain = "ظ", shift = "؟" } diff --git a/data/wordlists/CREDITS.md b/data/wordlists/CREDITS.md index 33d81f40..8667c568 100644 --- a/data/wordlists/CREDITS.md +++ b/data/wordlists/CREDITS.md @@ -71,6 +71,7 @@ was wrong about at least Russian, which is BSD. | `it_it` | `it_IT/it_IT.dic` | GPL-3 | 3.3 M | | `pt_pt` | `pt_PT/pt_PT.dic` | GPL-2-or-later | 1.3 M | | `pt_br` | `pt_BR/pt_BR.dic` | LGPL-3 / MPL | 8.1 M | +| `ar` | `ar/ar.dic` | GPL-2.0+ / LGPL-2.1+ / MPL-1.1+ (Ayaspell; pick one) | 2.5 M | ### Hebrew is the row to look at twice @@ -88,6 +89,20 @@ plausibility detector separates he-IL from every other layout on script alone and the dictionary is a refinement, not the load-bearing signal. +### Arabic ships stems too + +Ayaspell's `ar.aff` compresses its flags into a 333-entry `AF` alias +table under `FLAG long`, and its rules attach the article and the +clitic particles (ال و ف ب ل ك) as prefixes — the same combinatorial +shape as Hebrew. `ar` is therefore processed with +`ExpandMode::StemsOnly`, and the article forms come back through the +source's `Extras` instead of the affix rules: every stem is also +emitted behind `ال`, `وال`, `بال`, `فال`, `كال` and `لل`, then every +form holding `لا` (or `لأ`, `لإ`, `لآ`) also in the one-code-point +spelling the `B` key types. That gives 2.5 M forms from 466 k `.dic` +lines. A word with a bare one-letter clitic (`وكتاب`, `بسم`) still +misses the dictionary and is scored on plausibility. + ### Hebrew also ships stems, not surface forms `he_IL.aff` is not an inflection table — it is 3335 **prefix** rules diff --git a/data/wordlists/ar-stop.txt b/data/wordlists/ar-stop.txt new file mode 100644 index 00000000..f88eafe5 --- /dev/null +++ b/data/wordlists/ar-stop.txt @@ -0,0 +1,77 @@ +# Hand-curated 1- and 2-letter Arabic stop words (compile-time +# baseline). +# +# `DictionaryDetector` consults this list **instead of** the embedded +# FST for buffers ≤2 letters long, because the FST ships too many +# spurious short "words" that would block real corrections. +# +# Runtime extension (no rebuild needed): drop matching files into +# `/poltertype/wordlists/`: +# * `ar-stop.txt` → merged into THIS list at startup / reload +# * `ar-extras.txt` → merged into the user-overlay HashSet +# * `ar.txt` → same as -extras, just a different filename +# All three accept the same format and reload via "Reload Settings". +# +# Format: one word per line; blank lines and `#` comments ignored. +# +# Every entry was checked against `en_us-stop.txt` through the Arabic +# (101) key positions: none of them is what an English stop word types +# as on this layout, so none can veto an English → Arabic correction of +# a real English word. (`فخ` = "to" and `هب` = "if" are left out for +# exactly that reason.) + +# 1-letter — deliberately empty. Arabic's one-letter forms +# (و ف ب ل ك) are mostly prefixes written attached to the next word, +# and listing them would veto legitimate corrections. + +# 2-letter — particles, pronouns, prepositions and a few common nouns. +# Hamza spellings are listed both ways (أن / ان) because informal +# typing often drops the hamza. +في +من +عن +ما +لا +لم +لن +إن +أن +ان +إذ +اذ +هو +هي +هم +هن +مع +يا +كم +قد +ثم +أو +او +بل +لو +كل +أي +اي +به +بك +بي +له +لك +لي +أب +اب +أخ +اخ +أم +ام +يد +دم +فم +عم +حب +سن +جو +ذا diff --git a/data/wordlists/ar.txt.gz b/data/wordlists/ar.txt.gz new file mode 100644 index 00000000..185a81f1 Binary files /dev/null and b/data/wordlists/ar.txt.gz differ diff --git a/installers/wix/main.wxs b/installers/wix/main.wxs index 32f219b5..a8f0bd34 100644 --- a/installers/wix/main.wxs +++ b/installers/wix/main.wxs @@ -180,6 +180,9 @@ + @@ -269,6 +272,9 @@ + @@ -315,6 +321,9 @@ + @@ -362,6 +371,9 @@ + diff --git a/xtask/src/consts.rs b/xtask/src/consts.rs index 30ac1155..01fbc0f1 100644 --- a/xtask/src/consts.rs +++ b/xtask/src/consts.rs @@ -1,7 +1,7 @@ //! Upstream dictionary source URLs. use crate::enums::ExpandMode; -use crate::types::HunspellSource; +use crate::types::{Extras, HunspellSource}; pub(crate) const EN_URL: &str = "https://raw.githubusercontent.com/dwyl/english-words/master/words_alpha.txt"; @@ -17,6 +17,13 @@ pub(crate) const UK_README_URL: &str = /// Keep in lock-step with `poltertype-core/build.rs::LAYOUTS`: a /// language with a layout but no entry here gets an empty FST and /// plausibility-only detection, which is legal but rarely intended. +/// The [`Extras`] of a dictionary that needs none. +pub(crate) const NO_EXTRAS: Extras = Extras { + prefixes: &[], + respell: &[], + drop: &[], +}; + pub(crate) const HUNSPELL_SOURCES: &[HunspellSource] = &[ HunspellSource { base: "uk_UA", @@ -24,6 +31,7 @@ pub(crate) const HUNSPELL_SOURCES: &[HunspellSource] = &[ aff: "https://raw.githubusercontent.com/LibreOffice/dictionaries/master/uk_UA/uk_UA.aff", output: "uk_ua.txt.gz", expand: ExpandMode::Full, + extras: NO_EXTRAS, }, HunspellSource { base: "ru_RU", @@ -31,6 +39,7 @@ pub(crate) const HUNSPELL_SOURCES: &[HunspellSource] = &[ aff: "https://raw.githubusercontent.com/LibreOffice/dictionaries/master/ru_RU/ru_RU.aff", output: "ru_ru.txt.gz", expand: ExpandMode::Full, + extras: NO_EXTRAS, }, HunspellSource { base: "de_DE_frami", @@ -38,6 +47,7 @@ pub(crate) const HUNSPELL_SOURCES: &[HunspellSource] = &[ aff: "https://raw.githubusercontent.com/LibreOffice/dictionaries/master/de/de_DE_frami.aff", output: "de_de.txt.gz", expand: ExpandMode::Full, + extras: NO_EXTRAS, }, HunspellSource { base: "es_ES", @@ -45,6 +55,7 @@ pub(crate) const HUNSPELL_SOURCES: &[HunspellSource] = &[ aff: "https://raw.githubusercontent.com/LibreOffice/dictionaries/master/es/es_ES.aff", output: "es_es.txt.gz", expand: ExpandMode::Full, + extras: NO_EXTRAS, }, // French moved down a level upstream (`fr_FR/` → `fr_FR/ // dictionaries/`) and the old URLs 404'd for months, because the @@ -57,6 +68,7 @@ pub(crate) const HUNSPELL_SOURCES: &[HunspellSource] = &[ aff: "https://raw.githubusercontent.com/LibreOffice/dictionaries/master/fr_FR/dictionaries/fr.aff", output: "fr_fr.txt.gz", expand: ExpandMode::Full, + extras: NO_EXTRAS, }, HunspellSource { base: "pl_PL", @@ -64,6 +76,7 @@ pub(crate) const HUNSPELL_SOURCES: &[HunspellSource] = &[ aff: "https://raw.githubusercontent.com/LibreOffice/dictionaries/master/pl_PL/pl_PL.aff", output: "pl_pl.txt.gz", expand: ExpandMode::Full, + extras: NO_EXTRAS, }, HunspellSource { base: "cs_CZ", @@ -71,6 +84,7 @@ pub(crate) const HUNSPELL_SOURCES: &[HunspellSource] = &[ aff: "https://raw.githubusercontent.com/LibreOffice/dictionaries/master/cs_CZ/cs_CZ.aff", output: "cs_cz.txt.gz", expand: ExpandMode::Full, + extras: NO_EXTRAS, }, HunspellSource { base: "el_GR", @@ -78,6 +92,7 @@ pub(crate) const HUNSPELL_SOURCES: &[HunspellSource] = &[ aff: "https://raw.githubusercontent.com/LibreOffice/dictionaries/master/el_GR/el_GR.aff", output: "el_gr.txt.gz", expand: ExpandMode::Full, + extras: NO_EXTRAS, }, HunspellSource { base: "he_IL", @@ -85,6 +100,29 @@ pub(crate) const HUNSPELL_SOURCES: &[HunspellSource] = &[ aff: "https://raw.githubusercontent.com/LibreOffice/dictionaries/master/he_IL/he_IL.aff", output: "he_il.txt.gz", expand: ExpandMode::StemsOnly, + extras: NO_EXTRAS, + }, + HunspellSource { + base: "ar", + dic: "https://raw.githubusercontent.com/LibreOffice/dictionaries/master/ar/ar.dic", + aff: "https://raw.githubusercontent.com/LibreOffice/dictionaries/master/ar/ar.aff", + output: "ar.txt.gz", + expand: ExpandMode::StemsOnly, + extras: Extras { + // The definite article and the clitics that fuse with it. + // Stems carry none of them, and `ال` starts a large share + // of running Arabic text. Safe against English: every one + // ends in `hg` / `gg` on the en-US keys, which no English + // word starts with. + prefixes: &["ال", "وال", "بال", "فال", "كال", "لل"], + // The lam-alef keys type one presentation-form code point + // (see `ar.toml`), so `hgsbl` renders as `السﻻم`. + respell: &[("لا", "ﻻ"), ("لأ", "ﻷ"), ("لإ", "ﻹ"), ("لآ", "ﻵ")], + // `حصي` is what `pwd` types as on Arabic 101, and the + // standard spelling of "pebbles" is `حصى` — kept out so the + // shell entry in `en_us-extras.txt` does not veto it. + drop: &["حصي"], + }, }, HunspellSource { base: "tr_TR", @@ -92,6 +130,7 @@ pub(crate) const HUNSPELL_SOURCES: &[HunspellSource] = &[ aff: "https://raw.githubusercontent.com/LibreOffice/dictionaries/master/tr_TR/tr_TR.aff", output: "tr_tr.txt.gz", expand: ExpandMode::Full, + extras: NO_EXTRAS, }, HunspellSource { base: "bg_BG", @@ -99,6 +138,7 @@ pub(crate) const HUNSPELL_SOURCES: &[HunspellSource] = &[ aff: "https://raw.githubusercontent.com/LibreOffice/dictionaries/master/bg_BG/bg_BG.aff", output: "bg_bg.txt.gz", expand: ExpandMode::Full, + extras: NO_EXTRAS, }, HunspellSource { base: "it_IT", @@ -106,6 +146,7 @@ pub(crate) const HUNSPELL_SOURCES: &[HunspellSource] = &[ aff: "https://raw.githubusercontent.com/LibreOffice/dictionaries/master/it_IT/it_IT.aff", output: "it_it.txt.gz", expand: ExpandMode::Full, + extras: NO_EXTRAS, }, HunspellSource { base: "pt_PT", @@ -113,6 +154,7 @@ pub(crate) const HUNSPELL_SOURCES: &[HunspellSource] = &[ aff: "https://raw.githubusercontent.com/LibreOffice/dictionaries/master/pt_PT/pt_PT.aff", output: "pt_pt.txt.gz", expand: ExpandMode::Full, + extras: NO_EXTRAS, }, HunspellSource { base: "pt_BR", @@ -120,6 +162,7 @@ pub(crate) const HUNSPELL_SOURCES: &[HunspellSource] = &[ aff: "https://raw.githubusercontent.com/LibreOffice/dictionaries/master/pt_BR/pt_BR.aff", output: "pt_br.txt.gz", expand: ExpandMode::Full, + extras: NO_EXTRAS, }, ]; diff --git a/xtask/src/types.rs b/xtask/src/types.rs index 3c420854..3a869d27 100644 --- a/xtask/src/types.rs +++ b/xtask/src/types.rs @@ -15,4 +15,24 @@ pub(crate) struct HunspellSource { /// Whether to run the affix rules. [`ExpandMode::Full`] unless the /// dictionary's affix table is pathological — see the enum. pub(crate) expand: ExpandMode, + /// Language-specific touch-ups to the processed list — + /// [`NO_EXTRAS`](crate::consts::NO_EXTRAS) for nearly every source. + pub(crate) extras: Extras, +} + +/// Applied in field order after the `.dic` is processed. Every entry +/// names its reason where it is listed. +pub(crate) struct Extras { + /// Attached to every form as well as kept off it: for a stems-only + /// list whose language writes its article as a prefix, so `كتاب` + /// also yields `الكتاب`. Without them the commonest form of a noun + /// is the one the dictionary lacks. + pub(crate) prefixes: &'static [&'static str], + /// `(from, to)`: every form containing `from` is kept and also + /// emitted with each `from` replaced by `to`. For a key that types + /// in one code point what the dictionary spells in two. + pub(crate) respell: &'static [(&'static str, &'static str)], + /// Forms removed last: real but rare words that would shadow a far + /// more common entry of another layout. + pub(crate) drop: &'static [&'static str], } diff --git a/xtask/src/wordlists.rs b/xtask/src/wordlists.rs index c32da526..c58f937d 100644 --- a/xtask/src/wordlists.rs +++ b/xtask/src/wordlists.rs @@ -80,7 +80,9 @@ pub(crate) fn fetch_hunspell(src_dir: &Path, wl_dir: &Path, source: &HunspellSou } let out = wl_dir.join(source.output); - if let Err(e) = process_hunspell_with_aff(&dic_path, &aff_path, &out, source.expand) { + if let Err(e) = + process_hunspell_with_aff(&dic_path, &aff_path, &out, source.expand, &source.extras) + { eprintln!(" process {} failed: {e}", dic_path.display()); return false; } @@ -116,14 +118,21 @@ pub(crate) fn encoding_of_aff(bytes: &[u8]) -> Result { // first line and hide a `SET` sitting there — pt_BR ships one. let bytes = bytes.strip_prefix(&[0xEF, 0xBB, 0xBF]).unwrap_or(bytes); - // Latin-1 view of the first 2 KB: safe on any byte, which matters - // because the encoding is what we are still looking for. - let preview: String = bytes.iter().take(2048).map(|&b| b as char).collect(); + // Latin-1 view of the whole file: safe on any byte, which matters + // because the encoding is what we are still looking for. Not just + // the head — ar.aff declares `SET` on line 360, after its `AF` table — + // and the separator may be a tab (`SET\tUTF-8`), as ar.aff writes it. + let preview: String = bytes.iter().map(|&b| b as char).collect(); let declared = preview .lines() .filter_map(|l| l.split('#').next()) - .find_map(|l| l.trim().strip_prefix("SET ")) - .map(|s| s.trim().to_uppercase()); + .find_map(|l| { + let mut words = l.split_whitespace(); + (words.next() == Some("SET")) + .then(|| words.next()) + .flatten() + }) + .map(str::to_uppercase); let normalized = declared .as_deref() @@ -212,12 +221,23 @@ pub(crate) fn process_hunspell_with_aff( aff: &Path, output: &Path, mode: ExpandMode, + extras: &Extras, ) -> Result<()> { // The .aff declares the encoding for both halves of the pair. let encoding = detect_encoding(aff)?; - let aff_text = read_hunspell_text(aff, encoding)?; - let parsed = - hunspell::Aff::parse(&aff_text).with_context(|| format!("parse {}", aff.display()))?; + // Stems-only never reads a rule, so it never needs the rules parsed: + // ar.aff uses `FLAG long` with an `AF` alias table, which the + // expander does not model and has no reason to for stems. + let parsed = match mode { + ExpandMode::Full => { + let aff_text = read_hunspell_text(aff, encoding)?; + Some( + hunspell::Aff::parse(&aff_text) + .with_context(|| format!("parse {}", aff.display()))?, + ) + } + ExpandMode::StemsOnly => None, + }; let dic_text = read_hunspell_text(dic, encoding)?; let mut words: BTreeSet = BTreeSet::new(); @@ -238,9 +258,9 @@ pub(crate) fn process_hunspell_with_aff( continue; } stem_count += 1; - let forms = match mode { - ExpandMode::Full => parsed.expand(stem, flags), - ExpandMode::StemsOnly => std::iter::once(stem.to_owned()).collect(), + let forms = match &parsed { + Some(parsed) => parsed.expand(stem, flags), + None => std::iter::once(stem.to_owned()).collect(), }; for form in forms { let lower: String = form.chars().flat_map(char::to_lowercase).collect(); @@ -253,6 +273,7 @@ pub(crate) fn process_hunspell_with_aff( } } } + apply_extras(&mut words, extras); write_sorted(output, &words)?; let name = output.file_name().and_then(|s| s.to_str()).unwrap_or("?"); @@ -296,3 +317,26 @@ pub(crate) fn http_get(url: &str) -> Result> { resp.into_reader().read_to_end(&mut buf)?; Ok(buf) } + +/// [`Extras`] in field order: prefixes, then respellings of the +/// prefixed forms too, then drops — so a dropped form cannot come back +/// through a prefix or a respelling. +fn apply_extras(words: &mut BTreeSet, extras: &Extras) { + let prefixed: Vec = extras + .prefixes + .iter() + .flat_map(|p| words.iter().map(move |w| format!("{p}{w}"))) + .collect(); + words.extend(prefixed); + for (from, to) in extras.respell { + let respelled: Vec = words + .iter() + .filter(|w| w.contains(from)) + .map(|w| w.replace(from, to)) + .collect(); + words.extend(respelled); + } + for form in extras.drop { + words.remove(*form); + } +} diff --git a/xtask/src/wordlists/tests.rs b/xtask/src/wordlists/tests.rs index 13b32c1f..a240bdf8 100644 --- a/xtask/src/wordlists/tests.rs +++ b/xtask/src/wordlists/tests.rs @@ -64,6 +64,23 @@ fn leading_bom_does_not_hide_the_set_line() { ); } +/// ar's `.aff` writes `SET\tUTF-8` on line 360, after a 333-entry `AF` +/// table — past any head-of-file preview and not matched by `"SET "`. +#[test] +fn tab_separated_set_deep_in_the_file_is_found() { + let mut text = String::from("FLAG\tlong\n"); + for i in 0..400 { + text.push_str(&format!( + "AF TaTbTcTdTeTfThTiTjTkTlTmTnToTpTqTrTsTtTuTvTxTy # {i}\n" + )); + } + text.push_str("SET\tUTF-8\nTRY\tabc\n"); + assert_eq!( + encoding_of_aff(text.as_bytes()).expect("late, tab-separated SET must be found"), + Encoding::Utf8 + ); +} + /// Both of these used to silently mean "decode as Latin-1", which is /// the behaviour that shipped broken dictionaries. They must fail. #[test] @@ -74,3 +91,19 @@ fn unknown_and_missing_encodings_are_errors() { let err = encoding_of_aff(b"TRY abc\nSFX A Y 0\n").expect_err("missing SET must fail"); assert!(err.to_string().contains("SET"), "explains itself: {err}"); } + +#[test] +fn extras_prefix_then_respell_then_drop() { + let mut words: BTreeSet = ["سلام", "كتاب", "حصي"].map(String::from).into(); + let extras = Extras { + prefixes: &["ال"], + respell: &[("لا", "ﻻ")], + drop: &["حصي"], + }; + apply_extras(&mut words, &extras); + for want in ["سلام", "السلام", "سﻻم", "السﻻم", "الكتاب", "الحصي"] + { + assert!(words.contains(want), "missing `{want}`"); + } + assert!(!words.contains("حصي"), "a dropped form must stay dropped"); +}