//! What writing system a text is in. //! //! For most of the world this is the whole answer: a text in Hangul is //! Korean, a text in Hiragana is Japanese. For the Latin and Cyrillic //! alphabets it narrows the field and no more, and the words have to decide. use serde::{Deserialize, Serialize}; /// A writing system, as far as telling languages apart needs one. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] pub enum Script { /// The alphabet most of Europe writes in. Latin, /// Russian, Ukrainian, Bulgarian, Serbian and their neighbours. Cyrillic, /// Greek. Greek, /// Arabic, Persian, Urdu. Arabic, /// Hebrew and Yiddish. Hebrew, /// Chinese characters, which Japanese also uses. Han, /// Japanese syllabaries, which only Japanese uses. Kana, /// Korean. Hangul, /// Devanagari: Hindi, Marathi, Sanskrit. Devanagari, /// Something else, or too little to tell. Other, } /// How much of a text has to be in one script before it counts as that /// script. /// /// Well under half, because a book carries page numbers, names and quotations /// in other alphabets, and an English word in a Russian paragraph does not /// make it English. const SHARE: f64 = 0.25; /// The script a text is written in, and what share of its letters agree. /// /// Counts letters only: digits and punctuation belong to every language and /// would drown the signal in a text full of citations. pub fn detect(text: &str) -> (Script, f64) { let mut counts = [0usize; 10]; let mut letters = 0usize; for c in text.chars() { let Some(script) = of(c) else { continue }; letters += 1; counts[index(script)] += 1; } if letters == 0 { return (Script::Other, 0.0); } // Kana before Han: a Japanese text is mostly Han characters with kana // between them, and it is the kana that say which language it is. if counts[index(Script::Kana)] as f64 / letters as f64 > 0.05 { return ( Script::Kana, counts[index(Script::Kana)] as f64 / letters as f64, ); } let (at, count) = counts .iter() .enumerate() .max_by_key(|(_, count)| **count) .expect("ten scripts"); let share = *count as f64 / letters as f64; match share >= SHARE { true => (from_index(at), share), false => (Script::Other, share), } } fn index(script: Script) -> usize { match script { Script::Latin => 0, Script::Cyrillic => 1, Script::Greek => 2, Script::Arabic => 3, Script::Hebrew => 4, Script::Han => 5, Script::Kana => 6, Script::Hangul => 7, Script::Devanagari => 8, Script::Other => 9, } } fn from_index(at: usize) -> Script { [ Script::Latin, Script::Cyrillic, Script::Greek, Script::Arabic, Script::Hebrew, Script::Han, Script::Kana, Script::Hangul, Script::Devanagari, Script::Other, ][at] } /// The script one character belongs to, or `None` if it is not a letter. /// /// By code point range rather than by a Unicode property table, because the /// ranges that matter here are few, stable, and cheaper than carrying the /// table. pub fn of(c: char) -> Option