diff --git a/Cargo.lock b/Cargo.lock index 5082389..f791068 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -907,6 +907,18 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "langid" +version = "0.1.0" +dependencies = [ + "catena-cli", + "clap", + "corpus-core", + "schemars", + "serde", + "serde_json", +] + [[package]] name = "lazy_static" version = "1.5.0" @@ -1699,6 +1711,20 @@ version = "1.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fd3ca314f692efd6c868f8408f53fe444634a845f96c028b97d35f6a1f79f0ee" +[[package]] +name = "tocfind" +version = "0.1.0" +dependencies = [ + "catena-cli", + "clap", + "corpus-core", + "lopdf", + "schemars", + "serde", + "serde_json", + "zip", +] + [[package]] name = "ttf-parser" version = "0.25.1" diff --git a/Cargo.toml b/Cargo.toml index 5ba60b7..b986cb7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,6 +30,8 @@ seriesof = { path = "crates/seriesof", version = "0.1.0" } idnorm = { path = "crates/idnorm", version = "0.1.0" } bibfetch = { path = "crates/bibfetch", version = "0.1.0" } bookmeta = { path = "crates/bookmeta", version = "0.1.0" } +langid = { path = "crates/langid", version = "0.1.0" } +tocfind = { path = "crates/tocfind", version = "0.1.0" } booksniff = { path = "crates/booksniff", version = "0.1.0" } emb = { path = "crates/emb", version = "0.1.0" } diff --git a/README.md b/README.md index 9ecebc0..94e651c 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,8 @@ Each tool installs by name, so you can take one without taking the rest. |---|---| | [`booksniff`](crates/booksniff/README.md) | What a file actually is, whatever its name claims | | [`bookmeta`](crates/bookmeta/README.md) | The metadata a file already carries | +| [`tocfind`](crates/tocfind/README.md) | A book's own table of contents | +| [`langid`](crates/langid/README.md) | What language a document is in | | [`pdfgutter`](crates/pdfgutter/README.md) | Books scanned two pages to a sheet, split back into pages | | [`pdftriage`](crates/pdftriage/README.md) | Whether a PDF's text layer is worth reading, or whether it has to be OCRed | | [`bibclean`](crates/bibclean/README.md) | Catalogue records audited and repaired, with a reason for every change | diff --git a/crates/langid/Cargo.toml b/crates/langid/Cargo.toml new file mode 100644 index 0000000..7287244 --- /dev/null +++ b/crates/langid/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "langid" +description = "What language a document is in, from its script and its commonest words." +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] +catena-cli.workspace = true +clap.workspace = true +corpus-core.workspace = true +schemars.workspace = true +serde.workspace = true +serde_json.workspace = true + +[lints] +workspace = true diff --git a/crates/langid/README.md b/crates/langid/README.md new file mode 100644 index 0000000..e417b80 --- /dev/null +++ b/crates/langid/README.md @@ -0,0 +1,123 @@ +# langid + +What language a document is in. + +Part of [catena](../../README.md). + +## What it does + +Two passes, because one will not do. The writing system settles most of the +world outright — a text in Hangul is Korean — and for the Latin and Cyrillic +alphabets it only narrows the field, so the commonest words decide between +what is left. + +No model, no download, no network. Function words make up a third of any +running text, and counting them is decisive on a paragraph. On a single line +it is not, which is why a close call is refused rather than guessed at. + +## When you would use it + +- **Before OCR**, to know which language to hand it. +- **Before embedding**, where a multilingual shelf wants different models. +- **Filling the `language` field** on a `doc` record, which nothing else sets. +- **Finding the odd book out** in a library you assumed was all one language. + +## Install + +```sh +cargo install --git https://git.pumpin.online/funkpower/catena langid +``` + +## Example + +``` +$ langid converted/*.md | jq -r '[.name, (.confidence*100|round|tostring + "%"), .path] | @tsv' +English 77% converted/enlightenment.md +English 76% converted/hagakure.md +French 81% converted/meditations.md +Latin 68% converted/summa.md +``` + +Just the code: + +``` +$ langid --quiet book.md +en +``` + +The whole record, which carries the bibliographic code the `doc` record +wants: + +``` +$ langid book.md +{"v":1,"path":"book.md","language":"en","bibliographic":"eng","name":"English","script":"Latin","confidence":0.77} +``` + +The ones that are not what you assumed: + +```sh +langid converted/*.md | jq -r 'select(.language != "en") | [.path, .name] | @tsv' +``` + +Reading from a pipe: + +```sh +pdftotext -l 5 scan.pdf - | langid --quiet +``` + +## Options + +| Option | Effect | +|---|---| +| `--quiet` | Print the code alone, one per line | +| `--man` | Write this page as a manual page, in roff | + +## Exit codes + +| Code | Meaning | +|---|---| +| `0` | Every file was placed | +| `1` | At least one could not be | +| `64` | The command line was wrong | +| `74` | A file could not be read | + +## How it decides + +**The script first.** Kana, Hangul, Hebrew, Arabic and Devanagari each belong +to one language as far as a library is concerned, and no amount of word +counting improves on that. Kana is checked before Han, because Japanese is +mostly Han characters with kana between them and it is the kana that tell it +from Chinese. + +A quarter of a text's letters agreeing is enough to call it a script: a book +carries page numbers, names and quotations in other alphabets, and an English +word in a Russian paragraph does not make the paragraph English. + +**Then the words.** Thirteen languages across Latin, Cyrillic and Greek, each +with its thirty commonest function words. The score is the share of a text's +words found in each list, and the lists are the same length so the scores +compare. + +**A close call is no answer.** The winner must be a quarter ahead of the +runner-up. `de` is among the commonest words of four languages at once, and on +its own settles none of them — so a text that short comes back with the script +named, no language, and a note saying which two it could not choose between. + +Only the first twenty thousand characters are read. A book's language does not +change after the first few pages. + +## As a library + +```rust +let found = langid::detect("book.md", text); +assert_eq!(found.language.as_deref(), Some("en")); +``` + +`langid::script::detect` answers the writing system on its own, and +`langid::words::score` ranks every language of a given script. + +## See also + +- [`booksniff`](../booksniff/README.md) — what the file is. +- [`bookmeta`](../bookmeta/README.md) — what the file says its language is, + which is worth more when it says anything. diff --git a/crates/langid/src/lib.rs b/crates/langid/src/lib.rs new file mode 100644 index 0000000..a8eb40a --- /dev/null +++ b/crates/langid/src/lib.rs @@ -0,0 +1,235 @@ +//! What language a document is in. +//! +//! Two passes, because one does not do. The writing system settles most of +//! the world outright — a text in Hangul is Korean — and for the Latin and +//! Cyrillic alphabets it only narrows the field, so the commonest words +//! decide between what is left. +//! +//! No model, no download, no network. Function words make up a third of any +//! running text, and on a paragraph or more counting them is decisive. On a +//! single line it is not, which is why every answer carries a confidence and +//! why "could not tell" is one of the answers. + +pub mod script; +pub mod words; + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +pub use script::Script; + +/// How much text is worth looking at. +/// +/// A book's language does not change after the first few pages, and reading +/// all of it to say so is waste. +pub const SAMPLE_CHARS: usize = 20_000; + +/// What language a text turned out to be. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct Verdict { + /// Record version. + #[serde(default = "version")] + pub v: u32, + /// What was read. + pub path: String, + /// ISO 639-1, the two-letter code. Absent when it could not be told. + #[serde(skip_serializing_if = "Option::is_none")] + pub language: Option, + /// ISO 639-2/B, which is what the `doc` record carries. + #[serde(skip_serializing_if = "Option::is_none")] + pub bibliographic: Option, + /// What the language is called in English. + #[serde(skip_serializing_if = "Option::is_none")] + pub name: Option, + /// The writing system, which is an answer even when the language is not. + pub script: String, + /// How far ahead the winner was, from zero to one. One means nothing else + /// scored at all. + #[serde(with = "corpus_core::finite")] + #[schemars(with = "f64")] + pub confidence: f64, + /// What was worth saying about the reading. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub notes: Vec, +} + +fn version() -> u32 { + corpus_core::V +} + +impl corpus_core::Record for Verdict { + fn version(&self) -> u32 { + self.v + } +} + +/// Read a text's language. +pub fn detect(path: &str, text: &str) -> Verdict { + let sample: String = text.chars().take(SAMPLE_CHARS).collect(); + let (script, share) = script::detect(&sample); + + let mut verdict = Verdict { + v: corpus_core::V, + path: path.to_string(), + language: None, + bibliographic: None, + name: None, + script: name_of(script).to_string(), + confidence: 0.0, + notes: Vec::new(), + }; + + // A script only one language uses is the whole answer, and no amount of + // word counting would improve on it. + if let Some(sole) = sole_language(script) { + verdict.language = Some(sole.0.to_string()); + verdict.bibliographic = Some(sole.1.to_string()); + verdict.name = Some(sole.2.to_string()); + verdict.confidence = share; + verdict + .notes + .push(format!("{} is written only in {}", sole.2, name_of(script))); + return verdict; + } + + let scored = words::score(&sample, script); + let Some((best, best_score)) = scored.first() else { + verdict.notes.push(match script { + Script::Other => "no letters to go on".to_string(), + script => format!( + "nothing here knows the languages written in {}", + name_of(script) + ), + }); + return verdict; + }; + + if *best_score == 0.0 { + verdict + .notes + .push("none of the commonest words of any language it knows".to_string()); + return verdict; + } + + // How far ahead of the runner-up, so a close call can be refused. + let runner_up = scored.get(1).map_or(0.0, |(_, score)| *score); + let ratio = match runner_up > 0.0 { + true => best_score / runner_up, + false => f64::INFINITY, + }; + if ratio < words::MARGIN { + verdict.notes.push(format!( + "{} and {} score too alike to choose between", + best.name, scored[1].0.name + )); + return verdict; + } + + verdict.language = Some(best.code.to_string()); + verdict.bibliographic = Some(best.bibliographic.to_string()); + verdict.name = Some(best.name.to_string()); + verdict.confidence = match runner_up > 0.0 { + true => (1.0 - runner_up / best_score).clamp(0.0, 1.0), + false => 1.0, + }; + verdict +} + +/// The one language a script is used for, where there is only one. +fn sole_language(script: Script) -> Option<(&'static str, &'static str, &'static str)> { + Some(match script { + Script::Kana => ("ja", "jpn", "Japanese"), + Script::Han => ("zh", "chi", "Chinese"), + Script::Hangul => ("ko", "kor", "Korean"), + Script::Hebrew => ("he", "heb", "Hebrew"), + Script::Arabic => ("ar", "ara", "Arabic"), + Script::Devanagari => ("hi", "hin", "Hindi"), + _ => return None, + }) +} + +/// What a script is called. +pub fn name_of(script: Script) -> &'static str { + match script { + Script::Latin => "Latin", + Script::Cyrillic => "Cyrillic", + Script::Greek => "Greek", + Script::Arabic => "Arabic", + Script::Hebrew => "Hebrew", + Script::Han => "Han", + Script::Kana => "Kana", + Script::Hangul => "Hangul", + Script::Devanagari => "Devanagari", + Script::Other => "unknown", + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_paragraph_of_english_is_english() { + let found = detect( + "x.md", + "The obligation ran both ways, and contemporaries said so plainly whenever they were asked about it at all.", + ); + assert_eq!(found.language.as_deref(), Some("en")); + assert_eq!(found.bibliographic.as_deref(), Some("eng")); + assert_eq!(found.script, "Latin"); + assert!(found.confidence > 0.0); + } + + #[test] + fn a_script_only_one_language_uses_needs_no_words() { + for (text, code) in [ + ("안녕하세요 세계 반갑습니다 오늘은", "ko"), + ("これは日本語の文章です", "ja"), + ("מרחבי שלום עולם וברוך הבא", "he"), + ] { + assert_eq!(detect("x", text).language.as_deref(), Some(code), "{text}"); + } + } + + #[test] + fn nothing_to_read_is_no_answer_rather_than_a_wrong_one() { + let found = detect("x", "12345 -- ..."); + assert_eq!(found.language, None); + assert_eq!(found.script, "unknown"); + assert_eq!(found.notes, ["no letters to go on"]); + } + + #[test] + fn latin_words_no_language_claims_are_refused() { + let found = detect("x", "zzzz qqqq xxxx vvvv wwww kkkk jjjj"); + assert_eq!(found.language, None); + assert!(found.notes[0].starts_with("none of the commonest words")); + } + + #[test] + fn a_close_call_is_refused_rather_than_guessed() { + // "de" is among the commonest words of four of these languages at + // once, and on its own settles none of them. + let found = detect("x", "de de de"); + assert_eq!(found.language, None, "{:?}", found.notes); + assert!(found.notes[0].contains("too alike"), "{:?}", found.notes); + } + + #[test] + fn only_the_opening_is_read() { + let long = format!( + "{}{}", + "The obligation ran both ways. ".repeat(2000), + "xyzzy" + ); + assert!(long.len() > SAMPLE_CHARS * 2); + assert_eq!(detect("x", &long).language.as_deref(), Some("en")); + } + + #[test] + fn the_script_is_reported_even_when_the_language_is_not() { + let found = detect("x", "zzzz qqqq xxxx vvvv"); + assert_eq!(found.script, "Latin"); + assert_eq!(found.language, None); + } +} diff --git a/crates/langid/src/main.rs b/crates/langid/src/main.rs new file mode 100644 index 0000000..1e0c928 --- /dev/null +++ b/crates/langid/src/main.rs @@ -0,0 +1,93 @@ +//! What language a document is in. + +use std::path::PathBuf; +use std::process::ExitCode; + +use clap::Parser; +use corpus_core::exit; + +/// Say what language a text is in, one record per file. +/// +/// The writing system settles most of the world outright; for the Latin and +/// Cyrillic alphabets the commonest words decide between what is left. +/// +/// No model, no download, no network. A paragraph is plenty; a single line +/// often is not, and a close call is refused rather than guessed at. +#[derive(Parser)] +#[command( + version, + about, + after_help = "\ +Exit codes: 0 every file was placed, 1 at least one could not be, +74 a file could not be read. + +Example: + + langid book.md + langid converted/*.md | jq -r 'select(.language != \"en\") | [.path, .name] | @tsv'" +)] +struct Args { + /// Files to read. Reads stdin when absent. + paths: Vec, + + /// Print the code alone, for one file. + #[arg(long)] + quiet: bool, +} + +fn main() -> ExitCode { + let args: Args = match catena_cli::parse() { + Ok(args) => args, + Err(code) => return code, + }; + + let mut verdicts = Vec::new(); + let mut unplaced = 0usize; + + if args.paths.is_empty() { + let text = match catena_cli::read_lossy(None) { + Ok(text) => text, + Err((code, why)) => { + eprintln!("langid: {why}"); + return ExitCode::from(code); + } + }; + verdicts.push(langid::detect("-", &text)); + } else { + for path in &args.paths { + let text = match catena_cli::read_lossy(Some(path)) { + Ok(text) => text, + Err((code, why)) => { + eprintln!("langid: {why}"); + return ExitCode::from(code); + } + }; + verdicts.push(langid::detect(&path.display().to_string(), &text)); + } + } + unplaced += verdicts.iter().filter(|v| v.language.is_none()).count(); + + if args.quiet { + let codes: String = verdicts + .iter() + .map(|v| format!("{}\n", v.language.as_deref().unwrap_or(""))) + .collect(); + let wrote = catena_cli::text("langid", &codes); + if format!("{wrote:?}") != format!("{:?}", ExitCode::from(exit::OK)) { + return wrote; + } + return ExitCode::from(match unplaced { + 0 => exit::OK, + _ => exit::NO, + }); + } + + let wrote = catena_cli::stream("langid", &verdicts); + if format!("{wrote:?}") != format!("{:?}", ExitCode::from(exit::OK)) { + return wrote; + } + ExitCode::from(match unplaced { + 0 => exit::OK, + _ => exit::NO, + }) +} diff --git a/crates/langid/src/script.rs b/crates/langid/src/script.rs new file mode 100644 index 0000000..35439b2 --- /dev/null +++ b/crates/langid/src/script.rs @@ -0,0 +1,175 @@ +//! What writing system a text is in. +//! +//! For most of the world this is the whole answer: a text in Hangul is +//! Korean, a text in Hiragana is Japanese. For the Latin and Cyrillic +//! alphabets it narrows the field and no more, and the words have to decide. + +use serde::{Deserialize, Serialize}; + +/// A writing system, as far as telling languages apart needs one. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum Script { + /// The alphabet most of Europe writes in. + Latin, + /// Russian, Ukrainian, Bulgarian, Serbian and their neighbours. + Cyrillic, + /// Greek. + Greek, + /// Arabic, Persian, Urdu. + Arabic, + /// Hebrew and Yiddish. + Hebrew, + /// Chinese characters, which Japanese also uses. + Han, + /// Japanese syllabaries, which only Japanese uses. + Kana, + /// Korean. + Hangul, + /// Devanagari: Hindi, Marathi, Sanskrit. + Devanagari, + /// Something else, or too little to tell. + Other, +} + +/// How much of a text has to be in one script before it counts as that +/// script. +/// +/// Well under half, because a book carries page numbers, names and quotations +/// in other alphabets, and an English word in a Russian paragraph does not +/// make it English. +const SHARE: f64 = 0.25; + +/// The script a text is written in, and what share of its letters agree. +/// +/// Counts letters only: digits and punctuation belong to every language and +/// would drown the signal in a text full of citations. +pub fn detect(text: &str) -> (Script, f64) { + let mut counts = [0usize; 10]; + let mut letters = 0usize; + + for c in text.chars() { + let Some(script) = of(c) else { continue }; + letters += 1; + counts[index(script)] += 1; + } + if letters == 0 { + return (Script::Other, 0.0); + } + + // Kana before Han: a Japanese text is mostly Han characters with kana + // between them, and it is the kana that say which language it is. + if counts[index(Script::Kana)] as f64 / letters as f64 > 0.05 { + return ( + Script::Kana, + counts[index(Script::Kana)] as f64 / letters as f64, + ); + } + + let (at, count) = counts + .iter() + .enumerate() + .max_by_key(|(_, count)| **count) + .expect("ten scripts"); + let share = *count as f64 / letters as f64; + match share >= SHARE { + true => (from_index(at), share), + false => (Script::Other, share), + } +} + +fn index(script: Script) -> usize { + match script { + Script::Latin => 0, + Script::Cyrillic => 1, + Script::Greek => 2, + Script::Arabic => 3, + Script::Hebrew => 4, + Script::Han => 5, + Script::Kana => 6, + Script::Hangul => 7, + Script::Devanagari => 8, + Script::Other => 9, + } +} + +fn from_index(at: usize) -> Script { + [ + Script::Latin, + Script::Cyrillic, + Script::Greek, + Script::Arabic, + Script::Hebrew, + Script::Han, + Script::Kana, + Script::Hangul, + Script::Devanagari, + Script::Other, + ][at] +} + +/// The script one character belongs to, or `None` if it is not a letter. +/// +/// By code point range rather than by a Unicode property table, because the +/// ranges that matter here are few, stable, and cheaper than carrying the +/// table. +pub fn of(c: char) -> Option