From 4d14544409f5e3e72a5c89df91f27cf88664a146 Mon Sep 17 00:00:00 2001 From: funkpower Date: Sat, 19 Sep 2026 19:09:27 -0400 Subject: [PATCH] bibclean: an abstract that arrived as an index --- crates/bibclean/README.md | 15 +++++ crates/bibclean/src/abstracts.rs | 99 ++++++++++++++++++++++++++++ crates/bibclean/src/lib.rs | 1 + parity/cases/abstracts/cases.ndjson | 19 ++++++ parity/golden/abstracts/cases.ndjson | 19 ++++++ parity/oracle/abstracts.py | 44 +++++++++++++ parity/src/bin/regen.rs | 15 ++++- parity/tests/abstracts.rs | 62 +++++++++++++++++ 8 files changed, 273 insertions(+), 1 deletion(-) create mode 100644 crates/bibclean/src/abstracts.rs create mode 100644 parity/cases/abstracts/cases.ndjson create mode 100644 parity/golden/abstracts/cases.ndjson create mode 100644 parity/oracle/abstracts.py create mode 100644 parity/tests/abstracts.rs diff --git a/crates/bibclean/README.md b/crates/bibclean/README.md index 601ef3f..87b3c23 100644 --- a/crates/bibclean/README.md +++ b/crates/bibclean/README.md @@ -341,6 +341,21 @@ count against a heading, and spaced initials count for one. A respelling is trusted further than an inversion: `strict_norm` at 0.95, `last_first` at 0.9, a publisher family at 0.8, a junk subject at 0.92. +## Abstracts that arrived inside out + +Some catalogues publish an abstract as an inverted index — each word mapped to +the positions it appears at — rather than as text. The reason is copyright: an +index is a set of facts about a document rather than a copy of it, and may be +redistributed where the paragraph may not. + +```rust +let prose = bibclean::abstracts::from_inverted(&index); +``` + +Sorting by position and joining gives the original back, punctuation and all, +because those travelled attached to the words. An index with nothing usable in +it gives `None`, which is how a record with no abstract is usually spelled. + ## As a library ```rust diff --git a/crates/bibclean/src/abstracts.rs b/crates/bibclean/src/abstracts.rs new file mode 100644 index 0000000..29f2f62 --- /dev/null +++ b/crates/bibclean/src/abstracts.rs @@ -0,0 +1,99 @@ +//! An abstract that arrived inside out. +//! +//! Some catalogues publish an abstract as an *inverted index* — a map of each +//! word to the positions it appears at — rather than as text. The reason is +//! copyright: an index is a set of facts about a document rather than a copy +//! of it, and may be redistributed where the paragraph may not. +//! +//! Turning one back into prose is sorting by position and joining. What comes +//! out is the original with its punctuation and spacing intact, because those +//! travelled attached to the words. + +use std::collections::HashMap; + +/// An abstract as its catalogue publishes it: word to the positions it holds. +pub type Inverted = HashMap>; + +/// Put an inverted abstract back in order. +/// +/// `None` for an index with nothing usable in it, which is how a record with +/// no abstract at all is usually spelled. +/// +/// A word at two positions appears at both. Positions with no word, and gaps +/// in the numbering, are simply not represented — the order is what is known, +/// not the exact offsets. +pub fn from_inverted(index: &Inverted) -> Option { + let mut positions: Vec<(i64, &str)> = index + .iter() + .flat_map(|(word, at)| at.iter().map(move |at| (*at, word.as_str()))) + .collect(); + if positions.is_empty() { + return None; + } + + // By position only, and stably, so two words claiming one position come + // out in the order the index listed them rather than an arbitrary one. + positions.sort_by_key(|(at, _)| *at); + Some( + positions + .into_iter() + .map(|(_, word)| word) + .collect::>() + .join(" "), + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn inverted(pairs: &[(&str, &[i64])]) -> Inverted { + pairs + .iter() + .map(|(word, at)| (word.to_string(), at.to_vec())) + .collect() + } + + #[test] + fn words_come_back_in_the_order_their_positions_give() { + let index = inverted(&[("obligation", &[1]), ("The", &[0]), ("ran", &[2])]); + assert_eq!(from_inverted(&index).as_deref(), Some("The obligation ran")); + } + + #[test] + fn a_word_at_two_positions_appears_at_both() { + let index = inverted(&[("the", &[0, 2]), ("cat", &[1]), ("mat", &[3])]); + assert_eq!(from_inverted(&index).as_deref(), Some("the cat the mat")); + } + + #[test] + fn punctuation_rides_along_with_the_word_it_was_attached_to() { + let index = inverted(&[("ways,", &[1]), ("Both", &[0]), ("plainly.", &[2])]); + assert_eq!( + from_inverted(&index).as_deref(), + Some("Both ways, plainly.") + ); + } + + #[test] + fn an_index_with_nothing_in_it_is_no_abstract() { + assert_eq!(from_inverted(&Inverted::new()), None); + assert_eq!(from_inverted(&inverted(&[("word", &[])])), None); + } + + #[test] + fn gaps_and_odd_positions_are_an_order_rather_than_an_offset() { + // Catalogues skip numbers and occasionally start below zero; what is + // known is which word comes first, not where it sat. + let index = inverted(&[("second", &[40]), ("first", &[-3]), ("third", &[900])]); + assert_eq!(from_inverted(&index).as_deref(), Some("first second third")); + } + + #[test] + fn a_one_word_abstract_is_still_an_abstract() { + assert_eq!( + from_inverted(&inverted(&[("Abstract", &[0])])).as_deref(), + Some("Abstract") + ); + } +} diff --git a/crates/bibclean/src/lib.rs b/crates/bibclean/src/lib.rs index 8a2538f..93b001a 100644 --- a/crates/bibclean/src/lib.rs +++ b/crates/bibclean/src/lib.rs @@ -11,6 +11,7 @@ //! which is the difference between a tool that cleans a catalogue and one //! that damages it in a new way. +pub mod abstracts; pub mod author; mod entities4; pub mod fix; diff --git a/parity/cases/abstracts/cases.ndjson b/parity/cases/abstracts/cases.ndjson new file mode 100644 index 0000000..c0e11a2 --- /dev/null +++ b/parity/cases/abstracts/cases.ndjson @@ -0,0 +1,19 @@ +{"name": "ordinary", "index": {"The": [0], "obligation": [1], "ran": [2], "both": [3], "ways": [4]}} +{"name": "repeated-word", "index": {"the": [0, 2], "cat": [1], "mat": [3]}} +{"name": "punctuation", "index": {"Both": [0], "ways,": [1], "plainly.": [2]}} +{"name": "empty", "index": {}} +{"name": "no-positions", "index": {"word": []}} +{"name": "one-word", "index": {"Abstract": [0]}} +{"name": "gaps", "index": {"second": [40], "first": [-3], "third": [900]}} +{"name": "out-of-order-keys", "index": {"zebra": [5], "apple": [0], "mango": [3]}} +{"name": "same-position", "index": {"alpha": [1], "beta": [1]}} +{"name": "unicode", "index": {"\u039a\u03b1\u03bb\u03b7\u03bc\u03ad\u03c1\u03b1": [0], "\u03ba\u03cc\u03c3\u03bc\u03b5": [1]}} +{"name": "long-runs", "index": {"a": [0, 2, 4, 6, 8, 10, 12, 14, 16, 18], "b": [1, 3, 5, 7, 9, 11, 13, 15, 17, 19]}} +{"name": "big-positions", "index": {"x": [2147483648], "y": [0]}} +{"name": "negative-only", "index": {"only": [-5]}} +{"name": "random-0", "index": {"that": [23], "small": [37], "is": [3], "research": [32], "but": [13], "effect": [2]}} +{"name": "random-1", "index": {"small": [4], "real": [15]}} +{"name": "random-2", "index": {"real": [3], "small": [36]}} +{"name": "random-3", "index": {"the": [36], "research": [37]}} +{"name": "random-4", "index": {"research": [9], "the": [34], "real": [7], "effect": [36], "shows": [19], "that": [11], "is": [6]}} +{"name": "random-5", "index": {"is": [4], "shows": [36], "effect": [3], "real": [39]}} diff --git a/parity/golden/abstracts/cases.ndjson b/parity/golden/abstracts/cases.ndjson new file mode 100644 index 0000000..1559d43 --- /dev/null +++ b/parity/golden/abstracts/cases.ndjson @@ -0,0 +1,19 @@ +{"abstract": "The obligation ran both ways", "name": "ordinary"} +{"abstract": "the cat the mat", "name": "repeated-word"} +{"abstract": "Both ways, plainly.", "name": "punctuation"} +{"abstract": null, "name": "empty"} +{"abstract": null, "name": "no-positions"} +{"abstract": "Abstract", "name": "one-word"} +{"abstract": "first second third", "name": "gaps"} +{"abstract": "apple mango zebra", "name": "out-of-order-keys"} +{"abstract": "alpha beta", "name": "same-position"} +{"abstract": "\u039a\u03b1\u03bb\u03b7\u03bc\u03ad\u03c1\u03b1 \u03ba\u03cc\u03c3\u03bc\u03b5", "name": "unicode"} +{"abstract": "a b a b a b a b a b a b a b a b a b a b", "name": "long-runs"} +{"abstract": "y x", "name": "big-positions"} +{"abstract": "only", "name": "negative-only"} +{"abstract": "effect is but that research small", "name": "random-0"} +{"abstract": "small real", "name": "random-1"} +{"abstract": "real small", "name": "random-2"} +{"abstract": "the research", "name": "random-3"} +{"abstract": "is real research that shows the effect", "name": "random-4"} +{"abstract": "effect is shows real", "name": "random-5"} diff --git a/parity/oracle/abstracts.py b/parity/oracle/abstracts.py new file mode 100644 index 0000000..b6b88b0 --- /dev/null +++ b/parity/oracle/abstracts.py @@ -0,0 +1,44 @@ +"""Turn an inverted abstract back into prose the way the policy pipeline did. + +The function below is the original, copied out of the journals module verbatim +— importing it opens a database connection and a catalogue client, and neither +decides anything. + + python3 oracle/abstracts.py < cases.ndjson + +One case per line: {name, index}. Writes one JSON answer per line. +""" + +from __future__ import annotations + +import json +import sys + + +def _decode_abstract(idx: dict | None) -> str | None: + if not isinstance(idx, dict) or not idx: + return None + positions: list[tuple[int, str]] = [] + for word, locs in idx.items(): + if not isinstance(locs, list): + continue + for p in locs: + if isinstance(p, int): + positions.append((p, str(word))) + if not positions: + return None + positions.sort(key=lambda t: t[0]) + return " ".join(w for _, w in positions) + + +for line in sys.stdin: + if not line.strip(): + continue + case = json.loads(line) + sys.stdout.write( + json.dumps( + {"name": case["name"], "abstract": _decode_abstract(case["index"])}, + sort_keys=True, + ) + + "\n" + ) diff --git a/parity/src/bin/regen.rs b/parity/src/bin/regen.rs index b18d07b..41def92 100644 --- a/parity/src/bin/regen.rs +++ b/parity/src/bin/regen.rs @@ -19,7 +19,7 @@ type Written = Result, String>; type Job = (&'static str, fn() -> Written); fn main() -> ExitCode { - let jobs: [Job; 19] = [ + let jobs: [Job; 20] = [ ("tokens", tokens), ("unescape", unescape), ("chunkmd", chunkmd), @@ -39,6 +39,7 @@ fn main() -> ExitCode { ("pdfgutter", pdfgutter), ("seriesof", seriesof), ("subjects", subjects), + ("abstracts", abstracts), ]; for (name, job) in jobs { @@ -221,6 +222,18 @@ fn cueseg() -> Written { Ok(written) } +fn abstracts() -> Written { + let dir = parity::golden("abstracts"); + std::fs::create_dir_all(&dir).map_err(|e| e.to_string())?; + + let cases = parity::dir().join("cases/abstracts/cases.ndjson"); + let text = std::fs::read_to_string(&cases).map_err(|e| e.to_string())?; + let answers = parity::run_oracle("abstracts.py", &text)?; + let path = dir.join("cases.ndjson"); + std::fs::write(&path, answers).map_err(|e| e.to_string())?; + Ok(vec![path.display().to_string()]) +} + fn subjects() -> Written { let dir = parity::golden("subjects"); std::fs::create_dir_all(&dir).map_err(|e| e.to_string())?; diff --git a/parity/tests/abstracts.rs b/parity/tests/abstracts.rs new file mode 100644 index 0000000..9ca9210 --- /dev/null +++ b/parity/tests/abstracts.rs @@ -0,0 +1,62 @@ +//! An inverted abstract has to come back as the same prose the Python gave. +//! +//! What the catalogue publishes is an index rather than the paragraph, and +//! what a reader sees on a search result is whatever this reconstructs. A +//! word in the wrong place is a sentence that reads as though it means +//! something and does not. + +use bibclean::abstracts::{self, Inverted}; +use serde_json::Value; + +fn records(text: &str) -> Vec { + text.lines() + .filter(|line| !line.trim().is_empty()) + .map(|line| serde_json::from_str(line).expect("valid JSON")) + .collect() +} + +fn cases() -> Vec { + let text = std::fs::read_to_string(parity::dir().join("cases/abstracts/cases.ndjson")) + .expect("a readable case file"); + records(&text) +} + +#[test] +fn every_abstract_reads_back_the_same_as_python() { + let golden = records(&parity::read_golden_text("abstracts/cases.ndjson")); + let cases = cases(); + assert_eq!(cases.len(), golden.len(), "case count moved — regenerate"); + + let mut wrong: Vec = Vec::new(); + for (case, theirs) in cases.iter().zip(&golden) { + let name = case["name"].as_str().expect("a name"); + assert_eq!(theirs["name"].as_str(), Some(name), "cases out of order"); + + let index: Inverted = + serde_json::from_value(case["index"].clone()).expect("an index of positions"); + let ours = abstracts::from_inverted(&index); + if ours.as_deref() != theirs["abstract"].as_str() { + wrong.push(format!( + "{name}: python {}, rust {ours:?}", + theirs["abstract"] + )); + } + } + + assert!( + wrong.is_empty(), + "{} abstracts differ:\n{}", + wrong.len(), + wrong.join("\n") + ); +} + +/// Catches a golden file drifting away from a Python that has moved on. +#[test] +#[cfg(feature = "python")] +fn the_golden_file_still_matches_live_python() { + let cases = std::fs::read_to_string(parity::dir().join("cases/abstracts/cases.ndjson")) + .expect("a readable case file"); + let live = parity::run_oracle("abstracts.py", &cases).expect("the oracle runs"); + assert_eq!(live, parity::read_golden_text("abstracts/cases.ndjson")); +}