bibclean: an abstract that arrived as an index

crates/bibclean/README.md

@@ -341,6 +341,21 @@ count against a heading, and spaced initials count for one.
 A respelling is trusted further than an inversion: `strict_norm` at 0.95,
 `last_first` at 0.9, a publisher family at 0.8, a junk subject at 0.92.
 
+## Abstracts that arrived inside out
+
+Some catalogues publish an abstract as an inverted index — each word mapped to
+the positions it appears at — rather than as text. The reason is copyright: an
+index is a set of facts about a document rather than a copy of it, and may be
+redistributed where the paragraph may not.
+
+```rust
+let prose = bibclean::abstracts::from_inverted(&index);
+```
+
+Sorting by position and joining gives the original back, punctuation and all,
+because those travelled attached to the words. An index with nothing usable in
+it gives `None`, which is how a record with no abstract is usually spelled.
+
 ## As a library
 
 ```rust

crates/bibclean/src/abstracts.rs

@@ -0,0 +1,99 @@
+//! An abstract that arrived inside out.
+//!
+//! Some catalogues publish an abstract as an *inverted index* — a map of each
+//! word to the positions it appears at — rather than as text. The reason is
+//! copyright: an index is a set of facts about a document rather than a copy
+//! of it, and may be redistributed where the paragraph may not.
+//!
+//! Turning one back into prose is sorting by position and joining. What comes
+//! out is the original with its punctuation and spacing intact, because those
+//! travelled attached to the words.
+
+use std::collections::HashMap;
+
+/// An abstract as its catalogue publishes it: word to the positions it holds.
+pub type Inverted = HashMap<String, Vec<i64>>;
+
+/// Put an inverted abstract back in order.
+///
+/// `None` for an index with nothing usable in it, which is how a record with
+/// no abstract at all is usually spelled.
+///
+/// A word at two positions appears at both. Positions with no word, and gaps
+/// in the numbering, are simply not represented — the order is what is known,
+/// not the exact offsets.
+pub fn from_inverted(index: &Inverted) -> Option<String> {
+    let mut positions: Vec<(i64, &str)> = index
+        .iter()
+        .flat_map(|(word, at)| at.iter().map(move |at| (*at, word.as_str())))
+        .collect();
+    if positions.is_empty() {
+        return None;
+    }
+
+    // By position only, and stably, so two words claiming one position come
+    // out in the order the index listed them rather than an arbitrary one.
+    positions.sort_by_key(|(at, _)| *at);
+    Some(
+        positions
+            .into_iter()
+            .map(|(_, word)| word)
+            .collect::<Vec<_>>()
+            .join(" "),
+    )
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn inverted(pairs: &[(&str, &[i64])]) -> Inverted {
+        pairs
+            .iter()
+            .map(|(word, at)| (word.to_string(), at.to_vec()))
+            .collect()
+    }
+
+    #[test]
+    fn words_come_back_in_the_order_their_positions_give() {
+        let index = inverted(&[("obligation", &[1]), ("The", &[0]), ("ran", &[2])]);
+        assert_eq!(from_inverted(&index).as_deref(), Some("The obligation ran"));
+    }
+
+    #[test]
+    fn a_word_at_two_positions_appears_at_both() {
+        let index = inverted(&[("the", &[0, 2]), ("cat", &[1]), ("mat", &[3])]);
+        assert_eq!(from_inverted(&index).as_deref(), Some("the cat the mat"));
+    }
+
+    #[test]
+    fn punctuation_rides_along_with_the_word_it_was_attached_to() {
+        let index = inverted(&[("ways,", &[1]), ("Both", &[0]), ("plainly.", &[2])]);
+        assert_eq!(
+            from_inverted(&index).as_deref(),
+            Some("Both ways, plainly.")
+        );
+    }
+
+    #[test]
+    fn an_index_with_nothing_in_it_is_no_abstract() {
+        assert_eq!(from_inverted(&Inverted::new()), None);
+        assert_eq!(from_inverted(&inverted(&[("word", &[])])), None);
+    }
+
+    #[test]
+    fn gaps_and_odd_positions_are_an_order_rather_than_an_offset() {
+        // Catalogues skip numbers and occasionally start below zero; what is
+        // known is which word comes first, not where it sat.
+        let index = inverted(&[("second", &[40]), ("first", &[-3]), ("third", &[900])]);
+        assert_eq!(from_inverted(&index).as_deref(), Some("first second third"));
+    }
+
+    #[test]
+    fn a_one_word_abstract_is_still_an_abstract() {
+        assert_eq!(
+            from_inverted(&inverted(&[("Abstract", &[0])])).as_deref(),
+            Some("Abstract")
+        );
+    }
+}

crates/bibclean/src/lib.rs

@@ -11,6 +11,7 @@
 //! which is the difference between a tool that cleans a catalogue and one
 //! that damages it in a new way.
 
+pub mod abstracts;
 pub mod author;
 mod entities4;
 pub mod fix;

parity/cases/abstracts/cases.ndjson

@@ -0,0 +1,19 @@
+{"name": "ordinary", "index": {"The": [0], "obligation": [1], "ran": [2], "both": [3], "ways": [4]}}
+{"name": "repeated-word", "index": {"the": [0, 2], "cat": [1], "mat": [3]}}
+{"name": "punctuation", "index": {"Both": [0], "ways,": [1], "plainly.": [2]}}
+{"name": "empty", "index": {}}
+{"name": "no-positions", "index": {"word": []}}
+{"name": "one-word", "index": {"Abstract": [0]}}
+{"name": "gaps", "index": {"second": [40], "first": [-3], "third": [900]}}
+{"name": "out-of-order-keys", "index": {"zebra": [5], "apple": [0], "mango": [3]}}
+{"name": "same-position", "index": {"alpha": [1], "beta": [1]}}
+{"name": "unicode", "index": {"\u039a\u03b1\u03bb\u03b7\u03bc\u03ad\u03c1\u03b1": [0], "\u03ba\u03cc\u03c3\u03bc\u03b5": [1]}}
+{"name": "long-runs", "index": {"a": [0, 2, 4, 6, 8, 10, 12, 14, 16, 18], "b": [1, 3, 5, 7, 9, 11, 13, 15, 17, 19]}}
+{"name": "big-positions", "index": {"x": [2147483648], "y": [0]}}
+{"name": "negative-only", "index": {"only": [-5]}}
+{"name": "random-0", "index": {"that": [23], "small": [37], "is": [3], "research": [32], "but": [13], "effect": [2]}}
+{"name": "random-1", "index": {"small": [4], "real": [15]}}
+{"name": "random-2", "index": {"real": [3], "small": [36]}}
+{"name": "random-3", "index": {"the": [36], "research": [37]}}
+{"name": "random-4", "index": {"research": [9], "the": [34], "real": [7], "effect": [36], "shows": [19], "that": [11], "is": [6]}}
+{"name": "random-5", "index": {"is": [4], "shows": [36], "effect": [3], "real": [39]}}

parity/golden/abstracts/cases.ndjson

@@ -0,0 +1,19 @@
+{"abstract": "The obligation ran both ways", "name": "ordinary"}
+{"abstract": "the cat the mat", "name": "repeated-word"}
+{"abstract": "Both ways, plainly.", "name": "punctuation"}
+{"abstract": null, "name": "empty"}
+{"abstract": null, "name": "no-positions"}
+{"abstract": "Abstract", "name": "one-word"}
+{"abstract": "first second third", "name": "gaps"}
+{"abstract": "apple mango zebra", "name": "out-of-order-keys"}
+{"abstract": "alpha beta", "name": "same-position"}
+{"abstract": "\u039a\u03b1\u03bb\u03b7\u03bc\u03ad\u03c1\u03b1 \u03ba\u03cc\u03c3\u03bc\u03b5", "name": "unicode"}
+{"abstract": "a b a b a b a b a b a b a b a b a b a b", "name": "long-runs"}
+{"abstract": "y x", "name": "big-positions"}
+{"abstract": "only", "name": "negative-only"}
+{"abstract": "effect is but that research small", "name": "random-0"}
+{"abstract": "small real", "name": "random-1"}
+{"abstract": "real small", "name": "random-2"}
+{"abstract": "the research", "name": "random-3"}
+{"abstract": "is real research that shows the effect", "name": "random-4"}
+{"abstract": "effect is shows real", "name": "random-5"}

parity/oracle/abstracts.py

@@ -0,0 +1,44 @@
+"""Turn an inverted abstract back into prose the way the policy pipeline did.
+
+The function below is the original, copied out of the journals module verbatim
+— importing it opens a database connection and a catalogue client, and neither
+decides anything.
+
+    python3 oracle/abstracts.py < cases.ndjson
+
+One case per line: {name, index}. Writes one JSON answer per line.
+"""
+
+from __future__ import annotations
+
+import json
+import sys
+
+
+def _decode_abstract(idx: dict | None) -> str | None:
+    if not isinstance(idx, dict) or not idx:
+        return None
+    positions: list[tuple[int, str]] = []
+    for word, locs in idx.items():
+        if not isinstance(locs, list):
+            continue
+        for p in locs:
+            if isinstance(p, int):
+                positions.append((p, str(word)))
+    if not positions:
+        return None
+    positions.sort(key=lambda t: t[0])
+    return " ".join(w for _, w in positions)
+
+
+for line in sys.stdin:
+    if not line.strip():
+        continue
+    case = json.loads(line)
+    sys.stdout.write(
+        json.dumps(
+            {"name": case["name"], "abstract": _decode_abstract(case["index"])},
+            sort_keys=True,
+        )
+        + "\n"
+    )

parity/src/bin/regen.rs

@@ -19,7 +19,7 @@ type Written = Result<Vec<String>, String>;
 type Job = (&'static str, fn() -> Written);
 
 fn main() -> ExitCode {
-    let jobs: [Job; 19] = [
+    let jobs: [Job; 20] = [
         ("tokens", tokens),
         ("unescape", unescape),
         ("chunkmd", chunkmd),
@@ -39,6 +39,7 @@ fn main() -> ExitCode {
         ("pdfgutter", pdfgutter),
         ("seriesof", seriesof),
         ("subjects", subjects),
+        ("abstracts", abstracts),
     ];
 
     for (name, job) in jobs {
@@ -221,6 +222,18 @@ fn cueseg() -> Written {
     Ok(written)
 }
 
+fn abstracts() -> Written {
+    let dir = parity::golden("abstracts");
+    std::fs::create_dir_all(&dir).map_err(|e| e.to_string())?;
+
+    let cases = parity::dir().join("cases/abstracts/cases.ndjson");
+    let text = std::fs::read_to_string(&cases).map_err(|e| e.to_string())?;
+    let answers = parity::run_oracle("abstracts.py", &text)?;
+    let path = dir.join("cases.ndjson");
+    std::fs::write(&path, answers).map_err(|e| e.to_string())?;
+    Ok(vec![path.display().to_string()])
+}
+
 fn subjects() -> Written {
     let dir = parity::golden("subjects");
     std::fs::create_dir_all(&dir).map_err(|e| e.to_string())?;

parity/tests/abstracts.rs

@@ -0,0 +1,62 @@
+//! An inverted abstract has to come back as the same prose the Python gave.
+//!
+//! What the catalogue publishes is an index rather than the paragraph, and
+//! what a reader sees on a search result is whatever this reconstructs. A
+//! word in the wrong place is a sentence that reads as though it means
+//! something and does not.
+
+use bibclean::abstracts::{self, Inverted};
+use serde_json::Value;
+
+fn records(text: &str) -> Vec<Value> {
+    text.lines()
+        .filter(|line| !line.trim().is_empty())
+        .map(|line| serde_json::from_str(line).expect("valid JSON"))
+        .collect()
+}
+
+fn cases() -> Vec<Value> {
+    let text = std::fs::read_to_string(parity::dir().join("cases/abstracts/cases.ndjson"))
+        .expect("a readable case file");
+    records(&text)
+}
+
+#[test]
+fn every_abstract_reads_back_the_same_as_python() {
+    let golden = records(&parity::read_golden_text("abstracts/cases.ndjson"));
+    let cases = cases();
+    assert_eq!(cases.len(), golden.len(), "case count moved — regenerate");
+
+    let mut wrong: Vec<String> = Vec::new();
+    for (case, theirs) in cases.iter().zip(&golden) {
+        let name = case["name"].as_str().expect("a name");
+        assert_eq!(theirs["name"].as_str(), Some(name), "cases out of order");
+
+        let index: Inverted =
+            serde_json::from_value(case["index"].clone()).expect("an index of positions");
+        let ours = abstracts::from_inverted(&index);
+        if ours.as_deref() != theirs["abstract"].as_str() {
+            wrong.push(format!(
+                "{name}: python {}, rust {ours:?}",
+                theirs["abstract"]
+            ));
+        }
+    }
+
+    assert!(
+        wrong.is_empty(),
+        "{} abstracts differ:\n{}",
+        wrong.len(),
+        wrong.join("\n")
+    );
+}
+
+/// Catches a golden file drifting away from a Python that has moved on.
+#[test]
+#[cfg(feature = "python")]
+fn the_golden_file_still_matches_live_python() {
+    let cases = std::fs::read_to_string(parity::dir().join("cases/abstracts/cases.ndjson"))
+        .expect("a readable case file");
+    let live = parity::run_oracle("abstracts.py", &cases).expect("the oracle runs");
+    assert_eq!(live, parity::read_golden_text("abstracts/cases.ndjson"));
+}