bibclean: an abstract that arrived as an index

parity/cases/abstracts/cases.ndjson

@@ -0,0 +1,19 @@
+{"name": "ordinary", "index": {"The": [0], "obligation": [1], "ran": [2], "both": [3], "ways": [4]}}
+{"name": "repeated-word", "index": {"the": [0, 2], "cat": [1], "mat": [3]}}
+{"name": "punctuation", "index": {"Both": [0], "ways,": [1], "plainly.": [2]}}
+{"name": "empty", "index": {}}
+{"name": "no-positions", "index": {"word": []}}
+{"name": "one-word", "index": {"Abstract": [0]}}
+{"name": "gaps", "index": {"second": [40], "first": [-3], "third": [900]}}
+{"name": "out-of-order-keys", "index": {"zebra": [5], "apple": [0], "mango": [3]}}
+{"name": "same-position", "index": {"alpha": [1], "beta": [1]}}
+{"name": "unicode", "index": {"\u039a\u03b1\u03bb\u03b7\u03bc\u03ad\u03c1\u03b1": [0], "\u03ba\u03cc\u03c3\u03bc\u03b5": [1]}}
+{"name": "long-runs", "index": {"a": [0, 2, 4, 6, 8, 10, 12, 14, 16, 18], "b": [1, 3, 5, 7, 9, 11, 13, 15, 17, 19]}}
+{"name": "big-positions", "index": {"x": [2147483648], "y": [0]}}
+{"name": "negative-only", "index": {"only": [-5]}}
+{"name": "random-0", "index": {"that": [23], "small": [37], "is": [3], "research": [32], "but": [13], "effect": [2]}}
+{"name": "random-1", "index": {"small": [4], "real": [15]}}
+{"name": "random-2", "index": {"real": [3], "small": [36]}}
+{"name": "random-3", "index": {"the": [36], "research": [37]}}
+{"name": "random-4", "index": {"research": [9], "the": [34], "real": [7], "effect": [36], "shows": [19], "that": [11], "is": [6]}}
+{"name": "random-5", "index": {"is": [4], "shows": [36], "effect": [3], "real": [39]}}

parity/golden/abstracts/cases.ndjson

@@ -0,0 +1,19 @@
+{"abstract": "The obligation ran both ways", "name": "ordinary"}
+{"abstract": "the cat the mat", "name": "repeated-word"}
+{"abstract": "Both ways, plainly.", "name": "punctuation"}
+{"abstract": null, "name": "empty"}
+{"abstract": null, "name": "no-positions"}
+{"abstract": "Abstract", "name": "one-word"}
+{"abstract": "first second third", "name": "gaps"}
+{"abstract": "apple mango zebra", "name": "out-of-order-keys"}
+{"abstract": "alpha beta", "name": "same-position"}
+{"abstract": "\u039a\u03b1\u03bb\u03b7\u03bc\u03ad\u03c1\u03b1 \u03ba\u03cc\u03c3\u03bc\u03b5", "name": "unicode"}
+{"abstract": "a b a b a b a b a b a b a b a b a b a b", "name": "long-runs"}
+{"abstract": "y x", "name": "big-positions"}
+{"abstract": "only", "name": "negative-only"}
+{"abstract": "effect is but that research small", "name": "random-0"}
+{"abstract": "small real", "name": "random-1"}
+{"abstract": "real small", "name": "random-2"}
+{"abstract": "the research", "name": "random-3"}
+{"abstract": "is real research that shows the effect", "name": "random-4"}
+{"abstract": "effect is shows real", "name": "random-5"}

parity/oracle/abstracts.py

@@ -0,0 +1,44 @@
+"""Turn an inverted abstract back into prose the way the policy pipeline did.
+
+The function below is the original, copied out of the journals module verbatim
+— importing it opens a database connection and a catalogue client, and neither
+decides anything.
+
+    python3 oracle/abstracts.py < cases.ndjson
+
+One case per line: {name, index}. Writes one JSON answer per line.
+"""
+
+from __future__ import annotations
+
+import json
+import sys
+
+
+def _decode_abstract(idx: dict | None) -> str | None:
+    if not isinstance(idx, dict) or not idx:
+        return None
+    positions: list[tuple[int, str]] = []
+    for word, locs in idx.items():
+        if not isinstance(locs, list):
+            continue
+        for p in locs:
+            if isinstance(p, int):
+                positions.append((p, str(word)))
+    if not positions:
+        return None
+    positions.sort(key=lambda t: t[0])
+    return " ".join(w for _, w in positions)
+
+
+for line in sys.stdin:
+    if not line.strip():
+        continue
+    case = json.loads(line)
+    sys.stdout.write(
+        json.dumps(
+            {"name": case["name"], "abstract": _decode_abstract(case["index"])},
+            sort_keys=True,
+        )
+        + "\n"
+    )

parity/src/bin/regen.rs

@@ -19,7 +19,7 @@ type Written = Result<Vec<String>, String>;
 type Job = (&'static str, fn() -> Written);
 
 fn main() -> ExitCode {
-    let jobs: [Job; 19] = [
+    let jobs: [Job; 20] = [
         ("tokens", tokens),
         ("unescape", unescape),
         ("chunkmd", chunkmd),
@@ -39,6 +39,7 @@ fn main() -> ExitCode {
         ("pdfgutter", pdfgutter),
         ("seriesof", seriesof),
         ("subjects", subjects),
+        ("abstracts", abstracts),
     ];
 
     for (name, job) in jobs {
@@ -221,6 +222,18 @@ fn cueseg() -> Written {
     Ok(written)
 }
 
+fn abstracts() -> Written {
+    let dir = parity::golden("abstracts");
+    std::fs::create_dir_all(&dir).map_err(|e| e.to_string())?;
+
+    let cases = parity::dir().join("cases/abstracts/cases.ndjson");
+    let text = std::fs::read_to_string(&cases).map_err(|e| e.to_string())?;
+    let answers = parity::run_oracle("abstracts.py", &text)?;
+    let path = dir.join("cases.ndjson");
+    std::fs::write(&path, answers).map_err(|e| e.to_string())?;
+    Ok(vec![path.display().to_string()])
+}
+
 fn subjects() -> Written {
     let dir = parity::golden("subjects");
     std::fs::create_dir_all(&dir).map_err(|e| e.to_string())?;

parity/tests/abstracts.rs

@@ -0,0 +1,62 @@
+//! An inverted abstract has to come back as the same prose the Python gave.
+//!
+//! What the catalogue publishes is an index rather than the paragraph, and
+//! what a reader sees on a search result is whatever this reconstructs. A
+//! word in the wrong place is a sentence that reads as though it means
+//! something and does not.
+
+use bibclean::abstracts::{self, Inverted};
+use serde_json::Value;
+
+fn records(text: &str) -> Vec<Value> {
+    text.lines()
+        .filter(|line| !line.trim().is_empty())
+        .map(|line| serde_json::from_str(line).expect("valid JSON"))
+        .collect()
+}
+
+fn cases() -> Vec<Value> {
+    let text = std::fs::read_to_string(parity::dir().join("cases/abstracts/cases.ndjson"))
+        .expect("a readable case file");
+    records(&text)
+}
+
+#[test]
+fn every_abstract_reads_back_the_same_as_python() {
+    let golden = records(&parity::read_golden_text("abstracts/cases.ndjson"));
+    let cases = cases();
+    assert_eq!(cases.len(), golden.len(), "case count moved — regenerate");
+
+    let mut wrong: Vec<String> = Vec::new();
+    for (case, theirs) in cases.iter().zip(&golden) {
+        let name = case["name"].as_str().expect("a name");
+        assert_eq!(theirs["name"].as_str(), Some(name), "cases out of order");
+
+        let index: Inverted =
+            serde_json::from_value(case["index"].clone()).expect("an index of positions");
+        let ours = abstracts::from_inverted(&index);
+        if ours.as_deref() != theirs["abstract"].as_str() {
+            wrong.push(format!(
+                "{name}: python {}, rust {ours:?}",
+                theirs["abstract"]
+            ));
+        }
+    }
+
+    assert!(
+        wrong.is_empty(),
+        "{} abstracts differ:\n{}",
+        wrong.len(),
+        wrong.join("\n")
+    );
+}
+
+/// Catches a golden file drifting away from a Python that has moved on.
+#[test]
+#[cfg(feature = "python")]
+fn the_golden_file_still_matches_live_python() {
+    let cases = std::fs::read_to_string(parity::dir().join("cases/abstracts/cases.ndjson"))
+        .expect("a readable case file");
+    let live = parity::run_oracle("abstracts.py", &cases).expect("the oracle runs");
+    assert_eq!(live, parity::read_golden_text("abstracts/cases.ndjson"));
+}