mirror of
https://git.pumpin.online/funkpower/catena.git
synced 2026-10-22 18:57:14 +00:00
vocabmerge: the near-duplicate headings a catalogue accumulates
This commit is contained in:
funkpower
@@ -0,0 +1,15 @@
|
||||
{"kind": "author", "items": [{"id": 1, "name": "Dostoevsky, Fyodor", "count": 3}, {"id": 2, "name": "Dostoevsky| Fyodor", "count": 1}, {"id": 3, "name": "Fyodor Dostoevsky", "count": 11}]}
|
||||
{"kind": "author", "items": [{"id": 1, "name": "Fyodor Dostoevsky", "count": 11}, {"id": 2, "name": "Fyodor Dostoëvsky", "count": 2}, {"id": 3, "name": "Dostoevsky, Fyodor", "count": 3}]}
|
||||
{"kind": "author", "items": [{"id": 1, "name": "C.S. Lewis", "count": 8}, {"id": 2, "name": "C. S. Lewis", "count": 12}, {"id": 3, "name": "Lewis, C. S.", "count": 4}, {"id": 4, "name": "CLIVE STAPLES LEWIS", "count": 1}]}
|
||||
{"kind": "author", "items": [{"id": 1, "name": "Smith, John (editor)", "count": 2}, {"id": 2, "name": "John Smith", "count": 5}, {"id": 3, "name": "Smith, John Jr.", "count": 1}]}
|
||||
{"kind": "author", "items": [{"id": 1, "name": "Plato", "count": 30}, {"id": 2, "name": "Aristotle", "count": 25}]}
|
||||
{"kind": "author", "items": [{"id": 1, "name": "Arendt, Hannah;", "count": 1}, {"id": 2, "name": "Hannah Arendt", "count": 9}]}
|
||||
{"kind": "author", "items": [{"id": 1, "name": "Étienne Gilson", "count": 4}, {"id": 2, "name": "Etienne Gilson", "count": 2}]}
|
||||
{"kind": "publisher", "items": [{"id": 1, "name": "Oxford University Press", "count": 40}, {"id": 2, "name": "Oxford University Press, Inc.", "count": 2}, {"id": 3, "name": "Oxford Univ. Press", "count": 1}, {"id": 4, "name": "Routledge", "count": 9}]}
|
||||
{"kind": "publisher", "items": [{"id": 1, "name": "Penguin Books", "count": 20}, {"id": 2, "name": "Penguin Books Ltd", "count": 3}, {"id": 3, "name": "Penguin", "count": 7}]}
|
||||
{"kind": "publisher", "items": [{"id": 1, "name": "Brill", "count": 5}]}
|
||||
{"kind": "tag", "items": [{"id": 1, "name": "Philosophy", "count": 30}, {"id": 2, "name": "[Philosophy]", "count": 1}, {"id": 3, "name": "Philosophy.", "count": 2}, {"id": 4, "name": "philosophy", "count": 4}]}
|
||||
{"kind": "tag", "items": [{"id": 1, "name": "18th Century", "count": 12}, {"id": 2, "name": "z-lib.org", "count": 300}, {"id": 3, "name": "9780198237907", "count": 1}, {"id": 4, "name": "https://example.org/x", "count": 2}, {"id": 5, "name": "1998", "count": 5}, {"id": 6, "name": "2024-01-15", "count": 1}, {"id": 7, "name": "451", "count": 2}, {"id": 8, "name": "1:23:45", "count": 1}, {"id": 9, "name": "[kobo] favourites", "count": 4}, {"id": 10, "name": "a;b;c;d", "count": 1}, {"id": 11, "name": "---", "count": 1}, {"id": 12, "name": "#genre", "count": 9}, {"id": 13, "name": "ISBN-13", "count": 3}, {"id": 14, "name": "World War, 1939-1945", "count": 40}]}
|
||||
{"kind": "tag", "items": [{"id": 1, "name": "A very long blurb used as a tag that goes on and on and on and should not be here at all really", "count": 1}, {"id": 2, "name": "Church history", "count": 8}]}
|
||||
{"kind": "tag", "items": [{"id": 1, "name": "[alexskobo] to read", "count": 6}, {"id": 2, "name": "Fiction", "count": 40}]}
|
||||
{"kind": "author", "items": []}
|
||||
@@ -0,0 +1,15 @@
|
||||
{"canonical": "Fyodor Dostoevsky", "keys": [{"author_tokens": ["dostoevsky", "fyodor"], "junk": null, "norm": "dostoevsky fyodor", "pub_key": "dostoevsky fyodor", "tag_key": "dostoevsky fyodor"}, {"author_tokens": ["dostoevsky", "fyodor"], "junk": null, "norm": "dostoevsky fyodor", "pub_key": "dostoevsky fyodor", "tag_key": "dostoevsky fyodor"}, {"author_tokens": ["fyodor", "dostoevsky"], "junk": null, "norm": "fyodor dostoevsky", "pub_key": "fyodor dostoevsky", "tag_key": "fyodor dostoevsky"}], "proposals": [{"action": "merge", "confidence": 0.9, "drop": "Dostoevsky, Fyodor", "drop_count": 3, "drop_id": 1, "evidence": "3 books as 'Dostoevsky, Fyodor' → 'Fyodor Dostoevsky' (11)", "keep": "Fyodor Dostoevsky", "keep_count": 11, "keep_id": 3, "method": "last_first"}, {"action": "merge", "confidence": 0.9, "drop": "Dostoevsky| Fyodor", "drop_count": 1, "drop_id": 2, "evidence": "1 books as 'Dostoevsky| Fyodor' → 'Fyodor Dostoevsky' (11)", "keep": "Fyodor Dostoevsky", "keep_count": 11, "keep_id": 3, "method": "last_first"}]}
|
||||
{"canonical": "Fyodor Dostoevsky", "keys": [{"author_tokens": ["fyodor", "dostoevsky"], "junk": null, "norm": "fyodor dostoevsky", "pub_key": "fyodor dostoevsky", "tag_key": "fyodor dostoevsky"}, {"author_tokens": ["fyodor", "dostoevsky"], "junk": null, "norm": "fyodor dostoevsky", "pub_key": "fyodor dostoevsky", "tag_key": "fyodor dostoevsky"}, {"author_tokens": ["dostoevsky", "fyodor"], "junk": null, "norm": "dostoevsky fyodor", "pub_key": "dostoevsky fyodor", "tag_key": "dostoevsky fyodor"}], "proposals": [{"action": "merge", "confidence": 0.95, "drop": "Fyodor Dostoëvsky", "drop_count": 2, "drop_id": 2, "evidence": "2 books as 'Fyodor Dostoëvsky' → 'Fyodor Dostoevsky' (11)", "keep": "Fyodor Dostoevsky", "keep_count": 11, "keep_id": 1, "method": "strict_norm"}, {"action": "merge", "confidence": 0.9, "drop": "Dostoevsky, Fyodor", "drop_count": 3, "drop_id": 3, "evidence": "3 books as 'Dostoevsky, Fyodor' → 'Fyodor Dostoevsky' (11)", "keep": "Fyodor Dostoevsky", "keep_count": 11, "keep_id": 1, "method": "last_first"}]}
|
||||
{"canonical": "C. S. Lewis", "keys": [{"author_tokens": ["c", "s", "lewis"], "junk": null, "norm": "c s lewis", "pub_key": "c s lewis", "tag_key": "c s lewis"}, {"author_tokens": ["c", "s", "lewis"], "junk": null, "norm": "c s lewis", "pub_key": "c s lewis", "tag_key": "c s lewis"}, {"author_tokens": ["lewis", "c", "s"], "junk": null, "norm": "lewis c s", "pub_key": "lewis c s", "tag_key": "lewis c s"}, {"author_tokens": ["clive", "staples", "lewis"], "junk": null, "norm": "clive staples lewis", "pub_key": "clive staples lewis", "tag_key": "clive staples lewis"}], "proposals": [{"action": "merge", "confidence": 0.95, "drop": "C.S. Lewis", "drop_count": 8, "drop_id": 1, "evidence": "8 books as 'C.S. Lewis' → 'C. S. Lewis' (12)", "keep": "C. S. Lewis", "keep_count": 12, "keep_id": 2, "method": "strict_norm"}, {"action": "merge", "confidence": 0.9, "drop": "Lewis, C. S.", "drop_count": 4, "drop_id": 3, "evidence": "4 books as 'Lewis, C. S.' → 'C. S. Lewis' (12)", "keep": "C. S. Lewis", "keep_count": 12, "keep_id": 2, "method": "last_first"}]}
|
||||
{"canonical": "John Smith", "keys": [{"author_tokens": ["smith", "john"], "junk": null, "norm": "smith john editor", "pub_key": "smith john editor", "tag_key": "smith john editor"}, {"author_tokens": ["john", "smith"], "junk": null, "norm": "john smith", "pub_key": "john smith", "tag_key": "john smith"}, {"author_tokens": ["smith", "john"], "junk": null, "norm": "smith john jr", "pub_key": "smith john jr", "tag_key": "smith john jr"}], "proposals": [{"action": "merge", "confidence": 0.9, "drop": "Smith, John (editor)", "drop_count": 2, "drop_id": 1, "evidence": "2 books as 'Smith, John (editor)' → 'John Smith' (5)", "keep": "John Smith", "keep_count": 5, "keep_id": 2, "method": "last_first"}, {"action": "merge", "confidence": 0.9, "drop": "Smith, John Jr.", "drop_count": 1, "drop_id": 3, "evidence": "1 books as 'Smith, John Jr.' → 'John Smith' (5)", "keep": "John Smith", "keep_count": 5, "keep_id": 2, "method": "last_first"}]}
|
||||
{"canonical": "Plato", "keys": [{"author_tokens": ["plato"], "junk": null, "norm": "plato", "pub_key": "plato", "tag_key": "plato"}, {"author_tokens": ["aristotle"], "junk": null, "norm": "aristotle", "pub_key": "aristotle", "tag_key": "aristotle"}], "proposals": []}
|
||||
{"canonical": "Hannah Arendt", "keys": [{"author_tokens": ["arendt", "hannah"], "junk": null, "norm": "arendt hannah", "pub_key": "arendt hannah", "tag_key": "arendt hannah"}, {"author_tokens": ["hannah", "arendt"], "junk": null, "norm": "hannah arendt", "pub_key": "hannah arendt", "tag_key": "hannah arendt"}], "proposals": [{"action": "merge", "confidence": 0.9, "drop": "Arendt, Hannah;", "drop_count": 1, "drop_id": 1, "evidence": "1 books as 'Arendt, Hannah;' → 'Hannah Arendt' (9)", "keep": "Hannah Arendt", "keep_count": 9, "keep_id": 2, "method": "last_first"}]}
|
||||
{"canonical": "Étienne Gilson", "keys": [{"author_tokens": ["etienne", "gilson"], "junk": null, "norm": "etienne gilson", "pub_key": "etienne gilson", "tag_key": "etienne gilson"}, {"author_tokens": ["etienne", "gilson"], "junk": null, "norm": "etienne gilson", "pub_key": "etienne gilson", "tag_key": "etienne gilson"}], "proposals": [{"action": "merge", "confidence": 0.95, "drop": "Etienne Gilson", "drop_count": 2, "drop_id": 2, "evidence": "2 books as 'Etienne Gilson' → 'Étienne Gilson' (4)", "keep": "Étienne Gilson", "keep_count": 4, "keep_id": 1, "method": "strict_norm"}]}
|
||||
{"canonical": "Oxford University Press", "keys": [{"author_tokens": ["oxford", "university", "press"], "junk": null, "norm": "oxford university press", "pub_key": "oxford", "tag_key": "oxford university press"}, {"author_tokens": ["oxford", "university", "press", "inc"], "junk": null, "norm": "oxford university press inc", "pub_key": "oxford", "tag_key": "oxford university press inc"}, {"author_tokens": ["oxford", "univ", "press"], "junk": null, "norm": "oxford univ press", "pub_key": "oxford univ", "tag_key": "oxford univ press"}, {"author_tokens": ["routledge"], "junk": null, "norm": "routledge", "pub_key": "routledge", "tag_key": "routledge"}], "proposals": [{"action": "merge", "confidence": 0.8, "drop": "Oxford University Press, Inc.", "drop_count": 2, "drop_id": 2, "evidence": "2 books as 'Oxford University Press, Inc.' → 'Oxford University Press' (40)", "keep": "Oxford University Press", "keep_count": 40, "keep_id": 1, "method": "publisher_family"}]}
|
||||
{"canonical": "Penguin Books", "keys": [{"author_tokens": ["penguin", "books"], "junk": null, "norm": "penguin books", "pub_key": "penguin", "tag_key": "penguin books"}, {"author_tokens": ["penguin", "books", "ltd"], "junk": null, "norm": "penguin books ltd", "pub_key": "penguin", "tag_key": "penguin books ltd"}, {"author_tokens": ["penguin"], "junk": null, "norm": "penguin", "pub_key": "penguin", "tag_key": "penguin"}], "proposals": [{"action": "merge", "confidence": 0.8, "drop": "Penguin Books Ltd", "drop_count": 3, "drop_id": 2, "evidence": "3 books as 'Penguin Books Ltd' → 'Penguin Books' (20)", "keep": "Penguin Books", "keep_count": 20, "keep_id": 1, "method": "publisher_family"}, {"action": "merge", "confidence": 0.8, "drop": "Penguin", "drop_count": 7, "drop_id": 3, "evidence": "7 books as 'Penguin' → 'Penguin Books' (20)", "keep": "Penguin Books", "keep_count": 20, "keep_id": 1, "method": "publisher_family"}]}
|
||||
{"canonical": "Brill", "keys": [{"author_tokens": ["brill"], "junk": null, "norm": "brill", "pub_key": "brill", "tag_key": "brill"}], "proposals": []}
|
||||
{"canonical": "Philosophy", "keys": [{"author_tokens": ["philosophy"], "junk": null, "norm": "philosophy", "pub_key": "philosophy", "tag_key": "philosophy"}, {"author_tokens": ["philosophy"], "junk": null, "norm": "philosophy", "pub_key": "philosophy", "tag_key": "philosophy"}, {"author_tokens": ["philosophy"], "junk": null, "norm": "philosophy", "pub_key": "philosophy", "tag_key": "philosophy"}, {"author_tokens": ["philosophy"], "junk": null, "norm": "philosophy", "pub_key": "philosophy", "tag_key": "philosophy"}], "proposals": [{"action": "merge", "confidence": 0.9, "drop": "[Philosophy]", "drop_count": 1, "drop_id": 2, "evidence": "1 books as '[Philosophy]' → 'Philosophy' (30)", "keep": "Philosophy", "keep_count": 30, "keep_id": 1, "method": "tag_norm"}, {"action": "merge", "confidence": 0.9, "drop": "Philosophy.", "drop_count": 2, "drop_id": 3, "evidence": "2 books as 'Philosophy.' → 'Philosophy' (30)", "keep": "Philosophy", "keep_count": 30, "keep_id": 1, "method": "tag_norm"}, {"action": "merge", "confidence": 0.9, "drop": "philosophy", "drop_count": 4, "drop_id": 4, "evidence": "4 books as 'philosophy' → 'Philosophy' (30)", "keep": "Philosophy", "keep_count": 30, "keep_id": 1, "method": "tag_norm"}]}
|
||||
{"canonical": "z-lib.org", "keys": [{"author_tokens": ["18th", "century"], "junk": null, "norm": "18th century", "pub_key": "18th century", "tag_key": "18th century"}, {"author_tokens": ["z", "lib", "org"], "junk": "format-or-watermark", "norm": "z lib org", "pub_key": "z lib org", "tag_key": "z lib org"}, {"author_tokens": ["9780198237907"], "junk": "isbn", "norm": "9780198237907", "pub_key": "9780198237907", "tag_key": "9780198237907"}, {"author_tokens": ["https", "example", "org", "x"], "junk": "url", "norm": "https example org x", "pub_key": "https example org x", "tag_key": "https example org x"}, {"author_tokens": ["1998"], "junk": "date", "norm": "1998", "pub_key": "1998", "tag_key": "1998"}, {"author_tokens": ["2024", "01", "15"], "junk": "date", "norm": "2024 01 15", "pub_key": "2024 01 15", "tag_key": "2024 01 15"}, {"author_tokens": ["451"], "junk": "numeric-id", "norm": "451", "pub_key": "451", "tag_key": "451"}, {"author_tokens": ["1", "23", "45"], "junk": "timestamp", "norm": "1 23 45", "pub_key": "1 23 45", "tag_key": "1 23 45"}, {"author_tokens": ["kobo", "favourites"], "junk": "device-tag", "norm": "kobo favourites", "pub_key": "kobo favourites", "tag_key": "kobo favourites"}, {"author_tokens": ["a", "b", "c", "d"], "junk": "exploded-list", "norm": "a b c d", "pub_key": "a b c d", "tag_key": "a b c d"}, {"author_tokens": [], "junk": "punctuation-only", "norm": "", "pub_key": "", "tag_key": ""}, {"author_tokens": ["genre"], "junk": "format-or-watermark", "norm": "genre", "pub_key": "genre", "tag_key": "genre"}, {"author_tokens": ["isbn", "13"], "junk": "catalog-field", "norm": "isbn 13", "pub_key": "isbn 13", "tag_key": "isbn 13"}, {"author_tokens": ["world", "war", "1939", "1945"], "junk": null, "norm": "world war 1939 1945", "pub_key": "world war 1939 1945", "tag_key": "world war 1939 1945"}], "proposals": [{"action": "delete", "confidence": 0.92, "drop": "z-lib.org", "drop_count": 300, "drop_id": 2, "evidence": "format-or-watermark: 'z-lib.org' on 300 books", "keep": null, "keep_count": 0, "keep_id": null, "method": "junk_tag:format-or-watermark"}, {"action": "delete", "confidence": 0.92, "drop": "9780198237907", "drop_count": 1, "drop_id": 3, "evidence": "isbn: '9780198237907' on 1 books", "keep": null, "keep_count": 0, "keep_id": null, "method": "junk_tag:isbn"}, {"action": "delete", "confidence": 0.92, "drop": "https://example.org/x", "drop_count": 2, "drop_id": 4, "evidence": "url: 'https://example.org/x' on 2 books", "keep": null, "keep_count": 0, "keep_id": null, "method": "junk_tag:url"}, {"action": "delete", "confidence": 0.92, "drop": "1998", "drop_count": 5, "drop_id": 5, "evidence": "date: '1998' on 5 books", "keep": null, "keep_count": 0, "keep_id": null, "method": "junk_tag:date"}, {"action": "delete", "confidence": 0.92, "drop": "2024-01-15", "drop_count": 1, "drop_id": 6, "evidence": "date: '2024-01-15' on 1 books", "keep": null, "keep_count": 0, "keep_id": null, "method": "junk_tag:date"}, {"action": "delete", "confidence": 0.92, "drop": "451", "drop_count": 2, "drop_id": 7, "evidence": "numeric-id: '451' on 2 books", "keep": null, "keep_count": 0, "keep_id": null, "method": "junk_tag:numeric-id"}, {"action": "delete", "confidence": 0.92, "drop": "1:23:45", "drop_count": 1, "drop_id": 8, "evidence": "timestamp: '1:23:45' on 1 books", "keep": null, "keep_count": 0, "keep_id": null, "method": "junk_tag:timestamp"}, {"action": "delete", "confidence": 0.92, "drop": "[kobo] favourites", "drop_count": 4, "drop_id": 9, "evidence": "device-tag: '[kobo] favourites' on 4 books", "keep": null, "keep_count": 0, "keep_id": null, "method": "junk_tag:device-tag"}, {"action": "delete", "confidence": 0.92, "drop": "a;b;c;d", "drop_count": 1, "drop_id": 10, "evidence": "exploded-list: 'a;b;c;d' on 1 books", "keep": null, "keep_count": 0, "keep_id": null, "method": "junk_tag:exploded-list"}, {"action": "delete", "confidence": 0.92, "drop": "---", "drop_count": 1, "drop_id": 11, "evidence": "punctuation-only: '---' on 1 books", "keep": null, "keep_count": 0, "keep_id": null, "method": "junk_tag:punctuation-only"}, {"action": "delete", "confidence": 0.92, "drop": "#genre", "drop_count": 9, "drop_id": 12, "evidence": "format-or-watermark: '#genre' on 9 books", "keep": null, "keep_count": 0, "keep_id": null, "method": "junk_tag:format-or-watermark"}, {"action": "delete", "confidence": 0.92, "drop": "ISBN-13", "drop_count": 3, "drop_id": 13, "evidence": "catalog-field: 'ISBN-13' on 3 books", "keep": null, "keep_count": 0, "keep_id": null, "method": "junk_tag:catalog-field"}]}
|
||||
{"canonical": "Church history", "keys": [{"author_tokens": ["a", "very", "long", "blurb", "used", "as", "a", "tag", "that", "goes", "on", "and", "on", "and", "on", "and", "should", "not", "be", "here", "at", "all", "really"], "junk": "blurb", "norm": "a very long blurb used as a tag that goes on and on and on and should not be here at all really", "pub_key": "a very long blurb used as a tag that goes on and on and on and should not be here at all really", "tag_key": "a very long blurb used as a tag that goes on and on and on and should not be here at all really"}, {"author_tokens": ["church", "history"], "junk": null, "norm": "church history", "pub_key": "church history", "tag_key": "church history"}], "proposals": [{"action": "delete", "confidence": 0.92, "drop": "A very long blurb used as a tag that goes on and on and on and should not be here at all really", "drop_count": 1, "drop_id": 1, "evidence": "blurb: 'A very long blurb used as a tag that goes on and on and on and should not be here at all really' on 1 books", "keep": null, "keep_count": 0, "keep_id": null, "method": "junk_tag:blurb"}]}
|
||||
{"canonical": "Fiction", "keys": [{"author_tokens": ["alexskobo", "to", "read"], "junk": "device-tag", "norm": "alexskobo to read", "pub_key": "alexskobo to read", "tag_key": "alexskobo to read"}, {"author_tokens": ["fiction"], "junk": null, "norm": "fiction", "pub_key": "fiction", "tag_key": "fiction"}], "proposals": [{"action": "delete", "confidence": 0.92, "drop": "[alexskobo] to read", "drop_count": 6, "drop_id": 1, "evidence": "device-tag: '[alexskobo] to read' on 6 books", "keep": null, "keep_count": 0, "keep_id": null, "method": "junk_tag:device-tag"}]}
|
||||
{"canonical": null, "keys": [], "proposals": []}
|
||||
@@ -0,0 +1,73 @@
|
||||
"""Cluster vocabulary the way the Python pipeline clusters it.
|
||||
|
||||
Reads one `{"kind", "items": [{"id","name","count"}]}` object per line and
|
||||
writes the proposals each list produces.
|
||||
|
||||
CATENA_PARITY_PYTHONPATH=/path/to/book-pipeline/src \
|
||||
python3 oracle/vocab.py < cases/vocab/lists.ndjson
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
for path in os.environ.get("CATENA_PARITY_PYTHONPATH", "").split(os.pathsep):
|
||||
if path:
|
||||
sys.path.insert(0, path)
|
||||
|
||||
from book_pipeline import vocab # noqa: E402
|
||||
|
||||
|
||||
def main() -> int:
|
||||
out = sys.stdout
|
||||
for line in sys.stdin.buffer.read().decode("utf-8").split("\n"):
|
||||
if not line.strip():
|
||||
continue
|
||||
case = json.loads(line)
|
||||
items = [
|
||||
vocab.VocabItem(calibre_id=i["id"], name=i["name"], book_count=i["count"])
|
||||
for i in case["items"]
|
||||
]
|
||||
proposals = vocab.cluster(case["kind"], items)
|
||||
out.write(
|
||||
json.dumps(
|
||||
{
|
||||
"keys": [
|
||||
{
|
||||
"norm": vocab._norm(i.name),
|
||||
"author_tokens": list(vocab._author_tokens(i.name)),
|
||||
"pub_key": vocab._pub_key(i.name),
|
||||
"tag_key": vocab._tag_key(i.name),
|
||||
"junk": vocab.junk_tag_reason(i.name),
|
||||
}
|
||||
for i in items
|
||||
],
|
||||
"canonical": (
|
||||
vocab.pick_canonical(case["kind"], items).name if items else None
|
||||
),
|
||||
"proposals": [
|
||||
{
|
||||
"action": p.action,
|
||||
"keep": p.keep_name,
|
||||
"drop": p.drop_name,
|
||||
"keep_id": p.keep_calibre_id,
|
||||
"drop_id": p.drop_calibre_id,
|
||||
"keep_count": p.book_count_keep,
|
||||
"drop_count": p.book_count_drop,
|
||||
"method": p.method,
|
||||
"confidence": p.confidence,
|
||||
"evidence": p.evidence,
|
||||
}
|
||||
for p in proposals
|
||||
],
|
||||
},
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
)
|
||||
+ "\n"
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
+14
-1
@@ -19,7 +19,7 @@ type Written = Result<Vec<String>, String>;
|
||||
type Job = (&'static str, fn() -> Written);
|
||||
|
||||
fn main() -> ExitCode {
|
||||
let jobs: [Job; 13] = [
|
||||
let jobs: [Job; 14] = [
|
||||
("tokens", tokens),
|
||||
("unescape", unescape),
|
||||
("chunkmd", chunkmd),
|
||||
@@ -33,6 +33,7 @@ fn main() -> ExitCode {
|
||||
("bibclean", bibclean),
|
||||
("slug", slug),
|
||||
("author", author),
|
||||
("vocab", vocab),
|
||||
];
|
||||
|
||||
for (name, job) in jobs {
|
||||
@@ -76,6 +77,18 @@ fn unescape() -> Written {
|
||||
Ok(vec![path.display().to_string()])
|
||||
}
|
||||
|
||||
fn vocab() -> Written {
|
||||
let dir = parity::golden("vocab");
|
||||
std::fs::create_dir_all(&dir).map_err(|e| e.to_string())?;
|
||||
|
||||
let cases = parity::dir().join("cases/vocab/lists.ndjson");
|
||||
let cases = std::fs::read_to_string(&cases).map_err(|e| e.to_string())?;
|
||||
let judgements = parity::run_oracle("vocab.py", &cases)?;
|
||||
let path = dir.join("lists.ndjson");
|
||||
std::fs::write(&path, judgements).map_err(|e| e.to_string())?;
|
||||
Ok(vec![path.display().to_string()])
|
||||
}
|
||||
|
||||
fn author() -> Written {
|
||||
let dir = parity::golden("author");
|
||||
std::fs::create_dir_all(&dir).map_err(|e| e.to_string())?;
|
||||
|
||||
@@ -0,0 +1,139 @@
|
||||
//! Vocabulary clustering must match the Python exactly.
|
||||
//!
|
||||
//! A merge proposal is a suggestion to collapse two catalogue headings into
|
||||
//! one. Proposing a different set than the system being replaced means either
|
||||
//! merging two real people or leaving a person split in half, and the first
|
||||
//! is much harder to undo than it is to do.
|
||||
|
||||
use bibclean::vocab::{Item, Kind, Rules};
|
||||
use serde_json::Value;
|
||||
|
||||
fn records(path: &str) -> Vec<Value> {
|
||||
let text = std::fs::read_to_string(parity::dir().join(path)).expect("a readable file");
|
||||
text.lines()
|
||||
.filter(|line| !line.trim().is_empty())
|
||||
.map(|line| serde_json::from_str(line).expect("valid JSON"))
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn kind_of(case: &Value) -> Kind {
|
||||
match case["kind"].as_str().expect("a kind") {
|
||||
"author" => Kind::Author,
|
||||
"publisher" => Kind::Publisher,
|
||||
"tag" => Kind::Tag,
|
||||
other => panic!("unknown kind {other}"),
|
||||
}
|
||||
}
|
||||
|
||||
fn items_of(case: &Value) -> Vec<Item> {
|
||||
case["items"]
|
||||
.as_array()
|
||||
.expect("a list")
|
||||
.iter()
|
||||
.map(|item| Item {
|
||||
id: item["id"].as_i64().expect("an id"),
|
||||
name: item["name"].as_str().expect("a name").to_string(),
|
||||
count: item["count"].as_i64().expect("a count"),
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// The device named after its owner is supplied rather than compiled in, so
|
||||
/// the oracle's built-in list has to be handed over for the comparison.
|
||||
fn rules() -> Rules {
|
||||
Rules {
|
||||
device_prefixes: vec!["alexskobo".into()],
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_cluster_matches_python() {
|
||||
let cases = records("cases/vocab/lists.ndjson");
|
||||
let golden = records("golden/vocab/lists.ndjson");
|
||||
assert_eq!(cases.len(), golden.len(), "case count moved — regenerate");
|
||||
|
||||
let mut wrong: Vec<String> = Vec::new();
|
||||
for (at, (case, theirs)) in cases.iter().zip(&golden).enumerate() {
|
||||
let kind = kind_of(case);
|
||||
let items = items_of(case);
|
||||
|
||||
let mut differs = |what: &str, ours: String, theirs: &Value| {
|
||||
wrong.push(format!("list {at} {what}: python {theirs}, rust {ours}"));
|
||||
};
|
||||
|
||||
// The keys every grouping rests on.
|
||||
for (item, theirs) in items.iter().zip(theirs["keys"].as_array().expect("a list")) {
|
||||
let ours = bibclean::vocab::normalise(&item.name);
|
||||
if Some(ours.as_str()) != theirs["norm"].as_str() {
|
||||
differs("norm", ours, &theirs["norm"]);
|
||||
}
|
||||
let ours = Value::from(bibclean::vocab::author_tokens(&item.name));
|
||||
if ours != theirs["author_tokens"] {
|
||||
differs("author_tokens", ours.to_string(), &theirs["author_tokens"]);
|
||||
}
|
||||
let ours = bibclean::vocab::publisher_key(&item.name);
|
||||
if Some(ours.as_str()) != theirs["pub_key"].as_str() {
|
||||
differs("pub_key", ours, &theirs["pub_key"]);
|
||||
}
|
||||
let ours = bibclean::vocab::tag_key(&item.name);
|
||||
if Some(ours.as_str()) != theirs["tag_key"].as_str() {
|
||||
differs("tag_key", ours, &theirs["tag_key"]);
|
||||
}
|
||||
let ours = bibclean::vocab::junk_reason(&item.name, &rules());
|
||||
if ours != theirs["junk"].as_str() {
|
||||
differs("junk", format!("{ours:?}"), &theirs["junk"]);
|
||||
}
|
||||
}
|
||||
|
||||
if !items.is_empty() {
|
||||
let ours = bibclean::vocab::pick_canonical(kind, &items).name;
|
||||
if Some(ours.as_str()) != theirs["canonical"].as_str() {
|
||||
differs("canonical", ours, &theirs["canonical"]);
|
||||
}
|
||||
}
|
||||
|
||||
let ours: Vec<Value> = bibclean::vocab::cluster(kind, &items, &rules())
|
||||
.iter()
|
||||
.map(|proposal| {
|
||||
serde_json::json!({
|
||||
"action": proposal.action,
|
||||
"keep": proposal.keep,
|
||||
"drop": proposal.drop,
|
||||
"keep_id": proposal.keep_id,
|
||||
"drop_id": proposal.drop_id,
|
||||
"keep_count": proposal.keep_count,
|
||||
"drop_count": proposal.drop_count,
|
||||
"method": proposal.method,
|
||||
"confidence": proposal.confidence,
|
||||
"evidence": proposal.evidence,
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
let ours = Value::Array(ours);
|
||||
if ours != theirs["proposals"] {
|
||||
differs("proposals", ours.to_string(), &theirs["proposals"]);
|
||||
}
|
||||
}
|
||||
|
||||
assert!(
|
||||
wrong.is_empty(),
|
||||
"{} judgements differ:\n{}",
|
||||
wrong.len(),
|
||||
wrong
|
||||
.iter()
|
||||
.take(10)
|
||||
.map(|line| line.chars().take(400).collect::<String>())
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n")
|
||||
);
|
||||
}
|
||||
|
||||
/// Catches a golden file drifting away from a Python that has moved on.
|
||||
#[test]
|
||||
#[cfg(feature = "python")]
|
||||
fn the_golden_file_still_matches_live_python() {
|
||||
let cases = std::fs::read_to_string(parity::dir().join("cases/vocab/lists.ndjson"))
|
||||
.expect("a readable case file");
|
||||
let live = parity::run_oracle("vocab.py", &cases).expect("the oracle runs");
|
||||
assert_eq!(live, parity::read_golden_text("vocab/lists.ndjson"));
|
||||
}
|
||||
Reference in New Issue
Block a user