From 577b3818685fe0f730ed44391f4665019a5f007f Mon Sep 17 00:00:00 2001 From: funkpower Date: Sat, 19 Sep 2026 20:43:16 -0400 Subject: [PATCH] bindings: how to verify a swap, without the scaffolding for one --- crates/bindings/README.md | 31 ++--- tools/shadow.py | 281 -------------------------------------- 2 files changed, 12 insertions(+), 300 deletions(-) delete mode 100755 tools/shadow.py diff --git a/crates/bindings/README.md b/crates/bindings/README.md index 56db696..a581961 100644 --- a/crates/bindings/README.md +++ b/crates/bindings/README.md @@ -162,28 +162,21 @@ it; the gate checks it separately against `wasm32-unknown-unknown`. A binding that returns *nearly* the same answer as the code it replaces is worse than no binding: the pipeline keeps running and the corpus quietly -changes underneath it. `tools/shadow.py` is the check — it runs both -implementations over a directory of real files and reports every place they -disagree. +changes underneath it. -``` -$ tools/shadow.py --stage chunk --corpus /path/to/converted -shadow: chunk — book_pipeline.chunk.chunk_markdown against catena, over 60 files +So before swapping anything, run both implementations over a directory of real +files and compare the answers field by field — not a sample, and not the test +fixtures, which are the inputs somebody already thought of. The things that +find divergences are the documents nobody chose. -shadow: 60 agreed, 0 disagreed, 0 could not be read -``` +Two details worth the trouble: -It writes nothing, touches no database, and exits `1` if anything disagreed, -so it drops into a script. Where the two differ on a long text it says where — -`differ at character 2038 of 2038/1641` — rather than printing two -identical-looking prefixes. - -It also refuses to run against a checkout that has moved on from the commit -the port was made against, because that measures the drift rather than the -port and reports failures that are nobody's bug. `--any-commit` overrides it. - -Run it before the swap and again after. The stages it knows are `chunk`, -`scrub` and `tokens`; adding one is a three-line entry in `STAGES`. +- **Compare against the code you are actually replacing.** A checkout that has + moved on since the port measures the drift rather than the port, and reports + failures that are nobody's bug. +- **Check that the comparison can fail.** Feed one side a deliberately wrong + setting and confirm it is reported. A check that cannot fail proves nothing, + and one that silently compares nothing looks exactly like success. ## As a Rust library diff --git a/tools/shadow.py b/tools/shadow.py deleted file mode 100755 index add6be1..0000000 --- a/tools/shadow.py +++ /dev/null @@ -1,281 +0,0 @@ -#!/usr/bin/env python3 -"""Run a Python pipeline stage and its catena binding over the same corpus, -and report every place they disagree. - -This is the thing to run before swapping a stage, and again after. It writes -nothing, changes nothing, and touches no database: it reads a directory of -real files, runs both implementations over each, and compares the answers -field by field. - - tools/shadow.py --stage chunk --corpus /path/to/converted - tools/shadow.py --stage scrub --corpus /path/to/converted --limit 200 - -Exit codes follow the suite's own: 0 they agree everywhere, 1 they disagree -somewhere, 69 something needed is not installed, 74 the corpus could not be -read. - -The Python side is imported from `book_pipeline`, so PYTHONPATH must point at -its `src`. The catena side is the installed wheel. -""" - -from __future__ import annotations - -import argparse -import json -import subprocess -import sys -from pathlib import Path - -# The commit each stage was ported from. A comparison against a working tree -# that has moved on measures the drift rather than the port, and reports -# failures that are nobody's bug. -PORTED_FROM = "7499e3257572968b437ecfdc79feccc91b96585b" - -OK, DIFFER, UNAVAILABLE, IO = 0, 1, 69, 74 - - -def die(code: int, message: str) -> int: - print(f"shadow: {message}", file=sys.stderr) - return code - - -# --------------------------------------------------------------------------- -# The stages -# --------------------------------------------------------------------------- - - -def chunk_python(text: str) -> list[dict]: - from book_pipeline import chunk as chunker - - return [ - { - "idx": c.chunk_idx, - "page": c.page, - "chapter": c.chapter_title, - "text": c.text, - "tokens": c.token_count, - } - for c in chunker.chunk_markdown(text) - ] - - -def chunk_catena(text: str) -> list[dict]: - import catena - - return [ - { - "idx": c["idx"], - "page": c["page"], - "chapter": c["chapter"], - "text": c["text"], - "tokens": c["tokens"], - } - for c in catena.chunk_markdown(text) - ] - - -def scrub_python(text: str) -> str: - from book_pipeline.convert import _epub_postprocess - - return _epub_postprocess(text) - - -def scrub_catena(text: str) -> str: - import catena - - return catena.scrub(text) - - -def tokens_python(text: str) -> int: - from book_pipeline.chunk import _ntok - - return _ntok(text) - - -def tokens_catena(text: str) -> int: - import catena - - return catena.count_tokens(text) - - -STAGES = { - "chunk": (chunk_python, chunk_catena, "book_pipeline.chunk.chunk_markdown"), - "scrub": (scrub_python, scrub_catena, "book_pipeline.convert._epub_postprocess"), - "tokens": (tokens_python, tokens_catena, "book_pipeline.chunk._ntok"), -} - - -# --------------------------------------------------------------------------- -# Comparing -# --------------------------------------------------------------------------- - - -def differences(theirs, ours) -> list[str]: - """Every way two answers disagree, in a form that says what to look at.""" - if isinstance(theirs, str) or isinstance(theirs, int): - if theirs == ours: - return [] - return [f"python {shorten(theirs)!r} != catena {shorten(ours)!r}"] - - out: list[str] = [] - if len(theirs) != len(ours): - out.append(f"{len(theirs)} records from python, {len(ours)} from catena") - for i, (t, o) in enumerate(zip(theirs, ours)): - for field in t: - if t[field] != o.get(field): - out.append(f"[{i}].{field}: {describe(t[field], o.get(field))}") - return out - - -def describe(theirs, ours) -> str: - """How two values differ, shown where they actually differ. - - Two chunk texts usually share a long opening and part company somewhere in - the middle. Printing the first ninety characters of each shows two - identical-looking strings and says nothing. - """ - if not (isinstance(theirs, str) and isinstance(ours, str)): - return f"python {theirs!r} != catena {ours!r}" - - at = first_difference(theirs, ours) - if at is None: - return f"python {len(theirs)} characters != catena {len(ours)}" - return ( - f"differ at character {at} of {len(theirs)}/{len(ours)}: " - f"python {window(theirs, at)!r} != catena {window(ours, at)!r}" - ) - - -def first_difference(a: str, b: str) -> int | None: - for i, (x, y) in enumerate(zip(a, b)): - if x != y: - return i - return len(a) if len(a) != len(b) else None - - -def window(text: str, at: int, width: int = 40) -> str: - """The text around a position, with a marker for what came before.""" - start = max(0, at - width // 4) - piece = text[start : at + width] - return ("…" if start else "") + piece + ("…" if at + width < len(text) else "") - - -def corpus_files(root: Path, limit: int | None) -> list[Path]: - files = sorted(p for p in root.rglob("*.md") if p.is_file()) - return files[:limit] if limit else files - - -def ported_commit_matches() -> tuple[bool, str]: - """Whether the book_pipeline being imported is the one catena was ported - from.""" - try: - import book_pipeline - except ImportError: - return False, "book_pipeline is not importable" - repo = Path(book_pipeline.__file__).resolve().parents[2] - try: - head = subprocess.run( - ["git", "-C", str(repo), "rev-parse", "HEAD"], - capture_output=True, - text=True, - check=True, - ).stdout.strip() - except (subprocess.CalledProcessError, FileNotFoundError): - return False, f"{repo} is not a git checkout" - if head == PORTED_FROM: - return True, head - return False, f"{repo} is at {head[:8]}, ported from {PORTED_FROM[:8]}" - - -def main() -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--stage", required=True, choices=sorted(STAGES)) - parser.add_argument("--corpus", required=True, type=Path) - parser.add_argument("--limit", type=int, help="Look at this many files") - parser.add_argument( - "--show", type=int, default=5, help="Differences to print per file" - ) - parser.add_argument("--json", action="store_true", help="Emit a record per file") - parser.add_argument( - "--any-commit", - action="store_true", - help="Compare even where the Python has moved since the port", - ) - args = parser.parse_args() - - try: - import catena # noqa: F401 - except ImportError: - return die(UNAVAILABLE, "catena is not installed: pip install catena") - - matches, where = ported_commit_matches() - if not matches: - message = f"the Python is not the port's source — {where}" - if not args.any_commit: - return die( - UNAVAILABLE, - f"{message}\n " - "`git pull` it, or pass --any-commit to compare anyway", - ) - print(f"shadow: {message}", file=sys.stderr) - - if not args.corpus.is_dir(): - return die(IO, f"{args.corpus}: not a directory") - files = corpus_files(args.corpus, args.limit) - if not files: - return die(IO, f"{args.corpus}: no markdown in it") - - theirs_fn, ours_fn, which = STAGES[args.stage] - print( - f"shadow: {args.stage} — {which} against catena, over {len(files)} files", - file=sys.stderr, - ) - - disagreed = 0 - failed = 0 - for path in files: - try: - text = path.read_text(encoding="utf-8", errors="replace") - except OSError as e: - print(f"shadow: {path}: {e}", file=sys.stderr) - failed += 1 - continue - - try: - theirs, ours = theirs_fn(text), ours_fn(text) - except Exception as e: # noqa: BLE001 — one bad file must not stop the run - print(f"shadow: {path}: {type(e).__name__}: {e}", file=sys.stderr) - failed += 1 - continue - - found = differences(theirs, ours) - if args.json: - print( - json.dumps( - { - "path": str(path), - "stage": args.stage, - "agrees": not found, - "differences": found[: args.show], - } - ) - ) - elif found: - print(f"\n{path} ({len(found)} differences)") - for line in found[: args.show]: - print(f" {line}") - if len(found) > args.show: - print(f" … and {len(found) - args.show} more") - if found: - disagreed += 1 - - agreed = len(files) - disagreed - failed - print( - f"\nshadow: {agreed} agreed, {disagreed} disagreed, {failed} could not be read", - file=sys.stderr, - ) - return DIFFER if disagreed else OK - - -if __name__ == "__main__": - raise SystemExit(main())