From 645116d5ed2664b89e5e0159dc78741c74da2b86 Mon Sep 17 00:00:00 2001 From: funkpower Date: Sat, 19 Sep 2026 20:40:15 -0400 Subject: [PATCH] shadow: run both implementations over a corpus and diff them --- crates/bindings/README.md | 27 +++ tools/__pycache__/shadow.cpython-312.pyc | Bin 0 -> 12473 bytes tools/shadow.py | 281 +++++++++++++++++++++++ 3 files changed, 308 insertions(+) create mode 100644 tools/__pycache__/shadow.cpython-312.pyc create mode 100755 tools/shadow.py diff --git a/crates/bindings/README.md b/crates/bindings/README.md index 657ed02..56db696 100644 --- a/crates/bindings/README.md +++ b/crates/bindings/README.md @@ -158,6 +158,33 @@ be the flag. module compiles to nothing, so `--all-features` on a host build does not reach it; the gate checks it separately against `wasm32-unknown-unknown`. +## Swapping a stage over + +A binding that returns *nearly* the same answer as the code it replaces is +worse than no binding: the pipeline keeps running and the corpus quietly +changes underneath it. `tools/shadow.py` is the check — it runs both +implementations over a directory of real files and reports every place they +disagree. + +``` +$ tools/shadow.py --stage chunk --corpus /path/to/converted +shadow: chunk — book_pipeline.chunk.chunk_markdown against catena, over 60 files + +shadow: 60 agreed, 0 disagreed, 0 could not be read +``` + +It writes nothing, touches no database, and exits `1` if anything disagreed, +so it drops into a script. Where the two differ on a long text it says where — +`differ at character 2038 of 2038/1641` — rather than printing two +identical-looking prefixes. + +It also refuses to run against a checkout that has moved on from the commit +the port was made against, because that measures the drift rather than the +port and reports failures that are nobody's bug. `--any-commit` overrides it. + +Run it before the swap and again after. The stages it knows are `chunk`, +`scrub` and `tokens`; adding one is a three-line entry in `STAGES`. + ## As a Rust library The same surface, without a binding in the way: diff --git a/tools/__pycache__/shadow.cpython-312.pyc b/tools/__pycache__/shadow.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7c318fc77301a09b4b455fafb57367ec791903e3 GIT binary patch literal 12473 zcmbt)eQ;A(mgjrY)3;=O_zN2_FJKG`#s&fg10exy2ZzLv*pQCXj3etkV;Na;-g}O* zTJHArcI}#kDnuo-C<9ECNv2~eOf_AT-S$>zrQRu4lO#ayXGv$ow zn(p*J`#bmPNwV>z)3w*uz4zVEbI-l^+;e{S{x6Tm$-%X5`_ag+H*(y+(1+nHdW2uH z@*H=Clem6P;w4j*A2s!xc#4~&=6*8^g?@pBmVOHht^HPnLew^D@3*6@rQac0zb$ZF zzq8-f@9y_V^^)xeLVty{RY4ZoqHddj&qchtZS!K0j#=Hig_~j(c`ICWge5liF}R zCXPkMeknD~q9+Bt|q<42Lv177_;|F)0!|E5=`z6;T^TsnDn_hU3cExZ3Qb z!iqc=S2R&Zfk|;J8Vbu)Zc>yYYUr#Y%Pmf4-*7|~@lW-&VQNl`iwcMsl!xL9+MEcD zjZtz)C!(2A5jq=+fX*RJR$9bvO`K37nyflwan?k$7# zT@HnZb+W?o(Xo(%E@(8Nm^vXVs&gnJM)H9jgAvfA23`qF1J zCd-nHWy9t|jKna^XjGP(#l5>(Z-s$|RJ}O$=#HTDUK&O8CVVks2>)=gu_XNkxvD9;^?yR%no!)o*-|GKZ*pYL2r;jfop9>cU ziw_5Wj1hm22m~_;Wp0$8;xRCOq#(ADRdh+t5MmwNB~#dR7K}Y}kmIIINz;h6SgaU9 zs=fG8j3AXA7R3Y>L8^Os3Vk89;=xqOe9dy%bn%dY{-;d6@Oc)S;DSOAn|~}E3)|i| zEO~Q8I-eKD!0x;avT6)+EH7yCc`a|%;^*X;8nov<%T~8igFf>Wtbu{ika7-!BbImQ z-)K##qkK=Hu>mcjMP+3zrMXy(1K43PEoj!$&?)2$I_1ME*p2xXl>ilX>kW=E`L8r0 zsTDu+IY=hr5khkz6bhlG5LzjeY-}Ret}>B!eIk`LD8~4$)KiSO ztk6NqK`CO6T+aKV!fKJj4?aR4MJjIV+BI1{C zKohoLiN8$B7z^7!Adig?4vfWB4Wc3pNdh)hFI&JZg(FM}qT!%P=NiQe0;jP4*bz2R zSby7){WKNv!K3xJ_t&m<>AnTm#;mYWsY2qil~Sa=89Cp4rSx()Ooc6KV)K-@Fe+{o zh@`1ojmW~J?Lc-f6(M@qqHD5#RAInx{JDLYg#6UAZ0YUNv18&how7S%^H1PE+Pe^~%zUb3o-lwKj znJ-LPk^)jsBE^~%l1wlsEhB!Tl~S0r8m-VMRQ}@H)yE-Ol4j6nOCmyfISS7T^Gc3Md50VuAC(nYc6oCmGL|== zm9@Mbra@SV49YU-r-uCjWjI~SwoHK3z={v87(~Yl1KC*4d*cIu?mYlEhiu1BF{Y?6$bT|{pbY}L>ZoK)e zTcJ72kB4vbzp(tB<+l8xdtv+2+2)??+7lo9PJU_OYHAnFT+OaUJLmRJKdH-95_g89 zoxvoA?;A)ArCeh0t4I_(%mj`k0*nRh|6WcjYRcvNjg+A2`u{+l(4H>InX)}i?MXt? zgneTvD^V^DtYztCHL=giBv7(wq%hf4%Hhj%t~+Tbg#@tW7#XD4MlXk=<1kV6wFDz? z7@9AcXg~+BCSjb0wQ(4g1S`;LD_GclsL9R&k@RX+99Q*1FpE$zgzZMq1=c5E#gVCq8L2p`{yq59Z=1NK!h;qe%){@_Z=I4=MyBe_3lI!97 zR<5Ql?SJhwB<0SlFMm|M`QjrpFTL896naYuD(!-!7@mL==ByD#w zq(Ix0#1QZwH9}Zjvp5j}fENy7j=ARn z)9Vofz&g(H=r+lnfj!I&bqMp7&gcXhTUC<(ulyAr+EksuO>%#2p5T>JC=bBI7EuQn zc4f+&T6X05v1L|`owj7McA!5}-EyV)wpgolg_5aKLcACou z9-nX6k@nv@{FCkOb5uOmdTsdU@v1XM~cU8 zvCZOquLglh3?rH19c9Y=s_>UZvTBfxx`T>em~zP`+0R-e2mRf2zDHXNZX}?G<^D)@ z`IjCy76~aXTD_6aOd~;ZBuzt5yCoMPq>_ikj8sw2A?_`eHPWtIsO+XPbF7h5%si~% zN%I7E-uyf_!6UZ__l=O?Fl|4Ya47Q2+o9d4a$Y#rdAOqwi?|{T&~aRm2cXA~jcYn82zXKspX-9AQD`zG zq77&jNJY7XjnbiJd_l`FQPEr=Q8z$>70P<**454E=v0)rqR@Qj?E`Ep3=HILxEqN_ zUk2z&*Ca7b%{$fc!2)8GH{+f~S563!MILht}PU^@KYdP=l!p7@~Yh5Jg z^(5EH6VLW_b_{eqd*W#vnv5EOuAuV5FkSJGNHtftvdci_iJ*PR`}O&~&Bxa`kJq zi!RRLS@dys_r>pgWUJ44D^qWcQ14w%f8>=uYyW;*BzX92k&e3EGzGl+&4Xjbo&n}s*t>r@239`wv5|f zsd7cRqHHy2#Q9u^l601;BrfUzrq*6Bnr|>KHyEX)HI$3{DbYw7p=Gqf-K;YdKX#4NxFQ>$BU>I!Ge)4UFQaO! zQ4S+mr5#XOzl_ptUtb=r5URPt8Po!Nvi_01uU*!AaowUHI{#NVx^MV4*71~W%0A^- z#?@8JV|4414*2V}okmk701W93*KOZ?CZwtBT(d4I=ng% zZ$4Dyi^sv?Ho%tj82b@l_R`msZ_2BYt;t-{oAecH`oRe@Kvy`Wvow0`F(Y&2aU)g= zwWCHn={9;_ox4fDn@jphYaQt@>XgEIPHMbX1S^e}crKPq`jte98fkOV4bC=|aolf| zM*5cP7UK(R`)hiG?TA~9sX$rFTay9WJEi&;1F#%RGh6Iu^%0!IsFclR?FGMiT{6Z% z{lnMIrQVZ%)s*5dT9Zy`Qvv|hloMZ`WfT>m&!m%DNV>?Msw|^nTe1@EHYc6%w9Hjb|#j z;`sqc#0~c3m`t^pZIq~eN=LMTok4FLC0LDFODO)IS@B^xI+n1a6CiqDkma0M-%$dL z2@_}U6+}EbU^SuCv5GcH9tw>|HRZoh9W*e4HwjQB@peZZ(3EjGQPW9}TtWrNywm{9 zIlyxmn_ktueLJYaV;j-|6KzNJ=R5Syr3_#+-k=OaCm+RwAHZ}G0NM-@Bqfr828nG2 z&D~JOjfXBk-y&ni(~^REKTd}j0g5N}XI6zeltK;S?TreM9bNqM^y^~cF(;H(-MvWa z_}F+f`Xb&fHH&~S#zLx!M?Ixc5Z4%pNg!uz5?2ulwkW5#h*kdm?1@d;Q>R2FB{5Kh z9u&|8MMN8CCOqLE-Y`w z^OCrtDhH{5f{20wy?H2LCV2W_WzUNAn}IyuOXGae#yH| z^&CEZxcjNYN1p0rMnxVE;wB5$hC=ASLfWQ6_hNW2zyO216=(+@q2*28Cm3v?j8Ksj zd55VS47rdrz@CZNPW0+VkE8Q^SZ3Q*-y1NtVnDP82H1T|USM0pB8`uZsk&_?&~D3H zJG#5NI-kv(P8<(b<*jTWc!E+K3Z4V$77<`BLfRG5Kb3Qg3anFL-UCCiyp8UHD1t(f z@NnKyc6oP>!DVL}odxyfyWtg&Mi0rn!K(7l6>p)@`=*Chn5B19=VH68dc^~cl zEq>}xp*I0e@l@aSG|qb(-><#nX`k-O3C_Dh?YvN%cFcBs(Dd_~pSEO$+B-sbDZ@G2 z_kq9?A1x(1Zkn^ZU`V0pQ^nMr-S@hhIh<`ij>{a&n){r|VqLUy7FVGQ`|P^x!9G^$ zR9U5(oOf-`)46D4bp+ILa_+!g*SdMvx&>DQ>_owAIl+HwufA{RYPKy@H~(qR^b<3u zK5+$dRR`xz%{`F~bj;Xb1z!5zE8n|US#!^~Hdk4n3Ig+92S}Ua{m1wvi%?au$O)c` z`%cbLk+P>T+1@}&XQ7+$#^6lB${R`d$S^I&{#ioq* zZsM)Pytr##SU=r8Gm*C7jaSYYNbSC||MLD+b6U>enk@V9<3F#wed@#d+wOnp%G!JH zd3=}pFZBP$0!SV#@du{6bDoOnu6w@f8DY_6aa89jYp)!-d}wy>Tzl%!LS^Sn$D$j_ zpR8@n^j&@I#uK+X-g`RLG1GP5&H2}*yD}XMzTk`nPsuK|zk+f2@A<0euI_VjQ>No> zNA97`@7BLnpKIEd3Ffvm&%Q9L{b=&WWOhsYmkvvH&63&X5B$dL@Kk(R#Z^?Idg@3@ zOAV!uzGKdGzU@pWXCKW5+V8mc=hm!GpUVjEI^J?Dta&&kfZ((#y*u5MI`^?}Bi2UU zw$HtR)Sg!p>5ezM-{}6xyP1kO-+3f6{`US`!cQDOcFb>T&u-j5w>ew$_#NNjPhGy8 zyE^T^(DO?7Vkgh}YwjQ8xe9-(`qln)aNe^iGc@mM25X3|RjJddW9jZp1VDm&?)2QT zY{%)W{kd{>eu;&&;UWqL9@jj+x|Q|b z>6?A%?dR@p-ZQ^>Pqwk`*5I6f&YTS#{P`ol*#CF?KX&&#*w9z%5p3e09TJ|}!Tn-W z<%`wTWODlHIWmRP0Trrl6hnOUZm>g98J7NiEG@@!RNRhC1=7CIB z?7>u${Lo-nbqA{skm^h!MXzI3P8Yj*t{9^lUpeVDYEtY0))(ZzAUz;xFC&UB7W5JP zNijzCjCLaw7sz8r!%ry2DA5N(?@|p?5J%5+de@Nb6k@p69*U76SuaP`5<`uHMl^x`f z-mUB5e$&5j-cLE#r=0&&&iN~D4Frz2ecJU|!^893+G*2t z*Ua&BV99FYeM?nt-n!JlJTt#($!+Ev$g3r9GoC7X*QTG!KHQV_oXAzwQ?xhh>C1W7 zrAM>dk7X;mzqAQX)3kMIO$F~=+UP@LO#;8;eucn)i(jfR@!OZ2mHbouQuA{>zhx2E g_kw#S@sYXi6KB=szSmAKIM-!`b&w)LJsafz1EOM;8~^|S literal 0 HcmV?d00001 diff --git a/tools/shadow.py b/tools/shadow.py new file mode 100755 index 0000000..add6be1 --- /dev/null +++ b/tools/shadow.py @@ -0,0 +1,281 @@ +#!/usr/bin/env python3 +"""Run a Python pipeline stage and its catena binding over the same corpus, +and report every place they disagree. + +This is the thing to run before swapping a stage, and again after. It writes +nothing, changes nothing, and touches no database: it reads a directory of +real files, runs both implementations over each, and compares the answers +field by field. + + tools/shadow.py --stage chunk --corpus /path/to/converted + tools/shadow.py --stage scrub --corpus /path/to/converted --limit 200 + +Exit codes follow the suite's own: 0 they agree everywhere, 1 they disagree +somewhere, 69 something needed is not installed, 74 the corpus could not be +read. + +The Python side is imported from `book_pipeline`, so PYTHONPATH must point at +its `src`. The catena side is the installed wheel. +""" + +from __future__ import annotations + +import argparse +import json +import subprocess +import sys +from pathlib import Path + +# The commit each stage was ported from. A comparison against a working tree +# that has moved on measures the drift rather than the port, and reports +# failures that are nobody's bug. +PORTED_FROM = "7499e3257572968b437ecfdc79feccc91b96585b" + +OK, DIFFER, UNAVAILABLE, IO = 0, 1, 69, 74 + + +def die(code: int, message: str) -> int: + print(f"shadow: {message}", file=sys.stderr) + return code + + +# --------------------------------------------------------------------------- +# The stages +# --------------------------------------------------------------------------- + + +def chunk_python(text: str) -> list[dict]: + from book_pipeline import chunk as chunker + + return [ + { + "idx": c.chunk_idx, + "page": c.page, + "chapter": c.chapter_title, + "text": c.text, + "tokens": c.token_count, + } + for c in chunker.chunk_markdown(text) + ] + + +def chunk_catena(text: str) -> list[dict]: + import catena + + return [ + { + "idx": c["idx"], + "page": c["page"], + "chapter": c["chapter"], + "text": c["text"], + "tokens": c["tokens"], + } + for c in catena.chunk_markdown(text) + ] + + +def scrub_python(text: str) -> str: + from book_pipeline.convert import _epub_postprocess + + return _epub_postprocess(text) + + +def scrub_catena(text: str) -> str: + import catena + + return catena.scrub(text) + + +def tokens_python(text: str) -> int: + from book_pipeline.chunk import _ntok + + return _ntok(text) + + +def tokens_catena(text: str) -> int: + import catena + + return catena.count_tokens(text) + + +STAGES = { + "chunk": (chunk_python, chunk_catena, "book_pipeline.chunk.chunk_markdown"), + "scrub": (scrub_python, scrub_catena, "book_pipeline.convert._epub_postprocess"), + "tokens": (tokens_python, tokens_catena, "book_pipeline.chunk._ntok"), +} + + +# --------------------------------------------------------------------------- +# Comparing +# --------------------------------------------------------------------------- + + +def differences(theirs, ours) -> list[str]: + """Every way two answers disagree, in a form that says what to look at.""" + if isinstance(theirs, str) or isinstance(theirs, int): + if theirs == ours: + return [] + return [f"python {shorten(theirs)!r} != catena {shorten(ours)!r}"] + + out: list[str] = [] + if len(theirs) != len(ours): + out.append(f"{len(theirs)} records from python, {len(ours)} from catena") + for i, (t, o) in enumerate(zip(theirs, ours)): + for field in t: + if t[field] != o.get(field): + out.append(f"[{i}].{field}: {describe(t[field], o.get(field))}") + return out + + +def describe(theirs, ours) -> str: + """How two values differ, shown where they actually differ. + + Two chunk texts usually share a long opening and part company somewhere in + the middle. Printing the first ninety characters of each shows two + identical-looking strings and says nothing. + """ + if not (isinstance(theirs, str) and isinstance(ours, str)): + return f"python {theirs!r} != catena {ours!r}" + + at = first_difference(theirs, ours) + if at is None: + return f"python {len(theirs)} characters != catena {len(ours)}" + return ( + f"differ at character {at} of {len(theirs)}/{len(ours)}: " + f"python {window(theirs, at)!r} != catena {window(ours, at)!r}" + ) + + +def first_difference(a: str, b: str) -> int | None: + for i, (x, y) in enumerate(zip(a, b)): + if x != y: + return i + return len(a) if len(a) != len(b) else None + + +def window(text: str, at: int, width: int = 40) -> str: + """The text around a position, with a marker for what came before.""" + start = max(0, at - width // 4) + piece = text[start : at + width] + return ("…" if start else "") + piece + ("…" if at + width < len(text) else "") + + +def corpus_files(root: Path, limit: int | None) -> list[Path]: + files = sorted(p for p in root.rglob("*.md") if p.is_file()) + return files[:limit] if limit else files + + +def ported_commit_matches() -> tuple[bool, str]: + """Whether the book_pipeline being imported is the one catena was ported + from.""" + try: + import book_pipeline + except ImportError: + return False, "book_pipeline is not importable" + repo = Path(book_pipeline.__file__).resolve().parents[2] + try: + head = subprocess.run( + ["git", "-C", str(repo), "rev-parse", "HEAD"], + capture_output=True, + text=True, + check=True, + ).stdout.strip() + except (subprocess.CalledProcessError, FileNotFoundError): + return False, f"{repo} is not a git checkout" + if head == PORTED_FROM: + return True, head + return False, f"{repo} is at {head[:8]}, ported from {PORTED_FROM[:8]}" + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--stage", required=True, choices=sorted(STAGES)) + parser.add_argument("--corpus", required=True, type=Path) + parser.add_argument("--limit", type=int, help="Look at this many files") + parser.add_argument( + "--show", type=int, default=5, help="Differences to print per file" + ) + parser.add_argument("--json", action="store_true", help="Emit a record per file") + parser.add_argument( + "--any-commit", + action="store_true", + help="Compare even where the Python has moved since the port", + ) + args = parser.parse_args() + + try: + import catena # noqa: F401 + except ImportError: + return die(UNAVAILABLE, "catena is not installed: pip install catena") + + matches, where = ported_commit_matches() + if not matches: + message = f"the Python is not the port's source — {where}" + if not args.any_commit: + return die( + UNAVAILABLE, + f"{message}\n " + "`git pull` it, or pass --any-commit to compare anyway", + ) + print(f"shadow: {message}", file=sys.stderr) + + if not args.corpus.is_dir(): + return die(IO, f"{args.corpus}: not a directory") + files = corpus_files(args.corpus, args.limit) + if not files: + return die(IO, f"{args.corpus}: no markdown in it") + + theirs_fn, ours_fn, which = STAGES[args.stage] + print( + f"shadow: {args.stage} — {which} against catena, over {len(files)} files", + file=sys.stderr, + ) + + disagreed = 0 + failed = 0 + for path in files: + try: + text = path.read_text(encoding="utf-8", errors="replace") + except OSError as e: + print(f"shadow: {path}: {e}", file=sys.stderr) + failed += 1 + continue + + try: + theirs, ours = theirs_fn(text), ours_fn(text) + except Exception as e: # noqa: BLE001 — one bad file must not stop the run + print(f"shadow: {path}: {type(e).__name__}: {e}", file=sys.stderr) + failed += 1 + continue + + found = differences(theirs, ours) + if args.json: + print( + json.dumps( + { + "path": str(path), + "stage": args.stage, + "agrees": not found, + "differences": found[: args.show], + } + ) + ) + elif found: + print(f"\n{path} ({len(found)} differences)") + for line in found[: args.show]: + print(f" {line}") + if len(found) > args.show: + print(f" … and {len(found) - args.show} more") + if found: + disagreed += 1 + + agreed = len(files) - disagreed - failed + print( + f"\nshadow: {agreed} agreed, {disagreed} disagreed, {failed} could not be read", + file=sys.stderr, + ) + return DIFFER if disagreed else OK + + +if __name__ == "__main__": + raise SystemExit(main())