diff --git a/README.md b/README.md index 43b4b43..1af00b4 100644 --- a/README.md +++ b/README.md @@ -19,20 +19,38 @@ cargo install --git https://git.pumpin.online/funkpower/catena Each tool installs by name, so you can take one without taking the rest. +## The tools + +| | | +|---|---| +| [`booksniff`](docs/booksniff.md) | What a file actually is, whatever its name claims | +| [`pdftriage`](docs/pdftriage.md) | Whether a PDF's text layer is worth reading, or whether it has to be OCRed | +| [`bibclean`](docs/bibclean.md) | Catalogue records audited and repaired, with a reason for every change | +| [`mdscrub`](docs/mdscrub.md) | Converted markdown in, clean markdown out | +| [`chunkmd`](docs/chunkmd.md) | Markdown into passages that carry their page, chapter and role | +| [`cuenorm`](docs/cuenorm.md) | Any subtitle format in, one normalised cue stream out | +| [`cuepack`](docs/cuepack.md) | Cues into passages worth embedding, keeping the times they were said at | +| [`cueseg`](docs/cueseg.md) | A timed transcript into topic chapters | +| [`emb`](docs/emb.md) | Lines into vectors, against an embedding server | +| [`rrf`](docs/rrf.md) | Several ranked lists into one, without the database | + +Full documentation, including the libraries underneath, is in +**[docs/](docs/README.md)**. + ## A worked pipeline Converted markdown in, retrievable passages out: ``` $ pandoc -t commonmark chapter.xhtml | mdscrub | chunkmd --min-tokens 20 -{"v":1,"idx":0,"text":"The obligation ran both ways, and contemporaries said so plainly\nwhenever they were asked.4 A lord who could not\nprotect was a lord who could not command…","tokens":57,"page":41,"chapter":"III. The Second Estate","role":"body","start_time":null,"end_time":null,"flags":[]} -{"v":1,"idx":1,"text":"What the rolls do not say, and what has to be inferred, is how often the\nobligation was actually tested…","tokens":41,"page":42,"chapter":"III. The Second Estate","role":"body","start_time":null,"end_time":null,"flags":[]} +{"v":1,"idx":0,"text":"The obligation ran both ways, and contemporaries said so plainly\nwhenever they were asked.4 A lord who could not\nprotect was a lord who could not command…","tokens":56,"page":41,"chapter":"III. The Second Estate","role":"body","start_time":null,"end_time":null,"flags":[]} +{"v":1,"idx":1,"text":"What the rolls do not say, and what has to be inferred, is how often the\nobligation was actually tested…","tokens":143,"page":42,"chapter":"III. The Second Estate","role":"body","start_time":null,"end_time":null,"flags":[]} ``` A recording instead of a book, with chapter marks a scrubber can use: ``` -$ cuenorm episode.vtt | cueseg --duration 7245 +$ cuenorm episode.vtt | cueseg --duration 7200 {"v":1,"start":0.0,"end":1190.6,"title":"The Mill Stood At The Northern End","excerpt":"the mill stood at the northern end of the valley where…"} {"v":1,"start":1190.6,"end":2381.3,"title":"Parliament Dissolved Itself Again","excerpt":"parliament dissolved itself again amid furious objections…"} ``` @@ -49,91 +67,45 @@ garbled, needs OCR $ for f in *.pdf; do pdftriage --quiet "$f" && ocrmypdf "$f" "ocr/$f"; done ``` -## The tools - -| | | -|---|---| -| `bibclean` | Catalogue records audited and repaired, with a reason for every change | -| `booksniff` | What a file actually is, whatever its name claims | -| `mdscrub` | Converted markdown in, clean markdown out: residual HTML, control characters, headings, page markers | -| `chunkmd` | Markdown into passages that carry their page, chapter and role | -| `pdftriage` | Whether a PDF's text layer is healthy, garbled, sparse, permuted, or absent — and so whether OCR is worth its cost | -| `cuenorm` | Any subtitle format in, one normalised cue stream out | -| `cueseg` | A timed transcript into topic chapters | -| `cuepack` | Cues into passages worth embedding, keeping the times they were said at | -| `emb` | Lines into vectors, against an embedding server | -| `rrf` | Several ranked lists into one, without the database | - ## The contract -Every binary in this workspace obeys all of this, or it does not ship. +Every binary in this workspace obeys all of this, or it does not ship. The +whole of it is in [docs/contract.md](docs/contract.md). 1. **One job.** A tool that needs a `--mode` flag to change what it *is* is two tools. 2. **stdin to stdout.** Input on stdin or a path argument, result on stdout, - diagnostics on stderr only — never content and chatter on the same stream. + diagnostics on stderr only. 3. **Streams, not documents.** Anything list-shaped is NDJSON: one record per - line, flushed as it is produced, so a slow stage never blocks the pipe. -4. **Predicates answer with exit codes.** `0` yes, `1` no, `2` could not tell. - Failures use the `sysexits` range — `64` bad usage, `65` bad input, `69` a - missing external program, `70` the tool's own fault, `74` I/O — so "no" and - "broke" are never the same number. A consumer closing the pipe is not a - failure: `… | head -1` exits `0`, like every other filter. -5. **No network, no database, no config file, no state.** Two exceptions, both - declared in `--help`: `emb`, whose whole job is an HTTP call, and a tool - that shells out to a named external program and fails clearly when it is - absent — `pdftriage` needs `pdftotext`. + line, flushed as it is produced. +4. **Predicates answer with exit codes.** `0` yes, `1` no, `2` could not tell; + failures use the `sysexits` range, so "no" and "broke" are never the same + number. +5. **No network, no database, no config file, no state**, with two declared + exceptions. 6. **No ambient knowledge.** No hostnames, no local paths, no private - vocabularies compiled in. General scheme data — Bible book names, ISBN check - digits — may be; anything local is a `--rules file.json`. + vocabularies compiled in. 7. **Deterministic.** Same bytes in, same bytes out, across runs and machines. 8. **`--json` everywhere.** The human-readable default is for a person at a - terminal; `--json` is the contract, and it does not change shape without a - major version. + terminal; `--json` is the contract. ## The records -Three shapes travel between the tools, defined in `corpus-core` and versioned -by a `v` field. +Three shapes travel between the tools, versioned, defined in `corpus-core` and +described in [docs/records.md](docs/records.md). -**`doc`** — what a file is. - -```json -{"v":1,"path":"scan.pdf","kind":"pdf","declared_ext":"pdf","bytes":9261184, - "pages":412,"language":"eng","text_layer":"garbled","needs_ocr":true,"notes":[]} ``` - -**`chunk`** — a retrievable passage with enough provenance to cite it. - -```json -{"v":1,"idx":7,"text":"…","tokens":512,"page":37,"chapter":"III. The Second Estate", - "role":"body","start_time":null,"end_time":null,"flags":[]} -``` - -`page` and `chapter` for documents, `start_time` and `end_time` for -recordings. One shape either way, so a single index serves a book and a -lecture and returns both with a location you can cite. - -**`hit`** — one scored candidate, on its way into a fusion. - -```json -{"v":1,"id":"c81a","channel":"vec","rank":1,"score":0.83,"payload":{}} -``` - -The JSON Schema for all three: - -```sh -cargo run -p corpus-core -- schema all +$ cargo run -p corpus-core -- schema all ``` ## Build ```sh -cargo test -cargo clippy --all-targets -- -D warnings -cargo fmt --check +./check.sh ``` +Formatting, lints, tests and documentation, which is the same gate CI runs. + ## Licence MPL-2.0, library crates and binaries alike. The notice lives in `LICENSE` and diff --git a/check.sh b/check.sh index 18cf180..c5237d2 100755 --- a/check.sh +++ b/check.sh @@ -21,6 +21,13 @@ cargo test --workspace say "docs" RUSTDOCFLAGS="-D warnings" cargo doc --workspace --no-deps --quiet +say "documentation links" +if command -v python3 >/dev/null 2>&1; then + python3 tools/check-links.py +else + echo "skipped: no python3" +fi + # Differential tests against the Python these tools were ported from. Skipped # unless an interpreter with the dependencies is named, since most machines do # not have one. diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000..e470620 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,79 @@ +# catena + +Small programs that each do one part of turning a pile of files into a +searchable corpus, and hand the next one text on stdout. + +Every one of them is a filter: bytes in, records out, diagnostics on stderr, +and an exit code that means something. None of them needs a database, a GPU, a +queue or a config file. + +## The tools + +### Working out what you have + +| | | +|---|---| +| [`booksniff`](booksniff.md) | What a file actually is, whatever its name claims | +| [`pdftriage`](pdftriage.md) | Whether a PDF's text layer is worth reading, or whether it has to be OCRed | +| [`bibclean`](bibclean.md) | Catalogue records audited and repaired, with a reason for every change | + +### Turning documents into passages + +| | | +|---|---| +| [`mdscrub`](mdscrub.md) | Converted markdown in, clean markdown out | +| [`chunkmd`](chunkmd.md) | Markdown into passages that carry their page, chapter and role | + +### Turning recordings into passages + +| | | +|---|---| +| [`cuenorm`](cuenorm.md) | Any subtitle format in, one normalised cue stream out | +| [`cuepack`](cuepack.md) | Cues into passages worth embedding, keeping the times they were said at | +| [`cueseg`](cueseg.md) | A timed transcript into topic chapters | + +### Searching + +| | | +|---|---| +| [`emb`](emb.md) | Lines into vectors, against an embedding server | +| [`rrf`](rrf.md) | Several ranked lists into one, without the database | + +## The libraries underneath + +Every tool is a library crate with a thin command on top, so anything a +command does can be done in process instead. + +| | | +|---|---| +| [`corpus-core`](corpus-core.md) | The record types and the stream plumbing everything shares | +| [`textkit`](textkit.md) | Token counting, sentence and paragraph splitting | +| [`cuekit`](cuekit.md) | The cue model, and the three subtitle formats it is read from | +| [`catena-cli`](catena-cli.md) | The parts of a filter every command does the same way | + +## Reference + +- [The records](records.md) — the shapes that travel between the tools. +- [The contract](contract.md) — what every one of these programs promises. + +## A worked pipeline + +A book, from a converted chapter to passages an index can hold: + +``` +$ mdscrub chapter.md | chunkmd --min-tokens 20 +{"v":1,"idx":0,"text":"The obligation ran both ways, and contemporaries said so plainly…","tokens":56,"page":41,"chapter":"III. The Second Estate","role":"body","start_time":null,"end_time":null,"flags":[]} +{"v":1,"idx":1,"text":"What the rolls do not say, and what has to be inferred…","tokens":143,"page":42,"chapter":"III. The Second Estate","role":"body","start_time":null,"end_time":null,"flags":[]} +``` + +A recording, through the same shape: + +``` +$ cuenorm episode.vtt | cuepack --seconds 10 +{"v":1,"idx":0,"text":"The mill stood at the northern end of the valley, where the fall of the river was steepest and the ice came latest.","tokens":27,"page":null,"chapter":null,"role":"body","start_time":1.0,"end_time":8.0,"flags":[]} +{"v":1,"idx":1,"text":"Its wheel was undershot, which was wasteful,","tokens":11,"page":null,"chapter":null,"role":"body","start_time":11.0,"end_time":19.0,"flags":[]} +``` + +Both produce the same record, so one index answers across a book and a lecture +and returns both with a location you can cite — a page number for one, a +timestamp for the other. diff --git a/docs/bibclean.md b/docs/bibclean.md new file mode 100644 index 0000000..0d99705 --- /dev/null +++ b/docs/bibclean.md @@ -0,0 +1,154 @@ +# bibclean + +Catalogue records audited and repaired, with a reason for every change. + +Back to [the index](README.md). + +## What it does + +Reads catalogue records, names everything wrong with each one, weighs it, and +makes the repairs that need no lookup. + +Nothing is rewritten silently. Every change comes back with the fault that +prompted it, so a run can be read before it is agreed to. + +## When you would use it + +- **Before trusting a catalogue you inherited.** Metadata rots in recognisable + ways, and the ways are worth naming: a title that is really a filename, an + author field holding a publisher, a scanner's name where the work's name + should be. +- **Finding the worst records first.** Every fault carries a weight, so a + thousand records sort by how much attention they deserve and a limited pass + spends its time where it matters. +- **Recovering identifiers you already have.** A file is often named exactly + its ISBN, which is a free and certain identifier for a record that has none + — certain because the check digit says so. +- **As a gate in a pipeline.** It exits `1` when anything needed attention, + so a nightly job can stop and ask. + +## Install + +```sh +cargo install --git https://git.pumpin.online/funkpower/catena bibclean +``` + +## Example + +Five records, four of which have something wrong with them: + +``` +$ bibclean --flagged books.ndjson +{"v":1,"title":"The Human Condition","author":"Arendt, Hannah","isbn":"9780226586601","path":"/library/9780226586601.pdf","reasons":["allcaps_title","missing_isbn"],"severity":2,"changes":[{"field":"title","from":"THE HUMAN CONDITION","to":"The Human Condition","why":"allcaps_title"},{"field":"isbn","from":"","to":"9780226586601","why":"missing_isbn"}]} +{"v":1,"title":"Untitled","author":"Arguing About War","isbn":null,"path":null,"reasons":["placeholder_title","title_in_author","missing_isbn"],"severity":7,"changes":[]} +{"v":1,"title":"Microsoft Word - ch3.doc","author":"Oxford University Press","isbn":null,"path":null,"reasons":["filename_title","junk_title","publisher_as_author","missing_isbn"],"severity":9,"changes":[]} +{"v":1,"title":"The Fall: And the Rise","author":"Camus, Albert","isbn":null,"path":null,"shelf":"upstairs","reasons":["underscore_title","missing_isbn"],"severity":1,"changes":[{"field":"title","from":"The_Fall_ And the Rise_txt","to":"The Fall: And the Rise","why":"underscore_title"}]} +``` + +Three things to notice: the ISBN was recovered from the filename and the +check digit confirmed it; the second record was not touched, because what it +needs is a lookup and guessing would be worse; and `"shelf": "upstairs"` came +through untouched, because fields this does not know about are carried. + +The worst ten records, in order: + +```sh +bibclean --flagged books.ndjson | jq -s 'sort_by(-.severity) | .[:10] | .[]' -c +``` + +Just the changes, for review: + +```sh +bibclean books.ndjson | jq -r '.changes[] | "\(.why)\t\(.from)\t→\t\(.to)"' +``` + +## Options + +| Option | Effect | +|---|---| +| `--flagged` | Write only the records that need attention | + +A missing ISBN alone does not count as needing attention: nearly every book +lacks one. + +## Exit codes + +| Code | Meaning | +|---|---| +| `0` | Nothing needed attention | +| `1` | Something did | +| `64` | The command line was wrong | +| `65` | A record could not be read | +| `74` | The file could not be read | + +## Input + +One JSON record per line. `title`, `author`, `isbn` and `path` are read; +everything else is carried through untouched, so a record can go through this +and back into whatever it came from. + +```json +{"title":"THE HUMAN CONDITION","author":"Arendt, Hannah","path":"/library/9780226586601.pdf"} +``` + +## The faults it names + +| Reason | Weight | What it means | +|---|---|---| +| `placeholder_title` | 4 | The whole title is "Untitled", "Document", "N/A" | +| `placeholder_author` | 4 | The same, in the author field | +| `path_title` | 4 | A file path sitting in the title | +| `numeric_author` | 4 | An author field opening with a bare year or series number | +| `unknown_author` | 3 | "Unknown", "Anonymous", empty | +| `filename_title` | 3 | A bare filename, extension and all | +| `junk_title` | 3 | The name of the tool that made the file, not the work | +| `publisher_as_author` | 3 | An imprint in the author field | +| `title_in_author` | 3 | The title is a placeholder and the real title is in the author field | +| `allcaps_title` | 2 | Shouting | +| `lowercase_title` | 2 | A filename stub: "heidegger areopagite" | +| `missing_apostrophe` | 2 | "Lossky s Argument" | +| `possible_swap` | 2 | Title and author look to be the wrong way round | +| `underscore_title` | 1 | Underscores standing in for spaces and colons | +| `missing_isbn` | 0 | Reported, but not a reason to do anything | + +Each test is deliberately conservative. A real book called "The Document" or +"A Test of Faith" must not be swept up, so the placeholder tests only fire +when the *entire* field is the placeholder; and "Oxford University Press" in +an author field is wrong while "Oxford, Michael" is a person. + +## The repairs it makes + +| Fault | Repair | +|---|---| +| `underscore_title` | A trailing format name comes off, an underscore before a subtitle becomes a colon, the rest become spaces | +| `allcaps_title` | Title-cased, keeping small words small, hyphenated parts cased on both sides, and accented first letters capitalised rather than skipped | +| `missing_isbn` | Filled from the filename, if a checksum-valid ISBN is in it | + +Everything else is named and left alone, because the fix needs a lookup and a +wrong guess in a catalogue is harder to find than a missing value. + +## As a library + +```rust +let cleaned = bibclean::clean(&book); +if bibclean::reasons::worth_fixing(&cleaned.reasons) { + for change in &cleaned.changes { + println!("{}: {} → {} ({})", change.field, change.from, change.to, change.why); + } +} +``` + +The pieces are usable on their own: `bibclean::isbn::valid`, +`bibclean::isbn::from_filename`, `bibclean::fix::title_case`, +`bibclean::matching::same_work`. + +`same_work` is worth knowing about separately. It compares vocabulary rather +than characters, because character similarity is not usable for titles: "The +Cambridge Companion to Aquinas" and "The Cambridge Companion to Hume" are +eighty-six per cent the same string, and believing that attaches one book's +description to another. + +## See also + +- [`booksniff`](booksniff.md) — what the file behind the record actually is. +- [The records](records.md) — the shapes the rest of the suite uses. diff --git a/docs/booksniff.md b/docs/booksniff.md new file mode 100644 index 0000000..27e458e --- /dev/null +++ b/docs/booksniff.md @@ -0,0 +1,118 @@ +# booksniff + +What a file actually is, whatever its name claims. + +Back to [the index](README.md). + +## What it does + +Reads the first few bytes of a file, and an archive's directory at the back, +and says what format it really is. It never reads the whole file, so a +directory of nine-hundred-megabyte scans costs the same as a directory of +anything else. + +Writes one [`doc` record](records.md#doc--what-a-file-is) per file, with a +note for anything worth remarking on: a name that disagrees with the bytes, a +file too small to be a book, a name that says "sample". + +## When you would use it + +- **Before a conversion step**, so a converter is never handed a format it + cannot read. Most conversion failures are this, and the error a converter + gives for it is rarely the error you want. +- **Auditing a library you did not build**, where files have been renamed, + re-saved, and moved between machines that disagreed about extensions. +- **Finding the files that are not books** — the samples, the readmes, the + cover images that ended up in a folder of PDFs. + +A library accumulates files that lie: a DjVu named `.pdf`, a Word document +named `.epub`, an EPUB saved as `.zip`, a single PDF that somebody zipped and +renamed back to `.pdf`. + +## Install + +```sh +cargo install --git https://git.pumpin.online/funkpower/catena booksniff +``` + +## Example + +Three files, one of which is not what it says it is: + +``` +$ booksniff --plain book.pdf atlas.pdf chapter.md +pdf book.pdf +djvu atlas.pdf + named .pdf, but the bytes say DjVu +txt chapter.md + 925 bytes, which is too small to be a book +``` + +The record form, which is what a pipeline wants: + +``` +$ booksniff book.pdf +{"v":1,"path":"book.pdf","kind":"pdf","declared_ext":"pdf","bytes":17595,"pages":null,"language":null,"text_layer":null,"needs_ocr":null,"notes":[]} +``` + +Finding everything in a folder that is not really a PDF: + +```sh +booksniff library/*.pdf | jq -r 'select(.kind != "pdf") | "\(.kind)\t\(.path)"' +``` + +## Options + +| Option | Effect | +|---|---| +| `--plain` | Print a line a person can read instead of the record | + +## Exit codes + +| Code | Meaning | +|---|---| +| `0` | Every file was identified | +| `1` | At least one was not | +| `64` | The command line was wrong | +| `74` | A file could not be read | + +A file that cannot be read is reported on stderr and the rest still run, so +one bad file does not stop a sweep. + +## What it recognises + +| Kind | How | +|---|---| +| `pdf` | `%PDF-` | +| `djvu` | The IFF container tag | +| `ole2` | The compound-file header every binary Word document starts with | +| `epub` | A zip holding `mimetype` and `META-INF/container.xml` | +| `zip-wrapped-pdf` | A zip whose only real member is one PDF | +| `azw3`, `mobi` | A Palm database tagged `BOOKMOBI`, told apart by the format version inside the first record | +| `html` | Markup at the front, after any byte-order mark | +| `txt` | Text that is none of the above | +| `unknown` | Anything else, with a note saying what was seen | + +An archive holding several PDFs is a collection, not a wrapped book: choosing +among them would be a guess, and naming one chapter as the whole work is worse +than refusing. + +## As a library + +```rust +use booksniff::bytes::File; + +let file = File::open(std::path::Path::new("atlas.pdf"))?; +let doc = booksniff::sniff(&file, "atlas.pdf"); +assert_eq!(doc.kind, corpus_core::Kind::Djvu); +``` + +`sniff` works over anything implementing `booksniff::bytes::Bytes`, which is +implemented for `&[u8]` as well as for a file — so a caller that already has +the bytes need not write them to disk first. + +## See also + +- [`pdftriage`](pdftriage.md) — once you know it is a PDF, whether its text is + worth reading. +- [The records](records.md) — the shape this writes. diff --git a/docs/catena-cli.md b/docs/catena-cli.md new file mode 100644 index 0000000..9c0f4f3 --- /dev/null +++ b/docs/catena-cli.md @@ -0,0 +1,121 @@ +# catena-cli + +The parts of a filter every command does the same way. + +Back to [the index](README.md). + +## What it is + +A library. Not a framework — a handful of functions, each of which exists +because getting it wrong is easy and the consequence is a pipeline that +misreports why it stopped. + +Every command in this workspace is built on it, which is why they all behave +the same at the edges. + +## When you would use it + +- **Writing a filter that joins this pipeline**, and wanting it to behave like + the others without rereading [the contract](contract.md). +- **Anywhere a small Unix filter is wanted**, in or out of this workspace. + +## Parsing a command line + +```rust +fn main() -> std::process::ExitCode { + let args: Args = match catena_cli::parse() { + Ok(args) => args, + Err(code) => return code, + }; + // … +} +``` + +`--help` and `--version` are not errors and leave with `0`; anything else +wrong with the arguments leaves with `64`, which is what the rest of the suite +means by "the command line was wrong". Argument parsers tend to pick their own +number for this, and a pipeline that routes on exit codes then has two +different meanings for the same one. + +## Reading the input + +```rust +let text = match catena_cli::read_text(args.path.as_deref()) { + Ok(text) => text, + Err((code, why)) => { + eprintln!("mytool: {why}"); + return ExitCode::from(code); + } +}; +``` + +`read_text` tells a file that cannot be read from one that is not text: the +first is an I/O failure (`74`) and the second is bad input (`65`). A caller +routing on the exit code needs to tell those apart — one is worth retrying and +the other never is. + +`read_lossy` is the same but replaces anything that is not UTF-8, for formats +that routinely arrive in another encoding. A subtitle file downloaded from +anywhere is as likely to be Latin-1, and a cue with one wrong character still +has its timing and most of its words. + +Both read stdin when given no path. + +## Writing the output + +```rust +catena_cli::stream("mytool", &records) // NDJSON, one per line, flushed +catena_cli::text("mytool", &cleaned) // plain text +``` + +Both return the exit code to leave with. Both treat a consumer that closed the +pipe as the end of the job rather than as a failure — `… | head -1` exits `0`, +like every other filter — and both flush as they go, so a slow stage +downstream sees a record as soon as it exists. + +For a loop that needs to do something between records: + +```rust +let mut out = std::io::BufWriter::new(std::io::stdout().lock()); +for record in records { + // … +} +catena_cli::finish("mytool", out) +``` + +## A whole filter + +```rust +use std::process::ExitCode; +use clap::Parser; + +#[derive(Parser)] +#[command(version, about)] +struct Args { + path: Option, +} + +fn main() -> ExitCode { + let args: Args = match catena_cli::parse() { + Ok(args) => args, + Err(code) => return code, + }; + let text = match catena_cli::read_text(args.path.as_deref()) { + Ok(text) => text, + Err((code, why)) => { + eprintln!("mytool: {why}"); + return ExitCode::from(code); + } + }; + catena_cli::stream("mytool", &do_the_work(&text)) +} +``` + +That is the whole shape. Everything a tool here actually *does* lives in its +library crate; the command is argument parsing, input, output and an exit +code, and should stay under a hundred and fifty lines. + +## See also + +- [The contract](contract.md) — what these functions are enforcing. +- [`corpus-core`](corpus-core.md) — the records and the exit-code constants. diff --git a/docs/chunkmd.md b/docs/chunkmd.md new file mode 100644 index 0000000..a8185fd --- /dev/null +++ b/docs/chunkmd.md @@ -0,0 +1,131 @@ +# chunkmd + +Markdown into passages that carry their page, chapter and role. + +Back to [the index](README.md). + +## What it does + +Cuts a document into pieces of about the size an embedding model wants, and +puts on each one the page it came from, the chapter it is in, and what part of +the work it belongs to. + +Writes [`chunk` records](records.md#chunk--a-retrievable-passage), one per +line. + +## When you would use it + +- **Before embedding anything.** A whole chapter is too big to embed usefully + and a sentence is too small to mean anything; this is the step in between. +- **When a result has to be citable.** A passage a reader cannot locate in the + book is a passage they cannot check, so the page and chapter are carried + from the first stage rather than reconstructed later. +- **To keep the apparatus out of search results.** An index, a table of + contents and a bibliography all match a query beautifully and answer + nothing — they are lists of the words somebody is searching for. Marking + them lets a search exclude them while keeping them addressable for the + consumers that want them. + +## Install + +```sh +cargo install --git https://git.pumpin.online/funkpower/catena chunkmd +``` + +## Example + +``` +$ mdscrub chapter.md | chunkmd --min-tokens 20 +{"v":1,"idx":0,"text":"The obligation ran both ways, and contemporaries said so plainly\nwhenever they were asked.4 A lord who could not\nprotect was a lord who could not command, and the men who owed him service\nsaid as much in language that has survived in the court rolls.","tokens":56,"page":41,"chapter":"III. The Second Estate","role":"body","start_time":null,"end_time":null,"flags":[]} +{"v":1,"idx":1,"text":"What the rolls do not say, and what has to be inferred, is how often the\nobligation was actually tested…","tokens":143,"page":42,"chapter":"III. The Second Estate","role":"body","start_time":null,"end_time":null,"flags":[]} +``` + +Just the text, for an embedder: + +```sh +mdscrub chapter.md | chunkmd | jq -r .text | emb --model bge-m3 --dim 1024 +``` + +Only the body, leaving the bibliography and the index behind: + +```sh +chunkmd book.md | jq -c 'select(.role == "body")' +``` + +## Options + +| Option | Default | Effect | +|---|---|---| +| `--target ` | `512` | Flush a chunk at this many tokens | +| `--overlap ` | `64` | Carry this much of each chunk into the next | +| `--min-tokens ` | `50` | Drop chunks smaller than this | +| `--hard-max ` | `768` | Never emit a chunk larger than this | + +The defaults suit a model with a 512-token comfortable window. A larger model +wants roughly double everything: + +```sh +chunkmd --target 1024 --overlap 128 --min-tokens 80 --hard-max 1280 +``` + +Budgets that cannot produce anything sensible — an overlap at or above the +target, a hard maximum below the target — are refused before any input is +read. + +## Exit codes + +| Code | Meaning | +|---|---| +| `0` | Done | +| `64` | The command line was wrong | +| `65` | The input was not text | +| `74` | The file could not be read | + +## How it cuts + +Page first, paragraph second, sentence only when a paragraph is itself too +big. + +1. **Split on page markers**, so a chunk never spans a page and its page + number is exact rather than inferred. Text before the first marker is kept, + with no page number — it is still the book. +2. **Pack paragraphs** toward the token target within each page, carrying an + overlap forward so a sentence that answers a question is not orphaned from + the sentence that asked it. +3. **Enforce the hard maximum** on token boundaries, whatever the structure + said. +4. **Drop what is too small** to be worth an embedding. +5. **Classify roles** across the whole document at once, because deciding a + stray chunk needs to see its neighbours: a short run of body chunks + sandwiched between two bibliography chunks is part of the bibliography. + +A chunk whose text is a page of numbered notes is flagged `footnote_citations`, +because footnotes are citations in disguise — they read as body text, rank as +body text, and answer a query with a publisher and a page number. + +## The roles + +| Role | What it is | +|---|---| +| `body` | Running text | +| `bibliography` | A works-cited or references list | +| `index` | A back-of-book index | +| `toc` | A table of contents | +| `frontmatter` | Title page, copyright page, dedication | + +## As a library + +```rust +use chunkmd::Budgets; + +for chunk in chunkmd::chunk(&markdown, &Budgets::default()) { + println!("{} tokens on page {:?}", chunk.tokens, chunk.page); +} +``` + +## See also + +- [`mdscrub`](mdscrub.md) — what to run before this. +- [`cuepack`](cuepack.md) — the same record, from a recording. +- [`emb`](emb.md) — turning the text into vectors. +- [The records](records.md) — the shape this writes. diff --git a/docs/contract.md b/docs/contract.md new file mode 100644 index 0000000..aaf36ca --- /dev/null +++ b/docs/contract.md @@ -0,0 +1,78 @@ +# The contract + +Every program here obeys all of this, or it does not ship. It is worth reading +once, because it is what lets these be used together without reading any of +them. + +Back to [the index](README.md). + +## One job + +A tool that needs a `--mode` flag to change what it *is* is two tools. + +## stdin to stdout + +Input on stdin or a path argument, result on stdout, diagnostics on stderr +only — never content and chatter on the same stream. A tool that takes a path +instead of a stream says why in its `--help`; [`pdftriage`](pdftriage.md) does, +because it reads two specific page ranges rather than the whole file. + +## Streams, not documents + +Anything list-shaped is NDJSON: one [record](records.md) per line, flushed as +it is produced, so a slow stage never blocks the pipe behind it. + +The reader is strict on purpose. It refuses a record whose version it does not +know, it stops at the first bad line rather than reading past a disagreement +about the format, and it bounds how much one line may cost — a filter pointed +at a binary file should fail, not allocate the file. + +## Predicates answer with exit codes + +| Code | Meaning | +|---|---| +| `0` | Yes, or done | +| `1` | No | +| `2` | Could not tell | +| `64` | The command line was wrong | +| `65` | The input was not what the tool can work on | +| `69` | A program it needs is not installed | +| `70` | The tool's own fault | +| `74` | Reading or writing failed | + +The failure codes start where `sysexits.h` puts them so that "no" and "broke" +are never the same number. That is what makes this work: + +```sh +for f in *.pdf; do pdftriage --quiet "$f" && ocrmypdf "$f" "ocr/$f"; done +``` + +A consumer closing the pipe is not a failure. `… | head -1` exits `0`, like +every other filter. + +## No network, no database, no config file, no state + +Two exceptions, both declared in `--help`: + +- [`emb`](emb.md), whose whole job is an HTTP call. +- [`pdftriage`](pdftriage.md), which shells out to `pdftotext` and `pdfinfo` + and exits `69` when either is missing. + +## No ambient knowledge + +No hostnames, no local paths, no private vocabularies compiled in. General +scheme data — ISBN check digits, the HTML character references — may be; +anything local is a `--rules` file. + +## Deterministic + +Same bytes in, same bytes out, across runs and machines. No hash-order +dependence, no clock, no locale reads. Floats that JSON cannot carry are +refused where they are produced rather than written as `null` and lost. + +## `--json` everywhere + +The human-readable default is for a person at a terminal; `--json` is the +contract, and it does not change shape without a major version. Tools whose +only sensible output is a stream of records write that stream and have no +`--json` to add. diff --git a/docs/corpus-core.md b/docs/corpus-core.md new file mode 100644 index 0000000..3e403ac --- /dev/null +++ b/docs/corpus-core.md @@ -0,0 +1,142 @@ +# corpus-core + +The record types and the stream plumbing everything shares. + +Back to [the index](README.md). + +## What it is + +A library, not a command — though it carries a small one for printing +schemas. It holds the three [record types](records.md), the NDJSON reader and +writer, and the exit-code conventions. + +Everything else in the workspace depends on it, which is what makes the tools +composable: they agree about the records before they agree about anything +else. + +## When you would use it + +- **Reading or writing these records in your own program**, without shelling + out to any of the tools. +- **Generating bindings in another language**, from the JSON Schema. +- **Writing a new filter** that joins the pipeline. + +## The records + +[`Doc`](records.md#doc--what-a-file-is), +[`Chunk`](records.md#chunk--a-retrievable-passage) and +[`Hit`](records.md#hit--one-scored-candidate). Each derives `Serialize`, +`Deserialize` and `JsonSchema`, and implements `Record`, which is how a reader +knows what version it is looking at. + +```rust +use corpus_core::{Chunk, Kind, Doc}; + +let mut chunk = Chunk::new(0, "the mill stood at the northern end", 8); +chunk.page = Some(41); +chunk.chapter = Some("III. The Second Estate".into()); +chunk.flag(corpus_core::chunk::flags::FOOTNOTE_CITATIONS); + +let doc = Doc::new("scan.pdf", Kind::Pdf, 9_261_184); +``` + +Every field is emitted, including the empty ones, so a consumer can rely on +the shape rather than on what a particular producer happened to fill in. + +## Reading and writing a stream + +```rust +use corpus_core::ndjson; + +for record in ndjson::read::(std::io::stdin().lock()) { + let chunk = record?; + println!("{} tokens", chunk.tokens); +} + +ndjson::write_line(&mut std::io::stdout(), &chunk)?; +``` + +The reader is strict, and each strictness is there because the alternative is +a silent failure: + +- **A record from a version it does not know is refused.** A reader that + parses a newer record into whatever lines up will process truncated records + rather than failing. +- **It stops at the first bad line.** A bad line means the producer and the + consumer disagree about the format, and reading on buries that. +- **Lines are bounded.** A filter pointed at a binary file should fail, not + allocate the file. The limit is sixteen megabytes and can be changed with + `read_with_limit`. +- **Blank lines are skipped and a byte-order mark is ignored**, so a stream + that was concatenated or written on Windows still reads. +- **Every record is flushed as it is written**, so a slow stage downstream + sees a record as soon as it exists. + +An error knows whether it was a closed pipe: + +```rust +match ndjson::write_line(&mut out, &chunk) { + Err(e) if e.is_broken_pipe() => return Ok(()), // `| head` is normal + other => other?, +} +``` + +## Floats JSON cannot carry + +JSON has no spelling for NaN or infinity, and the usual behaviour is to write +them as `null` — which is a silent corruption in both directions: a score +written as `null` fails to read back and kills the stream at the consumer, and +a timestamp written as `null` reads back as "no timestamp" with nothing to say +it was ever anything else. + +So a non-finite number is refused where it is produced, which is the only +place anyone can still fix it. Use the same guard on your own records: + +```rust +#[derive(Serialize, Deserialize)] +struct Mine { + #[serde(with = "corpus_core::finite")] + score: f64, + #[serde(default, with = "corpus_core::finite::option")] + when: Option, +} +``` + +## Exit codes + +```rust +use corpus_core::exit; + +std::process::ExitCode::from(exit::OK) // 0 +std::process::ExitCode::from(exit::NO) // 1 +std::process::ExitCode::from(exit::INDETERMINATE) // 2 +std::process::ExitCode::from(exit::USAGE) // 64 +``` + +`Verdict` is the three-answer form for a predicate: + +```rust +use corpus_core::exit::Verdict; + +let answer = Verdict::from(needs_ocr); +std::process::ExitCode::from(answer) +``` + +See [the contract](contract.md#predicates-answer-with-exit-codes) for the +whole table. + +## The schemas + +``` +$ cargo run -p corpus-core -- schema chunk > chunk.schema.json +$ cargo run -p corpus-core -- schema all +``` + +`doc`, `chunk`, `hit` or `all`. The output is byte-identical across runs, so +it can be committed and diffed. + +## See also + +- [The records](records.md) — the shapes, described. +- [`catena-cli`](catena-cli.md) — the rest of what a filter needs. +- [`textkit`](textkit.md) — counting and splitting text. diff --git a/docs/cuekit.md b/docs/cuekit.md new file mode 100644 index 0000000..f27b66e --- /dev/null +++ b/docs/cuekit.md @@ -0,0 +1,104 @@ +# cuekit + +The cue model, and the three subtitle formats it is read from. + +Back to [the index](README.md). + +## What it is + +A library. Three formats go in — SubRip, WebVTT and SubStation Alpha — and +one [`cue`](records.md#cue--one-line-of-a-transcript) stream comes out. +Everything downstream works on cues and never has to know which of the three a +recording arrived with. + +[`cuenorm`](cuenorm.md) is a command over this. + +## When you would use it + +- **Reading subtitles in your own program**, without shelling out. +- **Adding a fourth format**, or handling one of these three differently. +- **Anywhere timed text is the input** and the format is whatever somebody + happened to have. + +## Reading a file + +```rust +let cues = cuekit::parse(&content); +``` + +The format is worked out from the content, not from the name, because a +subtitle file is named for the format somebody expected it to be in. When it +is already known: + +```rust +use cuekit::Format; + +let cues = cuekit::parse_as(&content, Format::WebVtt); +assert_eq!(cuekit::detect(&content), Format::WebVtt); +``` + +The parsers are also available directly — `cuekit::srt::parse` (which reads +WebVTT too) and `cuekit::ass::parse`. + +## The cue + +```rust +use cuekit::Cue; + +let cue = Cue::new(12.5, 15.0, "the mill stood at the northern end"); +assert_eq!(cue.seconds(), 2.5); +``` + +Deliberately smaller than a [chunk](records.md#chunk--a-retrievable-passage). +A cue is what a subtitle file holds — a few seconds, often half a sentence — +and turning cues into passages worth retrieving is a later stage's job. + +`seconds()` is negative if the file says a cue ends before it starts, which +some do. + +## Merging the repeats + +```rust +let merged = cuekit::merge_repeats(&cues); +``` + +Consecutive cues with identical text become one, keeping the earliest start +and the latest end. + +Automatic captioning emits a rolling window: the same line appears in cue +after cue as the window slides, so a ten-minute stretch of speech can arrive +as three times as many cues as there are sentences. Indexed as they come, +every one of those is a separate hit for the same words at almost the same +timestamp. + +## Cleaning a cue's text + +```rust +assert_eq!(cuekit::text::clean("[Music] The mill stood"), "The mill stood"); +``` + +Styling and speaker tags go; bracketed sound cues — `[Music]`, `[Applause]`, +`[inaudible]` — go with them, because they are not speech and a search that +matches them matches every recording with a theme tune. + +## Timestamps + +```rust +assert_eq!(cuekit::srt::timestamp("00:01:23,456"), 83.456); +assert_eq!(cuekit::ass::timestamp("0:01:23.45"), 83.45); +``` + +Both return `NaN` rather than an error for something that is not a timestamp, +which is what lets a malformed block be dropped whole instead of guessed at. + +`cuekit::jsnum` holds the number parsing these use. It reproduces the two +JavaScript conversions the original parser was written against, which disagree +about almost everything an ill-formed timestamp can contain — an empty string +is zero to one of them and not a number to the other. Reproducing both is what +makes a malformed file drop the same cues here as there. + +## See also + +- [`cuenorm`](cuenorm.md) — the command over this. +- [`cuepack`](cuepack.md) and [`cueseg`](cueseg.md) — what reads cues. +- [The records](records.md) — the cue shape on the wire. diff --git a/docs/cuenorm.md b/docs/cuenorm.md new file mode 100644 index 0000000..d23f852 --- /dev/null +++ b/docs/cuenorm.md @@ -0,0 +1,132 @@ +# cuenorm + +Any subtitle format in, one normalised cue stream out. + +Back to [the index](README.md). + +## What it does + +Reads a subtitle file — SubRip, WebVTT or SubStation Alpha — and writes one +[`cue` record](records.md#cue--one-line-of-a-transcript) per line: a start, an +end, and what was said. + +The format is worked out from the content, not from the name, because a +subtitle file is named for the format somebody expected it to be in. + +## When you would use it + +- **At the front of anything that reads recordings.** Everything downstream + works on cues and never has to know which of the three formats a recording + arrived with. +- **Cleaning up automatic captioning.** It repeats each line across + overlapping windows, so the same sentence arrives three or four times over; + indexed as they come, every one is a separate hit for the same words at + almost the same timestamp, which buries everything else in a result list. +- **Getting markup and sound cues out of the way.** `` tags, + styling, `[Music]`, `[Applause]` — none of it is speech, and a search that + matches `[Music]` matches every recording with a theme tune. + +## Install + +```sh +cargo install --git https://git.pumpin.online/funkpower/catena cuenorm +``` + +## Example + +A WebVTT file with a speaker tag, a sound cue, and a repeated line: + +``` +$ cat episode.vtt +WEBVTT + +00:00:01.000 --> 00:00:04.500 +The mill stood at the northern end of the valley, + +00:00:04.500 --> 00:00:08.000 +where the fall of the river was steepest and the ice came latest. + +00:00:08.000 --> 00:00:11.000 +[Music] + +00:00:11.000 --> 00:00:15.500 +Its wheel was undershot, which was wasteful, + +00:00:15.000 --> 00:00:19.000 +Its wheel was undershot, which was wasteful, + +00:00:19.000 --> 00:00:24.000 +and nobody replaced it for two hundred years. +``` + +``` +$ cuenorm episode.vtt +{"v":1,"start":1.0,"end":4.5,"text":"The mill stood at the northern end of the valley,"} +{"v":1,"start":4.5,"end":8.0,"text":"where the fall of the river was steepest and the ice came latest."} +{"v":1,"start":11.0,"end":19.0,"text":"Its wheel was undershot, which was wasteful,"} +{"v":1,"start":19.0,"end":24.0,"text":"and nobody replaced it for two hundred years."} +``` + +The speaker tag is gone, `[Music]` left no cue at all, and the repeated line +became one cue spanning both — `11.0` to `19.0`, the earliest start and the +latest end. + +In a pipeline: + +```sh +cuenorm episode.vtt | cuepack --seconds 75 > passages.ndjson +cuenorm episode.srt | cueseg --duration 7245 > chapters.ndjson +``` + +## Options + +| Option | Effect | +|---|---| +| `--keep-repeats` | Keep consecutive repeats instead of merging them | + +## Exit codes + +| Code | Meaning | +|---|---| +| `0` | Done | +| `64` | The command line was wrong | +| `74` | The file could not be read | + +Bytes that are not UTF-8 are replaced rather than refused. A subtitle file +downloaded from anywhere is as likely to be Latin-1, and a cue with one wrong +character still has its timing and most of its words. + +## The formats + +| Format | Recognised by | +|---|---| +| WebVTT | The `WEBVTT` line at the front | +| SubStation Alpha | A `[Script Info]` or `[Events]` section | +| SubRip | Neither of the above | + +SubStation Alpha events are rows under a `Format:` line that names the +columns, so the column order is read rather than assumed. Override blocks +(`{\an8}`), line breaks (`\N`) and drawing commands are handled: a drawing +command is a shape, not speech, and reads as a string of coordinates if it is +kept. + +A WebVTT cue line carries positioning settings after the end time, which are +ignored — that is also what lets an old SubRip file with screen coordinates +after its timestamps read at all. + +## As a library + +```rust +let cues = cuekit::merge_repeats(&cuekit::parse(&content)); +for cue in &cues { + println!("{:.1}–{:.1} {}", cue.start, cue.end, cue.text); +} +``` + +See [`cuekit`](cuekit.md) for the parsers one at a time. + +## See also + +- [`cuepack`](cuepack.md) — cues into passages worth embedding. +- [`cueseg`](cueseg.md) — cues into topic chapters. +- [`cuekit`](cuekit.md) — the library this is a command over. diff --git a/docs/cuepack.md b/docs/cuepack.md new file mode 100644 index 0000000..dc5391e --- /dev/null +++ b/docs/cuepack.md @@ -0,0 +1,105 @@ +# cuepack + +Cues into passages worth embedding, keeping the times they were said at. + +Back to [the index](README.md). + +## What it does + +Packs consecutive cues into passages of about a minute of continuous speech, +and writes each one as a [`chunk` +record](records.md#chunk--a-retrievable-passage) carrying the start of its +first cue and the end of its last. + +## When you would use it + +- **Before embedding a transcript.** A cue is a few seconds of speech and + often half a sentence; an embedding of one says almost nothing, and a search + over them returns fragments. +- **When a result has to point somewhere.** Because the passage keeps the + first cue's start and the last cue's end, a hit can be cited to the second + and a player can jump to it. +- **To index recordings and books together.** The record is the same one + [`chunkmd`](chunkmd.md) writes, so one index answers across both and returns + each with a location: a page number for one, a timestamp for the other. + +## Install + +```sh +cargo install --git https://git.pumpin.online/funkpower/catena cuepack +``` + +## Example + +``` +$ cuenorm episode.vtt | cuepack --seconds 10 +{"v":1,"idx":0,"text":"The mill stood at the northern end of the valley, where the fall of the river was steepest and the ice came latest.","tokens":27,"page":null,"chapter":null,"role":"body","start_time":1.0,"end_time":8.0,"flags":[]} +{"v":1,"idx":1,"text":"Its wheel was undershot, which was wasteful,","tokens":11,"page":null,"chapter":null,"role":"body","start_time":11.0,"end_time":19.0,"flags":[]} +{"v":1,"idx":2,"text":"and nobody replaced it for two hundred years.","tokens":9,"page":null,"chapter":null,"role":"body","start_time":19.0,"end_time":24.0,"flags":[]} +``` + +Straight to an embedder: + +```sh +cuenorm episode.vtt | cuepack | jq -r .text | emb --model bge-m3 --dim 1024 +``` + +It also reads a subtitle file directly, so the middle step can be skipped when +nothing else needs the cues: + +```sh +cuepack episode.srt > passages.ndjson +``` + +## Options + +| Option | Default | Effect | +|---|---|---| +| `--seconds ` | `75` | Flush a passage once it covers this much audio | +| `--max-tokens ` | `512` | Never let a passage exceed this many tokens | + +Seventy-five seconds is about a paragraph of speech: long enough to hold an +argument, short enough that a result points at a place in the recording rather +than at a region of it. + +## Exit codes + +| Code | Meaning | +|---|---| +| `0` | Done | +| `64` | The command line was wrong | +| `65` | The cue stream could not be read | +| `74` | The file could not be read | + +## How it packs + +A passage is flushed when the window from its first cue's start to the current +cue's end reaches the seconds budget, or when adding the current cue would +take it past the token budget. The cue that trips the budget starts the next +passage rather than finishing this one. + +Immediately repeated lines are dropped rather than packed. Automatic +captioning repeats each line across overlapping windows, and a passage padded +with its own echo embeds as something nobody said. + +## Input + +Either the cue stream [`cuenorm`](cuenorm.md) writes, or a subtitle file. A +cue stream begins with a JSON object and a subtitle file does not, which is +enough to tell them apart. + +## As a library + +```rust +use cuepack::Budgets; + +for passage in cuepack::pack(&cues, &Budgets::default()) { + println!("{:?}–{:?}: {}", passage.start_time, passage.end_time, passage.text); +} +``` + +## See also + +- [`cuenorm`](cuenorm.md) — what to run before this. +- [`cueseg`](cueseg.md) — the other thing to do with a cue stream. +- [`chunkmd`](chunkmd.md) — the same record, from a book. diff --git a/docs/cueseg.md b/docs/cueseg.md new file mode 100644 index 0000000..f087bd1 --- /dev/null +++ b/docs/cueseg.md @@ -0,0 +1,119 @@ +# cueseg + +A timed transcript into topic chapters. + +Back to [the index](README.md). + +## What it does + +Finds where a recording changes subject, and writes one [`segment` +record](records.md#segment--one-topic-chapter) per chapter: a start, an end, a +title taken from the chapter's own first sentence, and an excerpt. + +No model and no network. The text is hashed into a small lexical vector a +window at a time, and a topic boundary is where two consecutive windows stop +resembling each other. + +## When you would use it + +- **Chapter marks for a player.** A three-hour recording with no marks is + unnavigable; the boundaries this finds are exactly the marks a scrubber + needs. +- **Feeding a summariser one subject at a time.** A whole transcript is too + much context and a paragraph is not enough; a chapter is the unit that gives + a useful answer. +- **Navigation in a search result.** Knowing which chapter a hit falls in is + often more useful than the timestamp alone. +- **Titles before anybody has written any.** The extracted titles are meant to + be replaced by something better later, and to be usable until they are. + +## Install + +```sh +cargo install --git https://git.pumpin.online/funkpower/catena cueseg +``` + +## Example + +A two-hour recording of two subjects: + +``` +$ cueseg --duration 7200 transcript.ndjson +{"v":1,"start":0.0,"end":1190.6666666666667,"title":"The Mill Stood At The Northern End Of","excerpt":"the mill stood at the northern end of the valley where the fall of the river was steepest and the ice came latest…"} +{"v":1,"start":1190.6666666666667,"end":2381.3333333333335,"title":"On One Household And The Benefit Was Spread","excerpt":"on one household and the benefit was spread across four hundred families…"} +{"v":1,"start":2381.3333333333335,"end":3572.0,"title":"Stood At The Northern End Of The Valley","excerpt":"stood at the northern end of the valley where the fall of the river was steepest…"} +{"v":1,"start":3572.0,"end":4479.0,"title":"Steepest And The Ice Came Latest Its Wheel","excerpt":"steepest and the ice came latest its wheel was undershot which was wasteful…"} +{"v":1,"start":4479.0,"end":5386.0,"title":"It From The Journals Of Either House Parliament","excerpt":"it from the journals of either house parliament dissolved itself again amid furious objections…"} +{"v":1,"start":5386.0,"end":6293.0,"title":"At All Which Left The Committee With No","excerpt":"at all which left the committee with no record of the vote…"} +{"v":1,"start":6293.0,"end":7200.0,"title":"House Parliament Dissolved Itself Again Amid Furious Objections","excerpt":"house parliament dissolved itself again amid furious objections from every bench…"} +``` + +Straight from a subtitle file: + +```sh +cuenorm episode.vtt | cueseg --duration 7245 > chapters.ndjson +``` + +As chapter marks for a player: + +```sh +cueseg --duration 7245 transcript.ndjson | jq -r '"\(.start | floor)\t\(.title)"' +``` + +## Options + +| Option | Effect | +|---|---| +| `--duration ` | How long the recording is | + +Without it, the duration is taken from the end of the last cue. That is +earlier than the end of the file whenever a recording ends in music or +silence, and a chapter that claims to run to the end of the transcript rather +than the end of the file is a mark that lands in the wrong place. + +## Exit codes + +| Code | Meaning | +|---|---| +| `0` | Done | +| `64` | The command line was wrong, or the duration was not a length | +| `65` | The cue stream could not be read | +| `74` | The file could not be read | + +## How it finds a boundary + +1. **Windows.** Cues are gathered into stretches of about seventy-five + seconds — long enough to have a subject, short enough that a boundary + lands within a minute of the real one. +2. **Vectors.** Each window's content words are hashed into a small vector of + counts. Filler, discourse markers and the vague nouns that carry a + conversation are dropped: leaving them in makes every window resemble every + other. +3. **Valleys.** A boundary is where two consecutive windows resemble each + other less than their neighbours do, and less than the recording's own + average by half a standard deviation. Valleys are taken deepest first, each + at least a chapter's length from the ones already taken. +4. **A floor and a ceiling.** Between four and twenty chapters, targeting one + every fifteen minutes. A chapter mark every two minutes is not navigation, + it is noise; and a chapter that runs too long is no more navigable than no + chapter at all, so an over-long span is divided evenly whether or not the + text suggested a break there. +5. **A fallback.** A recording where nothing stands out is cut evenly, which + is better than one chapter of three hours. + +## Input + +Either the cue stream [`cuenorm`](cuenorm.md) writes, or a subtitle file. + +## As a library + +```rust +for chapter in cueseg::segment(&cues, 7245.0) { + println!("{:.0}s {}", chapter.start, chapter.title); +} +``` + +## See also + +- [`cuenorm`](cuenorm.md) — what to run before this. +- [`cuepack`](cuepack.md) — the other thing to do with a cue stream. diff --git a/docs/emb.md b/docs/emb.md new file mode 100644 index 0000000..bf85d78 --- /dev/null +++ b/docs/emb.md @@ -0,0 +1,129 @@ +# emb + +Lines into vectors, against an embedding server. + +Back to [the index](README.md). + +## What it does + +Sends each line of its input to an HTTP embedding endpoint and writes the +vector that comes back, one JSON array per line, in the order the lines +arrived. + +It never loads a model. An embedding model is half a gigabyte of weights and a +GPU, which belongs behind a server where it can be shared — what this owns is +everything around the call: batching, a defensive truncation, a retry, and the +assertion that the vectors are the width the index expects. + +## When you would use it + +- **At the end of an ingest pipeline**, where passages become rows. +- **Re-embedding an existing corpus** after changing model, where the thing + that matters is that every vector is the same width as the last. +- **Anywhere a GPU is somebody else's problem.** One server, many machines + running this. + +## Install + +```sh +cargo install --git https://git.pumpin.online/funkpower/catena emb +``` + +## Example + +``` +$ emb --host http://127.0.0.1:11434 --model bge-m3 --dim 4 lines.txt +[0.0123,-0.0456,0.0789,0.0012] +[0.0123,-0.0456,0.0789,0.0012] +``` + +Asking for a width the server does not produce is refused, rather than +half-filling an index: + +``` +$ emb --host http://127.0.0.1:11434 --model bge-m3 --dim 1024 lines.txt +emb: vectors are 4 wide, expected 1024 +$ echo $? +65 +``` + +At the end of a pipeline: + +```sh +mdscrub chapter.md | chunkmd | jq -r .text \ + | emb --model bge-m3 --dim 1024 > vectors.ndjson +``` + +Because the output lines up with the input line for line, the two can be put +back together: + +```sh +paste -d'\t' <(chunkmd book.md | jq -c .) vectors.ndjson +``` + +## Options + +| Option | Default | Effect | +|---|---|---| +| `--host ` | `http://localhost:11434` | The server to ask | +| `--model ` | required | The model to ask for | +| `--dim ` | — | Refuse vectors that are not this wide | +| `--batch ` | `32` | Texts per request | +| `--max-input-tokens ` | `1800` | Cut each text to this many tokens before sending it; `0` sends them whole | +| `--timeout ` | `600` | How long to wait for one request | +| `--attempts ` | `3` | How many times to try a failing request | + +The input cap is a defensive one, not a model limit. A server returns an error +for an input past its context length, and that error fails the whole batch — +so one oversized passage costs thirty-one good ones. + +The timeout is generous because a cold server loading a model onto a GPU can +take minutes before it answers at all. + +## Exit codes + +| Code | Meaning | +|---|---| +| `0` | Every line was embedded | +| `64` | The command line was wrong | +| `65` | The server answered something that was not an embedding, or was the wrong width | +| `69` | The server could not be reached | +| `74` | The input could not be read | + +A width mismatch and a missing vector are refused immediately and not retried: +the server answered, it just did not answer the question. + +## The request + +`POST {host}/api/embed`, with: + +```json +{"model": "bge-m3", "input": ["the mill stood", "at the northern end"]} +``` + +and expecting: + +```json +{"embeddings": [[0.0123, -0.0456, …], [0.0021, 0.0310, …]]} +``` + +which is the shape Ollama uses and several other servers speak. + +## Empty lines + +Sent as they are. Dropping them would make the output stop lining up with the +input, which is the one property that makes this composable. + +## As a library + +```rust +let endpoint = emb::Endpoint { dim: Some(1024), ..emb::Endpoint::new(host, "bge-m3") }; +let agent = ureq::Agent::new_with_defaults(); +let vectors = emb::embed_batch(&agent, &endpoint, &texts)?; +``` + +## See also + +- [`chunkmd`](chunkmd.md) and [`cuepack`](cuepack.md) — where the text comes + from. +- [`rrf`](rrf.md) — what to do with the results of searching those vectors. diff --git a/docs/mdscrub.md b/docs/mdscrub.md new file mode 100644 index 0000000..b4389b9 --- /dev/null +++ b/docs/mdscrub.md @@ -0,0 +1,144 @@ +# mdscrub + +Converted markdown in, clean markdown out. + +Back to [the index](README.md). + +## What it does + +Takes the markdown that falls out of an EPUB or a PDF converter and removes +everything that means nothing: styling tags that survived, control characters +a text column will reject, headings wrapped in links to their own anchors, +page anchors that should be page markers. + +Safe to run twice — the second run changes nothing. + +## When you would use it + +- **Between a converter and a chunker.** Converted markdown is readable but + not clean, and every piece of leftover markup ends up in an embedding, in a + full-text index, and in every result a reader sees. +- **Backfilling text you already stored.** Running it over text that was + ingested before a cleaning step existed is safe, because it is idempotent. +- **Getting rid of the things that break a database write.** A null byte in + extracted text is rejected outright by most text columns; the rest of the + C0 controls are storable and are extraction garbage. + +## Install + +```sh +cargo install --git https://git.pumpin.online/funkpower/catena mdscrub +``` + +## Example + +A chapter as a converter leaves it: + +``` +$ cat chapter.md +# [III. The Second Estate](#ch3) {.unnumbered} + +The obligation ran both ways, and contemporaries said so plainly +whenever they were asked.4 A lord who could not +protect was a lord who could not command… + + +… +## [Notes]{.section-title} + +5. Maitland, *Domesday Book and Beyond* (Cambridge, 1897), 220–24. +``` + +and after: + +``` +$ mdscrub chapter.md +# III. The Second Estate + + + +The obligation ran both ways, and contemporaries said so plainly +whenever they were asked.4 A lord who could not +protect was a lord who could not command… + + +… +## Notes + +5. Maitland, *Domesday Book and Beyond* (Cambridge, 1897), 220–24. +``` + +The heading is readable, its page anchor became a marker on its own line, the +superscript note reference became the digit it was, and `–` became the +dash it names. + +In a pipeline: + +```sh +pandoc -t commonmark chapter.xhtml | mdscrub | chunkmd > chunks.ndjson +``` + +## Options + +| Option | Effect | +|---|---| +| `--pages` | First turn page separators and form feeds into page markers | + +`--pages` is for text straight out of a PDF or DjVu extractor, which marks its +page boundaries with a form feed or a rule of dashes. It is off by default +because only the program that just read a PDF knows that a form feed in the +text is a page break rather than a stray control character — in ordinary +markdown it is a control character, and in a code listing it is part of the +listing. + +## Exit codes + +| Code | Meaning | +|---|---| +| `0` | Done | +| `64` | The command line was wrong | +| `65` | The input was not text | +| `74` | The file could not be read | + +## What it does, pass by pass + +1. **Headings.** A heading wrapped in a link to its own anchor, an attribute + block, a bracketed span, bold markers around a title set in capitals — all + removed, leaving the title. A page anchor inside a heading is moved to a + marker on the line after it, so the heading keeps its position relative to + the body text. +2. **Page anchors.** `` in the body becomes + ``, with the number exactly as it was written. +3. **Character references.** `<sup>107` is decoded first, because a tag + escaped twice is invisible to any pattern looking for a tag — which is how + markup survives into text. +4. **Tags.** Styling tags go and their text stays. A tag that stands for a + line or block boundary — `
`, `

`, `

  • `, a table cell — becomes a + space, because deleting it welds the words either side together: + `APPENDICES
    OF
    LIST` would otherwise come out as `APPENDICESOFLIST`, + which is a word no reader can search for. Genuinely inline tags go to + nothing, so `claim16` stays `claim16`. +5. **Fenced code.** Left alone, so a book that quotes markup keeps its + examples. A decorative rule of tildes is not a fence. +6. **Control characters.** Removed, tab and newline excepted. +7. **Blank lines.** Runs collapse to one. + +HTML comments survive every pass, which is what lets page markers through. + +## As a library + +```rust +let cleaned = mdscrub::scrub(&markdown); + +// And, for text straight out of an extractor: +let cleaned = mdscrub::scrub(&mdscrub::pages::normalize(&raw)); +``` + +The passes are also available one at a time — `mdscrub::html::strip_tags`, +`mdscrub::heading::clean`, `mdscrub::unescape::unescape` — for a caller that +wants only one of them. + +## See also + +- [`chunkmd`](chunkmd.md) — what to do with the markdown once it is clean. +- [`pdftriage`](pdftriage.md) — whether the text was worth extracting at all. diff --git a/docs/pdftriage.md b/docs/pdftriage.md new file mode 100644 index 0000000..b2b9b15 --- /dev/null +++ b/docs/pdftriage.md @@ -0,0 +1,136 @@ +# pdftriage + +Whether a PDF's text layer is worth reading, or whether it has to be OCRed. + +Back to [the index](README.md). + +## What it does + +Extracts a sample of text from the middle of a PDF and judges it: healthy, +garbled, sparse, reflected-font, or absent — and therefore whether OCR is +worth its cost. + +It never runs OCR, never writes to the file, and reads two page ranges rather +than the whole document. + +## When you would use it + +- **Deciding what to OCR.** OCR is the expensive step in any ingest: minutes + of GPU time a page, and a result that is worse than a text layer the file + already had. This is the question worth answering carefully before spending + that. +- **Finding books that are already broken in an index.** A garbled layer + extracts as characters, not words, so it indexes silently and returns + nothing anybody searched for. +- **Recovering a layer that only looks dead.** Some PDFs store every character + code reflected about a fixed point, so the text extracts as punctuation + soup — `ײ¬®±¼«½¬·±²` is "Introduction". That is recoverable in full, which + is strictly better than an OCR pass that could only approximate it. + +## Install + +```sh +cargo install --git https://git.pumpin.online/funkpower/catena pdftriage +``` + +Needs `pdftotext` and `pdfinfo` on `PATH`. Both ship with poppler. Without +either, the tool exits `69` and says so rather than guessing. + +## Example + +A book whose text is fine: + +``` +$ pdftriage book.pdf +healthy, no OCR needed + 1805 characters on the opening pages + 1805 characters over 5 sample pages, ascii text score 1.00 + letters 0.98, latin-1 0.00 + 361 characters a page + english function words 0.44 +$ echo $? +1 +``` + +A scan with no text at all: + +``` +$ pdftriage scan.pdf +absent, needs OCR + 0 characters on the opening pages +$ echo $? +0 +``` + +The exit code is the answer, so a shell can act on it directly: + +```sh +for f in *.pdf; do pdftriage --quiet "$f" && ocrmypdf "$f" "ocr/$f"; done +``` + +The record form: + +``` +$ pdftriage --json book.pdf +{"v":1,"path":"book.pdf","kind":"pdf","declared_ext":"pdf","bytes":17595,"pages":20,"language":null,"text_layer":"healthy","needs_ocr":false,"notes":["1805 characters on the opening pages","1805 characters over 5 sample pages, ascii text score 1.00","letters 0.98, latin-1 0.00","361 characters a page","english function words 0.44"]} +``` + +## Options + +| Option | Effect | +|---|---| +| `--json` | Emit a [`doc` record](records.md#doc--what-a-file-is) instead of a readable verdict | +| `-q`, `--quiet` | Write no verdict; answer with the exit code alone. Diagnostics still go to stderr, because a silent failure is worse than a quiet one | + +## Exit codes + +| Code | Meaning | +|---|---| +| `0` | The layer needs OCR | +| `1` | It does not | +| `2` | Could not tell — the extractor could not read the file | +| `64` | The command line was wrong | +| `69` | `pdftotext` or `pdfinfo` is not installed | +| `74` | The file could not be read | + +## The verdicts + +| Verdict | What it means | OCR? | +|---|---|---| +| `healthy` | Extractable and readable as it stands | no | +| `reflected-font` | Permuted by a non-standard font encoding, and recoverable without OCR once the permutation is undone | no | +| `garbled` | Present but mojibake — extraction yields characters, not words | yes | +| `sparse` | Present but too thin to be the whole page: a scan whose layer holds only the page numbers and the running heads | yes | +| `absent` | No text at all | yes | + +The sample comes from the middle of the document, not the front. Front matter +is often a bare scan in a book whose body carries good text, and often cleanly +typeset boilerplate in a book whose body is damaged — judging a book by its +title page gets both wrong. + +The thresholds are measurements rather than guesses. Each one sits in the +middle of a gap observed across a few thousand real PDFs, and the reason is +recorded beside it in the source. + +## As a library + +The judging is a pure function over already-extracted text, so it can be +tested and used without a PDF anywhere in sight: + +```rust +use pdftriage::{Limits, Sample}; + +let found = pdftriage::triage( + &Sample { opening, middle, middle_pages: 5, pages: Some(412) }, + &Limits::default(), +); +println!("{:?} needs_ocr={}", found.layer, found.needs_ocr); +``` + +Running `pdftotext` is the command's job, not the library's. + +## See also + +- [`booksniff`](booksniff.md) — whether it is a PDF at all. +- [`mdscrub`](mdscrub.md) — cleaning up the text once it has been extracted. +- [The records](records.md) — the shape `--json` writes. diff --git a/docs/records.md b/docs/records.md new file mode 100644 index 0000000..3135df3 --- /dev/null +++ b/docs/records.md @@ -0,0 +1,116 @@ +# The records + +Five shapes travel between these tools. Each carries a version field `v`, so a +reader can refuse a stream it does not understand rather than parsing it into +whatever happens to line up. + +Every list-shaped stream is [NDJSON](contract.md#streams-not-documents): one +record per line, flushed as it is produced. + +Back to [the index](README.md). + +## `doc` — what a file is + +```json +{"v":1,"path":"scan.pdf","kind":"pdf","declared_ext":"pdf","bytes":9261184, + "pages":412,"language":"eng","text_layer":"garbled","needs_ocr":true,"notes":[]} +``` + +| Field | Meaning | +|---|---| +| `path` | Where the file was read from, as given | +| `kind` | `pdf`, `epub`, `djvu`, `azw3`, `mobi`, `txt`, `html`, `ole2`, `zip-wrapped-pdf`, `unknown` | +| `declared_ext` | The extension the name claimed, lowercased, without the dot | +| `bytes` | Size on disk | +| `pages` | Page count, for paginated formats | +| `language` | Language code, when one was detected | +| `text_layer` | `healthy`, `garbled`, `sparse`, `reflected-font`, `absent` | +| `needs_ocr` | Whether the file has to be OCRed to be searchable | +| `notes` | Observations, in the order they were made | + +A tool fills what it can and leaves the rest null: a format sniffer fills +`kind` and leaves `text_layer` empty, a triage tool does the reverse. + +Written by [`booksniff`](booksniff.md) and [`pdftriage`](pdftriage.md). + +## `chunk` — a retrievable passage + +```json +{"v":1,"idx":7,"text":"…","tokens":512,"page":37,"chapter":"III. The Second Estate", + "role":"body","start_time":null,"end_time":null,"flags":[]} +``` + +| Field | Meaning | +|---|---| +| `idx` | Position in the stream this chunk was cut from, from zero | +| `text` | The passage | +| `tokens` | Token count of `text` | +| `page` | Page the passage starts on, for paginated sources | +| `chapter` | Nearest enclosing heading, as written | +| `role` | `body`, `bibliography`, `index`, `toc`, `frontmatter` | +| `start_time`, `end_time` | Seconds from the start, for recordings | +| `flags` | Set tags, sorted and deduplicated | + +`page` and `chapter` for documents, `start_time` and `end_time` for +recordings. One shape either way, which is what lets a single index serve a +book and a lecture and cite both. + +Written by [`chunkmd`](chunkmd.md) and [`cuepack`](cuepack.md). + +## `hit` — one scored candidate + +```json +{"v":1,"id":"c81a","channel":"vec","rank":1,"score":0.83,"payload":{}} +``` + +| Field | Meaning | +|---|---| +| `id` | Identifier of the thing retrieved, opaque to fusion | +| `channel` | `vec`, `fts` or `meta` | +| `rank` | Position within that channel's own ranking, from one | +| `score` | The channel's raw score, on whatever scale it uses | +| `payload` | Anything the producer wants carried through untouched | + +`rank` is what fusion consumes. `score` is carried along because it is useful +to a person reading a trace and useless for comparing across channels. + +Read and written by [`rrf`](rrf.md). + +## `cue` — one line of a transcript + +```json +{"v":1,"start":12.5,"end":15.0,"text":"the mill stood at the northern end"} +``` + +Smaller than a chunk on purpose: a cue is what a subtitle file holds, a few +seconds and often half a sentence. + +Written by [`cuenorm`](cuenorm.md), read by [`cuepack`](cuepack.md) and +[`cueseg`](cueseg.md). + +## `segment` — one topic chapter + +```json +{"v":1,"start":0.0,"end":1190.7,"title":"The Mill Stood At The Northern End", + "excerpt":"the mill stood at the northern end of the valley where…"} +``` + +The marks a scrubber needs, and the spans a summariser should be given one at +a time. + +Written by [`cueseg`](cueseg.md). + +## Getting the schemas + +The JSON Schema for the first three is printed by the record library itself: + +``` +$ cargo run -p corpus-core -- schema chunk +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "Chunk", + … +} +``` + +See [`corpus-core`](corpus-core.md). diff --git a/docs/rrf.md b/docs/rrf.md new file mode 100644 index 0000000..9ee37aa --- /dev/null +++ b/docs/rrf.md @@ -0,0 +1,114 @@ +# rrf + +Several ranked lists into one, without the database. + +Back to [the index](README.md). + +## What it does + +Reciprocal rank fusion. Reads [`hit` +records](records.md#hit--one-scored-candidate) from any number of retrieval +channels and writes one ranked list, best first. + +## When you would use it + +- **Hybrid search, without adopting anything.** Vector search and full-text + search disagree about everything except what matters, and their scores are + not comparable — a cosine distance and a text rank are different quantities + on different scales. Fusion sidesteps that by throwing the scores away and + keeping only the ranks: a result both channels put near the top beats one + that either channel loves alone. +- **Making a metadata match count for more.** When somebody searches for a + book by its title, the book rarely repeats its own title in its body, so the + channel that matched the title is the one telling the truth. That is what + the weights are for. +- **Merging results across corpora**, or across shards, or across any two + systems that both return ranked lists. + +This is the one tool here worth calling in process rather than as a +subprocess: by the time a hit reaches it the data is already yours, so the +isolation a separate process buys is not worth a spawn on an interactive +search path. + +## Install + +```sh +cargo install --git https://git.pumpin.online/funkpower/catena rrf +``` + +## Example + +Three channels, overlapping: + +``` +$ cat vector.ndjson fts.ndjson meta.ndjson | rrf +{"v":1,"id":"e901","channel":"meta","rank":1,"score":0.040719196192490745,"payload":{"title":"The Human Condition"}} +{"v":1,"id":"c81a","channel":"vec","rank":2,"score":0.032266458495966696,"payload":{}} +{"v":1,"id":"b455","channel":"fts","rank":3,"score":0.032266458495966696,"payload":{}} +{"v":1,"id":"7f20","channel":"vec","rank":4,"score":0.016129032258064516,"payload":{}} +``` + +`e901` was third in full-text search and first in metadata, and the metadata +weight carries it to the top. `c81a` and `b455` tie, and the tie is broken by +which was seen first, so the same input always gives the same output. + +The top ten: + +```sh +cat *.ndjson | rrf --limit 10 +``` + +Trusting full-text matching more than usual: + +```sh +cat *.ndjson | rrf --weight fts=2 --weight meta=1 +``` + +## Options + +| Option | Default | Effect | +|---|---|---| +| `-n`, `--limit ` | all | How many results to keep | +| `--rrf-k ` | `60` | The rank constant | +| `--weight ` | `vec=1 fts=1 meta=1.5` | What a channel's rank is worth. Repeatable | + +The rank constant flattens the difference between rank one and rank two. Sixty +is what the method's own paper uses and what every other implementation uses, +which makes fused scores comparable across systems — worth more than tuning +it. + +## Exit codes + +| Code | Meaning | +|---|---| +| `0` | Done | +| `64` | The command line was wrong | +| `65` | A hit could not be read | +| `74` | The input could not be read | + +## Input and output + +One [`hit`](records.md#hit--one-scored-candidate) per line, in any order and +interleaved if you like — each carries the rank its own channel gave it, so +the stream does not have to be grouped. + +The record that comes back for an id is the one that ranked it best, carrying +its payload, with `rank` and `score` replaced by its place and score in the +fused list. That makes `channel` on the output informative: it names which +channel liked this result most. + +## The arithmetic + +Each hit contributes `weight / (k + rank)` to its id's score. Nothing else is +used — not the channel's own score, which is exactly what is not comparable. + +## As a library + +```rust +let fused = rrf::fuse(&hits, &rrf::Weights::default()); +``` + +## See also + +- [`emb`](emb.md) — producing the vectors one of the channels searches. +- [The records](records.md) — the shape this reads and writes. diff --git a/docs/textkit.md b/docs/textkit.md new file mode 100644 index 0000000..ca2bf77 --- /dev/null +++ b/docs/textkit.md @@ -0,0 +1,121 @@ +# textkit + +Token counting, sentence and paragraph splitting. + +Back to [the index](README.md). + +## What it is + +A library. Two jobs, both of which have to give the same answer everywhere or +chunk boundaries move: counting tokens the way an embedding stack counts them, +and cutting text into sentences and paragraphs. + +## When you would use it + +- **Deciding how much text fits** in a model's context, before sending it. +- **Writing your own chunker** that has to agree with + [`chunkmd`](chunkmd.md) about where a boundary is. +- **Porting anything from Python** where the exact behaviour of `\s`, `\w` or + `\b` matters more than it looks like it should. + +## Counting tokens + +```rust +let tokens = textkit::count("the mill stood at the northern end"); +let ids = textkit::encode("…"); +let text = textkit::decode_lossy(&ids[..100]); +``` + +`cl100k_base`, loaded from data compiled into the binary — no download, no +cache directory, nothing to go stale. Counting against a different tokeniser +than the one an index was built with silently moves every chunk boundary, +which is why this is one function and not a parameter. + +Special-token spellings are treated as ordinary text. Text out of a PDF can +contain anything, including the literal string `<|endoftext|>`, and encoding +it as a control token would be both wrong and a way for a document to reach +into a model's input. + +Cutting a list of tokens at an arbitrary index can split a character in half, +so decoding a slice is lossy rather than fallible: + +```rust +// The last resort of a chunker: guarantees a maximum, respects nothing else. +for piece in textkit::split_at_token_limit(&text, 512) { + assert!(textkit::count(&piece) <= 512); +} +``` + +## Splitting text + +```rust +for paragraph in textkit::paragraphs(&markdown) { + for sentence in textkit::sentences(paragraph) { + // … + } +} +``` + +Both return borrowed slices, trimmed, with the empty pieces dropped. + +A paragraph break is a newline, any run of whitespace, and a newline — so one +blank line and four of them separate equally, and trailing spaces on an +otherwise blank line do not stop it being blank. + +A sentence break is whitespace following `.`, `!` or `?`. Crude on purpose: it +does not know that `Mr.` or `p. 41` are not sentence ends. It is the fallback +path for a paragraph too large to chunk whole, where a wrong cut costs a +little context and a clever cut that disagrees with an existing index costs a +re-embedding. + +## Whitespace + +```rust +let trimmed = text.trim_matches(textkit::is_space); +``` + +`is_space` is whitespace by Python's definition: the Unicode `White_Space` +property *and* the four ASCII information separators `U+001C`–`U+001F`. +Trimming with the narrower definition leaves a separator on the front of a +piece, which changes its token count, which moves a chunk boundary. + +## Numbers + +```rust +assert_eq!(textkit::decimal(""), 412); +assert_eq!(textkit::decimal("٤٢"), 42); +``` + +Every decimal digit, not only the ASCII ones. Every Unicode decimal block runs +zero to nine in consecutive code points, which is what makes this possible +without a lookup table. + +## Patterns + +Character classes that mean in Rust what they mean in Python: + +```rust +use textkit::pattern::{SPACE, WORD, WORD_START, WORD_END}; + +let year = fancy_regex::Regex::new(&format!(r"{WORD_START}(19|20)\d{{2}}{WORD_END}"))?; +let heading = regex::Regex::new(&format!(r"(?m)^(#{{1,3}}){SPACE}+(.+)$"))?; +``` + +The two regex engines disagree, and every disagreement is a heuristic that +fires on one side and not the other: + +- Python's `\s` also matches `U+001C`–`U+001F`. +- Python's `\w` is the letters, the numbers and the underscore. Rust's adds + every combining mark and every connector punctuation mark, and leaves out + the letter-like and other numbers — so `Gödel` written in decomposed form is + one word to Rust and two to Python, and `²` is a word character to Python + and not to Rust. +- `\b` is defined in terms of `\w`, so it inherits all of that. + +The boundary forms are lookarounds, so a pattern using them needs an engine +that has them. + +## See also + +- [`corpus-core`](corpus-core.md) — the records these help produce. +- [`chunkmd`](chunkmd.md) — the chunker built on this. diff --git a/tools/check-links.py b/tools/check-links.py new file mode 100755 index 0000000..29502d7 --- /dev/null +++ b/tools/check-links.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 +"""Check that every link between the markdown files goes somewhere. + +Documentation that links to a page that has been renamed is worse than +documentation that does not link at all, because the reader trusts it. Run +from the repository root: + + python3 tools/check-links.py +""" + +import pathlib +import re +import sys + +LINK = re.compile(r"\[[^\]]*\]\(([^)]+)\)") + + +def prose(text: str) -> str: + """The document with its fenced code blocks taken out. + + An example that happens to contain a link is an example, not a link. + """ + out, fenced = [], False + for line in text.split("\n"): + if line.startswith("```"): + fenced = not fenced + continue + if not fenced: + out.append(line) + return "\n".join(out) + + +def anchors(text: str) -> set[str]: + """The headings of a document, as the fragments that reach them.""" + found = set() + for line in prose(text).split("\n"): + if line.startswith("#"): + title = line.lstrip("#").strip().lower() + found.add(re.sub(r"[^a-z0-9 -]", "", title).replace(" ", "-")) + return found + + +def main() -> int: + root = pathlib.Path(".").resolve() + # Fixtures are not documentation: a parity case is deliberately malformed + # markdown, and a link in one is part of what is being tested. + skip = {"target", "cases", "golden"} + files = sorted( + path + for path in root.rglob("*.md") + if skip.isdisjoint(path.parts) and not path.name.startswith(".") + ) + headings = {path: anchors(path.read_text(encoding="utf-8")) for path in files} + + broken = [] + for path in files: + for target in LINK.findall(prose(path.read_text(encoding="utf-8"))): + if target.startswith(("http://", "https://", "mailto:")): + continue + where, _, fragment = target.partition("#") + dest = (path.parent / where).resolve() if where else path + if not dest.exists(): + broken.append(f"{path.relative_to(root)} -> {target}: no such file") + elif fragment and dest in headings and fragment not in headings[dest]: + broken.append(f"{path.relative_to(root)} -> {target}: no such heading") + + for line in broken: + print(f"broken link: {line}", file=sys.stderr) + print(f"{len(files)} files, {len(broken)} broken links") + return 1 if broken else 0 + + +if __name__ == "__main__": + raise SystemExit(main())