mdscrub: residual html, control characters, headings, page markers

crates/chunkmd/src/pages.rs

@@ -76,7 +76,7 @@ pub fn split_by_page(markdown: &str) -> Vec<Page<'_>> {
         let text = trim(&markdown[start..end]);
         if !text.is_empty() {
             pages.push(Page {
-                number: Some(page_number(&markdown[marker.start()..marker.end()])),
+                number: Some(textkit::decimal(&markdown[marker.start()..marker.end()])),
                 text,
                 offset: start,
             });
@@ -138,7 +138,7 @@ impl PageIndex {
     pub fn new(markdown: &str) -> Self {
         Self(
             PAGE.find_iter(markdown)
-                .map(|marker| (marker.start(), page_number(marker.as_str())))
+                .map(|marker| (marker.start(), textkit::decimal(marker.as_str())))
                 .collect(),
         )
     }
@@ -193,20 +193,6 @@ pub fn page_chapters(chapters: &[Chapter]) -> HashMap<u32, &str> {
     out
 }
 
-/// Read the digits out of a page marker.
-///
-/// Digits are matched by the Unicode class, so a marker could in principle
-/// carry non-ASCII decimal digits; those count the same way here as they
-/// would in Python. A number too large for the field saturates rather than
-/// wrapping — see `parity/KNOWN.md`.
-fn page_number(marker: &str) -> u32 {
-    let mut number: u32 = 0;
-    for digit in marker.chars().filter_map(|c| c.to_digit(10)) {
-        number = number.saturating_mul(10).saturating_add(digit);
-    }
-    number
-}
-
 fn trim(text: &str) -> &str {
     text.trim_matches(textkit::is_space)
 }

crates/mdscrub/Cargo.toml

@@ -0,0 +1,17 @@
+[package]
+name = "mdscrub"
+description = "Cleans converted markdown: residual HTML, control characters, headings, page markers."
+version.workspace = true
+edition.workspace = true
+rust-version.workspace = true
+license.workspace = true
+repository.workspace = true
+
+[dependencies]
+clap.workspace = true
+corpus-core.workspace = true
+regex.workspace = true
+textkit.workspace = true
+
+[lints]
+workspace = true

crates/mdscrub/src/heading.rs

@@ -0,0 +1,149 @@
+//! Making a heading line readable again.
+//!
+//! A converted heading arrives carrying whatever the source used to style it:
+//! link wrappers around the whole title, attribute blocks, anchor spans, bold
+//! markers around a title that was set in capitals. The heading text is what
+//! every chunk in the chapter will be labelled with, so it is worth cleaning
+//! properly rather than passing through.
+
+use std::sync::LazyLock;
+
+use regex::Regex;
+
+fn pattern(source: &str) -> Regex {
+    Regex::new(source).expect("a valid pattern")
+}
+
+/// `[text](url)` — a heading wrapped in a link to its own anchor.
+static LINK: LazyLock<Regex> = LazyLock::new(|| pattern(r"\[([^\]]*)\]\([^)]*\)"));
+
+/// `[text]{.cls}` — pandoc's bracketed inline span.
+static BRACKETED: LazyLock<Regex> = LazyLock::new(|| pattern(r"\[([^\]]+)\]\{[^}]*\}"));
+
+/// A trailing `{.attr}` block.
+static ATTRIBUTE_TAIL: LazyLock<Regex> = LazyLock::new(|| pattern(r"\s*\{[^}]*\}\s*$"));
+
+/// Any inline tag. Broader than the one used on body text, because a heading
+/// has no code fences to protect.
+static TAG: LazyLock<Regex> = LazyLock::new(|| pattern(r"<[^>]+>"));
+
+static SPACE_RUN: LazyLock<Regex> = LazyLock::new(|| pattern(r"\s+"));
+
+/// A heading line, and the text after its hashes.
+static HEADING: LazyLock<Regex> = LazyLock::new(|| pattern(r"^(#{1,6})\s+(.*)$"));
+
+/// A page anchor: `<span id="…page42">`, `<a id="page-42">`, and the rest of
+/// the shapes an EPUB uses to say where a printed page began.
+pub static PAGE_ANCHOR: LazyLock<Regex> = LazyLock::new(|| {
+    pattern(
+        r#"(?i)<(?:span|a|div)\b[^>]*\bid\s*=\s*["'][^"']*?page[-_]?(\d+)[^"']*["'][^>]*>(?:</(?:span|a|div)>)?"#,
+    )
+});
+
+/// Clean one heading's text.
+pub fn clean(raw: &str) -> String {
+    let text = LINK.replace_all(raw, "$1");
+    let text = BRACKETED.replace_all(&text, "$1");
+    let text = ATTRIBUTE_TAIL.replace_all(&text, "");
+    let text = TAG.replace_all(&text, "");
+    let text = SPACE_RUN.replace_all(&text, " ");
+    let text = text.trim_matches(textkit::is_space);
+    // Emphasis markers around a title that was set in capitals say nothing
+    // once the title is a label on a search result.
+    text.replace("**", "").replace("__", "")
+}
+
+/// Clean every heading in a document, moving any page anchor found in a
+/// heading to a marker on its own line just after it.
+///
+/// After the heading rather than before, so that the heading keeps its offset
+/// relative to the body text that follows, and the chapter title stays whole.
+pub fn clean_all(markdown: &str) -> String {
+    let mut out: Vec<String> = Vec::new();
+    for line in markdown.split('\n') {
+        let Some(found) = HEADING.captures(line) else {
+            out.push(line.to_string());
+            continue;
+        };
+        let hashes = &found[1];
+        let raw = &found[2];
+        let pages: Vec<u32> = PAGE_ANCHOR
+            .captures_iter(raw)
+            .map(|anchor| textkit::decimal(&anchor[1]))
+            .collect();
+        let cleaned = clean(&PAGE_ANCHOR.replace_all(raw, ""));
+        if !cleaned.is_empty() {
+            out.push(format!("{hashes} {cleaned}"));
+        }
+        for page in pages {
+            out.push(String::new());
+            out.push(format!("<!-- Page {page} -->"));
+        }
+    }
+    out.join("\n")
+}
+
+/// Turn the page anchors left in the body into page markers.
+pub fn anchors_to_markers(markdown: &str) -> String {
+    PAGE_ANCHOR
+        .replace_all(markdown, |found: &regex::Captures| {
+            format!("\n\n<!-- Page {} -->\n\n", textkit::decimal(&found[1]))
+        })
+        .into_owned()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn a_link_wrapper_comes_off() {
+        assert_eq!(clean("[I. The Setting](#ch1)"), "I. The Setting");
+    }
+
+    #[test]
+    fn a_bracketed_span_comes_off() {
+        assert_eq!(clean("[The Quarrel]{.chapter-title}"), "The Quarrel");
+    }
+
+    #[test]
+    fn a_trailing_attribute_block_comes_off() {
+        assert_eq!(clean("The Break {#ch3 .unnumbered}"), "The Break");
+    }
+
+    #[test]
+    fn tags_come_off_and_whitespace_collapses() {
+        assert_eq!(clean("<span id=\"x\">The   Break</span>  "), "The Break");
+    }
+
+    #[test]
+    fn emphasis_markers_come_off() {
+        assert_eq!(clean("**THE SETTLEMENT**"), "THE SETTLEMENT");
+    }
+
+    #[test]
+    fn a_page_anchor_in_a_heading_becomes_a_marker_after_it() {
+        let cleaned = clean_all("# <span id=\"page42\"></span>The Quarrel");
+        assert_eq!(cleaned, "# The Quarrel\n\n<!-- Page 42 -->");
+    }
+
+    #[test]
+    fn a_heading_that_was_only_an_anchor_leaves_only_the_marker() {
+        let cleaned = clean_all("## <a id=\"page-7\"></a>");
+        assert_eq!(cleaned, "\n<!-- Page 7 -->");
+    }
+
+    #[test]
+    fn body_anchors_become_markers() {
+        let cleaned = anchors_to_markers("text <span id=\"pg_page19\"></span> more");
+        assert!(cleaned.contains("<!-- Page 19 -->"));
+    }
+
+    #[test]
+    fn a_line_that_is_not_a_heading_is_untouched() {
+        assert_eq!(
+            clean_all("#no space is not a heading"),
+            "#no space is not a heading"
+        );
+    }
+}

crates/mdscrub/src/html.rs

@@ -0,0 +1,122 @@
+//! Removing the HTML that survives conversion to markdown.
+
+use std::sync::LazyLock;
+
+use regex::Regex;
+
+/// An opening or closing inline tag.
+///
+/// Converted markdown is full of them: pandoc's commonmark output keeps an
+/// EPUB's styling spans and anchors, and a PDF converter emits anchor spans
+/// and superscript note references. None of it means anything once the page
+/// anchors have been pulled out, and all of it pollutes an embedding, a
+/// full-text index, and every result a reader sees.
+///
+/// Comments are deliberately not matched: `<!-- Page 41 -->` has to survive.
+static TAG: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"</?[A-Za-z][^>]*>").expect("valid"));
+
+/// A tag that stands for a line break rather than for styling.
+static BREAK: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"(?i)<br\s*/?>").expect("valid"));
+
+/// The start of a fenced code block.
+static FENCE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"^\s*(```|~~~)").expect("valid"));
+
+/// Control characters that must never reach a text column.
+///
+/// A database will reject `0x00` outright. The other C0 controls — tab,
+/// newline and carriage return excepted — are storable and are extraction
+/// garbage: bad scans and malformed PDFs emit them routinely.
+static CONTROL: LazyLock<Regex> =
+    LazyLock::new(|| Regex::new(r"[\x00-\x08\x0b\x0c\x0e-\x1f]").expect("valid"));
+
+/// Three or more newlines, left behind wherever a tag-only line was removed.
+static BLANK_RUN: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"\n{3,}").expect("valid"));
+
+/// Strip inline HTML, keeping the text inside it and the HTML comments.
+///
+/// Fenced code blocks are left alone, so a book that quotes HTML keeps its
+/// examples. `<br>` becomes a line break rather than nothing, because
+/// deleting it welds the words on either side into one — which is a spelling
+/// mistake in the index that no reader can search for.
+pub fn strip_tags(markdown: &str) -> String {
+    let mut out = Vec::new();
+    let mut in_fence = false;
+    for line in markdown.split('\n') {
+        if FENCE.is_match(line) {
+            in_fence = !in_fence;
+            out.push(line.to_string());
+            continue;
+        }
+        if in_fence {
+            out.push(line.to_string());
+        } else {
+            let line = BREAK.replace_all(line, "\n");
+            out.push(TAG.replace_all(&line, "").into_owned());
+        }
+    }
+    out.join("\n")
+}
+
+/// Remove the control characters a text column cannot hold.
+pub fn scrub_control(text: &str) -> String {
+    CONTROL.replace_all(text, "").into_owned()
+}
+
+/// Collapse runs of blank lines to a single blank line.
+pub fn collapse_blank_runs(text: &str) -> String {
+    BLANK_RUN.replace_all(text, "\n\n").into_owned()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn styling_tags_go_and_their_text_stays() {
+        assert_eq!(
+            strip_tags("<span class=\"c1\">The</span> second <em>estate</em>"),
+            "The second estate"
+        );
+    }
+
+    #[test]
+    fn html_comments_survive() {
+        assert_eq!(strip_tags("<!-- Page 41 -->"), "<!-- Page 41 -->");
+    }
+
+    #[test]
+    fn a_break_tag_does_not_weld_the_words_either_side_of_it() {
+        assert_eq!(strip_tags("first<br>second"), "first\nsecond");
+        assert_eq!(strip_tags("first<br />second"), "first\nsecond");
+        assert_eq!(strip_tags("first<BR/>second"), "first\nsecond");
+    }
+
+    #[test]
+    fn fenced_code_keeps_its_tags() {
+        let markdown = "before <b>x</b>\n\n```html\n<span>kept</span>\n```\n\nafter <b>y</b>";
+        let cleaned = strip_tags(markdown);
+        assert!(cleaned.contains("<span>kept</span>"));
+        assert!(cleaned.contains("before x"));
+        assert!(cleaned.contains("after y"));
+    }
+
+    #[test]
+    fn a_tag_split_across_two_lines_is_left_alone() {
+        // Matching per line is what keeps a stray `<` in prose from eating
+        // the rest of the document.
+        let markdown = "a < b and\nc > d";
+        assert_eq!(strip_tags(markdown), markdown);
+    }
+
+    #[test]
+    fn control_characters_go_but_tabs_and_newlines_stay() {
+        assert_eq!(scrub_control("a\u{0}b\u{1}c"), "abc");
+        assert_eq!(scrub_control("a\tb\nc\r\n"), "a\tb\nc\r\n");
+    }
+
+    #[test]
+    fn blank_line_runs_collapse_to_one() {
+        assert_eq!(collapse_blank_runs("a\n\n\n\n\nb"), "a\n\nb");
+        assert_eq!(collapse_blank_runs("a\n\nb"), "a\n\nb");
+    }
+}

crates/mdscrub/src/lib.rs

@@ -0,0 +1,69 @@
+//! Cleans converted markdown.
+//!
+//! Conversion from a PDF or an EPUB leaves markdown that is readable but not
+//! clean: styling tags that survived, control characters a text column will
+//! reject, headings wrapped in links to their own anchors, and page
+//! boundaries spelled three different ways. None of it means anything
+//! downstream, and all of it ends up in an index.
+//!
+//! The order of the passes matters. Page boundaries are normalised first,
+//! because one of the things that marks a page boundary — a form feed — is
+//! also a control character that the later pass deletes.
+
+pub mod heading;
+pub mod html;
+pub mod pages;
+
+/// Clean a converted markdown document.
+///
+/// Safe to run twice: the second run changes nothing.
+pub fn scrub(markdown: &str) -> String {
+    let text = pages::normalize(markdown);
+    let text = heading::clean_all(&text);
+    let text = heading::anchors_to_markers(&text);
+    let text = html::strip_tags(&text);
+    let text = html::scrub_control(&text);
+    let text = html::collapse_blank_runs(&text);
+    format!("{}\n", text.trim_matches(textkit::is_space))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn a_converted_heading_comes_out_readable() {
+        let markdown = "# [<span id=\"page12\"></span>I. The Setting](#ch1) {.unnumbered}";
+        assert_eq!(scrub(markdown), "# I. The Setting\n\n<!-- Page 12 -->\n");
+    }
+
+    #[test]
+    fn running_it_twice_changes_nothing() {
+        let markdown = "<!-- Page 3 -->\n\n# The <em>Quarrel</em>\n\n\n\nA <b>mill</b>\u{0} \
+                        stood here.<br>It still does.\n\n<span id=\"page4\"></span>\n\nOn.";
+        let once = scrub(markdown);
+        assert_eq!(scrub(&once), once);
+    }
+
+    #[test]
+    fn a_page_marker_survives_every_pass() {
+        assert!(scrub("<!-- Page 41 -->\n\ntext").contains("<!-- Page 41 -->"));
+    }
+
+    #[test]
+    fn the_result_always_ends_in_exactly_one_newline() {
+        for markdown in ["", "text", "text\n\n\n", "  \n"] {
+            let cleaned = scrub(markdown);
+            assert!(cleaned.ends_with('\n'), "{markdown:?}");
+            assert!(!cleaned.ends_with("\n\n"), "{markdown:?}");
+        }
+    }
+
+    #[test]
+    fn form_feeds_become_pages_rather_than_disappearing() {
+        // The control-character pass would delete a form feed, so page
+        // normalisation has to happen first or the boundary is lost.
+        let cleaned = scrub("one\u{c}two");
+        assert!(cleaned.contains("<!-- Page 2 -->"), "{cleaned:?}");
+    }
+}

crates/mdscrub/src/main.rs

@@ -0,0 +1,62 @@
+//! Cleans converted markdown.
+
+use std::io::{Read, Write};
+use std::path::{Path, PathBuf};
+use std::process::ExitCode;
+
+use clap::Parser;
+use corpus_core::exit;
+
+/// Clean converted markdown: drop residual HTML and control characters,
+/// make headings readable, and spell every page boundary the same way.
+///
+/// Reads markdown on stdin, or from a path, and writes it to stdout. Safe to
+/// run twice.
+#[derive(Parser)]
+#[command(
+    version,
+    about,
+    long_about = None,
+    after_help = "\
+Example:
+
+    pandoc -t commonmark book.epub | mdscrub | chunkmd > chunks.ndjson"
+)]
+struct Args {
+    /// Markdown file to read. Reads stdin when absent.
+    path: Option<PathBuf>,
+}
+
+fn main() -> ExitCode {
+    let args = Args::parse();
+
+    let markdown = match read(args.path.as_deref()) {
+        Ok(markdown) => markdown,
+        Err(e) => {
+            eprintln!("mdscrub: {e}");
+            return ExitCode::from(exit::IO);
+        }
+    };
+
+    let cleaned = mdscrub::scrub(&markdown);
+    match std::io::stdout().write_all(cleaned.as_bytes()) {
+        Ok(()) => ExitCode::from(exit::OK),
+        // A consumer that stopped reading is not a failure.
+        Err(e) if e.kind() == std::io::ErrorKind::BrokenPipe => ExitCode::from(exit::OK),
+        Err(e) => {
+            eprintln!("mdscrub: {e}");
+            ExitCode::from(exit::IO)
+        }
+    }
+}
+
+fn read(path: Option<&Path>) -> std::io::Result<String> {
+    match path {
+        Some(path) => std::fs::read_to_string(path),
+        None => {
+            let mut markdown = String::new();
+            std::io::stdin().read_to_string(&mut markdown)?;
+            Ok(markdown)
+        }
+    }
+}

crates/mdscrub/src/pages.rs

@@ -0,0 +1,112 @@
+//! Turning whatever a converter used for page boundaries into one marker.
+
+use std::sync::LazyLock;
+
+use regex::Regex;
+
+/// The separator a paginating PDF converter writes between pages, like
+/// `{0}------------------------------`. Its numbering is zero-based.
+static SEPARATOR: LazyLock<Regex> =
+    LazyLock::new(|| Regex::new(r"\{(\d+)\}-{20,}[^\S\n]*\n").expect("a valid pattern"));
+
+/// A form feed, which is what a plain text extractor leaves between pages.
+const FORM_FEED: char = '\u{c}';
+
+/// Rewrite page boundaries as `<!-- Page N -->` markers.
+///
+/// Separators first, then form feeds, and text with neither is returned
+/// unchanged — a cleanup pass has to be safe to run twice, and inventing a
+/// page marker for a document that has none would not be.
+pub fn normalize(text: &str) -> String {
+    if SEPARATOR.is_match(text) {
+        return from_separators(text);
+    }
+    if text.contains(FORM_FEED) {
+        return from_form_feeds(text);
+    }
+    text.to_string()
+}
+
+fn from_separators(text: &str) -> String {
+    let mut pages: Vec<String> = Vec::new();
+    let mut at = 0;
+    let mut number = 1;
+
+    for found in SEPARATOR.captures_iter(text) {
+        let whole = found.get(0).expect("group 0");
+        let before = trim(&text[at..whole.start()]);
+        if !before.is_empty() {
+            pages.push(format!("<!-- Page {number} -->\n\n{before}"));
+        }
+        // The separator carries the number of the page it *ends*, counted
+        // from zero, so the page that follows it is that number plus one.
+        number = textkit::decimal(&found[1]).saturating_add(1);
+        at = whole.end();
+    }
+
+    let last = trim(&text[at..]);
+    if !last.is_empty() {
+        pages.push(format!("<!-- Page {number} -->\n\n{last}"));
+    }
+    pages.join("\n\n")
+}
+
+fn from_form_feeds(text: &str) -> String {
+    text.split(FORM_FEED)
+        .map(trim)
+        .enumerate()
+        .filter(|(_, page)| !page.is_empty())
+        .map(|(at, page)| format!("<!-- Page {} -->\n\n{page}", at + 1))
+        .collect::<Vec<_>>()
+        .join("\n\n")
+}
+
+fn trim(text: &str) -> &str {
+    text.trim_matches(textkit::is_space)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn separators_become_markers_numbered_from_one() {
+        let text = "first page\n{0}------------------------------\nsecond page\n\
+                    {1}------------------------------\nthird page";
+        assert_eq!(
+            normalize(text),
+            "<!-- Page 1 -->\n\nfirst page\n\n\
+             <!-- Page 1 -->\n\nsecond page\n\n\
+             <!-- Page 2 -->\n\nthird page"
+        );
+    }
+
+    #[test]
+    fn form_feeds_become_markers() {
+        let text = "one\u{c}two\u{c}three";
+        assert_eq!(
+            normalize(text),
+            "<!-- Page 1 -->\n\none\n\n<!-- Page 2 -->\n\ntwo\n\n<!-- Page 3 -->\n\nthree"
+        );
+    }
+
+    #[test]
+    fn an_empty_page_is_dropped_and_the_numbering_carries_on() {
+        assert_eq!(
+            normalize("one\u{c}\u{c}three"),
+            "<!-- Page 1 -->\n\none\n\n<!-- Page 3 -->\n\nthree"
+        );
+    }
+
+    #[test]
+    fn text_with_no_page_boundaries_is_left_exactly_as_it_was() {
+        let text = "# A Heading\n\nSome prose, with no pagination at all.\n";
+        assert_eq!(normalize(text), text);
+    }
+
+    #[test]
+    fn a_short_dash_run_is_not_a_separator() {
+        let text = "before\n{0}-----\nafter";
+        assert_eq!(normalize(text), text);
+    }
+}

crates/textkit/src/lib.rs

@@ -10,8 +10,10 @@
 //! chunk ends and a chunk boundary that moves is a chunk that no longer
 //! matches what is already in an index.
 
+pub mod number;
 pub mod segment;
 pub mod tokens;
 
+pub use number::decimal;
 pub use segment::{is_space, paragraphs, sentences};
 pub use tokens::{count, decode_lossy, encode, split_at_token_limit};

crates/textkit/src/number.rs

@@ -0,0 +1,91 @@
+//! Reading numbers out of text the way Python reads them.
+
+/// The value of a run of decimal digits, Unicode included.
+///
+/// Digits are matched by the Unicode class in the patterns these tools port,
+/// and Python's `int()` accepts every decimal digit, not just the ASCII ones
+/// — so `page٤٢` is page forty-two there and has to be here too. Non-digits
+/// are skipped, and a number too large for the field saturates rather than
+/// wrapping.
+pub fn decimal(text: &str) -> u32 {
+    let mut value: u32 = 0;
+    for digit in text.chars().filter_map(digit_value) {
+        value = value.saturating_mul(10).saturating_add(digit);
+    }
+    value
+}
+
+/// What one decimal digit is worth, in any script.
+///
+/// [`char::to_digit`] knows only the ASCII digits. Every Unicode decimal
+/// block, though, runs zero to nine in consecutive codepoints, so counting
+/// back to the first non-digit gives the value without a lookup table.
+fn digit_value(c: char) -> Option<u32> {
+    if let Some(digit) = c.to_digit(10) {
+        return Some(digit);
+    }
+    if !c.is_numeric() {
+        return None;
+    }
+    let code = c as u32;
+    let mut value = 9;
+    for back in 1..=9 {
+        let previous = code.checked_sub(back).and_then(char::from_u32);
+        match previous {
+            Some(previous) if previous.is_numeric() => continue,
+            _ => {
+                value = back - 1;
+                break;
+            }
+        }
+    }
+    // Confirm it really is a run of ten. Fractions and the other numeric
+    // characters sit in short runs, and would otherwise be read as digits:
+    // `½` follows `¼`, which would make it a one.
+    let zero = code - value;
+    let ten = (zero..zero + 10).all(|code| char::from_u32(code).is_some_and(char::is_numeric));
+    ten.then_some(value)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn ascii_digits_read_as_themselves() {
+        assert_eq!(decimal("<!-- Page 412 -->"), 412);
+    }
+
+    #[test]
+    fn other_decimal_digits_read_too() {
+        assert_eq!(decimal("٤٢"), 42);
+        assert_eq!(decimal("４２"), 42);
+    }
+
+    #[test]
+    fn every_digit_of_a_script_is_worth_what_it_looks_like() {
+        for (zero, name) in [('\u{660}', "arabic-indic"), ('\u{ff10}', "fullwidth")] {
+            for offset in 0..10u32 {
+                let digit = char::from_u32(zero as u32 + offset).expect("a digit");
+                assert_eq!(digit_value(digit), Some(offset), "{name} {digit}");
+            }
+        }
+    }
+
+    #[test]
+    fn a_character_that_is_not_a_digit_is_worth_nothing() {
+        for c in ['x', ' ', '-', '½'] {
+            assert_eq!(digit_value(c), None, "{c}");
+        }
+    }
+
+    #[test]
+    fn nothing_reads_as_zero() {
+        assert_eq!(decimal("page"), 0);
+    }
+
+    #[test]
+    fn an_absurd_number_saturates_rather_than_wrapping() {
+        assert_eq!(decimal(&"9".repeat(40)), u32::MAX);
+    }
+}