diff --git a/crates/chunkmd/src/pages.rs b/crates/chunkmd/src/pages.rs index ff315c6..eca70da 100644 --- a/crates/chunkmd/src/pages.rs +++ b/crates/chunkmd/src/pages.rs @@ -4,6 +4,7 @@ //! and chapter boundaries as headings. Both are needed on every chunk: a //! passage a reader cannot locate in the book is a passage they cannot check. +use std::collections::BTreeMap; use std::sync::LazyLock; use regex::Regex; @@ -117,12 +118,22 @@ pub fn chapters(markdown: &str) -> Vec { return Vec::new(); } - let shallowest = headings + // The shallowest level that divides the document, which is not always the + // shallowest level in it. A converted EPUB carries the book's title as a + // lone `#` above sixty-two `##` chapters, and taking the lone one makes + // the whole book a single chapter — so a level used once is a title and a + // level used twice is a chapter map. + let mut counts: BTreeMap = BTreeMap::new(); + for (level, _, _) in &headings { + *counts.entry(*level).or_default() += 1; + } + let dividing = counts .iter() - .map(|(level, _, _)| *level) - .min() + .find(|(_, used)| **used > 1) + .map(|(level, _)| *level) + .or_else(|| counts.keys().next().copied()) .expect("a heading"); - headings.retain(|(level, _, _)| *level == shallowest); + headings.retain(|(level, _, _)| *level == dividing); let page_at = PageIndex::new(markdown); let mut chapters = Vec::with_capacity(headings.len()); @@ -245,6 +256,22 @@ mod tests { assert_eq!(pages[0].number, Some(7)); } + #[test] + fn a_lone_shallow_heading_is_a_title_rather_than_a_chapter_map() { + // What a converted EPUB looks like: the book's title as the only + // `#`, and every actual chapter a `##` under it. Taking the lone + // one makes the whole book a single chapter. + let markdown = "# The Whole Book\n\n## Chapter One\n\ntext\n\n## Chapter Two\n\ntext"; + let found = chapters(markdown); + assert_eq!( + found + .iter() + .map(|one| one.title.as_str()) + .collect::>(), + ["Chapter One", "Chapter Two"] + ); + } + #[test] fn chapters_come_from_the_shallowest_heading_level_present() { let markdown = "## Alpha\n\ntext\n\n### Sub\n\ntext\n\n## Beta\n\ntext"; diff --git a/crates/mdscrub/src/heading.rs b/crates/mdscrub/src/heading.rs index 2d86409..10cc0b1 100644 --- a/crates/mdscrub/src/heading.rs +++ b/crates/mdscrub/src/heading.rs @@ -18,8 +18,10 @@ fn pattern(source: &str) -> Regex { /// `[text](url)` — a heading wrapped in a link to its own anchor. static LINK: LazyLock = LazyLock::new(|| pattern(r"\[([^\]]*)\]\([^)]*\)")); -/// `[text]{.cls}` — pandoc's bracketed inline span. -static BRACKETED: LazyLock = LazyLock::new(|| pattern(r"\[([^\]]+)\]\{[^}]*\}")); +/// `[text]{.cls}` — pandoc's bracketed inline span. The text may be empty: +/// `[]{#chapter-3}` is how an anchor with nothing in it arrives, and a +/// heading that keeps one is a chapter title with an element id in it. +static BRACKETED: LazyLock = LazyLock::new(|| pattern(r"\[([^\]]*)\]\{[^}]*\}")); /// A trailing `{.attr}` block. static ATTRIBUTE_TAIL: LazyLock = diff --git a/docs/decisions.md b/docs/decisions.md index d8da4e1..6532633 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -93,6 +93,11 @@ library. The chapter a passage carries is the nearest heading above the point the passage *starts*, found by byte offset into the source. +The heading level that divides a document is the shallowest one used more +than once, not simply the shallowest one present: a converted EPUB carries the +book's title as a lone `#` above sixty-two `##` chapters, and a level used +once is a title rather than a chapter map. + The offset has to be the passage's own. A document with no page markers is a single page beginning at zero, so asking where the *page* started puts every chunk in the book under whichever heading came first in the file — which also diff --git a/examples/rust/Cargo.lock b/examples/rust/Cargo.lock new file mode 100644 index 0000000..3a634fd --- /dev/null +++ b/examples/rust/Cargo.lock @@ -0,0 +1,603 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys", +] + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "bibclean" +version = "0.1.0" +dependencies = [ + "catena-cli", + "clap", + "corpus-core", + "deunicode", + "fancy-regex 0.16.2", + "regex", + "schemars", + "serde", + "serde_json", + "textkit", + "unicode-normalization", +] + +[[package]] +name = "bit-set" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" +dependencies = [ + "bit-vec", +] + +[[package]] +name = "bit-vec" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" + +[[package]] +name = "booksniff" +version = "0.1.0" +dependencies = [ + "catena-cli", + "clap", + "corpus-core", + "serde_json", + "textkit", +] + +[[package]] +name = "bstr" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bb31b46c14244e20ee9984b11bf5c992b91fb6939fea616e3512c8baecdbe5f" +dependencies = [ + "memchr", + "regex-automata", + "serde_core", +] + +[[package]] +name = "catena-cli" +version = "0.1.0" +dependencies = [ + "clap", + "clap_mangen", + "corpus-core", + "serde", +] + +[[package]] +name = "catena-example" +version = "0.0.0" +dependencies = [ + "booksniff", + "chunkmd", + "citefind", + "corpus-core", + "mdscrub", +] + +[[package]] +name = "chunkmd" +version = "0.1.0" +dependencies = [ + "catena-cli", + "clap", + "corpus-core", + "fancy-regex 0.16.2", + "regex", + "textkit", +] + +[[package]] +name = "citefind" +version = "0.1.0" +dependencies = [ + "catena-cli", + "chunkmd", + "clap", + "corpus-core", + "idnorm", + "regex", + "schemars", + "serde", + "serde_json", +] + +[[package]] +name = "clap" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa8876b300ab35ba921adea3dfd70157a46249b33f95c9084ae5709785478946" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0797fb7aeb1406c84efac526901f7ec3ead2124f946b494e72879d4b54704d" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9c751b79415d4e559e3d1fcf128e09e720eb673a06d26cf6f392d37d75b66e0" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "clap_lex" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c133bc6a41be0d194c306b5506d15e6feeea7b1d6604bd3f8310dfb2ca96486" + +[[package]] +name = "clap_mangen" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "211d617eaa4b735c96c9e0228fcbdb5120ef623f2b8cb67ffb84c3e02dbc28a4" +dependencies = [ + "clap", + "roff", +] + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "corpus-core" +version = "0.1.0" +dependencies = [ + "schemars", + "serde", + "serde_json", + "thiserror", +] + +[[package]] +name = "deunicode" +version = "1.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abd57806937c9cc163efc8ea3910e00a62e2aeb0b8119f1793a978088f8f6b04" + +[[package]] +name = "dyn-clone" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" + +[[package]] +name = "fancy-regex" +version = "0.16.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "998b056554fbe42e03ae0e152895cd1a7e1002aec800fdc6635d20270260c46f" +dependencies = [ + "bit-set", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "fancy-regex" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72cf461f865c862bb7dc573f643dd6a2b6842f7c30b07882b56bd148cc2761b8" +dependencies = [ + "bit-set", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "idnorm" +version = "0.1.0" +dependencies = [ + "bibclean", + "catena-cli", + "clap", + "corpus-core", + "fancy-regex 0.16.2", + "schemars", + "serde", + "serde_json", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "mdscrub" +version = "0.1.0" +dependencies = [ + "catena-cli", + "clap", + "corpus-core", + "fancy-regex 0.16.2", + "regex", + "textkit", +] + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "ref-cast" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e440fb4e4b4147295338efb76001ab9e4efc0e5839df2c47fc5ac2381d365c3" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92ecd8964f8453721699a1ed72037b0db49ce2f5a5138486ee89bed6f67cdf3a" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "roff" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "323c417e1d9665a65b263ec744ba09030cfb277e9daa0b018a4ab62e57bc8189" + +[[package]] +name = "rustc-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" + +[[package]] +name = "schemars" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "687274d293b6cdc6e73e0fee520bf2049650090d7164f87672d212a3c530cf4a" +dependencies = [ + "dyn-clone", + "ref-cast", + "schemars_derive", + "serde", + "serde_json", +] + +[[package]] +name = "schemars_derive" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d98c67716b46af2f0b8cf752abc930f6f9aecfbf671ecfb531db8a31dbe4e2ba" +dependencies = [ + "proc-macro2", + "quote", + "serde_derive_internals", + "syn", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_derive_internals" +version = "0.30.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f852137cce035d6a4df67ccce505ff6b3e9fd3a10e3e52b24dc71e650bb1a9bd" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "syn" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "textkit" +version = "0.1.0" +dependencies = [ + "tiktoken-rs", +] + +[[package]] +name = "thiserror" +version = "2.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tiktoken-rs" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "027853bbf8c7763b77c5c595f1c271c7d536ced7d6f83452911b944621e57fc2" +dependencies = [ + "anyhow", + "base64", + "bstr", + "fancy-regex 0.17.0", + "lazy_static", + "regex", + "rustc-hash", +] + +[[package]] +name = "tinyvec" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd3ca314f692efd6c868f8408f53fe444634a845f96c028b97d35f6a1f79f0ee" + +[[package]] +name = "unicode-ident" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954" + +[[package]] +name = "unicode-normalization" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/examples/rust/Cargo.toml b/examples/rust/Cargo.toml new file mode 100644 index 0000000..b6db021 --- /dev/null +++ b/examples/rust/Cargo.toml @@ -0,0 +1,16 @@ +# Standalone so that `cargo run` here does not need the workspace above it. +[workspace] + +[package] +name = "catena-example" +version = "0.0.0" +edition = "2024" +rust-version = "1.85" +publish = false + +[dependencies] +booksniff = { path = "../../crates/booksniff" } +chunkmd = { path = "../../crates/chunkmd" } +citefind = { path = "../../crates/citefind" } +corpus-core = { path = "../../crates/corpus-core" } +mdscrub = { path = "../../crates/mdscrub" } diff --git a/examples/rust/src/main.rs b/examples/rust/src/main.rs new file mode 100644 index 0000000..3150570 --- /dev/null +++ b/examples/rust/src/main.rs @@ -0,0 +1,119 @@ +//! A converted book, read the whole way through in one process. +//! +//! The commands are filters and a subprocess is usually the right answer, but +//! when the caller already holds the text there is nothing to serialise: the +//! same libraries the commands are built from take a `&str` and hand back the +//! records. This walks a file from "what even is it" to passages worth +//! embedding and the references at the back, without a pipe anywhere. +//! +//! cargo run -- book.md +//! cargo run # reads the sample below + +use std::path::Path; + +/// A page of a converted book: a heading, two passages and a short +/// bibliography, which is enough to show every stage doing something. +const SAMPLE: &str = "\ +# III. The Second Estate + +The obligation ran both ways, and contemporaries said so plainly whenever +they were asked.[^4] A lord who could not protect was a lord who could not +command, and the rolls record the consequences. + +What the rolls do not say, and what has to be inferred, is how often the +obligation was actually tested. + +# Bibliography + +Bloch, Marc. *Feudal Society*. Translated by L. A. Manyon. London: Routledge +and Kegan Paul, 1961. + +Duby, Georges. *The Three Orders: Feudal Society Imagined*. Chicago: +University of Chicago Press, 1980. + +Reynolds, Susan. *Fiefs and Vassals: The Medieval Evidence Reinterpreted*. +Oxford: Clarendon Press, 1994. +"; + +fn main() { + let (name, raw) = match std::env::args().nth(1) { + Some(path) => { + let bytes = std::fs::read(&path).unwrap_or_else(|why| { + eprintln!("catena-example: {path}: {why}"); + std::process::exit(74); + }); + (path, bytes) + } + None => ("sample.md".to_string(), SAMPLE.as_bytes().to_vec()), + }; + + // What the file actually is, whatever it is called. Worth asking first: + // a `.pdf` that is really a DjVu fails four stages later, in a way that + // looks like a bug in the converter. + let doc = booksniff::sniff(&raw.as_slice(), &name); + let stem = Path::new(&name).file_name().unwrap_or_default(); + println!("{:?} is a {:?}, {} bytes", stem, doc.kind, doc.bytes); + for note in &doc.notes { + println!(" note: {note}"); + } + + // Converted markdown carries the converter's leftovers. Scrubbing before + // chunking means the passages are the book's words and not the tool's. + let markdown = String::from_utf8_lossy(&raw); + let clean = mdscrub::scrub(&markdown); + + // The defaults are sized for a book: 512 tokens a chunk, which would put + // this whole sample in one. A page needs a page's budget. + let budgets = match std::env::args().nth(1) { + Some(_) => chunkmd::Budgets::default(), + None => chunkmd::Budgets { + target: 120, + overlap: 0, + min_tokens: 10, + ..chunkmd::Budgets::default() + }, + }; + let chunks = chunkmd::chunk(&clean, &budgets); + println!("\n{} passage(s):", chunks.len()); + for chunk in &chunks { + let chapter = chunk.chapter.as_deref().unwrap_or("—"); + let opening: String = chunk.text.chars().take(64).collect(); + println!( + " [{}] {:?} {} tokens, under {chapter:?}\n {}…", + chunk.idx, + chunk.role, + chunk.tokens, + opening.replace('\n', " ") + ); + } + + // The references at the back. A passage whose role is `bibliography` is + // where they live; a note at the foot of a page is the other place, and + // `chunkmd` flags those so this does not have to guess. + let references: Vec = chunks + .iter() + .filter(|chunk| chunk.role == corpus_core::Role::Bibliography) + .flat_map(|chunk| citefind::read(&chunk.text)) + .filter(|found| !found.continuation) + .collect(); + + println!("\n{} reference(s):", references.len()); + for found in &references { + let who = match found.authors.is_empty() { + true => "—".to_string(), + false => found.authors.join("; "), + }; + println!( + " {who}\n {} ({}) — {:?}", + found.title.as_deref().unwrap_or("—"), + found + .year + .map(|year| year.to_string()) + .unwrap_or_else(|| "no year".into()), + found.kind + ); + if !found.unsettled.is_empty() { + println!(" could not read: {}", found.unsettled.join(", ")); + } + } +}