mirror of
https://git.pumpin.online/funkpower/catena.git
synced 2026-10-22 18:57:14 +00:00
chunkmd, mdscrub: the chapter a converted book is actually divided by
Two things between a converted EPUB and its chapter titles.
The heading level taken as the chapter map was the shallowest present. Pandoc
writes the book's title as the only `#` and every real chapter as a `##`, so
the shallowest level had one heading in it and the whole book came back as a
single chapter. The dividing level is now the shallowest one used more than
once — a level used once is a title.
And an empty bracketed span survived the heading cleaner, because the pattern
wanted at least one character between the brackets. Pandoc writes an anchor
as exactly that: `## []{#link2H_INTR} Introduction`.
Gibbon's first volume, converted from the Project Gutenberg EPUB: 2 distinct
chapter titles before, 61 after, and none of them carrying an element id.
This commit is contained in:
funkpower
@@ -4,6 +4,7 @@
|
||||
//! and chapter boundaries as headings. Both are needed on every chunk: a
|
||||
//! passage a reader cannot locate in the book is a passage they cannot check.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::sync::LazyLock;
|
||||
|
||||
use regex::Regex;
|
||||
@@ -117,12 +118,22 @@ pub fn chapters(markdown: &str) -> Vec<Chapter> {
|
||||
return Vec::new();
|
||||
}
|
||||
|
||||
let shallowest = headings
|
||||
// The shallowest level that divides the document, which is not always the
|
||||
// shallowest level in it. A converted EPUB carries the book's title as a
|
||||
// lone `#` above sixty-two `##` chapters, and taking the lone one makes
|
||||
// the whole book a single chapter — so a level used once is a title and a
|
||||
// level used twice is a chapter map.
|
||||
let mut counts: BTreeMap<usize, usize> = BTreeMap::new();
|
||||
for (level, _, _) in &headings {
|
||||
*counts.entry(*level).or_default() += 1;
|
||||
}
|
||||
let dividing = counts
|
||||
.iter()
|
||||
.map(|(level, _, _)| *level)
|
||||
.min()
|
||||
.find(|(_, used)| **used > 1)
|
||||
.map(|(level, _)| *level)
|
||||
.or_else(|| counts.keys().next().copied())
|
||||
.expect("a heading");
|
||||
headings.retain(|(level, _, _)| *level == shallowest);
|
||||
headings.retain(|(level, _, _)| *level == dividing);
|
||||
|
||||
let page_at = PageIndex::new(markdown);
|
||||
let mut chapters = Vec::with_capacity(headings.len());
|
||||
@@ -245,6 +256,22 @@ mod tests {
|
||||
assert_eq!(pages[0].number, Some(7));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_lone_shallow_heading_is_a_title_rather_than_a_chapter_map() {
|
||||
// What a converted EPUB looks like: the book's title as the only
|
||||
// `#`, and every actual chapter a `##` under it. Taking the lone
|
||||
// one makes the whole book a single chapter.
|
||||
let markdown = "# The Whole Book\n\n## Chapter One\n\ntext\n\n## Chapter Two\n\ntext";
|
||||
let found = chapters(markdown);
|
||||
assert_eq!(
|
||||
found
|
||||
.iter()
|
||||
.map(|one| one.title.as_str())
|
||||
.collect::<Vec<_>>(),
|
||||
["Chapter One", "Chapter Two"]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chapters_come_from_the_shallowest_heading_level_present() {
|
||||
let markdown = "## Alpha\n\ntext\n\n### Sub\n\ntext\n\n## Beta\n\ntext";
|
||||
|
||||
@@ -18,8 +18,10 @@ fn pattern(source: &str) -> Regex {
|
||||
/// `[text](url)` — a heading wrapped in a link to its own anchor.
|
||||
static LINK: LazyLock<Regex> = LazyLock::new(|| pattern(r"\[([^\]]*)\]\([^)]*\)"));
|
||||
|
||||
/// `[text]{.cls}` — pandoc's bracketed inline span.
|
||||
static BRACKETED: LazyLock<Regex> = LazyLock::new(|| pattern(r"\[([^\]]+)\]\{[^}]*\}"));
|
||||
/// `[text]{.cls}` — pandoc's bracketed inline span. The text may be empty:
|
||||
/// `[]{#chapter-3}` is how an anchor with nothing in it arrives, and a
|
||||
/// heading that keeps one is a chapter title with an element id in it.
|
||||
static BRACKETED: LazyLock<Regex> = LazyLock::new(|| pattern(r"\[([^\]]*)\]\{[^}]*\}"));
|
||||
|
||||
/// A trailing `{.attr}` block.
|
||||
static ATTRIBUTE_TAIL: LazyLock<Regex> =
|
||||
|
||||
@@ -93,6 +93,11 @@ library.
|
||||
The chapter a passage carries is the nearest heading above the point the
|
||||
passage *starts*, found by byte offset into the source.
|
||||
|
||||
The heading level that divides a document is the shallowest one used more
|
||||
than once, not simply the shallowest one present: a converted EPUB carries the
|
||||
book's title as a lone `#` above sixty-two `##` chapters, and a level used
|
||||
once is a title rather than a chapter map.
|
||||
|
||||
The offset has to be the passage's own. A document with no page markers is a
|
||||
single page beginning at zero, so asking where the *page* started puts every
|
||||
chunk in the book under whichever heading came first in the file — which also
|
||||
|
||||
Generated
+603
@@ -0,0 +1,603 @@
|
||||
# This file is automatically @generated by Cargo.
|
||||
# It is not intended for manual editing.
|
||||
version = 4
|
||||
|
||||
[[package]]
|
||||
name = "aho-corasick"
|
||||
version = "1.1.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba"
|
||||
dependencies = [
|
||||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anstream"
|
||||
version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d"
|
||||
dependencies = [
|
||||
"anstyle",
|
||||
"anstyle-parse",
|
||||
"anstyle-query",
|
||||
"anstyle-wincon",
|
||||
"colorchoice",
|
||||
"is_terminal_polyfill",
|
||||
"utf8parse",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anstyle"
|
||||
version = "1.0.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000"
|
||||
|
||||
[[package]]
|
||||
name = "anstyle-parse"
|
||||
version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e"
|
||||
dependencies = [
|
||||
"utf8parse",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anstyle-query"
|
||||
version = "1.1.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc"
|
||||
dependencies = [
|
||||
"windows-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anstyle-wincon"
|
||||
version = "3.0.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d"
|
||||
dependencies = [
|
||||
"anstyle",
|
||||
"once_cell_polyfill",
|
||||
"windows-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anyhow"
|
||||
version = "1.0.104"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470"
|
||||
|
||||
[[package]]
|
||||
name = "base64"
|
||||
version = "0.22.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
|
||||
|
||||
[[package]]
|
||||
name = "bibclean"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"catena-cli",
|
||||
"clap",
|
||||
"corpus-core",
|
||||
"deunicode",
|
||||
"fancy-regex 0.16.2",
|
||||
"regex",
|
||||
"schemars",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"textkit",
|
||||
"unicode-normalization",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bit-set"
|
||||
version = "0.8.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3"
|
||||
dependencies = [
|
||||
"bit-vec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bit-vec"
|
||||
version = "0.8.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7"
|
||||
|
||||
[[package]]
|
||||
name = "booksniff"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"catena-cli",
|
||||
"clap",
|
||||
"corpus-core",
|
||||
"serde_json",
|
||||
"textkit",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bstr"
|
||||
version = "1.13.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6bb31b46c14244e20ee9984b11bf5c992b91fb6939fea616e3512c8baecdbe5f"
|
||||
dependencies = [
|
||||
"memchr",
|
||||
"regex-automata",
|
||||
"serde_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "catena-cli"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"clap",
|
||||
"clap_mangen",
|
||||
"corpus-core",
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "catena-example"
|
||||
version = "0.0.0"
|
||||
dependencies = [
|
||||
"booksniff",
|
||||
"chunkmd",
|
||||
"citefind",
|
||||
"corpus-core",
|
||||
"mdscrub",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "chunkmd"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"catena-cli",
|
||||
"clap",
|
||||
"corpus-core",
|
||||
"fancy-regex 0.16.2",
|
||||
"regex",
|
||||
"textkit",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "citefind"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"catena-cli",
|
||||
"chunkmd",
|
||||
"clap",
|
||||
"corpus-core",
|
||||
"idnorm",
|
||||
"regex",
|
||||
"schemars",
|
||||
"serde",
|
||||
"serde_json",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "clap"
|
||||
version = "4.6.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "aa8876b300ab35ba921adea3dfd70157a46249b33f95c9084ae5709785478946"
|
||||
dependencies = [
|
||||
"clap_builder",
|
||||
"clap_derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "clap_builder"
|
||||
version = "4.6.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ec0797fb7aeb1406c84efac526901f7ec3ead2124f946b494e72879d4b54704d"
|
||||
dependencies = [
|
||||
"anstream",
|
||||
"anstyle",
|
||||
"clap_lex",
|
||||
"strsim",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "clap_derive"
|
||||
version = "4.6.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f9c751b79415d4e559e3d1fcf128e09e720eb673a06d26cf6f392d37d75b66e0"
|
||||
dependencies = [
|
||||
"heck",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "clap_lex"
|
||||
version = "1.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1c133bc6a41be0d194c306b5506d15e6feeea7b1d6604bd3f8310dfb2ca96486"
|
||||
|
||||
[[package]]
|
||||
name = "clap_mangen"
|
||||
version = "0.3.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "211d617eaa4b735c96c9e0228fcbdb5120ef623f2b8cb67ffb84c3e02dbc28a4"
|
||||
dependencies = [
|
||||
"clap",
|
||||
"roff",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "colorchoice"
|
||||
version = "1.0.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570"
|
||||
|
||||
[[package]]
|
||||
name = "corpus-core"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"schemars",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"thiserror",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "deunicode"
|
||||
version = "1.6.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "abd57806937c9cc163efc8ea3910e00a62e2aeb0b8119f1793a978088f8f6b04"
|
||||
|
||||
[[package]]
|
||||
name = "dyn-clone"
|
||||
version = "1.0.20"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555"
|
||||
|
||||
[[package]]
|
||||
name = "fancy-regex"
|
||||
version = "0.16.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "998b056554fbe42e03ae0e152895cd1a7e1002aec800fdc6635d20270260c46f"
|
||||
dependencies = [
|
||||
"bit-set",
|
||||
"regex-automata",
|
||||
"regex-syntax",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "fancy-regex"
|
||||
version = "0.17.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "72cf461f865c862bb7dc573f643dd6a2b6842f7c30b07882b56bd148cc2761b8"
|
||||
dependencies = [
|
||||
"bit-set",
|
||||
"regex-automata",
|
||||
"regex-syntax",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "heck"
|
||||
version = "0.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
|
||||
|
||||
[[package]]
|
||||
name = "idnorm"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"bibclean",
|
||||
"catena-cli",
|
||||
"clap",
|
||||
"corpus-core",
|
||||
"fancy-regex 0.16.2",
|
||||
"schemars",
|
||||
"serde",
|
||||
"serde_json",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "is_terminal_polyfill"
|
||||
version = "1.70.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695"
|
||||
|
||||
[[package]]
|
||||
name = "itoa"
|
||||
version = "1.0.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
|
||||
|
||||
[[package]]
|
||||
name = "lazy_static"
|
||||
version = "1.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
|
||||
|
||||
[[package]]
|
||||
name = "mdscrub"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"catena-cli",
|
||||
"clap",
|
||||
"corpus-core",
|
||||
"fancy-regex 0.16.2",
|
||||
"regex",
|
||||
"textkit",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "memchr"
|
||||
version = "2.8.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"
|
||||
|
||||
[[package]]
|
||||
name = "once_cell_polyfill"
|
||||
version = "1.70.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe"
|
||||
|
||||
[[package]]
|
||||
name = "proc-macro2"
|
||||
version = "1.0.107"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9"
|
||||
dependencies = [
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quote"
|
||||
version = "1.0.47"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ref-cast"
|
||||
version = "1.0.27"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7e440fb4e4b4147295338efb76001ab9e4efc0e5839df2c47fc5ac2381d365c3"
|
||||
dependencies = [
|
||||
"ref-cast-impl",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ref-cast-impl"
|
||||
version = "1.0.27"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "92ecd8964f8453721699a1ed72037b0db49ce2f5a5138486ee89bed6f67cdf3a"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "regex"
|
||||
version = "1.13.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d"
|
||||
dependencies = [
|
||||
"aho-corasick",
|
||||
"memchr",
|
||||
"regex-automata",
|
||||
"regex-syntax",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "regex-automata"
|
||||
version = "0.4.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2"
|
||||
dependencies = [
|
||||
"aho-corasick",
|
||||
"memchr",
|
||||
"regex-syntax",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "regex-syntax"
|
||||
version = "0.8.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4"
|
||||
|
||||
[[package]]
|
||||
name = "roff"
|
||||
version = "1.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "323c417e1d9665a65b263ec744ba09030cfb277e9daa0b018a4ab62e57bc8189"
|
||||
|
||||
[[package]]
|
||||
name = "rustc-hash"
|
||||
version = "2.1.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d"
|
||||
|
||||
[[package]]
|
||||
name = "schemars"
|
||||
version = "1.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "687274d293b6cdc6e73e0fee520bf2049650090d7164f87672d212a3c530cf4a"
|
||||
dependencies = [
|
||||
"dyn-clone",
|
||||
"ref-cast",
|
||||
"schemars_derive",
|
||||
"serde",
|
||||
"serde_json",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "schemars_derive"
|
||||
version = "1.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d98c67716b46af2f0b8cf752abc930f6f9aecfbf671ecfb531db8a31dbe4e2ba"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"serde_derive_internals",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde"
|
||||
version = "1.0.229"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba"
|
||||
dependencies = [
|
||||
"serde_core",
|
||||
"serde_derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_core"
|
||||
version = "1.0.229"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48"
|
||||
dependencies = [
|
||||
"serde_derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_derive"
|
||||
version = "1.0.229"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_derive_internals"
|
||||
version = "0.30.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f852137cce035d6a4df67ccce505ff6b3e9fd3a10e3e52b24dc71e650bb1a9bd"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_json"
|
||||
version = "1.0.151"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14"
|
||||
dependencies = [
|
||||
"itoa",
|
||||
"memchr",
|
||||
"serde",
|
||||
"serde_core",
|
||||
"zmij",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "strsim"
|
||||
version = "0.11.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "3.0.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "textkit"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"tiktoken-rs",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "thiserror"
|
||||
version = "2.0.20"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f"
|
||||
dependencies = [
|
||||
"thiserror-impl",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "thiserror-impl"
|
||||
version = "2.0.20"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tiktoken-rs"
|
||||
version = "0.12.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "027853bbf8c7763b77c5c595f1c271c7d536ced7d6f83452911b944621e57fc2"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"base64",
|
||||
"bstr",
|
||||
"fancy-regex 0.17.0",
|
||||
"lazy_static",
|
||||
"regex",
|
||||
"rustc-hash",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tinyvec"
|
||||
version = "1.13.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fd3ca314f692efd6c868f8408f53fe444634a845f96c028b97d35f6a1f79f0ee"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-ident"
|
||||
version = "1.0.26"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-normalization"
|
||||
version = "0.1.25"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8"
|
||||
dependencies = [
|
||||
"tinyvec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "utf8parse"
|
||||
version = "0.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
|
||||
|
||||
[[package]]
|
||||
name = "windows-link"
|
||||
version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"
|
||||
|
||||
[[package]]
|
||||
name = "windows-sys"
|
||||
version = "0.61.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc"
|
||||
dependencies = [
|
||||
"windows-link",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zmij"
|
||||
version = "1.0.23"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b"
|
||||
@@ -0,0 +1,16 @@
|
||||
# Standalone so that `cargo run` here does not need the workspace above it.
|
||||
[workspace]
|
||||
|
||||
[package]
|
||||
name = "catena-example"
|
||||
version = "0.0.0"
|
||||
edition = "2024"
|
||||
rust-version = "1.85"
|
||||
publish = false
|
||||
|
||||
[dependencies]
|
||||
booksniff = { path = "../../crates/booksniff" }
|
||||
chunkmd = { path = "../../crates/chunkmd" }
|
||||
citefind = { path = "../../crates/citefind" }
|
||||
corpus-core = { path = "../../crates/corpus-core" }
|
||||
mdscrub = { path = "../../crates/mdscrub" }
|
||||
@@ -0,0 +1,119 @@
|
||||
//! A converted book, read the whole way through in one process.
|
||||
//!
|
||||
//! The commands are filters and a subprocess is usually the right answer, but
|
||||
//! when the caller already holds the text there is nothing to serialise: the
|
||||
//! same libraries the commands are built from take a `&str` and hand back the
|
||||
//! records. This walks a file from "what even is it" to passages worth
|
||||
//! embedding and the references at the back, without a pipe anywhere.
|
||||
//!
|
||||
//! cargo run -- book.md
|
||||
//! cargo run # reads the sample below
|
||||
|
||||
use std::path::Path;
|
||||
|
||||
/// A page of a converted book: a heading, two passages and a short
|
||||
/// bibliography, which is enough to show every stage doing something.
|
||||
const SAMPLE: &str = "\
|
||||
# III. The Second Estate
|
||||
|
||||
The obligation ran both ways, and contemporaries said so plainly whenever
|
||||
they were asked.[^4] A lord who could not protect was a lord who could not
|
||||
command, and the rolls record the consequences.
|
||||
|
||||
What the rolls do not say, and what has to be inferred, is how often the
|
||||
obligation was actually tested. <!-- a converter's leftover -->
|
||||
|
||||
# Bibliography
|
||||
|
||||
Bloch, Marc. *Feudal Society*. Translated by L. A. Manyon. London: Routledge
|
||||
and Kegan Paul, 1961.
|
||||
|
||||
Duby, Georges. *The Three Orders: Feudal Society Imagined*. Chicago:
|
||||
University of Chicago Press, 1980.
|
||||
|
||||
Reynolds, Susan. *Fiefs and Vassals: The Medieval Evidence Reinterpreted*.
|
||||
Oxford: Clarendon Press, 1994.
|
||||
";
|
||||
|
||||
fn main() {
|
||||
let (name, raw) = match std::env::args().nth(1) {
|
||||
Some(path) => {
|
||||
let bytes = std::fs::read(&path).unwrap_or_else(|why| {
|
||||
eprintln!("catena-example: {path}: {why}");
|
||||
std::process::exit(74);
|
||||
});
|
||||
(path, bytes)
|
||||
}
|
||||
None => ("sample.md".to_string(), SAMPLE.as_bytes().to_vec()),
|
||||
};
|
||||
|
||||
// What the file actually is, whatever it is called. Worth asking first:
|
||||
// a `.pdf` that is really a DjVu fails four stages later, in a way that
|
||||
// looks like a bug in the converter.
|
||||
let doc = booksniff::sniff(&raw.as_slice(), &name);
|
||||
let stem = Path::new(&name).file_name().unwrap_or_default();
|
||||
println!("{:?} is a {:?}, {} bytes", stem, doc.kind, doc.bytes);
|
||||
for note in &doc.notes {
|
||||
println!(" note: {note}");
|
||||
}
|
||||
|
||||
// Converted markdown carries the converter's leftovers. Scrubbing before
|
||||
// chunking means the passages are the book's words and not the tool's.
|
||||
let markdown = String::from_utf8_lossy(&raw);
|
||||
let clean = mdscrub::scrub(&markdown);
|
||||
|
||||
// The defaults are sized for a book: 512 tokens a chunk, which would put
|
||||
// this whole sample in one. A page needs a page's budget.
|
||||
let budgets = match std::env::args().nth(1) {
|
||||
Some(_) => chunkmd::Budgets::default(),
|
||||
None => chunkmd::Budgets {
|
||||
target: 120,
|
||||
overlap: 0,
|
||||
min_tokens: 10,
|
||||
..chunkmd::Budgets::default()
|
||||
},
|
||||
};
|
||||
let chunks = chunkmd::chunk(&clean, &budgets);
|
||||
println!("\n{} passage(s):", chunks.len());
|
||||
for chunk in &chunks {
|
||||
let chapter = chunk.chapter.as_deref().unwrap_or("—");
|
||||
let opening: String = chunk.text.chars().take(64).collect();
|
||||
println!(
|
||||
" [{}] {:?} {} tokens, under {chapter:?}\n {}…",
|
||||
chunk.idx,
|
||||
chunk.role,
|
||||
chunk.tokens,
|
||||
opening.replace('\n', " ")
|
||||
);
|
||||
}
|
||||
|
||||
// The references at the back. A passage whose role is `bibliography` is
|
||||
// where they live; a note at the foot of a page is the other place, and
|
||||
// `chunkmd` flags those so this does not have to guess.
|
||||
let references: Vec<citefind::Reference> = chunks
|
||||
.iter()
|
||||
.filter(|chunk| chunk.role == corpus_core::Role::Bibliography)
|
||||
.flat_map(|chunk| citefind::read(&chunk.text))
|
||||
.filter(|found| !found.continuation)
|
||||
.collect();
|
||||
|
||||
println!("\n{} reference(s):", references.len());
|
||||
for found in &references {
|
||||
let who = match found.authors.is_empty() {
|
||||
true => "—".to_string(),
|
||||
false => found.authors.join("; "),
|
||||
};
|
||||
println!(
|
||||
" {who}\n {} ({}) — {:?}",
|
||||
found.title.as_deref().unwrap_or("—"),
|
||||
found
|
||||
.year
|
||||
.map(|year| year.to_string())
|
||||
.unwrap_or_else(|| "no year".into()),
|
||||
found.kind
|
||||
);
|
||||
if !found.unsettled.is_empty() {
|
||||
println!(" could not read: {}", found.unsettled.join(", "));
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user