mirror of
https://git.pumpin.online/funkpower/catena.git
synced 2026-10-23 03:07:13 +00:00
chunkmd: page and chapter aware chunking, roles, footnote flag
This commit is contained in:
funkpower
Generated
+167
-1
@@ -11,6 +11,56 @@ dependencies = [
|
|||||||
"memchr",
|
"memchr",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "anstream"
|
||||||
|
version = "1.0.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d"
|
||||||
|
dependencies = [
|
||||||
|
"anstyle",
|
||||||
|
"anstyle-parse",
|
||||||
|
"anstyle-query",
|
||||||
|
"anstyle-wincon",
|
||||||
|
"colorchoice",
|
||||||
|
"is_terminal_polyfill",
|
||||||
|
"utf8parse",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "anstyle"
|
||||||
|
version = "1.0.14"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "anstyle-parse"
|
||||||
|
version = "1.0.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e"
|
||||||
|
dependencies = [
|
||||||
|
"utf8parse",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "anstyle-query"
|
||||||
|
version = "1.1.5"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc"
|
||||||
|
dependencies = [
|
||||||
|
"windows-sys",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "anstyle-wincon"
|
||||||
|
version = "3.0.11"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d"
|
||||||
|
dependencies = [
|
||||||
|
"anstyle",
|
||||||
|
"once_cell_polyfill",
|
||||||
|
"windows-sys",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "anyhow"
|
name = "anyhow"
|
||||||
version = "1.0.104"
|
version = "1.0.104"
|
||||||
@@ -49,6 +99,63 @@ dependencies = [
|
|||||||
"serde_core",
|
"serde_core",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "chunkmd"
|
||||||
|
version = "0.1.0"
|
||||||
|
dependencies = [
|
||||||
|
"clap",
|
||||||
|
"corpus-core",
|
||||||
|
"fancy-regex 0.16.2",
|
||||||
|
"regex",
|
||||||
|
"textkit",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "clap"
|
||||||
|
version = "4.6.7"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "aa8876b300ab35ba921adea3dfd70157a46249b33f95c9084ae5709785478946"
|
||||||
|
dependencies = [
|
||||||
|
"clap_builder",
|
||||||
|
"clap_derive",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "clap_builder"
|
||||||
|
version = "4.6.7"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "ec0797fb7aeb1406c84efac526901f7ec3ead2124f946b494e72879d4b54704d"
|
||||||
|
dependencies = [
|
||||||
|
"anstream",
|
||||||
|
"anstyle",
|
||||||
|
"clap_lex",
|
||||||
|
"strsim",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "clap_derive"
|
||||||
|
version = "4.6.7"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "f9c751b79415d4e559e3d1fcf128e09e720eb673a06d26cf6f392d37d75b66e0"
|
||||||
|
dependencies = [
|
||||||
|
"heck",
|
||||||
|
"proc-macro2",
|
||||||
|
"quote",
|
||||||
|
"syn",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "clap_lex"
|
||||||
|
version = "1.1.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "1c133bc6a41be0d194c306b5506d15e6feeea7b1d6604bd3f8310dfb2ca96486"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "colorchoice"
|
||||||
|
version = "1.0.5"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "corpus-core"
|
name = "corpus-core"
|
||||||
version = "0.1.0"
|
version = "0.1.0"
|
||||||
@@ -65,6 +172,17 @@ version = "1.0.20"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555"
|
checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "fancy-regex"
|
||||||
|
version = "0.16.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "998b056554fbe42e03ae0e152895cd1a7e1002aec800fdc6635d20270260c46f"
|
||||||
|
dependencies = [
|
||||||
|
"bit-set",
|
||||||
|
"regex-automata",
|
||||||
|
"regex-syntax",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "fancy-regex"
|
name = "fancy-regex"
|
||||||
version = "0.17.0"
|
version = "0.17.0"
|
||||||
@@ -76,6 +194,18 @@ dependencies = [
|
|||||||
"regex-syntax",
|
"regex-syntax",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "heck"
|
||||||
|
version = "0.5.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "is_terminal_polyfill"
|
||||||
|
version = "1.70.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "itoa"
|
name = "itoa"
|
||||||
version = "1.0.18"
|
version = "1.0.18"
|
||||||
@@ -94,10 +224,19 @@ version = "2.8.3"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"
|
checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "once_cell_polyfill"
|
||||||
|
version = "1.70.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "parity"
|
name = "parity"
|
||||||
version = "0.1.0"
|
version = "0.1.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
|
"chunkmd",
|
||||||
|
"corpus-core",
|
||||||
|
"serde_json",
|
||||||
"textkit",
|
"textkit",
|
||||||
]
|
]
|
||||||
|
|
||||||
@@ -253,6 +392,12 @@ dependencies = [
|
|||||||
"zmij",
|
"zmij",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "strsim"
|
||||||
|
version = "0.11.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "syn"
|
name = "syn"
|
||||||
version = "3.0.6"
|
version = "3.0.6"
|
||||||
@@ -300,7 +445,7 @@ dependencies = [
|
|||||||
"anyhow",
|
"anyhow",
|
||||||
"base64",
|
"base64",
|
||||||
"bstr",
|
"bstr",
|
||||||
"fancy-regex",
|
"fancy-regex 0.17.0",
|
||||||
"lazy_static",
|
"lazy_static",
|
||||||
"regex",
|
"regex",
|
||||||
"rustc-hash",
|
"rustc-hash",
|
||||||
@@ -312,6 +457,27 @@ version = "1.0.26"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954"
|
checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "utf8parse"
|
||||||
|
version = "0.2.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "windows-link"
|
||||||
|
version = "0.2.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "windows-sys"
|
||||||
|
version = "0.61.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc"
|
||||||
|
dependencies = [
|
||||||
|
"windows-link",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "zmij"
|
name = "zmij"
|
||||||
version = "1.0.23"
|
version = "1.0.23"
|
||||||
|
|||||||
@@ -12,8 +12,11 @@ repository = "https://git.pumpin.online/funkpower/catena"
|
|||||||
[workspace.dependencies]
|
[workspace.dependencies]
|
||||||
corpus-core = { path = "crates/corpus-core", version = "0.1.0" }
|
corpus-core = { path = "crates/corpus-core", version = "0.1.0" }
|
||||||
textkit = { path = "crates/textkit", version = "0.1.0" }
|
textkit = { path = "crates/textkit", version = "0.1.0" }
|
||||||
|
chunkmd = { path = "crates/chunkmd", version = "0.1.0" }
|
||||||
|
|
||||||
clap = { version = "4", features = ["derive"] }
|
clap = { version = "4", features = ["derive"] }
|
||||||
|
fancy-regex = "0.16"
|
||||||
|
regex = "1"
|
||||||
schemars = "1"
|
schemars = "1"
|
||||||
serde = { version = "1", features = ["derive"] }
|
serde = { version = "1", features = ["derive"] }
|
||||||
serde_json = "1"
|
serde_json = "1"
|
||||||
|
|||||||
@@ -0,0 +1,18 @@
|
|||||||
|
[package]
|
||||||
|
name = "chunkmd"
|
||||||
|
description = "Cuts markdown into retrievable passages that carry their page, chapter and role."
|
||||||
|
version.workspace = true
|
||||||
|
edition.workspace = true
|
||||||
|
rust-version.workspace = true
|
||||||
|
license.workspace = true
|
||||||
|
repository.workspace = true
|
||||||
|
|
||||||
|
[dependencies]
|
||||||
|
clap.workspace = true
|
||||||
|
corpus-core.workspace = true
|
||||||
|
fancy-regex.workspace = true
|
||||||
|
regex.workspace = true
|
||||||
|
textkit.workspace = true
|
||||||
|
|
||||||
|
[lints]
|
||||||
|
workspace = true
|
||||||
@@ -0,0 +1,179 @@
|
|||||||
|
//! Spotting a chunk that is really a page of numbered notes.
|
||||||
|
//!
|
||||||
|
//! Footnotes and endnotes are citations in disguise: they read as body text,
|
||||||
|
//! rank as body text, and answer a query with a publisher and a page number.
|
||||||
|
//! Flagging them lets a consumer treat them as the bibliography they are
|
||||||
|
//! without moving them out of the body, where their page provenance lives.
|
||||||
|
|
||||||
|
use std::sync::LazyLock;
|
||||||
|
|
||||||
|
use fancy_regex::Regex as FancyRegex;
|
||||||
|
use regex::Regex;
|
||||||
|
|
||||||
|
/// A private-use codepoint marking the start of a note's text.
|
||||||
|
///
|
||||||
|
/// EPUB3 note definitions carry a structural signal — `epub:type="footnote"`
|
||||||
|
/// and the ARIA `doc-footnote` role — that is reliable in a way the visible
|
||||||
|
/// marker never is: `[65]`, `fn39.`, a bare superscript, all of them and
|
||||||
|
/// none of them. A converter that can see that structure marks it with this
|
||||||
|
/// character, which survives conversion to markdown verbatim. When it is
|
||||||
|
/// present, nothing below has to guess.
|
||||||
|
///
|
||||||
|
/// It is stripped from the text a chunk carries, after the flag is set.
|
||||||
|
pub const SENTINEL: char = '\u{e000}';
|
||||||
|
|
||||||
|
/// How far past its marker a note's citation signal may sit, in characters.
|
||||||
|
///
|
||||||
|
/// Citation signals sit near the front of an entry. The cap stops a marker
|
||||||
|
/// followed by pages of prose from matching a signal far away. HTML class
|
||||||
|
/// soup inflates entries, so it is generous.
|
||||||
|
const ENTRY_WINDOW: usize = 500;
|
||||||
|
|
||||||
|
/// At least this many numbered-note lines have to carry a citation signal.
|
||||||
|
///
|
||||||
|
/// Two, because one avoids a false positive from a stray "1. " list item,
|
||||||
|
/// and a real page of notes almost always has more.
|
||||||
|
const MIN_CITED_NOTES: usize = 2;
|
||||||
|
|
||||||
|
/// The start of a numbered note, in any of the shapes conversion leaves
|
||||||
|
/// behind. A marker only counts at a line start or after a run of two or
|
||||||
|
/// more spaces — several notes can share one physical line — so that a
|
||||||
|
/// superscript *reference* to a note, mid-sentence, does not qualify.
|
||||||
|
static NOTE_MARKER: LazyLock<FancyRegex> = LazyLock::new(|| {
|
||||||
|
FancyRegex::new(
|
||||||
|
r"(?xm)
|
||||||
|
(?: ^[ \t]{0,8} (?:-[ \t]+)? # line start, optional '- ' list bullet
|
||||||
|
| (?<=\s\s) # or after a 2+ space run
|
||||||
|
| (?<=\ -\ ) # or after ' - ' (inline list separator)
|
||||||
|
)
|
||||||
|
(?:
|
||||||
|
\d{1,3} \\?[.)] [ \t]+ \S # plain '12. ' or escaped '12\. '
|
||||||
|
| \[ \d{1,3} \] [ \t]* \S # bracket-numbered '[12] Author'
|
||||||
|
| \d{1,3} [ \t]+ \S # bare '12 ' from a page bottom
|
||||||
|
| [¹²³⁰-⁹]+ \s* \S # unicode superscript marker
|
||||||
|
| (?:fn|ftn|note) \d{1,4} [.)] [ \t]+ \S # id-style 'fn39. '
|
||||||
|
| (?:<span[^>]*>\s*)* # outer span wrap
|
||||||
|
<a\b[^>]*> \s* (?:<span[^>]*>\s*)* (?:<sup>\s*)?
|
||||||
|
\d{1,3} \s* [.)]?
|
||||||
|
\s* (?:</sup>\s*)? (?:</span>\s*)* </a>
|
||||||
|
(?:\s*</span>)* \s* [.)]? # anchor forms
|
||||||
|
| \[ \s* (?:<span[^>]*>)? \s* \d{1,3}
|
||||||
|
\s* (?:</span>)? \s* \] \( \#[^)]* \) # [42](#...)
|
||||||
|
| <sup> \s* \d{1,3} \s* </sup> # bare sup note
|
||||||
|
)",
|
||||||
|
)
|
||||||
|
.expect("a valid pattern")
|
||||||
|
});
|
||||||
|
|
||||||
|
/// What makes a note bibliographic rather than discursive.
|
||||||
|
static CITE_SIGNAL: LazyLock<Regex> = LazyLock::new(|| {
|
||||||
|
Regex::new(
|
||||||
|
r"(?ix)
|
||||||
|
\((?:[^()]*\b)?(?:1[5-9]|20)\d{2}\) # parenthesised year: (Oxford, 1998)
|
||||||
|
| \bpp?\.\s*\d # p. 12 / pp. 12-34
|
||||||
|
| \bvol\.\s*\d # vol. 2
|
||||||
|
| \bIbid\b | \bop\.\s*cit\b | \bloc\.\s*cit\b
|
||||||
|
| \bet\s+al\b
|
||||||
|
| \b(?:University|Clarendon|Academic)\s+Press\b | \bPress,\s
|
||||||
|
| \beds?\.\s | \btrans\.\s
|
||||||
|
| \*[^*\n]{3,80}\*\s*,?\s*\d # short form: *Title*, 183
|
||||||
|
| <span[^>]*italic[^>]*> # HTML-italic title survives conversion
|
||||||
|
| ,\s+\d{1,4}\s*[.;] # trailing page: '..., 183.'
|
||||||
|
| https?:// # web citations (URLs, DOIs)",
|
||||||
|
)
|
||||||
|
.expect("a valid pattern")
|
||||||
|
});
|
||||||
|
|
||||||
|
/// Whether a chunk carries footnote or endnote citation material.
|
||||||
|
///
|
||||||
|
/// The structural signal first, when the source had one. Otherwise each note
|
||||||
|
/// entry runs from its marker to the next one, capped at a few hundred
|
||||||
|
/// characters, and enough of them have to look like citations.
|
||||||
|
pub fn looks_like_citations(text: &str) -> bool {
|
||||||
|
if text.contains(SENTINEL) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
let starts: Vec<usize> = NOTE_MARKER
|
||||||
|
.find_iter(text)
|
||||||
|
.filter_map(Result::ok)
|
||||||
|
.map(|found| found.start())
|
||||||
|
.collect();
|
||||||
|
if starts.len() < MIN_CITED_NOTES {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut cited = 0;
|
||||||
|
for (at, start) in starts.iter().enumerate() {
|
||||||
|
let end = starts.get(at + 1).copied().unwrap_or(text.len());
|
||||||
|
let entry: String = text[*start..end].chars().take(ENTRY_WINDOW).collect();
|
||||||
|
if CITE_SIGNAL.is_match(&entry) {
|
||||||
|
cited += 1;
|
||||||
|
if cited >= MIN_CITED_NOTES {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
false
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Remove the structural marker from text on its way out.
|
||||||
|
pub fn strip_sentinel(text: &str) -> String {
|
||||||
|
text.replace(SENTINEL, "")
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn the_structural_marker_settles_it_on_its_own() {
|
||||||
|
assert!(looks_like_citations(&format!("{SENTINEL}See above.")));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn two_numbered_notes_with_citation_signals_count() {
|
||||||
|
let text = "12. Arendt, The Human Condition (Chicago, 1958), 44.\n\
|
||||||
|
13. Bloch, Feudal Society, vol. 2, p. 17.";
|
||||||
|
assert!(looks_like_citations(text));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn one_note_is_not_enough() {
|
||||||
|
assert!(!looks_like_citations(
|
||||||
|
"12. Arendt, The Human Condition (Chicago, 1958), 44."
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_numbered_list_without_citations_is_not_notes() {
|
||||||
|
let text = "1. Boil the water.\n2. Add the salt.\n3. Wait.";
|
||||||
|
assert!(!looks_like_citations(text));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_mid_sentence_reference_is_not_a_note() {
|
||||||
|
let text = "As shown elsewhere 12. and again 13. the argument holds \
|
||||||
|
(Oxford, 1998) and p. 12.";
|
||||||
|
assert!(!looks_like_citations(text));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_signal_far_past_its_marker_does_not_count() {
|
||||||
|
let filler = "word ".repeat(200);
|
||||||
|
let text = format!("12. {filler}(Oxford, 1998)\n13. {filler}p. 44");
|
||||||
|
assert!(!looks_like_citations(&text));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn anchor_wrapped_markers_are_recognised() {
|
||||||
|
let text = "<a href=\"#ref1\"><sup>1</sup></a>. Arendt, (Chicago, 1958), 44.\n\
|
||||||
|
<a href=\"#ref2\"><sup>2</sup></a>. Bloch, p. 17.";
|
||||||
|
assert!(looks_like_citations(text));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn the_marker_is_removed_from_the_text_that_ships() {
|
||||||
|
assert_eq!(strip_sentinel(&format!("{SENTINEL}note")), "note");
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,187 @@
|
|||||||
|
//! Cuts markdown into retrievable passages that carry their page, chapter
|
||||||
|
//! and role.
|
||||||
|
//!
|
||||||
|
//! The order of the passes is load-bearing and is not obvious:
|
||||||
|
//!
|
||||||
|
//! 1. Split on page markers, so a chunk never spans a page and its page
|
||||||
|
//! number is exact rather than inferred.
|
||||||
|
//! 2. Pack paragraphs toward a token target within each page, carrying an
|
||||||
|
//! overlap forward.
|
||||||
|
//! 3. Enforce the hard maximum on token boundaries, whatever the structure
|
||||||
|
//! said.
|
||||||
|
//! 4. Drop what is too small to be worth an embedding.
|
||||||
|
//! 5. Flag the note chunks, *before* the structural marker is stripped out
|
||||||
|
//! of their text.
|
||||||
|
//! 6. Classify roles across the whole document at once, because smoothing a
|
||||||
|
//! stray chunk needs to see its neighbours.
|
||||||
|
|
||||||
|
pub mod footnote;
|
||||||
|
pub mod pack;
|
||||||
|
pub mod pages;
|
||||||
|
pub mod role;
|
||||||
|
|
||||||
|
use corpus_core::{Chunk, chunk::flags};
|
||||||
|
|
||||||
|
pub use pack::Budgets;
|
||||||
|
|
||||||
|
/// Cut `markdown` into chunks.
|
||||||
|
pub fn chunk(markdown: &str, budgets: &Budgets) -> Vec<Chunk> {
|
||||||
|
let pages = pages::split_by_page(markdown);
|
||||||
|
let chapters = pages::chapters(markdown);
|
||||||
|
let by_page = pages::page_chapters(&chapters);
|
||||||
|
let by_offset = pages::ChapterIndex::new(&chapters);
|
||||||
|
|
||||||
|
let mut chunks: Vec<Chunk> = Vec::new();
|
||||||
|
for page in pages {
|
||||||
|
for packed in pack::page(page.text, budgets) {
|
||||||
|
for piece in pack::enforce_hard_max(&packed, budgets.hard_max) {
|
||||||
|
let tokens = textkit::count(&piece);
|
||||||
|
if tokens < budgets.min_tokens {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
// A page-based chapter when the document has page markers;
|
||||||
|
// the byte offset otherwise, which is what carries EPUBs
|
||||||
|
// that have headings but no page anchors.
|
||||||
|
let chapter = page
|
||||||
|
.number
|
||||||
|
.and_then(|number| by_page.get(&number).copied())
|
||||||
|
.or_else(|| by_offset.at(page.offset));
|
||||||
|
|
||||||
|
let idx = chunks.len() as u32;
|
||||||
|
let notes = footnote::looks_like_citations(&piece);
|
||||||
|
let mut chunk = Chunk::new(idx, footnote::strip_sentinel(&piece), tokens as u32);
|
||||||
|
chunk.page = page.number;
|
||||||
|
chunk.chapter = chapter.map(str::to_string);
|
||||||
|
if notes {
|
||||||
|
chunk.flag(flags::FOOTNOTE_CITATIONS);
|
||||||
|
}
|
||||||
|
chunks.push(chunk);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut roles: Vec<_> = chunks
|
||||||
|
.iter()
|
||||||
|
.map(|chunk| role::classify(&chunk.text, chunk.chapter.as_deref()))
|
||||||
|
.collect();
|
||||||
|
role::smooth(&mut roles);
|
||||||
|
for (chunk, role) in chunks.iter_mut().zip(roles) {
|
||||||
|
chunk.role = role;
|
||||||
|
}
|
||||||
|
|
||||||
|
chunks
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
use corpus_core::Role;
|
||||||
|
|
||||||
|
fn small() -> Budgets {
|
||||||
|
Budgets {
|
||||||
|
target: 20,
|
||||||
|
overlap: 4,
|
||||||
|
min_tokens: 1,
|
||||||
|
hard_max: 40,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn chunks_are_numbered_from_zero_without_gaps() {
|
||||||
|
let markdown = "# One\n\nalpha beta\n\n<!-- Page 2 -->\n\ngamma delta\n\nepsilon zeta";
|
||||||
|
let chunks = chunk(markdown, &small());
|
||||||
|
for (at, chunk) in chunks.iter().enumerate() {
|
||||||
|
assert_eq!(chunk.idx, at as u32);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_chunk_carries_the_page_it_came_from() {
|
||||||
|
let markdown = "<!-- Page 7 -->\n\nalpha beta gamma";
|
||||||
|
let chunks = chunk(markdown, &small());
|
||||||
|
assert_eq!(chunks[0].page, Some(7));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_chunk_carries_its_chapter() {
|
||||||
|
let markdown = "<!-- Page 1 -->\n\n# The Second Estate\n\nalpha beta gamma";
|
||||||
|
let chunks = chunk(markdown, &small());
|
||||||
|
assert_eq!(chunks[0].chapter.as_deref(), Some("The Second Estate"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_chapter_is_found_by_offset_when_there_are_no_page_markers() {
|
||||||
|
let markdown = "# The Second Estate\n\nalpha beta gamma";
|
||||||
|
let chunks = chunk(markdown, &small());
|
||||||
|
assert_eq!(chunks[0].page, None);
|
||||||
|
assert_eq!(chunks[0].chapter.as_deref(), Some("The Second Estate"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_chunk_below_the_minimum_is_dropped() {
|
||||||
|
let budgets = Budgets {
|
||||||
|
min_tokens: 100,
|
||||||
|
..small()
|
||||||
|
};
|
||||||
|
assert!(chunk("alpha beta", &budgets).is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn nothing_exceeds_the_hard_maximum() {
|
||||||
|
let markdown = "word ".repeat(500);
|
||||||
|
let budgets = Budgets {
|
||||||
|
target: 30,
|
||||||
|
overlap: 5,
|
||||||
|
min_tokens: 1,
|
||||||
|
hard_max: 35,
|
||||||
|
};
|
||||||
|
for chunk in chunk(&markdown, &budgets) {
|
||||||
|
assert!(chunk.tokens <= 35, "{} tokens", chunk.tokens);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_note_chunk_is_flagged_and_its_marker_is_stripped() {
|
||||||
|
let markdown = format!(
|
||||||
|
"{}12. Arendt, The Human Condition (Chicago, 1958), 44.\n\n\
|
||||||
|
13. Bloch, Feudal Society, vol. 2, p. 17.",
|
||||||
|
footnote::SENTINEL
|
||||||
|
);
|
||||||
|
let chunks = chunk(&markdown, &small());
|
||||||
|
assert!(chunks[0].has_flag(flags::FOOTNOTE_CITATIONS));
|
||||||
|
assert!(!chunks[0].text.contains(footnote::SENTINEL));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn roles_are_smoothed_across_the_whole_document() {
|
||||||
|
// Two bibliography-shaped pages with an ambiguous one between them.
|
||||||
|
let entries = "Arendt, Hannah. The Human Condition. Chicago: University Press, 1958.\n\n\
|
||||||
|
Bloch, Marc. Feudal Society. London: Routledge, 1961.\n\n\
|
||||||
|
Cassirer, Ernst. The Myth of the State. New Haven: Yale Press, 1946.";
|
||||||
|
let markdown = format!(
|
||||||
|
"<!-- Page 1 -->\n\n{entries}\n\n<!-- Page 2 -->\n\nhttps://example.org/a\n\n\
|
||||||
|
<!-- Page 3 -->\n\n{entries}"
|
||||||
|
);
|
||||||
|
let chunks = chunk(
|
||||||
|
&markdown,
|
||||||
|
&Budgets {
|
||||||
|
target: 200,
|
||||||
|
overlap: 0,
|
||||||
|
min_tokens: 1,
|
||||||
|
hard_max: 400,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
assert!(chunks.len() >= 3, "{chunks:#?}");
|
||||||
|
assert!(
|
||||||
|
chunks.iter().all(|chunk| chunk.role == Role::Bibliography),
|
||||||
|
"{:?}",
|
||||||
|
chunks.iter().map(|c| c.role).collect::<Vec<_>>()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn empty_input_produces_no_chunks() {
|
||||||
|
assert!(chunk("", &Budgets::default()).is_empty());
|
||||||
|
assert!(chunk(" \n\n ", &Budgets::default()).is_empty());
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,130 @@
|
|||||||
|
//! Cuts markdown into retrievable passages that carry their page, chapter
|
||||||
|
//! and role.
|
||||||
|
|
||||||
|
use std::io::{BufWriter, Read, Write};
|
||||||
|
use std::path::PathBuf;
|
||||||
|
use std::process::ExitCode;
|
||||||
|
|
||||||
|
use chunkmd::Budgets;
|
||||||
|
use clap::Parser;
|
||||||
|
use corpus_core::{exit, ndjson};
|
||||||
|
|
||||||
|
/// Cut markdown into chunks, one JSON record per line.
|
||||||
|
///
|
||||||
|
/// Reads markdown on stdin, or from a path. Page markers (`<!-- Page 41 -->`)
|
||||||
|
/// and headings become the page and chapter on every chunk, so a passage can
|
||||||
|
/// always be cited back to where it came from.
|
||||||
|
#[derive(Parser)]
|
||||||
|
#[command(
|
||||||
|
version,
|
||||||
|
about,
|
||||||
|
long_about = None,
|
||||||
|
after_help = "\
|
||||||
|
Example:
|
||||||
|
|
||||||
|
mdscrub book.md | chunkmd --target 1024 --overlap 128 > chunks.ndjson
|
||||||
|
|
||||||
|
The defaults suit a model with a 512-token window; a larger model wants
|
||||||
|
--target 1024 --overlap 128 --min-tokens 80 --hard-max 1280."
|
||||||
|
)]
|
||||||
|
struct Args {
|
||||||
|
/// Markdown file to read. Reads stdin when absent.
|
||||||
|
path: Option<PathBuf>,
|
||||||
|
|
||||||
|
/// Flush a chunk at this many tokens.
|
||||||
|
#[arg(long, default_value_t = Budgets::default().target, value_name = "TOKENS")]
|
||||||
|
target: usize,
|
||||||
|
|
||||||
|
/// Carry this much of each chunk into the next one.
|
||||||
|
#[arg(long, default_value_t = Budgets::default().overlap, value_name = "TOKENS")]
|
||||||
|
overlap: usize,
|
||||||
|
|
||||||
|
/// Drop chunks smaller than this.
|
||||||
|
#[arg(long, default_value_t = Budgets::default().min_tokens, value_name = "TOKENS")]
|
||||||
|
min_tokens: usize,
|
||||||
|
|
||||||
|
/// Never emit a chunk larger than this.
|
||||||
|
#[arg(long, default_value_t = Budgets::default().hard_max, value_name = "TOKENS")]
|
||||||
|
hard_max: usize,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn main() -> ExitCode {
|
||||||
|
let args = Args::parse();
|
||||||
|
let budgets = Budgets {
|
||||||
|
target: args.target,
|
||||||
|
overlap: args.overlap,
|
||||||
|
min_tokens: args.min_tokens,
|
||||||
|
hard_max: args.hard_max,
|
||||||
|
};
|
||||||
|
if let Err(why) = sane(&budgets) {
|
||||||
|
eprintln!("chunkmd: {why}");
|
||||||
|
return ExitCode::from(exit::USAGE);
|
||||||
|
}
|
||||||
|
|
||||||
|
let markdown = match read(args.path.as_deref()) {
|
||||||
|
Ok(markdown) => markdown,
|
||||||
|
Err(e) => {
|
||||||
|
eprintln!("chunkmd: {e}");
|
||||||
|
return ExitCode::from(exit::IO);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
let mut out = BufWriter::new(std::io::stdout().lock());
|
||||||
|
for chunk in chunkmd::chunk(&markdown, &budgets) {
|
||||||
|
if let Err(e) = ndjson::write_line(&mut out, &chunk) {
|
||||||
|
// A consumer that stopped reading is not a failure: `| head` is a
|
||||||
|
// normal way to look at a stream.
|
||||||
|
if e.is_broken_pipe() {
|
||||||
|
return ExitCode::from(exit::OK);
|
||||||
|
}
|
||||||
|
eprintln!("chunkmd: {e}");
|
||||||
|
return ExitCode::from(exit::IO);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
match out.flush() {
|
||||||
|
Ok(()) => ExitCode::from(exit::OK),
|
||||||
|
Err(e) if e.kind() == std::io::ErrorKind::BrokenPipe => ExitCode::from(exit::OK),
|
||||||
|
Err(e) => {
|
||||||
|
eprintln!("chunkmd: {e}");
|
||||||
|
ExitCode::from(exit::IO)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Budgets that cannot produce anything sensible, rejected before reading
|
||||||
|
/// input rather than after.
|
||||||
|
fn sane(budgets: &Budgets) -> Result<(), String> {
|
||||||
|
if budgets.target == 0 || budgets.hard_max == 0 {
|
||||||
|
return Err("--target and --hard-max must be above zero".into());
|
||||||
|
}
|
||||||
|
if budgets.overlap >= budgets.target {
|
||||||
|
return Err(format!(
|
||||||
|
"--overlap {} must be below --target {}, or a chunk is all overlap",
|
||||||
|
budgets.overlap, budgets.target
|
||||||
|
));
|
||||||
|
}
|
||||||
|
if budgets.hard_max < budgets.target {
|
||||||
|
return Err(format!(
|
||||||
|
"--hard-max {} must be at least --target {}",
|
||||||
|
budgets.hard_max, budgets.target
|
||||||
|
));
|
||||||
|
}
|
||||||
|
if budgets.min_tokens > budgets.hard_max {
|
||||||
|
return Err(format!(
|
||||||
|
"--min-tokens {} is above --hard-max {}, so every chunk would be dropped",
|
||||||
|
budgets.min_tokens, budgets.hard_max
|
||||||
|
));
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn read(path: Option<&std::path::Path>) -> std::io::Result<String> {
|
||||||
|
match path {
|
||||||
|
Some(path) => std::fs::read_to_string(path),
|
||||||
|
None => {
|
||||||
|
let mut markdown = String::new();
|
||||||
|
std::io::stdin().read_to_string(&mut markdown)?;
|
||||||
|
Ok(markdown)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,186 @@
|
|||||||
|
//! Packing paragraphs into chunks of about the size an embedding model wants.
|
||||||
|
//!
|
||||||
|
//! Page first, paragraph second, sentence only when a paragraph is itself too
|
||||||
|
//! big. Every flush carries a tail of the previous chunk forward, so a
|
||||||
|
//! sentence that answers a question is not orphaned from the sentence that
|
||||||
|
//! asked it.
|
||||||
|
|
||||||
|
/// The token budgets a chunker works to.
|
||||||
|
///
|
||||||
|
/// The defaults suit a model with a 512-token comfortable window. A model
|
||||||
|
/// with a larger one wants roughly double everything — the numbers below are
|
||||||
|
/// ratios that have held up, not constants with a life of their own.
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
|
pub struct Budgets {
|
||||||
|
/// Where a chunk is flushed, in tokens.
|
||||||
|
pub target: usize,
|
||||||
|
/// How much of the previous chunk is carried into the next one.
|
||||||
|
pub overlap: usize,
|
||||||
|
/// Chunks smaller than this are dropped: a fragment retrieves badly and
|
||||||
|
/// costs an embedding.
|
||||||
|
pub min_tokens: usize,
|
||||||
|
/// No chunk may exceed this, whatever structure says.
|
||||||
|
pub hard_max: usize,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Default for Budgets {
|
||||||
|
fn default() -> Self {
|
||||||
|
Self {
|
||||||
|
target: 512,
|
||||||
|
overlap: 64,
|
||||||
|
min_tokens: 50,
|
||||||
|
hard_max: 768,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Pack one page's text into chunk-sized pieces.
|
||||||
|
pub fn page(text: &str, budgets: &Budgets) -> Vec<String> {
|
||||||
|
let paragraphs = textkit::paragraphs(text);
|
||||||
|
if paragraphs.is_empty() {
|
||||||
|
return Vec::new();
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut chunks = Vec::new();
|
||||||
|
let mut buf: Vec<String> = Vec::new();
|
||||||
|
let mut buf_tokens = 0usize;
|
||||||
|
|
||||||
|
for paragraph in paragraphs {
|
||||||
|
let paragraph_tokens = textkit::count(paragraph);
|
||||||
|
|
||||||
|
if paragraph_tokens > budgets.hard_max {
|
||||||
|
// A paragraph nothing can hold: break it at sentences instead.
|
||||||
|
// Note that a flush from here joins with a space, while the final
|
||||||
|
// flush below joins with a blank line — the seam a chunk was cut
|
||||||
|
// at is visible in how its pieces are rejoined.
|
||||||
|
for sentence in textkit::sentences(paragraph) {
|
||||||
|
let sentence_tokens = textkit::count(sentence);
|
||||||
|
if buf_tokens + sentence_tokens > budgets.target && !buf.is_empty() {
|
||||||
|
chunks.push(buf.join(" "));
|
||||||
|
buf = carry_tail(&buf, budgets.overlap);
|
||||||
|
buf_tokens = textkit::count(&buf.join(" "));
|
||||||
|
}
|
||||||
|
buf.push(sentence.to_string());
|
||||||
|
buf_tokens += sentence_tokens;
|
||||||
|
}
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if buf_tokens + paragraph_tokens > budgets.target && !buf.is_empty() {
|
||||||
|
chunks.push(buf.join("\n\n"));
|
||||||
|
buf = carry_tail(&buf, budgets.overlap);
|
||||||
|
buf_tokens = textkit::count(&buf.join("\n\n"));
|
||||||
|
}
|
||||||
|
buf.push(paragraph.to_string());
|
||||||
|
buf_tokens += paragraph_tokens;
|
||||||
|
}
|
||||||
|
|
||||||
|
if !buf.is_empty() {
|
||||||
|
chunks.push(buf.join("\n\n"));
|
||||||
|
}
|
||||||
|
chunks
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Keep the last pieces of a buffer, up to `overlap` tokens of context.
|
||||||
|
///
|
||||||
|
/// Always keeps at least one piece, even when that piece alone is over
|
||||||
|
/// budget: an overlap of nothing is a chunk boundary with no bridge across
|
||||||
|
/// it, which is the thing overlap exists to prevent.
|
||||||
|
fn carry_tail(buf: &[String], overlap: usize) -> Vec<String> {
|
||||||
|
let mut out: Vec<String> = Vec::new();
|
||||||
|
let mut total = 0usize;
|
||||||
|
for piece in buf.iter().rev() {
|
||||||
|
let tokens = textkit::count(piece);
|
||||||
|
if total + tokens > overlap && !out.is_empty() {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
out.insert(0, piece.clone());
|
||||||
|
total += tokens;
|
||||||
|
}
|
||||||
|
out
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Split anything still over the hard maximum, on token boundaries.
|
||||||
|
///
|
||||||
|
/// Defence in depth. The structural passes above should have handled it; this
|
||||||
|
/// is what guarantees the promise regardless.
|
||||||
|
pub fn enforce_hard_max(text: &str, hard_max: usize) -> Vec<String> {
|
||||||
|
textkit::split_at_token_limit(text, hard_max)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
fn budgets(target: usize, overlap: usize) -> Budgets {
|
||||||
|
Budgets {
|
||||||
|
target,
|
||||||
|
overlap,
|
||||||
|
..Budgets::default()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn short_text_is_one_chunk() {
|
||||||
|
let chunks = page("one\n\ntwo", &Budgets::default());
|
||||||
|
assert_eq!(chunks, ["one\n\ntwo"]);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn paragraphs_are_rejoined_with_a_blank_line() {
|
||||||
|
let chunks = page("alpha\n\n\n\nbeta", &Budgets::default());
|
||||||
|
assert_eq!(chunks, ["alpha\n\nbeta"]);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_full_buffer_flushes_and_carries_a_tail() {
|
||||||
|
let text = "one two three\n\nfour five six\n\nseven eight nine";
|
||||||
|
let chunks = page(text, &budgets(6, 4));
|
||||||
|
assert!(chunks.len() > 1, "{chunks:?}");
|
||||||
|
// Every chunk after the first repeats the end of the one before it.
|
||||||
|
for pair in chunks.windows(2) {
|
||||||
|
let earlier_tail = pair[0].rsplit("\n\n").next().expect("a paragraph");
|
||||||
|
assert!(
|
||||||
|
pair[1].starts_with(earlier_tail),
|
||||||
|
"{:?} does not carry {:?}",
|
||||||
|
pair[1],
|
||||||
|
earlier_tail
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn an_oversized_paragraph_is_cut_at_sentences() {
|
||||||
|
let paragraph = "Alpha one. Beta two. Gamma three. Delta four. Epsilon five.";
|
||||||
|
let chunks = page(
|
||||||
|
paragraph,
|
||||||
|
&Budgets {
|
||||||
|
target: 6,
|
||||||
|
overlap: 2,
|
||||||
|
min_tokens: 1,
|
||||||
|
hard_max: 8,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
assert!(chunks.len() > 1, "{chunks:?}");
|
||||||
|
assert!(chunks[0].contains(' '));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn an_empty_page_packs_to_nothing() {
|
||||||
|
assert!(page(" \n\n ", &Budgets::default()).is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn the_tail_keeps_one_piece_even_when_it_is_over_budget() {
|
||||||
|
let buf = vec!["a longer paragraph than the overlap allows".to_string()];
|
||||||
|
assert_eq!(carry_tail(&buf, 1).len(), 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn nothing_leaves_the_packer_over_the_hard_maximum() {
|
||||||
|
let long = "word ".repeat(2000);
|
||||||
|
for piece in enforce_hard_max(&long, 100) {
|
||||||
|
assert!(textkit::count(&piece) <= 100);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,294 @@
|
|||||||
|
//! Finding where the pages and the chapters are.
|
||||||
|
//!
|
||||||
|
//! Converted markdown carries page boundaries as `<!-- Page 41 -->` comments
|
||||||
|
//! and chapter boundaries as headings. Both are needed on every chunk: a
|
||||||
|
//! passage a reader cannot locate in the book is a passage they cannot check.
|
||||||
|
|
||||||
|
use std::collections::HashMap;
|
||||||
|
use std::sync::LazyLock;
|
||||||
|
|
||||||
|
use regex::Regex;
|
||||||
|
|
||||||
|
static PAGE: LazyLock<Regex> =
|
||||||
|
LazyLock::new(|| Regex::new(r"<!--\s*Page\s+(\d+)\s*-->").expect("a valid pattern"));
|
||||||
|
|
||||||
|
static HEADING: LazyLock<Regex> =
|
||||||
|
LazyLock::new(|| Regex::new(r"(?m)^(#{1,3})\s+(.+)$").expect("a valid pattern"));
|
||||||
|
|
||||||
|
/// One page's worth of text, and where it started in the source.
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
|
pub struct Page<'a> {
|
||||||
|
/// The page number from the marker, absent for text that came before the
|
||||||
|
/// first marker.
|
||||||
|
pub number: Option<u32>,
|
||||||
|
/// The page's text, trimmed.
|
||||||
|
pub text: &'a str,
|
||||||
|
/// Byte offset in the source markdown where the page's body starts, used
|
||||||
|
/// to find the chapter when there are no page markers to go on.
|
||||||
|
pub offset: usize,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A heading and the span of the document it covers.
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
|
pub struct Chapter {
|
||||||
|
/// The heading text, trimmed.
|
||||||
|
pub title: String,
|
||||||
|
/// Page the heading falls on, when the document has page markers.
|
||||||
|
pub start_page: Option<u32>,
|
||||||
|
/// Page the next heading falls on, or the last page.
|
||||||
|
pub end_page: Option<u32>,
|
||||||
|
/// Byte offset of the heading in the source markdown.
|
||||||
|
pub start_offset: usize,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Split markdown on its page markers.
|
||||||
|
///
|
||||||
|
/// Text *before* the first marker is still the book. Starting the walk at the
|
||||||
|
/// first marker silently deleted it — which cost one EPUB 362k of its 364k
|
||||||
|
/// characters, because it carried exactly one stray anchor near the very end.
|
||||||
|
/// Nothing surfaced the loss: the book stayed ingested, with a single chunk.
|
||||||
|
/// So the prologue is emitted with no page number, since we genuinely do not
|
||||||
|
/// know its page, and the offset-based chapter lookup still tags it.
|
||||||
|
pub fn split_by_page(markdown: &str) -> Vec<Page<'_>> {
|
||||||
|
let markers: Vec<_> = PAGE.find_iter(markdown).collect();
|
||||||
|
if markers.is_empty() {
|
||||||
|
return vec![Page {
|
||||||
|
number: None,
|
||||||
|
text: markdown,
|
||||||
|
offset: 0,
|
||||||
|
}];
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut pages = Vec::new();
|
||||||
|
let head = trim(&markdown[..markers[0].start()]);
|
||||||
|
if !head.is_empty() {
|
||||||
|
pages.push(Page {
|
||||||
|
number: None,
|
||||||
|
text: head,
|
||||||
|
offset: 0,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
for (at, marker) in markers.iter().enumerate() {
|
||||||
|
let start = marker.end();
|
||||||
|
let end = markers
|
||||||
|
.get(at + 1)
|
||||||
|
.map_or(markdown.len(), |next| next.start());
|
||||||
|
let text = trim(&markdown[start..end]);
|
||||||
|
if !text.is_empty() {
|
||||||
|
pages.push(Page {
|
||||||
|
number: Some(page_number(&markdown[marker.start()..marker.end()])),
|
||||||
|
text,
|
||||||
|
offset: start,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
pages
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Extract the headings that act as chapters, with their page and byte spans.
|
||||||
|
///
|
||||||
|
/// A document that has H1s is chaptered by its H1s; one that only has H2s is
|
||||||
|
/// chaptered by those. Mixing levels would make a chapter list where a
|
||||||
|
/// section and the chapter containing it sit side by side.
|
||||||
|
pub fn chapters(markdown: &str) -> Vec<Chapter> {
|
||||||
|
let mut headings: Vec<(usize, String, usize)> = Vec::new();
|
||||||
|
for found in HEADING.captures_iter(markdown) {
|
||||||
|
let title = trim(found.get(2).expect("group 2").as_str());
|
||||||
|
if title.is_empty() {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
headings.push((
|
||||||
|
found.get(1).expect("group 1").len(),
|
||||||
|
title.to_string(),
|
||||||
|
found.get(0).expect("group 0").start(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
if headings.is_empty() {
|
||||||
|
return Vec::new();
|
||||||
|
}
|
||||||
|
|
||||||
|
let shallowest = headings
|
||||||
|
.iter()
|
||||||
|
.map(|(level, _, _)| *level)
|
||||||
|
.min()
|
||||||
|
.expect("a heading");
|
||||||
|
headings.retain(|(level, _, _)| *level == shallowest);
|
||||||
|
|
||||||
|
let page_at = PageIndex::new(markdown);
|
||||||
|
let mut chapters = Vec::with_capacity(headings.len());
|
||||||
|
for (at, (_, title, offset)) in headings.iter().enumerate() {
|
||||||
|
let end = headings
|
||||||
|
.get(at + 1)
|
||||||
|
.map_or(markdown.len(), |(_, _, next)| *next);
|
||||||
|
chapters.push(Chapter {
|
||||||
|
title: title.clone(),
|
||||||
|
start_page: page_at.at(*offset),
|
||||||
|
end_page: page_at.at(end),
|
||||||
|
start_offset: *offset,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
chapters
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Which page a byte offset falls on.
|
||||||
|
pub struct PageIndex(Vec<(usize, u32)>);
|
||||||
|
|
||||||
|
impl PageIndex {
|
||||||
|
/// Index the page markers in `markdown`.
|
||||||
|
pub fn new(markdown: &str) -> Self {
|
||||||
|
Self(
|
||||||
|
PAGE.find_iter(markdown)
|
||||||
|
.map(|marker| (marker.start(), page_number(marker.as_str())))
|
||||||
|
.collect(),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The last page marker at or before `offset`.
|
||||||
|
pub fn at(&self, offset: usize) -> Option<u32> {
|
||||||
|
let found = self.0.partition_point(|(start, _)| *start <= offset);
|
||||||
|
found.checked_sub(1).map(|at| self.0[at].1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Which chapter a byte offset falls in.
|
||||||
|
pub struct ChapterIndex<'a>(&'a [Chapter]);
|
||||||
|
|
||||||
|
impl<'a> ChapterIndex<'a> {
|
||||||
|
/// Index a chapter list, which must be in document order.
|
||||||
|
pub fn new(chapters: &'a [Chapter]) -> Self {
|
||||||
|
Self(chapters)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The last chapter starting at or before `offset`.
|
||||||
|
pub fn at(&self, offset: usize) -> Option<&'a str> {
|
||||||
|
let found = self.0.partition_point(|c| c.start_offset <= offset);
|
||||||
|
found.checked_sub(1).map(|at| self.0[at].title.as_str())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Map each page number to the chapter covering it.
|
||||||
|
///
|
||||||
|
/// Where chapters overlap — which happens when two headings land on one page
|
||||||
|
/// — the later one wins, because that is the one a reader scrolling down
|
||||||
|
/// would be under.
|
||||||
|
pub fn page_chapters(chapters: &[Chapter]) -> HashMap<u32, &str> {
|
||||||
|
let mut out = HashMap::new();
|
||||||
|
for chapter in chapters {
|
||||||
|
let Some(start) = chapter.start_page else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
// A zero end page means the same thing as no end page: this chapter
|
||||||
|
// is one page long as far as anyone can tell.
|
||||||
|
let end = match chapter.end_page {
|
||||||
|
Some(0) | None => start,
|
||||||
|
Some(end) => end,
|
||||||
|
};
|
||||||
|
if end < start {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
for page in start..=end {
|
||||||
|
out.insert(page, chapter.title.as_str());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
out
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Read the digits out of a page marker.
|
||||||
|
///
|
||||||
|
/// Digits are matched by the Unicode class, so a marker could in principle
|
||||||
|
/// carry non-ASCII decimal digits; those count the same way here as they
|
||||||
|
/// would in Python. A number too large for the field saturates rather than
|
||||||
|
/// wrapping — see `parity/KNOWN.md`.
|
||||||
|
fn page_number(marker: &str) -> u32 {
|
||||||
|
let mut number: u32 = 0;
|
||||||
|
for digit in marker.chars().filter_map(|c| c.to_digit(10)) {
|
||||||
|
number = number.saturating_mul(10).saturating_add(digit);
|
||||||
|
}
|
||||||
|
number
|
||||||
|
}
|
||||||
|
|
||||||
|
fn trim(text: &str) -> &str {
|
||||||
|
text.trim_matches(textkit::is_space)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn markdown_with_no_markers_is_one_unnumbered_page() {
|
||||||
|
let pages = split_by_page("just text");
|
||||||
|
assert_eq!(pages.len(), 1);
|
||||||
|
assert_eq!(pages[0].number, None);
|
||||||
|
assert_eq!(pages[0].text, "just text");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn text_before_the_first_marker_is_kept() {
|
||||||
|
let pages = split_by_page("prologue\n\n<!-- Page 19 -->\n\nbody");
|
||||||
|
assert_eq!(pages.len(), 2);
|
||||||
|
assert_eq!(pages[0].number, None);
|
||||||
|
assert_eq!(pages[0].text, "prologue");
|
||||||
|
assert_eq!(pages[1].number, Some(19));
|
||||||
|
assert_eq!(pages[1].text, "body");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn an_empty_page_is_dropped() {
|
||||||
|
let pages = split_by_page("<!-- Page 1 -->\n\n<!-- Page 2 -->\n\nbody");
|
||||||
|
assert_eq!(pages.len(), 1);
|
||||||
|
assert_eq!(pages[0].number, Some(2));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn markers_may_be_spelled_loosely() {
|
||||||
|
let pages = split_by_page("<!--Page 7-->\n\nbody");
|
||||||
|
assert_eq!(pages[0].number, Some(7));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn chapters_come_from_the_shallowest_heading_level_present() {
|
||||||
|
let markdown = "## Alpha\n\ntext\n\n### Sub\n\ntext\n\n## Beta\n\ntext";
|
||||||
|
let found = chapters(markdown);
|
||||||
|
assert_eq!(
|
||||||
|
found.iter().map(|c| c.title.as_str()).collect::<Vec<_>>(),
|
||||||
|
["Alpha", "Beta"]
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_heading_with_nothing_after_it_is_not_a_chapter() {
|
||||||
|
assert!(chapters("# \n\n").is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_hash_on_its_own_line_takes_the_next_line_as_its_title() {
|
||||||
|
// The whitespace between the hashes and the title may include the
|
||||||
|
// newline, so a bare `#` adopts what follows it. Faithful to the
|
||||||
|
// Python, and the case does not arise in converted markdown.
|
||||||
|
let found = chapters("# \n\ntext");
|
||||||
|
assert_eq!(found.len(), 1);
|
||||||
|
assert_eq!(found[0].title, "text");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn chapters_carry_the_pages_they_span() {
|
||||||
|
let markdown = "<!-- Page 1 -->\n\n# One\n\ntext\n\n<!-- Page 4 -->\n\n# Two\n\ntext";
|
||||||
|
let found = chapters(markdown);
|
||||||
|
assert_eq!(found[0].start_page, Some(1));
|
||||||
|
assert_eq!(found[0].end_page, Some(4));
|
||||||
|
assert_eq!(found[1].start_page, Some(4));
|
||||||
|
let by_page = page_chapters(&found);
|
||||||
|
assert_eq!(by_page.get(&1), Some(&"One"));
|
||||||
|
assert_eq!(by_page.get(&3), Some(&"One"));
|
||||||
|
assert_eq!(by_page.get(&4), Some(&"Two"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn an_offset_before_the_first_heading_has_no_chapter() {
|
||||||
|
let found = chapters("front\n\n# One\n\ntext");
|
||||||
|
let index = ChapterIndex::new(&found);
|
||||||
|
assert_eq!(index.at(0), None);
|
||||||
|
assert_eq!(index.at(10), Some("One"));
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,317 @@
|
|||||||
|
//! Telling body text from the apparatus around it.
|
||||||
|
//!
|
||||||
|
//! Retrieval wants body text. An index, a table of contents and a
|
||||||
|
//! bibliography all match a query beautifully and answer nothing — they are
|
||||||
|
//! lists of the words a reader is searching for. Marking them lets a search
|
||||||
|
//! exclude them while keeping them addressable for the consumers that do want
|
||||||
|
//! them: citation extraction reads bibliographies on purpose.
|
||||||
|
//!
|
||||||
|
//! Two signals, in order. The chapter title, when the document's headings
|
||||||
|
//! said what the section is; then the shape of the content, for back-matter
|
||||||
|
//! buried inside a body chapter — common in EPUBs whose bibliography heading
|
||||||
|
//! is not at the level the chapter map used.
|
||||||
|
|
||||||
|
use std::sync::LazyLock;
|
||||||
|
|
||||||
|
use corpus_core::Role;
|
||||||
|
use regex::Regex;
|
||||||
|
|
||||||
|
fn pattern(source: &str) -> Regex {
|
||||||
|
Regex::new(source).expect("a valid pattern")
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Chapter titles that give the section away, tried in this order.
|
||||||
|
static BY_TITLE: LazyLock<Vec<(Role, Regex)>> = LazyLock::new(|| {
|
||||||
|
vec![
|
||||||
|
(
|
||||||
|
Role::Bibliography,
|
||||||
|
pattern(
|
||||||
|
r"(?i)(bibliograph|works\s+cited|literature\s+cited|further\s+reading|^\s*references\s*$|^\s*(end)?notes\s*$)",
|
||||||
|
),
|
||||||
|
),
|
||||||
|
(Role::Index, pattern(r"(?i)^\s*index(\s+of\s+\w+)?\s*$")),
|
||||||
|
(Role::Toc, pattern(r"(?i)^\s*(table\s+of\s+)?contents\s*$")),
|
||||||
|
(
|
||||||
|
Role::Frontmatter,
|
||||||
|
pattern(
|
||||||
|
r"(?i)(copyright|title\s*page|colophon|^\s*dedication\s*$|^\s*epigraph\s*$|about\s+the\s+(author|publisher)|acknowledg)",
|
||||||
|
),
|
||||||
|
),
|
||||||
|
]
|
||||||
|
});
|
||||||
|
|
||||||
|
/// A standalone bibliography heading inside the chunk's own text —
|
||||||
|
/// "Bibliography", "## Works Cited", "Selected Bibliography".
|
||||||
|
static INTEXT_BIBLIO_HEADING: LazyLock<Regex> = LazyLock::new(|| {
|
||||||
|
pattern(
|
||||||
|
r"(?im)^\s*#{0,6}\s*(select(?:ed)?\s+)?(bibliography|works\s+cited|references|sources)\s*$",
|
||||||
|
)
|
||||||
|
});
|
||||||
|
|
||||||
|
/// One bibliography entry: "Lastname, First…", or the dash run that stands
|
||||||
|
/// for "the same author again".
|
||||||
|
static BIBLIO_ENTRY: LazyLock<Regex> =
|
||||||
|
LazyLock::new(|| pattern(r"^(?:[A-Z][\w'’\-]+,\s+\S|[—–-]{2,3}\s*[.,])"));
|
||||||
|
|
||||||
|
/// The continuation of a wrapped entry: "New York: Springer, 2013."
|
||||||
|
static BIBLIO_CONTINUATION: LazyLock<Regex> =
|
||||||
|
LazyLock::new(|| pattern(r"\b(University|Press|Publish|Books|Verlag|Editions|& Co|Inc\.)\b"));
|
||||||
|
|
||||||
|
static YEAR: LazyLock<Regex> = LazyLock::new(|| pattern(r"\b(1[5-9]\d{2}|20\d{2})\b"));
|
||||||
|
|
||||||
|
/// An index line: a short lead, then a list of page numbers.
|
||||||
|
static INDEX_LINE: LazyLock<Regex> =
|
||||||
|
LazyLock::new(|| pattern(r"^.{1,70}?,\s+[ivxlc\d]{1,4}(?:[,–-]\s*[ivxlc\d]{1,4})*\s*$"));
|
||||||
|
|
||||||
|
/// A contents line: dot leaders to a page number, or a link to an internal
|
||||||
|
/// anchor, which is what a converted EPUB's table of contents becomes.
|
||||||
|
static TOC_LINE: LazyLock<Regex> = LazyLock::new(|| pattern(r"(\.{3,}\s*\d{1,4}\s*$|\]\(#)"));
|
||||||
|
|
||||||
|
/// A wrapped entry long enough that its publisher name is probably a
|
||||||
|
/// coincidence rather than a citation, in characters.
|
||||||
|
const CONTINUATION_MAX: usize = 200;
|
||||||
|
|
||||||
|
/// How many entry-shaped paragraphs make a bibliography, and what share of
|
||||||
|
/// the paragraphs they have to be.
|
||||||
|
const BIBLIO_MIN_ENTRIES: usize = 3;
|
||||||
|
const BIBLIO_SHARE: f64 = 0.55;
|
||||||
|
|
||||||
|
/// The same, for an in-text bibliography heading, which is evidence on its
|
||||||
|
/// own and so needs less corroboration.
|
||||||
|
const BIBLIO_MIN_ENTRIES_WITH_HEADING: usize = 2;
|
||||||
|
|
||||||
|
/// How many list-shaped lines make an index or a table of contents.
|
||||||
|
const LIST_MIN_LINES: usize = 5;
|
||||||
|
const LIST_SHARE: f64 = 0.5;
|
||||||
|
|
||||||
|
/// Classify one chunk.
|
||||||
|
pub fn classify(text: &str, chapter: Option<&str>) -> Role {
|
||||||
|
if let Some(chapter) = chapter.filter(|title| !title.is_empty()) {
|
||||||
|
for (role, pattern) in BY_TITLE.iter() {
|
||||||
|
if pattern.is_match(chapter) {
|
||||||
|
return *role;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Content-shape fallbacks. Cheap line scans; a chunk is a thousand
|
||||||
|
// tokens at the outside.
|
||||||
|
let lines: Vec<&str> = text
|
||||||
|
.split('\n')
|
||||||
|
.map(trim)
|
||||||
|
.filter(|line| !line.is_empty())
|
||||||
|
.collect();
|
||||||
|
if lines.is_empty() {
|
||||||
|
return Role::Body;
|
||||||
|
}
|
||||||
|
let paragraphs = textkit::paragraphs(text);
|
||||||
|
|
||||||
|
let entryish = paragraphs
|
||||||
|
.iter()
|
||||||
|
.filter(|paragraph| looks_like_entry(paragraph))
|
||||||
|
.count();
|
||||||
|
let enough_entries =
|
||||||
|
entryish >= BIBLIO_MIN_ENTRIES && entryish as f64 >= BIBLIO_SHARE * paragraphs.len() as f64;
|
||||||
|
let headed =
|
||||||
|
INTEXT_BIBLIO_HEADING.is_match(text) && entryish >= BIBLIO_MIN_ENTRIES_WITH_HEADING;
|
||||||
|
if enough_entries || headed {
|
||||||
|
return Role::Bibliography;
|
||||||
|
}
|
||||||
|
|
||||||
|
if dominates(
|
||||||
|
lines
|
||||||
|
.iter()
|
||||||
|
.filter(|line| INDEX_LINE.is_match(line))
|
||||||
|
.count(),
|
||||||
|
lines.len(),
|
||||||
|
) {
|
||||||
|
return Role::Index;
|
||||||
|
}
|
||||||
|
|
||||||
|
if dominates(
|
||||||
|
lines.iter().filter(|line| TOC_LINE.is_match(line)).count(),
|
||||||
|
lines.len(),
|
||||||
|
) {
|
||||||
|
return Role::Toc;
|
||||||
|
}
|
||||||
|
|
||||||
|
Role::Body
|
||||||
|
}
|
||||||
|
|
||||||
|
fn looks_like_entry(paragraph: &str) -> bool {
|
||||||
|
let year = YEAR.is_match(paragraph);
|
||||||
|
let publisher = BIBLIO_CONTINUATION.is_match(paragraph);
|
||||||
|
if BIBLIO_ENTRY.is_match(paragraph) && (year || publisher) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
publisher && year && paragraph.chars().count() < CONTINUATION_MAX
|
||||||
|
}
|
||||||
|
|
||||||
|
fn dominates(matching: usize, total: usize) -> bool {
|
||||||
|
matching >= LIST_MIN_LINES && matching as f64 >= LIST_SHARE * total as f64
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Fix classification stragglers.
|
||||||
|
///
|
||||||
|
/// A run of at most two body chunks sandwiched between chunks of the same
|
||||||
|
/// non-body role inherits that role. This catches the boundary chunks of a
|
||||||
|
/// bibliography or an index, whose own content shape is ambiguous because
|
||||||
|
/// the entries there are URL-heavy, or split across the seam.
|
||||||
|
pub fn smooth(roles: &mut [Role]) {
|
||||||
|
const MAX_RUN: usize = 2;
|
||||||
|
|
||||||
|
let mut at = 0;
|
||||||
|
while at < roles.len() {
|
||||||
|
if roles[at] != Role::Body {
|
||||||
|
at += 1;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let mut end = at;
|
||||||
|
while end < roles.len() && roles[end] == Role::Body {
|
||||||
|
end += 1;
|
||||||
|
}
|
||||||
|
let before = at.checked_sub(1).map(|before| roles[before]);
|
||||||
|
let after = roles.get(end).copied();
|
||||||
|
if let Some(role) = before
|
||||||
|
&& end - at <= MAX_RUN
|
||||||
|
&& before == after
|
||||||
|
&& role != Role::Body
|
||||||
|
{
|
||||||
|
roles[at..end].fill(role);
|
||||||
|
}
|
||||||
|
at = end;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn trim(text: &str) -> &str {
|
||||||
|
text.trim_matches(textkit::is_space)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_chapter_title_decides_when_it_can() {
|
||||||
|
for (title, role) in [
|
||||||
|
("Selected Bibliography", Role::Bibliography),
|
||||||
|
("Works Cited", Role::Bibliography),
|
||||||
|
("References", Role::Bibliography),
|
||||||
|
("Notes", Role::Bibliography),
|
||||||
|
("Endnotes", Role::Bibliography),
|
||||||
|
("Index", Role::Index),
|
||||||
|
("Index of Names", Role::Index),
|
||||||
|
("Table of Contents", Role::Toc),
|
||||||
|
("Contents", Role::Toc),
|
||||||
|
("Copyright", Role::Frontmatter),
|
||||||
|
("Acknowledgements", Role::Frontmatter),
|
||||||
|
("About the Author", Role::Frontmatter),
|
||||||
|
("III. The Second Estate", Role::Body),
|
||||||
|
] {
|
||||||
|
assert_eq!(
|
||||||
|
classify("ordinary text", Some(title)),
|
||||||
|
role,
|
||||||
|
"for {title:?}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_title_that_merely_mentions_references_in_a_sentence_is_not_a_bibliography() {
|
||||||
|
assert_eq!(
|
||||||
|
classify("text", Some("What the references tell us")),
|
||||||
|
Role::Body
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn entry_shaped_paragraphs_make_a_bibliography_without_a_title() {
|
||||||
|
let text = "Arendt, Hannah. The Human Condition. Chicago: University Press, 1958.\n\n\
|
||||||
|
Bloch, Marc. Feudal Society. London: Routledge, 1961.\n\n\
|
||||||
|
Cassirer, Ernst. The Myth of the State. New Haven: Yale Press, 1946.";
|
||||||
|
assert_eq!(classify(text, None), Role::Bibliography);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn an_in_text_heading_lowers_the_bar_to_two_entries() {
|
||||||
|
let text = "## Works Cited\n\n\
|
||||||
|
Arendt, Hannah. The Human Condition. Chicago: University Press, 1958.\n\n\
|
||||||
|
Bloch, Marc. Feudal Society. London: Routledge, 1961.";
|
||||||
|
assert_eq!(classify(text, None), Role::Bibliography);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn page_number_lists_make_an_index() {
|
||||||
|
let text = "agriculture, 12, 34, 102\n\
|
||||||
|
bishops, 7\n\
|
||||||
|
charters, 88, 91\n\
|
||||||
|
demesne, 14-16\n\
|
||||||
|
estates, 3, 44\n\
|
||||||
|
fealty, 201";
|
||||||
|
assert_eq!(classify(text, None), Role::Index);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn dot_leaders_make_a_table_of_contents() {
|
||||||
|
let text = "One. The Setting......1\n\
|
||||||
|
Two. The Quarrel......22\n\
|
||||||
|
Three. The Break......45\n\
|
||||||
|
Four. The Peace......70\n\
|
||||||
|
Five. After......99";
|
||||||
|
assert_eq!(classify(text, None), Role::Toc);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn ordinary_prose_is_body() {
|
||||||
|
let text = "The second estate held its land in exchange for service, and \
|
||||||
|
the service was military. That much is agreed; what follows is not.";
|
||||||
|
assert_eq!(classify(text, None), Role::Body);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn empty_text_is_body() {
|
||||||
|
assert_eq!(classify(" \n ", None), Role::Body);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_short_body_run_inside_a_bibliography_is_absorbed() {
|
||||||
|
let mut roles = vec![
|
||||||
|
Role::Bibliography,
|
||||||
|
Role::Body,
|
||||||
|
Role::Body,
|
||||||
|
Role::Bibliography,
|
||||||
|
];
|
||||||
|
smooth(&mut roles);
|
||||||
|
assert_eq!(roles, [Role::Bibliography; 4]);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_long_body_run_is_left_alone() {
|
||||||
|
let mut roles = vec![
|
||||||
|
Role::Bibliography,
|
||||||
|
Role::Body,
|
||||||
|
Role::Body,
|
||||||
|
Role::Body,
|
||||||
|
Role::Bibliography,
|
||||||
|
];
|
||||||
|
let before = roles.clone();
|
||||||
|
smooth(&mut roles);
|
||||||
|
assert_eq!(roles, before);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_run_between_different_roles_is_left_alone() {
|
||||||
|
let mut roles = vec![Role::Bibliography, Role::Body, Role::Index];
|
||||||
|
let before = roles.clone();
|
||||||
|
smooth(&mut roles);
|
||||||
|
assert_eq!(roles, before);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn body_at_either_end_is_left_alone() {
|
||||||
|
let mut roles = vec![Role::Body, Role::Index, Role::Body];
|
||||||
|
let before = roles.clone();
|
||||||
|
smooth(&mut roles);
|
||||||
|
assert_eq!(roles, before);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -14,6 +14,9 @@ publish = false
|
|||||||
python = []
|
python = []
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
|
chunkmd.workspace = true
|
||||||
|
corpus-core.workspace = true
|
||||||
|
serde_json.workspace = true
|
||||||
textkit.workspace = true
|
textkit.workspace = true
|
||||||
|
|
||||||
[lints]
|
[lints]
|
||||||
|
|||||||
@@ -0,0 +1,24 @@
|
|||||||
|
## Select Bibliography
|
||||||
|
|
||||||
|
Arendt, Hannah. The Human Condition. Chicago: University of Chicago Press, 1958.
|
||||||
|
|
||||||
|
Bloch, Marc. Feudal Society. Translated by L. A. Manyon. London: Routledge, 1961.
|
||||||
|
|
||||||
|
Cassirer, Ernst. The Myth of the State. New Haven: Yale University Press, 1946.
|
||||||
|
|
||||||
|
———. An Essay on Man. New Haven: Yale University Press, 1944.
|
||||||
|
|
||||||
|
Duby, Georges. The Three Orders: Feudal Society Imagined. Chicago: University
|
||||||
|
of Chicago Press, 1980.
|
||||||
|
|
||||||
|
Ganshof, F. L. Feudalism. Translated by Philip Grierson. New York: Harper,
|
||||||
|
1961.
|
||||||
|
|
||||||
|
Hilton, R. H. Bond Men Made Free. London: Temple Smith, 1973.
|
||||||
|
|
||||||
|
Maitland, F. W. Domesday Book and Beyond. Cambridge: Cambridge University
|
||||||
|
Press, 1897.
|
||||||
|
|
||||||
|
Postan, M. M. The Medieval Economy and Society. London: Weidenfeld, 1972.
|
||||||
|
|
||||||
|
Vinogradoff, Paul. Villainage in England. Oxford: Clarendon Press, 1892.
|
||||||
@@ -0,0 +1,12 @@
|
|||||||
|
# Contents
|
||||||
|
|
||||||
|
Preface......vii
|
||||||
|
One. The Setting......1
|
||||||
|
Two. The Quarrel......22
|
||||||
|
Three. The Break......45
|
||||||
|
Four. The Settlement......70
|
||||||
|
Five. After the Settlement......99
|
||||||
|
Appendix A. The Renders......121
|
||||||
|
Appendix B. The Mills......134
|
||||||
|
Bibliography......150
|
||||||
|
Index......161
|
||||||
@@ -0,0 +1,29 @@
|
|||||||
|
## Chapter One
|
||||||
|
|
||||||
|
There are no page markers in this text, because the format it was converted
|
||||||
|
from had none. A chunker that needs pages will produce nothing useful here,
|
||||||
|
and one that falls back to character position will find the chapter correctly.
|
||||||
|
|
||||||
|
The chapter itself concerns the difference between a right and a habit. A
|
||||||
|
right can be pleaded; a habit can only be described. The men in these records
|
||||||
|
pleaded habits as though they were rights, and the courts, with a patience
|
||||||
|
that is easy to underestimate, let them.
|
||||||
|
|
||||||
|
## Chapter Two
|
||||||
|
|
||||||
|
The second chapter takes the same question into a period where the records are
|
||||||
|
better. Better records make the problem worse rather than easier, because they
|
||||||
|
show how much of what looked like custom was in fact a settlement of a dispute
|
||||||
|
that had gone the other way a generation earlier.
|
||||||
|
|
||||||
|
Custom, in this reading, is the name given to the outcome of a quarrel nobody
|
||||||
|
alive remembers having. It is stable exactly as long as the memory of the
|
||||||
|
quarrel is dead and the conditions that produced the settlement persist. When
|
||||||
|
either changes, custom turns out to have been a treaty all along.
|
||||||
|
|
||||||
|
## Chapter Three
|
||||||
|
|
||||||
|
What remains is to say what this means for the wider argument, which is less
|
||||||
|
than it might appear. The valley was not typical. It was chosen because its
|
||||||
|
documents survive, and documents survive where somebody had a reason to keep
|
||||||
|
them, which is itself a fact about the place.
|
||||||
@@ -0,0 +1,22 @@
|
|||||||
|
# Index
|
||||||
|
|
||||||
|
abbey, 44, 47, 91
|
||||||
|
advowson, 12
|
||||||
|
agriculture, 12, 34, 102
|
||||||
|
assize, 7, 19-22
|
||||||
|
bishops, 7
|
||||||
|
boon-work, 88
|
||||||
|
charters, 88, 91
|
||||||
|
commons, 14, 15, 16
|
||||||
|
demesne, 14-16
|
||||||
|
escheat, 120
|
||||||
|
estates, 3, 44
|
||||||
|
fealty, 201
|
||||||
|
frankpledge, 55, 56
|
||||||
|
heriot, 61
|
||||||
|
homage, 3, 4, 201
|
||||||
|
manor, 1-8, 14, 33
|
||||||
|
merchet, 62
|
||||||
|
mill, 71-79, 84
|
||||||
|
villein, 90, 91, 104
|
||||||
|
wardship, 118
|
||||||
@@ -0,0 +1,11 @@
|
|||||||
|
## Notes
|
||||||
|
|
||||||
|
12. Arendt, *The Human Condition* (Chicago: University of Chicago Press, 1958), 44.
|
||||||
|
|
||||||
|
13. Bloch, *Feudal Society*, vol. 2, p. 17. See also Duby, op. cit., 203.
|
||||||
|
|
||||||
|
14. The point is made at length in Maitland, *Domesday Book and Beyond* (Cambridge, 1897), 220-24, and again, with qualifications, in Vinogradoff, *Villainage in England* (Oxford: Clarendon Press, 1892), 88.
|
||||||
|
|
||||||
|
15. Ibid., 91.
|
||||||
|
|
||||||
|
16. https://example.org/records/valley, consulted 12 March.
|
||||||
@@ -0,0 +1,11 @@
|
|||||||
|
## Notes
|
||||||
|
|
||||||
|
12. Arendt, *The Human Condition* (Chicago: University of Chicago Press, 1958), 44.
|
||||||
|
|
||||||
|
13. Bloch, *Feudal Society*, vol. 2, p. 17. See also Duby, op. cit., 203.
|
||||||
|
|
||||||
|
14. The point is made at length in Maitland, *Domesday Book and Beyond* (Cambridge, 1897), 220-24, and again, with qualifications, in Vinogradoff, *Villainage in England* (Oxford: Clarendon Press, 1892), 88.
|
||||||
|
|
||||||
|
15. Ibid., 91.
|
||||||
|
|
||||||
|
16. https://example.org/records/valley, consulted 12 March.
|
||||||
@@ -0,0 +1,3 @@
|
|||||||
|
<!-- Page 40 -->
|
||||||
|
|
||||||
|
One sentence follows another without a paragraph break for the whole of this page, which is the case the sentence splitter exists for. The mill stood at the northern end of the valley, where the fall of the river was steepest and the ice came latest. Its wheel was undershot, which was wasteful, and nobody replaced it for two hundred years because the cost of replacing it fell on one household and the benefit was spread across four hundred. The abbey's mill, two miles downstream, was overshot and newer and better, and the abbey charged more to use it, which the valley resented and paid. When the lord's wheel finally failed, in a winter the chronicle calls hard without saying how hard, the valley ground at the abbey for eleven months and discovered that it preferred the abbey's flour. The lord rebuilt anyway. He had to: the right to compel grinding was worth more than the flour, because it was the visible sign that he could compel anything at all. A right that is never exercised is a right that is being tested, and he knew it, and so did they. The rebuilt mill ran for another century and was still running, badly, when the survey of 1439 recorded it as ruinous and in use. That phrase, ruinous and in use, is as good a description of the whole arrangement as anything a historian has managed since.
|
||||||
@@ -0,0 +1,43 @@
|
|||||||
|
This preface was set before the pagination began, and any tool that starts
|
||||||
|
reading at the first page marker will lose it. That has happened, and it cost
|
||||||
|
a book all but two thousand of its characters.
|
||||||
|
|
||||||
|
<!-- Page 1 -->
|
||||||
|
|
||||||
|
# I. The Setting
|
||||||
|
|
||||||
|
The valley is twelve miles long and never more than two wide. At its northern
|
||||||
|
end the river enters through a gorge that no road followed until the last
|
||||||
|
century; at its southern end it leaves through a marsh that no road follows
|
||||||
|
yet. Everything that happened here happened to people who could not easily
|
||||||
|
leave, and who knew it.
|
||||||
|
|
||||||
|
The population in the first surviving survey is given as four hundred and six
|
||||||
|
households, which at the usual multiplier means somewhere near two thousand
|
||||||
|
people. That figure held, within a tenth either way, for three hundred years.
|
||||||
|
|
||||||
|
<!-- Page 2 -->
|
||||||
|
|
||||||
|
What changed was not how many people lived in the valley but what they owed
|
||||||
|
for the privilege. The earliest schedule lists eleven separate renders, seven
|
||||||
|
of them in kind. The latest lists two, both in coin. Between those two
|
||||||
|
documents lies the whole of the argument.
|
||||||
|
|
||||||
|
It would be convenient to say that the change was gradual. It was not. Two
|
||||||
|
thirds of it happens in a single generation, in the middle of the fourteenth
|
||||||
|
century, for reasons the documents do not give and which have to be supplied
|
||||||
|
from outside.
|
||||||
|
|
||||||
|
<!-- Page 3 -->
|
||||||
|
|
||||||
|
# II. The Quarrel
|
||||||
|
|
||||||
|
The quarrel began over a mill. It nearly always does. A mill is the one piece
|
||||||
|
of capital in a village that everybody must use and only one person can own,
|
||||||
|
and the law of the mill is therefore the law of the place in miniature.
|
||||||
|
|
||||||
|
The lord's mill stood at the northern end, where the fall was steepest. The
|
||||||
|
abbey's mill stood two miles downstream, on land the abbey had been given by a
|
||||||
|
man whose grandson disputed the gift. Both houses claimed the right to compel
|
||||||
|
the valley's tenants to grind at their own stones, and both were prepared to
|
||||||
|
break the other's.
|
||||||
@@ -0,0 +1,32 @@
|
|||||||
|
The argument of this chapter is that the obligation ran both ways, and that
|
||||||
|
contemporaries said so plainly whenever they were asked. A lord who could not
|
||||||
|
protect was a lord who could not command, and the men who owed him service
|
||||||
|
said as much in language that has survived in the court rolls.
|
||||||
|
|
||||||
|
What the rolls do not say, and what has to be inferred, is how often the
|
||||||
|
obligation was actually tested. A tenant who withheld service in a year of
|
||||||
|
famine was not making a constitutional argument; he was hungry. The clerk who
|
||||||
|
recorded the withholding had no vocabulary for the difference, and so the
|
||||||
|
record flattens the two into one offence with one penalty.
|
||||||
|
|
||||||
|
This flattening is the central difficulty of the sources. It is not that they
|
||||||
|
lie. It is that the categories they were written in were designed to produce a
|
||||||
|
number at the bottom of a column, and a number at the bottom of a column
|
||||||
|
cannot distinguish a rebellion from a bad harvest. Everything that follows
|
||||||
|
depends on reading around that limitation rather than through it.
|
||||||
|
|
||||||
|
Three approaches have been tried. The first counts the offences and treats the
|
||||||
|
count as a proxy for discontent, which assumes the clerk's attention was
|
||||||
|
constant across years when we know it was not. The second reads the language
|
||||||
|
of the entries for signs of unusual trouble, which works where a clerk was
|
||||||
|
talkative and fails entirely where he was not. The third, adopted here,
|
||||||
|
compares the same manor across decades and treats changes in what was recorded
|
||||||
|
as evidence about the recorder as much as about the recorded.
|
||||||
|
|
||||||
|
None of the three is satisfactory, and the third has the particular weakness
|
||||||
|
that it needs a long run of documents from a single place. There are not many
|
||||||
|
such runs. The ones that exist are concentrated in the south and east, which
|
||||||
|
is to say in exactly the regions where the argument is least likely to be
|
||||||
|
surprising. A reader who wants to know what happened in the north will find
|
||||||
|
this chapter unsatisfying, and the reason is that the documents do not exist,
|
||||||
|
not that the question is uninteresting.
|
||||||
@@ -0,0 +1,17 @@
|
|||||||
|
<!-- Page 5 -->
|
||||||
|
|
||||||
|
# Ἀθῆναι and Μόσχα
|
||||||
|
|
||||||
|
Le café était naïve, and the first ligature in that sentence is a single
|
||||||
|
codepoint rather than two letters, which is the sort of thing that moves a
|
||||||
|
token boundary without moving a character boundary.
|
||||||
|
|
||||||
|
東京から来た手紙には、日付が二つ書いてあった。One was the date it was written
|
||||||
|
and the other was the date it was meant to be read, and the difference between
|
||||||
|
them is the whole subject of this section.
|
||||||
|
|
||||||
|
مرحبا — the paragraph above is followed by a right-to-left greeting, then by a
|
||||||
|
combining acute on a bare e: é. None of this is unusual in a converted
|
||||||
|
book, and all of it has broken a chunker at some point.
|
||||||
|
|
||||||
|
Straße, Ægypt, first, ॐ. The list is arbitrary. Its purpose is to be awkward.
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
{"v":1,"idx":0,"text":"## Select Bibliography\n\nArendt, Hannah. The Human Condition. Chicago: University of Chicago Press, 1958.\n\nBloch, Marc. Feudal Society. Translated by L. A. Manyon. London: Routledge, 1961.\n\nCassirer, Ernst. The Myth of the State. New Haven: Yale University Press, 1946.\n\n———. An Essay on Man. New Haven: Yale University Press, 1944.\n\nDuby, Georges. The Three Orders: Feudal Society Imagined. Chicago: University\nof Chicago Press, 1980.\n\nGanshof, F. L. Feudalism. Translated by Philip Grierson. New York: Harper,\n1961.\n\nHilton, R. H. Bond Men Made Free. London: Temple Smith, 1973.\n\nMaitland, F. W. Domesday Book and Beyond. Cambridge: Cambridge University\nPress, 1897.\n\nPostan, M. M. The Medieval Economy and Society. London: Weidenfeld, 1972.\n\nVinogradoff, Paul. Villainage in England. Oxford: Clarendon Press, 1892.","tokens":247,"page":null,"chapter":"Select Bibliography","role":"bibliography","start_time":null,"end_time":null,"flags":[]}
|
||||||
@@ -0,0 +1,9 @@
|
|||||||
|
{"v":1,"idx":0,"text":"## Select Bibliography\n\nArendt, Hannah. The Human Condition. Chicago: University of Chicago Press, 1958.\n\nBloch, Marc. Feudal Society. Translated by L. A. Manyon. London: Routledge, 1961.","tokens":55,"page":null,"chapter":"Select Bibliography","role":"bibliography","start_time":null,"end_time":null,"flags":[]}
|
||||||
|
{"v":1,"idx":1,"text":"Bloch, Marc. Feudal Society. Translated by L. A. Manyon. London: Routledge, 1961.\n\nCassirer, Ernst. The Myth of the State. New Haven: Yale University Press, 1946.","tokens":53,"page":null,"chapter":"Select Bibliography","role":"bibliography","start_time":null,"end_time":null,"flags":[]}
|
||||||
|
{"v":1,"idx":2,"text":"Cassirer, Ernst. The Myth of the State. New Haven: Yale University Press, 1946.\n\n———. An Essay on Man. New Haven: Yale University Press, 1944.","tokens":43,"page":null,"chapter":"Select Bibliography","role":"bibliography","start_time":null,"end_time":null,"flags":[]}
|
||||||
|
{"v":1,"idx":3,"text":"———. An Essay on Man. New Haven: Yale University Press, 1944.\n\nDuby, Georges. The Three Orders: Feudal Society Imagined. Chicago: University\nof Chicago Press, 1980.","tokens":47,"page":null,"chapter":"Select Bibliography","role":"bibliography","start_time":null,"end_time":null,"flags":[]}
|
||||||
|
{"v":1,"idx":4,"text":"Duby, Georges. The Three Orders: Feudal Society Imagined. Chicago: University\nof Chicago Press, 1980.\n\nGanshof, F. L. Feudalism. Translated by Philip Grierson. New York: Harper,\n1961.","tokens":56,"page":null,"chapter":"Select Bibliography","role":"bibliography","start_time":null,"end_time":null,"flags":[]}
|
||||||
|
{"v":1,"idx":5,"text":"Ganshof, F. L. Feudalism. Translated by Philip Grierson. New York: Harper,\n1961.\n\nHilton, R. H. Bond Men Made Free. London: Temple Smith, 1973.","tokens":49,"page":null,"chapter":"Select Bibliography","role":"bibliography","start_time":null,"end_time":null,"flags":[]}
|
||||||
|
{"v":1,"idx":6,"text":"Hilton, R. H. Bond Men Made Free. London: Temple Smith, 1973.\n\nMaitland, F. W. Domesday Book and Beyond. Cambridge: Cambridge University\nPress, 1897.","tokens":47,"page":null,"chapter":"Select Bibliography","role":"bibliography","start_time":null,"end_time":null,"flags":[]}
|
||||||
|
{"v":1,"idx":7,"text":"Maitland, F. W. Domesday Book and Beyond. Cambridge: Cambridge University\nPress, 1897.\n\nPostan, M. M. The Medieval Economy and Society. London: Weidenfeld, 1972.","tokens":49,"page":null,"chapter":"Select Bibliography","role":"bibliography","start_time":null,"end_time":null,"flags":[]}
|
||||||
|
{"v":1,"idx":8,"text":"Postan, M. M. The Medieval Economy and Society. London: Weidenfeld, 1972.\n\nVinogradoff, Paul. Villainage in England. Oxford: Clarendon Press, 1892.","tokens":46,"page":null,"chapter":"Select Bibliography","role":"bibliography","start_time":null,"end_time":null,"flags":[]}
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
{"v":1,"idx":0,"text":"# Contents\n\nPreface......vii\nOne. The Setting......1\nTwo. The Quarrel......22\nThree. The Break......45\nFour. The Settlement......70\nFive. After the Settlement......99\nAppendix A. The Renders......121\nAppendix B. The Mills......134\nBibliography......150\nIndex......161","tokens":73,"page":null,"chapter":"Contents","role":"toc","start_time":null,"end_time":null,"flags":[]}
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
{"v":1,"idx":0,"text":"# Contents\n\nPreface......vii\nOne. The Setting......1\nTwo. The Quarrel......22\nThree. The Break......45\nFour. The Settlement......70\nFive. After the Settlement......99\nAppendix A. The Renders......121\nAppendix B. The Mills......134\nBibliography......150\nIndex......161","tokens":73,"page":null,"chapter":"Contents","role":"toc","start_time":null,"end_time":null,"flags":[]}
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
{"v":1,"idx":0,"text":"## Chapter One\n\nThere are no page markers in this text, because the format it was converted\nfrom had none. A chunker that needs pages will produce nothing useful here,\nand one that falls back to character position will find the chapter correctly.\n\nThe chapter itself concerns the difference between a right and a habit. A\nright can be pleaded; a habit can only be described. The men in these records\npleaded habits as though they were rights, and the courts, with a patience\nthat is easy to underestimate, let them.\n\n## Chapter Two\n\nThe second chapter takes the same question into a period where the records are\nbetter. Better records make the problem worse rather than easier, because they\nshow how much of what looked like custom was in fact a settlement of a dispute\nthat had gone the other way a generation earlier.\n\nCustom, in this reading, is the name given to the outcome of a quarrel nobody\nalive remembers having. It is stable exactly as long as the memory of the\nquarrel is dead and the conditions that produced the settlement persist. When\neither changes, custom turns out to have been a treaty all along.\n\n## Chapter Three\n\nWhat remains is to say what this means for the wider argument, which is less\nthan it might appear. The valley was not typical. It was chosen because its\ndocuments survive, and documents survive where somebody had a reason to keep\nthem, which is itself a fact about the place.","tokens":298,"page":null,"chapter":"Chapter One","role":"body","start_time":null,"end_time":null,"flags":[]}
|
||||||
@@ -0,0 +1,9 @@
|
|||||||
|
{"v":1,"idx":0,"text":"## Chapter One\n\nThere are no page markers in this text, because the format it was converted\nfrom had none. A chunker that needs pages will produce nothing useful here,\nand one that falls back to character position will find the chapter correctly.","tokens":50,"page":null,"chapter":"Chapter One","role":"body","start_time":null,"end_time":null,"flags":[]}
|
||||||
|
{"v":1,"idx":1,"text":"There are no page markers in this text, because the format it was converted\nfrom had none. A chunker that needs pages will produce nothing useful here,\nand one that falls back to character position will find the chapter correctly.\n\nThe chapter itself concerns the difference between a right and a habit. A\nright can be pleaded; a habit can only be described. The men in these records\npleaded habits as though they were rights, and the","tokens":90,"page":null,"chapter":"Chapter One","role":"body","start_time":null,"end_time":null,"flags":[]}
|
||||||
|
{"v":1,"idx":2,"text":" courts, with a patience\nthat is easy to underestimate, let them.","tokens":15,"page":null,"chapter":"Chapter One","role":"body","start_time":null,"end_time":null,"flags":[]}
|
||||||
|
{"v":1,"idx":3,"text":"The chapter itself concerns the difference between a right and a habit. A\nright can be pleaded; a habit can only be described. The men in these records\npleaded habits as though they were rights, and the courts, with a patience\nthat is easy to underestimate, let them.\n\n## Chapter Two","tokens":62,"page":null,"chapter":"Chapter One","role":"body","start_time":null,"end_time":null,"flags":[]}
|
||||||
|
{"v":1,"idx":4,"text":"## Chapter Two\n\nThe second chapter takes the same question into a period where the records are\nbetter. Better records make the problem worse rather than easier, because they\nshow how much of what looked like custom was in fact a settlement of a dispute\nthat had gone the other way a generation earlier.","tokens":61,"page":null,"chapter":"Chapter One","role":"body","start_time":null,"end_time":null,"flags":[]}
|
||||||
|
{"v":1,"idx":5,"text":"The second chapter takes the same question into a period where the records are\nbetter. Better records make the problem worse rather than easier, because they\nshow how much of what looked like custom was in fact a settlement of a dispute\nthat had gone the other way a generation earlier.\n\nCustom, in this reading, is the name given to the outcome of a quarrel nobody\nalive remembers having. It is stable exactly as long as the memory of","tokens":90,"page":null,"chapter":"Chapter One","role":"body","start_time":null,"end_time":null,"flags":[]}
|
||||||
|
{"v":1,"idx":6,"text":" the\nquarrel is dead and the conditions that produced the settlement persist. When\neither changes, custom turns out to have been a treaty all along.","tokens":32,"page":null,"chapter":"Chapter One","role":"body","start_time":null,"end_time":null,"flags":[]}
|
||||||
|
{"v":1,"idx":7,"text":"Custom, in this reading, is the name given to the outcome of a quarrel nobody\nalive remembers having. It is stable exactly as long as the memory of the\nquarrel is dead and the conditions that produced the settlement persist. When\neither changes, custom turns out to have been a treaty all along.\n\n## Chapter Three","tokens":68,"page":null,"chapter":"Chapter One","role":"body","start_time":null,"end_time":null,"flags":[]}
|
||||||
|
{"v":1,"idx":8,"text":"## Chapter Three\n\nWhat remains is to say what this means for the wider argument, which is less\nthan it might appear. The valley was not typical. It was chosen because its\ndocuments survive, and documents survive where somebody had a reason to keep\nthem, which is itself a fact about the place.","tokens":63,"page":null,"chapter":"Chapter One","role":"body","start_time":null,"end_time":null,"flags":[]}
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
{"v":1,"idx":0,"text":"# Index\n\nabbey, 44, 47, 91\nadvowson, 12\nagriculture, 12, 34, 102\nassize, 7, 19-22\nbishops, 7\nboon-work, 88\ncharters, 88, 91\ncommons, 14, 15, 16\ndemesne, 14-16\nescheat, 120\nestates, 3, 44\nfealty, 201\nfrankpledge, 55, 56\nheriot, 61\nhomage, 3, 4, 201\nmanor, 1-8, 14, 33\nmerchet, 62\nmill, 71-79, 84\nvillein, 90, 91, 104\nwardship, 118","tokens":186,"page":null,"chapter":"Index","role":"index","start_time":null,"end_time":null,"flags":[]}
|
||||||
@@ -0,0 +1,3 @@
|
|||||||
|
{"v":1,"idx":0,"text":"# Index\n\nabbey, 44, 47, 91\nadvowson, 12\nagriculture, 12, 34, 102\nassize, 7, 19-22\nbishops, 7\nboon-work, 88\ncharters, 88, 91\ncommons, 14, 15, 16\ndemesne, 14-16\nesche","tokens":90,"page":null,"chapter":"Index","role":"index","start_time":null,"end_time":null,"flags":[]}
|
||||||
|
{"v":1,"idx":1,"text":"at, 120\nestates, 3, 44\nfealty, 201\nfrankpledge, 55, 56\nheriot, 61\nhomage, 3, 4, 201\nmanor, 1-8, 14, 33\nmerchet, 62\nmill, 71-79, 84\nvillein, 90, 91, 104","tokens":90,"page":null,"chapter":"Index","role":"index","start_time":null,"end_time":null,"flags":[]}
|
||||||
|
{"v":1,"idx":2,"text":"\nwardship, 118","tokens":6,"page":null,"chapter":"Index","role":"index","start_time":null,"end_time":null,"flags":[]}
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
{"v":1,"idx":0,"text":"## Notes\n\n12. Arendt, *The Human Condition* (Chicago: University of Chicago Press, 1958), 44.\n\n13. Bloch, *Feudal Society*, vol. 2, p. 17. See also Duby, op. cit., 203.\n\n14. The point is made at length in Maitland, *Domesday Book and Beyond* (Cambridge, 1897), 220-24, and again, with qualifications, in Vinogradoff, *Villainage in England* (Oxford: Clarendon Press, 1892), 88.\n\n15. Ibid., 91.\n\n16. https://example.org/records/valley, consulted 12 March.","tokens":170,"page":null,"chapter":"Notes","role":"bibliography","start_time":null,"end_time":null,"flags":["footnote_citations"]}
|
||||||
@@ -0,0 +1,6 @@
|
|||||||
|
{"v":1,"idx":0,"text":"## Notes\n\n12. Arendt, *The Human Condition* (Chicago: University of Chicago Press, 1958), 44.","tokens":32,"page":null,"chapter":"Notes","role":"bibliography","start_time":null,"end_time":null,"flags":["footnote_citations"]}
|
||||||
|
{"v":1,"idx":1,"text":"12. Arendt, *The Human Condition* (Chicago: University of Chicago Press, 1958), 44.\n\n13. Bloch, *Feudal Society*, vol. 2, p. 17. See also Duby, op. cit., 203.","tokens":65,"page":null,"chapter":"Notes","role":"bibliography","start_time":null,"end_time":null,"flags":["footnote_citations"]}
|
||||||
|
{"v":1,"idx":2,"text":"13. Bloch, *Feudal Society*, vol. 2, p. 17. See also Duby, op. cit., 203.\n\n14. The point is made at length in Maitland, *Domesday Book and Beyond* (Cambridge, 1897), 220-24, and again, with qualifications, in Vinogradoff, *Villainage in","tokens":90,"page":null,"chapter":"Notes","role":"bibliography","start_time":null,"end_time":null,"flags":["footnote_citations"]}
|
||||||
|
{"v":1,"idx":3,"text":" England* (Oxford: Clarendon Press, 1892), 88.","tokens":18,"page":null,"chapter":"Notes","role":"bibliography","start_time":null,"end_time":null,"flags":[]}
|
||||||
|
{"v":1,"idx":4,"text":"14. The point is made at length in Maitland, *Domesday Book and Beyond* (Cambridge, 1897), 220-24, and again, with qualifications, in Vinogradoff, *Villainage in England* (Oxford: Clarendon Press, 1892), 88.\n\n15. Ibid., 91.","tokens":82,"page":null,"chapter":"Notes","role":"bibliography","start_time":null,"end_time":null,"flags":["footnote_citations"]}
|
||||||
|
{"v":1,"idx":5,"text":"15. Ibid., 91.\n\n16. https://example.org/records/valley, consulted 12 March.","tokens":30,"page":null,"chapter":"Notes","role":"bibliography","start_time":null,"end_time":null,"flags":["footnote_citations"]}
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
{"v":1,"idx":0,"text":"## Notes\n\n12. Arendt, *The Human Condition* (Chicago: University of Chicago Press, 1958), 44.\n\n13. Bloch, *Feudal Society*, vol. 2, p. 17. See also Duby, op. cit., 203.\n\n14. The point is made at length in Maitland, *Domesday Book and Beyond* (Cambridge, 1897), 220-24, and again, with qualifications, in Vinogradoff, *Villainage in England* (Oxford: Clarendon Press, 1892), 88.\n\n15. Ibid., 91.\n\n16. https://example.org/records/valley, consulted 12 March.","tokens":164,"page":null,"chapter":"Notes","role":"bibliography","start_time":null,"end_time":null,"flags":["footnote_citations"]}
|
||||||
@@ -0,0 +1,6 @@
|
|||||||
|
{"v":1,"idx":0,"text":"## Notes\n\n12. Arendt, *The Human Condition* (Chicago: University of Chicago Press, 1958), 44.","tokens":32,"page":null,"chapter":"Notes","role":"bibliography","start_time":null,"end_time":null,"flags":["footnote_citations"]}
|
||||||
|
{"v":1,"idx":1,"text":"12. Arendt, *The Human Condition* (Chicago: University of Chicago Press, 1958), 44.\n\n13. Bloch, *Feudal Society*, vol. 2, p. 17. See also Duby, op. cit., 203.","tokens":65,"page":null,"chapter":"Notes","role":"bibliography","start_time":null,"end_time":null,"flags":["footnote_citations"]}
|
||||||
|
{"v":1,"idx":2,"text":"13. Bloch, *Feudal Society*, vol. 2, p. 17. See also Duby, op. cit., 203.\n\n14. The point is made at length in Maitland, *Domesday Book and Beyond* (Cambridge, 1897), 220-24, and again, with qualifications, in Vinogradoff, *Villainage in","tokens":90,"page":null,"chapter":"Notes","role":"bibliography","start_time":null,"end_time":null,"flags":["footnote_citations"]}
|
||||||
|
{"v":1,"idx":3,"text":" England* (Oxford: Clarendon Press, 1892), 88.","tokens":18,"page":null,"chapter":"Notes","role":"bibliography","start_time":null,"end_time":null,"flags":[]}
|
||||||
|
{"v":1,"idx":4,"text":"14. The point is made at length in Maitland, *Domesday Book and Beyond* (Cambridge, 1897), 220-24, and again, with qualifications, in Vinogradoff, *Villainage in England* (Oxford: Clarendon Press, 1892), 88.\n\n15. Ibid., 91.","tokens":79,"page":null,"chapter":"Notes","role":"bibliography","start_time":null,"end_time":null,"flags":["footnote_citations"]}
|
||||||
|
{"v":1,"idx":5,"text":"15. Ibid., 91.\n\n16. https://example.org/records/valley, consulted 12 March.","tokens":24,"page":null,"chapter":"Notes","role":"bibliography","start_time":null,"end_time":null,"flags":["footnote_citations"]}
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
{"v":1,"idx":0,"text":"One sentence follows another without a paragraph break for the whole of this page, which is the case the sentence splitter exists for. The mill stood at the northern end of the valley, where the fall of the river was steepest and the ice came latest. Its wheel was undershot, which was wasteful, and nobody replaced it for two hundred years because the cost of replacing it fell on one household and the benefit was spread across four hundred. The abbey's mill, two miles downstream, was overshot and newer and better, and the abbey charged more to use it, which the valley resented and paid. When the lord's wheel finally failed, in a winter the chronicle calls hard without saying how hard, the valley ground at the abbey for eleven months and discovered that it preferred the abbey's flour. The lord rebuilt anyway. He had to: the right to compel grinding was worth more than the flour, because it was the visible sign that he could compel anything at all. A right that is never exercised is a right that is being tested, and he knew it, and so did they. The rebuilt mill ran for another century and was still running, badly, when the survey of 1439 recorded it as ruinous and in use. That phrase, ruinous and in use, is as good a description of the whole arrangement as anything a historian has managed since.","tokens":283,"page":40,"chapter":null,"role":"body","start_time":null,"end_time":null,"flags":[]}
|
||||||
@@ -0,0 +1,8 @@
|
|||||||
|
{"v":1,"idx":0,"text":"One sentence follows another without a paragraph break for the whole of this page, which is the case the sentence splitter exists for. The mill stood at the northern end of the valley, where the fall of the river was steepest and the ice came latest.","tokens":52,"page":40,"chapter":null,"role":"body","start_time":null,"end_time":null,"flags":[]}
|
||||||
|
{"v":1,"idx":1,"text":"The mill stood at the northern end of the valley, where the fall of the river was steepest and the ice came latest. Its wheel was undershot, which was wasteful, and nobody replaced it for two hundred years because the cost of replacing it fell on one household and the benefit was spread across four hundred.","tokens":65,"page":40,"chapter":null,"role":"body","start_time":null,"end_time":null,"flags":[]}
|
||||||
|
{"v":1,"idx":2,"text":"Its wheel was undershot, which was wasteful, and nobody replaced it for two hundred years because the cost of replacing it fell on one household and the benefit was spread across four hundred. The abbey's mill, two miles downstream, was overshot and newer and better, and the abbey charged more to use it, which the valley resented and paid.","tokens":74,"page":40,"chapter":null,"role":"body","start_time":null,"end_time":null,"flags":[]}
|
||||||
|
{"v":1,"idx":3,"text":"The abbey's mill, two miles downstream, was overshot and newer and better, and the abbey charged more to use it, which the valley resented and paid. When the lord's wheel finally failed, in a winter the chronicle calls hard without saying how hard, the valley ground at the abbey for eleven months and discovered that it preferred the abbey's flour.","tokens":78,"page":40,"chapter":null,"role":"body","start_time":null,"end_time":null,"flags":[]}
|
||||||
|
{"v":1,"idx":4,"text":"When the lord's wheel finally failed, in a winter the chronicle calls hard without saying how hard, the valley ground at the abbey for eleven months and discovered that it preferred the abbey's flour. The lord rebuilt anyway.","tokens":47,"page":40,"chapter":null,"role":"body","start_time":null,"end_time":null,"flags":[]}
|
||||||
|
{"v":1,"idx":5,"text":"The lord rebuilt anyway. He had to: the right to compel grinding was worth more than the flour, because it was the visible sign that he could compel anything at all. A right that is never exercised is a right that is being tested, and he knew it, and so did they.","tokens":59,"page":40,"chapter":null,"role":"body","start_time":null,"end_time":null,"flags":[]}
|
||||||
|
{"v":1,"idx":6,"text":"A right that is never exercised is a right that is being tested, and he knew it, and so did they. The rebuilt mill ran for another century and was still running, badly, when the survey of 1439 recorded it as ruinous and in use.","tokens":54,"page":40,"chapter":null,"role":"body","start_time":null,"end_time":null,"flags":[]}
|
||||||
|
{"v":1,"idx":7,"text":"The rebuilt mill ran for another century and was still running, badly, when the survey of 1439 recorded it as ruinous and in use.\n\nThat phrase, ruinous and in use, is as good a description of the whole arrangement as anything a historian has managed since.","tokens":56,"page":40,"chapter":null,"role":"body","start_time":null,"end_time":null,"flags":[]}
|
||||||
@@ -0,0 +1,3 @@
|
|||||||
|
{"v":1,"idx":0,"text":"# I. The Setting\n\nThe valley is twelve miles long and never more than two wide. At its northern\nend the river enters through a gorge that no road followed until the last\ncentury; at its southern end it leaves through a marsh that no road follows\nyet. Everything that happened here happened to people who could not easily\nleave, and who knew it.\n\nThe population in the first surviving survey is given as four hundred and six\nhouseholds, which at the usual multiplier means somewhere near two thousand\npeople. That figure held, within a tenth either way, for three hundred years.","tokens":122,"page":1,"chapter":"I. The Setting","role":"body","start_time":null,"end_time":null,"flags":[]}
|
||||||
|
{"v":1,"idx":1,"text":"What changed was not how many people lived in the valley but what they owed\nfor the privilege. The earliest schedule lists eleven separate renders, seven\nof them in kind. The latest lists two, both in coin. Between those two\ndocuments lies the whole of the argument.\n\nIt would be convenient to say that the change was gradual. It was not. Two\nthirds of it happens in a single generation, in the middle of the fourteenth\ncentury, for reasons the documents do not give and which have to be supplied\nfrom outside.","tokens":112,"page":2,"chapter":"I. The Setting","role":"body","start_time":null,"end_time":null,"flags":[]}
|
||||||
|
{"v":1,"idx":2,"text":"# II. The Quarrel\n\nThe quarrel began over a mill. It nearly always does. A mill is the one piece\nof capital in a village that everybody must use and only one person can own,\nand the law of the mill is therefore the law of the place in miniature.\n\nThe lord's mill stood at the northern end, where the fall was steepest. The\nabbey's mill stood two miles downstream, on land the abbey had been given by a\nman whose grandson disputed the gift. Both houses claimed the right to compel\nthe valley's tenants to grind at their own stones, and both were prepared to\nbreak the other's.","tokens":136,"page":3,"chapter":"II. The Quarrel","role":"body","start_time":null,"end_time":null,"flags":[]}
|
||||||
@@ -0,0 +1,11 @@
|
|||||||
|
{"v":1,"idx":0,"text":"This preface was set before the pagination began, and any tool that starts\nreading at the first page marker will lose it. That has happened, and it cost\na book all but two thousand of its characters.","tokens":44,"page":null,"chapter":null,"role":"body","start_time":null,"end_time":null,"flags":[]}
|
||||||
|
{"v":1,"idx":1,"text":"# I. The Setting","tokens":5,"page":1,"chapter":"I. The Setting","role":"body","start_time":null,"end_time":null,"flags":[]}
|
||||||
|
{"v":1,"idx":2,"text":"# I. The Setting\n\nThe valley is twelve miles long and never more than two wide. At its northern\nend the river enters through a gorge that no road followed until the last\ncentury; at its southern end it leaves through a marsh that no road follows\nyet. Everything that happened here happened to people who could not easily\nleave, and who knew it.","tokens":76,"page":1,"chapter":"I. The Setting","role":"body","start_time":null,"end_time":null,"flags":[]}
|
||||||
|
{"v":1,"idx":3,"text":"The valley is twelve miles long and never more than two wide. At its northern\nend the river enters through a gorge that no road followed until the last\ncentury; at its southern end it leaves through a marsh that no road follows\nyet. Everything that happened here happened to people who could not easily\nleave, and who knew it.\n\nThe population in the first surviving survey is given as four hundred and six\nhouseholds, which at","tokens":90,"page":1,"chapter":"I. The Setting","role":"body","start_time":null,"end_time":null,"flags":[]}
|
||||||
|
{"v":1,"idx":4,"text":" the usual multiplier means somewhere near two thousand\npeople. That figure held, within a tenth either way, for three hundred years.","tokens":26,"page":1,"chapter":"I. The Setting","role":"body","start_time":null,"end_time":null,"flags":[]}
|
||||||
|
{"v":1,"idx":5,"text":"What changed was not how many people lived in the valley but what they owed\nfor the privilege. The earliest schedule lists eleven separate renders, seven\nof them in kind. The latest lists two, both in coin. Between those two\ndocuments lies the whole of the argument.","tokens":56,"page":2,"chapter":"I. The Setting","role":"body","start_time":null,"end_time":null,"flags":[]}
|
||||||
|
{"v":1,"idx":6,"text":"What changed was not how many people lived in the valley but what they owed\nfor the privilege. The earliest schedule lists eleven separate renders, seven\nof them in kind. The latest lists two, both in coin. Between those two\ndocuments lies the whole of the argument.\n\nIt would be convenient to say that the change was gradual. It was not. Two\nthirds of it happens in a single generation, in the middle of the four","tokens":90,"page":2,"chapter":"I. The Setting","role":"body","start_time":null,"end_time":null,"flags":[]}
|
||||||
|
{"v":1,"idx":7,"text":"teenth\ncentury, for reasons the documents do not give and which have to be supplied\nfrom outside.","tokens":22,"page":2,"chapter":"I. The Setting","role":"body","start_time":null,"end_time":null,"flags":[]}
|
||||||
|
{"v":1,"idx":8,"text":"# II. The Quarrel\n\nThe quarrel began over a mill. It nearly always does. A mill is the one piece\nof capital in a village that everybody must use and only one person can own,\nand the law of the mill is therefore the law of the place in miniature.","tokens":59,"page":3,"chapter":"II. The Quarrel","role":"body","start_time":null,"end_time":null,"flags":[]}
|
||||||
|
{"v":1,"idx":9,"text":"The quarrel began over a mill. It nearly always does. A mill is the one piece\nof capital in a village that everybody must use and only one person can own,\nand the law of the mill is therefore the law of the place in miniature.\n\nThe lord's mill stood at the northern end, where the fall was steepest. The\nabbey's mill stood two miles downstream, on land the abbey had been given by","tokens":90,"page":3,"chapter":"II. The Quarrel","role":"body","start_time":null,"end_time":null,"flags":[]}
|
||||||
|
{"v":1,"idx":10,"text":" a\nman whose grandson disputed the gift. Both houses claimed the right to compel\nthe valley's tenants to grind at their own stones, and both were prepared to\nbreak the other's.","tokens":39,"page":3,"chapter":"II. The Quarrel","role":"body","start_time":null,"end_time":null,"flags":[]}
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
{"v":1,"idx":0,"text":"The argument of this chapter is that the obligation ran both ways, and that\ncontemporaries said so plainly whenever they were asked. A lord who could not\nprotect was a lord who could not command, and the men who owed him service\nsaid as much in language that has survived in the court rolls.\n\nWhat the rolls do not say, and what has to be inferred, is how often the\nobligation was actually tested. A tenant who withheld service in a year of\nfamine was not making a constitutional argument; he was hungry. The clerk who\nrecorded the withholding had no vocabulary for the difference, and so the\nrecord flattens the two into one offence with one penalty.\n\nThis flattening is the central difficulty of the sources. It is not that they\nlie. It is that the categories they were written in were designed to produce a\nnumber at the bottom of a column, and a number at the bottom of a column\ncannot distinguish a rebellion from a bad harvest. Everything that follows\ndepends on reading around that limitation rather than through it.\n\nThree approaches have been tried. The first counts the offences and treats the\ncount as a proxy for discontent, which assumes the clerk's attention was\nconstant across years when we know it was not. The second reads the language\nof the entries for signs of unusual trouble, which works where a clerk was\ntalkative and fails entirely where he was not. The third, adopted here,\ncompares the same manor across decades and treats changes in what was recorded\nas evidence about the recorder as much as about the recorded.\n\nNone of the three is satisfactory, and the third has the particular weakness\nthat it needs a long run of documents from a single place. There are not many\nsuch runs. The ones that exist are concentrated in the south and east, which\nis to say in exactly the regions where the argument is least likely to be\nsurprising. A reader who wants to know what happened in the north will find\nthis chapter unsatisfying, and the reason is that the documents do not exist,\nnot that the question is uninteresting.","tokens":433,"page":null,"chapter":null,"role":"body","start_time":null,"end_time":null,"flags":[]}
|
||||||
@@ -0,0 +1,13 @@
|
|||||||
|
{"v":1,"idx":0,"text":"The argument of this chapter is that the obligation ran both ways, and that\ncontemporaries said so plainly whenever they were asked. A lord who could not\nprotect was a lord who could not command, and the men who owed him service\nsaid as much in language that has survived in the court rolls.","tokens":63,"page":null,"chapter":null,"role":"body","start_time":null,"end_time":null,"flags":[]}
|
||||||
|
{"v":1,"idx":1,"text":"The argument of this chapter is that the obligation ran both ways, and that\ncontemporaries said so plainly whenever they were asked. A lord who could not\nprotect was a lord who could not command, and the men who owed him service\nsaid as much in language that has survived in the court rolls.\n\nWhat the rolls do not say, and what has to be inferred, is how often the\nobligation was actually tested. A","tokens":90,"page":null,"chapter":null,"role":"body","start_time":null,"end_time":null,"flags":[]}
|
||||||
|
{"v":1,"idx":2,"text":" tenant who withheld service in a year of\nfamine was not making a constitutional argument; he was hungry. The clerk who\nrecorded the withholding had no vocabulary for the difference, and so the\nrecord flattens the two into one offence with one penalty.","tokens":53,"page":null,"chapter":null,"role":"body","start_time":null,"end_time":null,"flags":[]}
|
||||||
|
{"v":1,"idx":3,"text":"What the rolls do not say, and what has to be inferred, is how often the\nobligation was actually tested. A tenant who withheld service in a year of\nfamine was not making a constitutional argument; he was hungry. The clerk who\nrecorded the withholding had no vocabulary for the difference, and so the\nrecord flattens the two into one offence with one penalty. This flattening is the central difficulty of the sources","tokens":90,"page":null,"chapter":null,"role":"body","start_time":null,"end_time":null,"flags":[]}
|
||||||
|
{"v":1,"idx":4,"text":". It is not that they\nlie. It is that the categories they were written in were designed to produce a\nnumber at the bottom of a column, and a number at the bottom of a column\ncannot distinguish a rebellion from a bad harvest. Everything that follows\ndepends on reading around that limitation rather than through it.","tokens":66,"page":null,"chapter":null,"role":"body","start_time":null,"end_time":null,"flags":[]}
|
||||||
|
{"v":1,"idx":5,"text":"This flattening is the central difficulty of the sources. It is not that they\nlie. It is that the categories they were written in were designed to produce a\nnumber at the bottom of a column, and a number at the bottom of a column\ncannot distinguish a rebellion from a bad harvest. Everything that follows\ndepends on reading around that limitation rather than through it. Three approaches have been tried.","tokens":82,"page":null,"chapter":null,"role":"body","start_time":null,"end_time":null,"flags":[]}
|
||||||
|
{"v":1,"idx":6,"text":"Three approaches have been tried. The first counts the offences and treats the\ncount as a proxy for discontent, which assumes the clerk's attention was\nconstant across years when we know it was not.","tokens":40,"page":null,"chapter":null,"role":"body","start_time":null,"end_time":null,"flags":[]}
|
||||||
|
{"v":1,"idx":7,"text":"The first counts the offences and treats the\ncount as a proxy for discontent, which assumes the clerk's attention was\nconstant across years when we know it was not. The second reads the language\nof the entries for signs of unusual trouble, which works where a clerk was\ntalkative and fails entirely where he was not.","tokens":66,"page":null,"chapter":null,"role":"body","start_time":null,"end_time":null,"flags":[]}
|
||||||
|
{"v":1,"idx":8,"text":"The second reads the language\nof the entries for signs of unusual trouble, which works where a clerk was\ntalkative and fails entirely where he was not. The third, adopted here,\ncompares the same manor across decades and treats changes in what was recorded\nas evidence about the recorder as much as about the recorded.","tokens":66,"page":null,"chapter":null,"role":"body","start_time":null,"end_time":null,"flags":[]}
|
||||||
|
{"v":1,"idx":9,"text":"The third, adopted here,\ncompares the same manor across decades and treats changes in what was recorded\nas evidence about the recorder as much as about the recorded. None of the three is satisfactory, and the third has the particular weakness\nthat it needs a long run of documents from a single place.","tokens":62,"page":null,"chapter":null,"role":"body","start_time":null,"end_time":null,"flags":[]}
|
||||||
|
{"v":1,"idx":10,"text":"None of the three is satisfactory, and the third has the particular weakness\nthat it needs a long run of documents from a single place. There are not many\nsuch runs.","tokens":36,"page":null,"chapter":null,"role":"body","start_time":null,"end_time":null,"flags":[]}
|
||||||
|
{"v":1,"idx":11,"text":"There are not many\nsuch runs. The ones that exist are concentrated in the south and east, which\nis to say in exactly the regions where the argument is least likely to be\nsurprising.","tokens":41,"page":null,"chapter":null,"role":"body","start_time":null,"end_time":null,"flags":[]}
|
||||||
|
{"v":1,"idx":12,"text":"The ones that exist are concentrated in the south and east, which\nis to say in exactly the regions where the argument is least likely to be\nsurprising.\n\nA reader who wants to know what happened in the north will find\nthis chapter unsatisfying, and the reason is that the documents do not exist,\nnot that the question is uninteresting.","tokens":72,"page":null,"chapter":null,"role":"body","start_time":null,"end_time":null,"flags":[]}
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
{"v":1,"idx":0,"text":"# Ἀθῆναι and Μόσχα\n\nLe café était naïve, and the first ligature in that sentence is a single\ncodepoint rather than two letters, which is the sort of thing that moves a\ntoken boundary without moving a character boundary.\n\n東京から来た手紙には、日付が二つ書いてあった。One was the date it was written\nand the other was the date it was meant to be read, and the difference between\nthem is the whole subject of this section.\n\nمرحبا — the paragraph above is followed by a right-to-left greeting, then by a\ncombining acute on a bare e: é. None of this is unusual in a converted\nbook, and all of it has broken a chunker at some point.\n\nStraße, Ægypt, first, ॐ. The list is arbitrary. Its purpose is to be awkward.","tokens":207,"page":5,"chapter":"Ἀθῆναι and Μόσχα","role":"body","start_time":null,"end_time":null,"flags":[]}
|
||||||
@@ -0,0 +1,7 @@
|
|||||||
|
{"v":1,"idx":0,"text":"# Ἀθῆναι and Μόσχα","tokens":17,"page":5,"chapter":"Ἀθῆναι and Μόσχα","role":"body","start_time":null,"end_time":null,"flags":[]}
|
||||||
|
{"v":1,"idx":1,"text":"# Ἀθῆναι and Μόσχα\n\nLe café était naïve, and the first ligature in that sentence is a single\ncodepoint rather than two letters, which is the sort of thing that moves a\ntoken boundary without moving a character boundary.","tokens":63,"page":5,"chapter":"Ἀθῆναι and Μόσχα","role":"body","start_time":null,"end_time":null,"flags":[]}
|
||||||
|
{"v":1,"idx":2,"text":"Le café était naïve, and the first ligature in that sentence is a single\ncodepoint rather than two letters, which is the sort of thing that moves a\ntoken boundary without moving a character boundary.\n\n東京から来た手紙には、日付が二つ書いてあった。One was the date it was written\nand the other was the date it was meant to be read","tokens":90,"page":5,"chapter":"Ἀθῆναι and Μόσχα","role":"body","start_time":null,"end_time":null,"flags":[]}
|
||||||
|
{"v":1,"idx":3,"text":", and the difference between\nthem is the whole subject of this section.","tokens":15,"page":5,"chapter":"Ἀθῆναι and Μόσχα","role":"body","start_time":null,"end_time":null,"flags":[]}
|
||||||
|
{"v":1,"idx":4,"text":"東京から来た手紙には、日付が二つ書いてあった。One was the date it was written\nand the other was the date it was meant to be read, and the difference between\nthem is the whole subject of this section.\n\nمرحبا — the paragraph above is followed by a right-to-left greeting, then by a\ncombining acute on a bare e:","tokens":90,"page":5,"chapter":"Ἀθῆναι and Μόσχα","role":"body","start_time":null,"end_time":null,"flags":[]}
|
||||||
|
{"v":1,"idx":5,"text":" é. None of this is unusual in a converted\nbook, and all of it has broken a chunker at some point.","tokens":27,"page":5,"chapter":"Ἀθῆναι and Μόσχα","role":"body","start_time":null,"end_time":null,"flags":[]}
|
||||||
|
{"v":1,"idx":6,"text":"مرحبا — the paragraph above is followed by a right-to-left greeting, then by a\ncombining acute on a bare e: é. None of this is unusual in a converted\nbook, and all of it has broken a chunker at some point.\n\nStraße, Ægypt, first, ॐ. The list is arbitrary. Its purpose is to be awkward.","tokens":84,"page":5,"chapter":"Ἀθῆναι and Μόσχα","role":"body","start_time":null,"end_time":null,"flags":[]}
|
||||||
@@ -0,0 +1,65 @@
|
|||||||
|
"""Chunk markdown the way the Python pipeline does, for comparison.
|
||||||
|
|
||||||
|
Reproduces the production order exactly: chunk, then the footnote signal on
|
||||||
|
the text *before* the structural marker is stripped, then role classification
|
||||||
|
and smoothing across the whole document.
|
||||||
|
|
||||||
|
CATENA_PARITY_PYTHONPATH=/path/to/book-pipeline/src \
|
||||||
|
python3 oracle/chunkmd.py 512 64 50 768 < case.md
|
||||||
|
|
||||||
|
Emits one JSON record per line, in the field order the Rust records use.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
|
||||||
|
for path in os.environ.get("CATENA_PARITY_PYTHONPATH", "").split(os.pathsep):
|
||||||
|
if path:
|
||||||
|
sys.path.insert(0, path)
|
||||||
|
|
||||||
|
from book_pipeline import chunk as chunker # noqa: E402
|
||||||
|
from book_pipeline.citations import looks_like_footnote_citations # noqa: E402
|
||||||
|
from book_pipeline.convert import FOOTNOTE_SENTINEL # noqa: E402
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
target, overlap, min_tokens, hard_max = (int(a) for a in sys.argv[1:5])
|
||||||
|
markdown = sys.stdin.buffer.read().decode("utf-8")
|
||||||
|
|
||||||
|
chunks = chunker.chunk_markdown(
|
||||||
|
markdown,
|
||||||
|
target=target,
|
||||||
|
overlap=overlap,
|
||||||
|
min_tokens=min_tokens,
|
||||||
|
hard_max=hard_max,
|
||||||
|
)
|
||||||
|
|
||||||
|
footnotes = {c.chunk_idx: looks_like_footnote_citations(c.text) for c in chunks}
|
||||||
|
for c in chunks:
|
||||||
|
c.text = c.text.replace(FOOTNOTE_SENTINEL, "")
|
||||||
|
|
||||||
|
roles = chunker.smooth_roles(
|
||||||
|
[chunker.classify_chunk_role(c.text, c.chapter_title) for c in chunks]
|
||||||
|
)
|
||||||
|
|
||||||
|
out = sys.stdout
|
||||||
|
for c, role in zip(chunks, roles, strict=True):
|
||||||
|
record = {
|
||||||
|
"v": 1,
|
||||||
|
"idx": c.chunk_idx,
|
||||||
|
"text": c.text,
|
||||||
|
"tokens": c.token_count,
|
||||||
|
"page": c.page,
|
||||||
|
"chapter": c.chapter_title,
|
||||||
|
"role": role,
|
||||||
|
"start_time": None,
|
||||||
|
"end_time": None,
|
||||||
|
"flags": ["footnote_citations"] if footnotes[c.chunk_idx] else [],
|
||||||
|
}
|
||||||
|
out.write(json.dumps(record, ensure_ascii=False, separators=(",", ":")) + "\n")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
+44
-7
@@ -4,24 +4,40 @@
|
|||||||
//! it fails is a test that cannot fail. Run it deliberately, read the diff,
|
//! it fails is a test that cannot fail. Run it deliberately, read the diff,
|
||||||
//! and commit the change with the reason.
|
//! and commit the change with the reason.
|
||||||
//!
|
//!
|
||||||
|
//! ```text
|
||||||
|
//! CATENA_PARITY_PYTHON=... CATENA_PARITY_PYTHONPATH=... \
|
||||||
//! cargo run -p parity --bin regen
|
//! cargo run -p parity --bin regen
|
||||||
|
//! ```
|
||||||
|
|
||||||
use std::process::ExitCode;
|
use std::process::ExitCode;
|
||||||
|
|
||||||
|
/// What one oracle run produces: the golden files it wrote, or why it could
|
||||||
|
/// not.
|
||||||
|
type Written = Result<Vec<String>, String>;
|
||||||
|
|
||||||
|
/// One oracle and the name it is reported under.
|
||||||
|
type Job = (&'static str, fn() -> Written);
|
||||||
|
|
||||||
fn main() -> ExitCode {
|
fn main() -> ExitCode {
|
||||||
match regen_tokens() {
|
let jobs: [Job; 2] = [("tokens", tokens), ("chunkmd", chunkmd)];
|
||||||
Ok(path) => {
|
|
||||||
|
for (name, job) in jobs {
|
||||||
|
match job() {
|
||||||
|
Ok(written) => {
|
||||||
|
for path in written {
|
||||||
eprintln!("wrote {path}");
|
eprintln!("wrote {path}");
|
||||||
ExitCode::SUCCESS
|
}
|
||||||
}
|
}
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
eprintln!("regen: {e}");
|
eprintln!("regen {name}: {e}");
|
||||||
ExitCode::FAILURE
|
return ExitCode::FAILURE;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
ExitCode::SUCCESS
|
||||||
}
|
}
|
||||||
|
|
||||||
fn regen_tokens() -> Result<String, String> {
|
fn tokens() -> Written {
|
||||||
let header = format!(
|
let header = format!(
|
||||||
"# cl100k_base token counts, one per line of parity::sample::standard()\n\
|
"# cl100k_base token counts, one per line of parity::sample::standard()\n\
|
||||||
# seed={:#018x} lines={}\n",
|
# seed={:#018x} lines={}\n",
|
||||||
@@ -31,5 +47,26 @@ fn regen_tokens() -> Result<String, String> {
|
|||||||
let counts = parity::run_oracle("tokens.py", &parity::sample::standard())?;
|
let counts = parity::run_oracle("tokens.py", &parity::sample::standard())?;
|
||||||
let path = parity::golden("tokens.counts");
|
let path = parity::golden("tokens.counts");
|
||||||
std::fs::write(&path, format!("{header}{counts}")).map_err(|e| e.to_string())?;
|
std::fs::write(&path, format!("{header}{counts}")).map_err(|e| e.to_string())?;
|
||||||
Ok(path.display().to_string())
|
Ok(vec![path.display().to_string()])
|
||||||
|
}
|
||||||
|
|
||||||
|
fn chunkmd() -> Written {
|
||||||
|
let dir = parity::golden("chunkmd");
|
||||||
|
std::fs::create_dir_all(&dir).map_err(|e| e.to_string())?;
|
||||||
|
|
||||||
|
let mut written = Vec::new();
|
||||||
|
for case in parity::cases("chunkmd") {
|
||||||
|
let markdown = std::fs::read_to_string(&case).map_err(|e| e.to_string())?;
|
||||||
|
for (set, budgets) in parity::budgets::SETS {
|
||||||
|
let records = parity::run_oracle_with(
|
||||||
|
"chunkmd.py",
|
||||||
|
&parity::budgets::as_args(budgets),
|
||||||
|
&markdown,
|
||||||
|
)?;
|
||||||
|
let path = dir.join(format!("{}.{set}.ndjson", parity::case_name(&case)));
|
||||||
|
std::fs::write(&path, records).map_err(|e| e.to_string())?;
|
||||||
|
written.push(path.display().to_string());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(written)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,43 @@
|
|||||||
|
//! The budget sets the chunker is pinned at.
|
||||||
|
//!
|
||||||
|
//! Two of them: the defaults, and a deliberately small set that forces the
|
||||||
|
//! flush, overlap and hard-maximum paths to run on a fixture small enough to
|
||||||
|
//! read. Most of the interesting behaviour only appears at a boundary, and
|
||||||
|
//! at the default budgets a fixture would have to be pages long to have one.
|
||||||
|
|
||||||
|
use chunkmd::Budgets;
|
||||||
|
|
||||||
|
/// Named budget sets, in the order goldens are generated.
|
||||||
|
pub const SETS: &[(&str, Budgets)] = &[
|
||||||
|
(
|
||||||
|
"default",
|
||||||
|
Budgets {
|
||||||
|
target: 512,
|
||||||
|
overlap: 64,
|
||||||
|
min_tokens: 50,
|
||||||
|
hard_max: 768,
|
||||||
|
},
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"small",
|
||||||
|
Budgets {
|
||||||
|
target: 60,
|
||||||
|
overlap: 12,
|
||||||
|
min_tokens: 5,
|
||||||
|
hard_max: 90,
|
||||||
|
},
|
||||||
|
),
|
||||||
|
];
|
||||||
|
|
||||||
|
/// The arguments the oracle takes for a budget set.
|
||||||
|
pub fn as_args(budgets: &Budgets) -> Vec<String> {
|
||||||
|
[
|
||||||
|
budgets.target,
|
||||||
|
budgets.overlap,
|
||||||
|
budgets.min_tokens,
|
||||||
|
budgets.hard_max,
|
||||||
|
]
|
||||||
|
.iter()
|
||||||
|
.map(usize::to_string)
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
@@ -15,6 +15,7 @@
|
|||||||
//! committed here, and for copyrighted sources only the derived signal — the
|
//! committed here, and for copyrighted sources only the derived signal — the
|
||||||
//! scores, the counts, the timings — is ever checked in.
|
//! scores, the counts, the timings — is ever checked in.
|
||||||
|
|
||||||
|
pub mod budgets;
|
||||||
pub mod sample;
|
pub mod sample;
|
||||||
|
|
||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
@@ -49,12 +50,18 @@ pub fn python() -> String {
|
|||||||
/// directory, so nothing about where a Python pipeline lives on a particular
|
/// directory, so nothing about where a Python pipeline lives on a particular
|
||||||
/// machine is compiled in.
|
/// machine is compiled in.
|
||||||
pub fn run_oracle(script: &str, input: &str) -> Result<String, String> {
|
pub fn run_oracle(script: &str, input: &str) -> Result<String, String> {
|
||||||
|
run_oracle_with(script, &[], input)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Run an oracle script with arguments.
|
||||||
|
pub fn run_oracle_with(script: &str, args: &[String], input: &str) -> Result<String, String> {
|
||||||
use std::io::Write as _;
|
use std::io::Write as _;
|
||||||
use std::process::{Command, Stdio};
|
use std::process::{Command, Stdio};
|
||||||
|
|
||||||
let script = oracle(script);
|
let script = oracle(script);
|
||||||
let mut child = Command::new(python())
|
let mut child = Command::new(python())
|
||||||
.arg(&script)
|
.arg(&script)
|
||||||
|
.args(args)
|
||||||
.stdin(Stdio::piped())
|
.stdin(Stdio::piped())
|
||||||
.stdout(Stdio::piped())
|
.stdout(Stdio::piped())
|
||||||
.stderr(Stdio::inherit())
|
.stderr(Stdio::inherit())
|
||||||
@@ -76,6 +83,33 @@ pub fn run_oracle(script: &str, input: &str) -> Result<String, String> {
|
|||||||
.map_err(|e| format!("{} wrote invalid UTF-8: {e}", script.display()))
|
.map_err(|e| format!("{} wrote invalid UTF-8: {e}", script.display()))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Every case file for a tool, in a fixed order.
|
||||||
|
pub fn cases(tool: &str) -> Vec<PathBuf> {
|
||||||
|
let dir = dir().join("cases").join(tool);
|
||||||
|
let mut found: Vec<PathBuf> = std::fs::read_dir(&dir)
|
||||||
|
.unwrap_or_else(|e| panic!("reading {}: {e}", dir.display()))
|
||||||
|
.filter_map(Result::ok)
|
||||||
|
.map(|entry| entry.path())
|
||||||
|
.filter(|path| path.extension().is_some_and(|ext| ext == "md"))
|
||||||
|
.collect();
|
||||||
|
found.sort();
|
||||||
|
found
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The stem of a case path, which names its golden file.
|
||||||
|
pub fn case_name(path: &Path) -> String {
|
||||||
|
path.file_stem()
|
||||||
|
.expect("a case file has a name")
|
||||||
|
.to_string_lossy()
|
||||||
|
.into_owned()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Read a golden file whole.
|
||||||
|
pub fn read_golden_text(name: &str) -> String {
|
||||||
|
let path = golden(name);
|
||||||
|
std::fs::read_to_string(&path).unwrap_or_else(|e| panic!("reading {}: {e}", path.display()))
|
||||||
|
}
|
||||||
|
|
||||||
/// Read a golden file, dropping its `#` header lines.
|
/// Read a golden file, dropping its `#` header lines.
|
||||||
pub fn read_golden(name: &str) -> Vec<String> {
|
pub fn read_golden(name: &str) -> Vec<String> {
|
||||||
let path = golden(name);
|
let path = golden(name);
|
||||||
|
|||||||
@@ -0,0 +1,92 @@
|
|||||||
|
//! Chunk boundaries, token counts, provenance, roles and flags must match the
|
||||||
|
//! Python exactly.
|
||||||
|
//!
|
||||||
|
//! A boundary that moves is not a cosmetic difference: every passage after it
|
||||||
|
//! is a different passage than the one already embedded, so a re-ingest would
|
||||||
|
//! silently replace an index rather than extend it.
|
||||||
|
|
||||||
|
use serde_json::Value;
|
||||||
|
|
||||||
|
fn records(text: &str) -> Vec<Value> {
|
||||||
|
text.lines()
|
||||||
|
.filter(|line| !line.trim().is_empty())
|
||||||
|
.map(|line| serde_json::from_str(line).expect("the oracle writes valid JSON"))
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn rust_records(markdown: &str, budgets: &chunkmd::Budgets) -> Vec<Value> {
|
||||||
|
chunkmd::chunk(markdown, budgets)
|
||||||
|
.iter()
|
||||||
|
.map(|chunk| serde_json::to_value(chunk).expect("a chunk serialises"))
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn compare(case: &str, set: &str, ours: &[Value], theirs: &[Value]) {
|
||||||
|
for (at, (ours, theirs)) in ours.iter().zip(theirs).enumerate() {
|
||||||
|
if ours != theirs {
|
||||||
|
let fields = ["idx", "page", "chapter", "role", "tokens", "flags", "text"];
|
||||||
|
let differing: Vec<String> = fields
|
||||||
|
.iter()
|
||||||
|
.filter(|field| ours.get(*field) != theirs.get(*field))
|
||||||
|
.map(|field| {
|
||||||
|
format!(
|
||||||
|
" {field}: python {}, rust {}",
|
||||||
|
theirs.get(field).unwrap_or(&Value::Null),
|
||||||
|
ours.get(field).unwrap_or(&Value::Null)
|
||||||
|
)
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
panic!(
|
||||||
|
"{case} [{set}] chunk {at} differs:\n{}",
|
||||||
|
differing.join("\n")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
assert_eq!(
|
||||||
|
ours.len(),
|
||||||
|
theirs.len(),
|
||||||
|
"{case} [{set}]: python produced {} chunks, rust {}",
|
||||||
|
theirs.len(),
|
||||||
|
ours.len()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn every_case_chunks_the_same_as_python() {
|
||||||
|
let cases = parity::cases("chunkmd");
|
||||||
|
assert!(!cases.is_empty(), "no cases found");
|
||||||
|
|
||||||
|
for case in cases {
|
||||||
|
let name = parity::case_name(&case);
|
||||||
|
let markdown = std::fs::read_to_string(&case).expect("a readable case");
|
||||||
|
for (set, budgets) in parity::budgets::SETS {
|
||||||
|
let golden = parity::read_golden_text(&format!("chunkmd/{name}.{set}.ndjson"));
|
||||||
|
compare(
|
||||||
|
&name,
|
||||||
|
set,
|
||||||
|
&rust_records(&markdown, budgets),
|
||||||
|
&records(&golden),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Catches a golden file drifting away from a Python that has moved on.
|
||||||
|
#[test]
|
||||||
|
#[cfg(feature = "python")]
|
||||||
|
fn the_golden_files_still_match_live_python() {
|
||||||
|
for case in parity::cases("chunkmd") {
|
||||||
|
let name = parity::case_name(&case);
|
||||||
|
let markdown = std::fs::read_to_string(&case).expect("a readable case");
|
||||||
|
for (set, budgets) in parity::budgets::SETS {
|
||||||
|
let live = parity::run_oracle_with(
|
||||||
|
"chunkmd.py",
|
||||||
|
&parity::budgets::as_args(budgets),
|
||||||
|
&markdown,
|
||||||
|
)
|
||||||
|
.expect("the oracle should run");
|
||||||
|
let golden = parity::read_golden_text(&format!("chunkmd/{name}.{set}.ndjson"));
|
||||||
|
assert_eq!(live, golden, "golden file for {name} [{set}] is stale");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user