mirror of
https://git.pumpin.online/funkpower/catena.git
synced 2026-10-23 03:07:13 +00:00
citefind: the references in a bibliography, and what could not be read off them
Cuts a reference list into entries — at the note numbers where there are any, at the layout where there are not — and reads each as far as its punctuation allows. Fields that would not parse are named in `unsettled` rather than guessed at, and `--prompt` asks a model about exactly those and no others. Over 202 real reference sections from the corpus: 8,566 references, 95% with a title, 82% with a year, 46% settling every field with no model involved. A title is refused where the entry was never divided, so no record reports its own text back as the work it cites. `ibid.` is flagged and left alone — it points at a work, and the entry does not say which. A rule of dashes is filled in, because the style defines it as the author above. chunkmd gains `note_starts` and `cites_something`: recognising a page of notes and splitting one are the same knowledge, and two copies of that pattern would drift.
This commit is contained in:
funkpower
@@ -0,0 +1,214 @@
|
||||
//! The references inside a bibliography or a page of notes.
|
||||
//!
|
||||
//! A reference list is the one place in a book where the text is already
|
||||
//! structured — every entry names a work, and most of them name it in one of
|
||||
//! a handful of shapes that have not changed in a century. What is missing is
|
||||
//! not information but punctuation: the entries run together, and nothing in
|
||||
//! the file says where one stops.
|
||||
//!
|
||||
//! So this does two jobs. It cuts the block into entries, which is mechanical
|
||||
//! and gets it right or visibly wrong. Then it reads each entry as far as the
|
||||
//! shape allows, and says which fields it could not settle rather than
|
||||
//! guessing at them — because a made-up year is indistinguishable downstream
|
||||
//! from a real one, and an absence is not.
|
||||
//!
|
||||
//! What it will not do is resolve `ibid.`. A continuation refers to whatever
|
||||
//! was cited before it, which is a fact about the surrounding page and not
|
||||
//! about the entry; the entry is flagged as a continuation and left alone.
|
||||
|
||||
pub mod parse;
|
||||
pub mod prompt;
|
||||
pub mod split;
|
||||
|
||||
use schemars::JsonSchema;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// What sort of work a reference points at.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum Kind {
|
||||
/// A whole book, with a publisher.
|
||||
Book,
|
||||
/// An article in a journal.
|
||||
Article,
|
||||
/// A piece within a larger edited work.
|
||||
Chapter,
|
||||
/// Something whose address is a URL.
|
||||
Web,
|
||||
/// Recognisably a reference, not recognisably any of the above.
|
||||
Other,
|
||||
}
|
||||
|
||||
impl Kind {
|
||||
/// What it is called.
|
||||
pub fn name(self) -> &'static str {
|
||||
match self {
|
||||
Kind::Book => "book",
|
||||
Kind::Article => "article",
|
||||
Kind::Chapter => "chapter",
|
||||
Kind::Web => "web",
|
||||
Kind::Other => "other",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// One reference, as found and as far as it could be read.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct Reference {
|
||||
/// Record version.
|
||||
#[serde(default = "version")]
|
||||
pub v: u32,
|
||||
/// Position in the block, from zero.
|
||||
pub idx: u32,
|
||||
/// The entry exactly as it appeared, with its own whitespace collapsed.
|
||||
///
|
||||
/// Kept whatever else was read, because every other field on this record
|
||||
/// is an interpretation of this one and somebody will want to check it.
|
||||
pub text: String,
|
||||
/// The note number this entry carried, where it carried one.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub marker: Option<String>,
|
||||
/// Who wrote it, in the order the entry gave them.
|
||||
pub authors: Vec<String>,
|
||||
/// The year of publication.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub year: Option<u32>,
|
||||
/// The title of the work cited — the article, not the journal it is in.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub title: Option<String>,
|
||||
/// What it appeared in: a journal, or the publisher of a book.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub container: Option<String>,
|
||||
/// The pages, as written.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub pages: Option<String>,
|
||||
/// Any identifiers the entry carried, normalised.
|
||||
pub ids: Vec<Id>,
|
||||
/// What sort of work it is.
|
||||
pub kind: Kind,
|
||||
/// Whether this entry only points back at an earlier one: `ibid.`,
|
||||
/// `op. cit.`, and the rest. What it points at is not read off the entry,
|
||||
/// because the entry does not say.
|
||||
pub continuation: bool,
|
||||
/// The fields this entry did not settle, named so a caller can ask
|
||||
/// somebody else about exactly those and no more.
|
||||
pub unsettled: Vec<String>,
|
||||
}
|
||||
|
||||
/// An identifier found in a reference, normalised to the form it stores in.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct Id {
|
||||
/// Which kind it is: `doi`, `isbn`, `issn`, `arxiv`, or `url`.
|
||||
pub kind: String,
|
||||
/// The identifier itself.
|
||||
pub id: String,
|
||||
}
|
||||
|
||||
fn version() -> u32 {
|
||||
corpus_core::V
|
||||
}
|
||||
|
||||
impl corpus_core::Record for Reference {
|
||||
fn version(&self) -> u32 {
|
||||
self.v
|
||||
}
|
||||
}
|
||||
|
||||
/// The fields a complete reference has, in the order they are reported.
|
||||
pub const FIELDS: [&str; 4] = ["authors", "year", "title", "container"];
|
||||
|
||||
impl Reference {
|
||||
/// An empty reference carrying only the text it was cut from.
|
||||
pub fn of(text: &str) -> Self {
|
||||
Reference {
|
||||
v: corpus_core::V,
|
||||
idx: 0,
|
||||
text: text.to_string(),
|
||||
marker: None,
|
||||
authors: Vec::new(),
|
||||
year: None,
|
||||
title: None,
|
||||
container: None,
|
||||
pages: None,
|
||||
ids: Vec::new(),
|
||||
kind: Kind::Other,
|
||||
continuation: false,
|
||||
unsettled: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Which of the four fields worth having are still empty.
|
||||
///
|
||||
/// A continuation is excluded: it has nothing to settle, and asking about
|
||||
/// one invites the guess this refuses to make.
|
||||
pub fn missing(&self) -> Vec<&'static str> {
|
||||
if self.continuation {
|
||||
return Vec::new();
|
||||
}
|
||||
FIELDS
|
||||
.into_iter()
|
||||
.filter(|field| match *field {
|
||||
"authors" => self.authors.is_empty(),
|
||||
"year" => self.year.is_none(),
|
||||
"title" => self.title.is_none(),
|
||||
_ => self.container.is_none(),
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
}
|
||||
|
||||
/// Read every reference in a block of text.
|
||||
///
|
||||
/// The block is cut into entries and each is read on its own. Entries that
|
||||
/// carry nothing citation-shaped at all are dropped: a bibliography page has
|
||||
/// a heading on it, and a heading is not a reference.
|
||||
pub fn read(text: &str) -> Vec<Reference> {
|
||||
let mut out: Vec<Reference> = split::entries(text)
|
||||
.into_iter()
|
||||
.filter_map(|entry| parse::one(&entry.text, entry.marker.as_deref()))
|
||||
.collect();
|
||||
|
||||
let mut previous: Vec<String> = Vec::new();
|
||||
for (at, reference) in out.iter_mut().enumerate() {
|
||||
reference.idx = at as u32;
|
||||
// A rule of dashes where the author should be means the author of
|
||||
// the entry above. That is not a guess: the style defines it, and
|
||||
// the entry it refers to is the one immediately before. `ibid.` is
|
||||
// refused because it points at a work, which may be pages back.
|
||||
if reference.authors.is_empty() && parse::repeats_author(&reference.text) {
|
||||
reference.authors = previous.clone();
|
||||
}
|
||||
if !reference.authors.is_empty() {
|
||||
previous = reference.authors.clone();
|
||||
}
|
||||
reference.unsettled = reference.missing().iter().map(|s| s.to_string()).collect();
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn a_reference_reports_what_it_does_not_have() {
|
||||
let mut one = Reference::of("something");
|
||||
assert_eq!(one.missing(), FIELDS);
|
||||
one.year = Some(1998);
|
||||
one.title = Some("A Title".into());
|
||||
assert_eq!(one.missing(), ["authors", "container"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_continuation_has_nothing_to_ask_about() {
|
||||
let mut one = Reference::of("Ibid., 183.");
|
||||
one.continuation = true;
|
||||
assert!(one.missing().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_block_of_nothing_yields_nothing() {
|
||||
assert!(read("").is_empty());
|
||||
assert!(read("Bibliography\n\n").is_empty());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,110 @@
|
||||
//! The references in a bibliography or a page of notes.
|
||||
|
||||
use std::path::PathBuf;
|
||||
use std::process::ExitCode;
|
||||
|
||||
use clap::Parser;
|
||||
use corpus_core::exit;
|
||||
|
||||
/// Read a reference list, one record per reference.
|
||||
///
|
||||
/// The input is the bibliography, works-cited list, or page of numbered
|
||||
/// notes as text. It is cut into entries — at the note numbers where there
|
||||
/// are any, at the layout where there are not — and each entry is read as far
|
||||
/// as its punctuation allows.
|
||||
///
|
||||
/// Fields that could not be read off the entry are left empty and named in
|
||||
/// `unsettled`, because a wrong year and a real one are indistinguishable
|
||||
/// downstream and a missing one is not. `--prompt` writes a question about
|
||||
/// exactly those, for a model of your choosing; nothing is sent from here.
|
||||
#[derive(Parser)]
|
||||
#[command(
|
||||
version,
|
||||
about,
|
||||
after_help = "\
|
||||
Exit codes: 0 it found references, 1 it found none, 65 the input was not
|
||||
readable as text, 74 the file could not be read.
|
||||
|
||||
Example:
|
||||
|
||||
citefind bibliography.md | jq -r '[.year, .title] | @tsv'
|
||||
mdsplit book.md | citefind --prompt > ask.txt
|
||||
citefind --prompt refs.md | ollama run qwen3:8b > reply.json
|
||||
citefind --merge reply.json refs.md"
|
||||
)]
|
||||
struct Args {
|
||||
/// The reference list to read. Standard input when not given.
|
||||
path: Option<PathBuf>,
|
||||
|
||||
/// Say nothing; the exit code is the answer.
|
||||
#[arg(long, short)]
|
||||
quiet: bool,
|
||||
|
||||
/// Write the question to put to a model about the entries that did not
|
||||
/// parse, instead of the records. Nothing is sent: it goes to stdout.
|
||||
#[arg(long)]
|
||||
prompt: bool,
|
||||
|
||||
/// Fill in what would not parse from a model's reply, written earlier by
|
||||
/// `--prompt`, and write the completed records.
|
||||
#[arg(long, value_name = "FILE")]
|
||||
merge: Option<PathBuf>,
|
||||
}
|
||||
|
||||
fn main() -> ExitCode {
|
||||
let args: Args = match catena_cli::parse() {
|
||||
Ok(args) => args,
|
||||
Err(code) => return code,
|
||||
};
|
||||
if args.prompt && args.merge.is_some() {
|
||||
eprintln!("citefind: --prompt composes a question and --merge answers one");
|
||||
return ExitCode::from(exit::USAGE);
|
||||
}
|
||||
|
||||
let text = match catena_cli::read_text(args.path.as_deref()) {
|
||||
Ok(text) => text,
|
||||
Err((code, why)) => {
|
||||
eprintln!("citefind: {why}");
|
||||
return ExitCode::from(code);
|
||||
}
|
||||
};
|
||||
|
||||
let mut references = citefind::read(&text);
|
||||
if references.is_empty() {
|
||||
if !args.quiet {
|
||||
eprintln!("citefind: nothing in this looks like a reference");
|
||||
}
|
||||
return ExitCode::from(exit::NO);
|
||||
}
|
||||
|
||||
if let Some(path) = args.merge.as_deref() {
|
||||
let reply = match std::fs::read_to_string(path) {
|
||||
Ok(reply) => reply,
|
||||
Err(e) => {
|
||||
eprintln!("citefind: {}: {e}", path.display());
|
||||
return ExitCode::from(exit::IO);
|
||||
}
|
||||
};
|
||||
match citefind::prompt::merge(&mut references, &reply) {
|
||||
Err(why) => {
|
||||
eprintln!("citefind: {why}");
|
||||
return ExitCode::from(exit::DATA);
|
||||
}
|
||||
Ok(0) => eprintln!("citefind: the reply filled nothing in"),
|
||||
Ok(filled) => eprintln!("citefind: {filled} entries filled in from the reply"),
|
||||
}
|
||||
}
|
||||
|
||||
if args.quiet {
|
||||
return ExitCode::from(exit::OK);
|
||||
}
|
||||
if args.prompt {
|
||||
return match citefind::prompt::compose(&references) {
|
||||
// Nothing unsettled is not a failure, and a prompt nobody needs
|
||||
// would waste a model's time.
|
||||
None => ExitCode::from(exit::NO),
|
||||
Some(prompt) => catena_cli::text("citefind", &prompt),
|
||||
};
|
||||
}
|
||||
catena_cli::stream("citefind", &references)
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,194 @@
|
||||
//! A question to put to a model, for the references that would not parse.
|
||||
//!
|
||||
//! Nothing here sends anything. The prompt is composed and printed, and what
|
||||
//! happens to it is the caller's business — their model, their machine, their
|
||||
//! GPU. The tool stays what every tool here is: the same output for the same
|
||||
//! input, on every machine, with no network and nothing to install.
|
||||
//!
|
||||
//! One prompt covers every unsettled entry rather than one prompt each. A
|
||||
//! reference list is long and mostly ordinary; the entries worth asking about
|
||||
//! are the mangled few, and they are cheaper to ask about together.
|
||||
|
||||
use crate::Reference;
|
||||
|
||||
/// The version of the prompt below.
|
||||
///
|
||||
/// Bumped when the wording changes in a way that could change an answer, so a
|
||||
/// caller keeping results can tell which prompt produced them.
|
||||
pub const PROMPT_VERSION: u32 = 1;
|
||||
|
||||
/// Compose one question covering every reference that did not fully parse.
|
||||
///
|
||||
/// `None` when every reference settled, because there is nothing to ask and
|
||||
/// a call to a model would be waste.
|
||||
pub fn compose(references: &[Reference]) -> Option<String> {
|
||||
let wanted: Vec<&Reference> = references
|
||||
.iter()
|
||||
.filter(|one| !one.missing().is_empty())
|
||||
.collect();
|
||||
if wanted.is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
let mut out = String::new();
|
||||
out.push_str(
|
||||
"Below are entries from the bibliography or notes of a book. Each was \
|
||||
read as far as its punctuation allowed, and the fields that could not \
|
||||
be read off it are listed with it.\n\n",
|
||||
);
|
||||
for one in &wanted {
|
||||
out.push_str(&format!("--- {}\n", one.idx));
|
||||
out.push_str(&format!("text: {}\n", one.text));
|
||||
let known = known(one);
|
||||
if !known.is_empty() {
|
||||
out.push_str(&known);
|
||||
}
|
||||
out.push_str(&format!("wanted: {}\n\n", one.missing().join(", ")));
|
||||
}
|
||||
|
||||
out.push_str(
|
||||
"Reply with one JSON object and nothing else. Its keys are the entry \
|
||||
numbers above, as strings; each value is an object using only these \
|
||||
keys, and only the ones listed as wanted for that entry: authors, \
|
||||
year, title, container. `authors` is a list of names as the entry \
|
||||
writes them. `year` is a four-digit number. `title` is the work \
|
||||
cited — for an article the article, not the journal. `container` is \
|
||||
the journal an article appeared in, or the publisher of a book.\n\n\
|
||||
Use null for anything the entry does not settle. A guess is worse \
|
||||
than an absence, because nobody downstream can tell them apart. Do \
|
||||
not resolve `ibid.` or `op. cit.` — an entry that only points at an \
|
||||
earlier one has nothing of its own to give.\n",
|
||||
);
|
||||
Some(out)
|
||||
}
|
||||
|
||||
/// What was read off the entry, as the prompt shows it.
|
||||
fn known(one: &Reference) -> String {
|
||||
let mut out = String::new();
|
||||
if !one.authors.is_empty() {
|
||||
out.push_str(&format!("authors: {}\n", one.authors.join("; ")));
|
||||
}
|
||||
if let Some(year) = one.year {
|
||||
out.push_str(&format!("year: {year}\n"));
|
||||
}
|
||||
if let Some(title) = &one.title {
|
||||
out.push_str(&format!("title: {title}\n"));
|
||||
}
|
||||
if let Some(container) = &one.container {
|
||||
out.push_str(&format!("container: {container}\n"));
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Take a model's reply and fill in the fields that were missing.
|
||||
///
|
||||
/// Only the missing ones, and only on entries that asked: a model that
|
||||
/// volunteers a better title for a reference whose title was read off the
|
||||
/// page is overruled, because the page is evidence and the model is a guess.
|
||||
/// A reply that is not one JSON object is refused rather than half-applied.
|
||||
pub fn merge(references: &mut [Reference], reply: &str) -> Result<usize, String> {
|
||||
let json = object_in(reply).ok_or("no JSON object in the reply")?;
|
||||
let value: serde_json::Value =
|
||||
serde_json::from_str(json).map_err(|e| format!("not readable as JSON: {e}"))?;
|
||||
let object = value.as_object().ok_or("the reply is not an object")?;
|
||||
|
||||
let mut filled = 0;
|
||||
for one in references.iter_mut() {
|
||||
let missing = one.missing();
|
||||
if missing.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let Some(answer) = object.get(&one.idx.to_string()) else {
|
||||
continue;
|
||||
};
|
||||
let mut touched = false;
|
||||
|
||||
if missing.contains(&"authors") {
|
||||
let authors: Vec<String> = answer
|
||||
.get("authors")
|
||||
.and_then(serde_json::Value::as_array)
|
||||
.map(|list| {
|
||||
list.iter()
|
||||
.filter_map(serde_json::Value::as_str)
|
||||
.map(|name| name.trim().to_string())
|
||||
.filter(|name| !name.is_empty())
|
||||
.collect()
|
||||
})
|
||||
.unwrap_or_default();
|
||||
if !authors.is_empty() {
|
||||
one.authors = authors;
|
||||
touched = true;
|
||||
}
|
||||
}
|
||||
if missing.contains(&"year") {
|
||||
let year = answer
|
||||
.get("year")
|
||||
.and_then(|found| match found {
|
||||
serde_json::Value::Number(number) => number.as_u64(),
|
||||
serde_json::Value::String(text) => text.trim().parse().ok(),
|
||||
_ => None,
|
||||
})
|
||||
.filter(|year| (1000..=2100).contains(year));
|
||||
if let Some(year) = year {
|
||||
one.year = Some(year as u32);
|
||||
touched = true;
|
||||
}
|
||||
}
|
||||
for (key, slot) in [("title", &mut one.title), ("container", &mut one.container)] {
|
||||
if !missing.contains(&key) {
|
||||
continue;
|
||||
}
|
||||
let found = answer
|
||||
.get(key)
|
||||
.and_then(serde_json::Value::as_str)
|
||||
.map(str::trim)
|
||||
.filter(|text| !text.is_empty());
|
||||
if let Some(found) = found {
|
||||
*slot = Some(found.to_string());
|
||||
touched = true;
|
||||
}
|
||||
}
|
||||
|
||||
if touched {
|
||||
filled += 1;
|
||||
}
|
||||
one.unsettled = one.missing().iter().map(|s| s.to_string()).collect();
|
||||
}
|
||||
Ok(filled)
|
||||
}
|
||||
|
||||
/// The first balanced `{…}` in a reply.
|
||||
///
|
||||
/// Models wrap JSON in prose and in code fences however they were asked not
|
||||
/// to, and refusing a good answer over a fence helps nobody.
|
||||
fn object_in(reply: &str) -> Option<&str> {
|
||||
let start = reply.find('{')?;
|
||||
let bytes = reply.as_bytes();
|
||||
let mut depth = 0usize;
|
||||
let mut in_string = false;
|
||||
let mut escaped = false;
|
||||
for at in start..bytes.len() {
|
||||
let byte = bytes[at];
|
||||
if in_string {
|
||||
match (escaped, byte) {
|
||||
(true, _) => escaped = false,
|
||||
(false, b'\\') => escaped = true,
|
||||
(false, b'"') => in_string = false,
|
||||
_ => {}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
match byte {
|
||||
b'"' => in_string = true,
|
||||
b'{' => depth += 1,
|
||||
b'}' => {
|
||||
depth -= 1;
|
||||
if depth == 0 {
|
||||
return Some(&reply[start..=at]);
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
@@ -0,0 +1,383 @@
|
||||
//! Cutting a block of references into the references in it.
|
||||
//!
|
||||
//! Two shapes arrive. A page of notes numbers its entries, and the numbering
|
||||
//! says exactly where each one starts — that case is decided, not guessed at,
|
||||
//! and the pattern that recognises it is the one `chunkmd` already uses to
|
||||
//! recognise a page of notes, so the two cannot drift apart.
|
||||
//!
|
||||
//! A bibliography numbers nothing. There the signal is layout: a blank line
|
||||
//! between entries, a hanging indent under each, or the surname-comma opening
|
||||
//! that nearly every author-first entry begins with. Whichever of those the
|
||||
//! block actually carries is the one used, and when it carries none of them
|
||||
//! the fallback is one entry per line, which is right for a list that was
|
||||
//! never wrapped and visibly wrong for one that was.
|
||||
|
||||
use std::sync::LazyLock;
|
||||
|
||||
use regex::Regex;
|
||||
|
||||
/// One entry, cut out of the block.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct Entry {
|
||||
/// The entry's text, with its marker removed and whitespace collapsed.
|
||||
pub text: String,
|
||||
/// The note number it carried, where it carried one.
|
||||
pub marker: Option<String>,
|
||||
}
|
||||
|
||||
/// At least this many numbered notes before the numbering is believed.
|
||||
///
|
||||
/// Two: one is a stray list item, and a real page of notes has more.
|
||||
const MIN_NOTES: usize = 2;
|
||||
|
||||
/// The opening of an author-first bibliography entry: a surname, a comma,
|
||||
/// and something after it. Deliberately narrow — it is being used to decide
|
||||
/// where an entry *starts*, and a loose version would cut entries in half at
|
||||
/// every "Oxford, 1998".
|
||||
static SURNAME_FIRST: LazyLock<Regex> = LazyLock::new(|| {
|
||||
Regex::new(r"^[\p{Lu}][\p{L}'’\-]{1,30}(?:\s+[\p{Lu}][\p{L}'’\-]{1,30}){0,2},\s+[\p{Lu}]")
|
||||
.expect("a valid pattern")
|
||||
});
|
||||
|
||||
/// A leading note number, once the tags around it are gone.
|
||||
///
|
||||
/// Three digits at most, which is what `chunkmd` counts as a note number.
|
||||
/// Four would swallow the year off the front of an author-date entry.
|
||||
static LEADING_NUMBER: LazyLock<Regex> =
|
||||
LazyLock::new(|| Regex::new(r"^\[?(\d{1,3})\]?\\?[.)]?\s+").expect("a valid pattern"));
|
||||
|
||||
/// An HTML tag, which conversion leaves wrapped around note numbers.
|
||||
static TAG: LazyLock<Regex> =
|
||||
LazyLock::new(|| Regex::new(r"</?[a-zA-Z][^>]*>").expect("a valid pattern"));
|
||||
|
||||
/// A markdown link left where a back-reference anchor was. The href points
|
||||
/// at a fragment in some EPUBs and at another file in most: `[42](#note-42)`
|
||||
/// and `[4.](part0011.html#ke_00001)` are the same marker.
|
||||
static ANCHOR_LINK: LazyLock<Regex> = LazyLock::new(|| {
|
||||
Regex::new(r"^\[\s*(\d{1,4})\s*\\?[.)]?\s*\]\([^)\s]*\)\s*[.)]?\s*").expect("a valid pattern")
|
||||
});
|
||||
|
||||
/// A marker `chunkmd` does not count, because it does not need to: a
|
||||
/// back-reference whose link points at another file rather than at a
|
||||
/// fragment. `chunkmd` is deciding whether a chunk is a page of notes, and
|
||||
/// one missed marker among thirty does not change that answer. Here a missed
|
||||
/// marker runs two references together, so these are added to its.
|
||||
static FILE_ANCHOR: LazyLock<Regex> = LazyLock::new(|| {
|
||||
Regex::new(r"(?m)^[ \t]{0,8}(?:[>•]+[ \t]*|-[ \t]+)?\[\s*\d{1,3}\s*\\?[.)]?\s*\]\([^)\s]*\)")
|
||||
.expect("a valid pattern")
|
||||
});
|
||||
|
||||
/// A bullet or a blockquote marker, which conversion puts in front of a note
|
||||
/// often enough to matter. A leading `*` is left alone: it opens an italic
|
||||
/// title as often as it opens a list.
|
||||
static BULLET: LazyLock<Regex> =
|
||||
LazyLock::new(|| Regex::new(r"^(?:[>•]+\s*|-\s+)+").expect("a valid pattern"));
|
||||
|
||||
/// An id-style marker: `fn39.`, `note12)`.
|
||||
static ID_MARKER: LazyLock<Regex> =
|
||||
LazyLock::new(|| Regex::new(r"^(?:fn|ftn|note)(\d{1,4})[.)]\s+").expect("a valid pattern"));
|
||||
|
||||
/// Cut a block into entries.
|
||||
pub fn entries(text: &str) -> Vec<Entry> {
|
||||
let numbered = numbered_notes(text);
|
||||
if numbered.len() >= MIN_NOTES {
|
||||
return numbered;
|
||||
}
|
||||
unnumbered(text)
|
||||
}
|
||||
|
||||
/// Entries cut at their note numbers.
|
||||
///
|
||||
/// An entry runs from its marker to the next one. Only entries that cite
|
||||
/// something are kept: a page of notes mixes bibliographic notes with
|
||||
/// discursive ones, and a remark about the argument is not a reference.
|
||||
fn numbered_notes(text: &str) -> Vec<Entry> {
|
||||
let mut starts = chunkmd::footnote::note_starts(text);
|
||||
starts.extend(FILE_ANCHOR.find_iter(text).map(|found| found.start()));
|
||||
starts.sort_unstable();
|
||||
starts.dedup();
|
||||
// Decided before the head is added, so that one stray `12.` in a
|
||||
// bibliography cannot turn the whole of it into two entries.
|
||||
if starts.len() < MIN_NOTES {
|
||||
return Vec::new();
|
||||
}
|
||||
// Whatever is in front of the first marker is the first note, when the
|
||||
// marker it carried was one neither pattern knows. Dropping it would
|
||||
// lose a reference silently, which is the one thing a cut must not do.
|
||||
if starts[0] > 0 {
|
||||
starts.insert(0, 0);
|
||||
}
|
||||
|
||||
let mut out = Vec::new();
|
||||
for (at, start) in starts.iter().enumerate() {
|
||||
let end = starts.get(at + 1).copied().unwrap_or(text.len());
|
||||
let raw = &text[*start..end];
|
||||
if !chunkmd::footnote::cites_something(raw) {
|
||||
continue;
|
||||
}
|
||||
out.push(strip_marker(raw));
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Entries cut at whatever the layout offers.
|
||||
fn unnumbered(text: &str) -> Vec<Entry> {
|
||||
let blocks: Vec<&str> = text
|
||||
.split("\n\n")
|
||||
.map(str::trim)
|
||||
.filter(|block| !block.is_empty())
|
||||
.collect();
|
||||
if blocks.len() >= 2 {
|
||||
return blocks.iter().map(|block| strip_marker(block)).collect();
|
||||
}
|
||||
|
||||
let lines: Vec<&str> = text
|
||||
.lines()
|
||||
.filter(|line| !line.trim().is_empty())
|
||||
.collect();
|
||||
if lines.len() < 2 {
|
||||
return lines.iter().map(|line| strip_marker(line)).collect();
|
||||
}
|
||||
|
||||
// A surname-comma opening on most lines means the list is one entry per
|
||||
// line and any line without it is a wrap of the line above. Without that,
|
||||
// fall back to the hanging indent, and without that to the lines
|
||||
// themselves.
|
||||
let openers = lines
|
||||
.iter()
|
||||
.filter(|line| SURNAME_FIRST.is_match(line.trim_start()))
|
||||
.count();
|
||||
let by_surname = openers >= 2 && openers * 2 >= lines.len();
|
||||
|
||||
let mut out: Vec<String> = Vec::new();
|
||||
for line in &lines {
|
||||
let indented = line.starts_with(' ') || line.starts_with('\t');
|
||||
let starts_entry = match by_surname {
|
||||
true => SURNAME_FIRST.is_match(line.trim_start()),
|
||||
false => !indented,
|
||||
};
|
||||
match (starts_entry, out.last_mut()) {
|
||||
(false, Some(previous)) => {
|
||||
previous.push(' ');
|
||||
previous.push_str(line.trim());
|
||||
}
|
||||
_ => out.push(line.trim().to_string()),
|
||||
}
|
||||
}
|
||||
out.iter().map(|entry| strip_marker(entry)).collect()
|
||||
}
|
||||
|
||||
/// Take the note number off the front of an entry, and tidy what is left.
|
||||
///
|
||||
/// The number arrives wrapped in whatever the converter left behind, so the
|
||||
/// patterns are matched against the opening with its tags removed — and the
|
||||
/// cut is then made in the original, so that an italic span marking a title
|
||||
/// further along survives.
|
||||
fn strip_marker(raw: &str) -> Entry {
|
||||
let text = tidy(raw);
|
||||
let text = BULLET.replace(&text, "").to_string();
|
||||
let (bare, from) = visible(&text, 160);
|
||||
|
||||
for pattern in [&*ANCHOR_LINK, &*ID_MARKER, &*LEADING_NUMBER] {
|
||||
let Some(found) = pattern.captures(&bare) else {
|
||||
continue;
|
||||
};
|
||||
let whole = found.get(0).expect("the whole match");
|
||||
let consumed = bare[..whole.end()].chars().count();
|
||||
let cut = from[consumed];
|
||||
return Entry {
|
||||
text: text[cut..].trim_start().to_string(),
|
||||
marker: Some(found[1].to_string()),
|
||||
};
|
||||
}
|
||||
|
||||
if let Some((marker, consumed)) = superscript(&bare) {
|
||||
return Entry {
|
||||
text: text[from[consumed]..].trim_start().to_string(),
|
||||
marker: Some(marker),
|
||||
};
|
||||
}
|
||||
|
||||
Entry { text, marker: None }
|
||||
}
|
||||
|
||||
/// The opening of an entry with its tags dropped, and where each character
|
||||
/// that was kept came from in the original.
|
||||
///
|
||||
/// The last element is where the kept text ends, so that a marker running to
|
||||
/// the end of the opening still has somewhere to cut.
|
||||
fn visible(text: &str, limit: usize) -> (String, Vec<usize>) {
|
||||
let mut out = String::new();
|
||||
let mut from = Vec::new();
|
||||
let mut at = 0;
|
||||
while at < text.len() && from.len() < limit {
|
||||
let rest = &text[at..];
|
||||
if rest.starts_with('<') {
|
||||
if let Some(close) = rest.find('>') {
|
||||
if TAG.is_match(&rest[..=close]) {
|
||||
at += close + 1;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
let ch = rest.chars().next().expect("a character");
|
||||
if !(out.is_empty() && ch.is_whitespace()) {
|
||||
out.push(ch);
|
||||
from.push(at);
|
||||
}
|
||||
at += ch.len_utf8();
|
||||
}
|
||||
from.push(at);
|
||||
(out, from)
|
||||
}
|
||||
|
||||
/// A leading run of superscript digits, as the number it spells and how many
|
||||
/// characters it took.
|
||||
fn superscript(text: &str) -> Option<(String, usize)> {
|
||||
const SUPER: [char; 10] = ['⁰', '¹', '²', '³', '⁴', '⁵', '⁶', '⁷', '⁸', '⁹'];
|
||||
let mut digits = String::new();
|
||||
let mut taken = 0;
|
||||
for ch in text.chars() {
|
||||
match SUPER.iter().position(|s| *s == ch) {
|
||||
Some(value) => {
|
||||
digits.push(char::from_digit(value as u32, 10)?);
|
||||
taken += 1;
|
||||
}
|
||||
None => break,
|
||||
}
|
||||
}
|
||||
if digits.is_empty() {
|
||||
return None;
|
||||
}
|
||||
// Step over the space the marker is separated from its entry by.
|
||||
let after: String = text
|
||||
.chars()
|
||||
.skip(taken)
|
||||
.take_while(|c| c.is_whitespace())
|
||||
.collect();
|
||||
Some((digits, taken + after.chars().count()))
|
||||
}
|
||||
|
||||
/// Collapse the whitespace a converted entry is wrapped with.
|
||||
fn tidy(text: &str) -> String {
|
||||
text.split_whitespace().collect::<Vec<_>>().join(" ")
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn texts(entries: &[Entry]) -> Vec<&str> {
|
||||
entries.iter().map(|entry| entry.text.as_str()).collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_page_of_numbered_notes_is_cut_at_its_numbers() {
|
||||
let page = "\
|
||||
1. John Smith, *The Title* (Oxford: Clarendon Press, 1998), 183.
|
||||
2. Jane Doe, *Another Book* (Cambridge: University Press, 2001), p. 12.";
|
||||
let found = entries(page);
|
||||
assert_eq!(found.len(), 2, "{found:#?}");
|
||||
assert_eq!(found[0].marker.as_deref(), Some("1"));
|
||||
assert_eq!(found[1].marker.as_deref(), Some("2"));
|
||||
assert!(found[0].text.starts_with("John Smith"), "{:?}", found[0]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_note_that_cites_nothing_is_not_a_reference() {
|
||||
let page = "\
|
||||
1. John Smith, *The Title* (Oxford: Clarendon Press, 1998), 183.
|
||||
2. I am indebted to the seminar for this point.
|
||||
3. Jane Doe, *Another Book* (Cambridge: University Press, 2001), p. 12.";
|
||||
let found = entries(page);
|
||||
assert_eq!(
|
||||
found
|
||||
.iter()
|
||||
.filter_map(|e| e.marker.clone())
|
||||
.collect::<Vec<_>>(),
|
||||
["1", "3"]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn blank_lines_cut_a_bibliography() {
|
||||
let block = "Smith, John. The Title. Oxford: Clarendon, 1998.\n\nDoe, Jane. Another.\nCambridge, 2001.";
|
||||
let found = entries(block);
|
||||
assert_eq!(found.len(), 2);
|
||||
assert_eq!(found[1].text, "Doe, Jane. Another. Cambridge, 2001.");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_surname_opening_cuts_a_list_that_was_wrapped() {
|
||||
let block = "\
|
||||
Smith, John. The Title of the Book That Ran On.
|
||||
Oxford: Clarendon Press, 1998.
|
||||
Doe, Jane. Another Work Entirely. Cambridge, 2001.";
|
||||
let found = entries(block);
|
||||
assert_eq!(
|
||||
texts(&found),
|
||||
[
|
||||
"Smith, John. The Title of the Book That Ran On. Oxford: Clarendon Press, 1998.",
|
||||
"Doe, Jane. Another Work Entirely. Cambridge, 2001.",
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_hanging_indent_cuts_a_list_with_no_surnames_to_go_on() {
|
||||
let block = "\
|
||||
The Oxford Companion to Everything. Oxford, 1998.
|
||||
Second line of the same entry.
|
||||
Another Anonymous Work. Cambridge, 2001.";
|
||||
let found = entries(block);
|
||||
assert_eq!(found.len(), 2, "{found:#?}");
|
||||
assert!(found[0].text.ends_with("same entry."), "{:?}", found[0]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_marker_wrapped_in_tags_is_still_a_marker() {
|
||||
let entry = strip_marker("<a href=\"#ref12\"><sup>12</sup></a>. Smith, *Title*, p. 4.");
|
||||
assert_eq!(entry.marker.as_deref(), Some("12"));
|
||||
assert_eq!(entry.text, "Smith, *Title*, p. 4.");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_anchor_link_marker_keeps_the_entry_and_not_the_link() {
|
||||
let entry = strip_marker("[42](#note-42). Smith, *Title*, p. 4.");
|
||||
assert_eq!(entry.marker.as_deref(), Some("42"));
|
||||
assert_eq!(entry.text, "Smith, *Title*, p. 4.");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_id_style_marker_is_read_as_its_number() {
|
||||
let entry = strip_marker("fn39. Smith, *Title*, p. 4.");
|
||||
assert_eq!(entry.marker.as_deref(), Some("39"));
|
||||
assert_eq!(entry.text, "Smith, *Title*, p. 4.");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_superscript_marker_is_read_as_its_number() {
|
||||
let entry = strip_marker("¹² Smith, *Title*, p. 4.");
|
||||
assert_eq!(entry.marker.as_deref(), Some("12"));
|
||||
assert_eq!(entry.text, "Smith, *Title*, p. 4.");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_entry_with_no_marker_keeps_all_of_itself() {
|
||||
let entry = strip_marker("Smith, John. The Title. Oxford, 1998.");
|
||||
assert_eq!(entry.marker, None);
|
||||
assert_eq!(entry.text, "Smith, John. The Title. Oxford, 1998.");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_year_in_a_place_of_publication_does_not_start_an_entry() {
|
||||
// "Oxford, 1998." would match a loose surname-comma rule.
|
||||
let block = "\
|
||||
Smith, John. A Title.
|
||||
Oxford, 1998.
|
||||
Doe, Jane. Another.
|
||||
Cambridge, 2001.";
|
||||
let found = entries(block);
|
||||
assert_eq!(found.len(), 2, "{found:#?}");
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user