From 1284136f694f7d7bebd7535f68e0612298620729 Mon Sep 17 00:00:00 2001 From: funkpower Date: Sun, 20 Sep 2026 16:06:22 -0400 Subject: [PATCH] chunkmd, citefind: the back-reference marker a whole class of EPUBs leaves MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit \[←788 \] \[[←193](index_split_011.xhtml#citation-193) \] An arrow pointing back to where the note was called, sometimes a link and sometimes not, inside escaped brackets. Neither pattern knew it, so a page of notes set this way read as body prose: the chunk was never flagged, the citation stage never selected it, and the book came out with nothing. Ten books of the corpus use it and eight of them had zero citations — including both volumes of *The Decline of the West*, whose 830 endnotes are exactly what this is for. 142 of the 199 affected chunks are those two books. Recorded in parity/KNOWN.md: the Python this was ported from did not know the marker either, and no fixture carries one, so the suite agrees either way. --- crates/chunkmd/src/footnote.rs | 4 ++++ crates/citefind/src/split.rs | 15 ++++++++++++++- parity/KNOWN.md | 24 ++++++++++++++++++++++++ 3 files changed, 42 insertions(+), 1 deletion(-) diff --git a/crates/chunkmd/src/footnote.rs b/crates/chunkmd/src/footnote.rs index 5ffd6b5..37bd9f0 100644 --- a/crates/chunkmd/src/footnote.rs +++ b/crates/chunkmd/src/footnote.rs @@ -60,6 +60,10 @@ static NOTE_MARKER: LazyLock = LazyLock::new(|| { | \[ {SPACE}* (?:]*>)? {SPACE}* \d{{1,3}} {SPACE}* (?:)? {SPACE}* \] \( \#[^)]* \) # [42](#...) | {SPACE}* \d{{1,3}} {SPACE}* # bare sup note + | \\?\[ {SPACE}* (?: \[ {SPACE}* )? # `\[←788 \]` and + [\u{{2190}}\u{{2191}}\u{{21D1}}\u{{23CE}}] # `\[[←193](file#frag) \]` + {SPACE}* \d{{1,4}} {SPACE}* + (?: \] \( [^)\s]* \) )? {SPACE}* \\?\] )" )) .expect("a valid pattern") diff --git a/crates/citefind/src/split.rs b/crates/citefind/src/split.rs index 2ef64c0..72e2df2 100644 --- a/crates/citefind/src/split.rs +++ b/crates/citefind/src/split.rs @@ -99,6 +99,19 @@ static GLUED_NUMBER: LazyLock = static COPYRIGHT_FOOTER: LazyLock = LazyLock::new(|| Regex::new(r"©").expect("a valid pattern")); +/// The back-reference marker a whole class of EPUB conversions leaves: +/// `\[←788 \]`, or `\[[←193](index_split_011.xhtml#citation-193) \]` where +/// the arrow is also a link back to where the note was called. +static ARROW_MARKER: LazyLock = LazyLock::new(|| { + Regex::new( + r"(?x) ^ \\?\[ \s* (?:\[\s*)? + [\x{2190}\x{2191}\x{21D1}\x{23CE}] \s* + (\d{1,4}) \s* + (?: \] \( [^)\s]* \) )? \s* \\?\] \s*", + ) + .expect("a valid pattern") +}); + /// A bullet or a blockquote marker, which conversion puts in front of a note /// often enough to matter. A leading `*` is left alone: it opens an italic /// title as often as it opens a list. @@ -266,7 +279,7 @@ fn strip_marker(raw: &str) -> Entry { let text = BULLET.replace(&text, "").to_string(); let (bare, from) = visible(&text, 160); - for pattern in [&*ANCHOR_LINK, &*ID_MARKER, &*LEADING_NUMBER] { + for pattern in [&*ARROW_MARKER, &*ANCHOR_LINK, &*ID_MARKER, &*LEADING_NUMBER] { let Some(found) = pattern.captures(&bare) else { continue; }; diff --git a/parity/KNOWN.md b/parity/KNOWN.md index 6481a57..bf07d67 100644 --- a/parity/KNOWN.md +++ b/parity/KNOWN.md @@ -343,3 +343,27 @@ were always joined onto an absolute root. **Measured:** 32 cases, including a file at the root, a name beginning with a dash, a name that is only a dot, a non-ASCII name and an extension that is only a dot. Ordinary paths — the overwhelming majority — are byte-identical. + +--- + +## chunkmd — the back-reference marker a whole class of EPUBs leaves + +`looks_like_citations` recognises one note marker the Python it was ported +from did not: + + \[←788 \] + \[[←193](index_split_011.xhtml#citation-193) \] + +An arrow pointing back to where the note was called, sometimes as a link and +sometimes not, wrapped in escaped brackets. It is what a common EPUB +conversion leaves behind, and ten books of the corpus use it — including both +volumes of *The Decline of the West*, whose endnotes run to 830 entries of +`Krumbacher, op. cit., pp. 969-90` and were being read as body prose. Every +one of those books had zero citations extracted. + +The parity suite does not cover the footnote signal and no fixture carries +the marker, so the tests agree either way. This is recorded because it is a +deliberate difference from the Python, not because anything caught it. + +`citefind` strips the same marker when it cuts a page of notes into notes. +