From 1bfca45b67bb50af390c2b0aea4fc17d33793220 Mon Sep 17 00:00:00 2001 From: funkpower Date: Sun, 20 Sep 2026 14:10:18 -0400 Subject: [PATCH] idnorm: a prefix is ASCII and the text it is cut from is not MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `10.1093/oso/` is exactly sixteen bytes, which is the length of `https://doi.org/`, so testing that prefix against it sliced into the middle of the third zero-width space and brought the process down. Found by running citefind over the corpus: the DOI is printed that way in a real book, because the typesetter marked where the line could break. Both prefix tests — DOI and arXiv — now ask for the head rather than cutting one, so a byte length that is not a character boundary is an answer of no rather than a panic. And an invisible character is not part of an identifier. Zero-width spaces, joiners, the soft hyphen, a byte-order mark left at the front of a fragment: text out of a PDF or an EPUB is full of them, they cannot be seen, and a DOI carrying one does not resolve. They are dropped before anything is read. --- crates/idnorm/src/arxiv.rs | 7 ++++- crates/idnorm/src/doi.rs | 10 ++++++- crates/idnorm/src/lib.rs | 57 ++++++++++++++++++++++++++++++++++++-- 3 files changed, 70 insertions(+), 4 deletions(-) diff --git a/crates/idnorm/src/arxiv.rs b/crates/idnorm/src/arxiv.rs index 12a0961..6883f2d 100644 --- a/crates/idnorm/src/arxiv.rs +++ b/crates/idnorm/src/arxiv.rs @@ -60,7 +60,12 @@ fn strip_prefix(text: &str) -> &str { "arxiv:", "arxiv ", ] { - if text.len() >= prefix.len() && text[..prefix.len()].eq_ignore_ascii_case(prefix) { + // See the note in `doi::parse`: a slice at the prefix's byte length + // can land inside a character the text happens to carry. + if text + .get(..prefix.len()) + .is_some_and(|head| head.eq_ignore_ascii_case(prefix)) + { return text[prefix.len()..].trim(); } } diff --git a/crates/idnorm/src/doi.rs b/crates/idnorm/src/doi.rs index db34f03..f9fc2ab 100644 --- a/crates/idnorm/src/doi.rs +++ b/crates/idnorm/src/doi.rs @@ -37,7 +37,15 @@ pub fn parse(text: &str) -> Option { let text = text.trim(); let mut without = text; for prefix in PREFIXES { - if without.len() >= prefix.len() && without[..prefix.len()].eq_ignore_ascii_case(prefix) { + // `get` rather than a slice: the prefixes are ASCII and the text is + // not, so cutting at the prefix's byte length can land inside a + // character. A DOI copied out of a PDF arrives full of zero-width + // spaces, and `10.1093/\u{200b}oso/` is exactly long enough to put + // one where `https://doi.org/` ends. + if without + .get(..prefix.len()) + .is_some_and(|head| head.eq_ignore_ascii_case(prefix)) + { without = &without[prefix.len()..]; break; } diff --git a/crates/idnorm/src/lib.rs b/crates/idnorm/src/lib.rs index 55fe9d7..e17b27a 100644 --- a/crates/idnorm/src/lib.rs +++ b/crates/idnorm/src/lib.rs @@ -50,7 +50,7 @@ pub struct Identifier { /// Record version. #[serde(default = "version")] pub v: u32, - /// The text it was read from. + /// The text it was read from, with any invisible characters removed. pub input: String, /// Which kind it turned out to be. pub kind: Kind, @@ -84,9 +84,26 @@ pub fn parse(text: &str) -> Option { parse_as(text, None) } +/// A character that takes no width and carries no meaning. +fn is_invisible(c: char) -> bool { + matches!( + c, + '\u{00ad}' // soft hyphen + | '\u{200b}'..='\u{200f}' // zero-width space, joiners, marks + | '\u{2060}'..='\u{2064}' // word joiner and friends + | '\u{feff}' // byte-order mark + ) +} + /// Read a string as one particular kind, or as whatever it looks like. pub fn parse_as(text: &str, kind: Option) -> Option { - let input = text.trim(); + // Text extracted from a PDF or an EPUB is full of characters that take + // no width and mean nothing: zero-width spaces and joiners where the + // typesetter allowed a line to break, a soft hyphen, a byte-order mark + // left at the front of a fragment. None of them is part of an + // identifier, and a DOI carrying one does not resolve. + let cleaned: String = text.chars().filter(|c| !is_invisible(*c)).collect(); + let input = cleaned.trim(); if input.is_empty() { return None; } @@ -165,6 +182,42 @@ fn isbn(input: &str) -> Option { mod tests { use super::*; + #[test] + fn an_identifier_copied_out_of_a_pdf_is_full_of_nothing() { + // A typesetter's zero-width spaces, where the line was allowed to + // break. The DOI does not resolve with them in it. + let found = + parse("DOI: 10.1093/\u{200b}oso/\u{200b}9780190065065.003.0001").expect("a doi"); + assert_eq!(found.id, "10.1093/oso/9780190065065.003.0001"); + assert_eq!(found.kind, Kind::Doi); + } + + #[test] + fn a_prefix_is_not_cut_off_inside_a_character() { + // `10.1093/\u{200b}oso/\u{200b}` is exactly as many bytes as + // `https://doi.org/`, and a slice at that length used to land in the + // middle of the third zero-width space and bring the process down. + for text in [ + "10.1093/\u{200b}oso/\u{200b}9780190065065", + "\u{feff}10.1000/182", + "arxiv:\u{200b}1706.03762", + "\u{200b}\u{200b}\u{200b}\u{200b}\u{200b}\u{200b}", + "\u{00ad}", + ] { + // The answer may be None; what it may not do is panic. + let _ = parse(text); + } + } + + #[test] + fn a_soft_hyphen_in_an_isbn_is_not_a_digit() { + let found = parse("978\u{00ad}0\u{00ad}306\u{00ad}40615\u{00ad}7").expect("an isbn"); + assert_eq!(found.kind, Kind::Isbn); + assert_eq!(found.checksum, Some(true)); + } + + use super::*; + #[test] fn a_doi_is_read_however_it_was_written() { for text in [