idnorm: a prefix is ASCII and the text it is cut from is not

`10.1093/<zwsp>oso/<zwsp>` is exactly sixteen bytes, which is the length of
`https://doi.org/`, so testing that prefix against it sliced into the middle
of the third zero-width space and brought the process down. Found by running
citefind over the corpus: the DOI is printed that way in a real book, because
the typesetter marked where the line could break.

Both prefix tests — DOI and arXiv — now ask for the head rather than cutting
one, so a byte length that is not a character boundary is an answer of no
rather than a panic.

And an invisible character is not part of an identifier. Zero-width spaces,
joiners, the soft hyphen, a byte-order mark left at the front of a fragment:
text out of a PDF or an EPUB is full of them, they cannot be seen, and a DOI
carrying one does not resolve. They are dropped before anything is read.

crates/idnorm/src/arxiv.rs

@@ -60,7 +60,12 @@ fn strip_prefix(text: &str) -> &str {
         "arxiv:",
         "arxiv ",
     ] {
-        if text.len() >= prefix.len() && text[..prefix.len()].eq_ignore_ascii_case(prefix) {
+        // See the note in `doi::parse`: a slice at the prefix's byte length
+        // can land inside a character the text happens to carry.
+        if text
+            .get(..prefix.len())
+            .is_some_and(|head| head.eq_ignore_ascii_case(prefix))
+        {
             return text[prefix.len()..].trim();
         }
     }

crates/idnorm/src/doi.rs

@@ -37,7 +37,15 @@ pub fn parse(text: &str) -> Option<String> {
     let text = text.trim();
     let mut without = text;
     for prefix in PREFIXES {
-        if without.len() >= prefix.len() && without[..prefix.len()].eq_ignore_ascii_case(prefix) {
+        // `get` rather than a slice: the prefixes are ASCII and the text is
+        // not, so cutting at the prefix's byte length can land inside a
+        // character. A DOI copied out of a PDF arrives full of zero-width
+        // spaces, and `10.1093/\u{200b}oso/` is exactly long enough to put
+        // one where `https://doi.org/` ends.
+        if without
+            .get(..prefix.len())
+            .is_some_and(|head| head.eq_ignore_ascii_case(prefix))
+        {
             without = &without[prefix.len()..];
             break;
         }

crates/idnorm/src/lib.rs

@@ -50,7 +50,7 @@ pub struct Identifier {
     /// Record version.
     #[serde(default = "version")]
     pub v: u32,
-    /// The text it was read from.
+    /// The text it was read from, with any invisible characters removed.
     pub input: String,
     /// Which kind it turned out to be.
     pub kind: Kind,
@@ -84,9 +84,26 @@ pub fn parse(text: &str) -> Option<Identifier> {
     parse_as(text, None)
 }
 
+/// A character that takes no width and carries no meaning.
+fn is_invisible(c: char) -> bool {
+    matches!(
+        c,
+        '\u{00ad}'                 // soft hyphen
+            | '\u{200b}'..='\u{200f}' // zero-width space, joiners, marks
+            | '\u{2060}'..='\u{2064}' // word joiner and friends
+            | '\u{feff}'               // byte-order mark
+    )
+}
+
 /// Read a string as one particular kind, or as whatever it looks like.
 pub fn parse_as(text: &str, kind: Option<Kind>) -> Option<Identifier> {
-    let input = text.trim();
+    // Text extracted from a PDF or an EPUB is full of characters that take
+    // no width and mean nothing: zero-width spaces and joiners where the
+    // typesetter allowed a line to break, a soft hyphen, a byte-order mark
+    // left at the front of a fragment. None of them is part of an
+    // identifier, and a DOI carrying one does not resolve.
+    let cleaned: String = text.chars().filter(|c| !is_invisible(*c)).collect();
+    let input = cleaned.trim();
     if input.is_empty() {
         return None;
     }
@@ -165,6 +182,42 @@ fn isbn(input: &str) -> Option<Identifier> {
 mod tests {
     use super::*;
 
+    #[test]
+    fn an_identifier_copied_out_of_a_pdf_is_full_of_nothing() {
+        // A typesetter's zero-width spaces, where the line was allowed to
+        // break. The DOI does not resolve with them in it.
+        let found =
+            parse("DOI: 10.1093/\u{200b}oso/\u{200b}9780190065065.003.0001").expect("a doi");
+        assert_eq!(found.id, "10.1093/oso/9780190065065.003.0001");
+        assert_eq!(found.kind, Kind::Doi);
+    }
+
+    #[test]
+    fn a_prefix_is_not_cut_off_inside_a_character() {
+        // `10.1093/\u{200b}oso/\u{200b}` is exactly as many bytes as
+        // `https://doi.org/`, and a slice at that length used to land in the
+        // middle of the third zero-width space and bring the process down.
+        for text in [
+            "10.1093/\u{200b}oso/\u{200b}9780190065065",
+            "\u{feff}10.1000/182",
+            "arxiv:\u{200b}1706.03762",
+            "\u{200b}\u{200b}\u{200b}\u{200b}\u{200b}\u{200b}",
+            "\u{00ad}",
+        ] {
+            // The answer may be None; what it may not do is panic.
+            let _ = parse(text);
+        }
+    }
+
+    #[test]
+    fn a_soft_hyphen_in_an_isbn_is_not_a_digit() {
+        let found = parse("978\u{00ad}0\u{00ad}306\u{00ad}40615\u{00ad}7").expect("an isbn");
+        assert_eq!(found.kind, Kind::Isbn);
+        assert_eq!(found.checksum, Some(true));
+    }
+
+    use super::*;
+
     #[test]
     fn a_doi_is_read_however_it_was_written() {
         for text in [