From da96196d9856c5d9243aea0b87735a3520554be2 Mon Sep 17 00:00:00 2001
From: funkpower <funkpower@pumpin.online>
Date: Sun, 20 Sep 2026 15:40:24 -0400
Subject: [PATCH] idnorm: a DOI may have brackets in it, and usually the ones
 Elsevier put there
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

`10.1016/S0140-6736(16)30370-1` came back as `10.1016/s0140-6736` — a
different DOI, and one that resolves, so nothing downstream could tell. The
pattern excluded brackets to keep the ones a sentence wraps round a DOI, and
Elsevier alone publishes hundreds of thousands of DOIs that contain them.

Brackets are taken now, and an unbalanced closer at the end is given back:
`(10.1234/foo)` in a sentence ends on a bracket the sentence opened, and
`10.1016/S0140-6736(16)30370-1` ends on digits with a matched pair in the
middle. Counting is what tells them apart.

Found by comparing idnorm against policy-pipeline's own DOI normaliser over
229,556 real documents.
---
 crates/idnorm/src/doi.rs | 63 +++++++++++++++++++++++++++++++++++++++-
 1 file changed, 62 insertions(+), 1 deletion(-)

diff --git a/crates/idnorm/src/doi.rs b/crates/idnorm/src/doi.rs
index f9fc2ab..3c32cc8 100644
--- a/crates/idnorm/src/doi.rs
+++ b/crates/idnorm/src/doi.rs
@@ -15,7 +15,12 @@ use fancy_regex::Regex;
 /// wrap a citation, because those are never part of one in practice and
 /// leaving them on makes two spellings of the same identifier.
 static DOI: LazyLock<Regex> = LazyLock::new(|| {
-    Regex::new(r#"(?i)\b(10\.\d{4,9}/[^\s"'<>,;()\[\]{}]+)"#).expect("a valid pattern")
+    // Brackets are part of a DOI far more often than they are punctuation
+    // around one: Elsevier alone publishes hundreds of thousands in the form
+    // `10.1016/S0140-6736(16)30370-1`. They are taken, and an unbalanced one
+    // at the end — the closing half of a bracket the sentence opened — is
+    // given back afterwards.
+    Regex::new(r#"(?i)\b(10\.\d{4,9}/[^\s"'<>,;]+)"#).expect("a valid pattern")
 });
 
 /// The prefixes a DOI arrives wearing.
@@ -58,6 +63,8 @@ pub fn parse(text: &str) -> Option<String> {
     // rather than to the identifier. A closing bracket does too, unless the
     // suffix opened one.
     let id = id.trim_end_matches(['.', ',', ';', ':']);
+    let id = trim_unclosed(id);
+    let id = id.trim_end_matches(['.', ',', ';', ':']);
     if id.len() < 6 || !id.contains('/') {
         return None;
     }
@@ -68,8 +75,62 @@ pub fn parse(text: &str) -> Option<String> {
     Some(id.to_lowercase())
 }
 
+/// Give back a closing bracket the DOI never opened.
+///
+/// `(10.1234/foo)` in a sentence ends on a bracket that belongs to the
+/// sentence; `10.1016/S0140-6736(16)30370-1` ends on digits and keeps the
+/// pair in the middle of it. Counting is what tells them apart.
+fn trim_unclosed(id: &str) -> &str {
+    let mut end = id.len();
+    loop {
+        let head = &id[..end];
+        let Some(last) = head.chars().last() else {
+            return head;
+        };
+        let opener = match last {
+            ')' => '(',
+            ']' => '[',
+            '}' => '{',
+            _ => return head,
+        };
+        let body = &head[..head.len() - last.len_utf8()];
+        if body.matches(opener).count() > body.matches(last).count() {
+            return head;
+        }
+        end -= last.len_utf8();
+    }
+}
+
 #[cfg(test)]
 mod tests {
+    #[test]
+    fn a_doi_may_have_brackets_in_it() {
+        // Elsevier publishes hundreds of thousands in this form, and the
+        // pattern used to stop at the first bracket and hand back
+        // `10.1016/s0140-6736` — a different, wrong, resolvable DOI.
+        assert_eq!(
+            parse("10.1016/S0140-6736(16)30370-1").as_deref(),
+            Some("10.1016/s0140-6736(16)30370-1")
+        );
+        assert_eq!(
+            parse("https://doi.org/10.1016/S0140-6736(23)00441-5").as_deref(),
+            Some("10.1016/s0140-6736(23)00441-5")
+        );
+        assert_eq!(parse("10.1234/a[b]c").as_deref(), Some("10.1234/a[b]c"));
+    }
+
+    #[test]
+    fn a_bracket_the_sentence_opened_is_given_back() {
+        for (text, want) in [
+            ("see (10.1234/foo) here", "10.1234/foo"),
+            ("10.1234/foo).", "10.1234/foo"),
+            ("[10.1234/foo]", "10.1234/foo"),
+            ("10.1234/foo(bar)", "10.1234/foo(bar)"),
+        ] {
+            assert_eq!(parse(text).as_deref(), Some(want), "{text}");
+        }
+    }
+
     use super::*;
 
     #[test]