papers-cli 2.15.3__tar.gz → 2.15.5__tar.gz

{papers_cli-2.15.3 → papers_cli-2.15.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: papers-cli
-Version: 2.15.3
+Version: 2.15.5
 Summary: utilities to keep your PDF library organized
 Author-email: Mahé Perrette <mahe.perrette@gmail.com>
 License: MIT

{papers_cli-2.15.3 → papers_cli-2.15.5}/papers/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '2.15.3'
-__version_tuple__ = version_tuple = (2, 15, 3)
+__version__ = version = '2.15.5'
+__version_tuple__ = version_tuple = (2, 15, 5)
 
-__commit_id__ = commit_id = 'gccc7b54c0'
+__commit_id__ = commit_id = 'gbad573f0d'

{papers_cli-2.15.3 → papers_cli-2.15.5}/papers/extract.py

@@ -110,49 +110,178 @@ def readpdf_image(pdf, first=None, last=None):
 
     return txt
 
-REGEXP = re.compile(r'[doi,doi.org/][\s\.\:]{0,2}(10\.\d{4}[\d\:\.\-\/a-z]+)[A-Z\s,\n]')
+# CrossRef DOI standard: /^10.\d{4,9}/[-._;()/:A-Z0-9]+$/i
+REGEXP = re.compile(r'[doi,doi.org/][\s\.\:]{0,2}(10\.\d{4,9}/[-._;()/:a-z0-9]+)')
 ARXIV = re.compile(r'arxiv:\s*(\d{4}\.\d{4,5})')
 
-def parse_doi(txt):
-    # based on: https://doeidoei.wordpress.com/2009/10/22/regular-expression-to-match-a-doi-digital-object-identifier/
-    # doi = r'[doi|DOI][\s\.\:]{0,2}(10\.\d{4}[\d\:\.\-\/a-z]+)[A-Z\s]'
+def _parse_doi_from_metadata_string(metadata):
+    """Extract DOI from XMP metadata string."""
+    patterns = [
+        r'<prism:doi>(10\.\d{4,9}/[-._;()/:a-z0-9]+)</prism:doi>',
+        r'<dc:identifier>doi:(10\.\d{4,9}/[-._;()/:a-z0-9]+)</dc:identifier>',
+        r'<pdfx:doi>(10\.\d{4,9}/[-._;()/:a-z0-9]+)</pdfx:doi>',
+        r'<crossmark:DOI>(10\.\d{4,9}/[-._;()/:a-z0-9]+)</crossmark:DOI>',
+    ]
+    for pattern in patterns:
+        match = re.search(pattern, metadata, re.IGNORECASE)
+        if match:
+            return match.group(1)
+    return None
 
-    # maybe try that? (need to convert to python-regex)
-    # https://www.crossref.org/blog/dois-and-matching-regular-expressions/
-    # a. /^10.\d{4,9}/[-._;()/:A-Z0-9]+$/i
-    # b. /^10.1002/[^\s]+$/i
-    # c. /^10.\d{4}/\d+-\d+X?(\d+)\d+<[\d\w]+:[\d\w]*>\d+.\d+.\w+;\d$/i
-    # d. /^10.1021/\w\w\d++$/i
-    # e. /^10.1207/[\w\d]+\&\d+_\d+$/i
+def parse_doi_from_pdf_metadata_poppler(pdf_path):
+    """Extract DOI from PDF metadata using pdfinfo (poppler-utils)."""
+    try:
+        result = sp.run(['pdfinfo', '-meta', pdf_path],
+                       capture_output=True, text=True, timeout=5)
+        if result.returncode == 0:
+            return _parse_doi_from_metadata_string(result.stdout)
+    except Exception:
+        pass
+    return None
 
-    matches = REGEXP.findall(' '+txt.lower()+' ')
+def parse_doi_from_pdf_metadata_fitz(pdf_path):
+    """Extract DOI from PDF metadata using PyMuPDF/fitz."""
+    import fitz
 
-    if not matches:
+    with fitz.open(pdf_path) as doc:
+        metadata = doc.metadata
 
-        # try arxiv pattern
-        match = ARXIV.search(txt.lower())
-        if match:
-            arxiv_id = match.group(1)
-            matches = [ f"10.48550/arXiv.{arxiv_id}" ]
+        # Try direct metadata fields first
+        if metadata:
+            # Check common metadata fields
+            for key in ['subject', 'keywords', 'title']:
+                value = metadata.get(key, '')
+                if value and '10.' in value:
+                    # Try to extract DOI from the field
+                    doi_match = re.search(r'10\.\d{4,9}/[-._;()/:a-z0-9]+', value, re.IGNORECASE)
+                    if doi_match:
+                        return doi_match.group(0)
 
-        else:
-            raise DOIParsingError('parse_doi::no matches')
+        # Fall back to XMP metadata
+        xmp = doc.get_xml_metadata() if hasattr(doc, 'get_xml_metadata') else doc.xref_get_key(-1, "Metadata")
+        if xmp:
+            xmp_str = xmp if isinstance(xmp, str) else xmp.decode('utf-8', errors='ignore')
+            return _parse_doi_from_metadata_string(xmp_str)
+
+
+def parse_doi_from_pdf_metadata(pdf_path):
+    """Extract DOI from PDF metadata (tries fitz, falls back to poppler)."""
+    # Try fitz first (no subprocess overhead)
+
+    try:
+        return parse_doi_from_pdf_metadata_fitz(pdf_path)
+    except ImportError:
+        # Fall back to poppler-utils
+        return parse_doi_from_pdf_metadata_poppler(pdf_path)
+
+def parse_doi(txt):
+    # Remove invisible Unicode characters and normalize dashes and slashes
+    # U+200B: zero-width space, U+00AD: soft hyphen, U+2013: en-dash, U+2014: em-dash
+    # \x02: STX (Start of Text) sometimes used instead of slash in PDFs
+    txt_clean = txt.replace('\u200b', '').replace('\xad', '').replace('\u2013', '-').replace('\u2014', '-').replace('\x02', '/')
+    txt_lower = txt_clean.lower()
 
-    match = matches[0]
+    # Handle DOI split at "10.\n<registrant>" (e.g., "10.\n1073/pnas.123")
+    # Requires 4-9 digits (valid registrant) followed by slash
+    txt_lower = re.sub(r'(10\.)\s*\n\s*(\d{4,9}/)', r'\1\2', txt_lower)
 
-    # clean expression
-    doi = match.replace('\n','').strip('.')
+    # Special case: PNAS DOIs may be split at "10.1073/pnas. \n<digits>"
+    # This is a very specific pattern that's safe to join
+    txt_lower = re.sub(r'(10\.1073/pnas\.)\s*\n\s*(\d)', r'\1\2', txt_lower)
 
-    if doi.lower().endswith('.received'):
-        doi = doi[:-len('.received')]
+    # Handle DOIs split at dash-newline-alphanumeric (common in many publishers)
+    # e.g., "10.1175/jcli-d-21-\n0636.1", "10.1175/JCLI-\nD-17-0112.s1"
+    txt_lower = re.sub(r'(10\.\d{4,9}/[-._;()/:a-z0-9]+-)\s*\n\s*([a-z0-9])', r'\1\2', txt_lower)
 
-    # quality check
+    # Don't join across newlines generally - it causes more problems than it solves
+    # The newline naturally stops incorrect matches from extending into following text
+    matches = REGEXP.findall(txt_lower)
+
+    if not matches:
+        # Try arxiv pattern
+        match = ARXIV.search(txt_lower)
+        if match:
+            return f"10.48550/arXiv.{match.group(1)}"
+
+        raise DOIParsingError('parse_doi::no matches')
+
+    # Start with first match as default
+    doi = matches[0]
+
+    # Prefer DOIs that appear in full URL context (e.g., www.pnas.org/cgi/doi/10.1073/...)
+    # These are more likely to be the article's own DOI rather than citations
+    if len(matches) > 1:
+        url_patterns = [
+            r'www\.[a-z]+\.org/[a-z/]+/(10\.\d{4,9}/[-._;()/:a-z0-9]+)',
+            r'doi\.org/(10\.\d{4,9}/[-._;()/:a-z0-9]+)',
+            r'dx\.doi\.org/(10\.\d{4,9}/[-._;()/:a-z0-9]+)',
+        ]
+        for pattern in url_patterns:
+            url_matches = re.findall(pattern, txt_lower)
+            # If any of our DOI matches appear in a URL, prefer the first one
+            for doi_match in matches:
+                if doi_match in url_matches:
+                    doi = doi_match
+                    break
+            if doi in url_matches:  # Found a URL match, stop searching patterns
+                break
+
+    # If multiple matches and first is supplemental, prefer non-supplemental with same base
+    # e.g., prefer "10.1175/jcli-d-16-0271.1" over "10.1175/jcli-d-16-0271.s1"
+    if len(matches) > 1 and doi == matches[0]:
+        # Check if first match is supplemental (.s<digit>)
+        if re.search(r'\.s\d+\.?$', matches[0]):
+            # Look for non-supplemental version with same base
+            base = re.sub(r'\.s\d+\.?$', '', matches[0])
+            for m in matches[1:]:
+                if m.startswith(base) and not re.search(r'\.s\d+\.?$', m):
+                    doi = m
+                    break
+
+    # Remove non-DOI suffixes: known publisher paths and common junk
+    # Strategy: if we find these patterns, remove them AND everything after
+    suffixes = [
+        # Publisher-specific paths
+        '/-/dcsupplemental',
+        '/-/dc',
+        # Common file extension
+        '.pdf',
+        # Document status markers that clearly shouldn't be in DOI
+        'preprint',
+        'received',
+        'published',
+        'edited',
+        'advance',
+        'full',
+        'abstract',
+        '-supplement',
+        'supplement',
+    ]
+
+    doi_lower = doi.lower()
+    for suffix in suffixes:
+        pos = doi_lower.find(suffix)
+        if pos > 0:  # Found suffix (not at start)
+            # For word-only suffixes, ensure they follow a valid DOI character
+            if suffix.isalpha():
+                prev_char = doi[pos-1]
+                # Accept if preceded by alphanumeric, dash, underscore, slash, or dot
+                if prev_char.isalnum() or prev_char in '.-_/':
+                    doi = doi[:pos]
+                    break
+            else:
+                # Non-word suffixes (paths, extensions) - just cut
+                doi = doi[:pos]
+                break
+
+    # Clean up trailing periods and colons
+    doi = doi.rstrip('.:')
+
+    # Quality check
     if len(doi) <= 8:
         raise DOIParsingError('failed to extract doi: '+doi)
 
     return doi
-
-
 def isvaliddoi(doi):
     try:
         doi2 = parse_doi('doi:'+doi)
@@ -178,6 +307,12 @@ def pdfhead(pdf, maxpages=10, minwords=200, image=False):
 
 
 def extract_pdf_doi(pdf, image=False):
+    # Try PDF metadata first (fast and reliable for many publishers)
+    metadata_doi = parse_doi_from_pdf_metadata(pdf)
+    if metadata_doi:
+        return metadata_doi
+
+    # Fall back to text extraction if metadata doesn't have DOI
     return parse_doi(pdfhead(pdf, image=image))
 
 

{papers_cli-2.15.3 → papers_cli-2.15.5}/papers_cli.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: papers-cli
-Version: 2.15.3
+Version: 2.15.5
 Summary: utilities to keep your PDF library organized
 Author-email: Mahé Perrette <mahe.perrette@gmail.com>
 License: MIT