papers-cli 2.17.6__tar.gz → 2.18.1__tar.gz

{papers_cli-2.17.6 → papers_cli-2.18.1}/.github/workflows/tests.yml

@@ -8,12 +8,11 @@ jobs:
   build:
 
     runs-on: ubuntu-latest
-    # 3.15t is allowed to fail: PyMuPDF does not support it yet
-    continue-on-error: ${{ matrix.python-version == '3.15t' }}
     strategy:
       fail-fast: false
       matrix:
-        python-version: [ "3.9", "3.10", "3.11", "3.12", "3.13", "3.14", "3.14t", "3.15", "3.15t" ]
+        # 3.15t: add when PyMuPDF supports it (https://github.com/pymupdf/PyMuPDF/issues/5170, #113)
+        python-version: [ "3.9", "3.10", "3.11", "3.12", "3.13", "3.14", "3.14t", "3.15" ]
 
     steps:
     - uses: actions/checkout@v6

{papers_cli-2.17.6 → papers_cli-2.18.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: papers-cli
-Version: 2.17.6
+Version: 2.18.1
 Summary: utilities to keep your PDF library organized
 Author-email: Mahé Perrette <mahe.perrette@gmail.com>
 License: MIT
@@ -13,7 +13,6 @@ Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: PyMuPDF<2
 Requires-Dist: unidecode
-Requires-Dist: crossrefapi
 Requires-Dist: bibtexparser>=2.0.0b2
 Requires-Dist: scholarly
 Requires-Dist: rapidfuzz

{papers_cli-2.17.6 → papers_cli-2.18.1}/docs/adding-entries.md

@@ -46,6 +46,25 @@ where, above, the `papers_test` directory contains a few PDF files. For each
 PDF, `papers` will attempt to extract the metadata and add the relevant file to
 the bibliography, and rename files into the files directory.
 
+The metadata of a PDF are found as follows:
+
+1. a DOI in the PDF metadata is used as is;
+2. otherwise, the DOIs printed in the text (first pages, and the last page,
+   where some publishers add a cover page) are looked up in turn, and the
+   first one whose title appears in the first pages is used: this skips the
+   DOIs of cited papers;
+3. otherwise, a crossref search on the text of the first pages, whose result
+   is also accepted only if its title appears in the text.
+
+If none matches, the PDF is not added: pass its DOI with `--doi` (see below),
+or, for a reference without DOI (older papers, books, reports...), describe it
+and attach the PDF:
+
+```
+papers add --title "Self-consistent equilibrium ocean tides" --author "Agnew, Duncan C. and Farrell, W. E." \
+    --year 1978 --journal "Geophysical Journal International" --attachment agnew1978.pdf --rename
+```
+
 ## Add an entry from its DOI
 
 If you already know the DOI of a PDF, and don't want to gamble the fulltext

{papers_cli-2.17.6 → papers_cli-2.18.1}/docs/configuration.md

@@ -28,7 +28,10 @@ Type `papers status -v` to check your configuration.
 
 You also notice a cache directory. All internet requests such as crossref
 requests are saved in the cache directory. This happens regardless of whether
-`papers` is installed or not.
+`papers` is installed or not. The metadata of each DOI is a json file under
+`doi/`, e.g. `doi/10.1126/science.1166510.json` (DOIs that do not exist are
+remembered for a day). Deleting the directory, or any file in it, is safe:
+the requests are made again when needed.
 
 ## Local install
 

{papers_cli-2.17.6 → papers_cli-2.18.1}/docs/features.md

@@ -3,7 +3,7 @@
 ## Current features
 
 - parse PDF to extract DOI
-- fetch bibtex entry from DOI (using crossref API)
+- fetch bibtex entry from DOI (using crossref API, or doi.org for DOIs registered elsewhere, e.g. DataCite)
 - fetch bibtex entry by fulltext search (using crossref API or google scholar)
 - create and maintain bibtex file
 - add entry as PDF (`papers add ...`)

{papers_cli-2.17.6 → papers_cli-2.18.1}/docs/installation.md

@@ -37,10 +37,8 @@ mitigations are built in:
 - python 3.9+
 - [PyMuPDF](https://github.com/pymupdf/PyMuPDF) (preferred) or
   [poppler-utils](https://en.wikipedia.org/wiki/Poppler_(software))
-  (only `pdftotext`; deprecated): convert PDF to text for parsing
+  (only `pdftotext` and `pdfinfo`; deprecated): convert PDF to text for parsing
 - [bibtexparser](https://bibtexparser.readthedocs.io): parse bibtex files
-- [crossrefapi](https://github.com/fabiobatalha/crossrefapi): make polite
-  requests to the crossref API
 - [scholarly](https://github.com/OrganicIrradiation/scholarly): interface for
   google scholar
 - [rapidfuzz](https://github.com/rhasspy/rapidfuzz): calculate score to sort

{papers_cli-2.17.6 → papers_cli-2.18.1}/papers/__main__.py

@@ -13,16 +13,16 @@ import fnmatch   # unix-like match
 
 import papers
 from papers import logger
-from papers.extract import extract_pdf_doi, isvaliddoi, extract_pdf_metadata
+from papers.extract import extract_pdf_doi, isvaliddoi, extract_pdf_metadata, prefetch_pdf_metadata
 from papers.extract import fetch_bibtex_by_doi, fetch_bibtex_by_fulltext_crossref, fetch_bibtex_by_fulltext_scholar
 from papers.encoding import parse_file, format_file, family_names, format_entries, standard_name, format_entry, parse_keywords, format_key
 from papers.config import (bcolors, Config, search_config, CONFIG_FILE, CONFIG_FILE_LOCAL,
                            DATA_DIR, CONFIG_FILE_LEGACY, CONFIG_FILE_LEGACY_XDG)
 from papers.duplicate import list_duplicates, list_uniques, edit_entries, title_id
-from papers.entries import get_entry_val, entry_content_equal
+from papers.entries import get_entry_val, entry_content_equal, entry_from_dict
 from papers.bib import (Biblio, FUZZY_RATIO, DEFAULT_SIMILARITY, entry_filecheck,
                         backupfile as backupfile_func, isvalidkey, DuplicateKeyError, clean_filesdir,
-                        are_duplicates, download_url, get_biblio)
+                        are_duplicates, download_url, get_biblio, scan_dir_walk)
 from papers.install import resolve_install, apply_install, InputAsker, DefaultAsker
 from papers.utils import view_pdf, open_folder, PapersExit, file_uri_to_path, find_masked_viewer
 from papers.backup import (silent_backup_bib, restore_from_backupdir,
@@ -329,9 +329,20 @@ def addcmd(parser, o, config):
             metadata = edit_entries(metadata)
             o.edit = False
 
-        entries.extend( biblio.insert_entry(metadata, **kw) )
+        entries.extend( biblio.insert_entry(entry_from_dict(metadata), rename=o.rename, copy=o.copy, **kw) )
 
 
+    # several PDFs: fetch their metadata concurrently first
+    if not o.doi and not o.scholar and not (o.no_query_doi and o.no_query_fulltext):
+        pdfs = []
+        for file in o.file:
+            if os.path.isdir(file) and o.recursive:
+                pdfs.extend(path for kind, path in scan_dir_walk(file) if kind == 'file' and path.lower().endswith('.pdf'))
+            elif os.path.isfile(file) and file.lower().endswith('.pdf'):
+                pdfs.append(file)
+        if len(pdfs) > 1:
+            prefetch_pdf_metadata(pdfs, search_doi=not o.no_query_doi, search_fulltext=not o.no_query_fulltext)
+
     for file in o.file:
         try:
             if os.path.isdir(file):

{papers_cli-2.17.6 → papers_cli-2.18.1}/papers/_version.py

@@ -18,7 +18,7 @@ version_tuple: tuple[int | str, ...]
 commit_id: str | None
 __commit_id__: str | None
 
-__version__ = version = '2.17.6'
-__version_tuple__ = version_tuple = (2, 17, 6)
+__version__ = version = '2.18.1'
+__version_tuple__ = version_tuple = (2, 18, 1)
 
-__commit_id__ = commit_id = 'g2f4e6bd97'
+__commit_id__ = commit_id = 'g0c4d92d88'

{papers_cli-2.17.6 → papers_cli-2.18.1}/papers/bib.py

@@ -260,6 +260,26 @@ def backupfile(bibtex):
 class DuplicateKeyError(ValueError):
     pass
 
+def scan_dir_walk(direc):
+    """the files scanned by Biblio.scan_dir: yield ('entry', dir) for entry
+    directories (with a hidden bibtex), ('file', path) for other files"""
+    top = os.path.abspath(direc)
+    for root, direcs, files in os.walk(direc):
+        dirname = os.path.basename(root)
+        # never skip the scan root itself: basename('.') == '.' looks hidden
+        if os.path.abspath(root) != top:
+            if dirname.startswith('.'): continue
+            if dirname.startswith('_'): continue
+
+        if os.path.exists(hidden_bibtex(root)):
+            yield 'entry', root
+            continue
+
+        for file in files:
+            if not file.startswith('.'):
+                yield 'file', os.path.join(root, file)
+
+
 class Biblio:
     """
     The bibtex object that we operate on, which is mainly used to read and write to dynamically, and can then send the changes to be stored in a specified bibtex file on disk.
@@ -575,36 +595,25 @@ class Biblio:
 
     def scan_dir_iter(self, direc, search_doi=True, search_fulltext=True, **kw):
 
-        top = os.path.abspath(direc)
-        for root, direcs, files in os.walk(direc):
-            dirname = os.path.basename(root)
-            # never skip the scan root itself: basename('.') == '.' looks hidden
-            if os.path.abspath(root) != top:
-                if dirname.startswith('.'): continue
-                if dirname.startswith('_'): continue
-
-            # maybe a special entry directory?
-            if os.path.exists(hidden_bibtex(root)):
+        for kind, path in scan_dir_walk(direc):
+            # a special entry directory
+            if kind == 'entry':
                 logger.debug('read from hidden bibtex')
                 try:
-                    entry = read_entry_dir(root, relative_to=self.relative_to)
+                    entry = read_entry_dir(path, relative_to=self.relative_to)
                     yield from self.insert_entry(entry, **kw)
                 except Exception as error:
-                    logger.warning(root+'::'+str(error))
+                    logger.warning(path+'::'+str(error))
                 continue
 
-            for file in files:
-                if file.startswith('.'):
-                    continue
-                path = os.path.join(root, file)
-                try:
-                    if file.lower().endswith('.pdf'):
-                        yield from self.add_pdf(path, search_doi=search_doi, search_fulltext=search_fulltext, **kw)
-                    elif file.lower().endswith('.bib'):
-                        yield from self.add_bibtex_file(path, **kw)
-                except Exception as error:
-                    logger.warning(path+'::'+str(error))
-                    continue
+            try:
+                if path.lower().endswith('.pdf'):
+                    yield from self.add_pdf(path, search_doi=search_doi, search_fulltext=search_fulltext, **kw)
+                elif path.lower().endswith('.bib'):
+                    yield from self.add_bibtex_file(path, **kw)
+            except Exception as error:
+                logger.warning(path+'::'+str(error))
+                continue
 
     def scan_dir(self, direc, **kw):
         " like scan_dir_iter but returns a list"

{papers_cli-2.17.6 → papers_cli-2.18.1}/papers/config.py

@@ -4,12 +4,10 @@ import copy
 import tempfile
 from pathlib import Path
 import hashlib
+import time
+from urllib.parse import quote
 import platformdirs
 
-try:
-    import fcntl  # not available on Windows
-except ImportError:
-    fcntl = None
 from papers.entries import parse_string
 from papers import logger
 from papers.filename import Format, NAMEFORMAT, KEYFORMAT
@@ -246,24 +244,17 @@ class Config:
         return '\n'.join(lines)
 
 
-def _init_cache():
-    if not os.path.exists(CACHE_DIR):
-        logger.info('make cache directory for DOI requests: '+CACHE_DIR)
-        os.makedirs(CACHE_DIR)
-
-
-@contextlib.contextmanager
-def _cache_lock(file):
-    """advisory inter-process lock on a sidecar file (no-op without fcntl)"""
-    if fcntl is None:
-        yield
-        return
-    with open(file + '.lock', 'w') as lockfile:
-        fcntl.flock(lockfile, fcntl.LOCK_EX)
-        try:
-            yield
-        finally:
-            fcntl.flock(lockfile, fcntl.LOCK_UN)
+def _cache_path(folder, key):
+    """one json file per key: <folder>/<prefix>/<rest>.json, split at the first
+    slash so that DOIs are grouped by registrant (10.1126/science.1166510 ->
+    10.1126/science.1166510.json); characters unsafe in file names are quoted"""
+    head, sep, tail = key.partition('/')
+    if not sep:
+        head, tail = key[:2], key
+    name = quote(tail, safe='')
+    if len(name) > 200:  # file name length limit
+        name = hashlib.sha256(tail.encode('utf-8')).hexdigest()
+    return os.path.join(folder, quote(head, safe=''), name + '.json')
 
 
 def _read_cache_file(file):
@@ -272,17 +263,18 @@ def _read_cache_file(file):
             return json.load(open(file))
         except (json.JSONDecodeError, OSError) as error:
             # a corrupt cache file must not break every query
-            logger.warning(f"unreadable cache file {file}: {error} -- starting over with an empty cache")
-    return {}
+            logger.warning(f"unreadable cache file {file}: {error} -- ignored")
+    return None
 
 
-def _write_cache_file(cache, file):
-    """atomic replace, so an interrupted write cannot truncate the cache"""
-    fd, tmp = tempfile.mkstemp(dir=os.path.dirname(file) or '.',
+def _write_cache_file(data, file):
+    """atomic replace, so an interrupted write cannot truncate the file"""
+    os.makedirs(os.path.dirname(file), exist_ok=True)
+    fd, tmp = tempfile.mkstemp(dir=os.path.dirname(file),
                                prefix=os.path.basename(file) + '.', suffix='.tmp')
     try:
         with os.fdopen(fd, 'w') as f:
-            json.dump(cache, f)
+            json.dump(data, f)
         os.replace(tmp, file)
     except BaseException:
         with contextlib.suppress(OSError):
@@ -290,33 +282,59 @@ def _write_cache_file(cache, file):
         raise
 
 
-def cached(file, hashed_key=False):
+def _migrate_legacy_cache(legacy, folder, migrate):
+    """split a cache of previous versions (one json file for all keys) into
+    the per-key files; the legacy file is kept as <legacy>.bak"""
+    data = _read_cache_file(legacy)
+    if data:
+        logger.info(f'migrate cache {legacy} to {folder}')
+        for key, value in data.items():
+            if migrate:
+                value = migrate(value)
+            if value is not None:
+                # lower-case: DOIs are case-insensitive, and looked up in lower case
+                _write_cache_file({'value': value}, _cache_path(folder, key.lower()))
+    with contextlib.suppress(OSError):  # another process may have migrated it
+        os.replace(legacy, legacy + '.bak')
+
+
+NOT_FOUND_TTL = 24 * 3600  # seconds
 
-    file = os.path.join(CACHE_DIR, file)
 
+def cached(name, hashed_key=False, legacy=None, migrate=None, trim=None, not_found=None):
+    """cache the results of fun(key) as json files under CACHE_DIR/name/
+
+    legacy: cache file of previous versions, migrated on first use (with
+        migrate(value) -> value, or None to drop it)
+    trim: applied to results before they are stored
+    not_found: exception class raised by fun when the key does not exist:
+        it is remembered (and raised again) for NOT_FOUND_TTL seconds
+    """
     def decorator(fun):
-        # load lazily on first call so that importing papers does not touch the cache
-        cache = None
-        def decorated(doi):
-            nonlocal cache
-            if cache is None:
-                _init_cache()
-                cache = _read_cache_file(file)
+        def decorated(arg):
+            key = arg
+            folder = os.path.join(CACHE_DIR, name)
+            if legacy and not DRYRUN and os.path.exists(os.path.join(CACHE_DIR, legacy)):
+                _migrate_legacy_cache(os.path.join(CACHE_DIR, legacy), folder, migrate)
             if hashed_key: # use hashed parameter as key (for full text query)
-                key = hashlib.sha256(doi.encode('utf-8')).hexdigest()[:6]
-            else:
-                key = doi
-            if key in cache:
-                logger.debug('load from cache: '+repr((file, key)))
-                return cache[key]
-            else:
-                res = cache[key] = fun(doi)
-                if not DRYRUN:
-                    with _cache_lock(file):
-                        # merge entries written meanwhile by other processes,
-                        # keeping our own values for keys present in both
-                        cache = {**_read_cache_file(file), **cache}
-                        _write_cache_file(cache, file)
+                key = hashlib.sha256(arg.encode('utf-8')).hexdigest()[:16]
+            file = _cache_path(folder, key)
+            data = _read_cache_file(file)
+            if data and 'value' in data:
+                logger.debug('load from cache: '+repr((name, key)))
+                return data['value']
+            if data and not_found and time.time() - data.get('not_found', 0) < NOT_FOUND_TTL:
+                raise not_found(f'{key}: not found (cached until {time.ctime(data["not_found"] + NOT_FOUND_TTL)})')
+            try:
+                res = fun(arg)
+            except Exception as error:
+                if not_found and isinstance(error, not_found) and not DRYRUN:
+                    _write_cache_file({'not_found': time.time()}, file)
+                raise
+            if trim:
+                res = trim(res)
+            if not DRYRUN:
+                _write_cache_file({'value': res}, file)
             return res
         return decorated
     return decorator