papers-cli 2.14__tar.gz → 2.14.1__tar.gz

{papers_cli-2.14 → papers_cli-2.14.1}/.github/workflows/tests.yml

@@ -22,6 +22,5 @@ jobs:
       run: |
         python -m pip install --upgrade pip
         pip install tox tox-gh-actions
-        sudo apt install poppler-utils
     - name: Test with tox
       run: tox

{papers_cli-2.14 → papers_cli-2.14.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: papers-cli
-Version: 2.14
+Version: 2.14.1
 Summary: utilities to keep your PDF library organized
 Author-email: Mahé Perrette <mahe.perrette@gmail.com>
 License: MIT
@@ -10,6 +10,7 @@ Classifier: Programming Language :: Python :: 3
 Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE
+Requires-Dist: PyMuPDF
 Requires-Dist: unidecode
 Requires-Dist: crossrefapi
 Requires-Dist: bibtexparser
@@ -44,7 +45,7 @@ command-line bibliography management tool. Aims:
 ## Dependencies
 
 - python 3.9+
-- [poppler-utils](https://en.wikipedia.org/wiki/Poppler_(software)) (only:`pdftotext`): convert PDF to text for parsing
+- [PyMuPDF](https://github.com/pymupdf/PyMuPDF) (preferred) or [poppler-utils](https://en.wikipedia.org/wiki/Poppler_(software)) (only:`pdftotext`; deprecated): convert PDF to text for parsing
 - [bibtexparser](https://bibtexparser.readthedocs.io) : parse bibtex files
 - [crossrefapi](https://github.com/fabiobatalha/crossrefapi) : make polite requests to crossref API
 - [scholarly](https://github.com/OrganicIrradiation/scholarly) : interface for google scholar
@@ -54,7 +55,6 @@ command-line bibliography management tool. Aims:
 ## Install
 
 - `pip install papers-cli`
-- install third-party dependencies (Ubuntu: `sudo apt install poppler-utils`)
 
 Note there is another project registered on pypi as papers, hence `papers-cli` for command-line-interface.
 
@@ -311,7 +311,7 @@ There is also a command specifically designed to clean up the zombie files and f
     papers filecheck --clean-filesdir
 
 That command will ask before removig anything, unless `--force` is passed. Currently
-it ignores hidden files and folders, and will only consider folder that have a `.{folder}.bib` file inside, 
+it ignores hidden files and folders, and will only consider folder that have a `.{folder}.bib` file inside,
 which is the convention `papers` follows to store multiple attachments. That command works best
 when the files are in their own folder, and not mixed up with other things, obviously.
 

{papers_cli-2.14 → papers_cli-2.14.1}/README.md

@@ -25,7 +25,7 @@ command-line bibliography management tool. Aims:
 ## Dependencies
 
 - python 3.9+
-- [poppler-utils](https://en.wikipedia.org/wiki/Poppler_(software)) (only:`pdftotext`): convert PDF to text for parsing
+- [PyMuPDF](https://github.com/pymupdf/PyMuPDF) (preferred) or [poppler-utils](https://en.wikipedia.org/wiki/Poppler_(software)) (only:`pdftotext`; deprecated): convert PDF to text for parsing
 - [bibtexparser](https://bibtexparser.readthedocs.io) : parse bibtex files
 - [crossrefapi](https://github.com/fabiobatalha/crossrefapi) : make polite requests to crossref API
 - [scholarly](https://github.com/OrganicIrradiation/scholarly) : interface for google scholar
@@ -35,7 +35,6 @@ command-line bibliography management tool. Aims:
 ## Install
 
 - `pip install papers-cli`
-- install third-party dependencies (Ubuntu: `sudo apt install poppler-utils`)
 
 Note there is another project registered on pypi as papers, hence `papers-cli` for command-line-interface.
 
@@ -292,7 +291,7 @@ There is also a command specifically designed to clean up the zombie files and f
     papers filecheck --clean-filesdir
 
 That command will ask before removig anything, unless `--force` is passed. Currently
-it ignores hidden files and folders, and will only consider folder that have a `.{folder}.bib` file inside, 
+it ignores hidden files and folders, and will only consider folder that have a `.{folder}.bib` file inside,
 which is the convention `papers` follows to store multiple attachments. That command works best
 when the files are in their own folder, and not mixed up with other things, obviously.
 

{papers_cli-2.14 → papers_cli-2.14.1}/papers/__main__.py

@@ -1,6 +1,7 @@
 """That is the script called by the papers cli command
 """
 import os
+import copy
 import sys
 from pathlib import Path
 import logging
@@ -80,6 +81,8 @@ def _backup_bib(biblio, config, message=None):
     shutil.copy(config.bibtex, config.backupfile)
     config.gitcmd(f"add {config.backupfile.name}")
 
+    biblio = copy.deepcopy(biblio)
+
     if config.backup_files:
         logger.info('backup bibliography with files')
         backupfilesdir = backupdir/"files"
@@ -561,6 +564,7 @@ def addcmd(parser, o, config):
     set_keyformat_config_from_cmd(o, config)
 
     biblio = get_biblio(config)
+    biblio_init = copy.deepcopy(biblio)
 
     entries = []
 
@@ -650,10 +654,33 @@ def addcmd(parser, o, config):
             return
 
         entries = [{k:v for k,v in e.items() if v != ""} for e in entries]
-        biblio.db.entries = otherentries + entries
+        biblio.db.entries = sorted(otherentries + entries, key=lambda e: biblio.key(e))
 
     savebib(biblio, config)
 
+    # compare entries to inform user
+    # this is not very efficient but I have yet to hear a complaint about speed
+
+    old_set = set(e["ID"] for e in biblio_init.entries)
+    old_entries_by_key = {e["ID"]:e for e in biblio_init.db.entries}
+    new_set = set(e["ID"] for e in biblio.entries)
+    new_entries_by_key = {e["ID"]:e for e in biblio.db.entries}
+    modified_set = set(e["ID"] for e in entries ).intersection(set.intersection(old_set, new_set))
+
+    for ID in sorted(old_set - new_set):
+        print(format_entry(biblio_init, old_entries_by_key[ID], prefix="Removed"))
+
+    for ID in sorted(new_set - old_set):
+        print(format_entry(biblio, new_entries_by_key[ID], prefix="Added"))
+
+    for ID in sorted(modified_set):
+        before = old_entries_by_key[ID]
+        after = new_entries_by_key[ID]
+        if before != after:
+            print(format_entry(biblio, after, prefix="Modified"))
+        else:
+            print(format_entry(biblio, after, prefix="Existing"))
+
     if o.open:
         for e in entries:
             view_entry_files(biblio, e)

{papers_cli-2.14 → papers_cli-2.14.1}/papers/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '2.14'
-__version_tuple__ = version_tuple = (2, 14)
+__version__ = version = '2.14.1'
+__version_tuple__ = version_tuple = (2, 14, 1)

{papers_cli-2.14 → papers_cli-2.14.1}/papers/bib.py

@@ -273,7 +273,6 @@ class Biblio:
         keys = [self.key(ei) for ei in self.db.entries]
         return bisect.bisect_left(keys, self.key(entry))
 
-
     def insert_entry(self, entry, update_key=False, check_duplicate=False, rename=False, copy=False, metadata={}, **checkopt):
         """
         """
@@ -296,20 +295,21 @@ class Biblio:
         if i < len(self.entries) and self.key(self.entries[i]) == self.key(entry):
             logger.info('key duplicate: '+self.key(self.entries[i]))
 
+            if self.entries[i]["title"] == entry["title"]:
+                logger.info('exact duplicate')
+                return [ self.entries[i] ]
+
             if update_key:
                 newkey = self.append_abc_to_key(entry)  # add abc
                 logger.info('update key: {} => {}'.format(entry['ID'], newkey))
                 entry['ID'] = newkey
 
             else:
-                print(format_entry(self, entry, prefix="New entry"))
-                print(format_entry(self, self.entries[i], prefix="Conflicts with"))
                 raise DuplicateKeyError('Key conflict. Try update_key is True or specify --key KEY explicitly')
 
         else:
             logger.info('new entry: '+self.key(entry))
 
-        print(format_entry(self, entry, prefix="Added"))
         self.entries.insert(i, entry)
 
         if rename: self.rename_entry_files(entry, copy=copy)
@@ -338,7 +338,6 @@ class Biblio:
             if entry == candidate:
                 logger.debug('exact duplicate')
                 if rename: self.rename_entry_files(candidate, copy=copy)
-                print(format_entry(self, entry, prefix="Existing"))
                 return [ entry ] # do nothing
 
             if update_key and entry['ID'] != candidate['ID']:
@@ -357,7 +356,6 @@ class Biblio:
                 logger.debug('fixed: exact duplicate')
                 entry = candidate
                 if rename: self.rename_entry_files(candidate, copy=copy)
-                print(format_entry(self, entry, prefix="Existing"))
                 return [ entry ] # do nothing
 
             logger.debug('conflict resolution: '+on_conflict)
@@ -438,10 +436,10 @@ class Biblio:
             files += attachments
 
         self.set_files(entry, [os.path.abspath(f) for f in files])
-        entry['ID'] = self.generate_key(entry)
+        entry['ID'] = self.keyformat(entry)
         logger.debug('generated PDF key: '+entry['ID'])
 
-        kw["update_key"] = False  # already updated
+        kw["update_key"] = True
         return self.insert_entry(entry, **kw)
 
 

{papers_cli-2.14 → papers_cli-2.14.1}/papers/config.py

@@ -143,9 +143,9 @@ class Config:
     def load(cls, the_file):
         js = json.load(open(the_file))
         if 'nameformat' in js:
-            js['nameformat'] = Format(**js.get('nameformat'))
+            js['nameformat'] = Format(**{**vars(NAMEFORMAT), **js.get('nameformat')})
         if 'keyformat' in js:
-            js['keyformat'] = Format(**js.get('keyformat'))
+            js['keyformat'] = Format(**{**vars(KEYFORMAT), **js.get('keyformat')})
         cfg = cls(file=the_file, **js)
         cfg._update_paths_to_absolute()
         return cfg

{papers_cli-2.14 → papers_cli-2.14.1}/papers/duplicate.py

@@ -367,12 +367,6 @@ def edit_entries(entries, diff=False, ndiff=False, editor=None):
     if res == 0:
         logger.info('sucessfully edited file, insert edited entries')
         db_updated = bibtexparser.loads(open(filename).read())
-        keys_before = [e.get('ID') for e in entries]
-        keys_after = [e.get('ID') for e in db_updated.entries]
-        if set(keys_before) != set(keys_after):
-            print(f"{bcolors.WARNING}Keys edited ->", bcolors.OKBLUE ," ".join(sorted(set(keys_before).difference(keys_after))), bcolors.ENDC,
-                  "=>", bcolors.OKBLUE,
-                  " ".join(sorted(set(keys_after).difference(keys_before))), bcolors.ENDC) if keys_after else '(deleted)'
         return db_updated.entries
 
     else:

{papers_cli-2.14 → papers_cli-2.14.1}/papers/extract.py

@@ -26,12 +26,28 @@ class DOIRequestError(ValueError):
 
 # PDF parsing / crossref requests
 # ===============================
+def readpdf_fitz(pdf_path, pages=None, first=None, last=None):
+    import fitz
 
-def readpdf(pdf, first=None, last=None):
-    # TODO the python package pdftotext can do this directly, with no temp file and no I/O.
-    if not os.path.isfile(pdf):
-        raise ValueError(repr(pdf) + ": not a file")
+    # Open the PDF file
+    document = fitz.open(pdf_path)
+    text = ""
+
+    # Iterate through the pages
+    for page_num in range(len(document)):
+        if pages is not None and page_num+1 not in pages:
+            continue
+        elif first is not None and page_num+1 < first:
+            continue
+        elif last is not None and page_num+1 > last:
+            continue
+        page = document.load_page(page_num)
+        text += page.get_text()
+
+    return text
 
+def readpdf_poputils(pdf, first=None, last=None, pages=None):
+    # DEPRECATED
     tmptxt = tempfile.mktemp(suffix='.txt')
 
     cmd = ['pdftotext']
@@ -46,6 +62,18 @@ def readpdf(pdf, first=None, last=None):
 
     return txt
 
+def readpdf(pdf, first=None, last=None):
+    # TODO the python package pdftotext can do this directly, with no temp file and no I/O.
+    if not os.path.isfile(pdf):
+        raise ValueError(repr(pdf) + ": not a file")
+    try:
+        return readpdf_fitz(pdf, first=first, last=last)
+    except ImportError:
+        logger.warning("PyMuPDF not installed, using pdftotext")
+        return readpdf_fitz(pdf, first=first, last=last)
+
+    return txt
+
 
 def readpdf_image(pdf, first=None, last=None):
 

{papers_cli-2.14 → papers_cli-2.14.1}/papers/filename.py

@@ -22,6 +22,10 @@ def _cite_author(names):
     else:
         return names[0]
 
+UNKNOWN_AUTHOR = 'unknown'
+UNKNOWN_YEAR = '0000'
+UNKNOWN_JOURNAL = None
+UNKNOWN_TITLE = ""
 
 def make_template_fields(
     entry,
@@ -31,6 +35,7 @@ def make_template_fields(
     title_length=100,
     author_sep="_",
     title_sep="-",
+    **ignore,
 ):
     """
     Available fields in output are explicitly listed here, and this is the single source of truth for these.
@@ -43,10 +48,12 @@ def make_template_fields(
     - Title: same as title by with capitalized words
     - year
     - ID : bibtex key
+    - doi : bibtex DOI
+    - doi- : slugified doi (for use in filenames)
     Each one of these needs a specific, explicit assignment below.
     """
     # names = bibtexparser.customization.getnames(entry.get('author','unknown').lower().split(' and '))
-    _names = family_names(entry.get("author", "unknown").lower())
+    _names = family_names(entry.get("author", UNKNOWN_AUTHOR).lower())
     _names = [slugify(nm) for nm in _names]
     author = author_sep.join([nm for nm in _names[:author_num]])
     Author = author_sep.join([nm.capitalize() for nm in _names[:author_num]])
@@ -54,13 +61,13 @@ def make_template_fields(
     authorX = AuthorX.lower()
 
     # a thing that's not a bibtex article won't have a journal
-    journal = entry.get("journal", None)
+    journal = entry.get("journal", UNKNOWN_JOURNAL)
 
-    year = str(entry.get("year", "0000"))
+    year = str(entry.get("year", UNKNOWN_YEAR))
 
     if not title_word_num or not entry.get("title", ""):
-        title = ""
-        Title = ""
+        title = UNKNOWN_TITLE
+        Title = UNKNOWN_TITLE
     else:
         titlewords = normalize(entry["title"]).lower().split()
         _titles = listtag(
@@ -84,15 +91,12 @@ def make_template_fields(
         "title": title,
         "Title": Title,
         "ID": entry.get("ID"),
+        "doi": entry.get("doi"),
+        "doi_": slugify(entry.get("doi", "")),
+        "doi_or_id": slugify(entry.get("doi", entry.get("ID", ""))),
     }
 
 
-def stringify_entry(entry, template, **opt):
-    fields = make_template_fields(entry, **opt)
-    res = template.format(**fields)
-    return res
-
-
 class Format:
     """
     Store formatting info as python template formatted with str.format() method. See make_template_fields for available fields.
@@ -113,7 +117,8 @@ class Format:
 
     """
     # def __init__(self, template="{author}{year}{title}", author_num=2, title_word_num=5, author_sep="_", title_sep="-")
-    def __init__(self, template, author_num=2, title_word_num=100, title_word_size=1, title_length=100, author_sep="_", title_sep="-"):
+    def __init__(self, template, author_num=2, title_word_num=100, title_word_size=1, title_length=100,
+                 author_sep="_", title_sep="-", unknown_strict=False, unknown_template=None):
         self.template = template
         self.author_num = author_num
         self.author_sep = author_sep
@@ -121,16 +126,30 @@ class Format:
         self.title_sep = title_sep
         self.title_word_num = title_word_num
         self.title_word_size = title_word_size
+        self.unknown_strict = unknown_strict
+        self.unknown_template = unknown_template
 
     def todict(self):
         return vars(self)
 
+    def is_unknown(self, entry):
+        conditions = ( entry.get("author", UNKNOWN_AUTHOR) == UNKNOWN_AUTHOR, entry.get("year", UNKNOWN_YEAR) == UNKNOWN_YEAR, entry.get("title", UNKNOWN_TITLE) == UNKNOWN_TITLE)
+        if self.unknown_strict:
+            return all(conditions)
+        else:
+            return any(conditions)
+
     def render(self, **entry):
-        return stringify_entry(entry, **vars(self))
+        fields = make_template_fields(entry, **vars(self))
+
+        if self.unknown_template and self.is_unknown(entry):
+            return self.unknown_template.format(**fields)
+
+        return self.template.format(**fields)
 
     def __call__(self, entry):
         return self.render(**entry)
 
 
-KEYFORMAT = Format(template='{author}{year}', author_num=2, author_sep="_")
-NAMEFORMAT = Format(template='{authorX}_{year}_{title}', author_sep="_", title_sep="-")
+KEYFORMAT = Format(template='{author}{year}', author_num=2, author_sep="_", unknown_template="{doi}")
+NAMEFORMAT = Format(template='{authorX}_{year}_{title}', author_sep="_", title_sep="-", unknown_template="unknown_{doi_or_id}")

{papers_cli-2.14 → papers_cli-2.14.1}/papers_cli.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: papers-cli
-Version: 2.14
+Version: 2.14.1
 Summary: utilities to keep your PDF library organized
 Author-email: Mahé Perrette <mahe.perrette@gmail.com>
 License: MIT
@@ -10,6 +10,7 @@ Classifier: Programming Language :: Python :: 3
 Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE
+Requires-Dist: PyMuPDF
 Requires-Dist: unidecode
 Requires-Dist: crossrefapi
 Requires-Dist: bibtexparser
@@ -44,7 +45,7 @@ command-line bibliography management tool. Aims:
 ## Dependencies
 
 - python 3.9+
-- [poppler-utils](https://en.wikipedia.org/wiki/Poppler_(software)) (only:`pdftotext`): convert PDF to text for parsing
+- [PyMuPDF](https://github.com/pymupdf/PyMuPDF) (preferred) or [poppler-utils](https://en.wikipedia.org/wiki/Poppler_(software)) (only:`pdftotext`; deprecated): convert PDF to text for parsing
 - [bibtexparser](https://bibtexparser.readthedocs.io) : parse bibtex files
 - [crossrefapi](https://github.com/fabiobatalha/crossrefapi) : make polite requests to crossref API
 - [scholarly](https://github.com/OrganicIrradiation/scholarly) : interface for google scholar
@@ -54,7 +55,6 @@ command-line bibliography management tool. Aims:
 ## Install
 
 - `pip install papers-cli`
-- install third-party dependencies (Ubuntu: `sudo apt install poppler-utils`)
 
 Note there is another project registered on pypi as papers, hence `papers-cli` for command-line-interface.
 
@@ -311,7 +311,7 @@ There is also a command specifically designed to clean up the zombie files and f
     papers filecheck --clean-filesdir
 
 That command will ask before removig anything, unless `--force` is passed. Currently
-it ignores hidden files and folders, and will only consider folder that have a `.{folder}.bib` file inside, 
+it ignores hidden files and folders, and will only consider folder that have a `.{folder}.bib` file inside,
 which is the convention `papers` follows to store multiple attachments. That command works best
 when the files are in their own folder, and not mixed up with other things, obviously.
 

{papers_cli-2.14 → papers_cli-2.14.1}/papers_cli.egg-info/requires.txt

@@ -1,3 +1,4 @@
+PyMuPDF
 unidecode
 crossrefapi
 bibtexparser

{papers_cli-2.14 → papers_cli-2.14.1}/pyproject.toml

@@ -16,6 +16,7 @@ classifiers = [
     "Programming Language :: Python :: 3",
 ]
 dependencies = [
+    "PyMuPDF",
     "unidecode",
     "crossrefapi",
     "bibtexparser",
@@ -58,6 +59,7 @@ python =
 commands = pytest --cov=papers --cov-append --cov-report=term-missing {posargs} -xv
 deps =
     bibtexparser < 2.0.0
+    PyMuPDF
     scholarly
     crossrefapi
     rapidfuzz

{papers_cli-2.14 → papers_cli-2.14.1}/requirements.txt

@@ -1,3 +1,4 @@
+PyMuPDF
 unidecode
 crossrefapi
 bibtexparser < 2.0.0

{papers_cli-2.14 → papers_cli-2.14.1}/tests/test_add.py

@@ -305,7 +305,7 @@ class TestAddConflict(BibTest):
  title = {Something else entirely}
 }"""
 
-    bibtex_conflict_key_fixed = """@article{perrette_author0000,
+    bibtex_conflict_key_fixed = """@article{10.5194/bg-8-515-2011XXX,
  author = {M. Perrette and Another author},
  doi = {10.5194/bg-8-515-2011XXX},
  title = {Something else entirely}
@@ -416,7 +416,7 @@ class TestAddConflict(BibTest):
         # key conflict and update entry
         open(self.otherbib, 'w').write(self.bibtex_conflict_key)
         paperscmd(f'add {self.otherbib} --bibtex {self.mybib} -u')
-        expected = self.bibtex+'\n\n'+self.bibtex_conflict_key_fixed
+        expected = self.bibtex_conflict_key_fixed+'\n\n'+self.bibtex
         self.assertMultiLineEqual(open(self.mybib).read().strip(), expected) # entries did not change
 
     def test_add_same_doi_unchecked(self):

{papers_cli-2.14 → papers_cli-2.14.1}/tests/test_duplicates.py

@@ -266,7 +266,7 @@ class TestAddResolveDuplicate(BibTest):
 
     def test_conflict_updated_from_original_but_originalkey(self):
 
-        expected = """@article{fieldRareYear,
+        expected = """@article{10.5194/bg-8-515-2011,
  author = {New Author Field},
  doi = {10.5194/bg-8-515-2011},
  journal = {ConflictJournal},