papers-cli 2.16__tar.gz → 2.16.2__tar.gz

{papers_cli-2.16 → papers_cli-2.16.2}/.github/workflows/pypi.yml

@@ -1,3 +1,7 @@
+# Publishes to PyPI on tag push (e.g. v2.16.1).
+# If this job fails with "400 Bad Request", the package may still be on PyPI: that usually
+# means the workflow ran twice (e.g. tag pushed twice or job re-run) and the second upload
+# failed because the version already exists. Use verbose: true output to see the real error.
 name: Publish to PyPI
 
 on:
@@ -34,3 +38,6 @@ jobs:
 
     - name: Publish to PyPI
       uses: pypa/gh-action-pypi-publish@release/v1
+      with:
+        # Show full PyPI error (e.g. "400 File already exists") when upload fails
+        verbose: true

{papers_cli-2.16 → papers_cli-2.16.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: papers-cli
-Version: 2.16
+Version: 2.16.2
 Summary: utilities to keep your PDF library organized
 Author-email: Mahé Perrette <mahe.perrette@gmail.com>
 License: MIT

{papers_cli-2.16 → papers_cli-2.16.2}/papers/__main__.py

@@ -18,7 +18,7 @@ from papers.extract import extract_pdf_doi, isvaliddoi, extract_pdf_metadata
 from papers.extract import fetch_bibtex_by_doi, fetch_bibtex_by_fulltext_crossref, fetch_bibtex_by_fulltext_scholar
 from papers.encoding import parse_file, format_file, family_names, format_entries, standard_name, format_entry, parse_keywords, format_key
 from papers.config import bcolors, Config, search_config, CONFIG_FILE, CONFIG_FILE_LOCAL, DATA_DIR, CONFIG_FILE_LEGACY, BACKUP_DIR
-from papers.duplicate import list_duplicates, list_uniques, edit_entries
+from papers.duplicate import list_duplicates, list_uniques, edit_entries, title_id
 from papers.entries import get_entry_val
 from papers.bib import (Biblio, FUZZY_RATIO, DEFAULT_SIMILARITY, entry_filecheck,
                         backupfile as backupfile_func, isvalidkey, DuplicateKeyError, clean_filesdir,

{papers_cli-2.16 → papers_cli-2.16.2}/papers/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '2.16'
-__version_tuple__ = version_tuple = (2, 16)
+__version__ = version = '2.16.2'
+__version_tuple__ = version_tuple = (2, 16, 2)
 
-__commit_id__ = commit_id = 'g8b22ae064'
+__commit_id__ = commit_id = 'g60ea4dbb9'

{papers_cli-2.16 → papers_cli-2.16.2}/papers/bib.py

@@ -35,7 +35,16 @@ from papers.filename import NAMEFORMAT, KEYFORMAT
 from papers.utils import bcolors, checksum, move as _move
 import papers.config
 
-from papers.duplicate import conflict_resolution_on_insert, entry_diff, merge_files, check_duplicates
+from papers.duplicate import (
+    conflict_resolution_on_insert,
+    entry_diff,
+    merge_files,
+    check_duplicates,
+    entry_id,
+    _build_duplicate_index,
+    _get_duplicate_candidates,
+    _add_to_duplicate_index,
+)
 
 # KEY GENERATION
 # ==============
@@ -67,41 +76,8 @@ def append_abc(key, keys=[]):
     return Key
 
 
-# DUPLICATE DEFINITION
-# ====================
-
-def _remove_unicode(s, replace='_'):
-    s2 = []
-    for c in s:
-        if ord(c) > 128:
-            c = replace
-        s2.append(c)
-    return ''.join(s2)
-
-
-def _simplify_string(s):
-    ' replace unicode, strip, lower case '
-    # try:
-    #     s = latex_to_unicode(s)
-    # except Exception as error:
-    #     raise
-    #     logger.warning('simplify string: failed to remove latex: '+str(error))
-    s = _remove_unicode(s)
-    return s.lower().strip()
-
-
-def author_id(e):
-    return _simplify_string(' '.join(family_names(get_entry_val(e, 'author', ''))))
-
-def title_id(e):
-    return _simplify_string(get_entry_val(e, 'title', ''))
-
-def entry_id(e):
-    """entry identifier which is not the bibtex key
-    """
-    authortitle = ''.join([author_id(e), title_id(e)])
-    return (get_entry_val(e, 'doi', '').lower(), authortitle)
-
+# DUPLICATE DEFINITION (entry_id, author_id, title_id live in papers.duplicate)
+# =============================================================================
 
 FUZZY_RATIO = 80
 
@@ -172,7 +148,6 @@ def are_duplicates(e1, e2, similarity=DEFAULT_SIMILARITY, fuzzy_ratio=FUZZY_RATI
     return score >= target
 
 
-
 def hidden_bibtex(direc):
     " save metadata for a bundle of files "
     dirname = os.path.basename(direc)
@@ -301,7 +276,10 @@ class Biblio:
         # additional checks on DOI etc...
         if check_duplicate:
             logger.debug('check duplicates : TRUE')
-            return self.insert_entry_check(entry, update_key=update_key, rename=rename, copy=copy, **checkopt)
+            return self.insert_entry_check(
+                entry, update_key=update_key, rename=rename, copy=copy,
+                duplicate_index=checkopt.pop('duplicate_index', None), **checkopt
+            )
         else:
             logger.debug('check duplicates : FALSE')
 
@@ -335,8 +313,17 @@ class Biblio:
 
         return [ entry ]
 
-    def insert_entry_check(self, entry, update_key=False, mergefiles=True, on_conflict='i', rename=False, copy=False):
-        duplicates = [e for e in self.entries if self.eq(e, entry)]
+    def insert_entry_check(self, entry, update_key=False, mergefiles=True, on_conflict='i', rename=False, copy=False, duplicate_index=None):
+        # FUZZY can match on token_set_ratio(authortitle) without same doi/authortitle, so index would miss some duplicates
+        if self.similarity == 'FUZZY':
+            candidates = self.entries
+        elif duplicate_index is not None:
+            by_doi, by_authortitle = duplicate_index
+            candidates = _get_duplicate_candidates(entry, by_doi, by_authortitle)
+        else:
+            by_doi, by_authortitle = _build_duplicate_index(self.entries)
+            candidates = _get_duplicate_candidates(entry, by_doi, by_authortitle)
+        duplicates = [e for e in candidates if self.eq(e, entry)]
 
         if not duplicates:
             logger.debug('not a duplicate')
@@ -426,6 +413,12 @@ class Biblio:
         if convert_to_unicode:
             bib = latex_to_unicode_library(bib)
         entries = []
+        # Build index once so duplicate check is O(candidates) not O(library size) per entry
+        check_dup = kw.get('check_duplicate', True)
+        if check_dup and bib.entries:
+            duplicate_index = _build_duplicate_index(self.entries)
+        else:
+            duplicate_index = None
         for e in bib.entries:
             files = []
             if "file" in e:
@@ -436,7 +429,14 @@ class Biblio:
             if files:
                 self.set_files(e, files)
 
-            entries.extend( self.insert_entry(e, **kw) )
+            run_kw = dict(kw)
+            if duplicate_index is not None:
+                run_kw['duplicate_index'] = duplicate_index
+            added = self.insert_entry(e, **run_kw)
+            entries.extend(added)
+            if duplicate_index is not None:
+                for new_e in added:
+                    _add_to_duplicate_index(new_e, duplicate_index[0], duplicate_index[1])
         return entries
 
 

{papers_cli-2.16 → papers_cli-2.16.2}/papers/duplicate.py

@@ -18,12 +18,81 @@ import logging
 logger = logging.getLogger(__name__)
 
 from papers.extract import isvaliddoi, fetch_entry
-from papers.encoding import parse_file, format_file, format_entries
+from papers.encoding import parse_file, format_file, format_entries, family_names
 from papers.entries import update_entry
 
 from papers.utils import bcolors, checksum
 
 
+# ENTRY IDENTITY (for duplicate indexing and comparison)
+# ======================================================
+
+
+def _remove_unicode(s, replace='_'):
+    s2 = []
+    for c in s:
+        if ord(c) > 128:
+            c = replace
+        s2.append(c)
+    return ''.join(s2)
+
+
+def _simplify_string(s):
+    """Replace unicode, strip, lower case."""
+    s = _remove_unicode(s)
+    return s.lower().strip()
+
+
+def author_id(e):
+    return _simplify_string(' '.join(family_names(get_entry_val(e, 'author', ''))))
+
+
+def title_id(e):
+    return _simplify_string(get_entry_val(e, 'title', ''))
+
+
+def entry_id(e):
+    """Entry identifier which is not the bibtex key. Returns (doi_lower, authortitle)."""
+    authortitle = ''.join([author_id(e), title_id(e)])
+    return (get_entry_val(e, 'doi', '').lower(), authortitle)
+
+
+def _build_duplicate_index(entries):
+    """Build (by_doi, by_authortitle) so duplicate checks only run on candidates. O(n)."""
+    by_doi = {}
+    by_authortitle = {}
+    for e in entries:
+        doi, at = entry_id(e)
+        if doi:
+            by_doi.setdefault(doi, []).append(e)
+        if at:
+            by_authortitle.setdefault(at, []).append(e)
+    return (by_doi, by_authortitle)
+
+
+def _get_duplicate_candidates(entry, by_doi, by_authortitle):
+    """Return list of entries that might duplicate `entry` (same doi or authortitle)."""
+    doi, at = entry_id(entry)
+    candidates = by_doi.get(doi, []) + by_authortitle.get(at, [])
+    # deduplicate by id(); same entry can be in both indices
+    seen = set()
+    out = []
+    for e in candidates:
+        if id(e) not in seen:
+            seen.add(id(e))
+            out.append(e)
+    return out
+
+
+def _add_to_duplicate_index(entry, by_doi, by_authortitle):
+    """Register one entry in the index (after it was added to the library)."""
+    doi, at = entry_id(entry)
+    if doi:
+        by_doi.setdefault(doi, []).append(entry)
+    if at:
+        by_authortitle.setdefault(at, []).append(entry)
+
+
 # SEARCH DUPLICATES
 # =================
 

{papers_cli-2.16 → papers_cli-2.16.2}/papers_cli.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: papers-cli
-Version: 2.16
+Version: 2.16.2
 Summary: utilities to keep your PDF library organized
 Author-email: Mahé Perrette <mahe.perrette@gmail.com>
 License: MIT

{papers_cli-2.16 → papers_cli-2.16.2}/tests/test_bib_unit.py

@@ -6,9 +6,6 @@ from papers.bib import (
     isvalidkey,
     compare_entries,
     are_duplicates,
-    author_id,
-    title_id,
-    entry_id,
     hidden_bibtex,
     backupfile as backupfile_fn,
     EXACT_DUPLICATES,
@@ -16,6 +13,7 @@ from papers.bib import (
     FAIR_DUPLICATES,
     PARTIAL_DUPLICATES,
 )
+from papers.duplicate import author_id, title_id, entry_id
 
 
 class TestHiddenBibtex(unittest.TestCase):

{papers_cli-2.16 → papers_cli-2.16.2}/tests/test_duplicates.py

@@ -17,6 +17,7 @@ from papers.duplicate import (
     MergedEntry,
     ConflictingField,
 )
+from papers.entries import entry_from_dict, get_entry_val
 from tests.common import PAPERSCMD, paperscmd, BibTest
 
 
@@ -204,6 +205,117 @@ class TestDuplicatesPartial(TestDuplicatesFair):
         self.assertTrue(self.isduplicate(self.reference, self.conflictdoi))
 
 
+class TestInsertEntryCheckSimilarityLevels(unittest.TestCase):
+    """
+    Test that insert_entry(..., check_duplicate=True) applies each similarity level
+    correctly. Protects the duplicate-check logic (and index optimization) from
+    regressions: duplicates must be merged, non-duplicates must be added.
+    """
+
+    def _add_entry(self, biblio, entry_dict, check_duplicate=True, on_conflict='u'):
+        entry = entry_from_dict(entry_dict)
+        return biblio.insert_entry(
+            entry, check_duplicate=check_duplicate, on_conflict=on_conflict
+        )
+
+    def test_exact_duplicate_merged_non_duplicate_added(self):
+        b = Biblio(similarity='EXACT')
+        e1 = {'ENTRYTYPE': 'article', 'ID': 'Key1', 'author': 'Smith', 'title': 'Same', 'doi': '10.1/a', 'year': '2020'}
+        self._add_entry(b, e1)
+        self.assertEqual(len(b.entries), 1)
+        # Same content (including same key) → entry_content_equal → exact duplicate → merge
+        e2 = {'ENTRYTYPE': 'article', 'ID': 'Key1', 'author': 'Smith', 'title': 'Same', 'doi': '10.1/a', 'year': '2020'}
+        merged = self._add_entry(b, e2)
+        self.assertEqual(len(b.entries), 1, 'EXACT: duplicate should be merged')
+        self.assertEqual(get_entry_val(merged[0], 'ID', '').lower(), 'key1', 'EXACT: should keep first entry')
+        # Different title → not duplicate → add
+        e3 = {'ENTRYTYPE': 'article', 'ID': 'Key3', 'author': 'Smith', 'title': 'Other', 'doi': '10.1/b', 'year': '2020'}
+        self._add_entry(b, e3)
+        self.assertEqual(len(b.entries), 2, 'EXACT: non-duplicate should be added')
+
+    def test_good_duplicate_merged_non_duplicate_added(self):
+        b = Biblio(similarity='GOOD')
+        e1 = {'ENTRYTYPE': 'article', 'ID': 'Key1', 'author': 'Smith, John', 'title': 'A paper', 'doi': '10.1/a', 'year': '2020'}
+        self._add_entry(b, e1)
+        self.assertEqual(len(b.entries), 1)
+        # Same (doi, authortitle), different key/year → GOOD duplicate → merge
+        e2 = {'ENTRYTYPE': 'article', 'ID': 'Key2', 'author': 'Smith, John', 'title': 'A paper', 'doi': '10.1/a', 'year': '2021'}
+        self._add_entry(b, e2)
+        self.assertEqual(len(b.entries), 1, 'GOOD: duplicate should be merged')
+        # Different authortitle → not duplicate
+        e3 = {'ENTRYTYPE': 'article', 'ID': 'Key3', 'author': 'Jones', 'title': 'Other', 'doi': '10.1/b', 'year': '2020'}
+        self._add_entry(b, e3)
+        self.assertEqual(len(b.entries), 2, 'GOOD: non-duplicate should be added')
+
+    def test_fair_duplicate_merged_non_duplicate_added(self):
+        b = Biblio(similarity='FAIR')
+        e1 = {'ENTRYTYPE': 'article', 'ID': 'Key1', 'author': 'Smith', 'title': 'Paper A', 'doi': '10.1/a', 'year': '2020'}
+        self._add_entry(b, e1)
+        self.assertEqual(len(b.entries), 1)
+        # Same doi, different author/title → FAIR duplicate → merge
+        e2 = {'ENTRYTYPE': 'article', 'ID': 'Key2', 'author': 'Jones', 'title': 'Paper B', 'doi': '10.1/a', 'year': '2021'}
+        self._add_entry(b, e2)
+        self.assertEqual(len(b.entries), 1, 'FAIR: duplicate (same doi) should be merged')
+        # Different doi and different authortitle → not duplicate
+        e3 = {'ENTRYTYPE': 'article', 'ID': 'Key3', 'author': 'Lee', 'title': 'Other', 'doi': '10.1/b', 'year': '2020'}
+        self._add_entry(b, e3)
+        self.assertEqual(len(b.entries), 2, 'FAIR: non-duplicate should be added')
+
+    def test_partial_duplicate_merged_non_duplicate_added(self):
+        b = Biblio(similarity='PARTIAL')
+        e1 = {'ENTRYTYPE': 'article', 'ID': 'Key1', 'author': 'Smith', 'title': 'Same title', 'doi': '10.1/a', 'year': '2020'}
+        self._add_entry(b, e1)
+        self.assertEqual(len(b.entries), 1)
+        # Same authortitle, different doi → PARTIAL duplicate → merge
+        e2 = {'ENTRYTYPE': 'article', 'ID': 'Key2', 'author': 'Smith', 'title': 'Same title', 'doi': '10.1/b', 'year': '2021'}
+        self._add_entry(b, e2)
+        self.assertEqual(len(b.entries), 1, 'PARTIAL: duplicate (same authortitle) should be merged')
+        # Different doi and different authortitle → not duplicate
+        e3 = {'ENTRYTYPE': 'article', 'ID': 'Key3', 'author': 'Jones', 'title': 'Other', 'doi': '10.1/c', 'year': '2020'}
+        self._add_entry(b, e3)
+        self.assertEqual(len(b.entries), 2, 'PARTIAL: non-duplicate should be added')
+
+    def test_fuzzy_duplicate_merged_non_duplicate_added(self):
+        b = Biblio(similarity='FUZZY')
+        # Same authortitle → GOOD duplicate (score 103 >= 100) → merge. Exercises FUZZY full-scan path.
+        e1 = {'ENTRYTYPE': 'article', 'ID': 'Key1', 'author': 'Smith, John', 'title': 'Climate change impacts', 'doi': '', 'year': '2020'}
+        self._add_entry(b, e1)
+        self.assertEqual(len(b.entries), 1)
+        e2 = {'ENTRYTYPE': 'article', 'ID': 'Key2', 'author': 'Smith, John', 'title': 'Climate change impacts', 'doi': '', 'year': '2021'}
+        self._add_entry(b, e2)
+        self.assertEqual(len(b.entries), 1, 'FUZZY: duplicate (same authortitle) should be merged')
+        # Clearly different → not duplicate
+        e3 = {'ENTRYTYPE': 'article', 'ID': 'Key3', 'author': 'Jones', 'title': 'Unrelated topic', 'doi': '10.1/x', 'year': '2020'}
+        self._add_entry(b, e3)
+        self.assertEqual(len(b.entries), 2, 'FUZZY: non-duplicate should be added')
+
+    def test_batch_add_with_index_same_result_as_single_add(self):
+        """Index optimization (add_bibtex) must yield same duplicate result as single inserts."""
+        bib_with_dup = """@article{Key1,
+ author = {Smith},
+ title = {Same},
+ doi = {10.1/a},
+ year = {2020}
+}
+@article{Key2,
+ author = {Smith},
+ title = {Same},
+ doi = {10.1/a},
+ year = {2021}
+}
+"""
+        b = Biblio(similarity='PARTIAL')
+        b.add_bibtex(bib_with_dup, check_duplicate=True, on_conflict='u')
+        # Key1 and Key2 are GOOD/PARTIAL duplicates (same doi, same authortitle) → merged to one
+        self.assertEqual(len(b.entries), 1, 'add_bibtex with duplicate pair should merge (index path)')
+        # Same bib added one-by-one should also merge
+        b2 = Biblio(similarity='PARTIAL')
+        db = bp_parse_string(bib_with_dup)
+        for e in db.entries:
+            b2.insert_entry(e, check_duplicate=True, on_conflict='u')
+        self.assertEqual(len(b2.entries), 1, 'single inserts with duplicate pair should merge (no-index path)')
+
+
 class TestDuplicates(TestDuplicatesPartial):
 
     @staticmethod