papers-cli 2.9.1__tar.gz → 2.11__tar.gz

{papers_cli-2.9.1 → papers_cli-2.11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: papers-cli
-Version: 2.9.1
+Version: 2.11
 Summary: utilities to keep your PDF library organized
 Author-email: Mahé Perrette <mahe.perrette@gmail.com>
 License: MIT

{papers_cli-2.9.1 → papers_cli-2.11}/papers/__main__.py

@@ -19,7 +19,7 @@ from papers.extract import fetch_bibtex_by_doi
 from papers.encoding import parse_file, format_file, family_names, format_entries, standard_name, format_entry, parse_keywords, format_key
 from papers.config import bcolors, Config, search_config, CONFIG_FILE, CONFIG_FILE_LOCAL, DATA_DIR, CONFIG_FILE_LEGACY, BACKUP_DIR
 from papers.duplicate import list_duplicates, list_uniques, edit_entries
-from papers.bib import Biblio, FUZZY_RATIO, DEFAULT_SIMILARITY, entry_filecheck, backupfile as backupfile_func, isvalidkey
+from papers.bib import Biblio, FUZZY_RATIO, DEFAULT_SIMILARITY, entry_filecheck, backupfile as backupfile_func, isvalidkey, DuplicateKeyError
 from papers.utils import move, checksum, view_pdf, open_folder
 from papers import __version__
 
@@ -538,30 +538,45 @@ def addcmd(parser, o, config):
 
     biblio = get_biblio(config)
 
-    kw = {'on_conflict':o.mode, 'check_duplicate':not o.no_check_duplicate,
-            'mergefiles':not o.no_merge_files, 'update_key':o.update_key}
-
     entries = []
 
+    metadata = {k: v for k, v in o.metadata or []}
+    if o.doi: metadata['doi'] = o.doi
+    if o.key: metadata['ID'] = o.key
+    if o.title: metadata['title'] = o.title
+    if o.author: metadata['author'] = o.author
+    if o.journal: metadata['journal'] = o.journal
+    if o.year: metadata['year'] = o.year
+    if o.type: metadata['ENTRYTYPE'] = o.type
+    if "author" in metadata:
+        metadata["author"] = standard_name(metadata["author"])
+    if o.attachment:
+        metadata['file'] = format_file(biblio.get_files(metadata) + o.attachment, relative_to=biblio.relative_to)
+
     if len(o.file) > 1:
-        if o.attachment:
-            logger.error('--attachment is only valid for one PDF / BIBTEX entry')
+        if metadata:
+            logger.error('--doi, --metadata, --key, --attachment and other metadata keys are only valid for one PDF / BIBTEX entry')
             raise PapersExit()
-        if o.doi:
-            logger.error('--doi is only valid for one added file')
+
+    kw = {'on_conflict':o.mode, 'check_duplicate':not o.no_check_duplicate,
+            'mergefiles':not o.no_merge_files, 'update_key':o.update_key, 'metadata':metadata}
+
+    if len(o.file) == 0 and o.doi and not o.no_query_doi:
+        entries.extend( biblio.fetch_doi(o.doi, attachments=o.attachment, rename=o.rename, copy=o.copy, **kw) )
+
+    elif len(o.file) == 0:
+        if not o.edit and not o.metadata and not o.key and not o.doi and not o.attachment and not o.title and not o.author and not o.journal and not o.year:
+            logger.error("No entry added: use --doi, --metadata, --key, --attachment, --title, --author, --journal, --year to add a new entry")
             raise PapersExit()
 
-    if len(o.file) == 0:
+        metadata.setdefault('ID', biblio.generate_key(metadata))
+        metadata.setdefault('ENTRYTYPE', 'article')
         if o.edit:
-            pass
-        elif not o.doi:
-            logger.error('Please provide either a PDF file or BIBTEX entry or specify `--doi DOI` or `--edit`')
-            raise PapersExit()
-        elif o.no_query_doi:
-            logger.error('If no file is present, --no-query-doi is not compatible with --doi')
-            raise PapersExit()
-        else:
-            entries.extend( biblio.fetch_doi(o.doi, attachments=o.attachment, rename=o.rename, copy=o.copy, **kw) )
+            metadata = edit_entries(metadata)
+            o.edit = False
+
+        entries.extend( biblio.insert_entry(metadata, **kw) )
+
 
     for file in o.file:
         try:
@@ -601,30 +616,18 @@ def addcmd(parser, o, config):
         entries = [e for e in biblio.entries if biblio.key(e) in unique_keys]
 
     if o.edit:
-        if len(entries) == 0:
-            entries = [{
-                "ID": "EDIT-NEW-KEY",
-                "ENTRYTYPE": "article",
-                "author": standard_name("John Doe and Jane Roe"),
-                "title": "",
-                "journal": "",
-                "year": "",
-                "doi": o.doi or "",
-                "file": format_file(o.attachment or [], relative_to=biblio.relative_to),
-                }]
+        entry_keys = [biblio.key(e) for e in entries]
+        otherentries = [e for e in biblio.entries if biblio.key(e) not in entry_keys]
 
         try:
-            otherentries = [e for e in biblio.entries if e not in entries]
             entries = edit_entries(entries)
-            entries = [{k:v for k,v in e.items() if v != ""} for e in entries]
-            biblio.db.entries = otherentries
-            for e in entries:
-                biblio.insert_entry(e, **kw)
-
         except Exception as error:
             logger.error(str(error))
             return
 
+        entries = [{k:v for k,v in e.items() if v != ""} for e in entries]
+        biblio.db.entries = otherentries + entries
+
     savebib(biblio, config)
 
     if o.open:
@@ -888,7 +891,6 @@ def statuscmd(parser, o, config):
     print(config.status(check_files=not o.no_check_files, verbose=o.verbose))
 
 
-
 def get_parser(config=None):
     if config is None:
         config = papers.config.Config()
@@ -1022,8 +1024,15 @@ def get_parser(config=None):
     grp.add_argument('--ignore-errors', action='store_true',
         help='ignore errors when adding multiple files')
 
-    grp = addp.add_argument_group('pdf metadata')
+    grp = addp.add_argument_group('metadata')
     grp.add_argument('--doi', help='provide DOI -- skip parsing PDF')
+    grp.add_argument('--key', help='manually set the key')
+    grp.add_argument('--type', help='document type')
+    grp.add_argument('--title', help='manually set the title')
+    grp.add_argument('--author', help='manually set the author')
+    grp.add_argument('--journal', help='manually set the journal')
+    grp.add_argument('--year', help='manually set the year')
+    grp.add_argument('--metadata', nargs="+", metavar="KEY=VALUE", type=lambda meta: meta.split('=', 1), help='the metadata fields manually')
     grp.add_argument('--no-query-doi', action='store_true', help='do not attempt to parse and query doi')
     grp.add_argument('--no-query-fulltext', action='store_true', help='do not attempt to query fulltext in case doi query fails')
     grp.add_argument('--scholar', action='store_true', help='use google scholar instead of crossref')
@@ -1298,7 +1307,7 @@ class PapersExit(Exception):
 def main_clean_exit(args=None):
     try:
         main(args)
-    except PapersExit as error:
+    except (PapersExit, DuplicateKeyError) as error:
         if logger.getEffectiveLevel() == logging.DEBUG:
             raise
         if error.args:

{papers_cli-2.9.1 → papers_cli-2.11}/papers/_version.py

@@ -12,5 +12,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '2.9.1'
-__version_tuple__ = version_tuple = (2, 9, 1)
+__version__ = version = '2.11'
+__version_tuple__ = version_tuple = (2, 11)

{papers_cli-2.9.1 → papers_cli-2.11}/papers/bib.py

@@ -279,9 +279,16 @@ class Biblio:
         return bisect.bisect_left(keys, self.key(entry))
 
 
-    def insert_entry(self, entry, update_key=False, check_duplicate=False, rename=False, copy=False, **checkopt):
+    def insert_entry(self, entry, update_key=False, check_duplicate=False, rename=False, copy=False, metadata={}, **checkopt):
         """
         """
+        if metadata:
+            files = self.get_files(entry) + self.get_files(metadata)
+            self.set_files(metadata, files)
+
+        if update_key:
+            entry['ID'] = self.generate_key(entry)
+
         # additional checks on DOI etc...
         if check_duplicate:
             logger.debug('check duplicates : TRUE')
@@ -298,15 +305,16 @@ class Biblio:
                 newkey = self.append_abc_to_key(entry)  # add abc
                 logger.info('update key: {} => {}'.format(entry['ID'], newkey))
                 entry['ID'] = newkey
-                print(format_entry(self, entry, prefix="Updated"))
 
             else:
-                raise DuplicateKeyError('this error can be avoided if update_key is True')
+                print(format_entry(self, entry, prefix="New entry"))
+                print(format_entry(self, self.entries[i], prefix="Conflicts with"))
+                raise DuplicateKeyError('Key conflict. Try update_key is True?')
 
         else:
             logger.info('new entry: '+self.key(entry))
-            print(format_entry(self, entry, prefix="Added"))
 
+        print(format_entry(self, entry, prefix="Added"))
         self.entries.insert(i, entry)
 
         if rename: self.rename_entry_files(entry, copy=copy)
@@ -413,8 +421,10 @@ class Biblio:
 
     def fetch_doi(self, doi, **kw):
         bibtex = fetch_bibtex_by_doi(doi)
+        kw["update_key"] = True  # fetched key is always updated
         return self.add_bibtex(bibtex, **kw)
 
+
     def add_pdf(self, pdf, attachments=None, search_doi=True, search_fulltext=True, scholar=False, doi=None, **kw):
 
         if doi:
@@ -436,9 +446,8 @@ class Biblio:
         entry['ID'] = self.generate_key(entry)
         logger.debug('generated PDF key: '+entry['ID'])
 
-        kw.pop('update_key', True)
-            # logger.warning('fetched key is always updated when adding PDF to existing bib')
-        return self.insert_entry(entry, update_key=True, **kw)
+        kw["update_key"] = False  # already updated
+        return self.insert_entry(entry, **kw)
 
 
     def scan_dir_iter(self, direc, search_doi=True, search_fulltext=True, **kw):

{papers_cli-2.9.1 → papers_cli-2.11}/papers/duplicate.py

@@ -341,7 +341,6 @@ def choose_entry_interactive(entries, extra=[], msg='', select=False, best=None)
     return _ask_pick_loop(entries, extra, select)
 
 
-
 def edit_entries(entries, diff=False, ndiff=False, editor=None):
     '''edit entries and insert result in database
     '''
@@ -367,12 +366,18 @@ def edit_entries(entries, diff=False, ndiff=False, editor=None):
 
     if res == 0:
         logger.info('sucessfully edited file, insert edited entries')
-        db = bibtexparser.loads(open(filename).read())
-        return db.entries
-
-        raise ValueError('error when editing entries file: '+filename)
-
+        db_updated = bibtexparser.loads(open(filename).read())
+        keys_before = [e.get('ID') for e in entries]
+        keys_after = [e.get('ID') for e in db_updated.entries]
+        if set(keys_before) != set(keys_after):
+            print(f"{bcolors.WARNING}Keys edited ->", bcolors.OKBLUE ," ".join(sorted(set(keys_before).difference(keys_after))), bcolors.ENDC,
+                  "=>", bcolors.OKBLUE,
+                  " ".join(sorted(set(keys_after).difference(keys_before))), bcolors.ENDC) if keys_after else '(deleted)'
+        return db_updated.entries
 
+    else:
+        logger.info('failed to edit file')
+        return db
 
 
 def score(e):

{papers_cli-2.9.1 → papers_cli-2.11}/papers/extract.py

@@ -14,7 +14,7 @@ from papers.encoding import family_names
 from bibtexparser.customization import convert_to_unicode
 
 my_etiquette = Etiquette('papers', papers.__version__, 'https://github.com/perrette/papers', 'mahe.perrette@gmail.com')
-
+work = Works(etiquette=my_etiquette)
 
 class DOIParsingError(ValueError):
     pass
@@ -208,27 +208,29 @@ def extract_pdf_metadata(pdf, search_doi=True, search_fulltext=True, maxpages=10
     txt = pdfhead(pdf, maxpages, minwords, image=image)
     return extract_txt_metadata(txt, search_doi, search_fulltext, **kw)
 
-
-
-@cached('crossref-bibtex.json')
-def fetch_bibtex_by_doi(doi):
-    url = "http://api.crossref.org/works/"+doi+"/transform/application/x-bibtex"
-    work = Works(etiquette=my_etiquette)
+@cached('crossref.json')
+def fetch_crossref_by_doi(doi):
+    url = "http://api.crossref.org/works/"+doi
     response = work.do_http_request('get', url, custom_header={'user-agent': str(work.etiquette)})
-    if response.ok:
-        bibtex = response.text.strip()
-        return bibtex
-    raise DOIRequestError(repr(doi)+': '+response.text)
+    try:
+        response.raise_for_status()
+    except Exception as error:
+        raise DOIRequestError(repr(doi)+': '+repr(error))
+    return response.json()
 
+# @cached('crossref-bibtex.json')
+def fetch_bibtex_by_doi(doi):
+    json_data = fetch_crossref_by_doi(doi)
+    return crossref_to_bibtex(json_data['message'])
 
-@cached('crossref.json')
+@cached('crossref-json.json')
 def fetch_json_by_doi(doi):
     url = "http://api.crossref.org/works/"+doi+"/transform/application/json"
-    work = Works(etiquette=my_etiquette)
     jsontxt = work.do_http_request('get', url, custom_header={'user-agent': str(work.etiquette)}).text
     return jsontxt.dumps(json)
 
 
+
 def _get_page_fast(pagerequest):
     """Return the data for a page on scholar.google.com"""
     from scholarly import scholarly
@@ -267,11 +269,6 @@ def fetch_bibtex_by_fulltext_scholar(txt, assess_results=True):
 
     return scholarly.bibtex(result)
 
-
-def _crossref_get_author(res, sep='; '):
-    return sep.join([p.get('given','') + p['family'] for p in res.get('author',[]) if 'family' in p])
-
-
 def _crossref_score(txt, r):
     # high score means high similarity
     from rapidfuzz.fuzz import token_set_ratio
@@ -285,56 +282,104 @@ def _crossref_score(txt, r):
         score += token_set_ratio(r['abstract'], txt)
     return score
 
-
-def crossref_to_bibtex(r):
-    """convert crossref result to bibtex
-    """
-    bib = {}
-
-    if 'author' in r:
-        family = lambda p: p['family'] if len(p['family'].split()) == 1 else '{'+p['family']+'}'
-        bib['author'] = ' and '.join([family(p) + ', '+ p.get('given','')
-            for p in r.get('author',[]) if 'family' in p])
-
-    # for k in ['issued','published-print', 'published-online']:
-    k = 'issued'
-    if k in r and 'date-parts' in r[k] and len(r[k]['date-parts'])>0:
-        date = r[k]['date-parts'][0]
-        bib['year'] = str(date[0])
-        if len(date) >= 2:
-            bib['month'] = str(date[1])
-        # break
-
-    if 'DOI' in r: bib['doi'] = r['DOI']
-    if 'URL' in r: bib['url'] = r['URL']
-    if 'title' in r: bib['title'] = r['title'][0]
-    if 'container-title' in r: bib['journal'] = r['container-title'][0]
-    if 'volume' in r: bib['volume'] = r['volume']
-    if 'issue' in r: bib['number'] = r['issue']
-    if 'page' in r: bib['pages'] = r['page']
-    if 'publisher' in r: bib['publisher'] = r['publisher']
-
-    # entry type
-    type = bib.get('type','journal-article')
-    type_mapping = {'journal-article':'article'}
-    bib['ENTRYTYPE'] = type_mapping.get(type, type)
-
-    # bibtex key
-    year = str(bib.get('year','0000'))
-    if 'author' in r:
-        ID = r['author'][0]['family'] + '_' + year
+def map_crossref_to_bibtex_type(crossref_type):
+    mapping = {
+        "journal-article": "article",
+        "book": "book",
+        "book-chapter": "incollection",  # or sometimes inbook
+        "proceedings-article": "inproceedings",
+        "conference-paper": "inproceedings",
+        "report": "techreport",
+        "thesis": "phdthesis",  # or "mastersthesis" based on additional info
+    }
+    return mapping.get(crossref_type, "misc")
+
+
+def format_authors(authors):
+    """Converts a list of author dicts to a BibTeX-friendly string."""
+    author_list = []
+    for author in authors:
+        given = author.get("given", "")
+        family = author.get("family", "")
+        # Format as "Family, Given" for BibTeX
+        formatted = f"{family}, {given}" if family else given
+        # author_list.append(unidecode(formatted))
+        author_list.append(formatted)
+    return " and ".join(author_list)
+
+
+def crossref_to_bibtex(message):
+    entry_type = message.get('type', 'misc')  # Default to 'misc' if type is not specified
+
+    # Common fields
+    bib_entry = {
+        'title': message.get('title', [''])[0],
+        'author': format_authors(message.get('author', [])),
+        'doi': message.get('DOI', ''),
+        'url': message.get('URL', ''),
+        'ENTRYTYPE': map_crossref_to_bibtex_type(entry_type),
+        'ID': message.get('DOI', '')
+    }
+
+    # Year: Look for publication date in several possible places
+    date_parts = None
+    for date_field in ["published-print", "published-online", "issued"]:
+        if date_field in message and "date-parts" in message[date_field]:
+            date_parts = message[date_field]["date-parts"][0]
+            break
+    if date_parts and len(date_parts) > 0:
+        bib_entry["year"] = str(date_parts[0])
     else:
-        ID = year
-    bib['ID'] = ID
-
+        bib_entry["year"] = "0000"
+
+    # Fields specific to entry types
+    if entry_type == 'journal-article':
+        bib_entry.update({
+            'journal': message.get('container-title', [''])[0],
+            'volume': message.get('volume', ''),
+            'number': message.get('issue', ''),
+            'pages': message.get('page', ''),
+        })
+    elif entry_type == 'book':
+        bib_entry.update({
+            'publisher': message.get('publisher', ''),
+            'address': message.get('publisher-location', ''),
+            'editor': format_authors(message.get('editor', [])),
+            'isbn': message.get('ISBN', [''])[0],
+        })
+
+
+    elif entry_type == 'proceedings-article':
+        bib_entry.update({
+            'booktitle': message.get('container-title', [''])[0],
+            'publisher': message.get('publisher', ''),
+            'editor': format_authors(message.get('editor', [])),
+        })
+    elif entry_type == 'report':
+        bib_entry.update({
+            'institution': message.get('institution', {}).get('name', ''),
+        })
+    elif entry_type == 'thesis':
+        bib_entry.update({
+            'school': message.get('institution', {}).get('name', ''),
+            'type': message.get('type', ''),
+        })
+
+    bib_entry = {k: v for k, v in bib_entry.items() if v}  # Remove empty fields
+
+    # Create a BibDatabase object
     db = bibtexparser.bibdatabase.BibDatabase()
-    db.entries.append(bib)
-    return bibtexparser.dumps(db)
+    db.entries = [bib_entry]
+
+    # Write to a BibTeX string
+    writer = bibtexparser.bwriter.BibTexWriter()
+    bibtex_str = writer.write(db)
+
+    return bibtex_str
 
 
 # @cached('crossref-bibtex-fulltext.json', hashed_key=True)
 def fetch_bibtex_by_fulltext_crossref(txt, **kw):
-    work = Works(etiquette=my_etiquette)
     logger.debug('crossref fulltext seach:\n'+txt)
 
     # get the most likely match of the first results
@@ -367,7 +412,6 @@ def fetch_bibtex_by_fulltext_crossref(txt, **kw):
     return crossref_to_bibtex(result).strip()
 
 
-
 def fetch_entry(e):
     if 'doi' in e and isvaliddoi(e['doi']):
         bibtex = fetch_bibtex_by_doi(e['doi'])

{papers_cli-2.9.1 → papers_cli-2.11}/papers_cli.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: papers-cli
-Version: 2.9.1
+Version: 2.11
 Summary: utilities to keep your PDF library organized
 Author-email: Mahé Perrette <mahe.perrette@gmail.com>
 License: MIT

{papers_cli-2.9.1 → papers_cli-2.11}/tests/common.py

@@ -77,23 +77,21 @@ def run(cmd, sp_cmd=None, **kw):
 def prepare_paper():
     pdf = downloadpdf('bg-8-515-2011.pdf')
     doi = '10.5194/bg-8-515-2011'
-    key = 'Perrette_2011'
+    key = '10.5194/bg-8-515-2011'
     newkey = 'perrette_yool2011'
     year = '2011'
-    bibtex = """@article{Perrette_2011,
+    bibtex = """@article{10.5194/bg-8-515-2011,
  author = {Perrette, M. and Yool, A. and Quartly, G. D. and Popova, E. E.},
  doi = {10.5194/bg-8-515-2011},
- issn = {1726-4189},
  journal = {Biogeosciences},
- month = {February},
  number = {2},
- pages = {515–524},
- publisher = {Copernicus GmbH},
+ pages = {515-524},
  title = {Near-ubiquity of ice-edge blooms in the Arctic},
- url = {http://dx.doi.org/10.5194/bg-8-515-2011},
+ url = {https://doi.org/10.5194/bg-8-515-2011},
  volume = {8},
  year = {2011}
-}"""
+}
+"""
 
     file_rename = "perrette_et_al_2011_near-ubiquity-of-ice-edge-blooms-in-the-arctic.pdf"
     # The above corresponds to --name-template "{authorX}_{year}_{title}"
@@ -104,20 +102,17 @@ def prepare_paper2():
     pdf = downloadpdf('esd-4-11-2013.pdf')
     si = downloadpdf('esd-4-11-2013-supplement.pdf')
     doi = '10.5194/esd-4-11-2013'
-    key = 'Perrette_2013'
+    key = '10.5194/esd-4-11-2013'
     newkey = 'perrette_landerer2013'
     year = '2013'
-    bibtex = """@article{Perrette_2013,
+    bibtex = """@article{10.5194/esd-4-11-2013,
  author = {Perrette, M. and Landerer, F. and Riva, R. and Frieler, K. and Meinshausen, M.},
  doi = {10.5194/esd-4-11-2013},
- issn = {2190-4987},
  journal = {Earth System Dynamics},
- month = {January},
  number = {1},
- pages = {11–29},
- publisher = {Copernicus GmbH},
+ pages = {11-29},
  title = {A scaling approach to project regional sea level rise and its uncertainties},
- url = {http://dx.doi.org/10.5194/esd-4-11-2013},
+ url = {https://doi.org/10.5194/esd-4-11-2013},
  volume = {4},
  year = {2013}
 }"""
@@ -185,19 +180,18 @@ class BibTest(unittest.TestCase):
 
 
 
-bibtex = """@article{Perrette_2011,
- author = {M. Perrette and A. Yool and G. D. Quartly and E. E. Popova},
+bibtex = """@article{10.5194/bg-8-515-2011,
+ author = {Perrette, M. and Yool, A. and Quartly, G. D. and Popova, E. E.},
  doi = {10.5194/bg-8-515-2011},
  journal = {Biogeosciences},
- link = {https://doi.org/10.5194%2Fbg-8-515-2011},
- month = {feb},
  number = {2},
- pages = {515--524},
- publisher = {Copernicus {GmbH}},
+ pages = {515-524},
  title = {Near-ubiquity of ice-edge blooms in the Arctic},
+ url = {https://doi.org/10.5194/bg-8-515-2011},
  volume = {8},
  year = {2011}
-}"""
+}
+"""
 
 bibtex2 = """@article{SomeOneElse2000,
  author = {Some One},

{papers_cli-2.9.1 → papers_cli-2.11}/tests/test_add.py

@@ -305,7 +305,7 @@ class TestAddConflict(BibTest):
  title = {Something else entirely}
 }"""
 
-    bibtex_conflict_key_fixed = """@article{Perrette_2011b,
+    bibtex_conflict_key_fixed = """@article{perrette_author0000,
  author = {M. Perrette and Another author},
  doi = {10.5194/bg-8-515-2011XXX},
  title = {Something else entirely}

{papers_cli-2.9.1 → papers_cli-2.11}/tests/test_duplicates.py

@@ -266,7 +266,7 @@ class TestAddResolveDuplicate(BibTest):
 
     def test_conflict_updated_from_original_but_originalkey(self):
 
-        expected = """@article{Perrette_2011,
+        expected = """@article{fieldRareYear,
  author = {New Author Field},
  doi = {10.5194/bg-8-515-2011},
  journal = {ConflictJournal},