quarto-utils 1.0.0__py3-none-any.whl

qtils/examples/update_refs.py

@@ -0,0 +1,18 @@
+#!/usr/bin/python3
+import sys
+from qtils.utils import update_references
+import pathlib as pl
+
+for x in sys.argv:
+    if not isinstance(x, str):
+        print("Inputs must be strings")
+        sys.exit(0)
+        
+# choose a paper!
+qmdfile = pl.Path(sys.argv[0])
+bibfile = pl.Path(sys.argv[1])
+
+print(qmdfile)
+print(bibfile)
+
+update_references(qmdfile,bibfile,inplace=True)

qtils/examples/update_refs_example.py

@@ -0,0 +1,7 @@
+from qtils.utils import update_references
+import pathlib as pl
+
+# choose a paper!
+qmdfile = pl.Path('test.qmd')
+bibfile = pl.Path('tmp.bib')
+update_references(qmdfile, bibfile)

qtils/tests/data/update_test.py

@@ -0,0 +1,5 @@
+from qtils.utils import update_references
+update_references(DATA_DIR / 'example.dois.qmd',
+                    DATA_DIR / 'references.bib',
+                    False,
+                    None)

qtils/tests/test_all.py

@@ -0,0 +1,95 @@
+import pytest
+import pathlib as pl
+import urllib.request
+from urllib.error import HTTPError, URLError
+import shutil, os
+from text_unidecode import unidecode
+
+DATA_DIR = pl.Path('qtils/tests/data')
+
+def test_url():
+    doi = 'hTTps://doi.org/10.1038/nclimate2425'
+    BASE_URL = 'https://dx.doi.org/'
+    url = BASE_URL + doi.lower().replace('https://doi.org','').replace(' ','')
+    req = urllib.request.Request(url,
+            headers={'Accept': 'text/bibliography; style=bibtex'} )
+    
+    with urllib.request.urlopen(req) as f:
+            bibtex = f.read().decode()
+ 
+
+def test_getbib():
+    from qtils.utils import doi2bib
+    bib = '\n\t'.join(doi2bib('hTTps://doi.org/10.1038/nclimate2425'))
+    refbib = unidecode(''.join([i for i in open(DATA_DIR / 'one_entry.tst', 'r').readlines()]))
+    assert refbib == bib
+
+def test_update_bib():
+    from qtils.utils import update_bibfile
+    # make a copy to test with 
+    shutil.copy2(DATA_DIR / 'ref.bib_backup', 
+                DATA_DIR / 'ref.bib')
+    # set up dois to update with
+    dois = ['junkus fail','10.1111/gwat.12536',
+            '10.1111/gwat.13083', 
+            '10.3133/tm7C9',
+            '10.1007/978-3-030-45367-1_2',
+            'https://doi.org/10.1080/02626660009492371']
+    
+    # run the update - one should fail
+    update_bibfile(DATA_DIR / 'ref.bib', dois)
+    
+
+def test_doi_parser():
+    from qtils.utils import _strip_doi
+    qfile = DATA_DIR / 'example.dois.qmd'
+    dat = qfile.read_text().split('_doi:')
+    dois = [_strip_doi(i) for i in dat[1:]]
+    assert dois == ['10.1145/2492517.2500290',
+                    'hTTps://doi.org/10.1038/nclimate2425',
+                    '10.1080/14650040590946584',
+                    '10.1029/WR020i004p00415',
+                    'https://doi.org/10.1577/1548-8659(2001)130<0809:SVIDOL>2.0.CO;2',
+                    'https://doi.org/10.1890/1051-0761(2006)016[2035:BIRFUP]2.0.CO;2',
+                    'https://doi.org/10.1641/0006-3568(2000)050[0053:EAECON]2.3.CO;2',
+                    'https://doi.org/10.2193/0022-541X(2006)70[255:EAAIHE]2.0.CO;2']
+    
+def test_underscore_warning():
+    from qtils.utils import update_references
+    if (DATA_DIR / 'underscore.bib').exists():
+        os.remove(DATA_DIR / 'underscore.bib')
+    update_references(DATA_DIR / 'underscore_test.qmd',
+                DATA_DIR / 'underscore.bib',
+                False,
+                DATA_DIR / 'underscoreout.qmd')
+    
+    
+def test_dois_to_bibtex_qmd():
+    from qtils.utils import update_references
+    # make a copy to test with 
+    shutil.copy2(DATA_DIR / 'ref.bib_backup', 
+                DATA_DIR / 'references.bib')
+    
+    # try with automatic update of filename
+    update_references(DATA_DIR / 'example.dois.qmd',
+                    DATA_DIR / 'references.bib',
+                    False,
+                    None)
+    
+    # now try inplace test
+    shutil.copy2(DATA_DIR / 'ref.bib_backup', 
+                DATA_DIR / 'references.bib')
+    shutil.copy2(DATA_DIR / 'example.dois.qmd', 
+                DATA_DIR / 'inplace.qmd')
+    update_references(DATA_DIR / 'inplace.qmd',
+                DATA_DIR / 'references.bib',
+                True,
+                None)
+    
+    # and, finally try writing to custom filename
+    update_references(DATA_DIR / 'inplace.qmd',
+                DATA_DIR / 'references.bib',
+                False,
+                DATA_DIR / 'newfile.qmd')
+    
+    

qtils/utils.py

@@ -0,0 +1,159 @@
+import urllib.request
+from urllib.error import HTTPError, URLError
+import pathlib as pl 
+from text_unidecode import unidecode
+
+BASE_URL = 'http://dx.doi.org/'
+
+
+def doi2bib(doi='10.1016/j.jhydrol.2014.04.061', skip_url=True):
+    """function to return a bibTex entry for a doi string
+
+
+    Args:
+        doi (str, optional): string for doi. Defaults to '10.1016/j.jhydrol.2014.04.061'.
+        skip_url (bool, optional): flag for skipping the url entry from bibTex. 
+                                    Defaults to True
+    Returns:
+        list: list of lines for bibTex entry, skipping url if skip_url is True
+    """
+    url = BASE_URL + doi.lower().replace('https://doi.org','').replace(' ','')
+    req = urllib.request.Request(url)
+    req.add_header('Accept', 'application/x-bibtex')
+    try:
+        with urllib.request.urlopen(req) as f:
+            bibtex = unidecode(f.read().decode())
+        return bibtex.strip().split(None, 1)
+        # return [i for i in bibtex.split('\n') if not i.strip().startswith('url')]
+    except HTTPError as e:
+        print(e)
+        if e.code == 404:
+            print('DOI not found.')
+        else:
+            print('Service unavailable.')
+        return f'FAIL'
+    except URLError as e:
+        return e.reason.strerror
+
+def update_bibfile(bib_file, dois):
+    """updates an bibliography file with a new list of dois. If the bibliography
+        file doesn't exist, it is created.
+
+    Args:
+        bib_file (str): Name of bib file to update
+        dois (list): List of dois (strings) to add to bib_file
+    Returns:
+        bib_dict (dict): Dictionary with keys of value dois and values of entry
+                        records in the resulting bib file
+    """
+    # read in the existing bib file for updating
+    bib_file = pl.Path(bib_file)
+    if not bib_file.exists():
+        bib_file.touch(exist_ok = False)
+    inbib = ' '.join(open(bib_file, 'r', encoding='utf-8').readlines())
+    
+    # get the bibtex entries
+    with open(bib_file, 'a') as ofp:
+        bibs = [doi2bib(cdoi) for cdoi in dois]
+        for cdoi, cbib in zip(dois,bibs):
+            if cbib == 'FAIL':
+                # doi2bib failed - just warn and move on
+                print (f'Could not obtain bibTex entry for doi="{cdoi}"')
+            elif 'failed' in cbib:
+                print(f'Could not obtain bibTex entry for doi="{cdoi}". Error message from urllib:\n{cbib}')
+            elif cbib[0].strip() in inbib:
+                # entry already in the bib file
+                print(f'doi: "{cdoi}" already in {bib_file}. Skipping')
+            else:
+                # write the new entry into the bib file
+                print(f'Adding doi: "{cdoi}" to bib file')
+                ofp.write('\n' + '\n '.join(cbib))
+    # make a dictionary with the dois and the key for references
+    bib_dict = {}
+    for cdoi,i in zip(dois,bibs):
+        if i != 'FAIL' and 'failed' not in i:
+            bib_dict[cdoi] = i[0].split('{')[1].strip().replace(',','')
+    return bib_dict
+                
+def _strip_doi(doistring):
+    """Function to get the doi ID out of a string
+
+    Args:
+        doistring (_type_): string containing a doi
+    """
+    doistring = doistring.strip().replace("\t",' ').replace("\n",' ')
+    locbraks = locbrakp = locsep = locsepend = locspace = 1e6
+    locbackslash = loccomma = locbrakpars = locbracketend= 1e6
+    if "; " in doistring:
+        locsep = doistring.index('; ')
+    if doistring.endswith(';'):
+        locsepend = doistring.index(';')
+    if doistring.endswith(']'):
+        locbracketend = doistring.index(']')
+    if '] ' in doistring:
+        locbraks = doistring.index('] ')
+    if '].' in doistring:
+        locbrakp = doistring.index('].')
+    if '])' in doistring:
+        locbrakpars =  doistring.index('])')
+    if ' ' in doistring:
+        locspace = doistring.index(' ')
+    if ',' in doistring:
+        loccomma = doistring.index(',')
+    if "\\" in doistring:
+        locbackslash = doistring.index('\\')
+        
+    doilimit = min((locbraks, locbrakp, locsep, locsepend, locbrakpars,
+                    locspace, locbackslash, loccomma, locbracketend))    
+    return(doistring[:doilimit])
+    
+def _write_updated_qmd(qmd_file, qmd_text, inplace=False, new_qmd_file=None):
+    if inplace == True:
+        with open(qmd_file, 'w') as ofp:
+            ofp.write(qmd_text)
+    elif new_qmd_file is None:
+        with open(pl.Path(f'{qmd_file.parent}/{qmd_file.stem}.updated.qmd'), 'w') as ofp:
+            ofp.write(qmd_text)
+    else:
+        with open(new_qmd_file, 'w') as ofp:
+            ofp.write(qmd_text)        
+
+def update_references(qmd_file, bibfile,
+                inplace=False, new_qmd_file=None):
+    """Updates an entire qmd file, replacing doi callouts (which must
+    be listed as `_doi:<actual_doi_here>` in the text) with references
+    to the updated bibliography file.
+
+    Args:
+        qmd_file (str,): input qmd file. 
+        bibfile (str, optional): original bib file.
+        inplace (bool, optional): Option to replace existing file or to save as a copy.
+            If False, and if new_qmd_file is provided, the new file is written to new_qmd_file.
+            If False and no new_wmd_file value is provided, the new file is written to a file
+            that simply updates qmd_file to include "updated" in the filename. Defaults to True.
+        new_qmd_file (_type_, optional): New file to which updated qmd file is written. 
+            Ignored if inplace==True. Defaults to None.
+    """
+    # make sure the paths are legit pathlib objects
+    qmd_file = pl.Path(qmd_file)
+    if new_qmd_file is not None:
+        new_qmd_file = pl.Path(new_qmd_file)
+    # first find the dois
+    qmd_text = qmd_file.read_text(encoding='utf-8')
+    if '\_doi' in qmd_text:
+        doiflag = '\_doi'
+    else:
+        doiflag = '_doi'
+    dois = qmd_text.split('_doi:')
+    if len(dois)>1:
+        dois = [_strip_doi(i) for i in dois[1:]]
+        # next update the references
+        bib_dict = update_bibfile(bibfile, dois)
+        for ck,cv in bib_dict.items():
+            qmd_text = qmd_text.replace(f'{doiflag}:{ck}',f'@{cv}')        
+    else:
+        print('no dois found. No updates to make.')
+
+    # finally write out the updated file
+    _write_updated_qmd(qmd_file, qmd_text, 
+                inplace=inplace, new_qmd_file=new_qmd_file)

quarto_utils-1.0.0.dist-info/METADATA

@@ -0,0 +1,130 @@
+Metadata-Version: 2.4
+Name: quarto-utils
+Version: 1.0.0
+Summary: Utilities to update references in Quarto using DOIs
+Home-page: https://github.com/mnfienen/quarto-utils
+Author: Michael N. Fienen, Richard A. Erickson
+Author-email: Michael Fienen <mnfienen@usgs.gov>, Richard Erickson <rerickson@usgs.gov>
+Maintainer-email: Michael Fienen <mnfienen@usgs.gov>
+Project-URL: documentation, https://github.com/DOI-USGS/quarto-utils
+Project-URL: repository, https://github.com/DOI-USGS/quarto-utils
+Keywords: quarto,references,doi
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Science/Research
+Classifier: Programming Language :: Python :: 3 :: Only
+Requires-Python: >=3.7
+Description-Content-Type: text/markdown
+License-File: LICENSE.md
+Requires-Dist: requests
+Requires-Dist: jupyter
+Requires-Dist: notebook
+Requires-Dist: ipykernel
+Requires-Dist: text-unidecode
+Requires-Dist: pytest
+Provides-Extra: test
+Requires-Dist: pytest; extra == "test"
+Requires-Dist: pytest-timeout; extra == "test"
+Dynamic: author
+Dynamic: home-page
+Dynamic: license-file
+Dynamic: requires-python
+
+# quarto-utils
+
+**Package note:** A static copy of this project exists on code.usgs.gov as the official release.
+The dynamic, living version of this project exists on GitHub at https://github.com/DOI-USGS/quarto-utils.
+The code.usgs.gov page will be updated with major releases with active development occuring on the GitHub repo.
+
+#### Authors:          Michael N. Fienen, Richard A. Erickson
+#### Point of contact: Michael N. Fienen (mnfienen@usgs.gov)
+#### Repository Type:  Formal Python package
+#### Year of Origin:   2024 (original publication)
+#### Year of Version:  2024
+#### Version:          1.0.0 
+#### Digital Object Identifier (DOI): https://doi.org/10.5066/P1GZPONT
+#### USGS Information Product Data System (IPDS) no.: IP-158658
+
+***
+
+_Suggested Citation:_
+
+Fienen MN, Erickson RA. 2024.
+quarto-utils
+U.S. Geological Survey software release. Reston, Va.,
+https://doi.org/10.5066/P1GZPONT.
+
+_Authors' [ORCID](https://orcid.org) nos.:_
+
+- Michael N. Fienen, [0000-0002-7756-4651](https://orcid.org/0000-0002-7756-4651)
+- Richard A. Erickson, [0000-0003-4649-482X](https://orcid.org/0000-0003-4649-482X)
+
+
+***
+***
+
+This python package works with Quarto MarkDown files to allow authors to supply `dois` for references while writing a document. Tools in this utilities package provide the ability to automatically query https://doi.org to return valid `BiBTeX` entries from `dois`. 
+
+Further utilities allow users to supply a list of `dois`, look up the `BiBTeX` entries, and update them into a new or existing `BiBTeX` file.
+
+Finally, this package allows authors to compose Quarto MarkDown documents (`.qmd` files) using tagged `dois` as references. From these, the code replaces such `dois` with valid `BiBTeX` references which are also downloaded into a `.bib` file.
+
+The jupyter notebook in `/qtils/examples/examples.ipynb` includes example applications of all the functions in the package.
+
+# Getting Started
+
+This code will work with python 3.10 and greater with `requests` and `text-unidecode` packages (optional additional packages include `jupyter notebook` to run the notebook example and `pytest` to evaluate the tests).
+
+To install, download this repo (such as `git clone ...`) and then run `python setup.py install` (or possibly `python3` depending upon your local path configurations).
+
+# SSL certificates
+
+Department of Interior users may experience SSL certificate errors due to The site https://code.usgs.gov/usgs/best-practices/-/blob/master/ssl/WorkingWithinSSLIntercept.md?ref_type=heads provides some best practices for working with these security settings.
+
+# Repository Files
+
+This repository contains a Quarto-based template for the _Journal of Fish and Wildlife Management_.
+To support this, the following files are located here:
+
+- `README.md` is this file.
+- `LICENSE.md` is the Official USGS License. 
+- `code.json` is the code metadata.
+- `CONTRIBUTING.md` describes how to contribute to this project.
+- `DISCLAIMER.md` is the standard USGS disclaimer.
+- `.gitignore` is a file telling git which files to not track.
+- `ci` a folder for automated testing of this repository and contains a `yaml` file.
+- `quarto-utils` a folder with the Python package an examples. This includes an `examples` folder and `test` folder.
+- `setup.py` is a Python file for the package.
+
+# Background knowledge
+
+This template assumes the user knows, or at least wants to learn, how to use Markdown-based text programs such as Quarto.
+Knowing basic Markdown commands will help.
+Additionally, knowing LaTeX will assist in helping with advanced formatting.
+
+# Explanation of examples and other tips
+
+The `test.qmd` and `example.dois.qmd` include references that Fienen created to demonstrate the package. 
+The content of the example is not meaningful, but the references are real - just inasmcuh as they can be pulled from the web.
+
+`test.qmd` becomes `test.updated.qmd` after processing and the second file should be rendered as a Quarto file.
+
+References in a bib files often require manual formatting.
+For example, proper nouns like Mississippi River may need an extra `{}` to be capitalized, for example `title  = {{Mississippi River:} my story on {Old Man River}}` to keep "Mississippi River" and "Old Man River capitalized.
+The automatic references also often do not have correct capitalization and these may need to be changed by the user.
+
+# Run time
+
+This code takes minimal run time (< 1 minute) under most situations.
+
+# Known Issues
+
+- In some cases, special characters (such as em-dash or en-dash) in BiBTeX will be encoded as short character strings when written to the bib file. These characters will be seen on render or may crash quarto rendering. Users are advised to inspect references after they are downloaded.
+- Some DOIs contain the underscore character (`_`) and this cause this package to not find any new DOIs or otherwise fail. If this occurs, use `\_doi` rather than `_doi` to tag your DOIs in the Quarto file.
+
+# Acknowledgments
+
+We thank the U.S. Geological Survey Biological Threats and Invasive Species Research Program and U.S. Geological Survey Water Mission Area Integrated Information Dissemination Division for funding.
+Any use of trade, firm, or product names is for descriptive purposes only and does not imply endorsement by the U.S. Government.
+The findings and opinions expressed in this manuscript are those of the authors and do not necessarily represent the views of the US Fish and Wildlife Service.
+
+[quarto]: https://quarto.org/

quarto_utils-1.0.0.dist-info/RECORD

@@ -0,0 +1,12 @@
+qtils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+qtils/utils.py,sha256=C6aoCB9PO1wAIRUOdJ8sETSeNLajjyaahccuwJ5Dn94,6352
+qtils/examples/update_refs.py,sha256=qOzh4TUsw2Tm00w245QVixySOfu6sB1u2fi7_ATdJ8g,373
+qtils/examples/update_refs_example.py,sha256=xjXjwpjSAPXAhc3RfhsdoGRog3-JUp3jextVkdkm5nk,176
+qtils/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+qtils/tests/test_all.py,sha256=AiHUc4kL7r0lQOepXfaBNXI2XxuPp2DDvSeZ_F5v3a8,3425
+qtils/tests/data/update_test.py,sha256=G8W0a6OMPQEoqsALb762IijZggiYYoFrVycYG91eiT0,192
+quarto_utils-1.0.0.dist-info/licenses/LICENSE.md,sha256=j3-fCgUr1LmIPipq2I71L94P7PBedauuZ0xMegcuMEM,1650
+quarto_utils-1.0.0.dist-info/METADATA,sha256=pY_WfKKJKSVuTpqs7OHQxqx6CdAUVm_C2K3l1p2bab8,6603
+quarto_utils-1.0.0.dist-info/WHEEL,sha256=K260EYznzXsJYBQGqmI8VTxEdiZYNvDZwW9cBh9-_MA,91
+quarto_utils-1.0.0.dist-info/top_level.txt,sha256=RFzShkQWZhr-_XWihGGl46u4k4EbmyPh_tWFyxv0VBo,6
+quarto_utils-1.0.0.dist-info/RECORD,,

quarto_utils-1.0.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (83.0.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

quarto_utils-1.0.0.dist-info/licenses/LICENSE.md

@@ -0,0 +1,46 @@
+License
+=======
+
+Unless otherwise noted, This project is in the public domain in the United
+States because it contains materials that originally came from the United
+States Geological Survey, an agency of the United States Department of
+Interior. For more information, see the official USGS copyright policy at
+https://www.usgs.gov/information-policies-and-instructions/copyrights-and-credits
+
+Additionally, we waive copyright and related rights in the work
+worldwide through the CC0 1.0 Universal public domain dedication.
+
+
+CC0 1.0 Universal Summary
+-------------------------
+
+This is a human-readable summary of the
+[Legal Code (read the full text)][1].
+
+
+### No Copyright
+
+The person who associated a work with this deed has dedicated the work to
+the public domain by waiving all of his or her rights to the work worldwide
+under copyright law, including all related and neighboring rights, to the
+extent allowed by law.
+
+You can copy, modify, distribute and perform the work, even for commercial
+purposes, all without asking permission.
+
+
+### Other Information
+
+In no way are the patent or trademark rights of any person affected by CC0,
+nor are the rights that other persons may have in the work or in how the
+work is used, such as publicity or privacy rights.
+
+Unless expressly stated otherwise, the person who associated a work with
+this deed makes no warranties about the work, and disclaims liability for
+all uses of the work, to the fullest extent permitted by applicable law.
+When using or citing the work, you should not imply endorsement by the
+author or the affirmer.
+
+
+
+[1]: https://creativecommons.org/publicdomain/zero/1.0/legalcode

quarto_utils-1.0.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+qtils