flm-citations 0.2.4__tar.gz

flm_citations-0.2.4/LICENSE.txt

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2022 Philippe Faist
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

flm_citations-0.2.4/PKG-INFO

@@ -0,0 +1,19 @@
+Metadata-Version: 2.1
+Name: flm-citations
+Version: 0.2.4
+Summary: Support for citations in FLM (see flm package)
+License: MIT
+Author: Philippe Faist
+Requires-Python: >=3.8,<4.0
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Requires-Dist: PyYAML (>=6.0,<7.0)
+Requires-Dist: arxiv (>=1.4.2,<2.0.0)
+Requires-Dist: backoff (>=2.1.2,<3.0.0)
+Requires-Dist: citeproc-py (>=0.6.0,<0.7.0)
+Requires-Dist: flm-core (>=0.3.0a10)
+Requires-Dist: requests (>=2.28.1,<3.0.0)

flm_citations-0.2.4/flm_citations/__init__.py

@@ -0,0 +1,25 @@
+
+
+### FIXME: Merge feature config of imported configs don't work yet ... :/
+### Need to fix this in flm.main.run().  Probably all $import's should be
+### processed first, before merging anything.
+
+flm_default_import_config = {
+    'flm': {
+        'features': {
+            'flm_citations': {},
+            # {
+            #     'sources': [
+            #         {
+            #             '$defaults': True,
+            #         },
+            #     ],
+            # },
+        },
+    },
+}
+
+
+def FeatureClass(*args, **kwargs):
+    from .feature import FeatureClass as _FeatureClass
+    return _FeatureClass(*args, **kwargs)

flm_citations-0.2.4/flm_citations/_cslformatter.py

@@ -0,0 +1,72 @@
+import re
+
+_escape_chars = {
+    '\\': r'\textbackslash',
+    '%': r'\%',
+    '#': r'\#',
+    '&': r'\&',
+    '$': r'\$',
+    '{': r'\{',
+    '}': r'\}',
+}
+
+def _multiple_replace(s, rep_dict):
+    # https://stackoverflow.com/a/15448887/1694896
+    pattern = re.compile("|".join([
+        re.escape(k)
+        for k in sorted(rep_dict.keys(),key=len,reverse=True)
+    ]), flags=re.DOTALL)
+    return pattern.sub(lambda x: rep_dict[x.group(0)], s)
+
+def preformat(text):
+    # if isinstance(text, list): # ???
+    #     return [preformat(z) for z in text]
+    text = str(text)
+
+    s = _multiple_replace(text, _escape_chars)
+
+    return s
+
+
+class FLMCommandWrapper(str):
+    cmd = None
+
+    @classmethod
+    def _wrap(cls, text):
+        text = preformat(text)
+        return f'\\{cls.cmd}{{{text}}}'
+
+    def __new__(cls, text):
+        return super(FLMCommandWrapper, cls).__new__(cls, cls._wrap(text))
+
+
+class Italic(FLMCommandWrapper):
+    cmd = 'textit'
+
+
+class Oblique(Italic):
+    pass
+
+
+class Bold(FLMCommandWrapper):
+    cmd = 'textbf'
+
+
+class Light(FLMCommandWrapper):
+    cmd = 'textit' # italic instead
+
+
+class Underline(FLMCommandWrapper):
+    cmd = 'textit' # italic instead
+
+
+class Superscript(FLMCommandWrapper):
+    cmd = 'textit' # italic instead
+
+
+class Subscript(FLMCommandWrapper):
+    cmd = 'textit' # italic instead
+
+
+class SmallCaps(FLMCommandWrapper):
+    cmd = 'textit' # italic instead

flm_citations-0.2.4/flm_citations/american-physical-society-et-al--patched.csl

@@ -0,0 +1,169 @@
+<?xml version="1.0" encoding="utf-8"?>
+<style xmlns="http://purl.org/net/xbiblio/csl" class="in-text" version="1.0" demote-non-dropping-particle="sort-only" default-locale="en-US">
+  <info>
+    <title>American Physical Society - et al. (if more than 3 authors)</title>
+    <title-short>APS-et-al</title-short>
+    <id>http://www.zotero.org/styles/american-physical-society-et-al</id>
+    <link href="http://www.zotero.org/styles/american-physical-society-et-al" rel="self"/>
+    <link href="http://www.zotero.org/styles/american-physics-society" rel="template"/>
+    <link href="https://cdn.journals.aps.org/files/styleguide-pr.pdf" rel="documentation"/>
+    <author>
+      <name>Jonas Lähnemann</name>
+      <email>jonas@pdi-berlin.de</email>
+      <name>Philippe Faist</name><!-- Patched (Feb 2023) -->
+    </author>
+    <category citation-format="numeric"/>
+    <category field="physics"/>
+    <summary>Short variant (first author et al., if more than 3 authors) of the American Physical Society (APS) citation style. Volume is bold and year comes last in normal brackets.  Patched by Philippe Faist in Feb 2023.</summary>
+    <updated>2020-10-01T22:47:43+00:00</updated>
+    <rights license="http://creativecommons.org/licenses/by-sa/3.0/">This work is licensed under a Creative Commons Attribution-ShareAlike 3.0 License</rights>
+  </info>
+  <macro name="author">
+    <names variable="author">
+      <name delimiter=", " initialize-with=". " and="text"/>
+      <label form="long" prefix=", " suffix=" "/>
+      <!-- <et-al font-style="italic"/> -->
+      <substitute>
+        <names variable="editor"/>
+        <names variable="translator"/>
+      </substitute>
+    </names>
+  </macro>
+  <macro name="editor">
+    <names variable="editor">
+      <label form="verb" suffix=" "/>
+      <name delimiter=", " initialize-with=". " and="text"/>
+      <!-- <et-al font-style="italic"/> -->
+    </names>
+  </macro>
+  <macro name="year-date">
+    <choose>
+      <if variable="issued">
+        <date variable="issued">
+          <date-part name="year"/>
+        </date>
+      </if>
+      <else>
+        <!-- <text term="no date" form="short"/> -->
+      </else>
+    </choose>
+  </macro>
+  <macro name="day-date">
+    <choose>
+      <if variable="issued">
+        <date variable="issued">
+          <date-part name="day" suffix=" "/>
+          <date-part name="month" form="long" suffix=" "/>
+          <date-part name="year"/>
+        </date>
+      </if>
+      <else>
+        <text term="no date" form="short"/>
+      </else>
+    </choose>
+  </macro>
+  <macro name="publisher">
+    <group prefix="(" suffix=")" delimiter=", ">
+      <text variable="publisher"/>
+      <text variable="publisher-place" text-case="title"/>
+      <text macro="year-date"/>
+    </group>
+  </macro>
+  <macro name="edition">
+    <choose>
+      <if is-numeric="edition">
+        <group delimiter=" ">
+          <number variable="edition" form="ordinal"/>
+          <text term="edition" form="short"/>
+        </group>
+      </if>
+      <else>
+        <text variable="edition"/>
+      </else>
+    </choose>
+  </macro>
+  <citation collapse="citation-number">
+    <sort>
+      <key variable="citation-number"/>
+    </sort>
+    <layout delimiter="," prefix="&#160;[" suffix="]">
+      <text variable="citation-number"/>
+    </layout>
+  </citation>
+  <bibliography et-al-min="7" et-al-use-first="1" entry-spacing="0" second-field-align="flush">
+    <layout suffix=".">
+      <!-- PATCH Feb 2023 (Philippe Faist): remove citation number -->
+      <!-- <text variable="citation-number" prefix="[" suffix="]"/> -->
+      <group delimiter=", "><!-- added -->
+      <text macro="author" /> <!-- suffix=", "/> -->
+      <choose>
+        <if type="bill book graphic legal_case legislation motion_picture report song" match="any">
+          <group delimiter=" ">
+            <group delimiter=", ">
+              <text variable="title" text-case="title" font-style="italic"/>
+              <text macro="edition"/>
+            </group>
+            <group delimiter=", ">
+              <text macro="publisher"/>
+              <group delimiter=" ">
+                <label variable="page" form="short"/>
+                <text variable="page"/>
+              </group>
+            </group>
+          </group>
+        </if>
+        <else-if type="chapter paper-conference" match="any">
+          <group delimiter=" ">
+            <text term="in"/>
+            <group delimiter=", ">
+              <text variable="container-title" form="short" text-case="title" font-style="italic"/>
+              <text macro="editor"/>
+              <text macro="edition"/>
+            </group>
+            <group delimiter=", ">
+              <text macro="publisher"/>
+              <group delimiter=" ">
+                <label variable="page" form="short"/>
+                <text variable="page"/>
+              </group>
+            </group>
+          </group>
+        </else-if>
+        <else-if type="patent">
+          <group delimiter=" ">
+            <text variable="number"/>
+            <text macro="day-date" prefix="(" suffix=")"/>
+          </group>
+        </else-if>
+        <else-if type="thesis">
+          <group delimiter=", ">
+            <text variable="title" text-case="title"/>
+            <text variable="genre"/>
+            <text variable="publisher"/>
+            <text macro="year-date"/>
+          </group>
+        </else-if>
+        <else>
+          <group delimiter=" ">
+            <choose>
+              <if type="article-journal">
+                <text variable="container-title" form="short"/>
+              </if>
+              <else>
+                <text variable="container-title"/>
+              </else>
+            </choose>
+            <group delimiter=", ">
+              <text variable="volume" font-weight="bold"/>
+              <group delimiter=" ">
+                <text variable="page-first" form="short"/>
+                <text macro="year-date" prefix="(" suffix=")"/>
+              </group>
+            </group>
+          </group>
+        </else>
+      </choose>
+      </group><!-- added -->
+    </layout>
+  </bibliography>
+</style>

flm_citations-0.2.4/flm_citations/citesources/__init__.py

@@ -0,0 +1 @@
+

flm_citations-0.2.4/flm_citations/citesources/arxiv.py

@@ -0,0 +1,195 @@
+import re
+
+import logging
+logger = logging.getLogger(__name__)
+
+import arxiv
+import yaml
+
+from .base import CitationSourceBase
+
+
+_rx_arxiv_id_from_entryid = re.compile(
+    # note: <arxivid> does NOT include the version number
+    r'https?://arxiv.org/abs/(?P<arxivid>.*?)(?P<versionnumstr>v(?P<versionnum>\d+))?$',
+    flags=re.IGNORECASE
+)
+
+_rx_ver = re.compile(r'v\d+$')
+
+class CitationSourceArxiv(CitationSourceBase):
+
+    def __init__(self, **kwargs):
+
+        self.chain_to_doi = kwargs.get('chain_to_doi', True)
+
+        override_options = {
+            'chains_to_sources': ['doi'] if self.chain_to_doi else [],
+            'source_name': 'ArXiv API citation info source',
+        }
+        default_options = {
+            'chunk_query_delay_ms': 3100,
+            'cite_prefix': 'arxiv',
+            'use_requests': True,
+        }
+
+        super().__init__(
+            override_options,
+            kwargs,
+            default_options,
+        )
+
+        self.override_arxiv_dois_file = self.options.get('override_arxiv_dois_file', None)
+        self.override_arxiv_dois = self.options.get('override_arxiv_dois', {})
+
+        # silence some arxiv.arxiv messages
+        if not self.options.get('keep_arxiv_arxiv_logging_info_output', False):
+            logging.getLogger('arxiv.arxiv').setLevel(level=logging.WARNING)
+
+
+    def source_initialize_run(self):
+
+        self._arxiv_to_doi_override = {}
+        if self.override_arxiv_dois_file:
+            content = self.fetch_url(self.override_arxiv_dois_file)
+            data = yaml.safe_load(content)
+            self._arxiv_to_doi_override.update(data)
+
+        self._arxiv_to_doi_override.update(self.override_arxiv_dois)
+
+        self.data_for_versionless_arxivid = {
+            arxivid: []
+            for arxivid in self.cite_key_list
+            if not _rx_ver.search(arxivid)
+        }
+    
+    def retrieve_chunk(self, arxivid_list):
+
+        if not arxivid_list:
+            return {}
+
+        #
+        # fetch meta-info from the arxiv for all encountered arXiv IDs, and
+        # build the associated citation text endnotes.
+        #
+        big_slow_client = arxiv.Client(
+            page_size=len(arxivid_list) + 100,
+            delay_seconds=4,
+            num_retries=5
+        )
+        searchobj = arxiv.Search(
+            id_list=arxivid_list,
+        )
+        for result in big_slow_client.results(searchobj):
+            #
+            # build citation from the arxiv meta-information
+            #
+
+            m = _rx_arxiv_id_from_entryid.match(result.entry_id)
+            if m is None:
+                logger.warning(f"Unable to parse arXiv ID from {result.entry_id!r}")
+                continue
+
+            arxivid = m.group('arxivid').lower()
+            versionnum = m.group('versionnum')
+            arxivver = int(versionnum) if versionnum else None
+
+            logger.debug(
+                f"Processing received information for ‘{arxivid}’ (got v{arxivver})"
+            )
+
+            doi = None
+            if result.doi is not None and result.doi:
+                doi = str(result.doi)
+
+            # override the given DOI in exceptional cases where it might be
+            # missing, incomplete, or incorrect:
+            if arxivid in self._arxiv_to_doi_override:
+                override_doi = self._arxiv_to_doi_override[arxivid]
+                logger.debug(f"Overriding reported DOI (‘{doi}’) for arXiv "
+                             f"item ‘{arxivid}’ with manually specified DOI "
+                             f"‘{override_doi}’ given in citation hints")
+                doi = override_doi
+
+            # construct CSL-JSON entry from the retrieved metadata
+            citeprocjsond = {
+                'type': 'article-journal',
+                'title': result.title,
+                'author': [
+                    {'name': a.name}
+                    for a in result.authors
+                ],
+                'published': {
+                    'date-parts': [[
+                        result.published.year,
+                        result.published.month,
+                        result.published.day,
+                    ]],
+                },
+                'doi': doi,
+                'arxivid': arxivid,
+                'arxiv_version_number': arxivver,
+            }
+            
+            if arxivid in self.cite_key_list:
+                self.citation_manager.store_citation(self.cite_prefix, arxivid, citeprocjsond)
+
+            if arxivid in self.data_for_versionless_arxivid:
+                self.data_for_versionless_arxivid[arxivid].append( citeprocjsond )
+
+        return
+
+    def source_finalize_run(self):
+
+        # all the queries for ids ran, now figure out which versions to link to
+        # when a non-versioned key is provided
+
+        for arxivid, versionslist in self.data_for_versionless_arxivid.items():
+
+            if not versionslist:
+                logger.error(
+                    f"No arXiv data received for arXiv id ‘{arxivid}’, what happened?!?"
+                )
+                raise ValueError(f"No arXiv data found for ‘{arxivid}’")
+
+            best = None
+            for current in versionslist:
+                if best is None and current is not None:
+                    best = current
+                    continue
+                if current is None:
+                    continue
+                # "arxiv_version_number === null" only happens if arXiv API
+                # responded with an article whose ID didn't have a version
+                # number -> use that one directly as it is the answer to our
+                # query!
+                if best['arxiv_version_number'] is None:
+                    continue
+                if current['arxiv_version_number'] is None:
+                    best = current
+                    continue
+                if current['arxiv_version_number'] >= best['arxiv_version_number']:
+                    best = current
+                    continue
+
+            self._store_citation(arxivid, best)
+
+    def _store_citation(self, arxivid, csljsondata):
+        doi = csljsondata.get('doi', None)
+        if doi and self.chain_to_doi:
+            # chain to DOI
+            self.citation_manager.store_citation_chained(
+                self.cite_prefix, arxivid,
+                'doi', doi,
+                {
+                    'arxivid': arxivid,
+                },
+            )
+        else:
+            self.citation_manager.store_citation(self.cite_prefix, arxivid, csljsondata)
+
+
+
+
+# for when using shorthane naming
+CitationSourceClass = CitationSourceArxiv

flm_citations-0.2.4/flm_citations/citesources/base.py

@@ -0,0 +1,145 @@
+import time
+from urllib.parse import urlparse
+
+import logging
+logger = logging.getLogger(__name__)
+
+import requests
+
+
+class CitationSourceBase:
+
+    def __init__(self, override_options, kwargs, default_options):
+        super().__init__()
+
+        self.options = dict(default_options)
+        self.options.update(kwargs)
+        self.options.update(override_options)
+
+        self.chunk_size = self.options.get('chunk_size', 512)
+        self.chunk_query_delay_ms = self.options.get('chunk_query_delay_ms', 1000)
+
+        self.cite_prefix = self.options['cite_prefix']
+
+        self.chains_to_sources = self.options.get('chains_to_sources', [])
+        self.source_name = self.options.get('source_name', '<unknown source>')
+
+        self.requests_session = None
+        if self.options.get('use_requests', False):
+            if 'requests_session' in self.options:
+                self.requests_session = self.options['requests_session']
+            else:
+                self.requests_session = requests.Session()
+
+        self.cite_key_list = None
+
+        self.citation_manager = None
+
+    def set_citation_manager(self, citation_manager):
+        self.citation_manager = citation_manager
+
+
+    def retrieve_citations(self, cite_key_list):
+
+        self.cite_key_list = cite_key_list
+
+        self.source_initialize_run()
+
+        total_retrieved = 0
+        remaining_keys = list(cite_key_list)
+
+        last_chunk_query_monotonic_s = None
+
+        if len(remaining_keys):
+            logger.info(f"{self.source_name}: there are "
+                        f"{len(remaining_keys)} citation(s) to query")
+        else:
+            logger.debug(f"{self.source_name}: there are no citations to query")
+
+        while len(remaining_keys) > 0:
+
+            # if applicable, wait before another chunk query call
+            if (self.chunk_query_delay_ms and last_chunk_query_monotonic_s is not None):
+                dt_s = (time.monotonic() - last_chunk_query_monotonic_s)
+                dt_s_to_wait = (self.chunk_query_delay_ms/1000) - dt_s
+                if dt_s_to_wait > 0:
+                    time.sleep(dt_s_to_wait)
+
+            # create a chunk of IDs to query
+            chunk_ids, remaining_keys = \
+                remaining_keys[:self.chunk_size], remaining_keys[self.chunk_size:]
+
+            last_chunk_query_monotonic_s = time.monotonic()
+            self.retrieve_chunk(chunk_ids)
+            
+            # bookkeeping & logging
+            total_retrieved += len(chunk_ids)
+            logger.info(f"{self.source_name}: {total_retrieved}/{len(cite_key_list)}")
+
+        self.source_finalize_run()
+
+
+    # helper for fetching URLs
+    def fetch_url(self, url, binary=False, json=False, **kwargs):
+
+        if url is None:
+            raise ValueError(f"fetch_url(): url is None!")
+
+        logger.debug(f"Fetching URL ‘{url}’ ...")
+
+        p = urlparse(url)
+
+        if not p.scheme or p.scheme == 'file':
+            # Read a local file.
+            if binary:
+                open_args, open_kwargs = ('rb',), {}
+            else:
+                open_args, open_kwargs = ('r',), { 'encoding': 'utf-8' }
+            with open(p.path, *open_args, **open_kwargs) as f:
+                return f.read()
+
+        req_kwargs = {}
+
+        method = kwargs.pop('method', 'get')
+        reqfn = getattr(self.requests_session, method)
+
+        if method == 'post':
+            req_kwargs['data'] = kwargs.pop('body', None)
+
+        # e.g., headers etc.
+        req_kwargs.update(kwargs)
+
+        # fire!
+        r = reqfn(url)
+
+        if not r.ok:
+            # raise errors
+            r.raise_for_status()
+
+        if binary:
+            return r.content
+
+        r.encoding = 'utf-8'
+
+        if json:
+            return r.json()
+
+        return r.text
+
+
+
+    # -------------
+
+    # can be reimplemented
+
+    def source_initialize_run(self):
+        pass
+
+    def source_finalize_run(self):
+        pass
+
+
+    # must be reimplemented
+
+    def retrieve_chunk(self, chunk_keys):
+        raise RuntimeError("The method retrieve_chunk() must be reimplemented by subclasses!")