idlib 0.0.1.dev22__py2.py3-none-any.whl → 0.0.1.dev23__py2.py3-none-any.whl

idlib/__init__.py

@@ -55,4 +55,4 @@ def get_right_id(uri):
     return pi
 
 
-__version__ = '0.0.1.dev22'
+__version__ = '0.0.1.dev23'

idlib/apis/protocols_io.py

@@ -3,7 +3,6 @@ import sys
 import pickle
 from urllib import parse
 from getpass import getpass
-from ..config import auth
 from ..utils import log as _log
 from . import log as _log
 
@@ -68,11 +67,16 @@ def get_auth_code(url):
 
 
 def get_protocols_io_auth(creds_file,
-                          store_file=auth.get_path('protocols-io-api-store-file'),
-                          # yes reading from the store file here at the top level
-                          # increases startup time, however if it is burried then
-                          # any config error is deferred until later in runtime
+                          store_file=None,
                           SCOPES='readwrite'):
+    if store_file is None:
+        # since this file is imported whenever idlib is imported
+        # the time cost is prohibitive and we will just have to
+        # deal with the configuration value failing later in time
+        # by e.g. having a separate check config process
+        from ..config import auth
+        store_file = auth.get_path('protocols-io-api-store-file')
+
     from google_auth_oauthlib.flow import InstalledAppFlow
     from google.auth.transport.requests import Request
     from idlib import exceptions as exc

idlib/core.py

@@ -3,19 +3,45 @@ from . import exceptions as exc
 from .utils import log
 
 
-def resolution_chain(iri):
-    for head in resolution_chain_responses(iri):
+def resolution_chain(iri, headers_fun=None):
+    for head in resolution_chain_responses(iri, headers_fun=headers_fun):
         yield head.url
 
 
-def try_get(s, head):
+_user_agent_idiocy = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:136.0) Gecko/20100101 Firefox/136.0'
+
+
+def try_get(s, head, headers_fun=None):
     # see whether the server has a bad/broken support for head requests
     # sometimes they return e.g. 400 instead of 405, and really they should
     # just work because they work correctly with get request ...
     head = s.get(head.url, stream=True)
+
     if head.ok:
         log.info(f'bad HEAD implementation {head.url}')
-    else:
+    elif head.status_code < 500:
+        head.close()
+        headers = {'User-Agent': _user_agent_idiocy}
+        head = s.get(head.url, headers=headers, stream=True)
+        if head.ok:
+            log.info(f'bad HEAD implementation AND bad User-Agent behavior {head.url}')
+        else:
+            headers = None
+            if headers_fun is not None:
+                # headers_fun is used to ensure that e.g. auth headers
+                # are only sent to the netloc's they are for
+                headers = headers_fun(head.url)
+                if headers is not None:
+                    head = s.get(head.url, headers=headers, stream=True)
+                    if head.ok:
+                        # we log a warning because dereferencing a public id
+                        # and encountering a permission boundary is bad
+                        # TODO likely need a way to fail on this condition
+                        # or at least indicate that it happend in band
+                        msg = f'additional headers were added for {head.url}'
+                        log.warning(msg)
+
+    if not head.ok:
         content = head.content
         if content:
             if head.headers['Content-Type'] == 'application/json':
@@ -29,14 +55,34 @@ def try_get(s, head):
     return head
 
 
-def resolution_chain_responses(iri, raise_on_final=True):
+def resolution_raise_logic(head):
+    """ broken out into its own function for reuse to avoid re-resolving
+        in cases where we need to interpose additional operations between
+        a dereference failure and raising the error
+    """
+    if not head.ok:
+        try:
+            head.raise_for_status()
+        except Exception as e:
+            if head.status_code == 404:
+                msg = f'Nothing found due to {head.status_code} at {head.url}\n'
+                raise exc.IdDoesNotExistError(msg) from e  # often a permissions issue
+            elif head.status_code >= 500:
+                msg = f'Remote in error due to {head.status_code} at {head.url}\n'
+                raise exc.RemoteError(msg) from e
+            elif head.status_code >= 400:
+                msg = f'Nothing found due to {head.status_code} at {head.url}\n'
+                raise exc.ResolutionError(msg) from e
+
+
+def resolution_chain_responses(iri, raise_on_final=True, headers_fun=None):
     #doi = doi  # TODO
     s = requests.Session()
     head = s.head(iri, allow_redirects=False)
     if head.status_code < 400:
         yield head
     else:
-        head = try_get(s, head)
+        head = try_get(s, head, headers_fun=headers_fun)
         yield head
 
     while head.is_redirect and head.status_code < 400:  # FIXME redirect loop issue
@@ -49,21 +95,14 @@ def resolution_chain_responses(iri, raise_on_final=True):
         if head.status_code < 400:
             yield head
         else:
-            head = try_get(s, head)
+            head = try_get(s, head, headers_fun=headers_fun)
             yield head
 
         if not head.is_redirect:
             break
 
     if raise_on_final:  # we still want the chain ... null pointer error comes later?
-        if head.status_code == 404:
-            head.raise_for_status()  # probably a permissions issue
-        elif head.status_code >= 500:
-            msg = f'Remote in error due to {head.status_code} at {head.url}\n'
-            raise exc.RemoteError(msg)
-        elif head.status_code >= 400:
-            msg = f'Nothing found due to {head.status_code} at {head.url}\n'
-            raise exc.ResolutionError(msg)
+        resolution_raise_logic(head)
 
     if head.status_code >= 400:
         # XXX this seems like the "right" thing to do, but it will

idlib/from_oq.py

@@ -2,6 +2,7 @@ import re
 import json
 from time import time, sleep
 from datetime import datetime
+from urllib.parse import urlparse
 import orthauth as oa
 import ontquery as oq  # temporary implementation detail
 import idlib
@@ -20,6 +21,11 @@ from idlib.utils import (log,
 from idlib.config import auth
 
 
+def path_older_than_day(path):
+    ps = path.stat()
+    return time() - ps.st_ctime > (60 * 60 * 24)
+
+
 # from neurondm.simple
 class OntTerm(oq.OntTerm):
     """ ask things about a term! """
@@ -345,6 +351,13 @@ def setup(cls, creds_file=None):
             log.warning(e)
             cls._pio_header = None
 
+    def rchf(url):
+        purl = urlparse(url)
+        if purl.netloc.endswith('protocols.io'):
+            return cls._pio_header
+
+    cls._resolution_chain_headers_fun = staticmethod(rchf)
+
     if not hasattr(idlib.Stream, '_requests'):
         idlib.Stream.__new__(cls)
 
@@ -645,7 +658,7 @@ class Pio(formats.Rdf, idlib.Stream):
         return data
 
     _apiuri_private_cache = {}  # XXX FIXME hack
-    def data4(self, fail_ok=False):
+    def data4(self, fail_ok=False, try_refresh_cache=False):
         if not hasattr(self, '_data'):
             self._data_in_error = True
             if not isinstance(self._progenitors, dict):
@@ -661,7 +674,7 @@ class Pio(formats.Rdf, idlib.Stream):
             # however this is not trivial to retrive
             # the /v3/protocols/{id} endpoint returns bad/mangled data and we need
             # the /v1/ data to be able to actually render to org
-            blob, path = self._get_data(apiuri)
+            blob, path = self._get_data(apiuri, _refresh_cache=try_refresh_cache)
             if 'stream-http' not in self._progenitors:
                 self._progenitors['path'] = path
 
@@ -690,6 +703,16 @@ class Pio(formats.Rdf, idlib.Stream):
                     if fail_ok: return
                     raise exc.IdDoesNotExistError(message)
                 elif sc in (250, 205):  # access requested, not authorized
+                    if path_older_than_day(path):
+                        # there seems to be a case where a protocol is shared
+                        # with us and there is an old cache, in that case we
+                        # hit an infinite loop because getting data for the
+                        # progenitor says we should be able to resolve the
+                        # final protocol now but it was cached on cooldown, if
+                        # we can get to the first progenitor then we likely
+                        # have access to the final url and get will succeed
+                        return self.data4(fail_ok=fail_ok, try_refresh_cache=True)
+
                     try:
                         # there might be a private id in the progenitor chain
                         nself = self.progenitor(type='id-converted-from')
@@ -811,7 +834,8 @@ class Pio(formats.Rdf, idlib.Stream):
                     try:
                         return self.data1(fail_ok=fail_ok)
                     except Exception as e1:
-                        raise exc.TransportError() from e1
+                        # FIXME error type ...
+                        raise exc.InbetweenError() from e1
             else:
                 return self.data4(fail_ok=fail_ok)
         except (AttributeError, NotImplementedError) as e:
@@ -2041,6 +2065,14 @@ class Pio(formats.Rdf, idlib.Stream):
                 if include_private and self.identifier.is_private():
                     out['uri_private'] = self.identifier  # FIXME some way to avoid leaking these if needed?
                 return out
+            except exc.MalformedIdentifierError as e:
+                out = super().asDict(include_description)
+                out['malformed'] = True
+                if 'errors' not in out:
+                    out['errors'] = []
+
+                out['errors'].append({'message': 'malformed'})
+                return out
             except exc.RemoteError as e:
                 # we don't have any metadata but we will return what little info we have
                 return super().asDict(include_description)

idlib/streams.py

@@ -41,6 +41,8 @@ class Stream:
 
     _progenitors = MappingProxyType({})  # this is a dict, but immutable to prevent accidents
 
+    _resolution_chain_headers_fun = None
+
     def __new__(cls, *args, **kwargs):
         return super().__new__(cls)
 
@@ -56,8 +58,9 @@ class Stream:
         import requests  # resolver dejoure
         Stream._requests = requests
 
-        from idlib.core import resolution_chain_responses
+        from idlib.core import resolution_chain_responses, resolution_raise_logic
         Stream._resolution_chain_responses = staticmethod(resolution_chain_responses)
+        Stream._resolution_raise_logic = staticmethod(resolution_raise_logic)
 
     @classmethod
     def fromJson(cls, blob):
@@ -608,8 +611,10 @@ class StreamUri(Stream):
                 dc[self.identifier_actionable] = tuple(
                     resp if resp is None else StringProgenitor(resp.url, progenitor=resp)
                     for resp in
-                    self._resolution_chain_responses(self.identifier_actionable,
-                                                     raise_on_final=False))
+                    self._resolution_chain_responses(
+                        self.identifier_actionable,
+                        raise_on_final=False,
+                        headers_fun=self._resolution_chain_headers_fun))
             except (exc.ResolutionError, exc.RemoteError) as e:
                 # TODO
                 # FIXME partial resolution is a complete nightmare
@@ -626,6 +631,10 @@ class StreamUri(Stream):
     def dereference(self, asType=None):
         drc = self.dereference_chain()
         uri = drc[-1]
+        if uri is None:
+            # null pointer
+            self._resolution_raise_logic(drc[-2]._progenitor)
+
         if asType:
             nid = asType(uri)
             if isinstance(nid, idlib.Stream):

idlib/systems/ror.py

@@ -97,7 +97,7 @@ class Ror(formats.Rdf, idlib.HelperNoData, idlib.Stream):
     def _checksum(self, cypher):  # FIXME unqualified checksum goes to ... metadata ???
         m = cypher()
         metadata = self.metadata()
-        name = metadata['name']
+        name = self._name(metadata)
         m.update(self.identifier.checksum(cypher))
         m.update(self.id_bound_metadata.checksum(cypher))
         m.update(name.encode())  # unix epoch -> ??
@@ -143,9 +143,19 @@ class Ror(formats.Rdf, idlib.HelperNoData, idlib.Stream):
                 # or the remote telling us that we are in error
                 raise exc.RemoteError(self.identifier) from e
 
+    @staticmethod
+    def _name(m):
+        if 'names' in m:
+            for name in m['names']:
+                if 'ror_display' in name['types']:
+                    return name['value']
+        else:
+            return m['name']
+
     @property
     def name(self):
-        return self.metadata()['name']
+        m = self.metadata()
+        return self._name(m)
 
     def asExternalId(self, id_class):
         eids = self.data['external_ids']
@@ -170,6 +180,11 @@ class Ror(formats.Rdf, idlib.HelperNoData, idlib.Stream):
         'Funder':     'Funder',  # FIXME funder is a role ...
         'Other':      'Institution',
     }
+    for _k, _v in list(_type_map.items()):
+        _type_map[_k.lower()] = _v
+    del _k
+    del _v
+
     @property
     def institutionTypes(self):
         metadata = self.metadata()
@@ -214,19 +229,29 @@ class Ror(formats.Rdf, idlib.HelperNoData, idlib.Stream):
     def synonyms_rdf(self, rdflib):  # FIXME annoying
         d = self.metadata()
         # FIXME how to deal with type conversion an a saner way ...
-        yield from [rdflib.Literal(s) for s in d['aliases']]
-        yield from [rdflib.Literal(s) for s in d['acronyms']]
-        yield from [rdflib.Literal(l['label'], lang=l['iso639']) for l in d['labels']]
+        if 'names' in d:
+            for name in d['names']:
+                if 'ror_display' not in name['types']:
+                    yield rdflib.Literal(name['value'], lang=name['lang'])
+        else:
+            yield from [rdflib.Literal(s) for s in d['aliases']]
+            yield from [rdflib.Literal(s) for s in d['acronyms']]
+            yield from [rdflib.Literal(l['label'], lang=l['iso639']) for l in d['labels']]
 
     @property
     def synonyms(self):
         out = []
         m = self.metadata()
-        for a in m['aliases'] + m['acronyms']:
-            out.append(a)
+        if 'names' in m:
+            for name in m['names']:
+                if 'ror_display' not in name['types']:
+                    out.append(name['value'])
+        else:
+            for a in m['aliases'] + m['acronyms']:
+                out.append(a)
 
-        for l in m['labels']:
-            out.append(l['label'])
+            for l in m['labels']:
+                out.append(l['label'])
 
         return out
 

idlib/systems/rrid.py

@@ -175,8 +175,52 @@ class Rrid(formats.Rdf, idlib.HelperNoData, idlib.Stream):
             if descs:
                 return sorted(descs, key=len, reverse=True)[0]
 
+    def proper_citation(self):
+        ms = self.metadata()
+        if ms:
+            cites = set(m['rrid']['properCitation'] for m in ms if 'rrid' in m and 'properCitation' in m['rrid'])
+            if cites:
+                return sorted(cites, key=len, reverse=True)[0]
+
+    # output streams
+
+    def _triples_gen(self,
+                     rdflib=None,
+                     rdf=None,
+                     rdfs=None,
+                     owl=None,
+                     NIFRID=None,
+                     TEMP=None,
+                     **kwargs):
+        """ implementation of method to produce a
+            triplified version of the record """
+        s = self.asUri(rdflib.URIRef)
+        yield s, rdf.type, owl.NamedIndividual
+        try:
+            pc = self.proper_citation()
+            if pc:
+                yield s, TEMP.properCitation, rdflib.Literal(pc)
+        except exc.ResolutionError as e:
+            log.exception(e)
+            yield s, TEMP.resolutionError, rdflib.Literal(True)
+            return
+        except exc.RemoteError as e:
+            log.exception(e)
+            yield s, TEMP.remoteError, rdflib.Literal(True)
+            return
+
+        yield s, rdfs.label, rdflib.Literal(self.label)
+
     # alternate representations
 
+    def asDict(self):
+        out = super().asDict()
+        pc = self.proper_citation()
+        if pc:
+            out['proper_citation'] = pc
+
+        return out
+
     def asUri(self, asType=None):
         # TODO n2t, identifiers.org
         # TODO TODO having an explicit model for resolver/metadata services

idlib/utils.py

@@ -124,7 +124,12 @@ def makeEnc(alphabet):
     def decode(chars, *, _index=index, _base=base):
         number = 0
         for c in chars:
-            number = number * _base + _index[c]
+            try:
+                cind = _index[c]
+            except KeyError as e:
+                msg = ''.join(chars)
+                raise exc.MalformedIdentifierError(msg) from e
+            number = number * _base + cind
 
         return number
 

{idlib-0.0.1.dev22.dist-info → idlib-0.0.1.dev23.dist-info}/METADATA

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: idlib
-Version: 0.0.1.dev22
+Version: 0.0.1.dev23
 Summary: A library for working with identifiers of all kinds.
 Home-page: https://github.com/tgbugs/idlib
 Author: Tom Gillespie
@@ -38,6 +38,19 @@ Provides-Extra: rdf
 Requires-Dist: pyontutils>=0.1.28; extra == "rdf"
 Provides-Extra: oauth
 Requires-Dist: google-auth-oauthlib; extra == "oauth"
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: keywords
+Dynamic: license
+Dynamic: license-file
+Dynamic: provides-extra
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
 
 # idlib
 [![PyPI version](https://badge.fury.io/py/idlib.svg)](https://pypi.org/project/idlib/)

{idlib-0.0.1.dev22.dist-info → idlib-0.0.1.dev23.dist-info}/RECORD

@@ -1,16 +1,16 @@
-idlib/__init__.py,sha256=FdowWsUq9bwbPqZPU2sTU1iWMxm1KetrH4kknNBlMmQ,1696
+idlib/__init__.py,sha256=PMFL7ZemlNHqabBZ_MyLbYcaxaGBU1ET-nN9BlflWHE,1696
 idlib/auth-config.py,sha256=tYM2aYCYJFG5NB4Af1dSn6YxhafttOp7W7aSc-QbVV8,460
 idlib/cache.py,sha256=lR1I3iEYvBhNeqoptS-Sj7SpIHt1K65nYlBrTK-ROVQ,3875
 idlib/config.py,sha256=Gli2tZlbKxBos8sXDoDK1g9mngfDw1ZWpBt_7MDZH-g,76
-idlib/core.py,sha256=DfJoPe_xqJJducri1ub7e_anJYuhJ37-ZxvZCuuOpaY,2704
+idlib/core.py,sha256=KVhWN6yQfIGA5rG_ydNVC25oMa88lYJwQtAjzzhqziQ,4569
 idlib/edit.py,sha256=s3yRtvNelQUA8fziu_igZeoFsW4xCK_kgfNVv3TS0iE,3635
 idlib/exceptions.py,sha256=QC3Jq98iwezcqB7SmDr-UEOqdNCnvEXVkKB0eJLIwZs,4714
-idlib/from_oq.py,sha256=HlgepGbgSUbOt5R79ALRhgiLOEvU6TSu3evB8zq7N84,94170
+idlib/from_oq.py,sha256=kt4BpbAZgLo1CwxwyR_swnlVMsekV7ePfZivq9V8yns,95599
 idlib/identifiers.py,sha256=s39bAS-fCQrlwAbMtaa7tjSnqKq4YPlfSbfJ6ju2o7I,3861
-idlib/streams.py,sha256=HGcs_S7bOiUYfiSjZNGLXo5bJI7yd0bYjc8BKkdqECI,26439
-idlib/utils.py,sha256=zItGz-lXovZytSENUZ9q7A7gLr5uK_9VMawokGSskbk,5127
+idlib/streams.py,sha256=QhOtT3xnp8FGVt7GEYAFAnd7PKYRKBPbOF6XGit63lQ,26765
+idlib/utils.py,sha256=q2hcz0Qtbya1UpPnlGisougg9EuDZkcAKXJQIN4OFQs,5306
 idlib/apis/__init__.py,sha256=q2-iSMUDPlYbjqGWa1mtbe2G95N7wbjWHGj30BUdVDM,88
-idlib/apis/protocols_io.py,sha256=3PGKBwe9P_DxhScIG-L3ZFtnEHxgHaccLhRGVPZuYHg,3956
+idlib/apis/protocols_io.py,sha256=PKcoSbO9AqTfBYKLWJQyO3vHjug9qEL826s7jY4LWWg,4026
 idlib/conventions/__init__.py,sha256=ug6W8eNHuq9JZy5bwrZ6Rjfu8r81Rrnm0nzApmvvMzw,204
 idlib/conventions/core.py,sha256=g0T1VoNpoC9afvUd0wp6Mu7tRxbPDFK59uwVXV-zTtA,940
 idlib/conventions/local.py,sha256=H1WnXTY06aLq0RB_SsTLVhMZiSP9GGvWuWm_zrFUNmQ,129
@@ -24,12 +24,12 @@ idlib/systems/handle.py,sha256=Gg97IuysdAdCX9j6EhuNfIddb1GDq0CmwW1SmuLkpgM,95
 idlib/systems/isni.py,sha256=TgiSTlDcSBLRro3gg_YfQJQkfaJ3saw0Au1OMhE23wU,89
 idlib/systems/orcid.py,sha256=aTVu_Q6fnaQvqmd6jeXdst6yKRTO-BEUMs7_4fdQK5E,7886
 idlib/systems/pmid.py,sha256=Na-4_n7qmJ3VAbG0RUwnNTg7o4Z8OkuL_Hluq_A3IO0,181
-idlib/systems/ror.py,sha256=xIS30mxKd0IcjwalBjnyUU7GKeeU5pxM3i9hSofG6u8,8129
-idlib/systems/rrid.py,sha256=XVHyHyiGC2OvbOSXmfU3_wswS6yUw65qZ3Pct0sjbfU,6886
+idlib/systems/ror.py,sha256=3pZ1aLg7NUh-mHQm-qu-hqy5WrJZprtj5lWIWRpP2-M,8894
+idlib/systems/rrid.py,sha256=RcDn3-BRmrKzzxDCypSWRQuivKvWqUqqohY8BeSqRF4,8278
 idlib/systems/urn.py,sha256=p222KjHgfF6y9srCm7kFWuWA9NK4lI-wEQUtSeuBqYk,113
 idlib/systems/web.py,sha256=vrSbQYWnxhe9pSgZcOqwLM3V6XQDLwv1QqWlhWWxbL0,1291
-idlib-0.0.1.dev22.dist-info/LICENSE,sha256=nEuJBsoHfpXGgf6ZyFGUhQJqxFWwhSYF-45pRAqTSb4,1080
-idlib-0.0.1.dev22.dist-info/METADATA,sha256=kjQ8xltmifA_qoV2GmEQAFeO5P3HzWVMc0MjiINC38o,2052
-idlib-0.0.1.dev22.dist-info/WHEEL,sha256=pxeNX5JdtCe58PUSYP9upmc7jdRPgvT0Gm9kb1SHlVw,109
-idlib-0.0.1.dev22.dist-info/top_level.txt,sha256=xIk87gJPbeCKih9BVMP8PlEFdpcsGplenz0VQhx5sP8,6
-idlib-0.0.1.dev22.dist-info/RECORD,,
+idlib-0.0.1.dev23.dist-info/licenses/LICENSE,sha256=nEuJBsoHfpXGgf6ZyFGUhQJqxFWwhSYF-45pRAqTSb4,1080
+idlib-0.0.1.dev23.dist-info/METADATA,sha256=leTa1rrRwvU4n5ucULjBXh9zzBHvsjXNE8nj3-qWPog,2330
+idlib-0.0.1.dev23.dist-info/WHEEL,sha256=JNWh1Fm1UdwIQV075glCn4MVuCRs0sotJIq-J6rbxCU,109
+idlib-0.0.1.dev23.dist-info/top_level.txt,sha256=xIk87gJPbeCKih9BVMP8PlEFdpcsGplenz0VQhx5sP8,6
+idlib-0.0.1.dev23.dist-info/RECORD,,

{idlib-0.0.1.dev22.dist-info → idlib-0.0.1.dev23.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.6.0)
+Generator: setuptools (80.9.0)
 Root-Is-Purelib: true
 Tag: py2-none-any
 Tag: py3-none-any