idlib 0.0.1.dev22__tar.gz → 0.0.1.dev23__tar.gz

{idlib-0.0.1.dev22 → idlib-0.0.1.dev23}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: idlib
-Version: 0.0.1.dev22
+Version: 0.0.1.dev23
 Summary: A library for working with identifiers of all kinds.
 Home-page: https://github.com/tgbugs/idlib
 Author: Tom Gillespie
@@ -38,6 +38,19 @@ Provides-Extra: rdf
 Requires-Dist: pyontutils>=0.1.28; extra == "rdf"
 Provides-Extra: oauth
 Requires-Dist: google-auth-oauthlib; extra == "oauth"
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: keywords
+Dynamic: license
+Dynamic: license-file
+Dynamic: provides-extra
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
 
 # idlib
 [![PyPI version](https://badge.fury.io/py/idlib.svg)](https://pypi.org/project/idlib/)

{idlib-0.0.1.dev22 → idlib-0.0.1.dev23}/idlib/__init__.py

@@ -55,4 +55,4 @@ def get_right_id(uri):
     return pi
 
 
-__version__ = '0.0.1.dev22'
+__version__ = '0.0.1.dev23'

{idlib-0.0.1.dev22 → idlib-0.0.1.dev23}/idlib/apis/protocols_io.py

@@ -3,7 +3,6 @@ import sys
 import pickle
 from urllib import parse
 from getpass import getpass
-from ..config import auth
 from ..utils import log as _log
 from . import log as _log
 
@@ -68,11 +67,16 @@ def get_auth_code(url):
 
 
 def get_protocols_io_auth(creds_file,
-                          store_file=auth.get_path('protocols-io-api-store-file'),
-                          # yes reading from the store file here at the top level
-                          # increases startup time, however if it is burried then
-                          # any config error is deferred until later in runtime
+                          store_file=None,
                           SCOPES='readwrite'):
+    if store_file is None:
+        # since this file is imported whenever idlib is imported
+        # the time cost is prohibitive and we will just have to
+        # deal with the configuration value failing later in time
+        # by e.g. having a separate check config process
+        from ..config import auth
+        store_file = auth.get_path('protocols-io-api-store-file')
+
     from google_auth_oauthlib.flow import InstalledAppFlow
     from google.auth.transport.requests import Request
     from idlib import exceptions as exc

idlib-0.0.1.dev23/idlib/core.py

@@ -0,0 +1,116 @@
+import requests
+from . import exceptions as exc
+from .utils import log
+
+
+def resolution_chain(iri, headers_fun=None):
+    for head in resolution_chain_responses(iri, headers_fun=headers_fun):
+        yield head.url
+
+
+_user_agent_idiocy = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:136.0) Gecko/20100101 Firefox/136.0'
+
+
+def try_get(s, head, headers_fun=None):
+    # see whether the server has a bad/broken support for head requests
+    # sometimes they return e.g. 400 instead of 405, and really they should
+    # just work because they work correctly with get request ...
+    head = s.get(head.url, stream=True)
+
+    if head.ok:
+        log.info(f'bad HEAD implementation {head.url}')
+    elif head.status_code < 500:
+        head.close()
+        headers = {'User-Agent': _user_agent_idiocy}
+        head = s.get(head.url, headers=headers, stream=True)
+        if head.ok:
+            log.info(f'bad HEAD implementation AND bad User-Agent behavior {head.url}')
+        else:
+            headers = None
+            if headers_fun is not None:
+                # headers_fun is used to ensure that e.g. auth headers
+                # are only sent to the netloc's they are for
+                headers = headers_fun(head.url)
+                if headers is not None:
+                    head = s.get(head.url, headers=headers, stream=True)
+                    if head.ok:
+                        # we log a warning because dereferencing a public id
+                        # and encountering a permission boundary is bad
+                        # TODO likely need a way to fail on this condition
+                        # or at least indicate that it happend in band
+                        msg = f'additional headers were added for {head.url}'
+                        log.warning(msg)
+
+    if not head.ok:
+        content = head.content
+        if content:
+            if head.headers['Content-Type'] == 'application/json':
+                try:
+                    j = head.json()
+                    log.error(f'{head.url} error json {j}')
+                except Exception as e:
+                    log.error(f'{head.url} error text {head.text}')
+
+    head.close()
+    return head
+
+
+def resolution_raise_logic(head):
+    """ broken out into its own function for reuse to avoid re-resolving
+        in cases where we need to interpose additional operations between
+        a dereference failure and raising the error
+    """
+    if not head.ok:
+        try:
+            head.raise_for_status()
+        except Exception as e:
+            if head.status_code == 404:
+                msg = f'Nothing found due to {head.status_code} at {head.url}\n'
+                raise exc.IdDoesNotExistError(msg) from e  # often a permissions issue
+            elif head.status_code >= 500:
+                msg = f'Remote in error due to {head.status_code} at {head.url}\n'
+                raise exc.RemoteError(msg) from e
+            elif head.status_code >= 400:
+                msg = f'Nothing found due to {head.status_code} at {head.url}\n'
+                raise exc.ResolutionError(msg) from e
+
+
+def resolution_chain_responses(iri, raise_on_final=True, headers_fun=None):
+    #doi = doi  # TODO
+    s = requests.Session()
+    head = s.head(iri, allow_redirects=False)
+    if head.status_code < 400:
+        yield head
+    else:
+        head = try_get(s, head, headers_fun=headers_fun)
+        yield head
+
+    while head.is_redirect and head.status_code < 400:  # FIXME redirect loop issue
+        yield head.next
+        try:
+            head = s.send(head.next)
+        except requests.exceptions.SSLError as e:
+            raise exc.InbetweenError(msg) from e
+
+        if head.status_code < 400:
+            yield head
+        else:
+            head = try_get(s, head, headers_fun=headers_fun)
+            yield head
+
+        if not head.is_redirect:
+            break
+
+    if raise_on_final:  # we still want the chain ... null pointer error comes later?
+        resolution_raise_logic(head)
+
+    if head.status_code >= 400:
+        # XXX this seems like the "right" thing to do, but it will
+        # probably break a bunch of stuff, but at least this way it
+        # will be visible now, obviously yeilding None as the final
+        # element in the reference chain conflates all the possible
+        # reasons so raising is a better solution and is the default
+        # for a reason but at least this way the user knows that the
+        # final element of the chain dereference to nothing instead of
+        # assuming that the last element succeeded
+        yield None

{idlib-0.0.1.dev22 → idlib-0.0.1.dev23}/idlib/from_oq.py

@@ -2,6 +2,7 @@ import re
 import json
 from time import time, sleep
 from datetime import datetime
+from urllib.parse import urlparse
 import orthauth as oa
 import ontquery as oq  # temporary implementation detail
 import idlib
@@ -20,6 +21,11 @@ from idlib.utils import (log,
 from idlib.config import auth
 
 
+def path_older_than_day(path):
+    ps = path.stat()
+    return time() - ps.st_ctime > (60 * 60 * 24)
+
+
 # from neurondm.simple
 class OntTerm(oq.OntTerm):
     """ ask things about a term! """
@@ -345,6 +351,13 @@ def setup(cls, creds_file=None):
             log.warning(e)
             cls._pio_header = None
 
+    def rchf(url):
+        purl = urlparse(url)
+        if purl.netloc.endswith('protocols.io'):
+            return cls._pio_header
+
+    cls._resolution_chain_headers_fun = staticmethod(rchf)
+
     if not hasattr(idlib.Stream, '_requests'):
         idlib.Stream.__new__(cls)
 
@@ -645,7 +658,7 @@ class Pio(formats.Rdf, idlib.Stream):
         return data
 
     _apiuri_private_cache = {}  # XXX FIXME hack
-    def data4(self, fail_ok=False):
+    def data4(self, fail_ok=False, try_refresh_cache=False):
         if not hasattr(self, '_data'):
             self._data_in_error = True
             if not isinstance(self._progenitors, dict):
@@ -661,7 +674,7 @@ class Pio(formats.Rdf, idlib.Stream):
             # however this is not trivial to retrive
             # the /v3/protocols/{id} endpoint returns bad/mangled data and we need
             # the /v1/ data to be able to actually render to org
-            blob, path = self._get_data(apiuri)
+            blob, path = self._get_data(apiuri, _refresh_cache=try_refresh_cache)
             if 'stream-http' not in self._progenitors:
                 self._progenitors['path'] = path
 
@@ -690,6 +703,16 @@ class Pio(formats.Rdf, idlib.Stream):
                     if fail_ok: return
                     raise exc.IdDoesNotExistError(message)
                 elif sc in (250, 205):  # access requested, not authorized
+                    if path_older_than_day(path):
+                        # there seems to be a case where a protocol is shared
+                        # with us and there is an old cache, in that case we
+                        # hit an infinite loop because getting data for the
+                        # progenitor says we should be able to resolve the
+                        # final protocol now but it was cached on cooldown, if
+                        # we can get to the first progenitor then we likely
+                        # have access to the final url and get will succeed
+                        return self.data4(fail_ok=fail_ok, try_refresh_cache=True)
+
                     try:
                         # there might be a private id in the progenitor chain
                         nself = self.progenitor(type='id-converted-from')
@@ -811,7 +834,8 @@ class Pio(formats.Rdf, idlib.Stream):
                     try:
                         return self.data1(fail_ok=fail_ok)
                     except Exception as e1:
-                        raise exc.TransportError() from e1
+                        # FIXME error type ...
+                        raise exc.InbetweenError() from e1
             else:
                 return self.data4(fail_ok=fail_ok)
         except (AttributeError, NotImplementedError) as e:
@@ -2041,6 +2065,14 @@ class Pio(formats.Rdf, idlib.Stream):
                 if include_private and self.identifier.is_private():
                     out['uri_private'] = self.identifier  # FIXME some way to avoid leaking these if needed?
                 return out
+            except exc.MalformedIdentifierError as e:
+                out = super().asDict(include_description)
+                out['malformed'] = True
+                if 'errors' not in out:
+                    out['errors'] = []
+
+                out['errors'].append({'message': 'malformed'})
+                return out
             except exc.RemoteError as e:
                 # we don't have any metadata but we will return what little info we have
                 return super().asDict(include_description)

{idlib-0.0.1.dev22 → idlib-0.0.1.dev23}/idlib/streams.py

@@ -41,6 +41,8 @@ class Stream:
 
     _progenitors = MappingProxyType({})  # this is a dict, but immutable to prevent accidents
 
+    _resolution_chain_headers_fun = None
+
     def __new__(cls, *args, **kwargs):
         return super().__new__(cls)
 
@@ -56,8 +58,9 @@ class Stream:
         import requests  # resolver dejoure
         Stream._requests = requests
 
-        from idlib.core import resolution_chain_responses
+        from idlib.core import resolution_chain_responses, resolution_raise_logic
         Stream._resolution_chain_responses = staticmethod(resolution_chain_responses)
+        Stream._resolution_raise_logic = staticmethod(resolution_raise_logic)
 
     @classmethod
     def fromJson(cls, blob):
@@ -608,8 +611,10 @@ class StreamUri(Stream):
                 dc[self.identifier_actionable] = tuple(
                     resp if resp is None else StringProgenitor(resp.url, progenitor=resp)
                     for resp in
-                    self._resolution_chain_responses(self.identifier_actionable,
-                                                     raise_on_final=False))
+                    self._resolution_chain_responses(
+                        self.identifier_actionable,
+                        raise_on_final=False,
+                        headers_fun=self._resolution_chain_headers_fun))
             except (exc.ResolutionError, exc.RemoteError) as e:
                 # TODO
                 # FIXME partial resolution is a complete nightmare
@@ -626,6 +631,10 @@ class StreamUri(Stream):
     def dereference(self, asType=None):
         drc = self.dereference_chain()
         uri = drc[-1]
+        if uri is None:
+            # null pointer
+            self._resolution_raise_logic(drc[-2]._progenitor)
+
         if asType:
             nid = asType(uri)
             if isinstance(nid, idlib.Stream):

{idlib-0.0.1.dev22 → idlib-0.0.1.dev23}/idlib/systems/ror.py

@@ -97,7 +97,7 @@ class Ror(formats.Rdf, idlib.HelperNoData, idlib.Stream):
     def _checksum(self, cypher):  # FIXME unqualified checksum goes to ... metadata ???
         m = cypher()
         metadata = self.metadata()
-        name = metadata['name']
+        name = self._name(metadata)
         m.update(self.identifier.checksum(cypher))
         m.update(self.id_bound_metadata.checksum(cypher))
         m.update(name.encode())  # unix epoch -> ??
@@ -143,9 +143,19 @@ class Ror(formats.Rdf, idlib.HelperNoData, idlib.Stream):
                 # or the remote telling us that we are in error
                 raise exc.RemoteError(self.identifier) from e
 
+    @staticmethod
+    def _name(m):
+        if 'names' in m:
+            for name in m['names']:
+                if 'ror_display' in name['types']:
+                    return name['value']
+        else:
+            return m['name']
+
     @property
     def name(self):
-        return self.metadata()['name']
+        m = self.metadata()
+        return self._name(m)
 
     def asExternalId(self, id_class):
         eids = self.data['external_ids']
@@ -170,6 +180,11 @@ class Ror(formats.Rdf, idlib.HelperNoData, idlib.Stream):
         'Funder':     'Funder',  # FIXME funder is a role ...
         'Other':      'Institution',
     }
+    for _k, _v in list(_type_map.items()):
+        _type_map[_k.lower()] = _v
+    del _k
+    del _v
+
     @property
     def institutionTypes(self):
         metadata = self.metadata()
@@ -214,19 +229,29 @@ class Ror(formats.Rdf, idlib.HelperNoData, idlib.Stream):
     def synonyms_rdf(self, rdflib):  # FIXME annoying
         d = self.metadata()
         # FIXME how to deal with type conversion an a saner way ...
-        yield from [rdflib.Literal(s) for s in d['aliases']]
-        yield from [rdflib.Literal(s) for s in d['acronyms']]
-        yield from [rdflib.Literal(l['label'], lang=l['iso639']) for l in d['labels']]
+        if 'names' in d:
+            for name in d['names']:
+                if 'ror_display' not in name['types']:
+                    yield rdflib.Literal(name['value'], lang=name['lang'])
+        else:
+            yield from [rdflib.Literal(s) for s in d['aliases']]
+            yield from [rdflib.Literal(s) for s in d['acronyms']]
+            yield from [rdflib.Literal(l['label'], lang=l['iso639']) for l in d['labels']]
 
     @property
     def synonyms(self):
         out = []
         m = self.metadata()
-        for a in m['aliases'] + m['acronyms']:
-            out.append(a)
+        if 'names' in m:
+            for name in m['names']:
+                if 'ror_display' not in name['types']:
+                    out.append(name['value'])
+        else:
+            for a in m['aliases'] + m['acronyms']:
+                out.append(a)
 
-        for l in m['labels']:
-            out.append(l['label'])
+            for l in m['labels']:
+                out.append(l['label'])
 
         return out
 

{idlib-0.0.1.dev22 → idlib-0.0.1.dev23}/idlib/systems/rrid.py

@@ -175,8 +175,52 @@ class Rrid(formats.Rdf, idlib.HelperNoData, idlib.Stream):
             if descs:
                 return sorted(descs, key=len, reverse=True)[0]
 
+    def proper_citation(self):
+        ms = self.metadata()
+        if ms:
+            cites = set(m['rrid']['properCitation'] for m in ms if 'rrid' in m and 'properCitation' in m['rrid'])
+            if cites:
+                return sorted(cites, key=len, reverse=True)[0]
+
+    # output streams
+
+    def _triples_gen(self,
+                     rdflib=None,
+                     rdf=None,
+                     rdfs=None,
+                     owl=None,
+                     NIFRID=None,
+                     TEMP=None,
+                     **kwargs):
+        """ implementation of method to produce a
+            triplified version of the record """
+        s = self.asUri(rdflib.URIRef)
+        yield s, rdf.type, owl.NamedIndividual
+        try:
+            pc = self.proper_citation()
+            if pc:
+                yield s, TEMP.properCitation, rdflib.Literal(pc)
+        except exc.ResolutionError as e:
+            log.exception(e)
+            yield s, TEMP.resolutionError, rdflib.Literal(True)
+            return
+        except exc.RemoteError as e:
+            log.exception(e)
+            yield s, TEMP.remoteError, rdflib.Literal(True)
+            return
+
+        yield s, rdfs.label, rdflib.Literal(self.label)
+
     # alternate representations
 
+    def asDict(self):
+        out = super().asDict()
+        pc = self.proper_citation()
+        if pc:
+            out['proper_citation'] = pc
+
+        return out
+
     def asUri(self, asType=None):
         # TODO n2t, identifiers.org
         # TODO TODO having an explicit model for resolver/metadata services

{idlib-0.0.1.dev22 → idlib-0.0.1.dev23}/idlib/utils.py

@@ -124,7 +124,12 @@ def makeEnc(alphabet):
     def decode(chars, *, _index=index, _base=base):
         number = 0
         for c in chars:
-            number = number * _base + _index[c]
+            try:
+                cind = _index[c]
+            except KeyError as e:
+                msg = ''.join(chars)
+                raise exc.MalformedIdentifierError(msg) from e
+            number = number * _base + cind
 
         return number
 

{idlib-0.0.1.dev22 → idlib-0.0.1.dev23}/idlib.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: idlib
-Version: 0.0.1.dev22
+Version: 0.0.1.dev23
 Summary: A library for working with identifiers of all kinds.
 Home-page: https://github.com/tgbugs/idlib
 Author: Tom Gillespie
@@ -38,6 +38,19 @@ Provides-Extra: rdf
 Requires-Dist: pyontutils>=0.1.28; extra == "rdf"
 Provides-Extra: oauth
 Requires-Dist: google-auth-oauthlib; extra == "oauth"
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: keywords
+Dynamic: license
+Dynamic: license-file
+Dynamic: provides-extra
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
 
 # idlib
 [![PyPI version](https://badge.fury.io/py/idlib.svg)](https://pypi.org/project/idlib/)

{idlib-0.0.1.dev22 → idlib-0.0.1.dev23}/test/test_identifiers.py

@@ -183,6 +183,7 @@ class TestPio(HelpTestStreams, unittest.TestCase):
         'https://www.protocols.io/view/21417',
         'https://www.protocols.io/api/v3/protocols/21417',
         'https://www.protocols.io/api/v4/protocols/21417',
+        'https://www.protocols.io/api/v4/protocols/126454',
 
         'pio.view:human-islet-microvasculature-immunofluorescence-in-y3tfynn/materials',
 
@@ -192,6 +193,11 @@ class TestPio(HelpTestStreams, unittest.TestCase):
         'https://www.protocols.io/view/18980',  # deleted
         'https://www.protocols.io/api/v3/protocols/18980',  # deleted
         'https://www.protocols.io/private/D327A68038D5B4ECB32964C22D904270',  # deleted
+
+        # malformed slug causing issues with asDict ... FIXME also possibly changes in Pio api
+        # because now the error says NotAuthorized ... which seems suspicious
+        'https://www.protocols.io/view/transfection-protocol-for-n2a-and-hek293-using-pro-75mhq50',
+        'https://www.protocols.io/api/v4/protocols/transfection-protocol-for-n2a-and-hek293-using-pro-75mhq50',
     ]
 
     @skipif_no_net

idlib-0.0.1.dev22/idlib/core.py

@@ -1,77 +0,0 @@
-import requests
-from . import exceptions as exc
-from .utils import log
-
-
-def resolution_chain(iri):
-    for head in resolution_chain_responses(iri):
-        yield head.url
-
-
-def try_get(s, head):
-    # see whether the server has a bad/broken support for head requests
-    # sometimes they return e.g. 400 instead of 405, and really they should
-    # just work because they work correctly with get request ...
-    head = s.get(head.url, stream=True)
-    if head.ok:
-        log.info(f'bad HEAD implementation {head.url}')
-    else:
-        content = head.content
-        if content:
-            if head.headers['Content-Type'] == 'application/json':
-                try:
-                    j = head.json()
-                    log.error(f'{head.url} error json {j}')
-                except Exception as e:
-                    log.error(f'{head.url} error text {head.text}')
-
-    head.close()
-    return head
-
-
-def resolution_chain_responses(iri, raise_on_final=True):
-    #doi = doi  # TODO
-    s = requests.Session()
-    head = s.head(iri, allow_redirects=False)
-    if head.status_code < 400:
-        yield head
-    else:
-        head = try_get(s, head)
-        yield head
-
-    while head.is_redirect and head.status_code < 400:  # FIXME redirect loop issue
-        yield head.next
-        try:
-            head = s.send(head.next)
-        except requests.exceptions.SSLError as e:
-            raise exc.InbetweenError(msg) from e
-
-        if head.status_code < 400:
-            yield head
-        else:
-            head = try_get(s, head)
-            yield head
-
-        if not head.is_redirect:
-            break
-
-    if raise_on_final:  # we still want the chain ... null pointer error comes later?
-        if head.status_code == 404:
-            head.raise_for_status()  # probably a permissions issue
-        elif head.status_code >= 500:
-            msg = f'Remote in error due to {head.status_code} at {head.url}\n'
-            raise exc.RemoteError(msg)
-        elif head.status_code >= 400:
-            msg = f'Nothing found due to {head.status_code} at {head.url}\n'
-            raise exc.ResolutionError(msg)
-
-    if head.status_code >= 400:
-        # XXX this seems like the "right" thing to do, but it will
-        # probably break a bunch of stuff, but at least this way it
-        # will be visible now, obviously yeilding None as the final
-        # element in the reference chain conflates all the possible
-        # reasons so raising is a better solution and is the default
-        # for a reason but at least this way the user knows that the
-        # final element of the chain dereference to nothing instead of
-        # assuming that the last element succeeded
-        yield None