idlib 0.0.1.dev22__tar.gz → 0.0.1.dev24__tar.gz

{idlib-0.0.1.dev22 → idlib-0.0.1.dev24}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: idlib
-Version: 0.0.1.dev22
+Version: 0.0.1.dev24
 Summary: A library for working with identifiers of all kinds.
 Home-page: https://github.com/tgbugs/idlib
 Author: Tom Gillespie
@@ -8,7 +8,6 @@ Author-email: tgbugs@gmail.com
 License: MIT
 Keywords: python persistent identifiers
 Classifier: Development Status :: 4 - Beta
-Classifier: License :: OSI Approved :: MIT License
 Classifier: Programming Language :: Python :: 3.7
 Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
@@ -16,6 +15,7 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: Implementation :: PyPy
 Classifier: Operating System :: POSIX :: Linux
@@ -30,14 +30,27 @@ Provides-Extra: test
 Requires-Dist: pytest; extra == "test"
 Requires-Dist: joblib>=1.1.0; extra == "test"
 Requires-Dist: beautifulsoup4[html5lib]; extra == "test"
-Requires-Dist: pyontutils>=0.1.28; extra == "test"
+Requires-Dist: pyontutils>=0.1.40; extra == "test"
 Requires-Dist: google-auth-oauthlib; extra == "test"
 Provides-Extra: org
 Requires-Dist: beautifulsoup4[html5lib]; extra == "org"
 Provides-Extra: rdf
-Requires-Dist: pyontutils>=0.1.28; extra == "rdf"
+Requires-Dist: pyontutils>=0.1.40; extra == "rdf"
 Provides-Extra: oauth
 Requires-Dist: google-auth-oauthlib; extra == "oauth"
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: keywords
+Dynamic: license
+Dynamic: license-file
+Dynamic: provides-extra
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
 
 # idlib
 [![PyPI version](https://badge.fury.io/py/idlib.svg)](https://pypi.org/project/idlib/)

{idlib-0.0.1.dev22 → idlib-0.0.1.dev24}/idlib/__init__.py

@@ -55,4 +55,4 @@ def get_right_id(uri):
     return pi
 
 
-__version__ = '0.0.1.dev22'
+__version__ = '0.0.1.dev24'

{idlib-0.0.1.dev22 → idlib-0.0.1.dev24}/idlib/apis/protocols_io.py

@@ -3,7 +3,6 @@ import sys
 import pickle
 from urllib import parse
 from getpass import getpass
-from ..config import auth
 from ..utils import log as _log
 from . import log as _log
 
@@ -68,11 +67,16 @@ def get_auth_code(url):
 
 
 def get_protocols_io_auth(creds_file,
-                          store_file=auth.get_path('protocols-io-api-store-file'),
-                          # yes reading from the store file here at the top level
-                          # increases startup time, however if it is burried then
-                          # any config error is deferred until later in runtime
+                          store_file=None,
                           SCOPES='readwrite'):
+    if store_file is None:
+        # since this file is imported whenever idlib is imported
+        # the time cost is prohibitive and we will just have to
+        # deal with the configuration value failing later in time
+        # by e.g. having a separate check config process
+        from ..config import auth
+        store_file = auth.get_path('protocols-io-api-store-file')
+
     from google_auth_oauthlib.flow import InstalledAppFlow
     from google.auth.transport.requests import Request
     from idlib import exceptions as exc

{idlib-0.0.1.dev22 → idlib-0.0.1.dev24}/idlib/cache.py

@@ -1,4 +1,5 @@
 import json
+import pickle
 import hashlib
 import inspect
 from pathlib import Path
@@ -66,6 +67,10 @@ def cache(folder, ser='json', clear_cache=False, create=False, return_path=False
         serialize = json.dump
         deserialize = json.load
         mode = 't'
+    elif ser == 'pickle':
+        serialize = pickle.dump
+        deserialize = pickle.load
+        mode = 'b'
     else:
         raise TypeError('Bad serialization format.')
 
@@ -77,9 +82,11 @@ def cache(folder, ser='json', clear_cache=False, create=False, return_path=False
         if not create:
             raise FileNotFoundError(f'Cache base folder does not exist! {folder}')
 
-        folder.mkdir(parents=True)
+        # TOCTOU race condition is hit idlib is imported by multi-process workers
+        folder.mkdir(parents=True, exist_ok=True)
 
     if clear_cache:
+        import shutil
         log.debug(f'clearing cache for {folder}')
         shutil.rmtree(folder)
         folder.mkdir()
@@ -100,8 +107,14 @@ def cache(folder, ser='json', clear_cache=False, create=False, return_path=False
             else:
                 output = function(*args, **kwargs)
                 if output is not None:
-                    with open(filepath, write_mode) as f:
-                        serialize(output, f)
+                    try:
+                        with open(filepath, write_mode) as f:
+                            serialize(output, f)
+                    except Exception as e:
+                        filepath.unlink()
+                        # TODO remove serialization on failure so we
+                        # don't think a real cache exists
+                        raise e
 
             if isinstance(output, dict) and COOLDOWN in output:
                 # a hack to put a dummy variable in the cache

idlib-0.0.1.dev24/idlib/core.py

@@ -0,0 +1,116 @@
+import requests
+from . import exceptions as exc
+from .utils import log
+
+
+def resolution_chain(iri, headers_fun=None):
+    for head in resolution_chain_responses(iri, headers_fun=headers_fun):
+        yield head.url
+
+
+_user_agent_idiocy = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:136.0) Gecko/20100101 Firefox/136.0'
+
+
+def try_get(s, head, headers_fun=None):
+    # see whether the server has a bad/broken support for head requests
+    # sometimes they return e.g. 400 instead of 405, and really they should
+    # just work because they work correctly with get request ...
+    head = s.get(head.url, stream=True)
+
+    if head.ok:
+        log.info(f'bad HEAD implementation {head.url}')
+    elif head.status_code < 500:
+        head.close()
+        headers = {'User-Agent': _user_agent_idiocy}
+        head = s.get(head.url, headers=headers, stream=True)
+        if head.ok:
+            log.info(f'bad HEAD implementation AND bad User-Agent behavior {head.url}')
+        else:
+            headers = None
+            if headers_fun is not None:
+                # headers_fun is used to ensure that e.g. auth headers
+                # are only sent to the netloc's they are for
+                headers = headers_fun(head.url)
+                if headers is not None:
+                    head = s.get(head.url, headers=headers, stream=True)
+                    if head.ok:
+                        # we log a warning because dereferencing a public id
+                        # and encountering a permission boundary is bad
+                        # TODO likely need a way to fail on this condition
+                        # or at least indicate that it happend in band
+                        msg = f'additional headers were added for {head.url}'
+                        log.warning(msg)
+
+    if not head.ok:
+        content = head.content
+        if content:
+            if head.headers['Content-Type'] == 'application/json':
+                try:
+                    j = head.json()
+                    log.error(f'{head.url} error json {j}')
+                except Exception as e:
+                    log.error(f'{head.url} error text {head.text}')
+
+    head.close()
+    return head
+
+
+def resolution_raise_logic(head):
+    """ broken out into its own function for reuse to avoid re-resolving
+        in cases where we need to interpose additional operations between
+        a dereference failure and raising the error
+    """
+    if not head.ok:
+        try:
+            head.raise_for_status()
+        except Exception as e:
+            if head.status_code == 404:
+                msg = f'Nothing found due to {head.status_code} at {head.url}\n'
+                raise exc.IdDoesNotExistError(msg) from e  # often a permissions issue
+            elif head.status_code >= 500:
+                msg = f'Remote in error due to {head.status_code} at {head.url}\n'
+                raise exc.RemoteError(msg) from e
+            elif head.status_code >= 400:
+                msg = f'Nothing found due to {head.status_code} at {head.url}\n'
+                raise exc.ResolutionError(msg) from e
+
+
+def resolution_chain_responses(iri, raise_on_final=True, headers_fun=None):
+    #doi = doi  # TODO
+    s = requests.Session()
+    head = s.head(iri, allow_redirects=False)
+    if head.status_code < 400:
+        yield head
+    else:
+        head = try_get(s, head, headers_fun=headers_fun)
+        yield head
+
+    while head.is_redirect and head.status_code < 400:  # FIXME redirect loop issue
+        yield head.next
+        try:
+            head = s.send(head.next)
+        except requests.exceptions.SSLError as e:
+            raise exc.InbetweenError(msg) from e
+
+        if head.status_code < 400:
+            yield head
+        else:
+            head = try_get(s, head, headers_fun=headers_fun)
+            yield head
+
+        if not head.is_redirect:
+            break
+
+    if raise_on_final:  # we still want the chain ... null pointer error comes later?
+        resolution_raise_logic(head)
+
+    if head.status_code >= 400:
+        # XXX this seems like the "right" thing to do, but it will
+        # probably break a bunch of stuff, but at least this way it
+        # will be visible now, obviously yeilding None as the final
+        # element in the reference chain conflates all the possible
+        # reasons so raising is a better solution and is the default
+        # for a reason but at least this way the user knows that the
+        # final element of the chain dereference to nothing instead of
+        # assuming that the last element succeeded
+        yield None

{idlib-0.0.1.dev22 → idlib-0.0.1.dev24}/idlib/from_oq.py

@@ -2,6 +2,7 @@ import re
 import json
 from time import time, sleep
 from datetime import datetime
+from urllib.parse import urlparse
 import orthauth as oa
 import ontquery as oq  # temporary implementation detail
 import idlib
@@ -20,6 +21,11 @@ from idlib.utils import (log,
 from idlib.config import auth
 
 
+def path_older_than_day(path):
+    ps = path.stat()
+    return time() - ps.st_ctime > (60 * 60 * 24)
+
+
 # from neurondm.simple
 class OntTerm(oq.OntTerm):
     """ ask things about a term! """
@@ -345,6 +351,13 @@ def setup(cls, creds_file=None):
             log.warning(e)
             cls._pio_header = None
 
+    def rchf(url):
+        purl = urlparse(url)
+        if purl.netloc.endswith('protocols.io'):
+            return cls._pio_header
+
+    cls._resolution_chain_headers_fun = staticmethod(rchf)
+
     if not hasattr(idlib.Stream, '_requests'):
         idlib.Stream.__new__(cls)
 
@@ -419,6 +432,10 @@ class Pio(formats.Rdf, idlib.Stream):
         if data:
             doi = data['doi']
             if doi:
+                if not doi.startswith('http'):
+                    # why is there no scheme? who knows!?
+                    doi = 'https://' + doi
+
                 return idlib.Doi(doi)
 
     @property
@@ -645,7 +662,7 @@ class Pio(formats.Rdf, idlib.Stream):
         return data
 
     _apiuri_private_cache = {}  # XXX FIXME hack
-    def data4(self, fail_ok=False):
+    def data4(self, fail_ok=False, try_refresh_cache=False):
         if not hasattr(self, '_data'):
             self._data_in_error = True
             if not isinstance(self._progenitors, dict):
@@ -661,7 +678,7 @@ class Pio(formats.Rdf, idlib.Stream):
             # however this is not trivial to retrive
             # the /v3/protocols/{id} endpoint returns bad/mangled data and we need
             # the /v1/ data to be able to actually render to org
-            blob, path = self._get_data(apiuri)
+            blob, path = self._get_data(apiuri, _refresh_cache=try_refresh_cache)
             if 'stream-http' not in self._progenitors:
                 self._progenitors['path'] = path
 
@@ -690,6 +707,16 @@ class Pio(formats.Rdf, idlib.Stream):
                     if fail_ok: return
                     raise exc.IdDoesNotExistError(message)
                 elif sc in (250, 205):  # access requested, not authorized
+                    if path_older_than_day(path):
+                        # there seems to be a case where a protocol is shared
+                        # with us and there is an old cache, in that case we
+                        # hit an infinite loop because getting data for the
+                        # progenitor says we should be able to resolve the
+                        # final protocol now but it was cached on cooldown, if
+                        # we can get to the first progenitor then we likely
+                        # have access to the final url and get will succeed
+                        return self.data4(fail_ok=fail_ok, try_refresh_cache=True)
+
                     try:
                         # there might be a private id in the progenitor chain
                         nself = self.progenitor(type='id-converted-from')
@@ -735,6 +762,11 @@ class Pio(formats.Rdf, idlib.Stream):
                     # and pio json mean there is an extra layer that needs
                     # interpretation ... i.e. we definitely got to the remote
                     raise NotImplementedError(msg)
+                elif sc == 1266:
+                    msg = 'rate limit exceeded'
+                    if fail_ok: return
+                    path.unlink()  # hopefully works next time?
+                    raise exc.AccessLimitError(msg)
                 else:
                     msg = f'unhandled pio status code {sc}\n' + message
                     raise NotImplementedError(msg)
@@ -793,8 +825,10 @@ class Pio(formats.Rdf, idlib.Stream):
                 # for how we got there, even if we toss the crazy slugs
                 self._progenitors = {}  # FIXME yes this blasts progens every time
                 uh = self.uri_human
+                _sigh = uh._progenitors.pop('id-converted-from')  # avoid infinite recursion
                 uai = uh.uri_api_int
                 d4 = uai.data4(fail_ok=fail_ok)
+                uh._progenitors['id-converted-from'] = _sigh
                 self._progenitors['id-converted-to'] = uai
                 if 'path' in uai._progenitors:
                     self._progenitors['path'] = uai._progenitors['path']
@@ -810,12 +844,18 @@ class Pio(formats.Rdf, idlib.Stream):
                 except Exception as e0:
                     try:
                         return self.data1(fail_ok=fail_ok)
+                    except exc.IdDoesNotExistError as ene1:
+                        raise ene1
                     except Exception as e1:
-                        raise exc.TransportError() from e1
+                        # FIXME error type ...
+                        raise exc.InbetweenError() from e1
             else:
                 return self.data4(fail_ok=fail_ok)
         except (AttributeError, NotImplementedError) as e:
             raise e
+        except exc.IdDoesNotExistError as e:
+            # don't wait for pioids that don't exist
+            raise e
         except Exception as e:
             wait_until = time() + self._wait_time
             self._data_in_error_sigh = wait_until, e
@@ -2041,6 +2081,14 @@ class Pio(formats.Rdf, idlib.Stream):
                 if include_private and self.identifier.is_private():
                     out['uri_private'] = self.identifier  # FIXME some way to avoid leaking these if needed?
                 return out
+            except exc.MalformedIdentifierError as e:
+                out = super().asDict(include_description)
+                out['malformed'] = True
+                if 'errors' not in out:
+                    out['errors'] = []
+
+                out['errors'].append({'message': 'malformed'})
+                return out
             except exc.RemoteError as e:
                 # we don't have any metadata but we will return what little info we have
                 return super().asDict(include_description)

{idlib-0.0.1.dev22 → idlib-0.0.1.dev24}/idlib/streams.py

@@ -41,6 +41,8 @@ class Stream:
 
     _progenitors = MappingProxyType({})  # this is a dict, but immutable to prevent accidents
 
+    _resolution_chain_headers_fun = None
+
     def __new__(cls, *args, **kwargs):
         return super().__new__(cls)
 
@@ -56,8 +58,9 @@ class Stream:
         import requests  # resolver dejoure
         Stream._requests = requests
 
-        from idlib.core import resolution_chain_responses
+        from idlib.core import resolution_chain_responses, resolution_raise_logic
         Stream._resolution_chain_responses = staticmethod(resolution_chain_responses)
+        Stream._resolution_raise_logic = staticmethod(resolution_raise_logic)
 
     @classmethod
     def fromJson(cls, blob):
@@ -437,7 +440,12 @@ class HelpTestStreams:
             HelpTestStreams._copy = copy
             import pickle
             HelpTestStreams._pickle = pickle
-            from joblib import Parallel, delayed
+            try:
+                from joblib import Parallel, delayed
+            except Exception as e:
+                log.exception(e)
+                from pyontutils.asyncd import Async as Parallel, deferred as delayed
+
             HelpTestStreams._Parallel = Parallel
             HelpTestStreams._delayed = delayed
 
@@ -461,7 +469,11 @@ class HelpTestStreams:
                 print(d.identifier.prefix)
                 print(d.identifier.suffix)
                 if d.identifier.prefix is None or d.identifier.suffix is None:
-                    bads.append(d.identifier.__dict__)
+                    if hasattr(d.identifier, 'iri') and hasattr(d.identifier, 'valid') and d.identifier.valid:
+                        # doi case
+                        pass
+                    else:
+                        bads.append(d.identifier.__dict__)
 
             try:
                 d.identifier_bound_metadata
@@ -498,9 +510,18 @@ class HelpTestStreams:
 
     def test_malformed(self):
         bads = []
-        for i in self.ids_bad:
+        vbads = []
+        pre_garbage = 'oops '
+        post_garbage = ' oops'
+        bad_ids = (
+            list(self.ids_bad) +
+            [pre_garbage + i for i in self.ids] +
+            [i + post_garbage for i in self.ids])
+        for i in bad_ids:
             try:
                 d = self.stream(i)
+                d.identifier.validate()
+                vbads.append(d)
                 d.data()
                 bads.append(d)
             except Exception as e:
@@ -508,6 +529,7 @@ class HelpTestStreams:
                 pass
 
         assert not bads, bads
+        assert not vbads, vbads
 
     def test_hash_eq_id(self):
         hrm = self.ids[0]
@@ -608,8 +630,10 @@ class StreamUri(Stream):
                 dc[self.identifier_actionable] = tuple(
                     resp if resp is None else StringProgenitor(resp.url, progenitor=resp)
                     for resp in
-                    self._resolution_chain_responses(self.identifier_actionable,
-                                                     raise_on_final=False))
+                    self._resolution_chain_responses(
+                        self.identifier_actionable,
+                        raise_on_final=False,
+                        headers_fun=self._resolution_chain_headers_fun))
             except (exc.ResolutionError, exc.RemoteError) as e:
                 # TODO
                 # FIXME partial resolution is a complete nightmare
@@ -626,6 +650,10 @@ class StreamUri(Stream):
     def dereference(self, asType=None):
         drc = self.dereference_chain()
         uri = drc[-1]
+        if uri is None:
+            # null pointer
+            self._resolution_raise_logic(drc[-2]._progenitor)
+
         if asType:
             nid = asType(uri)
             if isinstance(nid, idlib.Stream):

{idlib-0.0.1.dev22 → idlib-0.0.1.dev24}/idlib/systems/doi.py

@@ -1,3 +1,5 @@
+import re
+from urllib.parse import unquote, quote
 import ontquery as oq  # temp implementation detail
 import idlib
 from idlib import formats
@@ -26,8 +28,11 @@ class DoiId(oq.OntId, idlib.Identifier, idlib.Stream):  # also _technically_ a s
     _namespaces = _DoiPrefixes
     _local_conventions = _namespaces
     _id_class = str  # eventually we have to drop down to something
-    local_regex = '^10\\.[0-9]{4,9}[-._;()/:a-zA-Z0-9]+$'
-    canonical_regex = '^https://doi.org/10\\.[0-9]{4,9}[-._;()/:a-zA-Z0-9]+$'
+    # https://www.crossref.org/blog/dois-and-matching-regular-expressions/
+    # but we have cases that don't follow in our set ... so we may need a variant regex
+    # so we can warn if we see one
+    local_regex = '^10\\.[0-9]{4,9}/[-._;()<>/:a-zA-Z0-9]+$'
+    canonical_regex = '^https://doi.org/10\\.[0-9]{4,9}/[-._;()<>/:a-zA-Z0-9]+$'
 
     def __new__(cls, doi_in_various_states_of_mangling=None, iri=None):
         if doi_in_various_states_of_mangling is None and iri is not None:
@@ -59,22 +64,29 @@ class DoiId(oq.OntId, idlib.Identifier, idlib.Stream):  # also _technically_ a s
 
     @staticmethod
     def normalize(doi):
-        doi = doi.replace(' ', '')
-        if 'http' in doi or 'doi.org' in doi:
+        doi = doi.strip().replace(' :', ':').replace(': ', ':')
+        if doi.startswith('http') or doi.startswith('doi.org'):
             doi = '10.' + doi.split('.org/10.', 1)[-1]
+            # space is left in quote to ensure we produce an error because there should be no spaces
+            doi = quote(unquote(doi), safe='-._;()/: ')
         elif doi.startswith('doi:'):
-            doi = doi.strip('doi:')
+            doi = doi.strip('doi:').strip()
         elif doi.startswith('DOI:'):
-            doi = doi.strip('DOI:')
+            doi = doi.strip('DOI:').strip()
+
         return doi
 
     @property
     def valid(self):
-        return self.suffix is not None and self.suffix.startswith('10.')
+        if self.curie is None:
+            # we may have one of the pathalogical cases
+            return re.match(self.canonical_regex, unquote(self.iri))
+        else:
+            return self.suffix is not None and self.suffix.startswith('10.')
 
     def validate(self):
         if not self.valid:
-            raise exc.MalformedIdentifierError(f'{self._unnormalized} does not appear '
+            raise exc.MalformedIdentifierError(f'{self._unnormalized!r} does not appear '
                                                f'to be of type {self.__class__}')
 
 
@@ -97,6 +109,9 @@ class Doi(formats.Rdf, idlib.Stream):  # FIXME that 'has canonical representaito
     def __init__(self, doi_in_various_states_of_mangling=None, iri=None):
         self._identifier = self._id_class(doi_in_various_states_of_mangling, iri)
 
+    def __repr__(self):
+        return f'<idlib.{self.__class__.__name__}({self._identifier.asStr()!r})>'
+
     def __gt__(self, other):
         if isinstance(other, idlib.Stream):
             return self.identifier > other.identifier

{idlib-0.0.1.dev22 → idlib-0.0.1.dev24}/idlib/systems/ror.py

@@ -97,7 +97,7 @@ class Ror(formats.Rdf, idlib.HelperNoData, idlib.Stream):
     def _checksum(self, cypher):  # FIXME unqualified checksum goes to ... metadata ???
         m = cypher()
         metadata = self.metadata()
-        name = metadata['name']
+        name = self._name(metadata)
         m.update(self.identifier.checksum(cypher))
         m.update(self.id_bound_metadata.checksum(cypher))
         m.update(name.encode())  # unix epoch -> ??
@@ -143,9 +143,19 @@ class Ror(formats.Rdf, idlib.HelperNoData, idlib.Stream):
                 # or the remote telling us that we are in error
                 raise exc.RemoteError(self.identifier) from e
 
+    @staticmethod
+    def _name(m):
+        if 'names' in m:
+            for name in m['names']:
+                if 'ror_display' in name['types']:
+                    return name['value']
+        else:
+            return m['name']
+
     @property
     def name(self):
-        return self.metadata()['name']
+        m = self.metadata()
+        return self._name(m)
 
     def asExternalId(self, id_class):
         eids = self.data['external_ids']
@@ -170,6 +180,11 @@ class Ror(formats.Rdf, idlib.HelperNoData, idlib.Stream):
         'Funder':     'Funder',  # FIXME funder is a role ...
         'Other':      'Institution',
     }
+    for _k, _v in list(_type_map.items()):
+        _type_map[_k.lower()] = _v
+    del _k
+    del _v
+
     @property
     def institutionTypes(self):
         metadata = self.metadata()
@@ -214,19 +229,29 @@ class Ror(formats.Rdf, idlib.HelperNoData, idlib.Stream):
     def synonyms_rdf(self, rdflib):  # FIXME annoying
         d = self.metadata()
         # FIXME how to deal with type conversion an a saner way ...
-        yield from [rdflib.Literal(s) for s in d['aliases']]
-        yield from [rdflib.Literal(s) for s in d['acronyms']]
-        yield from [rdflib.Literal(l['label'], lang=l['iso639']) for l in d['labels']]
+        if 'names' in d:
+            for name in d['names']:
+                if 'ror_display' not in name['types']:
+                    yield rdflib.Literal(name['value'], lang=name['lang'])
+        else:
+            yield from [rdflib.Literal(s) for s in d['aliases']]
+            yield from [rdflib.Literal(s) for s in d['acronyms']]
+            yield from [rdflib.Literal(l['label'], lang=l['iso639']) for l in d['labels']]
 
     @property
     def synonyms(self):
         out = []
         m = self.metadata()
-        for a in m['aliases'] + m['acronyms']:
-            out.append(a)
+        if 'names' in m:
+            for name in m['names']:
+                if 'ror_display' not in name['types']:
+                    out.append(name['value'])
+        else:
+            for a in m['aliases'] + m['acronyms']:
+                out.append(a)
 
-        for l in m['labels']:
-            out.append(l['label'])
+            for l in m['labels']:
+                out.append(l['label'])
 
         return out
 

{idlib-0.0.1.dev22 → idlib-0.0.1.dev24}/idlib/systems/rrid.py

@@ -1,4 +1,5 @@
 import re
+from time import sleep
 import idlib
 from idlib import formats
 from idlib import streams
@@ -11,12 +12,12 @@ from idlib.config import auth
 
 class RridId(idlib.Identifier):
 
-    canonical_regex = 'RRID:([A-Za-z]+)_([A-Za-z0-9_:-]+)'
+    canonical_regex = '^RRID:([A-Za-z]+)_([A-Za-z0-9_:-]+)$'
     # NOTE: the canonical regex fails to capture a couple
     # of legacy cases e.g. RRID:WB-STRAIN_ and RRID:MGI:
     # yes I understand why MGI doesn't want RRID:MGI_MGI:
     # but what other identifier namespaces do they use?
-    _variant_regex = 'RRID:([A-Z]+):([A-Za-z0-9_:-]+)'  # legacy
+    _variant_regex = '^RRID:([A-Z]+):([A-Za-z0-9_:-]+)$'  # legacy
     _local_conventions = conv.Identity
 
     def __init__(self, rrid):
@@ -101,10 +102,31 @@ class Rrid(formats.Rdf, idlib.HelperNoData, idlib.Stream):
         elif self._COOLDOWN and self._resp_metadata.status_code == 404:
             msg = f'RRID failure: {self._resp_metadata.status_code} {self.asUri()}'
             return {COOLDOWN: msg,}
+        elif self._resp_metadata.status_code == 429:
+            headers = self._resp_metadata.headers
+            if 'Retry-After' in headers:
+                try:
+                    wait = int(headers['Retry-After'])
+                except ValueError as ve:
+                    try:
+                        self._resp_metadata.raise_for_status()
+                    except Exception as e:
+                        raise exc.ResolutionError(identifier) from e
+
+                if wait > 20:
+                    try:
+                        self._resp_metadata.raise_for_status()
+                    except Exception as e:
+                        raise exc.ResolutionError(identifier) from e
+
+                log.debug(f'waiting {wait}s for {identifier}')
+                sleep(wait)
+                return self._metadata(identifier)
+
         else:
             try:
                 self._resp_metadata.raise_for_status()
-            except BaseException as e:
+            except Exception as e:
                 raise exc.ResolutionError(identifier) from e
 
     @cache_result
@@ -175,8 +197,52 @@ class Rrid(formats.Rdf, idlib.HelperNoData, idlib.Stream):
             if descs:
                 return sorted(descs, key=len, reverse=True)[0]
 
+    def proper_citation(self):
+        ms = self.metadata()
+        if ms:
+            cites = set(m['rrid']['properCitation'] for m in ms if 'rrid' in m and 'properCitation' in m['rrid'])
+            if cites:
+                return sorted(cites, key=len, reverse=True)[0]
+
+    # output streams
+
+    def _triples_gen(self,
+                     rdflib=None,
+                     rdf=None,
+                     rdfs=None,
+                     owl=None,
+                     NIFRID=None,
+                     TEMP=None,
+                     **kwargs):
+        """ implementation of method to produce a
+            triplified version of the record """
+        s = self.asUri(rdflib.URIRef)
+        yield s, rdf.type, owl.NamedIndividual
+        try:
+            pc = self.proper_citation()
+            if pc:
+                yield s, TEMP.properCitation, rdflib.Literal(pc)
+        except exc.ResolutionError as e:
+            log.exception(e)
+            yield s, TEMP.resolutionError, rdflib.Literal(True)
+            return
+        except exc.RemoteError as e:
+            log.exception(e)
+            yield s, TEMP.remoteError, rdflib.Literal(True)
+            return
+
+        yield s, rdfs.label, rdflib.Literal(self.label)
+
     # alternate representations
 
+    def asDict(self):
+        out = super().asDict()
+        pc = self.proper_citation()
+        if pc:
+            out['proper_citation'] = pc
+
+        return out
+
     def asUri(self, asType=None):
         # TODO n2t, identifiers.org
         # TODO TODO having an explicit model for resolver/metadata services

{idlib-0.0.1.dev22 → idlib-0.0.1.dev24}/idlib/utils.py

@@ -124,7 +124,12 @@ def makeEnc(alphabet):
     def decode(chars, *, _index=index, _base=base):
         number = 0
         for c in chars:
-            number = number * _base + _index[c]
+            try:
+                cind = _index[c]
+            except KeyError as e:
+                msg = ''.join(chars)
+                raise exc.MalformedIdentifierError(msg) from e
+            number = number * _base + cind
 
         return number
 

{idlib-0.0.1.dev22 → idlib-0.0.1.dev24}/idlib.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: idlib
-Version: 0.0.1.dev22
+Version: 0.0.1.dev24
 Summary: A library for working with identifiers of all kinds.
 Home-page: https://github.com/tgbugs/idlib
 Author: Tom Gillespie
@@ -8,7 +8,6 @@ Author-email: tgbugs@gmail.com
 License: MIT
 Keywords: python persistent identifiers
 Classifier: Development Status :: 4 - Beta
-Classifier: License :: OSI Approved :: MIT License
 Classifier: Programming Language :: Python :: 3.7
 Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
@@ -16,6 +15,7 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: Implementation :: PyPy
 Classifier: Operating System :: POSIX :: Linux
@@ -30,14 +30,27 @@ Provides-Extra: test
 Requires-Dist: pytest; extra == "test"
 Requires-Dist: joblib>=1.1.0; extra == "test"
 Requires-Dist: beautifulsoup4[html5lib]; extra == "test"
-Requires-Dist: pyontutils>=0.1.28; extra == "test"
+Requires-Dist: pyontutils>=0.1.40; extra == "test"
 Requires-Dist: google-auth-oauthlib; extra == "test"
 Provides-Extra: org
 Requires-Dist: beautifulsoup4[html5lib]; extra == "org"
 Provides-Extra: rdf
-Requires-Dist: pyontutils>=0.1.28; extra == "rdf"
+Requires-Dist: pyontutils>=0.1.40; extra == "rdf"
 Provides-Extra: oauth
 Requires-Dist: google-auth-oauthlib; extra == "oauth"
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: keywords
+Dynamic: license
+Dynamic: license-file
+Dynamic: provides-extra
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
 
 # idlib
 [![PyPI version](https://badge.fury.io/py/idlib.svg)](https://pypi.org/project/idlib/)

{idlib-0.0.1.dev22 → idlib-0.0.1.dev24}/idlib.egg-info/requires.txt

@@ -8,11 +8,11 @@ google-auth-oauthlib
 beautifulsoup4[html5lib]
 
 [rdf]
-pyontutils>=0.1.28
+pyontutils>=0.1.40
 
 [test]
 pytest
 joblib>=1.1.0
 beautifulsoup4[html5lib]
-pyontutils>=0.1.28
+pyontutils>=0.1.40
 google-auth-oauthlib

{idlib-0.0.1.dev22 → idlib-0.0.1.dev24}/setup.py

@@ -19,7 +19,7 @@ with open('README.md', 'rt') as f:
     long_description = f.read()
 
 org_require = ['beautifulsoup4[html5lib]']
-rdf_require = ['pyontutils>=0.1.28']
+rdf_require = ['pyontutils>=0.1.40']
 oauth_require = ['google-auth-oauthlib']
 tests_require = (['pytest', 'joblib>=1.1.0'] +
                  org_require +
@@ -36,7 +36,6 @@ setup(name='idlib',
       license='MIT',
       classifiers=[
           'Development Status :: 4 - Beta',
-          'License :: OSI Approved :: MIT License',
           'Programming Language :: Python :: 3.7',
           'Programming Language :: Python :: 3.8',
           'Programming Language :: Python :: 3.9',
@@ -44,6 +43,7 @@ setup(name='idlib',
           'Programming Language :: Python :: 3.11',
           'Programming Language :: Python :: 3.12',
           'Programming Language :: Python :: 3.13',
+          'Programming Language :: Python :: 3.14',
           'Programming Language :: Python :: Implementation :: CPython',
           'Programming Language :: Python :: Implementation :: PyPy',
           'Operating System :: POSIX :: Linux',

{idlib-0.0.1.dev22 → idlib-0.0.1.dev24}/test/test_identifiers.py

@@ -36,6 +36,7 @@ class TestDoi(HelpTestStreams, unittest.TestCase):
     ids = [
         'https://doi.org/10.13003/5jchdy',
         'https://doi.org/10.1101/2020.10.19.343129',  # crossref 500 error
+        'https://doi.org/10.1002/(SICI)1096-9136(199608)13:8%3C700::AID-DIA174%3E3.0.CO;2-8',  # oof
     ]
     ids_bad = ['lol not an identifier']
     def test_version_f1000(self):
@@ -142,7 +143,10 @@ class TestRrid(HelpTestStreams, unittest.TestCase):
         'RRID:Addgene_19640',
         #'RRID:NCBITaxon_9606',  # TODO not in resolver yet
     ]
-    ids_bad = ['lol not an identifier']
+    ids_bad = [
+        'lol not an identifier',
+        'RRID:RGD_737929 Charles River RS:0001011',
+    ]
 
 
 @skipif_ci
@@ -183,6 +187,7 @@ class TestPio(HelpTestStreams, unittest.TestCase):
         'https://www.protocols.io/view/21417',
         'https://www.protocols.io/api/v3/protocols/21417',
         'https://www.protocols.io/api/v4/protocols/21417',
+        'https://www.protocols.io/api/v4/protocols/126454',
 
         'pio.view:human-islet-microvasculature-immunofluorescence-in-y3tfynn/materials',
 
@@ -192,6 +197,11 @@ class TestPio(HelpTestStreams, unittest.TestCase):
         'https://www.protocols.io/view/18980',  # deleted
         'https://www.protocols.io/api/v3/protocols/18980',  # deleted
         'https://www.protocols.io/private/D327A68038D5B4ECB32964C22D904270',  # deleted
+
+        # malformed slug causing issues with asDict ... FIXME also possibly changes in Pio api
+        # because now the error says NotAuthorized ... which seems suspicious
+        'https://www.protocols.io/view/transfection-protocol-for-n2a-and-hek293-using-pro-75mhq50',
+        'https://www.protocols.io/api/v4/protocols/transfection-protocol-for-n2a-and-hek293-using-pro-75mhq50',
     ]
 
     @skipif_no_net

idlib-0.0.1.dev22/idlib/core.py

@@ -1,77 +0,0 @@
-import requests
-from . import exceptions as exc
-from .utils import log
-
-
-def resolution_chain(iri):
-    for head in resolution_chain_responses(iri):
-        yield head.url
-
-
-def try_get(s, head):
-    # see whether the server has a bad/broken support for head requests
-    # sometimes they return e.g. 400 instead of 405, and really they should
-    # just work because they work correctly with get request ...
-    head = s.get(head.url, stream=True)
-    if head.ok:
-        log.info(f'bad HEAD implementation {head.url}')
-    else:
-        content = head.content
-        if content:
-            if head.headers['Content-Type'] == 'application/json':
-                try:
-                    j = head.json()
-                    log.error(f'{head.url} error json {j}')
-                except Exception as e:
-                    log.error(f'{head.url} error text {head.text}')
-
-    head.close()
-    return head
-
-
-def resolution_chain_responses(iri, raise_on_final=True):
-    #doi = doi  # TODO
-    s = requests.Session()
-    head = s.head(iri, allow_redirects=False)
-    if head.status_code < 400:
-        yield head
-    else:
-        head = try_get(s, head)
-        yield head
-
-    while head.is_redirect and head.status_code < 400:  # FIXME redirect loop issue
-        yield head.next
-        try:
-            head = s.send(head.next)
-        except requests.exceptions.SSLError as e:
-            raise exc.InbetweenError(msg) from e
-
-        if head.status_code < 400:
-            yield head
-        else:
-            head = try_get(s, head)
-            yield head
-
-        if not head.is_redirect:
-            break
-
-    if raise_on_final:  # we still want the chain ... null pointer error comes later?
-        if head.status_code == 404:
-            head.raise_for_status()  # probably a permissions issue
-        elif head.status_code >= 500:
-            msg = f'Remote in error due to {head.status_code} at {head.url}\n'
-            raise exc.RemoteError(msg)
-        elif head.status_code >= 400:
-            msg = f'Nothing found due to {head.status_code} at {head.url}\n'
-            raise exc.ResolutionError(msg)
-
-    if head.status_code >= 400:
-        # XXX this seems like the "right" thing to do, but it will
-        # probably break a bunch of stuff, but at least this way it
-        # will be visible now, obviously yeilding None as the final
-        # element in the reference chain conflates all the possible
-        # reasons so raising is a better solution and is the default
-        # for a reason but at least this way the user knows that the
-        # final element of the chain dereference to nothing instead of
-        # assuming that the last element succeeded
-        yield None