idlib 0.0.1.dev23__py2.py3-none-any.whl → 0.0.1.dev24__py2.py3-none-any.whl

idlib/__init__.py

@@ -55,4 +55,4 @@ def get_right_id(uri):
     return pi
 
 
-__version__ = '0.0.1.dev23'
+__version__ = '0.0.1.dev24'

idlib/cache.py

@@ -1,4 +1,5 @@
 import json
+import pickle
 import hashlib
 import inspect
 from pathlib import Path
@@ -66,6 +67,10 @@ def cache(folder, ser='json', clear_cache=False, create=False, return_path=False
         serialize = json.dump
         deserialize = json.load
         mode = 't'
+    elif ser == 'pickle':
+        serialize = pickle.dump
+        deserialize = pickle.load
+        mode = 'b'
     else:
         raise TypeError('Bad serialization format.')
 
@@ -77,9 +82,11 @@ def cache(folder, ser='json', clear_cache=False, create=False, return_path=False
         if not create:
             raise FileNotFoundError(f'Cache base folder does not exist! {folder}')
 
-        folder.mkdir(parents=True)
+        # TOCTOU race condition is hit idlib is imported by multi-process workers
+        folder.mkdir(parents=True, exist_ok=True)
 
     if clear_cache:
+        import shutil
         log.debug(f'clearing cache for {folder}')
         shutil.rmtree(folder)
         folder.mkdir()
@@ -100,8 +107,14 @@ def cache(folder, ser='json', clear_cache=False, create=False, return_path=False
             else:
                 output = function(*args, **kwargs)
                 if output is not None:
-                    with open(filepath, write_mode) as f:
-                        serialize(output, f)
+                    try:
+                        with open(filepath, write_mode) as f:
+                            serialize(output, f)
+                    except Exception as e:
+                        filepath.unlink()
+                        # TODO remove serialization on failure so we
+                        # don't think a real cache exists
+                        raise e
 
             if isinstance(output, dict) and COOLDOWN in output:
                 # a hack to put a dummy variable in the cache

idlib/from_oq.py

@@ -432,6 +432,10 @@ class Pio(formats.Rdf, idlib.Stream):
         if data:
             doi = data['doi']
             if doi:
+                if not doi.startswith('http'):
+                    # why is there no scheme? who knows!?
+                    doi = 'https://' + doi
+
                 return idlib.Doi(doi)
 
     @property
@@ -758,6 +762,11 @@ class Pio(formats.Rdf, idlib.Stream):
                     # and pio json mean there is an extra layer that needs
                     # interpretation ... i.e. we definitely got to the remote
                     raise NotImplementedError(msg)
+                elif sc == 1266:
+                    msg = 'rate limit exceeded'
+                    if fail_ok: return
+                    path.unlink()  # hopefully works next time?
+                    raise exc.AccessLimitError(msg)
                 else:
                     msg = f'unhandled pio status code {sc}\n' + message
                     raise NotImplementedError(msg)
@@ -816,8 +825,10 @@ class Pio(formats.Rdf, idlib.Stream):
                 # for how we got there, even if we toss the crazy slugs
                 self._progenitors = {}  # FIXME yes this blasts progens every time
                 uh = self.uri_human
+                _sigh = uh._progenitors.pop('id-converted-from')  # avoid infinite recursion
                 uai = uh.uri_api_int
                 d4 = uai.data4(fail_ok=fail_ok)
+                uh._progenitors['id-converted-from'] = _sigh
                 self._progenitors['id-converted-to'] = uai
                 if 'path' in uai._progenitors:
                     self._progenitors['path'] = uai._progenitors['path']
@@ -833,6 +844,8 @@ class Pio(formats.Rdf, idlib.Stream):
                 except Exception as e0:
                     try:
                         return self.data1(fail_ok=fail_ok)
+                    except exc.IdDoesNotExistError as ene1:
+                        raise ene1
                     except Exception as e1:
                         # FIXME error type ...
                         raise exc.InbetweenError() from e1
@@ -840,6 +853,9 @@ class Pio(formats.Rdf, idlib.Stream):
                 return self.data4(fail_ok=fail_ok)
         except (AttributeError, NotImplementedError) as e:
             raise e
+        except exc.IdDoesNotExistError as e:
+            # don't wait for pioids that don't exist
+            raise e
         except Exception as e:
             wait_until = time() + self._wait_time
             self._data_in_error_sigh = wait_until, e

idlib/streams.py

@@ -440,7 +440,12 @@ class HelpTestStreams:
             HelpTestStreams._copy = copy
             import pickle
             HelpTestStreams._pickle = pickle
-            from joblib import Parallel, delayed
+            try:
+                from joblib import Parallel, delayed
+            except Exception as e:
+                log.exception(e)
+                from pyontutils.asyncd import Async as Parallel, deferred as delayed
+
             HelpTestStreams._Parallel = Parallel
             HelpTestStreams._delayed = delayed
 
@@ -464,7 +469,11 @@ class HelpTestStreams:
                 print(d.identifier.prefix)
                 print(d.identifier.suffix)
                 if d.identifier.prefix is None or d.identifier.suffix is None:
-                    bads.append(d.identifier.__dict__)
+                    if hasattr(d.identifier, 'iri') and hasattr(d.identifier, 'valid') and d.identifier.valid:
+                        # doi case
+                        pass
+                    else:
+                        bads.append(d.identifier.__dict__)
 
             try:
                 d.identifier_bound_metadata
@@ -501,9 +510,18 @@ class HelpTestStreams:
 
     def test_malformed(self):
         bads = []
-        for i in self.ids_bad:
+        vbads = []
+        pre_garbage = 'oops '
+        post_garbage = ' oops'
+        bad_ids = (
+            list(self.ids_bad) +
+            [pre_garbage + i for i in self.ids] +
+            [i + post_garbage for i in self.ids])
+        for i in bad_ids:
             try:
                 d = self.stream(i)
+                d.identifier.validate()
+                vbads.append(d)
                 d.data()
                 bads.append(d)
             except Exception as e:
@@ -511,6 +529,7 @@ class HelpTestStreams:
                 pass
 
         assert not bads, bads
+        assert not vbads, vbads
 
     def test_hash_eq_id(self):
         hrm = self.ids[0]

idlib/systems/doi.py

@@ -1,3 +1,5 @@
+import re
+from urllib.parse import unquote, quote
 import ontquery as oq  # temp implementation detail
 import idlib
 from idlib import formats
@@ -26,8 +28,11 @@ class DoiId(oq.OntId, idlib.Identifier, idlib.Stream):  # also _technically_ a s
     _namespaces = _DoiPrefixes
     _local_conventions = _namespaces
     _id_class = str  # eventually we have to drop down to something
-    local_regex = '^10\\.[0-9]{4,9}[-._;()/:a-zA-Z0-9]+$'
-    canonical_regex = '^https://doi.org/10\\.[0-9]{4,9}[-._;()/:a-zA-Z0-9]+$'
+    # https://www.crossref.org/blog/dois-and-matching-regular-expressions/
+    # but we have cases that don't follow in our set ... so we may need a variant regex
+    # so we can warn if we see one
+    local_regex = '^10\\.[0-9]{4,9}/[-._;()<>/:a-zA-Z0-9]+$'
+    canonical_regex = '^https://doi.org/10\\.[0-9]{4,9}/[-._;()<>/:a-zA-Z0-9]+$'
 
     def __new__(cls, doi_in_various_states_of_mangling=None, iri=None):
         if doi_in_various_states_of_mangling is None and iri is not None:
@@ -59,22 +64,29 @@ class DoiId(oq.OntId, idlib.Identifier, idlib.Stream):  # also _technically_ a s
 
     @staticmethod
     def normalize(doi):
-        doi = doi.replace(' ', '')
-        if 'http' in doi or 'doi.org' in doi:
+        doi = doi.strip().replace(' :', ':').replace(': ', ':')
+        if doi.startswith('http') or doi.startswith('doi.org'):
             doi = '10.' + doi.split('.org/10.', 1)[-1]
+            # space is left in quote to ensure we produce an error because there should be no spaces
+            doi = quote(unquote(doi), safe='-._;()/: ')
         elif doi.startswith('doi:'):
-            doi = doi.strip('doi:')
+            doi = doi.strip('doi:').strip()
         elif doi.startswith('DOI:'):
-            doi = doi.strip('DOI:')
+            doi = doi.strip('DOI:').strip()
+
         return doi
 
     @property
     def valid(self):
-        return self.suffix is not None and self.suffix.startswith('10.')
+        if self.curie is None:
+            # we may have one of the pathalogical cases
+            return re.match(self.canonical_regex, unquote(self.iri))
+        else:
+            return self.suffix is not None and self.suffix.startswith('10.')
 
     def validate(self):
         if not self.valid:
-            raise exc.MalformedIdentifierError(f'{self._unnormalized} does not appear '
+            raise exc.MalformedIdentifierError(f'{self._unnormalized!r} does not appear '
                                                f'to be of type {self.__class__}')
 
 
@@ -97,6 +109,9 @@ class Doi(formats.Rdf, idlib.Stream):  # FIXME that 'has canonical representaito
     def __init__(self, doi_in_various_states_of_mangling=None, iri=None):
         self._identifier = self._id_class(doi_in_various_states_of_mangling, iri)
 
+    def __repr__(self):
+        return f'<idlib.{self.__class__.__name__}({self._identifier.asStr()!r})>'
+
     def __gt__(self, other):
         if isinstance(other, idlib.Stream):
             return self.identifier > other.identifier

idlib/systems/rrid.py

@@ -1,4 +1,5 @@
 import re
+from time import sleep
 import idlib
 from idlib import formats
 from idlib import streams
@@ -11,12 +12,12 @@ from idlib.config import auth
 
 class RridId(idlib.Identifier):
 
-    canonical_regex = 'RRID:([A-Za-z]+)_([A-Za-z0-9_:-]+)'
+    canonical_regex = '^RRID:([A-Za-z]+)_([A-Za-z0-9_:-]+)$'
     # NOTE: the canonical regex fails to capture a couple
     # of legacy cases e.g. RRID:WB-STRAIN_ and RRID:MGI:
     # yes I understand why MGI doesn't want RRID:MGI_MGI:
     # but what other identifier namespaces do they use?
-    _variant_regex = 'RRID:([A-Z]+):([A-Za-z0-9_:-]+)'  # legacy
+    _variant_regex = '^RRID:([A-Z]+):([A-Za-z0-9_:-]+)$'  # legacy
     _local_conventions = conv.Identity
 
     def __init__(self, rrid):
@@ -101,10 +102,31 @@ class Rrid(formats.Rdf, idlib.HelperNoData, idlib.Stream):
         elif self._COOLDOWN and self._resp_metadata.status_code == 404:
             msg = f'RRID failure: {self._resp_metadata.status_code} {self.asUri()}'
             return {COOLDOWN: msg,}
+        elif self._resp_metadata.status_code == 429:
+            headers = self._resp_metadata.headers
+            if 'Retry-After' in headers:
+                try:
+                    wait = int(headers['Retry-After'])
+                except ValueError as ve:
+                    try:
+                        self._resp_metadata.raise_for_status()
+                    except Exception as e:
+                        raise exc.ResolutionError(identifier) from e
+
+                if wait > 20:
+                    try:
+                        self._resp_metadata.raise_for_status()
+                    except Exception as e:
+                        raise exc.ResolutionError(identifier) from e
+
+                log.debug(f'waiting {wait}s for {identifier}')
+                sleep(wait)
+                return self._metadata(identifier)
+
         else:
             try:
                 self._resp_metadata.raise_for_status()
-            except BaseException as e:
+            except Exception as e:
                 raise exc.ResolutionError(identifier) from e
 
     @cache_result

{idlib-0.0.1.dev23.dist-info → idlib-0.0.1.dev24.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: idlib
-Version: 0.0.1.dev23
+Version: 0.0.1.dev24
 Summary: A library for working with identifiers of all kinds.
 Home-page: https://github.com/tgbugs/idlib
 Author: Tom Gillespie
@@ -8,7 +8,6 @@ Author-email: tgbugs@gmail.com
 License: MIT
 Keywords: python persistent identifiers
 Classifier: Development Status :: 4 - Beta
-Classifier: License :: OSI Approved :: MIT License
 Classifier: Programming Language :: Python :: 3.7
 Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
@@ -16,6 +15,7 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: Implementation :: PyPy
 Classifier: Operating System :: POSIX :: Linux
@@ -30,12 +30,12 @@ Provides-Extra: test
 Requires-Dist: pytest; extra == "test"
 Requires-Dist: joblib>=1.1.0; extra == "test"
 Requires-Dist: beautifulsoup4[html5lib]; extra == "test"
-Requires-Dist: pyontutils>=0.1.28; extra == "test"
+Requires-Dist: pyontutils>=0.1.40; extra == "test"
 Requires-Dist: google-auth-oauthlib; extra == "test"
 Provides-Extra: org
 Requires-Dist: beautifulsoup4[html5lib]; extra == "org"
 Provides-Extra: rdf
-Requires-Dist: pyontutils>=0.1.28; extra == "rdf"
+Requires-Dist: pyontutils>=0.1.40; extra == "rdf"
 Provides-Extra: oauth
 Requires-Dist: google-auth-oauthlib; extra == "oauth"
 Dynamic: author

{idlib-0.0.1.dev23.dist-info → idlib-0.0.1.dev24.dist-info}/RECORD

@@ -1,13 +1,13 @@
-idlib/__init__.py,sha256=PMFL7ZemlNHqabBZ_MyLbYcaxaGBU1ET-nN9BlflWHE,1696
+idlib/__init__.py,sha256=nM5sqe3xTkzp2ETq97yYQUeeoJ3sHfNb_CtVrzJ7moA,1696
 idlib/auth-config.py,sha256=tYM2aYCYJFG5NB4Af1dSn6YxhafttOp7W7aSc-QbVV8,460
-idlib/cache.py,sha256=lR1I3iEYvBhNeqoptS-Sj7SpIHt1K65nYlBrTK-ROVQ,3875
+idlib/cache.py,sha256=UN3ufibDtPHgdMqKEXqVvVhfUTod08Vs_xsG12D5kns,4396
 idlib/config.py,sha256=Gli2tZlbKxBos8sXDoDK1g9mngfDw1ZWpBt_7MDZH-g,76
 idlib/core.py,sha256=KVhWN6yQfIGA5rG_ydNVC25oMa88lYJwQtAjzzhqziQ,4569
 idlib/edit.py,sha256=s3yRtvNelQUA8fziu_igZeoFsW4xCK_kgfNVv3TS0iE,3635
 idlib/exceptions.py,sha256=QC3Jq98iwezcqB7SmDr-UEOqdNCnvEXVkKB0eJLIwZs,4714
-idlib/from_oq.py,sha256=kt4BpbAZgLo1CwxwyR_swnlVMsekV7ePfZivq9V8yns,95599
+idlib/from_oq.py,sha256=AVxs7zq8GlAmHOJAu8M3Wb3R5byTc5D4X7U47PzRldM,96351
 idlib/identifiers.py,sha256=s39bAS-fCQrlwAbMtaa7tjSnqKq4YPlfSbfJ6ju2o7I,3861
-idlib/streams.py,sha256=QhOtT3xnp8FGVt7GEYAFAnd7PKYRKBPbOF6XGit63lQ,26765
+idlib/streams.py,sha256=spUVSCyR0gugdZxi8orhTAkUKCAT-kq7J5dyBxl5_uo,27477
 idlib/utils.py,sha256=q2hcz0Qtbya1UpPnlGisougg9EuDZkcAKXJQIN4OFQs,5306
 idlib/apis/__init__.py,sha256=q2-iSMUDPlYbjqGWa1mtbe2G95N7wbjWHGj30BUdVDM,88
 idlib/apis/protocols_io.py,sha256=PKcoSbO9AqTfBYKLWJQyO3vHjug9qEL826s7jY4LWWg,4026
@@ -19,17 +19,17 @@ idlib/formats/__init__.py,sha256=yS6l0mbDwa2xjyxhgVzlqIf0LPqJmuhNVg8rfKSsh7M,528
 idlib/formats/rdf.py,sha256=yIKHPaPn-SUuUMkR4VD7KffVgQD8oNaZNxcqDBilxtg,466
 idlib/systems/__init__.py,sha256=3lWSK2DyzYX0d61DJo-gGytc-HoiOlNJl5so4s7Grrg,203
 idlib/systems/ark.py,sha256=i9JqmuTWsJABrLHR0-_FX_PBbbpeFa7X4CO65QhyOD8,111
-idlib/systems/doi.py,sha256=PQAstSUocPpK9Ug7L-eKvJrQhn3jRU9WuKITGyZoL9g,13100
+idlib/systems/doi.py,sha256=5Ut8u6m4X_RWJv6yGOTM1MktrJlWnORPfYdmeMT9m_Y,13858
 idlib/systems/handle.py,sha256=Gg97IuysdAdCX9j6EhuNfIddb1GDq0CmwW1SmuLkpgM,95
 idlib/systems/isni.py,sha256=TgiSTlDcSBLRro3gg_YfQJQkfaJ3saw0Au1OMhE23wU,89
 idlib/systems/orcid.py,sha256=aTVu_Q6fnaQvqmd6jeXdst6yKRTO-BEUMs7_4fdQK5E,7886
 idlib/systems/pmid.py,sha256=Na-4_n7qmJ3VAbG0RUwnNTg7o4Z8OkuL_Hluq_A3IO0,181
 idlib/systems/ror.py,sha256=3pZ1aLg7NUh-mHQm-qu-hqy5WrJZprtj5lWIWRpP2-M,8894
-idlib/systems/rrid.py,sha256=RcDn3-BRmrKzzxDCypSWRQuivKvWqUqqohY8BeSqRF4,8278
+idlib/systems/rrid.py,sha256=BtQzrS90sAGKvnoH5CQK0URxTEw1A0hwNaEngxuGLFs,9136
 idlib/systems/urn.py,sha256=p222KjHgfF6y9srCm7kFWuWA9NK4lI-wEQUtSeuBqYk,113
 idlib/systems/web.py,sha256=vrSbQYWnxhe9pSgZcOqwLM3V6XQDLwv1QqWlhWWxbL0,1291
-idlib-0.0.1.dev23.dist-info/licenses/LICENSE,sha256=nEuJBsoHfpXGgf6ZyFGUhQJqxFWwhSYF-45pRAqTSb4,1080
-idlib-0.0.1.dev23.dist-info/METADATA,sha256=leTa1rrRwvU4n5ucULjBXh9zzBHvsjXNE8nj3-qWPog,2330
-idlib-0.0.1.dev23.dist-info/WHEEL,sha256=JNWh1Fm1UdwIQV075glCn4MVuCRs0sotJIq-J6rbxCU,109
-idlib-0.0.1.dev23.dist-info/top_level.txt,sha256=xIk87gJPbeCKih9BVMP8PlEFdpcsGplenz0VQhx5sP8,6
-idlib-0.0.1.dev23.dist-info/RECORD,,
+idlib-0.0.1.dev24.dist-info/licenses/LICENSE,sha256=nEuJBsoHfpXGgf6ZyFGUhQJqxFWwhSYF-45pRAqTSb4,1080
+idlib-0.0.1.dev24.dist-info/METADATA,sha256=CW7UYjKLSBcisjzs4NZ-SkT-yIXKIxryIhuOO3D-mn8,2330
+idlib-0.0.1.dev24.dist-info/WHEEL,sha256=4YBfCYNH4wlLpv3pzq1hbEuIlXA4WJabKLFurZ7eTL0,109
+idlib-0.0.1.dev24.dist-info/top_level.txt,sha256=xIk87gJPbeCKih9BVMP8PlEFdpcsGplenz0VQhx5sP8,6
+idlib-0.0.1.dev24.dist-info/RECORD,,

{idlib-0.0.1.dev23.dist-info → idlib-0.0.1.dev24.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (80.9.0)
+Generator: setuptools (84.0.0)
 Root-Is-Purelib: true
 Tag: py2-none-any
 Tag: py3-none-any