dcor-shared 0.12.2__tar.gz → 0.12.3__tar.gz

{dcor_shared-0.12.2 → dcor_shared-0.12.3}/CHANGELOG

@@ -1,3 +1,5 @@
+0.12.3
+ - enh: improved resource availability check
 0.12.2
  - fix: testing did not append S3 resources
 0.12.1

{dcor_shared-0.12.2 → dcor_shared-0.12.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dcor_shared
-Version: 0.12.2
+Version: 0.12.3
 Summary: Functionalities shared by the DCOR CKAN extensions
 Author: Paul Müller
 Maintainer-email: Paul Müller <dev@craban.de>

{dcor_shared-0.12.2 → dcor_shared-0.12.3}/dcor_shared/__init__.py

@@ -2,9 +2,10 @@
 from .ckan import (
     get_ckan_config_option, get_resource_dc_config, get_resource_path, get_resource_info
 )
-from .data import DUMMY_BYTES, sha256sum, wait_for_resource
+from .data import DUMMY_BYTES, wait_for_resource
 from .dcinst import get_dc_instance
 from .mime import DC_MIME_TYPES, VALID_FORMATS
 from . import paths
 from .rqjob import RQJob, rqjob_register
+from .util import sha256sum
 from ._version import version as __version__

{dcor_shared-0.12.2 → dcor_shared-0.12.3}/dcor_shared/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.12.2'
-__version_tuple__ = version_tuple = (0, 12, 2)
+__version__ = version = '0.12.3'
+__version_tuple__ = version_tuple = (0, 12, 3)

{dcor_shared-0.12.2 → dcor_shared-0.12.3}/dcor_shared/data.py

@@ -1,24 +1,16 @@
 import functools
-import hashlib
 import pathlib
 import time
 
 from .ckan import get_resource_path
+from . import s3cc
+from .util import sha256sum  # noqa: F401
 
 
 #: Content of the dummy file created when importing data.
 DUMMY_BYTES = b"[Data import pending]"
 
 
-def sha256sum(path):
-    """Compute the SHA256 hash of a file in 1MB chunks"""
-    file_hash = hashlib.sha256()
-    with open(path, "rb") as fd:
-        while data := fd.read(2 ** 20):
-            file_hash.update(data)
-    return file_hash.hexdigest()
-
-
 @functools.lru_cache(maxsize=100)
 def wait_for_resource(resource_id: str,
                       timeout: float = 10):
@@ -67,13 +59,23 @@ def wait_for_resource(resource_id: str,
             time.sleep(5)
             continue
 
+        # object exists in database?
         s3_ok = res_dict.get("s3_available", None)
+        if s3_ok:
+            # If the resource is on S3, it is considered to be available.
+            break
+
+        # object exists on S3?
+        try:
+            s3_exist = s3cc.artifact_exists(res_dict["id"])
+        except BaseException:
+            s3_exist = False
+        if s3_exist:
+            break
+
         if time.time() - t0 > timeout:
             raise OSError("Data import seems to take too long "
                           "for '{}'!".format(path))
-        elif s3_ok is not None:
-            # If the dataset is on S3, it is considered to be available.
-            break
         elif not path.exists():
             time.sleep(5)
             continue

{dcor_shared-0.12.2 → dcor_shared-0.12.3}/dcor_shared/s3.py

@@ -14,7 +14,7 @@ import boto3
 import botocore.exceptions
 
 from .ckan import get_ckan_config_option
-from .data import sha256sum
+from .util import sha256sum
 
 
 def compute_checksum(bucket_name, object_name, max_size=None):

{dcor_shared-0.12.2 → dcor_shared-0.12.3}/dcor_shared/testing.py

@@ -8,6 +8,7 @@ import uuid
 import warnings
 
 import ckan.authz
+import ckan.model
 import ckan.tests.factories as factories
 import ckan.tests.helpers as helpers
 from ckan.tests.pytest_ckan.fixtures import FakeFileStorage
@@ -134,7 +135,9 @@ def make_dataset_via_s3(
     """
     if create_context is None:
         user = factories.User()
+        user_obj = ckan.model.User.by_name(user["name"])
         create_context = {'ignore_auth': False,
+                          'auth_user_obj': user_obj,
                           'user': user['name'],
                           'api_version': 3}
         user_id = user["id"]
@@ -144,10 +147,11 @@ def make_dataset_via_s3(
             create_context["user"], allow_none=True)
 
     if owner_org is None:
-        owner_org = factories.Organization(users=[{
-            'name': user_id,
-            'capacity': 'admin'
-        }])
+        owner_org = factories.Organization(
+            users=[{
+                'name': user_id,
+                'capacity': 'admin'
+            }])
 
     if "title" not in kwargs:
         kwargs["title"] = "test-dataset-S3"

dcor_shared-0.12.3/dcor_shared/util.py

@@ -0,0 +1,10 @@
+import hashlib
+
+
+def sha256sum(path):
+    """Compute the SHA256 hash of a file in 1MB chunks"""
+    file_hash = hashlib.sha256()
+    with open(path, "rb") as fd:
+        while data := fd.read(2 ** 20):
+            file_hash.update(data)
+    return file_hash.hexdigest()

{dcor_shared-0.12.2 → dcor_shared-0.12.3}/dcor_shared.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dcor_shared
-Version: 0.12.2
+Version: 0.12.3
 Summary: Functionalities shared by the DCOR CKAN extensions
 Author: Paul Müller
 Maintainer-email: Paul Müller <dev@craban.de>

{dcor_shared-0.12.2 → dcor_shared-0.12.3}/dcor_shared.egg-info/SOURCES.txt

@@ -18,6 +18,7 @@ dcor_shared/rqjob.py
 dcor_shared/s3.py
 dcor_shared/s3cc.py
 dcor_shared/testing.py
+dcor_shared/util.py
 dcor_shared.egg-info/PKG-INFO
 dcor_shared.egg-info/SOURCES.txt
 dcor_shared.egg-info/dependency_links.txt