dcor-shared 0.5.6__tar.gz → 0.6.0__tar.gz

{dcor_shared-0.5.6 → dcor_shared-0.6.0}/CHANGELOG

@@ -1,3 +1,8 @@
+0.6.0
+ - BREAKING CHANGE: wait_for_resource now requires the resource_id
+   instead of the resource path
+ - BREAKING CHANGE: ckan installation is now mandatory
+ - enh: cache calls to data.wait_for_resource
 0.5.6
  - maintenance release
 0.5.5

{dcor_shared-0.5.6 → dcor_shared-0.6.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dcor_shared
-Version: 0.5.6
+Version: 0.6.0
 Summary: Functionalities shared by the DCOR CKAN extensions
 Home-page: https://github.com/DCOR-dev/dcor_shared
 Author: Paul Müller

{dcor_shared-0.5.6 → dcor_shared-0.6.0}/dcor_shared/_version_save.py

@@ -1,3 +1,3 @@
 #!/usr/bin/env python
 # This file was created automatically
-longversion = '0.5.6'
+longversion = '0.6.0'

{dcor_shared-0.5.6 → dcor_shared-0.6.0}/dcor_shared/ckan.py

@@ -1,10 +1,7 @@
 import os
 import pathlib
 
-try:
-    from ckan import common as ckan_common
-except ImportError:
-    ckan_common = None
+from ckan import common as ckan_common
 
 from .parse import get_ini_config_option
 
@@ -16,7 +13,7 @@ def get_ckan_config_option(option):
     taken from there. Else, the configuration is taken from the
     configuration file returned by :func:`get_ckan_config_path`.
     """
-    if ckan_common and ckan_common.config:
+    if ckan_common.config:
         # Get from current configuration (The `get` method in CKAN 2.10
         # returns default values if not placeholder is specified)
         opt = ckan_common.config.get(option)

dcor_shared-0.6.0/dcor_shared/data.py

@@ -0,0 +1,93 @@
+import functools
+import hashlib
+import pathlib
+import time
+import warnings
+
+from ckan.common import config
+from ckan import logic
+
+from .ckan import get_resource_path
+
+
+#: Content of the dummy file created when importing data.
+DUMMY_BYTES = b"[Data import pending]"
+
+
+def sha256sum(path):
+    """Compute the SHA256 hash of a file in 1MB chunks"""
+    file_hash = hashlib.sha256()
+    with open(path, "rb") as fd:
+        while data := fd.read(2 ** 20):
+            file_hash.update(data)
+    return file_hash.hexdigest()
+
+
+@functools.lru_cache(maxsize=100)
+def wait_for_resource(resource_id: str,
+                      timeout: float = 10):
+    """Wait for resource to be available
+
+    This function can be used by other plugins to ensure that
+    a resource is available for processing.
+
+    There multiple ways for data to become available:
+
+    1. The ckanext-dcor_depot plugin imports data by touching
+       dummy files and then sym-linking to data on disk. Here
+       we just check that the file is not a dummy file anymore.
+    2. Legacy uploads via nginx/uwsgi directly into CKAN and onto
+       the local block storage worked the same way. We have to wait
+       for the dummy file to be replaced.
+    3. The new (2024) way of uploading data is via pre-signed URLs
+       to an S3 instance. Here, we have to make sure that the
+       upload is complete and the file exists. If this is the case,
+       then uploads should have already completed when this function
+       is called, so we only check for the existence of the resource
+       in ckan and whether the `s3_available` attribute is defined.
+    """
+    if len(resource_id) != 36:
+        warnings.warn("Please pass the CKAN resource id as `resource_id` "
+                      "instead of a string or a path. This will raise an "
+                      "exception in the future.",
+                      DeprecationWarning)
+        # The user passed a local path on the block storage. These look like
+        # this: /data/dcor-instance/resources/RE/SO/URCE-ID
+        resource_id = "".join(resource_id.split("/")[-3])
+
+    resource_show = logic.get_action("resource_show")
+    path = pathlib.Path(get_resource_path(resource_id))
+
+    dcor_depot_available = "dcor_depot" in config.get('ckan.plugins', "")
+    # Initially this was set to 10s, but if `/data` is mounted on a
+    # network share then this part here just takes too long.
+    t0 = time.time()
+    ld = len(DUMMY_BYTES)
+    while True:
+        res_dict = resource_show(context={'ignore_auth': True,
+                                          'user': 'default'},
+                                 data_dict={"id": resource_id})
+
+        s3_url = res_dict.get("s3_url", None)
+        if time.time() - t0 > timeout:
+            raise OSError("Data import seems to take too long "
+                          "for '{}'!".format(path))
+        elif s3_url is not None:
+            # If the dataset is on S3, it is considered to be available.
+            break
+        elif not path.exists():
+            time.sleep(0.05)
+            continue
+        elif dcor_depot_available and not path.is_symlink():
+            # Resource is only available when it is symlinked by
+            # the ckanext.dcor_depot `symlink_user_dataset` job
+            # (or by the ckanext.dcor_depot importers).
+            time.sleep(0.05)
+            continue
+        elif path.stat().st_size == ld and path.read_bytes() == DUMMY_BYTES:
+            # wait a bit
+            time.sleep(0.01)
+            continue
+        else:
+            # not a dummy file
+            break

{dcor_shared-0.5.6 → dcor_shared-0.6.0}/dcor_shared/paths.py

@@ -6,10 +6,7 @@ from .ckan import get_ckan_storage_path, get_ckan_webassets_path  # noqa: F401
 from .parse import get_ini_config_option
 
 
-try:
-    from ckan.common import config as ckan_config
-except ImportError:
-    ckan_config = None
+from ckan.common import config as ckan_config  # noqa: F401
 
 
 def get_dcor_depot_path():

{dcor_shared-0.5.6 → dcor_shared-0.6.0}/dcor_shared.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dcor-shared
-Version: 0.5.6
+Version: 0.6.0
 Summary: Functionalities shared by the DCOR CKAN extensions
 Home-page: https://github.com/DCOR-dev/dcor_shared
 Author: Paul Müller

dcor_shared-0.6.0/tests/test_data.py

@@ -0,0 +1,68 @@
+import pathlib
+import uuid
+
+from ckan import logic
+
+import pytest
+
+from dcor_shared import get_resource_path, sha256sum, wait_for_resource
+
+
+def test_sha256sum(tmp_path):
+    p = tmp_path / "test.txt"
+    p.write_text("Sum this up!")
+    ist = sha256sum(p)
+    soll = "d00df55b97a60c78bbb137540e1b60647a5e6b216262a95ab96cafd4519bcf6a"
+    assert ist == soll
+
+
+def test_wait_for_resource_s3(monkeypatch):
+    res_id = str(uuid.uuid4())
+    monkeypatch.setattr(logic, "get_action",
+                        lambda x: lambda context, data_dict: {
+                            "id": res_id,
+                            "s3_url": "https://bucket.name/object"})
+    # Should not raise an exception
+    wait_for_resource(res_id)
+
+
+@pytest.mark.ckan_config('ckan.plugins', '')  # disable plugins
+def test_wait_for_resource_no_s3_url_no_plugins(monkeypatch):
+    res_id = str(uuid.uuid4())
+    monkeypatch.setattr(logic, "get_action",
+                        lambda x: lambda context, data_dict: {
+                            "id": res_id})
+    # Create the actual path
+    rp = pathlib.Path(get_resource_path(res_id))
+    rp.parent.mkdir(parents=True, exist_ok=True)
+    rp.write_bytes(b"This is not a dummy file, but a fake DC file!")
+    # Should not raise an exception
+    wait_for_resource(res_id)
+
+
+@pytest.mark.ckan_config('ckan.plugins', 'dcor_depot')
+def test_wait_for_resource_no_s3_url_with_dcor_depot_plugin(monkeypatch):
+    res_id = str(uuid.uuid4())
+    monkeypatch.setattr(logic, "get_action",
+                        lambda x: lambda context, data_dict: {
+                            "id": res_id})
+    # Create the actual path
+    rp = pathlib.Path(get_resource_path(res_id))
+    rp.parent.mkdir(parents=True, exist_ok=True)
+    rp.write_bytes(b"This is not a dummy file, but a fake DC file!")
+    rp2 = rp.with_name("test.rtdc")
+    rp.rename(rp2)
+    rp.symlink_to(rp2)
+
+    # Should not raise an exception
+    wait_for_resource(res_id)
+
+
+def test_wait_resource_not_available(monkeypatch):
+    res_id = str(uuid.uuid4())
+    monkeypatch.setattr(logic, "get_action",
+                        lambda x: lambda context, data_dict: {
+                            "id": res_id})
+    # Should raise an exception
+    with pytest.raises(OSError, match="Data import seems to take too long"):
+        wait_for_resource(res_id, timeout=1)

dcor_shared-0.5.6/dcor_shared/data.py

@@ -1,59 +0,0 @@
-import hashlib
-import pathlib
-import time
-
-try:
-    from ckan.common import config
-except ImportError:
-    config = {}
-
-
-#: Content of the dummy file created when importing data.
-DUMMY_BYTES = b"[Data import pending]"
-
-
-def sha256sum(path):
-    """Compute the SHA256 hash of a file in 1MB chunks"""
-    file_hash = hashlib.sha256()
-    with open(path, "rb") as fd:
-        while data := fd.read(2 ** 20):
-            file_hash.update(data)
-    return file_hash.hexdigest()
-
-
-def wait_for_resource(path, timeout=10):
-    """Wait for resource if it is uploaded manually
-
-    This function can be used by other plugins to ensure that
-    a resource is available for processing.
-
-    The ckanext-dcor_depot plugin imports data by uploading
-    dummy files and then sym-linking to data on disk. Here
-    we just check that the file is not a dummy file anymore.
-    """
-    path = pathlib.Path(path)
-    dcor_depot_available = "dcor_depot" in config.get('ckan.plugins', "")
-    # Initially this was set to 10s, but if `/data` is mounted on a
-    # network share then this part here just takes too long.
-    t0 = time.time()
-    ld = len(DUMMY_BYTES)
-    while True:
-        if time.time() - t0 > timeout:
-            raise OSError("Data import seems to take too long "
-                          "for '{}'!".format(path))
-        elif not path.exists():
-            time.sleep(0.05)
-            continue
-        elif dcor_depot_available and not path.is_symlink():
-            # Resource is only available when it is symlinked by
-            # the ckanext.dcor_depot `symlink_user_dataset` job
-            # (or by the ckanext.dcor_depot importers).
-            time.sleep(0.05)
-            continue
-        elif path.stat().st_size == ld and path.read_bytes() == DUMMY_BYTES:
-            # wait a bit
-            time.sleep(0.01)
-            continue
-        else:
-            # not a dummy file
-            break

dcor_shared-0.5.6/tests/test_data.py

@@ -1,9 +0,0 @@
-from dcor_shared import sha256sum
-
-
-def test_sha256sum(tmp_path):
-    p = tmp_path / "test.txt"
-    p.write_text("Sum this up!")
-    ist = sha256sum(p)
-    soll = "d00df55b97a60c78bbb137540e1b60647a5e6b216262a95ab96cafd4519bcf6a"
-    assert ist == soll