dcor-shared 0.7.2__tar.gz → 0.7.4__tar.gz

{dcor_shared-0.7.2 → dcor_shared-0.7.4}/CHANGELOG

@@ -1,3 +1,8 @@
+0.7.4
+ - fix: disable basins when loading a dataset instance
+ - enh: implement s3cc.upload_artifact
+0.7.3
+ - enh: add get_dc_instance convenience method
 0.7.2
  - tests: fix invalid make_dataset method
 0.7.1

{dcor_shared-0.7.2 → dcor_shared-0.7.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dcor_shared
-Version: 0.7.2
+Version: 0.7.4
 Summary: Functionalities shared by the DCOR CKAN extensions
 Home-page: https://github.com/DCOR-dev/dcor_shared
 Author: Paul Müller

{dcor_shared-0.7.2 → dcor_shared-0.7.4}/dcor_shared/__init__.py

@@ -1,6 +1,7 @@
 # flake8: noqa: F401
 from .ckan import get_ckan_config_option, get_resource_path
 from .data import DUMMY_BYTES, sha256sum, wait_for_resource
+from .dcinst import get_dc_instance
 from .mime import DC_MIME_TYPES, VALID_FORMATS
 from . import paths
 from ._version import version as __version__

{dcor_shared-0.7.2 → dcor_shared-0.7.4}/dcor_shared/_version_save.py

@@ -1,3 +1,3 @@
 #!/usr/bin/env python
 # This file was created automatically
-longversion = '0.7.2'
+longversion = '0.7.4'

dcor_shared-0.7.4/dcor_shared/dcinst.py

@@ -0,0 +1,20 @@
+from dclab.rtdc_dataset import RTDC_HDF5
+
+from .ckan import get_resource_path
+from . import s3cc
+
+
+def get_dc_instance(rid):
+    """Return an instance of dclab's `RTDCBase` for a resource identifier"""
+    # Try local file first
+    path = get_resource_path(rid)
+    if path.is_file():
+        # Disable basins, because they could point to files on the local
+        # file system (security).
+        return RTDC_HDF5(path, enable_basins=False)
+    else:
+        # The resource must be on S3
+        if s3cc.artifact_exists(rid):
+            return s3cc.get_s3_dc_handle(rid)
+        else:
+            raise ValueError(f"Could not find resource {rid} anywhere")

{dcor_shared-0.7.2 → dcor_shared-0.7.4}/dcor_shared/s3cc.py

@@ -3,16 +3,32 @@
 Contains methods to directly interact with CKAN resources that are on S3
 via just the resource ID.
 """
+from __future__ import annotations
 import functools
-
+import pathlib
 from typing import Literal
+import warnings
 
 from dclab.rtdc_dataset import fmt_s3
 
 from .ckan import get_ckan_config_option
+from .data import sha256sum
 from . import s3
 
 
+def artifact_exists(
+        resource_id: str,
+        artifact: Literal["condensed", "preview", "resource"] = "resource"):
+    """Check whether an artifact is available on S3
+
+    The resource with the identifier `resource_id` must exist in the
+    CKAN database.
+    """
+    bucket_name, object_name = get_s3_bucket_object_for_artifact(
+        resource_id=resource_id, artifact=artifact)
+    return s3.object_exists(bucket_name=bucket_name, object_name=object_name)
+
+
 def compute_checksum(resource_id):
     """Compute the SHA256 checksum of the corresponding CKAN resource"""
     bucket_name, object_name = get_s3_bucket_object_for_artifact(
@@ -91,6 +107,9 @@ def get_s3_dc_handle(resource_id):
             "dcor_object_store.access_key_id"),
         secret_key=get_ckan_config_option(
             "dcor_object_store.secret_access_key"),
+        # Disable basins, because they could point to files on the
+        # local file system (security).
+        enable_basins=False,
     )
     return ds
 
@@ -130,11 +149,64 @@ def make_resource_public(resource_id: str,
 def object_exists(
         resource_id: str,
         artifact: Literal["condensed", "preview", "resource"] = "resource"):
-    """Check whether an object is available on S3
+    """Check whether an artifact is available on S3
 
     The resource with the identifier `resource_id` must exist in the
     CKAN database.
     """
+    warnings.warn("`s3cc.object_exists` is deprecated, please use"
+                  "`s3cc.artifact_exists` instead",
+                  DeprecationWarning)
+    return artifact_exists(resource_id, artifact)
+
+
+def upload_artifact(
+        resource_id: str,
+        path_artifact: str | pathlib.Path,
+        artifact: Literal["condensed", "preview", "resource"] = "resource",
+        sha256: str = None,
+        private: bool = None,
+        override: bool = False
+):
+    """Upload an artifact to S3
+
+    Parameters
+    ----------
+    resource_id: str
+        The resource identifier for the artifact
+    path_artifact: pathlib.Path
+        The path to the artifact file
+    artifact: str
+        The artifact type that the file represents
+    sha256: str
+        The SHA256 sum of `path_artifcat`, will be computed if not provided
+    private: bool
+        Whether the dataset that the resource belongs to is private.
+        Leave this blank if you don't know and we will do a database
+        look-up to determine the correct value.
+    override: bool
+        Whether to override a possibly existing object on S3.
+    """
     bucket_name, object_name = get_s3_bucket_object_for_artifact(
         resource_id=resource_id, artifact=artifact)
-    return s3.object_exists(bucket_name=bucket_name, object_name=object_name)
+
+    if private is None:
+        # User did not say whether the resource is private. We have to
+        # find out ourselves.
+        import ckan.logic
+        res_dict = ckan.logic.get_action('resource_show')(
+            context={'ignore_auth': True, 'user': 'default'},
+            data_dict={"id": resource_id})
+        ds_dict = ckan.logic.get_action('package_show')(
+            context={'ignore_auth': True, 'user': 'default'},
+            data_dict={'id': res_dict["package_id"]})
+        private = ds_dict["private"]
+
+    rid = resource_id
+    s3.upload_file(
+        bucket_name=bucket_name,
+        object_name=f"{artifact}/{rid[:3]}/{rid[3:6]}/{rid[6:]}",
+        path=path_artifact,
+        sha256=sha256 or sha256sum(path_artifact),
+        private=private,
+        override=override)

{dcor_shared-0.7.2 → dcor_shared-0.7.4}/dcor_shared.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dcor-shared
-Version: 0.7.2
+Version: 0.7.4
 Summary: Functionalities shared by the DCOR CKAN extensions
 Home-page: https://github.com/DCOR-dev/dcor_shared
 Author: Paul Müller

{dcor_shared-0.7.2 → dcor_shared-0.7.4}/dcor_shared.egg-info/SOURCES.txt

@@ -9,6 +9,7 @@ dcor_shared/_version.py
 dcor_shared/_version_save.py
 dcor_shared/ckan.py
 dcor_shared/data.py
+dcor_shared/dcinst.py
 dcor_shared/mime.py
 dcor_shared/parse.py
 dcor_shared/paths.py

dcor_shared-0.7.4/tests/test_data.py

@@ -0,0 +1,183 @@
+import atexit
+import pathlib
+import shutil
+from unittest import mock
+import uuid
+
+from ckan import logic
+import ckanext.dcor_schemas.plugin
+
+from dcor_shared import (
+    get_dc_instance, get_resource_path, s3, sha256sum, wait_for_resource
+)
+
+import pytest
+import ckan.tests.factories as factories
+from dcor_shared.testing import make_dataset, synchronous_enqueue_job
+
+data_path = pathlib.Path(__file__).parent / "data"
+
+
+@pytest.mark.ckan_config('ckan.plugins', 'dcor_depot dcor_schemas')
+@pytest.mark.usefixtures('clean_db', 'with_request_context')
+@mock.patch('ckan.plugins.toolkit.enqueue_job',
+            side_effect=synchronous_enqueue_job)
+def test_get_dc_instance_file(enqueue_job_mock, create_with_upload,
+                              monkeypatch):
+    monkeypatch.setattr(
+        ckanext.dcor_schemas.plugin,
+        'DISABLE_AFTER_DATASET_CREATE_FOR_CONCURRENT_JOB_TESTS',
+        True)
+
+    user = factories.User()
+    owner_org = factories.Organization(users=[{
+        'name': user['id'],
+        'capacity': 'admin'
+    }])
+    create_context = {'ignore_auth': False,
+                      'user': user['name'],
+                      'api_version': 3}
+    ds_dict, _ = make_dataset(
+        create_context, owner_org,
+        create_with_upload=create_with_upload,
+        resource_path=data_path / "calibration_beads_47.rtdc",
+        activate=True)
+    rid = ds_dict["resources"][0]["id"]
+    resource_path = pathlib.Path(get_resource_path(rid))
+    assert resource_path.exists(), "sanity check"
+    with get_dc_instance(rid) as ds:
+        assert str(ds.path) == str(resource_path)
+
+
+@pytest.mark.ckan_config('ckan.storage_path',
+                         '/tmp/test_dcor_shared/test_get_dc_instance')
+def test_get_dc_instance_file_fails_without_actual_resource():
+    tmp_path = pathlib.Path("/tmp/test_dcor_shared/test_get_dc_instance")
+    atexit.register(shutil.rmtree,
+                    tmp_path,
+                    ignore_errors=True)
+    rid = str(uuid.uuid4())
+    resource_path = tmp_path / rid[:3] / rid[3:6] / rid[6:]
+    resource_path.parent.mkdir(parents=True)
+    shutil.copy2(data_path / "calibration_beads_47.rtdc", resource_path)
+    with pytest.raises(logic.NotFound):
+        get_dc_instance(rid)
+
+
+@pytest.mark.ckan_config('ckan.plugins', 'dcor_depot dcor_schemas')
+@pytest.mark.usefixtures('clean_db', 'with_request_context')
+@mock.patch('ckan.plugins.toolkit.enqueue_job',
+            side_effect=synchronous_enqueue_job)
+def test_get_dc_instance_s3(enqueue_job_mock, create_with_upload,
+                            monkeypatch):
+    monkeypatch.setattr(
+        ckanext.dcor_schemas.plugin,
+        'DISABLE_AFTER_DATASET_CREATE_FOR_CONCURRENT_JOB_TESTS',
+        True)
+
+    user = factories.User()
+    owner_org = factories.Organization(users=[{
+        'name': user['id'],
+        'capacity': 'admin'
+    }])
+    create_context = {'ignore_auth': False,
+                      'user': user['name'],
+                      'api_version': 3}
+    ds_dict, _ = make_dataset(
+        create_context, owner_org,
+        create_with_upload=create_with_upload,
+        resource_path=data_path / "calibration_beads_47.rtdc",
+        activate=True)
+    res_dict = ds_dict["resources"][0]
+    rid = res_dict["id"]
+    resource_path = pathlib.Path(get_resource_path(rid))
+    # remove the file, so DCOR falls back to the S3 resource
+    resource_path.unlink()
+    assert not resource_path.exists(), "sanity check"
+    with get_dc_instance(rid) as ds:
+        assert ds.path.startswith("http")
+        assert res_dict["s3_available"]
+        assert res_dict["s3_url"] == ds.path
+
+
+@pytest.mark.ckan_config('ckan.plugins', 'dcor_depot dcor_schemas')
+@pytest.mark.usefixtures('clean_db', 'with_request_context')
+@mock.patch('ckan.plugins.toolkit.enqueue_job',
+            side_effect=synchronous_enqueue_job)
+def test_get_dc_instance_s3_fails_without_actual_resource(enqueue_job_mock):
+    user = factories.User()
+    owner_org = factories.Organization(users=[{
+        'name': user['id'],
+        'capacity': 'admin'
+    }])
+    rid = str(uuid.uuid4())
+    bucket_name = f"circle-{owner_org['id']}"
+    object_name = f"resource/{rid[:3]}/{rid[3:6]}/{rid[6:]}"
+    upload_path = data_path / "calibration_beads_47.rtdc"
+    s3.upload_file(bucket_name=bucket_name,
+                   object_name=object_name,
+                   path=upload_path,
+                   sha256=sha256sum(upload_path)
+                   )
+    with pytest.raises(logic.NotFound):
+        get_dc_instance(rid)
+
+
+def test_sha256sum(tmp_path):
+    p = tmp_path / "test.txt"
+    p.write_text("Sum this up!")
+    ist = sha256sum(p)
+    soll = "d00df55b97a60c78bbb137540e1b60647a5e6b216262a95ab96cafd4519bcf6a"
+    assert ist == soll
+
+
+def test_wait_for_resource_s3(monkeypatch):
+    res_id = str(uuid.uuid4())
+    monkeypatch.setattr(logic, "get_action",
+                        lambda x: lambda context, data_dict: {
+                            "id": res_id,
+                            "s3_available": True})
+    # Should not raise an exception
+    wait_for_resource(res_id)
+
+
+@pytest.mark.ckan_config('ckan.plugins', '')  # disable plugins
+def test_wait_for_resource_no_s3_url_no_plugins(monkeypatch):
+    res_id = str(uuid.uuid4())
+    monkeypatch.setattr(logic, "get_action",
+                        lambda x: lambda context, data_dict: {
+                            "id": res_id})
+    # Create the actual path
+    rp = pathlib.Path(get_resource_path(res_id))
+    rp.parent.mkdir(parents=True, exist_ok=True)
+    rp.write_bytes(b"This is not a dummy file, but a fake DC file!")
+    # Should not raise an exception
+    wait_for_resource(res_id)
+
+
+@pytest.mark.ckan_config('ckan.plugins', 'dcor_depot')
+def test_wait_for_resource_no_s3_url_with_dcor_depot_plugin(monkeypatch):
+    res_id = str(uuid.uuid4())
+    monkeypatch.setattr(logic, "get_action",
+                        lambda x: lambda context, data_dict: {
+                            "id": res_id})
+    # Create the actual path
+    rp = pathlib.Path(get_resource_path(res_id))
+    rp.parent.mkdir(parents=True, exist_ok=True)
+    rp.write_bytes(b"This is not a dummy file, but a fake DC file!")
+    rp2 = rp.with_name("test.rtdc")
+    rp.rename(rp2)
+    rp.symlink_to(rp2)
+
+    # Should not raise an exception
+    wait_for_resource(res_id)
+
+
+def test_wait_resource_not_available(monkeypatch):
+    res_id = str(uuid.uuid4())
+    monkeypatch.setattr(logic, "get_action",
+                        lambda x: lambda context, data_dict: {
+                            "id": res_id})
+    # Should raise an exception
+    with pytest.raises(OSError, match="Data import seems to take too long"):
+        wait_for_resource(res_id, timeout=1)

{dcor_shared-0.7.2 → dcor_shared-0.7.4}/tests/test_s3cc.py

@@ -16,6 +16,23 @@ import requests
 data_path = pathlib.Path(__file__).parent / "data"
 
 
+@pytest.mark.ckan_config('ckan.plugins', 'dcor_schemas')
+@pytest.mark.usefixtures('clean_db', 'with_request_context')
+@mock.patch('ckan.plugins.toolkit.enqueue_job',
+            side_effect=synchronous_enqueue_job)
+def test_artifact_exists(enqueue_job_mock):
+    rid, s3_url, _, org_dict = setup_s3_resource_on_ckan(private=True)
+    assert s3cc.artifact_exists(rid)
+    # Delete the object
+    s3_client, _, _ = s3.get_s3()
+    bucket_name, object_name = s3cc.get_s3_bucket_object_for_artifact(rid)
+    s3_client.delete_object(
+        Bucket=bucket_name,
+        Key=object_name
+    )
+    assert not s3cc.artifact_exists(rid)
+
+
 def setup_s3_resource_on_ckan(private=False):
     """Create an S3 resource in CKAN"""
     user = factories.User()
@@ -151,14 +168,38 @@ def test_make_resource_public(enqueue_job_mock):
 @pytest.mark.usefixtures('clean_db', 'with_request_context')
 @mock.patch('ckan.plugins.toolkit.enqueue_job',
             side_effect=synchronous_enqueue_job)
-def test_object_exists(enqueue_job_mock):
+def test_upload_artifact(enqueue_job_mock, tmp_path):
     rid, s3_url, _, org_dict = setup_s3_resource_on_ckan(private=True)
-    assert s3cc.object_exists(rid)
-    # Delete the object
-    s3_client, _, _ = s3.get_s3()
-    bucket_name, object_name = s3cc.get_s3_bucket_object_for_artifact(rid)
-    s3_client.delete_object(
-        Bucket=bucket_name,
-        Key=object_name
-    )
-    assert not s3cc.object_exists(rid)
+    path_fake_preview = tmp_path / "test_preview.jpg"
+    path_fake_preview.write_text("This is not a real image!")
+    # upload the preview
+    s3cc.upload_artifact(rid,
+                         path_artifact=path_fake_preview,
+                         artifact="preview")
+    # make sure that worked
+    assert s3cc.artifact_exists(rid, "preview")
+    # attempt to download the private artifact
+    resp1 = requests.get(s3_url.replace("resource", "preview"))
+    assert not resp1.ok, "sanity check"
+
+
+@pytest.mark.ckan_config('ckan.plugins', 'dcor_schemas')
+@pytest.mark.usefixtures('clean_db', 'with_request_context')
+@mock.patch('ckan.plugins.toolkit.enqueue_job',
+            side_effect=synchronous_enqueue_job)
+def test_upload_artifact_public(enqueue_job_mock, tmp_path):
+    rid, s3_url, _, org_dict = setup_s3_resource_on_ckan(private=True)
+    path_fake_preview = tmp_path / "test_preview.jpg"
+    path_fake_preview.write_text("This is not a real image!")
+    # upload the preview
+    s3cc.upload_artifact(rid,
+                         path_artifact=path_fake_preview,
+                         artifact="preview",
+                         # force public resource even though dataset is not
+                         # (this has no real-life use case)
+                         private=False)
+    # make sure that worked
+    assert s3cc.artifact_exists(rid, "preview")
+    # attempt to download the private artifact
+    resp1 = requests.get(s3_url.replace("resource", "preview"))
+    assert resp1.ok, "preview should be public"

dcor_shared-0.7.2/tests/test_data.py

@@ -1,68 +0,0 @@
-import pathlib
-import uuid
-
-from ckan import logic
-
-import pytest
-
-from dcor_shared import get_resource_path, sha256sum, wait_for_resource
-
-
-def test_sha256sum(tmp_path):
-    p = tmp_path / "test.txt"
-    p.write_text("Sum this up!")
-    ist = sha256sum(p)
-    soll = "d00df55b97a60c78bbb137540e1b60647a5e6b216262a95ab96cafd4519bcf6a"
-    assert ist == soll
-
-
-def test_wait_for_resource_s3(monkeypatch):
-    res_id = str(uuid.uuid4())
-    monkeypatch.setattr(logic, "get_action",
-                        lambda x: lambda context, data_dict: {
-                            "id": res_id,
-                            "s3_available": True})
-    # Should not raise an exception
-    wait_for_resource(res_id)
-
-
-@pytest.mark.ckan_config('ckan.plugins', '')  # disable plugins
-def test_wait_for_resource_no_s3_url_no_plugins(monkeypatch):
-    res_id = str(uuid.uuid4())
-    monkeypatch.setattr(logic, "get_action",
-                        lambda x: lambda context, data_dict: {
-                            "id": res_id})
-    # Create the actual path
-    rp = pathlib.Path(get_resource_path(res_id))
-    rp.parent.mkdir(parents=True, exist_ok=True)
-    rp.write_bytes(b"This is not a dummy file, but a fake DC file!")
-    # Should not raise an exception
-    wait_for_resource(res_id)
-
-
-@pytest.mark.ckan_config('ckan.plugins', 'dcor_depot')
-def test_wait_for_resource_no_s3_url_with_dcor_depot_plugin(monkeypatch):
-    res_id = str(uuid.uuid4())
-    monkeypatch.setattr(logic, "get_action",
-                        lambda x: lambda context, data_dict: {
-                            "id": res_id})
-    # Create the actual path
-    rp = pathlib.Path(get_resource_path(res_id))
-    rp.parent.mkdir(parents=True, exist_ok=True)
-    rp.write_bytes(b"This is not a dummy file, but a fake DC file!")
-    rp2 = rp.with_name("test.rtdc")
-    rp.rename(rp2)
-    rp.symlink_to(rp2)
-
-    # Should not raise an exception
-    wait_for_resource(res_id)
-
-
-def test_wait_resource_not_available(monkeypatch):
-    res_id = str(uuid.uuid4())
-    monkeypatch.setattr(logic, "get_action",
-                        lambda x: lambda context, data_dict: {
-                            "id": res_id})
-    # Should raise an exception
-    with pytest.raises(OSError, match="Data import seems to take too long"):
-        wait_for_resource(res_id, timeout=1)