dcor-shared 0.7.3__tar.gz → 0.7.4__tar.gz

{dcor_shared-0.7.3 → dcor_shared-0.7.4}/CHANGELOG

@@ -1,3 +1,6 @@
+0.7.4
+ - fix: disable basins when loading a dataset instance
+ - enh: implement s3cc.upload_artifact
 0.7.3
  - enh: add get_dc_instance convenience method
 0.7.2

{dcor_shared-0.7.3 → dcor_shared-0.7.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dcor_shared
-Version: 0.7.3
+Version: 0.7.4
 Summary: Functionalities shared by the DCOR CKAN extensions
 Home-page: https://github.com/DCOR-dev/dcor_shared
 Author: Paul Müller

{dcor_shared-0.7.3 → dcor_shared-0.7.4}/dcor_shared/__init__.py

@@ -1,6 +1,7 @@
 # flake8: noqa: F401
 from .ckan import get_ckan_config_option, get_resource_path
-from .data import DUMMY_BYTES, get_dc_instance, sha256sum, wait_for_resource
+from .data import DUMMY_BYTES, sha256sum, wait_for_resource
+from .dcinst import get_dc_instance
 from .mime import DC_MIME_TYPES, VALID_FORMATS
 from . import paths
 from ._version import version as __version__

{dcor_shared-0.7.3 → dcor_shared-0.7.4}/dcor_shared/_version_save.py

@@ -1,3 +1,3 @@
 #!/usr/bin/env python
 # This file was created automatically
-longversion = '0.7.3'
+longversion = '0.7.4'

{dcor_shared-0.7.3 → dcor_shared-0.7.4}/dcor_shared/data.py

@@ -4,30 +4,13 @@ import pathlib
 import time
 import warnings
 
-from dclab.rtdc_dataset import RTDC_HDF5
-
 from .ckan import get_resource_path
-from . import s3cc
 
 
 #: Content of the dummy file created when importing data.
 DUMMY_BYTES = b"[Data import pending]"
 
 
-def get_dc_instance(rid):
-    """Return an instance of dclab's `RTDCBase` for a resource identifier"""
-    # Try local file first
-    path = get_resource_path(rid)
-    if path.is_file():
-        return RTDC_HDF5(path)
-    else:
-        # The resource must be on S3
-        if s3cc.object_exists(rid):
-            return s3cc.get_s3_dc_handle(rid)
-        else:
-            raise ValueError(f"Could not find resource {rid} anywhere")
-
-
 def sha256sum(path):
     """Compute the SHA256 hash of a file in 1MB chunks"""
     file_hash = hashlib.sha256()

dcor_shared-0.7.4/dcor_shared/dcinst.py

@@ -0,0 +1,20 @@
+from dclab.rtdc_dataset import RTDC_HDF5
+
+from .ckan import get_resource_path
+from . import s3cc
+
+
+def get_dc_instance(rid):
+    """Return an instance of dclab's `RTDCBase` for a resource identifier"""
+    # Try local file first
+    path = get_resource_path(rid)
+    if path.is_file():
+        # Disable basins, because they could point to files on the local
+        # file system (security).
+        return RTDC_HDF5(path, enable_basins=False)
+    else:
+        # The resource must be on S3
+        if s3cc.artifact_exists(rid):
+            return s3cc.get_s3_dc_handle(rid)
+        else:
+            raise ValueError(f"Could not find resource {rid} anywhere")

{dcor_shared-0.7.3 → dcor_shared-0.7.4}/dcor_shared/s3cc.py

@@ -3,16 +3,32 @@
 Contains methods to directly interact with CKAN resources that are on S3
 via just the resource ID.
 """
+from __future__ import annotations
 import functools
-
+import pathlib
 from typing import Literal
+import warnings
 
 from dclab.rtdc_dataset import fmt_s3
 
 from .ckan import get_ckan_config_option
+from .data import sha256sum
 from . import s3
 
 
+def artifact_exists(
+        resource_id: str,
+        artifact: Literal["condensed", "preview", "resource"] = "resource"):
+    """Check whether an artifact is available on S3
+
+    The resource with the identifier `resource_id` must exist in the
+    CKAN database.
+    """
+    bucket_name, object_name = get_s3_bucket_object_for_artifact(
+        resource_id=resource_id, artifact=artifact)
+    return s3.object_exists(bucket_name=bucket_name, object_name=object_name)
+
+
 def compute_checksum(resource_id):
     """Compute the SHA256 checksum of the corresponding CKAN resource"""
     bucket_name, object_name = get_s3_bucket_object_for_artifact(
@@ -91,6 +107,9 @@ def get_s3_dc_handle(resource_id):
             "dcor_object_store.access_key_id"),
         secret_key=get_ckan_config_option(
             "dcor_object_store.secret_access_key"),
+        # Disable basins, because they could point to files on the
+        # local file system (security).
+        enable_basins=False,
     )
     return ds
 
@@ -130,11 +149,64 @@ def make_resource_public(resource_id: str,
 def object_exists(
         resource_id: str,
         artifact: Literal["condensed", "preview", "resource"] = "resource"):
-    """Check whether an object is available on S3
+    """Check whether an artifact is available on S3
 
     The resource with the identifier `resource_id` must exist in the
     CKAN database.
     """
+    warnings.warn("`s3cc.object_exists` is deprecated, please use"
+                  "`s3cc.artifact_exists` instead",
+                  DeprecationWarning)
+    return artifact_exists(resource_id, artifact)
+
+
+def upload_artifact(
+        resource_id: str,
+        path_artifact: str | pathlib.Path,
+        artifact: Literal["condensed", "preview", "resource"] = "resource",
+        sha256: str = None,
+        private: bool = None,
+        override: bool = False
+):
+    """Upload an artifact to S3
+
+    Parameters
+    ----------
+    resource_id: str
+        The resource identifier for the artifact
+    path_artifact: pathlib.Path
+        The path to the artifact file
+    artifact: str
+        The artifact type that the file represents
+    sha256: str
+        The SHA256 sum of `path_artifcat`, will be computed if not provided
+    private: bool
+        Whether the dataset that the resource belongs to is private.
+        Leave this blank if you don't know and we will do a database
+        look-up to determine the correct value.
+    override: bool
+        Whether to override a possibly existing object on S3.
+    """
     bucket_name, object_name = get_s3_bucket_object_for_artifact(
         resource_id=resource_id, artifact=artifact)
-    return s3.object_exists(bucket_name=bucket_name, object_name=object_name)
+
+    if private is None:
+        # User did not say whether the resource is private. We have to
+        # find out ourselves.
+        import ckan.logic
+        res_dict = ckan.logic.get_action('resource_show')(
+            context={'ignore_auth': True, 'user': 'default'},
+            data_dict={"id": resource_id})
+        ds_dict = ckan.logic.get_action('package_show')(
+            context={'ignore_auth': True, 'user': 'default'},
+            data_dict={'id': res_dict["package_id"]})
+        private = ds_dict["private"]
+
+    rid = resource_id
+    s3.upload_file(
+        bucket_name=bucket_name,
+        object_name=f"{artifact}/{rid[:3]}/{rid[3:6]}/{rid[6:]}",
+        path=path_artifact,
+        sha256=sha256 or sha256sum(path_artifact),
+        private=private,
+        override=override)

{dcor_shared-0.7.3 → dcor_shared-0.7.4}/dcor_shared.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dcor-shared
-Version: 0.7.3
+Version: 0.7.4
 Summary: Functionalities shared by the DCOR CKAN extensions
 Home-page: https://github.com/DCOR-dev/dcor_shared
 Author: Paul Müller

{dcor_shared-0.7.3 → dcor_shared-0.7.4}/dcor_shared.egg-info/SOURCES.txt

@@ -9,6 +9,7 @@ dcor_shared/_version.py
 dcor_shared/_version_save.py
 dcor_shared/ckan.py
 dcor_shared/data.py
+dcor_shared/dcinst.py
 dcor_shared/mime.py
 dcor_shared/parse.py
 dcor_shared/paths.py

{dcor_shared-0.7.3 → dcor_shared-0.7.4}/tests/test_s3cc.py

@@ -16,6 +16,23 @@ import requests
 data_path = pathlib.Path(__file__).parent / "data"
 
 
+@pytest.mark.ckan_config('ckan.plugins', 'dcor_schemas')
+@pytest.mark.usefixtures('clean_db', 'with_request_context')
+@mock.patch('ckan.plugins.toolkit.enqueue_job',
+            side_effect=synchronous_enqueue_job)
+def test_artifact_exists(enqueue_job_mock):
+    rid, s3_url, _, org_dict = setup_s3_resource_on_ckan(private=True)
+    assert s3cc.artifact_exists(rid)
+    # Delete the object
+    s3_client, _, _ = s3.get_s3()
+    bucket_name, object_name = s3cc.get_s3_bucket_object_for_artifact(rid)
+    s3_client.delete_object(
+        Bucket=bucket_name,
+        Key=object_name
+    )
+    assert not s3cc.artifact_exists(rid)
+
+
 def setup_s3_resource_on_ckan(private=False):
     """Create an S3 resource in CKAN"""
     user = factories.User()
@@ -151,14 +168,38 @@ def test_make_resource_public(enqueue_job_mock):
 @pytest.mark.usefixtures('clean_db', 'with_request_context')
 @mock.patch('ckan.plugins.toolkit.enqueue_job',
             side_effect=synchronous_enqueue_job)
-def test_object_exists(enqueue_job_mock):
+def test_upload_artifact(enqueue_job_mock, tmp_path):
     rid, s3_url, _, org_dict = setup_s3_resource_on_ckan(private=True)
-    assert s3cc.object_exists(rid)
-    # Delete the object
-    s3_client, _, _ = s3.get_s3()
-    bucket_name, object_name = s3cc.get_s3_bucket_object_for_artifact(rid)
-    s3_client.delete_object(
-        Bucket=bucket_name,
-        Key=object_name
-    )
-    assert not s3cc.object_exists(rid)
+    path_fake_preview = tmp_path / "test_preview.jpg"
+    path_fake_preview.write_text("This is not a real image!")
+    # upload the preview
+    s3cc.upload_artifact(rid,
+                         path_artifact=path_fake_preview,
+                         artifact="preview")
+    # make sure that worked
+    assert s3cc.artifact_exists(rid, "preview")
+    # attempt to download the private artifact
+    resp1 = requests.get(s3_url.replace("resource", "preview"))
+    assert not resp1.ok, "sanity check"
+
+
+@pytest.mark.ckan_config('ckan.plugins', 'dcor_schemas')
+@pytest.mark.usefixtures('clean_db', 'with_request_context')
+@mock.patch('ckan.plugins.toolkit.enqueue_job',
+            side_effect=synchronous_enqueue_job)
+def test_upload_artifact_public(enqueue_job_mock, tmp_path):
+    rid, s3_url, _, org_dict = setup_s3_resource_on_ckan(private=True)
+    path_fake_preview = tmp_path / "test_preview.jpg"
+    path_fake_preview.write_text("This is not a real image!")
+    # upload the preview
+    s3cc.upload_artifact(rid,
+                         path_artifact=path_fake_preview,
+                         artifact="preview",
+                         # force public resource even though dataset is not
+                         # (this has no real-life use case)
+                         private=False)
+    # make sure that worked
+    assert s3cc.artifact_exists(rid, "preview")
+    # attempt to download the private artifact
+    resp1 = requests.get(s3_url.replace("resource", "preview"))
+    assert resp1.ok, "preview should be public"