dcor-shared 0.8.0__tar.gz → 0.9.0__tar.gz

{dcor_shared-0.8.0 → dcor_shared-0.9.0}/CHANGELOG

@@ -1,3 +1,7 @@
+0.9.0
+ - feat: introduce `testing.make_dataset_via_s3` and
+   `testing.make_resource_via_s3` which uploads resources via S3
+ - enh: make sha256 optional in `s3.upload_file`
 0.8.0
  - feat: introduce `get_resource_dc_config`, `get_resource_info`, and
    `s3cc.get_s3_dc_handle_basin_based`

{dcor_shared-0.8.0 → dcor_shared-0.9.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dcor_shared
-Version: 0.8.0
+Version: 0.9.0
 Summary: Functionalities shared by the DCOR CKAN extensions
 Home-page: https://github.com/DCOR-dev/dcor_shared
 Author: Paul Müller

{dcor_shared-0.8.0 → dcor_shared-0.9.0}/dcor_shared/_version_save.py

@@ -1,3 +1,3 @@
 #!/usr/bin/env python
 # This file was created automatically
-longversion = '0.8.0'
+longversion = '0.9.0'

{dcor_shared-0.8.0 → dcor_shared-0.9.0}/dcor_shared/s3.py

@@ -1,4 +1,6 @@
 """Basic S3 utility functions"""
+from __future__ import annotations
+
 import functools
 import hashlib
 import json
@@ -10,6 +12,7 @@ import boto3
 import botocore.exceptions
 
 from .ckan import get_ckan_config_option
+from .data import sha256sum
 
 
 def compute_checksum(bucket_name, object_name, max_size=None):
@@ -124,8 +127,11 @@ def create_presigned_upload_url(bucket_name, object_name,
     return response["url"], response["fields"]
 
 
-def create_presigned_url(bucket_name, object_name, expiration=3600,
-                         filename=None):
+def create_presigned_url(
+        bucket_name: str,
+        object_name: str,
+        expiration: int = 3600,
+        filename: str | None = None):
     """Generate a presigned URL to share an S3 object
 
     Parameters
@@ -164,8 +170,11 @@ def create_presigned_url(bucket_name, object_name, expiration=3600,
 
 
 @functools.lru_cache()
-def create_presigned_url_until(bucket_name, object_name, expires_at,
-                               filename=None):
+def create_presigned_url_until(
+        bucket_name: str,
+        object_name: str,
+        expires_at: int,
+        filename: str | None = None) -> str:
     """Cached `create_presigned_url` with expiry time point
 
     Parameters
@@ -182,7 +191,7 @@ def create_presigned_url_until(bucket_name, object_name, expires_at,
 
     Returns
     -------
-    psurl: str
+    ps_url: str
         Presigned URL as string.
     """
     # Generate a presigned URL for the S3 object
@@ -192,12 +201,12 @@ def create_presigned_url_until(bucket_name, object_name, expires_at,
     if filename is not None:
         params["ResponseContentDisposition"] = \
             f"attachment; filename = {filename}"
-    psurl = s3_client.generate_presigned_url(
+    ps_url = s3_client.generate_presigned_url(
         'get_object',
         Params=params,
         ExpiresIn=expires_at - create_time())
     # The response contains the presigned URL
-    return psurl
+    return ps_url
 
 
 def create_time():
@@ -367,7 +376,7 @@ def require_bucket(bucket_name):
     return s3_bucket
 
 
-def upload_file(bucket_name, object_name, path, sha256, private=True,
+def upload_file(bucket_name, object_name, path, sha256=None, private=True,
                 override=False):
     """Upload a file to a bucket
 
@@ -380,7 +389,8 @@ def upload_file(bucket_name, object_name, path, sha256, private=True,
     path: str or pathlib.Path
         Local path of the file to be uploaded
     sha256: str
-        SHA256 checksum of the file to be uploaded
+        SHA256 checksum of the file to be uploaded, will be computed
+        if not provided
     private: bool
         Whether the object should remain private. If set to False,
         a tag "public:true" is added to the object which is picket up
@@ -393,6 +403,9 @@ def upload_file(bucket_name, object_name, path, sha256, private=True,
     s3_url: str
         URL to the S3 object
     """
+    if not sha256:
+        sha256 = sha256sum(path)
+
     path_size = pathlib.Path(path).stat().st_size
     s3_client, _, _ = get_s3()
     s3_bucket = require_bucket(bucket_name)

{dcor_shared-0.8.0 → dcor_shared-0.9.0}/dcor_shared/s3cc.py

@@ -18,7 +18,7 @@ import h5py
 from .ckan import (
     get_ckan_config_option, get_resource_dc_config, get_resource_info
 )
-from .data import sha256sum
+
 from . import s3
 
 
@@ -246,7 +246,7 @@ def upload_artifact(
         bucket_name=bucket_name,
         object_name=f"{artifact}/{rid[:3]}/{rid[3:6]}/{rid[6:]}",
         path=path_artifact,
-        sha256=sha256 or sha256sum(path_artifact),
+        sha256=sha256,
         private=private,
         override=override)
     return s3_url

{dcor_shared-0.8.0 → dcor_shared-0.9.0}/dcor_shared/testing.py

@@ -1,4 +1,10 @@
+from __future__ import annotations
+
 from io import BytesIO
+import numbers
+import pathlib
+from typing import Dict
+import uuid
 
 import ckan.authz
 import ckan.tests.factories as factories
@@ -9,6 +15,9 @@ from requests_toolbelt import MultipartEncoder, MultipartEncoderMonitor
 
 import pytest
 
+from .ckan import get_ckan_config_option
+from . import s3
+
 
 @pytest.fixture
 def create_with_upload_no_temp(clean_db, ckan_config, monkeypatch):
@@ -47,7 +56,8 @@ def make_dataset(create_context=None, owner_org=None, create_with_upload=None,
         user_id = user["id"]
     else:
         # get user ID from create_context
-        user_id = ckan.authz.get_user_id_for_username(create_context["user"])
+        user_id = ckan.authz.get_user_id_for_username(
+            create_context["user"], allow_none=True)
 
     if owner_org is None:
         owner_org = factories.Organization(users=[{
@@ -77,9 +87,11 @@ def make_dataset(create_context=None, owner_org=None, create_with_upload=None,
                            dataset_id=ds["id"])
 
     if activate:
-        helpers.call_action("package_patch", create_context,
-                            id=ds["id"],
-                            state="active")
+        revise_dict = {
+            "match": {"id": ds["id"]},
+            "update": {"state": "active"}
+        }
+        helpers.call_action("package_revise", create_context, **revise_dict)
 
     ds_dict = helpers.call_action("package_show", id=ds["id"])
 
@@ -91,6 +103,90 @@ def make_dataset(create_context=None, owner_org=None, create_with_upload=None,
         return ds_dict
 
 
+def make_dataset_via_s3(
+        create_context: Dict | None = None,
+        owner_org: Dict | None = None,
+        resource_path: str | pathlib.Path | None = None,
+        activate: bool = False,
+        **kwargs: str | numbers.Number):
+    """Create a dataset and upload the given resource directly via S3
+
+    For this action to work, the "ckanext-dcor_schemas" extension
+    must be loaded.
+
+    Parameters
+    ----------
+    create_context: dict
+        CKAN context for creating the dataset
+    owner_org: dict
+        CKAN dictionary of the owner organization
+    resource_path: str or pathlib.Path
+        path to the resource file to upload. If not specified, then
+        not resource is created for the dataset
+    activate: bool
+        whether to activate the dataset
+    kwargs: dict
+        keyword arguments passed to `package_create`
+    """
+    if create_context is None:
+        user = factories.User()
+        create_context = {'ignore_auth': False,
+                          'user': user['name'],
+                          'api_version': 3}
+        user_id = user["id"]
+    else:
+        # get user ID from create_context
+        user_id = ckan.authz.get_user_id_for_username(
+            create_context["user"], allow_none=True)
+
+    if owner_org is None:
+        owner_org = factories.Organization(users=[{
+            'name': user_id,
+            'capacity': 'admin'
+        }])
+
+    if "title" not in kwargs:
+        kwargs["title"] = "test-dataset-S3"
+    if "authors" not in kwargs:
+        kwargs["authors"] = "Peter Fly Pan"
+    if "license_id" not in kwargs:
+        kwargs["license_id"] = "CC-BY-4.0"
+    assert "state" not in kwargs, "must not be set"
+    assert "owner_org" not in kwargs, "must not be set"
+    # create a dataset
+    ds_dict = helpers.call_action("package_create", create_context,
+                                  owner_org=owner_org["name"],
+                                  state="draft",
+                                  **kwargs
+                                  )
+
+    if resource_path is not None:
+        rid = make_resource_via_s3(resource_path=resource_path,
+                                   organization_id=owner_org["id"],
+                                   dataset_id=ds_dict["id"],
+                                   create_context=create_context,
+                                   private=ds_dict.get("private", False)
+                                   )
+    else:
+        rid = None
+
+    if activate:
+        revise_dict = {
+            "match": {"id": ds_dict["id"]},
+            "update": {"state": "active"}
+        }
+        helpers.call_action("package_revise", create_context, **revise_dict)
+
+    ds_dict = helpers.call_action("package_show", id=ds_dict["id"])
+
+    if rid is not None:
+        # fetch resource dictionary
+        rs_dict = helpers.call_action("resource_show", id=rid)
+        return ds_dict, rs_dict
+    else:
+        return ds_dict
+
+
 def make_resource(resource_path, create_with_upload, create_context,
                   dataset_id):
     content = resource_path.read_bytes()
@@ -105,6 +201,33 @@ def make_resource(resource_path, create_with_upload, create_context,
     return resource
 
 
+def make_resource_via_s3(resource_path, organization_id, dataset_id,
+                         create_context, private=False):
+    """Upload a resource to S3 and register it with CKAN"""
+    bucket_name = get_ckan_config_option(
+        "dcor_object_store.bucket_name").format(
+        organization_id=organization_id)
+    rid = str(uuid.uuid4())
+    object_name = f"resource/{rid[:3]}/{rid[3:6]}/{rid[6:]}"
+
+    s3.upload_file(bucket_name=bucket_name,
+                   object_name=object_name,
+                   path=resource_path,
+                   private=private)
+
+    revise_dict = {
+        "match": {"id": dataset_id},
+        "update": {"resources": [{"id": rid,
+                                  "name": resource_path.name,
+                                  "s3_available": True,
+                                  }
+                                 ]
+                   }
+        }
+    helpers.call_action("package_revise", create_context, **revise_dict)
+    return rid
+
+
 def synchronous_enqueue_job(job_func, args=None, kwargs=None, title=None,
                             queue=None, rq_kwargs=None):
     """

{dcor_shared-0.8.0 → dcor_shared-0.9.0}/dcor_shared.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dcor-shared
-Version: 0.8.0
+Version: 0.9.0
 Summary: Functionalities shared by the DCOR CKAN extensions
 Home-page: https://github.com/DCOR-dev/dcor_shared
 Author: Paul Müller

{dcor_shared-0.8.0 → dcor_shared-0.9.0}/dcor_shared.egg-info/requires.txt

@@ -1,5 +1,5 @@
 boto3
-dclab[s3]>=0.57.3
+dclab[s3]>=0.57.5
 
 [tests]
 requests

{dcor_shared-0.8.0 → dcor_shared-0.9.0}/setup.py

@@ -44,7 +44,7 @@ setup(
     long_description=open('README.rst').read() if exists('README.rst') else '',
     install_requires=[
         "boto3",
-        "dclab[s3]>=0.57.3",
+        "dclab[s3]>=0.57.5",
     ],
     extras_require={
         # Required for the `dcor_shared.testing` submodule

dcor_shared-0.9.0/tests/test_testing.py

@@ -0,0 +1,85 @@
+import pathlib
+import pytest
+
+import ckan.tests.factories as factories
+import requests
+
+from dcor_shared.testing import make_dataset, make_dataset_via_s3
+from dcor_shared import s3cc, sha256sum
+
+data_path = pathlib.Path(__file__).parent / "data"
+
+
+@pytest.mark.ckan_config('ckan.plugins', 'dcor_schemas')
+@pytest.mark.usefixtures('clean_db', 'with_request_context')
+def test_make_dataset():
+    user = factories.User()
+    owner_org = factories.Organization(users=[{
+        'name': user['id'],
+        'capacity': 'admin'
+    }])
+    # Note: `call_action` bypasses authorization!
+    create_context = {'ignore_auth': False,
+                      'user': user['name'], 'api_version': 3}
+    ds_dict = make_dataset(create_context, owner_org,
+                           activate=False)
+    assert ds_dict["state"] == "draft"
+
+
+@pytest.mark.ckan_config('ckan.plugins', 'dcor_schemas')
+@pytest.mark.usefixtures('clean_db', 'with_request_context')
+def test_make_dataset_no_user_specified():
+    ds_dict = make_dataset(activate=False)
+    assert ds_dict["state"] == "draft"
+
+
+@pytest.mark.ckan_config('ckan.plugins', 'dcor_schemas')
+@pytest.mark.usefixtures('clean_db', 'with_request_context')
+def test_make_dataset_with_resource(create_with_upload):
+    # upload a *valid* [sic] .rtdc File (this is the control)
+    ds_dict, res_dict = make_dataset(
+        create_with_upload=create_with_upload,
+        resource_path=data_path / "calibration_beads_47.rtdc",
+        activate=True)
+    assert len(ds_dict["resources"]) == 1
+    assert "id" in res_dict
+    assert res_dict["package_id"] == ds_dict["id"]
+
+
+@pytest.mark.ckan_config('ckan.plugins', 'dcor_schemas')
+@pytest.mark.usefixtures('clean_db', 'with_request_context')
+def test_make_dataset_via_s3():
+    user = factories.User()
+    owner_org = factories.Organization(users=[{
+        'name': user['id'],
+        'capacity': 'admin'
+    }])
+    # Note: `call_action` bypasses authorization!
+    create_context = {'ignore_auth': False,
+                      'user': user['name'], 'api_version': 3}
+    ds_dict = make_dataset_via_s3(create_context, owner_org, activate=False)
+    assert ds_dict["state"] == "draft"
+
+
+@pytest.mark.ckan_config('ckan.plugins', 'dcor_schemas')
+@pytest.mark.usefixtures('clean_db', 'with_request_context')
+def test_make_dataset_with_resource_via_s3(tmp_path):
+    # upload a *valid* [sic] .rtdc File (this is the control)
+    ds_dict, res_dict = make_dataset_via_s3(
+        resource_path=data_path / "calibration_beads_47.rtdc",
+        activate=True,
+        private=False,
+    )
+    assert len(ds_dict["resources"]) == 1
+    assert "id" in res_dict
+    assert res_dict["package_id"] == ds_dict["id"]
+
+    # verify SHA256 sum of the resource
+    ps_url = s3cc.get_s3_url_for_artifact(res_dict["id"], artifact="resource")
+    response = requests.get(ps_url)
+    dl_path = tmp_path / "calbeads.rtdc"
+    with dl_path.open("wb") as fd:
+        fd.write(response.content)
+
+    assert sha256sum(dl_path) == \
+           "490efdf5d9bb4cd4b2a6bcf2fe54d4dc201c38530140bcb168980bf8bf846c73"

dcor_shared-0.8.0/tests/test_testing.py

@@ -1,44 +0,0 @@
-import pathlib
-import pytest
-
-import ckan.tests.factories as factories
-
-from dcor_shared.testing import make_dataset
-
-data_path = pathlib.Path(__file__).parent / "data"
-
-
-@pytest.mark.ckan_config('ckan.plugins', 'dcor_schemas')
-@pytest.mark.usefixtures('clean_db', 'with_request_context')
-def test_make_dataset():
-    user = factories.User()
-    owner_org = factories.Organization(users=[{
-        'name': user['id'],
-        'capacity': 'admin'
-    }])
-    # Note: `call_action` bypasses authorization!
-    create_context = {'ignore_auth': False,
-                      'user': user['name'], 'api_version': 3}
-    ds_dict = make_dataset(create_context, owner_org,
-                           activate=False)
-    assert ds_dict["state"] == "draft"
-
-
-@pytest.mark.ckan_config('ckan.plugins', 'dcor_schemas')
-@pytest.mark.usefixtures('clean_db', 'with_request_context')
-def test_make_dataset_no_user_specified():
-    ds_dict = make_dataset(activate=False)
-    assert ds_dict["state"] == "draft"
-
-
-@pytest.mark.ckan_config('ckan.plugins', 'dcor_schemas')
-@pytest.mark.usefixtures('clean_db', 'with_request_context')
-def test_make_dataset_with_resource(create_with_upload):
-    # upload a *valid* [sic] .rtdc File (this is the control)
-    ds_dict, res_dict = make_dataset(
-        create_with_upload=create_with_upload,
-        resource_path=data_path / "calibration_beads_47.rtdc",
-        activate=True)
-    assert len(ds_dict["resources"]) == 1
-    assert "id" in res_dict
-    assert res_dict["package_id"] == ds_dict["id"]