dcor-shared 0.7.4__tar.gz → 0.9.0__tar.gz

{dcor_shared-0.7.4 → dcor_shared-0.9.0}/CHANGELOG

@@ -1,3 +1,13 @@
+0.9.0
+ - feat: introduce `testing.make_dataset_via_s3` and
+   `testing.make_resource_via_s3` which uploads resources via S3
+ - enh: make sha256 optional in `s3.upload_file`
+0.8.0
+ - feat: introduce `get_resource_dc_config`, `get_resource_info`, and
+   `s3cc.get_s3_dc_handle_basin_based`
+ - enh: return S3 URL in s3cc.upload_artifact
+ - enh: make specifying owner organization and create context in
+   `testing.make_dataset` optional
 0.7.4
  - fix: disable basins when loading a dataset instance
  - enh: implement s3cc.upload_artifact

{dcor_shared-0.7.4 → dcor_shared-0.9.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dcor_shared
-Version: 0.7.4
+Version: 0.9.0
 Summary: Functionalities shared by the DCOR CKAN extensions
 Home-page: https://github.com/DCOR-dev/dcor_shared
 Author: Paul Müller

{dcor_shared-0.7.4 → dcor_shared-0.9.0}/dcor_shared/__init__.py

@@ -1,5 +1,7 @@
 # flake8: noqa: F401
-from .ckan import get_ckan_config_option, get_resource_path
+from .ckan import (
+    get_ckan_config_option, get_resource_dc_config, get_resource_path, get_resource_info
+)
 from .data import DUMMY_BYTES, sha256sum, wait_for_resource
 from .dcinst import get_dc_instance
 from .mime import DC_MIME_TYPES, VALID_FORMATS

{dcor_shared-0.7.4 → dcor_shared-0.9.0}/dcor_shared/_version_save.py

@@ -1,3 +1,3 @@
 #!/usr/bin/env python
 # This file was created automatically
-longversion = '0.7.4'
+longversion = '0.9.0'

{dcor_shared-0.7.4 → dcor_shared-0.9.0}/dcor_shared/ckan.py

@@ -41,7 +41,43 @@ def get_ckan_webassets_path():
     return pathlib.Path(get_ckan_config_option("ckan.webassets.path"))
 
 
-def get_resource_path(rid, create_dirs=False):
+def get_resource_dc_config(resource_id):
+    """Return the DC metadata for a resource identifier
+
+    For this to work, the dcor_schemas CKAN extension must be active.
+    """
+    _, res_dict = get_resource_info(resource_id)
+    # build metadata dictionary from resource metadata
+    meta = {}
+    for item in res_dict:
+        if item.startswith("dc:"):
+            _, sec, key = item.split(":", 2)
+            meta.setdefault(sec, {})
+            meta[sec][key] = res_dict[item]
+    return meta
+
+
+def get_resource_info(resource_id):
+    """Return resource and dataset dictionaries for a resource identifier
+
+    Return the dataset dictionary and the resource dictionary.
+    """
+    from ckan import logic
+    res_dict = logic.get_action("resource_show")(
+        context={'ignore_auth': True, 'user': 'default'},
+        data_dict={"id": resource_id})
+    ds_dict = logic.get_action("package_show")(
+        context={'ignore_auth': True, 'user': 'default'},
+        data_dict={"id": res_dict["package_id"]})
+    return ds_dict, res_dict
+
+
+def get_resource_path(resource_id, create_dirs=False):
+    """Return the expected local path for a resource identifier
+
+    If `create_dirs` is True, create the parent directory tree.
+    """
+    rid = resource_id
     resources_path = get_ckan_storage_path() / "resources"
     pdir = resources_path / rid[:3] / rid[3:6]
     path = pdir / rid[6:]

{dcor_shared-0.7.4 → dcor_shared-0.9.0}/dcor_shared/s3.py

@@ -1,4 +1,6 @@
 """Basic S3 utility functions"""
+from __future__ import annotations
+
 import functools
 import hashlib
 import json
@@ -10,6 +12,7 @@ import boto3
 import botocore.exceptions
 
 from .ckan import get_ckan_config_option
+from .data import sha256sum
 
 
 def compute_checksum(bucket_name, object_name, max_size=None):
@@ -124,8 +127,11 @@ def create_presigned_upload_url(bucket_name, object_name,
     return response["url"], response["fields"]
 
 
-def create_presigned_url(bucket_name, object_name, expiration=3600,
-                         filename=None):
+def create_presigned_url(
+        bucket_name: str,
+        object_name: str,
+        expiration: int = 3600,
+        filename: str | None = None):
     """Generate a presigned URL to share an S3 object
 
     Parameters
@@ -164,8 +170,11 @@ def create_presigned_url(bucket_name, object_name, expiration=3600,
 
 
 @functools.lru_cache()
-def create_presigned_url_until(bucket_name, object_name, expires_at,
-                               filename=None):
+def create_presigned_url_until(
+        bucket_name: str,
+        object_name: str,
+        expires_at: int,
+        filename: str | None = None) -> str:
     """Cached `create_presigned_url` with expiry time point
 
     Parameters
@@ -182,7 +191,7 @@ def create_presigned_url_until(bucket_name, object_name, expires_at,
 
     Returns
     -------
-    psurl: str
+    ps_url: str
         Presigned URL as string.
     """
     # Generate a presigned URL for the S3 object
@@ -192,12 +201,12 @@ def create_presigned_url_until(bucket_name, object_name, expires_at,
     if filename is not None:
         params["ResponseContentDisposition"] = \
             f"attachment; filename = {filename}"
-    psurl = s3_client.generate_presigned_url(
+    ps_url = s3_client.generate_presigned_url(
         'get_object',
         Params=params,
         ExpiresIn=expires_at - create_time())
     # The response contains the presigned URL
-    return psurl
+    return ps_url
 
 
 def create_time():
@@ -367,7 +376,7 @@ def require_bucket(bucket_name):
     return s3_bucket
 
 
-def upload_file(bucket_name, object_name, path, sha256, private=True,
+def upload_file(bucket_name, object_name, path, sha256=None, private=True,
                 override=False):
     """Upload a file to a bucket
 
@@ -380,7 +389,8 @@ def upload_file(bucket_name, object_name, path, sha256, private=True,
     path: str or pathlib.Path
         Local path of the file to be uploaded
     sha256: str
-        SHA256 checksum of the file to be uploaded
+        SHA256 checksum of the file to be uploaded, will be computed
+        if not provided
     private: bool
         Whether the object should remain private. If set to False,
         a tag "public:true" is added to the object which is picket up
@@ -393,6 +403,9 @@ def upload_file(bucket_name, object_name, path, sha256, private=True,
     s3_url: str
         URL to the S3 object
     """
+    if not sha256:
+        sha256 = sha256sum(path)
+
     path_size = pathlib.Path(path).stat().st_size
     s3_client, _, _ = get_s3()
     s3_bucket = require_bucket(bucket_name)

{dcor_shared-0.7.4 → dcor_shared-0.9.0}/dcor_shared/s3cc.py

@@ -4,15 +4,21 @@ Contains methods to directly interact with CKAN resources that are on S3
 via just the resource ID.
 """
 from __future__ import annotations
+import io
 import functools
 import pathlib
 from typing import Literal
+from urllib.parse import urlparse
 import warnings
 
-from dclab.rtdc_dataset import fmt_s3
+import dclab
+from dclab.rtdc_dataset import fmt_hdf5, fmt_s3
+import h5py
+
+from .ckan import (
+    get_ckan_config_option, get_resource_dc_config, get_resource_info
+)
 
-from .ckan import get_ckan_config_option
-from .data import sha256sum
 from . import s3
 
 
@@ -81,13 +87,7 @@ def get_s3_bucket_name_for_resource(resource_id):
     The resource with the identifier `resource_id` must exist in the
     CKAN database.
     """
-    import ckan.logic
-    res_dict = ckan.logic.get_action('resource_show')(
-        context={'ignore_auth': True, 'user': 'default'},
-        data_dict={"id": resource_id})
-    ds_dict = ckan.logic.get_action('package_show')(
-        context={'ignore_auth': True, 'user': 'default'},
-        data_dict={'id': res_dict["package_id"]})
+    ds_dict, _ = get_resource_info(resource_id)
     bucket_name = get_ckan_config_option(
         "dcor_object_store.bucket_name").format(
         organization_id=ds_dict["organization"]["id"])
@@ -97,6 +97,9 @@ def get_s3_bucket_name_for_resource(resource_id):
 def get_s3_dc_handle(resource_id):
     """Return an instance of :class:`RTDC_S3`
 
+    The data are accessed directly via S3 using DCOR's access credentials.
+    Use this if you need to access the original raw file.
+
     The resource with the identifier `resource_id` must exist in the
     CKAN database.
     """
@@ -114,6 +117,48 @@ def get_s3_dc_handle(resource_id):
     return ds
 
 
+def get_s3_dc_handle_basin_based(resource_id):
+    """Return a :class:`RTDC_HTTP`-basin-backed instance of :class:`RTDC_HDF5`
+
+    The returned instance does not contain any feature data, but has
+    basins defined that link to the original data on S3. The upside
+    over :func:`get_s3_dc_handle` is that the returned dataset
+    includes the basin with the condensed data and that the returned
+    instance does not contain the DCOR S3 credentials. The downside is
+    that initialization takes slightly longer and that, if private
+    resources are accessed, the presigned URLs in the basins are only
+    valid for a fixed time period.
+    """
+    ds_dict, res_dict = get_resource_info(resource_id)
+    basin_paths = []
+    for artifact in ["resource", "condensed"]:
+        if ds_dict["private"]:
+            bp = create_presigned_url(resource_id, artifact=artifact)
+        else:
+            bp = get_s3_url_for_artifact(resource_id, artifact=artifact)
+        basin_paths.append(bp)
+
+    fd = io.BytesIO()
+    with h5py.File(fd, "w", libver="latest") as hv:
+        # We don't use RTDCWriter as a context manager to avoid overhead
+        # during __exit__, but then we have to make sure "events" is there.
+        hv.require_group("events")
+        hw = dclab.RTDCWriter(hv)
+        hw.store_metadata(get_resource_dc_config(resource_id))
+        for bp in basin_paths:
+            urlp = urlparse(bp)  # e.g. http://localhost/bucket/resource/id
+            hw.store_basin(
+                basin_name=urlp.path.split("/")[2],  # e.g. "resource"
+                basin_format="http",
+                basin_type="remote",
+                basin_locs=[bp],
+                # Don't verify anything. This would only cost time,
+                # and we know these objects exist.
+                verify=False,
+                )
+    return fmt_hdf5.RTDC_HDF5(fd)
+
+
 def get_s3_url_for_artifact(
         resource_id: str,
         artifact: Literal["condensed", "preview", "resource"] = "resource"):
@@ -193,20 +238,15 @@ def upload_artifact(
     if private is None:
         # User did not say whether the resource is private. We have to
         # find out ourselves.
-        import ckan.logic
-        res_dict = ckan.logic.get_action('resource_show')(
-            context={'ignore_auth': True, 'user': 'default'},
-            data_dict={"id": resource_id})
-        ds_dict = ckan.logic.get_action('package_show')(
-            context={'ignore_auth': True, 'user': 'default'},
-            data_dict={'id': res_dict["package_id"]})
+        ds_dict, _ = get_resource_info(resource_id)
         private = ds_dict["private"]
 
     rid = resource_id
-    s3.upload_file(
+    s3_url = s3.upload_file(
         bucket_name=bucket_name,
         object_name=f"{artifact}/{rid[:3]}/{rid[3:6]}/{rid[6:]}",
         path=path_artifact,
-        sha256=sha256 or sha256sum(path_artifact),
+        sha256=sha256,
         private=private,
         override=override)
+    return s3_url

dcor_shared-0.9.0/dcor_shared/testing.py

@@ -0,0 +1,286 @@
+from __future__ import annotations
+
+from io import BytesIO
+import numbers
+import pathlib
+from typing import Dict
+import uuid
+
+import ckan.authz
+import ckan.tests.factories as factories
+import ckan.tests.helpers as helpers
+from ckan.tests.pytest_ckan.fixtures import FakeFileStorage
+import requests
+from requests_toolbelt import MultipartEncoder, MultipartEncoderMonitor
+
+import pytest
+
+from .ckan import get_ckan_config_option
+from . import s3
+
+
+@pytest.fixture
+def create_with_upload_no_temp(clean_db, ckan_config, monkeypatch):
+    """
+    Create upload without tempdir
+    """
+
+    def factory(data, filename, context=None, **kwargs):
+        if context is None:
+            context = {}
+        action = kwargs.pop("action", "resource_create")
+        field = kwargs.pop("upload_field_name", "upload")
+        test_file = BytesIO()
+        if type(data) is not bytes:
+            data = bytes(data, encoding="utf-8")
+        test_file.write(data)
+        test_file.seek(0)
+        test_resource = FakeFileStorage(test_file, filename)
+
+        params = {
+            field: test_resource,
+        }
+        params.update(kwargs)
+        return helpers.call_action(action, context, **params)
+    return factory
+
+
+def make_dataset(create_context=None, owner_org=None, create_with_upload=None,
+                 resource_path=None, activate=False, **kwargs):
+    """Create a dataset with a resource for testing"""
+    if create_context is None:
+        user = factories.User()
+        create_context = {'ignore_auth': False,
+                          'user': user['name'],
+                          'api_version': 3}
+        user_id = user["id"]
+    else:
+        # get user ID from create_context
+        user_id = ckan.authz.get_user_id_for_username(
+            create_context["user"], allow_none=True)
+
+    if owner_org is None:
+        owner_org = factories.Organization(users=[{
+            'name': user_id,
+            'capacity': 'admin'
+        }])
+
+    if "title" not in kwargs:
+        kwargs["title"] = "test-dataset"
+    if "authors" not in kwargs:
+        kwargs["authors"] = "Peter Pan"
+    if "license_id" not in kwargs:
+        kwargs["license_id"] = "CC-BY-4.0"
+    assert "state" not in kwargs, "must not be set"
+    assert "owner_org" not in kwargs, "must not be set"
+    # create a dataset
+    ds = helpers.call_action("package_create", create_context,
+                             owner_org=owner_org["name"],
+                             state="draft",
+                             **kwargs
+                             )
+
+    if create_with_upload is not None:
+        rs = make_resource(resource_path=resource_path,
+                           create_with_upload=create_with_upload,
+                           create_context=create_context,
+                           dataset_id=ds["id"])
+
+    if activate:
+        revise_dict = {
+            "match": {"id": ds["id"]},
+            "update": {"state": "active"}
+        }
+        helpers.call_action("package_revise", create_context, **revise_dict)
+
+    ds_dict = helpers.call_action("package_show", id=ds["id"])
+
+    if create_with_upload is not None:
+        # updated resource dictionary
+        rs_dict = helpers.call_action("resource_show", id=rs["id"])
+        return ds_dict, rs_dict
+    else:
+        return ds_dict
+
+
+def make_dataset_via_s3(
+        create_context: Dict | None = None,
+        owner_org: Dict | None = None,
+        resource_path: str | pathlib.Path | None = None,
+        activate: bool = False,
+        **kwargs: str | numbers.Number):
+    """Create a dataset and upload the given resource directly via S3
+
+    For this action to work, the "ckanext-dcor_schemas" extension
+    must be loaded.
+
+    Parameters
+    ----------
+    create_context: dict
+        CKAN context for creating the dataset
+    owner_org: dict
+        CKAN dictionary of the owner organization
+    resource_path: str or pathlib.Path
+        path to the resource file to upload. If not specified, then
+        not resource is created for the dataset
+    activate: bool
+        whether to activate the dataset
+    kwargs: dict
+        keyword arguments passed to `package_create`
+    """
+    if create_context is None:
+        user = factories.User()
+        create_context = {'ignore_auth': False,
+                          'user': user['name'],
+                          'api_version': 3}
+        user_id = user["id"]
+    else:
+        # get user ID from create_context
+        user_id = ckan.authz.get_user_id_for_username(
+            create_context["user"], allow_none=True)
+
+    if owner_org is None:
+        owner_org = factories.Organization(users=[{
+            'name': user_id,
+            'capacity': 'admin'
+        }])
+
+    if "title" not in kwargs:
+        kwargs["title"] = "test-dataset-S3"
+    if "authors" not in kwargs:
+        kwargs["authors"] = "Peter Fly Pan"
+    if "license_id" not in kwargs:
+        kwargs["license_id"] = "CC-BY-4.0"
+    assert "state" not in kwargs, "must not be set"
+    assert "owner_org" not in kwargs, "must not be set"
+    # create a dataset
+    ds_dict = helpers.call_action("package_create", create_context,
+                                  owner_org=owner_org["name"],
+                                  state="draft",
+                                  **kwargs
+                                  )
+
+    if resource_path is not None:
+        rid = make_resource_via_s3(resource_path=resource_path,
+                                   organization_id=owner_org["id"],
+                                   dataset_id=ds_dict["id"],
+                                   create_context=create_context,
+                                   private=ds_dict.get("private", False)
+                                   )
+    else:
+        rid = None
+
+    if activate:
+        revise_dict = {
+            "match": {"id": ds_dict["id"]},
+            "update": {"state": "active"}
+        }
+        helpers.call_action("package_revise", create_context, **revise_dict)
+
+    ds_dict = helpers.call_action("package_show", id=ds_dict["id"])
+
+    if rid is not None:
+        # fetch resource dictionary
+        rs_dict = helpers.call_action("resource_show", id=rid)
+        return ds_dict, rs_dict
+    else:
+        return ds_dict
+
+
+def make_resource(resource_path, create_with_upload, create_context,
+                  dataset_id):
+    content = resource_path.read_bytes()
+    rs = create_with_upload(
+        data=content,
+        filename='test.rtdc',
+        context=create_context,
+        package_id=dataset_id,
+        url="upload",
+    )
+    resource = helpers.call_action("resource_show", id=rs["id"])
+    return resource
+
+
+def make_resource_via_s3(resource_path, organization_id, dataset_id,
+                         create_context, private=False):
+    """Upload a resource to S3 and register it with CKAN"""
+    bucket_name = get_ckan_config_option(
+        "dcor_object_store.bucket_name").format(
+        organization_id=organization_id)
+    rid = str(uuid.uuid4())
+    object_name = f"resource/{rid[:3]}/{rid[3:6]}/{rid[6:]}"
+
+    s3.upload_file(bucket_name=bucket_name,
+                   object_name=object_name,
+                   path=resource_path,
+                   private=private)
+
+    revise_dict = {
+        "match": {"id": dataset_id},
+        "update": {"resources": [{"id": rid,
+                                  "name": resource_path.name,
+                                  "s3_available": True,
+                                  }
+                                 ]
+                   }
+        }
+    helpers.call_action("package_revise", create_context, **revise_dict)
+    return rid
+
+
+def synchronous_enqueue_job(job_func, args=None, kwargs=None, title=None,
+                            queue=None, rq_kwargs=None):
+    """
+    Synchronous mock for ``ckan.plugins.toolkit.enqueue_job``.
+
+    Due to the asynchronous nature of background jobs, code that uses them
+    needs to be handled specially when writing tests.
+
+    A common approach is to use the mock package to replace the
+    ckan.plugins.toolkit.enqueue_job function with a mock that executes jobs
+    synchronously instead of asynchronously
+
+    Also, since we are running the tests as root on a ckan instance that
+    is run by www-data, modifying files on disk in background jobs
+    (which were started by supervisor as www-data) does not work.
+    """
+    if rq_kwargs is None:
+        rq_kwargs = {}
+    args = args or []
+    kwargs = kwargs or {}
+    job_func(*args, **kwargs)
+
+
+def upload_presigned_to_s3(psurl, fields, path_to_upload):
+    """Helper function for uploading data to S3
+
+    This is exactly how DCOR-Aid would be uploading things (with the
+    requests_toolbelt package). This could have been a little simpler,
+    but for the sake of reproducibility, we do it the DCOR-Aid way.
+    """
+    # callback function for monitoring the upload progress
+    # open the input file for streaming
+    with path_to_upload.open("rb") as fd:
+        fields["file"] = (fields["key"], fd)
+        e = MultipartEncoder(fields=fields)
+        m = MultipartEncoderMonitor(
+            e, lambda monitor: print(f"Bytes: {monitor.bytes_read}"))
+        # Increase the read size to speed-up upload (the default chunk
+        # size for uploads in urllib is 8k which results in a lot of
+        # Python code being involved in uploading a 20GB file; Setting
+        # the chunk size to 4MB should increase the upload speed):
+        # https://github.com/requests/toolbelt/issues/75
+        # #issuecomment-237189952
+        m._read = m.read
+        m.read = lambda size: m._read(4 * 1024 * 1024)
+        # perform the actual upload
+        hrep = requests.post(
+            psurl,
+            data=m,
+            headers={'Content-Type': m.content_type},
+            verify=True,  # verify SSL connection
+            timeout=27.3,  # timeout to avoid freezing
+        )
+    if hrep.status_code != 204:
+        raise ValueError(
+            f"Upload failed with {hrep.status_code}: {hrep.reason}")

{dcor_shared-0.7.4 → dcor_shared-0.9.0}/dcor_shared.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dcor-shared
-Version: 0.7.4
+Version: 0.9.0
 Summary: Functionalities shared by the DCOR CKAN extensions
 Home-page: https://github.com/DCOR-dev/dcor_shared
 Author: Paul Müller

{dcor_shared-0.7.4 → dcor_shared-0.9.0}/dcor_shared.egg-info/SOURCES.txt

@@ -22,6 +22,7 @@ dcor_shared.egg-info/dependency_links.txt
 dcor_shared.egg-info/requires.txt
 dcor_shared.egg-info/top_level.txt
 tests/test_basic.py
+tests/test_ckan.py
 tests/test_data.py
 tests/test_s3.py
 tests/test_s3cc.py

{dcor_shared-0.7.4 → dcor_shared-0.9.0}/dcor_shared.egg-info/requires.txt

@@ -1,5 +1,5 @@
 boto3
-dclab[s3]>=0.57.3
+dclab[s3]>=0.57.5
 
 [tests]
 requests

{dcor_shared-0.7.4 → dcor_shared-0.9.0}/setup.py

@@ -44,7 +44,7 @@ setup(
     long_description=open('README.rst').read() if exists('README.rst') else '',
     install_requires=[
         "boto3",
-        "dclab[s3]>=0.57.3",
+        "dclab[s3]>=0.57.5",
     ],
     extras_require={
         # Required for the `dcor_shared.testing` submodule

dcor_shared-0.9.0/tests/test_ckan.py

@@ -0,0 +1,53 @@
+import pathlib
+from unittest import mock
+
+import ckanext.dcor_schemas.plugin
+
+from dcor_shared import get_resource_dc_config, get_resource_info
+
+import pytest
+from dcor_shared.testing import make_dataset, synchronous_enqueue_job
+
+data_path = pathlib.Path(__file__).parent / "data"
+
+
+@pytest.mark.ckan_config('ckan.plugins', 'dcor_schemas')
+@pytest.mark.usefixtures('clean_db', 'with_request_context')
+@mock.patch('ckan.plugins.toolkit.enqueue_job',
+            side_effect=synchronous_enqueue_job)
+def test_get_resource_dc_config(enqueue_job_mock, create_with_upload,
+                                monkeypatch):
+    monkeypatch.setattr(
+        ckanext.dcor_schemas.plugin,
+        'DISABLE_AFTER_DATASET_CREATE_FOR_CONCURRENT_JOB_TESTS',
+        True)
+
+    _, res_dict = make_dataset(
+        create_with_upload=create_with_upload,
+        resource_path=data_path / "calibration_beads_47.rtdc",
+        activate=True)
+
+    dc_config = get_resource_dc_config(res_dict["id"])
+    assert dc_config["experiment"]["event count"] == 47
+    assert dc_config["experiment"]["date"] == "2018-12-11"
+    assert dc_config["fluorescence"]["laser count"] == 2
+
+
+@pytest.mark.ckan_config('ckan.plugins', 'dcor_schemas')
+@pytest.mark.usefixtures('clean_db', 'with_request_context')
+@mock.patch('ckan.plugins.toolkit.enqueue_job',
+            side_effect=synchronous_enqueue_job)
+def test_get_resource_info(enqueue_job_mock, create_with_upload, monkeypatch):
+    monkeypatch.setattr(
+        ckanext.dcor_schemas.plugin,
+        'DISABLE_AFTER_DATASET_CREATE_FOR_CONCURRENT_JOB_TESTS',
+        True)
+
+    ds_dict, res_dict = make_dataset(
+        create_with_upload=create_with_upload,
+        resource_path=data_path / "calibration_beads_47.rtdc",
+        activate=True)
+
+    ds_dict_2, res_dict_2 = get_resource_info(res_dict["id"])
+    assert ds_dict == ds_dict_2
+    assert res_dict == res_dict_2

{dcor_shared-0.7.4 → dcor_shared-0.9.0}/tests/test_data.py

@@ -29,19 +29,11 @@ def test_get_dc_instance_file(enqueue_job_mock, create_with_upload,
         'DISABLE_AFTER_DATASET_CREATE_FOR_CONCURRENT_JOB_TESTS',
         True)
 
-    user = factories.User()
-    owner_org = factories.Organization(users=[{
-        'name': user['id'],
-        'capacity': 'admin'
-    }])
-    create_context = {'ignore_auth': False,
-                      'user': user['name'],
-                      'api_version': 3}
     ds_dict, _ = make_dataset(
-        create_context, owner_org,
         create_with_upload=create_with_upload,
         resource_path=data_path / "calibration_beads_47.rtdc",
         activate=True)
+
     rid = ds_dict["resources"][0]["id"]
     resource_path = pathlib.Path(get_resource_path(rid))
     assert resource_path.exists(), "sanity check"
@@ -75,19 +67,11 @@ def test_get_dc_instance_s3(enqueue_job_mock, create_with_upload,
         'DISABLE_AFTER_DATASET_CREATE_FOR_CONCURRENT_JOB_TESTS',
         True)
 
-    user = factories.User()
-    owner_org = factories.Organization(users=[{
-        'name': user['id'],
-        'capacity': 'admin'
-    }])
-    create_context = {'ignore_auth': False,
-                      'user': user['name'],
-                      'api_version': 3}
     ds_dict, _ = make_dataset(
-        create_context, owner_org,
         create_with_upload=create_with_upload,
         resource_path=data_path / "calibration_beads_47.rtdc",
         activate=True)
+
     res_dict = ds_dict["resources"][0]
     rid = res_dict["id"]
     resource_path = pathlib.Path(get_resource_path(rid))

{dcor_shared-0.7.4 → dcor_shared-0.9.0}/tests/test_s3cc.py

@@ -1,4 +1,5 @@
 import pathlib
+import shutil
 from unittest import mock
 
 import pytest
@@ -6,34 +7,21 @@ import pytest
 from ckan import model
 import ckan.tests.helpers as helpers
 import ckan.tests.factories as factories
+import ckanext.dcor_schemas.plugin
 
-from dcor_shared.testing import synchronous_enqueue_job, upload_presigned_to_s3
+from dcor_shared.testing import (
+    make_dataset, synchronous_enqueue_job, upload_presigned_to_s3
+)
 from dcor_shared import s3, s3cc, sha256sum
 
+import h5py
 import requests
 
 
 data_path = pathlib.Path(__file__).parent / "data"
 
 
-@pytest.mark.ckan_config('ckan.plugins', 'dcor_schemas')
-@pytest.mark.usefixtures('clean_db', 'with_request_context')
-@mock.patch('ckan.plugins.toolkit.enqueue_job',
-            side_effect=synchronous_enqueue_job)
-def test_artifact_exists(enqueue_job_mock):
-    rid, s3_url, _, org_dict = setup_s3_resource_on_ckan(private=True)
-    assert s3cc.artifact_exists(rid)
-    # Delete the object
-    s3_client, _, _ = s3.get_s3()
-    bucket_name, object_name = s3cc.get_s3_bucket_object_for_artifact(rid)
-    s3_client.delete_object(
-        Bucket=bucket_name,
-        Key=object_name
-    )
-    assert not s3cc.artifact_exists(rid)
-
-
-def setup_s3_resource_on_ckan(private=False):
+def setup_s3_resource_on_ckan(private=False, resource_path=None):
     """Create an S3 resource in CKAN"""
     user = factories.User()
     owner_org = factories.Organization(users=[{
@@ -51,10 +39,13 @@ def setup_s3_resource_on_ckan(private=False):
                                    organization_id=owner_org["id"],
                                    )
     rid = response["resource_id"]
+    if resource_path is None:
+        resource_path = data_path / "calibration_beads_47.rtdc"
+
     upload_presigned_to_s3(
         psurl=response["url"],
         fields=response["fields"],
-        path_to_upload=data_path / "calibration_beads_47.rtdc")
+        path_to_upload=resource_path)
 
     # Create the dataset
     pkg_dict = helpers.call_action("package_create",
@@ -82,6 +73,23 @@ def setup_s3_resource_on_ckan(private=False):
     return rid, s3_url, new_pkg_dict, owner_org
 
 
+@pytest.mark.ckan_config('ckan.plugins', 'dcor_schemas')
+@pytest.mark.usefixtures('clean_db', 'with_request_context')
+@mock.patch('ckan.plugins.toolkit.enqueue_job',
+            side_effect=synchronous_enqueue_job)
+def test_artifact_exists(enqueue_job_mock):
+    rid, s3_url, _, org_dict = setup_s3_resource_on_ckan(private=True)
+    assert s3cc.artifact_exists(rid)
+    # Delete the object
+    s3_client, _, _ = s3.get_s3()
+    bucket_name, object_name = s3cc.get_s3_bucket_object_for_artifact(rid)
+    s3_client.delete_object(
+        Bucket=bucket_name,
+        Key=object_name
+    )
+    assert not s3cc.artifact_exists(rid)
+
+
 @pytest.mark.ckan_config('ckan.plugins', 'dcor_schemas')
 @pytest.mark.usefixtures('clean_db', 'with_request_context')
 @mock.patch('ckan.plugins.toolkit.enqueue_job',
@@ -134,6 +142,44 @@ def test_get_s3_handle(enqueue_job_mock):
         assert len(ds) == 47
 
 
+@pytest.mark.ckan_config('ckan.plugins', 'dcor_depot dcor_schemas dc_serve')
+@pytest.mark.usefixtures('clean_db', 'with_request_context')
+@mock.patch('ckan.plugins.toolkit.enqueue_job',
+            side_effect=synchronous_enqueue_job)
+def test_get_s3_dc_handle_basin_based(enqueue_job_mock, create_with_upload,
+                                      monkeypatch, tmp_path):
+    monkeypatch.setattr(
+        ckanext.dcor_schemas.plugin,
+        'DISABLE_AFTER_DATASET_CREATE_FOR_CONCURRENT_JOB_TESTS',
+        True)
+
+    resource_path = tmp_path / "data.rtdc"
+    shutil.copy2(data_path / "calibration_beads_47.rtdc", resource_path)
+    with h5py.File(resource_path, "a") as h5:
+        del h5["events/volume"]
+
+    _, res_dict = make_dataset(
+        create_with_upload=create_with_upload,
+        resource_path=resource_path,
+        activate=True)
+    rid = res_dict["id"]
+
+    # sanity check with reference
+    with s3cc.get_s3_dc_handle(rid) as ds:
+        assert "volume" not in ds.features_innate
+        assert "image" in ds
+
+    # actual test, the condensed dataset containing "volume" should be
+    # in one of the basins.
+    with s3cc.get_s3_dc_handle_basin_based(rid) as ds2:
+        assert len(ds2.basins) == 2
+        assert "volume" in ds2
+        assert "volume" in ds2.features_basin
+        assert "volume" not in ds2.features_innate
+        assert "image" in ds2
+        assert len(ds2) == 47
+
+
 @pytest.mark.ckan_config('ckan.plugins', 'dcor_schemas')
 @pytest.mark.usefixtures('clean_db', 'with_request_context')
 @mock.patch('ckan.plugins.toolkit.enqueue_job',
@@ -173,9 +219,10 @@ def test_upload_artifact(enqueue_job_mock, tmp_path):
     path_fake_preview = tmp_path / "test_preview.jpg"
     path_fake_preview.write_text("This is not a real image!")
     # upload the preview
-    s3cc.upload_artifact(rid,
-                         path_artifact=path_fake_preview,
-                         artifact="preview")
+    s3_url2 = s3cc.upload_artifact(rid,
+                                   path_artifact=path_fake_preview,
+                                   artifact="preview")
+    assert s3_url == s3_url2.replace("preview", "resource")
     # make sure that worked
     assert s3cc.artifact_exists(rid, "preview")
     # attempt to download the private artifact

dcor_shared-0.9.0/tests/test_testing.py

@@ -0,0 +1,85 @@
+import pathlib
+import pytest
+
+import ckan.tests.factories as factories
+import requests
+
+from dcor_shared.testing import make_dataset, make_dataset_via_s3
+from dcor_shared import s3cc, sha256sum
+
+data_path = pathlib.Path(__file__).parent / "data"
+
+
+@pytest.mark.ckan_config('ckan.plugins', 'dcor_schemas')
+@pytest.mark.usefixtures('clean_db', 'with_request_context')
+def test_make_dataset():
+    user = factories.User()
+    owner_org = factories.Organization(users=[{
+        'name': user['id'],
+        'capacity': 'admin'
+    }])
+    # Note: `call_action` bypasses authorization!
+    create_context = {'ignore_auth': False,
+                      'user': user['name'], 'api_version': 3}
+    ds_dict = make_dataset(create_context, owner_org,
+                           activate=False)
+    assert ds_dict["state"] == "draft"
+
+
+@pytest.mark.ckan_config('ckan.plugins', 'dcor_schemas')
+@pytest.mark.usefixtures('clean_db', 'with_request_context')
+def test_make_dataset_no_user_specified():
+    ds_dict = make_dataset(activate=False)
+    assert ds_dict["state"] == "draft"
+
+
+@pytest.mark.ckan_config('ckan.plugins', 'dcor_schemas')
+@pytest.mark.usefixtures('clean_db', 'with_request_context')
+def test_make_dataset_with_resource(create_with_upload):
+    # upload a *valid* [sic] .rtdc File (this is the control)
+    ds_dict, res_dict = make_dataset(
+        create_with_upload=create_with_upload,
+        resource_path=data_path / "calibration_beads_47.rtdc",
+        activate=True)
+    assert len(ds_dict["resources"]) == 1
+    assert "id" in res_dict
+    assert res_dict["package_id"] == ds_dict["id"]
+
+
+@pytest.mark.ckan_config('ckan.plugins', 'dcor_schemas')
+@pytest.mark.usefixtures('clean_db', 'with_request_context')
+def test_make_dataset_via_s3():
+    user = factories.User()
+    owner_org = factories.Organization(users=[{
+        'name': user['id'],
+        'capacity': 'admin'
+    }])
+    # Note: `call_action` bypasses authorization!
+    create_context = {'ignore_auth': False,
+                      'user': user['name'], 'api_version': 3}
+    ds_dict = make_dataset_via_s3(create_context, owner_org, activate=False)
+    assert ds_dict["state"] == "draft"
+
+
+@pytest.mark.ckan_config('ckan.plugins', 'dcor_schemas')
+@pytest.mark.usefixtures('clean_db', 'with_request_context')
+def test_make_dataset_with_resource_via_s3(tmp_path):
+    # upload a *valid* [sic] .rtdc File (this is the control)
+    ds_dict, res_dict = make_dataset_via_s3(
+        resource_path=data_path / "calibration_beads_47.rtdc",
+        activate=True,
+        private=False,
+    )
+    assert len(ds_dict["resources"]) == 1
+    assert "id" in res_dict
+    assert res_dict["package_id"] == ds_dict["id"]
+
+    # verify SHA256 sum of the resource
+    ps_url = s3cc.get_s3_url_for_artifact(res_dict["id"], artifact="resource")
+    response = requests.get(ps_url)
+    dl_path = tmp_path / "calbeads.rtdc"
+    with dl_path.open("wb") as fd:
+        fd.write(response.content)
+
+    assert sha256sum(dl_path) == \
+           "490efdf5d9bb4cd4b2a6bcf2fe54d4dc201c38530140bcb168980bf8bf846c73"

dcor_shared-0.7.4/dcor_shared/testing.py

@@ -1,145 +0,0 @@
-from io import BytesIO
-
-import ckan.tests.helpers as helpers
-from ckan.tests.pytest_ckan.fixtures import FakeFileStorage
-import requests
-from requests_toolbelt import MultipartEncoder, MultipartEncoderMonitor
-
-import pytest
-
-
-@pytest.fixture
-def create_with_upload_no_temp(clean_db, ckan_config, monkeypatch):
-    """
-    Create upload without tempdir
-    """
-
-    def factory(data, filename, context=None, **kwargs):
-        if context is None:
-            context = {}
-        action = kwargs.pop("action", "resource_create")
-        field = kwargs.pop("upload_field_name", "upload")
-        test_file = BytesIO()
-        if type(data) is not bytes:
-            data = bytes(data, encoding="utf-8")
-        test_file.write(data)
-        test_file.seek(0)
-        test_resource = FakeFileStorage(test_file, filename)
-
-        params = {
-            field: test_resource,
-        }
-        params.update(kwargs)
-        return helpers.call_action(action, context, **params)
-    return factory
-
-
-def make_dataset(create_context, owner_org, create_with_upload=None,
-                 resource_path=None, activate=False, **kwargs):
-    if "title" not in kwargs:
-        kwargs["title"] = "test-dataset"
-    if "authors" not in kwargs:
-        kwargs["authors"] = "Peter Pan"
-    if "license_id" not in kwargs:
-        kwargs["license_id"] = "CC-BY-4.0"
-    assert "state" not in kwargs, "must not be set"
-    assert "owner_org" not in kwargs, "must not be set"
-    # create a dataset
-    ds = helpers.call_action("package_create", create_context,
-                             owner_org=owner_org["name"],
-                             state="draft",
-                             **kwargs
-                             )
-
-    if create_with_upload is not None:
-        rs = make_resource(resource_path=resource_path,
-                           create_with_upload=create_with_upload,
-                           create_context=create_context,
-                           dataset_id=ds["id"])
-
-    if activate:
-        helpers.call_action("package_patch", create_context,
-                            id=ds["id"],
-                            state="active")
-
-    ds_dict = helpers.call_action("package_show", id=ds["id"])
-
-    if create_with_upload is not None:
-        # updated resource dictionary
-        rs_dict = helpers.call_action("resource_show", id=rs["id"])
-        return ds_dict, rs_dict
-    else:
-        return ds_dict
-
-
-def make_resource(resource_path, create_with_upload, create_context,
-                  dataset_id):
-    content = resource_path.read_bytes()
-    rs = create_with_upload(
-        data=content,
-        filename='test.rtdc',
-        context=create_context,
-        package_id=dataset_id,
-        url="upload",
-    )
-    resource = helpers.call_action("resource_show", id=rs["id"])
-    return resource
-
-
-def synchronous_enqueue_job(job_func, args=None, kwargs=None, title=None,
-                            queue=None, rq_kwargs=None):
-    """
-    Synchronous mock for ``ckan.plugins.toolkit.enqueue_job``.
-
-
-    Due to the asynchronous nature of background jobs, code that uses them
-    needs to be handled specially when writing tests.
-
-    A common approach is to use the mock package to replace the
-    ckan.plugins.toolkit.enqueue_job function with a mock that executes jobs
-    synchronously instead of asynchronously
-
-    Also, since we are running the tests as root on a ckan instance that
-    is run by www-data, modifying files on disk in background jobs
-    (which were started by supervisor as www-data) does not work.
-    """
-    if rq_kwargs is None:
-        rq_kwargs = {}
-    args = args or []
-    kwargs = kwargs or {}
-    job_func(*args, **kwargs)
-
-
-def upload_presigned_to_s3(psurl, fields, path_to_upload):
-    """Helper function for uploading data to S3
-
-    This is exactly how DCOR-Aid would be uploading things (with the
-    requests_toolbelt package). This could have been a little simpler,
-    but for the sake of reproducibility, we do it the DCOR-Aid way.
-    """
-    # callback function for monitoring the upload progress
-    # open the input file for streaming
-    with path_to_upload.open("rb") as fd:
-        fields["file"] = (fields["key"], fd)
-        e = MultipartEncoder(fields=fields)
-        m = MultipartEncoderMonitor(
-            e, lambda monitor: print(f"Bytes: {monitor.bytes_read}"))
-        # Increase the read size to speed-up upload (the default chunk
-        # size for uploads in urllib is 8k which results in a lot of
-        # Python code being involved in uploading a 20GB file; Setting
-        # the chunk size to 4MB should increase the upload speed):
-        # https://github.com/requests/toolbelt/issues/75
-        # #issuecomment-237189952
-        m._read = m.read
-        m.read = lambda size: m._read(4 * 1024 * 1024)
-        # perform the actual upload
-        hrep = requests.post(
-            psurl,
-            data=m,
-            headers={'Content-Type': m.content_type},
-            verify=True,  # verify SSL connection
-            timeout=27.3,  # timeout to avoid freezing
-        )
-    if hrep.status_code != 204:
-        raise ValueError(
-            f"Upload failed with {hrep.status_code}: {hrep.reason}")

dcor_shared-0.7.4/tests/test_testing.py

@@ -1,46 +0,0 @@
-import pathlib
-import pytest
-
-import ckan.tests.factories as factories
-
-from dcor_shared.testing import make_dataset
-
-data_path = pathlib.Path(__file__).parent / "data"
-
-
-@pytest.mark.ckan_config('ckan.plugins', 'dcor_schemas')
-@pytest.mark.usefixtures('clean_db', 'with_request_context')
-def test_make_dataset():
-    user = factories.User()
-    owner_org = factories.Organization(users=[{
-        'name': user['id'],
-        'capacity': 'admin'
-    }])
-    # Note: `call_action` bypasses authorization!
-    create_context = {'ignore_auth': False,
-                      'user': user['name'], 'api_version': 3}
-    ds_dict = make_dataset(create_context, owner_org,
-                           activate=False)
-    assert ds_dict["state"] == "draft"
-
-
-@pytest.mark.ckan_config('ckan.plugins', 'dcor_schemas')
-@pytest.mark.usefixtures('clean_db', 'with_request_context')
-def test_make_dataset_with_resource(create_with_upload):
-    user = factories.User()
-    owner_org = factories.Organization(users=[{
-        'name': user['id'],
-        'capacity': 'admin'
-    }])
-    # Note: `call_action` bypasses authorization!
-    create_context = {'ignore_auth': False,
-                      'user': user['name'], 'api_version': 3}
-    # upload a *valid* [sic] .rtdc File (this is the control)
-    ds_dict, res_dict = make_dataset(
-        create_context, owner_org,
-        create_with_upload=create_with_upload,
-        resource_path=data_path / "calibration_beads_47.rtdc",
-        activate=True)
-    assert len(ds_dict["resources"]) == 1
-    assert "id" in res_dict
-    assert res_dict["package_id"] == ds_dict["id"]