dcor-shared 0.12.2__tar.gz → 0.13.0__tar.gz

{dcor_shared-0.12.2 → dcor_shared-0.13.0}/.github/workflows/check.yml

@@ -2,6 +2,20 @@ name: Checks
 on: [push, pull_request]
 
 jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+      - name: Install requirements
+        run: pip install flake8
+      - name: Check syntax
+        run: |
+          flake8 dcor_shared
+          flake8 tests
+
   test:
     strategy:
       matrix:

{dcor_shared-0.12.2 → dcor_shared-0.13.0}/CHANGELOG

@@ -1,3 +1,16 @@
+0.13.0
+ - BREAKING: remove `get_resource_path`, `get_dcor_depot_path`,
+   `get_dcor_users_depot_path`
+ - BREAKING: `wait_for_resource` and `get_dc_instance` now only check S3 data
+ - feat: add `is_resource_private` method
+ - fix: disable SHA256 verification in boto3 (not supported everywhere);
+   note that we are still doing SHA256 verification after the upload
+ - enh: add "artifact" kwarg to `get_s3_dc_handle`
+ - enh: partial resource ID in basin names for `get_s3_dc_handle_basin_based`
+ - enh: add "perishable" key for basins in `get_s3_dc_handle_basin_based`
+ - setup: bump dclab to 0.64.0
+0.12.3
+ - enh: improved resource availability check
 0.12.2
  - fix: testing did not append S3 resources
 0.12.1

{dcor_shared-0.12.2 → dcor_shared-0.13.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dcor_shared
-Version: 0.12.2
+Version: 0.13.0
 Summary: Functionalities shared by the DCOR CKAN extensions
 Author: Paul Müller
 Maintainer-email: Paul Müller <dev@craban.de>
@@ -17,7 +17,7 @@ Requires-Python: <4,>=3.8
 Description-Content-Type: text/x-rst
 License-File: LICENSE
 Requires-Dist: boto3
-Requires-Dist: dclab[s3]>=0.60.9
+Requires-Dist: dclab[s3]>=0.64.0
 Dynamic: license-file
 
 dcor_shared

{dcor_shared-0.12.2 → dcor_shared-0.13.0}/dcor_shared/__init__.py

@@ -1,10 +1,12 @@
 # flake8: noqa: F401
 from .ckan import (
-    get_ckan_config_option, get_resource_dc_config, get_resource_path, get_resource_info
+    get_ckan_config_option, get_ckan_storage_path, get_resource_dc_config,
+    get_resource_info, is_resource_private
 )
-from .data import DUMMY_BYTES, sha256sum, wait_for_resource
+from .data import wait_for_resource
 from .dcinst import get_dc_instance
 from .mime import DC_MIME_TYPES, VALID_FORMATS
 from . import paths
 from .rqjob import RQJob, rqjob_register
+from .util import sha256sum
 from ._version import version as __version__

{dcor_shared-0.12.2 → dcor_shared-0.13.0}/dcor_shared/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.12.2'
-__version_tuple__ = version_tuple = (0, 12, 2)
+__version__ = version = '0.13.0'
+__version_tuple__ = version_tuple = (0, 13, 0)

{dcor_shared-0.12.2 → dcor_shared-0.13.0}/dcor_shared/ckan.py

@@ -1,6 +1,5 @@
 import os
 import pathlib
-import warnings
 
 from .parse import get_ini_config_option
 
@@ -73,25 +72,9 @@ def get_resource_info(resource_id):
     return ds_dict, res_dict
 
 
-def get_resource_path(resource_id, create_dirs=False):
-    """Return the expected local path for a resource identifier
-
-    If `create_dirs` is True, create the parent directory tree.
-    """
-    warnings.warn("`get_resource_path` should not be used since DCOR moved "
-                  "to storing data solely on S3",
-                  DeprecationWarning)
-    rid = resource_id
-    resources_path = get_ckan_storage_path() / "resources"
-    pdir = resources_path / rid[:3] / rid[3:6]
-    path = pdir / rid[6:]
-    if create_dirs:
-        try:
-            pdir.mkdir(parents=True, exist_ok=True)
-            os.makedirs(pdir)
-            os.chown(pdir,
-                     os.stat(resources_path).st_uid,
-                     os.stat(resources_path).st_gid)
-        except OSError:
-            pass
-    return pathlib.Path(path)
+def is_resource_private(resource_id):
+    """Return true if a resource is private"""
+    from ckan import model
+    resource = model.Resource.get(resource_id)
+    dataset = model.Package.get(resource.package_id)
+    return dataset.private

dcor_shared-0.13.0/dcor_shared/data.py

@@ -0,0 +1,58 @@
+import functools
+import time
+
+from . import s3cc
+from .util import sha256sum  # noqa: F401
+
+
+@functools.lru_cache(maxsize=100)
+def wait_for_resource(resource_id: str,
+                      timeout: float = 10):
+    """Wait for resource to be available
+
+    This function can be used by other plugins to ensure that
+    a resource is available for processing.
+
+    The only way (since 2024) of uploading data is via pre-signed URLs
+    to an S3 instance. Here, we have to make sure that the
+    upload is complete and the file exists. If this is the case,
+    then uploads should have already completed when this function
+    is called, so we only check for the existence of the resource
+    in ckan and whether the `s3_available` attribute is defined.
+    """
+    from ckan import logic
+
+    if len(resource_id) != 36:
+        raise ValueError(f"Invalid resource id: {resource_id}")
+
+    resource_show = logic.get_action("resource_show")
+
+    t0 = time.time()
+    while True:
+        try:
+            res_dict = resource_show(context={'ignore_auth': True,
+                                              'user': 'default'},
+                                     data_dict={"id": resource_id})
+        except logic.NotFound:
+            # Other processes are still working on getting the resource
+            # online. We have to wait.
+            time.sleep(5)
+            continue
+
+        # object exists in database?
+        s3_ok = res_dict.get("s3_available", None)
+        if s3_ok:
+            # If the resource is on S3, it is considered to be available.
+            break
+
+        # object exists on S3?
+        try:
+            s3_exist = s3cc.artifact_exists(res_dict["id"])
+        except BaseException:
+            s3_exist = False
+        if s3_exist:
+            break
+
+        if time.time() - t0 > timeout:
+            raise OSError(f"Data import seems to take too long "
+                          f"for '{resource_id}'!")

dcor_shared-0.13.0/dcor_shared/dcinst.py

@@ -0,0 +1,10 @@
+from . import s3cc
+
+
+def get_dc_instance(rid):
+    """Return an instance of dclab's `RTDCBase` for a resource identifier"""
+    # The resource must be on S3
+    if s3cc.artifact_exists(rid):
+        return s3cc.get_s3_dc_handle(rid)
+    else:
+        raise ValueError(f"Could not find resource {rid} anywhere")

dcor_shared-0.13.0/dcor_shared/paths.py

@@ -0,0 +1,16 @@
+import pathlib
+
+from .ckan import get_ckan_storage_path, get_ckan_webassets_path  # noqa: F401
+from .ckan import get_ckan_config_path  # noqa: F401
+
+
+def get_nginx_config_path():
+    return pathlib.Path("/etc/nginx/sites-enabled/ckan")
+
+
+def get_supervisord_worker_config_path():
+    return pathlib.Path("/etc/supervisor/conf.d/ckan-worker.conf")
+
+
+def get_uwsgi_config_path():
+    return pathlib.Path("/etc/ckan/default/ckan-uwsgi.ini")

{dcor_shared-0.12.2 → dcor_shared-0.13.0}/dcor_shared/s3.py

@@ -11,10 +11,11 @@ from typing import List, Tuple
 import warnings
 
 import boto3
+from botocore.config import Config
 import botocore.exceptions
 
 from .ckan import get_ckan_config_option
-from .data import sha256sum
+from .util import sha256sum
 
 
 def compute_checksum(bucket_name, object_name, max_size=None):
@@ -305,6 +306,9 @@ def get_s3():
         use_ssl=ssl_verify,
         verify=ssl_verify,
         endpoint_url=get_ckan_config_option("dcor_object_store.endpoint_url"),
+        # https://github.com/boto/boto3/issues/4400#issuecomment-2600742103
+        config=Config(request_checksum_calculation="when_required",
+                      response_checksum_validation="when_required")
     )
     s3_resource = s3_session.resource(
         service_name="s3",
@@ -493,7 +497,7 @@ def upload_file(bucket_name, object_name, path, sha256=None, private=True,
 
     path_size = pathlib.Path(path).stat().st_size
     s3_client, _, _ = get_s3()
-    s3_bucket = require_bucket(bucket_name)
+    require_bucket(bucket_name)
 
     perform_upload = True
     if not override:
@@ -501,7 +505,8 @@ def upload_file(bucket_name, object_name, path, sha256=None, private=True,
                                            object_name=object_name)
 
     if perform_upload:
-        s3_bucket.upload_file(Filename=str(path),
+        s3_client.upload_file(Filename=str(path),
+                              Bucket=bucket_name,
                               Key=object_name,
                               # ExtraArgs={
                               # # verification of the upload (breaks OpenStack)

{dcor_shared-0.12.2 → dcor_shared-0.13.0}/dcor_shared/s3cc.py

@@ -8,7 +8,6 @@ import io
 import functools
 import pathlib
 from typing import Literal
-from urllib.parse import urlparse
 import warnings
 
 import dclab
@@ -16,7 +15,8 @@ from dclab.rtdc_dataset import fmt_hdf5, fmt_s3
 import h5py
 
 from .ckan import (
-    get_ckan_config_option, get_resource_dc_config, get_resource_info
+    get_ckan_config_option, get_resource_dc_config, get_resource_info,
+    is_resource_private
 )
 
 from . import s3
@@ -169,7 +169,9 @@ def get_s3_bucket_name_for_resource(resource_id):
     return bucket_name
 
 
-def get_s3_dc_handle(resource_id):
+def get_s3_dc_handle(
+        resource_id: str,
+        artifact: Literal["condensed", "preview", "resource"] = "resource"):
     """Return an instance of :class:`RTDC_S3`
 
     The data are accessed directly via S3 using DCOR's access credentials.
@@ -178,7 +180,7 @@ def get_s3_dc_handle(resource_id):
     The resource with the identifier `resource_id` must exist in the
     CKAN database.
     """
-    s3_url = get_s3_url_for_artifact(resource_id)
+    s3_url = get_s3_url_for_artifact(resource_id, artifact=artifact)
     ds = fmt_s3.RTDC_S3(
         url=s3_url,
         access_key_id=get_ckan_config_option(
@@ -203,11 +205,22 @@ def get_s3_dc_handle_basin_based(resource_id):
     that initialization takes slightly longer and that, if private
     resources are accessed, the presigned URLs in the basins are only
     valid for a fixed time period.
+
+    Parameters
+    ----------
+    resource_id : str
+        CKAN resource identifier
+
+    Returns
+    -------
+    ds: RTDCBase
+        basin-based dclab dataset
     """
-    ds_dict, res_dict = get_resource_info(resource_id)
+    artifacts = ["resource", "condensed"]
     basin_paths = []
-    for artifact in ["resource", "condensed"]:
-        if ds_dict["private"]:
+    private = is_resource_private(resource_id)
+    for artifact in artifacts:
+        if private:
             bp = create_presigned_url(resource_id, artifact=artifact)
         else:
             bp = get_s3_url_for_artifact(resource_id, artifact=artifact)
@@ -220,18 +233,19 @@ def get_s3_dc_handle_basin_based(resource_id):
         hv.require_group("events")
         hw = dclab.RTDCWriter(hv)
         hw.store_metadata(get_resource_dc_config(resource_id))
-        for bp in basin_paths:
-            urlp = urlparse(bp)  # e.g. http://localhost/bucket/resource/id
+        for bp, artifact in zip(basin_paths, artifacts):
             hw.store_basin(
-                basin_name=urlp.path.split("/")[2],  # e.g. "resource"
+                basin_name=f"{artifact}-{resource_id[:5]}",  # "resource-92a12"
                 basin_format="http",
                 basin_type="remote",
                 basin_locs=[bp],
                 # Don't verify anything. This would only cost time,
                 # and we know these objects exist.
                 verify=False,
+                perishable=private,
                 )
-    return fmt_hdf5.RTDC_HDF5(fd)
+    ds = fmt_hdf5.RTDC_HDF5(fd)
+    return ds
 
 
 def get_s3_url_for_artifact(

{dcor_shared-0.12.2 → dcor_shared-0.13.0}/dcor_shared/testing.py

@@ -8,6 +8,7 @@ import uuid
 import warnings
 
 import ckan.authz
+import ckan.model
 import ckan.tests.factories as factories
 import ckan.tests.helpers as helpers
 from ckan.tests.pytest_ckan.fixtures import FakeFileStorage
@@ -134,7 +135,9 @@ def make_dataset_via_s3(
     """
     if create_context is None:
         user = factories.User()
+        user_obj = ckan.model.User.by_name(user["name"])
         create_context = {'ignore_auth': False,
+                          'auth_user_obj': user_obj,
                           'user': user['name'],
                           'api_version': 3}
         user_id = user["id"]
@@ -144,10 +147,11 @@ def make_dataset_via_s3(
             create_context["user"], allow_none=True)
 
     if owner_org is None:
-        owner_org = factories.Organization(users=[{
-            'name': user_id,
-            'capacity': 'admin'
-        }])
+        owner_org = factories.Organization(
+            users=[{
+                'name': user_id,
+                'capacity': 'admin'
+            }])
 
     if "title" not in kwargs:
         kwargs["title"] = "test-dataset-S3"

dcor_shared-0.13.0/dcor_shared/util.py

@@ -0,0 +1,10 @@
+import hashlib
+
+
+def sha256sum(path):
+    """Compute the SHA256 hash of a file in 1MB chunks"""
+    file_hash = hashlib.sha256()
+    with open(path, "rb") as fd:
+        while data := fd.read(2 ** 20):
+            file_hash.update(data)
+    return file_hash.hexdigest()

{dcor_shared-0.12.2 → dcor_shared-0.13.0}/dcor_shared.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dcor_shared
-Version: 0.12.2
+Version: 0.13.0
 Summary: Functionalities shared by the DCOR CKAN extensions
 Author: Paul Müller
 Maintainer-email: Paul Müller <dev@craban.de>
@@ -17,7 +17,7 @@ Requires-Python: <4,>=3.8
 Description-Content-Type: text/x-rst
 License-File: LICENSE
 Requires-Dist: boto3
-Requires-Dist: dclab[s3]>=0.60.9
+Requires-Dist: dclab[s3]>=0.64.0
 Dynamic: license-file
 
 dcor_shared

{dcor_shared-0.12.2 → dcor_shared-0.13.0}/dcor_shared.egg-info/SOURCES.txt

@@ -18,13 +18,13 @@ dcor_shared/rqjob.py
 dcor_shared/s3.py
 dcor_shared/s3cc.py
 dcor_shared/testing.py
+dcor_shared/util.py
 dcor_shared.egg-info/PKG-INFO
 dcor_shared.egg-info/SOURCES.txt
 dcor_shared.egg-info/dependency_links.txt
 dcor_shared.egg-info/requires.txt
 dcor_shared.egg-info/top_level.txt
 tests/requirements.txt
-tests/test_basic.py
 tests/test_ckan.py
 tests/test_data.py
 tests/test_s3.py

dcor_shared-0.13.0/dcor_shared.egg-info/requires.txt

@@ -0,0 +1,2 @@
+boto3
+dclab[s3]>=0.64.0

{dcor_shared-0.12.2 → dcor_shared-0.13.0}/pyproject.toml

@@ -28,7 +28,7 @@ classifiers = [
 license = {text = "GNU Affero General Public License v3 or later (AGPLv3+)"}
 dependencies = [
     "boto3",
-    "dclab[s3]>=0.60.9",
+    "dclab[s3]>=0.64.0",
 ]
 dynamic = ["version"]
 

{dcor_shared-0.12.2 → dcor_shared-0.13.0}/tests/requirements.txt

@@ -2,3 +2,4 @@ pytest
 requests
 requests_toolbelt
 dclab[s3]
+pytest_factoryboy

{dcor_shared-0.12.2 → dcor_shared-0.13.0}/tests/test_ckan.py

@@ -1,7 +1,9 @@
 import pathlib
 from unittest import mock
 
-from dcor_shared import get_resource_dc_config, get_resource_info
+from dcor_shared import (
+    get_resource_dc_config, get_resource_info, is_resource_private
+)
 
 import pytest
 from dcor_shared.testing import make_dataset_via_s3, synchronous_enqueue_job
@@ -36,3 +38,14 @@ def test_get_resource_info(enqueue_job_mock):
     ds_dict_2, res_dict_2 = get_resource_info(res_dict["id"])
     assert ds_dict == ds_dict_2
     assert res_dict == res_dict_2
+
+
+@pytest.mark.ckan_config('ckan.plugins', 'dcor_schemas')
+@pytest.mark.usefixtures('clean_db', 'with_request_context')
+@pytest.mark.parametrize("private", [True, False])
+def test_is_resource_private(private):
+    _, res_dict = make_dataset_via_s3(
+        resource_path=data_path / "calibration_beads_47.rtdc",
+        private=private,
+        activate=True)
+    assert is_resource_private(res_dict["id"]) == private

{dcor_shared-0.12.2 → dcor_shared-0.13.0}/tests/test_s3cc.py

@@ -20,7 +20,9 @@ import requests
 data_path = pathlib.Path(__file__).parent / "data"
 
 
-def setup_s3_resource_on_ckan(private=False, resource_path=None):
+def setup_s3_resource_on_ckan(private: bool = False,
+                              resource_path: str | pathlib.Path = None
+                              ):
     """Create an S3 resource in CKAN"""
     if resource_path is None:
         resource_path = data_path / "calibration_beads_47.rtdc"
@@ -34,7 +36,9 @@ def setup_s3_resource_on_ckan(private=False, resource_path=None):
     }])
 
     test_context = {'ignore_auth': False,
-                    'user': user['name'], 'model': model, 'api_version': 3}
+                    'user': user['name'],
+                    'model': model,
+                    'api_version': 3}
 
     # Upload the resource to S3 (note that it is not required that the
     # dataset exists)
@@ -167,6 +171,23 @@ def test_get_s3_handle(enqueue_job_mock):
         assert len(ds) == 47
 
 
+@pytest.mark.ckan_config('ckan.plugins', 'dcor_schemas dc_serve')
+@pytest.mark.usefixtures('clean_db', 'with_request_context')
+@mock.patch('ckan.plugins.toolkit.enqueue_job',
+            side_effect=synchronous_enqueue_job)
+def test_get_s3_handle_condensed(enqueue_job_mock, tmp_path):
+    resource_path = tmp_path / "data.rtdc"
+    shutil.copy2(data_path / "calibration_beads_47.rtdc", resource_path)
+    _, res_dict = make_dataset_via_s3(
+        resource_path=resource_path,
+        activate=True)
+    rid = res_dict["id"]
+    expected_path = s3cc.get_s3_url_for_artifact(rid, artifact="condensed")
+    with s3cc.get_s3_dc_handle(rid, artifact="condensed") as ds:
+        assert ds.path == expected_path
+        assert len(ds) == 47
+
+
 @pytest.mark.ckan_config('ckan.plugins', 'dcor_depot dcor_schemas dc_serve')
 @pytest.mark.usefixtures('clean_db', 'with_request_context')
 @mock.patch('ckan.plugins.toolkit.enqueue_job',
@@ -198,6 +219,54 @@ def test_get_s3_dc_handle_basin_based(enqueue_job_mock, tmp_path):
         assert len(ds2) == 47
 
 
+@pytest.mark.ckan_config('ckan.plugins', 'dcor_depot dcor_schemas dc_serve')
+@pytest.mark.usefixtures('clean_db', 'with_request_context')
+@mock.patch('ckan.plugins.toolkit.enqueue_job',
+            side_effect=synchronous_enqueue_job)
+def test_get_s3_dc_handle_basin_based_public_urls(enqueue_job_mock, tmp_path):
+    resource_path = tmp_path / "data.rtdc"
+    shutil.copy2(data_path / "calibration_beads_47.rtdc", resource_path)
+    with h5py.File(resource_path, "a") as h5:
+        del h5["events/volume"]
+
+    _, res_dict = make_dataset_via_s3(
+        resource_path=resource_path,
+        private=False,
+        activate=True)
+    rid = res_dict["id"]
+
+    with s3cc.get_s3_dc_handle_basin_based(rid) as ds:
+        # get the basins
+        for bn_dict in ds.basins_get_dicts():
+            assert not bn_dict["perishable"]
+            for url in bn_dict["urls"]:
+                assert not url.lower().count("expires")
+
+
+@pytest.mark.ckan_config('ckan.plugins', 'dcor_depot dcor_schemas dc_serve')
+@pytest.mark.usefixtures('clean_db', 'with_request_context')
+@mock.patch('ckan.plugins.toolkit.enqueue_job',
+            side_effect=synchronous_enqueue_job)
+def test_get_s3_dc_handle_basin_based_private_urls(enqueue_job_mock, tmp_path):
+    resource_path = tmp_path / "data.rtdc"
+    shutil.copy2(data_path / "calibration_beads_47.rtdc", resource_path)
+    with h5py.File(resource_path, "a") as h5:
+        del h5["events/volume"]
+
+    _, res_dict = make_dataset_via_s3(
+        resource_path=resource_path,
+        private=True,
+        activate=True)
+    rid = res_dict["id"]
+
+    with s3cc.get_s3_dc_handle_basin_based(rid) as ds:
+        # get the basins
+        for bn_dict in ds.basins_get_dicts():
+            assert bn_dict["perishable"]
+            for url in bn_dict["urls"]:
+                assert url.lower().count("expires")
+
+
 @pytest.mark.ckan_config('ckan.plugins', 'dcor_schemas')
 @pytest.mark.usefixtures('clean_db', 'with_request_context')
 @mock.patch('ckan.plugins.toolkit.enqueue_job',

dcor_shared-0.12.2/dcor_shared/data.py

@@ -1,92 +0,0 @@
-import functools
-import hashlib
-import pathlib
-import time
-
-from .ckan import get_resource_path
-
-
-#: Content of the dummy file created when importing data.
-DUMMY_BYTES = b"[Data import pending]"
-
-
-def sha256sum(path):
-    """Compute the SHA256 hash of a file in 1MB chunks"""
-    file_hash = hashlib.sha256()
-    with open(path, "rb") as fd:
-        while data := fd.read(2 ** 20):
-            file_hash.update(data)
-    return file_hash.hexdigest()
-
-
-@functools.lru_cache(maxsize=100)
-def wait_for_resource(resource_id: str,
-                      timeout: float = 10):
-    """Wait for resource to be available
-
-    This function can be used by other plugins to ensure that
-    a resource is available for processing.
-
-    There multiple ways for data to become available:
-
-    1. The ckanext-dcor_depot plugin imports data by touching
-       dummy files and then sym-linking to data on disk. Here
-       we just check that the file is not a dummy file anymore.
-    2. Legacy uploads via nginx/uwsgi directly into CKAN and onto
-       the local block storage worked the same way. We have to wait
-       for the dummy file to be replaced.
-    3. The new (2024) way of uploading data is via pre-signed URLs
-       to an S3 instance. Here, we have to make sure that the
-       upload is complete and the file exists. If this is the case,
-       then uploads should have already completed when this function
-       is called, so we only check for the existence of the resource
-       in ckan and whether the `s3_available` attribute is defined.
-    """
-    from ckan.common import config
-    from ckan import logic
-
-    if len(resource_id) != 36:
-        raise ValueError(f"Invalid resource id: {resource_id}")
-
-    resource_show = logic.get_action("resource_show")
-    path = pathlib.Path(get_resource_path(resource_id))
-
-    dcor_depot_available = "dcor_depot" in config.get('ckan.plugins', "")
-    # Initially this was set to 10s, but if `/data` is mounted on a
-    # network share then this part here just takes too long.
-    t0 = time.time()
-    ld = len(DUMMY_BYTES)
-    while True:
-        try:
-            res_dict = resource_show(context={'ignore_auth': True,
-                                              'user': 'default'},
-                                     data_dict={"id": resource_id})
-        except logic.NotFound:
-            # Other processes are still working on getting the resource
-            # online. We have to wait.
-            time.sleep(5)
-            continue
-
-        s3_ok = res_dict.get("s3_available", None)
-        if time.time() - t0 > timeout:
-            raise OSError("Data import seems to take too long "
-                          "for '{}'!".format(path))
-        elif s3_ok is not None:
-            # If the dataset is on S3, it is considered to be available.
-            break
-        elif not path.exists():
-            time.sleep(5)
-            continue
-        elif dcor_depot_available and not path.is_symlink():
-            # Resource is only available when it is symlinked by
-            # the ckanext.dcor_depot `symlink_user_dataset` job
-            # (or by the ckanext.dcor_depot importers).
-            time.sleep(5)
-            continue
-        elif path.stat().st_size == ld and path.read_bytes() == DUMMY_BYTES:
-            # wait a bit
-            time.sleep(5)
-            continue
-        else:
-            # not a dummy file
-            break

dcor_shared-0.12.2/dcor_shared/dcinst.py

@@ -1,20 +0,0 @@
-from dclab.rtdc_dataset import RTDC_HDF5
-
-from .ckan import get_resource_path
-from . import s3cc
-
-
-def get_dc_instance(rid):
-    """Return an instance of dclab's `RTDCBase` for a resource identifier"""
-    # Try local file first
-    path = get_resource_path(rid)
-    if path.is_file():
-        # Disable basins, because they could point to files on the local
-        # file system (security).
-        return RTDC_HDF5(path, enable_basins=False)
-    else:
-        # The resource must be on S3
-        if s3cc.artifact_exists(rid):
-            return s3cc.get_s3_dc_handle(rid)
-        else:
-            raise ValueError(f"Could not find resource {rid} anywhere")

dcor_shared-0.12.2/dcor_shared/paths.py

@@ -1,30 +0,0 @@
-import pathlib
-
-from .ckan import get_ckan_config_path, get_ckan_config_option
-from .ckan import get_ckan_storage_path, get_ckan_webassets_path  # noqa: F401
-
-from .parse import get_ini_config_option
-
-
-def get_dcor_depot_path():
-    return pathlib.Path(get_ckan_config_option(
-        "ckanext.dcor_depot.depots_path"))
-
-
-def get_dcor_users_depot_path():
-    depot = get_dcor_depot_path()
-    return depot / get_ini_config_option(
-        "ckanext.dcor_depot.users_depot_name",
-        get_ckan_config_path())
-
-
-def get_nginx_config_path():
-    return pathlib.Path("/etc/nginx/sites-enabled/ckan")
-
-
-def get_supervisord_worker_config_path():
-    return pathlib.Path("/etc/supervisor/conf.d/ckan-worker.conf")
-
-
-def get_uwsgi_config_path():
-    return pathlib.Path("/etc/ckan/default/ckan-uwsgi.ini")

dcor_shared-0.12.2/dcor_shared.egg-info/requires.txt

@@ -1,2 +0,0 @@
-boto3
-dclab[s3]>=0.60.9

dcor_shared-0.12.2/tests/test_basic.py

@@ -1,5 +0,0 @@
-from dcor_shared import DUMMY_BYTES
-
-
-def test_dummy_bytes():
-    assert DUMMY_BYTES == b"[Data import pending]"