dcor-shared 0.12.3__tar.gz → 0.13.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. {dcor_shared-0.12.3 → dcor_shared-0.13.0}/.github/workflows/check.yml +14 -0
  2. {dcor_shared-0.12.3 → dcor_shared-0.13.0}/CHANGELOG +11 -0
  3. {dcor_shared-0.12.3 → dcor_shared-0.13.0}/PKG-INFO +2 -2
  4. {dcor_shared-0.12.3 → dcor_shared-0.13.0}/dcor_shared/__init__.py +3 -2
  5. {dcor_shared-0.12.3 → dcor_shared-0.13.0}/dcor_shared/_version.py +2 -2
  6. {dcor_shared-0.12.3 → dcor_shared-0.13.0}/dcor_shared/ckan.py +6 -23
  7. dcor_shared-0.13.0/dcor_shared/data.py +58 -0
  8. dcor_shared-0.13.0/dcor_shared/dcinst.py +10 -0
  9. dcor_shared-0.13.0/dcor_shared/paths.py +16 -0
  10. {dcor_shared-0.12.3 → dcor_shared-0.13.0}/dcor_shared/s3.py +7 -2
  11. {dcor_shared-0.12.3 → dcor_shared-0.13.0}/dcor_shared/s3cc.py +25 -11
  12. {dcor_shared-0.12.3 → dcor_shared-0.13.0}/dcor_shared.egg-info/PKG-INFO +2 -2
  13. {dcor_shared-0.12.3 → dcor_shared-0.13.0}/dcor_shared.egg-info/SOURCES.txt +0 -1
  14. dcor_shared-0.13.0/dcor_shared.egg-info/requires.txt +2 -0
  15. {dcor_shared-0.12.3 → dcor_shared-0.13.0}/pyproject.toml +1 -1
  16. {dcor_shared-0.12.3 → dcor_shared-0.13.0}/tests/requirements.txt +1 -0
  17. {dcor_shared-0.12.3 → dcor_shared-0.13.0}/tests/test_ckan.py +14 -1
  18. {dcor_shared-0.12.3 → dcor_shared-0.13.0}/tests/test_s3cc.py +71 -2
  19. dcor_shared-0.12.3/dcor_shared/data.py +0 -94
  20. dcor_shared-0.12.3/dcor_shared/dcinst.py +0 -20
  21. dcor_shared-0.12.3/dcor_shared/paths.py +0 -30
  22. dcor_shared-0.12.3/dcor_shared.egg-info/requires.txt +0 -2
  23. dcor_shared-0.12.3/tests/test_basic.py +0 -5
  24. {dcor_shared-0.12.3 → dcor_shared-0.13.0}/.github/workflows/deploy_pypi.yml +0 -0
  25. {dcor_shared-0.12.3 → dcor_shared-0.13.0}/.gitignore +0 -0
  26. {dcor_shared-0.12.3 → dcor_shared-0.13.0}/LICENSE +0 -0
  27. {dcor_shared-0.12.3 → dcor_shared-0.13.0}/MANIFEST.in +0 -0
  28. {dcor_shared-0.12.3 → dcor_shared-0.13.0}/README.rst +0 -0
  29. {dcor_shared-0.12.3 → dcor_shared-0.13.0}/dcor_shared/mime.py +0 -0
  30. {dcor_shared-0.12.3 → dcor_shared-0.13.0}/dcor_shared/parse.py +0 -0
  31. {dcor_shared-0.12.3 → dcor_shared-0.13.0}/dcor_shared/rqjob.py +0 -0
  32. {dcor_shared-0.12.3 → dcor_shared-0.13.0}/dcor_shared/testing.py +0 -0
  33. {dcor_shared-0.12.3 → dcor_shared-0.13.0}/dcor_shared/util.py +0 -0
  34. {dcor_shared-0.12.3 → dcor_shared-0.13.0}/dcor_shared.egg-info/dependency_links.txt +0 -0
  35. {dcor_shared-0.12.3 → dcor_shared-0.13.0}/dcor_shared.egg-info/top_level.txt +0 -0
  36. {dcor_shared-0.12.3 → dcor_shared-0.13.0}/setup.cfg +0 -0
  37. {dcor_shared-0.12.3 → dcor_shared-0.13.0}/tests/data/calibration_beads_47.rtdc +0 -0
  38. {dcor_shared-0.12.3 → dcor_shared-0.13.0}/tests/test_data.py +0 -0
  39. {dcor_shared-0.12.3 → dcor_shared-0.13.0}/tests/test_s3.py +0 -0
  40. {dcor_shared-0.12.3 → dcor_shared-0.13.0}/tests/test_testing.py +0 -0
@@ -2,6 +2,20 @@ name: Checks
2
2
  on: [push, pull_request]
3
3
 
4
4
  jobs:
5
+ lint:
6
+ runs-on: ubuntu-latest
7
+ steps:
8
+ - uses: actions/checkout@v4
9
+ - uses: actions/setup-python@v5
10
+ with:
11
+ python-version: '3.12'
12
+ - name: Install requirements
13
+ run: pip install flake8
14
+ - name: Check syntax
15
+ run: |
16
+ flake8 dcor_shared
17
+ flake8 tests
18
+
5
19
  test:
6
20
  strategy:
7
21
  matrix:
@@ -1,3 +1,14 @@
1
+ 0.13.0
2
+ - BREAKING: remove `get_resource_path`, `get_dcor_depot_path`,
3
+ `get_dcor_users_depot_path`
4
+ - BREAKING: `wait_for_resource` and `get_dc_instance` now only check S3 data
5
+ - feat: add `is_resource_private` method
6
+ - fix: disable SHA256 verification in boto3 (not supported everywhere);
7
+ note that we are still doing SHA256 verification after the upload
8
+ - enh: add "artifact" kwarg to `get_s3_dc_handle`
9
+ - enh: partial resource ID in basin names for `get_s3_dc_handle_basin_based`
10
+ - enh: add "perishable" key for basins in `get_s3_dc_handle_basin_based`
11
+ - setup: bump dclab to 0.64.0
1
12
  0.12.3
2
13
  - enh: improved resource availability check
3
14
  0.12.2
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: dcor_shared
3
- Version: 0.12.3
3
+ Version: 0.13.0
4
4
  Summary: Functionalities shared by the DCOR CKAN extensions
5
5
  Author: Paul Müller
6
6
  Maintainer-email: Paul Müller <dev@craban.de>
@@ -17,7 +17,7 @@ Requires-Python: <4,>=3.8
17
17
  Description-Content-Type: text/x-rst
18
18
  License-File: LICENSE
19
19
  Requires-Dist: boto3
20
- Requires-Dist: dclab[s3]>=0.60.9
20
+ Requires-Dist: dclab[s3]>=0.64.0
21
21
  Dynamic: license-file
22
22
 
23
23
  dcor_shared
@@ -1,8 +1,9 @@
1
1
  # flake8: noqa: F401
2
2
  from .ckan import (
3
- get_ckan_config_option, get_resource_dc_config, get_resource_path, get_resource_info
3
+ get_ckan_config_option, get_ckan_storage_path, get_resource_dc_config,
4
+ get_resource_info, is_resource_private
4
5
  )
5
- from .data import DUMMY_BYTES, wait_for_resource
6
+ from .data import wait_for_resource
6
7
  from .dcinst import get_dc_instance
7
8
  from .mime import DC_MIME_TYPES, VALID_FORMATS
8
9
  from . import paths
@@ -17,5 +17,5 @@ __version__: str
17
17
  __version_tuple__: VERSION_TUPLE
18
18
  version_tuple: VERSION_TUPLE
19
19
 
20
- __version__ = version = '0.12.3'
21
- __version_tuple__ = version_tuple = (0, 12, 3)
20
+ __version__ = version = '0.13.0'
21
+ __version_tuple__ = version_tuple = (0, 13, 0)
@@ -1,6 +1,5 @@
1
1
  import os
2
2
  import pathlib
3
- import warnings
4
3
 
5
4
  from .parse import get_ini_config_option
6
5
 
@@ -73,25 +72,9 @@ def get_resource_info(resource_id):
73
72
  return ds_dict, res_dict
74
73
 
75
74
 
76
- def get_resource_path(resource_id, create_dirs=False):
77
- """Return the expected local path for a resource identifier
78
-
79
- If `create_dirs` is True, create the parent directory tree.
80
- """
81
- warnings.warn("`get_resource_path` should not be used since DCOR moved "
82
- "to storing data solely on S3",
83
- DeprecationWarning)
84
- rid = resource_id
85
- resources_path = get_ckan_storage_path() / "resources"
86
- pdir = resources_path / rid[:3] / rid[3:6]
87
- path = pdir / rid[6:]
88
- if create_dirs:
89
- try:
90
- pdir.mkdir(parents=True, exist_ok=True)
91
- os.makedirs(pdir)
92
- os.chown(pdir,
93
- os.stat(resources_path).st_uid,
94
- os.stat(resources_path).st_gid)
95
- except OSError:
96
- pass
97
- return pathlib.Path(path)
75
+ def is_resource_private(resource_id):
76
+ """Return true if a resource is private"""
77
+ from ckan import model
78
+ resource = model.Resource.get(resource_id)
79
+ dataset = model.Package.get(resource.package_id)
80
+ return dataset.private
@@ -0,0 +1,58 @@
1
+ import functools
2
+ import time
3
+
4
+ from . import s3cc
5
+ from .util import sha256sum # noqa: F401
6
+
7
+
8
+ @functools.lru_cache(maxsize=100)
9
+ def wait_for_resource(resource_id: str,
10
+ timeout: float = 10):
11
+ """Wait for resource to be available
12
+
13
+ This function can be used by other plugins to ensure that
14
+ a resource is available for processing.
15
+
16
+ The only way (since 2024) of uploading data is via pre-signed URLs
17
+ to an S3 instance. Here, we have to make sure that the
18
+ upload is complete and the file exists. If this is the case,
19
+ then uploads should have already completed when this function
20
+ is called, so we only check for the existence of the resource
21
+ in ckan and whether the `s3_available` attribute is defined.
22
+ """
23
+ from ckan import logic
24
+
25
+ if len(resource_id) != 36:
26
+ raise ValueError(f"Invalid resource id: {resource_id}")
27
+
28
+ resource_show = logic.get_action("resource_show")
29
+
30
+ t0 = time.time()
31
+ while True:
32
+ try:
33
+ res_dict = resource_show(context={'ignore_auth': True,
34
+ 'user': 'default'},
35
+ data_dict={"id": resource_id})
36
+ except logic.NotFound:
37
+ # Other processes are still working on getting the resource
38
+ # online. We have to wait.
39
+ time.sleep(5)
40
+ continue
41
+
42
+ # object exists in database?
43
+ s3_ok = res_dict.get("s3_available", None)
44
+ if s3_ok:
45
+ # If the resource is on S3, it is considered to be available.
46
+ break
47
+
48
+ # object exists on S3?
49
+ try:
50
+ s3_exist = s3cc.artifact_exists(res_dict["id"])
51
+ except BaseException:
52
+ s3_exist = False
53
+ if s3_exist:
54
+ break
55
+
56
+ if time.time() - t0 > timeout:
57
+ raise OSError(f"Data import seems to take too long "
58
+ f"for '{resource_id}'!")
@@ -0,0 +1,10 @@
1
+ from . import s3cc
2
+
3
+
4
+ def get_dc_instance(rid):
5
+ """Return an instance of dclab's `RTDCBase` for a resource identifier"""
6
+ # The resource must be on S3
7
+ if s3cc.artifact_exists(rid):
8
+ return s3cc.get_s3_dc_handle(rid)
9
+ else:
10
+ raise ValueError(f"Could not find resource {rid} anywhere")
@@ -0,0 +1,16 @@
1
+ import pathlib
2
+
3
+ from .ckan import get_ckan_storage_path, get_ckan_webassets_path # noqa: F401
4
+ from .ckan import get_ckan_config_path # noqa: F401
5
+
6
+
7
+ def get_nginx_config_path():
8
+ return pathlib.Path("/etc/nginx/sites-enabled/ckan")
9
+
10
+
11
+ def get_supervisord_worker_config_path():
12
+ return pathlib.Path("/etc/supervisor/conf.d/ckan-worker.conf")
13
+
14
+
15
+ def get_uwsgi_config_path():
16
+ return pathlib.Path("/etc/ckan/default/ckan-uwsgi.ini")
@@ -11,6 +11,7 @@ from typing import List, Tuple
11
11
  import warnings
12
12
 
13
13
  import boto3
14
+ from botocore.config import Config
14
15
  import botocore.exceptions
15
16
 
16
17
  from .ckan import get_ckan_config_option
@@ -305,6 +306,9 @@ def get_s3():
305
306
  use_ssl=ssl_verify,
306
307
  verify=ssl_verify,
307
308
  endpoint_url=get_ckan_config_option("dcor_object_store.endpoint_url"),
309
+ # https://github.com/boto/boto3/issues/4400#issuecomment-2600742103
310
+ config=Config(request_checksum_calculation="when_required",
311
+ response_checksum_validation="when_required")
308
312
  )
309
313
  s3_resource = s3_session.resource(
310
314
  service_name="s3",
@@ -493,7 +497,7 @@ def upload_file(bucket_name, object_name, path, sha256=None, private=True,
493
497
 
494
498
  path_size = pathlib.Path(path).stat().st_size
495
499
  s3_client, _, _ = get_s3()
496
- s3_bucket = require_bucket(bucket_name)
500
+ require_bucket(bucket_name)
497
501
 
498
502
  perform_upload = True
499
503
  if not override:
@@ -501,7 +505,8 @@ def upload_file(bucket_name, object_name, path, sha256=None, private=True,
501
505
  object_name=object_name)
502
506
 
503
507
  if perform_upload:
504
- s3_bucket.upload_file(Filename=str(path),
508
+ s3_client.upload_file(Filename=str(path),
509
+ Bucket=bucket_name,
505
510
  Key=object_name,
506
511
  # ExtraArgs={
507
512
  # # verification of the upload (breaks OpenStack)
@@ -8,7 +8,6 @@ import io
8
8
  import functools
9
9
  import pathlib
10
10
  from typing import Literal
11
- from urllib.parse import urlparse
12
11
  import warnings
13
12
 
14
13
  import dclab
@@ -16,7 +15,8 @@ from dclab.rtdc_dataset import fmt_hdf5, fmt_s3
16
15
  import h5py
17
16
 
18
17
  from .ckan import (
19
- get_ckan_config_option, get_resource_dc_config, get_resource_info
18
+ get_ckan_config_option, get_resource_dc_config, get_resource_info,
19
+ is_resource_private
20
20
  )
21
21
 
22
22
  from . import s3
@@ -169,7 +169,9 @@ def get_s3_bucket_name_for_resource(resource_id):
169
169
  return bucket_name
170
170
 
171
171
 
172
- def get_s3_dc_handle(resource_id):
172
+ def get_s3_dc_handle(
173
+ resource_id: str,
174
+ artifact: Literal["condensed", "preview", "resource"] = "resource"):
173
175
  """Return an instance of :class:`RTDC_S3`
174
176
 
175
177
  The data are accessed directly via S3 using DCOR's access credentials.
@@ -178,7 +180,7 @@ def get_s3_dc_handle(resource_id):
178
180
  The resource with the identifier `resource_id` must exist in the
179
181
  CKAN database.
180
182
  """
181
- s3_url = get_s3_url_for_artifact(resource_id)
183
+ s3_url = get_s3_url_for_artifact(resource_id, artifact=artifact)
182
184
  ds = fmt_s3.RTDC_S3(
183
185
  url=s3_url,
184
186
  access_key_id=get_ckan_config_option(
@@ -203,11 +205,22 @@ def get_s3_dc_handle_basin_based(resource_id):
203
205
  that initialization takes slightly longer and that, if private
204
206
  resources are accessed, the presigned URLs in the basins are only
205
207
  valid for a fixed time period.
208
+
209
+ Parameters
210
+ ----------
211
+ resource_id : str
212
+ CKAN resource identifier
213
+
214
+ Returns
215
+ -------
216
+ ds: RTDCBase
217
+ basin-based dclab dataset
206
218
  """
207
- ds_dict, res_dict = get_resource_info(resource_id)
219
+ artifacts = ["resource", "condensed"]
208
220
  basin_paths = []
209
- for artifact in ["resource", "condensed"]:
210
- if ds_dict["private"]:
221
+ private = is_resource_private(resource_id)
222
+ for artifact in artifacts:
223
+ if private:
211
224
  bp = create_presigned_url(resource_id, artifact=artifact)
212
225
  else:
213
226
  bp = get_s3_url_for_artifact(resource_id, artifact=artifact)
@@ -220,18 +233,19 @@ def get_s3_dc_handle_basin_based(resource_id):
220
233
  hv.require_group("events")
221
234
  hw = dclab.RTDCWriter(hv)
222
235
  hw.store_metadata(get_resource_dc_config(resource_id))
223
- for bp in basin_paths:
224
- urlp = urlparse(bp) # e.g. http://localhost/bucket/resource/id
236
+ for bp, artifact in zip(basin_paths, artifacts):
225
237
  hw.store_basin(
226
- basin_name=urlp.path.split("/")[2], # e.g. "resource"
238
+ basin_name=f"{artifact}-{resource_id[:5]}", # "resource-92a12"
227
239
  basin_format="http",
228
240
  basin_type="remote",
229
241
  basin_locs=[bp],
230
242
  # Don't verify anything. This would only cost time,
231
243
  # and we know these objects exist.
232
244
  verify=False,
245
+ perishable=private,
233
246
  )
234
- return fmt_hdf5.RTDC_HDF5(fd)
247
+ ds = fmt_hdf5.RTDC_HDF5(fd)
248
+ return ds
235
249
 
236
250
 
237
251
  def get_s3_url_for_artifact(
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: dcor_shared
3
- Version: 0.12.3
3
+ Version: 0.13.0
4
4
  Summary: Functionalities shared by the DCOR CKAN extensions
5
5
  Author: Paul Müller
6
6
  Maintainer-email: Paul Müller <dev@craban.de>
@@ -17,7 +17,7 @@ Requires-Python: <4,>=3.8
17
17
  Description-Content-Type: text/x-rst
18
18
  License-File: LICENSE
19
19
  Requires-Dist: boto3
20
- Requires-Dist: dclab[s3]>=0.60.9
20
+ Requires-Dist: dclab[s3]>=0.64.0
21
21
  Dynamic: license-file
22
22
 
23
23
  dcor_shared
@@ -25,7 +25,6 @@ dcor_shared.egg-info/dependency_links.txt
25
25
  dcor_shared.egg-info/requires.txt
26
26
  dcor_shared.egg-info/top_level.txt
27
27
  tests/requirements.txt
28
- tests/test_basic.py
29
28
  tests/test_ckan.py
30
29
  tests/test_data.py
31
30
  tests/test_s3.py
@@ -0,0 +1,2 @@
1
+ boto3
2
+ dclab[s3]>=0.64.0
@@ -28,7 +28,7 @@ classifiers = [
28
28
  license = {text = "GNU Affero General Public License v3 or later (AGPLv3+)"}
29
29
  dependencies = [
30
30
  "boto3",
31
- "dclab[s3]>=0.60.9",
31
+ "dclab[s3]>=0.64.0",
32
32
  ]
33
33
  dynamic = ["version"]
34
34
 
@@ -2,3 +2,4 @@ pytest
2
2
  requests
3
3
  requests_toolbelt
4
4
  dclab[s3]
5
+ pytest_factoryboy
@@ -1,7 +1,9 @@
1
1
  import pathlib
2
2
  from unittest import mock
3
3
 
4
- from dcor_shared import get_resource_dc_config, get_resource_info
4
+ from dcor_shared import (
5
+ get_resource_dc_config, get_resource_info, is_resource_private
6
+ )
5
7
 
6
8
  import pytest
7
9
  from dcor_shared.testing import make_dataset_via_s3, synchronous_enqueue_job
@@ -36,3 +38,14 @@ def test_get_resource_info(enqueue_job_mock):
36
38
  ds_dict_2, res_dict_2 = get_resource_info(res_dict["id"])
37
39
  assert ds_dict == ds_dict_2
38
40
  assert res_dict == res_dict_2
41
+
42
+
43
+ @pytest.mark.ckan_config('ckan.plugins', 'dcor_schemas')
44
+ @pytest.mark.usefixtures('clean_db', 'with_request_context')
45
+ @pytest.mark.parametrize("private", [True, False])
46
+ def test_is_resource_private(private):
47
+ _, res_dict = make_dataset_via_s3(
48
+ resource_path=data_path / "calibration_beads_47.rtdc",
49
+ private=private,
50
+ activate=True)
51
+ assert is_resource_private(res_dict["id"]) == private
@@ -20,7 +20,9 @@ import requests
20
20
  data_path = pathlib.Path(__file__).parent / "data"
21
21
 
22
22
 
23
- def setup_s3_resource_on_ckan(private=False, resource_path=None):
23
+ def setup_s3_resource_on_ckan(private: bool = False,
24
+ resource_path: str | pathlib.Path = None
25
+ ):
24
26
  """Create an S3 resource in CKAN"""
25
27
  if resource_path is None:
26
28
  resource_path = data_path / "calibration_beads_47.rtdc"
@@ -34,7 +36,9 @@ def setup_s3_resource_on_ckan(private=False, resource_path=None):
34
36
  }])
35
37
 
36
38
  test_context = {'ignore_auth': False,
37
- 'user': user['name'], 'model': model, 'api_version': 3}
39
+ 'user': user['name'],
40
+ 'model': model,
41
+ 'api_version': 3}
38
42
 
39
43
  # Upload the resource to S3 (note that it is not required that the
40
44
  # dataset exists)
@@ -167,6 +171,23 @@ def test_get_s3_handle(enqueue_job_mock):
167
171
  assert len(ds) == 47
168
172
 
169
173
 
174
+ @pytest.mark.ckan_config('ckan.plugins', 'dcor_schemas dc_serve')
175
+ @pytest.mark.usefixtures('clean_db', 'with_request_context')
176
+ @mock.patch('ckan.plugins.toolkit.enqueue_job',
177
+ side_effect=synchronous_enqueue_job)
178
+ def test_get_s3_handle_condensed(enqueue_job_mock, tmp_path):
179
+ resource_path = tmp_path / "data.rtdc"
180
+ shutil.copy2(data_path / "calibration_beads_47.rtdc", resource_path)
181
+ _, res_dict = make_dataset_via_s3(
182
+ resource_path=resource_path,
183
+ activate=True)
184
+ rid = res_dict["id"]
185
+ expected_path = s3cc.get_s3_url_for_artifact(rid, artifact="condensed")
186
+ with s3cc.get_s3_dc_handle(rid, artifact="condensed") as ds:
187
+ assert ds.path == expected_path
188
+ assert len(ds) == 47
189
+
190
+
170
191
  @pytest.mark.ckan_config('ckan.plugins', 'dcor_depot dcor_schemas dc_serve')
171
192
  @pytest.mark.usefixtures('clean_db', 'with_request_context')
172
193
  @mock.patch('ckan.plugins.toolkit.enqueue_job',
@@ -198,6 +219,54 @@ def test_get_s3_dc_handle_basin_based(enqueue_job_mock, tmp_path):
198
219
  assert len(ds2) == 47
199
220
 
200
221
 
222
+ @pytest.mark.ckan_config('ckan.plugins', 'dcor_depot dcor_schemas dc_serve')
223
+ @pytest.mark.usefixtures('clean_db', 'with_request_context')
224
+ @mock.patch('ckan.plugins.toolkit.enqueue_job',
225
+ side_effect=synchronous_enqueue_job)
226
+ def test_get_s3_dc_handle_basin_based_public_urls(enqueue_job_mock, tmp_path):
227
+ resource_path = tmp_path / "data.rtdc"
228
+ shutil.copy2(data_path / "calibration_beads_47.rtdc", resource_path)
229
+ with h5py.File(resource_path, "a") as h5:
230
+ del h5["events/volume"]
231
+
232
+ _, res_dict = make_dataset_via_s3(
233
+ resource_path=resource_path,
234
+ private=False,
235
+ activate=True)
236
+ rid = res_dict["id"]
237
+
238
+ with s3cc.get_s3_dc_handle_basin_based(rid) as ds:
239
+ # get the basins
240
+ for bn_dict in ds.basins_get_dicts():
241
+ assert not bn_dict["perishable"]
242
+ for url in bn_dict["urls"]:
243
+ assert not url.lower().count("expires")
244
+
245
+
246
+ @pytest.mark.ckan_config('ckan.plugins', 'dcor_depot dcor_schemas dc_serve')
247
+ @pytest.mark.usefixtures('clean_db', 'with_request_context')
248
+ @mock.patch('ckan.plugins.toolkit.enqueue_job',
249
+ side_effect=synchronous_enqueue_job)
250
+ def test_get_s3_dc_handle_basin_based_private_urls(enqueue_job_mock, tmp_path):
251
+ resource_path = tmp_path / "data.rtdc"
252
+ shutil.copy2(data_path / "calibration_beads_47.rtdc", resource_path)
253
+ with h5py.File(resource_path, "a") as h5:
254
+ del h5["events/volume"]
255
+
256
+ _, res_dict = make_dataset_via_s3(
257
+ resource_path=resource_path,
258
+ private=True,
259
+ activate=True)
260
+ rid = res_dict["id"]
261
+
262
+ with s3cc.get_s3_dc_handle_basin_based(rid) as ds:
263
+ # get the basins
264
+ for bn_dict in ds.basins_get_dicts():
265
+ assert bn_dict["perishable"]
266
+ for url in bn_dict["urls"]:
267
+ assert url.lower().count("expires")
268
+
269
+
201
270
  @pytest.mark.ckan_config('ckan.plugins', 'dcor_schemas')
202
271
  @pytest.mark.usefixtures('clean_db', 'with_request_context')
203
272
  @mock.patch('ckan.plugins.toolkit.enqueue_job',
@@ -1,94 +0,0 @@
1
- import functools
2
- import pathlib
3
- import time
4
-
5
- from .ckan import get_resource_path
6
- from . import s3cc
7
- from .util import sha256sum # noqa: F401
8
-
9
-
10
- #: Content of the dummy file created when importing data.
11
- DUMMY_BYTES = b"[Data import pending]"
12
-
13
-
14
- @functools.lru_cache(maxsize=100)
15
- def wait_for_resource(resource_id: str,
16
- timeout: float = 10):
17
- """Wait for resource to be available
18
-
19
- This function can be used by other plugins to ensure that
20
- a resource is available for processing.
21
-
22
- There multiple ways for data to become available:
23
-
24
- 1. The ckanext-dcor_depot plugin imports data by touching
25
- dummy files and then sym-linking to data on disk. Here
26
- we just check that the file is not a dummy file anymore.
27
- 2. Legacy uploads via nginx/uwsgi directly into CKAN and onto
28
- the local block storage worked the same way. We have to wait
29
- for the dummy file to be replaced.
30
- 3. The new (2024) way of uploading data is via pre-signed URLs
31
- to an S3 instance. Here, we have to make sure that the
32
- upload is complete and the file exists. If this is the case,
33
- then uploads should have already completed when this function
34
- is called, so we only check for the existence of the resource
35
- in ckan and whether the `s3_available` attribute is defined.
36
- """
37
- from ckan.common import config
38
- from ckan import logic
39
-
40
- if len(resource_id) != 36:
41
- raise ValueError(f"Invalid resource id: {resource_id}")
42
-
43
- resource_show = logic.get_action("resource_show")
44
- path = pathlib.Path(get_resource_path(resource_id))
45
-
46
- dcor_depot_available = "dcor_depot" in config.get('ckan.plugins', "")
47
- # Initially this was set to 10s, but if `/data` is mounted on a
48
- # network share then this part here just takes too long.
49
- t0 = time.time()
50
- ld = len(DUMMY_BYTES)
51
- while True:
52
- try:
53
- res_dict = resource_show(context={'ignore_auth': True,
54
- 'user': 'default'},
55
- data_dict={"id": resource_id})
56
- except logic.NotFound:
57
- # Other processes are still working on getting the resource
58
- # online. We have to wait.
59
- time.sleep(5)
60
- continue
61
-
62
- # object exists in database?
63
- s3_ok = res_dict.get("s3_available", None)
64
- if s3_ok:
65
- # If the resource is on S3, it is considered to be available.
66
- break
67
-
68
- # object exists on S3?
69
- try:
70
- s3_exist = s3cc.artifact_exists(res_dict["id"])
71
- except BaseException:
72
- s3_exist = False
73
- if s3_exist:
74
- break
75
-
76
- if time.time() - t0 > timeout:
77
- raise OSError("Data import seems to take too long "
78
- "for '{}'!".format(path))
79
- elif not path.exists():
80
- time.sleep(5)
81
- continue
82
- elif dcor_depot_available and not path.is_symlink():
83
- # Resource is only available when it is symlinked by
84
- # the ckanext.dcor_depot `symlink_user_dataset` job
85
- # (or by the ckanext.dcor_depot importers).
86
- time.sleep(5)
87
- continue
88
- elif path.stat().st_size == ld and path.read_bytes() == DUMMY_BYTES:
89
- # wait a bit
90
- time.sleep(5)
91
- continue
92
- else:
93
- # not a dummy file
94
- break
@@ -1,20 +0,0 @@
1
- from dclab.rtdc_dataset import RTDC_HDF5
2
-
3
- from .ckan import get_resource_path
4
- from . import s3cc
5
-
6
-
7
- def get_dc_instance(rid):
8
- """Return an instance of dclab's `RTDCBase` for a resource identifier"""
9
- # Try local file first
10
- path = get_resource_path(rid)
11
- if path.is_file():
12
- # Disable basins, because they could point to files on the local
13
- # file system (security).
14
- return RTDC_HDF5(path, enable_basins=False)
15
- else:
16
- # The resource must be on S3
17
- if s3cc.artifact_exists(rid):
18
- return s3cc.get_s3_dc_handle(rid)
19
- else:
20
- raise ValueError(f"Could not find resource {rid} anywhere")
@@ -1,30 +0,0 @@
1
- import pathlib
2
-
3
- from .ckan import get_ckan_config_path, get_ckan_config_option
4
- from .ckan import get_ckan_storage_path, get_ckan_webassets_path # noqa: F401
5
-
6
- from .parse import get_ini_config_option
7
-
8
-
9
- def get_dcor_depot_path():
10
- return pathlib.Path(get_ckan_config_option(
11
- "ckanext.dcor_depot.depots_path"))
12
-
13
-
14
- def get_dcor_users_depot_path():
15
- depot = get_dcor_depot_path()
16
- return depot / get_ini_config_option(
17
- "ckanext.dcor_depot.users_depot_name",
18
- get_ckan_config_path())
19
-
20
-
21
- def get_nginx_config_path():
22
- return pathlib.Path("/etc/nginx/sites-enabled/ckan")
23
-
24
-
25
- def get_supervisord_worker_config_path():
26
- return pathlib.Path("/etc/supervisor/conf.d/ckan-worker.conf")
27
-
28
-
29
- def get_uwsgi_config_path():
30
- return pathlib.Path("/etc/ckan/default/ckan-uwsgi.ini")
@@ -1,2 +0,0 @@
1
- boto3
2
- dclab[s3]>=0.60.9
@@ -1,5 +0,0 @@
1
- from dcor_shared import DUMMY_BYTES
2
-
3
-
4
- def test_dummy_bytes():
5
- assert DUMMY_BYTES == b"[Data import pending]"
File without changes
File without changes
File without changes
File without changes
File without changes