dcor-shared 1.0.0__tar.gz → 1.0.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. {dcor_shared-1.0.0 → dcor_shared-1.0.1}/CHANGELOG +2 -0
  2. {dcor_shared-1.0.0 → dcor_shared-1.0.1}/PKG-INFO +1 -1
  3. {dcor_shared-1.0.0 → dcor_shared-1.0.1}/dcor_shared/_version.py +2 -2
  4. {dcor_shared-1.0.0 → dcor_shared-1.0.1}/dcor_shared/s3.py +23 -4
  5. {dcor_shared-1.0.0 → dcor_shared-1.0.1}/dcor_shared.egg-info/PKG-INFO +1 -1
  6. {dcor_shared-1.0.0 → dcor_shared-1.0.1}/tests/test_s3.py +17 -0
  7. {dcor_shared-1.0.0 → dcor_shared-1.0.1}/.github/workflows/check.yml +0 -0
  8. {dcor_shared-1.0.0 → dcor_shared-1.0.1}/.github/workflows/deploy_pypi.yml +0 -0
  9. {dcor_shared-1.0.0 → dcor_shared-1.0.1}/.gitignore +0 -0
  10. {dcor_shared-1.0.0 → dcor_shared-1.0.1}/LICENSE +0 -0
  11. {dcor_shared-1.0.0 → dcor_shared-1.0.1}/MANIFEST.in +0 -0
  12. {dcor_shared-1.0.0 → dcor_shared-1.0.1}/README.rst +0 -0
  13. {dcor_shared-1.0.0 → dcor_shared-1.0.1}/dcor_shared/__init__.py +0 -0
  14. {dcor_shared-1.0.0 → dcor_shared-1.0.1}/dcor_shared/ckan.py +0 -0
  15. {dcor_shared-1.0.0 → dcor_shared-1.0.1}/dcor_shared/data.py +0 -0
  16. {dcor_shared-1.0.0 → dcor_shared-1.0.1}/dcor_shared/dcinst.py +0 -0
  17. {dcor_shared-1.0.0 → dcor_shared-1.0.1}/dcor_shared/mime.py +0 -0
  18. {dcor_shared-1.0.0 → dcor_shared-1.0.1}/dcor_shared/parse.py +0 -0
  19. {dcor_shared-1.0.0 → dcor_shared-1.0.1}/dcor_shared/paths.py +0 -0
  20. {dcor_shared-1.0.0 → dcor_shared-1.0.1}/dcor_shared/rqjob.py +0 -0
  21. {dcor_shared-1.0.0 → dcor_shared-1.0.1}/dcor_shared/s3cc.py +0 -0
  22. {dcor_shared-1.0.0 → dcor_shared-1.0.1}/dcor_shared/testing.py +0 -0
  23. {dcor_shared-1.0.0 → dcor_shared-1.0.1}/dcor_shared/util.py +0 -0
  24. {dcor_shared-1.0.0 → dcor_shared-1.0.1}/dcor_shared.egg-info/SOURCES.txt +0 -0
  25. {dcor_shared-1.0.0 → dcor_shared-1.0.1}/dcor_shared.egg-info/dependency_links.txt +0 -0
  26. {dcor_shared-1.0.0 → dcor_shared-1.0.1}/dcor_shared.egg-info/requires.txt +0 -0
  27. {dcor_shared-1.0.0 → dcor_shared-1.0.1}/dcor_shared.egg-info/top_level.txt +0 -0
  28. {dcor_shared-1.0.0 → dcor_shared-1.0.1}/pyproject.toml +0 -0
  29. {dcor_shared-1.0.0 → dcor_shared-1.0.1}/setup.cfg +0 -0
  30. {dcor_shared-1.0.0 → dcor_shared-1.0.1}/tests/data/calibration_beads_47.rtdc +0 -0
  31. {dcor_shared-1.0.0 → dcor_shared-1.0.1}/tests/requirements.txt +0 -0
  32. {dcor_shared-1.0.0 → dcor_shared-1.0.1}/tests/test_ckan.py +0 -0
  33. {dcor_shared-1.0.0 → dcor_shared-1.0.1}/tests/test_data.py +0 -0
  34. {dcor_shared-1.0.0 → dcor_shared-1.0.1}/tests/test_s3cc.py +0 -0
  35. {dcor_shared-1.0.0 → dcor_shared-1.0.1}/tests/test_testing.py +0 -0
@@ -1,3 +1,5 @@
1
+ 1.0.1
2
+ - enh: introduce `for_circles_only` kwarg for `iter_buckets`
1
3
  1.0.0
2
4
  - ref: remove deprecations (`s3cc.object_exists`, `make_dataset`,
3
5
  `make_resource`, `create_context` arg in `make_resource_via_s3`)
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: dcor_shared
3
- Version: 1.0.0
3
+ Version: 1.0.1
4
4
  Summary: Functionalities shared by the DCOR CKAN extensions
5
5
  Author: Paul Müller
6
6
  Maintainer-email: Paul Müller <dev@craban.de>
@@ -17,5 +17,5 @@ __version__: str
17
17
  __version_tuple__: VERSION_TUPLE
18
18
  version_tuple: VERSION_TUPLE
19
19
 
20
- __version__ = version = '1.0.0'
21
- __version_tuple__ = version_tuple = (1, 0, 0)
20
+ __version__ = version = '1.0.1'
21
+ __version_tuple__ = version_tuple = (1, 0, 1)
@@ -6,6 +6,7 @@ import functools
6
6
  import hashlib
7
7
  import json
8
8
  import pathlib
9
+ import re
9
10
  import time
10
11
  import traceback
11
12
  from typing import List, Tuple
@@ -340,20 +341,38 @@ def is_available():
340
341
  return s3_key_id and s3_secret
341
342
 
342
343
 
343
- def iter_buckets():
344
- """Iterator returning all buckets for this DCOR instance"""
344
+ def iter_buckets(for_circles_only=True):
345
+ """Iterator returning all buckets for this DCOR instance
346
+
347
+ If `for_circles_only` is True (default), this method only returns
348
+ bucket names that belong to a circle via a regular expression, e.g.
349
+ "circle-13748aef-a5b5-47b4-b86d-8a0c1ff49035".
350
+ Otherwise, all buckets that match the `dcor_object_store.bucket_name`
351
+ prefix are returned.
352
+ The motivation of this argument is so that a DCOR instance can store
353
+ encrypted instance backups in a separate bucket while removing
354
+ orphaned S3 artifacts does not remove them.
355
+ """
345
356
  s3_client, _, _ = get_s3()
346
357
 
347
358
  # dcor_object_store.bucket_name
348
359
  bucket_prefix = get_ckan_config_option(
349
360
  "dcor_object_store.bucket_name").format(organization_id="")
350
361
 
362
+ if for_circles_only:
363
+ r4 = "[0-9a-f]{4}"
364
+ r8 = "[0-9a-f]{8}"
365
+ r12 = "[0-9a-f]{12}"
366
+ regex = re.compile(f"^{bucket_prefix}{r8}-{r4}-{r4}-{r4}-{r12}$")
367
+ else:
368
+ regex = re.compile(f"^{bucket_prefix}.*$")
369
+
351
370
  # Iterate over all buckets that match the schema of this DCOR instance.
352
371
  paginator = s3_client.get_paginator('list_buckets')
353
372
  for page in paginator.paginate():
354
373
  for bucket in page['Buckets']:
355
374
  bucket_name = bucket['Name']
356
- if bucket_name.startswith(bucket_prefix):
375
+ if regex.match(bucket_name):
357
376
  yield bucket_name
358
377
 
359
378
 
@@ -486,7 +505,7 @@ def prune_multipart_uploads(initiated_before_days: int = 5,
486
505
 
487
506
  ret_dict = {}
488
507
 
489
- for bucket_name in iter_buckets():
508
+ for bucket_name in iter_buckets(for_circles_only=False):
490
509
  to_prune = []
491
510
  bdict = {"found": 0,
492
511
  "ignored": 0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: dcor_shared
3
- Version: 1.0.0
3
+ Version: 1.0.1
4
4
  Summary: Functionalities shared by the DCOR CKAN extensions
5
5
  Author: Paul Müller
6
6
  Maintainer-email: Paul Müller <dev@craban.de>
@@ -44,6 +44,23 @@ def test_create_bucket_again():
44
44
  assert bucket3 is not bucket, "new object"
45
45
 
46
46
 
47
+ def iter_buckets_keep_non_circle():
48
+ bucket_prefix = get_ckan_config_option(
49
+ "dcor_object_store.bucket_name").format(organization_id="")
50
+ bucket_backup = f"{bucket_prefix}-backups"
51
+ s3.require_bucket(bucket_backup)
52
+ bucket_circle = f"{bucket_prefix}-13748aef-a5b5-a7b4-b86d-8a0c1ff49035"
53
+ s3.require_bucket(bucket_circle)
54
+
55
+ buckets_default = list(s3.iter_buckets())
56
+ assert bucket_backup not in buckets_default
57
+ assert bucket_circle in buckets_default
58
+
59
+ buckets_all = list(s3.iter_buckets(for_circles_only=False))
60
+ assert bucket_backup in buckets_all
61
+ assert bucket_circle in buckets_all
62
+
63
+
47
64
  def test_iter_bucket_objects():
48
65
  path = data_path / "calibration_beads_47.rtdc"
49
66
  bucket_name = f"test-circle-{uuid.uuid4()}"
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes