dcor-shared 1.0.0__tar.gz → 1.0.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {dcor_shared-1.0.0 → dcor_shared-1.0.1}/CHANGELOG +2 -0
- {dcor_shared-1.0.0 → dcor_shared-1.0.1}/PKG-INFO +1 -1
- {dcor_shared-1.0.0 → dcor_shared-1.0.1}/dcor_shared/_version.py +2 -2
- {dcor_shared-1.0.0 → dcor_shared-1.0.1}/dcor_shared/s3.py +23 -4
- {dcor_shared-1.0.0 → dcor_shared-1.0.1}/dcor_shared.egg-info/PKG-INFO +1 -1
- {dcor_shared-1.0.0 → dcor_shared-1.0.1}/tests/test_s3.py +17 -0
- {dcor_shared-1.0.0 → dcor_shared-1.0.1}/.github/workflows/check.yml +0 -0
- {dcor_shared-1.0.0 → dcor_shared-1.0.1}/.github/workflows/deploy_pypi.yml +0 -0
- {dcor_shared-1.0.0 → dcor_shared-1.0.1}/.gitignore +0 -0
- {dcor_shared-1.0.0 → dcor_shared-1.0.1}/LICENSE +0 -0
- {dcor_shared-1.0.0 → dcor_shared-1.0.1}/MANIFEST.in +0 -0
- {dcor_shared-1.0.0 → dcor_shared-1.0.1}/README.rst +0 -0
- {dcor_shared-1.0.0 → dcor_shared-1.0.1}/dcor_shared/__init__.py +0 -0
- {dcor_shared-1.0.0 → dcor_shared-1.0.1}/dcor_shared/ckan.py +0 -0
- {dcor_shared-1.0.0 → dcor_shared-1.0.1}/dcor_shared/data.py +0 -0
- {dcor_shared-1.0.0 → dcor_shared-1.0.1}/dcor_shared/dcinst.py +0 -0
- {dcor_shared-1.0.0 → dcor_shared-1.0.1}/dcor_shared/mime.py +0 -0
- {dcor_shared-1.0.0 → dcor_shared-1.0.1}/dcor_shared/parse.py +0 -0
- {dcor_shared-1.0.0 → dcor_shared-1.0.1}/dcor_shared/paths.py +0 -0
- {dcor_shared-1.0.0 → dcor_shared-1.0.1}/dcor_shared/rqjob.py +0 -0
- {dcor_shared-1.0.0 → dcor_shared-1.0.1}/dcor_shared/s3cc.py +0 -0
- {dcor_shared-1.0.0 → dcor_shared-1.0.1}/dcor_shared/testing.py +0 -0
- {dcor_shared-1.0.0 → dcor_shared-1.0.1}/dcor_shared/util.py +0 -0
- {dcor_shared-1.0.0 → dcor_shared-1.0.1}/dcor_shared.egg-info/SOURCES.txt +0 -0
- {dcor_shared-1.0.0 → dcor_shared-1.0.1}/dcor_shared.egg-info/dependency_links.txt +0 -0
- {dcor_shared-1.0.0 → dcor_shared-1.0.1}/dcor_shared.egg-info/requires.txt +0 -0
- {dcor_shared-1.0.0 → dcor_shared-1.0.1}/dcor_shared.egg-info/top_level.txt +0 -0
- {dcor_shared-1.0.0 → dcor_shared-1.0.1}/pyproject.toml +0 -0
- {dcor_shared-1.0.0 → dcor_shared-1.0.1}/setup.cfg +0 -0
- {dcor_shared-1.0.0 → dcor_shared-1.0.1}/tests/data/calibration_beads_47.rtdc +0 -0
- {dcor_shared-1.0.0 → dcor_shared-1.0.1}/tests/requirements.txt +0 -0
- {dcor_shared-1.0.0 → dcor_shared-1.0.1}/tests/test_ckan.py +0 -0
- {dcor_shared-1.0.0 → dcor_shared-1.0.1}/tests/test_data.py +0 -0
- {dcor_shared-1.0.0 → dcor_shared-1.0.1}/tests/test_s3cc.py +0 -0
- {dcor_shared-1.0.0 → dcor_shared-1.0.1}/tests/test_testing.py +0 -0
|
@@ -6,6 +6,7 @@ import functools
|
|
|
6
6
|
import hashlib
|
|
7
7
|
import json
|
|
8
8
|
import pathlib
|
|
9
|
+
import re
|
|
9
10
|
import time
|
|
10
11
|
import traceback
|
|
11
12
|
from typing import List, Tuple
|
|
@@ -340,20 +341,38 @@ def is_available():
|
|
|
340
341
|
return s3_key_id and s3_secret
|
|
341
342
|
|
|
342
343
|
|
|
343
|
-
def iter_buckets():
|
|
344
|
-
"""Iterator returning all buckets for this DCOR instance
|
|
344
|
+
def iter_buckets(for_circles_only=True):
|
|
345
|
+
"""Iterator returning all buckets for this DCOR instance
|
|
346
|
+
|
|
347
|
+
If `for_circles_only` is True (default), this method only returns
|
|
348
|
+
bucket names that belong to a circle via a regular expression, e.g.
|
|
349
|
+
"circle-13748aef-a5b5-47b4-b86d-8a0c1ff49035".
|
|
350
|
+
Otherwise, all buckets that match the `dcor_object_store.bucket_name`
|
|
351
|
+
prefix are returned.
|
|
352
|
+
The motivation of this argument is so that a DCOR instance can store
|
|
353
|
+
encrypted instance backups in a separate bucket while removing
|
|
354
|
+
orphaned S3 artifacts does not remove them.
|
|
355
|
+
"""
|
|
345
356
|
s3_client, _, _ = get_s3()
|
|
346
357
|
|
|
347
358
|
# dcor_object_store.bucket_name
|
|
348
359
|
bucket_prefix = get_ckan_config_option(
|
|
349
360
|
"dcor_object_store.bucket_name").format(organization_id="")
|
|
350
361
|
|
|
362
|
+
if for_circles_only:
|
|
363
|
+
r4 = "[0-9a-f]{4}"
|
|
364
|
+
r8 = "[0-9a-f]{8}"
|
|
365
|
+
r12 = "[0-9a-f]{12}"
|
|
366
|
+
regex = re.compile(f"^{bucket_prefix}{r8}-{r4}-{r4}-{r4}-{r12}$")
|
|
367
|
+
else:
|
|
368
|
+
regex = re.compile(f"^{bucket_prefix}.*$")
|
|
369
|
+
|
|
351
370
|
# Iterate over all buckets that match the schema of this DCOR instance.
|
|
352
371
|
paginator = s3_client.get_paginator('list_buckets')
|
|
353
372
|
for page in paginator.paginate():
|
|
354
373
|
for bucket in page['Buckets']:
|
|
355
374
|
bucket_name = bucket['Name']
|
|
356
|
-
if
|
|
375
|
+
if regex.match(bucket_name):
|
|
357
376
|
yield bucket_name
|
|
358
377
|
|
|
359
378
|
|
|
@@ -486,7 +505,7 @@ def prune_multipart_uploads(initiated_before_days: int = 5,
|
|
|
486
505
|
|
|
487
506
|
ret_dict = {}
|
|
488
507
|
|
|
489
|
-
for bucket_name in iter_buckets():
|
|
508
|
+
for bucket_name in iter_buckets(for_circles_only=False):
|
|
490
509
|
to_prune = []
|
|
491
510
|
bdict = {"found": 0,
|
|
492
511
|
"ignored": 0
|
|
@@ -44,6 +44,23 @@ def test_create_bucket_again():
|
|
|
44
44
|
assert bucket3 is not bucket, "new object"
|
|
45
45
|
|
|
46
46
|
|
|
47
|
+
def iter_buckets_keep_non_circle():
|
|
48
|
+
bucket_prefix = get_ckan_config_option(
|
|
49
|
+
"dcor_object_store.bucket_name").format(organization_id="")
|
|
50
|
+
bucket_backup = f"{bucket_prefix}-backups"
|
|
51
|
+
s3.require_bucket(bucket_backup)
|
|
52
|
+
bucket_circle = f"{bucket_prefix}-13748aef-a5b5-a7b4-b86d-8a0c1ff49035"
|
|
53
|
+
s3.require_bucket(bucket_circle)
|
|
54
|
+
|
|
55
|
+
buckets_default = list(s3.iter_buckets())
|
|
56
|
+
assert bucket_backup not in buckets_default
|
|
57
|
+
assert bucket_circle in buckets_default
|
|
58
|
+
|
|
59
|
+
buckets_all = list(s3.iter_buckets(for_circles_only=False))
|
|
60
|
+
assert bucket_backup in buckets_all
|
|
61
|
+
assert bucket_circle in buckets_all
|
|
62
|
+
|
|
63
|
+
|
|
47
64
|
def test_iter_bucket_objects():
|
|
48
65
|
path = data_path / "calibration_beads_47.rtdc"
|
|
49
66
|
bucket_name = f"test-circle-{uuid.uuid4()}"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|