dcor-shared 1.0.0__tar.gz → 1.0.2__tar.gz

{dcor_shared-1.0.0 → dcor_shared-1.0.2}/CHANGELOG

@@ -1,3 +1,7 @@
+1.0.2
+ - ref: disable threading and use 1GiB threshold for multipart uploads
+1.0.1
+ - enh: introduce `for_circles_only` kwarg for `iter_buckets`
 1.0.0
  - ref: remove deprecations (`s3cc.object_exists`, `make_dataset`,
    `make_resource`, `create_context` arg in `make_resource_via_s3`)

{dcor_shared-1.0.0 → dcor_shared-1.0.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dcor_shared
-Version: 1.0.0
+Version: 1.0.2
 Summary: Functionalities shared by the DCOR CKAN extensions
 Author: Paul Müller
 Maintainer-email: Paul Müller <dev@craban.de>

{dcor_shared-1.0.0 → dcor_shared-1.0.2}/dcor_shared/_version.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '1.0.0'
-__version_tuple__ = version_tuple = (1, 0, 0)
+__version__ = version = '1.0.2'
+__version_tuple__ = version_tuple = (1, 0, 2)

{dcor_shared-1.0.0 → dcor_shared-1.0.2}/dcor_shared/s3.py

@@ -6,12 +6,14 @@ import functools
 import hashlib
 import json
 import pathlib
+import re
 import time
 import traceback
 from typing import List, Tuple
 import warnings
 
 import boto3
+from boto3.s3.transfer import TransferConfig
 from botocore.config import Config
 import botocore.exceptions
 
@@ -340,20 +342,38 @@ def is_available():
     return s3_key_id and s3_secret
 
 
-def iter_buckets():
-    """Iterator returning all buckets for this DCOR instance"""
+def iter_buckets(for_circles_only=True):
+    """Iterator returning all buckets for this DCOR instance
+
+    If `for_circles_only` is True (default), this method only returns
+    bucket names that belong to a circle via a regular expression, e.g.
+    "circle-13748aef-a5b5-47b4-b86d-8a0c1ff49035".
+    Otherwise, all buckets that match the `dcor_object_store.bucket_name`
+    prefix are returned.
+    The motivation of this argument is so that a DCOR instance can store
+    encrypted instance backups in a separate bucket while removing
+    orphaned S3 artifacts does not remove them.
+    """
     s3_client, _, _ = get_s3()
 
     # dcor_object_store.bucket_name
     bucket_prefix = get_ckan_config_option(
         "dcor_object_store.bucket_name").format(organization_id="")
 
+    if for_circles_only:
+        r4 = "[0-9a-f]{4}"
+        r8 = "[0-9a-f]{8}"
+        r12 = "[0-9a-f]{12}"
+        regex = re.compile(f"^{bucket_prefix}{r8}-{r4}-{r4}-{r4}-{r12}$")
+    else:
+        regex = re.compile(f"^{bucket_prefix}.*$")
+
     # Iterate over all buckets that match the schema of this DCOR instance.
     paginator = s3_client.get_paginator('list_buckets')
     for page in paginator.paginate():
         for bucket in page['Buckets']:
             bucket_name = bucket['Name']
-            if bucket_name.startswith(bucket_prefix):
+            if regex.match(bucket_name):
                 yield bucket_name
 
 
@@ -486,7 +506,7 @@ def prune_multipart_uploads(initiated_before_days: int = 5,
 
     ret_dict = {}
 
-    for bucket_name in iter_buckets():
+    for bucket_name in iter_buckets(for_circles_only=False):
         to_prune = []
         bdict = {"found": 0,
                  "ignored": 0
@@ -658,9 +678,17 @@ def upload_file(bucket_name, object_name, path, sha256=None, private=True,
                                            object_name=object_name)
 
     if perform_upload:
+        config = TransferConfig(
+            # 1GiB threshold for multipart uploads
+            multipart_threshold=1 * 1024 * 1024 * 1024,
+            multipart_chunksize=1 * 1024 * 1024 * 1024,
+            # Disable threads
+            use_threads=False
+        )
         s3_client.upload_file(Filename=str(path),
                               Bucket=bucket_name,
                               Key=object_name,
+                              Config=config,
                               # ExtraArgs={
                               # # verification of the upload (breaks OpenStack)
                               # "ChecksumAlgorithm": "SHA256",

{dcor_shared-1.0.0 → dcor_shared-1.0.2}/dcor_shared.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: dcor_shared
-Version: 1.0.0
+Version: 1.0.2
 Summary: Functionalities shared by the DCOR CKAN extensions
 Author: Paul Müller
 Maintainer-email: Paul Müller <dev@craban.de>

{dcor_shared-1.0.0 → dcor_shared-1.0.2}/tests/test_s3.py

@@ -44,6 +44,23 @@ def test_create_bucket_again():
     assert bucket3 is not bucket, "new object"
 
 
+def iter_buckets_keep_non_circle():
+    bucket_prefix = get_ckan_config_option(
+        "dcor_object_store.bucket_name").format(organization_id="")
+    bucket_backup = f"{bucket_prefix}-backups"
+    s3.require_bucket(bucket_backup)
+    bucket_circle = f"{bucket_prefix}-13748aef-a5b5-a7b4-b86d-8a0c1ff49035"
+    s3.require_bucket(bucket_circle)
+
+    buckets_default = list(s3.iter_buckets())
+    assert bucket_backup not in buckets_default
+    assert bucket_circle in buckets_default
+
+    buckets_all = list(s3.iter_buckets(for_circles_only=False))
+    assert bucket_backup in buckets_all
+    assert bucket_circle in buckets_all
+
+
 def test_iter_bucket_objects():
     path = data_path / "calibration_beads_47.rtdc"
     bucket_name = f"test-circle-{uuid.uuid4()}"