dcor-shared 0.5.4__tar.gz → 0.5.6__tar.gz

{dcor_shared-0.5.4 → dcor_shared-0.5.6}/CHANGELOG

@@ -1,3 +1,8 @@
+0.5.6
+ - maintenance release
+0.5.5
+ - ref: factor out s3.object_exists method
+ - ref: factor out s3.compute_checksum method
 0.5.4
  - enh: cache presigned URLs with 10% tolerance
 0.5.3

{dcor_shared-0.5.4 → dcor_shared-0.5.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dcor_shared
-Version: 0.5.4
+Version: 0.5.6
 Summary: Functionalities shared by the DCOR CKAN extensions
 Home-page: https://github.com/DCOR-dev/dcor_shared
 Author: Paul Müller

{dcor_shared-0.5.4 → dcor_shared-0.5.6}/dcor_shared/_version_save.py

@@ -1,3 +1,3 @@
 #!/usr/bin/env python
 # This file was created automatically
-longversion = '0.5.4'
+longversion = '0.5.6'

{dcor_shared-0.5.4 → dcor_shared-0.5.6}/dcor_shared/s3.py

@@ -11,6 +11,36 @@ import botocore.exceptions
 from .ckan import get_ckan_config_option
 
 
+def compute_checksum(bucket_name, object_name, max_size=None):
+    """Compute the SHA256 checksum of an S3 object
+
+    The hash is computed in memory as the file is downloaded in chunks.
+    """
+    s3_client, _, s3_resource = get_s3()
+    if max_size is None:
+        obj = s3_resource.Object(bucket_name=bucket_name, key=object_name)
+        max_size = obj.content_length
+    hasher = hashlib.sha256()
+    # This is an increment of 1MB. If you change this, please also update
+    # the tests for large uploads.
+    increment = 2 ** 20
+    start_byte = 0
+    stop_byte = min(increment, max_size)
+    while start_byte < max_size:
+        resp = s3_client.get_object(
+            Bucket=bucket_name,
+            Key=object_name,
+            Range=f"bytes={start_byte}-{stop_byte}")
+        content = resp['Body'].read()
+        if not content:
+            break
+        hasher.update(content)
+        start_byte = stop_byte + 1  # range is inclusive
+        stop_byte = min(max_size, stop_byte + increment)
+    s3_sha256 = hasher.hexdigest()
+    return s3_sha256
+
+
 def create_presigned_url(bucket_name, object_name, expiration=3600,
                          filename=None):
     """Generate a presigned URL to share an S3 object
@@ -165,6 +195,19 @@ def make_object_public(bucket_name, object_name, missing_ok=False):
             )
 
 
+def object_exists(bucket_name, object_name):
+    """Check whether an object exists"""
+    s3_client, _, _ = get_s3()
+    try:
+        s3_client.head_object(Bucket=bucket_name, Key=object_name)
+    except (s3_client.exceptions.NoSuchKey,
+            s3_client.exceptions.ClientError):
+        obj_exists = False
+    else:
+        obj_exists = True
+    return obj_exists
+
+
 @functools.lru_cache()
 def require_bucket(bucket_name):
     """Create an S3 bucket if it does not exist yet
@@ -272,14 +315,8 @@ def upload_file(bucket_name, object_name, path, sha256, private=True,
 
     perform_upload = True
     if not override:
-        try:
-            s3_client.head_object(Bucket=bucket_name, Key=object_name)
-        except (s3_client.exceptions.NoSuchKey,
-                s3_client.exceptions.ClientError):
-            object_exists = False
-        else:
-            object_exists = True
-        perform_upload = not object_exists
+        perform_upload = not object_exists(bucket_name=bucket_name,
+                                           object_name=object_name)
 
     if perform_upload:
         s3_bucket.upload_file(Filename=str(path),
@@ -292,26 +329,9 @@ def upload_file(bucket_name, object_name, path, sha256, private=True,
                               # }
                               )
         # Make sure the upload worked properly by computing the SHA256 sum.
-        # Download the file directly into the hasher.
-        hasher = hashlib.sha256()
-        # This is an increment of 1MB. If you change this, please also update
-        # the tests for large uploads.
-        increment = 2**20
-        start_byte = 0
-        max_size = path_size
-        stop_byte = min(increment, max_size)
-        while start_byte < max_size:
-            resp = s3_client.get_object(
-                Bucket=bucket_name,
-                Key=object_name,
-                Range=f"bytes={start_byte}-{stop_byte}")
-            content = resp['Body'].read()
-            if not content:
-                break
-            hasher.update(content)
-            start_byte = stop_byte + 1  # range is inclusive
-            stop_byte = min(max_size, stop_byte + increment)
-        s3_sha256 = hasher.hexdigest()
+        s3_sha256 = compute_checksum(bucket_name=bucket_name,
+                                     object_name=object_name,
+                                     max_size=path_size)
         if sha256 != s3_sha256:
             raise ValueError(
                 f"Checksum mismatch for {bucket_name}:{object_name}!")

{dcor_shared-0.5.4 → dcor_shared-0.5.6}/dcor_shared.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: dcor-shared
-Version: 0.5.4
+Version: 0.5.6
 Summary: Functionalities shared by the DCOR CKAN extensions
 Home-page: https://github.com/DCOR-dev/dcor_shared
 Author: Paul Müller

{dcor_shared-0.5.4 → dcor_shared-0.5.6}/tests/test_s3.py

@@ -1,4 +1,5 @@
-import mock
+import hashlib
+from unittest import mock
 import pathlib
 import uuid
 
@@ -11,12 +12,30 @@ from dcor_shared import s3, sha256sum
 data_path = pathlib.Path(__file__).parent / "data"
 
 
+def test_compute_checksum():
+    path = data_path / "calibration_beads_47.rtdc"
+    bucket_name = f"test-circle-{uuid.uuid4()}"
+    rid = str(uuid.uuid4())
+    object_name = f"resource/{rid[:3]}/{rid[3:6]}/{rid[6:]}"
+    s3.upload_file(
+        bucket_name=bucket_name,
+        object_name=object_name,
+        path=path,
+        sha256=sha256sum(path),
+        private=True)
+
+    hash_exp = hashlib.sha256(path.read_bytes()).hexdigest()
+    hash_act = s3.compute_checksum(bucket_name=bucket_name,
+                                   object_name=object_name)
+    assert hash_exp == hash_act
+
+
 def test_create_bucket_again():
     bucket_name = f"test-circle-{uuid.uuid4()}"
     bucket = s3.require_bucket(bucket_name)
     # this is cached
     bucket2 = s3.require_bucket(bucket_name)
-    assert bucket2 is bucket, "chached"
+    assert bucket2 is bucket, "cached"
     s3.require_bucket.cache_clear()
     bucket3 = s3.require_bucket(bucket_name)
     assert bucket3 is not bucket, "new object"
@@ -77,6 +96,27 @@ def test_make_object_public_no_such_key(tmp_path):
                               missing_ok=False)
 
 
+def test_object_exists():
+    path = data_path / "calibration_beads_47.rtdc"
+    bucket_name = f"test-circle-{uuid.uuid4()}"
+    rid = str(uuid.uuid4())
+    object_name = f"resource/{rid[:3]}/{rid[3:6]}/{rid[6:]}"
+    s3.upload_file(
+        bucket_name=bucket_name,
+        object_name=object_name,
+        path=path,
+        sha256=sha256sum(path),
+        private=True)
+
+    assert s3.object_exists(bucket_name=bucket_name,
+                            object_name=object_name)
+    # sanity checks
+    assert not s3.object_exists(bucket_name=bucket_name,
+                                object_name=f"peter/pan-{uuid.uuid4()}")
+    assert not s3.object_exists(bucket_name=f"hansgunter-{uuid.uuid4()}",
+                                object_name=object_name)
+
+
 def test_presigned_url(tmp_path):
     path = data_path / "calibration_beads_47.rtdc"
     bucket_name = f"test-circle-{uuid.uuid4()}"