hexkit 3.6.0__tar.gz → 3.8.0__tar.gz

{hexkit-3.6.0/src/hexkit.egg-info → hexkit-3.8.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hexkit
-Version: 3.6.0
+Version: 3.8.0
 Summary: A Toolkit for Building Microservices using the Hexagonal Architecture
 Author-email: "German Human Genome Phenome Archive (GHGA)" <contact@ghga.de>
 License: Apache 2.0

{hexkit-3.6.0 → hexkit-3.8.0}/pyproject.toml

@@ -23,7 +23,7 @@ classifiers = [
     "Intended Audience :: Developers",
 ]
 name = "hexkit"
-version = "3.6.0"
+version = "3.8.0"
 description = "A Toolkit for Building Microservices using the Hexagonal Architecture"
 dependencies = [
     "pydantic >=2, <3",

{hexkit-3.6.0 → hexkit-3.8.0}/src/hexkit/protocols/objstorage.py

@@ -122,6 +122,7 @@ class ObjectStorageProtocol(ABC):
         object_id: str,
         part_number: int,
         expires_after: int = 3600,
+        part_md5: Optional[str] = None,
     ) -> str:
         """Given a id of an instantiated multipart upload along with the corresponding
         bucket and object ID, it returns a presigned URL for uploading a file part with the
@@ -137,6 +138,7 @@ class ObjectStorageProtocol(ABC):
             object_id=object_id,
             part_number=part_number,
             expires_after=expires_after,
+            part_md5=part_md5,
         )
 
     async def abort_multipart_upload(
@@ -207,6 +209,12 @@ class ObjectStorageProtocol(ABC):
             bucket_id=bucket_id, object_id=object_id, object_md5sum=object_md5sum
         )
 
+    async def get_object_etag(self, *, bucket_id: str, object_id: str) -> str:
+        """Returns the etag of an object."""
+        self._validate_bucket_id(bucket_id)
+        self._validate_object_id(object_id)
+        return await self._get_object_etag(bucket_id=bucket_id, object_id=object_id)
+
     async def get_object_size(self, *, bucket_id: str, object_id: str) -> int:
         """Returns the size of an object in bytes."""
         self._validate_bucket_id(bucket_id)
@@ -220,9 +228,15 @@ class ObjectStorageProtocol(ABC):
         source_object_id: str,
         dest_bucket_id: str,
         dest_object_id: str,
+        abort_failed: bool = True,
     ) -> None:
         """Copy an object from one bucket (`source_bucket_id` and `source_object_id`) to
         another bucket (`dest_bucket_id` and `dest_object_id`).
+
+        If `abort_failed` is set to true (default), a failed copy operation tries to
+        abort the ongoing multipart upload it created (if using multipart mode).
+        This only works reliably as long as there are no other ongoing multipart operations for
+        the same destination bucket and object ID, in which case this should be set to false.
         """
         self._validate_bucket_id(source_bucket_id)
         self._validate_object_id(source_object_id)
@@ -233,6 +247,7 @@ class ObjectStorageProtocol(ABC):
             source_object_id=source_object_id,
             dest_bucket_id=dest_bucket_id,
             dest_object_id=dest_object_id,
+            abort_failed=abort_failed,
         )
 
     async def delete_object(self, *, bucket_id: str, object_id: str) -> None:
@@ -336,6 +351,7 @@ class ObjectStorageProtocol(ABC):
         object_id: str,
         part_number: int,
         expires_after: int = 3600,
+        part_md5: Optional[str] = None,
     ) -> str:
         """
         Given a id of an instantiated multipart upload along with the corresponding
@@ -402,6 +418,15 @@ class ObjectStorageProtocol(ABC):
         """
         ...
 
+    @abstractmethod
+    async def _get_object_etag(self, *, bucket_id: str, object_id: str) -> str:
+        """Return the etag of an object.
+
+        *To be implemented by the provider. Input validation is done outside of this
+        method.*
+        """
+        ...
+
     @abstractmethod
     async def _get_object_size(self, *, bucket_id: str, object_id: str) -> int:
         """
@@ -435,11 +460,17 @@ class ObjectStorageProtocol(ABC):
         source_object_id: str,
         dest_bucket_id: str,
         dest_object_id: str,
+        abort_failed: bool = True,
     ) -> None:
         """
         Copy an object from one bucket (`source_bucket_id` and `source_object_id`) to
         another bucket (`dest_bucket_id` and `dest_object_id`).
 
+        If `abort_failed` is set to true (default), a failed copy operation tries to
+        abort the ongoing multipart upload it created (if using multipart mode).
+        This only works reliably as long as there are no other ongoing multipart operations for
+        the same destination bucket and object ID, in which case this should be set to false.
+
         *To be implemented by the provider. Input validation is done outside of this
         method.*
         """

{hexkit-3.6.0 → hexkit-3.8.0}/src/hexkit/providers/akafka/testcontainer.py

@@ -151,7 +151,7 @@ class KafkaSSLContainer(DockerContainer):
             with tarfile.TarFile(fileobj=archive, mode="w") as tar:
                 tarinfo = tarfile.TarInfo(name=path)
                 tarinfo.size = len(content)
-                tarinfo.mtime = time.time()  # type: ignore
+                tarinfo.mtime = time.time()
                 tar.addfile(tarinfo, BytesIO(content))
             archive.seek(0)
             self.get_wrapped_container().put_archive("/", archive)

{hexkit-3.6.0 → hexkit-3.8.0}/src/hexkit/providers/s3/provider.py

@@ -22,6 +22,7 @@ Utilities for testing are located in `./testutils.py`.
 # ruff: noqa: PLR0913
 
 import asyncio
+import re
 from functools import lru_cache
 from pathlib import Path
 from typing import Any, Optional
@@ -31,6 +32,7 @@ import botocore.client
 import botocore.config
 import botocore.configloader
 import botocore.exceptions
+import botocore.handlers
 from boto3.s3.transfer import TransferConfig
 from pydantic import Field, SecretStr
 from pydantic_settings import BaseSettings
@@ -39,6 +41,10 @@ from hexkit.protocols.objstorage import ObjectStorageProtocol, PresignedPostURL
 from hexkit.utils import calc_part_size
 
 __all__ = ["ObjectStorageProtocol", "PresignedPostURL"]
+# Allow colon character in bucket names to accommodate Ceph multi tenancy S3
+botocore.handlers.VALID_BUCKET = re.compile(
+    r"^(?:[a-zA-Z0-9_]{1,191}:)?[a-z0-9\-]{3,63}$"
+)
 
 
 class S3Config(BaseSettings):
@@ -116,6 +122,8 @@ def read_aws_config_ini(aws_config_ini: Path) -> botocore.config.Config:
 class S3ObjectStorage(ObjectStorageProtocol):
     """S3-based provider implementing the ObjectStorageProtocol."""
 
+    _re_bucket_id = botocore.handlers.VALID_BUCKET
+
     def __init__(
         self,
         *,
@@ -223,27 +231,33 @@ class S3ObjectStorage(ObjectStorageProtocol):
 
     async def _does_bucket_exist(self, bucket_id: str) -> bool:
         """Check whether a bucket with the specified ID (`bucket_id`) exists.
-        Return `True` if it exists and `False` otherwise.
+
+        Returns `True` if it exists and `False` otherwise.
+
+        Note that this method does not need to have the permission to list buckets.
         """
         try:
-            bucket_list = await asyncio.to_thread(self._client.list_buckets)
+            await asyncio.to_thread(self._client.head_bucket, Bucket=bucket_id)
         except botocore.exceptions.ClientError as error:
+            if error.response["Error"]["Code"] == "404":
+                return False
             raise self._translate_s3_client_errors(
                 error, bucket_id=bucket_id
             ) from error
-
-        return any(bucket["Name"] == bucket_id for bucket in bucket_list["Buckets"])
+        return True
 
     async def _assert_bucket_exists(self, bucket_id: str) -> None:
-        """Checks if the bucket with specified ID (`bucket_id`) exists and throws an
-        BucketNotFoundError otherwise.
+        """Assert that the bucket with the specified ID (`bucket_id`) exists.
+
+        If it does not exist, a BucketNotFoundError is raised.
         """
         if not await self.does_bucket_exist(bucket_id):
             raise self.BucketNotFoundError(bucket_id=bucket_id)
 
     async def _assert_bucket_not_exists(self, bucket_id: str) -> None:
-        """Checks if the bucket with specified ID (`bucket_id`) exists. If so, it throws
-        an BucketAlreadyExistsError.
+        """Assert that the bucket with the specified ID (`bucket_id`) does not exist.
+
+        If it exists, a BucketAlreadyExistsError is raised.
         """
         if await self.does_bucket_exist(bucket_id):
             raise self.BucketAlreadyExistsError(bucket_id=bucket_id)
@@ -274,7 +288,7 @@ class S3ObjectStorage(ObjectStorageProtocol):
         await self._assert_bucket_exists(bucket_id)
 
         try:
-            bucket = self._resource.Bucket(bucket_id)
+            bucket = self._resource.Bucket(bucket_id)  # pyright: ignore
             content = await asyncio.to_thread(bucket.objects.all)
             if delete_content:
                 await asyncio.to_thread(content.delete)
@@ -289,7 +303,7 @@ class S3ObjectStorage(ObjectStorageProtocol):
         await self._assert_bucket_exists(bucket_id)
 
         try:
-            bucket = self._resource.Bucket(bucket_id)
+            bucket = self._resource.Bucket(bucket_id)  # pyright: ignore
             content = await asyncio.to_thread(bucket.objects.all)
             return [object_summary.key for object_summary in content]
         except botocore.exceptions.ClientError as error:
@@ -306,17 +320,18 @@ class S3ObjectStorage(ObjectStorageProtocol):
         Return `True` if checks succeed and `False` otherwise.
         """
         if object_md5sum is not None:
-            raise NotImplementedError("Md5 checking is not yet implemented.")
+            raise NotImplementedError("MD5 checking is not yet implemented.")
 
         try:
-            _ = await asyncio.to_thread(
-                self._client.head_object,
-                Bucket=bucket_id,
-                Key=object_id,
+            await asyncio.to_thread(
+                self._client.head_object, Bucket=bucket_id, Key=object_id
             )
-        except botocore.exceptions.ClientError:
-            return False
-
+        except botocore.exceptions.ClientError as error:
+            if error.response["Error"]["Code"] == "404":
+                return False
+            raise self._translate_s3_client_errors(
+                error, bucket_id=bucket_id, object_id=object_id
+            ) from error
         return True
 
     async def _assert_object_exists(self, *, bucket_id: str, object_id: str) -> None:
@@ -451,7 +466,7 @@ class S3ObjectStorage(ObjectStorageProtocol):
             )
 
     async def _init_multipart_upload(self, *, bucket_id: str, object_id: str) -> str:
-        """Initiates a mulipart upload procedure. Returns the upload ID."""
+        """Initiates a multipart upload procedure. Returns the upload ID."""
         await self._assert_no_multipart_upload(bucket_id=bucket_id, object_id=object_id)
 
         try:
@@ -473,6 +488,7 @@ class S3ObjectStorage(ObjectStorageProtocol):
         object_id: str,
         part_number: int,
         expires_after: int = 3600,
+        part_md5: Optional[str] = None,
     ) -> str:
         """Given a id of an instantiated multipart upload along with the corresponding
         bucket and object ID, it returns a presigned URL for uploading a file part with the
@@ -491,16 +507,20 @@ class S3ObjectStorage(ObjectStorageProtocol):
             upload_id=upload_id, bucket_id=bucket_id, object_id=object_id
         )
 
+        params = {
+            "Bucket": bucket_id,
+            "Key": object_id,
+            "UploadId": upload_id,
+            "PartNumber": part_number,
+        }
+        # add additional parameters if any were passed
+        if part_md5:
+            params["ContentMD5"] = part_md5
         try:
             return await asyncio.to_thread(
                 self._client.generate_presigned_url,
                 ClientMethod="upload_part",
-                Params={
-                    "Bucket": bucket_id,
-                    "Key": object_id,
-                    "UploadId": upload_id,
-                    "PartNumber": part_number,
-                },
+                Params=params,
                 ExpiresIn=expires_after,
             )
         except botocore.exceptions.ClientError as error:
@@ -757,6 +777,22 @@ class S3ObjectStorage(ObjectStorageProtocol):
 
         return presigned_url
 
+    async def _get_object_etag(self, *, bucket_id: str, object_id: str) -> str:
+        """Return the etag of an object."""
+        await self._assert_object_exists(bucket_id=bucket_id, object_id=object_id)
+
+        object_metadata = await self._get_object_metadata(
+            bucket_id=bucket_id, object_id=object_id
+        )
+
+        if "ETag" not in object_metadata:
+            raise self.ObjectError(
+                f"Could not get the etag of the object with ID '{object_id}' in"
+                + f" bucket '{bucket_id}'."
+            )
+
+        return object_metadata["ETag"]
+
     async def _get_object_size(self, *, bucket_id: str, object_id: str) -> int:
         """Returns the size of an object in bytes."""
         await self._assert_object_exists(bucket_id=bucket_id, object_id=object_id)
@@ -795,9 +831,15 @@ class S3ObjectStorage(ObjectStorageProtocol):
         source_object_id: str,
         dest_bucket_id: str,
         dest_object_id: str,
+        abort_failed: bool = True,
     ) -> None:
         """Copy an object from one bucket (`source_bucket_id` and `source_object_id`) to
         another bucket (`dest_bucket_id` and `dest_object_id`).
+
+        If `abort_failed` is set to true (default), a failed copy operation tries to
+        abort the ongoing multipart upload it created (if using multipart mode).
+        This only works reliably as long as there are no other ongoing multipart operations for
+        the same destination bucket and object ID, in which case this should be set to false.
         """
         file_size = await self._get_object_size(
             bucket_id=source_bucket_id, object_id=source_object_id
@@ -825,6 +867,22 @@ class S3ObjectStorage(ObjectStorageProtocol):
                 Config=transfer_config,
             )
         except botocore.exceptions.ClientError as error:
+            if abort_failed:
+                # try to find and abort the multipart operation, if multipart copy mode is used
+                # There should only be one ongoing multipart upload/copy at at a time as long
+                # as a new object ID is generated for each attempt
+                try:
+                    upload_ids = await self._list_multipart_upload_for_object(
+                        bucket_id=dest_bucket_id, object_id=dest_object_id
+                    )
+                    if len(upload_ids) == 1:
+                        await self._abort_multipart_upload(
+                            upload_id=upload_ids[0],
+                            bucket_id=dest_bucket_id,
+                            object_id=dest_object_id,
+                        )
+                except botocore.exceptions.ClientError:
+                    pass
             raise self._translate_s3_client_errors(error) from error
 
     async def _delete_object(self, *, bucket_id: str, object_id: str) -> None:

{hexkit-3.6.0 → hexkit-3.8.0}/src/hexkit/providers/s3/testutils/_fixtures.py

@@ -24,7 +24,7 @@ from collections.abc import AsyncGenerator, Generator
 from contextlib import contextmanager
 from pathlib import Path
 from tempfile import NamedTemporaryFile
-from typing import Any, NamedTuple, Optional, Union
+from typing import NamedTuple, Optional, Union
 
 try:
     from typing import Self
@@ -64,7 +64,7 @@ __all__ = [
 ]
 
 
-LOCALSTACK_IMAGE = "localstack/localstack:3.5.0"
+LOCALSTACK_IMAGE = "localstack/localstack:3.8.1"
 
 TEST_FILE_DIR = Path(__file__).parent.parent.resolve() / "test_files"
 
@@ -223,19 +223,23 @@ class S3ContainerFixture(LocalStackContainer):
 
     def __init__(
         self,
-        port: int = 4566,
+        image: str = LOCALSTACK_IMAGE,
+        edge_port: int = 4566,
         region_name: Optional[str] = None,
-        **kwargs: Any,
+        **kwargs,
     ) -> None:
         """Initialize the container."""
-        super().__init__(image=LOCALSTACK_IMAGE)
+        super().__init__(
+            image=image, edge_port=edge_port, region_name=region_name, **kwargs
+        )
 
     def __enter__(self) -> Self:
         """Enter the container context."""
         super().__enter__()
-        s3_endpoint_url = self.get_url()
+        url = self.get_url()
+        url = url.replace("localhost", "127.0.0.1")
         s3_config = S3Config(  # type: ignore [call-arg]
-            s3_endpoint_url=s3_endpoint_url,
+            s3_endpoint_url=url,
             s3_access_key_id="test",
             s3_secret_access_key=SecretStr("test"),
         )
@@ -375,8 +379,8 @@ class FederatedS3Fixture:
 
         # Add the dummy items
         for bucket, objects in contents.items():
-            for object in objects:
-                with temp_file_object(bucket, object, 1) as file:
+            for obj in objects:
+                with temp_file_object(bucket, obj, 1) as file:
                     await storage.populate_file_objects([file])
 
 
@@ -418,10 +422,7 @@ def _s3_multi_container_fixture(
 
     if not storage_aliases:
         raise RuntimeError("The 'storage_aliases' list must not be empty.")
-    s3_containers = {
-        alias: S3ContainerFixture(name=f"{alias}_s3_container")
-        for alias in storage_aliases
-    }
+    s3_containers = {alias: S3ContainerFixture() for alias in storage_aliases}
     with S3MultiContainerFixture(s3_containers) as s3_multi_container:
         yield s3_multi_container
 

{hexkit-3.6.0 → hexkit-3.8.0/src/hexkit.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hexkit
-Version: 3.6.0
+Version: 3.8.0
 Summary: A Toolkit for Building Microservices using the Hexagonal Architecture
 Author-email: "German Human Genome Phenome Archive (GHGA)" <contact@ghga.de>
 License: Apache 2.0