hexkit 3.3.0__tar.gz → 3.5.0__tar.gz

{hexkit-3.3.0/src/hexkit.egg-info → hexkit-3.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hexkit
-Version: 3.3.0
+Version: 3.5.0
 Summary: A Toolkit for Building Microservices using the Hexagonal Architecture
 Author-email: "German Human Genome Phenome Archive (GHGA)" <contact@ghga.de>
 License: Apache 2.0

{hexkit-3.3.0 → hexkit-3.5.0}/pyproject.toml

@@ -23,7 +23,7 @@ classifiers = [
     "Intended Audience :: Developers",
 ]
 name = "hexkit"
-version = "3.3.0"
+version = "3.5.0"
 description = "A Toolkit for Building Microservices using the Hexagonal Architecture"
 dependencies = [
     "pydantic >=2, <3",

{hexkit-3.3.0 → hexkit-3.5.0}/src/hexkit/config.py

@@ -17,13 +17,13 @@
 
 import os
 from pathlib import Path
-from typing import Any, Callable, Final, Optional
+from typing import Callable, Final, Optional
 
-import yaml
 from pydantic_settings import (
     BaseSettings,
     PydanticBaseSettingsSource,
     SettingsConfigDict,
+    YamlConfigSettingsSource,
 )
 
 # Default config prefix:
@@ -79,29 +79,6 @@ def get_default_config_yaml(prefix: str) -> Optional[Path]:
     return None
 
 
-def yaml_settings_factory(
-    config_yaml: Optional[Path] = None,
-) -> Callable[[], dict[str, Any]]:
-    """
-    A factory of source methods for pydantic's BaseSettings Config that load
-    settings from a yaml file.
-
-    Args:
-        config_yaml (str, Optional):
-            Path to the yaml file to read from.
-    """
-
-    def yaml_settings() -> dict[str, Any]:
-        """Source method for loading pydantic BaseSettings from a yaml file"""
-        if config_yaml is None:
-            return {}
-
-        with open(config_yaml, encoding="utf8") as yaml_file:
-            return yaml.safe_load(yaml_file)
-
-    return yaml_settings
-
-
 def config_from_yaml(
     prefix: str = DEFAULT_CONFIG_PREFIX,
 ) -> Callable:
@@ -119,7 +96,7 @@ def config_from_yaml(
 
     Args:
         prefix: (str, optional):
-            When defining parameters via enviroment variables, all variables
+            When defining parameters via environment variables, all variables
             have to be prefixed with this string following this pattern
             "{prefix}_{actual_variable_name}". Moreover, this prefix is used
             to derive the default location for the config yaml file
@@ -161,7 +138,7 @@ def config_from_yaml(
                 raise ConfigYamlDoesNotExist(path=config_yaml)
 
             class ModSettings(settings):
-                """Modifies the orginal Settings class provided by the user"""
+                """Modifies the original Settings class provided by the user"""
 
                 model_config = SettingsConfigDict(frozen=True, env_prefix=f"{prefix}_")
 
@@ -174,12 +151,12 @@ def config_from_yaml(
                     dotenv_settings: PydanticBaseSettingsSource,
                     file_secret_settings: PydanticBaseSettingsSource,
                 ):
-                    """Add custom yaml source"""
+                    """Add custom yaml source, but maintain the priority of sources."""
                     return (
                         init_settings,
                         env_settings,
                         file_secret_settings,
-                        yaml_settings_factory(config_yaml),
+                        YamlConfigSettingsSource(settings_cls, config_yaml),
                     )
 
             # construct settings class:

{hexkit-3.3.0 → hexkit-3.5.0}/src/hexkit/protocols/dao.py

@@ -61,7 +61,7 @@ class ResourceAlreadyExistsError(RuntimeError):
     """Raised when a resource did unexpectedly exist."""
 
     def __init__(self, *, id_: str):
-        message = f'The resource with the id "{id_}" does already exist.'
+        message = f'The resource with the id "{id_}" already exists.'
         super().__init__(message)
 
 

{hexkit-3.3.0 → hexkit-3.5.0}/src/hexkit/protocols/objstorage.py

@@ -457,7 +457,10 @@ class ObjectStorageProtocol(ABC):
 
     # Validation logic for input parameter:
     # (is typically only used by the protocol but may also be used in
-    # provider-specific code)
+    # provider-specific code or overwritten by the provider)
+
+    _re_bucket_id = re.compile(r"^[a-z0-9\-]{3,63}$")
+    _re_bucket_id_msg = "must consist of 3-63 lowercase letters, digits or hyphens"
 
     @classmethod
     def _validate_bucket_id(cls, bucket_id: str):
@@ -466,43 +469,43 @@ class ObjectStorageProtocol(ABC):
         https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucketnamingrules.html
         Raises BucketIdValidationError if not valid.
         """
-        if not 3 <= len(bucket_id) < 64:
+        if not cls._re_bucket_id.match(bucket_id):
             raise cls.BucketIdValidationError(
                 bucket_id=bucket_id,
-                reason="must be between 3 and 63 characters long",
-            )
-        if not re.match(r"^[a-z0-9\-]*$", bucket_id):
-            raise cls.BucketIdValidationError(
-                bucket_id=bucket_id,
-                reason="only lowercase letters, digits and hyphens (-) are allowed",
+                reason=cls._re_bucket_id_msg,
             )
         if bucket_id.startswith("-") or bucket_id.endswith("-"):
             raise cls.BucketIdValidationError(
                 bucket_id=bucket_id,
-                reason="may not start or end with a hyphen (-).",
+                reason="may not start or end with a hyphen",
             )
 
+    _re_object_id = re.compile(r"^[a-zA-Z0-9\-_.]{3,255}$")
+    _re_object_id_msg = (
+        "must consist of 3-255 letters, digits, hyphens, underscores or dots"
+    )
+
     @classmethod
     def _validate_object_id(cls, object_id: str):
-        """Check whether a object id follows the recommended naming pattern.
+        """Check whether an object id follows the recommended naming pattern.
         This is roughly based on (plus some additional restrictions):
         https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-keys.html
         Raises ObjectIdValidationError if not valid.
+
+        Note that the default implementation does not allow object IDs to contain
+        forward slashes (/) which are sometimes used to represent directories.
+        Providers can override this method or the above constants to allow slashes,
+        other characters or longer object names.
         """
-        if not 3 <= len(object_id) < 64:
-            raise cls.ObjectIdValidationError(
-                object_id=object_id,
-                reason="must be between 3 and 63 characters long",
-            )
-        if not re.match(r"^[a-zA-Z0-9\-\.]*$", object_id):
+        if not cls._re_object_id.match(object_id):
             raise cls.ObjectIdValidationError(
                 object_id=object_id,
-                reason="only letters, digits, hyphens (-) and dots (.) are allowed",
+                reason=cls._re_object_id_msg,
             )
         if object_id.startswith(("-", ".")) or object_id.endswith(("-", ".")):
             raise cls.ObjectIdValidationError(
                 object_id=object_id,
-                reason="may not start or end with a hyphen (-) or a dot (.).",
+                reason="object names may not start or end with a hyphen or a dot",
             )
 
     # Exceptions that may be used by implementation:

{hexkit-3.3.0 → hexkit-3.5.0}/src/hexkit/providers/akafka/config.py

@@ -18,7 +18,7 @@
 
 from typing import Literal
 
-from pydantic import Field, SecretStr
+from pydantic import Field, PositiveInt, SecretStr
 from pydantic_settings import BaseSettings
 
 
@@ -76,3 +76,9 @@ class KafkaConfig(BaseSettings):
             + " newly correlation ID will be generated and used in the event header."
         ),
     )
+    kafka_max_message_size: PositiveInt = Field(
+        default=1024 * 1024,  # 1 MiB
+        description="The largest message size that can be transmitted, in bytes. Only"
+        + " services that have a need to send/receive larger messages should set this.",
+        examples=[1024 * 1024, 16 * 1024 * 1024],
+    )

{hexkit-3.3.0 → hexkit-3.5.0}/src/hexkit/providers/akafka/provider/eventpub.py

@@ -54,6 +54,7 @@ class KafkaProducerCompatible(Protocol):
         client_id: str,
         key_serializer: Callable[[Any], bytes],
         value_serializer: Callable[[Any], bytes],
+        max_request_size: int,
     ):
         """
         Initialize the producer with some config params.
@@ -67,6 +68,8 @@ class KafkaProducerCompatible(Protocol):
                 Function to serialize the keys into bytes.
             value_serializer:
                 Function to serialize the values into bytes.
+            max_request_size:
+                Maximum sendable message size.
         """
         ...
 
@@ -116,6 +119,7 @@ class KafkaEventPublisher(EventPublisherProtocol):
             value_serializer=lambda event_value: json.dumps(event_value).encode(
                 "ascii"
             ),
+            max_request_size=config.kafka_max_message_size,
         )
 
         try:

{hexkit-3.3.0 → hexkit-3.5.0}/src/hexkit/providers/akafka/provider/eventsub.py

@@ -90,6 +90,7 @@ class KafkaConsumerCompatible(Protocol):
         enable_auto_commit: bool,
         key_deserializer: Callable[[bytes], str],
         value_deserializer: Callable[[bytes], str],
+        max_partition_fetch_bytes: int,
     ):
         """
         Initialize the consumer with some config params.
@@ -108,6 +109,8 @@ class KafkaConsumerCompatible(Protocol):
                 Function to deserialize the keys into strings.
             value_serializer:
                 Function to deserialize the values into strings.
+            max_partition_fetch_bytes:
+                Maximum receivable message size
         """
         ...
 
@@ -176,6 +179,7 @@ class KafkaEventSubscriber(InboundProviderBase):
             value_deserializer=lambda event_value: json.loads(
                 event_value.decode("ascii")
             ),
+            max_partition_fetch_bytes=config.kafka_max_message_size,
         )
         try:
             await consumer.start()

{hexkit-3.3.0 → hexkit-3.5.0}/src/hexkit/providers/akafka/testutils.py

@@ -430,6 +430,7 @@ class KafkaFixture:
 
     async def clear_topics(
         self,
+        *,
         topics: Optional[Union[str, list[str]]] = None,
         exclude_internal: bool = True,
     ):

{hexkit-3.3.0 → hexkit-3.5.0}/src/hexkit/providers/mongodb/provider.py

@@ -31,6 +31,7 @@ from motor.core import AgnosticCollection
 from motor.motor_asyncio import AsyncIOMotorClient
 from pydantic import Field, SecretStr
 from pydantic_settings import BaseSettings
+from pymongo.errors import DuplicateKeyError
 
 from hexkit.protocols.dao import (
     DaoFactoryProtocol,
@@ -42,6 +43,7 @@ from hexkit.protocols.dao import (
     InvalidFindMappingError,
     MultipleHitsFoundError,
     NoHitsFoundError,
+    ResourceAlreadyExistsError,
     ResourceNotFoundError,
 )
 from hexkit.utils import FieldNotInModelError, validate_fields_in_model
@@ -386,7 +388,10 @@ class MongoDbDaoNaturalId(MongoDbDaoBase[Dto]):
                 when a resource with the ID specified in the dto does already exist.
         """
         document = self._dto_to_document(dto)
-        await self._collection.insert_one(document)
+        try:
+            await self._collection.insert_one(document)
+        except DuplicateKeyError as error:
+            raise ResourceAlreadyExistsError(id_=document["_id"]) from error
 
     async def upsert(self, dto: Dto) -> None:
         """Update the provided resource if it already exists, create it otherwise.

{hexkit-3.3.0 → hexkit-3.5.0}/src/hexkit/providers/mongodb/testutils.py

@@ -61,24 +61,30 @@ class MongoDbFixture:
 
     def empty_collections(
         self,
+        *,
+        collections: Optional[Union[str, list[str]]] = None,
         exclude_collections: Optional[Union[str, list[str]]] = None,
     ):
-        """Drop all mongodb collections in the database.
+        """Drop the given mongodb collection(s) in the database.
 
+        If no collections are specified, all collections will be dropped.
         You can also specify collection(s) that should be excluded
         from the operation, i.e. collections that should be kept.
         """
         db_name = self.config.db_name
-        if exclude_collections is None:
-            exclude_collections = []
-        if isinstance(exclude_collections, str):
-            exclude_collections = [exclude_collections]
-        excluded_collections = set(exclude_collections)
         try:
-            collection_names = self.client[db_name].list_collection_names()
-            for collection_name in collection_names:
-                if collection_name not in excluded_collections:
-                    self.client[db_name].drop_collection(collection_name)
+            if collections is None:
+                collections = self.client[db_name].list_collection_names()
+            elif isinstance(collections, str):
+                collections = [collections]
+            if exclude_collections is None:
+                exclude_collections = []
+            elif isinstance(exclude_collections, str):
+                exclude_collections = [exclude_collections]
+            excluded_collections = set(exclude_collections)
+            for collection in collections:
+                if collection not in excluded_collections:
+                    self.client[db_name].drop_collection(collection)
         except (ExecutionTimeout, OperationFailure) as error:
             raise RuntimeError(
                 f"Could not drop collection(s) of Mongo database {db_name}"

{hexkit-3.3.0 → hexkit-3.5.0}/src/hexkit/providers/mongokafka/provider.py

@@ -20,6 +20,7 @@ Require dependencies of the `akafka` and `mongodb` extras.
 """
 
 import json
+import logging
 from collections.abc import AsyncIterator, Awaitable, Collection, Mapping
 from contextlib import AbstractAsyncContextManager, asynccontextmanager, contextmanager
 from typing import Any, Callable, Generic, Optional
@@ -27,6 +28,7 @@ from typing import Any, Callable, Generic, Optional
 from aiokafka import AIOKafkaProducer
 from motor.core import AgnosticCollection
 from motor.motor_asyncio import AsyncIOMotorClient
+from pydantic import field_validator
 
 from hexkit.correlation import get_correlation_id, set_correlation_id
 from hexkit.custom_types import JsonObject
@@ -396,6 +398,16 @@ class MongoKafkaDaoPublisher(Generic[Dto]):
 class MongoKafkaConfig(MongoDbConfig, KafkaConfig):
     """Config parameters and their defaults."""
 
+    @field_validator("kafka_max_message_size", mode="after")
+    @classmethod
+    def validate_max_message_size(cls, value: int) -> int:
+        """Validate the maximum message size."""
+        if value > 2**24:  # 16 MiB
+            logging.warning(
+                f"Max message size ({value}) exceeds the 16 MiB document size limit for MongoDB!"
+            )
+        return value
+
 
 class MongoKafkaDaoPublisherFactory(DaoPublisherFactoryProtocol):
     """A provider implementing the DaoPublisherFactoryProtocol based on MongoDB and

{hexkit-3.3.0 → hexkit-3.5.0}/src/hexkit/providers/s3/testutils/__init__.py

@@ -15,7 +15,7 @@
 
 """S3-related test fixtures and support functions."""
 
-from ._fixtures import (  # noqa: F401
+from ._fixtures import (
     LOCALSTACK_IMAGE,
     MEBIBYTE,
     TEST_FILE_DIR,
@@ -36,8 +36,8 @@ from ._fixtures import (  # noqa: F401
     tmp_file,
     upload_file,
 )
-from ._typical_workflow import typical_workflow  # noqa: F401
-from ._utils import (  # noqa: F401
+from ._typical_workflow import typical_workflow
+from ._utils import (
     check_part_size,
     download_and_check_test_file,
     multipart_upload_file,
@@ -45,3 +45,32 @@ from ._utils import (  # noqa: F401
     upload_part_of_size,
     upload_part_via_url,
 )
+
+__all__ = [
+    "LOCALSTACK_IMAGE",
+    "MEBIBYTE",
+    "TEST_FILE_DIR",
+    "TEST_FILE_PATHS",
+    "FileObject",
+    "S3ContainerFixture",
+    "S3Fixture",
+    "calc_md5",
+    "clean_s3_fixture",
+    "get_clean_s3_fixture",
+    "get_persistent_s3_fixture",
+    "get_s3_container_fixture",
+    "persistent_s3_fixture",
+    "populate_storage",
+    "s3_container_fixture",
+    "s3_fixture",
+    "temp_file_object",
+    "tmp_file",
+    "upload_file",
+    "typical_workflow",
+    "check_part_size",
+    "download_and_check_test_file",
+    "multipart_upload_file",
+    "upload_part",
+    "upload_part_of_size",
+    "upload_part_via_url",
+]

{hexkit-3.3.0 → hexkit-3.5.0}/src/hexkit/providers/s3/testutils/_fixtures.py

@@ -24,7 +24,7 @@ from collections.abc import AsyncGenerator, Generator
 from contextlib import contextmanager
 from pathlib import Path
 from tempfile import NamedTemporaryFile
-from typing import Any, NamedTuple, Optional
+from typing import Any, NamedTuple, Optional, Union
 
 try:
     from typing import Self
@@ -93,11 +93,10 @@ class S3Fixture:
         self.config = config
         self.storage = storage
 
-    def get_buckets(self) -> set[str]:
+    def get_buckets(self) -> list[str]:
         """Return a list of the buckets currently existing in the S3 object storage."""
         response = self.storage._client.list_buckets()
-        buckets = {bucket["Name"] for bucket in response["Buckets"]}
-        return buckets
+        return [bucket["Name"] for bucket in response["Buckets"]]
 
     async def populate_buckets(self, buckets: list[str]):
         """Populate the storage with buckets."""
@@ -111,19 +110,66 @@ class S3Fixture:
             self.storage, bucket_fixtures=[], object_fixtures=file_objects
         )
 
-    async def empty_buckets(self, buckets_to_exclude: Optional[list[str]] = None):
-        """Clean the test artifacts or files from the populated buckets."""
-        for bucket in self.get_buckets().difference(buckets_to_exclude or []):
+    async def empty_buckets(
+        self,
+        *,
+        buckets: Optional[Union[str, list[str]]] = None,
+        exclude_buckets: Optional[Union[str, list[str]]] = None,
+    ):
+        """Remove all test objects from the given bucket(s).
+
+        If no buckets are specified, all existing buckets will be emptied,
+        therefore we recommend to always specify a bucket list.
+        You can also specify bucket(s) that should be excluded
+        from the operation, i.e. buckets that should not be emptied.
+        """
+        if buckets is None:
+            buckets = self.get_buckets()
+        elif isinstance(buckets, str):
+            buckets = [buckets]
+        if exclude_buckets is None:
+            exclude_buckets = []
+        elif isinstance(exclude_buckets, str):
+            exclude_buckets = [exclude_buckets]
+        excluded_buckets = set(exclude_buckets)
+        for bucket in buckets:
+            if bucket in excluded_buckets:
+                continue
             # Get list of all objects in the bucket
             object_ids = await self.storage.list_all_object_ids(bucket_id=bucket)
             # Delete all of these objects
             for object_id in object_ids:
+                # Note that id validation errors can be raised here if the bucket
+                # was populated by other means than the S3 storage fixture.
+                # We intentionally do not catch these errors because they
+                # usually mean that you're operating on the wrong buckets.
                 await self.storage.delete_object(bucket_id=bucket, object_id=object_id)
 
-    async def delete_buckets(self, buckets_to_exclude: Optional[list[str]] = None):
-        """Delete the populated buckets."""
-        for bucket in self.get_buckets().difference(buckets_to_exclude or []):
-            await self.storage.delete_bucket(bucket, delete_content=True)
+    async def delete_buckets(
+        self,
+        *,
+        buckets: Optional[Union[str, list[str]]] = None,
+        exclude_buckets: Optional[Union[str, list[str]]] = None,
+    ):
+        """Delete the given bucket(s).
+
+        If no buckets are specified, all existing buckets will be deleted,
+        therefore we recommend to always specify a bucket list.
+        You can also specify bucket(s) that should be excluded
+        from the operation, i.e. buckets that should be kept.
+        """
+        if buckets is None:
+            buckets = self.get_buckets()
+        elif isinstance(buckets, str):
+            buckets = [buckets]
+        if exclude_buckets is None:
+            exclude_buckets = []
+        elif isinstance(exclude_buckets, str):
+            exclude_buckets = [exclude_buckets]
+        excluded_buckets = set(exclude_buckets)
+        for bucket in buckets:
+            if bucket not in excluded_buckets:
+                await self.storage.delete_bucket(bucket, delete_content=True)
 
     async def get_initialized_upload(self) -> UploadDetails:
         """Initialize a new empty multipart upload process.

{hexkit-3.3.0 → hexkit-3.5.0}/src/hexkit/providers/testing/eventpub.py

@@ -26,6 +26,8 @@ from typing import NamedTuple, Optional
 from hexkit.custom_types import JsonObject
 from hexkit.protocols.eventpub import EventPublisherProtocol
 
+__all__ = ["InMemEventStore", "InMemEventPublisher", "TopicExhaustedError"]
+
 
 class TopicExhaustedError(RuntimeError):
     """Thrown when now more event are queued in a topic."""

{hexkit-3.3.0 → hexkit-3.5.0/src/hexkit.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hexkit
-Version: 3.3.0
+Version: 3.5.0
 Summary: A Toolkit for Building Microservices using the Hexagonal Architecture
 Author-email: "German Human Genome Phenome Archive (GHGA)" <contact@ghga.de>
 License: Apache 2.0