hexkit 3.0.2__tar.gz → 3.2.0__tar.gz

{hexkit-3.0.2/src/hexkit.egg-info → hexkit-3.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hexkit
-Version: 3.0.2
+Version: 3.2.0
 Summary: A Toolkit for Building Microservices using the Hexagonal Architecture
 Author-email: "German Human Genome Phenome Archive (GHGA)" <contact@ghga.de>
 License: Apache 2.0
@@ -23,21 +23,21 @@ Requires-Dist: pydantic_settings<3,>=2
 Requires-Dist: PyYAML<7,>=6.0
 Provides-Extra: akafka
 Requires-Dist: aiokafka~=0.10.0; extra == "akafka"
-Requires-Dist: jsonschema<5,>=4.21; extra == "akafka"
+Requires-Dist: jsonschema<5,>=4.22; extra == "akafka"
 Provides-Extra: s3
-Requires-Dist: boto3<2,>=1.26.50; extra == "s3"
-Requires-Dist: botocore<2,>=1.29.50; extra == "s3"
+Requires-Dist: boto3<2,>=1.34.104; extra == "s3"
+Requires-Dist: botocore<2,>=1.34.104; extra == "s3"
 Provides-Extra: mongodb
-Requires-Dist: motor<4,>=3.1.1; extra == "mongodb"
+Requires-Dist: motor<4,>=3.4.0; extra == "mongodb"
 Provides-Extra: test-akafka
 Requires-Dist: hexkit[akafka]; extra == "test-akafka"
-Requires-Dist: testcontainers[kafka]<5,>=4.3.3; extra == "test-akafka"
+Requires-Dist: testcontainers[kafka]<5,>=4.4.1; extra == "test-akafka"
 Provides-Extra: test-s3
 Requires-Dist: hexkit[s3]; extra == "test-s3"
-Requires-Dist: testcontainers<5,>=4.3.3; extra == "test-s3"
+Requires-Dist: testcontainers<5,>=4.4.1; extra == "test-s3"
 Provides-Extra: test-mongodb
 Requires-Dist: hexkit[mongodb]; extra == "test-mongodb"
-Requires-Dist: testcontainers[mongo]<5,>=4.3.3; extra == "test-mongodb"
+Requires-Dist: testcontainers[mongo]<5,>=4.4.1; extra == "test-mongodb"
 Provides-Extra: test
 Requires-Dist: hexkit[test-akafka,test-mongodb,test-s3]; extra == "test"
 Provides-Extra: all

{hexkit-3.0.2 → hexkit-3.2.0}/pyproject.toml

@@ -23,7 +23,7 @@ classifiers = [
     "Intended Audience :: Developers",
 ]
 name = "hexkit"
-version = "3.0.2"
+version = "3.2.0"
 description = "A Toolkit for Building Microservices using the Hexagonal Architecture"
 dependencies = [
     "pydantic >=2, <3",
@@ -37,26 +37,26 @@ text = "Apache 2.0"
 [project.optional-dependencies]
 akafka = [
     "aiokafka~=0.10.0",
-    "jsonschema >=4.21, <5",
+    "jsonschema >=4.22, <5",
 ]
 s3 = [
-    "boto3 >=1.26.50, <2",
-    "botocore >=1.29.50, <2",
+    "boto3 >=1.34.104, <2",
+    "botocore >=1.34.104, <2",
 ]
 mongodb = [
-    "motor >=3.1.1, <4",
+    "motor >=3.4.0, <4",
 ]
 test-akafka = [
     "hexkit[akafka]",
-    "testcontainers[kafka] >=4.3.3, <5",
+    "testcontainers[kafka] >=4.4.1, <5",
 ]
 test-s3 = [
     "hexkit[s3]",
-    "testcontainers >=4.3.3, <5",
+    "testcontainers >=4.4.1, <5",
 ]
 test-mongodb = [
     "hexkit[mongodb]",
-    "testcontainers[mongo] >=4.3.3, <5",
+    "testcontainers[mongo] >=4.4.1, <5",
 ]
 test = [
     "hexkit[test-akafka,test-s3,test-mongodb]",

{hexkit-3.0.2 → hexkit-3.2.0}/src/hexkit/protocols/daopub.py

@@ -62,7 +62,7 @@ class DaoPublisherFactoryProtocol(DaoFactoryBase, ABC):
         dto_model: type[Dto],
         id_field: str,
         fields_to_index: Optional[Collection[str]] = None,
-        dto_to_event: Callable[[Dto], JsonObject],
+        dto_to_event: Callable[[Dto], Optional[JsonObject]],
         event_topic: str,
         autopublish: bool = True,
     ) -> DaoPublisher[Dto]:
@@ -82,6 +82,7 @@ class DaoPublisherFactoryProtocol(DaoFactoryBase, ABC):
                 `id_field`. Defaults to None.
             dto_to_event:
                 A function that takes a DTO and returns the payload for an event.
+                If the returned payload is None, the event will not be published.
             event_topic:
                 The topic to which events should be published.
             autopublish:
@@ -120,7 +121,7 @@ class DaoPublisherFactoryProtocol(DaoFactoryBase, ABC):
         dto_model: type[Dto],
         id_field: str,
         fields_to_index: Optional[Collection[str]],
-        dto_to_event: Callable[[Dto], JsonObject],
+        dto_to_event: Callable[[Dto], Optional[JsonObject]],
         event_topic: str,
         autopublish: bool,
     ) -> DaoPublisher[Dto]:

{hexkit-3.0.2 → hexkit-3.2.0}/src/hexkit/protocols/objstorage.py

@@ -517,16 +517,16 @@ class ObjectStorageProtocol(ABC):
         """Thrown when trying to access a bucket with an ID that doesn't exist."""
 
         def __init__(self, bucket_id: Optional[str]):
-            in_bucket = f" in bucket with ID '{bucket_id}'" if bucket_id else ""
-            message = f"The bucket{in_bucket} does not exist."
+            with_id = f" with ID '{bucket_id}'" if bucket_id else ""
+            message = f"The bucket{with_id} does not exist."
             super().__init__(message)
 
     class BucketAlreadyExistsError(BucketError):
         """Thrown when trying to create a bucket with an ID that already exists."""
 
         def __init__(self, bucket_id: Optional[str]):
-            in_bucket = f" in bucket with ID '{bucket_id}'" if bucket_id else ""
-            message = f"The bucket{in_bucket} already exists."
+            with_id = f" with ID '{bucket_id}'" if bucket_id else ""
+            message = f"The bucket{with_id} already exists."
             super().__init__(message)
 
     class BucketNotEmptyError(BucketError):

{hexkit-3.0.2 → hexkit-3.2.0}/src/hexkit/providers/akafka/testcontainer.py

@@ -27,7 +27,7 @@ from testcontainers.core.waiting_utils import wait_for_logs
 
 __all__ = ["KafkaSSLContainer"]
 
-DEFAULT_IMAGE = "confluentinc/cp-kafka:7.6.0"
+DEFAULT_IMAGE = "confluentinc/cp-kafka:7.6.1"
 
 DEFAULT_PORT = 9093  # default port for the Kafka container
 BROKER_PORT = 9092  # auxiliary port for inter broker listener

{hexkit-3.0.2 → hexkit-3.2.0}/src/hexkit/providers/akafka/testutils.py

@@ -20,12 +20,19 @@ Please note, only use for testing purposes.
 """
 
 import json
-from collections.abc import AsyncGenerator, Sequence
+from collections.abc import AsyncGenerator, Generator, Sequence
 from contextlib import asynccontextmanager
 from dataclasses import dataclass
 from functools import partial
-from typing import Callable, Optional, Union
+from typing import Any, Callable, Optional, Union
 
+try:
+    from typing import Self
+except ImportError:  # Python < 3.11
+    from typing_extensions import Self
+
+
+import pytest
 import pytest_asyncio
 from aiokafka import AIOKafkaConsumer, TopicPartition
 from aiokafka.admin import AIOKafkaAdminClient
@@ -41,6 +48,26 @@ from hexkit.providers.akafka.provider import (
 )
 from hexkit.providers.akafka.testcontainer import DEFAULT_IMAGE as KAFKA_IMAGE
 
+__all__ = [
+    "KAFKA_IMAGE",
+    "EventBase",
+    "EventRecorder",
+    "ExpectedEvent",
+    "RecordedEvent",
+    "ValidationError",
+    "KafkaConfig",
+    "KafkaContainerFixture",
+    "KafkaEventPublisher",
+    "KafkaFixture",
+    "get_kafka_container_fixture",
+    "get_clean_kafka_fixture",
+    "get_persistent_kafka_fixture",
+    "kafka_container_fixture",
+    "clean_kafka_fixture",
+    "persistent_kafka_fixture",
+    "kafka_fixture",
+]
+
 
 @dataclass(frozen=True)
 class EventBase:
@@ -339,7 +366,7 @@ class EventRecorder:
 
 
 class KafkaFixture:
-    """Yielded by the `kafka_fixture` function"""
+    """A fixture with utility methods for tests that use Apache Kafka."""
 
     def __init__(
         self,
@@ -468,49 +495,115 @@ class KafkaFixture:
         )
 
 
-async def kafka_fixture_function() -> AsyncGenerator[KafkaFixture, None]:
-    """Pytest fixture for tests depending on the Kafka-base providers.
+class KafkaContainerFixture(KafkaContainer):
+    """Kafka test container with configuration and command execution."""
 
-    **Do not call directly** Instead, use get_kafka_fixture()
-    """
-    with KafkaContainer(image=KAFKA_IMAGE) as kafka:
-        kafka_servers = [kafka.get_bootstrap_server()]
-        config = KafkaConfig(
+    kafka_config: KafkaConfig
+
+    def __init__(self, port: int = 9093, **kwargs: Any) -> None:
+        """Initialize the container."""
+        super().__init__(image=KAFKA_IMAGE, port=port, **kwargs)
+
+    def __enter__(self) -> Self:
+        """Enter the container context."""
+        super().__enter__()
+        kafka_servers = [self.get_bootstrap_server()]
+        self.kafka_config = KafkaConfig(
             service_name="test_publisher",
             service_instance_id="001",
             kafka_servers=kafka_servers,
         )
+        return self
 
-        def wrapped_exec_run(command: str, run_in_shell: bool):
-            """Run the given command in the kafka testcontainer.
+    def wrapped_exec_run(self, command: str, run_in_shell: bool):
+        """Run the given command in the kafka testcontainer.
 
-            Args:
-              - `command`: The full command to run.
-              - `run_in_shell`: If True, will run the command in a shell.
+        Args:
+            - `command`: The full command to run.
+            - `run_in_shell`: If True, will run the command in a shell.
 
-            Raises:
-              - `RuntimeError`: when the exit code returned by the command is not zero.
-            """
-            cmd = ["sh", "-c", command] if run_in_shell else command
-            exit_code, output = kafka.get_wrapped_container().exec_run(cmd)
+        Raises:
+            - `RuntimeError`: when the exit code returned by the command is not zero.
+        """
+        cmd = ["sh", "-c", command] if run_in_shell else command
+        exit_code, output = self.get_wrapped_container().exec_run(cmd)
 
-            if exit_code != 0:
-                raise RuntimeError(
-                    f"result: {exit_code}, output: {output.decode('utf-8')}"
-                )
+        if exit_code != 0:
+            raise RuntimeError(f"result: {exit_code}, output: {output.decode('utf-8')}")
 
-        async with KafkaEventPublisher.construct(config=config) as publisher:
-            yield KafkaFixture(
-                config=config,
-                kafka_servers=kafka_servers,
-                publisher=publisher,
-                cmd_exec_func=wrapped_exec_run,
-            )
+
+def _kafka_container_fixture() -> Generator[KafkaContainerFixture, None, None]:
+    """Fixture function for getting a running Kafka test container."""
+    with KafkaContainerFixture() as kafka_container:
+        yield kafka_container
+
+
+def get_kafka_container_fixture(
+    scope: PytestScope = "session", name: str = "kafka_container"
+):
+    """Get a Kafka test container fixture with desired scope and name.
+
+    By default, the session scope is used for Kafka test containers.
+    """
+    return pytest.fixture(_kafka_container_fixture, scope=scope, name=name)
+
+
+kafka_container_fixture = get_kafka_container_fixture()
 
 
-def get_kafka_fixture(scope: PytestScope = "function"):
-    """Produce a kafka fixture with desired scope. Default is the function scope."""
-    return pytest_asyncio.fixture(kafka_fixture_function, scope=scope)
+async def _persistent_kafka_fixture(
+    kafka_container: KafkaContainerFixture,
+) -> AsyncGenerator[KafkaFixture, None]:
+    """Fixture function that gets a persistent Kafka fixture.
+
+    The state of Kafka is not cleaned up by this function.
+    """
+    config = kafka_container.kafka_config
+    async with KafkaEventPublisher.construct(config=config) as publisher:
+        kafka_fixture = KafkaFixture(
+            config=config,
+            kafka_servers=config.kafka_servers,
+            cmd_exec_func=kafka_container.wrapped_exec_run,
+            publisher=publisher,
+        )
+        yield kafka_fixture
+
+
+def get_persistent_kafka_fixture(scope: PytestScope = "function", name: str = "kafka"):
+    """Get a Kafka fixture with desired scope and name.
+
+    The state of the Kafka test container is persisted across tests.
+
+    By default, the function scope is used for this fixture,
+    while the session scope is used for the underlying Kafka test container.
+    """
+    return pytest_asyncio.fixture(_persistent_kafka_fixture, scope=scope, name=name)
+
+
+persistent_kafka_fixture = get_persistent_kafka_fixture()
+
+
+async def _clean_kafka_fixture(
+    kafka_container: KafkaContainerFixture,
+) -> AsyncGenerator[KafkaFixture, None]:
+    """Fixture function that gets a clean Kafka fixture.
+
+    The clean state is achieved by clearing all Kafka topics upfront.
+    """
+    async for kafka_fixture in _persistent_kafka_fixture(kafka_container):
+        await kafka_fixture.clear_topics()
+        yield kafka_fixture
+
+
+def get_clean_kafka_fixture(scope: PytestScope = "function", name: str = "kafka"):
+    """Get a Kafka fixture with desired scope and name.
+
+    The state of Kafka is reset by clearing all topics before running tests.
+
+    By default, the function scope is used for this fixture,
+    while the session scope is used for the underlying Kafka test container.
+    """
+    return pytest_asyncio.fixture(_clean_kafka_fixture, scope=scope, name=name)
 
 
-kafka_fixture = get_kafka_fixture()
+kafka_fixture = clean_kafka_fixture = get_clean_kafka_fixture()

hexkit-3.2.0/src/hexkit/providers/mongodb/testutils.py

@@ -0,0 +1,176 @@
+# Copyright 2021 - 2024 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
+# for the German Human Genome-Phenome Archive (GHGA)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""Utilities for testing code that uses the MongoDbDaoFactory provider.
+
+Please note, only use for testing purposes.
+"""
+
+from collections.abc import Generator
+from dataclasses import dataclass
+from typing import Optional, Union
+
+import pytest
+from pydantic import SecretStr
+from pymongo import MongoClient
+from pymongo.errors import ExecutionTimeout, OperationFailure
+from testcontainers.mongodb import MongoDbContainer
+
+from hexkit.custom_types import PytestScope
+from hexkit.providers.mongodb.provider import MongoDbConfig, MongoDbDaoFactory
+
+MONGODB_IMAGE = "mongo:7.0.9"
+
+__all__ = [
+    "MONGODB_IMAGE",
+    "MongoClient",
+    "MongoDbConfig",
+    "MongoDbContainerFixture",
+    "MongoDbDaoFactory",
+    "MongoDbFixture",
+    "get_mongodb_container_fixture",
+    "get_persistent_mongodb_fixture",
+    "get_clean_mongodb_fixture",
+    "mongodb_container_fixture",
+    "clean_mongodb_fixture",
+    "persistent_mongodb_fixture",
+    "mongodb_fixture",
+]
+
+
+@dataclass(frozen=True)
+class MongoDbFixture:
+    """A fixture with utility methods for tests that use MongoDB"""
+
+    client: MongoClient
+    config: MongoDbConfig
+    dao_factory: MongoDbDaoFactory
+
+    def empty_collections(
+        self,
+        exclude_collections: Optional[Union[str, list[str]]] = None,
+    ):
+        """Drop all mongodb collections in the database.
+
+        You can also specify collection(s) that should be excluded
+        from the operation, i.e. collections that should be kept.
+        """
+        db_name = self.config.db_name
+        if exclude_collections is None:
+            exclude_collections = []
+        if isinstance(exclude_collections, str):
+            exclude_collections = [exclude_collections]
+        excluded_collections = set(exclude_collections)
+        try:
+            collection_names = self.client[db_name].list_collection_names()
+            for collection_name in collection_names:
+                if collection_name not in excluded_collections:
+                    self.client[db_name].drop_collection(collection_name)
+        except (ExecutionTimeout, OperationFailure) as error:
+            raise RuntimeError(
+                f"Could not drop collection(s) of Mongo database {db_name}"
+            ) from error
+
+
+class MongoDbContainerFixture(MongoDbContainer):
+    """MongoDB test container with MongoDB configuration."""
+
+    mongodb_config: MongoDbConfig
+
+
+def _mongodb_container_fixture() -> Generator[MongoDbContainerFixture, None, None]:
+    """Fixture function for getting a running MongoDB test container."""
+    with MongoDbContainerFixture(image=MONGODB_IMAGE) as mongodb_container:
+        db_connection_str = mongodb_container.get_connection_url()
+        mongodb_config = MongoDbConfig(
+            db_connection_str=SecretStr(db_connection_str), db_name="test"
+        )
+        mongodb_container.mongodb_config = mongodb_config
+        yield mongodb_container
+
+
+def get_mongodb_container_fixture(
+    scope: PytestScope = "session", name: str = "mongodb_container"
+):
+    """Get a MongoDB test container fixture with desired scope and name.
+
+    By default, the session scope is used for MongoDB test containers.
+    """
+    return pytest.fixture(_mongodb_container_fixture, scope=scope, name=name)
+
+
+mongodb_container_fixture = get_mongodb_container_fixture()
+
+
+def _persistent_mongodb_fixture(
+    mongodb_container: MongoDbContainerFixture,
+) -> Generator[MongoDbFixture, None, None]:
+    """Fixture function that gets a persistent MongoDB fixture.
+
+    The state of the MongoDB is not cleaned up by the function.
+    """
+    config = mongodb_container.mongodb_config
+    dao_factory = MongoDbDaoFactory(config=config)
+    client = mongodb_container.get_connection_client()
+    yield MongoDbFixture(
+        client=client,
+        config=config,
+        dao_factory=dao_factory,
+    )
+
+    client.close()
+
+
+def get_persistent_mongodb_fixture(
+    scope: PytestScope = "function", name: str = "mongodb"
+):
+    """Get a MongoDB fixture with desired scope and name.
+
+    The state of the MongoDB test container is persisted across tests.
+
+    By default, the function scope is used for this fixture,
+    while the session scope is used for the underlying MongoDB test container.
+    """
+    return pytest.fixture(_persistent_mongodb_fixture, scope=scope, name=name)
+
+
+persistent_mongodb_fixture = get_persistent_mongodb_fixture()
+
+
+def _clean_mongodb_fixture(
+    mongodb_container: MongoDbContainerFixture,
+) -> Generator[MongoDbFixture, None, None]:
+    """Fixture function that gets a clean MongoDB fixture.
+
+    The clean state is achieved by emptying all MongoDB collections upfront.
+    """
+    for mongodb_fixture in _persistent_mongodb_fixture(mongodb_container):
+        mongodb_fixture.empty_collections()
+        yield mongodb_fixture
+
+
+def get_clean_mongodb_fixture(scope: PytestScope = "function", name: str = "mongodb"):
+    """Get a MongoDB fixture with desired scope and name.
+
+    The state of the MongoDB is reset by emptying all collections before running tests.
+
+    By default, the function scope is used for this fixture,
+    while the session scope is used for the underlying MongoDB test container.
+    """
+    return pytest.fixture(_clean_mongodb_fixture, scope=scope, name=name)
+
+
+mongodb_fixture = clean_mongodb_fixture = get_clean_mongodb_fixture()

{hexkit-3.0.2 → hexkit-3.2.0}/src/hexkit/providers/mongokafka/provider.py

@@ -28,6 +28,7 @@ from aiokafka import AIOKafkaProducer
 from motor.core import AgnosticCollection
 from motor.motor_asyncio import AsyncIOMotorClient
 
+from hexkit.correlation import get_correlation_id, set_correlation_id
 from hexkit.custom_types import JsonObject
 from hexkit.protocols.dao import (
     DaoNaturalId,
@@ -86,7 +87,13 @@ def dto_to_document(
     """
     document = json.loads(dto.model_dump_json())
     document["_id"] = document.pop(id_field)
-    document["__metadata__"] = {"deleted": False, "published": published}
+
+    correlation_id = get_correlation_id()
+    document["__metadata__"] = {
+        "deleted": False,
+        "published": published,
+        "correlation_id": correlation_id,
+    }
 
     return document
 
@@ -94,7 +101,7 @@ def dto_to_document(
 def get_change_publish_func(
     id_field: str,
     event_topic: str,
-    dto_to_event: Callable[[Dto], JsonObject],
+    dto_to_event: Callable[[Dto], Optional[JsonObject]],
     event_publisher: EventPublisherProtocol,
     collection: AgnosticCollection,
 ) -> Callable[[Dto], Awaitable[None]]:
@@ -103,12 +110,13 @@ def get_change_publish_func(
     async def publish_change(dto: Dto) -> None:
         """Publishes a change event and marks the change as published."""
         payload = dto_to_event(dto)
-        await event_publisher.publish(
-            payload=payload,
-            type_=CHANGE_EVENT_TYPE,
-            key=getattr(dto, id_field),
-            topic=event_topic,
-        )
+        if payload is not None:
+            await event_publisher.publish(
+                payload=payload,
+                type_=CHANGE_EVENT_TYPE,
+                key=getattr(dto, id_field),
+                topic=event_topic,
+            )
 
         document = dto_to_document(dto, id_field=id_field, published=True)
         await collection.replace_one({"_id": document["_id"]}, document, upsert=True)
@@ -134,7 +142,15 @@ def get_delete_publish_func(
             topic=event_topic,
         )
 
-        document = {"_id": id_, "__metadata__": {"deleted": True, "published": True}}
+        correlation_id = get_correlation_id()  # Get active correlation first
+        document = {
+            "_id": id_,
+            "__metadata__": {
+                "deleted": True,
+                "published": True,
+                "correlation_id": correlation_id,
+            },
+        }
         await collection.replace_one({"_id": document["_id"]}, document)
 
     return publish_deletion
@@ -233,7 +249,9 @@ class MongoKafkaDaoPublisher(Generic[Dto]):
             ResourceNotFoundError:
                 when resource with the id specified in the dto was not found
         """
+        correlation_id = get_correlation_id()
         document = self._dao._dto_to_document(dto)
+        document["__metadata__.correlation_id"] = correlation_id
         result = await self._collection.replace_one(
             {"_id": document["_id"], "__metadata__.deleted": False}, document
         )
@@ -252,9 +270,14 @@ class MongoKafkaDaoPublisher(Generic[Dto]):
         Raises:
             ResourceNotFoundError: when resource with the specified id_ was not found
         """
+        correlation_id = get_correlation_id()
         document = {
             "_id": id_,
-            "__metadata__": {"deleted": True, "published": False},
+            "__metadata__": {
+                "deleted": True,
+                "published": False,
+                "correlation_id": correlation_id,
+            },
         }
         result = await self._collection.replace_one(
             {"_id": document["_id"], "__metadata__.deleted": False}, document
@@ -343,13 +366,15 @@ class MongoKafkaDaoPublisher(Generic[Dto]):
 
     async def publish_document(self, document: dict[str, Any]) -> None:
         """Publishes a document"""
-        if document.get("__metadata__", {}).get("deleted", False):
-            await self._publish_delete(document["_id"])
-        else:
-            dto = document_to_dto(
-                document, id_field=self._id_field, dto_model=self._dto_model
-            )
-            await self._publish_change(dto)
+        correlation_id = document.get("__metadata__", {}).get("correlation_id", "")
+        async with set_correlation_id(correlation_id=correlation_id):
+            if document.get("__metadata__", {}).get("deleted", False):
+                await self._publish_delete(document["_id"])
+            else:
+                dto = document_to_dto(
+                    document, id_field=self._id_field, dto_model=self._dto_model
+                )
+                await self._publish_change(dto)
 
     async def publish_pending(self) -> None:
         """Publishes all non-published changes."""
@@ -422,7 +447,7 @@ class MongoKafkaDaoPublisherFactory(DaoPublisherFactoryProtocol):
         dto_model: type[Dto],
         id_field: str,
         fields_to_index: Optional[Collection[str]],
-        dto_to_event: Callable[[Dto], JsonObject],
+        dto_to_event: Callable[[Dto], Optional[JsonObject]],
         event_topic: str,
         autopublish: bool,
     ) -> DaoPublisher[Dto]:

hexkit-3.2.0/src/hexkit/providers/mongokafka/testutils.py

@@ -0,0 +1,73 @@
+# Copyright 2021 - 2024 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
+# for the German Human Genome-Phenome Archive (GHGA)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""Utilities for testing code that uses the MongoKafkaDaoPublisherFactory provider.
+
+Please note, only use for testing purposes.
+"""
+
+from typing import NamedTuple
+
+import pytest
+
+from hexkit.providers.akafka.testutils import KafkaContainerFixture, KafkaFixture
+from hexkit.providers.mongodb.testutils import MongoDbContainerFixture, MongoDbFixture
+from hexkit.providers.mongokafka import MongoKafkaConfig
+
+__all__ = [
+    "MongoKafkaConfig",
+    "MongoKafkaFixture",
+    "mongo_kafka_config_fixture",
+    "mongo_kafka_fixture",
+]
+
+
+@pytest.fixture(name="mongo_kafka_config", scope="session")
+def mongo_kafka_config_fixture(
+    mongodb_container: MongoDbContainerFixture, kafka_container: KafkaContainerFixture
+) -> MongoKafkaConfig:
+    """Fixture to get the combined Mongo and Kafka configuration."""
+    return MongoKafkaConfig(
+        **mongodb_container.mongodb_config.model_dump(),
+        **kafka_container.kafka_config.model_dump(),
+    )
+
+
+class MongoKafkaFixture(NamedTuple):
+    """Combined MongoDB and Kafka fixture"""
+
+    mongodb: MongoDbFixture
+    kafka: KafkaFixture
+    config: MongoKafkaConfig
+
+
+@pytest.fixture(name="mongo_kafka", scope="function")
+def mongo_kafka_fixture(
+    mongodb: MongoDbFixture, kafka: KafkaFixture
+) -> MongoKafkaFixture:
+    """Get a MongoKafka fixture with desired scope and name.
+
+    The state of the MongoDb and Kafka is reset by emptying all MongoDB collections
+    and clearing all Kafka topics before running tests.
+
+    By default, the function scope is used for this fixture,
+    while the session scope is used for the underlying test containers.
+    """
+    config = MongoKafkaConfig(
+        **mongodb.config.model_dump(),
+        **kafka.config.model_dump(),
+    )
+    return MongoKafkaFixture(mongodb, kafka, config)

{hexkit-3.0.2 → hexkit-3.2.0}/src/hexkit/providers/s3/testutils.py

@@ -23,13 +23,14 @@ Please note, only use for testing purposes.
 
 import hashlib
 import os
-from collections.abc import Generator
+from collections.abc import AsyncGenerator, Generator
 from contextlib import contextmanager
 from pathlib import Path
 from tempfile import NamedTemporaryFile
 from typing import Optional
 
 import pytest
+import pytest_asyncio
 import requests
 from pydantic import BaseModel, SecretStr, computed_field
 from testcontainers.localstack import LocalStackContainer
@@ -38,7 +39,26 @@ from hexkit.custom_types import PytestScope
 from hexkit.protocols.objstorage import ObjectStorageProtocol, PresignedPostURL
 from hexkit.providers.s3.provider import S3Config, S3ObjectStorage
 
-LOCALSTACK_IMAGE = "localstack/localstack:0.14.5"
+__all__ = [
+    "LOCALSTACK_IMAGE",
+    "MEBIBYTE",
+    "TEST_FILE_DIR",
+    "TEST_FILE_PATHS",
+    "calc_md5",
+    "FileObject",
+    "S3Fixture",
+    "S3ContainerFixture",
+    "get_s3_container_fixture",
+    "get_clean_s3_fixture",
+    "get_persistent_s3_fixture",
+    "clean_s3_fixture",
+    "persistent_s3_fixture",
+    "s3_fixture",
+    "file_fixture",
+]
+
+
+LOCALSTACK_IMAGE = "localstack/localstack:3.4.0"
 
 TEST_FILE_DIR = Path(__file__).parent.resolve() / "test_files"
 
@@ -81,7 +101,7 @@ class FileObject(BaseModel):
 
 
 class S3Fixture:
-    """Yielded by the `s3_fixture` function"""
+    """A fixture with utility methods for tests that use S3 file storage"""
 
     def __init__(self, config: S3Config, storage: S3ObjectStorage):
         """Initialize with config."""
@@ -120,24 +140,100 @@ class S3Fixture:
             buckets.discard(bucket)
 
 
-def s3_fixture_function() -> Generator[S3Fixture, None, None]:
-    """Pytest fixture for tests depending on the S3ObjectStorage DAO.
+class S3ContainerFixture(LocalStackContainer):
+    """LocalStack test container with S3 configuration."""
+
+    s3_config: S3Config
+
+    _created_buckets: set[str]
+
+
+def _s3_container_fixture() -> Generator[S3ContainerFixture, None, None]:
+    """Fixture function for getting a running S3 test container."""
+    with S3ContainerFixture(image=LOCALSTACK_IMAGE) as s3_container:
+        s3_endpoint_url = s3_container.get_url()
+        s3_config = S3Config(  # type: ignore [call-arg]
+            s3_endpoint_url=s3_endpoint_url,
+            s3_access_key_id="test",
+            s3_secret_access_key=SecretStr("test"),
+        )
+        s3_container.s3_config = s3_config
+        s3_container._created_buckets = set()
+        yield s3_container
+
+
+def get_s3_container_fixture(
+    scope: PytestScope = "session", name: str = "s3_container"
+):
+    """Get a LocalStack test container fixture with desired scope and name.
+
+    By default, the session scope is used for LocalStack test containers.
+    """
+    return pytest.fixture(_s3_container_fixture, scope=scope, name=name)
+
+
+s3_container_fixture = get_s3_container_fixture()
+
 
-    **Do not call directly** Instead, use get_s3_fixture()
+def _persistent_s3_fixture(
+    s3_container: S3ContainerFixture,
+) -> Generator[S3Fixture, None, None]:
+    """Fixture function that gets a persistent S3 storage fixture.
+
+    The state of the S3 storage is not cleaned up by the function.
+    """
+    config = s3_container.s3_config
+    storage = S3ObjectStorage(config=config)
+    yield S3Fixture(config=config, storage=storage)
+
+
+def get_persistent_s3_fixture(scope: PytestScope = "function", name: str = "s3"):
+    """Get an S3 fixture with desired scope and name.
+
+    The state of the LocalStack test container is persisted across tests.
+
+    By default, the function scope is used for this fixture,
+    while the session scope is used for the underlying LocalStack test container.
+    """
+    return pytest.fixture(_persistent_s3_fixture, scope=scope, name=name)
+
+
+persistent_s3_fixture = get_persistent_s3_fixture()
+
+
+async def _clean_s3_fixture(
+    s3_container: S3ContainerFixture,
+) -> AsyncGenerator[S3Fixture, None]:
+    """Async fixture function that gets a clean S3 storage fixture.
+
+    The clean state is achieved by deleting all S3 buckets upfront.
     """
-    with LocalStackContainer(image=LOCALSTACK_IMAGE).with_services("s3") as localstack:
-        config = config_from_localstack_container(localstack)
+    for s3_fixture in _persistent_s3_fixture(s3_container):
+        container_buckets = s3_container._created_buckets
+        fixture_buckets = s3_fixture.storage._created_buckets
+        fixture_buckets.update(container_buckets)
+        fixture_buckets_before = fixture_buckets.copy()
+        await s3_fixture.storage.delete_created_buckets()
+        deleted_buckets = fixture_buckets_before - fixture_buckets
+        container_buckets.difference_update(deleted_buckets)
 
-        storage = S3ObjectStorage(config=config)
-        yield S3Fixture(config=config, storage=storage)
+        yield s3_fixture
 
+    container_buckets.update(fixture_buckets)
 
-def get_s3_fixture(scope: PytestScope = "function"):
-    """Produce an S3 fixture with desired scope. Default is the function scope."""
-    return pytest.fixture(s3_fixture_function, scope=scope)
+
+def get_clean_s3_fixture(scope: PytestScope = "function", name: str = "s3"):
+    """Get an S3 storage fixture with desired scope and name.
+
+    The state of the S3 storage is reset by deleting all buckets before running tests.
+
+    By default, the function scope is used for this fixture,
+    while the session scope is used for the underlying LocalStack test container.
+    """
+    return pytest_asyncio.fixture(_clean_s3_fixture, scope=scope, name=name)
 
 
-s3_fixture = get_s3_fixture()
+s3_fixture = clean_s3_fixture = get_clean_s3_fixture()
 
 
 @contextmanager
@@ -146,7 +242,7 @@ def temp_file_object(
     object_id: str = "default-test-object",
     size: int = 5 * MEBIBYTE,
 ) -> Generator[FileObject, None, None]:
-    """Generates a file object with the specified size in bytes."""
+    """Generate a file object with the specified size in bytes."""
     chunk_size = 1024
     chunk = b"\0" * chunk_size
     current_size = 0

{hexkit-3.0.2 → hexkit-3.2.0/src/hexkit.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hexkit
-Version: 3.0.2
+Version: 3.2.0
 Summary: A Toolkit for Building Microservices using the Hexagonal Architecture
 Author-email: "German Human Genome Phenome Archive (GHGA)" <contact@ghga.de>
 License: Apache 2.0
@@ -23,21 +23,21 @@ Requires-Dist: pydantic_settings<3,>=2
 Requires-Dist: PyYAML<7,>=6.0
 Provides-Extra: akafka
 Requires-Dist: aiokafka~=0.10.0; extra == "akafka"
-Requires-Dist: jsonschema<5,>=4.21; extra == "akafka"
+Requires-Dist: jsonschema<5,>=4.22; extra == "akafka"
 Provides-Extra: s3
-Requires-Dist: boto3<2,>=1.26.50; extra == "s3"
-Requires-Dist: botocore<2,>=1.29.50; extra == "s3"
+Requires-Dist: boto3<2,>=1.34.104; extra == "s3"
+Requires-Dist: botocore<2,>=1.34.104; extra == "s3"
 Provides-Extra: mongodb
-Requires-Dist: motor<4,>=3.1.1; extra == "mongodb"
+Requires-Dist: motor<4,>=3.4.0; extra == "mongodb"
 Provides-Extra: test-akafka
 Requires-Dist: hexkit[akafka]; extra == "test-akafka"
-Requires-Dist: testcontainers[kafka]<5,>=4.3.3; extra == "test-akafka"
+Requires-Dist: testcontainers[kafka]<5,>=4.4.1; extra == "test-akafka"
 Provides-Extra: test-s3
 Requires-Dist: hexkit[s3]; extra == "test-s3"
-Requires-Dist: testcontainers<5,>=4.3.3; extra == "test-s3"
+Requires-Dist: testcontainers<5,>=4.4.1; extra == "test-s3"
 Provides-Extra: test-mongodb
 Requires-Dist: hexkit[mongodb]; extra == "test-mongodb"
-Requires-Dist: testcontainers[mongo]<5,>=4.3.3; extra == "test-mongodb"
+Requires-Dist: testcontainers[mongo]<5,>=4.4.1; extra == "test-mongodb"
 Provides-Extra: test
 Requires-Dist: hexkit[test-akafka,test-mongodb,test-s3]; extra == "test"
 Provides-Extra: all

{hexkit-3.0.2 → hexkit-3.2.0}/src/hexkit.egg-info/SOURCES.txt

@@ -38,6 +38,7 @@ src/hexkit/providers/mongodb/provider.py
 src/hexkit/providers/mongodb/testutils.py
 src/hexkit/providers/mongokafka/__init__.py
 src/hexkit/providers/mongokafka/provider.py
+src/hexkit/providers/mongokafka/testutils.py
 src/hexkit/providers/s3/__init__.py
 src/hexkit/providers/s3/provider.py
 src/hexkit/providers/s3/testutils.py

{hexkit-3.0.2 → hexkit-3.2.0}/src/hexkit.egg-info/requires.txt

@@ -4,29 +4,29 @@ PyYAML<7,>=6.0
 
 [akafka]
 aiokafka~=0.10.0
-jsonschema<5,>=4.21
+jsonschema<5,>=4.22
 
 [all]
 hexkit[test]
 
 [mongodb]
-motor<4,>=3.1.1
+motor<4,>=3.4.0
 
 [s3]
-boto3<2,>=1.26.50
-botocore<2,>=1.29.50
+boto3<2,>=1.34.104
+botocore<2,>=1.34.104
 
 [test]
 hexkit[test-akafka,test-mongodb,test-s3]
 
 [test-akafka]
 hexkit[akafka]
-testcontainers[kafka]<5,>=4.3.3
+testcontainers[kafka]<5,>=4.4.1
 
 [test-mongodb]
 hexkit[mongodb]
-testcontainers[mongo]<5,>=4.3.3
+testcontainers[mongo]<5,>=4.4.1
 
 [test-s3]
 hexkit[s3]
-testcontainers<5,>=4.3.3
+testcontainers<5,>=4.4.1

hexkit-3.0.2/src/hexkit/providers/mongodb/testutils.py

@@ -1,101 +0,0 @@
-# Copyright 2021 - 2024 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
-# for the German Human Genome-Phenome Archive (GHGA)
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-"""Utilities for testing code that uses the MongoDbDaoFactory provider.
-
-Please note, only use for testing purposes.
-"""
-
-from collections.abc import Generator
-from dataclasses import dataclass
-from typing import Optional, Union
-
-import pytest
-from pymongo import MongoClient
-from pymongo.errors import ExecutionTimeout, OperationFailure
-from testcontainers.mongodb import MongoDbContainer
-
-from hexkit.custom_types import PytestScope
-from hexkit.providers.mongodb.provider import MongoDbConfig, MongoDbDaoFactory
-
-MONGODB_IMAGE = "mongo:6.0.3"
-
-
-@dataclass(frozen=True)
-class MongoDbFixture:
-    """Yielded by the `mongodb_fixture` function"""
-
-    client: MongoClient
-    config: MongoDbConfig
-    dao_factory: MongoDbDaoFactory
-
-    def empty_collections(
-        self,
-        exclude_collections: Optional[Union[str, list[str]]] = None,
-    ):
-        """Drop all mongodb collections in the database.
-
-        You can also specify collection(s) that should be excluded
-        from the operation, i.e. collections that should be kept.
-        """
-        db_name = self.config.db_name
-        if exclude_collections is None:
-            exclude_collections = []
-        if isinstance(exclude_collections, str):
-            exclude_collections = [exclude_collections]
-        excluded_collections = set(exclude_collections)
-        try:
-            collection_names = self.client[db_name].list_collection_names()
-            for collection_name in collection_names:
-                if collection_name not in excluded_collections:
-                    self.client[db_name].drop_collection(collection_name)
-        except (ExecutionTimeout, OperationFailure) as error:
-            raise RuntimeError(
-                f"Could not drop collection(s) of Mongo database {db_name}"
-            ) from error
-
-
-def config_from_mongodb_container(container: MongoDbContainer) -> MongoDbConfig:
-    """Prepares a MongoDbConfig from an instance of a MongoDbContainer container."""
-    db_connection_str = container.get_connection_url()
-    return MongoDbConfig(db_connection_str=db_connection_str, db_name="test")
-
-
-def mongodb_fixture_function() -> Generator[MongoDbFixture, None, None]:
-    """Pytest fixture for tests depending on the MongoDbDaoFactory DAO.
-
-    **Do not call directly** Instead, use get_mongodb_fixture()
-    """
-    with MongoDbContainer(image=MONGODB_IMAGE) as mongodb:
-        config = config_from_mongodb_container(mongodb)
-        dao_factory = MongoDbDaoFactory(config=config)
-        client = mongodb.get_connection_client()
-
-        yield MongoDbFixture(
-            client=client,
-            config=config,
-            dao_factory=dao_factory,
-        )
-
-        client.close()
-
-
-def get_mongodb_fixture(scope: PytestScope = "function"):
-    """Produce a MongoDb fixture with desired scope. Default is the function scope."""
-    return pytest.fixture(mongodb_fixture_function, scope=scope)
-
-
-mongodb_fixture = get_mongodb_fixture()