hexkit 2.1.0__tar.gz → 2.2.0__tar.gz

{hexkit-2.1.0/src/hexkit.egg-info → hexkit-2.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hexkit
-Version: 2.1.0
+Version: 2.2.0
 Summary: A Toolkit for Building Microservices using the Hexagonal Architecture
 Author-email: "German Human Genome Phenome Archive (GHGA)" <contact@ghga.de>
 License: Apache 2.0
@@ -30,13 +30,13 @@ Provides-Extra: mongodb
 Requires-Dist: motor<4,>=3.1.1; extra == "mongodb"
 Provides-Extra: test-akafka
 Requires-Dist: hexkit[akafka]; extra == "test-akafka"
-Requires-Dist: testcontainers[kafka]<4,>=3.7.1; extra == "test-akafka"
+Requires-Dist: testcontainers[kafka]<5,>=4.3.3; extra == "test-akafka"
 Provides-Extra: test-s3
 Requires-Dist: hexkit[s3]; extra == "test-s3"
-Requires-Dist: testcontainers<4,>=3.7.1; extra == "test-s3"
+Requires-Dist: testcontainers<5,>=4.3.3; extra == "test-s3"
 Provides-Extra: test-mongodb
 Requires-Dist: hexkit[mongodb]; extra == "test-mongodb"
-Requires-Dist: testcontainers[mongo]<4,>=3.7.1; extra == "test-mongodb"
+Requires-Dist: testcontainers[mongo]<5,>=4.3.3; extra == "test-mongodb"
 Provides-Extra: test
 Requires-Dist: hexkit[test-akafka,test-mongodb,test-s3]; extra == "test"
 Provides-Extra: all

{hexkit-2.1.0 → hexkit-2.2.0}/pyproject.toml

@@ -22,7 +22,7 @@ classifiers = [
     "Intended Audience :: Developers",
 ]
 name = "hexkit"
-version = "2.1.0"
+version = "2.2.0"
 description = "A Toolkit for Building Microservices using the Hexagonal Architecture"
 dependencies = [
     "pydantic >=2, <3",
@@ -47,15 +47,15 @@ mongodb = [
 ]
 test-akafka = [
     "hexkit[akafka]",
-    "testcontainers[kafka] >=3.7.1, <4",
+    "testcontainers[kafka] >=4.3.3, <5",
 ]
 test-s3 = [
     "hexkit[s3]",
-    "testcontainers >=3.7.1, <4",
+    "testcontainers >=4.3.3, <5",
 ]
 test-mongodb = [
     "hexkit[mongodb]",
-    "testcontainers[mongo] >=3.7.1, <4",
+    "testcontainers[mongo] >=4.3.3, <5",
 ]
 test = [
     "hexkit[test-akafka,test-s3,test-mongodb]",
@@ -80,6 +80,21 @@ exclude = [
     "build",
     "dist",
 ]
+line-length = 88
+src = [
+    "src",
+    "tests",
+    "examples",
+    "scripts",
+]
+target-version = "py39"
+
+[tool.ruff.lint]
+fixable = [
+    "UP",
+    "I",
+    "D",
+]
 ignore = [
     "E",
     "W",
@@ -97,7 +112,6 @@ ignore = [
     "D206",
     "D300",
 ]
-line-length = 88
 select = [
     "C90",
     "F",
@@ -111,23 +125,11 @@ select = [
     "SIM",
     "D",
 ]
-fixable = [
-    "UP",
-    "I",
-    "D",
-]
-src = [
-    "src",
-    "tests",
-    "examples",
-    "scripts",
-]
-target-version = "py39"
 
-[tool.ruff.mccabe]
+[tool.ruff.lint.mccabe]
 max-complexity = 10
 
-[tool.ruff.per-file-ignores]
+[tool.ruff.lint.per-file-ignores]
 "scripts/*" = [
     "PL",
     "S",
@@ -153,7 +155,7 @@ max-complexity = 10
     "D",
 ]
 
-[tool.ruff.pydocstyle]
+[tool.ruff.lint.pydocstyle]
 convention = "pep257"
 
 [tool.mypy]

{hexkit-2.1.0 → hexkit-2.2.0}/src/hexkit/protocols/dao.py

@@ -268,10 +268,8 @@ async def uuid4_id_generator() -> AsyncGenerator[str, None]:
         yield str(uuid4())
 
 
-class DaoFactoryProtocol(ABC):
-    """A protocol describing a factory to produce Data Access Objects (DAO) objects
-    that are enclosed in transactional scopes.
-    """
+class DaoFactoryBase:
+    """A base for Data Access Objects (DAO) Factory protocols."""
 
     class IdFieldNotFoundError(ValueError):
         """Raised when the dto_model did not contain the expected id_field."""
@@ -335,6 +333,32 @@ class DaoFactoryProtocol(ABC):
                 f"Provided index fields are invalid: {error}"
             ) from error
 
+    @classmethod
+    def _validate(
+        cls,
+        *,
+        dto_model: type[Dto],
+        id_field: str,
+        dto_creation_model: Optional[type[DtoCreation]],
+        fields_to_index: Optional[Collection[str]],
+    ) -> None:
+        """Validates the input parameters of the get_dao method."""
+        cls._validate_dto_model_id(dto_model=dto_model, id_field=id_field)
+        cls._validate_dto_creation_model(
+            dto_model=dto_model,
+            dto_creation_model=dto_creation_model,
+            id_field=id_field,
+        )
+        cls._validate_fields_to_index(
+            dto_model=dto_model, fields_to_index=fields_to_index
+        )
+
+
+class DaoFactoryProtocol(DaoFactoryBase, ABC):
+    """A protocol describing a factory to produce Data Access Objects (DAO) objects
+    that are enclosed in transactional scopes.
+    """
+
     @overload
     async def get_dao(
         self,
@@ -344,8 +368,7 @@ class DaoFactoryProtocol(ABC):
         id_field: str,
         fields_to_index: Optional[Collection[str]] = None,
         id_generator: Optional[AsyncGenerator[str, None]] = None,
-    ) -> DaoNaturalId[Dto]:
-        ...
+    ) -> DaoNaturalId[Dto]: ...
 
     @overload
     async def get_dao(
@@ -357,8 +380,7 @@ class DaoFactoryProtocol(ABC):
         dto_creation_model: type[DtoCreation],
         fields_to_index: Optional[Collection[str]] = None,
         id_generator: Optional[AsyncGenerator[str, None]] = None,
-    ) -> DaoSurrogateId[Dto, DtoCreation]:
-        ...
+    ) -> DaoSurrogateId[Dto, DtoCreation]: ...
 
     async def get_dao(
         self,
@@ -408,16 +430,11 @@ class DaoFactoryProtocol(ABC):
             self.IdFieldNotFoundError:
                 Raised when the dto_model did not contain the expected id_field.
         """
-        self._validate_dto_model_id(dto_model=dto_model, id_field=id_field)
-
-        self._validate_dto_creation_model(
+        self._validate(
             dto_model=dto_model,
-            dto_creation_model=dto_creation_model,
             id_field=id_field,
-        )
-
-        self._validate_fields_to_index(
-            dto_model=dto_model, fields_to_index=fields_to_index
+            dto_creation_model=dto_creation_model,
+            fields_to_index=fields_to_index,
         )
 
         if id_generator is None:
@@ -430,7 +447,6 @@ class DaoFactoryProtocol(ABC):
             id_field=id_field,
             fields_to_index=fields_to_index,
             dto_creation_model=dto_creation_model,
-            # (above behavior by mypy seems incorrect)
             id_generator=id_generator,
         )
 
@@ -445,8 +461,7 @@ class DaoFactoryProtocol(ABC):
         dto_creation_model: None,
         fields_to_index: Optional[Collection[str]],
         id_generator: AsyncGenerator[str, None],
-    ) -> DaoNaturalId[Dto]:
-        ...
+    ) -> DaoNaturalId[Dto]: ...
 
     @overload
     @abstractmethod
@@ -459,8 +474,7 @@ class DaoFactoryProtocol(ABC):
         dto_creation_model: type[DtoCreation],
         fields_to_index: Optional[Collection[str]],
         id_generator: AsyncGenerator[str, None],
-    ) -> DaoSurrogateId[Dto, DtoCreation]:
-        ...
+    ) -> DaoSurrogateId[Dto, DtoCreation]: ...
 
     @abstractmethod
     async def _get_dao(

hexkit-2.2.0/src/hexkit/protocols/dao_outbox.py

@@ -0,0 +1,130 @@
+# Copyright 2021 - 2023 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
+# for the German Human Genome-Phenome Archive (GHGA)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""Protocol for creating Data Access Objects to perform CRUD (plus find) interactions
+with the database plus automatically publish the changes as events using a modification
+of the outbox pattern.
+"""
+
+# ruff: noqa: PLR0913
+
+import typing
+from abc import ABC, abstractmethod
+from collections.abc import Collection
+from typing import Callable, Optional
+
+from hexkit.custom_types import JsonObject
+from hexkit.protocols.dao import (
+    DaoFactoryBase,
+    DaoNaturalId,
+    Dto,
+)
+
+
+class DaoOutbox(DaoNaturalId[Dto], typing.Protocol[Dto]):
+    """A Data Access Object (DAO) that automatically publishes changes according to the
+    outbox pattern.
+    """
+
+    async def publish_pending(self) -> None:
+        """Publishes all non-published changes."""
+        ...
+
+    async def republish(self) -> None:
+        """Republishes the state of all resources independent of whether they have
+        already been published or not.
+        """
+        ...
+
+
+class DaoOutboxFactoryProtocol(DaoFactoryBase, ABC):
+    """A protocol describing a factory to produce Data Access Objects (DAO) objects
+    which automatically publish changes according to the outbox pattern.
+    """
+
+    async def get_dao(
+        self,
+        *,
+        name: str,
+        dto_model: type[Dto],
+        id_field: str,
+        fields_to_index: Optional[Collection[str]] = None,
+        dto_to_event: Callable[[Dto], JsonObject],
+        event_topic: str,
+        autopublish: bool = True,
+    ) -> DaoOutbox[Dto]:
+        """Constructs an Outbox DAO for interacting with resources in a database.
+
+        Args:
+            name:
+                The name of the resource type (roughly equivalent to the name of a
+                database table or collection).
+            dto_model:
+                A DTO (Data Transfer Object) model describing the shape of resources.
+            id_field:
+                The name of the field of the `dto_model` that serves as resource ID.
+                (DAO implementation might use this field as primary key.)
+            fields_to_index:
+                Optionally, provide any fields that should be indexed in addition to the
+                `id_field`. Defaults to None.
+            dto_to_event:
+                A function that takes a DTO and returns the payload for an event.
+            event_topic:
+                The topic to which events should be published.
+            autopublish:
+                Whether to automatically publish changes. Defaults to True.
+
+        Returns:
+            A DAO of type DaoOutbox, which requires ID specification upon resource
+            creation.
+
+        Raises:
+            self.IdFieldNotFoundError:
+                Raised when the dto_model did not contain the expected id_field.
+        """
+        self._validate(
+            dto_model=dto_model,
+            id_field=id_field,
+            dto_creation_model=None,
+            fields_to_index=fields_to_index,
+        )
+
+        return await self._get_dao(
+            name=name,
+            dto_model=dto_model,
+            id_field=id_field,
+            fields_to_index=fields_to_index,
+            dto_to_event=dto_to_event,
+            event_topic=event_topic,
+            autopublish=autopublish,
+        )
+
+    @abstractmethod
+    async def _get_dao(
+        self,
+        *,
+        name: str,
+        dto_model: type[Dto],
+        id_field: str,
+        fields_to_index: Optional[Collection[str]],
+        dto_to_event: Callable[[Dto], JsonObject],
+        event_topic: str,
+        autopublish: bool,
+    ) -> DaoOutbox[Dto]:
+        """*To be implemented by the provider. Input validation is done outside of this
+        method.*
+        """
+        ...

{hexkit-2.1.0 → hexkit-2.2.0}/src/hexkit/providers/akafka/provider.py

@@ -29,7 +29,7 @@ from typing import Any, Callable, Literal, Optional, Protocol, TypeVar
 
 from aiokafka import AIOKafkaConsumer, AIOKafkaProducer
 from aiokafka.helpers import create_ssl_context
-from pydantic import Field
+from pydantic import Field, SecretStr
 from pydantic_settings import BaseSettings
 
 from hexkit.base import InboundProviderBase
@@ -93,7 +93,7 @@ class KafkaConfig(BaseSettings):
     kafka_ssl_keyfile: str = Field(
         default="", description="Optional filename containing the client private key."
     )
-    kafka_ssl_password: str = Field(
+    kafka_ssl_password: SecretStr = Field(
         default="",
         description="Optional password to be used for the client private key.",
     )
@@ -131,7 +131,7 @@ def generate_ssl_context(config: KafkaConfig) -> Optional[ssl.SSLContext]:
             cafile=config.kafka_ssl_cafile,
             certfile=config.kafka_ssl_certfile,
             keyfile=config.kafka_ssl_keyfile,
-            password=config.kafka_ssl_password,
+            password=config.kafka_ssl_password.get_secret_value(),
         )
         if config.kafka_security_protocol == "SSL"
         else None
@@ -259,7 +259,7 @@ class KafkaEventPublisher(EventPublisherProtocol):
                 raise
 
             correlation_id = new_correlation_id()
-            logging.info("Generated new correlation id: %s", correlation_id)
+            logging.info("Generated new correlation ID: %s", correlation_id)
 
         validate_correlation_id(correlation_id)
 
@@ -311,6 +311,7 @@ class KafkaConsumerCompatible(Protocol):
         client_id: str,
         group_id: str,
         auto_offset_reset: Literal["earliest"],
+        enable_auto_commit: bool,
         key_deserializer: Callable[[bytes], str],
         value_deserializer: Callable[[bytes], str],
     ):
@@ -334,6 +335,10 @@ class KafkaConsumerCompatible(Protocol):
         """
         ...
 
+    async def commit(self, offsets=None):
+        """Commit offsets to Kafka Broker."""
+        ...
+
     async def start(self) -> None:
         """Setup the consumer."""
         ...
@@ -390,6 +395,7 @@ class KafkaEventSubscriber(InboundProviderBase):
             client_id=client_id,
             group_id=config.service_name,
             auto_offset_reset="earliest",
+            enable_auto_commit=False,
             key_deserializer=lambda event_key: event_key.decode("ascii"),
             value_deserializer=lambda event_value: json.loads(
                 event_value.decode("ascii")
@@ -437,6 +443,8 @@ class KafkaEventSubscriber(InboundProviderBase):
             )
         except EventHeaderNotFoundError as err:
             logging.warning("Ignored an event: %s. %s", event_label, err.args[0])
+            # acknowledge event receipt
+            await self._consumer.commit()
             return
 
         if type_ in self._types_whitelist:
@@ -450,6 +458,8 @@ class KafkaEventSubscriber(InboundProviderBase):
                         type_=type_,
                         topic=event.topic,
                     )
+                    # acknowledge successfully processed event
+                    await self._consumer.commit()
             except Exception:
                 logging.error(
                     "A fatal error occurred while processing the event: %s",
@@ -459,6 +469,8 @@ class KafkaEventSubscriber(InboundProviderBase):
 
         else:
             logging.info("Ignored event of type %s: %s", type_, event_label)
+            # acknowledge event receipt
+            await self._consumer.commit()
 
     async def run(self, forever: bool = True) -> None:
         """

{hexkit-2.1.0 → hexkit-2.2.0}/src/hexkit/providers/akafka/testutils.py

@@ -18,6 +18,7 @@
 
 Please note, only use for testing purposes.
 """
+
 import json
 import warnings
 from collections.abc import AsyncGenerator, Sequence
@@ -301,6 +302,12 @@ class EventRecorder:
         consumer = self._get_consumer()
         await consumer.start()
 
+        # The number of events to consume is determined by the difference between the
+        # starting and final offsets. To avoid waiting for too many events (and thus
+        # waiting forever), seek_to_end so only events published inside the context of
+        # the event recorder are considered.
+        await consumer.seek_to_end()
+
         try:
             self._starting_offsets = await self._get_consumer_offsets(consumer=consumer)
         finally: