hexkit 1.0.0__tar.gz → 1.2.0__tar.gz

{hexkit-1.0.0/src/hexkit.egg-info → hexkit-1.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hexkit
-Version: 1.0.0
+Version: 1.2.0
 Summary: A Toolkit for Building Microservices using the Hexagonal Architecture
 Author-email: "German Human Genome Phenome Archive (GHGA)" <contact@ghga.de>
 License: Apache 2.0
@@ -31,13 +31,13 @@ Provides-Extra: mongodb
 Requires-Dist: motor<4,>=3.1.1; extra == "mongodb"
 Provides-Extra: test-akafka
 Requires-Dist: hexkit[akafka]; extra == "test-akafka"
-Requires-Dist: testcontainers[kafka]<4,>=3.4.1; extra == "test-akafka"
+Requires-Dist: testcontainers[kafka]<4,>=3.7.1; extra == "test-akafka"
 Provides-Extra: test-s3
 Requires-Dist: hexkit[s3]; extra == "test-s3"
-Requires-Dist: testcontainers<4,>=3.4.1; extra == "test-s3"
+Requires-Dist: testcontainers<4,>=3.7.1; extra == "test-s3"
 Provides-Extra: test-mongodb
 Requires-Dist: hexkit[mongodb]; extra == "test-mongodb"
-Requires-Dist: testcontainers[mongo]<4,>=3.4.1; extra == "test-mongodb"
+Requires-Dist: testcontainers[mongo]<4,>=3.7.1; extra == "test-mongodb"
 Provides-Extra: test
 Requires-Dist: hexkit[test-akafka,test-mongodb,test-s3]; extra == "test"
 Provides-Extra: all

{hexkit-1.0.0 → hexkit-1.2.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "hexkit"
-version = "1.0.0"
+version = "1.2.0"
 description = "A Toolkit for Building Microservices using the Hexagonal Architecture"
 readme = "README.md"
 authors = [
@@ -35,9 +35,9 @@ akafka = ["aiokafka~=0.8.0", "jsonschema >=4.17.3, <5"]
 s3 = ["boto3 >=1.26.50, <2", "botocore >=1.29.50, <2"]
 mongodb = ["motor >=3.1.1, <4"]
 
-test-akafka = ["hexkit[akafka]", "testcontainers[kafka] >=3.4.1, <4"]
-test-s3 = ["hexkit[s3]", "testcontainers >=3.4.1, <4"]
-test-mongodb = ["hexkit[mongodb]", "testcontainers[mongo] >=3.4.1, <4"]
+test-akafka = ["hexkit[akafka]", "testcontainers[kafka] >=3.7.1, <4"]
+test-s3 = ["hexkit[s3]", "testcontainers >=3.7.1, <4"]
+test-mongodb = ["hexkit[mongodb]", "testcontainers[mongo] >=3.7.1, <4"]
 test = ["hexkit[test-akafka,test-s3,test-mongodb]"]
 
 all = ["hexkit[test]"]

hexkit-1.2.0/src/hexkit/correlation.py

@@ -0,0 +1,97 @@
+# Copyright 2021 - 2023 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
+# for the German Human Genome-Phenome Archive (GHGA)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""Utilities related to correlation IDs"""
+
+import logging
+from contextlib import asynccontextmanager
+from contextvars import ContextVar
+from uuid import UUID, uuid4
+
+from hexkit.utils import set_context_var
+
+log = logging.getLogger(__name__)
+
+correlation_id_var: ContextVar[str] = ContextVar("correlation_id", default="")
+
+__all__ = [
+    "set_correlation_id",
+    "get_correlation_id",
+    "new_correlation_id",
+    "validate_correlation_id",
+    "CorrelationIdContextError",
+    "InvalidCorrelationIdError",
+]
+
+
+class CorrelationIdContextError(RuntimeError):
+    """Raised when the correlation ID ContextVar is unexpectedly not set."""
+
+
+class InvalidCorrelationIdError(RuntimeError):
+    """Raised when a correlation ID fails validation."""
+
+    def __init__(self, *, correlation_id: str):
+        message = f"Invalid correlation ID found: '{correlation_id}'"
+        super().__init__(message)
+
+
+def new_correlation_id() -> str:
+    """Generates a new correlation ID."""
+    return str(uuid4())
+
+
+def validate_correlation_id(correlation_id: str):
+    """Raises an error if the correlation ID is invalid.
+
+    Raises:
+        InvalidCorrelationIdError: If the correlation ID is empty or invalid.
+    """
+    try:
+        UUID(correlation_id)
+    except ValueError as err:
+        raise InvalidCorrelationIdError(correlation_id=correlation_id) from err
+
+
+@asynccontextmanager
+async def set_correlation_id(correlation_id: str):
+    """Set the correlation ID for the life of the context.
+
+    Raises:
+        InvalidCorrelationIdError: when the correlation ID is empty or invalid.
+    """
+    validate_correlation_id(correlation_id)
+
+    async with set_context_var(correlation_id_var, correlation_id):
+        log.info("Set context correlation ID to %s", correlation_id)
+        yield
+
+
+def get_correlation_id() -> str:
+    """Get the correlation ID.
+
+    This should only be called when the correlation ID ContextVar is expected to be set.
+
+    Raises:
+        CorrelationIdContextError: when the correlation ID ContextVar is not set.
+        InvalidCorrelationIdError: when the correlation ID is invalid.
+    """
+    if not (correlation_id := correlation_id_var.get()):
+        raise CorrelationIdContextError()
+
+    validate_correlation_id(correlation_id)
+
+    return correlation_id

{hexkit-1.0.0 → hexkit-1.2.0}/src/hexkit/inject.py

@@ -46,8 +46,6 @@ __all__ = [
     "Configurator",
 ]
 
-# pylint: disable=c-extension-no-member
-
 
 class NotConstructableError(TypeError):
     """Thrown when an AsyncContextConstructable was expected but not obtained."""
@@ -60,7 +58,7 @@ class AsyncInitShutdownError(TypeError):
 
 
 def assert_async_constructable(
-    constructable: Union[type[AsyncContextConstructable], type[AsyncConstructable]]
+    constructable: Union[type[AsyncContextConstructable], type[AsyncConstructable]],
 ):
     """
     Make sure that the provided object has a callable attribute `construct`.
@@ -108,9 +106,6 @@ class AsyncConstructor(dependency_injector.providers.Resource):
 
         return resource
 
-    # This pylint error is inherited from the dependency_injector framework, other
-    # DI providers use the same basic signature:
-    # pylint: disable=keyword-arg-before-vararg
     def __init__(
         self,
         provides: Optional[

{hexkit-1.0.0 → hexkit-1.2.0}/src/hexkit/protocols/dao.py

@@ -450,7 +450,7 @@ class DaoFactoryProtocol(ABC):
 
     @overload
     @abstractmethod
-    async def _get_dao(  # pylint: disable=arguments-differ
+    async def _get_dao(
         self,
         *,
         name: str,

{hexkit-1.0.0 → hexkit-1.2.0}/src/hexkit/protocols/objstorage.py

@@ -155,7 +155,6 @@ class ObjectStorageProtocol(ABC):
             upload_id=upload_id, bucket_id=bucket_id, object_id=object_id
         )
 
-    # pylint: disable=too-many-arguments
     async def complete_multipart_upload(
         self,
         *,
@@ -367,7 +366,6 @@ class ObjectStorageProtocol(ABC):
         """
         ...
 
-    # pylint: disable=too-many-arguments
     @abstractmethod
     async def _complete_multipart_upload(
         self,

{hexkit-1.0.0 → hexkit-1.2.0}/src/hexkit/providers/akafka/provider.py

@@ -25,7 +25,7 @@ import json
 import logging
 import ssl
 from contextlib import asynccontextmanager
-from typing import Any, Callable, Optional, Protocol, TypeVar
+from typing import Any, Callable, Literal, Optional, Protocol, TypeVar
 
 from aiokafka import AIOKafkaConsumer, AIOKafkaProducer
 from aiokafka.helpers import create_ssl_context
@@ -33,15 +33,17 @@ from pydantic import Field
 from pydantic_settings import BaseSettings
 
 from hexkit.base import InboundProviderBase
+from hexkit.correlation import (
+    CorrelationIdContextError,
+    get_correlation_id,
+    new_correlation_id,
+    set_correlation_id,
+    validate_correlation_id,
+)
 from hexkit.custom_types import Ascii, JsonObject
 from hexkit.protocols.eventpub import EventPublisherProtocol
 from hexkit.protocols.eventsub import EventSubscriberProtocol
 
-try:  # workaround for https://github.com/pydantic/pydantic/issues/5821
-    from typing_extensions import Literal
-except ImportError:
-    from typing import Literal  # type: ignore
-
 __all__ = [
     "KafkaConfig",
     "KafkaEventPublisher",
@@ -73,32 +75,45 @@ class KafkaConfig(BaseSettings):
         description="A list of connection strings to connect to Kafka bootstrap servers.",
     )
     kafka_security_protocol: Literal["PLAINTEXT", "SSL"] = Field(
-        "PLAINTEXT",
+        default="PLAINTEXT",
         description="Protocol used to communicate with brokers. "
         + "Valid values are: PLAINTEXT, SSL.",
     )
     kafka_ssl_cafile: str = Field(
-        "",
+        default="",
         description="Certificate Authority file path containing certificates"
-        + " used to sign broker certificates. If a CA not specified, the default"
+        + " used to sign broker certificates. If a CA is not specified, the default"
         + " system CA will be used if found by OpenSSL.",
     )
     kafka_ssl_certfile: str = Field(
-        "",
+        default="",
         description="Optional filename of client certificate, as well as any"
         + " CA certificates needed to establish the certificate's authenticity.",
     )
     kafka_ssl_keyfile: str = Field(
-        "", description="Optional filename containing the client private key."
+        default="", description="Optional filename containing the client private key."
     )
     kafka_ssl_password: str = Field(
-        "",
+        default="",
         description="Optional password to be used for the client private key.",
     )
+    generate_correlation_id: bool = Field(
+        default=True,
+        examples=[True, False],
+        description=(
+            "A flag, which, if False, will result in an error when trying to publish an"
+            + " event without a valid correlation ID set for the context. If True, the a"
+            + " newly correlation ID will be generated and used in the event header."
+        ),
+    )
 
 
-class EventTypeNotFoundError(RuntimeError):
-    """Thrown when no `type` was set in the headers of an event."""
+class EventHeaderNotFoundError(RuntimeError):
+    """Thrown when a given detail was not set in the headers of an event."""
+
+    def __init__(self, *, header_name):
+        message = f"No '{header_name}' was set in event header."
+        super().__init__(message)
 
 
 def generate_client_id(*, service_name: str, instance_id: str) -> str:
@@ -198,9 +213,13 @@ class KafkaEventPublisher(EventPublisherProtocol):
                 "ascii"
             ),
         )
+
         try:
             await producer.start()
-            yield cls(producer=producer)
+            yield cls(
+                producer=producer,
+                generate_correlation_id=config.generate_correlation_id,
+            )
         finally:
             await producer.stop()
 
@@ -208,13 +227,19 @@ class KafkaEventPublisher(EventPublisherProtocol):
         self,
         *,
         producer: KafkaProducerCompatible,
+        generate_correlation_id: bool,
     ):
         """Please do not call directly! Should be called by the `construct` method.
         Args:
             producer:
                 hands over a started AIOKafkaProducer.
+            generate_correlation_id:
+                whether to throw an error or generate a new ID if no correlation ID is
+                set for the context when trying to publish an event. An invalid ID will
+                result in an error in either case.
         """
         self._producer = producer
+        self._generate_correlation_id = generate_correlation_id
 
     async def _publish_validated(
         self, *, payload: JsonObject, type_: Ascii, key: Ascii, topic: Ascii
@@ -227,8 +252,21 @@ class KafkaEventPublisher(EventPublisherProtocol):
             key (str): The event type. ASCII characters only.
             topic (str): The event type. ASCII characters only.
         """
-        event_headers = [("type", type_.encode("ascii"))]
+        try:
+            correlation_id = get_correlation_id()
+        except CorrelationIdContextError:
+            if not self._generate_correlation_id:
+                raise
+
+            correlation_id = new_correlation_id()
+            logging.info("Generated new correlation id: %s", correlation_id)
+
+        validate_correlation_id(correlation_id)
 
+        event_headers = [
+            ("type", type_.encode("ascii")),
+            ("correlation_id", correlation_id.encode("ascii")),
+        ]
         await self._producer.send_and_wait(
             topic, key=key, value=payload, headers=event_headers
         )
@@ -245,12 +283,17 @@ class ConsumerEvent(Protocol):
     offset: int
 
 
-def get_event_type(event: ConsumerEvent) -> str:
-    """Extract the event type out of an ConsumerEvent."""
-    for header in event.headers:
-        if header[0] == "type":
-            return header[1].decode("ascii")
-    raise EventTypeNotFoundError()
+def headers_as_dict(event: ConsumerEvent) -> dict[str, str]:
+    """Extract the headers from a ConsumerEvent object and return them as a dict."""
+    return {name: value.decode("ascii") for name, value in event.headers}
+
+
+def get_header_value(header_name: str, headers: dict[str, str]) -> str:
+    """Extract the given value from the dict headers and raise an error if not found."""
+    try:
+        return headers[header_name]
+    except KeyError as err:
+        raise EventHeaderNotFoundError(header_name=header_name) from err
 
 
 KCC = TypeVar("KCC")
@@ -358,8 +401,6 @@ class KafkaEventSubscriber(InboundProviderBase):
         finally:
             await consumer.stop()
 
-    # pylint: disable=too-many-arguments
-    # (some arguments are only used for testing)
     def __init__(
         self, *, consumer: KafkaConsumerCompatible, translator: EventSubscriberProtocol
     ):
@@ -387,21 +428,28 @@ class KafkaEventSubscriber(InboundProviderBase):
     async def _consume_event(self, event: ConsumerEvent) -> None:
         """Consume an event by passing it down to the translator via the protocol."""
         event_label = self._get_event_label(event)
+        headers = headers_as_dict(event)
 
         try:
-            type_ = get_event_type(event)
-        except EventTypeNotFoundError:
-            logging.warning("Ignored an event without type: %s", event_label)
+            type_ = get_header_value(header_name="type", headers=headers)
+            correlation_id = get_header_value(
+                header_name="correlation_id", headers=headers
+            )
+        except EventHeaderNotFoundError as err:
+            logging.warning("Ignored an event: %s. %s", event_label, err.args[0])
             return
 
         if type_ in self._types_whitelist:
             logging.info('Consuming event of type "%s": %s', type_, event_label)
 
             try:
-                # blocks until event processing is completed:
-                await self._translator.consume(
-                    payload=event.value, type_=type_, topic=event.topic
-                )
+                async with set_correlation_id(correlation_id):
+                    # blocks until event processing is completed:
+                    await self._translator.consume(
+                        payload=event.value,
+                        type_=type_,
+                        topic=event.topic,
+                    )
             except Exception:
                 logging.error(
                     "A fatal error occurred while processing the event: %s",

{hexkit-1.0.0 → hexkit-1.2.0}/src/hexkit/providers/akafka/testutils.py

@@ -36,7 +36,8 @@ from hexkit.providers.akafka.provider import (
     ConsumerEvent,
     KafkaConfig,
     KafkaEventPublisher,
-    get_event_type,
+    get_header_value,
+    headers_as_dict,
 )
 from hexkit.providers.akafka.testcontainer import DEFAULT_IMAGE as KAFKA_IMAGE
 
@@ -282,7 +283,7 @@ class EventRecorder:
         return [
             RecordedEvent(
                 payload=raw_event.value,
-                type_=get_event_type(raw_event),
+                type_=get_header_value("type", headers=headers_as_dict(raw_event)),
                 key=raw_event.key,
             )
             for raw_event in raw_events
@@ -406,7 +407,7 @@ async def kafka_fixture_function() -> AsyncGenerator[KafkaFixture, None]:
     """
     with KafkaContainer(image=KAFKA_IMAGE) as kafka:
         kafka_servers = [kafka.get_bootstrap_server()]
-        config = KafkaConfig(  # type: ignore
+        config = KafkaConfig(
             service_name="test_publisher",
             service_instance_id="001",
             kafka_servers=kafka_servers,

{hexkit-1.0.0 → hexkit-1.2.0}/src/hexkit/providers/mongodb/provider.py

@@ -111,7 +111,7 @@ class MongoDbDaoBase(ABC, Generic[Dto]):
         Raises:
             ResourceNotFoundError: when resource with the specified id_ was not found
         """
-        document = await self._collection.find_one(  # type: ignore
+        document = await self._collection.find_one(
             {"_id": id_},
             session=self._session,
         )
@@ -406,7 +406,7 @@ class MongoDbDaoFactory(DaoFactoryProtocol):
         self._config = config
 
         # get a database-specific client:
-        self._client = AsyncIOMotorClient(
+        self._client = AsyncIOMotorClient(  # type: ignore
             self._config.db_connection_str.get_secret_value()
         )
         self._db = self._client[self._config.db_name]
@@ -428,7 +428,7 @@ class MongoDbDaoFactory(DaoFactoryProtocol):
         ...
 
     @overload
-    async def _get_dao(  # pylint: disable=arguments-differ
+    async def _get_dao(
         self,
         *,
         name: str,

{hexkit-1.0.0 → hexkit-1.2.0}/src/hexkit/providers/s3/provider.py

@@ -97,7 +97,6 @@ class S3Config(BaseSettings):
         description=(
             "Path to a config file for specifying more advanced S3 parameters."
             + " This should follow the format described here:"
-            # pylint: disable=line-too-long
             + " https://boto3.amazonaws.com/v1/documentation/api/latest/guide/configuration.html#using-a-configuration-file"
         ),
     )
@@ -114,12 +113,10 @@ def read_aws_config_ini(aws_config_ini: Path) -> botocore.config.Config:
     return botocore.config.Config(**config_profile)
 
 
-class S3ObjectStorage(
-    ObjectStorageProtocol
-):  # pylint: disable=too-many-instance-attributes
+class S3ObjectStorage(ObjectStorageProtocol):
     """S3-based provider implementing the ObjectStorageProtocol."""
 
-    def __init__(  # pylint: disable=too-many-arguments
+    def __init__(
         self,
         *,
         config: S3Config,
@@ -548,7 +545,6 @@ class S3ObjectStorage(
                 object_id=object_id,
             ) from error
 
-    # pylint: disable=too-many-arguments
     async def _check_uploaded_parts(
         self,
         *,
@@ -629,7 +625,6 @@ class S3ObjectStorage(
                     + f" first part which had only {first_part_size}.",
                 )
 
-    # pylint: disable=too-many-arguments
     async def _abort_multipart_upload(
         self,
         *,
@@ -686,7 +681,6 @@ class S3ObjectStorage(
                 upload_id=upload_id, bucket_id=bucket_id, object_id=object_id
             )
 
-    # pylint: disable=too-many-arguments
     async def _complete_multipart_upload(
         self,
         *,

{hexkit-1.0.0 → hexkit-1.2.0}/src/hexkit/providers/s3/testutils.py

@@ -200,7 +200,6 @@ def check_part_size(file_path: Path, anticipated_size: int) -> None:
         )
 
 
-# pylint: disable=too-many-arguments
 async def upload_part(
     storage_dao: ObjectStorageProtocol,
     upload_id: str,
@@ -220,7 +219,6 @@ async def upload_part(
     response.raise_for_status()
 
 
-# pylint: disable=too-many-arguments
 async def upload_part_of_size(
     storage_dao: ObjectStorageProtocol,
     upload_id: str,
@@ -372,7 +370,6 @@ async def prepare_non_completed_upload(s3_fixture_: S3Fixture):
 # This workflow is defined as a seperate function so that it can also be used
 # outside of the `tests` package e.g. to test the compliance of an S3-compatible
 # object storage implementation:
-# pylint: disable=too-many-arguments
 async def typical_workflow(
     storage_client: ObjectStorageProtocol,
     test_file_path: Path,

{hexkit-1.0.0 → hexkit-1.2.0}/src/hexkit/utils.py

@@ -16,7 +16,9 @@
 """General utilities that don't require heavy dependencies."""
 
 from collections.abc import Collection
-from typing import Optional
+from contextlib import asynccontextmanager
+from contextvars import ContextVar
+from typing import Any, Optional
 
 from pydantic import BaseModel
 
@@ -111,3 +113,14 @@ def validate_fields_in_model(
     if not fields_set.issubset(existing_fields_set):
         unexpected_fields = fields_set.difference(existing_fields_set)
         raise FieldNotInModelError(model=model, unexpected_field=unexpected_fields)
+
+
+@asynccontextmanager
+async def set_context_var(context_var: ContextVar, value: Any):
+    """An async context manager to simplify the use of ContextVars.
+
+    The value will be reset upon exiting the context.
+    """
+    token = context_var.set(value)
+    yield
+    context_var.reset(token)

{hexkit-1.0.0 → hexkit-1.2.0/src/hexkit.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hexkit
-Version: 1.0.0
+Version: 1.2.0
 Summary: A Toolkit for Building Microservices using the Hexagonal Architecture
 Author-email: "German Human Genome Phenome Archive (GHGA)" <contact@ghga.de>
 License: Apache 2.0
@@ -31,13 +31,13 @@ Provides-Extra: mongodb
 Requires-Dist: motor<4,>=3.1.1; extra == "mongodb"
 Provides-Extra: test-akafka
 Requires-Dist: hexkit[akafka]; extra == "test-akafka"
-Requires-Dist: testcontainers[kafka]<4,>=3.4.1; extra == "test-akafka"
+Requires-Dist: testcontainers[kafka]<4,>=3.7.1; extra == "test-akafka"
 Provides-Extra: test-s3
 Requires-Dist: hexkit[s3]; extra == "test-s3"
-Requires-Dist: testcontainers<4,>=3.4.1; extra == "test-s3"
+Requires-Dist: testcontainers<4,>=3.7.1; extra == "test-s3"
 Provides-Extra: test-mongodb
 Requires-Dist: hexkit[mongodb]; extra == "test-mongodb"
-Requires-Dist: testcontainers[mongo]<4,>=3.4.1; extra == "test-mongodb"
+Requires-Dist: testcontainers[mongo]<4,>=3.7.1; extra == "test-mongodb"
 Provides-Extra: test
 Requires-Dist: hexkit[test-akafka,test-mongodb,test-s3]; extra == "test"
 Provides-Extra: all

{hexkit-1.0.0 → hexkit-1.2.0}/src/hexkit.egg-info/SOURCES.txt

@@ -6,6 +6,7 @@ src/hexkit/__init__.py
 src/hexkit/__main__.py
 src/hexkit/base.py
 src/hexkit/config.py
+src/hexkit/correlation.py
 src/hexkit/custom_types.py
 src/hexkit/inject.py
 src/hexkit/utils.py

{hexkit-1.0.0 → hexkit-1.2.0}/src/hexkit.egg-info/requires.txt

@@ -22,12 +22,12 @@ hexkit[test-akafka,test-mongodb,test-s3]
 
 [test-akafka]
 hexkit[akafka]
-testcontainers[kafka]<4,>=3.4.1
+testcontainers[kafka]<4,>=3.7.1
 
 [test-mongodb]
 hexkit[mongodb]
-testcontainers[mongo]<4,>=3.4.1
+testcontainers[mongo]<4,>=3.7.1
 
 [test-s3]
 hexkit[s3]
-testcontainers<4,>=3.4.1
+testcontainers<4,>=3.7.1