hexkit 3.2.2__tar.gz → 3.3.0__tar.gz

{hexkit-3.2.2/src/hexkit.egg-info → hexkit-3.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hexkit
-Version: 3.2.2
+Version: 3.3.0
 Summary: A Toolkit for Building Microservices using the Hexagonal Architecture
 Author-email: "German Human Genome Phenome Archive (GHGA)" <contact@ghga.de>
 License: Apache 2.0
@@ -22,22 +22,22 @@ Requires-Dist: pydantic<3,>=2
 Requires-Dist: pydantic_settings<3,>=2
 Requires-Dist: PyYAML<7,>=6.0
 Provides-Extra: akafka
-Requires-Dist: aiokafka~=0.10.0; extra == "akafka"
+Requires-Dist: aiokafka~=0.11.0; extra == "akafka"
 Requires-Dist: jsonschema<5,>=4.22; extra == "akafka"
 Provides-Extra: s3
-Requires-Dist: boto3<2,>=1.34.104; extra == "s3"
-Requires-Dist: botocore<2,>=1.34.104; extra == "s3"
+Requires-Dist: boto3<2,>=1.34.140; extra == "s3"
+Requires-Dist: botocore<2,>=1.34.140; extra == "s3"
 Provides-Extra: mongodb
-Requires-Dist: motor<4,>=3.4.0; extra == "mongodb"
+Requires-Dist: motor<4,>=3.5.0; extra == "mongodb"
 Provides-Extra: test-akafka
 Requires-Dist: hexkit[akafka]; extra == "test-akafka"
-Requires-Dist: testcontainers[kafka]<5,>=4.4.1; extra == "test-akafka"
+Requires-Dist: testcontainers[kafka]<5,>=4.7.1; extra == "test-akafka"
 Provides-Extra: test-s3
 Requires-Dist: hexkit[s3]; extra == "test-s3"
-Requires-Dist: testcontainers<5,>=4.4.1; extra == "test-s3"
+Requires-Dist: testcontainers<5,>=4.7.1; extra == "test-s3"
 Provides-Extra: test-mongodb
 Requires-Dist: hexkit[mongodb]; extra == "test-mongodb"
-Requires-Dist: testcontainers[mongo]<5,>=4.4.1; extra == "test-mongodb"
+Requires-Dist: testcontainers[mongo]<5,>=4.7.1; extra == "test-mongodb"
 Provides-Extra: test
 Requires-Dist: hexkit[test-akafka,test-mongodb,test-s3]; extra == "test"
 Provides-Extra: all

{hexkit-3.2.2 → hexkit-3.3.0}/pyproject.toml

@@ -23,7 +23,7 @@ classifiers = [
     "Intended Audience :: Developers",
 ]
 name = "hexkit"
-version = "3.2.2"
+version = "3.3.0"
 description = "A Toolkit for Building Microservices using the Hexagonal Architecture"
 dependencies = [
     "pydantic >=2, <3",
@@ -36,27 +36,27 @@ text = "Apache 2.0"
 
 [project.optional-dependencies]
 akafka = [
-    "aiokafka~=0.10.0",
+    "aiokafka~=0.11.0",
     "jsonschema >=4.22, <5",
 ]
 s3 = [
-    "boto3 >=1.34.104, <2",
-    "botocore >=1.34.104, <2",
+    "boto3 >=1.34.140, <2",
+    "botocore >=1.34.140, <2",
 ]
 mongodb = [
-    "motor >=3.4.0, <4",
+    "motor >=3.5.0, <4",
 ]
 test-akafka = [
     "hexkit[akafka]",
-    "testcontainers[kafka] >=4.4.1, <5",
+    "testcontainers[kafka] >=4.7.1, <5",
 ]
 test-s3 = [
     "hexkit[s3]",
-    "testcontainers >=4.4.1, <5",
+    "testcontainers >=4.7.1, <5",
 ]
 test-mongodb = [
     "hexkit[mongodb]",
-    "testcontainers[mongo] >=4.4.1, <5",
+    "testcontainers[mongo] >=4.7.1, <5",
 ]
 test = [
     "hexkit[test-akafka,test-s3,test-mongodb]",

{hexkit-3.2.2 → hexkit-3.3.0}/src/hexkit/log.py

@@ -17,10 +17,9 @@
 """Configurable logging tools with JSON output."""
 
 import json
-from collections import OrderedDict
 from datetime import datetime, timezone
 from logging import Formatter, Logger, LogRecord, StreamHandler, addLevelName, getLogger
-from typing import Literal, Optional
+from typing import Any, Literal, Optional
 
 from pydantic import Field
 from pydantic_settings import BaseSettings
@@ -38,7 +37,7 @@ __all__ = [
 
 
 DUMMY_RECORD = LogRecord("dummy", 0, "dummy", 0, None, None, None, None, None)
-RESERVED_RECORD_KEYS = DUMMY_RECORD.__dict__.keys()
+RESERVED_RECORD_KEYS = set(DUMMY_RECORD.__dict__)
 
 # Add TRACE log level
 addLevelName(5, "TRACE")
@@ -77,11 +76,20 @@ class LoggingConfig(BaseSettings):
             + " correlation_id, and details"
         ),
     )
+    log_traceback: bool = Field(
+        default=True,
+        description="Whether to include exception tracebacks in log messages.",
+    )
 
 
 class JsonFormatter(Formatter):
     """A formatter class that outputs logs in JSON format."""
 
+    def __init__(self, include_traceback=True, *args, **kwargs):
+        """Initialize the formatter."""
+        super().__init__(*args, **kwargs)
+        self._include_traceback = include_traceback
+
     def format(self, record: LogRecord) -> str:
         """Format the specified record as a JSON string.
 
@@ -94,10 +102,11 @@ class JsonFormatter(Formatter):
             - correlation_id: The correlation ID, if set, from the current context.
             - message: The message that was logged, formatted with any arguments.
             - details: Any additional values included at time of logging.
+            - exception: If an exception was logged, the exception info.
         """
         # Create a log record dictionary
         log_record = record.__dict__
-        output: OrderedDict[str, str] = OrderedDict()
+        output: dict[str, Any] = dict()
 
         output["timestamp"] = log_record["timestamp"]
         output["service"] = log_record["service"]
@@ -108,6 +117,18 @@ class JsonFormatter(Formatter):
         output["message"] = record.getMessage()  # construct msg str with any args
         output["details"] = log_record["details"]
 
+        if log_record["exc_info"]:
+            exc_type, exc_value = log_record["exc_info"][:2]
+            exception = {
+                "type": exc_type.__name__,
+                "message": str(exc_value),
+            }
+            if self._include_traceback:
+                exc_text = log_record["exc_text"]
+                if exc_text:
+                    exception["traceback"] = exc_text
+            output["exception"] = exception
+
         # Convert to JSON string
         return json.dumps(output)
 
@@ -149,7 +170,11 @@ def configure_logging(*, config: LoggingConfig, logger: Optional[Logger] = None)
     logger as well. Will also log the complete configuration of the service with
     secret values hidden, if it's passed to this function and inherits from LoggingConfig.
     """
-    formatter = Formatter(config.log_format) if config.log_format else JsonFormatter()
+    formatter = (
+        Formatter(config.log_format)
+        if config.log_format
+        else JsonFormatter(include_traceback=config.log_traceback)
+    )
 
     handler = RecordCompiler(config=config)
     handler.setLevel(config.log_level)

{hexkit-3.2.2 → hexkit-3.3.0}/src/hexkit/providers/akafka/provider/eventsub.py

@@ -124,7 +124,7 @@ class KafkaConsumerCompatible(Protocol):
         ...
 
     def __aiter__(self: KCC) -> KCC:
-        """Returns an async iterator for iterating through events."""  #
+        """Returns an async iterator for iterating through events."""
         ...
 
     async def __anext__(self) -> ConsumerEvent:

{hexkit-3.2.2 → hexkit-3.3.0}/src/hexkit/providers/akafka/testutils.py

@@ -24,7 +24,7 @@ from collections.abc import AsyncGenerator, Generator, Sequence
 from contextlib import asynccontextmanager
 from dataclasses import dataclass
 from functools import partial
-from typing import Any, Callable, Optional, Union
+from typing import Any, Optional, Union
 
 try:
     from typing import Self
@@ -35,7 +35,7 @@ except ImportError:  # Python < 3.11
 import pytest
 import pytest_asyncio
 from aiokafka import AIOKafkaConsumer, TopicPartition
-from aiokafka.admin import AIOKafkaAdminClient
+from aiokafka.admin import AIOKafkaAdminClient, RecordsToDelete
 from testcontainers.kafka import KafkaContainer
 
 from hexkit.custom_types import Ascii, JsonObject, PytestScope
@@ -83,16 +83,26 @@ class ExpectedEvent(EventBase):
 
     Please note, the key type is optional. If it is set to `None` (the default), the
     event key will be ignored when compared to the recording.
+
+    The `headers` value is treated the same as `key` -- it is ignored in comparison if
+    it is set to `None`.
     """
 
     key: Optional[Ascii] = None
+    headers: Optional[dict[str, str]] = None
 
 
 @dataclass(frozen=True)
 class RecordedEvent(EventBase):
-    """Used by the EventRecorder class to describe events recorded in a specific topic."""
+    """Used by the EventRecorder class to describe events recorded in a specific topic.
+
+    The event type information is stored in the header but extracted into its own field
+    because of its importance for the comparison. Because of this, it is also removed
+    from the headers.
+    """
 
     key: Ascii
+    headers: Optional[dict[str, str]] = None
 
 
 class ValidationError(RuntimeError):
@@ -134,11 +144,11 @@ def check_recorded_events(
         expected_events=expected_events,
     )
 
-    n_recorded_events = len(recorded_events)
-    n_expected_events = len(expected_events)
-    if n_recorded_events != n_expected_events:
+    n_recorded = len(recorded_events)
+    n_expected = len(expected_events)
+    if n_recorded != n_expected:
         raise get_detailed_error(
-            details=f"expected {n_expected_events} events but recorded {n_recorded_events}"
+            details=f"expected {n_expected} events but recorded {n_recorded}"
         )
 
     def get_field_mismatch_error(field, index):
@@ -156,6 +166,11 @@ def check_recorded_events(
             raise get_field_mismatch_error(field="type", index=index)
         if expected_event.key is not None and recorded_event.key != expected_event.key:
             raise get_field_mismatch_error(field="key", index=index)
+        if (
+            expected_event.headers is not None
+            and recorded_event.headers != expected_event.headers
+        ):
+            raise get_field_mismatch_error(field="headers", index=index)
 
 
 class EventRecorder:
@@ -186,10 +201,16 @@ class EventRecorder:
         *,
         kafka_servers: list[str],
         topic: Ascii,
+        capture_headers: bool = False,
     ):
-        """Initialize with connection details."""
+        """Initialize with connection details.
+
+        Set `capture_headers` to `True` if you want to capture the headers of the
+        recorded events, otherwise they will be left empty.
+        """
         self._kafka_servers = kafka_servers
         self._topic = topic
+        self._capture_headers = capture_headers
 
         self._starting_offsets: Optional[dict[str, int]] = None
         self._recorded_events: Optional[Sequence[RecordedEvent]] = None
@@ -302,19 +323,24 @@ class EventRecorder:
 
         # consume all the available events (but no more, as this would lead to infinite
         # waiting):
-        raw_events = [
+        raw_events: list[ConsumerEvent] = [
             await self._consume_event(consumer=consumer) for _ in range(event_count)
         ]
 
-        # discard all events that do not match the key of interest:
-        return [
-            RecordedEvent(
+        recorded_events: list[RecordedEvent] = []
+        for raw_event in raw_events:
+            headers = headers_as_dict(raw_event)
+            type_ = get_header_value("type", headers=headers)
+            del headers["type"]
+
+            recorded_event = RecordedEvent(
                 payload=raw_event.value,
-                type_=get_header_value("type", headers=headers_as_dict(raw_event)),
+                type_=type_,
                 key=raw_event.key,
+                headers=headers if self._capture_headers else None,
             )
-            for raw_event in raw_events
-        ]
+            recorded_events.append(recorded_event)
+        return recorded_events
 
     async def start_recording(self) -> None:
         """Start looking for the expected events from now on."""
@@ -374,13 +400,11 @@ class KafkaFixture:
         config: KafkaConfig,
         kafka_servers: list[str],
         publisher: KafkaEventPublisher,
-        cmd_exec_func: Callable[[str, bool], None],
     ):
         """Initialize with connection details and a ready-to-use publisher."""
         self.config = config
         self.kafka_servers = kafka_servers
         self.publisher = publisher
-        self._cmd_exec_func = cmd_exec_func
 
     async def publish_event(
         self, *, payload: JsonObject, type_: Ascii, topic: Ascii, key: Ascii = "test"
@@ -388,106 +412,73 @@ class KafkaFixture:
         """A convenience method to publish a test event."""
         await self.publisher.publish(payload=payload, type_=type_, key=key, topic=topic)
 
-    def record_events(self, *, in_topic: Ascii) -> EventRecorder:
+    def record_events(
+        self, *, in_topic: Ascii, capture_headers: bool = False
+    ) -> EventRecorder:
         """Constructs an EventRecorder object that can be used in an async with block to
         record events in the specified topic upon __aenter__ and stops the recording
         upon __aexit__.
-        """
-        return EventRecorder(kafka_servers=self.kafka_servers, topic=in_topic)
-
-    def _build_record_deletion_config(self, partitions: JsonObject) -> JsonObject:
-        """Build the config required to run the kafka-delete-records script."""
-        # The required JSON config has a schema specified as follows:
-        deletion_config: dict[str, Union[list, int]] = {
-            "partitions": [],  # {topic:str, partition: int, offset: int}
-            "version": 1,
-        }
 
-        # Add the partition offset info for each topic
-        for item in partitions:
-            for partition in item["partitions"]:  # type: ignore
-                deletion_config["partitions"].append(  # type: ignore
-                    {
-                        "topic": item["topic"],  # type: ignore
-                        "partition": partition["partition"],  # type: ignore
-                        "offset": -1,  # -1 instructs kafka to delete all records
-                    }
-                )
-
-        return deletion_config
-
-    def _build_record_deletion_command(self, delete_config: JsonObject) -> str:
-        """Build the command string used to run kafka-delete-records.
-
-        The configuration is dumped to a file with an echo command, and then the
-        delete command is called using that file.
+        Set `capture_events` to `True` if you want to capture the headers of the
+        recorded events, otherwise they will be left empty.
         """
-        file_name = "record-deletion.json"
-        json_data = json.dumps(delete_config)
-
-        # Build the two command strings that write the config file and run the deletion
-        echo_command = f"echo '{json_data}' > {file_name}"
-        deletion_command = (
-            "kafka-delete-records --bootstrap-server localhost:9092 "
-            + f"--offset-json-file {file_name}"
+        return EventRecorder(
+            kafka_servers=self.kafka_servers,
+            topic=in_topic,
+            capture_headers=capture_headers,
         )
-        return f"{echo_command} && {deletion_command}"
 
-    async def _get_topic_description(
+    async def clear_topics(
         self,
-        topics: Optional[Union[list[str], str]] = None,
+        topics: Optional[Union[str, list[str]]] = None,
         exclude_internal: bool = True,
-    ) -> JsonObject:
-        """Get a decription of the given topic(s).
+    ):
+        """Clear messages from given topic(s).
 
-        If no topics are specified, all topics will be covered, except internal topics
+        If no topics are specified, all topics will be cleared, except internal topics
         unless otherwise specified.
         """
         admin_client = AIOKafkaAdminClient(bootstrap_servers=self.kafka_servers)
         await admin_client.start()
         try:
             if topics is None:
-                # if topics is None, the admin client gets all topics
-                if exclude_internal:
-                    topics = [
-                        topic
-                        for topic in await admin_client.list_topics()
-                        if not topic.startswith("__")
-                    ]
+                topics = await admin_client.list_topics()
             elif isinstance(topics, str):
                 topics = [topics]
-            return await admin_client.describe_topics(topics)
+            if exclude_internal:
+                topics = [topic for topic in topics if not topic.startswith("__")]
+            topics_info = await admin_client.describe_topics(topics)
+            records_to_delete = {
+                TopicPartition(
+                    topic=topic_info["topic"], partition=partition_info["partition"]
+                ): RecordsToDelete(before_offset=-1)
+                for topic_info in topics_info
+                for partition_info in topic_info["partitions"]
+            }
+            await admin_client.delete_records(records_to_delete, timeout_ms=10000)
         finally:
             await admin_client.close()
 
-    async def clear_topics(
-        self,
-        topics: Optional[Union[str, list[str]]] = None,
-        exclude_internal: bool = True,
-    ):
-        """Clear messages from given topic(s).
-
-        If no topics are specified, all topics will be cleared, except internal topics
-        unless otherwise specified.
-        """
-        # Get the description of the topics to be deleted
-        partition_info = await self._get_topic_description(topics, exclude_internal)
-
-        # Build the command line and then run it in a shell
-        deletion_config = self._build_record_deletion_config(partition_info)
-        command = self._build_record_deletion_command(deletion_config)
-
-        self._cmd_exec_func(command, True)
-
     @asynccontextmanager
     async def expect_events(
-        self, events: Sequence[ExpectedEvent], *, in_topic: Ascii
+        self,
+        events: Sequence[ExpectedEvent],
+        *,
+        in_topic: Ascii,
+        capture_headers: bool = False,
     ) -> AsyncGenerator[EventRecorder, None]:
         """Can be used in an async with block to record events in the specified topic
         (on __aenter__) and check that they match the specified sequence of expected
         events (on __aexit__).
+
+        Set `capture_headers` to `True` if you want to capture the headers of the
+        recorded events, otherwise they will be left empty. Providing the headers in the
+        expected events without setting `capture_headers` to will result in a
+        ValidationError.
         """
-        async with self.record_events(in_topic=in_topic) as event_recorder:
+        async with self.record_events(
+            in_topic=in_topic, capture_headers=capture_headers
+        ) as event_recorder:
             yield event_recorder
 
         check_recorded_events(
@@ -529,7 +520,8 @@ class KafkaContainerFixture(KafkaContainer):
         exit_code, output = self.get_wrapped_container().exec_run(cmd)
 
         if exit_code != 0:
-            raise RuntimeError(f"result: {exit_code}, output: {output.decode('utf-8')}")
+            output_str = output.decode("utf-8")
+            raise RuntimeError(f"result: {exit_code}, output: {output_str}")
 
 
 def _kafka_container_fixture() -> Generator[KafkaContainerFixture, None, None]:
@@ -563,7 +555,6 @@ async def _persistent_kafka_fixture(
         kafka_fixture = KafkaFixture(
             config=config,
             kafka_servers=config.kafka_servers,
-            cmd_exec_func=kafka_container.wrapped_exec_run,
             publisher=publisher,
         )
         yield kafka_fixture

{hexkit-3.2.2 → hexkit-3.3.0}/src/hexkit/providers/mongodb/provider.py

@@ -440,7 +440,7 @@ class MongoDbDaoFactory(DaoFactoryProtocol):
         self._config = config
 
         # get a database-specific client:
-        self._client = AsyncIOMotorClient(
+        self._client: AsyncIOMotorClient = AsyncIOMotorClient(
             self._config.db_connection_str.get_secret_value()
         )
         self._db = self._client[self._config.db_name]

{hexkit-3.2.2 → hexkit-3.3.0}/src/hexkit/providers/mongokafka/provider.py

@@ -430,7 +430,7 @@ class MongoKafkaDaoPublisherFactory(DaoPublisherFactoryProtocol):
         self._config = config
 
         # get a database-specific client:
-        self._client = AsyncIOMotorClient(
+        self._client: AsyncIOMotorClient = AsyncIOMotorClient(
             self._config.db_connection_str.get_secret_value()
         )
         self._db = self._client[self._config.db_name]

{hexkit-3.2.2 → hexkit-3.3.0}/src/hexkit/providers/s3/provider.py

@@ -163,17 +163,9 @@ class S3ObjectStorage(ObjectStorageProtocol):
             config=self._advanced_config,
         )
 
-        self._created_buckets: set[str] = set()
-
     def __repr__(self) -> str:
         return f"{self.__class__.__qualname__}(config={repr(self._config)})"
 
-    async def delete_created_buckets(self):
-        """Delete all the buckets that have been created by the provider."""
-        buckets = self._created_buckets
-        while buckets:
-            await self._delete_bucket(bucket_id=buckets.pop(), delete_content=True)
-
     @staticmethod
     def _format_s3_error_code(error_code: str):
         """Format a message to describe an S3 error code."""
@@ -269,8 +261,6 @@ class S3ObjectStorage(ObjectStorageProtocol):
             raise self._translate_s3_client_errors(
                 error, bucket_id=bucket_id
             ) from error
-        else:
-            self._created_buckets.add(bucket_id)
 
     async def _delete_bucket(
         self, bucket_id: str, *, delete_content: bool = False
@@ -293,8 +283,6 @@ class S3ObjectStorage(ObjectStorageProtocol):
             raise self._translate_s3_client_errors(
                 error, bucket_id=bucket_id
             ) from error
-        else:
-            self._created_buckets.discard(bucket_id)
 
     async def _list_all_object_ids(self, *, bucket_id: str) -> list[str]:
         """Retrieve a list of IDs for all objects currently present in the specified bucket"""

hexkit-3.3.0/src/hexkit/providers/s3/testutils/__init__.py

@@ -0,0 +1,47 @@
+# Copyright 2021 - 2024 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
+# for the German Human Genome-Phenome Archive (GHGA)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""S3-related test fixtures and support functions."""
+
+from ._fixtures import (  # noqa: F401
+    LOCALSTACK_IMAGE,
+    MEBIBYTE,
+    TEST_FILE_DIR,
+    TEST_FILE_PATHS,
+    FileObject,
+    S3ContainerFixture,
+    S3Fixture,
+    calc_md5,
+    clean_s3_fixture,
+    get_clean_s3_fixture,
+    get_persistent_s3_fixture,
+    get_s3_container_fixture,
+    persistent_s3_fixture,
+    populate_storage,
+    s3_container_fixture,
+    s3_fixture,
+    temp_file_object,
+    tmp_file,
+    upload_file,
+)
+from ._typical_workflow import typical_workflow  # noqa: F401
+from ._utils import (  # noqa: F401
+    check_part_size,
+    download_and_check_test_file,
+    multipart_upload_file,
+    upload_part,
+    upload_part_of_size,
+    upload_part_via_url,
+)