hexkit 4.3.1__tar.gz → 4.5.0__tar.gz

{hexkit-4.3.1/src/hexkit.egg-info → hexkit-4.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hexkit
-Version: 4.3.1
+Version: 4.5.0
 Summary: A Toolkit for Building Microservices using the Hexagonal Architecture
 Author-email: "German Human Genome Phenome Archive (GHGA)" <contact@ghga.de>
 License: Apache 2.0
@@ -31,6 +31,16 @@ Requires-Dist: boto3<2,>=1.37; extra == "s3"
 Requires-Dist: botocore<2,>=1.37; extra == "s3"
 Provides-Extra: mongodb
 Requires-Dist: motor<4,>=3.7; extra == "mongodb"
+Provides-Extra: opentelemetry
+Requires-Dist: opentelemetry-sdk<2,>=1.31.1; extra == "opentelemetry"
+Requires-Dist: opentelemetry-exporter-otlp-proto-http<2,>=1.31.1; extra == "opentelemetry"
+Requires-Dist: opentelemetry-distro>=0.52b1; extra == "opentelemetry"
+Requires-Dist: opentelemetry-instrumentation>=0.52b1; extra == "opentelemetry"
+Requires-Dist: opentelemetry-instrumentation-aiokafka>=0.52b1; extra == "opentelemetry"
+Requires-Dist: opentelemetry-instrumentation-botocore>=0.52b1; extra == "opentelemetry"
+Requires-Dist: opentelemetry-instrumentation-fastapi>=0.52b1; extra == "opentelemetry"
+Requires-Dist: opentelemetry-instrumentation-httpx>=0.52b1; extra == "opentelemetry"
+Requires-Dist: opentelemetry-instrumentation-pymongo>=0.52b1; extra == "opentelemetry"
 Provides-Extra: test-akafka
 Requires-Dist: hexkit[akafka]; extra == "test-akafka"
 Requires-Dist: testcontainers[kafka]<5,>=4.9; extra == "test-akafka"
@@ -44,6 +54,7 @@ Provides-Extra: test
 Requires-Dist: hexkit[test-akafka,test-mongodb,test-s3]; extra == "test"
 Provides-Extra: all
 Requires-Dist: hexkit[test]; extra == "all"
+Requires-Dist: hexkit[opentelemetry]; extra == "all"
 Dynamic: license-file
 
 ![tests](https://github.com/ghga-de/hexkit/actions/workflows/tests.yaml/badge.svg)

{hexkit-4.3.1 → hexkit-4.5.0}/pyproject.toml

@@ -24,10 +24,10 @@ classifiers = [
     "Intended Audience :: Developers",
 ]
 name = "hexkit"
-version = "4.3.1"
+version = "4.5.0"
 description = "A Toolkit for Building Microservices using the Hexagonal Architecture"
 dependencies = [
-    "opentelemetry-api>=1.31.1, <2",
+    "opentelemetry-api >=1.31.1, <2",
     "pydantic >=2.10, <3",
     "pydantic_settings >=2.8, <3",
     "PyYAML >=6, <7",
@@ -48,6 +48,17 @@ s3 = [
 mongodb = [
     "motor >=3.7, <4",
 ]
+opentelemetry = [
+    "opentelemetry-sdk >=1.31.1, <2",
+    "opentelemetry-exporter-otlp-proto-http >=1.31.1, <2",
+    "opentelemetry-distro >=0.52b1",
+    "opentelemetry-instrumentation >=0.52b1",
+    "opentelemetry-instrumentation-aiokafka >=0.52b1",
+    "opentelemetry-instrumentation-botocore >=0.52b1",
+    "opentelemetry-instrumentation-fastapi >=0.52b1",
+    "opentelemetry-instrumentation-httpx >=0.52b1",
+    "opentelemetry-instrumentation-pymongo >=0.52b1",
+]
 test-akafka = [
     "hexkit[akafka]",
     "testcontainers[kafka] >=4.9, <5",
@@ -65,6 +76,7 @@ test = [
 ]
 all = [
     "hexkit[test]",
+    "hexkit[opentelemetry]",
 ]
 
 [project.urls]

hexkit-4.5.0/src/hexkit/opentelemetry_setup.py

@@ -0,0 +1,63 @@
+# Copyright 2021 - 2025 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
+# for the German Human Genome-Phenome Archive (GHGA)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""OpenTelemetry specific configuration code. This is gated behind the opentelemetry extra."""
+
+import os
+from typing import Callable
+
+from opentelemetry import trace
+from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
+from opentelemetry.sdk.environment_variables import OTEL_EXPORTER_OTLP_PROTOCOL
+from opentelemetry.sdk.resources import SERVICE_NAME, Resource
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import BatchSpanProcessor
+
+
+def configure_tracer(service_name: str, protocol: str = "http/protobuf"):
+    """Set up a global tracer for a specific service using the given exporter protocol."""
+    # opentelemetry distro sets this to grpc, but in the current context http/protobuf is preferred
+    os.environ.setdefault(OTEL_EXPORTER_OTLP_PROTOCOL, protocol)
+
+    resource = Resource(attributes={SERVICE_NAME: service_name})
+    trace_provider = TracerProvider(resource=resource)
+    processor = BatchSpanProcessor(OTLPSpanExporter())
+    trace_provider.add_span_processor(processor)
+    trace.set_tracer_provider(trace_provider)
+
+
+class SpanTracer:
+    """Custom tracer class providing a decorator to autpopulate span names."""
+
+    def __init__(self, name):
+        self.tracer = trace.get_tracer(name)
+
+    def start_span(
+        self, *, record_exception: bool = False, set_status_on_exception: bool = False
+    ):
+        """Decorator function starting a span populated with the __qualname__ of the wrapped function"""
+
+        def outer_wrapper(function: Callable):
+            def traced_function(*args, **kwargs):
+                name = function.__qualname__
+                with self.tracer.start_as_current_span(
+                    name,
+                    record_exception=record_exception,
+                    set_status_on_exception=set_status_on_exception,
+                ):
+                    return function(*args, **kwargs)
+
+            return traced_function
+
+        return outer_wrapper

{hexkit-4.3.1 → hexkit-4.5.0}/src/hexkit/providers/akafka/testutils.py

@@ -19,12 +19,15 @@
 Please note, only use for testing purposes.
 """
 
+import asyncio
 import json
 from collections.abc import AsyncGenerator, Generator, Mapping, Sequence
 from contextlib import asynccontextmanager
 from dataclasses import dataclass
 from functools import partial
-from typing import Any, Optional, Union
+from typing import Any, Optional, Union, cast
+
+from aiokafka.admin.config_resource import ConfigResource, ConfigResourceType
 
 try:
     from typing import Self
@@ -411,6 +414,7 @@ class KafkaFixture:
         self.config = config
         self.kafka_servers = kafka_servers
         self.publisher = publisher
+        self.admin_client: Optional[AIOKafkaAdminClient] = None
 
     async def publish_event(
         self,
@@ -442,6 +446,80 @@ class KafkaFixture:
             capture_headers=capture_headers,
         )
 
+    @asynccontextmanager
+    async def get_admin_client(self):
+        """An async context manager that provides a running AIOKafkaAdminClient
+        as a singleton instance.
+
+        If a running client already exists, it is yielded directly.
+        If not, this is considered the root use of the CM, so we start a new client,
+        yield it, then close the client within a `finally` block.
+        """
+        if self.admin_client:
+            yield self.admin_client
+            return
+
+        # No running client exists, so create one and make sure to clean up afterward.
+        self.admin_client = AIOKafkaAdminClient(bootstrap_servers=self.kafka_servers)
+        await self.admin_client.start()
+        try:
+            yield self.admin_client
+        finally:
+            await self.admin_client.close()
+            self.admin_client = None
+
+    async def get_cleanup_policy(self, *, topic: str) -> Optional[str]:
+        """Get the current cleanup policy for a topic or None if the topic doesn't exist"""
+        return await self.get_topic_config(topic=topic, config_name="cleanup.policy")
+
+    async def get_topic_config(self, *, topic: str, config_name: str) -> Optional[str]:
+        """Get the current config for a topic or None if the topic/config doesn't exist"""
+        async with self.get_admin_client() as admin_client:
+            # fetch a list containing a response for each requested topic
+            config_list = await admin_client.describe_configs(
+                [ConfigResource(ConfigResourceType.TOPIC, name=topic)]
+            )
+
+        # We only requested one topic, so get the first/only element from the list
+        config_response = config_list[0]
+
+        # Each config response has a 'resources' attribute
+        resources = config_response.resources
+
+        # The resources attr is a list containing one tuple for each requested topic.
+        #  We only requested one topic, so our target is the first element.
+        requested_resource = resources[0]
+
+        # The resource itself is a list, and the topic config is at the end
+        configs = requested_resource[-1]
+        value = None
+        config_name = config_name.lower()
+
+        # The `configs` item is a list of tuples, where each tuple is a config item
+        # The first item is the config name, the second is the value, and the rest are ancillary
+        for item in configs:
+            if item[0].lower() == config_name:
+                value = item[1]
+                break
+        return value
+
+    async def set_cleanup_policy(self, *, topic: str, policy: str):
+        """Set the cleanup policy for the given topic. The topic must already exist."""
+        await self.set_topic_config(topic=topic, config={"cleanup.policy": policy})
+
+    async def set_topic_config(self, *, topic: str, config: dict[str, Any]):
+        """Set an arbitrary config value(s) for a topic"""
+        async with self.get_admin_client() as admin_client:
+            await admin_client.alter_configs(
+                config_resources=[
+                    ConfigResource(
+                        ConfigResourceType.TOPIC,
+                        name=topic,
+                        configs=config,
+                    )
+                ]
+            )
+
     async def clear_topics(
         self,
         *,
@@ -453,26 +531,49 @@ class KafkaFixture:
         If no topics are specified, all topics will be cleared, except internal topics
         unless otherwise specified.
         """
-        admin_client = AIOKafkaAdminClient(bootstrap_servers=self.kafka_servers)
-        await admin_client.start()
-        try:
-            if topics is None:
-                topics = await admin_client.list_topics()
+        async with self.get_admin_client() as admin_client:
+            available_topics = await admin_client.list_topics()
+            if not topics:
+                topics = available_topics
             elif isinstance(topics, str):
                 topics = [topics]
+            topics = cast(list, topics)
             if exclude_internal:
                 topics = [topic for topic in topics if not topic.startswith("__")]
+
+            # Filter out any topics that don't exist
+            topics = [topic for topic in topics if topic in available_topics]
+            if not topics:
+                return
+
+            # Record the current cleanup policies before modifying them
             topics_info = await admin_client.describe_topics(topics)
-            records_to_delete = {
-                TopicPartition(
-                    topic=topic_info["topic"], partition=partition_info["partition"]
-                ): RecordsToDelete(before_offset=-1)
-                for topic_info in topics_info
-                for partition_info in topic_info["partitions"]
-            }
-            await admin_client.delete_records(records_to_delete, timeout_ms=10000)
-        finally:
-            await admin_client.close()
+            for topic_info in topics_info:
+                topic = topic_info["topic"]
+                original_policy = await self.get_cleanup_policy(topic=topic)
+                if original_policy and "compact" in original_policy.split(","):
+                    await self.set_cleanup_policy(topic=topic, policy="delete")
+                    await asyncio.sleep(0.2)  # brief pause to wait for policy changes
+
+                records_to_delete = {
+                    TopicPartition(
+                        topic=topic, partition=partition_info["partition"]
+                    ): RecordsToDelete(before_offset=-1)
+                    for partition_info in topic_info["partitions"]
+                }
+
+                # Delete any events in the topic
+                try:
+                    await admin_client.delete_records(
+                        records_to_delete, timeout_ms=10000
+                    )
+                finally:
+                    # Always set compacted topics back to their original policy
+                    if original_policy:
+                        await self.set_cleanup_policy(
+                            topic=topic,
+                            policy=original_policy,
+                        )
 
     @asynccontextmanager
     async def expect_events(

{hexkit-4.3.1 → hexkit-4.5.0}/src/hexkit/providers/s3/provider.py

@@ -409,20 +409,26 @@ class S3ObjectStorage(ObjectStorageProtocol):
 
         (S3 allows multiple ongoing multi-part uploads.)
         """
+        uploads = []
         try:
-            uploads_info = await asyncio.to_thread(
-                self._client.list_multipart_uploads,
+            response_iter = await asyncio.to_thread(
+                self._client.get_paginator("list_multipart_uploads").paginate,
                 Bucket=bucket_id,
             )
+            for response_page in response_iter:
+                uploads.extend(
+                    [
+                        upload["UploadId"]
+                        for upload in response_page.get("Uploads", [])
+                        if upload["Key"] == object_id
+                    ]
+                )
         except botocore.exceptions.ClientError as error:
             raise self._translate_s3_client_errors(
                 error, bucket_id=bucket_id, object_id=object_id
             ) from error
 
-        upload_list = uploads_info.get("Uploads", [])
-        return [
-            upload["UploadId"] for upload in upload_list if upload["Key"] == object_id
-        ]
+        return uploads
 
     async def _assert_no_multipart_upload(self, *, bucket_id: str, object_id: str):
         """Ensure that there are no active multi-part uploads for the given object."""

{hexkit-4.3.1 → hexkit-4.5.0/src/hexkit.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hexkit
-Version: 4.3.1
+Version: 4.5.0
 Summary: A Toolkit for Building Microservices using the Hexagonal Architecture
 Author-email: "German Human Genome Phenome Archive (GHGA)" <contact@ghga.de>
 License: Apache 2.0
@@ -31,6 +31,16 @@ Requires-Dist: boto3<2,>=1.37; extra == "s3"
 Requires-Dist: botocore<2,>=1.37; extra == "s3"
 Provides-Extra: mongodb
 Requires-Dist: motor<4,>=3.7; extra == "mongodb"
+Provides-Extra: opentelemetry
+Requires-Dist: opentelemetry-sdk<2,>=1.31.1; extra == "opentelemetry"
+Requires-Dist: opentelemetry-exporter-otlp-proto-http<2,>=1.31.1; extra == "opentelemetry"
+Requires-Dist: opentelemetry-distro>=0.52b1; extra == "opentelemetry"
+Requires-Dist: opentelemetry-instrumentation>=0.52b1; extra == "opentelemetry"
+Requires-Dist: opentelemetry-instrumentation-aiokafka>=0.52b1; extra == "opentelemetry"
+Requires-Dist: opentelemetry-instrumentation-botocore>=0.52b1; extra == "opentelemetry"
+Requires-Dist: opentelemetry-instrumentation-fastapi>=0.52b1; extra == "opentelemetry"
+Requires-Dist: opentelemetry-instrumentation-httpx>=0.52b1; extra == "opentelemetry"
+Requires-Dist: opentelemetry-instrumentation-pymongo>=0.52b1; extra == "opentelemetry"
 Provides-Extra: test-akafka
 Requires-Dist: hexkit[akafka]; extra == "test-akafka"
 Requires-Dist: testcontainers[kafka]<5,>=4.9; extra == "test-akafka"
@@ -44,6 +54,7 @@ Provides-Extra: test
 Requires-Dist: hexkit[test-akafka,test-mongodb,test-s3]; extra == "test"
 Provides-Extra: all
 Requires-Dist: hexkit[test]; extra == "all"
+Requires-Dist: hexkit[opentelemetry]; extra == "all"
 Dynamic: license-file
 
 ![tests](https://github.com/ghga-de/hexkit/actions/workflows/tests.yaml/badge.svg)

{hexkit-4.3.1 → hexkit-4.5.0}/src/hexkit.egg-info/SOURCES.txt

@@ -9,6 +9,7 @@ src/hexkit/config.py
 src/hexkit/correlation.py
 src/hexkit/custom_types.py
 src/hexkit/log.py
+src/hexkit/opentelemetry_setup.py
 src/hexkit/py.typed
 src/hexkit/utils.py
 src/hexkit.egg-info/PKG-INFO

{hexkit-4.3.1 → hexkit-4.5.0}/src/hexkit.egg-info/requires.txt

@@ -9,10 +9,22 @@ jsonschema<5,>=4.23
 
 [all]
 hexkit[test]
+hexkit[opentelemetry]
 
 [mongodb]
 motor<4,>=3.7
 
+[opentelemetry]
+opentelemetry-sdk<2,>=1.31.1
+opentelemetry-exporter-otlp-proto-http<2,>=1.31.1
+opentelemetry-distro>=0.52b1
+opentelemetry-instrumentation>=0.52b1
+opentelemetry-instrumentation-aiokafka>=0.52b1
+opentelemetry-instrumentation-botocore>=0.52b1
+opentelemetry-instrumentation-fastapi>=0.52b1
+opentelemetry-instrumentation-httpx>=0.52b1
+opentelemetry-instrumentation-pymongo>=0.52b1
+
 [s3]
 boto3<2,>=1.37
 botocore<2,>=1.37