hexkit 4.4.0__tar.gz → 4.6.0__tar.gz

{hexkit-4.4.0/src/hexkit.egg-info → hexkit-4.6.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hexkit
-Version: 4.4.0
+Version: 4.6.0
 Summary: A Toolkit for Building Microservices using the Hexagonal Architecture
 Author-email: "German Human Genome Phenome Archive (GHGA)" <contact@ghga.de>
 License: Apache 2.0

{hexkit-4.4.0 → hexkit-4.6.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [build-system]
 requires = [
-    "setuptools>=78.1",
+    "setuptools>=80.3",
 ]
 build-backend = "setuptools.build_meta"
 
@@ -24,7 +24,7 @@ classifiers = [
     "Intended Audience :: Developers",
 ]
 name = "hexkit"
-version = "4.4.0"
+version = "4.6.0"
 description = "A Toolkit for Building Microservices using the Hexagonal Architecture"
 dependencies = [
     "opentelemetry-api >=1.31.1, <2",

hexkit-4.6.0/src/hexkit/opentelemetry_setup.py

@@ -0,0 +1,170 @@
+# Copyright 2021 - 2025 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
+# for the German Human Genome-Phenome Archive (GHGA)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""OpenTelemetry specific configuration code. This is gated behind the opentelemetry extra."""
+
+import logging
+import os
+from typing import Annotated, Callable, Literal, Optional
+
+from opentelemetry import trace
+from opentelemetry.environment_variables import (
+    OTEL_LOGS_EXPORTER,
+    OTEL_METRICS_EXPORTER,
+    OTEL_TRACES_EXPORTER,
+)
+from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
+from opentelemetry.sdk.environment_variables import (
+    OTEL_EXPORTER_OTLP_PROTOCOL,
+    OTEL_SDK_DISABLED,
+)
+from opentelemetry.sdk.resources import SERVICE_NAME, Resource
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import BatchSpanProcessor
+from opentelemetry.sdk.trace.sampling import ParentBasedTraceIdRatio
+from pydantic import Field
+from pydantic_settings import BaseSettings
+from typing_extensions import deprecated
+
+TRACER: Optional["SpanTracer"] = None
+
+logger = logging.getLogger(__name__)
+
+
+class SpanTracer:
+    """Custom tracer class providing a decorator to autpopulate span names."""
+
+    def __init__(self, name):
+        self.tracer = trace.get_tracer(name)
+
+    def start_span(
+        self, *, record_exception: bool = False, set_status_on_exception: bool = False
+    ):
+        """Decorator function starting a span populated with the __qualname__ of the wrapped function"""
+
+        def outer_wrapper(function: Callable):
+            def traced_function(*args, **kwargs):
+                name = function.__qualname__
+                with self.tracer.start_as_current_span(
+                    name,
+                    record_exception=record_exception,
+                    set_status_on_exception=set_status_on_exception,
+                ):
+                    return function(*args, **kwargs)
+
+            return traced_function
+
+        return outer_wrapper
+
+
+class OpenTelemetryConfig(BaseSettings):
+    """OpenTelemetry specific configuration options"""
+
+    enable_opentelemetry: bool = Field(
+        default=False,
+        description="If set to true, this will run necessary setup code."
+        "If set to false, environment variables are set that should also effectively "
+        "disable autoinstrumentation.",
+    )
+    samplig_rate: Annotated[float, Field(strict=True, ge=0, le=1)] = Field(
+        default=1.0,
+        description="Determines which proportion of spans should be sampled. "
+        "A value of 1.0 means all and is equivalent to the previous behaviour. "
+        "Setting this to 0 will result in no spans being sampled, but this does not "
+        "automatically set `enable_opentelemetry` to False.",
+    )
+    protocol: Literal["grpc", "http/protobuf"] = Field(
+        default="http/protobuf",
+        description="Specifies which protocol should be used by exporters.",
+    )
+
+
+def configure_opentelemetry(*, service_name: str, config: OpenTelemetryConfig):
+    """Configure all needed parts of OpenTelemetry.
+
+    Setup of the TracerProvider is done programmatically, all other configuration exports
+    OpenTelemetry specific environment variables.
+    """
+    configure_tracer(service_name=service_name, config=config)
+
+
+@deprecated(
+    "This function will be removed in v5. Use `configure_opentelemetry instead.`"
+)
+def configure_tracer(*, service_name: str, config: OpenTelemetryConfig):
+    """Configure all needed parts of OpenTelemetry.
+
+    Setup of the TracerProvider is done programmatically, all other configuration exports
+    OpenTelemetry specific environment variables.
+    """
+    global TRACER
+    # opentelemetry distro sets this to grpc, but in the current context http/protobuf is preferred
+    os.environ[OTEL_EXPORTER_OTLP_PROTOCOL] = config.protocol
+    # Disable OpenTelemetry metrics and logs explicitly as they are not processed in the backend currently
+    # This overwrites the defaults of `otlp` set in opentelemetry distro
+    os.environ[OTEL_METRICS_EXPORTER] = "none"
+    os.environ[OTEL_LOGS_EXPORTER] = "none"
+
+    if config.enable_opentelemetry:
+        if TRACER is not None:
+            logger.warning(
+                "OpenTelemetry configuration code should only be run once. "
+                "If it has been run with a different service name than %s before, "
+                "the tracer and resource name will likely be wrong in some cases.",
+                service_name,
+            )
+        resource = Resource(attributes={SERVICE_NAME: service_name})
+        # Replace the default static sampler with a probabilistic one that honors parent
+        # span sampling decisions
+        # This should consistently yield full traces within a service but not necessarily
+        # across service boundaries
+        # With the default sampling rate, behaviour does not change, but this allows to
+        # introduce head sampling by adjusting a config option on the service side later on
+        sampler = ParentBasedTraceIdRatio(rate=config.samplig_rate)
+
+        # Initialize service specific TracerProvider
+        trace_provider = TracerProvider(resource=resource, sampler=sampler)
+        processor = BatchSpanProcessor(OTLPSpanExporter())
+        trace_provider.add_span_processor(processor)
+        trace.set_tracer_provider(trace_provider)
+        TRACER = SpanTracer(service_name)
+    else:
+        # Currently OTEL_SDK_DISABLED doesn't seem to be honored by all implementations yet
+        # It seems to be working well enough for the Python implementation, but to be on
+        # the safe side, let's explicitly disable the trace exporter for now
+        os.environ[OTEL_TRACES_EXPORTER] = "none"
+        os.environ[OTEL_SDK_DISABLED] = "true"
+
+
+def start_span(
+    *,
+    record_exception: bool = False,
+    set_status_on_exception: bool = False,
+) -> Callable:
+    """Returns decorated or undecorated function depending on if TRACER is instantiated.
+
+    Should be used as a decorator.
+    """
+
+    def wrapper(function: Callable):
+        # Caller did not have any time to populate from config yet or otel is disabled
+        if TRACER is None:
+            return function
+        # Return decorated function
+        return TRACER.start_span(
+            record_exception=record_exception,
+            set_status_on_exception=set_status_on_exception,
+        )(function)
+
+    return wrapper

{hexkit-4.4.0 → hexkit-4.6.0}/src/hexkit/providers/akafka/testutils.py

@@ -19,12 +19,15 @@
 Please note, only use for testing purposes.
 """
 
+import asyncio
 import json
 from collections.abc import AsyncGenerator, Generator, Mapping, Sequence
 from contextlib import asynccontextmanager
 from dataclasses import dataclass
 from functools import partial
-from typing import Any, Optional, Union
+from typing import Any, Optional, Union, cast
+
+from aiokafka.admin.config_resource import ConfigResource, ConfigResourceType
 
 try:
     from typing import Self
@@ -411,6 +414,7 @@ class KafkaFixture:
         self.config = config
         self.kafka_servers = kafka_servers
         self.publisher = publisher
+        self.admin_client: Optional[AIOKafkaAdminClient] = None
 
     async def publish_event(
         self,
@@ -442,6 +446,80 @@ class KafkaFixture:
             capture_headers=capture_headers,
         )
 
+    @asynccontextmanager
+    async def get_admin_client(self):
+        """An async context manager that provides a running AIOKafkaAdminClient
+        as a singleton instance.
+
+        If a running client already exists, it is yielded directly.
+        If not, this is considered the root use of the CM, so we start a new client,
+        yield it, then close the client within a `finally` block.
+        """
+        if self.admin_client:
+            yield self.admin_client
+            return
+
+        # No running client exists, so create one and make sure to clean up afterward.
+        self.admin_client = AIOKafkaAdminClient(bootstrap_servers=self.kafka_servers)
+        await self.admin_client.start()
+        try:
+            yield self.admin_client
+        finally:
+            await self.admin_client.close()
+            self.admin_client = None
+
+    async def get_cleanup_policy(self, *, topic: str) -> Optional[str]:
+        """Get the current cleanup policy for a topic or None if the topic doesn't exist"""
+        return await self.get_topic_config(topic=topic, config_name="cleanup.policy")
+
+    async def get_topic_config(self, *, topic: str, config_name: str) -> Optional[str]:
+        """Get the current config for a topic or None if the topic/config doesn't exist"""
+        async with self.get_admin_client() as admin_client:
+            # fetch a list containing a response for each requested topic
+            config_list = await admin_client.describe_configs(
+                [ConfigResource(ConfigResourceType.TOPIC, name=topic)]
+            )
+
+        # We only requested one topic, so get the first/only element from the list
+        config_response = config_list[0]
+
+        # Each config response has a 'resources' attribute
+        resources = config_response.resources
+
+        # The resources attr is a list containing one tuple for each requested topic.
+        #  We only requested one topic, so our target is the first element.
+        requested_resource = resources[0]
+
+        # The resource itself is a list, and the topic config is at the end
+        configs = requested_resource[-1]
+        value = None
+        config_name = config_name.lower()
+
+        # The `configs` item is a list of tuples, where each tuple is a config item
+        # The first item is the config name, the second is the value, and the rest are ancillary
+        for item in configs:
+            if item[0].lower() == config_name:
+                value = item[1]
+                break
+        return value
+
+    async def set_cleanup_policy(self, *, topic: str, policy: str):
+        """Set the cleanup policy for the given topic. The topic must already exist."""
+        await self.set_topic_config(topic=topic, config={"cleanup.policy": policy})
+
+    async def set_topic_config(self, *, topic: str, config: dict[str, Any]):
+        """Set an arbitrary config value(s) for a topic"""
+        async with self.get_admin_client() as admin_client:
+            await admin_client.alter_configs(
+                config_resources=[
+                    ConfigResource(
+                        ConfigResourceType.TOPIC,
+                        name=topic,
+                        configs=config,
+                    )
+                ]
+            )
+
     async def clear_topics(
         self,
         *,
@@ -453,26 +531,49 @@ class KafkaFixture:
         If no topics are specified, all topics will be cleared, except internal topics
         unless otherwise specified.
         """
-        admin_client = AIOKafkaAdminClient(bootstrap_servers=self.kafka_servers)
-        await admin_client.start()
-        try:
-            if topics is None:
-                topics = await admin_client.list_topics()
+        async with self.get_admin_client() as admin_client:
+            available_topics = await admin_client.list_topics()
+            if not topics:
+                topics = available_topics
             elif isinstance(topics, str):
                 topics = [topics]
+            topics = cast(list, topics)
             if exclude_internal:
                 topics = [topic for topic in topics if not topic.startswith("__")]
+
+            # Filter out any topics that don't exist
+            topics = [topic for topic in topics if topic in available_topics]
+            if not topics:
+                return
+
+            # Record the current cleanup policies before modifying them
             topics_info = await admin_client.describe_topics(topics)
-            records_to_delete = {
-                TopicPartition(
-                    topic=topic_info["topic"], partition=partition_info["partition"]
-                ): RecordsToDelete(before_offset=-1)
-                for topic_info in topics_info
-                for partition_info in topic_info["partitions"]
-            }
-            await admin_client.delete_records(records_to_delete, timeout_ms=10000)
-        finally:
-            await admin_client.close()
+            for topic_info in topics_info:
+                topic = topic_info["topic"]
+                original_policy = await self.get_cleanup_policy(topic=topic)
+                if original_policy and "compact" in original_policy.split(","):
+                    await self.set_cleanup_policy(topic=topic, policy="delete")
+                    await asyncio.sleep(0.2)  # brief pause to wait for policy changes
+
+                records_to_delete = {
+                    TopicPartition(
+                        topic=topic, partition=partition_info["partition"]
+                    ): RecordsToDelete(before_offset=-1)
+                    for partition_info in topic_info["partitions"]
+                }
+
+                # Delete any events in the topic
+                try:
+                    await admin_client.delete_records(
+                        records_to_delete, timeout_ms=10000
+                    )
+                finally:
+                    # Always set compacted topics back to their original policy
+                    if original_policy:
+                        await self.set_cleanup_policy(
+                            topic=topic,
+                            policy=original_policy,
+                        )
 
     @asynccontextmanager
     async def expect_events(

{hexkit-4.4.0 → hexkit-4.6.0/src/hexkit.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hexkit
-Version: 4.4.0
+Version: 4.6.0
 Summary: A Toolkit for Building Microservices using the Hexagonal Architecture
 Author-email: "German Human Genome Phenome Archive (GHGA)" <contact@ghga.de>
 License: Apache 2.0

hexkit-4.4.0/src/hexkit/opentelemetry_setup.py

@@ -1,30 +0,0 @@
-# Copyright 2021 - 2025 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
-# for the German Human Genome-Phenome Archive (GHGA)
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""OpenTelemetry specific configuration code. This is gated behind the opentelemetry extra."""
-
-from opentelemetry import trace
-from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
-from opentelemetry.sdk.resources import SERVICE_NAME, Resource
-from opentelemetry.sdk.trace import TracerProvider
-from opentelemetry.sdk.trace.export import BatchSpanProcessor
-
-
-def configure_tracer(service_name: str):
-    """Set up a global tracer for a specific service."""
-    resource = Resource(attributes={SERVICE_NAME: service_name})
-    trace_provider = TracerProvider(resource=resource)
-    processor = BatchSpanProcessor(OTLPSpanExporter())
-    trace_provider.add_span_processor(processor)
-    trace.set_tracer_provider(trace_provider)