buz 2.14.1__tar.gz → 2.14.3__tar.gz

{buz-2.14.1 → buz-2.14.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: buz
-Version: 2.14.1
+Version: 2.14.3
 Summary: Buz is a set of light, simple and extensible implementations of event, command and query buses.
 License: MIT
 Author: Luis Pintado Lozano
@@ -22,6 +22,7 @@ Classifier: Typing :: Typed
 Provides-Extra: aiokafka
 Provides-Extra: kombu
 Provides-Extra: pypendency
+Requires-Dist: aiohttp (>=3.11.13,<4.0.0)
 Requires-Dist: aiokafka[lz4] (==0.12.0) ; extra == "aiokafka"
 Requires-Dist: asgiref (>=3.8.1,<4.0.0) ; extra == "aiokafka"
 Requires-Dist: asyncio (>=3.4.3,<4.0.0) ; extra == "aiokafka"

{buz-2.14.1 → buz-2.14.3}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "buz"
-version = "2.14.1"
+version = "2.14.3"
 description = "Buz is a set of light, simple and extensible implementations of event, command and query buses."
 readme = "README.md"
 authors = ["Luis Pintado Lozano <luis.pintado.lozano@gmail.com>", "Gerardo Parra <gprauxiliar@gmail.com>"]
@@ -31,6 +31,7 @@ asyncio = { version ="^3.4.3", optional = true }
 aiokafka = {version = "==0.12.0", extras = ["lz4"], optional = true}
 asgiref = {version = "^3.8.1", optional = true}
 cachetools = "^5.5.0"
+aiohttp = "^3.11.13"
 
 [tool.poetry.dev-dependencies]
 black = "^23.3"

{buz-2.14.1 → buz-2.14.3}/src/buz/event/infrastructure/buz_kafka/base_buz_aiokafka_async_consumer.py

@@ -1,3 +1,4 @@
+import json
 import traceback
 from abc import abstractmethod
 from asyncio import Lock, Task, create_task, gather, Semaphore, Event as AsyncIOEvent, sleep
@@ -5,6 +6,7 @@ from datetime import timedelta, datetime
 from itertools import cycle
 from logging import Logger
 from typing import AsyncIterator, Optional, Sequence, Type, TypeVar
+from aiohttp import web
 
 from aiokafka import TopicPartition
 from aiokafka.coordinator.assignors.abstract import AbstractPartitionAssignor
@@ -43,6 +45,7 @@ class BaseBuzAIOKafkaAsyncConsumer(AsyncConsumer):
         kafka_partition_assignors: tuple[Type[AbstractPartitionAssignor], ...] = (),
         subscribers: Sequence[MetaSubscriber],
         logger: Logger,
+        health_check_port: Optional[int],
         consumer_initial_offset_position: ConsumerInitialOffsetPosition,
         auto_create_topic_configuration: Optional[AutoCreateTopicConfiguration] = None,
         seconds_between_executions_if_there_are_no_tasks_in_the_queue: int = 1,
@@ -55,6 +58,7 @@ class BaseBuzAIOKafkaAsyncConsumer(AsyncConsumer):
         self.__kafka_partition_assignors = kafka_partition_assignors
         self.__subscribers = subscribers
         self._logger = logger
+        self.__health_check_port = health_check_port
         self.__consumer_initial_offset_position = consumer_initial_offset_position
         self.__max_records_retrieved_per_poll = 1
         self.__executor_per_consumer_mapper: dict[AIOKafkaConsumer, KafkaEventSubscriberExecutor] = {}
@@ -78,12 +82,26 @@ class BaseBuzAIOKafkaAsyncConsumer(AsyncConsumer):
         )
         self.__seconds_between_polls_if_there_are_no_new_tasks = seconds_between_polls_if_there_are_no_new_tasks
         self.__polling_tasks_semaphore = Semaphore(max_number_of_concurrent_polling_tasks)
-
         self.__task_execution_mutex = Lock()
 
+    async def configure_health_check_server(self, health_check_port: int) -> web.TCPSite:
+        self._logger.info(f"Starting health check server on port {health_check_port}")
+        app = web.Application()
+        app.router.add_get("/health", lambda request: self.__health_check())
+        runner = web.AppRunner(app)
+        await runner.setup()
+        site = web.TCPSite(runner, "localhost", health_check_port)
+        await site.start()
+        return site
+
     async def run(self) -> None:
         start_time = datetime.now()
         await self.__generate_kafka_consumers()
+        health_check_server: Optional[web.TCPSite] = None
+
+        if self.__health_check_port is not None:
+            health_check_server = await self.configure_health_check_server(self.__health_check_port)
+
         self.__initial_coroutines_created_elapsed_time = datetime.now() - start_time
 
         if len(self.__executor_per_consumer_mapper) == 0:
@@ -94,6 +112,9 @@ class BaseBuzAIOKafkaAsyncConsumer(AsyncConsumer):
         worker_errors = await self.__run_worker()
         self.__events_processed_elapsed_time = datetime.now() - start_consumption_time
 
+        if health_check_server is not None:
+            await health_check_server.stop()
+
         await self.__handle_graceful_stop(worker_errors)
 
     async def __handle_graceful_stop(self, worker_errors: tuple[Optional[Exception], Optional[Exception]]) -> None:
@@ -270,6 +291,15 @@ class BaseBuzAIOKafkaAsyncConsumer(AsyncConsumer):
         for kafka_consumer in self.__queue_per_consumer_mapper.keys():
             await kafka_consumer.stop()
 
+    async def __health_check(self) -> web.Response:
+        health_information = {
+            "subscribers": [subscriber.fqn() for subscriber in self.__subscribers],
+            "number_of_subscribers": len(self.__subscribers),
+            "event_processed": self.__events_processed,
+        }
+
+        return web.Response(text=json.dumps(health_information), content_type="application/json")
+
     def __print_statistics(self) -> None:
         self._logger.info("Number of subscribers: %d", len(self.__subscribers))
         self._logger.info(f"Start kafka consumers elapsed time: {self.__start_kafka_consumers_elapsed_time}")

{buz-2.14.1 → buz-2.14.3}/src/buz/event/infrastructure/buz_kafka/buz_aiokafka_async_consumer.py

@@ -4,7 +4,6 @@ from typing import Optional, Sequence, Type, TypeVar
 from aiokafka.coordinator.assignors.abstract import AbstractPartitionAssignor
 
 from buz.event import Event
-
 from buz.event.async_subscriber import AsyncSubscriber
 from buz.event.infrastructure.buz_kafka.base_buz_aiokafka_async_consumer import BaseBuzAIOKafkaAsyncConsumer
 from buz.event.infrastructure.buz_kafka.consume_strategy.consume_strategy import KafkaConsumeStrategy
@@ -15,11 +14,11 @@ from buz.event.meta_subscriber import MetaSubscriber
 from buz.event.middleware.async_consume_middleware import AsyncConsumeMiddleware
 from buz.event.strategies.retry.consume_retrier import ConsumeRetrier
 from buz.event.strategies.retry.reject_callback import RejectCallback
-
 from buz.kafka.domain.models.auto_create_topic_configuration import AutoCreateTopicConfiguration
 from buz.kafka.domain.models.consumer_initial_offset_position import ConsumerInitialOffsetPosition
 from buz.kafka.domain.models.kafka_connection_config import KafkaConnectionConfig
 from buz.kafka.domain.services.kafka_admin_client import KafkaAdminClient
+from buz.kafka.infrastructure.deserializers.byte_deserializer import ByteDeserializer
 from buz.kafka.infrastructure.deserializers.bytes_to_message_deserializer import BytesToMessageDeserializer
 from buz.kafka.infrastructure.deserializers.implementations.json_bytes_to_message_deserializer import (
     JSONBytesToMessageDeserializer,
@@ -52,6 +51,7 @@ class BuzAIOKafkaAsyncConsumer(BaseBuzAIOKafkaAsyncConsumer):
         seconds_between_polls_if_there_are_tasks_in_the_queue: int = 1,
         seconds_between_polls_if_there_are_no_new_tasks: int = 1,
         max_number_of_concurrent_polling_tasks: int = 20,
+        health_check_port: Optional[int] = None,
     ):
         super().__init__(
             connection_config=connection_config,
@@ -68,6 +68,7 @@ class BuzAIOKafkaAsyncConsumer(BaseBuzAIOKafkaAsyncConsumer):
             seconds_between_polls_if_there_are_tasks_in_the_queue=seconds_between_polls_if_there_are_tasks_in_the_queue,
             seconds_between_polls_if_there_are_no_new_tasks=seconds_between_polls_if_there_are_no_new_tasks,
             max_number_of_concurrent_polling_tasks=max_number_of_concurrent_polling_tasks,
+            health_check_port=health_check_port,
         )
         self.__on_fail_strategy = on_fail_strategy
         self.__consume_middlewares = consume_middlewares
@@ -84,7 +85,9 @@ class BuzAIOKafkaAsyncConsumer(BaseBuzAIOKafkaAsyncConsumer):
                 f"Subscriber {subscriber.__class__.__name__} is not a subclass of Subscriber, probably you are trying to use a synchronous subscriber"
             )
 
-        byte_deserializer = self._deserializers_per_subscriber.get(subscriber) or JSONBytesToMessageDeserializer(
+        byte_deserializer: ByteDeserializer[Event] = self._deserializers_per_subscriber.get(
+            subscriber
+        ) or JSONBytesToMessageDeserializer(
             # todo: it looks like in next python versions the inference engine is powerful enough to ensure this type, so we can remove it when we upgrade the python version of the library
             event_class=subscriber.handles()  # type: ignore
         )

{buz-2.14.1 → buz-2.14.3}/src/buz/event/infrastructure/buz_kafka/buz_aiokafka_multi_threaded_consumer.py

@@ -19,6 +19,7 @@ from buz.kafka.domain.models.auto_create_topic_configuration import AutoCreateTo
 from buz.kafka.domain.models.consumer_initial_offset_position import ConsumerInitialOffsetPosition
 from buz.kafka.domain.models.kafka_connection_config import KafkaConnectionConfig
 from buz.kafka.domain.services.kafka_admin_client import KafkaAdminClient
+from buz.kafka.infrastructure.deserializers.byte_deserializer import ByteDeserializer
 from buz.kafka.infrastructure.deserializers.bytes_to_message_deserializer import BytesToMessageDeserializer
 from buz.kafka.infrastructure.deserializers.implementations.json_bytes_to_message_deserializer import (
     JSONBytesToMessageDeserializer,
@@ -51,6 +52,7 @@ class BuzAIOKafkaMultiThreadedConsumer(BaseBuzAIOKafkaAsyncConsumer):
         seconds_between_polls_if_there_are_tasks_in_the_queue: int = 1,
         seconds_between_polls_if_there_are_no_new_tasks: int = 1,
         max_number_of_concurrent_polling_tasks: int = 20,
+        health_check_port: Optional[int] = 3123,
     ):
         super().__init__(
             connection_config=connection_config,
@@ -67,6 +69,7 @@ class BuzAIOKafkaMultiThreadedConsumer(BaseBuzAIOKafkaAsyncConsumer):
             seconds_between_polls_if_there_are_tasks_in_the_queue=seconds_between_polls_if_there_are_tasks_in_the_queue,
             seconds_between_polls_if_there_are_no_new_tasks=seconds_between_polls_if_there_are_no_new_tasks,
             max_number_of_concurrent_polling_tasks=max_number_of_concurrent_polling_tasks,
+            health_check_port=health_check_port,
         )
         self.__on_fail_strategy = on_fail_strategy
         self.__consume_middlewares = consume_middlewares
@@ -83,7 +86,9 @@ class BuzAIOKafkaMultiThreadedConsumer(BaseBuzAIOKafkaAsyncConsumer):
                 f"Subscriber {subscriber.__class__.__name__} is not a subclass of Subscriber, probably you are trying to use an asynchronous subscriber"
             )
 
-        byte_deserializer = self._deserializers_per_subscriber.get(subscriber) or JSONBytesToMessageDeserializer(
+        byte_deserializer: ByteDeserializer[Event] = self._deserializers_per_subscriber.get(
+            subscriber
+        ) or JSONBytesToMessageDeserializer(
             # todo: it looks like in next python versions the inference engine is powerful enough to ensure this type, so we can remove it when we upgrade the python version of the library
             event_class=subscriber.handles()  # type: ignore
         )

buz-2.14.3/src/buz/event/infrastructure/buz_kafka/exceptions/retry_exception.py

@@ -0,0 +1,12 @@
+class ConsumerRetryException(Exception):
+    def __init__(
+        self,
+        *,
+        event_id: str,
+        subscriber_fqn: str,
+        number_of_executions: int,
+    ) -> None:
+        super().__init__(
+            f"An exception happened during the consumption of the event '{event_id}' by the subscriber '{subscriber_fqn}' "
+            + f"during execution number '{number_of_executions}'. Retrying the consumption..."
+        )

{buz-2.14.1 → buz-2.14.3}/src/buz/event/infrastructure/buz_kafka/kafka_event_async_subscriber_executor.py

@@ -6,6 +6,7 @@ from typing import Optional, Sequence, cast
 from buz.event import Event
 from buz.event.async_subscriber import AsyncSubscriber
 from buz.event.infrastructure.buz_kafka.consume_strategy.kafka_on_fail_strategy import KafkaOnFailStrategy
+from buz.event.infrastructure.buz_kafka.exceptions.retry_exception import ConsumerRetryException
 from buz.event.infrastructure.buz_kafka.kafka_event_subscriber_executor import KafkaEventSubscriberExecutor
 from buz.event.middleware.async_consume_middleware import AsyncConsumeMiddleware
 from buz.event.middleware.async_consume_middleware_chain_resolver import AsyncConsumeMiddlewareChainResolver
@@ -26,7 +27,7 @@ class KafkaEventAsyncSubscriberExecutor(KafkaEventSubscriberExecutor):
         logger: Logger,
         consume_middlewares: Optional[Sequence[AsyncConsumeMiddleware]] = None,
         seconds_between_retries: float = 5,
-        byte_deserializer: ByteDeserializer,
+        byte_deserializer: ByteDeserializer[Event],
         header_deserializer: KafkaHeaderSerializer,
         on_fail_strategy: KafkaOnFailStrategy,
         consume_retrier: Optional[ConsumeRetrier] = None,
@@ -53,10 +54,17 @@ class KafkaEventAsyncSubscriberExecutor(KafkaEventSubscriberExecutor):
 
             kafka_record_value = cast(bytes, kafka_poll_record.value)
 
+            deserialized_value = self.__byte_deserializer.deserialize(kafka_record_value)
+
+            self.__logger.info(
+                f"consuming the event '{deserialized_value.id}' by the subscriber '{self.__subscriber.fqn()}', "
+                + f"topic: '{kafka_poll_record.topic}', partition: '{kafka_poll_record.partition}', offset: '{kafka_poll_record.offset}'"
+            )
+
             await self.__consumption_callback(
                 self.__subscriber,
                 KafkaConsumerRecord(
-                    value=self.__byte_deserializer.deserialize(kafka_record_value),
+                    value=deserialized_value,
                     headers=self.__header_deserializer.deserialize(kafka_poll_record.headers),
                 ),
             )
@@ -78,20 +86,28 @@ class KafkaEventAsyncSubscriberExecutor(KafkaEventSubscriberExecutor):
 
     async def __perform_consume(self, event: Event, subscriber: AsyncSubscriber) -> None:
         should_retry = True
+        number_of_executions = 0
         while should_retry is True:
             try:
+                number_of_executions += 1
                 await subscriber.consume(event)
                 return
             except Exception as exception:
-                self.__logger.exception(
-                    f"Event {event.id} could not be consumed by the subscriber, error: {traceback.format_exc()}"
-                )
-
                 if self.__should_retry(event, subscriber) is True:
+                    self.__logger.warning(
+                        ConsumerRetryException(
+                            number_of_executions=number_of_executions,
+                            event_id=event.id,
+                            subscriber_fqn=subscriber.fqn(),
+                        ),
+                        exc_info=exception,
+                    )
                     self.__register_retry(event, subscriber)
                     await sleep(self.__seconds_between_retires)
                     continue
 
+                self.__logger.exception(exception)
+
                 if self.__reject_callback:
                     self.__reject_callback.on_reject(event=event, subscribers=[subscriber], exception=exception)
 

{buz-2.14.1 → buz-2.14.3}/src/buz/event/infrastructure/buz_kafka/kafka_event_sync_subscriber_executor.py

@@ -4,6 +4,7 @@ import time
 from typing import Optional, Sequence, cast
 from buz.event import Event
 from buz.event.infrastructure.buz_kafka.consume_strategy.kafka_on_fail_strategy import KafkaOnFailStrategy
+from buz.event.infrastructure.buz_kafka.exceptions.retry_exception import ConsumerRetryException
 from buz.event.infrastructure.buz_kafka.kafka_event_subscriber_executor import KafkaEventSubscriberExecutor
 from buz.event.middleware.consume_middleware import ConsumeMiddleware
 from buz.event.middleware.consume_middleware_chain_resolver import ConsumeMiddlewareChainResolver
@@ -25,7 +26,7 @@ class KafkaEventSyncSubscriberExecutor(KafkaEventSubscriberExecutor):
         logger: Logger,
         consume_middlewares: Optional[Sequence[ConsumeMiddleware]] = None,
         seconds_between_retries: float = 5,
-        byte_deserializer: ByteDeserializer,
+        byte_deserializer: ByteDeserializer[Event],
         header_deserializer: KafkaHeaderSerializer,
         on_fail_strategy: KafkaOnFailStrategy,
         consume_retrier: Optional[ConsumeRetrier] = None,
@@ -52,12 +53,19 @@ class KafkaEventSyncSubscriberExecutor(KafkaEventSubscriberExecutor):
 
             kafka_record_value = cast(bytes, kafka_poll_record.value)
 
+            deserialized_value = self.__byte_deserializer.deserialize(kafka_record_value)
+
+            self.__logger.info(
+                f"consuming the event '{deserialized_value.id}' by the subscriber '{self.__subscriber.fqn()}', "
+                + f"topic: '{kafka_poll_record.topic}', partition: '{kafka_poll_record.partition}', offset: '{kafka_poll_record.offset}'"
+            )
+
             await get_running_loop().run_in_executor(
                 None,
                 lambda: self.__execution_callback(
                     self.__subscriber,
                     KafkaConsumerRecord(
-                        value=self.__byte_deserializer.deserialize(kafka_record_value),
+                        value=deserialized_value,
                         headers=self.__header_deserializer.deserialize(kafka_poll_record.headers),
                     ),
                 ),
@@ -75,13 +83,22 @@ class KafkaEventSyncSubscriberExecutor(KafkaEventSubscriberExecutor):
 
     def __perform_consume(self, event: Event, subscriber: Subscriber) -> None:
         should_retry = True
+        execution_number = 0
         while should_retry is True:
             try:
+                execution_number += 1
                 subscriber.consume(event)
                 return
             except Exception as exception:
-                self.__logger.warning(f"Event {event.id} could not be consumed by the subscriber {subscriber.fqn}")
                 if self.__should_retry(event, subscriber) is True:
+                    self.__logger.warning(
+                        ConsumerRetryException(
+                            number_of_executions=execution_number,
+                            event_id=event.id,
+                            subscriber_fqn=subscriber.fqn(),
+                        ),
+                        exc_info=exception,
+                    )
                     self.__register_retry(event, subscriber)
                     time.sleep(self.__seconds_between_retires)
                     continue

{buz-2.14.1 → buz-2.14.3}/src/buz/event/meta_subscriber.py

@@ -1,5 +1,5 @@
 from abc import ABC, abstractmethod
-from typing import Awaitable, Union
+from typing import Awaitable, Type, Union
 
 from buz import Handler
 from buz.event import Event
@@ -9,3 +9,8 @@ class MetaSubscriber(Handler, ABC):
     @abstractmethod
     def consume(self, event: Event) -> Union[None, Awaitable[None]]:
         pass
+
+    @classmethod
+    @abstractmethod
+    def handles(cls) -> Type[Event]:
+        pass

buz-2.14.3/src/buz/kafka/domain/exceptions/not_all_partition_assigned_exception.py

@@ -0,0 +1,13 @@
+from __future__ import annotations
+
+
+class NotAllPartitionAssignedException(Exception):
+    def __init__(
+        self,
+        *,
+        topic_name: str,
+        consumer_group: str,
+    ) -> None:
+        super().__init__(
+            f'Not all the partitions in the consumer group "{consumer_group}" were assigned in the topic "{topic_name}". Please disconnect the rest of subscribers'
+        )

{buz-2.14.1 → buz-2.14.3}/src/buz/kafka/infrastructure/kafka_python/kafka_python_admin_client.py

@@ -228,7 +228,10 @@ class KafkaPythonAdminClient(KafkaAdminClient):
 
             # We need all the partitions in order to update the offsets
             if len(consumer.assignment()) != len(topic_partitions):
-                raise NotAllPartitionAssignedException(topic)
+                raise NotAllPartitionAssignedException(
+                    topic_name=topic,
+                    consumer_group=consumer_group,
+                )
 
             # This could produce a race condition, but it is a limitation of kafka admin (we are not able to check if all the partition are assigned using the manual assignment)
             # https://github.com/dpkp/kafka-python/blob/master/kafka/consumer/group.py#L430

buz-2.14.1/src/buz/kafka/domain/exceptions/not_all_partition_assigned_exception.py

@@ -1,8 +0,0 @@
-from __future__ import annotations
-
-
-class NotAllPartitionAssignedException(Exception):
-    def __init__(self, topic_name: str) -> None:
-        super().__init__(
-            f'Not all the partition were assigned for the topic "{topic_name}", please disconnect the rest of subscribers'
-        )