buz 2.14.0__tar.gz → 2.14.2__tar.gz

{buz-2.14.0 → buz-2.14.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: buz
-Version: 2.14.0
+Version: 2.14.2
 Summary: Buz is a set of light, simple and extensible implementations of event, command and query buses.
 License: MIT
 Author: Luis Pintado Lozano

{buz-2.14.0 → buz-2.14.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "buz"
-version = "2.14.0"
+version = "2.14.2"
 description = "Buz is a set of light, simple and extensible implementations of event, command and query buses."
 readme = "README.md"
 authors = ["Luis Pintado Lozano <luis.pintado.lozano@gmail.com>", "Gerardo Parra <gprauxiliar@gmail.com>"]

{buz-2.14.0 → buz-2.14.2}/src/buz/event/infrastructure/buz_kafka/buz_aiokafka_async_consumer.py

@@ -4,7 +4,6 @@ from typing import Optional, Sequence, Type, TypeVar
 from aiokafka.coordinator.assignors.abstract import AbstractPartitionAssignor
 
 from buz.event import Event
-
 from buz.event.async_subscriber import AsyncSubscriber
 from buz.event.infrastructure.buz_kafka.base_buz_aiokafka_async_consumer import BaseBuzAIOKafkaAsyncConsumer
 from buz.event.infrastructure.buz_kafka.consume_strategy.consume_strategy import KafkaConsumeStrategy
@@ -15,11 +14,11 @@ from buz.event.meta_subscriber import MetaSubscriber
 from buz.event.middleware.async_consume_middleware import AsyncConsumeMiddleware
 from buz.event.strategies.retry.consume_retrier import ConsumeRetrier
 from buz.event.strategies.retry.reject_callback import RejectCallback
-
 from buz.kafka.domain.models.auto_create_topic_configuration import AutoCreateTopicConfiguration
 from buz.kafka.domain.models.consumer_initial_offset_position import ConsumerInitialOffsetPosition
 from buz.kafka.domain.models.kafka_connection_config import KafkaConnectionConfig
 from buz.kafka.domain.services.kafka_admin_client import KafkaAdminClient
+from buz.kafka.infrastructure.deserializers.byte_deserializer import ByteDeserializer
 from buz.kafka.infrastructure.deserializers.bytes_to_message_deserializer import BytesToMessageDeserializer
 from buz.kafka.infrastructure.deserializers.implementations.json_bytes_to_message_deserializer import (
     JSONBytesToMessageDeserializer,
@@ -84,7 +83,9 @@ class BuzAIOKafkaAsyncConsumer(BaseBuzAIOKafkaAsyncConsumer):
                 f"Subscriber {subscriber.__class__.__name__} is not a subclass of Subscriber, probably you are trying to use a synchronous subscriber"
             )
 
-        byte_deserializer = self._deserializers_per_subscriber.get(subscriber) or JSONBytesToMessageDeserializer(
+        byte_deserializer: ByteDeserializer[Event] = self._deserializers_per_subscriber.get(
+            subscriber
+        ) or JSONBytesToMessageDeserializer(
             # todo: it looks like in next python versions the inference engine is powerful enough to ensure this type, so we can remove it when we upgrade the python version of the library
             event_class=subscriber.handles()  # type: ignore
         )

{buz-2.14.0 → buz-2.14.2}/src/buz/event/infrastructure/buz_kafka/buz_aiokafka_multi_threaded_consumer.py

@@ -19,6 +19,7 @@ from buz.kafka.domain.models.auto_create_topic_configuration import AutoCreateTo
 from buz.kafka.domain.models.consumer_initial_offset_position import ConsumerInitialOffsetPosition
 from buz.kafka.domain.models.kafka_connection_config import KafkaConnectionConfig
 from buz.kafka.domain.services.kafka_admin_client import KafkaAdminClient
+from buz.kafka.infrastructure.deserializers.byte_deserializer import ByteDeserializer
 from buz.kafka.infrastructure.deserializers.bytes_to_message_deserializer import BytesToMessageDeserializer
 from buz.kafka.infrastructure.deserializers.implementations.json_bytes_to_message_deserializer import (
     JSONBytesToMessageDeserializer,
@@ -83,7 +84,9 @@ class BuzAIOKafkaMultiThreadedConsumer(BaseBuzAIOKafkaAsyncConsumer):
                 f"Subscriber {subscriber.__class__.__name__} is not a subclass of Subscriber, probably you are trying to use an asynchronous subscriber"
             )
 
-        byte_deserializer = self._deserializers_per_subscriber.get(subscriber) or JSONBytesToMessageDeserializer(
+        byte_deserializer: ByteDeserializer[Event] = self._deserializers_per_subscriber.get(
+            subscriber
+        ) or JSONBytesToMessageDeserializer(
             # todo: it looks like in next python versions the inference engine is powerful enough to ensure this type, so we can remove it when we upgrade the python version of the library
             event_class=subscriber.handles()  # type: ignore
         )

buz-2.14.2/src/buz/event/infrastructure/buz_kafka/exceptions/retry_exception.py

@@ -0,0 +1,12 @@
+class ConsumerRetryException(Exception):
+    def __init__(
+        self,
+        *,
+        event_id: str,
+        subscriber_fqn: str,
+        number_of_executions: int,
+    ) -> None:
+        super().__init__(
+            f"An exception happened during the consumption of the event '{event_id}' by the subscriber '{subscriber_fqn}' "
+            + f"during execution number '{number_of_executions}'. Retrying the consumption..."
+        )

{buz-2.14.0 → buz-2.14.2}/src/buz/event/infrastructure/buz_kafka/kafka_event_async_subscriber_executor.py

@@ -6,6 +6,7 @@ from typing import Optional, Sequence, cast
 from buz.event import Event
 from buz.event.async_subscriber import AsyncSubscriber
 from buz.event.infrastructure.buz_kafka.consume_strategy.kafka_on_fail_strategy import KafkaOnFailStrategy
+from buz.event.infrastructure.buz_kafka.exceptions.retry_exception import ConsumerRetryException
 from buz.event.infrastructure.buz_kafka.kafka_event_subscriber_executor import KafkaEventSubscriberExecutor
 from buz.event.middleware.async_consume_middleware import AsyncConsumeMiddleware
 from buz.event.middleware.async_consume_middleware_chain_resolver import AsyncConsumeMiddlewareChainResolver
@@ -26,7 +27,7 @@ class KafkaEventAsyncSubscriberExecutor(KafkaEventSubscriberExecutor):
         logger: Logger,
         consume_middlewares: Optional[Sequence[AsyncConsumeMiddleware]] = None,
         seconds_between_retries: float = 5,
-        byte_deserializer: ByteDeserializer,
+        byte_deserializer: ByteDeserializer[Event],
         header_deserializer: KafkaHeaderSerializer,
         on_fail_strategy: KafkaOnFailStrategy,
         consume_retrier: Optional[ConsumeRetrier] = None,
@@ -53,10 +54,17 @@ class KafkaEventAsyncSubscriberExecutor(KafkaEventSubscriberExecutor):
 
             kafka_record_value = cast(bytes, kafka_poll_record.value)
 
+            deserialized_value = self.__byte_deserializer.deserialize(kafka_record_value)
+
+            self.__logger.info(
+                f"consuming the event '{deserialized_value.id}' by the subscriber '{self.__subscriber.fqn()}', "
+                + f"topic: '{kafka_poll_record.topic}', partition: '{kafka_poll_record.partition}', offset: '{kafka_poll_record.offset}'"
+            )
+
             await self.__consumption_callback(
                 self.__subscriber,
                 KafkaConsumerRecord(
-                    value=self.__byte_deserializer.deserialize(kafka_record_value),
+                    value=deserialized_value,
                     headers=self.__header_deserializer.deserialize(kafka_poll_record.headers),
                 ),
             )
@@ -78,20 +86,28 @@ class KafkaEventAsyncSubscriberExecutor(KafkaEventSubscriberExecutor):
 
     async def __perform_consume(self, event: Event, subscriber: AsyncSubscriber) -> None:
         should_retry = True
+        number_of_executions = 0
         while should_retry is True:
             try:
+                number_of_executions += 1
                 await subscriber.consume(event)
                 return
             except Exception as exception:
-                self.__logger.exception(
-                    f"Event {event.id} could not be consumed by the subscriber, error: {traceback.format_exc()}"
-                )
-
                 if self.__should_retry(event, subscriber) is True:
+                    self.__logger.warning(
+                        ConsumerRetryException(
+                            number_of_executions=number_of_executions,
+                            event_id=event.id,
+                            subscriber_fqn=subscriber.fqn(),
+                        ),
+                        exc_info=exception,
+                    )
                     self.__register_retry(event, subscriber)
                     await sleep(self.__seconds_between_retires)
                     continue
 
+                self.__logger.exception(exception)
+
                 if self.__reject_callback:
                     self.__reject_callback.on_reject(event=event, subscribers=[subscriber], exception=exception)
 

{buz-2.14.0 → buz-2.14.2}/src/buz/event/infrastructure/buz_kafka/kafka_event_sync_subscriber_executor.py

@@ -4,6 +4,7 @@ import time
 from typing import Optional, Sequence, cast
 from buz.event import Event
 from buz.event.infrastructure.buz_kafka.consume_strategy.kafka_on_fail_strategy import KafkaOnFailStrategy
+from buz.event.infrastructure.buz_kafka.exceptions.retry_exception import ConsumerRetryException
 from buz.event.infrastructure.buz_kafka.kafka_event_subscriber_executor import KafkaEventSubscriberExecutor
 from buz.event.middleware.consume_middleware import ConsumeMiddleware
 from buz.event.middleware.consume_middleware_chain_resolver import ConsumeMiddlewareChainResolver
@@ -25,7 +26,7 @@ class KafkaEventSyncSubscriberExecutor(KafkaEventSubscriberExecutor):
         logger: Logger,
         consume_middlewares: Optional[Sequence[ConsumeMiddleware]] = None,
         seconds_between_retries: float = 5,
-        byte_deserializer: ByteDeserializer,
+        byte_deserializer: ByteDeserializer[Event],
         header_deserializer: KafkaHeaderSerializer,
         on_fail_strategy: KafkaOnFailStrategy,
         consume_retrier: Optional[ConsumeRetrier] = None,
@@ -52,12 +53,19 @@ class KafkaEventSyncSubscriberExecutor(KafkaEventSubscriberExecutor):
 
             kafka_record_value = cast(bytes, kafka_poll_record.value)
 
+            deserialized_value = self.__byte_deserializer.deserialize(kafka_record_value)
+
+            self.__logger.info(
+                f"consuming the event '{deserialized_value.id}' by the subscriber '{self.__subscriber.fqn()}', "
+                + f"topic: '{kafka_poll_record.topic}', partition: '{kafka_poll_record.partition}', offset: '{kafka_poll_record.offset}'"
+            )
+
             await get_running_loop().run_in_executor(
                 None,
                 lambda: self.__execution_callback(
                     self.__subscriber,
                     KafkaConsumerRecord(
-                        value=self.__byte_deserializer.deserialize(kafka_record_value),
+                        value=deserialized_value,
                         headers=self.__header_deserializer.deserialize(kafka_poll_record.headers),
                     ),
                 ),
@@ -75,13 +83,22 @@ class KafkaEventSyncSubscriberExecutor(KafkaEventSubscriberExecutor):
 
     def __perform_consume(self, event: Event, subscriber: Subscriber) -> None:
         should_retry = True
+        execution_number = 0
         while should_retry is True:
             try:
+                execution_number += 1
                 subscriber.consume(event)
                 return
             except Exception as exception:
-                self.__logger.warning(f"Event {event.id} could not be consumed by the subscriber {subscriber.fqn}")
                 if self.__should_retry(event, subscriber) is True:
+                    self.__logger.warning(
+                        ConsumerRetryException(
+                            number_of_executions=execution_number,
+                            event_id=event.id,
+                            subscriber_fqn=subscriber.fqn(),
+                        ),
+                        exc_info=exception,
+                    )
                     self.__register_retry(event, subscriber)
                     time.sleep(self.__seconds_between_retires)
                     continue

{buz-2.14.0 → buz-2.14.2}/src/buz/event/meta_subscriber.py

@@ -1,5 +1,5 @@
 from abc import ABC, abstractmethod
-from typing import Awaitable, Union
+from typing import Awaitable, Type, Union
 
 from buz import Handler
 from buz.event import Event
@@ -9,3 +9,8 @@ class MetaSubscriber(Handler, ABC):
     @abstractmethod
     def consume(self, event: Event) -> Union[None, Awaitable[None]]:
         pass
+
+    @classmethod
+    @abstractmethod
+    def handles(cls) -> Type[Event]:
+        pass

buz-2.14.2/src/buz/kafka/domain/exceptions/not_all_partition_assigned_exception.py

@@ -0,0 +1,13 @@
+from __future__ import annotations
+
+
+class NotAllPartitionAssignedException(Exception):
+    def __init__(
+        self,
+        *,
+        topic_name: str,
+        consumer_group: str,
+    ) -> None:
+        super().__init__(
+            f'Not all the partitions in the consumer group "{consumer_group}" were assigned in the topic "{topic_name}". Please disconnect the rest of subscribers'
+        )

buz-2.14.2/src/buz/kafka/domain/exceptions/not_valid_partition_number_exception.py

@@ -0,0 +1,10 @@
+class NotValidPartitionNumberException(Exception):
+    def __init__(
+        self,
+        *,
+        partition_number: int,
+        min_partition_number: int,
+    ) -> None:
+        super().__init__(
+            f'"{partition_number}" is not a valid partition number, the minimum partition number is "{min_partition_number}"'
+        )

{buz-2.14.0 → buz-2.14.2}/src/buz/kafka/domain/services/kafka_admin_client.py

@@ -40,6 +40,10 @@ class KafkaAdminClient(ConnectionManager, ABC):
     ) -> set[str]:
         pass
 
+    @abstractmethod
+    def get_number_of_partitions(self, topic: str) -> int:
+        pass
+
     # This function moves the following offset from the provided date
     # if there are no messages with a date greater than the provided offset
     # the offset will be moved to the end
@@ -52,3 +56,12 @@ class KafkaAdminClient(ConnectionManager, ABC):
         target_datetime: datetime,
     ) -> None:
         pass
+
+    @abstractmethod
+    def increase_topic_partitions_and_set_offset_of_related_consumer_groups_to_the_beginning_of_the_new_ones(
+        self,
+        *,
+        topic: str,
+        new_number_of_partitions: int,
+    ) -> None:
+        pass