buz 2.13.1rc6__tar.gz → 2.13.1rc7__tar.gz

{buz-2.13.1rc6 → buz-2.13.1rc7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: buz
-Version: 2.13.1rc6
+Version: 2.13.1rc7
 Summary: Buz is a set of light, simple and extensible implementations of event, command and query buses.
 License: MIT
 Author: Luis Pintado Lozano

{buz-2.13.1rc6 → buz-2.13.1rc7}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "buz"
-version = "2.13.1rc6"
+version = "2.13.1rc7"
 description = "Buz is a set of light, simple and extensible implementations of event, command and query buses."
 readme = "README.md"
 authors = ["Luis Pintado Lozano <luis.pintado.lozano@gmail.com>", "Gerardo Parra <gprauxiliar@gmail.com>"]

{buz-2.13.1rc6 → buz-2.13.1rc7}/src/buz/event/dead_letter_queue/dlq_record.py

@@ -8,6 +8,7 @@ DlqRecordId = UUID
 
 @dataclass
 class DlqRecord:
+    __EXCEPTION_MESSAGE_MAX_CHARACTERS: ClassVar[int] = 300
     DATE_TIME_FORMAT: ClassVar[str] = "%Y-%m-%d %H:%M:%S.%f"
 
     id: DlqRecordId
@@ -18,6 +19,18 @@ class DlqRecord:
     exception_message: str
     last_failed_at: datetime
 
+    def __post_init__(self) -> None:
+        self.exception_message = self.__add_ellipsis(self.exception_message)
+
+    def __add_ellipsis(self, message: str) -> str:
+        if len(message) <= self.__EXCEPTION_MESSAGE_MAX_CHARACTERS:
+            return message
+        return message[: self.__EXCEPTION_MESSAGE_MAX_CHARACTERS - 3] + "..."
+
+    def set_exception(self, exception: Exception) -> None:
+        self.exception_type = type(exception).__name__
+        self.exception_message = self.__add_ellipsis(str(exception))
+
     def mark_as_failed(self) -> None:
         self.last_failed_at = datetime.now()
 

{buz-2.13.1rc6 → buz-2.13.1rc7}/src/buz/event/infrastructure/buz_kafka/base_buz_aiokafka_async_consumer.py

@@ -60,7 +60,6 @@ class BaseBuzAIOKafkaAsyncConsumer(AsyncConsumer):
         self._logger = logger
         self.__consumer_initial_offset_position = consumer_initial_offset_position
         self.__max_records_retrieved_per_poll = 1
-        self.__subscriber_per_consumer_mapper: dict[AIOKafkaConsumer, MetaSubscriber] = {}
         self.__executor_per_consumer_mapper: dict[AIOKafkaConsumer, KafkaEventSubscriberExecutor] = {}
         self.__queue_per_consumer_mapper: dict[
             AIOKafkaConsumer, MultiqueueRepository[TopicPartition, KafkaPollRecord]
@@ -96,8 +95,9 @@ class BaseBuzAIOKafkaAsyncConsumer(AsyncConsumer):
         self.__initial_coroutines_created_elapsed_time = datetime.now() - start_time
 
         start_consumption_time = datetime.now()
-        self._logger.info("Starting to consume events")
+
         worker_errors = await self.__run_worker()
+
         self.__events_processed_elapsed_time = datetime.now() - start_consumption_time
 
         await self.__handle_graceful_stop(worker_errors)
@@ -131,9 +131,9 @@ class BaseBuzAIOKafkaAsyncConsumer(AsyncConsumer):
             polling_task_exception = await self.__await_exception(polling_task)
             return (consume_events_exception, polling_task_exception)
 
-    async def __await_exception(self, future: Task) -> Optional[Exception]:
+    async def __await_exception(self, task: Task) -> Optional[Exception]:
         try:
-            await future
+            await task
             return None
         except Exception as exception:
             return exception
@@ -143,11 +143,11 @@ class BaseBuzAIOKafkaAsyncConsumer(AsyncConsumer):
 
     async def __generate_kafka_consumers(self):
         start_time = datetime.now()
-        tasks = [self.__initialize_kafka_consumer_for_subscriber(subscriber) for subscriber in self.__subscribers]
+        tasks = [self.__generate_kafka_consumer_for_subscriber(subscriber) for subscriber in self.__subscribers]
         await gather(*tasks)
         self.__start_kafka_consumers_elapsed_time = datetime.now() - start_time
 
-    async def __initialize_kafka_consumer_for_subscriber(self, subscriber: MetaSubscriber) -> None:
+    async def __generate_kafka_consumer_for_subscriber(self, subscriber: MetaSubscriber) -> None:
         try:
             executor = await self._create_kafka_consumer_executor(subscriber)
             topics = self.__consume_strategy.get_topics(subscriber)
@@ -167,73 +167,71 @@ class BaseBuzAIOKafkaAsyncConsumer(AsyncConsumer):
 
             self.__queue_per_consumer_mapper[kafka_consumer] = InMemoryMultiqueueRepository()
 
-            # The purpose of this block is to initialize the consumer without delay the execution of the polling thread
-            # If some of the subscribers haven't as many partitions as subscriber they will not be able to be initialized
-            # (but maybe they should be initialized in the future)
-            create_task(self.__init_consumer(kafka_consumer, subscriber))
         except Exception:
             self._logger.exception(
                 f"Unexpected error during Kafka subscriber '{subscriber.fqn()}' creation. Skipping it: {traceback.format_exc()}"
             )
 
-    async def __init_consumer(self, consumer: AIOKafkaConsumer, subscriber: MetaSubscriber) -> None:
+    @abstractmethod
+    async def _create_kafka_consumer_executor(self, subscriber: MetaSubscriber) -> KafkaEventSubscriberExecutor:
+        pass
+
+    async def __polling_task(self) -> None:
+        self._logger.info("Initializing subscribers")
+        try:
+            polling_task_per_consumer = [
+                create_task(self.__polling_consuming_tasks(consumer))
+                for consumer, subscriber in self.__queue_per_consumer_mapper.items()
+            ]
+
+            await gather(*polling_task_per_consumer)
+
+        except Exception:
+            self._logger.error(f"Polling task failed with exception: {traceback.format_exc()}")
+            self.__should_stop.set()
+
+    async def __polling_consuming_tasks(self, consumer: AIOKafkaConsumer) -> None:
+        queue = self.__queue_per_consumer_mapper[consumer]
+
         try:
+            self._logger.info(
+                f"initializing consumer group: '{consumer.get_consumer_group()}' subscribed to the topics: '{consumer.get_topics()}'"
+            )
             await consumer.init()
-            self.__subscriber_per_consumer_mapper[consumer] = subscriber
-            self._logger.info(f"initialized '{subscriber.fqn()}'")
+            self._logger.info(f"initialized '{consumer.get_consumer_group()}'")
         except Exception:
             self._logger.exception(
-                f"Unexpected error during Kafka subscriber '{subscriber.fqn()}' initialization. Skipping it: {traceback.format_exc()}"
+                f"Unexpected error during Kafka subscriber '{consumer.get_consumer_group()}' initialization. Skipping it: {traceback.format_exc()}"
             )
 
-    @abstractmethod
-    async def _create_kafka_consumer_executor(self, subscriber: MetaSubscriber) -> KafkaEventSubscriberExecutor:
-        pass
+        while not self.__should_stop.is_set():
+            total_size = sum([queue.get_total_size() for queue in self.__queue_per_consumer_mapper.values()])
+            if total_size >= self.__max_queue_size:
+                await sleep(self.__seconds_between_polls_if_there_are_tasks_in_the_queue)
+                continue
 
-    async def __polling_task(self) -> None:
-        try:
-            while not self.__should_stop.is_set():
-                total_size = sum([queue.get_total_size() for queue in self.__queue_per_consumer_mapper.values()])
-                if total_size >= self.__max_queue_size:
-                    await sleep(self.__seconds_between_polls_if_there_are_tasks_in_the_queue)
-                    continue
-
-                raw_consuming_tasks = await gather(
-                    *[
-                        self.__polling_consuming_tasks(kafka_consumer=consumer)
-                        for consumer, subscriber in self.__subscriber_per_consumer_mapper.items()
-                    ]
+            async with self.__polling_tasks_semaphore:
+                kafka_poll_records = await consumer.poll(
+                    number_of_messages_to_poll=self.__max_records_retrieved_per_poll,
                 )
 
-                poll_results: list[ConsumingTask] = [
-                    consuming_task for consuming_tasks in raw_consuming_tasks for consuming_task in consuming_tasks
-                ]
-                if len(poll_results) == 0:
-                    await sleep(self.__seconds_between_polls_if_there_are_no_new_tasks)
-
-                for poll_result in poll_results:
-                    queue = self.__queue_per_consumer_mapper[poll_result.consumer]
+                for kafka_poll_record in kafka_poll_records:
                     queue.push(
                         key=TopicPartition(
-                            topic=poll_result.kafka_poll_record.topic, partition=poll_result.kafka_poll_record.partition
+                            topic=kafka_poll_record.topic,
+                            partition=kafka_poll_record.partition,
                         ),
-                        record=poll_result.kafka_poll_record,
+                        record=kafka_poll_record,
                     )
 
-        except Exception:
-            self._logger.error(f"Polling task failed with exception: {traceback.format_exc()}")
-            self.__should_stop.set()
+            if len(kafka_poll_records) == 0:
+                await sleep(self.__seconds_between_polls_if_there_are_no_new_tasks)
 
         return
 
-    async def __polling_consuming_tasks(self, kafka_consumer: AIOKafkaConsumer) -> list[ConsumingTask]:
-        async with self.__polling_tasks_semaphore:
-            results = await kafka_consumer.poll(
-                number_of_messages_to_poll=self.__max_records_retrieved_per_poll,
-            )
-        return [ConsumingTask(kafka_consumer, result) for result in results]
-
     async def __consume_events_task(self) -> None:
+        self._logger.info("Initializing consuming task")
+
         blocked_tasks_iterator = self.generate_blocked_consuming_tasks_iterator()
 
         async for consuming_task in blocked_tasks_iterator:

{buz-2.13.1rc6 → buz-2.13.1rc7}/src/buz/kafka/infrastructure/aiokafka/aiokafka_consumer.py

@@ -2,7 +2,7 @@ from __future__ import annotations
 
 from logging import Logger
 from ssl import SSLContext
-from typing import Awaitable, Callable, Optional, cast
+from typing import Awaitable, Callable, Optional, Sequence, cast
 
 from aiokafka import AIOKafkaConsumer as AIOKafkaNativeConsumer, TopicPartition, OffsetAndMetadata
 from aiokafka.helpers import create_ssl_context
@@ -56,6 +56,12 @@ class AIOKafkaConsumer:
         self.__check_kafka_admin_client_is_needed()
         self.__consumer = self.__generate_consumer()
 
+    def get_topics(self) -> Sequence[str]:
+        return list(self.__topics)
+
+    def get_consumer_group(self) -> str:
+        return self.__consumer_group
+
     def __check_kafka_admin_client_is_needed(self) -> None:
         if self.__kafka_admin_client is None and self.__auto_create_topic_configuration is not None:
             raise KafkaEventBusConfigNotValidException(