buz 2.13.1rc6__py3-none-any.whl → 2.13.1rc7__py3-none-any.whl

buz/event/dead_letter_queue/dlq_record.py

@@ -8,6 +8,7 @@ DlqRecordId = UUID
 
 @dataclass
 class DlqRecord:
+    __EXCEPTION_MESSAGE_MAX_CHARACTERS: ClassVar[int] = 300
     DATE_TIME_FORMAT: ClassVar[str] = "%Y-%m-%d %H:%M:%S.%f"
 
     id: DlqRecordId
@@ -18,6 +19,18 @@ class DlqRecord:
     exception_message: str
     last_failed_at: datetime
 
+    def __post_init__(self) -> None:
+        self.exception_message = self.__add_ellipsis(self.exception_message)
+
+    def __add_ellipsis(self, message: str) -> str:
+        if len(message) <= self.__EXCEPTION_MESSAGE_MAX_CHARACTERS:
+            return message
+        return message[: self.__EXCEPTION_MESSAGE_MAX_CHARACTERS - 3] + "..."
+
+    def set_exception(self, exception: Exception) -> None:
+        self.exception_type = type(exception).__name__
+        self.exception_message = self.__add_ellipsis(str(exception))
+
     def mark_as_failed(self) -> None:
         self.last_failed_at = datetime.now()
 

buz/event/infrastructure/buz_kafka/base_buz_aiokafka_async_consumer.py

@@ -60,7 +60,6 @@ class BaseBuzAIOKafkaAsyncConsumer(AsyncConsumer):
         self._logger = logger
         self.__consumer_initial_offset_position = consumer_initial_offset_position
         self.__max_records_retrieved_per_poll = 1
-        self.__subscriber_per_consumer_mapper: dict[AIOKafkaConsumer, MetaSubscriber] = {}
         self.__executor_per_consumer_mapper: dict[AIOKafkaConsumer, KafkaEventSubscriberExecutor] = {}
         self.__queue_per_consumer_mapper: dict[
             AIOKafkaConsumer, MultiqueueRepository[TopicPartition, KafkaPollRecord]
@@ -96,8 +95,9 @@ class BaseBuzAIOKafkaAsyncConsumer(AsyncConsumer):
         self.__initial_coroutines_created_elapsed_time = datetime.now() - start_time
 
         start_consumption_time = datetime.now()
-        self._logger.info("Starting to consume events")
+
         worker_errors = await self.__run_worker()
+
         self.__events_processed_elapsed_time = datetime.now() - start_consumption_time
 
         await self.__handle_graceful_stop(worker_errors)
@@ -131,9 +131,9 @@ class BaseBuzAIOKafkaAsyncConsumer(AsyncConsumer):
             polling_task_exception = await self.__await_exception(polling_task)
             return (consume_events_exception, polling_task_exception)
 
-    async def __await_exception(self, future: Task) -> Optional[Exception]:
+    async def __await_exception(self, task: Task) -> Optional[Exception]:
         try:
-            await future
+            await task
             return None
         except Exception as exception:
             return exception
@@ -143,11 +143,11 @@ class BaseBuzAIOKafkaAsyncConsumer(AsyncConsumer):
 
     async def __generate_kafka_consumers(self):
         start_time = datetime.now()
-        tasks = [self.__initialize_kafka_consumer_for_subscriber(subscriber) for subscriber in self.__subscribers]
+        tasks = [self.__generate_kafka_consumer_for_subscriber(subscriber) for subscriber in self.__subscribers]
         await gather(*tasks)
         self.__start_kafka_consumers_elapsed_time = datetime.now() - start_time
 
-    async def __initialize_kafka_consumer_for_subscriber(self, subscriber: MetaSubscriber) -> None:
+    async def __generate_kafka_consumer_for_subscriber(self, subscriber: MetaSubscriber) -> None:
         try:
             executor = await self._create_kafka_consumer_executor(subscriber)
             topics = self.__consume_strategy.get_topics(subscriber)
@@ -167,73 +167,71 @@ class BaseBuzAIOKafkaAsyncConsumer(AsyncConsumer):
 
             self.__queue_per_consumer_mapper[kafka_consumer] = InMemoryMultiqueueRepository()
 
-            # The purpose of this block is to initialize the consumer without delay the execution of the polling thread
-            # If some of the subscribers haven't as many partitions as subscriber they will not be able to be initialized
-            # (but maybe they should be initialized in the future)
-            create_task(self.__init_consumer(kafka_consumer, subscriber))
         except Exception:
             self._logger.exception(
                 f"Unexpected error during Kafka subscriber '{subscriber.fqn()}' creation. Skipping it: {traceback.format_exc()}"
             )
 
-    async def __init_consumer(self, consumer: AIOKafkaConsumer, subscriber: MetaSubscriber) -> None:
+    @abstractmethod
+    async def _create_kafka_consumer_executor(self, subscriber: MetaSubscriber) -> KafkaEventSubscriberExecutor:
+        pass
+
+    async def __polling_task(self) -> None:
+        self._logger.info("Initializing subscribers")
+        try:
+            polling_task_per_consumer = [
+                create_task(self.__polling_consuming_tasks(consumer))
+                for consumer, subscriber in self.__queue_per_consumer_mapper.items()
+            ]
+
+            await gather(*polling_task_per_consumer)
+
+        except Exception:
+            self._logger.error(f"Polling task failed with exception: {traceback.format_exc()}")
+            self.__should_stop.set()
+
+    async def __polling_consuming_tasks(self, consumer: AIOKafkaConsumer) -> None:
+        queue = self.__queue_per_consumer_mapper[consumer]
+
         try:
+            self._logger.info(
+                f"initializing consumer group: '{consumer.get_consumer_group()}' subscribed to the topics: '{consumer.get_topics()}'"
+            )
             await consumer.init()
-            self.__subscriber_per_consumer_mapper[consumer] = subscriber
-            self._logger.info(f"initialized '{subscriber.fqn()}'")
+            self._logger.info(f"initialized '{consumer.get_consumer_group()}'")
         except Exception:
             self._logger.exception(
-                f"Unexpected error during Kafka subscriber '{subscriber.fqn()}' initialization. Skipping it: {traceback.format_exc()}"
+                f"Unexpected error during Kafka subscriber '{consumer.get_consumer_group()}' initialization. Skipping it: {traceback.format_exc()}"
             )
 
-    @abstractmethod
-    async def _create_kafka_consumer_executor(self, subscriber: MetaSubscriber) -> KafkaEventSubscriberExecutor:
-        pass
+        while not self.__should_stop.is_set():
+            total_size = sum([queue.get_total_size() for queue in self.__queue_per_consumer_mapper.values()])
+            if total_size >= self.__max_queue_size:
+                await sleep(self.__seconds_between_polls_if_there_are_tasks_in_the_queue)
+                continue
 
-    async def __polling_task(self) -> None:
-        try:
-            while not self.__should_stop.is_set():
-                total_size = sum([queue.get_total_size() for queue in self.__queue_per_consumer_mapper.values()])
-                if total_size >= self.__max_queue_size:
-                    await sleep(self.__seconds_between_polls_if_there_are_tasks_in_the_queue)
-                    continue
-
-                raw_consuming_tasks = await gather(
-                    *[
-                        self.__polling_consuming_tasks(kafka_consumer=consumer)
-                        for consumer, subscriber in self.__subscriber_per_consumer_mapper.items()
-                    ]
+            async with self.__polling_tasks_semaphore:
+                kafka_poll_records = await consumer.poll(
+                    number_of_messages_to_poll=self.__max_records_retrieved_per_poll,
                 )
 
-                poll_results: list[ConsumingTask] = [
-                    consuming_task for consuming_tasks in raw_consuming_tasks for consuming_task in consuming_tasks
-                ]
-                if len(poll_results) == 0:
-                    await sleep(self.__seconds_between_polls_if_there_are_no_new_tasks)
-
-                for poll_result in poll_results:
-                    queue = self.__queue_per_consumer_mapper[poll_result.consumer]
+                for kafka_poll_record in kafka_poll_records:
                     queue.push(
                         key=TopicPartition(
-                            topic=poll_result.kafka_poll_record.topic, partition=poll_result.kafka_poll_record.partition
+                            topic=kafka_poll_record.topic,
+                            partition=kafka_poll_record.partition,
                         ),
-                        record=poll_result.kafka_poll_record,
+                        record=kafka_poll_record,
                     )
 
-        except Exception:
-            self._logger.error(f"Polling task failed with exception: {traceback.format_exc()}")
-            self.__should_stop.set()
+            if len(kafka_poll_records) == 0:
+                await sleep(self.__seconds_between_polls_if_there_are_no_new_tasks)
 
         return
 
-    async def __polling_consuming_tasks(self, kafka_consumer: AIOKafkaConsumer) -> list[ConsumingTask]:
-        async with self.__polling_tasks_semaphore:
-            results = await kafka_consumer.poll(
-                number_of_messages_to_poll=self.__max_records_retrieved_per_poll,
-            )
-        return [ConsumingTask(kafka_consumer, result) for result in results]
-
     async def __consume_events_task(self) -> None:
+        self._logger.info("Initializing consuming task")
+
         blocked_tasks_iterator = self.generate_blocked_consuming_tasks_iterator()
 
         async for consuming_task in blocked_tasks_iterator:

buz/kafka/infrastructure/aiokafka/aiokafka_consumer.py

@@ -2,7 +2,7 @@ from __future__ import annotations
 
 from logging import Logger
 from ssl import SSLContext
-from typing import Awaitable, Callable, Optional, cast
+from typing import Awaitable, Callable, Optional, Sequence, cast
 
 from aiokafka import AIOKafkaConsumer as AIOKafkaNativeConsumer, TopicPartition, OffsetAndMetadata
 from aiokafka.helpers import create_ssl_context
@@ -56,6 +56,12 @@ class AIOKafkaConsumer:
         self.__check_kafka_admin_client_is_needed()
         self.__consumer = self.__generate_consumer()
 
+    def get_topics(self) -> Sequence[str]:
+        return list(self.__topics)
+
+    def get_consumer_group(self) -> str:
+        return self.__consumer_group
+
     def __check_kafka_admin_client_is_needed(self) -> None:
         if self.__kafka_admin_client is None and self.__auto_create_topic_configuration is not None:
             raise KafkaEventBusConfigNotValidException(

{buz-2.13.1rc6.dist-info → buz-2.13.1rc7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: buz
-Version: 2.13.1rc6
+Version: 2.13.1rc7
 Summary: Buz is a set of light, simple and extensible implementations of event, command and query buses.
 License: MIT
 Author: Luis Pintado Lozano

{buz-2.13.1rc6.dist-info → buz-2.13.1rc7.dist-info}/RECORD

@@ -33,7 +33,7 @@ buz/event/base_subscriber.py,sha256=nQ3fXdpFmU0S0Vpg-0ozH6_IG5zFe1B-Lsn4CNdfsds,
 buz/event/consumer.py,sha256=Rf3ZtodM_637ifk5Y2CmEVQ7caYhBgXQ9gm7D57raP0,181
 buz/event/dead_letter_queue/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 buz/event/dead_letter_queue/dlq_criteria.py,sha256=hxcV-BMayKTEc5suEfQZhEYkc14H7kZo_4YfDzcJTTY,290
-buz/event/dead_letter_queue/dlq_record.py,sha256=Puwb1wB0y4b0bJwi5PcjbMoWb0snAqDhmxNvWfRa4Mc,757
+buz/event/dead_letter_queue/dlq_record.py,sha256=wEa9CdWkHmxHQVwoHFjWeEU6sjNOi7X8dLr1E-gVmDc,1341
 buz/event/dead_letter_queue/dlq_repository.py,sha256=8XsXSfO2OzEq4qfQ_v0E0OExintDYI1g55Qu3PtoxKI,630
 buz/event/event.py,sha256=x1MCBydn3qk3AkvamsAwCG-nfxR9OyP4l1UNXtnhUwU,189
 buz/event/event_bus.py,sha256=DNr1cRLxYcn9qCu4_BKecpQHAx9D_PTxLnWXN2qVhFE,293
@@ -45,7 +45,7 @@ buz/event/exceptions/term_signal_interruption_exception.py,sha256=RkRRF0v_K9Hg48
 buz/event/exceptions/worker_execution_exception.py,sha256=6mgztvXOCG_9VZ_Jptkk72kZtNWQ2CPuQ3TjXEWFE14,123
 buz/event/infrastructure/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 buz/event/infrastructure/buz_kafka/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-buz/event/infrastructure/buz_kafka/base_buz_aiokafka_async_consumer.py,sha256=75I_wUUuQXAneEKYGjnboRHWwSYJ4UADSpcG-gnClSo,14641
+buz/event/infrastructure/buz_kafka/base_buz_aiokafka_async_consumer.py,sha256=6V54oD8dpdLyrBel98qlvt8ZMOCnukxfL-7aRreDaPI,13915
 buz/event/infrastructure/buz_kafka/buz_aiokafka_async_consumer.py,sha256=dqQDv7taAmINE9G2geMDExbcvSlntP09_rQ0JRbc4Rw,5507
 buz/event/infrastructure/buz_kafka/buz_aiokafka_multi_threaded_consumer.py,sha256=yrEU51OBjvLjCfYJFJPxux1bcIhoTVMw1Jf0HJMWbb0,5449
 buz/event/infrastructure/buz_kafka/buz_kafka_event_bus.py,sha256=f94fmS4AVfb3LQsp49e-4Cqzj00IqxHDzuUvDbN4u2s,4258
@@ -149,7 +149,7 @@ buz/kafka/domain/services/kafka_admin_test_client.py,sha256=91l_vFIo1yhJLQQCC_Om
 buz/kafka/domain/services/kafka_producer.py,sha256=CTiwGYwuzdJY5aeb2WFbJlyCpZ0YyhzcgKQYyogKzUM,401
 buz/kafka/infrastructure/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 buz/kafka/infrastructure/aiokafka/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-buz/kafka/infrastructure/aiokafka/aiokafka_consumer.py,sha256=zGalbfVPEr-08Uv0weSnkKayr2T76F8luO-UhfMNgx4,8560
+buz/kafka/infrastructure/aiokafka/aiokafka_consumer.py,sha256=ringqMhgtVzStx_k8Y2os1nlOIKmAx5XwQroTqMQbmg,8728
 buz/kafka/infrastructure/aiokafka/rebalance/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 buz/kafka/infrastructure/aiokafka/rebalance/kafka_callback_rebalancer.py,sha256=3l7NkTrCt3rBktVIS73cTmCOvv6eFguoCbGMYIUfCFc,1774
 buz/kafka/infrastructure/aiokafka/translators/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -230,7 +230,7 @@ buz/serializer/message_to_json_bytes_serializer.py,sha256=RGZJ64t4t4Pz2FCASZZCv-
 buz/wrapper/__init__.py,sha256=GnRdJFcncn-qp0hzDG9dBHLmTJSbHFVjE_yr-MdW_n4,77
 buz/wrapper/async_to_sync.py,sha256=OfK-vrVUhuN-LLLvekLdMbQYtH0ue5lfbvuasj6ovMI,698
 buz/wrapper/event_loop.py,sha256=pfBJ1g-8A2a3YgW8Gf9Fg0kkewoh3-wgTy2KIFDyfHk,266
-buz-2.13.1rc6.dist-info/LICENSE,sha256=Jytu2S-2SPEgsB0y6BF-_LUxIWY7402fl0JSh36TLZE,1062
-buz-2.13.1rc6.dist-info/METADATA,sha256=bE9USoD5Gh7NRbTZIClXx8Zvqi482Z6Y_AER8JhHeMM,1620
-buz-2.13.1rc6.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-buz-2.13.1rc6.dist-info/RECORD,,
+buz-2.13.1rc7.dist-info/LICENSE,sha256=Jytu2S-2SPEgsB0y6BF-_LUxIWY7402fl0JSh36TLZE,1062
+buz-2.13.1rc7.dist-info/METADATA,sha256=P_PoCYhT33nqbij8OAWpKZ0bE7cBL6ensd1sMRfzdoQ,1620
+buz-2.13.1rc7.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+buz-2.13.1rc7.dist-info/RECORD,,