buz 2.13.1rc6__tar.gz → 2.13.1rc8__tar.gz

{buz-2.13.1rc6 → buz-2.13.1rc8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: buz
-Version: 2.13.1rc6
+Version: 2.13.1rc8
 Summary: Buz is a set of light, simple and extensible implementations of event, command and query buses.
 License: MIT
 Author: Luis Pintado Lozano

{buz-2.13.1rc6 → buz-2.13.1rc8}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "buz"
-version = "2.13.1rc6"
+version = "2.13.1rc8"
 description = "Buz is a set of light, simple and extensible implementations of event, command and query buses."
 readme = "README.md"
 authors = ["Luis Pintado Lozano <luis.pintado.lozano@gmail.com>", "Gerardo Parra <gprauxiliar@gmail.com>"]
@@ -37,6 +37,7 @@ black = "^23.3"
 mypy = "^1.2"
 flake8 = "^5.0.4"
 tenacity = "^8.5.0"
+freezegun = "^1.5.1"
 types-cachetools = "^5.5.0.20240820"
 
 [tool.poetry.extras]

{buz-2.13.1rc6 → buz-2.13.1rc8}/src/buz/event/dead_letter_queue/dlq_record.py

@@ -8,6 +8,7 @@ DlqRecordId = UUID
 
 @dataclass
 class DlqRecord:
+    __EXCEPTION_MESSAGE_MAX_CHARACTERS: ClassVar[int] = 300
     DATE_TIME_FORMAT: ClassVar[str] = "%Y-%m-%d %H:%M:%S.%f"
 
     id: DlqRecordId
@@ -18,6 +19,18 @@ class DlqRecord:
     exception_message: str
     last_failed_at: datetime
 
+    def __post_init__(self) -> None:
+        self.exception_message = self.__add_ellipsis(self.exception_message)
+
+    def __add_ellipsis(self, message: str) -> str:
+        if len(message) <= self.__EXCEPTION_MESSAGE_MAX_CHARACTERS:
+            return message
+        return message[: self.__EXCEPTION_MESSAGE_MAX_CHARACTERS - 3] + "..."
+
+    def set_exception(self, exception: Exception) -> None:
+        self.exception_type = type(exception).__name__
+        self.exception_message = self.__add_ellipsis(str(exception))
+
     def mark_as_failed(self) -> None:
         self.last_failed_at = datetime.now()
 

{buz-2.13.1rc6 → buz-2.13.1rc8}/src/buz/event/infrastructure/buz_kafka/base_buz_aiokafka_async_consumer.py

@@ -1,5 +1,5 @@
-from abc import abstractmethod
 import traceback
+from abc import abstractmethod
 from asyncio import Lock, Task, create_task, gather, Semaphore, Event as AsyncIOEvent, sleep
 from datetime import timedelta, datetime
 from itertools import cycle
@@ -28,7 +28,6 @@ from buz.kafka.infrastructure.aiokafka.aiokafka_consumer import AIOKafkaConsumer
 from buz.queue.in_memory.in_memory_multiqueue_repository import InMemoryMultiqueueRepository
 from buz.queue.multiqueue_repository import MultiqueueRepository
 
-
 T = TypeVar("T", bound=Event)
 
 
@@ -60,7 +59,6 @@ class BaseBuzAIOKafkaAsyncConsumer(AsyncConsumer):
         self._logger = logger
         self.__consumer_initial_offset_position = consumer_initial_offset_position
         self.__max_records_retrieved_per_poll = 1
-        self.__subscriber_per_consumer_mapper: dict[AIOKafkaConsumer, MetaSubscriber] = {}
         self.__executor_per_consumer_mapper: dict[AIOKafkaConsumer, KafkaEventSubscriberExecutor] = {}
         self.__queue_per_consumer_mapper: dict[
             AIOKafkaConsumer, MultiqueueRepository[TopicPartition, KafkaPollRecord]
@@ -88,15 +86,13 @@ class BaseBuzAIOKafkaAsyncConsumer(AsyncConsumer):
     async def run(self) -> None:
         start_time = datetime.now()
         await self.__generate_kafka_consumers()
+        self.__initial_coroutines_created_elapsed_time = datetime.now() - start_time
 
         if len(self.__executor_per_consumer_mapper) == 0:
             self._logger.error("There are no valid subscribers to execute, finalizing consumer")
             return
 
-        self.__initial_coroutines_created_elapsed_time = datetime.now() - start_time
-
         start_consumption_time = datetime.now()
-        self._logger.info("Starting to consume events")
         worker_errors = await self.__run_worker()
         self.__events_processed_elapsed_time = datetime.now() - start_consumption_time
 
@@ -112,9 +108,9 @@ class BaseBuzAIOKafkaAsyncConsumer(AsyncConsumer):
         if self.__exceptions_are_thrown(worker_errors):
             consume_events_exception, polling_task_exception = worker_errors
             if consume_events_exception:
-                self._logger.error(consume_events_exception)
+                self._logger.exception(consume_events_exception)
             if polling_task_exception:
-                self._logger.error(polling_task_exception)
+                self._logger.exception(polling_task_exception)
 
             raise WorkerExecutionException("The worker was closed by an unexpected exception")
 
@@ -131,9 +127,9 @@ class BaseBuzAIOKafkaAsyncConsumer(AsyncConsumer):
             polling_task_exception = await self.__await_exception(polling_task)
             return (consume_events_exception, polling_task_exception)
 
-    async def __await_exception(self, future: Task) -> Optional[Exception]:
+    async def __await_exception(self, task: Task) -> Optional[Exception]:
         try:
-            await future
+            await task
             return None
         except Exception as exception:
             return exception
@@ -143,11 +139,11 @@ class BaseBuzAIOKafkaAsyncConsumer(AsyncConsumer):
 
     async def __generate_kafka_consumers(self):
         start_time = datetime.now()
-        tasks = [self.__initialize_kafka_consumer_for_subscriber(subscriber) for subscriber in self.__subscribers]
+        tasks = [self.__generate_kafka_consumer_for_subscriber(subscriber) for subscriber in self.__subscribers]
         await gather(*tasks)
         self.__start_kafka_consumers_elapsed_time = datetime.now() - start_time
 
-    async def __initialize_kafka_consumer_for_subscriber(self, subscriber: MetaSubscriber) -> None:
+    async def __generate_kafka_consumer_for_subscriber(self, subscriber: MetaSubscriber) -> None:
         try:
             executor = await self._create_kafka_consumer_executor(subscriber)
             topics = self.__consume_strategy.get_topics(subscriber)
@@ -167,103 +163,92 @@ class BaseBuzAIOKafkaAsyncConsumer(AsyncConsumer):
 
             self.__queue_per_consumer_mapper[kafka_consumer] = InMemoryMultiqueueRepository()
 
-            # The purpose of this block is to initialize the consumer without delay the execution of the polling thread
-            # If some of the subscribers haven't as many partitions as subscriber they will not be able to be initialized
-            # (but maybe they should be initialized in the future)
-            create_task(self.__init_consumer(kafka_consumer, subscriber))
         except Exception:
             self._logger.exception(
                 f"Unexpected error during Kafka subscriber '{subscriber.fqn()}' creation. Skipping it: {traceback.format_exc()}"
             )
 
-    async def __init_consumer(self, consumer: AIOKafkaConsumer, subscriber: MetaSubscriber) -> None:
-        try:
-            await consumer.init()
-            self.__subscriber_per_consumer_mapper[consumer] = subscriber
-            self._logger.info(f"initialized '{subscriber.fqn()}'")
-        except Exception:
-            self._logger.exception(
-                f"Unexpected error during Kafka subscriber '{subscriber.fqn()}' initialization. Skipping it: {traceback.format_exc()}"
-            )
-
     @abstractmethod
     async def _create_kafka_consumer_executor(self, subscriber: MetaSubscriber) -> KafkaEventSubscriberExecutor:
         pass
 
     async def __polling_task(self) -> None:
+        self._logger.info("Initializing subscribers")
         try:
-            while not self.__should_stop.is_set():
-                total_size = sum([queue.get_total_size() for queue in self.__queue_per_consumer_mapper.values()])
-                if total_size >= self.__max_queue_size:
-                    await sleep(self.__seconds_between_polls_if_there_are_tasks_in_the_queue)
-                    continue
-
-                raw_consuming_tasks = await gather(
-                    *[
-                        self.__polling_consuming_tasks(kafka_consumer=consumer)
-                        for consumer, subscriber in self.__subscriber_per_consumer_mapper.items()
-                    ]
-                )
+            polling_task_per_consumer = [
+                create_task(self.__polling_consuming_tasks(consumer))
+                for consumer, subscriber in self.__queue_per_consumer_mapper.items()
+            ]
 
-                poll_results: list[ConsumingTask] = [
-                    consuming_task for consuming_tasks in raw_consuming_tasks for consuming_task in consuming_tasks
-                ]
-                if len(poll_results) == 0:
-                    await sleep(self.__seconds_between_polls_if_there_are_no_new_tasks)
-
-                for poll_result in poll_results:
-                    queue = self.__queue_per_consumer_mapper[poll_result.consumer]
-                    queue.push(
-                        key=TopicPartition(
-                            topic=poll_result.kafka_poll_record.topic, partition=poll_result.kafka_poll_record.partition
-                        ),
-                        record=poll_result.kafka_poll_record,
-                    )
+            await gather(*polling_task_per_consumer)
 
         except Exception:
             self._logger.error(f"Polling task failed with exception: {traceback.format_exc()}")
             self.__should_stop.set()
 
-        return
+    async def __polling_consuming_tasks(self, consumer: AIOKafkaConsumer) -> None:
+        queue = self.__queue_per_consumer_mapper[consumer]
 
-    async def __polling_consuming_tasks(self, kafka_consumer: AIOKafkaConsumer) -> list[ConsumingTask]:
-        async with self.__polling_tasks_semaphore:
-            results = await kafka_consumer.poll(
-                number_of_messages_to_poll=self.__max_records_retrieved_per_poll,
+        try:
+            self._logger.info(
+                f"initializing consumer group: '{consumer.get_consumer_group()}' subscribed to the topics: '{consumer.get_topics()}'"
             )
-        return [ConsumingTask(kafka_consumer, result) for result in results]
+            await consumer.init()
+            self._logger.info(f"initialized '{consumer.get_consumer_group()}'")
+        except Exception:
+            self._logger.exception(
+                f"Unexpected error during Kafka subscriber '{consumer.get_consumer_group()}' initialization. Skipping it: {traceback.format_exc()}"
+            )
+
+        while not self.__should_stop.is_set():
+            total_size = sum([queue.get_total_size() for queue in self.__queue_per_consumer_mapper.values()])
+            if total_size >= self.__max_queue_size:
+                await sleep(self.__seconds_between_polls_if_there_are_tasks_in_the_queue)
+                continue
+
+            async with self.__polling_tasks_semaphore:
+                kafka_poll_records = await consumer.poll(
+                    number_of_messages_to_poll=self.__max_records_retrieved_per_poll,
+                )
+
+                for kafka_poll_record in kafka_poll_records:
+                    queue.push(
+                        key=TopicPartition(
+                            topic=kafka_poll_record.topic,
+                            partition=kafka_poll_record.partition,
+                        ),
+                        record=kafka_poll_record,
+                    )
+
+            if len(kafka_poll_records) == 0:
+                await sleep(self.__seconds_between_polls_if_there_are_no_new_tasks)
 
     async def __consume_events_task(self) -> None:
-        blocked_tasks_iterator = self.generate_blocked_consuming_tasks_iterator()
+        self._logger.info("Initializing consuming task")
+        blocked_tasks_iterator = self.__generate_blocked_consuming_tasks_iterator()
 
         async for consuming_task in blocked_tasks_iterator:
             consumer = consuming_task.consumer
             kafka_poll_record = consuming_task.kafka_poll_record
-            executor = self.__executor_per_consumer_mapper[consuming_task.consumer]
 
+            executor = self.__executor_per_consumer_mapper[consumer]
             await executor.consume(kafka_poll_record=kafka_poll_record)
-
             await consumer.commit_poll_record(kafka_poll_record)
 
             self.__events_processed += 1
 
     # This iterator return a blocked task, that will be blocked for other process (like rebalancing), until the next task will be requested
-    async def generate_blocked_consuming_tasks_iterator(self) -> AsyncIterator[ConsumingTask]:
+    async def __generate_blocked_consuming_tasks_iterator(self) -> AsyncIterator[ConsumingTask]:
         consumer_queues_cyclic_iterator = cycle(self.__queue_per_consumer_mapper.items())
         last_consumer, _ = next(consumer_queues_cyclic_iterator)
 
         while not self.__should_stop.is_set():
-            all_queues_are_empty = all(
-                [queue.is_totally_empty() for queue in self.__queue_per_consumer_mapper.values()]
-            )
-
-            if all_queues_are_empty:
+            if await self.__all_queues_are_empty():
                 await sleep(self.__seconds_between_executions_if_there_are_no_tasks_in_the_queue)
                 continue
 
             async with self.__task_execution_mutex:
                 consumer: Optional[AIOKafkaConsumer] = None
-                kafka_poll_record: Optional[KafkaPollRecord] = None
 
                 while consumer != last_consumer:
                     consumer, queue = next(consumer_queues_cyclic_iterator)
@@ -274,7 +259,8 @@ class BaseBuzAIOKafkaAsyncConsumer(AsyncConsumer):
                         last_consumer = consumer
                         break
 
-        return
+    async def __all_queues_are_empty(self) -> bool:
+        return all([queue.is_totally_empty() for queue in self.__queue_per_consumer_mapper.values()])
 
     async def __on_partition_revoked(self, consumer: AIOKafkaConsumer, topics_partitions: set[TopicPartition]) -> None:
         async with self.__task_execution_mutex:

{buz-2.13.1rc6 → buz-2.13.1rc8}/src/buz/event/infrastructure/buz_kafka/buz_kafka_event_bus.py

@@ -14,6 +14,7 @@ from buz.event.middleware.publish_middleware_chain_resolver import PublishMiddle
 from buz.kafka import (
     KafkaPythonProducer,
 )
+from buz.kafka.domain.exceptions.topic_already_created_exception import KafkaTopicsAlreadyCreatedException
 from buz.kafka.domain.models.auto_create_topic_configuration import AutoCreateTopicConfiguration
 from buz.kafka.domain.models.create_kafka_topic import CreateKafkaTopic
 from buz.kafka.domain.services.kafka_admin_client import KafkaAdminClient
@@ -53,18 +54,22 @@ class BuzKafkaEventBus(EventBus):
             topic = self.__publish_strategy.get_topic(event)
 
             if self.__auto_create_topic_configuration is not None and self.__is_topic_created(topic) is False:
-                self.__get_kafka_admin_client().create_topics(
-                    topics=[
-                        CreateKafkaTopic(
-                            name=topic,
-                            partitions=self.__auto_create_topic_configuration.partitions,
-                            replication_factor=self.__auto_create_topic_configuration.replication_factor,
-                            configs=self.__auto_create_topic_configuration.configs,
-                        )
-                    ]
-                )
-                self.__logger.info(f"Created missing topic: {topic}")
-                self.__topics_checked[topic] = True
+                try:
+                    self.__logger.info(f"Creating missing topic: {topic}..")
+                    self.__get_kafka_admin_client().create_topics(
+                        topics=[
+                            CreateKafkaTopic(
+                                name=topic,
+                                partitions=self.__auto_create_topic_configuration.partitions,
+                                replication_factor=self.__auto_create_topic_configuration.replication_factor,
+                                configs=self.__auto_create_topic_configuration.configs,
+                            )
+                        ]
+                    )
+                    self.__logger.info(f"Created missing topic: {topic}")
+                    self.__topics_checked[topic] = True
+                except KafkaTopicsAlreadyCreatedException:
+                    pass
 
             headers = self.__get_event_headers(event)
             self.__producer.produce(

{buz-2.13.1rc6 → buz-2.13.1rc8}/src/buz/kafka/infrastructure/aiokafka/aiokafka_consumer.py

@@ -2,7 +2,7 @@ from __future__ import annotations
 
 from logging import Logger
 from ssl import SSLContext
-from typing import Awaitable, Callable, Optional, cast
+from typing import Awaitable, Callable, Optional, Sequence, cast
 
 from aiokafka import AIOKafkaConsumer as AIOKafkaNativeConsumer, TopicPartition, OffsetAndMetadata
 from aiokafka.helpers import create_ssl_context
@@ -56,6 +56,12 @@ class AIOKafkaConsumer:
         self.__check_kafka_admin_client_is_needed()
         self.__consumer = self.__generate_consumer()
 
+    def get_topics(self) -> Sequence[str]:
+        return list(self.__topics)
+
+    def get_consumer_group(self) -> str:
+        return self.__consumer_group
+
     def __check_kafka_admin_client_is_needed(self) -> None:
         if self.__kafka_admin_client is None and self.__auto_create_topic_configuration is not None:
             raise KafkaEventBusConfigNotValidException(
@@ -117,7 +123,7 @@ class AIOKafkaConsumer:
         try:
             self.__logger.info(f"Creating missing topics: {non_created_topics}...")
             kafka_admin_client.create_topics(topics=topics_to_create)
-            self.__logger.info(f"Created missing topics: {non_created_topics}...")
+            self.__logger.info(f"Created missing topics: {non_created_topics}")
         except KafkaTopicsAlreadyCreatedException:
             # there is a possibility to have a race condition between the check and the creation
             # but it does not matters, the important part is that the topic is created

{buz-2.13.1rc6 → buz-2.13.1rc8}/src/buz/kafka/infrastructure/cdc/cdc_message.py

@@ -3,9 +3,11 @@ from dataclasses import dataclass
 
 @dataclass(frozen=True)
 class CDCPayload:
+    DATE_TIME_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ"
+
     payload: str  # json encoded
     event_id: str  # uuid
-    created_at: str  # date and hour ISO 8601
+    created_at: str
     event_fqn: str
 
     def validate(self) -> None:

{buz-2.13.1rc6 → buz-2.13.1rc8}/src/buz/kafka/infrastructure/deserializers/implementations/cdc/cdc_record_bytes_to_event_deserializer.py

@@ -1,16 +1,17 @@
 from __future__ import annotations
 
+from datetime import datetime
 from typing import TypeVar, Type, Generic
 
 import orjson
 from dacite import from_dict
 
-from buz.kafka.infrastructure.deserializers.implementations.cdc.not_valid_cdc_message_exception import (
-    NotValidCDCMessageException,
-)
 from buz.event import Event
 from buz.kafka.infrastructure.cdc.cdc_message import CDCMessage, CDCPayload
 from buz.kafka.infrastructure.deserializers.bytes_to_message_deserializer import BytesToMessageDeserializer
+from buz.kafka.infrastructure.deserializers.implementations.cdc.not_valid_cdc_message_exception import (
+    NotValidCDCMessageException,
+)
 
 T = TypeVar("T", bound=Event)
 
@@ -27,12 +28,16 @@ class CDCRecordBytesToEventDeserializer(BytesToMessageDeserializer[Event], Gener
             cdc_message = self.__get_outbox_record_as_dict(decoded_string)
             return self.__event_class.restore(
                 id=cdc_message.payload.event_id,
-                created_at=cdc_message.payload.created_at,
+                created_at=self.__get_created_at_in_event_format(cdc_message.payload.created_at),
                 **orjson.loads(cdc_message.payload.payload),
             )
         except Exception as exception:
             raise NotValidCDCMessageException(decoded_string, exception) from exception
 
+    def __get_created_at_in_event_format(self, cdc_payload_created_at: str) -> str:
+        created_at_datetime = datetime.strptime(cdc_payload_created_at, CDCPayload.DATE_TIME_FORMAT)
+        return created_at_datetime.strftime(Event.DATE_TIME_FORMAT)
+
     def __get_outbox_record_as_dict(self, decoded_string: str) -> CDCMessage:
         decoded_record: dict = orjson.loads(decoded_string)
 

{buz-2.13.1rc6 → buz-2.13.1rc8}/src/buz/kafka/infrastructure/serializers/implementations/cdc_record_bytes_to_event_serializer.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 from dataclasses import asdict
+from datetime import datetime
 
 from buz.event import Event
 from buz.kafka.infrastructure.cdc.cdc_message import CDCMessage, CDCPayload
@@ -16,13 +17,17 @@ class CDCRecordBytesToEventSerializer(ByteSerializer):
         cdc_message: CDCMessage = CDCMessage(
             payload=CDCPayload(
                 event_id=data.id,
-                created_at=data.created_at,
+                created_at=self.__adapt_created_to_cdc_format(data.created_at),
                 event_fqn=data.fqn(),
                 payload=self.__serialize_payload(data),
             )
         )
         return self.__json_serializer.serialize(asdict(cdc_message))
 
+    def __adapt_created_to_cdc_format(self, created_at: str) -> str:
+        created_at_datetime = datetime.strptime(created_at, Event.DATE_TIME_FORMAT)
+        return created_at_datetime.strftime(CDCPayload.DATE_TIME_FORMAT)
+
     def __serialize_payload(self, event: Event) -> str:
         # Remove id and created at, because Transactional outbox is not adding them
         payload = asdict(event)

{buz-2.13.1rc6 → buz-2.13.1rc8}/src/buz/queue/in_memory/in_memory_multiqueue_repository.py

@@ -1,37 +1,20 @@
-from threading import Lock
 from queue import Queue, Empty
 from typing import Optional, TypeVar, cast
 
 from buz.queue.multiqueue_repository import MultiqueueRepository
 
-
 K = TypeVar("K")
 R = TypeVar("R")
 
 
-def self_mutex(method):
-    def call(self, *args, **kwargs):
-        lock: Lock = self._get_method_lock()  # type: ignore
-        with lock:
-            return method(self, *args, **kwargs)
-
-    return call
-
-
 class InMemoryMultiqueueRepository(MultiqueueRepository[K, R]):
     def __init__(self):
         self.__queues = cast(dict[K, Queue[R]], {})
-        self.__mutex = Lock()
         self.__last_key_index = 0
 
-    def _get_method_lock(self) -> Lock:
-        return self.__mutex
-
-    @self_mutex
     def clear(self, key: K) -> None:
         self.__queues.pop(key, None)
 
-    @self_mutex
     def push(self, key: K, record: R) -> None:
         if key not in self.__queues:
             self.__add_key(key)
@@ -41,7 +24,6 @@ class InMemoryMultiqueueRepository(MultiqueueRepository[K, R]):
     def __add_key(self, key: K) -> None:
         self.__queues[key] = Queue[R]()
 
-    @self_mutex
     def pop(self) -> Optional[R]:
         if not self.__queues:
             return None
@@ -65,10 +47,8 @@ class InMemoryMultiqueueRepository(MultiqueueRepository[K, R]):
 
         return None
 
-    @self_mutex
     def get_total_size(self) -> int:
         return sum([queue.qsize() for queue in self.__queues.values()])
 
-    @self_mutex
     def is_totally_empty(self) -> bool:
         return all([queue.empty() for queue in self.__queues.values()])