buz 2.13.0rc1__py3-none-any.whl → 2.13.1__py3-none-any.whl

buz/event/async_event_bus.py

@@ -0,0 +1,15 @@
+from abc import ABC, abstractmethod
+from typing import Collection
+
+from buz.event import Event
+from buz.kafka.infrastructure.interfaces.async_connection_manager import AsyncConnectionManager
+
+
+class AsyncEventBus(AsyncConnectionManager, ABC):
+    @abstractmethod
+    async def publish(self, event: Event) -> None:
+        pass
+
+    @abstractmethod
+    async def bulk_publish(self, events: Collection[Event]) -> None:
+        pass

buz/event/dead_letter_queue/dlq_record.py

@@ -8,6 +8,7 @@ DlqRecordId = UUID
 
 @dataclass
 class DlqRecord:
+    __EXCEPTION_MESSAGE_MAX_CHARACTERS: ClassVar[int] = 300
     DATE_TIME_FORMAT: ClassVar[str] = "%Y-%m-%d %H:%M:%S.%f"
 
     id: DlqRecordId
@@ -18,6 +19,18 @@ class DlqRecord:
     exception_message: str
     last_failed_at: datetime
 
+    def __post_init__(self) -> None:
+        self.exception_message = self.__add_ellipsis(self.exception_message)
+
+    def __add_ellipsis(self, message: str) -> str:
+        if len(message) <= self.__EXCEPTION_MESSAGE_MAX_CHARACTERS:
+            return message
+        return message[: self.__EXCEPTION_MESSAGE_MAX_CHARACTERS - 3] + "..."
+
+    def set_exception(self, exception: Exception) -> None:
+        self.exception_type = type(exception).__name__
+        self.exception_message = self.__add_ellipsis(str(exception))
+
     def mark_as_failed(self) -> None:
         self.last_failed_at = datetime.now()
 

buz/event/infrastructure/buz_kafka/async_buz_kafka_event_bus.py

@@ -0,0 +1,107 @@
+from logging import Logger
+from typing import Collection, Optional
+
+from buz.event import Event
+from buz.event.async_event_bus import AsyncEventBus
+from buz.event.exceptions.event_not_published_exception import EventNotPublishedException
+from buz.event.infrastructure.buz_kafka.exceptions.kafka_event_bus_config_not_valid_exception import (
+    KafkaEventBusConfigNotValidException,
+)
+from buz.event.infrastructure.buz_kafka.publish_strategy.publish_strategy import KafkaPublishStrategy
+from buz.event.middleware.async_publish_middleware import AsyncPublishMiddleware
+from buz.event.middleware.async_publish_middleware_chain_resolver import AsyncPublishMiddlewareChainResolver
+from buz.kafka.domain.exceptions.topic_already_created_exception import KafkaTopicsAlreadyCreatedException
+from buz.kafka.domain.models.auto_create_topic_configuration import AutoCreateTopicConfiguration
+from buz.kafka.domain.models.create_kafka_topic import CreateKafkaTopic
+from buz.kafka.domain.services.async_kafka_producer import AsyncKafkaProducer
+from buz.kafka.domain.services.kafka_admin_client import KafkaAdminClient
+
+
+class AsyncBuzKafkaEventBus(AsyncEventBus):
+    def __init__(
+        self,
+        *,
+        publish_strategy: KafkaPublishStrategy,
+        producer: AsyncKafkaProducer,
+        logger: Logger,
+        kafka_admin_client: Optional[KafkaAdminClient] = None,
+        publish_middlewares: Optional[list[AsyncPublishMiddleware]] = None,
+        auto_create_topic_configuration: Optional[AutoCreateTopicConfiguration] = None,
+    ):
+        self.__publish_middleware_chain_resolver = AsyncPublishMiddlewareChainResolver(publish_middlewares or [])
+        self.__publish_strategy = publish_strategy
+        self.__producer = producer
+        self.__topics_checked: dict[str, bool] = {}
+        self.__kafka_admin_client = kafka_admin_client
+        self.__auto_create_topic_configuration = auto_create_topic_configuration
+        self.__logger = logger
+        self.__check_kafka_admin_client_is_needed()
+
+    def __check_kafka_admin_client_is_needed(self) -> None:
+        if self.__kafka_admin_client is None and self.__auto_create_topic_configuration is not None:
+            raise KafkaEventBusConfigNotValidException(
+                "A KafkaAdminClient is needed to create topics when 'auto_create_topic_configuration' is set."
+            )
+
+    async def publish(self, event: Event) -> None:
+        await self.__publish_middleware_chain_resolver.resolve(event, self.__perform_publish)
+
+    async def __perform_publish(self, event: Event) -> None:
+        try:
+            topic = self.__publish_strategy.get_topic(event)
+
+            if self.__auto_create_topic_configuration is not None and self.__is_topic_created(topic) is False:
+                try:
+                    self.__logger.info(f"Creating missing topic: {topic}..")
+                    self.__get_kafka_admin_client().create_topics(
+                        topics=[
+                            CreateKafkaTopic(
+                                name=topic,
+                                partitions=self.__auto_create_topic_configuration.partitions,
+                                replication_factor=self.__auto_create_topic_configuration.replication_factor,
+                                configs=self.__auto_create_topic_configuration.configs,
+                            )
+                        ]
+                    )
+                    self.__logger.info(f"Created missing topic: {topic}")
+                    self.__topics_checked[topic] = True
+                except KafkaTopicsAlreadyCreatedException:
+                    pass
+
+            headers = self.__get_event_headers(event)
+            await self.__producer.produce(
+                message=event,
+                headers=headers,
+                topic=topic,
+            )
+        except Exception as exc:
+            raise EventNotPublishedException(event) from exc
+
+    def __get_kafka_admin_client(self) -> KafkaAdminClient:
+        if self.__kafka_admin_client is None:
+            raise KafkaEventBusConfigNotValidException("KafkaAdminClient is not set.")
+        return self.__kafka_admin_client
+
+    def __is_topic_created(self, topic: str) -> bool:
+        is_topic_created = self.__topics_checked.get(topic, None)
+
+        if is_topic_created is not None:
+            return is_topic_created
+
+        is_topic_created = self.__get_kafka_admin_client().is_topic_created(topic)
+        self.__topics_checked[topic] = is_topic_created
+
+        return is_topic_created
+
+    async def bulk_publish(self, events: Collection[Event]) -> None:
+        for event in events:
+            await self.publish(event)
+
+    def __get_event_headers(self, event: Event) -> dict:
+        return {"id": event.id}
+
+    async def connect(self) -> None:
+        await self.__producer.connect()
+
+    async def disconnect(self) -> None:
+        await self.__producer.disconnect()

buz/event/infrastructure/buz_kafka/base_buz_aiokafka_async_consumer.py

@@ -1,10 +1,10 @@
-from abc import abstractmethod
 import traceback
-from asyncio import Lock, gather, Semaphore, Event as AsyncIOEvent, sleep
+from abc import abstractmethod
+from asyncio import Lock, Task, create_task, gather, Semaphore, Event as AsyncIOEvent, sleep
 from datetime import timedelta, datetime
 from itertools import cycle
 from logging import Logger
-from typing import AsyncIterator, Coroutine, Optional, Sequence, Type, TypeVar
+from typing import AsyncIterator, Optional, Sequence, Type, TypeVar
 
 from aiokafka import TopicPartition
 from aiokafka.coordinator.assignors.abstract import AbstractPartitionAssignor
@@ -17,18 +17,15 @@ from buz.event.infrastructure.buz_kafka.consume_strategy.consume_strategy import
 from buz.event.infrastructure.buz_kafka.kafka_event_subscriber_executor import KafkaEventSubscriberExecutor
 from buz.event.infrastructure.models.consuming_task import ConsumingTask
 from buz.event.meta_subscriber import MetaSubscriber
-from buz.kafka import (
-    KafkaConnectionConfig,
-    ConsumerInitialOffsetPosition,
-)
 from buz.kafka.domain.models.auto_create_topic_configuration import AutoCreateTopicConfiguration
+from buz.kafka.domain.models.consumer_initial_offset_position import ConsumerInitialOffsetPosition
+from buz.kafka.domain.models.kafka_connection_config import KafkaConnectionConfig
 from buz.kafka.domain.models.kafka_poll_record import KafkaPollRecord
 from buz.kafka.domain.services.kafka_admin_client import KafkaAdminClient
 from buz.kafka.infrastructure.aiokafka.aiokafka_consumer import AIOKafkaConsumer
 from buz.queue.in_memory.in_memory_multiqueue_repository import InMemoryMultiqueueRepository
 from buz.queue.multiqueue_repository import MultiqueueRepository
 
-
 T = TypeVar("T", bound=Event)
 
 
@@ -60,7 +57,6 @@ class BaseBuzAIOKafkaAsyncConsumer(AsyncConsumer):
         self._logger = logger
         self.__consumer_initial_offset_position = consumer_initial_offset_position
         self.__max_records_retrieved_per_poll = 1
-        self.__subscriber_per_consumer_mapper: dict[AIOKafkaConsumer, MetaSubscriber] = {}
         self.__executor_per_consumer_mapper: dict[AIOKafkaConsumer, KafkaEventSubscriberExecutor] = {}
         self.__queue_per_consumer_mapper: dict[
             AIOKafkaConsumer, MultiqueueRepository[TopicPartition, KafkaPollRecord]
@@ -88,16 +84,13 @@ class BaseBuzAIOKafkaAsyncConsumer(AsyncConsumer):
     async def run(self) -> None:
         start_time = datetime.now()
         await self.__generate_kafka_consumers()
+        self.__initial_coroutines_created_elapsed_time = datetime.now() - start_time
 
-        if len(self.__subscriber_per_consumer_mapper) == 0:
+        if len(self.__executor_per_consumer_mapper) == 0:
             self._logger.error("There are no valid subscribers to execute, finalizing consumer")
             return
 
-        self.__create_queue_repository_per_consumer()
-        self.__initial_coroutines_created_elapsed_time = datetime.now() - start_time
-
         start_consumption_time = datetime.now()
-        self._logger.info("Starting to consume events")
         worker_errors = await self.__run_worker()
         self.__events_processed_elapsed_time = datetime.now() - start_consumption_time
 
@@ -112,16 +105,13 @@ class BaseBuzAIOKafkaAsyncConsumer(AsyncConsumer):
 
         if self.__exceptions_are_thrown(worker_errors):
             consume_events_exception, polling_task_exception = worker_errors
-            if consume_events_exception:
-                self._logger.error(consume_events_exception)
-            if polling_task_exception:
-                self._logger.error(polling_task_exception)
-
-            raise WorkerExecutionException("The worker was closed by an unexpected exception")
+            raise WorkerExecutionException(
+                "The worker was closed by an unexpected exception"
+            ) from consume_events_exception or polling_task_exception
 
     async def __run_worker(self) -> tuple[Optional[Exception], Optional[Exception]]:
-        consume_events_task = self.__consume_events_task()
-        polling_task = self.__polling_task()
+        consume_events_task = create_task(self.__consume_events_task())
+        polling_task = create_task(self.__polling_task())
 
         try:
             await gather(consume_events_task, polling_task)
@@ -132,9 +122,9 @@ class BaseBuzAIOKafkaAsyncConsumer(AsyncConsumer):
             polling_task_exception = await self.__await_exception(polling_task)
             return (consume_events_exception, polling_task_exception)
 
-    async def __await_exception(self, future: Coroutine) -> Optional[Exception]:
+    async def __await_exception(self, task: Task) -> Optional[Exception]:
         try:
-            await future
+            await task
             return None
         except Exception as exception:
             return exception
@@ -144,11 +134,11 @@ class BaseBuzAIOKafkaAsyncConsumer(AsyncConsumer):
 
     async def __generate_kafka_consumers(self):
         start_time = datetime.now()
-        tasks = [self.__initialize_kafka_consumer_for_subscriber(subscriber) for subscriber in self.__subscribers]
+        tasks = [self.__generate_kafka_consumer_for_subscriber(subscriber) for subscriber in self.__subscribers]
         await gather(*tasks)
         self.__start_kafka_consumers_elapsed_time = datetime.now() - start_time
 
-    async def __initialize_kafka_consumer_for_subscriber(self, subscriber: MetaSubscriber) -> None:
+    async def __generate_kafka_consumer_for_subscriber(self, subscriber: MetaSubscriber) -> None:
         try:
             executor = await self._create_kafka_consumer_executor(subscriber)
             topics = self.__consume_strategy.get_topics(subscriber)
@@ -164,98 +154,96 @@ class BaseBuzAIOKafkaAsyncConsumer(AsyncConsumer):
                 on_partition_revoked=self.__on_partition_revoked,
             )
 
-            self.__subscriber_per_consumer_mapper[kafka_consumer] = subscriber
-
             self.__executor_per_consumer_mapper[kafka_consumer] = executor
 
-            await kafka_consumer.init()
+            self.__queue_per_consumer_mapper[kafka_consumer] = InMemoryMultiqueueRepository()
+
         except Exception:
             self._logger.exception(
-                f"Unexpected error during Kafka subscriber '{subscriber.fqn()}' initialization. Skipping it: {traceback.format_exc()}"
+                f"Unexpected error during Kafka subscriber '{subscriber.fqn()}' creation. Skipping it: {traceback.format_exc()}"
             )
 
     @abstractmethod
     async def _create_kafka_consumer_executor(self, subscriber: MetaSubscriber) -> KafkaEventSubscriberExecutor:
         pass
 
-    def __create_queue_repository_per_consumer(self) -> None:
-        for kafka_consumer in self.__subscriber_per_consumer_mapper.keys():
-            self.__queue_per_consumer_mapper[kafka_consumer] = InMemoryMultiqueueRepository()
-
     async def __polling_task(self) -> None:
+        self._logger.info("Initializing subscribers")
         try:
-            while not self.__should_stop.is_set():
-                total_size = sum([queue.get_total_size() for queue in self.__queue_per_consumer_mapper.values()])
-                if total_size >= self.__max_queue_size:
-                    await sleep(self.__seconds_between_polls_if_there_are_tasks_in_the_queue)
-                    continue
-
-                raw_consuming_tasks = await gather(
-                    *[
-                        self.__polling_consuming_tasks(kafka_consumer=consumer)
-                        for consumer, subscriber in self.__subscriber_per_consumer_mapper.items()
-                    ]
-                )
-
-                poll_results: list[ConsumingTask] = [
-                    consuming_task for consuming_tasks in raw_consuming_tasks for consuming_task in consuming_tasks
-                ]
-                if len(poll_results) == 0:
-                    await sleep(self.__seconds_between_polls_if_there_are_no_new_tasks)
+            polling_task_per_consumer = [
+                create_task(self.__polling_consuming_tasks(consumer))
+                for consumer, subscriber in self.__queue_per_consumer_mapper.items()
+            ]
 
-                for poll_result in poll_results:
-                    queue = self.__queue_per_consumer_mapper[poll_result.consumer]
-                    queue.push(
-                        key=TopicPartition(
-                            topic=poll_result.kafka_poll_record.topic, partition=poll_result.kafka_poll_record.partition
-                        ),
-                        record=poll_result.kafka_poll_record,
-                    )
+            await gather(*polling_task_per_consumer)
 
         except Exception:
             self._logger.error(f"Polling task failed with exception: {traceback.format_exc()}")
             self.__should_stop.set()
 
-        return
+    async def __polling_consuming_tasks(self, consumer: AIOKafkaConsumer) -> None:
+        queue = self.__queue_per_consumer_mapper[consumer]
 
-    async def __polling_consuming_tasks(self, kafka_consumer: AIOKafkaConsumer) -> list[ConsumingTask]:
-        async with self.__polling_tasks_semaphore:
-            results = await kafka_consumer.poll(
-                number_of_messages_to_poll=self.__max_records_retrieved_per_poll,
+        try:
+            self._logger.info(
+                f"initializing consumer group: '{consumer.get_consumer_group()}' subscribed to the topics: '{consumer.get_topics()}'"
             )
-        return [ConsumingTask(kafka_consumer, result) for result in results]
+            await consumer.init()
+            self._logger.info(f"initialized '{consumer.get_consumer_group()}'")
+        except Exception:
+            self._logger.exception(
+                f"Unexpected error during Kafka subscriber '{consumer.get_consumer_group()}' initialization. Skipping it: {traceback.format_exc()}"
+            )
+
+        while not self.__should_stop.is_set():
+            total_size = sum([queue.get_total_size() for queue in self.__queue_per_consumer_mapper.values()])
+            if total_size >= self.__max_queue_size:
+                await sleep(self.__seconds_between_polls_if_there_are_tasks_in_the_queue)
+                continue
+
+            async with self.__polling_tasks_semaphore:
+                kafka_poll_records = await consumer.poll(
+                    number_of_messages_to_poll=self.__max_records_retrieved_per_poll,
+                )
+
+                for kafka_poll_record in kafka_poll_records:
+                    queue.push(
+                        key=TopicPartition(
+                            topic=kafka_poll_record.topic,
+                            partition=kafka_poll_record.partition,
+                        ),
+                        record=kafka_poll_record,
+                    )
+
+            if len(kafka_poll_records) == 0:
+                await sleep(self.__seconds_between_polls_if_there_are_no_new_tasks)
 
     async def __consume_events_task(self) -> None:
-        blocked_tasks_iterator = self.generate_blocked_consuming_tasks_iterator()
+        self._logger.info("Initializing consuming task")
+        blocked_tasks_iterator = self.__generate_blocked_consuming_tasks_iterator()
 
         async for consuming_task in blocked_tasks_iterator:
             consumer = consuming_task.consumer
             kafka_poll_record = consuming_task.kafka_poll_record
-            executor = self.__executor_per_consumer_mapper[consuming_task.consumer]
 
+            executor = self.__executor_per_consumer_mapper[consumer]
             await executor.consume(kafka_poll_record=kafka_poll_record)
-
             await consumer.commit_poll_record(kafka_poll_record)
 
             self.__events_processed += 1
 
     # This iterator return a blocked task, that will be blocked for other process (like rebalancing), until the next task will be requested
-    async def generate_blocked_consuming_tasks_iterator(self) -> AsyncIterator[ConsumingTask]:
+    async def __generate_blocked_consuming_tasks_iterator(self) -> AsyncIterator[ConsumingTask]:
         consumer_queues_cyclic_iterator = cycle(self.__queue_per_consumer_mapper.items())
         last_consumer, _ = next(consumer_queues_cyclic_iterator)
 
         while not self.__should_stop.is_set():
-            all_queues_are_empty = all(
-                [queue.is_totally_empty() for queue in self.__queue_per_consumer_mapper.values()]
-            )
-
-            if all_queues_are_empty:
+            if await self.__all_queues_are_empty():
                 await sleep(self.__seconds_between_executions_if_there_are_no_tasks_in_the_queue)
                 continue
 
             async with self.__task_execution_mutex:
                 consumer: Optional[AIOKafkaConsumer] = None
-                kafka_poll_record: Optional[KafkaPollRecord] = None
 
                 while consumer != last_consumer:
                     consumer, queue = next(consumer_queues_cyclic_iterator)
@@ -266,7 +254,8 @@ class BaseBuzAIOKafkaAsyncConsumer(AsyncConsumer):
                         last_consumer = consumer
                         break
 
-        return
+    async def __all_queues_are_empty(self) -> bool:
+        return all([queue.is_totally_empty() for queue in self.__queue_per_consumer_mapper.values()])
 
     async def __on_partition_revoked(self, consumer: AIOKafkaConsumer, topics_partitions: set[TopicPartition]) -> None:
         async with self.__task_execution_mutex:
@@ -278,7 +267,7 @@ class BaseBuzAIOKafkaAsyncConsumer(AsyncConsumer):
         self._logger.info("Worker stop requested. Waiting for finalize the current task")
 
     async def __manage_kafka_consumers_stopping(self) -> None:
-        for kafka_consumer in self.__subscriber_per_consumer_mapper.keys():
+        for kafka_consumer in self.__queue_per_consumer_mapper.keys():
             await kafka_consumer.stop()
 
     def __print_statistics(self) -> None:

buz/event/infrastructure/buz_kafka/buz_aiokafka_async_consumer.py

@@ -15,11 +15,10 @@ from buz.event.meta_subscriber import MetaSubscriber
 from buz.event.middleware.async_consume_middleware import AsyncConsumeMiddleware
 from buz.event.strategies.retry.consume_retrier import ConsumeRetrier
 from buz.event.strategies.retry.reject_callback import RejectCallback
-from buz.kafka import (
-    KafkaConnectionConfig,
-    ConsumerInitialOffsetPosition,
-)
+
 from buz.kafka.domain.models.auto_create_topic_configuration import AutoCreateTopicConfiguration
+from buz.kafka.domain.models.consumer_initial_offset_position import ConsumerInitialOffsetPosition
+from buz.kafka.domain.models.kafka_connection_config import KafkaConnectionConfig
 from buz.kafka.domain.services.kafka_admin_client import KafkaAdminClient
 from buz.kafka.infrastructure.deserializers.bytes_to_message_deserializer import BytesToMessageDeserializer
 from buz.kafka.infrastructure.deserializers.implementations.json_bytes_to_message_deserializer import (

buz/event/infrastructure/buz_kafka/buz_aiokafka_multi_threaded_consumer.py

@@ -15,11 +15,9 @@ from buz.event.meta_subscriber import MetaSubscriber
 from buz.event.middleware.consume_middleware import ConsumeMiddleware
 from buz.event.strategies.retry.consume_retrier import ConsumeRetrier
 from buz.event.strategies.retry.reject_callback import RejectCallback
-from buz.kafka import (
-    KafkaConnectionConfig,
-    ConsumerInitialOffsetPosition,
-)
 from buz.kafka.domain.models.auto_create_topic_configuration import AutoCreateTopicConfiguration
+from buz.kafka.domain.models.consumer_initial_offset_position import ConsumerInitialOffsetPosition
+from buz.kafka.domain.models.kafka_connection_config import KafkaConnectionConfig
 from buz.kafka.domain.services.kafka_admin_client import KafkaAdminClient
 from buz.kafka.infrastructure.deserializers.bytes_to_message_deserializer import BytesToMessageDeserializer
 from buz.kafka.infrastructure.deserializers.implementations.json_bytes_to_message_deserializer import (

buz/event/infrastructure/buz_kafka/buz_kafka_event_bus.py

@@ -11,12 +11,11 @@ from buz.event.middleware import (
     PublishMiddleware,
 )
 from buz.event.middleware.publish_middleware_chain_resolver import PublishMiddlewareChainResolver
-from buz.kafka import (
-    KafkaPythonProducer,
-)
+from buz.kafka.domain.exceptions.topic_already_created_exception import KafkaTopicsAlreadyCreatedException
 from buz.kafka.domain.models.auto_create_topic_configuration import AutoCreateTopicConfiguration
 from buz.kafka.domain.models.create_kafka_topic import CreateKafkaTopic
 from buz.kafka.domain.services.kafka_admin_client import KafkaAdminClient
+from buz.kafka.domain.services.kafka_producer import KafkaProducer
 
 
 class BuzKafkaEventBus(EventBus):
@@ -24,7 +23,7 @@ class BuzKafkaEventBus(EventBus):
         self,
         *,
         publish_strategy: KafkaPublishStrategy,
-        producer: KafkaPythonProducer,
+        producer: KafkaProducer,
         logger: Logger,
         kafka_admin_client: Optional[KafkaAdminClient] = None,
         publish_middlewares: Optional[list[PublishMiddleware]] = None,
@@ -53,18 +52,22 @@ class BuzKafkaEventBus(EventBus):
             topic = self.__publish_strategy.get_topic(event)
 
             if self.__auto_create_topic_configuration is not None and self.__is_topic_created(topic) is False:
-                self.__get_kafka_admin_client().create_topics(
-                    topics=[
-                        CreateKafkaTopic(
-                            name=topic,
-                            partitions=self.__auto_create_topic_configuration.partitions,
-                            replication_factor=self.__auto_create_topic_configuration.replication_factor,
-                            configs=self.__auto_create_topic_configuration.configs,
-                        )
-                    ]
-                )
-                self.__logger.info(f"Created missing topic: {topic}")
-                self.__topics_checked[topic] = True
+                try:
+                    self.__logger.info(f"Creating missing topic: {topic}..")
+                    self.__get_kafka_admin_client().create_topics(
+                        topics=[
+                            CreateKafkaTopic(
+                                name=topic,
+                                partitions=self.__auto_create_topic_configuration.partitions,
+                                replication_factor=self.__auto_create_topic_configuration.replication_factor,
+                                configs=self.__auto_create_topic_configuration.configs,
+                            )
+                        ]
+                    )
+                    self.__logger.info(f"Created missing topic: {topic}")
+                    self.__topics_checked[topic] = True
+                except KafkaTopicsAlreadyCreatedException:
+                    pass
 
             headers = self.__get_event_headers(event)
             self.__producer.produce(

buz/event/infrastructure/buz_kafka/kafka_event_sync_subscriber_executor.py

@@ -67,11 +67,6 @@ class KafkaEventSyncSubscriberExecutor(KafkaEventSubscriberExecutor):
             self.__logger.error(
                 f'The message "{str(kafka_poll_record.value)}" is not valid, it will be consumed but not processed'
             )
-        except Exception as exception:
-            if self.__on_fail_strategy == KafkaOnFailStrategy.CONSUME_ON_FAIL:
-                self.__logger.error(f"Error consuming event: {exception}")
-                return
-            raise exception
 
     def __execution_callback(self, subscriber: Subscriber, message: KafkaConsumerRecord[Event]) -> None:
         self.__consume_middleware_chain_resolver.resolve(
@@ -86,13 +81,13 @@ class KafkaEventSyncSubscriberExecutor(KafkaEventSubscriberExecutor):
                 return
             except Exception as exception:
                 self.__logger.warning(f"Event {event.id} could not be consumed by the subscriber {subscriber.fqn}")
-                self.__logger.error(exception, exc_info=True)
-
                 if self.__should_retry(event, subscriber) is True:
                     self.__register_retry(event, subscriber)
                     time.sleep(self.__seconds_between_retires)
                     continue
 
+                self.__logger.exception(exception)
+
                 if self.__reject_callback:
                     self.__reject_callback.on_reject(event=event, subscribers=[subscriber], exception=exception)
 

buz/event/infrastructure/kombu/kombu_consumer.py

@@ -135,6 +135,7 @@ class KombuConsumer(ConsumerMixin, Consumer):
             message.requeue()
             return
 
+        self.__logger.exception(exception)
         self.__reject_message(message, event, subscribers, exception)
 
     def __reject_message(

buz/event/middleware/async_publish_middleware.py

@@ -0,0 +1,13 @@
+from abc import abstractmethod
+from typing import Awaitable, Callable
+
+from buz.event import Event
+from buz.middleware import Middleware
+
+AsyncPublishCallable = Callable[[Event], Awaitable[None]]
+
+
+class AsyncPublishMiddleware(Middleware):
+    @abstractmethod
+    async def on_publish(self, event: Event, publish: AsyncPublishCallable) -> None:
+        pass

buz/event/middleware/async_publish_middleware_chain_resolver.py

@@ -0,0 +1,22 @@
+from buz.event import Event
+from buz.event.middleware.async_publish_middleware import AsyncPublishCallable, AsyncPublishMiddleware
+from buz.middleware import MiddlewareChainBuilder
+
+
+class AsyncPublishMiddlewareChainResolver:
+    def __init__(self, middlewares: list[AsyncPublishMiddleware]):
+        self.__middlewares = middlewares
+        self.__middleware_chain_builder: MiddlewareChainBuilder[
+            AsyncPublishCallable, AsyncPublishMiddleware
+        ] = MiddlewareChainBuilder(middlewares)
+
+    async def resolve(self, event: Event, publish: AsyncPublishCallable) -> None:
+        chain_callable: AsyncPublishCallable = self.__middleware_chain_builder.get_chain_callable(
+            publish, self.__get_middleware_callable
+        )
+        await chain_callable(event)
+
+    def __get_middleware_callable(
+        self, middleware: AsyncPublishMiddleware, publish_callable: AsyncPublishCallable
+    ) -> AsyncPublishCallable:
+        return lambda event: middleware.on_publish(event, publish_callable)

buz/kafka/__init__.py

@@ -1,18 +1,32 @@
+from buz.kafka.domain.exceptions.not_all_partition_assigned_exception import NotAllPartitionAssignedException
+from buz.kafka.domain.exceptions.not_valid_kafka_message_exception import NotValidKafkaMessageException
 from buz.kafka.domain.exceptions.topic_already_created_exception import KafkaTopicsAlreadyCreatedException
+from buz.kafka.domain.exceptions.topic_not_found_exception import TopicNotFoundException
+from buz.kafka.domain.models.auto_create_topic_configuration import AutoCreateTopicConfiguration
 from buz.kafka.domain.models.consumer_initial_offset_position import ConsumerInitialOffsetPosition
 from buz.kafka.domain.models.kafka_connection_config import KafkaConnectionConfig
+from buz.kafka.domain.models.kafka_connection_credentials import KafkaConnectionCredentials
+from buz.kafka.domain.models.kafka_connection_plain_text_credentials import KafkaConnectionPlainTextCredentials
+from buz.kafka.domain.models.kafka_connection_sasl_credentials import KafkaConnectionSaslCredentials
 from buz.kafka.domain.models.kafka_consumer_record import KafkaConsumerRecord
+from buz.kafka.domain.models.kafka_supported_sasl_mechanisms import KafkaSupportedSaslMechanisms
 from buz.kafka.domain.models.kafka_supported_security_protocols import KafkaSupportedSecurityProtocols
 from buz.kafka.domain.models.create_kafka_topic import CreateKafkaTopic
 from buz.kafka.domain.services.kafka_admin_client import KafkaAdminClient
 from buz.kafka.domain.services.kafka_admin_test_client import KafkaAdminTestClient
 from buz.kafka.domain.services.kafka_producer import KafkaProducer
-from buz.kafka.infrastructure.kafka_python.factories.kafka_python_producer_factory import KafkaPythonProducerFactory
+from buz.kafka.infrastructure.aiokafka.aiokafka_producer import AIOKafkaProducer
 from buz.kafka.infrastructure.kafka_python.kafka_python_admin_client import KafkaPythonAdminClient
 from buz.kafka.infrastructure.kafka_python.kafka_python_admin_test_client import KafkaPythonAdminTestClient
 from buz.kafka.infrastructure.kafka_python.kafka_python_producer import KafkaPythonProducer
 from buz.kafka.infrastructure.serializers.byte_serializer import ByteSerializer
 from buz.kafka.infrastructure.serializers.implementations.json_byte_serializer import JSONByteSerializer
+from buz.kafka.domain.models.kafka_supported_compression_type import KafkaSupportedCompressionType
+from buz.event.infrastructure.buz_kafka.exceptions.kafka_event_bus_config_not_valid_exception import (
+    KafkaEventBusConfigNotValidException,
+)
+from buz.event.infrastructure.buz_kafka.async_buz_kafka_event_bus import AsyncBuzKafkaEventBus
+from buz.event.infrastructure.buz_kafka.buz_kafka_event_bus import BuzKafkaEventBus
 
 
 __all__ = [
@@ -22,7 +36,6 @@ __all__ = [
     "KafkaAdminTestClient",
     "KafkaPythonAdminClient",
     "KafkaPythonAdminTestClient",
-    "KafkaPythonProducerFactory",
     "KafkaTopicsAlreadyCreatedException",
     "KafkaConsumerRecord",
     "CreateKafkaTopic",
@@ -31,4 +44,17 @@ __all__ = [
     "ByteSerializer",
     "JSONByteSerializer",
     "ConsumerInitialOffsetPosition",
+    "KafkaSupportedCompressionType",
+    "KafkaEventBusConfigNotValidException",
+    "AsyncBuzKafkaEventBus",
+    "BuzKafkaEventBus",
+    "AutoCreateTopicConfiguration",
+    "NotAllPartitionAssignedException",
+    "NotValidKafkaMessageException",
+    "TopicNotFoundException",
+    "KafkaConnectionCredentials",
+    "KafkaConnectionPlainTextCredentials",
+    "KafkaConnectionSaslCredentials",
+    "KafkaSupportedSaslMechanisms",
+    "AIOKafkaProducer",
 ]

buz/kafka/domain/exceptions/not_all_partition_assigned_exception.py

@@ -0,0 +1,8 @@
+from __future__ import annotations
+
+
+class NotAllPartitionAssignedException(Exception):
+    def __init__(self, topic_name: str) -> None:
+        super().__init__(
+            f'Not all the partition were assigned for the topic "{topic_name}", please disconnect the rest of subscribers'
+        )