buz 2.14.2__tar.gz → 2.15.0__tar.gz

{buz-2.14.2 → buz-2.15.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: buz
-Version: 2.14.2
+Version: 2.15.0
 Summary: Buz is a set of light, simple and extensible implementations of event, command and query buses.
 License: MIT
 Author: Luis Pintado Lozano
@@ -22,6 +22,7 @@ Classifier: Typing :: Typed
 Provides-Extra: aiokafka
 Provides-Extra: kombu
 Provides-Extra: pypendency
+Requires-Dist: aiohttp (>=3.11.13,<4.0.0)
 Requires-Dist: aiokafka[lz4] (==0.12.0) ; extra == "aiokafka"
 Requires-Dist: asgiref (>=3.8.1,<4.0.0) ; extra == "aiokafka"
 Requires-Dist: asyncio (>=3.4.3,<4.0.0) ; extra == "aiokafka"

{buz-2.14.2 → buz-2.15.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "buz"
-version = "2.14.2"
+version = "2.15.0"
 description = "Buz is a set of light, simple and extensible implementations of event, command and query buses."
 readme = "README.md"
 authors = ["Luis Pintado Lozano <luis.pintado.lozano@gmail.com>", "Gerardo Parra <gprauxiliar@gmail.com>"]
@@ -31,6 +31,7 @@ asyncio = { version ="^3.4.3", optional = true }
 aiokafka = {version = "==0.12.0", extras = ["lz4"], optional = true}
 asgiref = {version = "^3.8.1", optional = true}
 cachetools = "^5.5.0"
+aiohttp = "^3.11.13"
 
 [tool.poetry.dev-dependencies]
 black = "^23.3"

{buz-2.14.2 → buz-2.15.0}/src/buz/event/exceptions/subscribers_not_found_exception.py

@@ -1,11 +1,15 @@
 from typing import Optional, Sequence
 
-from buz.event import Event, Subscriber
+from buz.event import Event
+from buz.event.meta_subscriber import MetaSubscriber
 
 
 class SubscribersNotFoundException(Exception):
     def __init__(
-        self, event: Event, allowed_subscriber_fqns: set[str], event_subscribers: Optional[Sequence[Subscriber]] = None
+        self,
+        event: Event,
+        allowed_subscriber_fqns: set[str],
+        event_subscribers: Optional[Sequence[MetaSubscriber]] = None,
     ) -> None:
         self.event = event
         self.allowed_subscriber_fqns = allowed_subscriber_fqns

{buz-2.14.2 → buz-2.15.0}/src/buz/event/infrastructure/buz_kafka/base_buz_aiokafka_async_consumer.py

@@ -1,3 +1,4 @@
+import json
 import traceback
 from abc import abstractmethod
 from asyncio import Lock, Task, create_task, gather, Semaphore, Event as AsyncIOEvent, sleep
@@ -5,6 +6,7 @@ from datetime import timedelta, datetime
 from itertools import cycle
 from logging import Logger
 from typing import AsyncIterator, Optional, Sequence, Type, TypeVar
+from aiohttp import web
 
 from aiokafka import TopicPartition
 from aiokafka.coordinator.assignors.abstract import AbstractPartitionAssignor
@@ -43,6 +45,7 @@ class BaseBuzAIOKafkaAsyncConsumer(AsyncConsumer):
         kafka_partition_assignors: tuple[Type[AbstractPartitionAssignor], ...] = (),
         subscribers: Sequence[MetaSubscriber],
         logger: Logger,
+        health_check_port: Optional[int],
         consumer_initial_offset_position: ConsumerInitialOffsetPosition,
         auto_create_topic_configuration: Optional[AutoCreateTopicConfiguration] = None,
         seconds_between_executions_if_there_are_no_tasks_in_the_queue: int = 1,
@@ -55,6 +58,7 @@ class BaseBuzAIOKafkaAsyncConsumer(AsyncConsumer):
         self.__kafka_partition_assignors = kafka_partition_assignors
         self.__subscribers = subscribers
         self._logger = logger
+        self.__health_check_port = health_check_port
         self.__consumer_initial_offset_position = consumer_initial_offset_position
         self.__max_records_retrieved_per_poll = 1
         self.__executor_per_consumer_mapper: dict[AIOKafkaConsumer, KafkaEventSubscriberExecutor] = {}
@@ -78,12 +82,26 @@ class BaseBuzAIOKafkaAsyncConsumer(AsyncConsumer):
         )
         self.__seconds_between_polls_if_there_are_no_new_tasks = seconds_between_polls_if_there_are_no_new_tasks
         self.__polling_tasks_semaphore = Semaphore(max_number_of_concurrent_polling_tasks)
-
         self.__task_execution_mutex = Lock()
 
+    async def configure_health_check_server(self, health_check_port: int) -> web.TCPSite:
+        self._logger.info(f"Starting health check server on port {health_check_port}")
+        app = web.Application()
+        app.router.add_get("/health", lambda request: self.__health_check())
+        runner = web.AppRunner(app)
+        await runner.setup()
+        site = web.TCPSite(runner, "localhost", health_check_port)
+        await site.start()
+        return site
+
     async def run(self) -> None:
         start_time = datetime.now()
         await self.__generate_kafka_consumers()
+        health_check_server: Optional[web.TCPSite] = None
+
+        if self.__health_check_port is not None:
+            health_check_server = await self.configure_health_check_server(self.__health_check_port)
+
         self.__initial_coroutines_created_elapsed_time = datetime.now() - start_time
 
         if len(self.__executor_per_consumer_mapper) == 0:
@@ -94,6 +112,9 @@ class BaseBuzAIOKafkaAsyncConsumer(AsyncConsumer):
         worker_errors = await self.__run_worker()
         self.__events_processed_elapsed_time = datetime.now() - start_consumption_time
 
+        if health_check_server is not None:
+            await health_check_server.stop()
+
         await self.__handle_graceful_stop(worker_errors)
 
     async def __handle_graceful_stop(self, worker_errors: tuple[Optional[Exception], Optional[Exception]]) -> None:
@@ -270,6 +291,15 @@ class BaseBuzAIOKafkaAsyncConsumer(AsyncConsumer):
         for kafka_consumer in self.__queue_per_consumer_mapper.keys():
             await kafka_consumer.stop()
 
+    async def __health_check(self) -> web.Response:
+        health_information = {
+            "subscribers": [subscriber.fqn() for subscriber in self.__subscribers],
+            "number_of_subscribers": len(self.__subscribers),
+            "event_processed": self.__events_processed,
+        }
+
+        return web.Response(text=json.dumps(health_information), content_type="application/json")
+
     def __print_statistics(self) -> None:
         self._logger.info("Number of subscribers: %d", len(self.__subscribers))
         self._logger.info(f"Start kafka consumers elapsed time: {self.__start_kafka_consumers_elapsed_time}")

{buz-2.14.2 → buz-2.15.0}/src/buz/event/infrastructure/buz_kafka/buz_aiokafka_async_consumer.py

@@ -51,6 +51,7 @@ class BuzAIOKafkaAsyncConsumer(BaseBuzAIOKafkaAsyncConsumer):
         seconds_between_polls_if_there_are_tasks_in_the_queue: int = 1,
         seconds_between_polls_if_there_are_no_new_tasks: int = 1,
         max_number_of_concurrent_polling_tasks: int = 20,
+        health_check_port: Optional[int] = None,
     ):
         super().__init__(
             connection_config=connection_config,
@@ -67,6 +68,7 @@ class BuzAIOKafkaAsyncConsumer(BaseBuzAIOKafkaAsyncConsumer):
             seconds_between_polls_if_there_are_tasks_in_the_queue=seconds_between_polls_if_there_are_tasks_in_the_queue,
             seconds_between_polls_if_there_are_no_new_tasks=seconds_between_polls_if_there_are_no_new_tasks,
             max_number_of_concurrent_polling_tasks=max_number_of_concurrent_polling_tasks,
+            health_check_port=health_check_port,
         )
         self.__on_fail_strategy = on_fail_strategy
         self.__consume_middlewares = consume_middlewares

{buz-2.14.2 → buz-2.15.0}/src/buz/event/infrastructure/buz_kafka/buz_aiokafka_multi_threaded_consumer.py

@@ -52,6 +52,7 @@ class BuzAIOKafkaMultiThreadedConsumer(BaseBuzAIOKafkaAsyncConsumer):
         seconds_between_polls_if_there_are_tasks_in_the_queue: int = 1,
         seconds_between_polls_if_there_are_no_new_tasks: int = 1,
         max_number_of_concurrent_polling_tasks: int = 20,
+        health_check_port: Optional[int] = 3123,
     ):
         super().__init__(
             connection_config=connection_config,
@@ -68,6 +69,7 @@ class BuzAIOKafkaMultiThreadedConsumer(BaseBuzAIOKafkaAsyncConsumer):
             seconds_between_polls_if_there_are_tasks_in_the_queue=seconds_between_polls_if_there_are_tasks_in_the_queue,
             seconds_between_polls_if_there_are_no_new_tasks=seconds_between_polls_if_there_are_no_new_tasks,
             max_number_of_concurrent_polling_tasks=max_number_of_concurrent_polling_tasks,
+            health_check_port=health_check_port,
         )
         self.__on_fail_strategy = on_fail_strategy
         self.__consume_middlewares = consume_middlewares

{buz-2.14.2 → buz-2.15.0}/src/buz/event/infrastructure/kombu/kombu_consumer.py

@@ -1,3 +1,4 @@
+import asyncio
 from logging import Logger
 from typing import Optional, Callable, cast
 
@@ -5,11 +6,14 @@ from kombu import Connection, Queue, Consumer as MessageConsumer, Message
 from kombu.mixins import ConsumerMixin
 from kombu.transport.pyamqp import Channel
 
-from buz.event import Event, Subscriber
+from buz.event import Event
+from buz.event.async_subscriber import AsyncSubscriber
 from buz.event.infrastructure.kombu.allowed_kombu_serializer import AllowedKombuSerializer
 from buz.event.exceptions.term_signal_interruption_exception import TermSignalInterruptionException
+from buz.event.meta_subscriber import MetaSubscriber
 from buz.event.middleware import ConsumeMiddleware, ConsumeMiddlewareChainResolver
 from buz.event.middleware.exceptions.event_already_in_progress_exception import EventAlreadyInProgressException
+from buz.event.subscriber import Subscriber
 from buz.locator import Locator
 from buz.event.consumer import Consumer
 from buz.event.exceptions.event_restore_exception import EventRestoreException
@@ -27,7 +31,7 @@ class KombuConsumer(ConsumerMixin, Consumer):
         queues_mapping: QueueToSubscriberFqnMapping,
         serializer: AllowedKombuSerializer,
         prefetch_count: int,
-        locator: Locator[Event, Subscriber],
+        locator: Locator[Event, MetaSubscriber],
         logger: Logger,
         consume_retrier: Optional[ConsumeRetrier] = None,
         reject_callback: Optional[RejectCallback] = None,
@@ -83,7 +87,7 @@ class KombuConsumer(ConsumerMixin, Consumer):
         except Exception as exc:
             raise EventRestoreException(body, str(message)) from exc
 
-    def __subscribers(self, event: Event, allowed_subscriber_fqns: set[str]) -> list[Subscriber]:
+    def __subscribers(self, event: Event, allowed_subscriber_fqns: set[str]) -> list[MetaSubscriber]:
         event_subscribers = None
         try:
             event_subscribers = self.__locator.get(event)
@@ -97,22 +101,41 @@ class KombuConsumer(ConsumerMixin, Consumer):
         except Exception as exc:
             raise SubscribersNotFoundException(event, allowed_subscriber_fqns, event_subscribers) from exc
 
-    def __consume_event(self, message: Message, event: Event, subscribers: list[Subscriber]) -> None:
+    def __consume_event(self, message: Message, event: Event, meta_subscribers: list[MetaSubscriber]) -> None:
         try:
             if self.should_stop is True:
                 raise TermSignalInterruptionException()
 
-            for subscriber in subscribers:
+            for meta_subscriber in meta_subscribers:
+                # The problem here is that the chain resolver works with syncsubscribers, an asyncsubscriber would require of a async function,
+                # but we are using run-until-complete to run the async function, so we are not really using the async function, we are just running it as a sync function, so we can cast the asyncsubscriber to a subscriber
+                subscriber = cast(Subscriber, meta_subscriber)
                 self.__consume_middleware_chain_resolver.resolve(event, subscriber, self.__perform_consume)
             message.ack()
         except Exception as exc:
-            self.__on_consume_exception(message, event, subscribers, exc)
+            self.__on_consume_exception(message, event, meta_subscribers, exc)
 
-    def __perform_consume(self, event: Event, subscriber: Subscriber) -> None:
-        subscriber.consume(event)
+    def __perform_consume(self, event: Event, subscriber: MetaSubscriber) -> None:
+        if isinstance(subscriber, AsyncSubscriber):
+            self.__get_or_create_event_loop().run_until_complete(subscriber.consume(event))
+            return
+
+        if isinstance(subscriber, Subscriber):
+            return subscriber.consume(event)
+
+        raise TypeError(f"Subscriber {subscriber} is not an instance of Subscriber or AsyncSubscriber")
+
+    def __get_or_create_event_loop(self) -> asyncio.AbstractEventLoop:
+        try:
+            return asyncio.get_event_loop()
+        except RuntimeError:
+            self.__logger.warning("The current thread does not have an event loop attached. Creating new event loop.")
+            event_loop = asyncio.new_event_loop()
+            asyncio.set_event_loop(event_loop)
+            return event_loop
 
     def __on_consume_exception(
-        self, message: Message, event: Event, subscribers: list[Subscriber], exception: Exception
+        self, message: Message, event: Event, subscribers: list[MetaSubscriber], exception: Exception
     ) -> None:
         if isinstance(exception, EventAlreadyInProgressException) or isinstance(
             exception, TermSignalInterruptionException
@@ -139,7 +162,7 @@ class KombuConsumer(ConsumerMixin, Consumer):
         self.__reject_message(message, event, subscribers, exception)
 
     def __reject_message(
-        self, message: Message, event: Event, subscribers: list[Subscriber], exception: Exception
+        self, message: Message, event: Event, subscribers: list[MetaSubscriber], exception: Exception
     ) -> None:
         message.reject()
         if self.__reject_callback is not None:

{buz-2.14.2 → buz-2.15.0}/src/buz/event/middleware/__init__.py

@@ -1,3 +1,4 @@
+from buz.event.middleware.base_async_consume_middleware import BaseAsyncConsumeMiddleware
 from buz.event.middleware.publish_middleware import PublishMiddleware, PublishCallable
 from buz.event.middleware.base_publish_middleware import BasePublishMiddleware
 from buz.event.middleware.publish_middleware_chain_resolver import PublishMiddlewareChainResolver
@@ -14,4 +15,5 @@ __all__ = [
     "ConsumeCallable",
     "BaseConsumeMiddleware",
     "ConsumeMiddlewareChainResolver",
+    "BaseAsyncConsumeMiddleware",
 ]

buz-2.15.0/src/buz/event/middleware/base_async_consume_middleware.py

@@ -0,0 +1,20 @@
+from abc import abstractmethod
+
+from buz.event import Event
+from buz.event.async_subscriber import AsyncSubscriber
+from buz.event.middleware.async_consume_middleware import AsyncConsumeMiddleware, AsyncConsumeCallable
+
+
+class BaseAsyncConsumeMiddleware(AsyncConsumeMiddleware):
+    async def on_consume(self, event: Event, subscriber: AsyncSubscriber, consume: AsyncConsumeCallable) -> None:
+        await self._before_on_consume(event, subscriber)
+        await consume(event, subscriber)
+        await self._after_on_consume(event, subscriber)
+
+    @abstractmethod
+    async def _before_on_consume(self, event: Event, subscriber: AsyncSubscriber) -> None:
+        pass
+
+    @abstractmethod
+    async def _after_on_consume(self, event: Event, subscriber: AsyncSubscriber) -> None:
+        pass