buz 2.15.7__tar.gz → 2.15.9__tar.gz

{buz-2.15.7 → buz-2.15.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: buz
-Version: 2.15.7
+Version: 2.15.9
 Summary: Buz is a set of light, simple and extensible implementations of event, command and query buses.
 License: MIT
 Author: Luis Pintado Lozano

{buz-2.15.7 → buz-2.15.9}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "buz"
-version = "2.15.7"
+version = "2.15.9"
 description = "Buz is a set of light, simple and extensible implementations of event, command and query buses."
 readme = "README.md"
 authors = ["Luis Pintado Lozano <luis.pintado.lozano@gmail.com>", "Gerardo Parra <gprauxiliar@gmail.com>"]

{buz-2.15.7 → buz-2.15.9}/src/buz/event/infrastructure/buz_kafka/base_buz_aiokafka_async_consumer.py

@@ -1,3 +1,4 @@
+from collections import defaultdict
 import json
 import traceback
 from abc import abstractmethod
@@ -39,6 +40,7 @@ class BaseBuzAIOKafkaAsyncConsumer(AsyncConsumer):
     __SECONDS_BETWEEN_EXECUTIONS_IF_THERE_ARE_NO_TASKS_IN_THE_QUEUE = 1
     __SECONDS_BETWEEN_POLLS_IF_THERE_ARE_TASKS_IN_THE_QUEUE = 1
     __SECONDS_BETWEEN_POLLS_IF_THERE_ARE_NO_NEW_TASKS = 1
+    __SECONDS_TO_WAIT_BETWEEN_REBALANCING_IN_PROGRESS = 0.1
     __MAX_NUMBER_OF_CONCURRENT_POLLING_TASKS = 20
 
     def __init__(
@@ -63,6 +65,7 @@ class BaseBuzAIOKafkaAsyncConsumer(AsyncConsumer):
         max_poll_interval_ms: Optional[int] = None,
         heartbeat_interval_ms: Optional[int] = None,
         wait_for_connection_to_cluster_ms: Optional[int] = None,
+        worker_instance_id: Optional[str] = None,
     ):
         self.__connection_config = connection_config
         self.__consume_strategy = consume_strategy
@@ -102,9 +105,11 @@ class BaseBuzAIOKafkaAsyncConsumer(AsyncConsumer):
             max_number_of_concurrent_polling_tasks or self.__MAX_NUMBER_OF_CONCURRENT_POLLING_TASKS
         )
         self.__wait_for_connection_to_cluster_ms: Optional[int] = wait_for_connection_to_cluster_ms
+        self.__worker_instance_id: Optional[str] = worker_instance_id
         self.__polling_tasks_semaphore = Semaphore(self.__max_number_of_concurrent_polling_tasks)
-        self.__task_execution_mutex = Lock()
+        self.__consumer_and_partition_mutex: dict[str, Lock] = defaultdict(Lock)
         self.__is_worked_initialized = False
+        self.__number_of_rebalancing_processes_in_progress: int = 0
 
     async def configure_http_check_server(self, health_check_port: int) -> web.TCPSite:
         self._logger.info(f"Starting health check server on port {health_check_port}")
@@ -148,6 +153,7 @@ class BaseBuzAIOKafkaAsyncConsumer(AsyncConsumer):
     def __print_worker_configuration(self) -> None:
         self._logger.info(
             f"Consumer configuration:\n"
+            f"  - Worker instance id: {self.__worker_instance_id}\n"
             f"  - Consume strategy: {self.__consume_strategy}\n"
             f"  - Max queue size: {self.__max_queue_size}\n"
             f"  - Max records retrieved per poll: {self.__max_records_retrieved_per_poll}\n"
@@ -212,6 +218,7 @@ class BaseBuzAIOKafkaAsyncConsumer(AsyncConsumer):
         topics = self.__consume_strategy.get_topics(subscriber)
         kafka_consumer = AIOKafkaConsumer(
             consumer_group=self.__consume_strategy.get_subscription_group(subscriber),
+            group_instance_id=self.__generate_consumer_group_instance_id(subscriber),
             topics=topics,
             connection_config=self.__connection_config,
             initial_offset_position=self.__consumer_initial_offset_position,
@@ -240,6 +247,11 @@ class BaseBuzAIOKafkaAsyncConsumer(AsyncConsumer):
             f"Initialized consumer group: '{kafka_consumer.get_consumer_group()}' subscribed to the topics: '{kafka_consumer.get_topics()}'"
         )
 
+    def __generate_consumer_group_instance_id(self, subscriber: MetaSubscriber) -> Optional[str]:
+        if self.__worker_instance_id is None:
+            return None
+        return f"{subscriber.fqn()}-{self.__worker_instance_id}"
+
     @abstractmethod
     async def _create_kafka_consumer_executor(self, subscriber: MetaSubscriber) -> KafkaEventSubscriberExecutor:
         pass
@@ -303,29 +315,59 @@ class BaseBuzAIOKafkaAsyncConsumer(AsyncConsumer):
         last_consumer, _ = next(consumer_queues_cyclic_iterator)
 
         while not self.__should_stop.is_set():
+            if self.is_rebalancing_in_progress():
+                self._logger.info("Waiting for rebalancing")
+                await sleep(self.__SECONDS_TO_WAIT_BETWEEN_REBALANCING_IN_PROGRESS)
+                continue
+
             if await self.__all_queues_are_empty():
                 await sleep(self.__seconds_between_executions_if_there_are_no_tasks_in_the_queue)
                 continue
 
-            async with self.__task_execution_mutex:
-                consumer: Optional[AIOKafkaConsumer] = None
+            consumer: Optional[AIOKafkaConsumer] = None
+
+            while consumer != last_consumer:
+                consumer, queue = next(consumer_queues_cyclic_iterator)
+                kafka_poll_record = queue.pop()
 
-                while consumer != last_consumer:
-                    consumer, queue = next(consumer_queues_cyclic_iterator)
-                    kafka_poll_record = queue.pop()
+                if kafka_poll_record is not None:
+                    consuming_task = ConsumingTask(consumer, kafka_poll_record)
+                    async with self.__get_partition_mutex(
+                        consumer_fqn=consuming_task.consumer.get_consumer_group(),
+                        partition=kafka_poll_record.partition,
+                    ):
+                        yield consuming_task
 
-                    if kafka_poll_record is not None:
-                        yield ConsumingTask(consumer, kafka_poll_record)
-                        last_consumer = consumer
-                        break
+                    last_consumer = consumer
+                    break
+
+    def __get_partition_mutex(self, consumer_fqn: str, partition: int) -> Lock:
+        mutex_key = f"consumer_{consumer_fqn}_partition_{partition}"
+        return self.__consumer_and_partition_mutex[mutex_key]
 
     async def __all_queues_are_empty(self) -> bool:
         return all([queue.is_totally_empty() for queue in self.__queue_per_consumer_mapper.values()])
 
     async def __on_partition_revoked(self, consumer: AIOKafkaConsumer, topics_partitions: set[TopicPartition]) -> None:
-        async with self.__task_execution_mutex:
-            for topic_partition in topics_partitions:
+        self._logger.info(f"rebalancing in progress, revoking partitions {topics_partitions}")
+
+        self.increase_number_of_rebalancing_in_progress()
+        for topic_partition in topics_partitions:
+            async with self.__get_partition_mutex(
+                consumer_fqn=consumer.get_consumer_group(),
+                partition=topic_partition.partition,
+            ):
                 self.__queue_per_consumer_mapper[consumer].clear(topic_partition)
+        self.decrease_number_of_rebalancing_in_progress()
+
+    def increase_number_of_rebalancing_in_progress(self) -> None:
+        self.__number_of_rebalancing_processes_in_progress += 1
+
+    def decrease_number_of_rebalancing_in_progress(self) -> None:
+        self.__number_of_rebalancing_processes_in_progress -= 1
+
+    def is_rebalancing_in_progress(self) -> bool:
+        return self.__number_of_rebalancing_processes_in_progress > 0
 
     def request_stop(self) -> None:
         self.__should_stop.set()

{buz-2.15.7 → buz-2.15.9}/src/buz/event/infrastructure/buz_kafka/buz_aiokafka_async_consumer.py

@@ -43,6 +43,7 @@ class BuzAIOKafkaAsyncConsumer(BaseBuzAIOKafkaAsyncConsumer):
         logger: Logger,
         consumer_initial_offset_position: ConsumerInitialOffsetPosition,
         deserializers_per_subscriber: dict[MetaSubscriber, BytesToMessageDeserializer[T]],
+        worker_instance_id: Optional[str] = None,
         consume_middlewares: Optional[Sequence[AsyncConsumeMiddleware]] = None,
         consume_retrier: Optional[ConsumeRetrier] = None,
         reject_callback: Optional[RejectCallback] = None,
@@ -77,6 +78,7 @@ class BuzAIOKafkaAsyncConsumer(BaseBuzAIOKafkaAsyncConsumer):
             heartbeat_interval_ms=heartbeat_interval_ms,
             health_check_port=health_check_port,
             wait_for_connection_to_cluster_ms=wait_for_connection_to_cluster_ms,
+            worker_instance_id=worker_instance_id,
         )
         self.__on_fail_strategy = on_fail_strategy
         self.__consume_middlewares = consume_middlewares

{buz-2.15.7 → buz-2.15.9}/src/buz/event/infrastructure/buz_kafka/buz_aiokafka_multi_threaded_consumer.py

@@ -44,6 +44,7 @@ class BuzAIOKafkaMultiThreadedConsumer(BaseBuzAIOKafkaAsyncConsumer):
         logger: Logger,
         consumer_initial_offset_position: ConsumerInitialOffsetPosition,
         deserializers_per_subscriber: dict[Subscriber, BytesToMessageDeserializer[T]],
+        worker_instance_id: Optional[str] = None,
         consume_middlewares: Optional[Sequence[ConsumeMiddleware]] = None,
         consume_retrier: Optional[ConsumeRetrier] = None,
         reject_callback: Optional[RejectCallback] = None,
@@ -78,6 +79,7 @@ class BuzAIOKafkaMultiThreadedConsumer(BaseBuzAIOKafkaAsyncConsumer):
             max_poll_interval_ms=max_poll_interval_ms,
             heartbeat_interval_ms=heartbeat_interval_ms,
             wait_for_connection_to_cluster_ms=wait_for_connection_to_cluster_ms,
+            worker_instance_id=worker_instance_id,
         )
         self.__on_fail_strategy = on_fail_strategy
         self.__consume_middlewares = consume_middlewares

{buz-2.15.7 → buz-2.15.9}/src/buz/kafka/infrastructure/aiokafka/aiokafka_consumer.py

@@ -43,6 +43,7 @@ class AIOKafkaConsumer:
         on_partition_revoked: Callable[[AIOKafkaConsumer, set[TopicPartition]], Awaitable[None]],
         auto_create_topic_configuration: Optional[AutoCreateTopicConfiguration] = None,
         wait_for_connection_to_cluster_ms: Optional[int] = None,
+        group_instance_id: Optional[str] = None,
     ) -> None:
         self.__consumer_group = consumer_group
         self.__topics = topics
@@ -57,6 +58,7 @@ class AIOKafkaConsumer:
         self.__max_poll_interval_ms = max_poll_interval_ms
         self.__heartbeat_interval_ms = heartbeat_interval_ms
         self.__wait_for_connection_to_cluster_ms = wait_for_connection_to_cluster_ms
+        self.__group_instance_id = group_instance_id
         self.__check_kafka_admin_client_is_needed()
         self.__consumer = self.__generate_consumer()
 
@@ -90,6 +92,7 @@ class AIOKafkaConsumer:
 
         consumer = AIOKafkaNativeConsumer(
             None,
+            group_instance_id=self.__group_instance_id,
             ssl_context=ssl_context,
             bootstrap_servers=",".join(self.__connection_config.bootstrap_servers),
             security_protocol=self.__connection_config.credentials.security_protocol.value,
@@ -163,10 +166,7 @@ class AIOKafkaConsumer:
         self,
         topics_partitions: set[TopicPartition],
     ) -> None:
-        # It could happen that the new partition assigned it was a previous one processed by this consumer, so we need to seek to the last committed offset
-        self.__consumer.pause(*topics_partitions)
-        await self.__consumer.seek_to_committed(*topics_partitions)
-        self.__consumer.resume(*topics_partitions)
+        return
 
     async def __on_partitions_revoked(
         self,