buz 2.15.7__tar.gz → 2.15.8rc1__tar.gz

{buz-2.15.7 → buz-2.15.8rc1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: buz
-Version: 2.15.7
+Version: 2.15.8rc1
 Summary: Buz is a set of light, simple and extensible implementations of event, command and query buses.
 License: MIT
 Author: Luis Pintado Lozano

{buz-2.15.7 → buz-2.15.8rc1}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "buz"
-version = "2.15.7"
+version = "2.15.8-rc1"
 description = "Buz is a set of light, simple and extensible implementations of event, command and query buses."
 readme = "README.md"
 authors = ["Luis Pintado Lozano <luis.pintado.lozano@gmail.com>", "Gerardo Parra <gprauxiliar@gmail.com>"]

{buz-2.15.7 → buz-2.15.8rc1}/src/buz/event/infrastructure/buz_kafka/base_buz_aiokafka_async_consumer.py

@@ -1,3 +1,4 @@
+from collections import defaultdict
 import json
 import traceback
 from abc import abstractmethod
@@ -39,6 +40,7 @@ class BaseBuzAIOKafkaAsyncConsumer(AsyncConsumer):
     __SECONDS_BETWEEN_EXECUTIONS_IF_THERE_ARE_NO_TASKS_IN_THE_QUEUE = 1
     __SECONDS_BETWEEN_POLLS_IF_THERE_ARE_TASKS_IN_THE_QUEUE = 1
     __SECONDS_BETWEEN_POLLS_IF_THERE_ARE_NO_NEW_TASKS = 1
+    __SECONDS_TO_WAIT_BETWEEN_REBALANCING_IN_PROGRESS = 0.1
     __MAX_NUMBER_OF_CONCURRENT_POLLING_TASKS = 20
 
     def __init__(
@@ -103,8 +105,9 @@ class BaseBuzAIOKafkaAsyncConsumer(AsyncConsumer):
         )
         self.__wait_for_connection_to_cluster_ms: Optional[int] = wait_for_connection_to_cluster_ms
         self.__polling_tasks_semaphore = Semaphore(self.__max_number_of_concurrent_polling_tasks)
-        self.__task_execution_mutex = Lock()
+        self.__consumer_and_partition_mutex: dict[str, Lock] = defaultdict(Lock)
         self.__is_worked_initialized = False
+        self.__number_of_rebalancing_processes_in_progress: int = 0
 
     async def configure_http_check_server(self, health_check_port: int) -> web.TCPSite:
         self._logger.info(f"Starting health check server on port {health_check_port}")
@@ -303,29 +306,59 @@ class BaseBuzAIOKafkaAsyncConsumer(AsyncConsumer):
         last_consumer, _ = next(consumer_queues_cyclic_iterator)
 
         while not self.__should_stop.is_set():
+            if self.is_rebalancing_in_progress():
+                self._logger.info("Waiting for rebalancing")
+                await sleep(self.__SECONDS_TO_WAIT_BETWEEN_REBALANCING_IN_PROGRESS)
+                continue
+
             if await self.__all_queues_are_empty():
                 await sleep(self.__seconds_between_executions_if_there_are_no_tasks_in_the_queue)
                 continue
 
-            async with self.__task_execution_mutex:
-                consumer: Optional[AIOKafkaConsumer] = None
+            consumer: Optional[AIOKafkaConsumer] = None
+
+            while consumer != last_consumer:
+                consumer, queue = next(consumer_queues_cyclic_iterator)
+                kafka_poll_record = queue.pop()
+
+                if kafka_poll_record is not None:
+                    consuming_task = ConsumingTask(consumer, kafka_poll_record)
+                    async with self.__get_partition_mutex(
+                        consumer_fqn=consuming_task.consumer.get_consumer_group(),
+                        partition=kafka_poll_record.partition,
+                    ):
+                        yield consuming_task
 
-                while consumer != last_consumer:
-                    consumer, queue = next(consumer_queues_cyclic_iterator)
-                    kafka_poll_record = queue.pop()
+                    last_consumer = consumer
+                    break
 
-                    if kafka_poll_record is not None:
-                        yield ConsumingTask(consumer, kafka_poll_record)
-                        last_consumer = consumer
-                        break
+    def __get_partition_mutex(self, consumer_fqn: str, partition: int) -> Lock:
+        mutex_key = f"consumer_{consumer_fqn}_partition_{partition}"
+        return self.__consumer_and_partition_mutex[mutex_key]
 
     async def __all_queues_are_empty(self) -> bool:
         return all([queue.is_totally_empty() for queue in self.__queue_per_consumer_mapper.values()])
 
     async def __on_partition_revoked(self, consumer: AIOKafkaConsumer, topics_partitions: set[TopicPartition]) -> None:
-        async with self.__task_execution_mutex:
-            for topic_partition in topics_partitions:
+        self._logger.info(f"rebalancing in progress, revoking partitions {topics_partitions}")
+
+        self.increase_number_of_rebalancing_in_progress()
+        for topic_partition in topics_partitions:
+            async with self.__get_partition_mutex(
+                consumer_fqn=consumer.get_consumer_group(),
+                partition=topic_partition.partition,
+            ):
                 self.__queue_per_consumer_mapper[consumer].clear(topic_partition)
+        self.decrease_number_of_rebalancing_in_progress()
+
+    def increase_number_of_rebalancing_in_progress(self) -> None:
+        self.__number_of_rebalancing_processes_in_progress += 1
+
+    def decrease_number_of_rebalancing_in_progress(self) -> None:
+        self.__number_of_rebalancing_processes_in_progress -= 1
+
+    def is_rebalancing_in_progress(self) -> bool:
+        return self.__number_of_rebalancing_processes_in_progress > 0
 
     def request_stop(self) -> None:
         self.__should_stop.set()

{buz-2.15.7 → buz-2.15.8rc1}/src/buz/kafka/infrastructure/aiokafka/aiokafka_consumer.py

@@ -163,10 +163,7 @@ class AIOKafkaConsumer:
         self,
         topics_partitions: set[TopicPartition],
     ) -> None:
-        # It could happen that the new partition assigned it was a previous one processed by this consumer, so we need to seek to the last committed offset
-        self.__consumer.pause(*topics_partitions)
-        await self.__consumer.seek_to_committed(*topics_partitions)
-        self.__consumer.resume(*topics_partitions)
+        return
 
     async def __on_partitions_revoked(
         self,