buz 2.15.6__py3-none-any.whl → 2.15.8rc1__py3-none-any.whl

buz/event/infrastructure/buz_kafka/base_buz_aiokafka_async_consumer.py

@@ -1,3 +1,4 @@
+from collections import defaultdict
 import json
 import traceback
 from abc import abstractmethod
@@ -39,6 +40,7 @@ class BaseBuzAIOKafkaAsyncConsumer(AsyncConsumer):
     __SECONDS_BETWEEN_EXECUTIONS_IF_THERE_ARE_NO_TASKS_IN_THE_QUEUE = 1
     __SECONDS_BETWEEN_POLLS_IF_THERE_ARE_TASKS_IN_THE_QUEUE = 1
     __SECONDS_BETWEEN_POLLS_IF_THERE_ARE_NO_NEW_TASKS = 1
+    __SECONDS_TO_WAIT_BETWEEN_REBALANCING_IN_PROGRESS = 0.1
     __MAX_NUMBER_OF_CONCURRENT_POLLING_TASKS = 20
 
     def __init__(
@@ -103,8 +105,9 @@ class BaseBuzAIOKafkaAsyncConsumer(AsyncConsumer):
         )
         self.__wait_for_connection_to_cluster_ms: Optional[int] = wait_for_connection_to_cluster_ms
         self.__polling_tasks_semaphore = Semaphore(self.__max_number_of_concurrent_polling_tasks)
-        self.__task_execution_mutex = Lock()
+        self.__consumer_and_partition_mutex: dict[str, Lock] = defaultdict(Lock)
         self.__is_worked_initialized = False
+        self.__number_of_rebalancing_processes_in_progress: int = 0
 
     async def configure_http_check_server(self, health_check_port: int) -> web.TCPSite:
         self._logger.info(f"Starting health check server on port {health_check_port}")
@@ -114,7 +117,7 @@ class BaseBuzAIOKafkaAsyncConsumer(AsyncConsumer):
 
         runner = web.AppRunner(app)
         await runner.setup()
-        site = web.TCPSite(runner, "localhost", health_check_port)
+        site = web.TCPSite(runner, "0.0.0.0", health_check_port)
         await site.start()
         return site
 
@@ -156,16 +159,13 @@ class BaseBuzAIOKafkaAsyncConsumer(AsyncConsumer):
             f"  - Session timeout ms: {self.__session_timeout_ms}\n"
             f"  - Max poll interval ms: {self.__max_poll_interval_ms}\n"
             f"  - Heartbeat interval ms: {self.__heartbeat_interval_ms}\n"
-            f"  - Seconds between executions if there are no tasks in the queue: "
-            f"{self.__seconds_between_executions_if_there_are_no_tasks_in_the_queue}\n"
-            f"  - Seconds between polls if there are tasks in the queue: "
-            f"{self.__seconds_between_polls_if_there_are_tasks_in_the_queue}\n"
-            f"  - Seconds between polls if there are no new tasks: "
-            f"{self.__seconds_between_polls_if_there_are_no_new_tasks}\n"
+            f"  - Seconds between executions if there are no tasks in the queue: {self.__seconds_between_executions_if_there_are_no_tasks_in_the_queue}\n"
+            f"  - Seconds between polls if there are tasks in the queue: {self.__seconds_between_polls_if_there_are_tasks_in_the_queue}\n"
+            f"  - Seconds between polls if there are no new tasks: {self.__seconds_between_polls_if_there_are_no_new_tasks}\n"
             f"  - Max number of concurrent polling tasks: {self.__max_number_of_concurrent_polling_tasks}\n"
             f"  - Wait for connection to cluster ms: {self.__wait_for_connection_to_cluster_ms}\n"
             f"  - Health check port: {self.__health_check_port}\n"
-            f"  - Number of subscribers: {len(self.__subscribers)}"
+            f"  - Number of subscribers: {len(self.__subscribers)}",
         )
 
     async def __handle_graceful_stop(self, worker_errors: tuple[Optional[Exception], Optional[Exception]]) -> None:
@@ -234,7 +234,7 @@ class BaseBuzAIOKafkaAsyncConsumer(AsyncConsumer):
         self.__queue_per_consumer_mapper[kafka_consumer] = InMemoryMultiqueueRepository()
 
         self._logger.info(
-            f"initializing consumer group: '{kafka_consumer.get_consumer_group()}' subscribed to the topics: '{kafka_consumer.get_topics()}'"
+            f"Initializing consumer group: '{kafka_consumer.get_consumer_group()}' subscribed to the topics: '{kafka_consumer.get_topics()}'"
         )
 
         await kafka_consumer.init()
@@ -306,29 +306,59 @@ class BaseBuzAIOKafkaAsyncConsumer(AsyncConsumer):
         last_consumer, _ = next(consumer_queues_cyclic_iterator)
 
         while not self.__should_stop.is_set():
+            if self.is_rebalancing_in_progress():
+                self._logger.info("Waiting for rebalancing")
+                await sleep(self.__SECONDS_TO_WAIT_BETWEEN_REBALANCING_IN_PROGRESS)
+                continue
+
             if await self.__all_queues_are_empty():
                 await sleep(self.__seconds_between_executions_if_there_are_no_tasks_in_the_queue)
                 continue
 
-            async with self.__task_execution_mutex:
-                consumer: Optional[AIOKafkaConsumer] = None
+            consumer: Optional[AIOKafkaConsumer] = None
+
+            while consumer != last_consumer:
+                consumer, queue = next(consumer_queues_cyclic_iterator)
+                kafka_poll_record = queue.pop()
+
+                if kafka_poll_record is not None:
+                    consuming_task = ConsumingTask(consumer, kafka_poll_record)
+                    async with self.__get_partition_mutex(
+                        consumer_fqn=consuming_task.consumer.get_consumer_group(),
+                        partition=kafka_poll_record.partition,
+                    ):
+                        yield consuming_task
 
-                while consumer != last_consumer:
-                    consumer, queue = next(consumer_queues_cyclic_iterator)
-                    kafka_poll_record = queue.pop()
+                    last_consumer = consumer
+                    break
 
-                    if kafka_poll_record is not None:
-                        yield ConsumingTask(consumer, kafka_poll_record)
-                        last_consumer = consumer
-                        break
+    def __get_partition_mutex(self, consumer_fqn: str, partition: int) -> Lock:
+        mutex_key = f"consumer_{consumer_fqn}_partition_{partition}"
+        return self.__consumer_and_partition_mutex[mutex_key]
 
     async def __all_queues_are_empty(self) -> bool:
         return all([queue.is_totally_empty() for queue in self.__queue_per_consumer_mapper.values()])
 
     async def __on_partition_revoked(self, consumer: AIOKafkaConsumer, topics_partitions: set[TopicPartition]) -> None:
-        async with self.__task_execution_mutex:
-            for topic_partition in topics_partitions:
+        self._logger.info(f"rebalancing in progress, revoking partitions {topics_partitions}")
+
+        self.increase_number_of_rebalancing_in_progress()
+        for topic_partition in topics_partitions:
+            async with self.__get_partition_mutex(
+                consumer_fqn=consumer.get_consumer_group(),
+                partition=topic_partition.partition,
+            ):
                 self.__queue_per_consumer_mapper[consumer].clear(topic_partition)
+        self.decrease_number_of_rebalancing_in_progress()
+
+    def increase_number_of_rebalancing_in_progress(self) -> None:
+        self.__number_of_rebalancing_processes_in_progress += 1
+
+    def decrease_number_of_rebalancing_in_progress(self) -> None:
+        self.__number_of_rebalancing_processes_in_progress -= 1
+
+    def is_rebalancing_in_progress(self) -> bool:
+        return self.__number_of_rebalancing_processes_in_progress > 0
 
     def request_stop(self) -> None:
         self.__should_stop.set()

buz/kafka/infrastructure/aiokafka/aiokafka_consumer.py

@@ -152,7 +152,7 @@ class AIOKafkaConsumer:
             ),
         )
 
-        self.__logger.info(f"initializing connection of consumer with group_id={self.__consumer_group}")
+        self.__logger.info(f"Initializing connection of consumer with group_id={self.__consumer_group}")
 
         if self.__wait_for_connection_to_cluster_ms is not None:
             await asyncio.wait_for(self.__consumer.start(), self.__wait_for_connection_to_cluster_ms / 1000)
@@ -163,10 +163,7 @@ class AIOKafkaConsumer:
         self,
         topics_partitions: set[TopicPartition],
     ) -> None:
-        # It could happen that the new partition assigned it was a previous one processed by this consumer, so we need to seek to the last committed offset
-        self.__consumer.pause(*topics_partitions)
-        await self.__consumer.seek_to_committed(*topics_partitions)
-        self.__consumer.resume(*topics_partitions)
+        return
 
     async def __on_partitions_revoked(
         self,

{buz-2.15.6.dist-info → buz-2.15.8rc1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: buz
-Version: 2.15.6
+Version: 2.15.8rc1
 Summary: Buz is a set of light, simple and extensible implementations of event, command and query buses.
 License: MIT
 Author: Luis Pintado Lozano

{buz-2.15.6.dist-info → buz-2.15.8rc1.dist-info}/RECORD

@@ -47,7 +47,7 @@ buz/event/exceptions/worker_execution_exception.py,sha256=6mgztvXOCG_9VZ_Jptkk72
 buz/event/infrastructure/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 buz/event/infrastructure/buz_kafka/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 buz/event/infrastructure/buz_kafka/async_buz_kafka_event_bus.py,sha256=SyLblUVlwWOaNfZzK7vL6Ee4m-85vZVCH0rjOgqVAww,4913
-buz/event/infrastructure/buz_kafka/base_buz_aiokafka_async_consumer.py,sha256=y8ZgJ11yfd9aARnZifTBNntlbDlj5mTiTtEsA5_72Dw,18672
+buz/event/infrastructure/buz_kafka/base_buz_aiokafka_async_consumer.py,sha256=13CqXqXk_1tgVJ6SpXpW4nr2SMA7ujpm8l4q3Yhtdsg,20179
 buz/event/infrastructure/buz_kafka/buz_aiokafka_async_consumer.py,sha256=lLHUnf9DpfAyB0PN7oBjn-Eyyl0zCHsSnrb6QUigruk,6345
 buz/event/infrastructure/buz_kafka/buz_aiokafka_multi_threaded_consumer.py,sha256=CiuNTns6eifR2JNGhXGNTdWURu-IQVIioe5n_pMSt9s,6288
 buz/event/infrastructure/buz_kafka/buz_kafka_event_bus.py,sha256=ymRSvcYVgbVCPgHN6rMBVBHQ5heCSwCDl6EffyqGVX8,4601
@@ -164,7 +164,7 @@ buz/kafka/domain/services/kafka_admin_test_client.py,sha256=91l_vFIo1yhJLQQCC_Om
 buz/kafka/domain/services/kafka_producer.py,sha256=8bLTV328orrPHcARzkc6no4vyJzrArVtCsjmSRXDjos,506
 buz/kafka/infrastructure/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 buz/kafka/infrastructure/aiokafka/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-buz/kafka/infrastructure/aiokafka/aiokafka_consumer.py,sha256=QyY7f2_E92wfX3Ru2IpLXQvC5Wj814SgDrcwe8inpTA,9102
+buz/kafka/infrastructure/aiokafka/aiokafka_consumer.py,sha256=9ORj5couBePn5LtIlTXUlMF2hSOuZ5vmgG0RUbw-BVM,8792
 buz/kafka/infrastructure/aiokafka/aiokafka_producer.py,sha256=LteHKIHpT6MKplwmwsPYMsd2GWNJCzus65XDHCIdoN8,3823
 buz/kafka/infrastructure/aiokafka/rebalance/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 buz/kafka/infrastructure/aiokafka/rebalance/kafka_callback_rebalancer.py,sha256=3l7NkTrCt3rBktVIS73cTmCOvv6eFguoCbGMYIUfCFc,1774
@@ -250,7 +250,7 @@ buz/serializer/message_to_json_bytes_serializer.py,sha256=RGZJ64t4t4Pz2FCASZZCv-
 buz/wrapper/__init__.py,sha256=GnRdJFcncn-qp0hzDG9dBHLmTJSbHFVjE_yr-MdW_n4,77
 buz/wrapper/async_to_sync.py,sha256=OfK-vrVUhuN-LLLvekLdMbQYtH0ue5lfbvuasj6ovMI,698
 buz/wrapper/event_loop.py,sha256=pfBJ1g-8A2a3YgW8Gf9Fg0kkewoh3-wgTy2KIFDyfHk,266
-buz-2.15.6.dist-info/LICENSE,sha256=Jytu2S-2SPEgsB0y6BF-_LUxIWY7402fl0JSh36TLZE,1062
-buz-2.15.6.dist-info/METADATA,sha256=u5zUavFByKN26H1zpAbr9Mmyo3TPvdq0_9_sGyVCMvM,1597
-buz-2.15.6.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-buz-2.15.6.dist-info/RECORD,,
+buz-2.15.8rc1.dist-info/LICENSE,sha256=Jytu2S-2SPEgsB0y6BF-_LUxIWY7402fl0JSh36TLZE,1062
+buz-2.15.8rc1.dist-info/METADATA,sha256=EcA5CSIiqruT_bLWz0GS6LaFdRbbbJiG_Stg9zspIug,1600
+buz-2.15.8rc1.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+buz-2.15.8rc1.dist-info/RECORD,,