buz 2.15.5__py3-none-any.whl → 2.15.7__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- buz/event/infrastructure/buz_kafka/base_buz_aiokafka_async_consumer.py +117 -60
- buz/event/infrastructure/buz_kafka/buz_aiokafka_async_consumer.py +12 -4
- buz/event/infrastructure/buz_kafka/buz_aiokafka_multi_threaded_consumer.py +13 -5
- buz/kafka/infrastructure/aiokafka/aiokafka_consumer.py +17 -8
- {buz-2.15.5.dist-info → buz-2.15.7.dist-info}/METADATA +1 -1
- {buz-2.15.5.dist-info → buz-2.15.7.dist-info}/RECORD +8 -8
- {buz-2.15.5.dist-info → buz-2.15.7.dist-info}/LICENSE +0 -0
- {buz-2.15.5.dist-info → buz-2.15.7.dist-info}/WHEEL +0 -0
|
@@ -33,6 +33,13 @@ T = TypeVar("T", bound=Event)
|
|
|
33
33
|
|
|
34
34
|
class BaseBuzAIOKafkaAsyncConsumer(AsyncConsumer):
|
|
35
35
|
__FALLBACK_PARTITION_ASSIGNORS = (RoundRobinPartitionAssignor,)
|
|
36
|
+
__DEFAULT_MAX_POLL_INTERVAL = 30 * 60 * 1000
|
|
37
|
+
__DEFAULT_SESSION_TIMEOUT_MS = 1000 * 120
|
|
38
|
+
__DEFAULT_HEARTBEAT_INTERVAL = int(__DEFAULT_SESSION_TIMEOUT_MS / 5)
|
|
39
|
+
__SECONDS_BETWEEN_EXECUTIONS_IF_THERE_ARE_NO_TASKS_IN_THE_QUEUE = 1
|
|
40
|
+
__SECONDS_BETWEEN_POLLS_IF_THERE_ARE_TASKS_IN_THE_QUEUE = 1
|
|
41
|
+
__SECONDS_BETWEEN_POLLS_IF_THERE_ARE_NO_NEW_TASKS = 1
|
|
42
|
+
__MAX_NUMBER_OF_CONCURRENT_POLLING_TASKS = 20
|
|
36
43
|
|
|
37
44
|
def __init__(
|
|
38
45
|
self,
|
|
@@ -48,10 +55,14 @@ class BaseBuzAIOKafkaAsyncConsumer(AsyncConsumer):
|
|
|
48
55
|
health_check_port: Optional[int],
|
|
49
56
|
consumer_initial_offset_position: ConsumerInitialOffsetPosition,
|
|
50
57
|
auto_create_topic_configuration: Optional[AutoCreateTopicConfiguration] = None,
|
|
51
|
-
seconds_between_executions_if_there_are_no_tasks_in_the_queue: int =
|
|
52
|
-
seconds_between_polls_if_there_are_tasks_in_the_queue: int =
|
|
53
|
-
seconds_between_polls_if_there_are_no_new_tasks: int =
|
|
54
|
-
max_number_of_concurrent_polling_tasks: int =
|
|
58
|
+
seconds_between_executions_if_there_are_no_tasks_in_the_queue: Optional[int] = None,
|
|
59
|
+
seconds_between_polls_if_there_are_tasks_in_the_queue: Optional[int] = None,
|
|
60
|
+
seconds_between_polls_if_there_are_no_new_tasks: Optional[int] = None,
|
|
61
|
+
max_number_of_concurrent_polling_tasks: Optional[int] = None,
|
|
62
|
+
session_timeout_ms: Optional[int] = None,
|
|
63
|
+
max_poll_interval_ms: Optional[int] = None,
|
|
64
|
+
heartbeat_interval_ms: Optional[int] = None,
|
|
65
|
+
wait_for_connection_to_cluster_ms: Optional[int] = None,
|
|
55
66
|
):
|
|
56
67
|
self.__connection_config = connection_config
|
|
57
68
|
self.__consume_strategy = consume_strategy
|
|
@@ -60,47 +71,62 @@ class BaseBuzAIOKafkaAsyncConsumer(AsyncConsumer):
|
|
|
60
71
|
self._logger = logger
|
|
61
72
|
self.__health_check_port = health_check_port
|
|
62
73
|
self.__consumer_initial_offset_position = consumer_initial_offset_position
|
|
63
|
-
self.__max_records_retrieved_per_poll = 1
|
|
64
74
|
self.__executor_per_consumer_mapper: dict[AIOKafkaConsumer, KafkaEventSubscriberExecutor] = {}
|
|
65
75
|
self.__queue_per_consumer_mapper: dict[
|
|
66
76
|
AIOKafkaConsumer, MultiqueueRepository[TopicPartition, KafkaPollRecord]
|
|
67
77
|
] = {}
|
|
68
|
-
self.__max_records_retrieved_per_poll = max_records_retrieved_per_poll
|
|
69
|
-
self.
|
|
70
|
-
self.
|
|
78
|
+
self.__max_records_retrieved_per_poll: int = max_records_retrieved_per_poll
|
|
79
|
+
self.__session_timeout_ms: int = session_timeout_ms or self.__DEFAULT_SESSION_TIMEOUT_MS
|
|
80
|
+
self.__max_poll_interval_ms: int = max_poll_interval_ms or self.__DEFAULT_MAX_POLL_INTERVAL
|
|
81
|
+
self.__heartbeat_interval_ms: int = heartbeat_interval_ms or self.__DEFAULT_HEARTBEAT_INTERVAL
|
|
82
|
+
self.__max_queue_size: int = max_queue_size
|
|
83
|
+
self.__should_stop: AsyncIOEvent = AsyncIOEvent()
|
|
71
84
|
self.__start_kafka_consumers_elapsed_time: Optional[timedelta] = None
|
|
72
85
|
self.__initial_coroutines_created_elapsed_time: Optional[timedelta] = None
|
|
73
86
|
self.__events_processed: int = 0
|
|
74
87
|
self.__events_processed_elapsed_time: timedelta = timedelta()
|
|
75
|
-
self.__kafka_admin_client = kafka_admin_client
|
|
76
|
-
self.__auto_create_topic_configuration = auto_create_topic_configuration
|
|
77
|
-
self.__seconds_between_executions_if_there_are_no_tasks_in_the_queue = (
|
|
88
|
+
self.__kafka_admin_client: Optional[KafkaAdminClient] = kafka_admin_client
|
|
89
|
+
self.__auto_create_topic_configuration: Optional[AutoCreateTopicConfiguration] = auto_create_topic_configuration
|
|
90
|
+
self.__seconds_between_executions_if_there_are_no_tasks_in_the_queue: int = (
|
|
78
91
|
seconds_between_executions_if_there_are_no_tasks_in_the_queue
|
|
92
|
+
or self.__SECONDS_BETWEEN_EXECUTIONS_IF_THERE_ARE_NO_TASKS_IN_THE_QUEUE
|
|
79
93
|
)
|
|
80
|
-
self.__seconds_between_polls_if_there_are_tasks_in_the_queue = (
|
|
94
|
+
self.__seconds_between_polls_if_there_are_tasks_in_the_queue: int = (
|
|
81
95
|
seconds_between_polls_if_there_are_tasks_in_the_queue
|
|
96
|
+
or self.__SECONDS_BETWEEN_POLLS_IF_THERE_ARE_TASKS_IN_THE_QUEUE
|
|
82
97
|
)
|
|
83
|
-
self.__seconds_between_polls_if_there_are_no_new_tasks =
|
|
84
|
-
|
|
98
|
+
self.__seconds_between_polls_if_there_are_no_new_tasks: int = (
|
|
99
|
+
seconds_between_polls_if_there_are_no_new_tasks or self.__SECONDS_BETWEEN_POLLS_IF_THERE_ARE_NO_NEW_TASKS
|
|
100
|
+
)
|
|
101
|
+
self.__max_number_of_concurrent_polling_tasks: int = (
|
|
102
|
+
max_number_of_concurrent_polling_tasks or self.__MAX_NUMBER_OF_CONCURRENT_POLLING_TASKS
|
|
103
|
+
)
|
|
104
|
+
self.__wait_for_connection_to_cluster_ms: Optional[int] = wait_for_connection_to_cluster_ms
|
|
105
|
+
self.__polling_tasks_semaphore = Semaphore(self.__max_number_of_concurrent_polling_tasks)
|
|
85
106
|
self.__task_execution_mutex = Lock()
|
|
107
|
+
self.__is_worked_initialized = False
|
|
86
108
|
|
|
87
|
-
async def
|
|
109
|
+
async def configure_http_check_server(self, health_check_port: int) -> web.TCPSite:
|
|
88
110
|
self._logger.info(f"Starting health check server on port {health_check_port}")
|
|
89
111
|
app = web.Application()
|
|
90
112
|
app.router.add_get("/health", lambda request: self.__health_check())
|
|
113
|
+
app.router.add_get("/ready", lambda request: self.__is_ready())
|
|
114
|
+
|
|
91
115
|
runner = web.AppRunner(app)
|
|
92
116
|
await runner.setup()
|
|
93
|
-
site = web.TCPSite(runner, "
|
|
117
|
+
site = web.TCPSite(runner, "0.0.0.0", health_check_port)
|
|
94
118
|
await site.start()
|
|
95
119
|
return site
|
|
96
120
|
|
|
97
121
|
async def run(self) -> None:
|
|
122
|
+
self.__print_worker_configuration()
|
|
98
123
|
start_time = datetime.now()
|
|
99
|
-
await self.__generate_kafka_consumers()
|
|
100
124
|
health_check_server: Optional[web.TCPSite] = None
|
|
101
125
|
|
|
102
126
|
if self.__health_check_port is not None:
|
|
103
|
-
health_check_server = await self.
|
|
127
|
+
health_check_server = await self.configure_http_check_server(self.__health_check_port)
|
|
128
|
+
|
|
129
|
+
await self.__generate_kafka_consumers()
|
|
104
130
|
|
|
105
131
|
self.__initial_coroutines_created_elapsed_time = datetime.now() - start_time
|
|
106
132
|
|
|
@@ -108,14 +134,36 @@ class BaseBuzAIOKafkaAsyncConsumer(AsyncConsumer):
|
|
|
108
134
|
self._logger.error("There are no valid subscribers to execute, finalizing consumer")
|
|
109
135
|
return
|
|
110
136
|
|
|
137
|
+
self.__is_worked_initialized = True
|
|
138
|
+
|
|
111
139
|
start_consumption_time = datetime.now()
|
|
112
140
|
worker_errors = await self.__run_worker()
|
|
113
141
|
self.__events_processed_elapsed_time = datetime.now() - start_consumption_time
|
|
114
142
|
|
|
143
|
+
await self.__handle_graceful_stop(worker_errors)
|
|
144
|
+
|
|
115
145
|
if health_check_server is not None:
|
|
116
146
|
await health_check_server.stop()
|
|
117
147
|
|
|
118
|
-
|
|
148
|
+
def __print_worker_configuration(self) -> None:
|
|
149
|
+
self._logger.info(
|
|
150
|
+
f"Consumer configuration:\n"
|
|
151
|
+
f" - Consume strategy: {self.__consume_strategy}\n"
|
|
152
|
+
f" - Max queue size: {self.__max_queue_size}\n"
|
|
153
|
+
f" - Max records retrieved per poll: {self.__max_records_retrieved_per_poll}\n"
|
|
154
|
+
f" - Kafka partition assignors: {self.__kafka_partition_assignors}\n"
|
|
155
|
+
f" - Consumer initial offset position: {self.__consumer_initial_offset_position}\n"
|
|
156
|
+
f" - Session timeout ms: {self.__session_timeout_ms}\n"
|
|
157
|
+
f" - Max poll interval ms: {self.__max_poll_interval_ms}\n"
|
|
158
|
+
f" - Heartbeat interval ms: {self.__heartbeat_interval_ms}\n"
|
|
159
|
+
f" - Seconds between executions if there are no tasks in the queue: {self.__seconds_between_executions_if_there_are_no_tasks_in_the_queue}\n"
|
|
160
|
+
f" - Seconds between polls if there are tasks in the queue: {self.__seconds_between_polls_if_there_are_tasks_in_the_queue}\n"
|
|
161
|
+
f" - Seconds between polls if there are no new tasks: {self.__seconds_between_polls_if_there_are_no_new_tasks}\n"
|
|
162
|
+
f" - Max number of concurrent polling tasks: {self.__max_number_of_concurrent_polling_tasks}\n"
|
|
163
|
+
f" - Wait for connection to cluster ms: {self.__wait_for_connection_to_cluster_ms}\n"
|
|
164
|
+
f" - Health check port: {self.__health_check_port}\n"
|
|
165
|
+
f" - Number of subscribers: {len(self.__subscribers)}",
|
|
166
|
+
)
|
|
119
167
|
|
|
120
168
|
async def __handle_graceful_stop(self, worker_errors: tuple[Optional[Exception], Optional[Exception]]) -> None:
|
|
121
169
|
self._logger.info("Stopping kafka consumers...")
|
|
@@ -160,36 +208,44 @@ class BaseBuzAIOKafkaAsyncConsumer(AsyncConsumer):
|
|
|
160
208
|
self.__start_kafka_consumers_elapsed_time = datetime.now() - start_time
|
|
161
209
|
|
|
162
210
|
async def __generate_kafka_consumer_for_subscriber(self, subscriber: MetaSubscriber) -> None:
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
self.
|
|
179
|
-
|
|
180
|
-
self.__queue_per_consumer_mapper[kafka_consumer] = InMemoryMultiqueueRepository()
|
|
211
|
+
executor = await self._create_kafka_consumer_executor(subscriber)
|
|
212
|
+
topics = self.__consume_strategy.get_topics(subscriber)
|
|
213
|
+
kafka_consumer = AIOKafkaConsumer(
|
|
214
|
+
consumer_group=self.__consume_strategy.get_subscription_group(subscriber),
|
|
215
|
+
topics=topics,
|
|
216
|
+
connection_config=self.__connection_config,
|
|
217
|
+
initial_offset_position=self.__consumer_initial_offset_position,
|
|
218
|
+
partition_assignors=self.__kafka_partition_assignors + self.__FALLBACK_PARTITION_ASSIGNORS,
|
|
219
|
+
logger=self._logger,
|
|
220
|
+
kafka_admin_client=self.__kafka_admin_client,
|
|
221
|
+
auto_create_topic_configuration=self.__auto_create_topic_configuration,
|
|
222
|
+
on_partition_revoked=self.__on_partition_revoked,
|
|
223
|
+
session_timeout_ms=self.__session_timeout_ms,
|
|
224
|
+
max_poll_interval_ms=self.__max_poll_interval_ms,
|
|
225
|
+
heartbeat_interval_ms=self.__heartbeat_interval_ms,
|
|
226
|
+
wait_for_connection_to_cluster_ms=self.__wait_for_connection_to_cluster_ms,
|
|
227
|
+
)
|
|
181
228
|
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
229
|
+
self.__executor_per_consumer_mapper[kafka_consumer] = executor
|
|
230
|
+
|
|
231
|
+
self.__queue_per_consumer_mapper[kafka_consumer] = InMemoryMultiqueueRepository()
|
|
232
|
+
|
|
233
|
+
self._logger.info(
|
|
234
|
+
f"Initializing consumer group: '{kafka_consumer.get_consumer_group()}' subscribed to the topics: '{kafka_consumer.get_topics()}'"
|
|
235
|
+
)
|
|
236
|
+
|
|
237
|
+
await kafka_consumer.init()
|
|
238
|
+
|
|
239
|
+
self._logger.info(
|
|
240
|
+
f"Initialized consumer group: '{kafka_consumer.get_consumer_group()}' subscribed to the topics: '{kafka_consumer.get_topics()}'"
|
|
241
|
+
)
|
|
186
242
|
|
|
187
243
|
@abstractmethod
|
|
188
244
|
async def _create_kafka_consumer_executor(self, subscriber: MetaSubscriber) -> KafkaEventSubscriberExecutor:
|
|
189
245
|
pass
|
|
190
246
|
|
|
191
247
|
async def __polling_task(self) -> None:
|
|
192
|
-
self._logger.info("
|
|
248
|
+
self._logger.info("Creating polling tasks")
|
|
193
249
|
try:
|
|
194
250
|
polling_task_per_consumer = [
|
|
195
251
|
create_task(self.__polling_consuming_tasks(consumer))
|
|
@@ -204,18 +260,6 @@ class BaseBuzAIOKafkaAsyncConsumer(AsyncConsumer):
|
|
|
204
260
|
|
|
205
261
|
async def __polling_consuming_tasks(self, consumer: AIOKafkaConsumer) -> None:
|
|
206
262
|
queue = self.__queue_per_consumer_mapper[consumer]
|
|
207
|
-
|
|
208
|
-
try:
|
|
209
|
-
self._logger.info(
|
|
210
|
-
f"initializing consumer group: '{consumer.get_consumer_group()}' subscribed to the topics: '{consumer.get_topics()}'"
|
|
211
|
-
)
|
|
212
|
-
await consumer.init()
|
|
213
|
-
self._logger.info(f"initialized '{consumer.get_consumer_group()}'")
|
|
214
|
-
except Exception:
|
|
215
|
-
self._logger.exception(
|
|
216
|
-
f"Unexpected error during Kafka subscriber '{consumer.get_consumer_group()}' initialization. Skipping it: {traceback.format_exc()}"
|
|
217
|
-
)
|
|
218
|
-
|
|
219
263
|
while not self.__should_stop.is_set():
|
|
220
264
|
total_size = sum([queue.get_total_size() for queue in self.__queue_per_consumer_mapper.values()])
|
|
221
265
|
if total_size >= self.__max_queue_size:
|
|
@@ -240,7 +284,7 @@ class BaseBuzAIOKafkaAsyncConsumer(AsyncConsumer):
|
|
|
240
284
|
await sleep(self.__seconds_between_polls_if_there_are_no_new_tasks)
|
|
241
285
|
|
|
242
286
|
async def __consume_events_task(self) -> None:
|
|
243
|
-
self._logger.info("
|
|
287
|
+
self._logger.info("Creating consuming task")
|
|
244
288
|
blocked_tasks_iterator = self.__generate_blocked_consuming_tasks_iterator()
|
|
245
289
|
|
|
246
290
|
async for consuming_task in blocked_tasks_iterator:
|
|
@@ -300,9 +344,22 @@ class BaseBuzAIOKafkaAsyncConsumer(AsyncConsumer):
|
|
|
300
344
|
|
|
301
345
|
return web.Response(text=json.dumps(health_information), content_type="application/json")
|
|
302
346
|
|
|
347
|
+
async def __is_ready(self) -> web.Response:
|
|
348
|
+
is_ready = self.__is_worked_initialized
|
|
349
|
+
status_code = 200 if is_ready else 503
|
|
350
|
+
|
|
351
|
+
self._logger.info(f"Health check is_ready: {is_ready}, status_code: {status_code}")
|
|
352
|
+
|
|
353
|
+
return web.Response(
|
|
354
|
+
text=json.dumps({"is_ready": is_ready}), content_type="application/json", status=status_code
|
|
355
|
+
)
|
|
356
|
+
|
|
303
357
|
def __print_statistics(self) -> None:
|
|
304
|
-
self._logger.info(
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
358
|
+
self._logger.info(
|
|
359
|
+
f"System startup summary:\n"
|
|
360
|
+
f" - Number of subscribers: {len(self.__subscribers)}\n"
|
|
361
|
+
f" - Start Kafka consumers elapsed time: {self.__start_kafka_consumers_elapsed_time}\n"
|
|
362
|
+
f" - Initial coroutines created elapsed time: {self.__initial_coroutines_created_elapsed_time}\n"
|
|
363
|
+
f" - Events processed: {self.__events_processed}\n"
|
|
364
|
+
f" - Events processed elapsed time: {self.__events_processed_elapsed_time}"
|
|
365
|
+
)
|
|
@@ -47,11 +47,15 @@ class BuzAIOKafkaAsyncConsumer(BaseBuzAIOKafkaAsyncConsumer):
|
|
|
47
47
|
consume_retrier: Optional[ConsumeRetrier] = None,
|
|
48
48
|
reject_callback: Optional[RejectCallback] = None,
|
|
49
49
|
auto_create_topic_configuration: Optional[AutoCreateTopicConfiguration] = None,
|
|
50
|
-
seconds_between_executions_if_there_are_no_tasks_in_the_queue: int =
|
|
51
|
-
seconds_between_polls_if_there_are_tasks_in_the_queue: int =
|
|
52
|
-
seconds_between_polls_if_there_are_no_new_tasks: int =
|
|
53
|
-
max_number_of_concurrent_polling_tasks: int =
|
|
50
|
+
seconds_between_executions_if_there_are_no_tasks_in_the_queue: Optional[int] = None,
|
|
51
|
+
seconds_between_polls_if_there_are_tasks_in_the_queue: Optional[int] = None,
|
|
52
|
+
seconds_between_polls_if_there_are_no_new_tasks: Optional[int] = None,
|
|
53
|
+
max_number_of_concurrent_polling_tasks: Optional[int] = None,
|
|
54
|
+
session_timeout_ms: Optional[int] = None,
|
|
55
|
+
max_poll_interval_ms: Optional[int] = None,
|
|
56
|
+
heartbeat_interval_ms: Optional[int] = None,
|
|
54
57
|
health_check_port: Optional[int] = None,
|
|
58
|
+
wait_for_connection_to_cluster_ms: Optional[int] = None,
|
|
55
59
|
):
|
|
56
60
|
super().__init__(
|
|
57
61
|
connection_config=connection_config,
|
|
@@ -68,7 +72,11 @@ class BuzAIOKafkaAsyncConsumer(BaseBuzAIOKafkaAsyncConsumer):
|
|
|
68
72
|
seconds_between_polls_if_there_are_tasks_in_the_queue=seconds_between_polls_if_there_are_tasks_in_the_queue,
|
|
69
73
|
seconds_between_polls_if_there_are_no_new_tasks=seconds_between_polls_if_there_are_no_new_tasks,
|
|
70
74
|
max_number_of_concurrent_polling_tasks=max_number_of_concurrent_polling_tasks,
|
|
75
|
+
session_timeout_ms=session_timeout_ms,
|
|
76
|
+
max_poll_interval_ms=max_poll_interval_ms,
|
|
77
|
+
heartbeat_interval_ms=heartbeat_interval_ms,
|
|
71
78
|
health_check_port=health_check_port,
|
|
79
|
+
wait_for_connection_to_cluster_ms=wait_for_connection_to_cluster_ms,
|
|
72
80
|
)
|
|
73
81
|
self.__on_fail_strategy = on_fail_strategy
|
|
74
82
|
self.__consume_middlewares = consume_middlewares
|
|
@@ -48,11 +48,15 @@ class BuzAIOKafkaMultiThreadedConsumer(BaseBuzAIOKafkaAsyncConsumer):
|
|
|
48
48
|
consume_retrier: Optional[ConsumeRetrier] = None,
|
|
49
49
|
reject_callback: Optional[RejectCallback] = None,
|
|
50
50
|
auto_create_topic_configuration: Optional[AutoCreateTopicConfiguration] = None,
|
|
51
|
-
seconds_between_executions_if_there_are_no_tasks_in_the_queue: int =
|
|
52
|
-
seconds_between_polls_if_there_are_tasks_in_the_queue: int =
|
|
53
|
-
seconds_between_polls_if_there_are_no_new_tasks: int =
|
|
54
|
-
max_number_of_concurrent_polling_tasks: int =
|
|
55
|
-
|
|
51
|
+
seconds_between_executions_if_there_are_no_tasks_in_the_queue: Optional[int] = None,
|
|
52
|
+
seconds_between_polls_if_there_are_tasks_in_the_queue: Optional[int] = None,
|
|
53
|
+
seconds_between_polls_if_there_are_no_new_tasks: Optional[int] = None,
|
|
54
|
+
max_number_of_concurrent_polling_tasks: Optional[int] = None,
|
|
55
|
+
session_timeout_ms: Optional[int] = None,
|
|
56
|
+
max_poll_interval_ms: Optional[int] = None,
|
|
57
|
+
heartbeat_interval_ms: Optional[int] = None,
|
|
58
|
+
health_check_port: Optional[int] = None,
|
|
59
|
+
wait_for_connection_to_cluster_ms: Optional[int] = None,
|
|
56
60
|
):
|
|
57
61
|
super().__init__(
|
|
58
62
|
connection_config=connection_config,
|
|
@@ -70,6 +74,10 @@ class BuzAIOKafkaMultiThreadedConsumer(BaseBuzAIOKafkaAsyncConsumer):
|
|
|
70
74
|
seconds_between_polls_if_there_are_no_new_tasks=seconds_between_polls_if_there_are_no_new_tasks,
|
|
71
75
|
max_number_of_concurrent_polling_tasks=max_number_of_concurrent_polling_tasks,
|
|
72
76
|
health_check_port=health_check_port,
|
|
77
|
+
session_timeout_ms=session_timeout_ms,
|
|
78
|
+
max_poll_interval_ms=max_poll_interval_ms,
|
|
79
|
+
heartbeat_interval_ms=heartbeat_interval_ms,
|
|
80
|
+
wait_for_connection_to_cluster_ms=wait_for_connection_to_cluster_ms,
|
|
73
81
|
)
|
|
74
82
|
self.__on_fail_strategy = on_fail_strategy
|
|
75
83
|
self.__consume_middlewares = consume_middlewares
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
|
+
import asyncio
|
|
3
4
|
from logging import Logger
|
|
4
5
|
from typing import Awaitable, Callable, Optional, Sequence, cast
|
|
5
6
|
|
|
@@ -25,8 +26,6 @@ from buz.kafka.infrastructure.aiokafka.translators.consumer_initial_offset_posit
|
|
|
25
26
|
|
|
26
27
|
class AIOKafkaConsumer:
|
|
27
28
|
__DEFAULT_POLL_TIMEOUT_MS = 0
|
|
28
|
-
__DEFAULT_SESSION_TIMEOUT_MS = 1000 * 120
|
|
29
|
-
__DEFAULT_MAX_POLL_INTERVAL = 30 * 60 * 1000
|
|
30
29
|
|
|
31
30
|
def __init__(
|
|
32
31
|
self,
|
|
@@ -38,9 +37,12 @@ class AIOKafkaConsumer:
|
|
|
38
37
|
initial_offset_position: ConsumerInitialOffsetPosition,
|
|
39
38
|
partition_assignors: tuple,
|
|
40
39
|
logger: Logger,
|
|
41
|
-
session_timeout_ms: int
|
|
40
|
+
session_timeout_ms: int,
|
|
41
|
+
max_poll_interval_ms: int,
|
|
42
|
+
heartbeat_interval_ms: int,
|
|
42
43
|
on_partition_revoked: Callable[[AIOKafkaConsumer, set[TopicPartition]], Awaitable[None]],
|
|
43
44
|
auto_create_topic_configuration: Optional[AutoCreateTopicConfiguration] = None,
|
|
45
|
+
wait_for_connection_to_cluster_ms: Optional[int] = None,
|
|
44
46
|
) -> None:
|
|
45
47
|
self.__consumer_group = consumer_group
|
|
46
48
|
self.__topics = topics
|
|
@@ -52,6 +54,9 @@ class AIOKafkaConsumer:
|
|
|
52
54
|
self.__session_timeout_ms = session_timeout_ms
|
|
53
55
|
self.__auto_create_topic_configuration = auto_create_topic_configuration
|
|
54
56
|
self.__on_partitions_revoked_callback = on_partition_revoked
|
|
57
|
+
self.__max_poll_interval_ms = max_poll_interval_ms
|
|
58
|
+
self.__heartbeat_interval_ms = heartbeat_interval_ms
|
|
59
|
+
self.__wait_for_connection_to_cluster_ms = wait_for_connection_to_cluster_ms
|
|
55
60
|
self.__check_kafka_admin_client_is_needed()
|
|
56
61
|
self.__consumer = self.__generate_consumer()
|
|
57
62
|
|
|
@@ -98,10 +103,10 @@ class AIOKafkaConsumer:
|
|
|
98
103
|
self.__initial_offset_position
|
|
99
104
|
),
|
|
100
105
|
session_timeout_ms=self.__session_timeout_ms,
|
|
101
|
-
heartbeat_interval_ms=
|
|
106
|
+
heartbeat_interval_ms=self.__heartbeat_interval_ms,
|
|
102
107
|
partition_assignment_strategy=list(self.__partition_assignors),
|
|
103
|
-
max_poll_interval_ms=self.
|
|
104
|
-
rebalance_timeout_ms=self.
|
|
108
|
+
max_poll_interval_ms=self.__max_poll_interval_ms,
|
|
109
|
+
rebalance_timeout_ms=self.__max_poll_interval_ms,
|
|
105
110
|
)
|
|
106
111
|
|
|
107
112
|
return consumer
|
|
@@ -147,8 +152,12 @@ class AIOKafkaConsumer:
|
|
|
147
152
|
),
|
|
148
153
|
)
|
|
149
154
|
|
|
150
|
-
self.__logger.info(f"
|
|
151
|
-
|
|
155
|
+
self.__logger.info(f"Initializing connection of consumer with group_id={self.__consumer_group}")
|
|
156
|
+
|
|
157
|
+
if self.__wait_for_connection_to_cluster_ms is not None:
|
|
158
|
+
await asyncio.wait_for(self.__consumer.start(), self.__wait_for_connection_to_cluster_ms / 1000)
|
|
159
|
+
else:
|
|
160
|
+
await self.__consumer.start()
|
|
152
161
|
|
|
153
162
|
async def __on_partitions_assigned(
|
|
154
163
|
self,
|
|
@@ -47,9 +47,9 @@ buz/event/exceptions/worker_execution_exception.py,sha256=6mgztvXOCG_9VZ_Jptkk72
|
|
|
47
47
|
buz/event/infrastructure/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
48
48
|
buz/event/infrastructure/buz_kafka/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
49
49
|
buz/event/infrastructure/buz_kafka/async_buz_kafka_event_bus.py,sha256=SyLblUVlwWOaNfZzK7vL6Ee4m-85vZVCH0rjOgqVAww,4913
|
|
50
|
-
buz/event/infrastructure/buz_kafka/base_buz_aiokafka_async_consumer.py,sha256=
|
|
51
|
-
buz/event/infrastructure/buz_kafka/buz_aiokafka_async_consumer.py,sha256=
|
|
52
|
-
buz/event/infrastructure/buz_kafka/buz_aiokafka_multi_threaded_consumer.py,sha256=
|
|
50
|
+
buz/event/infrastructure/buz_kafka/base_buz_aiokafka_async_consumer.py,sha256=FDIz3Usv4ZQrA1_jmfAwv_Nmml3QDHBGt6CyRK0IWiU,18623
|
|
51
|
+
buz/event/infrastructure/buz_kafka/buz_aiokafka_async_consumer.py,sha256=lLHUnf9DpfAyB0PN7oBjn-Eyyl0zCHsSnrb6QUigruk,6345
|
|
52
|
+
buz/event/infrastructure/buz_kafka/buz_aiokafka_multi_threaded_consumer.py,sha256=CiuNTns6eifR2JNGhXGNTdWURu-IQVIioe5n_pMSt9s,6288
|
|
53
53
|
buz/event/infrastructure/buz_kafka/buz_kafka_event_bus.py,sha256=ymRSvcYVgbVCPgHN6rMBVBHQ5heCSwCDl6EffyqGVX8,4601
|
|
54
54
|
buz/event/infrastructure/buz_kafka/consume_strategy/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
55
55
|
buz/event/infrastructure/buz_kafka/consume_strategy/consume_strategy.py,sha256=RqlXe5W2S6rH3FTr--tcxzFJTAVLb-Dhl7m6qjgNz2M,331
|
|
@@ -164,7 +164,7 @@ buz/kafka/domain/services/kafka_admin_test_client.py,sha256=91l_vFIo1yhJLQQCC_Om
|
|
|
164
164
|
buz/kafka/domain/services/kafka_producer.py,sha256=8bLTV328orrPHcARzkc6no4vyJzrArVtCsjmSRXDjos,506
|
|
165
165
|
buz/kafka/infrastructure/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
166
166
|
buz/kafka/infrastructure/aiokafka/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
167
|
-
buz/kafka/infrastructure/aiokafka/aiokafka_consumer.py,sha256=
|
|
167
|
+
buz/kafka/infrastructure/aiokafka/aiokafka_consumer.py,sha256=xk_812Ah4hxSnAYnpHgbAnwIcvzaCmPY7rBtxo7UcpM,9102
|
|
168
168
|
buz/kafka/infrastructure/aiokafka/aiokafka_producer.py,sha256=LteHKIHpT6MKplwmwsPYMsd2GWNJCzus65XDHCIdoN8,3823
|
|
169
169
|
buz/kafka/infrastructure/aiokafka/rebalance/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
170
170
|
buz/kafka/infrastructure/aiokafka/rebalance/kafka_callback_rebalancer.py,sha256=3l7NkTrCt3rBktVIS73cTmCOvv6eFguoCbGMYIUfCFc,1774
|
|
@@ -250,7 +250,7 @@ buz/serializer/message_to_json_bytes_serializer.py,sha256=RGZJ64t4t4Pz2FCASZZCv-
|
|
|
250
250
|
buz/wrapper/__init__.py,sha256=GnRdJFcncn-qp0hzDG9dBHLmTJSbHFVjE_yr-MdW_n4,77
|
|
251
251
|
buz/wrapper/async_to_sync.py,sha256=OfK-vrVUhuN-LLLvekLdMbQYtH0ue5lfbvuasj6ovMI,698
|
|
252
252
|
buz/wrapper/event_loop.py,sha256=pfBJ1g-8A2a3YgW8Gf9Fg0kkewoh3-wgTy2KIFDyfHk,266
|
|
253
|
-
buz-2.15.
|
|
254
|
-
buz-2.15.
|
|
255
|
-
buz-2.15.
|
|
256
|
-
buz-2.15.
|
|
253
|
+
buz-2.15.7.dist-info/LICENSE,sha256=Jytu2S-2SPEgsB0y6BF-_LUxIWY7402fl0JSh36TLZE,1062
|
|
254
|
+
buz-2.15.7.dist-info/METADATA,sha256=-987wNb7KXdP6U8KtilvaxhsAI8ugvrJ5PzVDQfQUzg,1597
|
|
255
|
+
buz-2.15.7.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
|
|
256
|
+
buz-2.15.7.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|