buz 2.12.0rc2__py3-none-any.whl → 2.13.1rc1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. buz/event/__init__.py +8 -0
  2. buz/event/async_subscriber.py +10 -0
  3. buz/event/async_worker.py +6 -3
  4. buz/event/base_async_subscriber.py +8 -0
  5. buz/event/base_subscriber.py +4 -19
  6. buz/event/dead_letter_queue/dlq_criteria.py +11 -0
  7. buz/event/dead_letter_queue/dlq_record.py +31 -0
  8. buz/event/dead_letter_queue/dlq_repository.py +23 -0
  9. buz/event/exceptions/worker_execution_exception.py +3 -0
  10. buz/event/infrastructure/buz_kafka/base_buz_aiokafka_async_consumer.py +289 -0
  11. buz/event/infrastructure/buz_kafka/buz_aiokafka_async_consumer.py +102 -0
  12. buz/event/infrastructure/buz_kafka/buz_aiokafka_multi_threaded_consumer.py +42 -197
  13. buz/event/infrastructure/buz_kafka/consume_strategy/consume_strategy.py +3 -3
  14. buz/event/infrastructure/buz_kafka/consume_strategy/topic_and_subscription_group_per_subscriber_kafka_consumer_strategy.py +3 -3
  15. buz/event/infrastructure/buz_kafka/kafka_event_async_subscriber_executor.py +113 -0
  16. buz/event/infrastructure/buz_kafka/kafka_event_subscriber_executor.py +4 -103
  17. buz/event/infrastructure/buz_kafka/kafka_event_sync_subscriber_executor.py +114 -0
  18. buz/event/infrastructure/kombu/kombu_consumer.py +6 -4
  19. buz/event/infrastructure/models/consuming_task.py +9 -0
  20. buz/event/meta_base_subscriber.py +23 -0
  21. buz/event/meta_subscriber.py +11 -0
  22. buz/event/middleware/async_consume_middleware.py +14 -0
  23. buz/event/middleware/async_consume_middleware_chain_resolver.py +29 -0
  24. buz/event/middleware/consume_middleware_chain_resolver.py +2 -1
  25. buz/event/strategies/retry/consume_retrier.py +5 -3
  26. buz/event/strategies/retry/consumed_event_retry_repository.py +6 -2
  27. buz/event/strategies/retry/max_retries_consume_retrier.py +9 -5
  28. buz/event/strategies/retry/reject_callback.py +4 -2
  29. buz/event/subscriber.py +3 -9
  30. buz/event/transactional_outbox/outbox_criteria/outbox_criteria.py +1 -0
  31. buz/event/worker.py +6 -3
  32. buz/kafka/__init__.py +0 -2
  33. buz/kafka/domain/services/kafka_admin_test_client.py +2 -2
  34. buz/kafka/infrastructure/aiokafka/{aiokafka_multi_threaded_consumer.py → aiokafka_consumer.py} +48 -25
  35. buz/kafka/infrastructure/aiokafka/rebalance/kafka_callback_rebalancer.py +42 -0
  36. buz/kafka/infrastructure/deserializers/implementations/json_bytes_to_message_deserializer.py +2 -2
  37. buz/kafka/infrastructure/kafka_python/kafka_python_admin_test_client.py +4 -2
  38. buz/middleware/middleware_chain_builder.py +2 -2
  39. buz/queue/in_memory/in_memory_multiqueue_repository.py +74 -0
  40. buz/{event/infrastructure/buz_kafka → queue/in_memory}/in_memory_queue_repository.py +2 -1
  41. buz/queue/multiqueue_repository.py +27 -0
  42. {buz-2.12.0rc2.dist-info → buz-2.13.1rc1.dist-info}/METADATA +2 -2
  43. {buz-2.12.0rc2.dist-info → buz-2.13.1rc1.dist-info}/RECORD +48 -35
  44. buz/kafka/domain/services/kafka_consumer.py +0 -25
  45. buz/kafka/infrastructure/aiokafka/factories/__init__.py +0 -0
  46. buz/kafka/infrastructure/aiokafka/factories/kafka_python_multi_threaded_consumer_factory.py +0 -55
  47. buz/kafka/infrastructure/aiokafka/rebalance/rebalance_ready.py +0 -7
  48. buz/kafka/infrastructure/aiokafka/rebalance/simple_kafka_lock_rebalancer.py +0 -18
  49. /buz/event/{domain → dead_letter_queue}/__init__.py +0 -0
  50. /buz/{event/domain/queue → queue}/__init__.py +0 -0
  51. /buz/{event/domain/queue → queue}/queue_repository.py +0 -0
  52. {buz-2.12.0rc2.dist-info → buz-2.13.1rc1.dist-info}/LICENSE +0 -0
  53. {buz-2.12.0rc2.dist-info → buz-2.13.1rc1.dist-info}/WHEEL +0 -0
buz/event/__init__.py CHANGED
@@ -1,11 +1,19 @@
1
1
  from buz.event.event import Event
2
2
  from buz.event.subscriber import Subscriber
3
+ from buz.event.async_subscriber import AsyncSubscriber
3
4
  from buz.event.base_subscriber import BaseSubscriber
5
+ from buz.event.base_async_subscriber import BaseAsyncSubscriber
4
6
  from buz.event.event_bus import EventBus
7
+ from buz.event.meta_subscriber import MetaSubscriber
8
+ from buz.event.meta_base_subscriber import MetaBaseSubscriber
5
9
 
6
10
  __all__ = [
7
11
  "Event",
8
12
  "Subscriber",
13
+ "AsyncSubscriber",
9
14
  "BaseSubscriber",
15
+ "BaseAsyncSubscriber",
10
16
  "EventBus",
17
+ "MetaSubscriber",
18
+ "MetaBaseSubscriber",
11
19
  ]
@@ -0,0 +1,10 @@
1
+ from abc import ABC, abstractmethod
2
+
3
+ from buz.event import Event
4
+ from buz.event.meta_subscriber import MetaSubscriber
5
+
6
+
7
+ class AsyncSubscriber(MetaSubscriber, ABC):
8
+ @abstractmethod
9
+ async def consume(self, event: Event) -> None:
10
+ pass
buz/event/async_worker.py CHANGED
@@ -1,4 +1,5 @@
1
1
  import asyncio
2
+ from logging import Logger
2
3
  from signal import SIGTERM, SIGINT
3
4
 
4
5
  from buz.event.strategies.execution_strategy.async_execution_strategy import AsyncExecutionStrategy
@@ -7,18 +8,20 @@ from buz.event.strategies.execution_strategy.async_execution_strategy import Asy
7
8
  class AsyncWorker:
8
9
  def __init__(
9
10
  self,
11
+ logger: Logger,
10
12
  execution_strategy: AsyncExecutionStrategy,
11
13
  ) -> None:
12
14
  self.__execution_strategy = execution_strategy
15
+ self.__logger = logger
13
16
 
14
17
  async def start(self) -> None:
15
18
  loop = asyncio.get_running_loop()
16
19
  loop.add_signal_handler(SIGINT, self.stop)
17
20
  loop.add_signal_handler(SIGTERM, self.stop)
18
- print("Starting buz worker...")
21
+ self.__logger.info("Starting buz worker...")
19
22
  await self.__execution_strategy.start()
20
- print("Buz worker stopped gracefully")
23
+ self.__logger.info("Buz worker stopped gracefully")
21
24
 
22
25
  def stop(self) -> None:
23
- print("Stopping buz worker...")
26
+ self.__logger.info("Stopping buz worker...")
24
27
  self.__execution_strategy.request_stop()
@@ -0,0 +1,8 @@
1
+ from abc import ABC
2
+
3
+ from buz.event.async_subscriber import AsyncSubscriber
4
+ from buz.event.meta_base_subscriber import MetaBaseSubscriber
5
+
6
+
7
+ class BaseAsyncSubscriber(AsyncSubscriber, MetaBaseSubscriber, ABC):
8
+ pass
@@ -1,22 +1,7 @@
1
- from typing import Type, get_type_hints
2
-
3
- from buz.event import Event
1
+ from abc import ABC
4
2
  from buz.event import Subscriber
3
+ from buz.event.meta_base_subscriber import MetaBaseSubscriber
5
4
 
6
5
 
7
- class BaseSubscriber(Subscriber):
8
- @classmethod
9
- def fqn(cls) -> str:
10
- return f"subscriber.{cls.__module__}.{cls.__name__}"
11
-
12
- @classmethod
13
- def handles(cls) -> Type[Event]:
14
- consume_types = get_type_hints(cls.consume)
15
-
16
- if "event" not in consume_types:
17
- raise TypeError("event parameter not found in consume method")
18
-
19
- if not issubclass(consume_types["event"], Event):
20
- raise TypeError("event parameter is not an buz.event.Event subclass")
21
-
22
- return consume_types["event"]
6
+ class BaseSubscriber(Subscriber, MetaBaseSubscriber, ABC):
7
+ pass
@@ -0,0 +1,11 @@
1
+ from dataclasses import dataclass
2
+ from typing import ClassVar, Union
3
+
4
+ from buz.event.dead_letter_queue.dlq_record import DlqRecordId
5
+
6
+
7
+ @dataclass(frozen=True)
8
+ class DlqCriteria:
9
+ UNSET_VALUE: ClassVar[object] = object()
10
+
11
+ dlq_record_id: Union[DlqRecordId, None, object] = UNSET_VALUE
@@ -0,0 +1,31 @@
1
+ from dataclasses import dataclass, fields
2
+ from datetime import datetime
3
+ from typing import Any, ClassVar
4
+ from uuid import UUID
5
+
6
+ DlqRecordId = UUID
7
+
8
+
9
+ @dataclass
10
+ class DlqRecord:
11
+ DATE_TIME_FORMAT: ClassVar[str] = "%Y-%m-%d %H:%M:%S.%f"
12
+
13
+ id: DlqRecordId
14
+ event_id: UUID
15
+ subscriber_fqn: str
16
+ event_payload: dict
17
+ exception_type: str
18
+ exception_message: str
19
+ last_failed_at: datetime
20
+
21
+ def mark_as_failed(self) -> None:
22
+ self.last_failed_at = datetime.now()
23
+
24
+ def get_attrs(self) -> dict[str, Any]:
25
+ attrs = {}
26
+ for field in fields(self):
27
+ property_name = field.name
28
+ property_value = getattr(self, property_name)
29
+ attrs[property_name] = property_value
30
+
31
+ return attrs
@@ -0,0 +1,23 @@
1
+ from abc import ABC, abstractmethod
2
+ from typing import Sequence
3
+
4
+ from buz.event.dead_letter_queue.dlq_criteria import DlqCriteria
5
+ from buz.event.dead_letter_queue.dlq_record import DlqRecord, DlqRecordId
6
+
7
+
8
+ class DlqRepository(ABC):
9
+ @abstractmethod
10
+ def save(self, dlq_record: DlqRecord) -> None:
11
+ pass
12
+
13
+ @abstractmethod
14
+ def find_one_or_fail_by_criteria(self, criteria: DlqCriteria) -> DlqRecord:
15
+ pass
16
+
17
+ @abstractmethod
18
+ def delete(self, dlq_record_id: DlqRecordId) -> None:
19
+ pass
20
+
21
+ @abstractmethod
22
+ def bulk_delete(self, dlq_record_ids: Sequence[DlqRecordId]) -> None:
23
+ pass
@@ -0,0 +1,3 @@
1
+ class WorkerExecutionException(Exception):
2
+ def __init__(self, message: str) -> None:
3
+ super().__init__(message)
@@ -0,0 +1,289 @@
1
+ from abc import abstractmethod
2
+ import traceback
3
+ from asyncio import Lock, gather, Semaphore, Event as AsyncIOEvent, sleep
4
+ from datetime import timedelta, datetime
5
+ from itertools import cycle
6
+ from logging import Logger
7
+ from typing import AsyncIterator, Coroutine, Optional, Sequence, Type, TypeVar
8
+
9
+ from aiokafka import TopicPartition
10
+ from aiokafka.coordinator.assignors.abstract import AbstractPartitionAssignor
11
+ from aiokafka.coordinator.assignors.roundrobin import RoundRobinPartitionAssignor
12
+
13
+ from buz.event import Event
14
+ from buz.event.async_consumer import AsyncConsumer
15
+ from buz.event.exceptions.worker_execution_exception import WorkerExecutionException
16
+ from buz.event.infrastructure.buz_kafka.consume_strategy.consume_strategy import KafkaConsumeStrategy
17
+ from buz.event.infrastructure.buz_kafka.kafka_event_subscriber_executor import KafkaEventSubscriberExecutor
18
+ from buz.event.infrastructure.models.consuming_task import ConsumingTask
19
+ from buz.event.meta_subscriber import MetaSubscriber
20
+ from buz.kafka import (
21
+ KafkaConnectionConfig,
22
+ ConsumerInitialOffsetPosition,
23
+ )
24
+ from buz.kafka.domain.models.auto_create_topic_configuration import AutoCreateTopicConfiguration
25
+ from buz.kafka.domain.models.kafka_poll_record import KafkaPollRecord
26
+ from buz.kafka.domain.services.kafka_admin_client import KafkaAdminClient
27
+ from buz.kafka.infrastructure.aiokafka.aiokafka_consumer import AIOKafkaConsumer
28
+ from buz.queue.in_memory.in_memory_multiqueue_repository import InMemoryMultiqueueRepository
29
+ from buz.queue.multiqueue_repository import MultiqueueRepository
30
+
31
+
32
+ T = TypeVar("T", bound=Event)
33
+
34
+
35
+ class BaseBuzAIOKafkaAsyncConsumer(AsyncConsumer):
36
+ __FALLBACK_PARTITION_ASSIGNORS = (RoundRobinPartitionAssignor,)
37
+
38
+ def __init__(
39
+ self,
40
+ *,
41
+ connection_config: KafkaConnectionConfig,
42
+ kafka_admin_client: Optional[KafkaAdminClient],
43
+ consume_strategy: KafkaConsumeStrategy,
44
+ max_queue_size: int,
45
+ max_records_retrieved_per_poll: int,
46
+ kafka_partition_assignors: tuple[Type[AbstractPartitionAssignor], ...] = (),
47
+ subscribers: Sequence[MetaSubscriber],
48
+ logger: Logger,
49
+ consumer_initial_offset_position: ConsumerInitialOffsetPosition,
50
+ auto_create_topic_configuration: Optional[AutoCreateTopicConfiguration] = None,
51
+ seconds_between_executions_if_there_are_no_tasks_in_the_queue: int = 1,
52
+ seconds_between_polls_if_there_are_tasks_in_the_queue: int = 1,
53
+ seconds_between_polls_if_there_are_no_new_tasks: int = 1,
54
+ max_number_of_concurrent_polling_tasks: int = 20,
55
+ ):
56
+ self.__connection_config = connection_config
57
+ self.__consume_strategy = consume_strategy
58
+ self.__kafka_partition_assignors = kafka_partition_assignors
59
+ self.__subscribers = subscribers
60
+ self._logger = logger
61
+ self.__consumer_initial_offset_position = consumer_initial_offset_position
62
+ self.__max_records_retrieved_per_poll = 1
63
+ self.__subscriber_per_consumer_mapper: dict[AIOKafkaConsumer, MetaSubscriber] = {}
64
+ self.__executor_per_consumer_mapper: dict[AIOKafkaConsumer, KafkaEventSubscriberExecutor] = {}
65
+ self.__queue_per_consumer_mapper: dict[
66
+ AIOKafkaConsumer, MultiqueueRepository[TopicPartition, KafkaPollRecord]
67
+ ] = {}
68
+ self.__max_records_retrieved_per_poll = max_records_retrieved_per_poll
69
+ self.__max_queue_size = max_queue_size
70
+ self.__should_stop = AsyncIOEvent()
71
+ self.__start_kafka_consumers_elapsed_time: Optional[timedelta] = None
72
+ self.__initial_coroutines_created_elapsed_time: Optional[timedelta] = None
73
+ self.__events_processed: int = 0
74
+ self.__events_processed_elapsed_time: timedelta = timedelta()
75
+ self.__kafka_admin_client = kafka_admin_client
76
+ self.__auto_create_topic_configuration = auto_create_topic_configuration
77
+ self.__seconds_between_executions_if_there_are_no_tasks_in_the_queue = (
78
+ seconds_between_executions_if_there_are_no_tasks_in_the_queue
79
+ )
80
+ self.__seconds_between_polls_if_there_are_tasks_in_the_queue = (
81
+ seconds_between_polls_if_there_are_tasks_in_the_queue
82
+ )
83
+ self.__seconds_between_polls_if_there_are_no_new_tasks = seconds_between_polls_if_there_are_no_new_tasks
84
+ self.__polling_tasks_semaphore = Semaphore(max_number_of_concurrent_polling_tasks)
85
+
86
+ self.__task_execution_mutex = Lock()
87
+
88
+ async def run(self) -> None:
89
+ start_time = datetime.now()
90
+ await self.__generate_kafka_consumers()
91
+
92
+ if len(self.__subscriber_per_consumer_mapper) == 0:
93
+ self._logger.error("There are no valid subscribers to execute, finalizing consumer")
94
+ return
95
+
96
+ self.__create_queue_repository_per_consumer()
97
+ self.__initial_coroutines_created_elapsed_time = datetime.now() - start_time
98
+
99
+ start_consumption_time = datetime.now()
100
+ self._logger.info("Starting to consume events")
101
+ worker_errors = await self.__run_worker()
102
+ self.__events_processed_elapsed_time = datetime.now() - start_consumption_time
103
+
104
+ await self.__handle_graceful_stop(worker_errors)
105
+
106
+ async def __handle_graceful_stop(self, worker_errors: tuple[Optional[Exception], Optional[Exception]]) -> None:
107
+ self._logger.info("Stopping kafka consumers...")
108
+ await self.__manage_kafka_consumers_stopping()
109
+ self._logger.info("All kafka consumers stopped")
110
+
111
+ self.__print_statistics()
112
+
113
+ if self.__exceptions_are_thrown(worker_errors):
114
+ consume_events_exception, polling_task_exception = worker_errors
115
+ if consume_events_exception:
116
+ self._logger.error(consume_events_exception)
117
+ if polling_task_exception:
118
+ self._logger.error(polling_task_exception)
119
+
120
+ raise WorkerExecutionException("The worker was closed by an unexpected exception")
121
+
122
+ async def __run_worker(self) -> tuple[Optional[Exception], Optional[Exception]]:
123
+ consume_events_task = self.__consume_events_task()
124
+ polling_task = self.__polling_task()
125
+
126
+ try:
127
+ await gather(consume_events_task, polling_task)
128
+ return (None, None)
129
+ except Exception:
130
+ self.__should_stop.set()
131
+ consume_events_exception = await self.__await_exception(consume_events_task)
132
+ polling_task_exception = await self.__await_exception(polling_task)
133
+ return (consume_events_exception, polling_task_exception)
134
+
135
+ async def __await_exception(self, future: Coroutine) -> Optional[Exception]:
136
+ try:
137
+ await future
138
+ return None
139
+ except Exception as exception:
140
+ return exception
141
+
142
+ def __exceptions_are_thrown(self, worker_errors: tuple[Optional[Exception], Optional[Exception]]) -> bool:
143
+ return any([error is not None for error in worker_errors])
144
+
145
+ async def __generate_kafka_consumers(self):
146
+ start_time = datetime.now()
147
+ tasks = [self.__initialize_kafka_consumer_for_subscriber(subscriber) for subscriber in self.__subscribers]
148
+ await gather(*tasks)
149
+ self.__start_kafka_consumers_elapsed_time = datetime.now() - start_time
150
+
151
+ async def __initialize_kafka_consumer_for_subscriber(self, subscriber: MetaSubscriber) -> None:
152
+ try:
153
+ executor = await self._create_kafka_consumer_executor(subscriber)
154
+ topics = self.__consume_strategy.get_topics(subscriber)
155
+ kafka_consumer = AIOKafkaConsumer(
156
+ consumer_group=self.__consume_strategy.get_subscription_group(subscriber),
157
+ topics=topics,
158
+ connection_config=self.__connection_config,
159
+ initial_offset_position=self.__consumer_initial_offset_position,
160
+ partition_assignors=self.__kafka_partition_assignors + self.__FALLBACK_PARTITION_ASSIGNORS,
161
+ logger=self._logger,
162
+ kafka_admin_client=self.__kafka_admin_client,
163
+ auto_create_topic_configuration=self.__auto_create_topic_configuration,
164
+ on_partition_revoked=self.__on_partition_revoked,
165
+ )
166
+
167
+ self.__subscriber_per_consumer_mapper[kafka_consumer] = subscriber
168
+
169
+ self.__executor_per_consumer_mapper[kafka_consumer] = executor
170
+
171
+ await kafka_consumer.init()
172
+ except Exception:
173
+ self._logger.exception(
174
+ f"Unexpected error during Kafka subscriber '{subscriber.fqn()}' initialization. Skipping it: {traceback.format_exc()}"
175
+ )
176
+
177
+ @abstractmethod
178
+ async def _create_kafka_consumer_executor(self, subscriber: MetaSubscriber) -> KafkaEventSubscriberExecutor:
179
+ pass
180
+
181
+ def __create_queue_repository_per_consumer(self) -> None:
182
+ for kafka_consumer in self.__subscriber_per_consumer_mapper.keys():
183
+ self.__queue_per_consumer_mapper[kafka_consumer] = InMemoryMultiqueueRepository()
184
+
185
+ async def __polling_task(self) -> None:
186
+ try:
187
+ while not self.__should_stop.is_set():
188
+ total_size = sum([queue.get_total_size() for queue in self.__queue_per_consumer_mapper.values()])
189
+ if total_size >= self.__max_queue_size:
190
+ await sleep(self.__seconds_between_polls_if_there_are_tasks_in_the_queue)
191
+ continue
192
+
193
+ raw_consuming_tasks = await gather(
194
+ *[
195
+ self.__polling_consuming_tasks(kafka_consumer=consumer)
196
+ for consumer, subscriber in self.__subscriber_per_consumer_mapper.items()
197
+ ]
198
+ )
199
+
200
+ poll_results: list[ConsumingTask] = [
201
+ consuming_task for consuming_tasks in raw_consuming_tasks for consuming_task in consuming_tasks
202
+ ]
203
+ if len(poll_results) == 0:
204
+ await sleep(self.__seconds_between_polls_if_there_are_no_new_tasks)
205
+
206
+ for poll_result in poll_results:
207
+ queue = self.__queue_per_consumer_mapper[poll_result.consumer]
208
+ queue.push(
209
+ key=TopicPartition(
210
+ topic=poll_result.kafka_poll_record.topic, partition=poll_result.kafka_poll_record.partition
211
+ ),
212
+ record=poll_result.kafka_poll_record,
213
+ )
214
+
215
+ except Exception:
216
+ self._logger.error(f"Polling task failed with exception: {traceback.format_exc()}")
217
+ self.__should_stop.set()
218
+
219
+ return
220
+
221
+ async def __polling_consuming_tasks(self, kafka_consumer: AIOKafkaConsumer) -> list[ConsumingTask]:
222
+ async with self.__polling_tasks_semaphore:
223
+ results = await kafka_consumer.poll(
224
+ number_of_messages_to_poll=self.__max_records_retrieved_per_poll,
225
+ )
226
+ return [ConsumingTask(kafka_consumer, result) for result in results]
227
+
228
+ async def __consume_events_task(self) -> None:
229
+ blocked_tasks_iterator = self.generate_blocked_consuming_tasks_iterator()
230
+
231
+ async for consuming_task in blocked_tasks_iterator:
232
+ consumer = consuming_task.consumer
233
+ kafka_poll_record = consuming_task.kafka_poll_record
234
+ executor = self.__executor_per_consumer_mapper[consuming_task.consumer]
235
+
236
+ await executor.consume(kafka_poll_record=kafka_poll_record)
237
+
238
+ await consumer.commit_poll_record(kafka_poll_record)
239
+
240
+ self.__events_processed += 1
241
+
242
+ # This iterator return a blocked task, that will be blocked for other process (like rebalancing), until the next task will be requested
243
+ async def generate_blocked_consuming_tasks_iterator(self) -> AsyncIterator[ConsumingTask]:
244
+ consumer_queues_cyclic_iterator = cycle(self.__queue_per_consumer_mapper.items())
245
+ last_consumer, _ = next(consumer_queues_cyclic_iterator)
246
+
247
+ while not self.__should_stop.is_set():
248
+ all_queues_are_empty = all(
249
+ [queue.is_totally_empty() for queue in self.__queue_per_consumer_mapper.values()]
250
+ )
251
+
252
+ if all_queues_are_empty:
253
+ await sleep(self.__seconds_between_executions_if_there_are_no_tasks_in_the_queue)
254
+ continue
255
+
256
+ async with self.__task_execution_mutex:
257
+ consumer: Optional[AIOKafkaConsumer] = None
258
+ kafka_poll_record: Optional[KafkaPollRecord] = None
259
+
260
+ while consumer != last_consumer:
261
+ consumer, queue = next(consumer_queues_cyclic_iterator)
262
+ kafka_poll_record = queue.pop()
263
+
264
+ if kafka_poll_record is not None:
265
+ yield ConsumingTask(consumer, kafka_poll_record)
266
+ last_consumer = consumer
267
+ break
268
+
269
+ return
270
+
271
+ async def __on_partition_revoked(self, consumer: AIOKafkaConsumer, topics_partitions: set[TopicPartition]) -> None:
272
+ async with self.__task_execution_mutex:
273
+ for topic_partition in topics_partitions:
274
+ self.__queue_per_consumer_mapper[consumer].clear(topic_partition)
275
+
276
+ def request_stop(self) -> None:
277
+ self.__should_stop.set()
278
+ self._logger.info("Worker stop requested. Waiting for finalize the current task")
279
+
280
+ async def __manage_kafka_consumers_stopping(self) -> None:
281
+ for kafka_consumer in self.__subscriber_per_consumer_mapper.keys():
282
+ await kafka_consumer.stop()
283
+
284
+ def __print_statistics(self) -> None:
285
+ self._logger.info("Number of subscribers: %d", len(self.__subscribers))
286
+ self._logger.info(f"Start kafka consumers elapsed time: {self.__start_kafka_consumers_elapsed_time}")
287
+ self._logger.info(f"Initial coroutines created elapsed time: {self.__initial_coroutines_created_elapsed_time}")
288
+ self._logger.info(f"Events processed: {self.__events_processed}")
289
+ self._logger.info(f"Events processed elapsed time: {self.__events_processed_elapsed_time}")
@@ -0,0 +1,102 @@
1
+ from logging import Logger
2
+ from typing import Optional, Sequence, Type, TypeVar
3
+
4
+ from aiokafka.coordinator.assignors.abstract import AbstractPartitionAssignor
5
+
6
+ from buz.event import Event
7
+
8
+ from buz.event.async_subscriber import AsyncSubscriber
9
+ from buz.event.infrastructure.buz_kafka.base_buz_aiokafka_async_consumer import BaseBuzAIOKafkaAsyncConsumer
10
+ from buz.event.infrastructure.buz_kafka.consume_strategy.consume_strategy import KafkaConsumeStrategy
11
+ from buz.event.infrastructure.buz_kafka.consume_strategy.kafka_on_fail_strategy import KafkaOnFailStrategy
12
+ from buz.event.infrastructure.buz_kafka.kafka_event_async_subscriber_executor import KafkaEventAsyncSubscriberExecutor
13
+ from buz.event.infrastructure.buz_kafka.kafka_event_subscriber_executor import KafkaEventSubscriberExecutor
14
+ from buz.event.meta_subscriber import MetaSubscriber
15
+ from buz.event.middleware.async_consume_middleware import AsyncConsumeMiddleware
16
+ from buz.event.strategies.retry.consume_retrier import ConsumeRetrier
17
+ from buz.event.strategies.retry.reject_callback import RejectCallback
18
+ from buz.kafka import (
19
+ KafkaConnectionConfig,
20
+ ConsumerInitialOffsetPosition,
21
+ )
22
+ from buz.kafka.domain.models.auto_create_topic_configuration import AutoCreateTopicConfiguration
23
+ from buz.kafka.domain.services.kafka_admin_client import KafkaAdminClient
24
+ from buz.kafka.infrastructure.deserializers.bytes_to_message_deserializer import BytesToMessageDeserializer
25
+ from buz.kafka.infrastructure.deserializers.implementations.json_bytes_to_message_deserializer import (
26
+ JSONBytesToMessageDeserializer,
27
+ )
28
+ from buz.kafka.infrastructure.serializers.kafka_header_serializer import KafkaHeaderSerializer
29
+
30
+ T = TypeVar("T", bound=Event)
31
+
32
+
33
+ class BuzAIOKafkaAsyncConsumer(BaseBuzAIOKafkaAsyncConsumer):
34
+ def __init__(
35
+ self,
36
+ *,
37
+ connection_config: KafkaConnectionConfig,
38
+ kafka_admin_client: Optional[KafkaAdminClient],
39
+ consume_strategy: KafkaConsumeStrategy,
40
+ on_fail_strategy: KafkaOnFailStrategy,
41
+ max_queue_size: int,
42
+ max_records_retrieved_per_poll: int,
43
+ kafka_partition_assignors: tuple[Type[AbstractPartitionAssignor], ...] = (),
44
+ subscribers: Sequence[AsyncSubscriber],
45
+ logger: Logger,
46
+ consumer_initial_offset_position: ConsumerInitialOffsetPosition,
47
+ deserializers_per_subscriber: dict[MetaSubscriber, BytesToMessageDeserializer[T]],
48
+ consume_middlewares: Optional[Sequence[AsyncConsumeMiddleware]] = None,
49
+ consume_retrier: Optional[ConsumeRetrier] = None,
50
+ reject_callback: Optional[RejectCallback] = None,
51
+ auto_create_topic_configuration: Optional[AutoCreateTopicConfiguration] = None,
52
+ seconds_between_executions_if_there_are_no_tasks_in_the_queue: int = 1,
53
+ seconds_between_polls_if_there_are_tasks_in_the_queue: int = 1,
54
+ seconds_between_polls_if_there_are_no_new_tasks: int = 1,
55
+ max_number_of_concurrent_polling_tasks: int = 20,
56
+ ):
57
+ super().__init__(
58
+ connection_config=connection_config,
59
+ kafka_admin_client=kafka_admin_client,
60
+ consume_strategy=consume_strategy,
61
+ max_queue_size=max_queue_size,
62
+ max_records_retrieved_per_poll=max_records_retrieved_per_poll,
63
+ kafka_partition_assignors=kafka_partition_assignors,
64
+ subscribers=subscribers,
65
+ logger=logger,
66
+ consumer_initial_offset_position=consumer_initial_offset_position,
67
+ auto_create_topic_configuration=auto_create_topic_configuration,
68
+ seconds_between_executions_if_there_are_no_tasks_in_the_queue=seconds_between_executions_if_there_are_no_tasks_in_the_queue,
69
+ seconds_between_polls_if_there_are_tasks_in_the_queue=seconds_between_polls_if_there_are_tasks_in_the_queue,
70
+ seconds_between_polls_if_there_are_no_new_tasks=seconds_between_polls_if_there_are_no_new_tasks,
71
+ max_number_of_concurrent_polling_tasks=max_number_of_concurrent_polling_tasks,
72
+ )
73
+ self.__on_fail_strategy = on_fail_strategy
74
+ self.__consume_middlewares = consume_middlewares
75
+ self.__consume_retrier = consume_retrier
76
+ self.__reject_callback = reject_callback
77
+ self._deserializers_per_subscriber = deserializers_per_subscriber
78
+
79
+ async def _create_kafka_consumer_executor(
80
+ self,
81
+ subscriber: MetaSubscriber,
82
+ ) -> KafkaEventSubscriberExecutor:
83
+ if not isinstance(subscriber, AsyncSubscriber):
84
+ raise TypeError(
85
+ f"Subscriber {subscriber.__class__.__name__} is not a subclass of Subscriber, probably you are trying to use a synchronous subscriber"
86
+ )
87
+
88
+ byte_deserializer = self._deserializers_per_subscriber.get(subscriber) or JSONBytesToMessageDeserializer(
89
+ # todo: it looks like in next python versions the inference engine is powerful enough to ensure this type, so we can remove it when we upgrade the python version of the library
90
+ event_class=subscriber.handles() # type: ignore
91
+ )
92
+
93
+ return KafkaEventAsyncSubscriberExecutor(
94
+ logger=self._logger,
95
+ byte_deserializer=byte_deserializer,
96
+ header_deserializer=KafkaHeaderSerializer(),
97
+ on_fail_strategy=self.__on_fail_strategy,
98
+ subscriber=subscriber,
99
+ consume_middlewares=self.__consume_middlewares,
100
+ consume_retrier=self.__consume_retrier,
101
+ reject_callback=self.__reject_callback,
102
+ )