buz 2.12.0rc2__py3-none-any.whl → 2.13.1rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- buz/event/__init__.py +8 -0
- buz/event/async_subscriber.py +10 -0
- buz/event/async_worker.py +6 -3
- buz/event/base_async_subscriber.py +8 -0
- buz/event/base_subscriber.py +4 -19
- buz/event/dead_letter_queue/dlq_criteria.py +11 -0
- buz/event/dead_letter_queue/dlq_record.py +31 -0
- buz/event/dead_letter_queue/dlq_repository.py +23 -0
- buz/event/exceptions/worker_execution_exception.py +3 -0
- buz/event/infrastructure/buz_kafka/base_buz_aiokafka_async_consumer.py +289 -0
- buz/event/infrastructure/buz_kafka/buz_aiokafka_async_consumer.py +102 -0
- buz/event/infrastructure/buz_kafka/buz_aiokafka_multi_threaded_consumer.py +42 -197
- buz/event/infrastructure/buz_kafka/consume_strategy/consume_strategy.py +3 -3
- buz/event/infrastructure/buz_kafka/consume_strategy/topic_and_subscription_group_per_subscriber_kafka_consumer_strategy.py +3 -3
- buz/event/infrastructure/buz_kafka/kafka_event_async_subscriber_executor.py +113 -0
- buz/event/infrastructure/buz_kafka/kafka_event_subscriber_executor.py +4 -103
- buz/event/infrastructure/buz_kafka/kafka_event_sync_subscriber_executor.py +114 -0
- buz/event/infrastructure/kombu/kombu_consumer.py +6 -4
- buz/event/infrastructure/models/consuming_task.py +9 -0
- buz/event/meta_base_subscriber.py +23 -0
- buz/event/meta_subscriber.py +11 -0
- buz/event/middleware/async_consume_middleware.py +14 -0
- buz/event/middleware/async_consume_middleware_chain_resolver.py +29 -0
- buz/event/middleware/consume_middleware_chain_resolver.py +2 -1
- buz/event/strategies/retry/consume_retrier.py +5 -3
- buz/event/strategies/retry/consumed_event_retry_repository.py +6 -2
- buz/event/strategies/retry/max_retries_consume_retrier.py +9 -5
- buz/event/strategies/retry/reject_callback.py +4 -2
- buz/event/subscriber.py +3 -9
- buz/event/transactional_outbox/outbox_criteria/outbox_criteria.py +1 -0
- buz/event/worker.py +6 -3
- buz/kafka/__init__.py +0 -2
- buz/kafka/domain/services/kafka_admin_test_client.py +2 -2
- buz/kafka/infrastructure/aiokafka/{aiokafka_multi_threaded_consumer.py → aiokafka_consumer.py} +48 -25
- buz/kafka/infrastructure/aiokafka/rebalance/kafka_callback_rebalancer.py +42 -0
- buz/kafka/infrastructure/deserializers/implementations/json_bytes_to_message_deserializer.py +2 -2
- buz/kafka/infrastructure/kafka_python/kafka_python_admin_test_client.py +4 -2
- buz/middleware/middleware_chain_builder.py +2 -2
- buz/queue/in_memory/in_memory_multiqueue_repository.py +74 -0
- buz/{event/infrastructure/buz_kafka → queue/in_memory}/in_memory_queue_repository.py +2 -1
- buz/queue/multiqueue_repository.py +27 -0
- {buz-2.12.0rc2.dist-info → buz-2.13.1rc1.dist-info}/METADATA +2 -2
- {buz-2.12.0rc2.dist-info → buz-2.13.1rc1.dist-info}/RECORD +48 -35
- buz/kafka/domain/services/kafka_consumer.py +0 -25
- buz/kafka/infrastructure/aiokafka/factories/__init__.py +0 -0
- buz/kafka/infrastructure/aiokafka/factories/kafka_python_multi_threaded_consumer_factory.py +0 -55
- buz/kafka/infrastructure/aiokafka/rebalance/rebalance_ready.py +0 -7
- buz/kafka/infrastructure/aiokafka/rebalance/simple_kafka_lock_rebalancer.py +0 -18
- /buz/event/{domain → dead_letter_queue}/__init__.py +0 -0
- /buz/{event/domain/queue → queue}/__init__.py +0 -0
- /buz/{event/domain/queue → queue}/queue_repository.py +0 -0
- {buz-2.12.0rc2.dist-info → buz-2.13.1rc1.dist-info}/LICENSE +0 -0
- {buz-2.12.0rc2.dist-info → buz-2.13.1rc1.dist-info}/WHEEL +0 -0
buz/event/__init__.py
CHANGED
|
@@ -1,11 +1,19 @@
|
|
|
1
1
|
from buz.event.event import Event
|
|
2
2
|
from buz.event.subscriber import Subscriber
|
|
3
|
+
from buz.event.async_subscriber import AsyncSubscriber
|
|
3
4
|
from buz.event.base_subscriber import BaseSubscriber
|
|
5
|
+
from buz.event.base_async_subscriber import BaseAsyncSubscriber
|
|
4
6
|
from buz.event.event_bus import EventBus
|
|
7
|
+
from buz.event.meta_subscriber import MetaSubscriber
|
|
8
|
+
from buz.event.meta_base_subscriber import MetaBaseSubscriber
|
|
5
9
|
|
|
6
10
|
__all__ = [
|
|
7
11
|
"Event",
|
|
8
12
|
"Subscriber",
|
|
13
|
+
"AsyncSubscriber",
|
|
9
14
|
"BaseSubscriber",
|
|
15
|
+
"BaseAsyncSubscriber",
|
|
10
16
|
"EventBus",
|
|
17
|
+
"MetaSubscriber",
|
|
18
|
+
"MetaBaseSubscriber",
|
|
11
19
|
]
|
buz/event/async_worker.py
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import asyncio
|
|
2
|
+
from logging import Logger
|
|
2
3
|
from signal import SIGTERM, SIGINT
|
|
3
4
|
|
|
4
5
|
from buz.event.strategies.execution_strategy.async_execution_strategy import AsyncExecutionStrategy
|
|
@@ -7,18 +8,20 @@ from buz.event.strategies.execution_strategy.async_execution_strategy import Asy
|
|
|
7
8
|
class AsyncWorker:
|
|
8
9
|
def __init__(
|
|
9
10
|
self,
|
|
11
|
+
logger: Logger,
|
|
10
12
|
execution_strategy: AsyncExecutionStrategy,
|
|
11
13
|
) -> None:
|
|
12
14
|
self.__execution_strategy = execution_strategy
|
|
15
|
+
self.__logger = logger
|
|
13
16
|
|
|
14
17
|
async def start(self) -> None:
|
|
15
18
|
loop = asyncio.get_running_loop()
|
|
16
19
|
loop.add_signal_handler(SIGINT, self.stop)
|
|
17
20
|
loop.add_signal_handler(SIGTERM, self.stop)
|
|
18
|
-
|
|
21
|
+
self.__logger.info("Starting buz worker...")
|
|
19
22
|
await self.__execution_strategy.start()
|
|
20
|
-
|
|
23
|
+
self.__logger.info("Buz worker stopped gracefully")
|
|
21
24
|
|
|
22
25
|
def stop(self) -> None:
|
|
23
|
-
|
|
26
|
+
self.__logger.info("Stopping buz worker...")
|
|
24
27
|
self.__execution_strategy.request_stop()
|
buz/event/base_subscriber.py
CHANGED
|
@@ -1,22 +1,7 @@
|
|
|
1
|
-
from
|
|
2
|
-
|
|
3
|
-
from buz.event import Event
|
|
1
|
+
from abc import ABC
|
|
4
2
|
from buz.event import Subscriber
|
|
3
|
+
from buz.event.meta_base_subscriber import MetaBaseSubscriber
|
|
5
4
|
|
|
6
5
|
|
|
7
|
-
class BaseSubscriber(Subscriber):
|
|
8
|
-
|
|
9
|
-
def fqn(cls) -> str:
|
|
10
|
-
return f"subscriber.{cls.__module__}.{cls.__name__}"
|
|
11
|
-
|
|
12
|
-
@classmethod
|
|
13
|
-
def handles(cls) -> Type[Event]:
|
|
14
|
-
consume_types = get_type_hints(cls.consume)
|
|
15
|
-
|
|
16
|
-
if "event" not in consume_types:
|
|
17
|
-
raise TypeError("event parameter not found in consume method")
|
|
18
|
-
|
|
19
|
-
if not issubclass(consume_types["event"], Event):
|
|
20
|
-
raise TypeError("event parameter is not an buz.event.Event subclass")
|
|
21
|
-
|
|
22
|
-
return consume_types["event"]
|
|
6
|
+
class BaseSubscriber(Subscriber, MetaBaseSubscriber, ABC):
|
|
7
|
+
pass
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
from dataclasses import dataclass
|
|
2
|
+
from typing import ClassVar, Union
|
|
3
|
+
|
|
4
|
+
from buz.event.dead_letter_queue.dlq_record import DlqRecordId
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
@dataclass(frozen=True)
|
|
8
|
+
class DlqCriteria:
|
|
9
|
+
UNSET_VALUE: ClassVar[object] = object()
|
|
10
|
+
|
|
11
|
+
dlq_record_id: Union[DlqRecordId, None, object] = UNSET_VALUE
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
from dataclasses import dataclass, fields
|
|
2
|
+
from datetime import datetime
|
|
3
|
+
from typing import Any, ClassVar
|
|
4
|
+
from uuid import UUID
|
|
5
|
+
|
|
6
|
+
DlqRecordId = UUID
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
@dataclass
|
|
10
|
+
class DlqRecord:
|
|
11
|
+
DATE_TIME_FORMAT: ClassVar[str] = "%Y-%m-%d %H:%M:%S.%f"
|
|
12
|
+
|
|
13
|
+
id: DlqRecordId
|
|
14
|
+
event_id: UUID
|
|
15
|
+
subscriber_fqn: str
|
|
16
|
+
event_payload: dict
|
|
17
|
+
exception_type: str
|
|
18
|
+
exception_message: str
|
|
19
|
+
last_failed_at: datetime
|
|
20
|
+
|
|
21
|
+
def mark_as_failed(self) -> None:
|
|
22
|
+
self.last_failed_at = datetime.now()
|
|
23
|
+
|
|
24
|
+
def get_attrs(self) -> dict[str, Any]:
|
|
25
|
+
attrs = {}
|
|
26
|
+
for field in fields(self):
|
|
27
|
+
property_name = field.name
|
|
28
|
+
property_value = getattr(self, property_name)
|
|
29
|
+
attrs[property_name] = property_value
|
|
30
|
+
|
|
31
|
+
return attrs
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
from abc import ABC, abstractmethod
|
|
2
|
+
from typing import Sequence
|
|
3
|
+
|
|
4
|
+
from buz.event.dead_letter_queue.dlq_criteria import DlqCriteria
|
|
5
|
+
from buz.event.dead_letter_queue.dlq_record import DlqRecord, DlqRecordId
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class DlqRepository(ABC):
|
|
9
|
+
@abstractmethod
|
|
10
|
+
def save(self, dlq_record: DlqRecord) -> None:
|
|
11
|
+
pass
|
|
12
|
+
|
|
13
|
+
@abstractmethod
|
|
14
|
+
def find_one_or_fail_by_criteria(self, criteria: DlqCriteria) -> DlqRecord:
|
|
15
|
+
pass
|
|
16
|
+
|
|
17
|
+
@abstractmethod
|
|
18
|
+
def delete(self, dlq_record_id: DlqRecordId) -> None:
|
|
19
|
+
pass
|
|
20
|
+
|
|
21
|
+
@abstractmethod
|
|
22
|
+
def bulk_delete(self, dlq_record_ids: Sequence[DlqRecordId]) -> None:
|
|
23
|
+
pass
|
|
@@ -0,0 +1,289 @@
|
|
|
1
|
+
from abc import abstractmethod
|
|
2
|
+
import traceback
|
|
3
|
+
from asyncio import Lock, gather, Semaphore, Event as AsyncIOEvent, sleep
|
|
4
|
+
from datetime import timedelta, datetime
|
|
5
|
+
from itertools import cycle
|
|
6
|
+
from logging import Logger
|
|
7
|
+
from typing import AsyncIterator, Coroutine, Optional, Sequence, Type, TypeVar
|
|
8
|
+
|
|
9
|
+
from aiokafka import TopicPartition
|
|
10
|
+
from aiokafka.coordinator.assignors.abstract import AbstractPartitionAssignor
|
|
11
|
+
from aiokafka.coordinator.assignors.roundrobin import RoundRobinPartitionAssignor
|
|
12
|
+
|
|
13
|
+
from buz.event import Event
|
|
14
|
+
from buz.event.async_consumer import AsyncConsumer
|
|
15
|
+
from buz.event.exceptions.worker_execution_exception import WorkerExecutionException
|
|
16
|
+
from buz.event.infrastructure.buz_kafka.consume_strategy.consume_strategy import KafkaConsumeStrategy
|
|
17
|
+
from buz.event.infrastructure.buz_kafka.kafka_event_subscriber_executor import KafkaEventSubscriberExecutor
|
|
18
|
+
from buz.event.infrastructure.models.consuming_task import ConsumingTask
|
|
19
|
+
from buz.event.meta_subscriber import MetaSubscriber
|
|
20
|
+
from buz.kafka import (
|
|
21
|
+
KafkaConnectionConfig,
|
|
22
|
+
ConsumerInitialOffsetPosition,
|
|
23
|
+
)
|
|
24
|
+
from buz.kafka.domain.models.auto_create_topic_configuration import AutoCreateTopicConfiguration
|
|
25
|
+
from buz.kafka.domain.models.kafka_poll_record import KafkaPollRecord
|
|
26
|
+
from buz.kafka.domain.services.kafka_admin_client import KafkaAdminClient
|
|
27
|
+
from buz.kafka.infrastructure.aiokafka.aiokafka_consumer import AIOKafkaConsumer
|
|
28
|
+
from buz.queue.in_memory.in_memory_multiqueue_repository import InMemoryMultiqueueRepository
|
|
29
|
+
from buz.queue.multiqueue_repository import MultiqueueRepository
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
T = TypeVar("T", bound=Event)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class BaseBuzAIOKafkaAsyncConsumer(AsyncConsumer):
|
|
36
|
+
__FALLBACK_PARTITION_ASSIGNORS = (RoundRobinPartitionAssignor,)
|
|
37
|
+
|
|
38
|
+
def __init__(
|
|
39
|
+
self,
|
|
40
|
+
*,
|
|
41
|
+
connection_config: KafkaConnectionConfig,
|
|
42
|
+
kafka_admin_client: Optional[KafkaAdminClient],
|
|
43
|
+
consume_strategy: KafkaConsumeStrategy,
|
|
44
|
+
max_queue_size: int,
|
|
45
|
+
max_records_retrieved_per_poll: int,
|
|
46
|
+
kafka_partition_assignors: tuple[Type[AbstractPartitionAssignor], ...] = (),
|
|
47
|
+
subscribers: Sequence[MetaSubscriber],
|
|
48
|
+
logger: Logger,
|
|
49
|
+
consumer_initial_offset_position: ConsumerInitialOffsetPosition,
|
|
50
|
+
auto_create_topic_configuration: Optional[AutoCreateTopicConfiguration] = None,
|
|
51
|
+
seconds_between_executions_if_there_are_no_tasks_in_the_queue: int = 1,
|
|
52
|
+
seconds_between_polls_if_there_are_tasks_in_the_queue: int = 1,
|
|
53
|
+
seconds_between_polls_if_there_are_no_new_tasks: int = 1,
|
|
54
|
+
max_number_of_concurrent_polling_tasks: int = 20,
|
|
55
|
+
):
|
|
56
|
+
self.__connection_config = connection_config
|
|
57
|
+
self.__consume_strategy = consume_strategy
|
|
58
|
+
self.__kafka_partition_assignors = kafka_partition_assignors
|
|
59
|
+
self.__subscribers = subscribers
|
|
60
|
+
self._logger = logger
|
|
61
|
+
self.__consumer_initial_offset_position = consumer_initial_offset_position
|
|
62
|
+
self.__max_records_retrieved_per_poll = 1
|
|
63
|
+
self.__subscriber_per_consumer_mapper: dict[AIOKafkaConsumer, MetaSubscriber] = {}
|
|
64
|
+
self.__executor_per_consumer_mapper: dict[AIOKafkaConsumer, KafkaEventSubscriberExecutor] = {}
|
|
65
|
+
self.__queue_per_consumer_mapper: dict[
|
|
66
|
+
AIOKafkaConsumer, MultiqueueRepository[TopicPartition, KafkaPollRecord]
|
|
67
|
+
] = {}
|
|
68
|
+
self.__max_records_retrieved_per_poll = max_records_retrieved_per_poll
|
|
69
|
+
self.__max_queue_size = max_queue_size
|
|
70
|
+
self.__should_stop = AsyncIOEvent()
|
|
71
|
+
self.__start_kafka_consumers_elapsed_time: Optional[timedelta] = None
|
|
72
|
+
self.__initial_coroutines_created_elapsed_time: Optional[timedelta] = None
|
|
73
|
+
self.__events_processed: int = 0
|
|
74
|
+
self.__events_processed_elapsed_time: timedelta = timedelta()
|
|
75
|
+
self.__kafka_admin_client = kafka_admin_client
|
|
76
|
+
self.__auto_create_topic_configuration = auto_create_topic_configuration
|
|
77
|
+
self.__seconds_between_executions_if_there_are_no_tasks_in_the_queue = (
|
|
78
|
+
seconds_between_executions_if_there_are_no_tasks_in_the_queue
|
|
79
|
+
)
|
|
80
|
+
self.__seconds_between_polls_if_there_are_tasks_in_the_queue = (
|
|
81
|
+
seconds_between_polls_if_there_are_tasks_in_the_queue
|
|
82
|
+
)
|
|
83
|
+
self.__seconds_between_polls_if_there_are_no_new_tasks = seconds_between_polls_if_there_are_no_new_tasks
|
|
84
|
+
self.__polling_tasks_semaphore = Semaphore(max_number_of_concurrent_polling_tasks)
|
|
85
|
+
|
|
86
|
+
self.__task_execution_mutex = Lock()
|
|
87
|
+
|
|
88
|
+
async def run(self) -> None:
|
|
89
|
+
start_time = datetime.now()
|
|
90
|
+
await self.__generate_kafka_consumers()
|
|
91
|
+
|
|
92
|
+
if len(self.__subscriber_per_consumer_mapper) == 0:
|
|
93
|
+
self._logger.error("There are no valid subscribers to execute, finalizing consumer")
|
|
94
|
+
return
|
|
95
|
+
|
|
96
|
+
self.__create_queue_repository_per_consumer()
|
|
97
|
+
self.__initial_coroutines_created_elapsed_time = datetime.now() - start_time
|
|
98
|
+
|
|
99
|
+
start_consumption_time = datetime.now()
|
|
100
|
+
self._logger.info("Starting to consume events")
|
|
101
|
+
worker_errors = await self.__run_worker()
|
|
102
|
+
self.__events_processed_elapsed_time = datetime.now() - start_consumption_time
|
|
103
|
+
|
|
104
|
+
await self.__handle_graceful_stop(worker_errors)
|
|
105
|
+
|
|
106
|
+
async def __handle_graceful_stop(self, worker_errors: tuple[Optional[Exception], Optional[Exception]]) -> None:
|
|
107
|
+
self._logger.info("Stopping kafka consumers...")
|
|
108
|
+
await self.__manage_kafka_consumers_stopping()
|
|
109
|
+
self._logger.info("All kafka consumers stopped")
|
|
110
|
+
|
|
111
|
+
self.__print_statistics()
|
|
112
|
+
|
|
113
|
+
if self.__exceptions_are_thrown(worker_errors):
|
|
114
|
+
consume_events_exception, polling_task_exception = worker_errors
|
|
115
|
+
if consume_events_exception:
|
|
116
|
+
self._logger.error(consume_events_exception)
|
|
117
|
+
if polling_task_exception:
|
|
118
|
+
self._logger.error(polling_task_exception)
|
|
119
|
+
|
|
120
|
+
raise WorkerExecutionException("The worker was closed by an unexpected exception")
|
|
121
|
+
|
|
122
|
+
async def __run_worker(self) -> tuple[Optional[Exception], Optional[Exception]]:
|
|
123
|
+
consume_events_task = self.__consume_events_task()
|
|
124
|
+
polling_task = self.__polling_task()
|
|
125
|
+
|
|
126
|
+
try:
|
|
127
|
+
await gather(consume_events_task, polling_task)
|
|
128
|
+
return (None, None)
|
|
129
|
+
except Exception:
|
|
130
|
+
self.__should_stop.set()
|
|
131
|
+
consume_events_exception = await self.__await_exception(consume_events_task)
|
|
132
|
+
polling_task_exception = await self.__await_exception(polling_task)
|
|
133
|
+
return (consume_events_exception, polling_task_exception)
|
|
134
|
+
|
|
135
|
+
async def __await_exception(self, future: Coroutine) -> Optional[Exception]:
|
|
136
|
+
try:
|
|
137
|
+
await future
|
|
138
|
+
return None
|
|
139
|
+
except Exception as exception:
|
|
140
|
+
return exception
|
|
141
|
+
|
|
142
|
+
def __exceptions_are_thrown(self, worker_errors: tuple[Optional[Exception], Optional[Exception]]) -> bool:
|
|
143
|
+
return any([error is not None for error in worker_errors])
|
|
144
|
+
|
|
145
|
+
async def __generate_kafka_consumers(self):
|
|
146
|
+
start_time = datetime.now()
|
|
147
|
+
tasks = [self.__initialize_kafka_consumer_for_subscriber(subscriber) for subscriber in self.__subscribers]
|
|
148
|
+
await gather(*tasks)
|
|
149
|
+
self.__start_kafka_consumers_elapsed_time = datetime.now() - start_time
|
|
150
|
+
|
|
151
|
+
async def __initialize_kafka_consumer_for_subscriber(self, subscriber: MetaSubscriber) -> None:
|
|
152
|
+
try:
|
|
153
|
+
executor = await self._create_kafka_consumer_executor(subscriber)
|
|
154
|
+
topics = self.__consume_strategy.get_topics(subscriber)
|
|
155
|
+
kafka_consumer = AIOKafkaConsumer(
|
|
156
|
+
consumer_group=self.__consume_strategy.get_subscription_group(subscriber),
|
|
157
|
+
topics=topics,
|
|
158
|
+
connection_config=self.__connection_config,
|
|
159
|
+
initial_offset_position=self.__consumer_initial_offset_position,
|
|
160
|
+
partition_assignors=self.__kafka_partition_assignors + self.__FALLBACK_PARTITION_ASSIGNORS,
|
|
161
|
+
logger=self._logger,
|
|
162
|
+
kafka_admin_client=self.__kafka_admin_client,
|
|
163
|
+
auto_create_topic_configuration=self.__auto_create_topic_configuration,
|
|
164
|
+
on_partition_revoked=self.__on_partition_revoked,
|
|
165
|
+
)
|
|
166
|
+
|
|
167
|
+
self.__subscriber_per_consumer_mapper[kafka_consumer] = subscriber
|
|
168
|
+
|
|
169
|
+
self.__executor_per_consumer_mapper[kafka_consumer] = executor
|
|
170
|
+
|
|
171
|
+
await kafka_consumer.init()
|
|
172
|
+
except Exception:
|
|
173
|
+
self._logger.exception(
|
|
174
|
+
f"Unexpected error during Kafka subscriber '{subscriber.fqn()}' initialization. Skipping it: {traceback.format_exc()}"
|
|
175
|
+
)
|
|
176
|
+
|
|
177
|
+
@abstractmethod
|
|
178
|
+
async def _create_kafka_consumer_executor(self, subscriber: MetaSubscriber) -> KafkaEventSubscriberExecutor:
|
|
179
|
+
pass
|
|
180
|
+
|
|
181
|
+
def __create_queue_repository_per_consumer(self) -> None:
|
|
182
|
+
for kafka_consumer in self.__subscriber_per_consumer_mapper.keys():
|
|
183
|
+
self.__queue_per_consumer_mapper[kafka_consumer] = InMemoryMultiqueueRepository()
|
|
184
|
+
|
|
185
|
+
async def __polling_task(self) -> None:
|
|
186
|
+
try:
|
|
187
|
+
while not self.__should_stop.is_set():
|
|
188
|
+
total_size = sum([queue.get_total_size() for queue in self.__queue_per_consumer_mapper.values()])
|
|
189
|
+
if total_size >= self.__max_queue_size:
|
|
190
|
+
await sleep(self.__seconds_between_polls_if_there_are_tasks_in_the_queue)
|
|
191
|
+
continue
|
|
192
|
+
|
|
193
|
+
raw_consuming_tasks = await gather(
|
|
194
|
+
*[
|
|
195
|
+
self.__polling_consuming_tasks(kafka_consumer=consumer)
|
|
196
|
+
for consumer, subscriber in self.__subscriber_per_consumer_mapper.items()
|
|
197
|
+
]
|
|
198
|
+
)
|
|
199
|
+
|
|
200
|
+
poll_results: list[ConsumingTask] = [
|
|
201
|
+
consuming_task for consuming_tasks in raw_consuming_tasks for consuming_task in consuming_tasks
|
|
202
|
+
]
|
|
203
|
+
if len(poll_results) == 0:
|
|
204
|
+
await sleep(self.__seconds_between_polls_if_there_are_no_new_tasks)
|
|
205
|
+
|
|
206
|
+
for poll_result in poll_results:
|
|
207
|
+
queue = self.__queue_per_consumer_mapper[poll_result.consumer]
|
|
208
|
+
queue.push(
|
|
209
|
+
key=TopicPartition(
|
|
210
|
+
topic=poll_result.kafka_poll_record.topic, partition=poll_result.kafka_poll_record.partition
|
|
211
|
+
),
|
|
212
|
+
record=poll_result.kafka_poll_record,
|
|
213
|
+
)
|
|
214
|
+
|
|
215
|
+
except Exception:
|
|
216
|
+
self._logger.error(f"Polling task failed with exception: {traceback.format_exc()}")
|
|
217
|
+
self.__should_stop.set()
|
|
218
|
+
|
|
219
|
+
return
|
|
220
|
+
|
|
221
|
+
async def __polling_consuming_tasks(self, kafka_consumer: AIOKafkaConsumer) -> list[ConsumingTask]:
|
|
222
|
+
async with self.__polling_tasks_semaphore:
|
|
223
|
+
results = await kafka_consumer.poll(
|
|
224
|
+
number_of_messages_to_poll=self.__max_records_retrieved_per_poll,
|
|
225
|
+
)
|
|
226
|
+
return [ConsumingTask(kafka_consumer, result) for result in results]
|
|
227
|
+
|
|
228
|
+
async def __consume_events_task(self) -> None:
|
|
229
|
+
blocked_tasks_iterator = self.generate_blocked_consuming_tasks_iterator()
|
|
230
|
+
|
|
231
|
+
async for consuming_task in blocked_tasks_iterator:
|
|
232
|
+
consumer = consuming_task.consumer
|
|
233
|
+
kafka_poll_record = consuming_task.kafka_poll_record
|
|
234
|
+
executor = self.__executor_per_consumer_mapper[consuming_task.consumer]
|
|
235
|
+
|
|
236
|
+
await executor.consume(kafka_poll_record=kafka_poll_record)
|
|
237
|
+
|
|
238
|
+
await consumer.commit_poll_record(kafka_poll_record)
|
|
239
|
+
|
|
240
|
+
self.__events_processed += 1
|
|
241
|
+
|
|
242
|
+
# This iterator return a blocked task, that will be blocked for other process (like rebalancing), until the next task will be requested
|
|
243
|
+
async def generate_blocked_consuming_tasks_iterator(self) -> AsyncIterator[ConsumingTask]:
|
|
244
|
+
consumer_queues_cyclic_iterator = cycle(self.__queue_per_consumer_mapper.items())
|
|
245
|
+
last_consumer, _ = next(consumer_queues_cyclic_iterator)
|
|
246
|
+
|
|
247
|
+
while not self.__should_stop.is_set():
|
|
248
|
+
all_queues_are_empty = all(
|
|
249
|
+
[queue.is_totally_empty() for queue in self.__queue_per_consumer_mapper.values()]
|
|
250
|
+
)
|
|
251
|
+
|
|
252
|
+
if all_queues_are_empty:
|
|
253
|
+
await sleep(self.__seconds_between_executions_if_there_are_no_tasks_in_the_queue)
|
|
254
|
+
continue
|
|
255
|
+
|
|
256
|
+
async with self.__task_execution_mutex:
|
|
257
|
+
consumer: Optional[AIOKafkaConsumer] = None
|
|
258
|
+
kafka_poll_record: Optional[KafkaPollRecord] = None
|
|
259
|
+
|
|
260
|
+
while consumer != last_consumer:
|
|
261
|
+
consumer, queue = next(consumer_queues_cyclic_iterator)
|
|
262
|
+
kafka_poll_record = queue.pop()
|
|
263
|
+
|
|
264
|
+
if kafka_poll_record is not None:
|
|
265
|
+
yield ConsumingTask(consumer, kafka_poll_record)
|
|
266
|
+
last_consumer = consumer
|
|
267
|
+
break
|
|
268
|
+
|
|
269
|
+
return
|
|
270
|
+
|
|
271
|
+
async def __on_partition_revoked(self, consumer: AIOKafkaConsumer, topics_partitions: set[TopicPartition]) -> None:
|
|
272
|
+
async with self.__task_execution_mutex:
|
|
273
|
+
for topic_partition in topics_partitions:
|
|
274
|
+
self.__queue_per_consumer_mapper[consumer].clear(topic_partition)
|
|
275
|
+
|
|
276
|
+
def request_stop(self) -> None:
|
|
277
|
+
self.__should_stop.set()
|
|
278
|
+
self._logger.info("Worker stop requested. Waiting for finalize the current task")
|
|
279
|
+
|
|
280
|
+
async def __manage_kafka_consumers_stopping(self) -> None:
|
|
281
|
+
for kafka_consumer in self.__subscriber_per_consumer_mapper.keys():
|
|
282
|
+
await kafka_consumer.stop()
|
|
283
|
+
|
|
284
|
+
def __print_statistics(self) -> None:
|
|
285
|
+
self._logger.info("Number of subscribers: %d", len(self.__subscribers))
|
|
286
|
+
self._logger.info(f"Start kafka consumers elapsed time: {self.__start_kafka_consumers_elapsed_time}")
|
|
287
|
+
self._logger.info(f"Initial coroutines created elapsed time: {self.__initial_coroutines_created_elapsed_time}")
|
|
288
|
+
self._logger.info(f"Events processed: {self.__events_processed}")
|
|
289
|
+
self._logger.info(f"Events processed elapsed time: {self.__events_processed_elapsed_time}")
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
from logging import Logger
|
|
2
|
+
from typing import Optional, Sequence, Type, TypeVar
|
|
3
|
+
|
|
4
|
+
from aiokafka.coordinator.assignors.abstract import AbstractPartitionAssignor
|
|
5
|
+
|
|
6
|
+
from buz.event import Event
|
|
7
|
+
|
|
8
|
+
from buz.event.async_subscriber import AsyncSubscriber
|
|
9
|
+
from buz.event.infrastructure.buz_kafka.base_buz_aiokafka_async_consumer import BaseBuzAIOKafkaAsyncConsumer
|
|
10
|
+
from buz.event.infrastructure.buz_kafka.consume_strategy.consume_strategy import KafkaConsumeStrategy
|
|
11
|
+
from buz.event.infrastructure.buz_kafka.consume_strategy.kafka_on_fail_strategy import KafkaOnFailStrategy
|
|
12
|
+
from buz.event.infrastructure.buz_kafka.kafka_event_async_subscriber_executor import KafkaEventAsyncSubscriberExecutor
|
|
13
|
+
from buz.event.infrastructure.buz_kafka.kafka_event_subscriber_executor import KafkaEventSubscriberExecutor
|
|
14
|
+
from buz.event.meta_subscriber import MetaSubscriber
|
|
15
|
+
from buz.event.middleware.async_consume_middleware import AsyncConsumeMiddleware
|
|
16
|
+
from buz.event.strategies.retry.consume_retrier import ConsumeRetrier
|
|
17
|
+
from buz.event.strategies.retry.reject_callback import RejectCallback
|
|
18
|
+
from buz.kafka import (
|
|
19
|
+
KafkaConnectionConfig,
|
|
20
|
+
ConsumerInitialOffsetPosition,
|
|
21
|
+
)
|
|
22
|
+
from buz.kafka.domain.models.auto_create_topic_configuration import AutoCreateTopicConfiguration
|
|
23
|
+
from buz.kafka.domain.services.kafka_admin_client import KafkaAdminClient
|
|
24
|
+
from buz.kafka.infrastructure.deserializers.bytes_to_message_deserializer import BytesToMessageDeserializer
|
|
25
|
+
from buz.kafka.infrastructure.deserializers.implementations.json_bytes_to_message_deserializer import (
|
|
26
|
+
JSONBytesToMessageDeserializer,
|
|
27
|
+
)
|
|
28
|
+
from buz.kafka.infrastructure.serializers.kafka_header_serializer import KafkaHeaderSerializer
|
|
29
|
+
|
|
30
|
+
T = TypeVar("T", bound=Event)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class BuzAIOKafkaAsyncConsumer(BaseBuzAIOKafkaAsyncConsumer):
|
|
34
|
+
def __init__(
|
|
35
|
+
self,
|
|
36
|
+
*,
|
|
37
|
+
connection_config: KafkaConnectionConfig,
|
|
38
|
+
kafka_admin_client: Optional[KafkaAdminClient],
|
|
39
|
+
consume_strategy: KafkaConsumeStrategy,
|
|
40
|
+
on_fail_strategy: KafkaOnFailStrategy,
|
|
41
|
+
max_queue_size: int,
|
|
42
|
+
max_records_retrieved_per_poll: int,
|
|
43
|
+
kafka_partition_assignors: tuple[Type[AbstractPartitionAssignor], ...] = (),
|
|
44
|
+
subscribers: Sequence[AsyncSubscriber],
|
|
45
|
+
logger: Logger,
|
|
46
|
+
consumer_initial_offset_position: ConsumerInitialOffsetPosition,
|
|
47
|
+
deserializers_per_subscriber: dict[MetaSubscriber, BytesToMessageDeserializer[T]],
|
|
48
|
+
consume_middlewares: Optional[Sequence[AsyncConsumeMiddleware]] = None,
|
|
49
|
+
consume_retrier: Optional[ConsumeRetrier] = None,
|
|
50
|
+
reject_callback: Optional[RejectCallback] = None,
|
|
51
|
+
auto_create_topic_configuration: Optional[AutoCreateTopicConfiguration] = None,
|
|
52
|
+
seconds_between_executions_if_there_are_no_tasks_in_the_queue: int = 1,
|
|
53
|
+
seconds_between_polls_if_there_are_tasks_in_the_queue: int = 1,
|
|
54
|
+
seconds_between_polls_if_there_are_no_new_tasks: int = 1,
|
|
55
|
+
max_number_of_concurrent_polling_tasks: int = 20,
|
|
56
|
+
):
|
|
57
|
+
super().__init__(
|
|
58
|
+
connection_config=connection_config,
|
|
59
|
+
kafka_admin_client=kafka_admin_client,
|
|
60
|
+
consume_strategy=consume_strategy,
|
|
61
|
+
max_queue_size=max_queue_size,
|
|
62
|
+
max_records_retrieved_per_poll=max_records_retrieved_per_poll,
|
|
63
|
+
kafka_partition_assignors=kafka_partition_assignors,
|
|
64
|
+
subscribers=subscribers,
|
|
65
|
+
logger=logger,
|
|
66
|
+
consumer_initial_offset_position=consumer_initial_offset_position,
|
|
67
|
+
auto_create_topic_configuration=auto_create_topic_configuration,
|
|
68
|
+
seconds_between_executions_if_there_are_no_tasks_in_the_queue=seconds_between_executions_if_there_are_no_tasks_in_the_queue,
|
|
69
|
+
seconds_between_polls_if_there_are_tasks_in_the_queue=seconds_between_polls_if_there_are_tasks_in_the_queue,
|
|
70
|
+
seconds_between_polls_if_there_are_no_new_tasks=seconds_between_polls_if_there_are_no_new_tasks,
|
|
71
|
+
max_number_of_concurrent_polling_tasks=max_number_of_concurrent_polling_tasks,
|
|
72
|
+
)
|
|
73
|
+
self.__on_fail_strategy = on_fail_strategy
|
|
74
|
+
self.__consume_middlewares = consume_middlewares
|
|
75
|
+
self.__consume_retrier = consume_retrier
|
|
76
|
+
self.__reject_callback = reject_callback
|
|
77
|
+
self._deserializers_per_subscriber = deserializers_per_subscriber
|
|
78
|
+
|
|
79
|
+
async def _create_kafka_consumer_executor(
|
|
80
|
+
self,
|
|
81
|
+
subscriber: MetaSubscriber,
|
|
82
|
+
) -> KafkaEventSubscriberExecutor:
|
|
83
|
+
if not isinstance(subscriber, AsyncSubscriber):
|
|
84
|
+
raise TypeError(
|
|
85
|
+
f"Subscriber {subscriber.__class__.__name__} is not a subclass of Subscriber, probably you are trying to use a synchronous subscriber"
|
|
86
|
+
)
|
|
87
|
+
|
|
88
|
+
byte_deserializer = self._deserializers_per_subscriber.get(subscriber) or JSONBytesToMessageDeserializer(
|
|
89
|
+
# todo: it looks like in next python versions the inference engine is powerful enough to ensure this type, so we can remove it when we upgrade the python version of the library
|
|
90
|
+
event_class=subscriber.handles() # type: ignore
|
|
91
|
+
)
|
|
92
|
+
|
|
93
|
+
return KafkaEventAsyncSubscriberExecutor(
|
|
94
|
+
logger=self._logger,
|
|
95
|
+
byte_deserializer=byte_deserializer,
|
|
96
|
+
header_deserializer=KafkaHeaderSerializer(),
|
|
97
|
+
on_fail_strategy=self.__on_fail_strategy,
|
|
98
|
+
subscriber=subscriber,
|
|
99
|
+
consume_middlewares=self.__consume_middlewares,
|
|
100
|
+
consume_retrier=self.__consume_retrier,
|
|
101
|
+
reject_callback=self.__reject_callback,
|
|
102
|
+
)
|