buz 2.13.1rc7__tar.gz → 2.13.1rc9__tar.gz

{buz-2.13.1rc7 → buz-2.13.1rc9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: buz
-Version: 2.13.1rc7
+Version: 2.13.1rc9
 Summary: Buz is a set of light, simple and extensible implementations of event, command and query buses.
 License: MIT
 Author: Luis Pintado Lozano

{buz-2.13.1rc7 → buz-2.13.1rc9}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "buz"
-version = "2.13.1rc7"
+version = "2.13.1rc9"
 description = "Buz is a set of light, simple and extensible implementations of event, command and query buses."
 readme = "README.md"
 authors = ["Luis Pintado Lozano <luis.pintado.lozano@gmail.com>", "Gerardo Parra <gprauxiliar@gmail.com>"]
@@ -37,6 +37,7 @@ black = "^23.3"
 mypy = "^1.2"
 flake8 = "^5.0.4"
 tenacity = "^8.5.0"
+freezegun = "^1.5.1"
 types-cachetools = "^5.5.0.20240820"
 
 [tool.poetry.extras]

{buz-2.13.1rc7 → buz-2.13.1rc9}/src/buz/event/infrastructure/buz_kafka/base_buz_aiokafka_async_consumer.py

@@ -1,5 +1,5 @@
-from abc import abstractmethod
 import traceback
+from abc import abstractmethod
 from asyncio import Lock, Task, create_task, gather, Semaphore, Event as AsyncIOEvent, sleep
 from datetime import timedelta, datetime
 from itertools import cycle
@@ -28,7 +28,6 @@ from buz.kafka.infrastructure.aiokafka.aiokafka_consumer import AIOKafkaConsumer
 from buz.queue.in_memory.in_memory_multiqueue_repository import InMemoryMultiqueueRepository
 from buz.queue.multiqueue_repository import MultiqueueRepository
 
-
 T = TypeVar("T", bound=Event)
 
 
@@ -87,17 +86,14 @@ class BaseBuzAIOKafkaAsyncConsumer(AsyncConsumer):
     async def run(self) -> None:
         start_time = datetime.now()
         await self.__generate_kafka_consumers()
+        self.__initial_coroutines_created_elapsed_time = datetime.now() - start_time
 
         if len(self.__executor_per_consumer_mapper) == 0:
             self._logger.error("There are no valid subscribers to execute, finalizing consumer")
             return
 
-        self.__initial_coroutines_created_elapsed_time = datetime.now() - start_time
-
         start_consumption_time = datetime.now()
-
         worker_errors = await self.__run_worker()
-
         self.__events_processed_elapsed_time = datetime.now() - start_consumption_time
 
         await self.__handle_graceful_stop(worker_errors)
@@ -112,9 +108,9 @@ class BaseBuzAIOKafkaAsyncConsumer(AsyncConsumer):
         if self.__exceptions_are_thrown(worker_errors):
             consume_events_exception, polling_task_exception = worker_errors
             if consume_events_exception:
-                self._logger.error(consume_events_exception)
+                self._logger.exception(consume_events_exception)
             if polling_task_exception:
-                self._logger.error(polling_task_exception)
+                self._logger.exception(polling_task_exception)
 
             raise WorkerExecutionException("The worker was closed by an unexpected exception")
 
@@ -227,41 +223,32 @@ class BaseBuzAIOKafkaAsyncConsumer(AsyncConsumer):
             if len(kafka_poll_records) == 0:
                 await sleep(self.__seconds_between_polls_if_there_are_no_new_tasks)
 
-        return
-
     async def __consume_events_task(self) -> None:
         self._logger.info("Initializing consuming task")
-
-        blocked_tasks_iterator = self.generate_blocked_consuming_tasks_iterator()
+        blocked_tasks_iterator = self.__generate_blocked_consuming_tasks_iterator()
 
         async for consuming_task in blocked_tasks_iterator:
             consumer = consuming_task.consumer
             kafka_poll_record = consuming_task.kafka_poll_record
-            executor = self.__executor_per_consumer_mapper[consuming_task.consumer]
 
+            executor = self.__executor_per_consumer_mapper[consumer]
             await executor.consume(kafka_poll_record=kafka_poll_record)
-
             await consumer.commit_poll_record(kafka_poll_record)
 
             self.__events_processed += 1
 
     # This iterator return a blocked task, that will be blocked for other process (like rebalancing), until the next task will be requested
-    async def generate_blocked_consuming_tasks_iterator(self) -> AsyncIterator[ConsumingTask]:
+    async def __generate_blocked_consuming_tasks_iterator(self) -> AsyncIterator[ConsumingTask]:
         consumer_queues_cyclic_iterator = cycle(self.__queue_per_consumer_mapper.items())
         last_consumer, _ = next(consumer_queues_cyclic_iterator)
 
         while not self.__should_stop.is_set():
-            all_queues_are_empty = all(
-                [queue.is_totally_empty() for queue in self.__queue_per_consumer_mapper.values()]
-            )
-
-            if all_queues_are_empty:
+            if await self.__all_queues_are_empty():
                 await sleep(self.__seconds_between_executions_if_there_are_no_tasks_in_the_queue)
                 continue
 
             async with self.__task_execution_mutex:
                 consumer: Optional[AIOKafkaConsumer] = None
-                kafka_poll_record: Optional[KafkaPollRecord] = None
 
                 while consumer != last_consumer:
                     consumer, queue = next(consumer_queues_cyclic_iterator)
@@ -272,7 +259,8 @@ class BaseBuzAIOKafkaAsyncConsumer(AsyncConsumer):
                         last_consumer = consumer
                         break
 
-        return
+    async def __all_queues_are_empty(self) -> bool:
+        return all([queue.is_totally_empty() for queue in self.__queue_per_consumer_mapper.values()])
 
     async def __on_partition_revoked(self, consumer: AIOKafkaConsumer, topics_partitions: set[TopicPartition]) -> None:
         async with self.__task_execution_mutex:

{buz-2.13.1rc7 → buz-2.13.1rc9}/src/buz/event/infrastructure/buz_kafka/buz_kafka_event_bus.py

@@ -14,6 +14,7 @@ from buz.event.middleware.publish_middleware_chain_resolver import PublishMiddle
 from buz.kafka import (
     KafkaPythonProducer,
 )
+from buz.kafka.domain.exceptions.topic_already_created_exception import KafkaTopicsAlreadyCreatedException
 from buz.kafka.domain.models.auto_create_topic_configuration import AutoCreateTopicConfiguration
 from buz.kafka.domain.models.create_kafka_topic import CreateKafkaTopic
 from buz.kafka.domain.services.kafka_admin_client import KafkaAdminClient
@@ -53,18 +54,22 @@ class BuzKafkaEventBus(EventBus):
             topic = self.__publish_strategy.get_topic(event)
 
             if self.__auto_create_topic_configuration is not None and self.__is_topic_created(topic) is False:
-                self.__get_kafka_admin_client().create_topics(
-                    topics=[
-                        CreateKafkaTopic(
-                            name=topic,
-                            partitions=self.__auto_create_topic_configuration.partitions,
-                            replication_factor=self.__auto_create_topic_configuration.replication_factor,
-                            configs=self.__auto_create_topic_configuration.configs,
-                        )
-                    ]
-                )
-                self.__logger.info(f"Created missing topic: {topic}")
-                self.__topics_checked[topic] = True
+                try:
+                    self.__logger.info(f"Creating missing topic: {topic}..")
+                    self.__get_kafka_admin_client().create_topics(
+                        topics=[
+                            CreateKafkaTopic(
+                                name=topic,
+                                partitions=self.__auto_create_topic_configuration.partitions,
+                                replication_factor=self.__auto_create_topic_configuration.replication_factor,
+                                configs=self.__auto_create_topic_configuration.configs,
+                            )
+                        ]
+                    )
+                    self.__logger.info(f"Created missing topic: {topic}")
+                    self.__topics_checked[topic] = True
+                except KafkaTopicsAlreadyCreatedException:
+                    pass
 
             headers = self.__get_event_headers(event)
             self.__producer.produce(

{buz-2.13.1rc7 → buz-2.13.1rc9}/src/buz/event/infrastructure/buz_kafka/kafka_event_sync_subscriber_executor.py

@@ -67,11 +67,6 @@ class KafkaEventSyncSubscriberExecutor(KafkaEventSubscriberExecutor):
             self.__logger.error(
                 f'The message "{str(kafka_poll_record.value)}" is not valid, it will be consumed but not processed'
             )
-        except Exception as exception:
-            if self.__on_fail_strategy == KafkaOnFailStrategy.CONSUME_ON_FAIL:
-                self.__logger.error(f"Error consuming event: {exception}")
-                return
-            raise exception
 
     def __execution_callback(self, subscriber: Subscriber, message: KafkaConsumerRecord[Event]) -> None:
         self.__consume_middleware_chain_resolver.resolve(

buz-2.13.1rc9/src/buz/kafka/domain/exceptions/not_all_partition_assigned_exception.py

@@ -0,0 +1,8 @@
+from __future__ import annotations
+
+
+class NotAllPartitionAssignedException(Exception):
+    def __init__(self, topic_name: str) -> None:
+        super().__init__(
+            f'Not all the partition were assigned for the topic "{topic_name}", please disconnect the rest of subscribers'
+        )

buz-2.13.1rc9/src/buz/kafka/domain/exceptions/topic_not_found_exception.py

@@ -0,0 +1,6 @@
+from __future__ import annotations
+
+
+class TopicNotFoundException(Exception):
+    def __init__(self, topic_name: str) -> None:
+        super().__init__(f'The topic "{topic_name}", has not been found')

{buz-2.13.1rc7 → buz-2.13.1rc9}/src/buz/kafka/domain/services/kafka_admin_client.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 from abc import abstractmethod, ABC
+from datetime import datetime
 from typing import Sequence
 
 from buz.kafka.domain.models.create_kafka_topic import CreateKafkaTopic
@@ -37,3 +38,16 @@ class KafkaAdminClient(ABC):
         self,
     ) -> set[str]:
         pass
+
+    # This function moves the following offset from the provided date
+    # if there are no messages with a date greater than the provided offset
+    # the offset will be moved to the end
+    @abstractmethod
+    def move_offsets_to_datetime(
+        self,
+        *,
+        consumer_group: str,
+        topic: str,
+        target_datetime: datetime,
+    ) -> None:
+        pass

{buz-2.13.1rc7 → buz-2.13.1rc9}/src/buz/kafka/infrastructure/aiokafka/aiokafka_consumer.py

@@ -123,7 +123,7 @@ class AIOKafkaConsumer:
         try:
             self.__logger.info(f"Creating missing topics: {non_created_topics}...")
             kafka_admin_client.create_topics(topics=topics_to_create)
-            self.__logger.info(f"Created missing topics: {non_created_topics}...")
+            self.__logger.info(f"Created missing topics: {non_created_topics}")
         except KafkaTopicsAlreadyCreatedException:
             # there is a possibility to have a race condition between the check and the creation
             # but it does not matters, the important part is that the topic is created

{buz-2.13.1rc7 → buz-2.13.1rc9}/src/buz/kafka/infrastructure/cdc/cdc_message.py

@@ -3,9 +3,11 @@ from dataclasses import dataclass
 
 @dataclass(frozen=True)
 class CDCPayload:
+    DATE_TIME_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ"
+
     payload: str  # json encoded
     event_id: str  # uuid
-    created_at: str  # date and hour ISO 8601
+    created_at: str
     event_fqn: str
 
     def validate(self) -> None:

{buz-2.13.1rc7 → buz-2.13.1rc9}/src/buz/kafka/infrastructure/deserializers/implementations/cdc/cdc_record_bytes_to_event_deserializer.py

@@ -1,16 +1,17 @@
 from __future__ import annotations
 
+from datetime import datetime
 from typing import TypeVar, Type, Generic
 
 import orjson
 from dacite import from_dict
 
-from buz.kafka.infrastructure.deserializers.implementations.cdc.not_valid_cdc_message_exception import (
-    NotValidCDCMessageException,
-)
 from buz.event import Event
 from buz.kafka.infrastructure.cdc.cdc_message import CDCMessage, CDCPayload
 from buz.kafka.infrastructure.deserializers.bytes_to_message_deserializer import BytesToMessageDeserializer
+from buz.kafka.infrastructure.deserializers.implementations.cdc.not_valid_cdc_message_exception import (
+    NotValidCDCMessageException,
+)
 
 T = TypeVar("T", bound=Event)
 
@@ -27,12 +28,16 @@ class CDCRecordBytesToEventDeserializer(BytesToMessageDeserializer[Event], Gener
             cdc_message = self.__get_outbox_record_as_dict(decoded_string)
             return self.__event_class.restore(
                 id=cdc_message.payload.event_id,
-                created_at=cdc_message.payload.created_at,
+                created_at=self.__get_created_at_in_event_format(cdc_message.payload.created_at),
                 **orjson.loads(cdc_message.payload.payload),
             )
         except Exception as exception:
             raise NotValidCDCMessageException(decoded_string, exception) from exception
 
+    def __get_created_at_in_event_format(self, cdc_payload_created_at: str) -> str:
+        created_at_datetime = datetime.strptime(cdc_payload_created_at, CDCPayload.DATE_TIME_FORMAT)
+        return created_at_datetime.strftime(Event.DATE_TIME_FORMAT)
+
     def __get_outbox_record_as_dict(self, decoded_string: str) -> CDCMessage:
         decoded_record: dict = orjson.loads(decoded_string)
 

{buz-2.13.1rc7 → buz-2.13.1rc9}/src/buz/kafka/infrastructure/kafka_python/kafka_python_admin_client.py

@@ -1,19 +1,30 @@
 from __future__ import annotations
 
+from datetime import datetime
+from logging import Logger
 import re
-from typing import Any, Callable, Sequence
+from typing import Any, Callable, Optional, Sequence, cast
 
 from cachetools import TTLCache
-from kafka import KafkaClient
+from kafka import KafkaClient, KafkaConsumer
 from kafka.admin import KafkaAdminClient as KafkaPythonLibraryAdminClient, NewTopic
 from kafka.errors import TopicAlreadyExistsError
+from kafka.structs import TopicPartition, OffsetAndTimestamp
 
+from buz.kafka.domain.exceptions.not_all_partition_assigned_exception import NotAllPartitionAssignedException
 from buz.kafka.domain.exceptions.topic_already_created_exception import KafkaTopicsAlreadyCreatedException
+from buz.kafka.domain.exceptions.topic_not_found_exception import TopicNotFoundException
+from buz.kafka.domain.models.consumer_initial_offset_position import ConsumerInitialOffsetPosition
 from buz.kafka.domain.models.create_kafka_topic import CreateKafkaTopic
 from buz.kafka.domain.models.kafka_connection_config import KafkaConnectionConfig
 from buz.kafka.domain.services.kafka_admin_client import KafkaAdminClient
 
+from buz.kafka.infrastructure.kafka_python.translators.consumer_initial_offset_position_translator import (
+    KafkaPythonConsumerInitialOffsetPositionTranslator,
+)
+
 INTERNAL_KAFKA_TOPICS = {"__consumer_offsets", "_schema"}
+TOPIC_CACHE_KEY = "topics"
 
 
 class KafkaPythonAdminClient(KafkaAdminClient):
@@ -22,9 +33,11 @@ class KafkaPythonAdminClient(KafkaAdminClient):
     def __init__(
         self,
         *,
+        logger: Logger,
         config: KafkaConnectionConfig,
         cache_ttl_seconds: int = 0,
     ):
+        self._logger = logger
         self._config = config
         self._config_in_library_format = self.__get_kafka_config_in_library_format(config)
         self._kafka_admin = KafkaPythonLibraryAdminClient(**self._config_in_library_format)
@@ -75,14 +88,13 @@ class KafkaPythonAdminClient(KafkaAdminClient):
         self,
         topic: str,
     ) -> bool:
-        topics = self.get_topics()
-        return topic in topics
+        return topic in self.get_topics()
 
     def get_topics(
         self,
     ) -> set[str]:
         return self.__resolve_cached_property(
-            "topics", lambda: set(self._kafka_admin.list_topics()) - INTERNAL_KAFKA_TOPICS
+            TOPIC_CACHE_KEY, lambda: set(self._kafka_admin.list_topics()) - INTERNAL_KAFKA_TOPICS
         )
 
     def __resolve_cached_property(self, property_key: str, callback: Callable) -> Any:
@@ -101,6 +113,10 @@ class KafkaPythonAdminClient(KafkaAdminClient):
         self._kafka_admin.delete_topics(
             topics=topics,
         )
+        self.__remove_cache_property(TOPIC_CACHE_KEY)
+
+    def __remove_cache_property(self, property_key: str) -> None:
+        self.__ttl_cache.pop(property_key, None)
 
     def delete_subscription_groups(
         self,
@@ -119,3 +135,80 @@ class KafkaPythonAdminClient(KafkaAdminClient):
     def _wait_for_cluster_update(self) -> None:
         future = self._kafka_client.cluster.request_update()
         self._kafka_client.poll(future=future)
+
+    def move_offsets_to_datetime(
+        self,
+        *,
+        consumer_group: str,
+        topic: str,
+        target_datetime: datetime,
+    ) -> None:
+        consumer = KafkaConsumer(
+            group_id=consumer_group,
+            enable_auto_commit=False,
+            auto_offset_reset=KafkaPythonConsumerInitialOffsetPositionTranslator.to_kafka_supported_format(
+                ConsumerInitialOffsetPosition.BEGINNING
+            ),
+            **self._config_in_library_format,
+        )
+
+        partitions = consumer.partitions_for_topic(topic)
+
+        if partitions is None:
+            raise TopicNotFoundException(topic)
+
+        topic_partitions = [TopicPartition(topic, p) for p in partitions]
+        consumer.subscribe(topics=[topic])
+
+        self.__force_partition_assignment(consumer)
+
+        # We need all the partitions in order to update the offsets
+        if len(consumer.assignment()) != len(topic_partitions):
+            raise NotAllPartitionAssignedException(topic)
+
+        offsets_for_date = self.__get_first_offset_after_date(
+            consumer=consumer,
+            topic_partitions=topic_partitions,
+            target_datetime=target_datetime,
+        )
+
+        end_offsets = consumer.end_offsets(topic_partitions)
+
+        if end_offsets is None or len(end_offsets.keys()) != len(topic_partitions):
+            raise Exception(f'There was an error extracting the end offsets of the topic "{topic}"')
+
+        for topic_partition in topic_partitions:
+            offset_and_timestamp = offsets_for_date.get(topic_partition)
+            if offset_and_timestamp:
+                self._logger.info(f'moving "{topic_partition}" to the offset "{offset_and_timestamp.offset}"')
+                consumer.seek(topic_partition, offset_and_timestamp.offset)
+            else:
+                self._logger.info(
+                    f'moving "{topic_partition}" to the end of the topic because there are no messages later than "{target_datetime}"'
+                )
+                consumer.seek(topic_partition, end_offsets[topic_partition])
+
+        consumer.commit()
+        consumer.close()
+
+    def __get_first_offset_after_date(
+        self,
+        *,
+        consumer: KafkaConsumer,
+        topic_partitions: Sequence[TopicPartition],
+        target_datetime: datetime,
+    ) -> dict[TopicPartition, Optional[OffsetAndTimestamp]]:
+        offset_for_times: dict[TopicPartition, Optional[int]] = {}
+        timestamp_ms = int(target_datetime.timestamp() * 1000)
+
+        for topic_partition in topic_partitions:
+            offset_for_times[topic_partition] = timestamp_ms
+
+        return cast(
+            dict[TopicPartition, Optional[OffsetAndTimestamp]],
+            consumer.offsets_for_times(offset_for_times),
+        )
+
+    # We are not to commit the new offset, but we need to execute a polling in order to start the partition assignment
+    def __force_partition_assignment(self, consumer: KafkaConsumer) -> None:
+        consumer.poll(max_records=1, timeout_ms=0)

{buz-2.13.1rc7 → buz-2.13.1rc9}/src/buz/kafka/infrastructure/kafka_python/kafka_python_admin_test_client.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+from logging import Logger
 from typing import Optional
 
 from kafka import KafkaConsumer, KafkaProducer
@@ -22,8 +23,16 @@ CONSUMER_POLL_TIMEOUT_MS = 1000
 
 
 class KafkaPythonAdminTestClient(KafkaPythonAdminClient, KafkaAdminTestClient):
-    def __init__(self, *, config: KafkaConnectionConfig):
-        super().__init__(config=config)
+    def __init__(
+        self,
+        *,
+        logger: Logger,
+        config: KafkaConnectionConfig,
+    ):
+        super().__init__(
+            config=config,
+            logger=logger,
+        )
 
     def send_message_to_topic(
         self,

{buz-2.13.1rc7 → buz-2.13.1rc9}/src/buz/kafka/infrastructure/serializers/implementations/cdc_record_bytes_to_event_serializer.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 from dataclasses import asdict
+from datetime import datetime
 
 from buz.event import Event
 from buz.kafka.infrastructure.cdc.cdc_message import CDCMessage, CDCPayload
@@ -16,13 +17,17 @@ class CDCRecordBytesToEventSerializer(ByteSerializer):
         cdc_message: CDCMessage = CDCMessage(
             payload=CDCPayload(
                 event_id=data.id,
-                created_at=data.created_at,
+                created_at=self.__adapt_created_to_cdc_format(data.created_at),
                 event_fqn=data.fqn(),
                 payload=self.__serialize_payload(data),
             )
         )
         return self.__json_serializer.serialize(asdict(cdc_message))
 
+    def __adapt_created_to_cdc_format(self, created_at: str) -> str:
+        created_at_datetime = datetime.strptime(created_at, Event.DATE_TIME_FORMAT)
+        return created_at_datetime.strftime(CDCPayload.DATE_TIME_FORMAT)
+
     def __serialize_payload(self, event: Event) -> str:
         # Remove id and created at, because Transactional outbox is not adding them
         payload = asdict(event)

{buz-2.13.1rc7 → buz-2.13.1rc9}/src/buz/queue/in_memory/in_memory_multiqueue_repository.py

@@ -1,37 +1,20 @@
-from threading import Lock
 from queue import Queue, Empty
 from typing import Optional, TypeVar, cast
 
 from buz.queue.multiqueue_repository import MultiqueueRepository
 
-
 K = TypeVar("K")
 R = TypeVar("R")
 
 
-def self_mutex(method):
-    def call(self, *args, **kwargs):
-        lock: Lock = self._get_method_lock()  # type: ignore
-        with lock:
-            return method(self, *args, **kwargs)
-
-    return call
-
-
 class InMemoryMultiqueueRepository(MultiqueueRepository[K, R]):
     def __init__(self):
         self.__queues = cast(dict[K, Queue[R]], {})
-        self.__mutex = Lock()
         self.__last_key_index = 0
 
-    def _get_method_lock(self) -> Lock:
-        return self.__mutex
-
-    @self_mutex
     def clear(self, key: K) -> None:
         self.__queues.pop(key, None)
 
-    @self_mutex
     def push(self, key: K, record: R) -> None:
         if key not in self.__queues:
             self.__add_key(key)
@@ -41,7 +24,6 @@ class InMemoryMultiqueueRepository(MultiqueueRepository[K, R]):
     def __add_key(self, key: K) -> None:
         self.__queues[key] = Queue[R]()
 
-    @self_mutex
     def pop(self) -> Optional[R]:
         if not self.__queues:
             return None
@@ -65,10 +47,8 @@ class InMemoryMultiqueueRepository(MultiqueueRepository[K, R]):
 
         return None
 
-    @self_mutex
     def get_total_size(self) -> int:
         return sum([queue.qsize() for queue in self.__queues.values()])
 
-    @self_mutex
     def is_totally_empty(self) -> bool:
         return all([queue.empty() for queue in self.__queues.values()])