buz 2.13.1rc8__tar.gz → 2.13.1rc9__tar.gz

{buz-2.13.1rc8 → buz-2.13.1rc9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: buz
-Version: 2.13.1rc8
+Version: 2.13.1rc9
 Summary: Buz is a set of light, simple and extensible implementations of event, command and query buses.
 License: MIT
 Author: Luis Pintado Lozano

{buz-2.13.1rc8 → buz-2.13.1rc9}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "buz"
-version = "2.13.1rc8"
+version = "2.13.1rc9"
 description = "Buz is a set of light, simple and extensible implementations of event, command and query buses."
 readme = "README.md"
 authors = ["Luis Pintado Lozano <luis.pintado.lozano@gmail.com>", "Gerardo Parra <gprauxiliar@gmail.com>"]

{buz-2.13.1rc8 → buz-2.13.1rc9}/src/buz/event/infrastructure/buz_kafka/kafka_event_sync_subscriber_executor.py

@@ -67,11 +67,6 @@ class KafkaEventSyncSubscriberExecutor(KafkaEventSubscriberExecutor):
             self.__logger.error(
                 f'The message "{str(kafka_poll_record.value)}" is not valid, it will be consumed but not processed'
             )
-        except Exception as exception:
-            if self.__on_fail_strategy == KafkaOnFailStrategy.CONSUME_ON_FAIL:
-                self.__logger.error(f"Error consuming event: {exception}")
-                return
-            raise exception
 
     def __execution_callback(self, subscriber: Subscriber, message: KafkaConsumerRecord[Event]) -> None:
         self.__consume_middleware_chain_resolver.resolve(

buz-2.13.1rc9/src/buz/kafka/domain/exceptions/not_all_partition_assigned_exception.py

@@ -0,0 +1,8 @@
+from __future__ import annotations
+
+
+class NotAllPartitionAssignedException(Exception):
+    def __init__(self, topic_name: str) -> None:
+        super().__init__(
+            f'Not all the partition were assigned for the topic "{topic_name}", please disconnect the rest of subscribers'
+        )

buz-2.13.1rc9/src/buz/kafka/domain/exceptions/topic_not_found_exception.py

@@ -0,0 +1,6 @@
+from __future__ import annotations
+
+
+class TopicNotFoundException(Exception):
+    def __init__(self, topic_name: str) -> None:
+        super().__init__(f'The topic "{topic_name}", has not been found')

{buz-2.13.1rc8 → buz-2.13.1rc9}/src/buz/kafka/domain/services/kafka_admin_client.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 from abc import abstractmethod, ABC
+from datetime import datetime
 from typing import Sequence
 
 from buz.kafka.domain.models.create_kafka_topic import CreateKafkaTopic
@@ -37,3 +38,16 @@ class KafkaAdminClient(ABC):
         self,
     ) -> set[str]:
         pass
+
+    # This function moves the following offset from the provided date
+    # if there are no messages with a date greater than the provided offset
+    # the offset will be moved to the end
+    @abstractmethod
+    def move_offsets_to_datetime(
+        self,
+        *,
+        consumer_group: str,
+        topic: str,
+        target_datetime: datetime,
+    ) -> None:
+        pass

{buz-2.13.1rc8 → buz-2.13.1rc9}/src/buz/kafka/infrastructure/kafka_python/kafka_python_admin_client.py

@@ -1,19 +1,30 @@
 from __future__ import annotations
 
+from datetime import datetime
+from logging import Logger
 import re
-from typing import Any, Callable, Sequence
+from typing import Any, Callable, Optional, Sequence, cast
 
 from cachetools import TTLCache
-from kafka import KafkaClient
+from kafka import KafkaClient, KafkaConsumer
 from kafka.admin import KafkaAdminClient as KafkaPythonLibraryAdminClient, NewTopic
 from kafka.errors import TopicAlreadyExistsError
+from kafka.structs import TopicPartition, OffsetAndTimestamp
 
+from buz.kafka.domain.exceptions.not_all_partition_assigned_exception import NotAllPartitionAssignedException
 from buz.kafka.domain.exceptions.topic_already_created_exception import KafkaTopicsAlreadyCreatedException
+from buz.kafka.domain.exceptions.topic_not_found_exception import TopicNotFoundException
+from buz.kafka.domain.models.consumer_initial_offset_position import ConsumerInitialOffsetPosition
 from buz.kafka.domain.models.create_kafka_topic import CreateKafkaTopic
 from buz.kafka.domain.models.kafka_connection_config import KafkaConnectionConfig
 from buz.kafka.domain.services.kafka_admin_client import KafkaAdminClient
 
+from buz.kafka.infrastructure.kafka_python.translators.consumer_initial_offset_position_translator import (
+    KafkaPythonConsumerInitialOffsetPositionTranslator,
+)
+
 INTERNAL_KAFKA_TOPICS = {"__consumer_offsets", "_schema"}
+TOPIC_CACHE_KEY = "topics"
 
 
 class KafkaPythonAdminClient(KafkaAdminClient):
@@ -22,9 +33,11 @@ class KafkaPythonAdminClient(KafkaAdminClient):
     def __init__(
         self,
         *,
+        logger: Logger,
         config: KafkaConnectionConfig,
         cache_ttl_seconds: int = 0,
     ):
+        self._logger = logger
         self._config = config
         self._config_in_library_format = self.__get_kafka_config_in_library_format(config)
         self._kafka_admin = KafkaPythonLibraryAdminClient(**self._config_in_library_format)
@@ -75,14 +88,13 @@ class KafkaPythonAdminClient(KafkaAdminClient):
         self,
         topic: str,
     ) -> bool:
-        topics = self.get_topics()
-        return topic in topics
+        return topic in self.get_topics()
 
     def get_topics(
         self,
     ) -> set[str]:
         return self.__resolve_cached_property(
-            "topics", lambda: set(self._kafka_admin.list_topics()) - INTERNAL_KAFKA_TOPICS
+            TOPIC_CACHE_KEY, lambda: set(self._kafka_admin.list_topics()) - INTERNAL_KAFKA_TOPICS
         )
 
     def __resolve_cached_property(self, property_key: str, callback: Callable) -> Any:
@@ -101,6 +113,10 @@ class KafkaPythonAdminClient(KafkaAdminClient):
         self._kafka_admin.delete_topics(
             topics=topics,
         )
+        self.__remove_cache_property(TOPIC_CACHE_KEY)
+
+    def __remove_cache_property(self, property_key: str) -> None:
+        self.__ttl_cache.pop(property_key, None)
 
     def delete_subscription_groups(
         self,
@@ -119,3 +135,80 @@ class KafkaPythonAdminClient(KafkaAdminClient):
     def _wait_for_cluster_update(self) -> None:
         future = self._kafka_client.cluster.request_update()
         self._kafka_client.poll(future=future)
+
+    def move_offsets_to_datetime(
+        self,
+        *,
+        consumer_group: str,
+        topic: str,
+        target_datetime: datetime,
+    ) -> None:
+        consumer = KafkaConsumer(
+            group_id=consumer_group,
+            enable_auto_commit=False,
+            auto_offset_reset=KafkaPythonConsumerInitialOffsetPositionTranslator.to_kafka_supported_format(
+                ConsumerInitialOffsetPosition.BEGINNING
+            ),
+            **self._config_in_library_format,
+        )
+
+        partitions = consumer.partitions_for_topic(topic)
+
+        if partitions is None:
+            raise TopicNotFoundException(topic)
+
+        topic_partitions = [TopicPartition(topic, p) for p in partitions]
+        consumer.subscribe(topics=[topic])
+
+        self.__force_partition_assignment(consumer)
+
+        # We need all the partitions in order to update the offsets
+        if len(consumer.assignment()) != len(topic_partitions):
+            raise NotAllPartitionAssignedException(topic)
+
+        offsets_for_date = self.__get_first_offset_after_date(
+            consumer=consumer,
+            topic_partitions=topic_partitions,
+            target_datetime=target_datetime,
+        )
+
+        end_offsets = consumer.end_offsets(topic_partitions)
+
+        if end_offsets is None or len(end_offsets.keys()) != len(topic_partitions):
+            raise Exception(f'There was an error extracting the end offsets of the topic "{topic}"')
+
+        for topic_partition in topic_partitions:
+            offset_and_timestamp = offsets_for_date.get(topic_partition)
+            if offset_and_timestamp:
+                self._logger.info(f'moving "{topic_partition}" to the offset "{offset_and_timestamp.offset}"')
+                consumer.seek(topic_partition, offset_and_timestamp.offset)
+            else:
+                self._logger.info(
+                    f'moving "{topic_partition}" to the end of the topic because there are no messages later than "{target_datetime}"'
+                )
+                consumer.seek(topic_partition, end_offsets[topic_partition])
+
+        consumer.commit()
+        consumer.close()
+
+    def __get_first_offset_after_date(
+        self,
+        *,
+        consumer: KafkaConsumer,
+        topic_partitions: Sequence[TopicPartition],
+        target_datetime: datetime,
+    ) -> dict[TopicPartition, Optional[OffsetAndTimestamp]]:
+        offset_for_times: dict[TopicPartition, Optional[int]] = {}
+        timestamp_ms = int(target_datetime.timestamp() * 1000)
+
+        for topic_partition in topic_partitions:
+            offset_for_times[topic_partition] = timestamp_ms
+
+        return cast(
+            dict[TopicPartition, Optional[OffsetAndTimestamp]],
+            consumer.offsets_for_times(offset_for_times),
+        )
+
+    # We are not to commit the new offset, but we need to execute a polling in order to start the partition assignment
+    def __force_partition_assignment(self, consumer: KafkaConsumer) -> None:
+        consumer.poll(max_records=1, timeout_ms=0)

{buz-2.13.1rc8 → buz-2.13.1rc9}/src/buz/kafka/infrastructure/kafka_python/kafka_python_admin_test_client.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+from logging import Logger
 from typing import Optional
 
 from kafka import KafkaConsumer, KafkaProducer
@@ -22,8 +23,16 @@ CONSUMER_POLL_TIMEOUT_MS = 1000
 
 
 class KafkaPythonAdminTestClient(KafkaPythonAdminClient, KafkaAdminTestClient):
-    def __init__(self, *, config: KafkaConnectionConfig):
-        super().__init__(config=config)
+    def __init__(
+        self,
+        *,
+        logger: Logger,
+        config: KafkaConnectionConfig,
+    ):
+        super().__init__(
+            config=config,
+            logger=logger,
+        )
 
     def send_message_to_topic(
         self,