buz 2.17.0rc7__tar.gz → 2.18.0__tar.gz

{buz-2.17.0rc7 → buz-2.18.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: buz
-Version: 2.17.0rc7
+Version: 2.18.0
 Summary: Buz is a set of light, simple and extensible implementations of event, command and query buses.
 License: MIT
 Author: Luis Pintado Lozano

{buz-2.17.0rc7 → buz-2.18.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "buz"
-version = "2.17.0-rc7"
+version = "2.18.0"
 description = "Buz is a set of light, simple and extensible implementations of event, command and query buses."
 readme = "README.md"
 authors = [

{buz-2.17.0rc7 → buz-2.18.0}/src/buz/event/infrastructure/buz_kafka/buz_aiokafka_async_consumer.py

@@ -2,6 +2,12 @@ from logging import Logger
 from typing import Optional, Sequence, Type, TypeVar
 
 from aiokafka.coordinator.assignors.abstract import AbstractPartitionAssignor
+from buz.kafka.infrastructure.deserializers.implementations.cdc.cdc_record_bytes_to_cdc_payload_deserializer import (
+    CDCRecordBytesToCDCPayloadDeserializer,
+)
+from buz.kafka.infrastructure.deserializers.implementations.cdc.cdc_record_bytes_to_event_deserializer import (
+    CDCRecordBytesToEventDeserializer,
+)
 
 from buz.event import Event
 from buz.event.async_subscriber import AsyncSubscriber
@@ -102,6 +108,12 @@ class BuzAIOKafkaAsyncConsumer(BaseBuzAIOKafkaAsyncConsumer):
             event_class=subscriber.handles()  # type: ignore
         )
 
+        cdc_payload_deserializer = (
+            CDCRecordBytesToCDCPayloadDeserializer()
+            if isinstance(byte_deserializer, CDCRecordBytesToEventDeserializer)
+            else None
+        )
+
         return KafkaEventAsyncSubscriberExecutor(
             logger=self._logger,
             byte_deserializer=byte_deserializer,
@@ -111,4 +123,5 @@ class BuzAIOKafkaAsyncConsumer(BaseBuzAIOKafkaAsyncConsumer):
             consume_middlewares=self.__consume_middlewares,
             consume_retrier=self.__consume_retrier,
             reject_callback=self.__reject_callback,
+            cdc_payload_deserializer=cdc_payload_deserializer,
         )

{buz-2.17.0rc7 → buz-2.18.0}/src/buz/event/infrastructure/buz_kafka/buz_aiokafka_multi_threaded_consumer.py

@@ -2,6 +2,12 @@ from logging import Logger
 from typing import Optional, Sequence, Type, TypeVar
 
 from aiokafka.coordinator.assignors.abstract import AbstractPartitionAssignor
+from buz.kafka.infrastructure.deserializers.implementations.cdc.cdc_record_bytes_to_cdc_payload_deserializer import (
+    CDCRecordBytesToCDCPayloadDeserializer,
+)
+from buz.kafka.infrastructure.deserializers.implementations.cdc.cdc_record_bytes_to_event_deserializer import (
+    CDCRecordBytesToEventDeserializer,
+)
 
 from buz.event import Event, Subscriber
 from buz.event.infrastructure.buz_kafka.base_buz_aiokafka_async_consumer import (
@@ -103,6 +109,12 @@ class BuzAIOKafkaMultiThreadedConsumer(BaseBuzAIOKafkaAsyncConsumer):
             event_class=subscriber.handles()  # type: ignore
         )
 
+        cdc_payload_deserializer = (
+            CDCRecordBytesToCDCPayloadDeserializer()
+            if isinstance(byte_deserializer, CDCRecordBytesToEventDeserializer)
+            else None
+        )
+
         return KafkaEventSyncSubscriberExecutor(
             logger=self._logger,
             byte_deserializer=byte_deserializer,
@@ -112,4 +124,5 @@ class BuzAIOKafkaMultiThreadedConsumer(BaseBuzAIOKafkaAsyncConsumer):
             consume_middlewares=self.__consume_middlewares,
             consume_retrier=self.__consume_retrier,
             reject_callback=self.__reject_callback,
+            cdc_payload_deserializer=cdc_payload_deserializer,
         )

{buz-2.17.0rc7 → buz-2.18.0}/src/buz/event/infrastructure/buz_kafka/kafka_event_async_subscriber_executor.py

@@ -2,12 +2,16 @@ from asyncio import sleep
 from logging import Logger
 from typing import Optional, Sequence, cast
 
+from buz.event.infrastructure.models.process_context import ProcessContext
+
 from buz.event import Event
 from buz.event.async_subscriber import AsyncSubscriber
 from buz.event.infrastructure.buz_kafka.consume_strategy.kafka_on_fail_strategy import KafkaOnFailStrategy
 from buz.event.infrastructure.buz_kafka.exceptions.max_consumer_retry_exception import MaxConsumerRetryException
 from buz.event.infrastructure.buz_kafka.exceptions.retry_exception import ConsumerRetryException
 from buz.event.infrastructure.buz_kafka.kafka_event_subscriber_executor import KafkaEventSubscriberExecutor
+from buz.event.infrastructure.buz_kafka.models.kafka_delivery_context import KafkaDeliveryContext
+from buz.event.infrastructure.buz_kafka.models.cdc_process_context import CDCProcessContext
 from buz.event.infrastructure.models.execution_context import ExecutionContext
 from buz.event.middleware.async_consume_middleware import AsyncConsumeMiddleware
 from buz.event.middleware.async_consume_middleware_chain_resolver import AsyncConsumeMiddlewareChainResolver
@@ -16,8 +20,13 @@ from buz.event.strategies.retry.reject_callback import RejectCallback
 from buz.kafka.domain.exceptions.not_valid_kafka_message_exception import NotValidKafkaMessageException
 from buz.kafka.domain.models.kafka_consumer_record import KafkaConsumerRecord
 from buz.kafka.domain.models.kafka_poll_record import KafkaPollRecord
-from buz.event.infrastructure.buz_kafka.models.kafka_delivery_context import KafkaDeliveryContext
 from buz.kafka.infrastructure.deserializers.byte_deserializer import ByteDeserializer
+from buz.kafka.infrastructure.deserializers.implementations.cdc.cannot_decode_cdc_message_exception import (
+    CannotDecodeCDCMessageException,
+)
+from buz.kafka.infrastructure.deserializers.implementations.cdc.cdc_record_bytes_to_cdc_payload_deserializer import (
+    CDCRecordBytesToCDCPayloadDeserializer,
+)
 from buz.kafka.infrastructure.serializers.kafka_header_serializer import KafkaHeaderSerializer
 
 
@@ -34,6 +43,7 @@ class KafkaEventAsyncSubscriberExecutor(KafkaEventSubscriberExecutor):
         on_fail_strategy: KafkaOnFailStrategy,
         consume_retrier: Optional[ConsumeRetrier] = None,
         reject_callback: Optional[RejectCallback] = None,
+        cdc_payload_deserializer: Optional[CDCRecordBytesToCDCPayloadDeserializer] = None,
     ):
         self.__subscriber = subscriber
         self.__logger = logger
@@ -44,6 +54,7 @@ class KafkaEventAsyncSubscriberExecutor(KafkaEventSubscriberExecutor):
         self.__reject_callback = reject_callback
         self.__byte_deserializer = byte_deserializer
         self.__header_deserializer = header_deserializer
+        self.__cdc_payload_deserializer = cdc_payload_deserializer
 
     async def consume(
         self,
@@ -57,6 +68,15 @@ class KafkaEventAsyncSubscriberExecutor(KafkaEventSubscriberExecutor):
             kafka_record_value = cast(bytes, kafka_poll_record.value)
 
             deserialized_value = self.__byte_deserializer.deserialize(kafka_record_value)
+            process_context = ProcessContext()
+            if self.__cdc_payload_deserializer is not None:
+                try:
+                    deserialized_cdc_payload = self.__cdc_payload_deserializer.deserialize(kafka_record_value)
+                    process_context = CDCProcessContext(
+                        captured_at_ms=deserialized_cdc_payload.captured_at_ms,
+                    )
+                except CannotDecodeCDCMessageException as e:
+                    self.__logger.error(e.message)
 
             self.__logger.info(
                 f"consuming the event '{deserialized_value.id}' by the subscriber '{self.__subscriber.fqn()}', "
@@ -75,7 +95,8 @@ class KafkaEventAsyncSubscriberExecutor(KafkaEventSubscriberExecutor):
                         consumer_group=self.__subscriber.fqn(),
                         partition=kafka_poll_record.partition,
                         timestamp=kafka_poll_record.timestamp,
-                    )
+                    ),
+                    process_context=process_context,
                 ),
             )
         except NotValidKafkaMessageException:

{buz-2.17.0rc7 → buz-2.18.0}/src/buz/event/infrastructure/buz_kafka/kafka_event_sync_subscriber_executor.py

@@ -1,13 +1,17 @@
+import time
 from asyncio import get_running_loop
 from logging import Logger
-import time
 from typing import Optional, Sequence, cast
+
+from buz.event.infrastructure.models.process_context import ProcessContext
+
 from buz.event import Event
 from buz.event.infrastructure.buz_kafka.consume_strategy.kafka_on_fail_strategy import KafkaOnFailStrategy
 from buz.event.infrastructure.buz_kafka.exceptions.max_consumer_retry_exception import MaxConsumerRetryException
 from buz.event.infrastructure.buz_kafka.exceptions.retry_exception import ConsumerRetryException
 from buz.event.infrastructure.buz_kafka.kafka_event_subscriber_executor import KafkaEventSubscriberExecutor
 from buz.event.infrastructure.buz_kafka.models.kafka_delivery_context import KafkaDeliveryContext
+from buz.event.infrastructure.buz_kafka.models.cdc_process_context import CDCProcessContext
 from buz.event.infrastructure.models.execution_context import ExecutionContext
 from buz.event.middleware.consume_middleware import ConsumeMiddleware
 from buz.event.middleware.consume_middleware_chain_resolver import ConsumeMiddlewareChainResolver
@@ -18,6 +22,12 @@ from buz.kafka.domain.exceptions.not_valid_kafka_message_exception import NotVal
 from buz.kafka.domain.models.kafka_consumer_record import KafkaConsumerRecord
 from buz.kafka.domain.models.kafka_poll_record import KafkaPollRecord
 from buz.kafka.infrastructure.deserializers.byte_deserializer import ByteDeserializer
+from buz.kafka.infrastructure.deserializers.implementations.cdc.cannot_decode_cdc_message_exception import (
+    CannotDecodeCDCMessageException,
+)
+from buz.kafka.infrastructure.deserializers.implementations.cdc.cdc_record_bytes_to_cdc_payload_deserializer import (
+    CDCRecordBytesToCDCPayloadDeserializer,
+)
 from buz.kafka.infrastructure.serializers.kafka_header_serializer import KafkaHeaderSerializer
 
 
@@ -34,6 +44,7 @@ class KafkaEventSyncSubscriberExecutor(KafkaEventSubscriberExecutor):
         on_fail_strategy: KafkaOnFailStrategy,
         consume_retrier: Optional[ConsumeRetrier] = None,
         reject_callback: Optional[RejectCallback] = None,
+        cdc_payload_deserializer: Optional[CDCRecordBytesToCDCPayloadDeserializer] = None,
     ):
         self.__subscriber = subscriber
         self.__logger = logger
@@ -44,6 +55,7 @@ class KafkaEventSyncSubscriberExecutor(KafkaEventSubscriberExecutor):
         self.__reject_callback = reject_callback
         self.__byte_deserializer = byte_deserializer
         self.__header_deserializer = header_deserializer
+        self.__cdc_payload_deserializer = cdc_payload_deserializer
 
     async def consume(
         self,
@@ -57,6 +69,15 @@ class KafkaEventSyncSubscriberExecutor(KafkaEventSubscriberExecutor):
             kafka_record_value = cast(bytes, kafka_poll_record.value)
 
             deserialized_value = self.__byte_deserializer.deserialize(kafka_record_value)
+            process_context = ProcessContext()
+            if self.__cdc_payload_deserializer is not None:
+                try:
+                    deserialized_cdc_payload = self.__cdc_payload_deserializer.deserialize(kafka_record_value)
+                    process_context = CDCProcessContext(
+                        captured_at_ms=deserialized_cdc_payload.captured_at_ms,
+                    )
+                except CannotDecodeCDCMessageException as e:
+                    self.__logger.error(e.message)
 
             self.__logger.info(
                 f"consuming the event '{deserialized_value.id}' by the subscriber '{self.__subscriber.fqn()}', "
@@ -77,7 +98,8 @@ class KafkaEventSyncSubscriberExecutor(KafkaEventSubscriberExecutor):
                             consumer_group=self.__subscriber.fqn(),
                             partition=kafka_poll_record.partition,
                             timestamp=kafka_poll_record.timestamp,
-                        )
+                        ),
+                        process_context=process_context,
                     ),
                 ),
             )

buz-2.18.0/src/buz/event/infrastructure/buz_kafka/models/cdc_process_context.py

@@ -0,0 +1,9 @@
+from dataclasses import dataclass
+from typing import Optional
+
+from buz.event.infrastructure.models.process_context import ProcessContext
+
+
+@dataclass(frozen=True)
+class CDCProcessContext(ProcessContext):
+    captured_at_ms: Optional[int] = None

{buz-2.17.0rc7 → buz-2.18.0}/src/buz/event/infrastructure/kombu/kombu_consumer.py

@@ -2,6 +2,8 @@ import asyncio
 from logging import Logger
 from typing import Optional, Callable, cast
 
+from buz.event.infrastructure.models.process_context import ProcessContext
+
 from buz.event.infrastructure.kombu.models.kombu_delivery_context import KombuDeliveryContext
 from buz.event.infrastructure.models.execution_context import ExecutionContext
 from kombu import Connection, Queue, Consumer as MessageConsumer, Message
@@ -112,7 +114,10 @@ class KombuConsumer(ConsumerMixin, Consumer):
                 # The problem here is that the chain resolver works with syncsubscribers, an asyncsubscriber would require of a async function,
                 # but we are using run-until-complete to run the async function, so we are not really using the async function, we are just running it as a sync function, so we can cast the asyncsubscriber to a subscriber
                 subscriber = cast(Subscriber, meta_subscriber)
-                execution_context = ExecutionContext(delivery_context=KombuDeliveryContext())
+                execution_context = ExecutionContext(
+                    delivery_context=KombuDeliveryContext(),
+                    process_context=ProcessContext(),
+                )
                 self.__consume_middleware_chain_resolver.resolve(
                     event=event,
                     subscriber=subscriber,

{buz-2.17.0rc7 → buz-2.18.0}/src/buz/event/infrastructure/models/execution_context.py

@@ -1,8 +1,10 @@
 from dataclasses import dataclass
 
 from buz.event.infrastructure.models.delivery_context import DeliveryContext
+from buz.event.infrastructure.models.process_context import ProcessContext
 
 
 @dataclass(frozen=True)
 class ExecutionContext:
     delivery_context: DeliveryContext
+    process_context: ProcessContext

buz-2.18.0/src/buz/event/infrastructure/models/process_context.py

@@ -0,0 +1,6 @@
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class ProcessContext:
+    pass

{buz-2.17.0rc7 → buz-2.18.0}/src/buz/event/sync/sync_event_bus.py

@@ -1,5 +1,7 @@
 from typing import Optional, Iterable
 
+from buz.event.infrastructure.models.process_context import ProcessContext
+
 from buz.event import Event, EventBus, Subscriber
 from buz.event.infrastructure.models.execution_context import ExecutionContext
 from buz.event.middleware import (
@@ -28,7 +30,7 @@ class SyncEventBus(EventBus):
 
     def __perform_publish(self, event: Event) -> None:
         subscribers = self.__locator.get(event)
-        execution_context = ExecutionContext(delivery_context=SyncDeliveryContext())
+        execution_context = ExecutionContext(delivery_context=SyncDeliveryContext(), process_context=ProcessContext())
         for subscriber in subscribers:
             self.__consume_middleware_chain_resolver.resolve(
                 event=event,

{buz-2.17.0rc7 → buz-2.18.0}/src/buz/kafka/infrastructure/cdc/cdc_payload.py

@@ -11,6 +11,7 @@ class CDCPayload:
     created_at: str
     event_fqn: str
     metadata: Optional[str] = None  # json encoded
+    captured_at_ms: Optional[int] = None
 
     def validate(self) -> None:
         if not isinstance(self.payload, str):
@@ -23,6 +24,8 @@ class CDCPayload:
             raise ValueError("The event_fqn value is not a valid value")
         if self.metadata is not None and not isinstance(self.metadata, str):
             raise ValueError("The metadata value is not a valid value")
+        if self.captured_at_ms is not None and not isinstance(self.captured_at_ms, int):
+            raise ValueError("The captured_at_ms value is not a valid value")
 
     def __post_init__(self) -> None:
         self.validate()

buz-2.17.0rc7/src/buz/kafka/infrastructure/deserializers/implementations/cdc/not_valid_cdc_message_exception.py → buz-2.18.0/src/buz/kafka/infrastructure/deserializers/implementations/cdc/cannot_decode_cdc_message_exception.py

@@ -1,7 +1,7 @@
 from buz.kafka.domain.exceptions.not_valid_kafka_message_exception import NotValidKafkaMessageException
 
 
-class NotValidCDCMessageException(NotValidKafkaMessageException):
+class CannotDecodeCDCMessageException(NotValidKafkaMessageException):
     def __init__(self, message: str, exception: Exception) -> None:
         self.exception = exception
         self.message = message

buz-2.18.0/src/buz/kafka/infrastructure/deserializers/implementations/cdc/cannot_restore_event_from_cdc_payload_exception.py

@@ -0,0 +1,11 @@
+from buz.kafka.domain.exceptions.not_valid_kafka_message_exception import NotValidKafkaMessageException
+from buz.kafka.infrastructure.cdc.cdc_payload import CDCPayload
+
+
+class CannotRestoreEventFromCDCPayloadException(NotValidKafkaMessageException):
+    def __init__(self, cdc_payload: CDCPayload, exception: Exception) -> None:
+        self.exception = exception
+        self.cdc_payload = cdc_payload
+        super().__init__(
+            f'Couldn\'t build a message from CDC payload "{self.cdc_payload}". Raised from "{str(exception)}"'
+        )

buz-2.18.0/src/buz/kafka/infrastructure/deserializers/implementations/cdc/cdc_record_bytes_to_cdc_payload_deserializer.py

@@ -0,0 +1,35 @@
+from __future__ import annotations
+
+import orjson
+from cachetools import LRUCache
+from dacite import from_dict
+
+from buz.kafka.infrastructure.cdc.cdc_message import CDCPayload
+from buz.kafka.infrastructure.deserializers.implementations.cdc.cannot_decode_cdc_message_exception import (
+    CannotDecodeCDCMessageException,
+)
+
+
+class CDCRecordBytesToCDCPayloadDeserializer:
+    __STRING_ENCODING = "utf-8"
+    __cached_deserialization: LRUCache[str, CDCPayload] = LRUCache(maxsize=128)
+
+    def deserialize(self, data: bytes) -> CDCPayload:
+        decoded_string = data.decode(self.__STRING_ENCODING)
+        if decoded_string not in CDCRecordBytesToCDCPayloadDeserializer.__cached_deserialization:
+            try:
+                CDCRecordBytesToCDCPayloadDeserializer.__cached_deserialization[
+                    decoded_string
+                ] = self.__get_cdc_payload_from_string(decoded_string)
+            except Exception as exception:
+                raise CannotDecodeCDCMessageException(decoded_string, exception) from exception
+
+        return CDCRecordBytesToCDCPayloadDeserializer.__cached_deserialization[decoded_string]
+
+    def __get_cdc_payload_from_string(self, decoded_string: str) -> CDCPayload:
+        decoded_record: dict = orjson.loads(decoded_string)
+        payload = decoded_record.get("payload")
+        if not isinstance(payload, dict):
+            raise ValueError("The provided payload value is not valid")
+
+        return from_dict(CDCPayload, payload)

{buz-2.17.0rc7 → buz-2.18.0}/src/buz/kafka/infrastructure/deserializers/implementations/cdc/cdc_record_bytes_to_event_deserializer.py

@@ -4,13 +4,15 @@ from datetime import datetime
 from typing import Optional, TypeVar, Type, Generic
 
 import orjson
-from dacite import from_dict
 
 from buz.event import Event
 from buz.kafka.infrastructure.cdc.cdc_message import CDCPayload
 from buz.kafka.infrastructure.deserializers.bytes_to_message_deserializer import BytesToMessageDeserializer
-from buz.kafka.infrastructure.deserializers.implementations.cdc.not_valid_cdc_message_exception import (
-    NotValidCDCMessageException,
+from buz.kafka.infrastructure.deserializers.implementations.cdc.cannot_restore_event_from_cdc_payload_exception import (
+    CannotRestoreEventFromCDCPayloadException,
+)
+from buz.kafka.infrastructure.deserializers.implementations.cdc.cdc_record_bytes_to_cdc_payload_deserializer import (
+    CDCRecordBytesToCDCPayloadDeserializer,
 )
 
 T = TypeVar("T", bound=Event)
@@ -21,34 +23,25 @@ class CDCRecordBytesToEventDeserializer(BytesToMessageDeserializer[Event], Gener
 
     def __init__(self, event_class: Type[T]) -> None:
         self.__event_class = event_class
+        self.__cdc_record_bytes_to_cdc_payload_deserializer = CDCRecordBytesToCDCPayloadDeserializer()
 
     def deserialize(self, data: bytes) -> T:
-        decoded_string = data.decode(self.__STRING_ENCODING)
+        cdc_payload = self.__cdc_record_bytes_to_cdc_payload_deserializer.deserialize(data)
         try:
-            cdc_payload = self.__get_outbox_record_as_dict(decoded_string)
+            payload_dict = orjson.loads(cdc_payload.payload)
             return self.__event_class.restore(
                 id=cdc_payload.event_id,
                 created_at=self.__get_created_at_in_event_format(cdc_payload.created_at),
                 metadata=self.__deserialize_metadata(cdc_payload.metadata),
-                **orjson.loads(cdc_payload.payload),
+                **payload_dict,
             )
         except Exception as exception:
-            raise NotValidCDCMessageException(decoded_string, exception) from exception
+            raise CannotRestoreEventFromCDCPayloadException(cdc_payload, exception) from exception
 
     def __get_created_at_in_event_format(self, cdc_payload_created_at: str) -> str:
         created_at_datetime = datetime.strptime(cdc_payload_created_at, CDCPayload.DATE_TIME_FORMAT)
         return created_at_datetime.strftime(Event.DATE_TIME_FORMAT)
 
-    def __get_outbox_record_as_dict(self, decoded_string: str) -> CDCPayload:
-        decoded_record: dict = orjson.loads(decoded_string)
-
-        payload = decoded_record.get("payload")
-
-        if not isinstance(payload, dict):
-            raise ValueError("The provided payload value is not valid")
-
-        return from_dict(CDCPayload, payload)
-
     def __deserialize_metadata(self, metadata: Optional[str]) -> dict:
         if metadata is None:
             return {}