eventsourcing 9.3.5__py3-none-any.whl → 9.4.0a1__py3-none-any.whl

eventsourcing/persistence.py

@@ -7,10 +7,12 @@ from collections import deque
 from dataclasses import dataclass
 from datetime import datetime
 from decimal import Decimal
-from threading import Condition, Event, Lock, Semaphore, Timer
-from time import time
+from queue import Queue
+from threading import Condition, Event, Lock, Semaphore, Thread, Timer
+from time import monotonic, sleep, time
 from types import ModuleType
 from typing import (
+    TYPE_CHECKING,
     Any,
     Dict,
     Generic,
@@ -35,6 +37,9 @@ from eventsourcing.utils import (
     strtobool,
 )
 
+if TYPE_CHECKING:  # pragma: no cover
+    from typing_extensions import Self
+
 
 class Transcoding(ABC):
     """
@@ -58,17 +63,6 @@ class Transcoder(ABC):
     Abstract base class for transcoders.
     """
 
-    def __init__(self) -> None:
-        self.types: Dict[type, Transcoding] = {}
-        self.names: Dict[str, Transcoding] = {}
-
-    def register(self, transcoding: Transcoding) -> None:
-        """
-        Registers given transcoding with the transcoder.
-        """
-        self.types[transcoding.type] = transcoding
-        self.names[transcoding.name] = transcoding
-
     @abstractmethod
     def encode(self, obj: Any) -> bytes:
         """Encodes obj as bytes."""
@@ -84,7 +78,8 @@ class JSONTranscoder(Transcoder):
     """
 
     def __init__(self) -> None:
-        super().__init__()
+        self.types: Dict[type, Transcoding] = {}
+        self.names: Dict[str, Transcoding] = {}
         self.encoder = json.JSONEncoder(
             default=self._encode_obj,
             separators=(",", ":"),
@@ -92,6 +87,13 @@ class JSONTranscoder(Transcoder):
         )
         self.decoder = json.JSONDecoder(object_hook=self._decode_obj)
 
+    def register(self, transcoding: Transcoding) -> None:
+        """
+        Registers given transcoding with the transcoder.
+        """
+        self.types[transcoding.type] = transcoding
+        self.names[transcoding.name] = transcoding
+
     def encode(self, obj: Any) -> bytes:
         """
         Encodes given object as a bytes array.
@@ -200,19 +202,16 @@ class StoredEvent:
     Frozen dataclass that represents :class:`~eventsourcing.domain.DomainEvent`
     objects, such as aggregate :class:`~eventsourcing.domain.Aggregate.Event`
     objects and :class:`~eventsourcing.domain.Snapshot` objects.
-
-    Constructor parameters:
-
-    :param UUID originator_id: ID of the originating aggregate
-    :param int originator_version: version of the originating aggregate
-    :param str topic: topic of the domain event object class
-    :param bytes state: serialised state of the domain event object
     """
 
     originator_id: uuid.UUID
+    """ID of the originating aggregate."""
     originator_version: int
+    """Position in an aggregate sequence."""
     topic: str
+    """Topic of a domain event object class."""
     state: bytes
+    """Serialised state of a domain event object."""
 
 
 class Compressor(ABC):
@@ -406,10 +405,19 @@ class NotSupportedError(DatabaseError):
     """
 
 
+class WaitInterruptedError(PersistenceError):
+    """
+    Raised when waiting for a tracking record is interrupted.
+    """
+
+
+class Recorder:
+    pass
+
+
 class AggregateRecorder(ABC):
     """
-    Abstract base class for recorders that record and
-    retrieve stored events for domain model aggregates.
+    Abstract base class for inserting and selecting stored events.
     """
 
     @abstractmethod
@@ -442,71 +450,135 @@ class Notification(StoredEvent):
     """
 
     id: int
+    """Position in an application sequence."""
 
 
 class ApplicationRecorder(AggregateRecorder):
     """
-    Abstract base class for recorders that record and
-    retrieve stored events for domain model aggregates.
-
-    Extends the behaviour of aggregate recorders by
-    recording aggregate events in a total order that
-    allows the stored events also to be retrieved
-    as event notifications.
+    Abstract base class for recording events in both aggregate
+    and application sequences.
     """
 
     @abstractmethod
     def select_notifications(
         self,
-        start: int,
+        start: int | None,
         limit: int,
         stop: int | None = None,
         topics: Sequence[str] = (),
+        *,
+        inclusive_of_start: bool = True,
     ) -> List[Notification]:
         """
-        Returns a list of event notifications
-        from 'start', limited by 'limit' and
-        optionally by 'stop'.
+        Returns a list of Notification objects representing events from an
+        application sequence. If `inclusive_of_start` is True (the default),
+        the returned Notification objects will have IDs greater than or equal
+        to `start` and less than or equal to `stop`. If `inclusive_of_start`
+        is False, the Notification objects will have IDs greater than `start`
+        and less than or equal to `stop`.
         """
 
     @abstractmethod
-    def max_notification_id(self) -> int:
+    def max_notification_id(self) -> int | None:
+        """
+        Returns the largest notification ID in an application sequence,
+        or None if no stored events have been recorded.
         """
-        Returns the maximum notification ID.
+
+    @abstractmethod
+    def subscribe(self, gt: int | None = None) -> Subscription[ApplicationRecorder]:
         """
+        Returns an iterator of Notification objects representing events from an
+        application sequence.
 
+        The iterator will block after the last recorded event has been yielded, but
+        will then continue yielding newly recorded events when they are recorded.
 
-class ProcessRecorder(ApplicationRecorder):
-    """
-    Abstract base class for recorders that record and
-    retrieve stored events for domain model aggregates.
+        Notifications will have IDs greater than the optional `gt` argument.
+        """
 
-    Extends the behaviour of applications recorders by
-    recording aggregate events with tracking information
-    that records the position of a processed event
-    notification in a notification log.
+
+class TrackingRecorder(Recorder, ABC):
+    """
+    Abstract base class for recorders that record tracking
+    objects atomically with other state.
     """
 
     @abstractmethod
-    def max_tracking_id(self, application_name: str) -> int:
+    def insert_tracking(self, tracking: Tracking) -> None:
+        """
+        Records a tracking object.
+        """
+
+    @abstractmethod
+    def max_tracking_id(self, application_name: str) -> int | None:
         """
-        Returns the largest notification ID across all tracking records
-        for the named application. Returns zero if there are no tracking
-        records.
+        Returns the largest notification ID across all recorded tracking objects
+        for the named application, or None if no tracking objects have been recorded.
         """
 
     @abstractmethod
     def has_tracking_id(self, application_name: str, notification_id: int) -> bool:
         """
-        Returns true if a tracking record with the given application name
-        and notification ID exists, otherwise returns false.
+        Returns True if a tracking object with the given application name
+        and notification ID has been recorded, otherwise returns False.
         """
 
+    def wait(
+        self,
+        application_name: str,
+        notification_id: int,
+        timeout: float = 1.0,
+        interrupt: Event | None = None,
+    ) -> None:
+        """
+        Block until a tracking object with the given application name and
+        notification ID has been recorded.
+
+        Polls has_tracking_id() with exponential backoff until the timeout
+        is reached, or until the optional interrupt event is set.
+
+        The timeout argument should be a floating point number specifying a
+        timeout for the operation in seconds (or fractions thereof). The default
+        is 1.0 seconds.
+
+        Raises TimeoutError if the timeout is reached.
+
+        Raises WaitInterruptError if the `interrupt` is set before `timeout` is reached.
+        """
+        deadline = monotonic() + timeout
+        delay_ms = 1.0
+        while True:
+            if self.has_tracking_id(application_name, notification_id):
+                break
+            if interrupt:
+                if interrupt.wait(timeout=delay_ms / 1000):
+                    raise WaitInterruptedError
+            else:
+                sleep(delay_ms / 1000)
+            delay_ms *= 2
+            remaining = deadline - monotonic()
+            if remaining < 0:
+                msg = (
+                    f"Timed out waiting for notification {notification_id} "
+                    f"from application '{application_name}' to be processed"
+                )
+                raise TimeoutError(msg)
+            delay_ms = min(delay_ms, remaining * 1000)
+
+
+class ProcessRecorder(TrackingRecorder, ApplicationRecorder, ABC):
+    pass
+
 
 @dataclass(frozen=True)
 class Recording:
+    """Represents the recording of a domain event."""
+
     domain_event: DomainEventProtocol
+    """The domain event that has been recorded."""
     notification: Notification
+    """A Notification that represents the domain event in the application sequence."""
 
 
 class EventStore:
@@ -573,31 +645,39 @@ class EventStore:
 
 
 TInfrastructureFactory = TypeVar(
-    "TInfrastructureFactory", bound="InfrastructureFactory"
+    "TInfrastructureFactory", bound="InfrastructureFactory[Any]"
 )
 
+TTrackingRecorder = TypeVar("TTrackingRecorder", bound=TrackingRecorder)
+
 
-class InfrastructureFactory(ABC):
+class InfrastructureFactory(ABC, Generic[TTrackingRecorder]):
     """
     Abstract base class for infrastructure factories.
     """
 
     PERSISTENCE_MODULE = "PERSISTENCE_MODULE"
+    TRANSCODER_TOPIC = "TRANSCODER_TOPIC"
     MAPPER_TOPIC = "MAPPER_TOPIC"
     CIPHER_TOPIC = "CIPHER_TOPIC"
     COMPRESSOR_TOPIC = "COMPRESSOR_TOPIC"
     IS_SNAPSHOTTING_ENABLED = "IS_SNAPSHOTTING_ENABLED"
+    APPLICATION_RECORDER_TOPIC = "APPLICATION_RECORDER_TOPIC"
+    TRACKING_RECORDER_TOPIC = "TRACKING_RECORDER_TOPIC"
+    PROCESS_RECORDER_TOPIC = "PROCESS_RECORDER_TOPIC"
 
     @classmethod
     def construct(
-        cls: Type[TInfrastructureFactory], env: Environment
+        cls: Type[TInfrastructureFactory], env: Environment | None = None
     ) -> TInfrastructureFactory:
         """
         Constructs concrete infrastructure factory for given
         named application. Reads and resolves persistence
         topic from environment variable 'PERSISTENCE_MODULE'.
         """
-        factory_cls: Type[InfrastructureFactory]
+        factory_cls: Type[InfrastructureFactory[TrackingRecorder]]
+        if env is None:
+            env = Environment()
         topic = (
             env.get(
                 cls.PERSISTENCE_MODULE,
@@ -614,7 +694,9 @@ class InfrastructureFactory(ABC):
             or "eventsourcing.popo"
         )
         try:
-            obj: Type[InfrastructureFactory] | ModuleType = resolve_topic(topic)
+            obj: Type[InfrastructureFactory[TrackingRecorder]] | ModuleType = (
+                resolve_topic(topic)
+            )
         except TopicError as e:
             msg = (
                 "Failed to resolve persistence module topic: "
@@ -625,15 +707,15 @@ class InfrastructureFactory(ABC):
 
         if isinstance(obj, ModuleType):
             # Find the factory in the module.
-            factory_classes: List[Type[InfrastructureFactory]] = [
-                member
-                for member in obj.__dict__.values()
+            factory_classes: List[Type[InfrastructureFactory[TrackingRecorder]]] = []
+            for member in obj.__dict__.values():
                 if (
                     member is not InfrastructureFactory
-                    and isinstance(member, type)
-                    and issubclass(member, InfrastructureFactory)
-                )
-            ]
+                    and isinstance(member, type)  # Look for classes...
+                    and issubclass(member, InfrastructureFactory)  # Ignore base class.
+                    and member not in factory_classes  # Forgive aliases.
+                ):
+                    factory_classes.append(member)
             if len(factory_classes) == 1:
                 factory_cls = factory_classes[0]
             else:
@@ -661,18 +743,27 @@ class InfrastructureFactory(ABC):
         """
         Constructs a transcoder.
         """
-        # TODO: Implement support for TRANSCODER_TOPIC.
-        return JSONTranscoder()
+        transcoder_topic = self.env.get(self.TRANSCODER_TOPIC)
+        if transcoder_topic:
+            transcoder_class: Type[Transcoder] = resolve_topic(transcoder_topic)
+        else:
+            transcoder_class = JSONTranscoder
+        return transcoder_class()
 
     def mapper(
-        self, transcoder: Transcoder, mapper_class: Type[Mapper] = Mapper
+        self,
+        transcoder: Transcoder | None = None,
+        mapper_class: Type[Mapper] | None = None,
     ) -> Mapper:
         """
         Constructs a mapper.
         """
-        # TODO: Implement support for MAPPER_TOPIC.
+        if mapper_class is None:
+            mapper_topic = self.env.get(self.MAPPER_TOPIC)
+            mapper_class = resolve_topic(mapper_topic) if mapper_topic else Mapper
+
         return mapper_class(
-            transcoder=transcoder,
+            transcoder=transcoder or self.transcoder(),
             cipher=self.cipher(),
             compressor=self.compressor(),
         )
@@ -712,12 +803,18 @@ class InfrastructureFactory(ABC):
                 compressor = compressor_cls
         return compressor
 
-    @staticmethod
-    def event_store(**kwargs: Any) -> EventStore:
+    def event_store(
+        self,
+        mapper: Mapper | None = None,
+        recorder: AggregateRecorder | None = None,
+    ) -> EventStore:
         """
         Constructs an event store.
         """
-        return EventStore(**kwargs)
+        return EventStore(
+            mapper=mapper or self.mapper(),
+            recorder=recorder or self.application_recorder(),
+        )
 
     @abstractmethod
     def aggregate_recorder(self, purpose: str = "events") -> AggregateRecorder:
@@ -731,6 +828,14 @@ class InfrastructureFactory(ABC):
         Constructs an application recorder.
         """
 
+    @abstractmethod
+    def tracking_recorder(
+        self, tracking_recorder_class: Type[TTrackingRecorder] | None = None
+    ) -> TTrackingRecorder:
+        """
+        Constructs a tracking recorder.
+        """
+
     @abstractmethod
     def process_recorder(self) -> ProcessRecorder:
         """
@@ -747,7 +852,7 @@ class InfrastructureFactory(ABC):
 
     def close(self) -> None:
         """
-        Closes any database connections, or anything else that needs closing.
+        Closes any database connections, and anything else that needs closing.
         """
 
 
@@ -1197,3 +1302,119 @@ class ConnectionPool(ABC, Generic[TConnection]):
 
     def __del__(self) -> None:
         self.close()
+
+
+TApplicationRecorder_co = TypeVar(
+    "TApplicationRecorder_co", bound=ApplicationRecorder, covariant=True
+)
+
+
+class Subscription(Iterator[Notification], Generic[TApplicationRecorder_co]):
+    def __init__(
+        self, recorder: TApplicationRecorder_co, gt: int | None = None
+    ) -> None:
+        self._recorder = recorder
+        self._last_notification_id = gt
+        self._has_been_entered = False
+        self._has_been_stopped = False
+
+    def __enter__(self) -> Self:
+        if self._has_been_entered:
+            msg = "Already entered subscription context manager"
+            raise ProgrammingError(msg)
+        self._has_been_entered = True
+        return self
+
+    def __exit__(self, *args: object, **kwargs: Any) -> None:
+        if not self._has_been_entered:
+            msg = "Not already entered subscription context manager"
+            raise ProgrammingError(msg)
+        self.stop()
+
+    def stop(self) -> None:
+        """
+        Stops the subscription.
+        """
+        self._has_been_stopped = True
+
+    def __iter__(self) -> Self:
+        return self
+
+    @abstractmethod
+    def __next__(self) -> Notification:
+        """
+        Returns the next Notification object in the application sequence.
+        """
+
+
+class ListenNotifySubscription(Subscription[TApplicationRecorder_co]):
+    def __init__(
+        self, recorder: TApplicationRecorder_co, gt: int | None = None
+    ) -> None:
+        super().__init__(recorder=recorder, gt=gt)
+        self._select_limit = 500
+        self._notifications: List[Notification] = []
+        self._notifications_index: int = 0
+        self._notifications_queue: Queue[List[Notification]] = Queue(maxsize=10)
+        self._has_been_notified = Event()
+        self._thread_error: BaseException | None = None
+        self._pull_thread = Thread(target=self._loop_on_pull)
+        self._pull_thread.start()
+
+    def __exit__(self, *args: object, **kwargs: Any) -> None:
+        super().__exit__(*args, **kwargs)
+        self._pull_thread.join()
+
+    def stop(self) -> None:
+        """
+        Stops the subscription.
+        """
+        super().stop()
+        self._notifications_queue.put([])
+        self._has_been_notified.set()
+
+    def __next__(self) -> Notification:
+        # If necessary, get a new list of notifications from the recorder.
+        if (
+            self._notifications_index == len(self._notifications)
+            and not self._has_been_stopped
+        ):
+            self._notifications = self._notifications_queue.get()
+            self._notifications_index = 0
+
+        # Stop the iteration if necessary, maybe raise thread error.
+        if self._has_been_stopped or not self._notifications:
+            if self._thread_error is not None:
+                raise self._thread_error
+            raise StopIteration
+
+        # Return a notification from previously obtained list.
+        notification = self._notifications[self._notifications_index]
+        self._notifications_index += 1
+        return notification
+
+    def _loop_on_pull(self) -> None:
+        try:
+            self._pull()  # Already recorded events.
+            while not self._has_been_stopped:
+                self._has_been_notified.wait()
+                self._pull()  # Newly recorded events.
+        except BaseException as e:
+            if self._thread_error is None:
+                self._thread_error = e
+            self.stop()
+
+    def _pull(self) -> None:
+        while not self._has_been_stopped:
+            self._has_been_notified.clear()
+            notifications = self._recorder.select_notifications(
+                start=self._last_notification_id or 0,
+                limit=self._select_limit,
+                inclusive_of_start=False,
+            )
+            if len(notifications) > 0:
+                # print("Putting", len(notifications), "notifications into queue")
+                self._notifications_queue.put(notifications)
+                self._last_notification_id = notifications[-1].id
+            if len(notifications) < self._select_limit:
+                break