eventsourcing 9.4.0b1__tar.gz → 9.4.0b2__tar.gz

{eventsourcing-9.4.0b1 → eventsourcing-9.4.0b2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: eventsourcing
-Version: 9.4.0b1
+Version: 9.4.0b2
 Summary: Event sourcing in Python
 License: BSD 3-Clause
 Keywords: event sourcing,event store,domain driven design,domain-driven design,ddd,cqrs,cqs

{eventsourcing-9.4.0b1 → eventsourcing-9.4.0b2}/eventsourcing/persistence.py

@@ -61,6 +61,12 @@ class Transcoder(ABC):
         """Decodes obj from bytes."""
 
 
+class TranscodingNotRegisteredError(EventSourcingError, TypeError):
+    """
+    Raised when a transcoding isn't registered with JSONTranscoder.
+    """
+
+
 class JSONTranscoder(Transcoder):
     """
     Extensible transcoder that uses the Python :mod:`json` module.
@@ -104,7 +110,7 @@ class JSONTranscoder(Transcoder):
                 "serializable. Please define and register "
                 "a custom transcoding for this type."
             )
-            raise TypeError(msg) from None
+            raise TranscodingNotRegisteredError(msg) from None
         else:
             return {
                 "_type_": transcoding.name,
@@ -131,7 +137,7 @@ class JSONTranscoder(Transcoder):
                             "deserializable. Please register a "
                             "custom transcoding for this type."
                         )
-                        raise TypeError(msg) from e
+                        raise TranscodingNotRegisteredError(msg) from e
                     else:
                         return transcoding.decode(_data_)
         else:
@@ -245,6 +251,12 @@ class Cipher(ABC):
         """
 
 
+class MapperDeserialisationError(EventSourcingError, ValueError):
+    """
+    Raised when deserialization fails in a Mapper.
+    """
+
+
 class Mapper:
     """
     Converts between domain event objects and :class:`StoredEvent` objects.
@@ -290,11 +302,21 @@ class Mapper:
         Converts the given :class:`StoredEvent` to a domain event object.
         """
         stored_state = stored_event.state
-        if self.cipher:
-            stored_state = self.cipher.decrypt(stored_state)
-        if self.compressor:
-            stored_state = self.compressor.decompress(stored_state)
-        event_state: dict[str, Any] = self.transcoder.decode(stored_state)
+        try:
+            if self.cipher:
+                stored_state = self.cipher.decrypt(stored_state)
+            if self.compressor:
+                stored_state = self.compressor.decompress(stored_state)
+            event_state: dict[str, Any] = self.transcoder.decode(stored_state)
+        except Exception as e:
+            msg = (
+                f"Failed to deserialise state of stored event with "
+                f"topic '{stored_event.topic}', "
+                f"originator_id '{stored_event.originator_id}' and "
+                f"originator_version {stored_event.originator_version}: {e}"
+            )
+            raise MapperDeserialisationError(msg) from e
+
         event_state["originator_id"] = stored_event.originator_id
         event_state["originator_version"] = stored_event.originator_version
         cls = resolve_topic(stored_event.topic)
@@ -395,7 +417,7 @@ class Recorder:
     pass
 
 
-class AggregateRecorder(ABC):
+class AggregateRecorder(Recorder, ABC):
     """
     Abstract base class for inserting and selecting stored events.
     """
@@ -532,7 +554,8 @@ class TrackingRecorder(Recorder, ABC):
         Raises WaitInterruptError if the `interrupt` is set before `timeout` is reached.
         """
         deadline = monotonic() + timeout
-        delay_ms = 1.0
+        sleep_interval_ms = 100.0
+        max_sleep_interval_ms = 800.0
         while True:
             max_tracking_id = self.max_tracking_id(application_name)
             if notification_id is None or (
@@ -540,11 +563,10 @@ class TrackingRecorder(Recorder, ABC):
             ):
                 break
             if interrupt:
-                if interrupt.wait(timeout=delay_ms / 1000):
+                if interrupt.wait(timeout=sleep_interval_ms / 1000):
                     raise WaitInterruptedError
             else:
-                sleep(delay_ms / 1000)
-            delay_ms *= 2
+                sleep(sleep_interval_ms / 1000)
             remaining = deadline - monotonic()
             if remaining < 0:
                 msg = (
@@ -552,7 +574,9 @@ class TrackingRecorder(Recorder, ABC):
                     f"from application '{application_name}' to be processed"
                 )
                 raise TimeoutError(msg)
-            delay_ms = min(delay_ms, remaining * 1000)
+            sleep_interval_ms = min(
+                sleep_interval_ms * 2, remaining * 1000, max_sleep_interval_ms
+            )
 
 
 class ProcessRecorder(TrackingRecorder, ApplicationRecorder, ABC):

{eventsourcing-9.4.0b1 → eventsourcing-9.4.0b2}/eventsourcing/projection.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import contextlib
 import os
 import weakref
 from abc import ABC, abstractmethod
@@ -22,6 +23,8 @@ from eventsourcing.persistence import (
 from eventsourcing.utils import Environment, EnvType
 
 if TYPE_CHECKING:
+    from types import TracebackType
+
     from typing_extensions import Self
 
 
@@ -143,7 +146,8 @@ class ProjectionRunner(Generic[TApplication, TTrackingRecorder]):
         thread, calls projection's process_event() method for each event and tracking
         object pair received from the subscription.
         """
-        self._is_stopping = Event()
+        self._is_interrupted = Event()
+        self._has_called_stop = False
 
         self.app: TApplication = application_class(env)
 
@@ -165,17 +169,29 @@ class ProjectionRunner(Generic[TApplication, TTrackingRecorder]):
             gt=self.view.max_tracking_id(self.app.name),
             topics=self.projection.topics,
         )
-        self.thread_error: BaseException | None = None
-        self.processing_thread = Thread(
+        self._thread_error: BaseException | None = None
+        self._stop_thread = Thread(
+            target=self._stop_subscription_when_stopping,
+            kwargs={
+                "subscription": self.subscription,
+                "is_stopping": self._is_interrupted,
+            },
+        )
+        self._stop_thread.start()
+        self._processing_thread = Thread(
             target=self._process_events_loop,
             kwargs={
                 "subscription": self.subscription,
                 "projection": self.projection,
-                "is_stopping": self._is_stopping,
+                "is_stopping": self._is_interrupted,
                 "runner": weakref.ref(self),
             },
         )
-        self.processing_thread.start()
+        self._processing_thread.start()
+
+    @property
+    def is_interrupted(self) -> Event:
+        return self._is_interrupted
 
     def _construct_env(self, name: str, env: EnvType | None = None) -> Environment:
         """
@@ -188,11 +204,25 @@ class ProjectionRunner(Generic[TApplication, TTrackingRecorder]):
         return Environment(name, _env)
 
     def stop(self) -> None:
+        """
+        Sets the "interrupted" event.
+        """
+        self._has_called_stop = True
+        self._is_interrupted.set()
+
+    @staticmethod
+    def _stop_subscription_when_stopping(
+        subscription: ApplicationSubscription,
+        is_stopping: Event,
+    ) -> None:
         """
         Stops the application subscription, which will stop the event-processing thread.
         """
-        self._is_stopping.set()
-        self.subscription.stop()
+        try:
+            is_stopping.wait()
+        finally:
+            is_stopping.set()
+            subscription.stop()
 
     @staticmethod
     def _process_events_loop(
@@ -208,7 +238,7 @@ class ProjectionRunner(Generic[TApplication, TTrackingRecorder]):
         except BaseException as e:
             _runner = runner()  # get reference from weakref
             if _runner is not None:
-                _runner.thread_error = e
+                _runner._thread_error = e
             else:
                 msg = "ProjectionRunner was deleted before error could be assigned:\n"
                 msg += format_exc()
@@ -217,17 +247,21 @@ class ProjectionRunner(Generic[TApplication, TTrackingRecorder]):
                     RuntimeWarning,
                     stacklevel=2,
                 )
-
+        finally:
             is_stopping.set()
-            subscription.stop()
 
     def run_forever(self, timeout: float | None = None) -> None:
         """
         Blocks until timeout, or until the runner is stopped or errors. Re-raises
         any error otherwise exits normally
         """
-        if self._is_stopping.wait(timeout=timeout) and self.thread_error is not None:
-            raise self.thread_error
+        if (
+            self._is_interrupted.wait(timeout=timeout)
+            and self._thread_error is not None
+        ):
+            error = self._thread_error
+            self._thread_error = None
+            raise error
 
     def wait(self, notification_id: int | None, timeout: float = 1.0) -> None:
         """
@@ -239,24 +273,40 @@ class ProjectionRunner(Generic[TApplication, TTrackingRecorder]):
                 application_name=self.subscription.name,
                 notification_id=notification_id,
                 timeout=timeout,
-                interrupt=self._is_stopping,
+                interrupt=self._is_interrupted,
             )
         except WaitInterruptedError:
-            if self.thread_error is not None:
-                raise self.thread_error from None
+            if self._thread_error:
+                error = self._thread_error
+                self._thread_error = None
+                raise error from None
+            if self._has_called_stop:
+                return
+            raise
 
     def __enter__(self) -> Self:
         return self
 
-    def __exit__(self, *args: object, **kwargs: Any) -> None:
+    def __exit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc_val: BaseException | None,
+        exc_tb: TracebackType | None,
+    ) -> None:
         """
         Calls stop() and waits for the event-processing thread to exit.
         """
         self.stop()
-        self.processing_thread.join()
+        self._stop_thread.join()
+        self._processing_thread.join()
+        if self._thread_error:
+            error = self._thread_error
+            self._thread_error = None
+            raise error
 
     def __del__(self) -> None:
         """
         Calls stop().
         """
-        self.stop()
+        with contextlib.suppress(AttributeError):
+            self.stop()

{eventsourcing-9.4.0b1 → eventsourcing-9.4.0b2}/eventsourcing/tests/application.py

@@ -10,7 +10,7 @@ from decimal import Decimal
 from threading import Event, get_ident
 from time import sleep
 from timeit import timeit
-from typing import ClassVar
+from typing import Any, ClassVar
 from unittest import TestCase
 from uuid import UUID, uuid4
 
@@ -23,7 +23,7 @@ from eventsourcing.persistence import (
     Transcoding,
 )
 from eventsourcing.tests.domain import BankAccount, EmailAddress
-from eventsourcing.utils import get_topic
+from eventsourcing.utils import EnvType, get_topic
 
 TIMEIT_FACTOR = int(os.environ.get("TEST_TIMEIT_FACTOR", default=10))
 
@@ -344,96 +344,131 @@ class ApplicationTestCase(TestCase):
         app.repository.get(aggregate.id)
         self.assertEqual(aggregate, app.repository.cache.get(aggregate.id))
 
-    def test_application_fastforward_skipping_during_contention(self) -> None:
-        app = Application(
+    def test_check_aggregate_fastforwarding_nonblocking(self) -> None:
+        self._check_aggregate_fastforwarding_during_contention(
             env={
                 "AGGREGATE_CACHE_MAXSIZE": "10",
                 "AGGREGATE_CACHE_FASTFORWARD_SKIPPING": "y",
             }
         )
 
-        aggregate = Aggregate()
-        aggregate_id = aggregate.id
-        app.save(aggregate)
+    def test_check_aggregate_fastforwarding_blocking(self) -> None:
+        self._check_aggregate_fastforwarding_during_contention(
+            env={"AGGREGATE_CACHE_MAXSIZE": "10"}
+        )
+
+    def _check_aggregate_fastforwarding_during_contention(self, env: EnvType) -> None:
+        app = Application(env=env)
+
+        self.assertEqual(len(app.repository._fastforward_locks_inuse), 0)
+
+        # Create one aggregate.
+        original_aggregate = Aggregate()
+        app.save(original_aggregate)
+        obj_ids = set()
+
+        # Prime the cache.
+        app.repository.get(original_aggregate.id)
+
+        # Remember the aggregate ID.
+        aggregate_id = original_aggregate.id
 
         stopped = Event()
+        errors: list[BaseException] = []
+        successful_thread_ids = set()
 
-        # Trigger, save, get, check.
         def trigger_save_get_check() -> None:
             while not stopped.is_set():
                 try:
+                    # Get the aggregate.
                     aggregate: Aggregate = app.repository.get(aggregate_id)
+                    original_version = aggregate.version
+
+                    # Try to record a new event.
                     aggregate.trigger_event(Aggregate.Event)
-                    saved_version = aggregate.version
+                    # Give other threads a chance.
                     try:
                         app.save(aggregate)
                     except IntegrityError:
+                        # Start again if we didn't record a new event.
+                        # print("Got integrity error")
+                        sleep(0.001)
                         continue
-                    cached: Aggregate = app.repository.get(aggregate_id)
-                    if saved_version > cached.version:
-                        print(f"Skipped fast-forwarding at version {saved_version}")
-                        stopped.set()
-                    if aggregate.version % 1000 == 0:
-                        print("Version:", aggregate.version, get_ident())
-                    sleep(0.00)
-                except BaseException:
-                    print(traceback.format_exc())
-                    raise
 
-        executor = ThreadPoolExecutor(max_workers=100)
-        for _ in range(100):
-            executor.submit(trigger_save_get_check)
+                    # Get the aggregate from the cache.
+                    assert app.repository.cache is not None
+                    cached: Any = app.repository.cache.get(aggregate_id)
+                    obj_ids.add(id(cached))
 
-        if not stopped.wait(timeout=100):
-            stopped.set()
-            self.fail("Didn't skip fast forwarding before test timed out...")
-        executor.shutdown()
+                    if len(obj_ids) > 1:
+                        stopped.set()
+                        continue
 
-    def test_application_fastforward_blocking_during_contention(self) -> None:
-        app = Application(
-            env={
-                "AGGREGATE_CACHE_MAXSIZE": "10",
-            }
-        )
+                    # Fast-forward the cached aggregate.
+                    fastforwarded: Aggregate = app.repository.get(aggregate_id)
 
-        aggregate = Aggregate()
-        aggregate_id = aggregate.id
-        app.save(aggregate)
+                    # Check cached aggregate was fast-forwarded with recorded event.
+                    if fastforwarded.version < original_version:
+                        try:
+                            self.fail(
+                                f"Failed to fast-forward at version {original_version}"
+                            )
+                        except AssertionError as e:
+                            errors.append(e)
+                            stopped.set()
+                            continue
 
-        stopped = Event()
+                    # Monitor number of threads getting involved.
+                    thread_id = get_ident()
+                    successful_thread_ids.add(thread_id)
 
-        # Trigger, save, get, check.
-        def trigger_save_get_check() -> None:
-            while not stopped.is_set():
-                try:
-                    aggregate: Aggregate = app.repository.get(aggregate_id)
-                    aggregate.trigger_event(Aggregate.Event)
-                    saved_version = aggregate.version
-                    try:
-                        app.save(aggregate)
-                    except IntegrityError:
-                        continue
-                    cached: Aggregate = app.repository.get(aggregate_id)
-                    if saved_version > cached.version:
-                        print(f"Skipped fast-forwarding at version {saved_version}")
+                    # print("Version:", aggregate.version, thread_id)
+
+                    # See if we have done enough.
+                    if len(successful_thread_ids) > 10 and aggregate.version >= 25:
                         stopped.set()
-                    if aggregate.version % 1000 == 0:
-                        print("Version:", aggregate.version, get_ident())
-                    sleep(0.00)
-                except BaseException:
+                        continue
+
+                    sleep(0.0001)
+                    # sleep(0.001)
+                except BaseException as e:
+                    errors.append(e)
+                    stopped.set()
                     print(traceback.format_exc())
                     raise
 
         executor = ThreadPoolExecutor(max_workers=100)
+        futures = []
         for _ in range(100):
-            executor.submit(trigger_save_get_check)
-
-        if not stopped.wait(timeout=3):
-            stopped.set()
-        else:
-            self.fail("Wrongly skipped fast forwarding")
+            f = executor.submit(trigger_save_get_check)
+            futures.append(f)
+
+        # Run for three seconds.
+        stopped.wait(timeout=10)
+        for f in futures:
+            f.result()
+        # print("Got all results, shutting down executor")
         executor.shutdown()
 
+        try:
+            if errors:
+                raise errors[0]
+            if len(obj_ids) > 1:
+                self.fail(f"More than one instance used in the cache: {len(obj_ids)}")
+            if len(successful_thread_ids) < 3:
+                self.fail("Insufficient sharing across contentious threads")
+
+            final_aggregate: Aggregate = app.repository.get(aggregate_id)
+            # print("Final aggregate version:", final_aggregate.version)
+            if final_aggregate.version < 25:
+                self.fail(f"Insufficient version increment: {final_aggregate.version}")
+
+            self.assertEqual(len(app.repository._fastforward_locks_inuse), 0)
+
+        finally:
+            # print("Closing application")
+            app.close()
+
     def test_application_with_cached_aggregates_not_fastforward(self) -> None:
         app = Application(
             env={
@@ -441,13 +476,37 @@ class ApplicationTestCase(TestCase):
                 "AGGREGATE_CACHE_FASTFORWARD": "f",
             }
         )
-        aggregate = Aggregate()
-        app.save(aggregate)
+        aggregate1 = Aggregate()
+        app.save(aggregate1)
+        aggregate_id = aggregate1.id
+
         # Should put the aggregate in the cache.
         assert app.repository.cache is not None  # for mypy
-        self.assertEqual(aggregate, app.repository.cache.get(aggregate.id))
-        app.repository.get(aggregate.id)
-        self.assertEqual(aggregate, app.repository.cache.get(aggregate.id))
+        self.assertEqual(aggregate1, app.repository.cache.get(aggregate_id))
+        app.repository.get(aggregate_id)
+        self.assertEqual(aggregate1, app.repository.cache.get(aggregate_id))
+
+        aggregate2 = Aggregate()
+        aggregate2._id = aggregate_id
+        aggregate2.trigger_event(Aggregate.Event)
+
+        # This will replace object in cache.
+        app.save(aggregate2)
+
+        self.assertEqual(aggregate2.version, aggregate1.version + 1)
+        aggregate3: Aggregate = app.repository.get(aggregate_id)
+        self.assertEqual(aggregate3.version, aggregate3.version)
+        self.assertEqual(id(aggregate3.version), id(aggregate3.version))
+
+        # This will mess things up because the cache has a stale aggregate.
+        aggregate3.trigger_event(Aggregate.Event)
+        app.events.put(aggregate3.collect_events())
+
+        # And so using the aggregate to record new events will cause an IntegrityError.
+        aggregate4: Aggregate = app.repository.get(aggregate_id)
+        aggregate4.trigger_event(Aggregate.Event)
+        with self.assertRaises(IntegrityError):
+            app.save(aggregate4)
 
     def test_application_with_deepcopy_from_cache_arg(self) -> None:
         app = Application(

{eventsourcing-9.4.0b1 → eventsourcing-9.4.0b2}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "eventsourcing"
-version = "9.4.0b1"
+version = "9.4.0b2"
 
 description = "Event sourcing in Python"
 authors = [