python-event-sourcery 0.4.0a13__tar.gz → 0.4.0a14__tar.gz

{python_event_sourcery-0.4.0a13 → python_event_sourcery-0.4.0a14}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: python-event-sourcery
-Version: 0.4.0a13
+Version: 0.4.0a14
 Summary: 
 License: MIT
 Author: Your Name
@@ -19,4 +19,5 @@ Requires-Dist: SQLAlchemy (>=2.0) ; extra == "sqlalchemy"
 Requires-Dist: django (>=4,<6) ; extra == "django"
 Requires-Dist: esdbclient (>=1.0b3) ; extra == "esdb"
 Requires-Dist: more-itertools
+Requires-Dist: psycopg2-binary (>2,<3)
 Requires-Dist: pydantic (>=2.0,<3)

{python_event_sourcery-0.4.0a13 → python_event_sourcery-0.4.0a14}/event_sourcery/event_store/__init__.py

@@ -1,30 +1,31 @@
 __all__ = [
+    "DEFAULT_TENANT",
+    "NO_VERSIONING",
     "Backend",
     "BackendFactory",
+    "Context",
     "Dispatcher",
     "Entry",
     "Event",
-    "Context",
     "EventRegistry",
     "EventStore",
     "ExplicitVersioning",
     "InMemoryBackendFactory",
     "Listener",
-    "WrappedEvent",
-    "NO_VERSIONING",
     "Position",
     "RawEvent",
     "Recorded",
     "RecordedRaw",
     "StreamId",
     "StreamUUID",
+    "TenantId",
     "TransactionalBackend",
     "Versioning",
+    "WrappedEvent",
     "exceptions",
     "factory",
     "interfaces",
     "subscription",
-    "TenantId",
 ]
 
 from event_sourcery.event_store import exceptions, factory, interfaces, subscription
@@ -48,7 +49,7 @@ from event_sourcery.event_store.factory import (
 )
 from event_sourcery.event_store.in_memory import InMemoryBackendFactory
 from event_sourcery.event_store.stream_id import StreamId, StreamUUID
-from event_sourcery.event_store.tenant_id import TenantId
+from event_sourcery.event_store.tenant_id import DEFAULT_TENANT, TenantId
 from event_sourcery.event_store.versioning import (
     NO_VERSIONING,
     ExplicitVersioning,

{python_event_sourcery-0.4.0a13 → python_event_sourcery-0.4.0a14}/event_sourcery/event_store/event/dto.py

@@ -92,5 +92,10 @@ class Entry:
 
 @dataclasses.dataclass(frozen=True)
 class Recorded(Entry):
+    """DTO containing events saved in the event store.
+
+    It contains position of the event in the event store and tenant id.
+    """
+
     position: Position
     tenant_id: TenantId = DEFAULT_TENANT

{python_event_sourcery-0.4.0a13 → python_event_sourcery-0.4.0a14}/event_sourcery/event_store/stream_id.py

@@ -32,7 +32,7 @@ class StreamUUID(UUID):
         return uuid5(self.NAMESPACE, name)
 
     def __repr__(self) -> str:
-        return f"{type(self).__name__}" f"(hex={self!s}, name={self.name})"
+        return f"{type(self).__name__}(hex={self!s}, name={self.name})"
 
 
 @dataclass(frozen=True, repr=False, eq=False)

python_event_sourcery-0.4.0a14/event_sourcery_django/__init__.py

@@ -0,0 +1,80 @@
+__all__ = [
+    "Config",
+    "DjangoBackendFactory",
+]
+
+from dataclasses import dataclass, field
+from datetime import timedelta
+from typing import cast
+
+from pydantic import BaseModel, ConfigDict, PositiveInt
+from typing_extensions import Self
+
+from event_sourcery import event_store as es
+from event_sourcery.event_store import (
+    BackendFactory,
+    Dispatcher,
+    Event,
+    EventRegistry,
+    EventStore,
+)
+from event_sourcery.event_store.event import Serde
+from event_sourcery.event_store.factory import (
+    NoOutboxStorageStrategy,
+    TransactionalBackend,
+    no_filter,
+)
+from event_sourcery.event_store.interfaces import (
+    OutboxFiltererStrategy,
+    OutboxStorageStrategy,
+)
+from event_sourcery.event_store.outbox import Outbox
+
+
+class Config(BaseModel):
+    model_config = ConfigDict(extra="forbid", frozen=True)
+
+    outbox_attempts: PositiveInt = 3
+    gap_retry_interval: timedelta = timedelta(seconds=0.5)
+
+
+@dataclass(repr=False)
+class DjangoBackendFactory(BackendFactory):
+    _config: Config = field(default_factory=Config)
+    _serde: Serde = field(default_factory=lambda: Serde(Event.__registry__))
+    _outbox_strategy: OutboxStorageStrategy | None = None
+
+    def build(self) -> TransactionalBackend:
+        from event_sourcery_django.event_store import DjangoStorageStrategy
+        from event_sourcery_django.outbox import DjangoOutboxStorageStrategy
+        from event_sourcery_django.subscription import DjangoSubscriptionStrategy
+
+        outbox = cast(DjangoOutboxStorageStrategy | None, self._outbox_strategy)
+        backend = TransactionalBackend()
+        backend.serde = self._serde
+        backend.in_transaction = Dispatcher(backend.serde)
+        storage_strategy = DjangoStorageStrategy(backend.in_transaction, outbox)
+        backend.event_store = EventStore(storage_strategy, backend.serde)
+        backend.outbox = Outbox(outbox or NoOutboxStorageStrategy(), backend.serde)
+        backend.subscriber = es.subscription.SubscriptionBuilder(
+            _serde=backend.serde,
+            _strategy=DjangoSubscriptionStrategy(self._config.gap_retry_interval),
+        )
+        return backend
+
+    def with_event_registry(self, event_registry: EventRegistry) -> Self:
+        self._serde = Serde(event_registry)
+        return self
+
+    def with_outbox(self, filterer: OutboxFiltererStrategy = no_filter) -> Self:
+        from event_sourcery_django.outbox import DjangoOutboxStorageStrategy
+
+        self._outbox_strategy = DjangoOutboxStorageStrategy(
+            filterer,
+            self._config.outbox_attempts,
+        )
+        return self
+
+    def without_outbox(self, filterer: OutboxFiltererStrategy = no_filter) -> Self:
+        self._outbox_strategy = None
+        return self

python_event_sourcery-0.4.0a14/event_sourcery_django/dto.py

@@ -0,0 +1,83 @@
+from datetime import datetime, timezone
+from uuid import UUID
+
+from event_sourcery.event_store import RawEvent, RecordedRaw, StreamId
+from event_sourcery_django.models import Event, OutboxEntry, Snapshot, Stream
+
+
+def raw_event(from_entry: Event, in_stream: Stream) -> RawEvent:
+    return RawEvent(
+        uuid=from_entry.uuid,
+        stream_id=StreamId(
+            uuid=in_stream.uuid,
+            name=in_stream.name,
+            category=None if in_stream.category == "" else in_stream.category,
+        ),
+        created_at=from_entry.created_at,
+        version=from_entry.version,
+        name=from_entry.name,
+        data=from_entry.data,
+        context=from_entry.event_context,
+    )
+
+
+def entry(from_raw: RawEvent, to_stream: Stream) -> Event:
+    return Event(
+        uuid=from_raw.uuid,
+        created_at=from_raw.created_at,
+        name=from_raw.name,
+        data=from_raw.data,
+        event_context=from_raw.context,
+        version=from_raw.version,
+        stream=to_stream,
+    )
+
+
+def snapshot(from_raw: RawEvent, to_stream: Stream) -> Snapshot:
+    return Snapshot(
+        uuid=from_raw.uuid,
+        created_at=from_raw.created_at,
+        name=from_raw.name,
+        data=from_raw.data,
+        event_context=from_raw.context,
+        version=from_raw.version,
+        stream=to_stream,
+    )
+
+
+def outbox_entry(from_raw: RecordedRaw, max_attempts: int) -> OutboxEntry:
+    return OutboxEntry(
+        created_at=datetime.now(timezone.utc).replace(tzinfo=None),
+        data={
+            "created_at": from_raw.entry.created_at.isoformat(),
+            "uuid": str(from_raw.entry.uuid),
+            "stream_id": str(from_raw.entry.stream_id),
+            "version": from_raw.entry.version,
+            "name": from_raw.entry.name,
+            "data": from_raw.entry.data,
+            "context": from_raw.entry.context,
+            "tenant_id": from_raw.tenant_id,
+        },
+        stream_name=from_raw.entry.stream_id.name,
+        position=from_raw.position,
+        tries_left=max_attempts,
+    )
+
+
+def raw_outbox(from_entry: OutboxEntry) -> RecordedRaw:
+    return RecordedRaw(
+        entry=RawEvent(
+            uuid=UUID(from_entry.data["uuid"]),
+            stream_id=StreamId(
+                from_hex=from_entry.data["stream_id"],
+                name=from_entry.stream_name,
+            ),
+            created_at=datetime.fromisoformat(from_entry.data["created_at"]),
+            version=from_entry.data["version"],
+            name=from_entry.data["name"],
+            data=from_entry.data["data"],
+            context=from_entry.data["context"],
+        ),
+        position=from_entry.position,
+        tenant_id=from_entry.data["tenant_id"],
+    )

python_event_sourcery-0.4.0a14/event_sourcery_django/event_store.py

@@ -0,0 +1,151 @@
+from collections.abc import Sequence
+from dataclasses import dataclass, replace
+from typing import cast
+
+from more_itertools import first, first_true
+from typing_extensions import Self
+
+from event_sourcery.event_store import (
+    NO_VERSIONING,
+    Dispatcher,
+    Position,
+    RawEvent,
+    RecordedRaw,
+    StreamId,
+    TenantId,
+    Versioning,
+)
+from event_sourcery.event_store.exceptions import (
+    AnotherStreamWithThisNameButOtherIdExists,
+    ConcurrentStreamWriteError,
+)
+from event_sourcery.event_store.interfaces import StorageStrategy
+from event_sourcery.event_store.tenant_id import DEFAULT_TENANT
+from event_sourcery_django import dto, models
+from event_sourcery_django.outbox import DjangoOutboxStorageStrategy
+
+
+@dataclass(repr=True)
+class DjangoStorageStrategy(StorageStrategy):
+    _dispatcher: Dispatcher
+    _outbox: DjangoOutboxStorageStrategy | None = None
+    _tenant_id: TenantId = DEFAULT_TENANT
+
+    def fetch_events(
+        self,
+        stream_id: StreamId,
+        start: int | None = None,
+        stop: int | None = None,
+    ) -> list[RawEvent]:
+        try:
+            stream = models.Stream.objects.by_stream_id(
+                stream_id=stream_id,
+                tenant_id=self._tenant_id,
+            ).get()
+        except models.Stream.DoesNotExist:
+            return []
+
+        events_query = models.Event.objects.filter(stream=stream).order_by("version")
+
+        if start is not None:
+            events_query = events_query.filter(version__gte=start)
+
+        if stop is not None:
+            events_query = events_query.filter(version__lt=stop)
+
+        events: Sequence[models.Event | models.Snapshot]
+
+        snapshot_query = models.Snapshot.objects.filter(stream=stream).order_by(
+            "-created_at"
+        )
+        if start is not None:
+            snapshot_query = snapshot_query.filter(version__gte=start)
+        if stop is not None:
+            snapshot_query = snapshot_query.filter(version__lt=stop)
+        latest_snapshot = snapshot_query.first()
+        if latest_snapshot is None:
+            events = events_query.all()
+        else:
+            newer_events = events_query.filter(
+                version__gt=latest_snapshot.version
+            ).all()
+            events = [latest_snapshot, *list(newer_events)]
+
+        return [dto.raw_event(event, stream) for event in events]
+
+    def insert_events(
+        self,
+        stream_id: StreamId,
+        versioning: Versioning,
+        events: list[RawEvent],
+    ) -> None:
+        self._ensure_stream(stream_id=stream_id, versioning=versioning)
+        event = cast(RawEvent, first(events))
+        stream = models.Stream.objects.by_stream_id(
+            stream_id=event.stream_id,
+            tenant_id=self._tenant_id,
+        ).get()
+        entries = [dto.entry(event, stream) for event in events]
+        models.Event.objects.bulk_create(entries)
+        records = [
+            RecordedRaw(entry=raw, position=db.id, tenant_id=self._tenant_id)
+            for raw, db in zip(events, entries, strict=False)
+        ]
+        if self._outbox:
+            self._outbox.put_into_outbox(records)
+        self._dispatcher.dispatch(*records)
+
+    def _ensure_stream(self, stream_id: StreamId, versioning: Versioning) -> None:
+        initial_version = versioning.initial_version
+
+        matching_streams = models.Stream.objects.by_stream_id(
+            stream_id=stream_id,
+            tenant_id=self._tenant_id,
+        ).all()
+        if stream_id.name and matching_streams:
+            stream_with_same_name = first_true(
+                matching_streams, pred=lambda stream: stream.name == stream_id.name
+            )
+            if (
+                stream_with_same_name is not None
+                and stream_with_same_name.uuid != stream_id
+            ):
+                raise AnotherStreamWithThisNameButOtherIdExists()
+
+        model, created = models.Stream.objects.get_or_create(
+            uuid=stream_id,
+            name=stream_id.name,
+            category=stream_id.category or "",
+            tenant_id=self._tenant_id,
+            defaults={"version": initial_version},
+        )
+
+        versioning.validate_if_compatible(model.version)
+
+        if versioning.expected_version and versioning is not NO_VERSIONING:
+            result = models.Stream.objects.filter(
+                id=model.id, version=versioning.expected_version
+            ).update(version=versioning.initial_version)
+            if result != 1:
+                raise ConcurrentStreamWriteError
+
+    def save_snapshot(self, snapshot: RawEvent) -> None:
+        stream = models.Stream.objects.by_stream_id(
+            stream_id=snapshot.stream_id,
+            tenant_id=self._tenant_id,
+        ).get()
+        entry = dto.snapshot(from_raw=snapshot, to_stream=stream)
+        entry.save()
+
+    def delete_stream(self, stream_id: StreamId) -> None:
+        models.Stream.objects.by_stream_id(
+            stream_id=stream_id, tenant_id=self._tenant_id
+        ).delete()
+
+    @property
+    def current_position(self) -> Position | None:
+        last_event = models.Event.objects.last()
+        return last_event.id if last_event else Position(0)
+
+    def scoped_for_tenant(self, tenant_id: TenantId) -> Self:
+        return replace(self, _tenant_id=tenant_id)

python_event_sourcery-0.4.0a14/event_sourcery_django/migrations/0001_initial.py

@@ -0,0 +1,95 @@
+# Generated by Django 5.0.4 on 2024-04-17 19:26
+
+import uuid
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    initial = True
+
+    dependencies: list = []
+
+    operations = [
+        migrations.CreateModel(
+            name="OutboxEntry",
+            fields=[
+                ("id", models.BigAutoField(primary_key=True, serialize=False)),
+                ("created_at", models.DateTimeField()),
+                ("data", models.JSONField()),
+                (
+                    "stream_name",
+                    models.CharField(blank=True, max_length=255, null=True),
+                ),
+                ("position", models.IntegerField()),
+                ("tries_left", models.IntegerField(default=3)),
+            ],
+        ),
+        migrations.CreateModel(
+            name="Stream",
+            fields=[
+                ("id", models.BigAutoField(primary_key=True, serialize=False)),
+                ("uuid", models.UUIDField(default=uuid.uuid4, editable=False)),
+                ("name", models.CharField(blank=True, max_length=255, null=True)),
+                ("category", models.CharField(default="", max_length=255)),
+                ("tenant_id", models.CharField(max_length=255)),
+                ("version", models.BigIntegerField(blank=True, null=True)),
+            ],
+            options={
+                "unique_together": {
+                    ("name", "category", "tenant_id"),
+                    ("uuid", "category", "tenant_id"),
+                },
+            },
+        ),
+        migrations.CreateModel(
+            name="Snapshot",
+            fields=[
+                ("uuid", models.UUIDField(primary_key=True, serialize=False)),
+                ("version", models.IntegerField()),
+                ("name", models.CharField(max_length=50)),
+                ("data", models.JSONField()),
+                ("event_context", models.JSONField()),
+                ("created_at", models.DateTimeField()),
+                (
+                    "stream",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="snapshots",
+                        to="event_sourcery_django.stream",
+                    ),
+                ),
+            ],
+        ),
+        migrations.CreateModel(
+            name="Event",
+            fields=[
+                ("id", models.BigAutoField(primary_key=True, serialize=False)),
+                ("version", models.IntegerField(blank=True, null=True)),
+                (
+                    "uuid",
+                    models.UUIDField(default=uuid.uuid4, editable=False, unique=True),
+                ),
+                ("name", models.CharField(max_length=200)),
+                ("data", models.JSONField()),
+                ("event_context", models.JSONField()),
+                ("created_at", models.DateTimeField()),
+                (
+                    "stream",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="events",
+                        to="event_sourcery_django.stream",
+                    ),
+                ),
+            ],
+            options={
+                "indexes": [
+                    models.Index(
+                        fields=["stream", "version"], name="ix_events_stream_id_version"
+                    )
+                ],
+            },
+        ),
+    ]

python_event_sourcery-0.4.0a14/event_sourcery_django/models.py

@@ -0,0 +1,86 @@
+from typing import ClassVar
+from uuid import uuid4
+
+from django.core.exceptions import ObjectDoesNotExist
+from django.db import models
+
+from event_sourcery.event_store import StreamId, TenantId
+
+
+class StreamManager(models.Manager):
+    def by_stream_id(self, stream_id: StreamId, tenant_id: TenantId) -> models.QuerySet:
+        category = stream_id.category or ""
+        condition = models.Q(uuid=stream_id, category=category, tenant_id=tenant_id)
+        if stream_id.name:
+            condition = condition | models.Q(
+                name=stream_id.name,
+                category=category,
+                tenant_id=tenant_id,
+            )
+        return self.filter(condition)
+
+
+class Stream(models.Model):
+    DoesNotExist: ClassVar[ObjectDoesNotExist]
+
+    id = models.BigAutoField(primary_key=True)
+    uuid = models.UUIDField(default=uuid4, editable=False)
+    name = models.CharField(max_length=255, null=True, blank=True)
+    category = models.CharField(max_length=255, default="")
+    tenant_id = models.CharField(max_length=255)
+    version = models.BigIntegerField(null=True, blank=True)
+
+    objects = StreamManager()
+
+    class Meta:
+        unique_together = (
+            ("uuid", "category", "tenant_id"),
+            ("name", "category", "tenant_id"),
+        )
+
+
+class Event(models.Model):
+    objects: models.Manager
+
+    id = models.BigAutoField(primary_key=True)
+    version = models.IntegerField(null=True, blank=True)
+    uuid = models.UUIDField(default=uuid4, editable=False, unique=True)
+    name = models.CharField(max_length=200)
+    data = models.JSONField()
+    event_context = models.JSONField()
+    created_at = models.DateTimeField()
+    stream = models.ForeignKey(Stream, related_name="events", on_delete=models.CASCADE)
+
+    class Meta:
+        indexes = [
+            models.Index(
+                fields=["stream", "version"], name="ix_events_stream_id_version"
+            ),
+        ]
+
+
+class Snapshot(models.Model):
+    objects: models.Manager
+
+    uuid = models.UUIDField(primary_key=True)
+    version = models.IntegerField()
+    stream = models.ForeignKey(
+        Stream,
+        related_name="snapshots",
+        on_delete=models.CASCADE,
+    )
+    name = models.CharField(max_length=50)
+    data = models.JSONField()
+    event_context = models.JSONField()
+    created_at = models.DateTimeField()
+
+
+class OutboxEntry(models.Model):
+    objects: models.Manager
+
+    id = models.BigAutoField(primary_key=True)
+    created_at = models.DateTimeField()
+    data = models.JSONField()
+    stream_name = models.CharField(max_length=255, null=True, blank=True)
+    position = models.BigIntegerField()
+    tries_left = models.IntegerField()

python_event_sourcery-0.4.0a14/event_sourcery_django/outbox.py

@@ -0,0 +1,51 @@
+import logging
+from collections.abc import Iterator
+from contextlib import AbstractContextManager, contextmanager
+from dataclasses import dataclass
+
+from event_sourcery.event_store import RecordedRaw
+from event_sourcery.event_store.interfaces import (
+    OutboxFiltererStrategy,
+    OutboxStorageStrategy,
+)
+from event_sourcery_django import dto
+from event_sourcery_django.models import OutboxEntry
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass(repr=False)
+class DjangoOutboxStorageStrategy(OutboxStorageStrategy):
+    _filterer: OutboxFiltererStrategy
+    _max_publish_attempts: int
+
+    def put_into_outbox(self, records: list[RecordedRaw]) -> None:
+        OutboxEntry.objects.bulk_create(
+            dto.outbox_entry(record, self._max_publish_attempts)
+            for record in records
+            if self._filterer(record.entry)
+        )
+
+    def outbox_entries(
+        self, limit: int
+    ) -> Iterator[AbstractContextManager[RecordedRaw]]:
+        entries = (
+            OutboxEntry.objects.select_for_update(skip_locked=True)
+            .filter(tries_left__gt=0)
+            .order_by("id")[:limit]
+        )
+
+        for entry in entries:
+            yield self._publish_context(entry)
+
+    @contextmanager
+    def _publish_context(self, entry: OutboxEntry) -> Iterator[RecordedRaw]:
+        raw = dto.raw_outbox(entry)
+        try:
+            yield raw
+        except Exception:
+            logger.exception("Failed to publish message #%d", entry.id)
+            entry.tries_left -= 1
+            entry.save()
+        else:
+            entry.delete()

python_event_sourcery-0.4.0a14/event_sourcery_django/subscription.py

@@ -0,0 +1,164 @@
+import time
+from collections.abc import Iterator
+from dataclasses import dataclass
+from datetime import timedelta
+from typing import Protocol, cast
+
+from event_sourcery.event_store import Position, RecordedRaw
+from event_sourcery.event_store.interfaces import SubscriptionStrategy
+from event_sourcery_django import dto, models
+
+
+class DjangoSubscriptionStrategy(SubscriptionStrategy):
+    def __init__(self, gap_retry_interval: timedelta) -> None:
+        self._gap_retry_interval = gap_retry_interval
+
+    def subscribe_to_all(
+        self,
+        start_from: Position,
+        batch_size: int,
+        timelimit: timedelta,
+    ) -> Iterator[list[RecordedRaw]]:
+        return GapDetectingIterator(
+            get_batch=GetBatchToAll(batch_size),
+            gap_retry_interval=self._gap_retry_interval,
+            start_from=start_from,
+            batch_size=batch_size,
+            timelimit=timelimit,
+        )
+
+    def subscribe_to_category(
+        self,
+        start_from: Position,
+        batch_size: int,
+        timelimit: timedelta,
+        category: str,
+    ) -> Iterator[list[RecordedRaw]]:
+        return GapDetectingIterator(
+            get_batch=GetBatchToCategory(batch_size, category),
+            gap_retry_interval=self._gap_retry_interval,
+            start_from=start_from,
+            batch_size=batch_size,
+            timelimit=timelimit,
+        )
+
+    def subscribe_to_events(
+        self,
+        start_from: Position,
+        batch_size: int,
+        timelimit: timedelta,
+        events: list[str],
+    ) -> Iterator[list[RecordedRaw]]:
+        return GapDetectingIterator(
+            get_batch=GetBatchToEvents(batch_size, events),
+            gap_retry_interval=self._gap_retry_interval,
+            start_from=start_from,
+            batch_size=batch_size,
+            timelimit=timelimit,
+        )
+
+
+class GetBatch(Protocol):
+    def __call__(self, position: Position) -> list[models.Event]: ...
+
+
+class GetBatchToAll(GetBatch):
+    def __init__(self, batch_size: int) -> None:
+        self._batch_size = batch_size
+
+    def __call__(self, position: Position) -> list[models.Event]:
+        query = (
+            models.Event.objects.filter(id__gt=position)
+            .select_related("stream")
+            .order_by("id")
+        )
+
+        return list(query[: self._batch_size])
+
+
+class GetBatchToCategory(GetBatch):
+    def __init__(self, batch_size: int, category: str) -> None:
+        self._batch_size = batch_size
+        self._category = category
+
+    def __call__(self, position: Position) -> list[models.Event]:
+        query = (
+            models.Event.objects.filter(
+                id__gt=position, stream__category=self._category
+            )
+            .select_related("stream")
+            .order_by("id")
+        )
+
+        return list(query[: self._batch_size])
+
+
+class GetBatchToEvents(GetBatch):
+    def __init__(self, batch_size: int, events: list[str]) -> None:
+        self._batch_size = batch_size
+        self._events = events
+
+    def __call__(self, position: Position) -> list[models.Event]:
+        query = (
+            models.Event.objects.filter(id__gt=position, name__in=self._events)
+            .select_related("stream")
+            .order_by("id")
+        )
+
+        return list(query[: self._batch_size])
+
+
+@dataclass
+class Cursor:
+    position: Position
+
+    def advance(self, batch: list[models.Event]) -> None:
+        if len(batch) > 0:
+            self.position = batch[-1].id
+
+
+class GapDetectingIterator(Iterator[list[RecordedRaw]]):
+    def __init__(
+        self,
+        get_batch: GetBatch,
+        gap_retry_interval: timedelta,
+        start_from: Position,
+        batch_size: int,
+        timelimit: timedelta,
+    ) -> None:
+        self._get_batch = get_batch
+        self._gap_retry_interval = gap_retry_interval
+        self._cursor = Cursor(position=start_from)
+        self._batch_size = batch_size
+        self._timelimit = timelimit
+
+    def __next__(self) -> list[RecordedRaw]:
+        start = time.monotonic()
+        while True:
+            batch = self._get_batch(self._cursor.position)
+            if self._is_continuous(batch) and len(batch) == self._batch_size:
+                self._cursor.advance(batch)
+                return self._batch_to_recorded_raw(batch)
+            elif time.monotonic() - start > self._timelimit.total_seconds():
+                self._cursor.advance(batch)
+                return self._batch_to_recorded_raw(batch)
+            else:
+                time.sleep(self._gap_retry_interval.total_seconds())
+
+    @staticmethod
+    def _is_continuous(batch: list[models.Event]) -> bool:
+        if len(batch) < 2:
+            return False
+
+        return cast(bool, batch[-1].id - batch[0].id + 1 == len(batch))
+
+    @staticmethod
+    def _batch_to_recorded_raw(batch: list[models.Event]) -> list[RecordedRaw]:
+        return [
+            RecordedRaw(
+                entry=dto.raw_event(event, event.stream),
+                position=event.id,
+                tenant_id=event.stream.tenant_id,
+            )
+            for event in batch
+        ]

{python_event_sourcery-0.4.0a13 → python_event_sourcery-0.4.0a14}/event_sourcery_esdb/event_store.py

@@ -74,11 +74,14 @@ class ESDBStorageStrategy(StorageStrategy):
             self._append_events(stream_name, events=stream_events)
 
     def _append_events(self, name: stream.Name, events: list[RawEvent]) -> int:
-        return self._client.append_events(
-            str(name),
-            current_version=StreamState.ANY,
-            events=(dto.new_entry(e) for e in events),
-            timeout=self._timeout,
+        return cast(
+            int,
+            self._client.append_events(
+                str(name),
+                current_version=StreamState.ANY,
+                events=(dto.new_entry(e) for e in events),
+                timeout=self._timeout,
+            ),
         )
 
     def save_snapshot(self, snapshot: RawEvent) -> None:

{python_event_sourcery-0.4.0a13 → python_event_sourcery-0.4.0a14}/event_sourcery_esdb/stream.py

@@ -56,5 +56,5 @@ def scope(
     start = cast(
         Position | None, start_version and Position.from_version(start_version)
     )
-    stop = stop_version and Position.from_version(stop_version) or MAX_POSITION
+    stop = (stop_version and Position.from_version(stop_version)) or MAX_POSITION
     return start, stop - (start or 0)

{python_event_sourcery-0.4.0a13 → python_event_sourcery-0.4.0a14}/event_sourcery_sqlalchemy/models.py

@@ -45,14 +45,14 @@ class StreamIdComparator(Comparator[StreamId]):
     ) -> None:
         super().__init__(Composite(uuid, name, category))
 
-    def __eq__(self, other: Any) -> ColumnElement[bool]:  # type: ignore[override]
+    def __eq__(self, other: Any) -> bool:  # type: ignore
         uuid, name, category = cast(Composite, self.__clause_element__()).attrs
         same_uuid = cast(ColumnElement[bool], uuid == other)
         same_name = cast(
             ColumnElement[bool], name == other.name if other.name else true()
         )
         same_category = cast(ColumnElement[bool], category == (other.category or ""))
-        return and_(same_uuid, same_name, same_category)
+        return and_(same_uuid, same_name, same_category)  # type: ignore
 
 
 class Stream:

{python_event_sourcery-0.4.0a13 → python_event_sourcery-0.4.0a14}/pyproject.toml

@@ -3,12 +3,13 @@ name = "python-event-sourcery"
 requires-python = ">=3.10"
 
 [tool.poetry]
-version = "0.4.0a13"
+version = "0.4.0a14"
 description = ""
 authors = ["Your Name <you@example.com>"]
 license = "MIT"
 packages = [
     { include = "event_sourcery" },
+    { include = "event_sourcery_django" },
     { include = "event_sourcery_esdb" },
     { include = "event_sourcery_sqlalchemy" },
 ]
@@ -20,6 +21,7 @@ more-itertools = "*"
 SQLAlchemy = {version = ">=2.0", optional = true}
 esdbclient = {version = ">=1.0b3", optional = true}
 django = {version = ">=4,<6", optional = true}
+psycopg2-binary = ">2,<3"
 
 [tool.poetry.group.dev.dependencies]
 ruff = "*"
@@ -30,6 +32,9 @@ pytest-django = {version = "*"}
 mkdocs-material = "^8.3.9"
 psycopg2-binary = "^2.9.3"
 mkdocstrings = {version = "*", extras = ["python"]}
+pymdown-extensions = "*"
+time-machine = "*"
+pika = "*"
 
 [tool.poetry.extras]
 sqlalchemy = ["SQLAlchemy"]
@@ -116,6 +121,11 @@ ignore = [
 "event_sourcery_django/models.py" = [
     "RUF012",  # Mutable class attributes should be annotated with typing.ClassVar
 ]
+"docs/documentation/code/*" = [
+   "S101",  # Use of assert detected
+   "N806",  # Variables in function should be lowercase
+   "B018",  # Useless expressions
+]
 
 [tool.ruff.lint.pylint]
 max-args = 10

{python_event_sourcery-0.4.0a13 → python_event_sourcery-0.4.0a14}/event_sourcery/event_store/event/__init__.py

@@ -3,12 +3,12 @@ __all__ = [
     "Entry",
     "Event",
     "EventRegistry",
-    "WrappedEvent",
     "Position",
     "RawEvent",
     "Recorded",
     "RecordedRaw",
     "Serde",
+    "WrappedEvent",
 ]
 
 

{python_event_sourcery-0.4.0a13 → python_event_sourcery-0.4.0a14}/event_sourcery_sqlalchemy/__init__.py

@@ -1,9 +1,9 @@
 __all__ = [
     "Config",
+    "SQLAlchemyBackendFactory",
+    "SqlAlchemyStorageStrategy",
     "configure_models",
     "models",
-    "SqlAlchemyStorageStrategy",
-    "SQLAlchemyBackendFactory",
 ]
 
 from dataclasses import dataclass, field