python-event-sourcery 0.4.0a0__tar.gz → 0.4.0a1__tar.gz

{python_event_sourcery-0.4.0a0 → python_event_sourcery-0.4.0a1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: python-event-sourcery
-Version: 0.4.0a0
+Version: 0.4.0a1
 Summary: 
 License: MIT
 Author: Your Name
@@ -10,13 +10,12 @@ Classifier: License :: OSI Approved :: MIT License
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
-Provides-Extra: cloudevents
+Classifier: Programming Language :: Python :: 3.12
 Provides-Extra: django
-Provides-Extra: kombu
-Requires-Dist: SQLAlchemy (>=2.0)
-Requires-Dist: cloudevents ; extra == "cloudevents"
+Provides-Extra: esdb
+Provides-Extra: sqlalchemy
+Requires-Dist: SQLAlchemy (>=2.0) ; extra == "sqlalchemy"
 Requires-Dist: django (>=4,<6) ; extra == "django"
-Requires-Dist: esdbclient (>=1.0b3)
-Requires-Dist: kombu ; extra == "kombu"
+Requires-Dist: esdbclient (>=1.0b3) ; extra == "esdb"
 Requires-Dist: more-itertools
 Requires-Dist: pydantic (>=2.0,<3)

{python_event_sourcery-0.4.0a0 → python_event_sourcery-0.4.0a1}/event_sourcery_sqlalchemy/__init__.py

@@ -7,6 +7,7 @@ __all__ = [
 ]
 
 from dataclasses import dataclass
+from datetime import timedelta
 
 from pydantic import BaseModel, ConfigDict, PositiveInt
 from sqlalchemy.orm import Session
@@ -36,6 +37,7 @@ class Config(BaseModel):
     model_config = ConfigDict(extra="forbid", frozen=True)
 
     outbox_attempts: PositiveInt = 3
+    gap_retry_interval: timedelta = timedelta(seconds=0.5)
 
 
 @dataclass(repr=False)
@@ -63,7 +65,9 @@ class SQLAlchemyBackendFactory(BackendFactory):
         )
         backend.subscriber = es.subscription.SubscriptionBuilder(
             _serde=backend.serde,
-            _strategy=SqlAlchemySubscriptionStrategy(),
+            _strategy=SqlAlchemySubscriptionStrategy(
+                self._session, self._config.gap_retry_interval
+            ),
         )
         return backend
 

python_event_sourcery-0.4.0a1/event_sourcery_sqlalchemy/dto.py

@@ -0,0 +1,18 @@
+from event_sourcery.event_store import RawEvent, StreamId
+from event_sourcery_sqlalchemy.models import Event, Stream
+
+
+def raw_event(from_entry: Event, in_stream: Stream) -> RawEvent:
+    return RawEvent(
+        uuid=from_entry.uuid,
+        stream_id=StreamId(
+            uuid=in_stream.uuid,
+            name=in_stream.name,
+            category=None if in_stream.category == "" else in_stream.category,
+        ),
+        created_at=from_entry.created_at,
+        version=from_entry.version,
+        name=from_entry.name,
+        data=from_entry.data,
+        context=from_entry.event_context,
+    )

{python_event_sourcery-0.4.0a0 → python_event_sourcery-0.4.0a1}/event_sourcery_sqlalchemy/event_store.py

@@ -2,7 +2,7 @@ from dataclasses import dataclass
 from typing import Sequence, Union, cast
 
 from more_itertools import first_true
-from sqlalchemy import delete, select, update
+from sqlalchemy import delete, func, select, update
 from sqlalchemy.dialects.postgresql import insert as postgresql_insert
 from sqlalchemy.exc import NoResultFound
 from sqlalchemy.orm import Session
@@ -221,4 +221,6 @@ class SqlAlchemyStorageStrategy(StorageStrategy):
 
     @property
     def current_position(self) -> Position | None:
-        raise NotImplementedError
+        stmt = select(func.max(EventModel.id))
+        last_event = self._session.scalar(stmt)
+        return last_event or Position(0)

python_event_sourcery-0.4.0a1/event_sourcery_sqlalchemy/subscription.py

@@ -0,0 +1,176 @@
+import time
+from dataclasses import dataclass
+from datetime import timedelta
+from typing import Iterator, Protocol, cast
+
+from sqlalchemy import select
+from sqlalchemy.orm import Session
+
+from event_sourcery.event_store import Position, RecordedRaw
+from event_sourcery.event_store.interfaces import SubscriptionStrategy
+from event_sourcery_sqlalchemy import dto, models
+
+
+class SqlAlchemySubscriptionStrategy(SubscriptionStrategy):
+    def __init__(self, session: Session, gap_retry_interval: timedelta) -> None:
+        self._session = session
+        self._gap_retry_interval = gap_retry_interval
+
+    def subscribe_to_all(
+        self,
+        start_from: Position,
+        batch_size: int,
+        timelimit: timedelta,
+    ) -> Iterator[list[RecordedRaw]]:
+        return GapDetectingIterator(
+            get_batch=GetBatchToAll(self._session, batch_size),
+            gap_retry_interval=self._gap_retry_interval,
+            start_from=start_from,
+            batch_size=batch_size,
+            timelimit=timelimit,
+        )
+
+    def subscribe_to_category(
+        self,
+        start_from: Position,
+        batch_size: int,
+        timelimit: timedelta,
+        category: str,
+    ) -> Iterator[list[RecordedRaw]]:
+        return GapDetectingIterator(
+            get_batch=GetBatchToCategory(self._session, batch_size, category),
+            gap_retry_interval=self._gap_retry_interval,
+            start_from=start_from,
+            batch_size=batch_size,
+            timelimit=timelimit,
+        )
+
+    def subscribe_to_events(
+        self,
+        start_from: Position,
+        batch_size: int,
+        timelimit: timedelta,
+        events: list[str],
+    ) -> Iterator[list[RecordedRaw]]:
+        return GapDetectingIterator(
+            get_batch=GetBatchToEvents(self._session, batch_size, events),
+            gap_retry_interval=self._gap_retry_interval,
+            start_from=start_from,
+            batch_size=batch_size,
+            timelimit=timelimit,
+        )
+
+
+class GetBatch(Protocol):
+    def __call__(self, position: Position) -> list[models.Event]:
+        ...
+
+
+class GetBatchToAll(GetBatch):
+    def __init__(self, session: Session, batch_size: int) -> None:
+        self._session = session
+        self._batch_size = batch_size
+
+    def __call__(self, position: Position) -> list[models.Event]:
+        stmt = (
+            select(models.Event)
+            .join(models.Stream)
+            .where(models.Event.id > position)
+            .order_by(models.Event.id)
+            .limit(self._batch_size)
+        )
+
+        return list(self._session.scalars(stmt).all())
+
+
+class GetBatchToCategory(GetBatch):
+    def __init__(self, session: Session, batch_size: int, category: str) -> None:
+        self._session = session
+        self._batch_size = batch_size
+        self._category = category
+
+    def __call__(self, position: Position) -> list[models.Event]:
+        stmt = (
+            select(models.Event)
+            .join(models.Stream)
+            .where(models.Stream.category == self._category)
+            .where(models.Event.id > position)
+            .order_by(models.Event.id)
+            .limit(self._batch_size)
+        )
+
+        return list(self._session.scalars(stmt).all())
+
+
+class GetBatchToEvents(GetBatch):
+    def __init__(self, session: Session, batch_size: int, events: list[str]) -> None:
+        self._session = session
+        self._batch_size = batch_size
+        self._events = events
+
+    def __call__(self, position: Position) -> list[models.Event]:
+        stmt = (
+            select(models.Event)
+            .join(models.Stream)
+            .where(models.Event.name.in_(self._events))
+            .where(models.Event.id > position)
+            .order_by(models.Event.id)
+            .limit(self._batch_size)
+        )
+
+        return list(self._session.scalars(stmt).all())
+
+
+@dataclass
+class Cursor:
+    position: Position
+
+    def advance(self, batch: list[models.Event]) -> None:
+        if len(batch) > 0:
+            self.position = batch[-1].id
+
+
+class GapDetectingIterator(Iterator[list[RecordedRaw]]):
+    def __init__(
+        self,
+        get_batch: GetBatch,
+        gap_retry_interval: timedelta,
+        start_from: Position,
+        batch_size: int,
+        timelimit: timedelta,
+    ) -> None:
+        self._get_batch = get_batch
+        self._gap_retry_interval = gap_retry_interval
+        self._cursor = Cursor(position=start_from)
+        self._batch_size = batch_size
+        self._timelimit = timelimit
+
+    def __next__(self) -> list[RecordedRaw]:
+        start = time.monotonic()
+        while True:
+            batch = self._get_batch(self._cursor.position)
+            if self._is_continuous(batch) and len(batch) == self._batch_size:
+                self._cursor.advance(batch)
+                return self._batch_to_recorded_raw(batch)
+            elif time.monotonic() - start > self._timelimit.total_seconds():
+                self._cursor.advance(batch)
+                return self._batch_to_recorded_raw(batch)
+            else:
+                time.sleep(self._gap_retry_interval.total_seconds())
+
+    @staticmethod
+    def _is_continuous(batch: list[models.Event]) -> bool:
+        if len(batch) < 2:
+            return False
+
+        return cast(bool, batch[-1].id - batch[0].id + 1 == len(batch))
+
+    @staticmethod
+    def _batch_to_recorded_raw(batch: list[models.Event]) -> list[RecordedRaw]:
+        return [
+            RecordedRaw(
+                entry=dto.raw_event(event, event.stream),
+                position=event.id,
+            )
+            for event in batch
+        ]

{python_event_sourcery-0.4.0a0 → python_event_sourcery-0.4.0a1}/pyproject.toml

@@ -1,24 +1,21 @@
 [tool.poetry]
 name = "python-event-sourcery"
-version = "0.4.0a0"
+version = "0.4.0a1"
 description = ""
 authors = ["Your Name <you@example.com>"]
 license = "MIT"
 packages = [
     { include = "event_sourcery" },
     { include = "event_sourcery_esdb" },
-    { include = "event_sourcery_kombu" },
     { include = "event_sourcery_sqlalchemy" },
 ]
 
 [tool.poetry.dependencies]
 python = ">=3.10,<4.0"
 pydantic = ">=2.0,<3"
-SQLAlchemy = ">=2.0"
-esdbclient = ">=1.0b3"
-kombu = {version = "*", optional = true}
-cloudevents = {version = "*", optional = true}
 more-itertools = "*"
+SQLAlchemy = {version = ">=2.0", optional = true}
+esdbclient = {version = ">=1.0b3", optional = true}
 django = {version = ">=4,<6", optional = true}
 
 [tool.poetry.group.dev.dependencies]
@@ -32,10 +29,11 @@ pytest-cov = "*"
 mkdocs-material = "^8.3.9"
 psycopg2-binary = "^2.9.3"
 pytest-django = {version = "*"}
+flake8-pyproject = "*"
 
 [tool.poetry.extras]
-kombu = ["kombu"]
-cloudevents = ["cloudevents"]
+sqlalchemy = ["SQLAlchemy"]
+esdb = ["esdbclient"]
 django = ["django"]
 
 [tool.pytest]
@@ -57,7 +55,6 @@ source = [
     "event_sourcery_sqlalchemy",
 ]
 omit = [
-    "event_sourcery_kombu/*",
     "tests/*",
 ]
 
@@ -75,6 +72,22 @@ exclude_lines = [
     "TYPE_CHECKING",
 ]
 
+[tool.mypy]
+python_version = "3.10"
+ignore_missing_imports = true
+disallow_untyped_defs = true
+no_implicit_optional = true
+check_untyped_defs = true
+warn_return_any = true
+warn_unused_ignores = true
+show_error_codes = true
+
+[tool.flake8]
+max-line-length = 88
+
+[tool.isort]
+profile = "black"
+
 [build-system]
 requires = ["poetry-core>=1.0.0"]
 build-backend = "poetry.core.masonry.api"

python_event_sourcery-0.4.0a0/event_sourcery_kombu/__init__.py

@@ -1,155 +0,0 @@
-from dataclasses import dataclass, field
-from typing import Callable, Type
-
-from kombu import Connection, Exchange, Message, Queue
-from kombu.connection import ConnectionPool
-from kombu.pools import connections
-from pydantic import AmqpDsn, Field
-from pydantic_settings import BaseSettings
-
-__all__ = ["publish_event", "declare_exchange"]
-
-from event_sourcery.event_store import Event, Metadata, StreamId
-from event_sourcery.event_store.event.registry import event_name
-
-
-class BrokerSettings(BaseSettings):
-    BROKER_URL: AmqpDsn = Field(default="amqp://guest:guest@rabbitmq//")
-
-
-class PoolFactory:
-    _pool: ConnectionPool | None = None
-    _config: BrokerSettings | None = None
-
-    @classmethod
-    def configure(cls, broker_url: str) -> None:
-        if cls._pool is not None:
-            cls._pool.force_close_all()
-            cls._pool = None
-
-        cls._config = BrokerSettings(BROKER_URL=broker_url)
-
-    @classmethod
-    def get(cls) -> ConnectionPool:
-        if cls._config is None:
-            cls._config = BrokerSettings()
-
-        if cls._pool is None:
-            connection = Connection(
-                cls._config.BROKER_URL, transport_options={"confirm_publish": True}
-            )
-            cls._pool = connections[connection]
-        return cls._pool
-
-
-EVENT_SOURCERY_EXCHANGE = "event_sourcery.events"
-
-
-def publish_event(
-    metadata: Metadata, stream_name: str | None, stream_id: StreamId
-) -> None:
-    name = event_name(type(metadata.event))
-    stream_category = stream_name.split(".", maxsplit=1)[0] if stream_name else None
-    _publish(
-        routing_key=name,
-        headers={
-            "event": name,
-            "stream_name": stream_name,
-            "stream_category": stream_category,
-            "stream_id": str(stream_id),
-        },
-        message=metadata.json(),
-        exchange=EVENT_SOURCERY_EXCHANGE,
-    )
-
-
-def declare_exchange() -> None:
-    _declare(Exchange(name=EVENT_SOURCERY_EXCHANGE, type="headers"))
-
-
-def _publish(
-    routing_key: str,
-    message: str,
-    exchange: str,
-    headers: dict,
-) -> None:
-    with PoolFactory.get().acquire(block=True) as conn:
-        producer = conn.Producer()
-        producer.publish(
-            message,
-            exchange=exchange,
-            routing_key=routing_key,
-            headers=headers,
-            content_type="application/json",
-        )
-
-
-def _declare(queue_or_exchange: Exchange | Queue) -> None:
-    with PoolFactory.get().acquire(block=True) as conn:
-        queue_or_exchange(conn).declare()
-
-
-class InvalidSubscription(Exception):
-    pass
-
-
-@dataclass(frozen=True)
-class Subscription:
-    event_types: list[Type[Event]] = field(default_factory=list)
-    stream_categories: list[str] = field(default_factory=list)
-
-    def __post_init__(self) -> None:
-        if len(self.event_types) == 0 and len(self.stream_categories) == 0:
-            raise InvalidSubscription(
-                "At least one event type or stream category needs to be specified"
-            )
-
-
-def consume(
-    subscription: Subscription,
-    listener: Callable[[Metadata, StreamId], None],
-) -> None:
-    queue = Queue(f"event_sourcery.{listener.__name__}", durable=True)
-    _declare(queue)
-
-    def callback(body: dict, message: Message) -> None:
-        event_name = message.headers["event"]
-        event_type = Event.__registry__.type_for_name(event_name)
-
-        stream_id = StreamId(
-            from_hex=message.headers["stream_id"],
-            name=message.headers["stream_name"],
-        )
-        metadata = Metadata[event_type](**body)  # type: ignore
-
-        try:
-            listener(metadata, stream_id)
-        except Exception:  # TODO: DLQ, retries, error handling etc
-            message.reject()
-            raise
-        else:
-            message.ack()
-
-    with PoolFactory.get().acquire(block=True) as conn:
-        for event_type in subscription.event_types:
-            queue(conn).bind_to(
-                exchange=EVENT_SOURCERY_EXCHANGE,
-                arguments={
-                    "event": event_name(event_type),
-                },
-            )
-
-        for stream_category in subscription.stream_categories:
-            queue(conn).bind_to(
-                exchange=EVENT_SOURCERY_EXCHANGE,
-                arguments={
-                    "stream_category": stream_category,
-                },
-            )
-
-        with conn.Consumer([queue], callbacks=[callback]):
-            while True:
-                try:
-                    conn.drain_events()
-                except KeyboardInterrupt:
-                    return

python_event_sourcery-0.4.0a0/event_sourcery_sqlalchemy/subscription.py

@@ -1,33 +0,0 @@
-from datetime import timedelta
-from typing import Iterator
-
-from event_sourcery.event_store import Position, RecordedRaw
-from event_sourcery.event_store.interfaces import SubscriptionStrategy
-
-
-class SqlAlchemySubscriptionStrategy(SubscriptionStrategy):
-    def subscribe_to_all(
-        self,
-        start_from: Position,
-        batch_size: int,
-        timelimit: timedelta,
-    ) -> Iterator[list[RecordedRaw]]:
-        raise NotImplementedError
-
-    def subscribe_to_category(
-        self,
-        start_from: Position,
-        batch_size: int,
-        timelimit: timedelta,
-        category: str,
-    ) -> Iterator[list[RecordedRaw]]:
-        raise NotImplementedError
-
-    def subscribe_to_events(
-        self,
-        start_from: Position,
-        batch_size: int,
-        timelimit: timedelta,
-        events: list[str],
-    ) -> Iterator[list[RecordedRaw]]:
-        raise NotImplementedError