python-event-sourcery 0.4.0a0__py3-none-any.whl → 0.4.0a2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -7,6 +7,7 @@ __all__ = [
7
7
  ]
8
8
 
9
9
  from dataclasses import dataclass
10
+ from datetime import timedelta
10
11
 
11
12
  from pydantic import BaseModel, ConfigDict, PositiveInt
12
13
  from sqlalchemy.orm import Session
@@ -36,6 +37,7 @@ class Config(BaseModel):
36
37
  model_config = ConfigDict(extra="forbid", frozen=True)
37
38
 
38
39
  outbox_attempts: PositiveInt = 3
40
+ gap_retry_interval: timedelta = timedelta(seconds=0.5)
39
41
 
40
42
 
41
43
  @dataclass(repr=False)
@@ -63,7 +65,9 @@ class SQLAlchemyBackendFactory(BackendFactory):
63
65
  )
64
66
  backend.subscriber = es.subscription.SubscriptionBuilder(
65
67
  _serde=backend.serde,
66
- _strategy=SqlAlchemySubscriptionStrategy(),
68
+ _strategy=SqlAlchemySubscriptionStrategy(
69
+ self._session, self._config.gap_retry_interval
70
+ ),
67
71
  )
68
72
  return backend
69
73
 
@@ -0,0 +1,18 @@
1
+ from event_sourcery.event_store import RawEvent, StreamId
2
+ from event_sourcery_sqlalchemy.models import Event, Stream
3
+
4
+
5
+ def raw_event(from_entry: Event, in_stream: Stream) -> RawEvent:
6
+ return RawEvent(
7
+ uuid=from_entry.uuid,
8
+ stream_id=StreamId(
9
+ uuid=in_stream.uuid,
10
+ name=in_stream.name,
11
+ category=None if in_stream.category == "" else in_stream.category,
12
+ ),
13
+ created_at=from_entry.created_at,
14
+ version=from_entry.version,
15
+ name=from_entry.name,
16
+ data=from_entry.data,
17
+ context=from_entry.event_context,
18
+ )
@@ -2,7 +2,7 @@ from dataclasses import dataclass
2
2
  from typing import Sequence, Union, cast
3
3
 
4
4
  from more_itertools import first_true
5
- from sqlalchemy import delete, select, update
5
+ from sqlalchemy import delete, func, select, update
6
6
  from sqlalchemy.dialects.postgresql import insert as postgresql_insert
7
7
  from sqlalchemy.exc import NoResultFound
8
8
  from sqlalchemy.orm import Session
@@ -221,4 +221,6 @@ class SqlAlchemyStorageStrategy(StorageStrategy):
221
221
 
222
222
  @property
223
223
  def current_position(self) -> Position | None:
224
- raise NotImplementedError
224
+ stmt = select(func.max(EventModel.id))
225
+ last_event = self._session.scalar(stmt)
226
+ return last_event or Position(0)
@@ -1,18 +1,34 @@
1
+ import time
2
+ from dataclasses import dataclass
1
3
  from datetime import timedelta
2
- from typing import Iterator
4
+ from typing import Iterator, Protocol, cast
5
+
6
+ from sqlalchemy import select
7
+ from sqlalchemy.orm import Session
3
8
 
4
9
  from event_sourcery.event_store import Position, RecordedRaw
5
10
  from event_sourcery.event_store.interfaces import SubscriptionStrategy
11
+ from event_sourcery_sqlalchemy import dto, models
6
12
 
7
13
 
8
14
  class SqlAlchemySubscriptionStrategy(SubscriptionStrategy):
15
+ def __init__(self, session: Session, gap_retry_interval: timedelta) -> None:
16
+ self._session = session
17
+ self._gap_retry_interval = gap_retry_interval
18
+
9
19
  def subscribe_to_all(
10
20
  self,
11
21
  start_from: Position,
12
22
  batch_size: int,
13
23
  timelimit: timedelta,
14
24
  ) -> Iterator[list[RecordedRaw]]:
15
- raise NotImplementedError
25
+ return GapDetectingIterator(
26
+ get_batch=GetBatchToAll(self._session, batch_size),
27
+ gap_retry_interval=self._gap_retry_interval,
28
+ start_from=start_from,
29
+ batch_size=batch_size,
30
+ timelimit=timelimit,
31
+ )
16
32
 
17
33
  def subscribe_to_category(
18
34
  self,
@@ -21,7 +37,13 @@ class SqlAlchemySubscriptionStrategy(SubscriptionStrategy):
21
37
  timelimit: timedelta,
22
38
  category: str,
23
39
  ) -> Iterator[list[RecordedRaw]]:
24
- raise NotImplementedError
40
+ return GapDetectingIterator(
41
+ get_batch=GetBatchToCategory(self._session, batch_size, category),
42
+ gap_retry_interval=self._gap_retry_interval,
43
+ start_from=start_from,
44
+ batch_size=batch_size,
45
+ timelimit=timelimit,
46
+ )
25
47
 
26
48
  def subscribe_to_events(
27
49
  self,
@@ -30,4 +52,125 @@ class SqlAlchemySubscriptionStrategy(SubscriptionStrategy):
30
52
  timelimit: timedelta,
31
53
  events: list[str],
32
54
  ) -> Iterator[list[RecordedRaw]]:
33
- raise NotImplementedError
55
+ return GapDetectingIterator(
56
+ get_batch=GetBatchToEvents(self._session, batch_size, events),
57
+ gap_retry_interval=self._gap_retry_interval,
58
+ start_from=start_from,
59
+ batch_size=batch_size,
60
+ timelimit=timelimit,
61
+ )
62
+
63
+
64
+ class GetBatch(Protocol):
65
+ def __call__(self, position: Position) -> list[models.Event]:
66
+ ...
67
+
68
+
69
+ class GetBatchToAll(GetBatch):
70
+ def __init__(self, session: Session, batch_size: int) -> None:
71
+ self._session = session
72
+ self._batch_size = batch_size
73
+
74
+ def __call__(self, position: Position) -> list[models.Event]:
75
+ stmt = (
76
+ select(models.Event)
77
+ .join(models.Stream)
78
+ .where(models.Event.id > position)
79
+ .order_by(models.Event.id)
80
+ .limit(self._batch_size)
81
+ )
82
+
83
+ return list(self._session.scalars(stmt).all())
84
+
85
+
86
+ class GetBatchToCategory(GetBatch):
87
+ def __init__(self, session: Session, batch_size: int, category: str) -> None:
88
+ self._session = session
89
+ self._batch_size = batch_size
90
+ self._category = category
91
+
92
+ def __call__(self, position: Position) -> list[models.Event]:
93
+ stmt = (
94
+ select(models.Event)
95
+ .join(models.Stream)
96
+ .where(models.Stream.category == self._category)
97
+ .where(models.Event.id > position)
98
+ .order_by(models.Event.id)
99
+ .limit(self._batch_size)
100
+ )
101
+
102
+ return list(self._session.scalars(stmt).all())
103
+
104
+
105
+ class GetBatchToEvents(GetBatch):
106
+ def __init__(self, session: Session, batch_size: int, events: list[str]) -> None:
107
+ self._session = session
108
+ self._batch_size = batch_size
109
+ self._events = events
110
+
111
+ def __call__(self, position: Position) -> list[models.Event]:
112
+ stmt = (
113
+ select(models.Event)
114
+ .join(models.Stream)
115
+ .where(models.Event.name.in_(self._events))
116
+ .where(models.Event.id > position)
117
+ .order_by(models.Event.id)
118
+ .limit(self._batch_size)
119
+ )
120
+
121
+ return list(self._session.scalars(stmt).all())
122
+
123
+
124
+ @dataclass
125
+ class Cursor:
126
+ position: Position
127
+
128
+ def advance(self, batch: list[models.Event]) -> None:
129
+ if len(batch) > 0:
130
+ self.position = batch[-1].id
131
+
132
+
133
+ class GapDetectingIterator(Iterator[list[RecordedRaw]]):
134
+ def __init__(
135
+ self,
136
+ get_batch: GetBatch,
137
+ gap_retry_interval: timedelta,
138
+ start_from: Position,
139
+ batch_size: int,
140
+ timelimit: timedelta,
141
+ ) -> None:
142
+ self._get_batch = get_batch
143
+ self._gap_retry_interval = gap_retry_interval
144
+ self._cursor = Cursor(position=start_from)
145
+ self._batch_size = batch_size
146
+ self._timelimit = timelimit
147
+
148
+ def __next__(self) -> list[RecordedRaw]:
149
+ start = time.monotonic()
150
+ while True:
151
+ batch = self._get_batch(self._cursor.position)
152
+ if self._is_continuous(batch) and len(batch) == self._batch_size:
153
+ self._cursor.advance(batch)
154
+ return self._batch_to_recorded_raw(batch)
155
+ elif time.monotonic() - start > self._timelimit.total_seconds():
156
+ self._cursor.advance(batch)
157
+ return self._batch_to_recorded_raw(batch)
158
+ else:
159
+ time.sleep(self._gap_retry_interval.total_seconds())
160
+
161
+ @staticmethod
162
+ def _is_continuous(batch: list[models.Event]) -> bool:
163
+ if len(batch) < 2:
164
+ return False
165
+
166
+ return cast(bool, batch[-1].id - batch[0].id + 1 == len(batch))
167
+
168
+ @staticmethod
169
+ def _batch_to_recorded_raw(batch: list[models.Event]) -> list[RecordedRaw]:
170
+ return [
171
+ RecordedRaw(
172
+ entry=dto.raw_event(event, event.stream),
173
+ position=event.id,
174
+ )
175
+ for event in batch
176
+ ]
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: python-event-sourcery
3
- Version: 0.4.0a0
3
+ Version: 0.4.0a2
4
4
  Summary:
5
5
  License: MIT
6
6
  Author: Your Name
@@ -10,13 +10,12 @@ Classifier: License :: OSI Approved :: MIT License
10
10
  Classifier: Programming Language :: Python :: 3
11
11
  Classifier: Programming Language :: Python :: 3.10
12
12
  Classifier: Programming Language :: Python :: 3.11
13
- Provides-Extra: cloudevents
13
+ Classifier: Programming Language :: Python :: 3.12
14
14
  Provides-Extra: django
15
- Provides-Extra: kombu
16
- Requires-Dist: SQLAlchemy (>=2.0)
17
- Requires-Dist: cloudevents ; extra == "cloudevents"
15
+ Provides-Extra: esdb
16
+ Provides-Extra: sqlalchemy
17
+ Requires-Dist: SQLAlchemy (>=2.0) ; extra == "sqlalchemy"
18
18
  Requires-Dist: django (>=4,<6) ; extra == "django"
19
- Requires-Dist: esdbclient (>=1.0b3)
20
- Requires-Dist: kombu ; extra == "kombu"
19
+ Requires-Dist: esdbclient (>=1.0b3) ; extra == "esdb"
21
20
  Requires-Dist: more-itertools
22
21
  Requires-Dist: pydantic (>=2.0,<3)
@@ -28,17 +28,17 @@ event_sourcery_esdb/outbox.py,sha256=U2c7EPSOIORaB6qlwRITEvbz6w_-k1p4X3Yeb4UuMms
28
28
  event_sourcery_esdb/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
29
29
  event_sourcery_esdb/stream.py,sha256=CUnua_Yo3f3PtquYAT9p6plmRe_OkoCsAiAxfTc9MHM,1909
30
30
  event_sourcery_esdb/subscription.py,sha256=qXocdwo7RLtvtRulQVGnIsUu16HhtphFQWqUow6nkew,2568
31
- event_sourcery_kombu/__init__.py,sha256=xauf7mn_dIE9IKug-jWc9b10z6Uk_l7Q5HFnX7U5oBU,4571
32
- event_sourcery_sqlalchemy/__init__.py,sha256=q6pD6OvKrweLI7sBA5_YIGvq4AXG9yseqmqVbXjSlww,2752
31
+ event_sourcery_sqlalchemy/__init__.py,sha256=-5KUMID9y5TmSVSOtGwwW9L25f5yzVE7MLdPBa6ZkTA,2918
33
32
  event_sourcery_sqlalchemy/cursors_dao.py,sha256=mMR97qxT9ohJ1zCeFnyWo8TFjruPRPObO9i44N-915o,2925
34
- event_sourcery_sqlalchemy/event_store.py,sha256=J0bSNUZpLTuOIHAPEHNp5wPUdfh2pFTivB5tdcuXu_M,7868
33
+ event_sourcery_sqlalchemy/dto.py,sha256=6w5dkTBTXC1v7zFd295KKitqoMFNeYd2PkJUOfqNst8,605
34
+ event_sourcery_sqlalchemy/event_store.py,sha256=OzGyxVYFKA3hf-xSdRl88K7n28IJW-lACy1Bd7InamI,7976
35
35
  event_sourcery_sqlalchemy/guid.py,sha256=UN-doEr1RSgmugPp-Vaw0uixA7cabf6q7pgMscI1ymE,1247
36
36
  event_sourcery_sqlalchemy/jsonb.py,sha256=KIkzujpWotpbInSYCmVIV-qaG8XoYFCaYieYKcfrewQ,822
37
37
  event_sourcery_sqlalchemy/models.py,sha256=_jzRfAsKh6QOFXguPJAM5dYwfTf_iSY9zm7KyjblI60,6268
38
38
  event_sourcery_sqlalchemy/outbox.py,sha256=W5PkcAZDPZzLkjZJ2R0nmpbP9aYn9HwAOTMllFeNmPg,2918
39
39
  event_sourcery_sqlalchemy/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
40
- event_sourcery_sqlalchemy/subscription.py,sha256=ZXgzbqDdpNv9vYXTfUecR4kXYQBuJTn3z8etwfJ6U44,905
41
- python_event_sourcery-0.4.0a0.dist-info/LICENSE,sha256=ufzizVlKJ_TRa5OZMIHzSrciFMdlx7ec29nUPLSe_zQ,1077
42
- python_event_sourcery-0.4.0a0.dist-info/METADATA,sha256=8ZQE4d5W_BWgIMlPReWfa-l6Ubr79eHVkkyfEMZAk50,719
43
- python_event_sourcery-0.4.0a0.dist-info/WHEEL,sha256=d2fvjOD7sXsVzChCqf0Ty0JbHKBaLYwDbGQDwQTnJ50,88
44
- python_event_sourcery-0.4.0a0.dist-info/RECORD,,
40
+ event_sourcery_sqlalchemy/subscription.py,sha256=WYfJ2NdHuNsYJmZeC-5KWBjeoAKScQ0HjKUfUYL8DpQ,5591
41
+ python_event_sourcery-0.4.0a2.dist-info/LICENSE,sha256=ufzizVlKJ_TRa5OZMIHzSrciFMdlx7ec29nUPLSe_zQ,1077
42
+ python_event_sourcery-0.4.0a2.dist-info/METADATA,sha256=8pfLtJ3hfY2a19Koz_ePiFC6V-ZLOJbEPax21aa00ek,718
43
+ python_event_sourcery-0.4.0a2.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
44
+ python_event_sourcery-0.4.0a2.dist-info/RECORD,,
@@ -1,4 +1,4 @@
1
1
  Wheel-Version: 1.0
2
- Generator: poetry-core 1.7.0
2
+ Generator: poetry-core 1.9.0
3
3
  Root-Is-Purelib: true
4
4
  Tag: py3-none-any
@@ -1,155 +0,0 @@
1
- from dataclasses import dataclass, field
2
- from typing import Callable, Type
3
-
4
- from kombu import Connection, Exchange, Message, Queue
5
- from kombu.connection import ConnectionPool
6
- from kombu.pools import connections
7
- from pydantic import AmqpDsn, Field
8
- from pydantic_settings import BaseSettings
9
-
10
- __all__ = ["publish_event", "declare_exchange"]
11
-
12
- from event_sourcery.event_store import Event, Metadata, StreamId
13
- from event_sourcery.event_store.event.registry import event_name
14
-
15
-
16
- class BrokerSettings(BaseSettings):
17
- BROKER_URL: AmqpDsn = Field(default="amqp://guest:guest@rabbitmq//")
18
-
19
-
20
- class PoolFactory:
21
- _pool: ConnectionPool | None = None
22
- _config: BrokerSettings | None = None
23
-
24
- @classmethod
25
- def configure(cls, broker_url: str) -> None:
26
- if cls._pool is not None:
27
- cls._pool.force_close_all()
28
- cls._pool = None
29
-
30
- cls._config = BrokerSettings(BROKER_URL=broker_url)
31
-
32
- @classmethod
33
- def get(cls) -> ConnectionPool:
34
- if cls._config is None:
35
- cls._config = BrokerSettings()
36
-
37
- if cls._pool is None:
38
- connection = Connection(
39
- cls._config.BROKER_URL, transport_options={"confirm_publish": True}
40
- )
41
- cls._pool = connections[connection]
42
- return cls._pool
43
-
44
-
45
- EVENT_SOURCERY_EXCHANGE = "event_sourcery.events"
46
-
47
-
48
- def publish_event(
49
- metadata: Metadata, stream_name: str | None, stream_id: StreamId
50
- ) -> None:
51
- name = event_name(type(metadata.event))
52
- stream_category = stream_name.split(".", maxsplit=1)[0] if stream_name else None
53
- _publish(
54
- routing_key=name,
55
- headers={
56
- "event": name,
57
- "stream_name": stream_name,
58
- "stream_category": stream_category,
59
- "stream_id": str(stream_id),
60
- },
61
- message=metadata.json(),
62
- exchange=EVENT_SOURCERY_EXCHANGE,
63
- )
64
-
65
-
66
- def declare_exchange() -> None:
67
- _declare(Exchange(name=EVENT_SOURCERY_EXCHANGE, type="headers"))
68
-
69
-
70
- def _publish(
71
- routing_key: str,
72
- message: str,
73
- exchange: str,
74
- headers: dict,
75
- ) -> None:
76
- with PoolFactory.get().acquire(block=True) as conn:
77
- producer = conn.Producer()
78
- producer.publish(
79
- message,
80
- exchange=exchange,
81
- routing_key=routing_key,
82
- headers=headers,
83
- content_type="application/json",
84
- )
85
-
86
-
87
- def _declare(queue_or_exchange: Exchange | Queue) -> None:
88
- with PoolFactory.get().acquire(block=True) as conn:
89
- queue_or_exchange(conn).declare()
90
-
91
-
92
- class InvalidSubscription(Exception):
93
- pass
94
-
95
-
96
- @dataclass(frozen=True)
97
- class Subscription:
98
- event_types: list[Type[Event]] = field(default_factory=list)
99
- stream_categories: list[str] = field(default_factory=list)
100
-
101
- def __post_init__(self) -> None:
102
- if len(self.event_types) == 0 and len(self.stream_categories) == 0:
103
- raise InvalidSubscription(
104
- "At least one event type or stream category needs to be specified"
105
- )
106
-
107
-
108
- def consume(
109
- subscription: Subscription,
110
- listener: Callable[[Metadata, StreamId], None],
111
- ) -> None:
112
- queue = Queue(f"event_sourcery.{listener.__name__}", durable=True)
113
- _declare(queue)
114
-
115
- def callback(body: dict, message: Message) -> None:
116
- event_name = message.headers["event"]
117
- event_type = Event.__registry__.type_for_name(event_name)
118
-
119
- stream_id = StreamId(
120
- from_hex=message.headers["stream_id"],
121
- name=message.headers["stream_name"],
122
- )
123
- metadata = Metadata[event_type](**body) # type: ignore
124
-
125
- try:
126
- listener(metadata, stream_id)
127
- except Exception: # TODO: DLQ, retries, error handling etc
128
- message.reject()
129
- raise
130
- else:
131
- message.ack()
132
-
133
- with PoolFactory.get().acquire(block=True) as conn:
134
- for event_type in subscription.event_types:
135
- queue(conn).bind_to(
136
- exchange=EVENT_SOURCERY_EXCHANGE,
137
- arguments={
138
- "event": event_name(event_type),
139
- },
140
- )
141
-
142
- for stream_category in subscription.stream_categories:
143
- queue(conn).bind_to(
144
- exchange=EVENT_SOURCERY_EXCHANGE,
145
- arguments={
146
- "stream_category": stream_category,
147
- },
148
- )
149
-
150
- with conn.Consumer([queue], callbacks=[callback]):
151
- while True:
152
- try:
153
- conn.drain_events()
154
- except KeyboardInterrupt:
155
- return