xtr-orm-messenger 6.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- xtr_orm_messenger/.agents/skills/xtr-orm-messenger/SKILL.md +108 -0
- xtr_orm_messenger/__init__.py +60 -0
- xtr_orm_messenger/exception/__init__.py +9 -0
- xtr_orm_messenger/exception/invalid_orm_dsn_error.py +34 -0
- xtr_orm_messenger/exception/orm_messenger_error.py +15 -0
- xtr_orm_messenger/exception/transport_error.py +32 -0
- xtr_orm_messenger/py.typed +0 -0
- xtr_orm_messenger/recipe/__init__.py +15 -0
- xtr_orm_messenger/recipe/manifest.toml +21 -0
- xtr_orm_messenger/transport/__init__.py +38 -0
- xtr_orm_messenger/transport/_database_errors.py +71 -0
- xtr_orm_messenger/transport/connection.py +438 -0
- xtr_orm_messenger/transport/messages_table.py +51 -0
- xtr_orm_messenger/transport/messages_table_driver.py +39 -0
- xtr_orm_messenger/transport/messages_tables.py +41 -0
- xtr_orm_messenger/transport/orm_options.py +145 -0
- xtr_orm_messenger/transport/orm_received_stamp.py +22 -0
- xtr_orm_messenger/transport/orm_receiver.py +214 -0
- xtr_orm_messenger/transport/orm_sender.py +51 -0
- xtr_orm_messenger/transport/orm_transport.py +170 -0
- xtr_orm_messenger/transport/orm_transport_factory.py +139 -0
- xtr_orm_messenger/transport/postgresql_notifier.py +105 -0
- xtr_orm_messenger/transport/stored_message.py +38 -0
- xtr_orm_messenger-6.0.0.dist-info/METADATA +280 -0
- xtr_orm_messenger-6.0.0.dist-info/RECORD +28 -0
- xtr_orm_messenger-6.0.0.dist-info/WHEEL +4 -0
- xtr_orm_messenger-6.0.0.dist-info/entry_points.txt +3 -0
- xtr_orm_messenger-6.0.0.dist-info/licenses/LICENSE +22 -0
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
"""Building ``orm://`` transports on the connections xtr-orm configures.
|
|
2
|
+
|
|
3
|
+
The DSN names a connection, not a database: ``orm://default`` is the
|
|
4
|
+
connection the application configured as ``default``, ``orm://`` the default
|
|
5
|
+
connection, whatever it is called. The database's URL, its credentials and its
|
|
6
|
+
pool stay where the connection is configured, so a transport never repeats
|
|
7
|
+
them.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
from typing import TYPE_CHECKING, final
|
|
13
|
+
from urllib.parse import urlsplit
|
|
14
|
+
|
|
15
|
+
from typing_extensions import override
|
|
16
|
+
from xtr_messenger import PublisherClosingInterface, TransportFactoryInterface
|
|
17
|
+
|
|
18
|
+
from xtr_orm_messenger.exception import InvalidOrmDsnError
|
|
19
|
+
|
|
20
|
+
from .connection import Connection
|
|
21
|
+
from .orm_options import ORM_SCHEME, OrmOptions
|
|
22
|
+
from .orm_transport import OrmTransport
|
|
23
|
+
|
|
24
|
+
if TYPE_CHECKING:
|
|
25
|
+
from collections.abc import Mapping
|
|
26
|
+
|
|
27
|
+
from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession
|
|
28
|
+
from xtr_clock import ClockInterface
|
|
29
|
+
from xtr_messenger import Dsn, SenderInterface, SerializerInterface, TransportConfig
|
|
30
|
+
from xtr_orm import ConnectionRegistry
|
|
31
|
+
|
|
32
|
+
__all__ = ["OrmTransportFactory"]
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
@final
|
|
36
|
+
class OrmTransportFactory(TransportFactoryInterface, PublisherClosingInterface):
|
|
37
|
+
"""Builds ``orm://`` transports, each on the connection its DSN names.
|
|
38
|
+
|
|
39
|
+
One transport per name, built the first time and kept: the bus sending to
|
|
40
|
+
it and the worker draining it must hold the same one, as they would any
|
|
41
|
+
transport holding a connection.
|
|
42
|
+
|
|
43
|
+
Not discovered by DSN scheme like a transport needing nothing: it needs
|
|
44
|
+
the application's connections, which only the orm bundle holds. With a
|
|
45
|
+
container, that bundle registers it when this package is installed;
|
|
46
|
+
without one, pass it to the bus yourself::
|
|
47
|
+
|
|
48
|
+
MessageBusFactory(config, [OrmTransportFactory(registry, serializer)])
|
|
49
|
+
"""
|
|
50
|
+
|
|
51
|
+
__slots__ = ("_clock", "_made", "_registry", "_serializer")
|
|
52
|
+
|
|
53
|
+
def __init__(
|
|
54
|
+
self,
|
|
55
|
+
registry: ConnectionRegistry,
|
|
56
|
+
serializer: SerializerInterface,
|
|
57
|
+
clock: ClockInterface | None = None,
|
|
58
|
+
) -> None:
|
|
59
|
+
"""Build transports on ``registry``'s connections.
|
|
60
|
+
|
|
61
|
+
Args:
|
|
62
|
+
registry: The application's connections, by name.
|
|
63
|
+
serializer: What every transport encodes and decodes envelopes
|
|
64
|
+
with.
|
|
65
|
+
clock: What every transport reads the time from; the system clock
|
|
66
|
+
when left out.
|
|
67
|
+
"""
|
|
68
|
+
self._registry = registry
|
|
69
|
+
self._serializer = serializer
|
|
70
|
+
self._clock = clock
|
|
71
|
+
self._made: dict[str, OrmTransport] = {}
|
|
72
|
+
|
|
73
|
+
@override
|
|
74
|
+
def supports(self, dsn: Dsn) -> bool:
|
|
75
|
+
"""Recognise the ``orm`` scheme."""
|
|
76
|
+
return dsn.scheme == ORM_SCHEME
|
|
77
|
+
|
|
78
|
+
@override
|
|
79
|
+
def create(self, group: Mapping[str, TransportConfig]) -> Mapping[str, SenderInterface]:
|
|
80
|
+
"""Return the transport for each name, building it the first time.
|
|
81
|
+
|
|
82
|
+
Building one opens nothing: the connection's engine is asked for the
|
|
83
|
+
first time the transport runs a statement.
|
|
84
|
+
|
|
85
|
+
Raises:
|
|
86
|
+
InvalidOrmDsnError: When a DSN names a connection the registry
|
|
87
|
+
does not hold.
|
|
88
|
+
UnknownTransportOptionError: When a transport carries a setting
|
|
89
|
+
this one does not accept.
|
|
90
|
+
InvalidTransportOptionError: When a setting's value is not what it
|
|
91
|
+
takes.
|
|
92
|
+
"""
|
|
93
|
+
return {name: self.transport(name, config) for name, config in group.items()}
|
|
94
|
+
|
|
95
|
+
def transport(self, name: str, config: TransportConfig) -> OrmTransport:
|
|
96
|
+
"""Return the transport ``name``, building it from ``config`` the first time.
|
|
97
|
+
|
|
98
|
+
Raises:
|
|
99
|
+
InvalidOrmDsnError: When the DSN names a connection the registry
|
|
100
|
+
does not hold.
|
|
101
|
+
UnknownTransportOptionError: When ``config`` carries a setting this
|
|
102
|
+
transport does not accept.
|
|
103
|
+
InvalidTransportOptionError: When a setting's value is not what it
|
|
104
|
+
takes.
|
|
105
|
+
"""
|
|
106
|
+
made = self._made.get(name)
|
|
107
|
+
if made is not None:
|
|
108
|
+
return made
|
|
109
|
+
options = OrmOptions.from_settings(config.settings)
|
|
110
|
+
connection_name = urlsplit(config.parsed.connection).netloc or self._registry.default
|
|
111
|
+
if not self._registry.has(connection_name):
|
|
112
|
+
raise InvalidOrmDsnError(name, connection_name, self._registry.names())
|
|
113
|
+
registry = self._registry
|
|
114
|
+
|
|
115
|
+
async def engine() -> AsyncEngine:
|
|
116
|
+
return await registry.engine(connection_name)
|
|
117
|
+
|
|
118
|
+
async def session() -> AsyncSession:
|
|
119
|
+
return await registry.session(connection_name)
|
|
120
|
+
|
|
121
|
+
made = OrmTransport(
|
|
122
|
+
Connection(engine, options, session_factory=session, clock=self._clock),
|
|
123
|
+
self._serializer,
|
|
124
|
+
name=name,
|
|
125
|
+
clock=self._clock,
|
|
126
|
+
)
|
|
127
|
+
self._made[name] = made
|
|
128
|
+
return made
|
|
129
|
+
|
|
130
|
+
@override
|
|
131
|
+
async def close_publishers(self) -> None:
|
|
132
|
+
"""Release the connections the transports this factory built listen on.
|
|
133
|
+
|
|
134
|
+
Sending holds no connection of its own — the engine's pool does, and
|
|
135
|
+
the connection's owner disposes it — so this leaves only the
|
|
136
|
+
connection a PostgreSQL worker listens for notifications on.
|
|
137
|
+
"""
|
|
138
|
+
for transport in self._made.values():
|
|
139
|
+
await transport.close()
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
"""Waking a waiting worker the moment a message is sent, through PostgreSQL notifications.
|
|
2
|
+
|
|
3
|
+
Every message an ``orm://`` transport writes on PostgreSQL is announced on the
|
|
4
|
+
table's channel, its queue as the payload, in the transaction that wrote it. A
|
|
5
|
+
worker that found nothing due listens on that channel instead of sleeping
|
|
6
|
+
through its poll interval, and asks again as soon as its own queue is named.
|
|
7
|
+
|
|
8
|
+
Listening takes a connection of its own for as long as the worker runs: a
|
|
9
|
+
notification is delivered to the session that asked for it, so the connection
|
|
10
|
+
cannot go back to the pool between waits.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import asyncio
|
|
16
|
+
from typing import TYPE_CHECKING, Protocol, cast, final
|
|
17
|
+
|
|
18
|
+
if TYPE_CHECKING:
|
|
19
|
+
from collections.abc import Awaitable, Callable
|
|
20
|
+
|
|
21
|
+
from sqlalchemy.ext.asyncio import AsyncConnection, AsyncEngine
|
|
22
|
+
|
|
23
|
+
__all__ = ["PostgreSqlNotifier"]
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class _Listening(Protocol):
|
|
27
|
+
"""The part of the PostgreSQL driver's connection notifications go through."""
|
|
28
|
+
|
|
29
|
+
def add_listener(
|
|
30
|
+
self, channel: str, callback: Callable[[object, int, str, str], None]
|
|
31
|
+
) -> Awaitable[None]: ...
|
|
32
|
+
|
|
33
|
+
def remove_listener(
|
|
34
|
+
self, channel: str, callback: Callable[[object, int, str, str], None]
|
|
35
|
+
) -> Awaitable[None]: ...
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
@final
|
|
39
|
+
class PostgreSqlNotifier:
|
|
40
|
+
"""Listens on a table's channel and wakes a waiting worker when its queue is named."""
|
|
41
|
+
|
|
42
|
+
__slots__ = ("_channel", "_connection", "_driver", "_engine", "_queue", "_woken")
|
|
43
|
+
|
|
44
|
+
def __init__(self, engine: AsyncEngine, channel: str, queue: str) -> None:
|
|
45
|
+
"""Listen on ``channel`` through ``engine``, waking for ``queue``'s messages only.
|
|
46
|
+
|
|
47
|
+
Building one opens nothing; :meth:`listen` does.
|
|
48
|
+
"""
|
|
49
|
+
self._engine = engine
|
|
50
|
+
self._channel = channel
|
|
51
|
+
self._queue = queue
|
|
52
|
+
self._woken = asyncio.Event()
|
|
53
|
+
self._connection: AsyncConnection | None = None
|
|
54
|
+
self._driver: _Listening | None = None
|
|
55
|
+
|
|
56
|
+
async def listen(self) -> None:
|
|
57
|
+
"""Take a connection of the engine's and start listening on it.
|
|
58
|
+
|
|
59
|
+
A notification sent from here on is heard; one sent before is not,
|
|
60
|
+
which is why a worker asks for its messages once more before it first
|
|
61
|
+
waits.
|
|
62
|
+
"""
|
|
63
|
+
if self._driver is not None:
|
|
64
|
+
return
|
|
65
|
+
connection = await self._engine.connect()
|
|
66
|
+
try:
|
|
67
|
+
raw = await connection.get_raw_connection()
|
|
68
|
+
driver = cast("_Listening", raw.driver_connection)
|
|
69
|
+
await driver.add_listener(self._channel, self._heard)
|
|
70
|
+
except BaseException:
|
|
71
|
+
await connection.close()
|
|
72
|
+
raise
|
|
73
|
+
self._connection = connection
|
|
74
|
+
self._driver = driver
|
|
75
|
+
|
|
76
|
+
async def wait(self, seconds: float) -> None:
|
|
77
|
+
"""Return once the queue is named on the channel, or ``seconds`` pass.
|
|
78
|
+
|
|
79
|
+
A notification heard while nobody waited is kept, so the next wait
|
|
80
|
+
returns at once: the message it announced may still be unclaimed.
|
|
81
|
+
"""
|
|
82
|
+
try:
|
|
83
|
+
async with asyncio.timeout(seconds):
|
|
84
|
+
_ = await self._woken.wait()
|
|
85
|
+
except TimeoutError:
|
|
86
|
+
return
|
|
87
|
+
self._woken.clear()
|
|
88
|
+
|
|
89
|
+
async def close(self) -> None:
|
|
90
|
+
"""Stop listening and give the connection back."""
|
|
91
|
+
driver, connection = self._driver, self._connection
|
|
92
|
+
self._driver = None
|
|
93
|
+
self._connection = None
|
|
94
|
+
if driver is None or connection is None:
|
|
95
|
+
return
|
|
96
|
+
try:
|
|
97
|
+
await driver.remove_listener(self._channel, self._heard)
|
|
98
|
+
finally:
|
|
99
|
+
await connection.close()
|
|
100
|
+
|
|
101
|
+
def _heard(self, connection: object, pid: int, channel: str, payload: str) -> None:
|
|
102
|
+
"""Wake the waiting worker when the notification names its queue."""
|
|
103
|
+
del connection, pid, channel
|
|
104
|
+
if payload == self._queue:
|
|
105
|
+
self._woken.set()
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
"""One message as the table keeps it."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass
|
|
6
|
+
from typing import TYPE_CHECKING, final
|
|
7
|
+
|
|
8
|
+
if TYPE_CHECKING:
|
|
9
|
+
from collections.abc import Mapping
|
|
10
|
+
from datetime import datetime
|
|
11
|
+
|
|
12
|
+
__all__ = ["StoredMessage"]
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@final
|
|
16
|
+
@dataclass(frozen=True, slots=True)
|
|
17
|
+
class StoredMessage:
|
|
18
|
+
"""One row of the messages table, its envelope still encoded.
|
|
19
|
+
|
|
20
|
+
Attributes:
|
|
21
|
+
id: The row's identifier, as the transport hands it out.
|
|
22
|
+
body: The encoded message.
|
|
23
|
+
headers: The encoded envelope's headers: its type and its stamps.
|
|
24
|
+
queue: The queue it was sent to.
|
|
25
|
+
created_at: When it was sent, in UTC.
|
|
26
|
+
available_at: When it falls due, in UTC — later than ``created_at``
|
|
27
|
+
for a delayed message.
|
|
28
|
+
delivered_at: When a worker was last handed it, in UTC, or ``None``
|
|
29
|
+
while no worker has been.
|
|
30
|
+
"""
|
|
31
|
+
|
|
32
|
+
id: str
|
|
33
|
+
body: str
|
|
34
|
+
headers: Mapping[str, str]
|
|
35
|
+
queue: str
|
|
36
|
+
created_at: datetime
|
|
37
|
+
available_at: datetime
|
|
38
|
+
delivered_at: datetime | None
|
|
@@ -0,0 +1,280 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: xtr-orm-messenger
|
|
3
|
+
Version: 6.0.0
|
|
4
|
+
Summary: A database transport for xtr-messenger: messages kept in a table on an xtr-orm connection, claimed by workers, delayed, redelivered and woken by notifications.
|
|
5
|
+
Keywords: message-bus,message-queue,transport,database,sqlalchemy,asyncio
|
|
6
|
+
Author: Xterr
|
|
7
|
+
Author-email: Xterr <me@xterr.dev>
|
|
8
|
+
License-Expression: MIT
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Classifier: Development Status :: 3 - Alpha
|
|
11
|
+
Classifier: Framework :: AsyncIO
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
17
|
+
Classifier: Topic :: Database
|
|
18
|
+
Classifier: Typing :: Typed
|
|
19
|
+
Requires-Dist: msgspec>=0.18
|
|
20
|
+
Requires-Dist: sqlalchemy[asyncio]>=2.0.41
|
|
21
|
+
Requires-Dist: typing-extensions>=4.4
|
|
22
|
+
Requires-Dist: xtr-clock>=6.0,<7
|
|
23
|
+
Requires-Dist: xtr-messenger>=6.0,<7
|
|
24
|
+
Requires-Dist: xtr-orm>=6.0,<7
|
|
25
|
+
Requires-Python: >=3.11
|
|
26
|
+
Description-Content-Type: text/markdown
|
|
27
|
+
|
|
28
|
+
<div align="center">
|
|
29
|
+
|
|
30
|
+
# xtr-orm-messenger
|
|
31
|
+
|
|
32
|
+
**A database transport for xtr-messenger: messages kept in a table on an xtr-orm connection, claimed by workers, delayed, redelivered and woken by notifications.**
|
|
33
|
+
|
|
34
|
+
<img alt="python 3.11+" src="https://img.shields.io/badge/python-%E2%89%A5%203.11-3776AB?logo=python&logoColor=white">
|
|
35
|
+
<img alt="typed" src="https://img.shields.io/badge/typed-ty%20%2B%20basedpyright-1f6feb">
|
|
36
|
+
<img alt="license MIT" src="https://img.shields.io/badge/license-MIT-blue">
|
|
37
|
+
|
|
38
|
+
</div>
|
|
39
|
+
|
|
40
|
+
---
|
|
41
|
+
|
|
42
|
+
## Why?
|
|
43
|
+
|
|
44
|
+
A queue needs a broker, and a broker is one more server to run, back up and watch. An
|
|
45
|
+
application that already has a database can keep its messages there: in a table, on the
|
|
46
|
+
connection [xtr-orm](../xtr-orm) already configures, sent in the same transaction as the
|
|
47
|
+
changes they announce if it wants.
|
|
48
|
+
|
|
49
|
+
- 🗄️ **No broker** — `orm://default` is the application's own `default` connection.
|
|
50
|
+
- 🔒 **Workers never collide** — each claims the oldest due message `FOR UPDATE SKIP LOCKED`.
|
|
51
|
+
- ⏰ **Delays** — a `DelayStamp` makes a message due later.
|
|
52
|
+
- ♻️ **Nothing is lost to a dead worker** — a message never settled is handed out again.
|
|
53
|
+
- 🔔 **No polling on PostgreSQL** — a waiting worker is woken the moment a message is sent.
|
|
54
|
+
- 🤝 **Sent with the handler's changes** — `transactional=true` writes the message in the unit
|
|
55
|
+
of work's session, so it commits or rolls back with them.
|
|
56
|
+
|
|
57
|
+
## Install
|
|
58
|
+
|
|
59
|
+
```sh
|
|
60
|
+
uv add xtr-orm-messenger
|
|
61
|
+
uv add "xtr-messenger[orm]" # the same, from the message bus's side
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
It needs [xtr-messenger](../xtr-messenger) and [xtr-orm](../xtr-orm), which come with it. It
|
|
65
|
+
brings no database driver: a transport writes through the connection its DSN names, with the
|
|
66
|
+
driver the application installed for that connection — xtr-orm's `postgres`, `mysql` or
|
|
67
|
+
`sqlite` extra. On PostgreSQL through asyncpg, a waiting worker is woken by notifications; with
|
|
68
|
+
any other driver it polls. It ships no bundle: with a
|
|
69
|
+
container, the [orm bundle](../xtr-orm#message-bus-transport) registers the transport once this
|
|
70
|
+
package is installed.
|
|
71
|
+
|
|
72
|
+
## Quick start
|
|
73
|
+
|
|
74
|
+
Without a container, hand the factory the connections it builds on and the envelope serializer
|
|
75
|
+
the bus and its workers share:
|
|
76
|
+
|
|
77
|
+
```python
|
|
78
|
+
from xtr_messenger import (
|
|
79
|
+
JsonSerializer,
|
|
80
|
+
MessageBusConfig,
|
|
81
|
+
MessageBusFactory,
|
|
82
|
+
TransportConfig,
|
|
83
|
+
WorkerFactory,
|
|
84
|
+
)
|
|
85
|
+
from xtr_orm import ConnectionRegistry
|
|
86
|
+
from xtr_orm_messenger import OrmTransportFactory
|
|
87
|
+
from xtr_serializer import JsonEncoder, PydanticNormalizer, Serializer
|
|
88
|
+
|
|
89
|
+
registry = ConnectionRegistry()
|
|
90
|
+
registry.register("default", engine=engine, migrator=migrator, database=database)
|
|
91
|
+
|
|
92
|
+
config = MessageBusConfig(
|
|
93
|
+
transports={"jobs": TransportConfig("orm://default", queue="jobs")},
|
|
94
|
+
routing={IngestDocument: "jobs"},
|
|
95
|
+
)
|
|
96
|
+
wire = JsonSerializer(Serializer([PydanticNormalizer()], [JsonEncoder()]))
|
|
97
|
+
factory = OrmTransportFactory(registry, wire)
|
|
98
|
+
|
|
99
|
+
await MessageBusFactory(config, [factory]).bus().dispatch(IngestDocument(document_id=1))
|
|
100
|
+
await WorkerFactory(config, [factory]).worker(["jobs"]).run()
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
The bus writes the message into `messenger_messages`; the worker claims it, hands it to its
|
|
104
|
+
handlers and deletes it. With a [container](#use-in-an-application) all of that is wired for you.
|
|
105
|
+
|
|
106
|
+
## How it works
|
|
107
|
+
|
|
108
|
+
| Step | What happens |
|
|
109
|
+
| --- | --- |
|
|
110
|
+
| Send | One row: the encoded envelope, its queue, `created_at`, and `available_at` — later by the delay, rounded up to a whole second |
|
|
111
|
+
| Claim | In one transaction, the oldest row of the queue that is due and not handed out is selected `FOR UPDATE SKIP LOCKED` and stamped `delivered_at` |
|
|
112
|
+
| Handle | The worker dispatches it; the envelope carries `OrmReceivedStamp` and `TransportMessageIdStamp`, both the row's id |
|
|
113
|
+
| Settle | Acknowledged or rejected, the row is deleted |
|
|
114
|
+
| Redeliver | A row handed out more than `redeliver_timeout` seconds ago is claimable again: its worker is taken for dead |
|
|
115
|
+
|
|
116
|
+
A worker that finds nothing due waits `poll_interval` seconds before asking again. On
|
|
117
|
+
PostgreSQL through asyncpg it listens instead: every send announces itself on the table's
|
|
118
|
+
channel, naming its queue, and the worker asks again at once — or after
|
|
119
|
+
`check_delayed_interval`, for a delayed message falling due. Listening holds one connection of
|
|
120
|
+
the pool for as long as the worker runs.
|
|
121
|
+
|
|
122
|
+
SQLite has no row locks, so the lock clause is left out there. A second worker reaching for the
|
|
123
|
+
row another is claiming gets "database is locked", which counts as a conflict and is tried again.
|
|
124
|
+
|
|
125
|
+
Conflicts between workers — a deadlock, a lock wait that timed out — say nothing about the
|
|
126
|
+
message. A poll sits through 3 in a row before it raises; an acknowledgement or a rejection
|
|
127
|
+
tries again with a pause doubling from 100 ms. One whose database went away waits up to 5
|
|
128
|
+
seconds for it and is replayed once: losing an acknowledgement to a restart would hand the
|
|
129
|
+
message out again.
|
|
130
|
+
|
|
131
|
+
A message that does not decode can never be handled, so it is rejected at once, its reason
|
|
132
|
+
logged on the `messenger` logger — never its body.
|
|
133
|
+
|
|
134
|
+
## The DSN
|
|
135
|
+
|
|
136
|
+
```sh
|
|
137
|
+
# .env
|
|
138
|
+
MESSENGER_DSN=orm://default
|
|
139
|
+
```
|
|
140
|
+
|
|
141
|
+
The format is `orm://<connection_name>`: the name of a connection the application configures
|
|
142
|
+
in xtr-orm, for when it has several and the messages go on one other than `default`; `orm://`
|
|
143
|
+
alone is the default connection. The URL, the driver and the pool are the connection's. The
|
|
144
|
+
transport creates a table named `messenger_messages` the first time it runs. To name the table
|
|
145
|
+
otherwise, give `table_name` in the DSN:
|
|
146
|
+
|
|
147
|
+
```sh
|
|
148
|
+
# .env
|
|
149
|
+
MESSENGER_DSN=orm://default?table_name=your_custom_table_name
|
|
150
|
+
```
|
|
151
|
+
|
|
152
|
+
The queue a transport reads and writes is `queue` — `?queue=jobs`, or
|
|
153
|
+
`TransportConfig(dsn, queue="jobs")` — the setting every xtr-messenger transport takes for it.
|
|
154
|
+
|
|
155
|
+
## Options
|
|
156
|
+
|
|
157
|
+
Every option may go in the DSN's query string or in `TransportConfig.options`; `options` wins.
|
|
158
|
+
An unknown one is refused with `UnknownTransportOptionError`, a bad value with
|
|
159
|
+
`InvalidTransportOptionError`.
|
|
160
|
+
|
|
161
|
+
| Option | Default | What it sets |
|
|
162
|
+
| --- | --- | --- |
|
|
163
|
+
| `queue` | `default` | the queue this transport reads and writes; also `TransportConfig(queue=...)` |
|
|
164
|
+
| `table_name` | `messenger_messages` | the table, a plain identifier |
|
|
165
|
+
| `redeliver_timeout` | `3600` | seconds after which a message handed out and never settled is handed out again |
|
|
166
|
+
| `auto_setup` | `true` | create the table the first time an operation finds it missing |
|
|
167
|
+
| `poll_interval` | `1.0` | seconds a worker waits before asking again when nothing was due |
|
|
168
|
+
| `transactional` | `false` | write the message in the session of the unit of work under way |
|
|
169
|
+
| `use_notify` | `true` | on PostgreSQL through asyncpg, wait for notifications instead of polling |
|
|
170
|
+
| `check_delayed_interval` | `60000` | milliseconds a listening worker waits at most before asking anyway |
|
|
171
|
+
| `get_notify_timeout` | `60000` | milliseconds one wait for a notification lasts at most |
|
|
172
|
+
|
|
173
|
+
`orm://` with no name uses the default connection. A name the application did not configure is
|
|
174
|
+
refused with `InvalidOrmDsnError` when the transport is built.
|
|
175
|
+
|
|
176
|
+
### Sending with the handler's changes
|
|
177
|
+
|
|
178
|
+
With `transactional=true`, a message dispatched while a message is handled — or a request, or a
|
|
179
|
+
command: any unit of work — is written in that unit's session and sent only when the unit
|
|
180
|
+
commits. Run the `orm_transaction` middleware, or commit yourself: a session never commits on
|
|
181
|
+
its own. Outside a unit of work the message is written in a transaction of its own, as it is
|
|
182
|
+
without the option.
|
|
183
|
+
|
|
184
|
+
## Looking into a queue
|
|
185
|
+
|
|
186
|
+
`OrmTransport` is a `TransportInterface` with more to ask:
|
|
187
|
+
|
|
188
|
+
```python
|
|
189
|
+
transport = factory.transport("jobs", config.transports["jobs"])
|
|
190
|
+
|
|
191
|
+
await transport.count() # what a worker could claim now
|
|
192
|
+
async for envelope in transport.all(limit=10):
|
|
193
|
+
... # the same, oldest first
|
|
194
|
+
await transport.find(message_id) # one message, handed out or not
|
|
195
|
+
await transport.keepalive(envelope, seconds=600) # a long handler keeps its message
|
|
196
|
+
await transport.setup() # create the table ahead of time
|
|
197
|
+
```
|
|
198
|
+
|
|
199
|
+
`keepalive` restarts a message's redelivery clock; `seconds` is how long until the next call,
|
|
200
|
+
refused when longer than `redeliver_timeout`.
|
|
201
|
+
|
|
202
|
+
## Use in an application
|
|
203
|
+
|
|
204
|
+
Everything adding this package to an application takes — and, read backwards, what removing it
|
|
205
|
+
undoes. It ships no bundle: the [orm bundle](../xtr-orm#message-bus-transport) registers the
|
|
206
|
+
transport, built on its connections, when the messenger bundle is active too.
|
|
207
|
+
|
|
208
|
+
- **Install** — `uv add xtr-orm-messenger`, or `uv add "xtr-messenger[orm]"`, beside
|
|
209
|
+
[xtr-messenger](../xtr-messenger/README.md#use-in-an-application) and
|
|
210
|
+
[xtr-orm](../xtr-orm/README.md#use-in-an-application) set up as their *Use in an application*
|
|
211
|
+
says. The database driver is the connection's own.
|
|
212
|
+
- **Recipe** — `uv run xtr-recipes recipes:sync` writes nothing: there is no bundle to list, no
|
|
213
|
+
configuration file, environment variable or ignore line. It prints the step naming a
|
|
214
|
+
transport, which a recipe cannot make for you.
|
|
215
|
+
- **Configure** — name a transport `orm://default` in `<app>/config/packages/messenger.py` and
|
|
216
|
+
route messages to it, as below.
|
|
217
|
+
- **Run** — `<script> orm:migrations:diff` writes the messages table of every `orm://`
|
|
218
|
+
transport into its connection's next revision; until it is applied, the transport creates the
|
|
219
|
+
table the first time it runs, unless `auto_setup=false`. A worker is
|
|
220
|
+
`<script> messenger:consume <transport>`.
|
|
221
|
+
- **Remove** — drop the `orm://` transports and the routing to them, and a migration dropping
|
|
222
|
+
the table. Then `uv remove xtr-orm-messenger`.
|
|
223
|
+
- **Check** — `debug:config messenger` lists the `orm://` transports.
|
|
224
|
+
|
|
225
|
+
Everything else — the bundles, the connections, the bus — is xtr-orm's and xtr-messenger's.
|
|
226
|
+
|
|
227
|
+
```python
|
|
228
|
+
# app/config/packages/messenger.py
|
|
229
|
+
from xtr_dependency_injection import configure
|
|
230
|
+
from xtr_messenger.bundle import MessageBusConfig, TransportConfig
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
@configure
|
|
234
|
+
def messenger() -> MessageBusConfig:
|
|
235
|
+
return MessageBusConfig(
|
|
236
|
+
transports={"jobs": TransportConfig("orm://default?auto_setup=false", queue="jobs")},
|
|
237
|
+
routing={IngestDocument: "jobs"},
|
|
238
|
+
)
|
|
239
|
+
```
|
|
240
|
+
|
|
241
|
+
The orm bundle registers `OrmTransportFactory` with the messenger, built on its
|
|
242
|
+
`ConnectionRegistry` and the messenger's envelope serializer, and consulted ahead of the
|
|
243
|
+
factories found by DSN scheme. A bus and every worker of the container share it, and so share
|
|
244
|
+
each transport. When the kernel shuts down, the connections workers listened on are given back.
|
|
245
|
+
|
|
246
|
+
Without a container, `messages_tables(bus_config, connection, default_connection)` names the
|
|
247
|
+
tables the transports of a bus configuration keep on a connection, and `MessagesTableDriver`
|
|
248
|
+
adds one to the `MetaData` a connection's migrations compare with.
|
|
249
|
+
|
|
250
|
+
## Errors
|
|
251
|
+
|
|
252
|
+
Everything this library raises derives from `OrmMessengerError`, itself a
|
|
253
|
+
`MessageBusError` of [xtr-messenger](../xtr-messenger):
|
|
254
|
+
|
|
255
|
+
| Error | Raised when |
|
|
256
|
+
| --- | --- |
|
|
257
|
+
| `TransportError` | the database refuses what a transport asked — sending, claiming, settling, listing — or an envelope settled here was not received here; the driver's error is its cause |
|
|
258
|
+
| `InvalidOrmDsnError` | an `orm://` DSN names a connection the application does not configure; also a `LookupError` |
|
|
259
|
+
|
|
260
|
+
## Development
|
|
261
|
+
|
|
262
|
+
Developed in the [python-xtr](https://github.com/xterr/python-xtr) monorepo, under
|
|
263
|
+
`packages/xtr-orm-messenger`; run the commands below from there. The `python-xtr-orm-messenger`
|
|
264
|
+
repository is a read-only copy, so send issues and pull requests to the monorepo.
|
|
265
|
+
|
|
266
|
+
```sh
|
|
267
|
+
uv sync
|
|
268
|
+
uv run ruff check && uv run ruff format --check && uv run basedpyright && uv run ty check && uv run pytest
|
|
269
|
+
```
|
|
270
|
+
|
|
271
|
+
Every test runs on SQLite files. The PostgreSQL suite under `tests/postgresql/` runs when
|
|
272
|
+
`XTR_TEST_POSTGRES_URL` names a database, as an asyncpg URL:
|
|
273
|
+
|
|
274
|
+
```sh
|
|
275
|
+
XTR_TEST_POSTGRES_URL=postgresql+asyncpg://user:password@127.0.0.1/test uv run pytest tests/postgresql
|
|
276
|
+
```
|
|
277
|
+
|
|
278
|
+
## License
|
|
279
|
+
|
|
280
|
+
MIT — see [LICENSE](LICENSE).
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
xtr_orm_messenger/.agents/skills/xtr-orm-messenger/SKILL.md,sha256=vIGNYCfLifws5ubQezJ4JYOfOhH7NRAOKG31MfelN5A,5825
|
|
2
|
+
xtr_orm_messenger/__init__.py,sha256=tbPSycd8DO3r9bi2Hz0tj0l666nQYv-6ceOa9gtOKeE,1608
|
|
3
|
+
xtr_orm_messenger/exception/__init__.py,sha256=dn7qB6Duq4tZoZ_lzhIAolJppw4Po2uxxm4SCA-dvU4,337
|
|
4
|
+
xtr_orm_messenger/exception/invalid_orm_dsn_error.py,sha256=6Y7pYuxBLQerwaFRdVjnrYX1ql5kH_OlIRhAUGmuc2c,1184
|
|
5
|
+
xtr_orm_messenger/exception/orm_messenger_error.py,sha256=VJPV8NYC3lUtnsokxnLoLOGIumsdjYKJUiWZgCcDfoo,423
|
|
6
|
+
xtr_orm_messenger/exception/transport_error.py,sha256=mtm9VxWxFGtY_WpLc3nwyZzfGEp2iN6HaXXFKBz1G-A,1077
|
|
7
|
+
xtr_orm_messenger/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
8
|
+
xtr_orm_messenger/recipe/__init__.py,sha256=4-zd7NZR7zzRtFJ0VBQlKTrdauempyBdhCOIAQSo0TQ,589
|
|
9
|
+
xtr_orm_messenger/recipe/manifest.toml,sha256=aIpEH4mJLNF8Eu5YZYRD__LbJJsrnWz8fikZT9edKMg,1008
|
|
10
|
+
xtr_orm_messenger/transport/__init__.py,sha256=FpeDIXVO59z5mM7aGL3B1lIFg5r_qcuwCiPIoBsvt_Y,1185
|
|
11
|
+
xtr_orm_messenger/transport/_database_errors.py,sha256=CpD-6Hn3NmqYx8TPHcbcE9v9dZ45NGIZtJA45wTsqUM,2642
|
|
12
|
+
xtr_orm_messenger/transport/connection.py,sha256=FzuvYvbsTSWxwc2-ANfMv6bmWFMvubJpSYZ_fYKhqjI,18345
|
|
13
|
+
xtr_orm_messenger/transport/messages_table.py,sha256=yvexSueivWbMAgvmnucfZ64xvM2lXvSntsB8uV4rKYc,1679
|
|
14
|
+
xtr_orm_messenger/transport/messages_table_driver.py,sha256=bRXlDDkOp2UbpEcL6XyYmm-jXpwZ7gBesM_Lz-7pT3M,1208
|
|
15
|
+
xtr_orm_messenger/transport/messages_tables.py,sha256=VWjpCoMkz9z1v2-2EwRfKr4yEYbaXWCaqxR4S60lPkQ,1456
|
|
16
|
+
xtr_orm_messenger/transport/orm_options.py,sha256=6nEe_6dga1KA3RHEOp1_cCYF2BI6NHJxvXZAmgfHdkk,5831
|
|
17
|
+
xtr_orm_messenger/transport/orm_received_stamp.py,sha256=p8_vgWhz5cuj4EYia4d3VUvUHYIvw2qyUgSP2EFM8ik,557
|
|
18
|
+
xtr_orm_messenger/transport/orm_receiver.py,sha256=qk0cFdti-WfZfOdakbr3IRqvoPX6gNKAWS8QSc1Is5M,8184
|
|
19
|
+
xtr_orm_messenger/transport/orm_sender.py,sha256=ynAQGhwkRgqRKGEgDtu1AvgwlTIZN6Rxn9DBwfR1rh0,1820
|
|
20
|
+
xtr_orm_messenger/transport/orm_transport.py,sha256=TzUaNk8UBV_5vtvi__pNEcVLimojxA5I87yXzE7zEM0,6373
|
|
21
|
+
xtr_orm_messenger/transport/orm_transport_factory.py,sha256=IN3qGhrL0ocwMQJ94P_-zqIDN0eZg5RYZg_6lPO1kQs,5342
|
|
22
|
+
xtr_orm_messenger/transport/postgresql_notifier.py,sha256=hCI5FvhtO4yddiNZjKByJM4jR_--QJwPezAyYP0pJe0,3854
|
|
23
|
+
xtr_orm_messenger/transport/stored_message.py,sha256=X7FoOTPibIwDd7_Ud2H6AOMS58POoQRPxLo9T2TskwI,1078
|
|
24
|
+
xtr_orm_messenger-6.0.0.dist-info/licenses/LICENSE,sha256=HjmMt8uWRU2yv_ijhX_mqelIvKdeCgg-QlhxyxrwQ3c,1106
|
|
25
|
+
xtr_orm_messenger-6.0.0.dist-info/WHEEL,sha256=MWx7psoF7KNST9G72pGtiBYuaXRXQXBk9pw5R6FzYko,80
|
|
26
|
+
xtr_orm_messenger-6.0.0.dist-info/entry_points.txt,sha256=avZm_RpndK1frlSJqNatZqj3Om8-EamOU6CnDoaeAhg,56
|
|
27
|
+
xtr_orm_messenger-6.0.0.dist-info/METADATA,sha256=sPxhdGZeOwGwhD7PDbnX7A-mRhZyR4BAcejJ5nu9ASM,13007
|
|
28
|
+
xtr_orm_messenger-6.0.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 xterr
|
|
4
|
+
Copyright (c) 2018-present Fabien Potencier
|
|
5
|
+
|
|
6
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
7
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
8
|
+
in the Software without restriction, including without limitation the rights
|
|
9
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
10
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
11
|
+
furnished to do so, subject to the following conditions:
|
|
12
|
+
|
|
13
|
+
The above copyright notice and this permission notice shall be included in all
|
|
14
|
+
copies or substantial portions of the Software.
|
|
15
|
+
|
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
17
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
18
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
19
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
20
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
21
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
22
|
+
SOFTWARE.
|