qx-events 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- qx_events-0.2.0/.gitignore +56 -0
- qx_events-0.2.0/PKG-INFO +65 -0
- qx_events-0.2.0/README.md +49 -0
- qx_events-0.2.0/pyproject.toml +24 -0
- qx_events-0.2.0/src/qx/events/__init__.py +27 -0
- qx_events-0.2.0/src/qx/events/dispatcher/__init__.py +31 -0
- qx_events-0.2.0/src/qx/events/nats/__init__.py +235 -0
- qx_events-0.2.0/src/qx/events/outbox_relay/__init__.py +216 -0
- qx_events-0.2.0/src/qx/events/py.typed +0 -0
- qx_events-0.2.0/src/qx/events/registry/__init__.py +67 -0
- qx_events-0.2.0/tests/test_events_unit.py +70 -0
- qx_events-0.2.0/tests/test_outbox_relay_unit.py +152 -0
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
# Python
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[cod]
|
|
4
|
+
*.pyo
|
|
5
|
+
*.pyd
|
|
6
|
+
.Python
|
|
7
|
+
*.so
|
|
8
|
+
*.egg
|
|
9
|
+
*.egg-info/
|
|
10
|
+
dist/
|
|
11
|
+
build/
|
|
12
|
+
eggs/
|
|
13
|
+
.eggs/
|
|
14
|
+
sdist/
|
|
15
|
+
wheels/
|
|
16
|
+
*.egg-link
|
|
17
|
+
|
|
18
|
+
# Virtual environments
|
|
19
|
+
.venv/
|
|
20
|
+
venv/
|
|
21
|
+
env/
|
|
22
|
+
ENV/
|
|
23
|
+
|
|
24
|
+
# uv
|
|
25
|
+
.uv/
|
|
26
|
+
|
|
27
|
+
# Testing
|
|
28
|
+
.pytest_cache/
|
|
29
|
+
.coverage
|
|
30
|
+
htmlcov/
|
|
31
|
+
.tox/
|
|
32
|
+
|
|
33
|
+
# Type checking
|
|
34
|
+
.mypy_cache/
|
|
35
|
+
.ruff_cache/
|
|
36
|
+
|
|
37
|
+
# IDE
|
|
38
|
+
.idea/
|
|
39
|
+
.vscode/
|
|
40
|
+
*.swp
|
|
41
|
+
*.swo
|
|
42
|
+
|
|
43
|
+
# OS
|
|
44
|
+
.DS_Store
|
|
45
|
+
Thumbs.db
|
|
46
|
+
|
|
47
|
+
# Docker
|
|
48
|
+
*.env.local
|
|
49
|
+
|
|
50
|
+
# Dist artifacts
|
|
51
|
+
dist/
|
|
52
|
+
|
|
53
|
+
# VS Code extension build artifacts
|
|
54
|
+
extensions/vscode/node_modules/
|
|
55
|
+
extensions/vscode/dist/
|
|
56
|
+
extensions/vscode/*.vsix
|
qx_events-0.2.0/PKG-INFO
ADDED
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: qx-events
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: Qx events: NATS JetStream publisher/consumer, outbox relay, mediator dispatcher bridge
|
|
5
|
+
Author: Qx Engineering
|
|
6
|
+
License: MIT
|
|
7
|
+
Requires-Python: >=3.14
|
|
8
|
+
Requires-Dist: nats-py>=2.9.0
|
|
9
|
+
Requires-Dist: qx-cache
|
|
10
|
+
Requires-Dist: qx-core
|
|
11
|
+
Requires-Dist: qx-cqrs
|
|
12
|
+
Requires-Dist: qx-db
|
|
13
|
+
Requires-Dist: qx-di
|
|
14
|
+
Requires-Dist: qx-observability
|
|
15
|
+
Description-Content-Type: text/markdown
|
|
16
|
+
|
|
17
|
+
# qx-events
|
|
18
|
+
|
|
19
|
+
NATS JetStream publisher/consumer, transactional outbox relay, and mediator-bridge event dispatcher for the Qx framework.
|
|
20
|
+
|
|
21
|
+
## What lives here
|
|
22
|
+
|
|
23
|
+
- **`qx.events.OutboxRelay`** — polls `qx_outbox_events`, publishes unpublished events to NATS JetStream, and marks them delivered. Runs as a background task alongside the HTTP server or as a standalone process. Supports optional leader election so only one relay instance publishes at a time.
|
|
24
|
+
- **`qx.events.NatsPublisher`** — publishes a single `IntegrationEvent` to a JetStream subject derived from `event_name`.
|
|
25
|
+
- **`qx.events.NatsConsumer`** — durable pull consumer over a JetStream stream. Used by `WorkerRuntime` to fetch and ack/nak messages.
|
|
26
|
+
- **`qx.events.NatsSettings`** — Pydantic settings for NATS connection (URL, credentials, stream name, consumer name).
|
|
27
|
+
- **`qx.events.EventRegistry`** — maps `event_name` strings to concrete `IntegrationEvent` subclasses. Required by both the relay (for serialisation) and the worker (for deserialisation).
|
|
28
|
+
- **`qx.events.MediatorEventDispatcher`** — `EventDispatcher` implementation that routes domain events to their in-process handlers via the Mediator after a UnitOfWork commit.
|
|
29
|
+
- **`qx.events.create_nats_connection`** — async factory that opens a NATS connection with retry.
|
|
30
|
+
|
|
31
|
+
## Usage
|
|
32
|
+
|
|
33
|
+
### Outbox relay (alongside the API server)
|
|
34
|
+
|
|
35
|
+
```python
|
|
36
|
+
from qx.events import OutboxRelay, NatsPublisher, NatsSettings, EventRegistry
|
|
37
|
+
|
|
38
|
+
registry = EventRegistry()
|
|
39
|
+
registry.register(UserRegisteredIntegration)
|
|
40
|
+
|
|
41
|
+
publisher = NatsPublisher(nc, registry)
|
|
42
|
+
relay = OutboxRelay(session_factory, publisher, registry)
|
|
43
|
+
|
|
44
|
+
# Run in background
|
|
45
|
+
asyncio.create_task(relay.run())
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
### Consuming events in a worker
|
|
49
|
+
|
|
50
|
+
```python
|
|
51
|
+
from qx.events import NatsConsumer, NatsSettings
|
|
52
|
+
|
|
53
|
+
settings = NatsSettings(url="nats://localhost:4222", stream="events", consumer="identity-worker")
|
|
54
|
+
consumer = NatsConsumer(nc, settings)
|
|
55
|
+
|
|
56
|
+
# WorkerRuntime handles the fetch/ack loop
|
|
57
|
+
worker = WorkerRuntime(container, consumer, registry, mediator)
|
|
58
|
+
await worker.run()
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
## Design rules
|
|
62
|
+
|
|
63
|
+
- **At-least-once delivery** — the outbox guarantees every `INSERT`ed event is eventually published. Consumers are expected to be idempotent (use `IdempotencyStore` from `qx-cache`).
|
|
64
|
+
- **Transactional outbox** — `UnitOfWork` (in `qx-db`) writes events to `qx_outbox_events` in the same transaction as the aggregate; the relay reads and publishes asynchronously. No event is lost even if the process crashes between commit and publish.
|
|
65
|
+
- **Event envelope** — each NATS message carries `event_name`, `event_version`, payload JSON, `correlation_id`, `tenant_id`, and OTel trace context headers for full observability continuity.
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
# qx-events
|
|
2
|
+
|
|
3
|
+
NATS JetStream publisher/consumer, transactional outbox relay, and mediator-bridge event dispatcher for the Qx framework.
|
|
4
|
+
|
|
5
|
+
## What lives here
|
|
6
|
+
|
|
7
|
+
- **`qx.events.OutboxRelay`** — polls `qx_outbox_events`, publishes unpublished events to NATS JetStream, and marks them delivered. Runs as a background task alongside the HTTP server or as a standalone process. Supports optional leader election so only one relay instance publishes at a time.
|
|
8
|
+
- **`qx.events.NatsPublisher`** — publishes a single `IntegrationEvent` to a JetStream subject derived from `event_name`.
|
|
9
|
+
- **`qx.events.NatsConsumer`** — durable pull consumer over a JetStream stream. Used by `WorkerRuntime` to fetch and ack/nak messages.
|
|
10
|
+
- **`qx.events.NatsSettings`** — Pydantic settings for NATS connection (URL, credentials, stream name, consumer name).
|
|
11
|
+
- **`qx.events.EventRegistry`** — maps `event_name` strings to concrete `IntegrationEvent` subclasses. Required by both the relay (for serialisation) and the worker (for deserialisation).
|
|
12
|
+
- **`qx.events.MediatorEventDispatcher`** — `EventDispatcher` implementation that routes domain events to their in-process handlers via the Mediator after a UnitOfWork commit.
|
|
13
|
+
- **`qx.events.create_nats_connection`** — async factory that opens a NATS connection with retry.
|
|
14
|
+
|
|
15
|
+
## Usage
|
|
16
|
+
|
|
17
|
+
### Outbox relay (alongside the API server)
|
|
18
|
+
|
|
19
|
+
```python
|
|
20
|
+
from qx.events import OutboxRelay, NatsPublisher, NatsSettings, EventRegistry
|
|
21
|
+
|
|
22
|
+
registry = EventRegistry()
|
|
23
|
+
registry.register(UserRegisteredIntegration)
|
|
24
|
+
|
|
25
|
+
publisher = NatsPublisher(nc, registry)
|
|
26
|
+
relay = OutboxRelay(session_factory, publisher, registry)
|
|
27
|
+
|
|
28
|
+
# Run in background
|
|
29
|
+
asyncio.create_task(relay.run())
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
### Consuming events in a worker
|
|
33
|
+
|
|
34
|
+
```python
|
|
35
|
+
from qx.events import NatsConsumer, NatsSettings
|
|
36
|
+
|
|
37
|
+
settings = NatsSettings(url="nats://localhost:4222", stream="events", consumer="identity-worker")
|
|
38
|
+
consumer = NatsConsumer(nc, settings)
|
|
39
|
+
|
|
40
|
+
# WorkerRuntime handles the fetch/ack loop
|
|
41
|
+
worker = WorkerRuntime(container, consumer, registry, mediator)
|
|
42
|
+
await worker.run()
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
## Design rules
|
|
46
|
+
|
|
47
|
+
- **At-least-once delivery** — the outbox guarantees every `INSERT`ed event is eventually published. Consumers are expected to be idempotent (use `IdempotencyStore` from `qx-cache`).
|
|
48
|
+
- **Transactional outbox** — `UnitOfWork` (in `qx-db`) writes events to `qx_outbox_events` in the same transaction as the aggregate; the relay reads and publishes asynchronously. No event is lost even if the process crashes between commit and publish.
|
|
49
|
+
- **Event envelope** — each NATS message carries `event_name`, `event_version`, payload JSON, `correlation_id`, `tenant_id`, and OTel trace context headers for full observability continuity.
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "qx-events"
|
|
3
|
+
version = "0.2.0"
|
|
4
|
+
description = "Qx events: NATS JetStream publisher/consumer, outbox relay, mediator dispatcher bridge"
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
requires-python = ">=3.14"
|
|
7
|
+
license = { text = "MIT" }
|
|
8
|
+
authors = [{ name = "Qx Engineering" }]
|
|
9
|
+
dependencies = [
|
|
10
|
+
"qx-core",
|
|
11
|
+
"qx-di",
|
|
12
|
+
"qx-cqrs",
|
|
13
|
+
"qx-db",
|
|
14
|
+
"qx-cache",
|
|
15
|
+
"qx-observability",
|
|
16
|
+
"nats-py>=2.9.0",
|
|
17
|
+
]
|
|
18
|
+
|
|
19
|
+
[build-system]
|
|
20
|
+
requires = ["hatchling"]
|
|
21
|
+
build-backend = "hatchling.build"
|
|
22
|
+
|
|
23
|
+
[tool.hatch.build.targets.wheel]
|
|
24
|
+
packages = ["src/qx"]
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
"""Qx events: outbox relay, NATS publisher/consumer, mediator bridge."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from qx.events.dispatcher import MediatorEventDispatcher
|
|
6
|
+
from qx.events.nats import (
|
|
7
|
+
NatsConsumer,
|
|
8
|
+
NatsPublisher,
|
|
9
|
+
NatsSettings,
|
|
10
|
+
create_nats_connection,
|
|
11
|
+
)
|
|
12
|
+
from qx.events.outbox_relay import OutboxRelay
|
|
13
|
+
from qx.events.registry import EventRegistry, EventTypeNotRegistered
|
|
14
|
+
|
|
15
|
+
__version__ = "0.2.0"
|
|
16
|
+
|
|
17
|
+
__all__ = [
|
|
18
|
+
"EventRegistry",
|
|
19
|
+
"EventTypeNotRegistered",
|
|
20
|
+
"MediatorEventDispatcher",
|
|
21
|
+
"NatsConsumer",
|
|
22
|
+
"NatsPublisher",
|
|
23
|
+
"NatsSettings",
|
|
24
|
+
"OutboxRelay",
|
|
25
|
+
"__version__",
|
|
26
|
+
"create_nats_connection",
|
|
27
|
+
]
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
"""Bridge between ``qx-db.UnitOfWork`` and ``qx-cqrs.Mediator``.
|
|
2
|
+
|
|
3
|
+
The UoW depends on a thin ``EventDispatcher`` protocol (one method,
|
|
4
|
+
``publish(event)``). The Mediator is a richer object. This module supplies a
|
|
5
|
+
trivial adapter so wiring in service bootstrap is one line::
|
|
6
|
+
|
|
7
|
+
container.register_singleton(
|
|
8
|
+
EventDispatcher,
|
|
9
|
+
lambda m: MediatorEventDispatcher(m),
|
|
10
|
+
)
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
from typing import TYPE_CHECKING
|
|
16
|
+
|
|
17
|
+
if TYPE_CHECKING:
|
|
18
|
+
from qx.core import DomainEvent
|
|
19
|
+
from qx.cqrs import Mediator
|
|
20
|
+
|
|
21
|
+
__all__ = ["MediatorEventDispatcher"]
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class MediatorEventDispatcher:
|
|
25
|
+
"""Adapt ``Mediator.publish`` to the ``EventDispatcher`` protocol."""
|
|
26
|
+
|
|
27
|
+
def __init__(self, mediator: Mediator) -> None:
|
|
28
|
+
self._m = mediator
|
|
29
|
+
|
|
30
|
+
async def publish(self, event: DomainEvent) -> None:
|
|
31
|
+
await self._m.publish(event)
|
|
@@ -0,0 +1,235 @@
|
|
|
1
|
+
"""NATS JetStream integration.
|
|
2
|
+
|
|
3
|
+
Two responsibilities:
|
|
4
|
+
|
|
5
|
+
- ``NatsPublisher``: durable publish to a JetStream subject. Used by the outbox
|
|
6
|
+
relay (and only the outbox relay — application code never publishes
|
|
7
|
+
integration events directly; that's the whole point of the outbox).
|
|
8
|
+
|
|
9
|
+
- ``NatsConsumer``: durable pull-subscription wrapper. Used by worker runtimes
|
|
10
|
+
to receive integration events with at-least-once semantics, ack/nack control,
|
|
11
|
+
and per-message tracing/logging.
|
|
12
|
+
|
|
13
|
+
NATS connection management:
|
|
14
|
+
- Single connection per process (singleton in DI).
|
|
15
|
+
- Auto-reconnect on disconnect (NATS client default).
|
|
16
|
+
- JetStream context obtained from the connection lazily.
|
|
17
|
+
|
|
18
|
+
Subject convention: ``<service>.<event_name>``, e.g., ``identity.user.registered``.
|
|
19
|
+
Streams should be configured at deployment (declarative), not at runtime; this
|
|
20
|
+
module assumes the stream exists.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
from __future__ import annotations
|
|
24
|
+
|
|
25
|
+
import json
|
|
26
|
+
from contextlib import asynccontextmanager
|
|
27
|
+
from typing import TYPE_CHECKING, Any, ClassVar
|
|
28
|
+
|
|
29
|
+
import nats
|
|
30
|
+
from nats.js.api import AckPolicy, ConsumerConfig, DeliverPolicy
|
|
31
|
+
from pydantic_settings import BaseSettings, SettingsConfigDict
|
|
32
|
+
from qx.events.registry import EventRegistry, EventTypeNotRegistered
|
|
33
|
+
|
|
34
|
+
if TYPE_CHECKING:
|
|
35
|
+
from collections.abc import AsyncIterator
|
|
36
|
+
|
|
37
|
+
from nats.aio.client import Client as NatsClient
|
|
38
|
+
from nats.aio.msg import Msg
|
|
39
|
+
from nats.js import JetStreamContext
|
|
40
|
+
from qx.core import IntegrationEvent
|
|
41
|
+
|
|
42
|
+
__all__ = [
|
|
43
|
+
"NatsConsumer",
|
|
44
|
+
"NatsPublisher",
|
|
45
|
+
"NatsSettings",
|
|
46
|
+
"create_nats_connection",
|
|
47
|
+
]
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
class NatsSettings(BaseSettings):
|
|
51
|
+
model_config: ClassVar[SettingsConfigDict] = SettingsConfigDict(
|
|
52
|
+
env_prefix="QX_NATS__",
|
|
53
|
+
extra="ignore",
|
|
54
|
+
)
|
|
55
|
+
|
|
56
|
+
servers: tuple[str, ...] = ("nats://localhost:4222",)
|
|
57
|
+
max_reconnect_attempts: int = 60
|
|
58
|
+
reconnect_time_wait_seconds: float = 2.0
|
|
59
|
+
connect_timeout_seconds: float = 5.0
|
|
60
|
+
name: str | None = None # client name shown in `nats-server -m monitoring`
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
async def create_nats_connection(settings: NatsSettings) -> NatsClient:
|
|
64
|
+
"""Connect to NATS with sensible reconnect behavior."""
|
|
65
|
+
return await nats.connect(
|
|
66
|
+
servers=list(settings.servers),
|
|
67
|
+
max_reconnect_attempts=settings.max_reconnect_attempts,
|
|
68
|
+
reconnect_time_wait=settings.reconnect_time_wait_seconds,
|
|
69
|
+
connect_timeout=settings.connect_timeout_seconds,
|
|
70
|
+
name=settings.name,
|
|
71
|
+
# Don't drain on close — we want to surface drops as errors.
|
|
72
|
+
drain_timeout=10,
|
|
73
|
+
)
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
# ============================================================
|
|
77
|
+
# Publisher
|
|
78
|
+
# ============================================================
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
class NatsPublisher:
|
|
82
|
+
"""Publish ``IntegrationEvent`` payloads to JetStream.
|
|
83
|
+
|
|
84
|
+
Subject is computed as ``f"{prefix}.{event_name}"``. The ``prefix`` is
|
|
85
|
+
typically the service name; configure once at construction.
|
|
86
|
+
"""
|
|
87
|
+
|
|
88
|
+
def __init__(
|
|
89
|
+
self,
|
|
90
|
+
connection: NatsClient,
|
|
91
|
+
*,
|
|
92
|
+
subject_prefix: str,
|
|
93
|
+
) -> None:
|
|
94
|
+
self._nc = connection
|
|
95
|
+
self._prefix = subject_prefix
|
|
96
|
+
self._js: JetStreamContext | None = None
|
|
97
|
+
|
|
98
|
+
def _jetstream(self) -> JetStreamContext:
|
|
99
|
+
if self._js is None:
|
|
100
|
+
self._js = self._nc.jetstream()
|
|
101
|
+
return self._js
|
|
102
|
+
|
|
103
|
+
def _subject(self, event_name: str) -> str:
|
|
104
|
+
return f"{self._prefix}.{event_name}"
|
|
105
|
+
|
|
106
|
+
async def publish(self, event: IntegrationEvent) -> None:
|
|
107
|
+
"""Publish a single event. Waits for JetStream ack."""
|
|
108
|
+
subject = self._subject(event.event_name)
|
|
109
|
+
payload = json.dumps(event.envelope()).encode()
|
|
110
|
+
# Headers carry W3C trace context and the event metadata in a way that
|
|
111
|
+
# consumers can read without deserializing the body. Useful for routing
|
|
112
|
+
# and observability sidecars.
|
|
113
|
+
headers = {
|
|
114
|
+
"qx.event_name": event.event_name,
|
|
115
|
+
"qx.event_version": str(event.event_version),
|
|
116
|
+
"qx.event_id": str(event.event_id),
|
|
117
|
+
}
|
|
118
|
+
if event.correlation_id:
|
|
119
|
+
headers["qx.correlation_id"] = str(event.correlation_id)
|
|
120
|
+
if event.tenant_id:
|
|
121
|
+
headers["qx.tenant_id"] = str(event.tenant_id)
|
|
122
|
+
await self._jetstream().publish(subject, payload, headers=headers)
|
|
123
|
+
|
|
124
|
+
async def publish_raw(
|
|
125
|
+
self,
|
|
126
|
+
event_name: str,
|
|
127
|
+
envelope: dict[str, Any],
|
|
128
|
+
*,
|
|
129
|
+
headers: dict[str, str] | None = None,
|
|
130
|
+
) -> None:
|
|
131
|
+
"""Publish a pre-serialized envelope. Used by the outbox relay."""
|
|
132
|
+
subject = self._subject(event_name)
|
|
133
|
+
hdrs = dict(headers or {})
|
|
134
|
+
hdrs.setdefault("qx.event_name", event_name)
|
|
135
|
+
await self._jetstream().publish(subject, json.dumps(envelope).encode(), headers=hdrs)
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
# ============================================================
|
|
139
|
+
# Consumer
|
|
140
|
+
# ============================================================
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
class NatsConsumer:
|
|
144
|
+
"""Durable pull-subscription for an integration-event stream.
|
|
145
|
+
|
|
146
|
+
Typical usage in the worker runtime::
|
|
147
|
+
|
|
148
|
+
async for msg, event in consumer.messages():
|
|
149
|
+
try:
|
|
150
|
+
await dispatch(event)
|
|
151
|
+
await msg.ack()
|
|
152
|
+
except Exception:
|
|
153
|
+
await msg.nak()
|
|
154
|
+
"""
|
|
155
|
+
|
|
156
|
+
def __init__(
|
|
157
|
+
self,
|
|
158
|
+
connection: NatsClient,
|
|
159
|
+
registry: EventRegistry,
|
|
160
|
+
*,
|
|
161
|
+
stream: str,
|
|
162
|
+
durable_name: str,
|
|
163
|
+
subject_filter: str,
|
|
164
|
+
batch_size: int = 32,
|
|
165
|
+
fetch_timeout_seconds: float = 5.0,
|
|
166
|
+
max_deliver: int = 5,
|
|
167
|
+
ack_wait_seconds: int = 30,
|
|
168
|
+
) -> None:
|
|
169
|
+
self._nc = connection
|
|
170
|
+
self._registry = registry
|
|
171
|
+
self._stream = stream
|
|
172
|
+
self._durable = durable_name
|
|
173
|
+
self._subject_filter = subject_filter
|
|
174
|
+
self._batch_size = batch_size
|
|
175
|
+
self._fetch_timeout = fetch_timeout_seconds
|
|
176
|
+
self._max_deliver = max_deliver
|
|
177
|
+
self._ack_wait = ack_wait_seconds
|
|
178
|
+
self._js: JetStreamContext | None = None
|
|
179
|
+
|
|
180
|
+
async def _ensure_consumer(self) -> Any:
|
|
181
|
+
"""Create or attach to the durable pull consumer."""
|
|
182
|
+
if self._js is None:
|
|
183
|
+
self._js = self._nc.jetstream()
|
|
184
|
+
config = ConsumerConfig(
|
|
185
|
+
durable_name=self._durable,
|
|
186
|
+
filter_subject=self._subject_filter,
|
|
187
|
+
ack_policy=AckPolicy.EXPLICIT,
|
|
188
|
+
deliver_policy=DeliverPolicy.ALL,
|
|
189
|
+
max_deliver=self._max_deliver,
|
|
190
|
+
ack_wait=self._ack_wait,
|
|
191
|
+
)
|
|
192
|
+
# add_consumer is idempotent if the config matches.
|
|
193
|
+
await self._js.add_consumer(self._stream, config=config)
|
|
194
|
+
return await self._js.pull_subscribe(
|
|
195
|
+
self._subject_filter,
|
|
196
|
+
durable=self._durable,
|
|
197
|
+
stream=self._stream,
|
|
198
|
+
)
|
|
199
|
+
|
|
200
|
+
@asynccontextmanager
|
|
201
|
+
async def messages(self) -> AsyncIterator[Any]:
|
|
202
|
+
"""Yield batches of ``(msg, IntegrationEvent)`` tuples.
|
|
203
|
+
|
|
204
|
+
Caller is responsible for ack/nak. Failures during fetch propagate;
|
|
205
|
+
the worker runtime catches and re-tries with backoff.
|
|
206
|
+
"""
|
|
207
|
+
sub = await self._ensure_consumer()
|
|
208
|
+
try:
|
|
209
|
+
yield sub
|
|
210
|
+
finally:
|
|
211
|
+
await sub.unsubscribe()
|
|
212
|
+
|
|
213
|
+
def parse_message(self, msg: Msg) -> IntegrationEvent:
|
|
214
|
+
"""Deserialize a NATS message into a concrete IntegrationEvent."""
|
|
215
|
+
envelope = json.loads(msg.data.decode())
|
|
216
|
+
event_name = envelope.get("event_name") or (msg.headers or {}).get("qx.event_name", "")
|
|
217
|
+
event_version = int(envelope.get("event_version", 1))
|
|
218
|
+
try:
|
|
219
|
+
cls = self._registry.lookup(event_name, event_version)
|
|
220
|
+
except EventTypeNotRegistered:
|
|
221
|
+
raise
|
|
222
|
+
# The envelope format puts business payload under "payload".
|
|
223
|
+
payload = envelope.get("payload", {})
|
|
224
|
+
# Rebuild the event with its meta fields.
|
|
225
|
+
return cls.model_validate(
|
|
226
|
+
{
|
|
227
|
+
**payload,
|
|
228
|
+
"event_id": envelope.get("event_id"),
|
|
229
|
+
"occurred_at": envelope.get("occurred_at"),
|
|
230
|
+
"correlation_id": envelope.get("correlation_id"),
|
|
231
|
+
"causation_id": envelope.get("causation_id"),
|
|
232
|
+
"tenant_id": envelope.get("tenant_id"),
|
|
233
|
+
"actor_id": envelope.get("actor_id"),
|
|
234
|
+
}
|
|
235
|
+
)
|
|
@@ -0,0 +1,216 @@
|
|
|
1
|
+
"""Outbox relay worker.
|
|
2
|
+
|
|
3
|
+
Background worker that drains the ``qx_outbox_events`` table to the message
|
|
4
|
+
broker. Architecture:
|
|
5
|
+
|
|
6
|
+
1. Periodically (default every 500ms) take a small batch of unpublished rows
|
|
7
|
+
with ``SELECT ... FOR UPDATE SKIP LOCKED`` so concurrent relay instances
|
|
8
|
+
never claim the same row.
|
|
9
|
+
2. For each row, publish to NATS JetStream and wait for the stream's ack.
|
|
10
|
+
3. On success: mark ``published_at = now()``.
|
|
11
|
+
4. On failure: increment ``attempts``, log, leave for retry.
|
|
12
|
+
|
|
13
|
+
**HA sharding** — run N relay workers in parallel, each owning a hash-based
|
|
14
|
+
shard of the outbox table (e.g., one per Kubernetes pod replica)::
|
|
15
|
+
|
|
16
|
+
# pod-0
|
|
17
|
+
relay = OutboxRelay(engine, publisher, shard_id=0, shard_count=3)
|
|
18
|
+
|
|
19
|
+
# pod-1
|
|
20
|
+
relay = OutboxRelay(engine, publisher, shard_id=1, shard_count=3)
|
|
21
|
+
|
|
22
|
+
# pod-2
|
|
23
|
+
relay = OutboxRelay(engine, publisher, shard_id=2, shard_count=3)
|
|
24
|
+
|
|
25
|
+
Each relay adds ``ABS(HASHTEXT(id::text)::bigint) % shard_count = shard_id``
|
|
26
|
+
to the batch query, so shards are disjoint with no coordinator overhead.
|
|
27
|
+
|
|
28
|
+
**Single-instance mode** — the default (``shard_count=1``) omits the shard
|
|
29
|
+
filter entirely and optionally uses a ``DistributedLock`` to guarantee a
|
|
30
|
+
single active relay under multi-replica deployments.
|
|
31
|
+
|
|
32
|
+
The relay is **separate** from the consumer worker (``qx-worker``) by
|
|
33
|
+
design — relay is service-internal infrastructure; consumer runs the
|
|
34
|
+
business handlers.
|
|
35
|
+
"""
|
|
36
|
+
|
|
37
|
+
from __future__ import annotations
|
|
38
|
+
|
|
39
|
+
import asyncio
|
|
40
|
+
import contextlib
|
|
41
|
+
import json
|
|
42
|
+
from datetime import UTC, datetime
|
|
43
|
+
from typing import TYPE_CHECKING
|
|
44
|
+
|
|
45
|
+
from qx.db.outbox import OUTBOX_TABLE_NAME
|
|
46
|
+
from qx.observability import get_logger, trace_span
|
|
47
|
+
from sqlalchemy import text
|
|
48
|
+
|
|
49
|
+
if TYPE_CHECKING:
|
|
50
|
+
from qx.cache import DistributedLock
|
|
51
|
+
from qx.events.nats import NatsPublisher
|
|
52
|
+
from sqlalchemy.ext.asyncio import AsyncEngine
|
|
53
|
+
|
|
54
|
+
__all__ = ["OutboxRelay"]
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
class OutboxRelay:
|
|
58
|
+
"""Polls the outbox table and publishes pending events.
|
|
59
|
+
|
|
60
|
+
Lifecycle: ``await relay.run()`` blocks until ``await relay.stop()`` is
|
|
61
|
+
called from a signal handler.
|
|
62
|
+
"""
|
|
63
|
+
|
|
64
|
+
def __init__(
|
|
65
|
+
self,
|
|
66
|
+
engine: AsyncEngine,
|
|
67
|
+
publisher: NatsPublisher,
|
|
68
|
+
*,
|
|
69
|
+
leader_lock: DistributedLock | None = None,
|
|
70
|
+
shard_id: int = 0,
|
|
71
|
+
shard_count: int = 1,
|
|
72
|
+
table_name: str = OUTBOX_TABLE_NAME,
|
|
73
|
+
batch_size: int = 100,
|
|
74
|
+
poll_interval_seconds: float = 0.5,
|
|
75
|
+
idle_poll_interval_seconds: float = 2.0,
|
|
76
|
+
max_attempts: int = 10,
|
|
77
|
+
) -> None:
|
|
78
|
+
if shard_count < 1:
|
|
79
|
+
raise ValueError(f"shard_count must be >= 1, got {shard_count}")
|
|
80
|
+
if not (0 <= shard_id < shard_count):
|
|
81
|
+
raise ValueError(
|
|
82
|
+
f"shard_id must be in [0, shard_count), got shard_id={shard_id}, shard_count={shard_count}"
|
|
83
|
+
)
|
|
84
|
+
self._engine = engine
|
|
85
|
+
self._publisher = publisher
|
|
86
|
+
self._lock = leader_lock
|
|
87
|
+
self._shard_id = shard_id
|
|
88
|
+
self._shard_count = shard_count
|
|
89
|
+
self._table = table_name
|
|
90
|
+
self._batch = batch_size
|
|
91
|
+
self._poll = poll_interval_seconds
|
|
92
|
+
self._idle_poll = idle_poll_interval_seconds
|
|
93
|
+
self._max_attempts = max_attempts
|
|
94
|
+
self._stop = asyncio.Event()
|
|
95
|
+
self._log = get_logger(f"qx.outbox-relay[{shard_id}/{shard_count}]")
|
|
96
|
+
|
|
97
|
+
async def stop(self) -> None:
|
|
98
|
+
self._stop.set()
|
|
99
|
+
|
|
100
|
+
async def run(self) -> None:
|
|
101
|
+
"""Main loop. Acquires the leader lock (if any), polls, publishes, repeats."""
|
|
102
|
+
self._log.info("outbox-relay starting")
|
|
103
|
+
while not self._stop.is_set():
|
|
104
|
+
try:
|
|
105
|
+
if self._lock is not None:
|
|
106
|
+
async with self._lock.acquired(ttl_seconds=60) as got:
|
|
107
|
+
if not got:
|
|
108
|
+
# Another instance is leader; back off.
|
|
109
|
+
await self._sleep(2.0)
|
|
110
|
+
continue
|
|
111
|
+
# Renew while we're working.
|
|
112
|
+
await self._drain_loop()
|
|
113
|
+
else:
|
|
114
|
+
await self._drain_loop()
|
|
115
|
+
except asyncio.CancelledError:
|
|
116
|
+
raise
|
|
117
|
+
except Exception as exc:
|
|
118
|
+
self._log.error("outbox-relay error: %s", exc, exc_info=True)
|
|
119
|
+
await self._sleep(2.0)
|
|
120
|
+
self._log.info("outbox-relay stopped")
|
|
121
|
+
|
|
122
|
+
async def _drain_loop(self) -> None:
|
|
123
|
+
while not self._stop.is_set():
|
|
124
|
+
with trace_span("outbox.batch"):
|
|
125
|
+
published = await self._process_one_batch()
|
|
126
|
+
if published == 0:
|
|
127
|
+
# Nothing pending; back off for the idle interval.
|
|
128
|
+
await self._sleep(self._idle_poll)
|
|
129
|
+
else:
|
|
130
|
+
await self._sleep(self._poll)
|
|
131
|
+
|
|
132
|
+
def _shard_clause(self) -> str:
|
|
133
|
+
if self._shard_count == 1:
|
|
134
|
+
return ""
|
|
135
|
+
# Cast to bigint before ABS to avoid int4 overflow on INT_MIN.
|
|
136
|
+
return " AND ABS(HASHTEXT(id::text)::bigint) % :shard_count = :shard_id\n"
|
|
137
|
+
|
|
138
|
+
async def _process_one_batch(self) -> int:
|
|
139
|
+
async with self._engine.begin() as conn:
|
|
140
|
+
res = await conn.execute(
|
|
141
|
+
text(
|
|
142
|
+
f"""
|
|
143
|
+
SELECT id, event_name, event_version, payload,
|
|
144
|
+
correlation_id, tenant_id, attempts
|
|
145
|
+
FROM {self._table}
|
|
146
|
+
WHERE published_at IS NULL
|
|
147
|
+
AND attempts < :max_attempts
|
|
148
|
+
{self._shard_clause()}ORDER BY occurred_at ASC
|
|
149
|
+
LIMIT :limit
|
|
150
|
+
FOR UPDATE SKIP LOCKED
|
|
151
|
+
"""
|
|
152
|
+
),
|
|
153
|
+
{
|
|
154
|
+
"limit": self._batch,
|
|
155
|
+
"max_attempts": self._max_attempts,
|
|
156
|
+
"shard_count": self._shard_count,
|
|
157
|
+
"shard_id": self._shard_id,
|
|
158
|
+
},
|
|
159
|
+
)
|
|
160
|
+
rows = list(res.mappings())
|
|
161
|
+
if not rows:
|
|
162
|
+
return 0
|
|
163
|
+
|
|
164
|
+
published_ids: list[str] = []
|
|
165
|
+
failed: list[tuple[str, str]] = [] # (id, error)
|
|
166
|
+
for row in rows:
|
|
167
|
+
envelope = (
|
|
168
|
+
json.loads(row["payload"])
|
|
169
|
+
if isinstance(row["payload"], str)
|
|
170
|
+
else row["payload"]
|
|
171
|
+
)
|
|
172
|
+
headers = {"qx.event_name": row["event_name"]}
|
|
173
|
+
if row["correlation_id"]:
|
|
174
|
+
headers["qx.correlation_id"] = str(row["correlation_id"])
|
|
175
|
+
if row["tenant_id"]:
|
|
176
|
+
headers["qx.tenant_id"] = str(row["tenant_id"])
|
|
177
|
+
try:
|
|
178
|
+
await self._publisher.publish_raw(
|
|
179
|
+
row["event_name"],
|
|
180
|
+
envelope,
|
|
181
|
+
headers=headers,
|
|
182
|
+
)
|
|
183
|
+
published_ids.append(row["id"])
|
|
184
|
+
except Exception as exc:
|
|
185
|
+
failed.append((row["id"], f"{type(exc).__name__}: {exc}"))
|
|
186
|
+
|
|
187
|
+
if published_ids:
|
|
188
|
+
await conn.execute(
|
|
189
|
+
text(
|
|
190
|
+
f"""
|
|
191
|
+
UPDATE {self._table}
|
|
192
|
+
SET published_at = :now
|
|
193
|
+
WHERE id = ANY(:ids)
|
|
194
|
+
"""
|
|
195
|
+
),
|
|
196
|
+
{"now": datetime.now(UTC), "ids": published_ids},
|
|
197
|
+
)
|
|
198
|
+
|
|
199
|
+
for fid, err in failed:
|
|
200
|
+
await conn.execute(
|
|
201
|
+
text(
|
|
202
|
+
f"""
|
|
203
|
+
UPDATE {self._table}
|
|
204
|
+
SET attempts = attempts + 1,
|
|
205
|
+
last_error = :err
|
|
206
|
+
WHERE id = :id
|
|
207
|
+
"""
|
|
208
|
+
),
|
|
209
|
+
{"err": err[:2000], "id": fid},
|
|
210
|
+
)
|
|
211
|
+
|
|
212
|
+
return len(published_ids)
|
|
213
|
+
|
|
214
|
+
async def _sleep(self, seconds: float) -> None:
|
|
215
|
+
with contextlib.suppress(TimeoutError):
|
|
216
|
+
await asyncio.wait_for(self._stop.wait(), timeout=seconds)
|
|
File without changes
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
"""Event type registry.
|
|
2
|
+
|
|
3
|
+
Maps ``event_name`` strings to the concrete ``IntegrationEvent`` subclass so
|
|
4
|
+
the consumer side can deserialize incoming messages. Without this, the
|
|
5
|
+
worker would receive an opaque dict and the application would have to do its
|
|
6
|
+
own dispatch.
|
|
7
|
+
|
|
8
|
+
Services register their event types once at bootstrap::
|
|
9
|
+
|
|
10
|
+
registry = EventRegistry()
|
|
11
|
+
registry.register(UserRegistered)
|
|
12
|
+
registry.register(OrderPlaced)
|
|
13
|
+
|
|
14
|
+
container.register_instance(EventRegistry, registry)
|
|
15
|
+
|
|
16
|
+
When the worker pulls a NATS message, the registry resolves the class:
|
|
17
|
+
|
|
18
|
+
cls = registry.lookup("user.registered")
|
|
19
|
+
event = cls.model_validate(payload)
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
from __future__ import annotations
|
|
23
|
+
|
|
24
|
+
from typing import TYPE_CHECKING
|
|
25
|
+
|
|
26
|
+
if TYPE_CHECKING:
|
|
27
|
+
from qx.core import IntegrationEvent
|
|
28
|
+
|
|
29
|
+
__all__ = ["EventRegistry", "EventTypeNotRegistered"]
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class EventTypeNotRegistered(Exception):
|
|
33
|
+
"""Raised when an incoming event_name has no matching class."""
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class EventRegistry:
|
|
37
|
+
"""Mapping from ``event_name`` to ``IntegrationEvent`` subclass."""
|
|
38
|
+
|
|
39
|
+
def __init__(self) -> None:
|
|
40
|
+
self._by_name: dict[tuple[str, int], type[IntegrationEvent]] = {}
|
|
41
|
+
|
|
42
|
+
def register(self, event_cls: type[IntegrationEvent]) -> None:
|
|
43
|
+
if not event_cls.event_name:
|
|
44
|
+
raise ValueError(f"{event_cls.__name__} has no event_name; cannot register")
|
|
45
|
+
key = (event_cls.event_name, event_cls.event_version)
|
|
46
|
+
existing = self._by_name.get(key)
|
|
47
|
+
if existing is not None and existing is not event_cls:
|
|
48
|
+
raise ValueError(
|
|
49
|
+
f"event_name {event_cls.event_name!r} v{event_cls.event_version} "
|
|
50
|
+
f"already registered to {existing.__name__}"
|
|
51
|
+
)
|
|
52
|
+
self._by_name[key] = event_cls
|
|
53
|
+
|
|
54
|
+
def lookup(self, event_name: str, version: int = 1) -> type[IntegrationEvent]:
|
|
55
|
+
try:
|
|
56
|
+
return self._by_name[(event_name, version)]
|
|
57
|
+
except KeyError as e:
|
|
58
|
+
raise EventTypeNotRegistered(
|
|
59
|
+
f"No event registered for {event_name!r} v{version}. "
|
|
60
|
+
f"Did you call EventRegistry.register({event_name})?"
|
|
61
|
+
) from e
|
|
62
|
+
|
|
63
|
+
def __len__(self) -> int:
|
|
64
|
+
return len(self._by_name)
|
|
65
|
+
|
|
66
|
+
def known_event_names(self) -> list[str]:
|
|
67
|
+
return sorted({name for name, _ in self._by_name})
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
"""Unit tests for events package — registry + dispatcher bridge."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import ClassVar
|
|
6
|
+
|
|
7
|
+
import pytest
|
|
8
|
+
from qx.core import DomainEvent, IntegrationEvent
|
|
9
|
+
from qx.events import EventRegistry, EventTypeNotRegistered, MediatorEventDispatcher
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class UserRegistered(IntegrationEvent):
|
|
13
|
+
event_name: ClassVar[str] = "identity.user.registered"
|
|
14
|
+
event_version: ClassVar[int] = 1
|
|
15
|
+
email: str
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class OrderPlaced(IntegrationEvent):
|
|
19
|
+
event_name: ClassVar[str] = "commerce.order.placed"
|
|
20
|
+
event_version: ClassVar[int] = 2
|
|
21
|
+
total_cents: int
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def test_registry_round_trip() -> None:
|
|
25
|
+
r = EventRegistry()
|
|
26
|
+
r.register(UserRegistered)
|
|
27
|
+
r.register(OrderPlaced)
|
|
28
|
+
assert r.lookup("identity.user.registered", 1) is UserRegistered
|
|
29
|
+
assert r.lookup("commerce.order.placed", 2) is OrderPlaced
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def test_unknown_event_raises() -> None:
|
|
33
|
+
r = EventRegistry()
|
|
34
|
+
with pytest.raises(EventTypeNotRegistered):
|
|
35
|
+
r.lookup("missing", 1)
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def test_duplicate_registration_with_same_class_is_noop() -> None:
|
|
39
|
+
r = EventRegistry()
|
|
40
|
+
r.register(UserRegistered)
|
|
41
|
+
r.register(UserRegistered) # idempotent
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def test_duplicate_registration_with_different_class_raises() -> None:
|
|
45
|
+
class OtherUserRegistered(IntegrationEvent):
|
|
46
|
+
event_name: ClassVar[str] = "identity.user.registered"
|
|
47
|
+
email: str
|
|
48
|
+
|
|
49
|
+
r = EventRegistry()
|
|
50
|
+
r.register(UserRegistered)
|
|
51
|
+
with pytest.raises(ValueError, match="already registered"):
|
|
52
|
+
r.register(OtherUserRegistered)
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
async def test_mediator_dispatcher_bridge_calls_publish() -> None:
|
|
56
|
+
class FakeMediator:
|
|
57
|
+
def __init__(self) -> None:
|
|
58
|
+
self.published: list[DomainEvent] = []
|
|
59
|
+
|
|
60
|
+
async def publish(self, ev: DomainEvent) -> None:
|
|
61
|
+
self.published.append(ev)
|
|
62
|
+
|
|
63
|
+
class Evt(DomainEvent):
|
|
64
|
+
event_name: ClassVar[str] = "x.something"
|
|
65
|
+
payload: str
|
|
66
|
+
|
|
67
|
+
m = FakeMediator()
|
|
68
|
+
d = MediatorEventDispatcher(m) # type: ignore[arg-type]
|
|
69
|
+
await d.publish(Evt(payload="hi"))
|
|
70
|
+
assert len(m.published) == 1
|
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
"""OutboxRelay sharding unit tests — no real DB or broker."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from contextlib import asynccontextmanager
|
|
6
|
+
from unittest.mock import AsyncMock, MagicMock
|
|
7
|
+
|
|
8
|
+
import pytest
|
|
9
|
+
from qx.events import OutboxRelay
|
|
10
|
+
|
|
11
|
+
# ---- construction / validation ----
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def test_default_is_single_shard() -> None:
|
|
15
|
+
relay = OutboxRelay(_mock_engine(), MagicMock())
|
|
16
|
+
assert relay._shard_id == 0
|
|
17
|
+
assert relay._shard_count == 1
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def test_valid_shard_configuration() -> None:
|
|
21
|
+
relay = OutboxRelay(_mock_engine(), MagicMock(), shard_id=2, shard_count=3)
|
|
22
|
+
assert relay._shard_id == 2
|
|
23
|
+
assert relay._shard_count == 3
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def test_shard_count_zero_raises() -> None:
|
|
27
|
+
with pytest.raises(ValueError, match="shard_count must be >= 1"):
|
|
28
|
+
OutboxRelay(_mock_engine(), MagicMock(), shard_count=0)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def test_shard_id_equal_to_shard_count_raises() -> None:
|
|
32
|
+
with pytest.raises(ValueError, match="shard_id must be in"):
|
|
33
|
+
OutboxRelay(_mock_engine(), MagicMock(), shard_id=3, shard_count=3)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def test_shard_id_negative_raises() -> None:
|
|
37
|
+
with pytest.raises(ValueError, match="shard_id must be in"):
|
|
38
|
+
OutboxRelay(_mock_engine(), MagicMock(), shard_id=-1, shard_count=3)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
# ---- shard clause generation ----
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def test_single_shard_produces_no_clause() -> None:
|
|
45
|
+
relay = OutboxRelay(_mock_engine(), MagicMock(), shard_id=0, shard_count=1)
|
|
46
|
+
assert relay._shard_clause() == ""
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def test_multi_shard_produces_hashtext_clause() -> None:
|
|
50
|
+
relay = OutboxRelay(_mock_engine(), MagicMock(), shard_id=1, shard_count=4)
|
|
51
|
+
clause = relay._shard_clause()
|
|
52
|
+
assert "HASHTEXT" in clause
|
|
53
|
+
assert "shard_count" in clause
|
|
54
|
+
assert "shard_id" in clause
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
# ---- batch query SQL ----
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
@pytest.mark.anyio
|
|
61
|
+
async def test_single_shard_batch_query_has_no_hashtext() -> None:
|
|
62
|
+
conn, engine = _mock_conn_engine()
|
|
63
|
+
relay = OutboxRelay(engine, MagicMock(), shard_id=0, shard_count=1)
|
|
64
|
+
|
|
65
|
+
await relay._process_one_batch()
|
|
66
|
+
|
|
67
|
+
sql = _first_select_sql(conn)
|
|
68
|
+
assert "HASHTEXT" not in sql
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
@pytest.mark.anyio
|
|
72
|
+
async def test_multi_shard_batch_query_includes_hashtext_filter() -> None:
|
|
73
|
+
conn, engine = _mock_conn_engine()
|
|
74
|
+
relay = OutboxRelay(engine, MagicMock(), shard_id=2, shard_count=5)
|
|
75
|
+
|
|
76
|
+
await relay._process_one_batch()
|
|
77
|
+
|
|
78
|
+
sql = _first_select_sql(conn)
|
|
79
|
+
assert "HASHTEXT" in sql
|
|
80
|
+
assert "shard_count" in sql
|
|
81
|
+
assert "shard_id" in sql
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
@pytest.mark.anyio
|
|
85
|
+
async def test_batch_query_params_include_shard_values() -> None:
|
|
86
|
+
conn, engine = _mock_conn_engine()
|
|
87
|
+
relay = OutboxRelay(engine, MagicMock(), shard_id=1, shard_count=3)
|
|
88
|
+
|
|
89
|
+
await relay._process_one_batch()
|
|
90
|
+
|
|
91
|
+
params = conn.execute.call_args_list[0].args[1]
|
|
92
|
+
assert params["shard_count"] == 3
|
|
93
|
+
assert params["shard_id"] == 1
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
@pytest.mark.anyio
|
|
97
|
+
async def test_batch_query_always_has_skip_locked() -> None:
|
|
98
|
+
conn, engine = _mock_conn_engine()
|
|
99
|
+
relay = OutboxRelay(engine, MagicMock(), shard_id=0, shard_count=4)
|
|
100
|
+
|
|
101
|
+
await relay._process_one_batch()
|
|
102
|
+
|
|
103
|
+
sql = _first_select_sql(conn)
|
|
104
|
+
assert "FOR UPDATE SKIP LOCKED" in sql
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
@pytest.mark.anyio
|
|
108
|
+
async def test_returns_zero_when_no_rows() -> None:
|
|
109
|
+
_, engine = _mock_conn_engine()
|
|
110
|
+
relay = OutboxRelay(engine, MagicMock())
|
|
111
|
+
|
|
112
|
+
result = await relay._process_one_batch()
|
|
113
|
+
|
|
114
|
+
assert result == 0
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
# ---- helpers ----
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def _mock_engine() -> MagicMock:
|
|
121
|
+
conn = AsyncMock()
|
|
122
|
+
mock_result = MagicMock()
|
|
123
|
+
mock_result.mappings.return_value = []
|
|
124
|
+
conn.execute = AsyncMock(return_value=mock_result)
|
|
125
|
+
|
|
126
|
+
@asynccontextmanager
|
|
127
|
+
async def _begin():
|
|
128
|
+
yield conn
|
|
129
|
+
|
|
130
|
+
engine = MagicMock()
|
|
131
|
+
engine.begin = MagicMock(return_value=_begin())
|
|
132
|
+
return engine
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def _mock_conn_engine() -> tuple[AsyncMock, MagicMock]:
|
|
136
|
+
conn = AsyncMock()
|
|
137
|
+
mock_result = MagicMock()
|
|
138
|
+
mock_result.mappings.return_value = []
|
|
139
|
+
conn.execute = AsyncMock(return_value=mock_result)
|
|
140
|
+
|
|
141
|
+
@asynccontextmanager
|
|
142
|
+
async def _begin():
|
|
143
|
+
yield conn
|
|
144
|
+
|
|
145
|
+
engine = MagicMock()
|
|
146
|
+
engine.begin = MagicMock(return_value=_begin())
|
|
147
|
+
return conn, engine
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def _first_select_sql(conn: AsyncMock) -> str:
|
|
151
|
+
arg = conn.execute.call_args_list[0].args[0]
|
|
152
|
+
return arg.text if hasattr(arg, "text") else str(arg)
|