txoutbox 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- txoutbox-0.1.0/.gitignore +21 -0
- txoutbox-0.1.0/CHANGELOG.md +37 -0
- txoutbox-0.1.0/LICENSE +21 -0
- txoutbox-0.1.0/PKG-INFO +216 -0
- txoutbox-0.1.0/README.md +187 -0
- txoutbox-0.1.0/examples/sqlite_demo.py +66 -0
- txoutbox-0.1.0/pyproject.toml +77 -0
- txoutbox-0.1.0/src/txoutbox/__init__.py +27 -0
- txoutbox-0.1.0/src/txoutbox/adapters/__init__.py +7 -0
- txoutbox-0.1.0/src/txoutbox/adapters/memory.py +162 -0
- txoutbox-0.1.0/src/txoutbox/adapters/postgres.py +201 -0
- txoutbox-0.1.0/src/txoutbox/adapters/sqlite.py +216 -0
- txoutbox-0.1.0/src/txoutbox/backoff.py +38 -0
- txoutbox-0.1.0/src/txoutbox/message.py +33 -0
- txoutbox-0.1.0/src/txoutbox/poller.py +55 -0
- txoutbox-0.1.0/src/txoutbox/protocols.py +69 -0
- txoutbox-0.1.0/src/txoutbox/py.typed +0 -0
- txoutbox-0.1.0/src/txoutbox/relay.py +291 -0
- txoutbox-0.1.0/src/txoutbox/testing.py +143 -0
- txoutbox-0.1.0/tests/conftest.py +3 -0
- txoutbox-0.1.0/tests/test_adapters.py +73 -0
- txoutbox-0.1.0/tests/test_postgres.py +142 -0
- txoutbox-0.1.0/tests/test_primitives.py +71 -0
- txoutbox-0.1.0/tests/test_relay.py +252 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
# Python-generated files
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[oc]
|
|
4
|
+
build/
|
|
5
|
+
dist/
|
|
6
|
+
wheels/
|
|
7
|
+
*.egg-info
|
|
8
|
+
|
|
9
|
+
# Virtual environments
|
|
10
|
+
.venv
|
|
11
|
+
|
|
12
|
+
# Tool caches
|
|
13
|
+
.mypy_cache/
|
|
14
|
+
.ruff_cache/
|
|
15
|
+
.pytest_cache/
|
|
16
|
+
|
|
17
|
+
# Local databases
|
|
18
|
+
*.db
|
|
19
|
+
*.db-wal
|
|
20
|
+
*.db-shm
|
|
21
|
+
pgdata/
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to this project will be documented in this file.
|
|
4
|
+
|
|
5
|
+
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
|
|
6
|
+
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
7
|
+
|
|
8
|
+
## [Unreleased]
|
|
9
|
+
|
|
10
|
+
## [0.1.0] - 2026-09-12
|
|
11
|
+
|
|
12
|
+
### Added
|
|
13
|
+
|
|
14
|
+
- `Relay`: storage-agnostic Transactional Outbox relay for asyncio, with
|
|
15
|
+
adaptive polling that backs off while the table is empty and snaps back when
|
|
16
|
+
work appears, and a `wake()` hook for LISTEN/NOTIFY-style triggers.
|
|
17
|
+
- Per-key ordering: messages sharing a key are published sequentially in claim
|
|
18
|
+
order, while different keys are published concurrently up to `concurrency`.
|
|
19
|
+
A failure blocks the rest of its key group for the round so retries never
|
|
20
|
+
reorder a key.
|
|
21
|
+
- Retries with exponential backoff and full jitter (`Backoff`), attempt
|
|
22
|
+
counting at claim time, and dead-lettering after `max_attempts`.
|
|
23
|
+
- Leases: claimed rows are invisible to other workers until the lease expires,
|
|
24
|
+
so a crashed worker's batch is recovered automatically.
|
|
25
|
+
- Graceful shutdown: `stop()`, optional SIGINT/SIGTERM handling, and an async
|
|
26
|
+
context manager that runs the relay as a background task.
|
|
27
|
+
- `Hooks` for observability (`on_claimed`, `on_published`, `on_retry`,
|
|
28
|
+
`on_dead_letter`, `on_blocked`); hook errors never break delivery.
|
|
29
|
+
- Adapters: `MemoryStorage` and `MemoryPublisher` for tests and demos,
|
|
30
|
+
`SqliteStorage` on the standard library, and `PostgresStorage` on `asyncpg`
|
|
31
|
+
using `FOR UPDATE SKIP LOCKED` (extra `postgres`).
|
|
32
|
+
- `txoutbox.testing.StorageContract`: a reusable test suite that verifies any
|
|
33
|
+
`Storage` implementation against the relay's expectations, including lease
|
|
34
|
+
expiry, concurrent claims and strict per-key ordering.
|
|
35
|
+
|
|
36
|
+
[Unreleased]: https://github.com/OsmnvAslan/txoutbox/compare/v0.1.0...HEAD
|
|
37
|
+
[0.1.0]: https://github.com/OsmnvAslan/txoutbox/releases/tag/v0.1.0
|
txoutbox-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Aslan Osmanov
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
txoutbox-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,216 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: txoutbox
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Storage-agnostic Transactional Outbox relay for asyncio: bring your own table, bring your own broker.
|
|
5
|
+
Project-URL: Homepage, https://github.com/OsmnvAslan/txoutbox
|
|
6
|
+
Project-URL: Repository, https://github.com/OsmnvAslan/txoutbox
|
|
7
|
+
Project-URL: Issues, https://github.com/OsmnvAslan/txoutbox/issues
|
|
8
|
+
Project-URL: Changelog, https://github.com/OsmnvAslan/txoutbox/blob/main/CHANGELOG.md
|
|
9
|
+
Author-email: Aslan Osmanov <osmnvaslan@gmail.com>
|
|
10
|
+
License-Expression: MIT
|
|
11
|
+
License-File: LICENSE
|
|
12
|
+
Keywords: asyncio,event-driven,messaging,outbox,reliability,transactional-outbox
|
|
13
|
+
Classifier: Development Status :: 3 - Alpha
|
|
14
|
+
Classifier: Framework :: AsyncIO
|
|
15
|
+
Classifier: Intended Audience :: Developers
|
|
16
|
+
Classifier: Operating System :: OS Independent
|
|
17
|
+
Classifier: Programming Language :: Python :: 3
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
22
|
+
Classifier: Topic :: Software Development :: Libraries
|
|
23
|
+
Classifier: Topic :: System :: Distributed Computing
|
|
24
|
+
Classifier: Typing :: Typed
|
|
25
|
+
Requires-Python: >=3.11
|
|
26
|
+
Provides-Extra: postgres
|
|
27
|
+
Requires-Dist: asyncpg>=0.29; extra == 'postgres'
|
|
28
|
+
Description-Content-Type: text/markdown
|
|
29
|
+
|
|
30
|
+
# txoutbox
|
|
31
|
+
|
|
32
|
+
**Storage-agnostic Transactional Outbox relay for asyncio.** Bring your own table, bring your own broker.
|
|
33
|
+
|
|
34
|
+
[](https://pypi.org/project/txoutbox/)
|
|
35
|
+
[](https://pypi.org/project/txoutbox/)
|
|
36
|
+
[](https://github.com/OsmnvAslan/txoutbox/actions/workflows/ci.yml)
|
|
37
|
+
[](LICENSE)
|
|
38
|
+
|
|
39
|
+
Zero dependencies. Fully typed. Python 3.11+.
|
|
40
|
+
|
|
41
|
+
## The problem
|
|
42
|
+
|
|
43
|
+
Your service writes to the database and must tell the world about it: Kafka, NSQ, RabbitMQ, a webhook.
|
|
44
|
+
Publishing straight from request code is unreliable. If the broker is down after commit, the event is lost.
|
|
45
|
+
If you publish before commit and the transaction rolls back, you announced something that never happened.
|
|
46
|
+
|
|
47
|
+
The Transactional Outbox pattern fixes that: write the event to an `outbox` table **in the same transaction**
|
|
48
|
+
as your business data, and let a separate relay process deliver it. The idea is simple. The relay is not.
|
|
49
|
+
Every team writes one, and every team hits the same bugs: two workers grab the same row, a crashed worker
|
|
50
|
+
holds a row forever, events for one order get reordered, an empty table gets polled every 10 ms, retries
|
|
51
|
+
hammer a broker that is already down, and `SIGTERM` kills the process mid-batch.
|
|
52
|
+
|
|
53
|
+
Existing libraries solve this bundled with infrastructure: their table, their ORM, their broker, sometimes
|
|
54
|
+
their whole framework. If your stack is Django ORM + NSQ, raw asyncpg + webhooks, or an outbox table you
|
|
55
|
+
are not allowed to change, you are back to writing your own relay.
|
|
56
|
+
|
|
57
|
+
## What txoutbox does
|
|
58
|
+
|
|
59
|
+
Only the relay. You implement two small protocols, and txoutbox handles the hard parts:
|
|
60
|
+
|
|
61
|
+
| Concern | How |
|
|
62
|
+
| --- | --- |
|
|
63
|
+
| Concurrent workers | Leases: `claim()` hands each row to one worker for a limited time |
|
|
64
|
+
| Crashed workers | Leases expire, the row is claimed again, the attempt is counted |
|
|
65
|
+
| Ordering | Messages sharing a `key` are published sequentially, in insertion order; different keys run in parallel |
|
|
66
|
+
| Retries | Exponential backoff with full jitter, per message, scheduled in storage |
|
|
67
|
+
| Poison messages | Dead-lettered after `max_attempts` |
|
|
68
|
+
| Idle polling | Adaptive: sleep grows from 50 ms to 5 s while the table is empty, resets on work; `wake()` for LISTEN/NOTIFY |
|
|
69
|
+
| Broker hangs | Optional per-message `publish_timeout` |
|
|
70
|
+
| Shutdown | `SIGTERM`/`SIGINT` finish the round in flight, then return |
|
|
71
|
+
| Observability | `Hooks` with `on_published`, `on_retry`, `on_dead_letter`, `on_blocked`, `on_claimed` |
|
|
72
|
+
|
|
73
|
+
## Install
|
|
74
|
+
|
|
75
|
+
```bash
|
|
76
|
+
pip install txoutbox # core, zero dependencies
|
|
77
|
+
pip install "txoutbox[postgres]" # + asyncpg adapter
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
## 60 seconds
|
|
81
|
+
|
|
82
|
+
```python
|
|
83
|
+
import asyncio, sqlite3
|
|
84
|
+
from txoutbox import Relay, OutboxMessage
|
|
85
|
+
from txoutbox.adapters.sqlite import SqliteStorage
|
|
86
|
+
|
|
87
|
+
storage = SqliteStorage("app.db")
|
|
88
|
+
storage.create_schema()
|
|
89
|
+
|
|
90
|
+
# --- producer side: business row + outbox row, one transaction ---
|
|
91
|
+
conn = sqlite3.connect("app.db")
|
|
92
|
+
conn.execute("CREATE TABLE IF NOT EXISTS orders (id INTEGER PRIMARY KEY, total INTEGER)")
|
|
93
|
+
with conn:
|
|
94
|
+
cur = conn.execute("INSERT INTO orders (total) VALUES (42)")
|
|
95
|
+
storage.insert(conn, "orders.created", b'{"total": 42}', key=f"order-{cur.lastrowid}")
|
|
96
|
+
|
|
97
|
+
# --- relay side: your publisher is two lines ---
|
|
98
|
+
class PrintPublisher:
|
|
99
|
+
async def publish(self, message: OutboxMessage) -> None:
|
|
100
|
+
print(message.topic, message.key, message.payload)
|
|
101
|
+
|
|
102
|
+
async def main():
|
|
103
|
+
relay = Relay(storage, PrintPublisher())
|
|
104
|
+
await relay.run(handle_signals=True) # Ctrl-C to stop
|
|
105
|
+
|
|
106
|
+
asyncio.run(main())
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
Swap `SqliteStorage` for `PostgresStorage`, or your own adapter, and `PrintPublisher` for Kafka, NSQ, an HTTP client.
|
|
110
|
+
The relay does not change.
|
|
111
|
+
|
|
112
|
+
## The two protocols
|
|
113
|
+
|
|
114
|
+
```python
|
|
115
|
+
class Storage(Protocol):
|
|
116
|
+
async def claim(self, *, batch_size: int, lease: timedelta, worker_id: str) -> Sequence[OutboxMessage]: ...
|
|
117
|
+
async def ack(self, ids: Sequence[MessageId]) -> None: ...
|
|
118
|
+
async def nack(self, message_id: MessageId, *, error: str, retry_at: datetime) -> None: ...
|
|
119
|
+
async def dead_letter(self, message_id: MessageId, *, error: str) -> None: ...
|
|
120
|
+
|
|
121
|
+
class Publisher(Protocol):
|
|
122
|
+
async def publish(self, message: OutboxMessage) -> None: ... # raise to retry
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
`OutboxMessage` carries `id`, `topic`, `payload: bytes`, optional `key`, `headers`, `attempts`, `created_at`.
|
|
126
|
+
The `id` is opaque: the relay only hands it back to your storage.
|
|
127
|
+
|
|
128
|
+
### Writing your own storage adapter
|
|
129
|
+
|
|
130
|
+
Implement the four methods over your table, then let the contract tests check the tricky bits
|
|
131
|
+
(lease expiry, concurrent claims, retry scheduling, ordering):
|
|
132
|
+
|
|
133
|
+
```python
|
|
134
|
+
import pytest
|
|
135
|
+
from txoutbox.testing import StorageContract
|
|
136
|
+
|
|
137
|
+
class TestMyStorage(StorageContract):
|
|
138
|
+
@pytest.fixture
|
|
139
|
+
async def storage(self):
|
|
140
|
+
return MyStorage(...) # fresh, empty, per test
|
|
141
|
+
|
|
142
|
+
async def insert(self, storage, topic, payload, key=None):
|
|
143
|
+
return await storage.add(topic, payload, key=key)
|
|
144
|
+
```
|
|
145
|
+
|
|
146
|
+
The rules an adapter must follow are short:
|
|
147
|
+
|
|
148
|
+
1. `claim` returns due, unleased, pending rows in insertion order, leases them for `lease`, and increments
|
|
149
|
+
`attempts` **at claim time** (a crash mid-batch must burn an attempt).
|
|
150
|
+
2. `ack` marks rows delivered: delete them or flip a status column, your call.
|
|
151
|
+
3. `nack` releases the lease and stores `retry_at`; `dead_letter` releases the lease and parks the row.
|
|
152
|
+
4. Optional but recommended, *strict ordering*: hold back a row while an earlier pending row with the same
|
|
153
|
+
key is not claimable right now (leased elsewhere, or waiting for a retry). Both built-in SQL adapters do
|
|
154
|
+
this with one `NOT EXISTS` clause. The relay guarantees order within a batch on its own; strict ordering
|
|
155
|
+
in storage extends the guarantee across retries and across workers.
|
|
156
|
+
|
|
157
|
+
## Built-in adapters
|
|
158
|
+
|
|
159
|
+
| Adapter | Module | Notes |
|
|
160
|
+
| --- | --- | --- |
|
|
161
|
+
| `MemoryStorage`, `MemoryPublisher` | `txoutbox.adapters.memory` | Tests, demos, deterministic |
|
|
162
|
+
| `SqliteStorage` | `txoutbox.adapters.sqlite` | Stdlib `sqlite3`, `BEGIN IMMEDIATE`, WAL |
|
|
163
|
+
| `PostgresStorage` | `txoutbox.adapters.postgres` | asyncpg, `FOR UPDATE SKIP LOCKED`, extra `postgres` |
|
|
164
|
+
|
|
165
|
+
Both SQL adapters expose `insert(conn, ...)` so you write the outbox row **inside your own transaction**,
|
|
166
|
+
and `schema_sql()` / `create_schema()` so the table lives in your migrations.
|
|
167
|
+
|
|
168
|
+
## Configuration
|
|
169
|
+
|
|
170
|
+
```python
|
|
171
|
+
from datetime import timedelta
|
|
172
|
+
from txoutbox import Backoff, RelayConfig
|
|
173
|
+
|
|
174
|
+
RelayConfig(
|
|
175
|
+
batch_size=100, # rows per round
|
|
176
|
+
lease=timedelta(seconds=30), # must exceed the time to publish one round
|
|
177
|
+
concurrency=10, # ordering groups published in parallel
|
|
178
|
+
max_attempts=10, # then dead-letter
|
|
179
|
+
backoff=Backoff(base=timedelta(seconds=1), factor=2, maximum=timedelta(minutes=5), jitter=0.25),
|
|
180
|
+
publish_timeout=None, # seconds per publish, None = no limit
|
|
181
|
+
poll_min=0.05, poll_max=5.0, poll_factor=2.0,
|
|
182
|
+
storage_error_delay=1.0, # sleep after a storage exception
|
|
183
|
+
)
|
|
184
|
+
```
|
|
185
|
+
|
|
186
|
+
Three ways to run:
|
|
187
|
+
|
|
188
|
+
```python
|
|
189
|
+
await relay.run(handle_signals=True) # foreground loop, until SIGTERM/SIGINT or relay.stop()
|
|
190
|
+
|
|
191
|
+
async with Relay(storage, publisher): # background task for the lifetime of the block
|
|
192
|
+
await app.serve()
|
|
193
|
+
|
|
194
|
+
result = await relay.run_once() # one round; RoundResult(claimed, published, retried, dead_lettered, blocked)
|
|
195
|
+
```
|
|
196
|
+
|
|
197
|
+
## Delivery semantics
|
|
198
|
+
|
|
199
|
+
* **At-least-once.** A crash between `publish` and `ack` redelivers the batch after the lease expires.
|
|
200
|
+
Consumers must be idempotent; a message's `id` is a natural deduplication key.
|
|
201
|
+
* **Ordered per key.** Within one round, messages sharing a key are published one after another, and a
|
|
202
|
+
failure blocks the rest of that key for the round (they are rescheduled together with the failed one, without
|
|
203
|
+
burning an attempt). With strict-ordering storage, this holds across retries and workers too. Keyless
|
|
204
|
+
messages are independent.
|
|
205
|
+
* **Leases bound duplicates.** Keep `lease` well above `batch_size / concurrency * publish_time`; set
|
|
206
|
+
`publish_timeout` so a hanging broker cannot outlive the lease.
|
|
207
|
+
* **Storage bookkeeping failures are tolerated.** If `nack` itself fails, the lease expiry recovers the row.
|
|
208
|
+
|
|
209
|
+
## What txoutbox is not
|
|
210
|
+
|
|
211
|
+
Not a broker, not a task queue, not an ORM. It does not own your table and does not promise exactly-once.
|
|
212
|
+
Deduplication on the consumer side is a separate concern (see `inboxd`, the companion library).
|
|
213
|
+
|
|
214
|
+
## License
|
|
215
|
+
|
|
216
|
+
MIT
|
txoutbox-0.1.0/README.md
ADDED
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
# txoutbox
|
|
2
|
+
|
|
3
|
+
**Storage-agnostic Transactional Outbox relay for asyncio.** Bring your own table, bring your own broker.
|
|
4
|
+
|
|
5
|
+
[](https://pypi.org/project/txoutbox/)
|
|
6
|
+
[](https://pypi.org/project/txoutbox/)
|
|
7
|
+
[](https://github.com/OsmnvAslan/txoutbox/actions/workflows/ci.yml)
|
|
8
|
+
[](LICENSE)
|
|
9
|
+
|
|
10
|
+
Zero dependencies. Fully typed. Python 3.11+.
|
|
11
|
+
|
|
12
|
+
## The problem
|
|
13
|
+
|
|
14
|
+
Your service writes to the database and must tell the world about it: Kafka, NSQ, RabbitMQ, a webhook.
|
|
15
|
+
Publishing straight from request code is unreliable. If the broker is down after commit, the event is lost.
|
|
16
|
+
If you publish before commit and the transaction rolls back, you announced something that never happened.
|
|
17
|
+
|
|
18
|
+
The Transactional Outbox pattern fixes that: write the event to an `outbox` table **in the same transaction**
|
|
19
|
+
as your business data, and let a separate relay process deliver it. The idea is simple. The relay is not.
|
|
20
|
+
Every team writes one, and every team hits the same bugs: two workers grab the same row, a crashed worker
|
|
21
|
+
holds a row forever, events for one order get reordered, an empty table gets polled every 10 ms, retries
|
|
22
|
+
hammer a broker that is already down, and `SIGTERM` kills the process mid-batch.
|
|
23
|
+
|
|
24
|
+
Existing libraries solve this bundled with infrastructure: their table, their ORM, their broker, sometimes
|
|
25
|
+
their whole framework. If your stack is Django ORM + NSQ, raw asyncpg + webhooks, or an outbox table you
|
|
26
|
+
are not allowed to change, you are back to writing your own relay.
|
|
27
|
+
|
|
28
|
+
## What txoutbox does
|
|
29
|
+
|
|
30
|
+
Only the relay. You implement two small protocols, and txoutbox handles the hard parts:
|
|
31
|
+
|
|
32
|
+
| Concern | How |
|
|
33
|
+
| --- | --- |
|
|
34
|
+
| Concurrent workers | Leases: `claim()` hands each row to one worker for a limited time |
|
|
35
|
+
| Crashed workers | Leases expire, the row is claimed again, the attempt is counted |
|
|
36
|
+
| Ordering | Messages sharing a `key` are published sequentially, in insertion order; different keys run in parallel |
|
|
37
|
+
| Retries | Exponential backoff with full jitter, per message, scheduled in storage |
|
|
38
|
+
| Poison messages | Dead-lettered after `max_attempts` |
|
|
39
|
+
| Idle polling | Adaptive: sleep grows from 50 ms to 5 s while the table is empty, resets on work; `wake()` for LISTEN/NOTIFY |
|
|
40
|
+
| Broker hangs | Optional per-message `publish_timeout` |
|
|
41
|
+
| Shutdown | `SIGTERM`/`SIGINT` finish the round in flight, then return |
|
|
42
|
+
| Observability | `Hooks` with `on_published`, `on_retry`, `on_dead_letter`, `on_blocked`, `on_claimed` |
|
|
43
|
+
|
|
44
|
+
## Install
|
|
45
|
+
|
|
46
|
+
```bash
|
|
47
|
+
pip install txoutbox # core, zero dependencies
|
|
48
|
+
pip install "txoutbox[postgres]" # + asyncpg adapter
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
## 60 seconds
|
|
52
|
+
|
|
53
|
+
```python
|
|
54
|
+
import asyncio, sqlite3
|
|
55
|
+
from txoutbox import Relay, OutboxMessage
|
|
56
|
+
from txoutbox.adapters.sqlite import SqliteStorage
|
|
57
|
+
|
|
58
|
+
storage = SqliteStorage("app.db")
|
|
59
|
+
storage.create_schema()
|
|
60
|
+
|
|
61
|
+
# --- producer side: business row + outbox row, one transaction ---
|
|
62
|
+
conn = sqlite3.connect("app.db")
|
|
63
|
+
conn.execute("CREATE TABLE IF NOT EXISTS orders (id INTEGER PRIMARY KEY, total INTEGER)")
|
|
64
|
+
with conn:
|
|
65
|
+
cur = conn.execute("INSERT INTO orders (total) VALUES (42)")
|
|
66
|
+
storage.insert(conn, "orders.created", b'{"total": 42}', key=f"order-{cur.lastrowid}")
|
|
67
|
+
|
|
68
|
+
# --- relay side: your publisher is two lines ---
|
|
69
|
+
class PrintPublisher:
|
|
70
|
+
async def publish(self, message: OutboxMessage) -> None:
|
|
71
|
+
print(message.topic, message.key, message.payload)
|
|
72
|
+
|
|
73
|
+
async def main():
|
|
74
|
+
relay = Relay(storage, PrintPublisher())
|
|
75
|
+
await relay.run(handle_signals=True) # Ctrl-C to stop
|
|
76
|
+
|
|
77
|
+
asyncio.run(main())
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
Swap `SqliteStorage` for `PostgresStorage`, or your own adapter, and `PrintPublisher` for Kafka, NSQ, an HTTP client.
|
|
81
|
+
The relay does not change.
|
|
82
|
+
|
|
83
|
+
## The two protocols
|
|
84
|
+
|
|
85
|
+
```python
|
|
86
|
+
class Storage(Protocol):
|
|
87
|
+
async def claim(self, *, batch_size: int, lease: timedelta, worker_id: str) -> Sequence[OutboxMessage]: ...
|
|
88
|
+
async def ack(self, ids: Sequence[MessageId]) -> None: ...
|
|
89
|
+
async def nack(self, message_id: MessageId, *, error: str, retry_at: datetime) -> None: ...
|
|
90
|
+
async def dead_letter(self, message_id: MessageId, *, error: str) -> None: ...
|
|
91
|
+
|
|
92
|
+
class Publisher(Protocol):
|
|
93
|
+
async def publish(self, message: OutboxMessage) -> None: ... # raise to retry
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
`OutboxMessage` carries `id`, `topic`, `payload: bytes`, optional `key`, `headers`, `attempts`, `created_at`.
|
|
97
|
+
The `id` is opaque: the relay only hands it back to your storage.
|
|
98
|
+
|
|
99
|
+
### Writing your own storage adapter
|
|
100
|
+
|
|
101
|
+
Implement the four methods over your table, then let the contract tests check the tricky bits
|
|
102
|
+
(lease expiry, concurrent claims, retry scheduling, ordering):
|
|
103
|
+
|
|
104
|
+
```python
|
|
105
|
+
import pytest
|
|
106
|
+
from txoutbox.testing import StorageContract
|
|
107
|
+
|
|
108
|
+
class TestMyStorage(StorageContract):
|
|
109
|
+
@pytest.fixture
|
|
110
|
+
async def storage(self):
|
|
111
|
+
return MyStorage(...) # fresh, empty, per test
|
|
112
|
+
|
|
113
|
+
async def insert(self, storage, topic, payload, key=None):
|
|
114
|
+
return await storage.add(topic, payload, key=key)
|
|
115
|
+
```
|
|
116
|
+
|
|
117
|
+
The rules an adapter must follow are short:
|
|
118
|
+
|
|
119
|
+
1. `claim` returns due, unleased, pending rows in insertion order, leases them for `lease`, and increments
|
|
120
|
+
`attempts` **at claim time** (a crash mid-batch must burn an attempt).
|
|
121
|
+
2. `ack` marks rows delivered: delete them or flip a status column, your call.
|
|
122
|
+
3. `nack` releases the lease and stores `retry_at`; `dead_letter` releases the lease and parks the row.
|
|
123
|
+
4. Optional but recommended, *strict ordering*: hold back a row while an earlier pending row with the same
|
|
124
|
+
key is not claimable right now (leased elsewhere, or waiting for a retry). Both built-in SQL adapters do
|
|
125
|
+
this with one `NOT EXISTS` clause. The relay guarantees order within a batch on its own; strict ordering
|
|
126
|
+
in storage extends the guarantee across retries and across workers.
|
|
127
|
+
|
|
128
|
+
## Built-in adapters
|
|
129
|
+
|
|
130
|
+
| Adapter | Module | Notes |
|
|
131
|
+
| --- | --- | --- |
|
|
132
|
+
| `MemoryStorage`, `MemoryPublisher` | `txoutbox.adapters.memory` | Tests, demos, deterministic |
|
|
133
|
+
| `SqliteStorage` | `txoutbox.adapters.sqlite` | Stdlib `sqlite3`, `BEGIN IMMEDIATE`, WAL |
|
|
134
|
+
| `PostgresStorage` | `txoutbox.adapters.postgres` | asyncpg, `FOR UPDATE SKIP LOCKED`, extra `postgres` |
|
|
135
|
+
|
|
136
|
+
Both SQL adapters expose `insert(conn, ...)` so you write the outbox row **inside your own transaction**,
|
|
137
|
+
and `schema_sql()` / `create_schema()` so the table lives in your migrations.
|
|
138
|
+
|
|
139
|
+
## Configuration
|
|
140
|
+
|
|
141
|
+
```python
|
|
142
|
+
from datetime import timedelta
|
|
143
|
+
from txoutbox import Backoff, RelayConfig
|
|
144
|
+
|
|
145
|
+
RelayConfig(
|
|
146
|
+
batch_size=100, # rows per round
|
|
147
|
+
lease=timedelta(seconds=30), # must exceed the time to publish one round
|
|
148
|
+
concurrency=10, # ordering groups published in parallel
|
|
149
|
+
max_attempts=10, # then dead-letter
|
|
150
|
+
backoff=Backoff(base=timedelta(seconds=1), factor=2, maximum=timedelta(minutes=5), jitter=0.25),
|
|
151
|
+
publish_timeout=None, # seconds per publish, None = no limit
|
|
152
|
+
poll_min=0.05, poll_max=5.0, poll_factor=2.0,
|
|
153
|
+
storage_error_delay=1.0, # sleep after a storage exception
|
|
154
|
+
)
|
|
155
|
+
```
|
|
156
|
+
|
|
157
|
+
Three ways to run:
|
|
158
|
+
|
|
159
|
+
```python
|
|
160
|
+
await relay.run(handle_signals=True) # foreground loop, until SIGTERM/SIGINT or relay.stop()
|
|
161
|
+
|
|
162
|
+
async with Relay(storage, publisher): # background task for the lifetime of the block
|
|
163
|
+
await app.serve()
|
|
164
|
+
|
|
165
|
+
result = await relay.run_once() # one round; RoundResult(claimed, published, retried, dead_lettered, blocked)
|
|
166
|
+
```
|
|
167
|
+
|
|
168
|
+
## Delivery semantics
|
|
169
|
+
|
|
170
|
+
* **At-least-once.** A crash between `publish` and `ack` redelivers the batch after the lease expires.
|
|
171
|
+
Consumers must be idempotent; a message's `id` is a natural deduplication key.
|
|
172
|
+
* **Ordered per key.** Within one round, messages sharing a key are published one after another, and a
|
|
173
|
+
failure blocks the rest of that key for the round (they are rescheduled together with the failed one, without
|
|
174
|
+
burning an attempt). With strict-ordering storage, this holds across retries and workers too. Keyless
|
|
175
|
+
messages are independent.
|
|
176
|
+
* **Leases bound duplicates.** Keep `lease` well above `batch_size / concurrency * publish_time`; set
|
|
177
|
+
`publish_timeout` so a hanging broker cannot outlive the lease.
|
|
178
|
+
* **Storage bookkeeping failures are tolerated.** If `nack` itself fails, the lease expiry recovers the row.
|
|
179
|
+
|
|
180
|
+
## What txoutbox is not
|
|
181
|
+
|
|
182
|
+
Not a broker, not a task queue, not an ORM. It does not own your table and does not promise exactly-once.
|
|
183
|
+
Deduplication on the consumer side is a separate concern (see `inboxd`, the companion library).
|
|
184
|
+
|
|
185
|
+
## License
|
|
186
|
+
|
|
187
|
+
MIT
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
"""Run: uv run python examples/sqlite_demo.py (Ctrl-C to stop)
|
|
2
|
+
|
|
3
|
+
A producer loop inserts orders + outbox rows in one transaction every second,
|
|
4
|
+
while a relay publishes them to stdout. Kill it and restart it: nothing is lost,
|
|
5
|
+
nothing is reordered.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import asyncio
|
|
9
|
+
import json
|
|
10
|
+
import logging
|
|
11
|
+
import random
|
|
12
|
+
import sqlite3
|
|
13
|
+
|
|
14
|
+
from txoutbox import Hooks, OutboxMessage, Relay, RelayConfig
|
|
15
|
+
from txoutbox.adapters.sqlite import SqliteStorage
|
|
16
|
+
|
|
17
|
+
DB = "demo.db"
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class FlakyStdoutPublisher:
|
|
21
|
+
"""Fails 20% of the time so you can watch retries and backoff."""
|
|
22
|
+
|
|
23
|
+
async def publish(self, message: OutboxMessage) -> None:
|
|
24
|
+
if random.random() < 0.2:
|
|
25
|
+
raise ConnectionError("broker hiccup")
|
|
26
|
+
print(f"-> {message.topic} key={message.key} attempt={message.attempts} {message.payload.decode()}")
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class LogHooks(Hooks):
|
|
30
|
+
async def on_retry(self, message, error, retry_at):
|
|
31
|
+
print(f"!! retry {message.id} at {retry_at:%H:%M:%S}: {error}")
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
async def producer(storage: SqliteStorage) -> None:
|
|
35
|
+
conn = sqlite3.connect(DB)
|
|
36
|
+
conn.execute("CREATE TABLE IF NOT EXISTS orders (id INTEGER PRIMARY KEY, total INTEGER)")
|
|
37
|
+
while True:
|
|
38
|
+
with conn: # business row + outbox row commit together
|
|
39
|
+
cur = conn.execute("INSERT INTO orders (total) VALUES (?)", (random.randint(1, 100),))
|
|
40
|
+
order_id = cur.lastrowid
|
|
41
|
+
storage.insert(
|
|
42
|
+
conn, "orders.created", json.dumps({"order_id": order_id}).encode(), key=f"order-{order_id}"
|
|
43
|
+
)
|
|
44
|
+
await asyncio.sleep(1)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
async def main() -> None:
|
|
48
|
+
logging.basicConfig(level=logging.WARNING)
|
|
49
|
+
storage = SqliteStorage(DB)
|
|
50
|
+
storage.create_schema()
|
|
51
|
+
relay = Relay(
|
|
52
|
+
storage,
|
|
53
|
+
FlakyStdoutPublisher(),
|
|
54
|
+
config=RelayConfig(batch_size=10, poll_max=1.0),
|
|
55
|
+
hooks=LogHooks(),
|
|
56
|
+
)
|
|
57
|
+
producer_task = asyncio.create_task(producer(storage))
|
|
58
|
+
try:
|
|
59
|
+
await relay.run(handle_signals=True)
|
|
60
|
+
finally:
|
|
61
|
+
producer_task.cancel()
|
|
62
|
+
storage.close()
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
if __name__ == "__main__":
|
|
66
|
+
asyncio.run(main())
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "txoutbox"
|
|
3
|
+
version = "0.1.0"
|
|
4
|
+
description = "Storage-agnostic Transactional Outbox relay for asyncio: bring your own table, bring your own broker."
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
license = "MIT"
|
|
7
|
+
license-files = ["LICENSE"]
|
|
8
|
+
authors = [{ name = "Aslan Osmanov", email = "osmnvaslan@gmail.com" }]
|
|
9
|
+
requires-python = ">=3.11"
|
|
10
|
+
dependencies = []
|
|
11
|
+
keywords = ["outbox", "transactional-outbox", "asyncio", "messaging", "event-driven", "reliability"]
|
|
12
|
+
classifiers = [
|
|
13
|
+
"Development Status :: 3 - Alpha",
|
|
14
|
+
"Framework :: AsyncIO",
|
|
15
|
+
"Intended Audience :: Developers",
|
|
16
|
+
"Operating System :: OS Independent",
|
|
17
|
+
"Programming Language :: Python :: 3",
|
|
18
|
+
"Programming Language :: Python :: 3.11",
|
|
19
|
+
"Programming Language :: Python :: 3.12",
|
|
20
|
+
"Programming Language :: Python :: 3.13",
|
|
21
|
+
"Programming Language :: Python :: 3.14",
|
|
22
|
+
"Topic :: Software Development :: Libraries",
|
|
23
|
+
"Topic :: System :: Distributed Computing",
|
|
24
|
+
"Typing :: Typed",
|
|
25
|
+
]
|
|
26
|
+
|
|
27
|
+
[project.urls]
|
|
28
|
+
Homepage = "https://github.com/OsmnvAslan/txoutbox"
|
|
29
|
+
Repository = "https://github.com/OsmnvAslan/txoutbox"
|
|
30
|
+
Issues = "https://github.com/OsmnvAslan/txoutbox/issues"
|
|
31
|
+
Changelog = "https://github.com/OsmnvAslan/txoutbox/blob/main/CHANGELOG.md"
|
|
32
|
+
|
|
33
|
+
[project.optional-dependencies]
|
|
34
|
+
postgres = ["asyncpg>=0.29"]
|
|
35
|
+
|
|
36
|
+
[dependency-groups]
|
|
37
|
+
dev = [
|
|
38
|
+
"pytest>=8",
|
|
39
|
+
"pytest-asyncio>=0.24",
|
|
40
|
+
"mypy>=1.11",
|
|
41
|
+
"ruff>=0.6",
|
|
42
|
+
"asyncpg>=0.31.0",
|
|
43
|
+
"pgserver>=0.1.4; python_version < '3.13'",
|
|
44
|
+
]
|
|
45
|
+
|
|
46
|
+
[build-system]
|
|
47
|
+
requires = ["hatchling>=1.25"]
|
|
48
|
+
build-backend = "hatchling.build"
|
|
49
|
+
|
|
50
|
+
[tool.hatch.build.targets.wheel]
|
|
51
|
+
packages = ["src/txoutbox"]
|
|
52
|
+
|
|
53
|
+
[tool.hatch.build.targets.sdist]
|
|
54
|
+
exclude = [".github", ".python-version", "uv.lock"]
|
|
55
|
+
|
|
56
|
+
[tool.pytest.ini_options]
|
|
57
|
+
asyncio_mode = "auto"
|
|
58
|
+
asyncio_default_fixture_loop_scope = "function"
|
|
59
|
+
testpaths = ["tests"]
|
|
60
|
+
|
|
61
|
+
[tool.ruff]
|
|
62
|
+
line-length = 100
|
|
63
|
+
target-version = "py311"
|
|
64
|
+
|
|
65
|
+
[tool.ruff.lint]
|
|
66
|
+
select = ["E", "F", "I", "UP", "B", "SIM", "ASYNC", "RUF"]
|
|
67
|
+
|
|
68
|
+
[tool.ruff.lint.per-file-ignores]
|
|
69
|
+
"tests/*" = ["ASYNC110"]
|
|
70
|
+
|
|
71
|
+
[tool.mypy]
|
|
72
|
+
strict = true
|
|
73
|
+
python_version = "3.11"
|
|
74
|
+
|
|
75
|
+
[[tool.mypy.overrides]]
|
|
76
|
+
module = "asyncpg.*"
|
|
77
|
+
ignore_missing_imports = true
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
"""txoutbox: a storage-agnostic Transactional Outbox relay for asyncio.
|
|
2
|
+
|
|
3
|
+
Bring your own table, bring your own broker. Implement :class:`Storage` over your
|
|
4
|
+
database and :class:`Publisher` over your transport; :class:`Relay` does the rest.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from .backoff import Backoff
|
|
8
|
+
from .message import MessageId, OutboxMessage
|
|
9
|
+
from .poller import AdaptivePoller
|
|
10
|
+
from .protocols import Hooks, Publisher, Storage
|
|
11
|
+
from .relay import Relay, RelayConfig, RoundResult, default_worker_id
|
|
12
|
+
|
|
13
|
+
__all__ = [
|
|
14
|
+
"AdaptivePoller",
|
|
15
|
+
"Backoff",
|
|
16
|
+
"Hooks",
|
|
17
|
+
"MessageId",
|
|
18
|
+
"OutboxMessage",
|
|
19
|
+
"Publisher",
|
|
20
|
+
"Relay",
|
|
21
|
+
"RelayConfig",
|
|
22
|
+
"RoundResult",
|
|
23
|
+
"Storage",
|
|
24
|
+
"default_worker_id",
|
|
25
|
+
]
|
|
26
|
+
|
|
27
|
+
__version__ = "0.1.0"
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
"""Built-in storage and publisher adapters.
|
|
2
|
+
|
|
3
|
+
* :mod:`txoutbox.adapters.memory` - in-process, for tests and demos.
|
|
4
|
+
* :mod:`txoutbox.adapters.sqlite` - stdlib ``sqlite3``, for small services and examples.
|
|
5
|
+
* :mod:`txoutbox.adapters.postgres` - ``asyncpg`` with ``FOR UPDATE SKIP LOCKED``
|
|
6
|
+
(install extra ``postgres``).
|
|
7
|
+
"""
|