babelqueue 1.5.0__tar.gz → 1.7.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {babelqueue-1.5.0 → babelqueue-1.7.0}/CHANGELOG.md +28 -1
- {babelqueue-1.5.0 → babelqueue-1.7.0}/PKG-INFO +30 -4
- {babelqueue-1.5.0 → babelqueue-1.7.0}/README.md +29 -3
- {babelqueue-1.5.0 → babelqueue-1.7.0}/pyproject.toml +1 -1
- {babelqueue-1.5.0 → babelqueue-1.7.0}/src/babelqueue/__init__.py +6 -2
- babelqueue-1.7.0/src/babelqueue/exceptions.py +25 -0
- babelqueue-1.7.0/src/babelqueue/idempotency.py +95 -0
- babelqueue-1.7.0/src/babelqueue/redis_transport.py +254 -0
- babelqueue-1.7.0/src/babelqueue/schema.py +261 -0
- {babelqueue-1.5.0 → babelqueue-1.7.0}/tests/conformance/manifest.json +49 -0
- babelqueue-1.7.0/tests/test_idempotency.py +89 -0
- babelqueue-1.7.0/tests/test_redis_transport.py +396 -0
- babelqueue-1.7.0/tests/test_schema.py +145 -0
- babelqueue-1.7.0/tests/test_schema_conformance.py +33 -0
- babelqueue-1.5.0/src/babelqueue/exceptions.py +0 -11
- babelqueue-1.5.0/src/babelqueue/redis_transport.py +0 -53
- babelqueue-1.5.0/tests/test_redis_transport.py +0 -78
- {babelqueue-1.5.0 → babelqueue-1.7.0}/.github/FUNDING.yml +0 -0
- {babelqueue-1.5.0 → babelqueue-1.7.0}/.github/workflows/ci.yml +0 -0
- {babelqueue-1.5.0 → babelqueue-1.7.0}/.github/workflows/release.yml +0 -0
- {babelqueue-1.5.0 → babelqueue-1.7.0}/.gitignore +0 -0
- {babelqueue-1.5.0 → babelqueue-1.7.0}/LICENSE +0 -0
- {babelqueue-1.5.0 → babelqueue-1.7.0}/src/babelqueue/app.py +0 -0
- {babelqueue-1.5.0 → babelqueue-1.7.0}/src/babelqueue/artemis_transport.py +0 -0
- {babelqueue-1.5.0 → babelqueue-1.7.0}/src/babelqueue/asb_transport.py +0 -0
- {babelqueue-1.5.0 → babelqueue-1.7.0}/src/babelqueue/celery.py +0 -0
- {babelqueue-1.5.0 → babelqueue-1.7.0}/src/babelqueue/codec.py +0 -0
- {babelqueue-1.5.0 → babelqueue-1.7.0}/src/babelqueue/contracts.py +0 -0
- {babelqueue-1.5.0 → babelqueue-1.7.0}/src/babelqueue/dead_letter.py +0 -0
- {babelqueue-1.5.0 → babelqueue-1.7.0}/src/babelqueue/django/__init__.py +0 -0
- {babelqueue-1.5.0 → babelqueue-1.7.0}/src/babelqueue/django/apps.py +0 -0
- {babelqueue-1.5.0 → babelqueue-1.7.0}/src/babelqueue/django/management/__init__.py +0 -0
- {babelqueue-1.5.0 → babelqueue-1.7.0}/src/babelqueue/django/management/commands/__init__.py +0 -0
- {babelqueue-1.5.0 → babelqueue-1.7.0}/src/babelqueue/django/management/commands/babelqueue_worker.py +0 -0
- {babelqueue-1.5.0 → babelqueue-1.7.0}/src/babelqueue/kafka_transport.py +0 -0
- {babelqueue-1.5.0 → babelqueue-1.7.0}/src/babelqueue/pika_transport.py +0 -0
- {babelqueue-1.5.0 → babelqueue-1.7.0}/src/babelqueue/pulsar_transport.py +0 -0
- {babelqueue-1.5.0 → babelqueue-1.7.0}/src/babelqueue/py.typed +0 -0
- {babelqueue-1.5.0 → babelqueue-1.7.0}/src/babelqueue/routing.py +0 -0
- {babelqueue-1.5.0 → babelqueue-1.7.0}/src/babelqueue/sqs_transport.py +0 -0
- {babelqueue-1.5.0 → babelqueue-1.7.0}/src/babelqueue/transport.py +0 -0
- {babelqueue-1.5.0 → babelqueue-1.7.0}/tests/conformance/fixtures/dead-lettered.json +0 -0
- {babelqueue-1.5.0 → babelqueue-1.7.0}/tests/conformance/fixtures/invalid-missing-urn.json +0 -0
- {babelqueue-1.5.0 → babelqueue-1.7.0}/tests/conformance/fixtures/invalid-unknown-schema-version.json +0 -0
- {babelqueue-1.5.0 → babelqueue-1.7.0}/tests/conformance/fixtures/order-created.json +0 -0
- {babelqueue-1.5.0 → babelqueue-1.7.0}/tests/conformance/fixtures/unicode-and-numbers.json +0 -0
- {babelqueue-1.5.0 → babelqueue-1.7.0}/tests/conformance/fixtures/urn-alias.json +0 -0
- {babelqueue-1.5.0 → babelqueue-1.7.0}/tests/conformance/schema/message-envelope.schema.json +0 -0
- {babelqueue-1.5.0 → babelqueue-1.7.0}/tests/fixtures/dead-lettered.json +0 -0
- {babelqueue-1.5.0 → babelqueue-1.7.0}/tests/fixtures/order-created.json +0 -0
- {babelqueue-1.5.0 → babelqueue-1.7.0}/tests/test_app.py +0 -0
- {babelqueue-1.5.0 → babelqueue-1.7.0}/tests/test_artemis_conformance.py +0 -0
- {babelqueue-1.5.0 → babelqueue-1.7.0}/tests/test_artemis_transport.py +0 -0
- {babelqueue-1.5.0 → babelqueue-1.7.0}/tests/test_asb_conformance.py +0 -0
- {babelqueue-1.5.0 → babelqueue-1.7.0}/tests/test_asb_transport.py +0 -0
- {babelqueue-1.5.0 → babelqueue-1.7.0}/tests/test_celery.py +0 -0
- {babelqueue-1.5.0 → babelqueue-1.7.0}/tests/test_codec.py +0 -0
- {babelqueue-1.5.0 → babelqueue-1.7.0}/tests/test_conformance.py +0 -0
- {babelqueue-1.5.0 → babelqueue-1.7.0}/tests/test_dead_letter.py +0 -0
- {babelqueue-1.5.0 → babelqueue-1.7.0}/tests/test_django.py +0 -0
- {babelqueue-1.5.0 → babelqueue-1.7.0}/tests/test_kafka_conformance.py +0 -0
- {babelqueue-1.5.0 → babelqueue-1.7.0}/tests/test_kafka_transport.py +0 -0
- {babelqueue-1.5.0 → babelqueue-1.7.0}/tests/test_overhead.py +0 -0
- {babelqueue-1.5.0 → babelqueue-1.7.0}/tests/test_pika_transport.py +0 -0
- {babelqueue-1.5.0 → babelqueue-1.7.0}/tests/test_pulsar_conformance.py +0 -0
- {babelqueue-1.5.0 → babelqueue-1.7.0}/tests/test_pulsar_transport.py +0 -0
- {babelqueue-1.5.0 → babelqueue-1.7.0}/tests/test_sqs_conformance.py +0 -0
- {babelqueue-1.5.0 → babelqueue-1.7.0}/tests/test_sqs_transport.py +0 -0
|
@@ -7,6 +7,32 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
7
7
|
The envelope wire format is versioned separately by `meta.schema_version`
|
|
8
8
|
(currently **1**) — see the contract at [babelqueue.com](https://babelqueue.com).
|
|
9
9
|
|
|
10
|
+
## [Unreleased]
|
|
11
|
+
|
|
12
|
+
## [1.6.0] - 2026-06-14
|
|
13
|
+
|
|
14
|
+
### Added
|
|
15
|
+
- **Redis/Laravel reservation parity** — the Redis transport can now consume a
|
|
16
|
+
**shared** Laravel BabelQueue Redis queue using Laravel's reserved-set / reliable-queue
|
|
17
|
+
semantics, not just a Python-owned queue. Enable it with the `laravel=1` URL flag
|
|
18
|
+
(`redis://host:6379/0?laravel=1`) or `RedisTransport(..., laravel_compat=True)`. In this
|
|
19
|
+
mode the key layout is Laravel's stock Redis queue ([§1 of the broker-bindings
|
|
20
|
+
contract](https://babelqueue.com/docs/spec/1.x/broker-bindings#redis)): a `queues:<name>`
|
|
21
|
+
ready list, a `queues:<name>:reserved` **sorted set** scored by a `retry_after` deadline
|
|
22
|
+
(default 60s), a `queues:<name>:delayed` set, and a `queues:<name>:notify` wake-up list.
|
|
23
|
+
Reserve, ack and release run the **byte-for-byte same Lua scripts** Laravel uses, so the
|
|
24
|
+
reserved member a Python worker writes is identical to a Laravel worker's — either side can
|
|
25
|
+
ack (`ZREM`) the other's reservation, so a Python worker and a Laravel worker share one
|
|
26
|
+
Redis queue without losing or double-processing messages. Before each pop, expired
|
|
27
|
+
reserved/delayed jobs migrate back to the ready list, so a crashed worker's in-flight job
|
|
28
|
+
is re-reserved exactly as Laravel does. Tunable via `?prefix=` / `?retry_after=` (or the
|
|
29
|
+
`key_prefix` / `retry_after` constructor args). The default Python-owned reliable-queue
|
|
30
|
+
mode (`BLMOVE` + `<queue>:processing` list) is unchanged and stays the default, so existing
|
|
31
|
+
callers are unaffected. The reservation logic is fully unit-tested with an injected
|
|
32
|
+
in-memory Redis double (no `redis` package, no broker); a live cross-runtime PHP↔Python
|
|
33
|
+
shared-queue round-trip is covered by the integration suite where a real Redis is present.
|
|
34
|
+
The envelope is unchanged (`schema_version: 1`); this is purely additive. Ships as a MINOR.
|
|
35
|
+
|
|
10
36
|
## [1.5.0] - 2026-06-13
|
|
11
37
|
|
|
12
38
|
### Added
|
|
@@ -173,7 +199,8 @@ reference at [babelqueue.com](https://babelqueue.com).
|
|
|
173
199
|
- Pre-1.0: the public API may change before the `1.0.0` tag.
|
|
174
200
|
- The core has **zero runtime dependencies** (standard library only); Python `>=3.9`.
|
|
175
201
|
|
|
176
|
-
[Unreleased]: https://github.com/BabelQueue/babelqueue-python/compare/v1.
|
|
202
|
+
[Unreleased]: https://github.com/BabelQueue/babelqueue-python/compare/v1.6.0...HEAD
|
|
203
|
+
[1.6.0]: https://github.com/BabelQueue/babelqueue-python/compare/v1.5.0...v1.6.0
|
|
177
204
|
[1.2.0]: https://github.com/BabelQueue/babelqueue-python/compare/v1.1.0...v1.2.0
|
|
178
205
|
[1.1.0]: https://github.com/BabelQueue/babelqueue-python/compare/v1.0.0...v1.1.0
|
|
179
206
|
[1.0.0]: https://github.com/BabelQueue/babelqueue-python/compare/v0.5.0...v1.0.0
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: babelqueue
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.7.0
|
|
4
4
|
Summary: Polyglot Queues, Simplified — the Python core: the canonical BabelQueue wire-envelope codec, contracts and dead-letter helpers.
|
|
5
5
|
Project-URL: Homepage, https://babelqueue.com
|
|
6
6
|
Project-URL: Source, https://github.com/BabelQueue/babelqueue-python
|
|
@@ -164,9 +164,35 @@ app.run() # consume forever (Ctrl-C to stop)
|
|
|
164
164
|
- **Retry & dead-letter:** failures are retried up to `max_attempts` (bumping the
|
|
165
165
|
envelope's `attempts`); enable `dead_letter=True` to quarantine exhausted
|
|
166
166
|
messages on `<queue>.dlq`. `on_unknown_urn` = `fail` | `delete` | `release` | `dead_letter`.
|
|
167
|
-
- **Transports:** `redis://` (reliable-queue pattern
|
|
168
|
-
|
|
169
|
-
|
|
167
|
+
- **Transports:** `redis://` (reliable-queue pattern; add `?laravel=1` to share a
|
|
168
|
+
Redis queue with a Laravel BabelQueue worker using its reserved-set semantics —
|
|
169
|
+
see below), `amqp://` (RabbitMQ via `pika`, with the contract AMQP properties)
|
|
170
|
+
and `memory://` (in-process, great for tests/local). Bring your own by passing
|
|
171
|
+
`transport=...`.
|
|
172
|
+
|
|
173
|
+
### Sharing a Redis queue with Laravel
|
|
174
|
+
|
|
175
|
+
By default the Redis transport owns its queue end-to-end (`RPUSH` to produce;
|
|
176
|
+
`BLMOVE` into a `<queue>:processing` list to reserve; `LREM` on ack). To consume a
|
|
177
|
+
**shared** Laravel BabelQueue Redis queue instead, enable Laravel-compatible mode:
|
|
178
|
+
|
|
179
|
+
```python
|
|
180
|
+
app = BabelQueue("redis://localhost:6379/0?laravel=1", queue="orders")
|
|
181
|
+
# or: RedisTransport("redis://localhost:6379/0", laravel_compat=True)
|
|
182
|
+
```
|
|
183
|
+
|
|
184
|
+
This replicates Laravel's stock Redis reservation exactly ([§1 of the
|
|
185
|
+
broker-bindings contract](https://babelqueue.com/docs/spec/1.x/broker-bindings#redis)):
|
|
186
|
+
the ready list is `queues:<name>`, reservations move into the `queues:<name>:reserved`
|
|
187
|
+
**sorted set** scored by a `retry_after` deadline (default 60s), with a
|
|
188
|
+
`queues:<name>:delayed` set and a `queues:<name>:notify` wake-up list. Reserve, ack
|
|
189
|
+
and release run the **byte-for-byte same Lua scripts** Laravel uses, so the reserved
|
|
190
|
+
member a Python worker writes is identical to a Laravel worker's — either side can
|
|
191
|
+
ack (`ZREM`) the other's reservation, so a Python worker and a Laravel worker share
|
|
192
|
+
one queue without losing or double-processing messages. Before each pop, expired
|
|
193
|
+
reserved/delayed jobs migrate back to the ready list, so a crashed worker's
|
|
194
|
+
in-flight job is re-reserved. Options: `?prefix=queues:` and `?retry_after=60`
|
|
195
|
+
(or the `key_prefix` / `retry_after` constructor args).
|
|
170
196
|
|
|
171
197
|
## Framework adapters — Celery & Django
|
|
172
198
|
|
|
@@ -114,9 +114,35 @@ app.run() # consume forever (Ctrl-C to stop)
|
|
|
114
114
|
- **Retry & dead-letter:** failures are retried up to `max_attempts` (bumping the
|
|
115
115
|
envelope's `attempts`); enable `dead_letter=True` to quarantine exhausted
|
|
116
116
|
messages on `<queue>.dlq`. `on_unknown_urn` = `fail` | `delete` | `release` | `dead_letter`.
|
|
117
|
-
- **Transports:** `redis://` (reliable-queue pattern
|
|
118
|
-
|
|
119
|
-
|
|
117
|
+
- **Transports:** `redis://` (reliable-queue pattern; add `?laravel=1` to share a
|
|
118
|
+
Redis queue with a Laravel BabelQueue worker using its reserved-set semantics —
|
|
119
|
+
see below), `amqp://` (RabbitMQ via `pika`, with the contract AMQP properties)
|
|
120
|
+
and `memory://` (in-process, great for tests/local). Bring your own by passing
|
|
121
|
+
`transport=...`.
|
|
122
|
+
|
|
123
|
+
### Sharing a Redis queue with Laravel
|
|
124
|
+
|
|
125
|
+
By default the Redis transport owns its queue end-to-end (`RPUSH` to produce;
|
|
126
|
+
`BLMOVE` into a `<queue>:processing` list to reserve; `LREM` on ack). To consume a
|
|
127
|
+
**shared** Laravel BabelQueue Redis queue instead, enable Laravel-compatible mode:
|
|
128
|
+
|
|
129
|
+
```python
|
|
130
|
+
app = BabelQueue("redis://localhost:6379/0?laravel=1", queue="orders")
|
|
131
|
+
# or: RedisTransport("redis://localhost:6379/0", laravel_compat=True)
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
This replicates Laravel's stock Redis reservation exactly ([§1 of the
|
|
135
|
+
broker-bindings contract](https://babelqueue.com/docs/spec/1.x/broker-bindings#redis)):
|
|
136
|
+
the ready list is `queues:<name>`, reservations move into the `queues:<name>:reserved`
|
|
137
|
+
**sorted set** scored by a `retry_after` deadline (default 60s), with a
|
|
138
|
+
`queues:<name>:delayed` set and a `queues:<name>:notify` wake-up list. Reserve, ack
|
|
139
|
+
and release run the **byte-for-byte same Lua scripts** Laravel uses, so the reserved
|
|
140
|
+
member a Python worker writes is identical to a Laravel worker's — either side can
|
|
141
|
+
ack (`ZREM`) the other's reservation, so a Python worker and a Laravel worker share
|
|
142
|
+
one queue without losing or double-processing messages. Before each pop, expired
|
|
143
|
+
reserved/delayed jobs migrate back to the ready list, so a crashed worker's
|
|
144
|
+
in-flight job is re-reserved. Options: `?prefix=queues:` and `?retry_after=60`
|
|
145
|
+
(or the `key_prefix` / `retry_after` constructor args).
|
|
120
146
|
|
|
121
147
|
## Framework adapters — Celery & Django
|
|
122
148
|
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "babelqueue"
|
|
7
|
-
version = "1.
|
|
7
|
+
version = "1.7.0"
|
|
8
8
|
description = "Polyglot Queues, Simplified — the Python core: the canonical BabelQueue wire-envelope codec, contracts and dead-letter helpers."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.9"
|
|
@@ -11,15 +11,16 @@ and dead-letter helpers. Framework adapters (Celery, Django, ...) build on this.
|
|
|
11
11
|
|
|
12
12
|
from __future__ import annotations
|
|
13
13
|
|
|
14
|
-
from . import dead_letter
|
|
14
|
+
from . import dead_letter, idempotency
|
|
15
15
|
from .app import BabelQueue
|
|
16
16
|
from .codec import SCHEMA_VERSION, SOURCE_LANG, EnvelopeCodec
|
|
17
17
|
from .contracts import HasTraceId, PolyglotMessage
|
|
18
|
+
from .idempotency import IdempotencyStore, InMemoryStore
|
|
18
19
|
from .exceptions import BabelQueueError, UnknownUrnError
|
|
19
20
|
from .routing import UnknownUrnStrategy
|
|
20
21
|
from .transport import InMemoryTransport, ReceivedMessage, Transport
|
|
21
22
|
|
|
22
|
-
__version__ = "1.
|
|
23
|
+
__version__ = "1.6.0"
|
|
23
24
|
|
|
24
25
|
__all__ = [
|
|
25
26
|
"BabelQueue",
|
|
@@ -35,5 +36,8 @@ __all__ = [
|
|
|
35
36
|
"BabelQueueError",
|
|
36
37
|
"UnknownUrnError",
|
|
37
38
|
"dead_letter",
|
|
39
|
+
"idempotency",
|
|
40
|
+
"IdempotencyStore",
|
|
41
|
+
"InMemoryStore",
|
|
38
42
|
"__version__",
|
|
39
43
|
]
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
"""Exception hierarchy for BabelQueue."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class BabelQueueError(Exception):
|
|
7
|
+
"""Base for every recoverable BabelQueue error (bad config, empty URN, ...)."""
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class UnknownUrnError(BabelQueueError):
|
|
11
|
+
"""A consumed message carries a URN with no mapped handler (strategy "fail")."""
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class InvalidPayloadError(BabelQueueError):
|
|
15
|
+
"""A message's ``data`` does not match the JSON Schema registered for its URN (ADR-0024).
|
|
16
|
+
|
|
17
|
+
Raised by the producer-side :func:`babelqueue.schema.validate` and the consumer-side
|
|
18
|
+
:func:`babelqueue.schema.wrap`; the latter lets the runtime redeliver (and eventually
|
|
19
|
+
dead-letter) a poison message.
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
def __init__(self, urn: str, violation: str) -> None:
|
|
23
|
+
super().__init__(f"data for {urn!r} does not match its URN schema: {violation}")
|
|
24
|
+
self.urn = urn
|
|
25
|
+
self.violation = violation
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
"""Optional idempotency helper (ADR-0022): dedupe a consume handler on ``meta.id``.
|
|
2
|
+
|
|
3
|
+
The Python mirror of the PHP ``BabelQueue\\Idempotency`` and Go ``idempotency``
|
|
4
|
+
helpers. It wraps a handler so a message whose ``meta.id`` was already processed
|
|
5
|
+
successfully is skipped instead of run again, composing with the runtime's
|
|
6
|
+
ack-on-return / redeliver-on-raise contract::
|
|
7
|
+
|
|
8
|
+
from babelqueue import BabelQueue
|
|
9
|
+
from babelqueue.idempotency import InMemoryStore, wrap
|
|
10
|
+
|
|
11
|
+
app = BabelQueue("redis://localhost:6379/0", queue="orders")
|
|
12
|
+
store = InMemoryStore()
|
|
13
|
+
app.register("urn:babel:orders:created", wrap(store, on_order_created))
|
|
14
|
+
|
|
15
|
+
A previously-seen id returns early (the runtime acks it, so the broker stops
|
|
16
|
+
redelivering); a raising handler leaves the id unmarked so a redelivery runs it again
|
|
17
|
+
(retry / dead-letter still apply); a message with no usable ``meta.id`` runs unchanged.
|
|
18
|
+
This is "seen-set" post-success dedupe — not exactly-once and not in-flight concurrency
|
|
19
|
+
locking; a transactional / outbox mode is a documented future direction.
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
from __future__ import annotations
|
|
23
|
+
|
|
24
|
+
import functools
|
|
25
|
+
import threading
|
|
26
|
+
from typing import Any, Callable, Mapping, Protocol, runtime_checkable
|
|
27
|
+
|
|
28
|
+
Handler = Callable[..., None]
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
@runtime_checkable
|
|
32
|
+
class IdempotencyStore(Protocol):
|
|
33
|
+
"""A record of message ids already processed, keyed on ``meta.id``."""
|
|
34
|
+
|
|
35
|
+
def seen(self, message_id: str) -> bool:
|
|
36
|
+
"""Whether this id has already been processed (remembered)."""
|
|
37
|
+
|
|
38
|
+
def remember(self, message_id: str) -> None:
|
|
39
|
+
"""Record this id as processed."""
|
|
40
|
+
|
|
41
|
+
def forget(self, message_id: str) -> None:
|
|
42
|
+
"""Drop an id from the store (manual eviction; a backend may also expire ids)."""
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
class InMemoryStore:
|
|
46
|
+
"""Process-local, thread-safe :class:`IdempotencyStore`.
|
|
47
|
+
|
|
48
|
+
For tests and single-process consumers; it is not shared across workers and not
|
|
49
|
+
persistent — use a Redis- or database-backed store for production fleets.
|
|
50
|
+
"""
|
|
51
|
+
|
|
52
|
+
def __init__(self) -> None:
|
|
53
|
+
self._seen: set[str] = set()
|
|
54
|
+
self._lock = threading.Lock()
|
|
55
|
+
|
|
56
|
+
def seen(self, message_id: str) -> bool:
|
|
57
|
+
with self._lock:
|
|
58
|
+
return message_id in self._seen
|
|
59
|
+
|
|
60
|
+
def remember(self, message_id: str) -> None:
|
|
61
|
+
with self._lock:
|
|
62
|
+
self._seen.add(message_id)
|
|
63
|
+
|
|
64
|
+
def forget(self, message_id: str) -> None:
|
|
65
|
+
with self._lock:
|
|
66
|
+
self._seen.discard(message_id)
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def wrap(store: IdempotencyStore, handler: Handler) -> Handler:
|
|
70
|
+
"""Wrap ``handler`` so a message whose ``meta.id`` was already processed is skipped.
|
|
71
|
+
|
|
72
|
+
The returned callable keeps ``handler``'s signature (via :func:`functools.wraps`),
|
|
73
|
+
so the runtime's introspection still passes it the right number of positional args
|
|
74
|
+
(``data, meta`` or ``data, meta, envelope``).
|
|
75
|
+
"""
|
|
76
|
+
|
|
77
|
+
@functools.wraps(handler)
|
|
78
|
+
def wrapped(*args: Any) -> None:
|
|
79
|
+
meta = args[1] if len(args) > 1 and isinstance(args[1], Mapping) else {}
|
|
80
|
+
message_id = meta.get("id")
|
|
81
|
+
|
|
82
|
+
# No usable id → cannot dedupe; run the handler unchanged.
|
|
83
|
+
if not isinstance(message_id, str) or message_id == "":
|
|
84
|
+
handler(*args)
|
|
85
|
+
return
|
|
86
|
+
|
|
87
|
+
# Already processed on an earlier delivery: return so the runtime acks it.
|
|
88
|
+
if store.seen(message_id):
|
|
89
|
+
return
|
|
90
|
+
|
|
91
|
+
# First success wins; a raise here leaves the id unmarked → retry/DLQ apply.
|
|
92
|
+
handler(*args)
|
|
93
|
+
store.remember(message_id)
|
|
94
|
+
|
|
95
|
+
return wrapped
|
|
@@ -0,0 +1,254 @@
|
|
|
1
|
+
"""Redis transport. Requires the ``redis`` extra:
|
|
2
|
+
|
|
3
|
+
pip install "babelqueue[redis]"
|
|
4
|
+
|
|
5
|
+
Two reservation modes share one class, selected by ``laravel_compat``:
|
|
6
|
+
|
|
7
|
+
**Python-owned (default).** Producing is ``RPUSH queue body``; consuming atomically
|
|
8
|
+
moves the head to a per-queue processing list (``BLMOVE``) so an in-flight message
|
|
9
|
+
survives a worker crash, and ``ack`` removes it from that processing list. Simple,
|
|
10
|
+
self-contained, and the right choice for a queue this runtime owns end-to-end.
|
|
11
|
+
|
|
12
|
+
**Laravel-compatible (``laravel_compat=True``).** Replicates Laravel's stock Redis
|
|
13
|
+
queue reservation so a Python worker can share one Redis queue with a PHP/Laravel
|
|
14
|
+
worker without losing or double-processing messages (§1 of the broker-bindings
|
|
15
|
+
contract). The key layout is Laravel's: a ``queues:<name>`` ready list, a
|
|
16
|
+
``queues:<name>:reserved`` sorted set scored by a retry-after deadline, a
|
|
17
|
+
``queues:<name>:delayed`` sorted set, and a ``queues:<name>:notify`` wake-up list.
|
|
18
|
+
Reserve/ack/release run the **byte-for-byte same Lua scripts** Laravel uses, so the
|
|
19
|
+
reserved-set member a Python worker writes is identical to what a Laravel worker
|
|
20
|
+
writes — either side can therefore ack (``ZREM``) the other's reservation. Before
|
|
21
|
+
each pop the transport migrates expired reserved/delayed jobs back to the ready
|
|
22
|
+
list, so a crashed worker's in-flight job is re-reserved exactly as Laravel does.
|
|
23
|
+
|
|
24
|
+
The envelope is unchanged (``schema_version`` stays 1); Redis is purely additive.
|
|
25
|
+
|
|
26
|
+
URL form: ``redis://host:port/db[?laravel=1&prefix=queues:&retry_after=60]``.
|
|
27
|
+
For richer setups, build the transport directly and pass ``BabelQueue(transport=...)``.
|
|
28
|
+
"""
|
|
29
|
+
|
|
30
|
+
from __future__ import annotations
|
|
31
|
+
|
|
32
|
+
import time
|
|
33
|
+
from typing import Any, Optional
|
|
34
|
+
from urllib.parse import parse_qs, urlsplit
|
|
35
|
+
|
|
36
|
+
from .transport import ReceivedMessage, Transport
|
|
37
|
+
|
|
38
|
+
# Laravel's stock Redis queue Lua scripts (Illuminate\Queue\LuaScripts). Replicated
|
|
39
|
+
# verbatim so the reserved-set member byte form matches Laravel's — that exact match
|
|
40
|
+
# is what lets a Python and a Laravel worker ack each other's reservations.
|
|
41
|
+
|
|
42
|
+
# KEYS[1]=ready list, KEYS[2]=notify list; ARGV[1]=payload.
|
|
43
|
+
_PUSH = """
|
|
44
|
+
redis.call('rpush', KEYS[1], ARGV[1])
|
|
45
|
+
redis.call('rpush', KEYS[2], 1)
|
|
46
|
+
"""
|
|
47
|
+
|
|
48
|
+
# KEYS[1]=ready, KEYS[2]=reserved, KEYS[3]=notify; ARGV[1]=reserved-until score.
|
|
49
|
+
# Returns {original_job, reserved_member}; the reserved member is the attempts+1
|
|
50
|
+
# re-encoding stored in the reserved set and is the ack handle.
|
|
51
|
+
_POP = """
|
|
52
|
+
local job = redis.call('lpop', KEYS[1])
|
|
53
|
+
local reserved = false
|
|
54
|
+
|
|
55
|
+
if(job ~= false) then
|
|
56
|
+
reserved = cjson.decode(job)
|
|
57
|
+
reserved['attempts'] = reserved['attempts'] + 1
|
|
58
|
+
reserved = cjson.encode(reserved)
|
|
59
|
+
redis.call('zadd', KEYS[2], ARGV[1], reserved)
|
|
60
|
+
redis.call('lpop', KEYS[3])
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
return {job, reserved}
|
|
64
|
+
"""
|
|
65
|
+
|
|
66
|
+
# KEYS[1]=delayed, KEYS[2]=reserved; ARGV[1]=reserved member, ARGV[2]=available-at.
|
|
67
|
+
_RELEASE = """
|
|
68
|
+
redis.call('zrem', KEYS[2], ARGV[1])
|
|
69
|
+
redis.call('zadd', KEYS[1], ARGV[2], ARGV[1])
|
|
70
|
+
return true
|
|
71
|
+
"""
|
|
72
|
+
|
|
73
|
+
# KEYS[1]=from (reserved/delayed), KEYS[2]=ready, KEYS[3]=notify; ARGV[1]=now,
|
|
74
|
+
# ARGV[2]=batch size (-1 = all). Moves jobs whose score has expired back to ready.
|
|
75
|
+
_MIGRATE = """
|
|
76
|
+
local val = redis.call('zrangebyscore', KEYS[1], '-inf', ARGV[1], 'limit', 0, ARGV[2])
|
|
77
|
+
|
|
78
|
+
if(next(val) ~= nil) then
|
|
79
|
+
redis.call('zremrangebyrank', KEYS[1], 0, #val - 1)
|
|
80
|
+
|
|
81
|
+
for i = 1, #val, 100 do
|
|
82
|
+
redis.call('rpush', KEYS[2], unpack(val, i, math.min(i+99, #val)))
|
|
83
|
+
for j = i, math.min(i+99, #val) do
|
|
84
|
+
redis.call('rpush', KEYS[3], 1)
|
|
85
|
+
end
|
|
86
|
+
end
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
return val
|
|
90
|
+
"""
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
class RedisTransport(Transport):
|
|
94
|
+
def __init__(
|
|
95
|
+
self,
|
|
96
|
+
url: str = "redis://localhost:6379/0",
|
|
97
|
+
*,
|
|
98
|
+
client: Any = None,
|
|
99
|
+
processing_suffix: str = ":processing",
|
|
100
|
+
laravel_compat: bool = False,
|
|
101
|
+
key_prefix: str = "queues:",
|
|
102
|
+
retry_after: int = 60,
|
|
103
|
+
block_for: Optional[float] = None,
|
|
104
|
+
) -> None:
|
|
105
|
+
parts = urlsplit(url) if url else urlsplit("redis://")
|
|
106
|
+
q = parse_qs(parts.query)
|
|
107
|
+
|
|
108
|
+
self._processing_suffix = processing_suffix
|
|
109
|
+
self._laravel_compat = laravel_compat or _qbool(q, "laravel") or _qbool(q, "laravel_compat")
|
|
110
|
+
self._key_prefix = key_prefix if "prefix" not in q else (_q1(q, "prefix") or key_prefix)
|
|
111
|
+
self._retry_after = retry_after if "retry_after" not in q else _qint(q, "retry_after", retry_after)
|
|
112
|
+
self._block_for = block_for
|
|
113
|
+
|
|
114
|
+
if client is not None:
|
|
115
|
+
self._redis = client
|
|
116
|
+
return
|
|
117
|
+
try:
|
|
118
|
+
import redis
|
|
119
|
+
except ImportError as exc: # pragma: no cover - import guard
|
|
120
|
+
raise ImportError(
|
|
121
|
+
"RedisTransport requires the 'redis' package. Install with "
|
|
122
|
+
'pip install "babelqueue[redis]".'
|
|
123
|
+
) from exc
|
|
124
|
+
self._redis = redis.Redis.from_url(url, decode_responses=True)
|
|
125
|
+
|
|
126
|
+
# -- key helpers --------------------------------------------------------
|
|
127
|
+
|
|
128
|
+
def _processing(self, queue: str) -> str:
|
|
129
|
+
return f"{queue}{self._processing_suffix}"
|
|
130
|
+
|
|
131
|
+
def _ready(self, queue: str) -> str:
|
|
132
|
+
"""The Laravel ready-list key for a logical queue name (``queues:<name>``)."""
|
|
133
|
+
return f"{self._key_prefix}{queue}"
|
|
134
|
+
|
|
135
|
+
# -- Transport ----------------------------------------------------------
|
|
136
|
+
|
|
137
|
+
def publish(self, queue: str, body: str) -> None:
|
|
138
|
+
if not self._laravel_compat:
|
|
139
|
+
self._redis.rpush(queue, body)
|
|
140
|
+
return
|
|
141
|
+
ready = self._ready(queue)
|
|
142
|
+
self._redis.eval(_PUSH, 2, ready, f"{ready}:notify", body)
|
|
143
|
+
|
|
144
|
+
def pop(self, queue: str, timeout: float = 1.0) -> Optional[ReceivedMessage]:
|
|
145
|
+
if not self._laravel_compat:
|
|
146
|
+
return self._pop_owned(queue, timeout)
|
|
147
|
+
return self._pop_laravel(queue, timeout)
|
|
148
|
+
|
|
149
|
+
def ack(self, message: ReceivedMessage) -> None:
|
|
150
|
+
if not self._laravel_compat:
|
|
151
|
+
self._redis.lrem(self._processing(message.queue), 1, message.handle)
|
|
152
|
+
return
|
|
153
|
+
if not message.handle:
|
|
154
|
+
return
|
|
155
|
+
self._redis.zrem(f"{self._ready(message.queue)}:reserved", message.handle)
|
|
156
|
+
|
|
157
|
+
def release(self, message: ReceivedMessage, delay: int = 0) -> None:
|
|
158
|
+
"""Return a reserved Laravel job to the (delayed) ready set for retry.
|
|
159
|
+
|
|
160
|
+
No-op in Python-owned mode (the runtime re-publishes instead). In
|
|
161
|
+
Laravel-compatible mode this is ``ZREM :reserved`` + ``ZADD :delayed`` —
|
|
162
|
+
the exact bookkeeping Laravel's release performs.
|
|
163
|
+
"""
|
|
164
|
+
if not self._laravel_compat or not message.handle:
|
|
165
|
+
return
|
|
166
|
+
ready = self._ready(message.queue)
|
|
167
|
+
self._redis.eval(
|
|
168
|
+
_RELEASE, 2, f"{ready}:delayed", f"{ready}:reserved",
|
|
169
|
+
message.handle, int(time.time()) + int(delay),
|
|
170
|
+
)
|
|
171
|
+
|
|
172
|
+
def close(self) -> None: # pragma: no cover
|
|
173
|
+
self._redis.close()
|
|
174
|
+
|
|
175
|
+
# -- Python-owned reliable queue ----------------------------------------
|
|
176
|
+
|
|
177
|
+
def _pop_owned(self, queue: str, timeout: float) -> Optional[ReceivedMessage]:
|
|
178
|
+
# redis-py types the BLMOVE timeout as int, but Redis accepts a float
|
|
179
|
+
# (sub-second) timeout; passing it through is correct at runtime.
|
|
180
|
+
body = self._redis.blmove(queue, self._processing(queue), timeout, "LEFT", "RIGHT") # type: ignore[arg-type]
|
|
181
|
+
if body is None:
|
|
182
|
+
return None
|
|
183
|
+
text = _as_text(body)
|
|
184
|
+
return ReceivedMessage(body=text, queue=queue, handle=text)
|
|
185
|
+
|
|
186
|
+
# -- Laravel-compatible reservation -------------------------------------
|
|
187
|
+
|
|
188
|
+
def _pop_laravel(self, queue: str, timeout: float) -> Optional[ReceivedMessage]:
|
|
189
|
+
ready = self._ready(queue)
|
|
190
|
+
self._migrate(ready)
|
|
191
|
+
|
|
192
|
+
result = self._redis.eval(
|
|
193
|
+
_POP, 3, ready, f"{ready}:reserved", f"{ready}:notify",
|
|
194
|
+
int(time.time()) + self._retry_after,
|
|
195
|
+
)
|
|
196
|
+
job, reserved = _pop_result(result)
|
|
197
|
+
if job is None or reserved is None:
|
|
198
|
+
# Nothing ready right now; optionally block on the notify list so a
|
|
199
|
+
# concurrently-pushed job wakes us, then retry once (mirrors Laravel).
|
|
200
|
+
block = self._block_for if self._block_for is not None else timeout
|
|
201
|
+
if block and block > 0 and self._redis.blpop([f"{ready}:notify"], block):
|
|
202
|
+
result = self._redis.eval(
|
|
203
|
+
_POP, 3, ready, f"{ready}:reserved", f"{ready}:notify",
|
|
204
|
+
int(time.time()) + self._retry_after,
|
|
205
|
+
)
|
|
206
|
+
job, reserved = _pop_result(result)
|
|
207
|
+
if job is None or reserved is None:
|
|
208
|
+
return None
|
|
209
|
+
|
|
210
|
+
# The reserved member (attempts already incremented) is both the consumed
|
|
211
|
+
# body and the ack handle, so ZREM removes exactly this reservation.
|
|
212
|
+
return ReceivedMessage(body=_as_text(reserved), queue=queue, handle=_as_text(reserved))
|
|
213
|
+
|
|
214
|
+
def _migrate(self, ready: str) -> None:
|
|
215
|
+
now = int(time.time())
|
|
216
|
+
# Delayed jobs that are due, then reserved jobs whose retry-after lapsed
|
|
217
|
+
# (a crashed worker's in-flight job) — both move back to the ready list.
|
|
218
|
+
self._redis.eval(_MIGRATE, 3, f"{ready}:delayed", ready, f"{ready}:notify", now, -1)
|
|
219
|
+
self._redis.eval(_MIGRATE, 3, f"{ready}:reserved", ready, f"{ready}:notify", now, -1)
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
def _as_text(value: Any) -> str:
|
|
223
|
+
"""Normalise a Redis reply to ``str`` (bytes when ``decode_responses`` is off)."""
|
|
224
|
+
return value if isinstance(value, str) else value.decode()
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
def _pop_result(result: Any) -> tuple[Optional[Any], Optional[Any]]:
|
|
228
|
+
"""Unpack the ``{job, reserved}`` reply; ``false`` Lua values arrive as ``None``."""
|
|
229
|
+
if not result or len(result) < 2:
|
|
230
|
+
return None, None
|
|
231
|
+
job, reserved = result[0], result[1]
|
|
232
|
+
if not job or not reserved:
|
|
233
|
+
return None, None
|
|
234
|
+
return job, reserved
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
def _q1(q: dict, key: str) -> Optional[str]:
|
|
238
|
+
values = q.get(key)
|
|
239
|
+
return values[0] if values else None
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
def _qint(q: dict, key: str, default: int) -> int:
|
|
243
|
+
v = _q1(q, key)
|
|
244
|
+
if v is None:
|
|
245
|
+
return default
|
|
246
|
+
try:
|
|
247
|
+
return int(v)
|
|
248
|
+
except ValueError: # pragma: no cover - defensive
|
|
249
|
+
return default
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
def _qbool(q: dict, key: str) -> bool:
|
|
253
|
+
v = _q1(q, key)
|
|
254
|
+
return v is not None and v.lower() in ("1", "true", "yes", "on")
|