babelqueue 1.5.0__tar.gz → 1.7.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (68) hide show
  1. {babelqueue-1.5.0 → babelqueue-1.7.0}/CHANGELOG.md +28 -1
  2. {babelqueue-1.5.0 → babelqueue-1.7.0}/PKG-INFO +30 -4
  3. {babelqueue-1.5.0 → babelqueue-1.7.0}/README.md +29 -3
  4. {babelqueue-1.5.0 → babelqueue-1.7.0}/pyproject.toml +1 -1
  5. {babelqueue-1.5.0 → babelqueue-1.7.0}/src/babelqueue/__init__.py +6 -2
  6. babelqueue-1.7.0/src/babelqueue/exceptions.py +25 -0
  7. babelqueue-1.7.0/src/babelqueue/idempotency.py +95 -0
  8. babelqueue-1.7.0/src/babelqueue/redis_transport.py +254 -0
  9. babelqueue-1.7.0/src/babelqueue/schema.py +261 -0
  10. {babelqueue-1.5.0 → babelqueue-1.7.0}/tests/conformance/manifest.json +49 -0
  11. babelqueue-1.7.0/tests/test_idempotency.py +89 -0
  12. babelqueue-1.7.0/tests/test_redis_transport.py +396 -0
  13. babelqueue-1.7.0/tests/test_schema.py +145 -0
  14. babelqueue-1.7.0/tests/test_schema_conformance.py +33 -0
  15. babelqueue-1.5.0/src/babelqueue/exceptions.py +0 -11
  16. babelqueue-1.5.0/src/babelqueue/redis_transport.py +0 -53
  17. babelqueue-1.5.0/tests/test_redis_transport.py +0 -78
  18. {babelqueue-1.5.0 → babelqueue-1.7.0}/.github/FUNDING.yml +0 -0
  19. {babelqueue-1.5.0 → babelqueue-1.7.0}/.github/workflows/ci.yml +0 -0
  20. {babelqueue-1.5.0 → babelqueue-1.7.0}/.github/workflows/release.yml +0 -0
  21. {babelqueue-1.5.0 → babelqueue-1.7.0}/.gitignore +0 -0
  22. {babelqueue-1.5.0 → babelqueue-1.7.0}/LICENSE +0 -0
  23. {babelqueue-1.5.0 → babelqueue-1.7.0}/src/babelqueue/app.py +0 -0
  24. {babelqueue-1.5.0 → babelqueue-1.7.0}/src/babelqueue/artemis_transport.py +0 -0
  25. {babelqueue-1.5.0 → babelqueue-1.7.0}/src/babelqueue/asb_transport.py +0 -0
  26. {babelqueue-1.5.0 → babelqueue-1.7.0}/src/babelqueue/celery.py +0 -0
  27. {babelqueue-1.5.0 → babelqueue-1.7.0}/src/babelqueue/codec.py +0 -0
  28. {babelqueue-1.5.0 → babelqueue-1.7.0}/src/babelqueue/contracts.py +0 -0
  29. {babelqueue-1.5.0 → babelqueue-1.7.0}/src/babelqueue/dead_letter.py +0 -0
  30. {babelqueue-1.5.0 → babelqueue-1.7.0}/src/babelqueue/django/__init__.py +0 -0
  31. {babelqueue-1.5.0 → babelqueue-1.7.0}/src/babelqueue/django/apps.py +0 -0
  32. {babelqueue-1.5.0 → babelqueue-1.7.0}/src/babelqueue/django/management/__init__.py +0 -0
  33. {babelqueue-1.5.0 → babelqueue-1.7.0}/src/babelqueue/django/management/commands/__init__.py +0 -0
  34. {babelqueue-1.5.0 → babelqueue-1.7.0}/src/babelqueue/django/management/commands/babelqueue_worker.py +0 -0
  35. {babelqueue-1.5.0 → babelqueue-1.7.0}/src/babelqueue/kafka_transport.py +0 -0
  36. {babelqueue-1.5.0 → babelqueue-1.7.0}/src/babelqueue/pika_transport.py +0 -0
  37. {babelqueue-1.5.0 → babelqueue-1.7.0}/src/babelqueue/pulsar_transport.py +0 -0
  38. {babelqueue-1.5.0 → babelqueue-1.7.0}/src/babelqueue/py.typed +0 -0
  39. {babelqueue-1.5.0 → babelqueue-1.7.0}/src/babelqueue/routing.py +0 -0
  40. {babelqueue-1.5.0 → babelqueue-1.7.0}/src/babelqueue/sqs_transport.py +0 -0
  41. {babelqueue-1.5.0 → babelqueue-1.7.0}/src/babelqueue/transport.py +0 -0
  42. {babelqueue-1.5.0 → babelqueue-1.7.0}/tests/conformance/fixtures/dead-lettered.json +0 -0
  43. {babelqueue-1.5.0 → babelqueue-1.7.0}/tests/conformance/fixtures/invalid-missing-urn.json +0 -0
  44. {babelqueue-1.5.0 → babelqueue-1.7.0}/tests/conformance/fixtures/invalid-unknown-schema-version.json +0 -0
  45. {babelqueue-1.5.0 → babelqueue-1.7.0}/tests/conformance/fixtures/order-created.json +0 -0
  46. {babelqueue-1.5.0 → babelqueue-1.7.0}/tests/conformance/fixtures/unicode-and-numbers.json +0 -0
  47. {babelqueue-1.5.0 → babelqueue-1.7.0}/tests/conformance/fixtures/urn-alias.json +0 -0
  48. {babelqueue-1.5.0 → babelqueue-1.7.0}/tests/conformance/schema/message-envelope.schema.json +0 -0
  49. {babelqueue-1.5.0 → babelqueue-1.7.0}/tests/fixtures/dead-lettered.json +0 -0
  50. {babelqueue-1.5.0 → babelqueue-1.7.0}/tests/fixtures/order-created.json +0 -0
  51. {babelqueue-1.5.0 → babelqueue-1.7.0}/tests/test_app.py +0 -0
  52. {babelqueue-1.5.0 → babelqueue-1.7.0}/tests/test_artemis_conformance.py +0 -0
  53. {babelqueue-1.5.0 → babelqueue-1.7.0}/tests/test_artemis_transport.py +0 -0
  54. {babelqueue-1.5.0 → babelqueue-1.7.0}/tests/test_asb_conformance.py +0 -0
  55. {babelqueue-1.5.0 → babelqueue-1.7.0}/tests/test_asb_transport.py +0 -0
  56. {babelqueue-1.5.0 → babelqueue-1.7.0}/tests/test_celery.py +0 -0
  57. {babelqueue-1.5.0 → babelqueue-1.7.0}/tests/test_codec.py +0 -0
  58. {babelqueue-1.5.0 → babelqueue-1.7.0}/tests/test_conformance.py +0 -0
  59. {babelqueue-1.5.0 → babelqueue-1.7.0}/tests/test_dead_letter.py +0 -0
  60. {babelqueue-1.5.0 → babelqueue-1.7.0}/tests/test_django.py +0 -0
  61. {babelqueue-1.5.0 → babelqueue-1.7.0}/tests/test_kafka_conformance.py +0 -0
  62. {babelqueue-1.5.0 → babelqueue-1.7.0}/tests/test_kafka_transport.py +0 -0
  63. {babelqueue-1.5.0 → babelqueue-1.7.0}/tests/test_overhead.py +0 -0
  64. {babelqueue-1.5.0 → babelqueue-1.7.0}/tests/test_pika_transport.py +0 -0
  65. {babelqueue-1.5.0 → babelqueue-1.7.0}/tests/test_pulsar_conformance.py +0 -0
  66. {babelqueue-1.5.0 → babelqueue-1.7.0}/tests/test_pulsar_transport.py +0 -0
  67. {babelqueue-1.5.0 → babelqueue-1.7.0}/tests/test_sqs_conformance.py +0 -0
  68. {babelqueue-1.5.0 → babelqueue-1.7.0}/tests/test_sqs_transport.py +0 -0
@@ -7,6 +7,32 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
7
7
  The envelope wire format is versioned separately by `meta.schema_version`
8
8
  (currently **1**) — see the contract at [babelqueue.com](https://babelqueue.com).
9
9
 
10
+ ## [Unreleased]
11
+
12
+ ## [1.6.0] - 2026-06-14
13
+
14
+ ### Added
15
+ - **Redis/Laravel reservation parity** — the Redis transport can now consume a
16
+ **shared** Laravel BabelQueue Redis queue using Laravel's reserved-set / reliable-queue
17
+ semantics, not just a Python-owned queue. Enable it with the `laravel=1` URL flag
18
+ (`redis://host:6379/0?laravel=1`) or `RedisTransport(..., laravel_compat=True)`. In this
19
+ mode the key layout is Laravel's stock Redis queue ([§1 of the broker-bindings
20
+ contract](https://babelqueue.com/docs/spec/1.x/broker-bindings#redis)): a `queues:<name>`
21
+ ready list, a `queues:<name>:reserved` **sorted set** scored by a `retry_after` deadline
22
+ (default 60s), a `queues:<name>:delayed` set, and a `queues:<name>:notify` wake-up list.
23
+ Reserve, ack and release run the **byte-for-byte same Lua scripts** Laravel uses, so the
24
+ reserved member a Python worker writes is identical to a Laravel worker's — either side can
25
+ ack (`ZREM`) the other's reservation, so a Python worker and a Laravel worker share one
26
+ Redis queue without losing or double-processing messages. Before each pop, expired
27
+ reserved/delayed jobs migrate back to the ready list, so a crashed worker's in-flight job
28
+ is re-reserved exactly as Laravel does. Tunable via `?prefix=` / `?retry_after=` (or the
29
+ `key_prefix` / `retry_after` constructor args). The default Python-owned reliable-queue
30
+ mode (`BLMOVE` + `<queue>:processing` list) is unchanged and stays the default, so existing
31
+ callers are unaffected. The reservation logic is fully unit-tested with an injected
32
+ in-memory Redis double (no `redis` package, no broker); a live cross-runtime PHP↔Python
33
+ shared-queue round-trip is covered by the integration suite where a real Redis is present.
34
+ The envelope is unchanged (`schema_version: 1`); this is purely additive. Ships as a MINOR.
35
+
10
36
  ## [1.5.0] - 2026-06-13
11
37
 
12
38
  ### Added
@@ -173,7 +199,8 @@ reference at [babelqueue.com](https://babelqueue.com).
173
199
  - Pre-1.0: the public API may change before the `1.0.0` tag.
174
200
  - The core has **zero runtime dependencies** (standard library only); Python `>=3.9`.
175
201
 
176
- [Unreleased]: https://github.com/BabelQueue/babelqueue-python/compare/v1.2.0...HEAD
202
+ [Unreleased]: https://github.com/BabelQueue/babelqueue-python/compare/v1.6.0...HEAD
203
+ [1.6.0]: https://github.com/BabelQueue/babelqueue-python/compare/v1.5.0...v1.6.0
177
204
  [1.2.0]: https://github.com/BabelQueue/babelqueue-python/compare/v1.1.0...v1.2.0
178
205
  [1.1.0]: https://github.com/BabelQueue/babelqueue-python/compare/v1.0.0...v1.1.0
179
206
  [1.0.0]: https://github.com/BabelQueue/babelqueue-python/compare/v0.5.0...v1.0.0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: babelqueue
3
- Version: 1.5.0
3
+ Version: 1.7.0
4
4
  Summary: Polyglot Queues, Simplified — the Python core: the canonical BabelQueue wire-envelope codec, contracts and dead-letter helpers.
5
5
  Project-URL: Homepage, https://babelqueue.com
6
6
  Project-URL: Source, https://github.com/BabelQueue/babelqueue-python
@@ -164,9 +164,35 @@ app.run() # consume forever (Ctrl-C to stop)
164
164
  - **Retry & dead-letter:** failures are retried up to `max_attempts` (bumping the
165
165
  envelope's `attempts`); enable `dead_letter=True` to quarantine exhausted
166
166
  messages on `<queue>.dlq`. `on_unknown_urn` = `fail` | `delete` | `release` | `dead_letter`.
167
- - **Transports:** `redis://` (reliable-queue pattern), `amqp://` (RabbitMQ via
168
- `pika`, with the contract AMQP properties) and `memory://` (in-process, great for
169
- tests/local). Bring your own by passing `transport=...`.
167
+ - **Transports:** `redis://` (reliable-queue pattern; add `?laravel=1` to share a
168
+ Redis queue with a Laravel BabelQueue worker using its reserved-set semantics —
169
+ see below), `amqp://` (RabbitMQ via `pika`, with the contract AMQP properties)
170
+ and `memory://` (in-process, great for tests/local). Bring your own by passing
171
+ `transport=...`.
172
+
173
+ ### Sharing a Redis queue with Laravel
174
+
175
+ By default the Redis transport owns its queue end-to-end (`RPUSH` to produce;
176
+ `BLMOVE` into a `<queue>:processing` list to reserve; `LREM` on ack). To consume a
177
+ **shared** Laravel BabelQueue Redis queue instead, enable Laravel-compatible mode:
178
+
179
+ ```python
180
+ app = BabelQueue("redis://localhost:6379/0?laravel=1", queue="orders")
181
+ # or: RedisTransport("redis://localhost:6379/0", laravel_compat=True)
182
+ ```
183
+
184
+ This replicates Laravel's stock Redis reservation exactly ([§1 of the
185
+ broker-bindings contract](https://babelqueue.com/docs/spec/1.x/broker-bindings#redis)):
186
+ the ready list is `queues:<name>`, reservations move into the `queues:<name>:reserved`
187
+ **sorted set** scored by a `retry_after` deadline (default 60s), with a
188
+ `queues:<name>:delayed` set and a `queues:<name>:notify` wake-up list. Reserve, ack
189
+ and release run the **byte-for-byte same Lua scripts** Laravel uses, so the reserved
190
+ member a Python worker writes is identical to a Laravel worker's — either side can
191
+ ack (`ZREM`) the other's reservation, so a Python worker and a Laravel worker share
192
+ one queue without losing or double-processing messages. Before each pop, expired
193
+ reserved/delayed jobs migrate back to the ready list, so a crashed worker's
194
+ in-flight job is re-reserved. Options: `?prefix=queues:` and `?retry_after=60`
195
+ (or the `key_prefix` / `retry_after` constructor args).
170
196
 
171
197
  ## Framework adapters — Celery & Django
172
198
 
@@ -114,9 +114,35 @@ app.run() # consume forever (Ctrl-C to stop)
114
114
  - **Retry & dead-letter:** failures are retried up to `max_attempts` (bumping the
115
115
  envelope's `attempts`); enable `dead_letter=True` to quarantine exhausted
116
116
  messages on `<queue>.dlq`. `on_unknown_urn` = `fail` | `delete` | `release` | `dead_letter`.
117
- - **Transports:** `redis://` (reliable-queue pattern), `amqp://` (RabbitMQ via
118
- `pika`, with the contract AMQP properties) and `memory://` (in-process, great for
119
- tests/local). Bring your own by passing `transport=...`.
117
+ - **Transports:** `redis://` (reliable-queue pattern; add `?laravel=1` to share a
118
+ Redis queue with a Laravel BabelQueue worker using its reserved-set semantics —
119
+ see below), `amqp://` (RabbitMQ via `pika`, with the contract AMQP properties)
120
+ and `memory://` (in-process, great for tests/local). Bring your own by passing
121
+ `transport=...`.
122
+
123
+ ### Sharing a Redis queue with Laravel
124
+
125
+ By default the Redis transport owns its queue end-to-end (`RPUSH` to produce;
126
+ `BLMOVE` into a `<queue>:processing` list to reserve; `LREM` on ack). To consume a
127
+ **shared** Laravel BabelQueue Redis queue instead, enable Laravel-compatible mode:
128
+
129
+ ```python
130
+ app = BabelQueue("redis://localhost:6379/0?laravel=1", queue="orders")
131
+ # or: RedisTransport("redis://localhost:6379/0", laravel_compat=True)
132
+ ```
133
+
134
+ This replicates Laravel's stock Redis reservation exactly ([§1 of the
135
+ broker-bindings contract](https://babelqueue.com/docs/spec/1.x/broker-bindings#redis)):
136
+ the ready list is `queues:<name>`, reservations move into the `queues:<name>:reserved`
137
+ **sorted set** scored by a `retry_after` deadline (default 60s), with a
138
+ `queues:<name>:delayed` set and a `queues:<name>:notify` wake-up list. Reserve, ack
139
+ and release run the **byte-for-byte same Lua scripts** Laravel uses, so the reserved
140
+ member a Python worker writes is identical to a Laravel worker's — either side can
141
+ ack (`ZREM`) the other's reservation, so a Python worker and a Laravel worker share
142
+ one queue without losing or double-processing messages. Before each pop, expired
143
+ reserved/delayed jobs migrate back to the ready list, so a crashed worker's
144
+ in-flight job is re-reserved. Options: `?prefix=queues:` and `?retry_after=60`
145
+ (or the `key_prefix` / `retry_after` constructor args).
120
146
 
121
147
  ## Framework adapters — Celery & Django
122
148
 
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
4
4
 
5
5
  [project]
6
6
  name = "babelqueue"
7
- version = "1.5.0"
7
+ version = "1.7.0"
8
8
  description = "Polyglot Queues, Simplified — the Python core: the canonical BabelQueue wire-envelope codec, contracts and dead-letter helpers."
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.9"
@@ -11,15 +11,16 @@ and dead-letter helpers. Framework adapters (Celery, Django, ...) build on this.
11
11
 
12
12
  from __future__ import annotations
13
13
 
14
- from . import dead_letter
14
+ from . import dead_letter, idempotency
15
15
  from .app import BabelQueue
16
16
  from .codec import SCHEMA_VERSION, SOURCE_LANG, EnvelopeCodec
17
17
  from .contracts import HasTraceId, PolyglotMessage
18
+ from .idempotency import IdempotencyStore, InMemoryStore
18
19
  from .exceptions import BabelQueueError, UnknownUrnError
19
20
  from .routing import UnknownUrnStrategy
20
21
  from .transport import InMemoryTransport, ReceivedMessage, Transport
21
22
 
22
- __version__ = "1.2.0"
23
+ __version__ = "1.6.0"
23
24
 
24
25
  __all__ = [
25
26
  "BabelQueue",
@@ -35,5 +36,8 @@ __all__ = [
35
36
  "BabelQueueError",
36
37
  "UnknownUrnError",
37
38
  "dead_letter",
39
+ "idempotency",
40
+ "IdempotencyStore",
41
+ "InMemoryStore",
38
42
  "__version__",
39
43
  ]
@@ -0,0 +1,25 @@
1
+ """Exception hierarchy for BabelQueue."""
2
+
3
+ from __future__ import annotations
4
+
5
+
6
+ class BabelQueueError(Exception):
7
+ """Base for every recoverable BabelQueue error (bad config, empty URN, ...)."""
8
+
9
+
10
+ class UnknownUrnError(BabelQueueError):
11
+ """A consumed message carries a URN with no mapped handler (strategy "fail")."""
12
+
13
+
14
+ class InvalidPayloadError(BabelQueueError):
15
+ """A message's ``data`` does not match the JSON Schema registered for its URN (ADR-0024).
16
+
17
+ Raised by the producer-side :func:`babelqueue.schema.validate` and the consumer-side
18
+ :func:`babelqueue.schema.wrap`; the latter lets the runtime redeliver (and eventually
19
+ dead-letter) a poison message.
20
+ """
21
+
22
+ def __init__(self, urn: str, violation: str) -> None:
23
+ super().__init__(f"data for {urn!r} does not match its URN schema: {violation}")
24
+ self.urn = urn
25
+ self.violation = violation
@@ -0,0 +1,95 @@
1
+ """Optional idempotency helper (ADR-0022): dedupe a consume handler on ``meta.id``.
2
+
3
+ The Python mirror of the PHP ``BabelQueue\\Idempotency`` and Go ``idempotency``
4
+ helpers. It wraps a handler so a message whose ``meta.id`` was already processed
5
+ successfully is skipped instead of run again, composing with the runtime's
6
+ ack-on-return / redeliver-on-raise contract::
7
+
8
+ from babelqueue import BabelQueue
9
+ from babelqueue.idempotency import InMemoryStore, wrap
10
+
11
+ app = BabelQueue("redis://localhost:6379/0", queue="orders")
12
+ store = InMemoryStore()
13
+ app.register("urn:babel:orders:created", wrap(store, on_order_created))
14
+
15
+ A previously-seen id returns early (the runtime acks it, so the broker stops
16
+ redelivering); a raising handler leaves the id unmarked so a redelivery runs it again
17
+ (retry / dead-letter still apply); a message with no usable ``meta.id`` runs unchanged.
18
+ This is "seen-set" post-success dedupe — not exactly-once and not in-flight concurrency
19
+ locking; a transactional / outbox mode is a documented future direction.
20
+ """
21
+
22
+ from __future__ import annotations
23
+
24
+ import functools
25
+ import threading
26
+ from typing import Any, Callable, Mapping, Protocol, runtime_checkable
27
+
28
+ Handler = Callable[..., None]
29
+
30
+
31
+ @runtime_checkable
32
+ class IdempotencyStore(Protocol):
33
+ """A record of message ids already processed, keyed on ``meta.id``."""
34
+
35
+ def seen(self, message_id: str) -> bool:
36
+ """Whether this id has already been processed (remembered)."""
37
+
38
+ def remember(self, message_id: str) -> None:
39
+ """Record this id as processed."""
40
+
41
+ def forget(self, message_id: str) -> None:
42
+ """Drop an id from the store (manual eviction; a backend may also expire ids)."""
43
+
44
+
45
+ class InMemoryStore:
46
+ """Process-local, thread-safe :class:`IdempotencyStore`.
47
+
48
+ For tests and single-process consumers; it is not shared across workers and not
49
+ persistent — use a Redis- or database-backed store for production fleets.
50
+ """
51
+
52
+ def __init__(self) -> None:
53
+ self._seen: set[str] = set()
54
+ self._lock = threading.Lock()
55
+
56
+ def seen(self, message_id: str) -> bool:
57
+ with self._lock:
58
+ return message_id in self._seen
59
+
60
+ def remember(self, message_id: str) -> None:
61
+ with self._lock:
62
+ self._seen.add(message_id)
63
+
64
+ def forget(self, message_id: str) -> None:
65
+ with self._lock:
66
+ self._seen.discard(message_id)
67
+
68
+
69
+ def wrap(store: IdempotencyStore, handler: Handler) -> Handler:
70
+ """Wrap ``handler`` so a message whose ``meta.id`` was already processed is skipped.
71
+
72
+ The returned callable keeps ``handler``'s signature (via :func:`functools.wraps`),
73
+ so the runtime's introspection still passes it the right number of positional args
74
+ (``data, meta`` or ``data, meta, envelope``).
75
+ """
76
+
77
+ @functools.wraps(handler)
78
+ def wrapped(*args: Any) -> None:
79
+ meta = args[1] if len(args) > 1 and isinstance(args[1], Mapping) else {}
80
+ message_id = meta.get("id")
81
+
82
+ # No usable id → cannot dedupe; run the handler unchanged.
83
+ if not isinstance(message_id, str) or message_id == "":
84
+ handler(*args)
85
+ return
86
+
87
+ # Already processed on an earlier delivery: return so the runtime acks it.
88
+ if store.seen(message_id):
89
+ return
90
+
91
+ # First success wins; a raise here leaves the id unmarked → retry/DLQ apply.
92
+ handler(*args)
93
+ store.remember(message_id)
94
+
95
+ return wrapped
@@ -0,0 +1,254 @@
1
+ """Redis transport. Requires the ``redis`` extra:
2
+
3
+ pip install "babelqueue[redis]"
4
+
5
+ Two reservation modes share one class, selected by ``laravel_compat``:
6
+
7
+ **Python-owned (default).** Producing is ``RPUSH queue body``; consuming atomically
8
+ moves the head to a per-queue processing list (``BLMOVE``) so an in-flight message
9
+ survives a worker crash, and ``ack`` removes it from that processing list. Simple,
10
+ self-contained, and the right choice for a queue this runtime owns end-to-end.
11
+
12
+ **Laravel-compatible (``laravel_compat=True``).** Replicates Laravel's stock Redis
13
+ queue reservation so a Python worker can share one Redis queue with a PHP/Laravel
14
+ worker without losing or double-processing messages (§1 of the broker-bindings
15
+ contract). The key layout is Laravel's: a ``queues:<name>`` ready list, a
16
+ ``queues:<name>:reserved`` sorted set scored by a retry-after deadline, a
17
+ ``queues:<name>:delayed`` sorted set, and a ``queues:<name>:notify`` wake-up list.
18
+ Reserve/ack/release run the **byte-for-byte same Lua scripts** Laravel uses, so the
19
+ reserved-set member a Python worker writes is identical to what a Laravel worker
20
+ writes — either side can therefore ack (``ZREM``) the other's reservation. Before
21
+ each pop the transport migrates expired reserved/delayed jobs back to the ready
22
+ list, so a crashed worker's in-flight job is re-reserved exactly as Laravel does.
23
+
24
+ The envelope is unchanged (``schema_version`` stays 1); Redis is purely additive.
25
+
26
+ URL form: ``redis://host:port/db[?laravel=1&prefix=queues:&retry_after=60]``.
27
+ For richer setups, build the transport directly and pass ``BabelQueue(transport=...)``.
28
+ """
29
+
30
+ from __future__ import annotations
31
+
32
+ import time
33
+ from typing import Any, Optional
34
+ from urllib.parse import parse_qs, urlsplit
35
+
36
+ from .transport import ReceivedMessage, Transport
37
+
38
+ # Laravel's stock Redis queue Lua scripts (Illuminate\Queue\LuaScripts). Replicated
39
+ # verbatim so the reserved-set member byte form matches Laravel's — that exact match
40
+ # is what lets a Python and a Laravel worker ack each other's reservations.
41
+
42
+ # KEYS[1]=ready list, KEYS[2]=notify list; ARGV[1]=payload.
43
+ _PUSH = """
44
+ redis.call('rpush', KEYS[1], ARGV[1])
45
+ redis.call('rpush', KEYS[2], 1)
46
+ """
47
+
48
+ # KEYS[1]=ready, KEYS[2]=reserved, KEYS[3]=notify; ARGV[1]=reserved-until score.
49
+ # Returns {original_job, reserved_member}; the reserved member is the attempts+1
50
+ # re-encoding stored in the reserved set and is the ack handle.
51
+ _POP = """
52
+ local job = redis.call('lpop', KEYS[1])
53
+ local reserved = false
54
+
55
+ if(job ~= false) then
56
+ reserved = cjson.decode(job)
57
+ reserved['attempts'] = reserved['attempts'] + 1
58
+ reserved = cjson.encode(reserved)
59
+ redis.call('zadd', KEYS[2], ARGV[1], reserved)
60
+ redis.call('lpop', KEYS[3])
61
+ end
62
+
63
+ return {job, reserved}
64
+ """
65
+
66
+ # KEYS[1]=delayed, KEYS[2]=reserved; ARGV[1]=reserved member, ARGV[2]=available-at.
67
+ _RELEASE = """
68
+ redis.call('zrem', KEYS[2], ARGV[1])
69
+ redis.call('zadd', KEYS[1], ARGV[2], ARGV[1])
70
+ return true
71
+ """
72
+
73
+ # KEYS[1]=from (reserved/delayed), KEYS[2]=ready, KEYS[3]=notify; ARGV[1]=now,
74
+ # ARGV[2]=batch size (-1 = all). Moves jobs whose score has expired back to ready.
75
+ _MIGRATE = """
76
+ local val = redis.call('zrangebyscore', KEYS[1], '-inf', ARGV[1], 'limit', 0, ARGV[2])
77
+
78
+ if(next(val) ~= nil) then
79
+ redis.call('zremrangebyrank', KEYS[1], 0, #val - 1)
80
+
81
+ for i = 1, #val, 100 do
82
+ redis.call('rpush', KEYS[2], unpack(val, i, math.min(i+99, #val)))
83
+ for j = i, math.min(i+99, #val) do
84
+ redis.call('rpush', KEYS[3], 1)
85
+ end
86
+ end
87
+ end
88
+
89
+ return val
90
+ """
91
+
92
+
93
+ class RedisTransport(Transport):
94
+ def __init__(
95
+ self,
96
+ url: str = "redis://localhost:6379/0",
97
+ *,
98
+ client: Any = None,
99
+ processing_suffix: str = ":processing",
100
+ laravel_compat: bool = False,
101
+ key_prefix: str = "queues:",
102
+ retry_after: int = 60,
103
+ block_for: Optional[float] = None,
104
+ ) -> None:
105
+ parts = urlsplit(url) if url else urlsplit("redis://")
106
+ q = parse_qs(parts.query)
107
+
108
+ self._processing_suffix = processing_suffix
109
+ self._laravel_compat = laravel_compat or _qbool(q, "laravel") or _qbool(q, "laravel_compat")
110
+ self._key_prefix = key_prefix if "prefix" not in q else (_q1(q, "prefix") or key_prefix)
111
+ self._retry_after = retry_after if "retry_after" not in q else _qint(q, "retry_after", retry_after)
112
+ self._block_for = block_for
113
+
114
+ if client is not None:
115
+ self._redis = client
116
+ return
117
+ try:
118
+ import redis
119
+ except ImportError as exc: # pragma: no cover - import guard
120
+ raise ImportError(
121
+ "RedisTransport requires the 'redis' package. Install with "
122
+ 'pip install "babelqueue[redis]".'
123
+ ) from exc
124
+ self._redis = redis.Redis.from_url(url, decode_responses=True)
125
+
126
+ # -- key helpers --------------------------------------------------------
127
+
128
+ def _processing(self, queue: str) -> str:
129
+ return f"{queue}{self._processing_suffix}"
130
+
131
+ def _ready(self, queue: str) -> str:
132
+ """The Laravel ready-list key for a logical queue name (``queues:<name>``)."""
133
+ return f"{self._key_prefix}{queue}"
134
+
135
+ # -- Transport ----------------------------------------------------------
136
+
137
+ def publish(self, queue: str, body: str) -> None:
138
+ if not self._laravel_compat:
139
+ self._redis.rpush(queue, body)
140
+ return
141
+ ready = self._ready(queue)
142
+ self._redis.eval(_PUSH, 2, ready, f"{ready}:notify", body)
143
+
144
+ def pop(self, queue: str, timeout: float = 1.0) -> Optional[ReceivedMessage]:
145
+ if not self._laravel_compat:
146
+ return self._pop_owned(queue, timeout)
147
+ return self._pop_laravel(queue, timeout)
148
+
149
+ def ack(self, message: ReceivedMessage) -> None:
150
+ if not self._laravel_compat:
151
+ self._redis.lrem(self._processing(message.queue), 1, message.handle)
152
+ return
153
+ if not message.handle:
154
+ return
155
+ self._redis.zrem(f"{self._ready(message.queue)}:reserved", message.handle)
156
+
157
+ def release(self, message: ReceivedMessage, delay: int = 0) -> None:
158
+ """Return a reserved Laravel job to the (delayed) ready set for retry.
159
+
160
+ No-op in Python-owned mode (the runtime re-publishes instead). In
161
+ Laravel-compatible mode this is ``ZREM :reserved`` + ``ZADD :delayed`` —
162
+ the exact bookkeeping Laravel's release performs.
163
+ """
164
+ if not self._laravel_compat or not message.handle:
165
+ return
166
+ ready = self._ready(message.queue)
167
+ self._redis.eval(
168
+ _RELEASE, 2, f"{ready}:delayed", f"{ready}:reserved",
169
+ message.handle, int(time.time()) + int(delay),
170
+ )
171
+
172
+ def close(self) -> None: # pragma: no cover
173
+ self._redis.close()
174
+
175
+ # -- Python-owned reliable queue ----------------------------------------
176
+
177
+ def _pop_owned(self, queue: str, timeout: float) -> Optional[ReceivedMessage]:
178
+ # redis-py types the BLMOVE timeout as int, but Redis accepts a float
179
+ # (sub-second) timeout; passing it through is correct at runtime.
180
+ body = self._redis.blmove(queue, self._processing(queue), timeout, "LEFT", "RIGHT") # type: ignore[arg-type]
181
+ if body is None:
182
+ return None
183
+ text = _as_text(body)
184
+ return ReceivedMessage(body=text, queue=queue, handle=text)
185
+
186
+ # -- Laravel-compatible reservation -------------------------------------
187
+
188
+ def _pop_laravel(self, queue: str, timeout: float) -> Optional[ReceivedMessage]:
189
+ ready = self._ready(queue)
190
+ self._migrate(ready)
191
+
192
+ result = self._redis.eval(
193
+ _POP, 3, ready, f"{ready}:reserved", f"{ready}:notify",
194
+ int(time.time()) + self._retry_after,
195
+ )
196
+ job, reserved = _pop_result(result)
197
+ if job is None or reserved is None:
198
+ # Nothing ready right now; optionally block on the notify list so a
199
+ # concurrently-pushed job wakes us, then retry once (mirrors Laravel).
200
+ block = self._block_for if self._block_for is not None else timeout
201
+ if block and block > 0 and self._redis.blpop([f"{ready}:notify"], block):
202
+ result = self._redis.eval(
203
+ _POP, 3, ready, f"{ready}:reserved", f"{ready}:notify",
204
+ int(time.time()) + self._retry_after,
205
+ )
206
+ job, reserved = _pop_result(result)
207
+ if job is None or reserved is None:
208
+ return None
209
+
210
+ # The reserved member (attempts already incremented) is both the consumed
211
+ # body and the ack handle, so ZREM removes exactly this reservation.
212
+ return ReceivedMessage(body=_as_text(reserved), queue=queue, handle=_as_text(reserved))
213
+
214
+ def _migrate(self, ready: str) -> None:
215
+ now = int(time.time())
216
+ # Delayed jobs that are due, then reserved jobs whose retry-after lapsed
217
+ # (a crashed worker's in-flight job) — both move back to the ready list.
218
+ self._redis.eval(_MIGRATE, 3, f"{ready}:delayed", ready, f"{ready}:notify", now, -1)
219
+ self._redis.eval(_MIGRATE, 3, f"{ready}:reserved", ready, f"{ready}:notify", now, -1)
220
+
221
+
222
+ def _as_text(value: Any) -> str:
223
+ """Normalise a Redis reply to ``str`` (bytes when ``decode_responses`` is off)."""
224
+ return value if isinstance(value, str) else value.decode()
225
+
226
+
227
+ def _pop_result(result: Any) -> tuple[Optional[Any], Optional[Any]]:
228
+ """Unpack the ``{job, reserved}`` reply; ``false`` Lua values arrive as ``None``."""
229
+ if not result or len(result) < 2:
230
+ return None, None
231
+ job, reserved = result[0], result[1]
232
+ if not job or not reserved:
233
+ return None, None
234
+ return job, reserved
235
+
236
+
237
+ def _q1(q: dict, key: str) -> Optional[str]:
238
+ values = q.get(key)
239
+ return values[0] if values else None
240
+
241
+
242
+ def _qint(q: dict, key: str, default: int) -> int:
243
+ v = _q1(q, key)
244
+ if v is None:
245
+ return default
246
+ try:
247
+ return int(v)
248
+ except ValueError: # pragma: no cover - defensive
249
+ return default
250
+
251
+
252
+ def _qbool(q: dict, key: str) -> bool:
253
+ v = _q1(q, key)
254
+ return v is not None and v.lower() in ("1", "true", "yes", "on")