redis-message-queue 8.2.0__tar.gz → 8.2.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- redis_message_queue-8.2.2/.gitignore +167 -0
- redis_message_queue-8.2.0/README.md → redis_message_queue-8.2.2/PKG-INFO +75 -18
- redis_message_queue-8.2.0/PKG-INFO → redis_message_queue-8.2.2/README.md +52 -45
- {redis_message_queue-8.2.0 → redis_message_queue-8.2.2}/pyproject.toml +62 -17
- {redis_message_queue-8.2.0 → redis_message_queue-8.2.2}/redis_message_queue/_redis_gateway.py +3 -3
- {redis_message_queue-8.2.0 → redis_message_queue-8.2.2}/redis_message_queue/asyncio/_redis_gateway.py +3 -3
- {redis_message_queue-8.2.0 → redis_message_queue-8.2.2}/redis_message_queue/asyncio/redis_message_queue.py +26 -14
- {redis_message_queue-8.2.0 → redis_message_queue-8.2.2}/redis_message_queue/redis_message_queue.py +24 -8
- {redis_message_queue-8.2.0 → redis_message_queue-8.2.2}/LICENSE +0 -0
- {redis_message_queue-8.2.0 → redis_message_queue-8.2.2}/redis_message_queue/__init__.py +0 -0
- {redis_message_queue-8.2.0 → redis_message_queue-8.2.2}/redis_message_queue/_abstract_redis_gateway.py +0 -0
- {redis_message_queue-8.2.0 → redis_message_queue-8.2.2}/redis_message_queue/_callable_utils.py +0 -0
- {redis_message_queue-8.2.0 → redis_message_queue-8.2.2}/redis_message_queue/_config.py +0 -0
- {redis_message_queue-8.2.0 → redis_message_queue-8.2.2}/redis_message_queue/_event.py +0 -0
- {redis_message_queue-8.2.0 → redis_message_queue-8.2.2}/redis_message_queue/_exceptions.py +0 -0
- {redis_message_queue-8.2.0 → redis_message_queue-8.2.2}/redis_message_queue/_payload_limits.py +0 -0
- {redis_message_queue-8.2.0 → redis_message_queue-8.2.2}/redis_message_queue/_queue_key_manager.py +0 -0
- {redis_message_queue-8.2.0 → redis_message_queue-8.2.2}/redis_message_queue/_redis_cluster.py +0 -0
- {redis_message_queue-8.2.0 → redis_message_queue-8.2.2}/redis_message_queue/_stored_message.py +0 -0
- {redis_message_queue-8.2.0 → redis_message_queue-8.2.2}/redis_message_queue/asyncio/__init__.py +0 -0
- {redis_message_queue-8.2.0 → redis_message_queue-8.2.2}/redis_message_queue/asyncio/_abstract_redis_gateway.py +0 -0
- {redis_message_queue-8.2.0 → redis_message_queue-8.2.2}/redis_message_queue/interrupt_handler/__init__.py +0 -0
- {redis_message_queue-8.2.0 → redis_message_queue-8.2.2}/redis_message_queue/interrupt_handler/_event_driven.py +0 -0
- {redis_message_queue-8.2.0 → redis_message_queue-8.2.2}/redis_message_queue/interrupt_handler/_implementation.py +0 -0
- {redis_message_queue-8.2.0 → redis_message_queue-8.2.2}/redis_message_queue/interrupt_handler/_interface.py +0 -0
- {redis_message_queue-8.2.0 → redis_message_queue-8.2.2}/redis_message_queue/py.typed +0 -0
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
.DS_Store
|
|
2
|
+
|
|
3
|
+
# Claude Code
|
|
4
|
+
.claude/
|
|
5
|
+
|
|
6
|
+
# Byte-compiled / optimized / DLL files
|
|
7
|
+
__pycache__/
|
|
8
|
+
*.py[cod]
|
|
9
|
+
*$py.class
|
|
10
|
+
|
|
11
|
+
# C extensions
|
|
12
|
+
*.so
|
|
13
|
+
|
|
14
|
+
# Distribution / packaging
|
|
15
|
+
.Python
|
|
16
|
+
build/
|
|
17
|
+
develop-eggs/
|
|
18
|
+
dist/
|
|
19
|
+
downloads/
|
|
20
|
+
eggs/
|
|
21
|
+
.eggs/
|
|
22
|
+
lib/
|
|
23
|
+
lib64/
|
|
24
|
+
parts/
|
|
25
|
+
sdist/
|
|
26
|
+
var/
|
|
27
|
+
wheels/
|
|
28
|
+
share/python-wheels/
|
|
29
|
+
*.egg-info/
|
|
30
|
+
.installed.cfg
|
|
31
|
+
*.egg
|
|
32
|
+
MANIFEST
|
|
33
|
+
|
|
34
|
+
# PyInstaller
|
|
35
|
+
# Usually these files are written by a python script from a template
|
|
36
|
+
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
|
37
|
+
*.manifest
|
|
38
|
+
*.spec
|
|
39
|
+
|
|
40
|
+
# Installer logs
|
|
41
|
+
pip-log.txt
|
|
42
|
+
pip-delete-this-directory.txt
|
|
43
|
+
|
|
44
|
+
# Unit test / coverage reports
|
|
45
|
+
htmlcov/
|
|
46
|
+
.tox/
|
|
47
|
+
.nox/
|
|
48
|
+
.coverage
|
|
49
|
+
.coverage.*
|
|
50
|
+
.cache
|
|
51
|
+
nosetests.xml
|
|
52
|
+
coverage.xml
|
|
53
|
+
*.cover
|
|
54
|
+
*.py,cover
|
|
55
|
+
.hypothesis/
|
|
56
|
+
.pytest_cache/
|
|
57
|
+
cover/
|
|
58
|
+
|
|
59
|
+
# Translations
|
|
60
|
+
*.mo
|
|
61
|
+
*.pot
|
|
62
|
+
|
|
63
|
+
# Django stuff:
|
|
64
|
+
*.log
|
|
65
|
+
local_settings.py
|
|
66
|
+
db.sqlite3
|
|
67
|
+
db.sqlite3-journal
|
|
68
|
+
|
|
69
|
+
# Flask stuff:
|
|
70
|
+
instance/
|
|
71
|
+
.webassets-cache
|
|
72
|
+
|
|
73
|
+
# Scrapy stuff:
|
|
74
|
+
.scrapy
|
|
75
|
+
|
|
76
|
+
# Sphinx documentation
|
|
77
|
+
docs/_build/
|
|
78
|
+
|
|
79
|
+
# PyBuilder
|
|
80
|
+
.pybuilder/
|
|
81
|
+
target/
|
|
82
|
+
|
|
83
|
+
# Jupyter Notebook
|
|
84
|
+
.ipynb_checkpoints
|
|
85
|
+
|
|
86
|
+
# IPython
|
|
87
|
+
profile_default/
|
|
88
|
+
ipython_config.py
|
|
89
|
+
|
|
90
|
+
# pyenv
|
|
91
|
+
# For a library or package, you might want to ignore these files since the code is
|
|
92
|
+
# intended to run in multiple environments; otherwise, check them in:
|
|
93
|
+
.python-version
|
|
94
|
+
|
|
95
|
+
# pipenv
|
|
96
|
+
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
|
|
97
|
+
# However, in case of collaboration, if having platform-specific dependencies or dependencies
|
|
98
|
+
# having no cross-platform support, pipenv may install dependencies that don't work, or not
|
|
99
|
+
# install all needed dependencies.
|
|
100
|
+
#Pipfile.lock
|
|
101
|
+
|
|
102
|
+
# pdm
|
|
103
|
+
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
|
|
104
|
+
#pdm.lock
|
|
105
|
+
# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
|
|
106
|
+
# in version control.
|
|
107
|
+
# https://pdm.fming.dev/#use-with-ide
|
|
108
|
+
.pdm.toml
|
|
109
|
+
|
|
110
|
+
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
|
|
111
|
+
__pypackages__/
|
|
112
|
+
|
|
113
|
+
# Celery stuff
|
|
114
|
+
celerybeat-schedule
|
|
115
|
+
celerybeat.pid
|
|
116
|
+
|
|
117
|
+
# SageMath parsed files
|
|
118
|
+
*.sage.py
|
|
119
|
+
|
|
120
|
+
# Environments
|
|
121
|
+
.env
|
|
122
|
+
.venv
|
|
123
|
+
env/
|
|
124
|
+
venv/
|
|
125
|
+
ENV/
|
|
126
|
+
env.bak/
|
|
127
|
+
venv.bak/
|
|
128
|
+
|
|
129
|
+
# Spyder project settings
|
|
130
|
+
.spyderproject
|
|
131
|
+
.spyproject
|
|
132
|
+
|
|
133
|
+
# Rope project settings
|
|
134
|
+
.ropeproject
|
|
135
|
+
|
|
136
|
+
# mkdocs documentation
|
|
137
|
+
/site
|
|
138
|
+
|
|
139
|
+
# mypy
|
|
140
|
+
.mypy_cache/
|
|
141
|
+
.dmypy.json
|
|
142
|
+
dmypy.json
|
|
143
|
+
|
|
144
|
+
# Ruff
|
|
145
|
+
.ruff_cache/
|
|
146
|
+
|
|
147
|
+
# Coverage tooling output
|
|
148
|
+
lcov.info
|
|
149
|
+
|
|
150
|
+
# PyPI upload config — credentials, never commit
|
|
151
|
+
.pypirc
|
|
152
|
+
|
|
153
|
+
# Pyre type checker
|
|
154
|
+
.pyre/
|
|
155
|
+
|
|
156
|
+
# pytype static type analyzer
|
|
157
|
+
.pytype/
|
|
158
|
+
|
|
159
|
+
# Cython debug symbols
|
|
160
|
+
cython_debug/
|
|
161
|
+
|
|
162
|
+
# PyCharm
|
|
163
|
+
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
|
|
164
|
+
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
|
|
165
|
+
# and can be added to the global gitignore or merged into this file. For a more nuclear
|
|
166
|
+
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
|
|
167
|
+
#.idea/
|
|
@@ -1,6 +1,29 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: redis-message-queue
|
|
3
|
+
Version: 8.2.2
|
|
4
|
+
Summary: Python message queuing with Redis and message deduplication
|
|
5
|
+
Project-URL: Homepage, https://github.com/Elijas/redis-message-queue
|
|
6
|
+
Project-URL: Repository, https://github.com/Elijas/redis-message-queue
|
|
7
|
+
Project-URL: Issues, https://github.com/Elijas/redis-message-queue/issues
|
|
8
|
+
Author-email: Elijas <4084885+Elijas@users.noreply.github.com>
|
|
9
|
+
License: MIT
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Keywords: deduplication,message-queue,redis,task-queue
|
|
12
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
17
|
+
Classifier: Topic :: Software Development :: Libraries
|
|
18
|
+
Classifier: Topic :: System :: Distributed Computing
|
|
19
|
+
Requires-Python: <4.0,>=3.12
|
|
20
|
+
Requires-Dist: redis<8.0.0,>=5.0.1
|
|
21
|
+
Requires-Dist: tenacity>=8.1.0
|
|
22
|
+
Description-Content-Type: text/markdown
|
|
23
|
+
|
|
1
24
|
# redis-message-queue
|
|
2
25
|
|
|
3
|
-
[](https://pypi.org/project/redis-message-queue)
|
|
4
27
|
[](https://pypistats.org/packages/redis-message-queue)
|
|
5
28
|
[](LICENSE)
|
|
6
29
|
[](https://github.com/Elijas/redis-message-queue/issues)
|
|
@@ -11,7 +34,7 @@
|
|
|
11
34
|
**Lightweight Python message queuing with Redis and built-in publish-side deduplication.** Deduplicate publishes within a TTL window, with optional crash recovery — across any number of producers and consumers.
|
|
12
35
|
|
|
13
36
|
```bash
|
|
14
|
-
pip install "redis-message-queue>=8.
|
|
37
|
+
pip install "redis-message-queue>=8.2.2,<9.0.0"
|
|
15
38
|
```
|
|
16
39
|
|
|
17
40
|
Requires Redis server >= 6.2.
|
|
@@ -22,6 +45,7 @@ Redis must be running locally first: use `redis-server` or
|
|
|
22
45
|
`docker run -p 6379:6379 redis:7`.
|
|
23
46
|
|
|
24
47
|
```python
|
|
48
|
+
import json
|
|
25
49
|
from redis import Redis
|
|
26
50
|
from redis_message_queue import RedisMessageQueue
|
|
27
51
|
|
|
@@ -35,7 +59,8 @@ queue = RedisMessageQueue(
|
|
|
35
59
|
queue.publish({"id": "msg-1", "text": "hello"})
|
|
36
60
|
with queue.process_message() as message:
|
|
37
61
|
if message is not None:
|
|
38
|
-
|
|
62
|
+
payload = json.loads(message)
|
|
63
|
+
print(f"got {payload['text']}")
|
|
39
64
|
# Expected output: got hello
|
|
40
65
|
```
|
|
41
66
|
|
|
@@ -46,13 +71,15 @@ with queue.process_message() as message:
|
|
|
46
71
|
> synchronous. If your handler is `async def`, returns a coroutine, or returns
|
|
47
72
|
> any other awaitable, use `redis_message_queue.asyncio.RedisMessageQueue`.
|
|
48
73
|
> The sync context manager does not inspect the handler's return value; an
|
|
49
|
-
> unawaited coroutine can be dropped while the message is acked.
|
|
50
|
-
> callback
|
|
74
|
+
> unawaited coroutine can be dropped while the message is acked. For sync
|
|
75
|
+
> callback-style handlers, use `process_message_callback(handler)`: it checks
|
|
76
|
+
> for awaitable returns before acking and raises `TypeError` if one is returned.
|
|
51
77
|
|
|
52
78
|
### Async quickstart
|
|
53
79
|
|
|
54
80
|
```python
|
|
55
81
|
import asyncio
|
|
82
|
+
import json
|
|
56
83
|
from redis.asyncio import Redis
|
|
57
84
|
from redis_message_queue.asyncio import RedisMessageQueue
|
|
58
85
|
|
|
@@ -66,7 +93,8 @@ async def main():
|
|
|
66
93
|
)
|
|
67
94
|
await queue.publish({"id": "msg-1", "text": "hello"})
|
|
68
95
|
async with queue.process_message() as message:
|
|
69
|
-
|
|
96
|
+
payload = json.loads(message)
|
|
97
|
+
print(f"got {payload['text']}")
|
|
70
98
|
await client.aclose()
|
|
71
99
|
|
|
72
100
|
asyncio.run(main()) # Expected output: got hello
|
|
@@ -202,7 +230,9 @@ so concurrent publishers cannot race above the configured cap. Overload policies
|
|
|
202
230
|
- `drop_oldest` removes the oldest pending message (`RPOP`) before enqueueing the
|
|
203
231
|
new message. This is silent data loss by design; deduplication markers for
|
|
204
232
|
dropped messages are not removed, so a dropped duplicate may still be
|
|
205
|
-
suppressed until its dedup TTL expires.
|
|
233
|
+
suppressed until its dedup TTL expires. The current event contract emits
|
|
234
|
+
`publish/success` for the new message, but no separate `on_event` signal for
|
|
235
|
+
the dropped message.
|
|
206
236
|
- `block` retries the atomic check until space opens or
|
|
207
237
|
`pending_overload_block_timeout_seconds` elapses (default: 1.0), then raises
|
|
208
238
|
`QueueBackpressureError`.
|
|
@@ -474,6 +504,14 @@ gateway = RedisGateway(
|
|
|
474
504
|
queue = RedisMessageQueue("q", gateway=gateway)
|
|
475
505
|
```
|
|
476
506
|
|
|
507
|
+
When `gateway=` is supplied, queue-level constructor defaults are not copied
|
|
508
|
+
into the gateway. For example, `RedisMessageQueue(..., gateway=gateway)`
|
|
509
|
+
leaves visibility timeout and dead-letter routing disabled unless
|
|
510
|
+
`message_visibility_timeout_seconds` and `max_delivery_count` are configured on
|
|
511
|
+
the gateway itself. Passing the queue-level default values
|
|
512
|
+
`visibility_timeout_seconds=300` or `max_delivery_count=10` with `gateway=`
|
|
513
|
+
does not transfer those settings to the gateway.
|
|
514
|
+
|
|
477
515
|
The retry knobs configure an internal `tenacity` strategy: exponential
|
|
478
516
|
backoff with jitter, retry on transient Redis errors only, capped at
|
|
479
517
|
`retry_budget_seconds`. The budget is monotonic elapsed time from the first attempt (including attempt duration), not inter-attempt delay; it is unaffected by Python-host NTP jumps. A single attempt that takes longer than the budget results in zero retries. Setting `retry_budget_seconds=0` disables retry
|
|
@@ -706,8 +744,8 @@ queue = RedisMessageQueue("jobs", client=client, on_event=on_event)
|
|
|
706
744
|
```
|
|
707
745
|
|
|
708
746
|
Events cover publish, dedup hits, claim/empty polls, reclaim, ack/nack,
|
|
709
|
-
completed/failed cleanup, DLQ moves, heartbeat renewal, stale leases,
|
|
710
|
-
and trim failures, and retry attempts. Callback exceptions are logged and
|
|
747
|
+
completed/failed cleanup, DLQ moves, heartbeat renewal, stale leases, drain,
|
|
748
|
+
cleanup and trim failures, and retry attempts. Callback exceptions are logged and
|
|
711
749
|
reported with `RuntimeWarning`, but never propagate into queue operations.
|
|
712
750
|
`on_event` is telemetry only: use it for metrics, tracing, and logging, not for
|
|
713
751
|
sagas, follow-up writes, billing callbacks, or other correctness-critical
|
|
@@ -800,6 +838,23 @@ Pre-commit and mid-flight exceptions:
|
|
|
800
838
|
despite the exception. Treat them as "operation did not succeed from the
|
|
801
839
|
caller's perspective", not "Redis did not commit".
|
|
802
840
|
|
|
841
|
+
#### Drain events
|
|
842
|
+
|
|
843
|
+
`drain()` and `close()` on the sync queue, and `drain()` and `aclose()` on the
|
|
844
|
+
async queue, emit `drain` events:
|
|
845
|
+
|
|
846
|
+
- `drain/start` when the queue-local drain flag is set.
|
|
847
|
+
- `drain/success` when pending claim IDs were recovered or no gateway drain
|
|
848
|
+
hook is present.
|
|
849
|
+
- `drain/skipped` when the queue was already drained and the cached successful
|
|
850
|
+
result is returned.
|
|
851
|
+
- `drain/failure` when pending claim recovery times out or otherwise leaves
|
|
852
|
+
unresolved claim IDs.
|
|
853
|
+
|
|
854
|
+
Drain events use `timeout_seconds` for the caller-supplied timeout,
|
|
855
|
+
`pending_claim_ids` for the number of unresolved local claim IDs when known,
|
|
856
|
+
and `exception_type` / `error` on failure.
|
|
857
|
+
|
|
803
858
|
#### Intentionally silent paths
|
|
804
859
|
|
|
805
860
|
The following operations have no `on_event` surface by design:
|
|
@@ -814,9 +869,11 @@ The following operations have no `on_event` surface by design:
|
|
|
814
869
|
it back to pending, and returns `false`. Python translates that into
|
|
815
870
|
`claim_empty/skipped`, the same shape as an empty poll. This is intentional
|
|
816
871
|
fail-safe behavior; the message is not lost.
|
|
817
|
-
- **`
|
|
818
|
-
|
|
819
|
-
|
|
872
|
+
- **`drop_oldest` evictions:** when publish backpressure uses
|
|
873
|
+
`pending_overload_policy="drop_oldest"`, the oldest pending message is
|
|
874
|
+
discarded before the new message is enqueued. The successful enqueue emits
|
|
875
|
+
`publish/success`, but there is no separate drop event for the discarded
|
|
876
|
+
message in the current feature set.
|
|
820
877
|
- **Non-claim-loop retry attempts:** tenacity retries in deduplicated publish,
|
|
821
878
|
ack/remove, move-to-completed/failed, and lease renewal collapse into the
|
|
822
879
|
terminal operation's failure event. There is no per-attempt event for those
|
|
@@ -1015,7 +1072,7 @@ v6.0.0 is a non-breaking-defaults release that adds new public APIs. v5 code con
|
|
|
1015
1072
|
|
|
1016
1073
|
- `max_pending_length=N` caps pending-list depth; with `pending_overload_policy="raise"` (default) producers see `QueueBackpressureError` when the cap is hit; `"block"` waits up to `pending_overload_block_timeout_seconds`; `"drop_oldest"` evicts silently, so use it only when data loss is acceptable.
|
|
1017
1074
|
- `queue.drain(timeout=...)` (sync) and `await queue.aclose(timeout=...)` (async) are explicit graceful-shutdown hooks. They refuse new claims and recover pending claim IDs but do not cancel in-flight handlers; join or await your worker separately.
|
|
1018
|
-
- `on_event=callback` receives a `QueueEvent` dataclass for
|
|
1075
|
+
- `on_event=callback` receives a `QueueEvent` dataclass for publish/claim/ack/reclaim/dedup/cleanup/drain lifecycle events. Use it for metrics, tracing, and structured logging. See [`examples/production/observability.py`](examples/production/observability.py) for the adapter pattern.
|
|
1019
1076
|
- See [`examples/production/backpressure.py`](examples/production/backpressure.py) and [`examples/production/graceful_shutdown.py`](examples/production/graceful_shutdown.py) for sync production patterns, with async siblings under [`examples/production/asyncio/`](examples/production/asyncio/).
|
|
1020
1077
|
|
|
1021
1078
|
> When using a pre-fork app server (gunicorn `--preload`, uvicorn workers that import the app at master startup), call `make_queue()` from your worker startup hook - NOT at module import. See [Fork safety](#fork-safety-and-pre-fork-servers) for why.
|
|
@@ -1064,13 +1121,13 @@ Try the [examples](https://github.com/Elijas/redis-message-queue/tree/main/examp
|
|
|
1064
1121
|
|
|
1065
1122
|
```bash
|
|
1066
1123
|
# Two publishers
|
|
1067
|
-
|
|
1068
|
-
|
|
1124
|
+
uv run python -m examples.send_messages
|
|
1125
|
+
uv run python -m examples.send_messages
|
|
1069
1126
|
|
|
1070
1127
|
# Three consumers
|
|
1071
|
-
|
|
1072
|
-
|
|
1073
|
-
|
|
1128
|
+
uv run python -m examples.receive_messages
|
|
1129
|
+
uv run python -m examples.receive_messages
|
|
1130
|
+
uv run python -m examples.receive_messages
|
|
1074
1131
|
```
|
|
1075
1132
|
|
|
1076
1133
|

|
|
@@ -1,32 +1,6 @@
|
|
|
1
|
-
Metadata-Version: 2.4
|
|
2
|
-
Name: redis-message-queue
|
|
3
|
-
Version: 8.2.0
|
|
4
|
-
Summary: Python message queuing with Redis and message deduplication
|
|
5
|
-
License: MIT
|
|
6
|
-
License-File: LICENSE
|
|
7
|
-
Keywords: redis,message-queue,deduplication,task-queue
|
|
8
|
-
Author: Elijas
|
|
9
|
-
Author-email: 4084885+Elijas@users.noreply.github.com
|
|
10
|
-
Requires-Python: >=3.12,<4.0
|
|
11
|
-
Classifier: Development Status :: 5 - Production/Stable
|
|
12
|
-
Classifier: Intended Audience :: Developers
|
|
13
|
-
Classifier: License :: OSI Approved :: MIT License
|
|
14
|
-
Classifier: Programming Language :: Python :: 3
|
|
15
|
-
Classifier: Programming Language :: Python :: 3.12
|
|
16
|
-
Classifier: Programming Language :: Python :: 3.13
|
|
17
|
-
Classifier: Programming Language :: Python :: 3.14
|
|
18
|
-
Classifier: Topic :: Software Development :: Libraries
|
|
19
|
-
Classifier: Topic :: System :: Distributed Computing
|
|
20
|
-
Requires-Dist: redis (>=5.0.0,<8.0.0)
|
|
21
|
-
Requires-Dist: tenacity (>=8.1.0)
|
|
22
|
-
Project-URL: Homepage, https://github.com/Elijas/redis-message-queue
|
|
23
|
-
Project-URL: Issues, https://github.com/Elijas/redis-message-queue/issues
|
|
24
|
-
Project-URL: Repository, https://github.com/Elijas/redis-message-queue
|
|
25
|
-
Description-Content-Type: text/markdown
|
|
26
|
-
|
|
27
1
|
# redis-message-queue
|
|
28
2
|
|
|
29
|
-
[](https://pypi.org/project/redis-message-queue)
|
|
30
4
|
[](https://pypistats.org/packages/redis-message-queue)
|
|
31
5
|
[](LICENSE)
|
|
32
6
|
[](https://github.com/Elijas/redis-message-queue/issues)
|
|
@@ -37,7 +11,7 @@ Description-Content-Type: text/markdown
|
|
|
37
11
|
**Lightweight Python message queuing with Redis and built-in publish-side deduplication.** Deduplicate publishes within a TTL window, with optional crash recovery — across any number of producers and consumers.
|
|
38
12
|
|
|
39
13
|
```bash
|
|
40
|
-
pip install "redis-message-queue>=8.
|
|
14
|
+
pip install "redis-message-queue>=8.2.2,<9.0.0"
|
|
41
15
|
```
|
|
42
16
|
|
|
43
17
|
Requires Redis server >= 6.2.
|
|
@@ -48,6 +22,7 @@ Redis must be running locally first: use `redis-server` or
|
|
|
48
22
|
`docker run -p 6379:6379 redis:7`.
|
|
49
23
|
|
|
50
24
|
```python
|
|
25
|
+
import json
|
|
51
26
|
from redis import Redis
|
|
52
27
|
from redis_message_queue import RedisMessageQueue
|
|
53
28
|
|
|
@@ -61,7 +36,8 @@ queue = RedisMessageQueue(
|
|
|
61
36
|
queue.publish({"id": "msg-1", "text": "hello"})
|
|
62
37
|
with queue.process_message() as message:
|
|
63
38
|
if message is not None:
|
|
64
|
-
|
|
39
|
+
payload = json.loads(message)
|
|
40
|
+
print(f"got {payload['text']}")
|
|
65
41
|
# Expected output: got hello
|
|
66
42
|
```
|
|
67
43
|
|
|
@@ -72,13 +48,15 @@ with queue.process_message() as message:
|
|
|
72
48
|
> synchronous. If your handler is `async def`, returns a coroutine, or returns
|
|
73
49
|
> any other awaitable, use `redis_message_queue.asyncio.RedisMessageQueue`.
|
|
74
50
|
> The sync context manager does not inspect the handler's return value; an
|
|
75
|
-
> unawaited coroutine can be dropped while the message is acked.
|
|
76
|
-
> callback
|
|
51
|
+
> unawaited coroutine can be dropped while the message is acked. For sync
|
|
52
|
+
> callback-style handlers, use `process_message_callback(handler)`: it checks
|
|
53
|
+
> for awaitable returns before acking and raises `TypeError` if one is returned.
|
|
77
54
|
|
|
78
55
|
### Async quickstart
|
|
79
56
|
|
|
80
57
|
```python
|
|
81
58
|
import asyncio
|
|
59
|
+
import json
|
|
82
60
|
from redis.asyncio import Redis
|
|
83
61
|
from redis_message_queue.asyncio import RedisMessageQueue
|
|
84
62
|
|
|
@@ -92,7 +70,8 @@ async def main():
|
|
|
92
70
|
)
|
|
93
71
|
await queue.publish({"id": "msg-1", "text": "hello"})
|
|
94
72
|
async with queue.process_message() as message:
|
|
95
|
-
|
|
73
|
+
payload = json.loads(message)
|
|
74
|
+
print(f"got {payload['text']}")
|
|
96
75
|
await client.aclose()
|
|
97
76
|
|
|
98
77
|
asyncio.run(main()) # Expected output: got hello
|
|
@@ -228,7 +207,9 @@ so concurrent publishers cannot race above the configured cap. Overload policies
|
|
|
228
207
|
- `drop_oldest` removes the oldest pending message (`RPOP`) before enqueueing the
|
|
229
208
|
new message. This is silent data loss by design; deduplication markers for
|
|
230
209
|
dropped messages are not removed, so a dropped duplicate may still be
|
|
231
|
-
suppressed until its dedup TTL expires.
|
|
210
|
+
suppressed until its dedup TTL expires. The current event contract emits
|
|
211
|
+
`publish/success` for the new message, but no separate `on_event` signal for
|
|
212
|
+
the dropped message.
|
|
232
213
|
- `block` retries the atomic check until space opens or
|
|
233
214
|
`pending_overload_block_timeout_seconds` elapses (default: 1.0), then raises
|
|
234
215
|
`QueueBackpressureError`.
|
|
@@ -500,6 +481,14 @@ gateway = RedisGateway(
|
|
|
500
481
|
queue = RedisMessageQueue("q", gateway=gateway)
|
|
501
482
|
```
|
|
502
483
|
|
|
484
|
+
When `gateway=` is supplied, queue-level constructor defaults are not copied
|
|
485
|
+
into the gateway. For example, `RedisMessageQueue(..., gateway=gateway)`
|
|
486
|
+
leaves visibility timeout and dead-letter routing disabled unless
|
|
487
|
+
`message_visibility_timeout_seconds` and `max_delivery_count` are configured on
|
|
488
|
+
the gateway itself. Passing the queue-level default values
|
|
489
|
+
`visibility_timeout_seconds=300` or `max_delivery_count=10` with `gateway=`
|
|
490
|
+
does not transfer those settings to the gateway.
|
|
491
|
+
|
|
503
492
|
The retry knobs configure an internal `tenacity` strategy: exponential
|
|
504
493
|
backoff with jitter, retry on transient Redis errors only, capped at
|
|
505
494
|
`retry_budget_seconds`. The budget is monotonic elapsed time from the first attempt (including attempt duration), not inter-attempt delay; it is unaffected by Python-host NTP jumps. A single attempt that takes longer than the budget results in zero retries. Setting `retry_budget_seconds=0` disables retry
|
|
@@ -732,8 +721,8 @@ queue = RedisMessageQueue("jobs", client=client, on_event=on_event)
|
|
|
732
721
|
```
|
|
733
722
|
|
|
734
723
|
Events cover publish, dedup hits, claim/empty polls, reclaim, ack/nack,
|
|
735
|
-
completed/failed cleanup, DLQ moves, heartbeat renewal, stale leases,
|
|
736
|
-
and trim failures, and retry attempts. Callback exceptions are logged and
|
|
724
|
+
completed/failed cleanup, DLQ moves, heartbeat renewal, stale leases, drain,
|
|
725
|
+
cleanup and trim failures, and retry attempts. Callback exceptions are logged and
|
|
737
726
|
reported with `RuntimeWarning`, but never propagate into queue operations.
|
|
738
727
|
`on_event` is telemetry only: use it for metrics, tracing, and logging, not for
|
|
739
728
|
sagas, follow-up writes, billing callbacks, or other correctness-critical
|
|
@@ -826,6 +815,23 @@ Pre-commit and mid-flight exceptions:
|
|
|
826
815
|
despite the exception. Treat them as "operation did not succeed from the
|
|
827
816
|
caller's perspective", not "Redis did not commit".
|
|
828
817
|
|
|
818
|
+
#### Drain events
|
|
819
|
+
|
|
820
|
+
`drain()` and `close()` on the sync queue, and `drain()` and `aclose()` on the
|
|
821
|
+
async queue, emit `drain` events:
|
|
822
|
+
|
|
823
|
+
- `drain/start` when the queue-local drain flag is set.
|
|
824
|
+
- `drain/success` when pending claim IDs were recovered or no gateway drain
|
|
825
|
+
hook is present.
|
|
826
|
+
- `drain/skipped` when the queue was already drained and the cached successful
|
|
827
|
+
result is returned.
|
|
828
|
+
- `drain/failure` when pending claim recovery times out or otherwise leaves
|
|
829
|
+
unresolved claim IDs.
|
|
830
|
+
|
|
831
|
+
Drain events use `timeout_seconds` for the caller-supplied timeout,
|
|
832
|
+
`pending_claim_ids` for the number of unresolved local claim IDs when known,
|
|
833
|
+
and `exception_type` / `error` on failure.
|
|
834
|
+
|
|
829
835
|
#### Intentionally silent paths
|
|
830
836
|
|
|
831
837
|
The following operations have no `on_event` surface by design:
|
|
@@ -840,9 +846,11 @@ The following operations have no `on_event` surface by design:
|
|
|
840
846
|
it back to pending, and returns `false`. Python translates that into
|
|
841
847
|
`claim_empty/skipped`, the same shape as an empty poll. This is intentional
|
|
842
848
|
fail-safe behavior; the message is not lost.
|
|
843
|
-
- **`
|
|
844
|
-
|
|
845
|
-
|
|
849
|
+
- **`drop_oldest` evictions:** when publish backpressure uses
|
|
850
|
+
`pending_overload_policy="drop_oldest"`, the oldest pending message is
|
|
851
|
+
discarded before the new message is enqueued. The successful enqueue emits
|
|
852
|
+
`publish/success`, but there is no separate drop event for the discarded
|
|
853
|
+
message in the current feature set.
|
|
846
854
|
- **Non-claim-loop retry attempts:** tenacity retries in deduplicated publish,
|
|
847
855
|
ack/remove, move-to-completed/failed, and lease renewal collapse into the
|
|
848
856
|
terminal operation's failure event. There is no per-attempt event for those
|
|
@@ -1041,7 +1049,7 @@ v6.0.0 is a non-breaking-defaults release that adds new public APIs. v5 code con
|
|
|
1041
1049
|
|
|
1042
1050
|
- `max_pending_length=N` caps pending-list depth; with `pending_overload_policy="raise"` (default) producers see `QueueBackpressureError` when the cap is hit; `"block"` waits up to `pending_overload_block_timeout_seconds`; `"drop_oldest"` evicts silently, so use it only when data loss is acceptable.
|
|
1043
1051
|
- `queue.drain(timeout=...)` (sync) and `await queue.aclose(timeout=...)` (async) are explicit graceful-shutdown hooks. They refuse new claims and recover pending claim IDs but do not cancel in-flight handlers; join or await your worker separately.
|
|
1044
|
-
- `on_event=callback` receives a `QueueEvent` dataclass for
|
|
1052
|
+
- `on_event=callback` receives a `QueueEvent` dataclass for publish/claim/ack/reclaim/dedup/cleanup/drain lifecycle events. Use it for metrics, tracing, and structured logging. See [`examples/production/observability.py`](examples/production/observability.py) for the adapter pattern.
|
|
1045
1053
|
- See [`examples/production/backpressure.py`](examples/production/backpressure.py) and [`examples/production/graceful_shutdown.py`](examples/production/graceful_shutdown.py) for sync production patterns, with async siblings under [`examples/production/asyncio/`](examples/production/asyncio/).
|
|
1046
1054
|
|
|
1047
1055
|
> When using a pre-fork app server (gunicorn `--preload`, uvicorn workers that import the app at master startup), call `make_queue()` from your worker startup hook - NOT at module import. See [Fork safety](#fork-safety-and-pre-fork-servers) for why.
|
|
@@ -1090,14 +1098,13 @@ Try the [examples](https://github.com/Elijas/redis-message-queue/tree/main/examp
|
|
|
1090
1098
|
|
|
1091
1099
|
```bash
|
|
1092
1100
|
# Two publishers
|
|
1093
|
-
|
|
1094
|
-
|
|
1101
|
+
uv run python -m examples.send_messages
|
|
1102
|
+
uv run python -m examples.send_messages
|
|
1095
1103
|
|
|
1096
1104
|
# Three consumers
|
|
1097
|
-
|
|
1098
|
-
|
|
1099
|
-
|
|
1105
|
+
uv run python -m examples.receive_messages
|
|
1106
|
+
uv run python -m examples.receive_messages
|
|
1107
|
+
uv run python -m examples.receive_messages
|
|
1100
1108
|
```
|
|
1101
1109
|
|
|
1102
1110
|

|
|
1103
|
-
|
|
@@ -1,11 +1,11 @@
|
|
|
1
|
-
[
|
|
1
|
+
[project]
|
|
2
2
|
name = "redis-message-queue"
|
|
3
|
-
version = "8.2.
|
|
3
|
+
version = "8.2.2"
|
|
4
4
|
description = "Python message queuing with Redis and message deduplication"
|
|
5
|
-
authors = ["Elijas
|
|
5
|
+
authors = [{ name = "Elijas", email = "4084885+Elijas@users.noreply.github.com" }]
|
|
6
6
|
readme = "README.md"
|
|
7
|
-
license = "MIT"
|
|
8
|
-
|
|
7
|
+
license = { text = "MIT" }
|
|
8
|
+
license-files = ["LICENSE"]
|
|
9
9
|
keywords = ["redis", "message-queue", "deduplication", "task-queue"]
|
|
10
10
|
classifiers = [
|
|
11
11
|
"Development Status :: 5 - Production/Stable",
|
|
@@ -16,26 +16,65 @@ classifiers = [
|
|
|
16
16
|
"Topic :: Software Development :: Libraries",
|
|
17
17
|
"Topic :: System :: Distributed Computing",
|
|
18
18
|
]
|
|
19
|
+
requires-python = ">=3.12,<4.0"
|
|
20
|
+
dependencies = [
|
|
21
|
+
"redis>=5.0.1,<8.0.0",
|
|
22
|
+
"tenacity>=8.1.0",
|
|
23
|
+
]
|
|
19
24
|
|
|
20
|
-
[
|
|
25
|
+
[project.urls]
|
|
21
26
|
Homepage = "https://github.com/Elijas/redis-message-queue"
|
|
22
27
|
Repository = "https://github.com/Elijas/redis-message-queue"
|
|
23
28
|
Issues = "https://github.com/Elijas/redis-message-queue/issues"
|
|
24
29
|
|
|
25
|
-
[
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
30
|
+
[dependency-groups]
|
|
31
|
+
dev = [
|
|
32
|
+
"bump-my-version>=1.1.2",
|
|
33
|
+
"mypy",
|
|
34
|
+
"ruff",
|
|
35
|
+
]
|
|
36
|
+
test = [
|
|
37
|
+
"fakeredis[lua]",
|
|
38
|
+
"pytest",
|
|
39
|
+
"pytest-asyncio",
|
|
40
|
+
"pytest-cov",
|
|
41
|
+
]
|
|
42
|
+
|
|
43
|
+
[tool.uv]
|
|
44
|
+
default-groups = ["dev", "test"]
|
|
29
45
|
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
46
|
+
##############################
|
|
47
|
+
### BUMP VERSION
|
|
48
|
+
##############################
|
|
49
|
+
|
|
50
|
+
[tool.bumpversion]
|
|
51
|
+
current_version = "8.2.2"
|
|
52
|
+
parse = "(?P<major>\\d+)\\.(?P<minor>\\d+)\\.(?P<patch>\\d+)"
|
|
53
|
+
serialize = ["{major}.{minor}.{patch}"]
|
|
54
|
+
search = "{current_version}"
|
|
55
|
+
replace = "{new_version}"
|
|
56
|
+
regex = false
|
|
57
|
+
ignore_missing_version = false
|
|
58
|
+
ignore_missing_files = false
|
|
59
|
+
tag = false
|
|
60
|
+
sign_tags = false
|
|
61
|
+
tag_name = "v{new_version}"
|
|
62
|
+
tag_message = "Bump version: {current_version} -> {new_version}"
|
|
63
|
+
allow_dirty = true
|
|
64
|
+
commit = true
|
|
65
|
+
message = "chore(release): bump version from {current_version} to {new_version}"
|
|
66
|
+
moveable_tags = []
|
|
67
|
+
commit_args = ""
|
|
68
|
+
setup_hooks = []
|
|
69
|
+
pre_commit_hooks = []
|
|
70
|
+
post_commit_hooks = []
|
|
35
71
|
|
|
36
72
|
[build-system]
|
|
37
|
-
requires = ["
|
|
38
|
-
build-backend = "
|
|
73
|
+
requires = ["hatchling"]
|
|
74
|
+
build-backend = "hatchling.build"
|
|
75
|
+
|
|
76
|
+
[tool.hatch.build]
|
|
77
|
+
packages = ["redis_message_queue"]
|
|
39
78
|
|
|
40
79
|
[tool.ruff]
|
|
41
80
|
target-version = "py312"
|
|
@@ -45,6 +84,12 @@ line-length = 120
|
|
|
45
84
|
select = ["E", "F", "I", "W"]
|
|
46
85
|
ignore = ["E731"]
|
|
47
86
|
|
|
87
|
+
[tool.mypy]
|
|
88
|
+
python_version = "3.12"
|
|
89
|
+
files = ["redis_message_queue/"]
|
|
90
|
+
explicit_package_bases = true
|
|
91
|
+
show_error_codes = true
|
|
92
|
+
|
|
48
93
|
[tool.pytest.ini_options]
|
|
49
94
|
asyncio_default_fixture_loop_scope = "function"
|
|
50
95
|
filterwarnings = [
|
{redis_message_queue-8.2.0 → redis_message_queue-8.2.2}/redis_message_queue/_redis_gateway.py
RENAMED
|
@@ -301,8 +301,8 @@ class RedisGateway(AbstractRedisGateway):
|
|
|
301
301
|
|
|
302
302
|
def _emit_event(
|
|
303
303
|
self,
|
|
304
|
-
operation: EventOperation,
|
|
305
|
-
outcome: EventOutcome,
|
|
304
|
+
operation: EventOperation | str,
|
|
305
|
+
outcome: EventOutcome | str,
|
|
306
306
|
*,
|
|
307
307
|
message_id: str | None = None,
|
|
308
308
|
claim_id: str | None = None,
|
|
@@ -328,7 +328,7 @@ class RedisGateway(AbstractRedisGateway):
|
|
|
328
328
|
|
|
329
329
|
def _emit_repeated_event(
|
|
330
330
|
self,
|
|
331
|
-
operation: EventOperation,
|
|
331
|
+
operation: EventOperation | str,
|
|
332
332
|
attempts: list[_MessageAttemptEvent],
|
|
333
333
|
*,
|
|
334
334
|
destination_queue: str | None = None,
|
|
@@ -275,8 +275,8 @@ class RedisGateway(AbstractRedisGateway):
|
|
|
275
275
|
|
|
276
276
|
async def _emit_event(
|
|
277
277
|
self,
|
|
278
|
-
operation: EventOperation,
|
|
279
|
-
outcome: EventOutcome,
|
|
278
|
+
operation: EventOperation | str,
|
|
279
|
+
outcome: EventOutcome | str,
|
|
280
280
|
*,
|
|
281
281
|
message_id: str | None = None,
|
|
282
282
|
claim_id: str | None = None,
|
|
@@ -302,7 +302,7 @@ class RedisGateway(AbstractRedisGateway):
|
|
|
302
302
|
|
|
303
303
|
async def _emit_repeated_event(
|
|
304
304
|
self,
|
|
305
|
-
operation: EventOperation,
|
|
305
|
+
operation: EventOperation | str,
|
|
306
306
|
attempts: list[_MessageAttemptEvent],
|
|
307
307
|
*,
|
|
308
308
|
destination_queue: str | None = None,
|
|
@@ -11,7 +11,6 @@ from typing import AsyncIterator, Awaitable, Callable, Literal, Optional, TypeVa
|
|
|
11
11
|
import redis.asyncio
|
|
12
12
|
import redis.exceptions
|
|
13
13
|
|
|
14
|
-
from redis_message_queue._callable_utils import is_async_callable
|
|
15
14
|
from redis_message_queue._config import (
|
|
16
15
|
DEFAULT_PENDING_OVERLOAD_BLOCK_TIMEOUT_SECONDS,
|
|
17
16
|
validate_dedup_configuration,
|
|
@@ -332,8 +331,12 @@ def _derive_dead_letter_queue(name: str, key_separator: str) -> str:
|
|
|
332
331
|
return f"{name}{key_separator}{_AUTO_DEAD_LETTER_QUEUE_SUFFIX}"
|
|
333
332
|
|
|
334
333
|
|
|
335
|
-
def _bind_dead_letter_gateway_to_queue(
|
|
336
|
-
|
|
334
|
+
def _bind_dead_letter_gateway_to_queue(
|
|
335
|
+
gateway: AbstractRedisGateway,
|
|
336
|
+
queue_pending_key: str,
|
|
337
|
+
queue_processing_key: str,
|
|
338
|
+
) -> None:
|
|
339
|
+
"""Validate and bind a dead-letter-enabled gateway to this queue.
|
|
337
340
|
|
|
338
341
|
The check is not thread-safe: constructing ``RedisMessageQueue`` instances
|
|
339
342
|
concurrently on multiple threads with the same DLQ-enabled gateway can
|
|
@@ -346,6 +349,12 @@ def _bind_dead_letter_gateway_to_queue(gateway: AbstractRedisGateway, queue_pend
|
|
|
346
349
|
if max_delivery_count is None:
|
|
347
350
|
return
|
|
348
351
|
|
|
352
|
+
if gateway.dead_letter_queue in (queue_pending_key, queue_processing_key):
|
|
353
|
+
raise ConfigurationError(
|
|
354
|
+
"'dead_letter_queue' must be distinct from the queue's pending and processing Redis keys. "
|
|
355
|
+
"Use a separate Redis list key for poison messages."
|
|
356
|
+
)
|
|
357
|
+
|
|
349
358
|
bound_pending_key = getattr(gateway, _GATEWAY_BOUND_PENDING_QUEUE_ATTR, None)
|
|
350
359
|
if bound_pending_key is None:
|
|
351
360
|
setattr(gateway, _GATEWAY_BOUND_PENDING_QUEUE_ATTR, queue_pending_key)
|
|
@@ -451,7 +460,7 @@ class _LeaseHeartbeat:
|
|
|
451
460
|
stacklevel=2,
|
|
452
461
|
)
|
|
453
462
|
|
|
454
|
-
async def _emit(self, operation: EventOperation, outcome: EventOutcome, **kwargs: object) -> None:
|
|
463
|
+
async def _emit(self, operation: EventOperation | str, outcome: EventOutcome | str, **kwargs: object) -> None:
|
|
455
464
|
if self._emit_event is not None:
|
|
456
465
|
await self._emit_event(operation, outcome, **kwargs)
|
|
457
466
|
|
|
@@ -597,6 +606,11 @@ class RedisMessageQueue:
|
|
|
597
606
|
auto-derived dead-letter queue. Set it to ``None`` for unlimited
|
|
598
607
|
redelivery.
|
|
599
608
|
|
|
609
|
+
When ``gateway=`` is supplied, queue-level defaults are not transferred
|
|
610
|
+
to the gateway. Configure lease, dead-letter, and backpressure settings
|
|
611
|
+
such as ``message_visibility_timeout_seconds``, ``max_delivery_count``,
|
|
612
|
+
and ``max_pending_length`` on the gateway itself.
|
|
613
|
+
|
|
600
614
|
``deduplication=True`` requires ``get_deduplication_key`` to be a
|
|
601
615
|
callable that returns a non-empty string. Use a stable logical ID for
|
|
602
616
|
the deduplication keyspace.
|
|
@@ -636,11 +650,11 @@ class RedisMessageQueue:
|
|
|
636
650
|
``GracefulInterruptHandler()`` for prompt Ctrl-C / termination handling
|
|
637
651
|
in polling waits. ``on_heartbeat_failure`` is a zero-argument callable
|
|
638
652
|
or coroutine callable invoked when lease renewal fails. ``on_event`` is
|
|
639
|
-
telemetry only: an
|
|
640
|
-
lifecycle notifications. Callback failures are
|
|
641
|
-
RuntimeWarning without influencing ack/nack or
|
|
642
|
-
outcome. Do not use it for correctness-critical
|
|
643
|
-
writes.
|
|
653
|
+
telemetry only: a callable returning an awaitable and receiving
|
|
654
|
+
best-effort QueueEvent lifecycle notifications. Callback failures are
|
|
655
|
+
logged and converted to RuntimeWarning without influencing ack/nack or
|
|
656
|
+
any other message outcome. Do not use it for correctness-critical
|
|
657
|
+
callbacks or follow-up writes.
|
|
644
658
|
"""
|
|
645
659
|
self.key = QueueKeyManager(name, key_separator=key_separator)
|
|
646
660
|
if not isinstance(deduplication, bool):
|
|
@@ -742,8 +756,6 @@ class RedisMessageQueue:
|
|
|
742
756
|
)
|
|
743
757
|
if on_event is not None and not callable(on_event):
|
|
744
758
|
raise TypeError(f"'on_event' must be callable, got {type(on_event).__name__}.")
|
|
745
|
-
if on_event is not None and not is_async_callable(on_event):
|
|
746
|
-
raise TypeError("'on_event' must be an async callable.")
|
|
747
759
|
self._queue_name = name
|
|
748
760
|
self._on_event = on_event
|
|
749
761
|
# Queue-local soft-drain flag. See sync queue ``_draining`` docstring
|
|
@@ -825,7 +837,7 @@ class RedisMessageQueue:
|
|
|
825
837
|
"'max_pending_length' cannot be provided alongside 'gateway'."
|
|
826
838
|
" Configure publish backpressure on the gateway directly instead."
|
|
827
839
|
)
|
|
828
|
-
_bind_dead_letter_gateway_to_queue(gateway, self.key.pending)
|
|
840
|
+
_bind_dead_letter_gateway_to_queue(gateway, self.key.pending, self.key.processing)
|
|
829
841
|
self._max_delivery_count = None
|
|
830
842
|
self._redis = gateway
|
|
831
843
|
elif client is None:
|
|
@@ -867,8 +879,8 @@ class RedisMessageQueue:
|
|
|
867
879
|
|
|
868
880
|
async def _emit_event(
|
|
869
881
|
self,
|
|
870
|
-
operation: EventOperation,
|
|
871
|
-
outcome: EventOutcome,
|
|
882
|
+
operation: EventOperation | str,
|
|
883
|
+
outcome: EventOutcome | str,
|
|
872
884
|
*,
|
|
873
885
|
message_id: str | None = None,
|
|
874
886
|
claim_id: str | None = None,
|
{redis_message_queue-8.2.0 → redis_message_queue-8.2.2}/redis_message_queue/redis_message_queue.py
RENAMED
|
@@ -273,8 +273,12 @@ def _derive_dead_letter_queue(name: str, key_separator: str) -> str:
|
|
|
273
273
|
return f"{name}{key_separator}{_AUTO_DEAD_LETTER_QUEUE_SUFFIX}"
|
|
274
274
|
|
|
275
275
|
|
|
276
|
-
def _bind_dead_letter_gateway_to_queue(
|
|
277
|
-
|
|
276
|
+
def _bind_dead_letter_gateway_to_queue(
|
|
277
|
+
gateway: AbstractRedisGateway,
|
|
278
|
+
queue_pending_key: str,
|
|
279
|
+
queue_processing_key: str,
|
|
280
|
+
) -> None:
|
|
281
|
+
"""Validate and bind a dead-letter-enabled gateway to this queue.
|
|
278
282
|
|
|
279
283
|
The check is not thread-safe: constructing ``RedisMessageQueue`` instances
|
|
280
284
|
concurrently on multiple threads with the same DLQ-enabled gateway can
|
|
@@ -287,6 +291,12 @@ def _bind_dead_letter_gateway_to_queue(gateway: AbstractRedisGateway, queue_pend
|
|
|
287
291
|
if max_delivery_count is None:
|
|
288
292
|
return
|
|
289
293
|
|
|
294
|
+
if gateway.dead_letter_queue in (queue_pending_key, queue_processing_key):
|
|
295
|
+
raise ConfigurationError(
|
|
296
|
+
"'dead_letter_queue' must be distinct from the queue's pending and processing Redis keys. "
|
|
297
|
+
"Use a separate Redis list key for poison messages."
|
|
298
|
+
)
|
|
299
|
+
|
|
290
300
|
bound_pending_key = getattr(gateway, _GATEWAY_BOUND_PENDING_QUEUE_ATTR, None)
|
|
291
301
|
if bound_pending_key is None:
|
|
292
302
|
setattr(gateway, _GATEWAY_BOUND_PENDING_QUEUE_ATTR, queue_pending_key)
|
|
@@ -407,7 +417,7 @@ class _LeaseHeartbeat:
|
|
|
407
417
|
stacklevel=2,
|
|
408
418
|
)
|
|
409
419
|
|
|
410
|
-
def _emit(self, operation: EventOperation, outcome: EventOutcome, **kwargs: object) -> None:
|
|
420
|
+
def _emit(self, operation: EventOperation | str, outcome: EventOutcome | str, **kwargs: object) -> None:
|
|
411
421
|
if self._emit_event is not None:
|
|
412
422
|
self._emit_event(operation, outcome, **kwargs)
|
|
413
423
|
|
|
@@ -549,6 +559,11 @@ class RedisMessageQueue:
|
|
|
549
559
|
auto-derived dead-letter queue. Set it to ``None`` for unlimited
|
|
550
560
|
redelivery.
|
|
551
561
|
|
|
562
|
+
When ``gateway=`` is supplied, queue-level defaults are not transferred
|
|
563
|
+
to the gateway. Configure lease, dead-letter, and backpressure settings
|
|
564
|
+
such as ``message_visibility_timeout_seconds``, ``max_delivery_count``,
|
|
565
|
+
and ``max_pending_length`` on the gateway itself.
|
|
566
|
+
|
|
552
567
|
``deduplication=True`` requires ``get_deduplication_key`` to be a
|
|
553
568
|
callable that returns a non-empty string. Use a stable logical ID for
|
|
554
569
|
the deduplication keyspace.
|
|
@@ -786,7 +801,7 @@ class RedisMessageQueue:
|
|
|
786
801
|
"'max_pending_length' cannot be provided alongside 'gateway'."
|
|
787
802
|
" Configure publish backpressure on the gateway directly instead."
|
|
788
803
|
)
|
|
789
|
-
_bind_dead_letter_gateway_to_queue(gateway, self.key.pending)
|
|
804
|
+
_bind_dead_letter_gateway_to_queue(gateway, self.key.pending, self.key.processing)
|
|
790
805
|
self._max_delivery_count = None
|
|
791
806
|
self._redis = gateway
|
|
792
807
|
elif client is None:
|
|
@@ -827,8 +842,8 @@ class RedisMessageQueue:
|
|
|
827
842
|
|
|
828
843
|
def _emit_event(
|
|
829
844
|
self,
|
|
830
|
-
operation: EventOperation,
|
|
831
|
-
outcome: EventOutcome,
|
|
845
|
+
operation: EventOperation | str,
|
|
846
|
+
outcome: EventOutcome | str,
|
|
832
847
|
*,
|
|
833
848
|
message_id: str | None = None,
|
|
834
849
|
claim_id: str | None = None,
|
|
@@ -1031,8 +1046,9 @@ class RedisMessageQueue:
|
|
|
1031
1046
|
does not inspect handler return values; if your handler returns a
|
|
1032
1047
|
coroutine or other awaitable, the awaitable can be dropped while the
|
|
1033
1048
|
message is acked. Use ``redis_message_queue.asyncio.RedisMessageQueue``
|
|
1034
|
-
for async handlers.
|
|
1035
|
-
|
|
1049
|
+
for async handlers. For sync callback-style handlers, use
|
|
1050
|
+
``process_message_callback(handler)`` so awaitable returns are detected
|
|
1051
|
+
before acking.
|
|
1036
1052
|
|
|
1037
1053
|
If the process is killed mid-handler, the claimed message and lease
|
|
1038
1054
|
metadata remain in Redis until a later consumer claim triggers
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{redis_message_queue-8.2.0 → redis_message_queue-8.2.2}/redis_message_queue/_callable_utils.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{redis_message_queue-8.2.0 → redis_message_queue-8.2.2}/redis_message_queue/_payload_limits.py
RENAMED
|
File without changes
|
{redis_message_queue-8.2.0 → redis_message_queue-8.2.2}/redis_message_queue/_queue_key_manager.py
RENAMED
|
File without changes
|
{redis_message_queue-8.2.0 → redis_message_queue-8.2.2}/redis_message_queue/_redis_cluster.py
RENAMED
|
File without changes
|
{redis_message_queue-8.2.0 → redis_message_queue-8.2.2}/redis_message_queue/_stored_message.py
RENAMED
|
File without changes
|
{redis_message_queue-8.2.0 → redis_message_queue-8.2.2}/redis_message_queue/asyncio/__init__.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|