redis-message-queue 8.2.0__tar.gz → 8.2.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (26) hide show
  1. redis_message_queue-8.2.2/.gitignore +167 -0
  2. redis_message_queue-8.2.0/README.md → redis_message_queue-8.2.2/PKG-INFO +75 -18
  3. redis_message_queue-8.2.0/PKG-INFO → redis_message_queue-8.2.2/README.md +52 -45
  4. {redis_message_queue-8.2.0 → redis_message_queue-8.2.2}/pyproject.toml +62 -17
  5. {redis_message_queue-8.2.0 → redis_message_queue-8.2.2}/redis_message_queue/_redis_gateway.py +3 -3
  6. {redis_message_queue-8.2.0 → redis_message_queue-8.2.2}/redis_message_queue/asyncio/_redis_gateway.py +3 -3
  7. {redis_message_queue-8.2.0 → redis_message_queue-8.2.2}/redis_message_queue/asyncio/redis_message_queue.py +26 -14
  8. {redis_message_queue-8.2.0 → redis_message_queue-8.2.2}/redis_message_queue/redis_message_queue.py +24 -8
  9. {redis_message_queue-8.2.0 → redis_message_queue-8.2.2}/LICENSE +0 -0
  10. {redis_message_queue-8.2.0 → redis_message_queue-8.2.2}/redis_message_queue/__init__.py +0 -0
  11. {redis_message_queue-8.2.0 → redis_message_queue-8.2.2}/redis_message_queue/_abstract_redis_gateway.py +0 -0
  12. {redis_message_queue-8.2.0 → redis_message_queue-8.2.2}/redis_message_queue/_callable_utils.py +0 -0
  13. {redis_message_queue-8.2.0 → redis_message_queue-8.2.2}/redis_message_queue/_config.py +0 -0
  14. {redis_message_queue-8.2.0 → redis_message_queue-8.2.2}/redis_message_queue/_event.py +0 -0
  15. {redis_message_queue-8.2.0 → redis_message_queue-8.2.2}/redis_message_queue/_exceptions.py +0 -0
  16. {redis_message_queue-8.2.0 → redis_message_queue-8.2.2}/redis_message_queue/_payload_limits.py +0 -0
  17. {redis_message_queue-8.2.0 → redis_message_queue-8.2.2}/redis_message_queue/_queue_key_manager.py +0 -0
  18. {redis_message_queue-8.2.0 → redis_message_queue-8.2.2}/redis_message_queue/_redis_cluster.py +0 -0
  19. {redis_message_queue-8.2.0 → redis_message_queue-8.2.2}/redis_message_queue/_stored_message.py +0 -0
  20. {redis_message_queue-8.2.0 → redis_message_queue-8.2.2}/redis_message_queue/asyncio/__init__.py +0 -0
  21. {redis_message_queue-8.2.0 → redis_message_queue-8.2.2}/redis_message_queue/asyncio/_abstract_redis_gateway.py +0 -0
  22. {redis_message_queue-8.2.0 → redis_message_queue-8.2.2}/redis_message_queue/interrupt_handler/__init__.py +0 -0
  23. {redis_message_queue-8.2.0 → redis_message_queue-8.2.2}/redis_message_queue/interrupt_handler/_event_driven.py +0 -0
  24. {redis_message_queue-8.2.0 → redis_message_queue-8.2.2}/redis_message_queue/interrupt_handler/_implementation.py +0 -0
  25. {redis_message_queue-8.2.0 → redis_message_queue-8.2.2}/redis_message_queue/interrupt_handler/_interface.py +0 -0
  26. {redis_message_queue-8.2.0 → redis_message_queue-8.2.2}/redis_message_queue/py.typed +0 -0
@@ -0,0 +1,167 @@
1
+ .DS_Store
2
+
3
+ # Claude Code
4
+ .claude/
5
+
6
+ # Byte-compiled / optimized / DLL files
7
+ __pycache__/
8
+ *.py[cod]
9
+ *$py.class
10
+
11
+ # C extensions
12
+ *.so
13
+
14
+ # Distribution / packaging
15
+ .Python
16
+ build/
17
+ develop-eggs/
18
+ dist/
19
+ downloads/
20
+ eggs/
21
+ .eggs/
22
+ lib/
23
+ lib64/
24
+ parts/
25
+ sdist/
26
+ var/
27
+ wheels/
28
+ share/python-wheels/
29
+ *.egg-info/
30
+ .installed.cfg
31
+ *.egg
32
+ MANIFEST
33
+
34
+ # PyInstaller
35
+ # Usually these files are written by a python script from a template
36
+ # before PyInstaller builds the exe, so as to inject date/other infos into it.
37
+ *.manifest
38
+ *.spec
39
+
40
+ # Installer logs
41
+ pip-log.txt
42
+ pip-delete-this-directory.txt
43
+
44
+ # Unit test / coverage reports
45
+ htmlcov/
46
+ .tox/
47
+ .nox/
48
+ .coverage
49
+ .coverage.*
50
+ .cache
51
+ nosetests.xml
52
+ coverage.xml
53
+ *.cover
54
+ *.py,cover
55
+ .hypothesis/
56
+ .pytest_cache/
57
+ cover/
58
+
59
+ # Translations
60
+ *.mo
61
+ *.pot
62
+
63
+ # Django stuff:
64
+ *.log
65
+ local_settings.py
66
+ db.sqlite3
67
+ db.sqlite3-journal
68
+
69
+ # Flask stuff:
70
+ instance/
71
+ .webassets-cache
72
+
73
+ # Scrapy stuff:
74
+ .scrapy
75
+
76
+ # Sphinx documentation
77
+ docs/_build/
78
+
79
+ # PyBuilder
80
+ .pybuilder/
81
+ target/
82
+
83
+ # Jupyter Notebook
84
+ .ipynb_checkpoints
85
+
86
+ # IPython
87
+ profile_default/
88
+ ipython_config.py
89
+
90
+ # pyenv
91
+ # For a library or package, you might want to ignore these files since the code is
92
+ # intended to run in multiple environments; otherwise, check them in:
93
+ .python-version
94
+
95
+ # pipenv
96
+ # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
97
+ # However, in case of collaboration, if having platform-specific dependencies or dependencies
98
+ # having no cross-platform support, pipenv may install dependencies that don't work, or not
99
+ # install all needed dependencies.
100
+ #Pipfile.lock
101
+
102
+ # pdm
103
+ # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
104
+ #pdm.lock
105
+ # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
106
+ # in version control.
107
+ # https://pdm.fming.dev/#use-with-ide
108
+ .pdm.toml
109
+
110
+ # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
111
+ __pypackages__/
112
+
113
+ # Celery stuff
114
+ celerybeat-schedule
115
+ celerybeat.pid
116
+
117
+ # SageMath parsed files
118
+ *.sage.py
119
+
120
+ # Environments
121
+ .env
122
+ .venv
123
+ env/
124
+ venv/
125
+ ENV/
126
+ env.bak/
127
+ venv.bak/
128
+
129
+ # Spyder project settings
130
+ .spyderproject
131
+ .spyproject
132
+
133
+ # Rope project settings
134
+ .ropeproject
135
+
136
+ # mkdocs documentation
137
+ /site
138
+
139
+ # mypy
140
+ .mypy_cache/
141
+ .dmypy.json
142
+ dmypy.json
143
+
144
+ # Ruff
145
+ .ruff_cache/
146
+
147
+ # Coverage tooling output
148
+ lcov.info
149
+
150
+ # PyPI upload config — credentials, never commit
151
+ .pypirc
152
+
153
+ # Pyre type checker
154
+ .pyre/
155
+
156
+ # pytype static type analyzer
157
+ .pytype/
158
+
159
+ # Cython debug symbols
160
+ cython_debug/
161
+
162
+ # PyCharm
163
+ # JetBrains specific template is maintained in a separate JetBrains.gitignore that can
164
+ # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
165
+ # and can be added to the global gitignore or merged into this file. For a more nuclear
166
+ # option (not recommended) you can uncomment the following to ignore the entire idea folder.
167
+ #.idea/
@@ -1,6 +1,29 @@
1
+ Metadata-Version: 2.4
2
+ Name: redis-message-queue
3
+ Version: 8.2.2
4
+ Summary: Python message queuing with Redis and message deduplication
5
+ Project-URL: Homepage, https://github.com/Elijas/redis-message-queue
6
+ Project-URL: Repository, https://github.com/Elijas/redis-message-queue
7
+ Project-URL: Issues, https://github.com/Elijas/redis-message-queue/issues
8
+ Author-email: Elijas <4084885+Elijas@users.noreply.github.com>
9
+ License: MIT
10
+ License-File: LICENSE
11
+ Keywords: deduplication,message-queue,redis,task-queue
12
+ Classifier: Development Status :: 5 - Production/Stable
13
+ Classifier: Intended Audience :: Developers
14
+ Classifier: License :: OSI Approved :: MIT License
15
+ Classifier: Programming Language :: Python :: 3.12
16
+ Classifier: Programming Language :: Python :: 3.13
17
+ Classifier: Topic :: Software Development :: Libraries
18
+ Classifier: Topic :: System :: Distributed Computing
19
+ Requires-Python: <4.0,>=3.12
20
+ Requires-Dist: redis<8.0.0,>=5.0.1
21
+ Requires-Dist: tenacity>=8.1.0
22
+ Description-Content-Type: text/markdown
23
+
1
24
  # redis-message-queue
2
25
 
3
- [![PyPI Version](https://img.shields.io/badge/v8.0.1-version?color=43cd0f&style=flat&label=pypi)](https://pypi.org/project/redis-message-queue)
26
+ [![PyPI Version](https://img.shields.io/pypi/v/redis-message-queue?color=43cd0f&style=flat&label=pypi)](https://pypi.org/project/redis-message-queue)
4
27
  [![PyPI Downloads](https://img.shields.io/pypi/dm/redis-message-queue?color=43cd0f&style=flat&label=downloads)](https://pypistats.org/packages/redis-message-queue)
5
28
  [![License: MIT](https://img.shields.io/badge/License-MIT-43cd0f.svg?style=flat&label=license)](LICENSE)
6
29
  [![Maintained: yes](https://img.shields.io/badge/yes-43cd0f.svg?style=flat&label=maintained)](https://github.com/Elijas/redis-message-queue/issues)
@@ -11,7 +34,7 @@
11
34
  **Lightweight Python message queuing with Redis and built-in publish-side deduplication.** Deduplicate publishes within a TTL window, with optional crash recovery — across any number of producers and consumers.
12
35
 
13
36
  ```bash
14
- pip install "redis-message-queue>=8.0.0,<9.0.0"
37
+ pip install "redis-message-queue>=8.2.2,<9.0.0"
15
38
  ```
16
39
 
17
40
  Requires Redis server >= 6.2.
@@ -22,6 +45,7 @@ Redis must be running locally first: use `redis-server` or
22
45
  `docker run -p 6379:6379 redis:7`.
23
46
 
24
47
  ```python
48
+ import json
25
49
  from redis import Redis
26
50
  from redis_message_queue import RedisMessageQueue
27
51
 
@@ -35,7 +59,8 @@ queue = RedisMessageQueue(
35
59
  queue.publish({"id": "msg-1", "text": "hello"})
36
60
  with queue.process_message() as message:
37
61
  if message is not None:
38
- print(f"got {message['text']}")
62
+ payload = json.loads(message)
63
+ print(f"got {payload['text']}")
39
64
  # Expected output: got hello
40
65
  ```
41
66
 
@@ -46,13 +71,15 @@ with queue.process_message() as message:
46
71
  > synchronous. If your handler is `async def`, returns a coroutine, or returns
47
72
  > any other awaitable, use `redis_message_queue.asyncio.RedisMessageQueue`.
48
73
  > The sync context manager does not inspect the handler's return value; an
49
- > unawaited coroutine can be dropped while the message is acked. An ergonomic
50
- > callback API that detects this is planned for v8.1.
74
+ > unawaited coroutine can be dropped while the message is acked. For sync
75
+ > callback-style handlers, use `process_message_callback(handler)`: it checks
76
+ > for awaitable returns before acking and raises `TypeError` if one is returned.
51
77
 
52
78
  ### Async quickstart
53
79
 
54
80
  ```python
55
81
  import asyncio
82
+ import json
56
83
  from redis.asyncio import Redis
57
84
  from redis_message_queue.asyncio import RedisMessageQueue
58
85
 
@@ -66,7 +93,8 @@ async def main():
66
93
  )
67
94
  await queue.publish({"id": "msg-1", "text": "hello"})
68
95
  async with queue.process_message() as message:
69
- print(f"got {message['text']}")
96
+ payload = json.loads(message)
97
+ print(f"got {payload['text']}")
70
98
  await client.aclose()
71
99
 
72
100
  asyncio.run(main()) # Expected output: got hello
@@ -202,7 +230,9 @@ so concurrent publishers cannot race above the configured cap. Overload policies
202
230
  - `drop_oldest` removes the oldest pending message (`RPOP`) before enqueueing the
203
231
  new message. This is silent data loss by design; deduplication markers for
204
232
  dropped messages are not removed, so a dropped duplicate may still be
205
- suppressed until its dedup TTL expires.
233
+ suppressed until its dedup TTL expires. The current event contract emits
234
+ `publish/success` for the new message, but no separate `on_event` signal for
235
+ the dropped message.
206
236
  - `block` retries the atomic check until space opens or
207
237
  `pending_overload_block_timeout_seconds` elapses (default: 1.0), then raises
208
238
  `QueueBackpressureError`.
@@ -474,6 +504,14 @@ gateway = RedisGateway(
474
504
  queue = RedisMessageQueue("q", gateway=gateway)
475
505
  ```
476
506
 
507
+ When `gateway=` is supplied, queue-level constructor defaults are not copied
508
+ into the gateway. For example, `RedisMessageQueue(..., gateway=gateway)`
509
+ leaves visibility timeout and dead-letter routing disabled unless
510
+ `message_visibility_timeout_seconds` and `max_delivery_count` are configured on
511
+ the gateway itself. Passing the queue-level default values
512
+ `visibility_timeout_seconds=300` or `max_delivery_count=10` with `gateway=`
513
+ does not transfer those settings to the gateway.
514
+
477
515
  The retry knobs configure an internal `tenacity` strategy: exponential
478
516
  backoff with jitter, retry on transient Redis errors only, capped at
479
517
  `retry_budget_seconds`. The budget is monotonic elapsed time from the first attempt (including attempt duration), not inter-attempt delay; it is unaffected by Python-host NTP jumps. A single attempt that takes longer than the budget results in zero retries. Setting `retry_budget_seconds=0` disables retry
@@ -706,8 +744,8 @@ queue = RedisMessageQueue("jobs", client=client, on_event=on_event)
706
744
  ```
707
745
 
708
746
  Events cover publish, dedup hits, claim/empty polls, reclaim, ack/nack,
709
- completed/failed cleanup, DLQ moves, heartbeat renewal, stale leases, cleanup
710
- and trim failures, and retry attempts. Callback exceptions are logged and
747
+ completed/failed cleanup, DLQ moves, heartbeat renewal, stale leases, drain,
748
+ cleanup and trim failures, and retry attempts. Callback exceptions are logged and
711
749
  reported with `RuntimeWarning`, but never propagate into queue operations.
712
750
  `on_event` is telemetry only: use it for metrics, tracing, and logging, not for
713
751
  sagas, follow-up writes, billing callbacks, or other correctness-critical
@@ -800,6 +838,23 @@ Pre-commit and mid-flight exceptions:
800
838
  despite the exception. Treat them as "operation did not succeed from the
801
839
  caller's perspective", not "Redis did not commit".
802
840
 
841
+ #### Drain events
842
+
843
+ `drain()` and `close()` on the sync queue, and `drain()` and `aclose()` on the
844
+ async queue, emit `drain` events:
845
+
846
+ - `drain/start` when the queue-local drain flag is set.
847
+ - `drain/success` when pending claim IDs were recovered or no gateway drain
848
+ hook is present.
849
+ - `drain/skipped` when the queue was already drained and the cached successful
850
+ result is returned.
851
+ - `drain/failure` when pending claim recovery times out or otherwise leaves
852
+ unresolved claim IDs.
853
+
854
+ Drain events use `timeout_seconds` for the caller-supplied timeout,
855
+ `pending_claim_ids` for the number of unresolved local claim IDs when known,
856
+ and `exception_type` / `error` on failure.
857
+
803
858
  #### Intentionally silent paths
804
859
 
805
860
  The following operations have no `on_event` surface by design:
@@ -814,9 +869,11 @@ The following operations have no `on_event` surface by design:
814
869
  it back to pending, and returns `false`. Python translates that into
815
870
  `claim_empty/skipped`, the same shape as an empty poll. This is intentional
816
871
  fail-safe behavior; the message is not lost.
817
- - **`drain()` / `close()` / `aclose()` lifecycle:** explicit shutdown
818
- operations do not emit lifecycle events. Pending-claim-drain recovery work
819
- counts as `claim_reclaim` events when reached.
872
+ - **`drop_oldest` evictions:** when publish backpressure uses
873
+ `pending_overload_policy="drop_oldest"`, the oldest pending message is
874
+ discarded before the new message is enqueued. The successful enqueue emits
875
+ `publish/success`, but there is no separate drop event for the discarded
876
+ message in the current feature set.
820
877
  - **Non-claim-loop retry attempts:** tenacity retries in deduplicated publish,
821
878
  ack/remove, move-to-completed/failed, and lease renewal collapse into the
822
879
  terminal operation's failure event. There is no per-attempt event for those
@@ -1015,7 +1072,7 @@ v6.0.0 is a non-breaking-defaults release that adds new public APIs. v5 code con
1015
1072
 
1016
1073
  - `max_pending_length=N` caps pending-list depth; with `pending_overload_policy="raise"` (default) producers see `QueueBackpressureError` when the cap is hit; `"block"` waits up to `pending_overload_block_timeout_seconds`; `"drop_oldest"` evicts silently, so use it only when data loss is acceptable.
1017
1074
  - `queue.drain(timeout=...)` (sync) and `await queue.aclose(timeout=...)` (async) are explicit graceful-shutdown hooks. They refuse new claims and recover pending claim IDs but do not cancel in-flight handlers; join or await your worker separately.
1018
- - `on_event=callback` receives a `QueueEvent` dataclass for every publish/claim/ack/reclaim/dedup/cleanup lifecycle event. Use it for metrics, tracing, and structured logging. See [`examples/production/observability.py`](examples/production/observability.py) for the adapter pattern.
1075
+ - `on_event=callback` receives a `QueueEvent` dataclass for publish/claim/ack/reclaim/dedup/cleanup/drain lifecycle events. Use it for metrics, tracing, and structured logging. See [`examples/production/observability.py`](examples/production/observability.py) for the adapter pattern.
1019
1076
  - See [`examples/production/backpressure.py`](examples/production/backpressure.py) and [`examples/production/graceful_shutdown.py`](examples/production/graceful_shutdown.py) for sync production patterns, with async siblings under [`examples/production/asyncio/`](examples/production/asyncio/).
1020
1077
 
1021
1078
  > When using a pre-fork app server (gunicorn `--preload`, uvicorn workers that import the app at master startup), call `make_queue()` from your worker startup hook - NOT at module import. See [Fork safety](#fork-safety-and-pre-fork-servers) for why.
@@ -1064,13 +1121,13 @@ Try the [examples](https://github.com/Elijas/redis-message-queue/tree/main/examp
1064
1121
 
1065
1122
  ```bash
1066
1123
  # Two publishers
1067
- poetry run python -m examples.send_messages
1068
- poetry run python -m examples.send_messages
1124
+ uv run python -m examples.send_messages
1125
+ uv run python -m examples.send_messages
1069
1126
 
1070
1127
  # Three consumers
1071
- poetry run python -m examples.receive_messages
1072
- poetry run python -m examples.receive_messages
1073
- poetry run python -m examples.receive_messages
1128
+ uv run python -m examples.receive_messages
1129
+ uv run python -m examples.receive_messages
1130
+ uv run python -m examples.receive_messages
1074
1131
  ```
1075
1132
 
1076
1133
  ![GitHub Repo stars](https://img.shields.io/github/stars/elijas/redis-message-queue?style=flat&color=fcfcfc&labelColor=white&logo=github&logoColor=black&label=stars)
@@ -1,32 +1,6 @@
1
- Metadata-Version: 2.4
2
- Name: redis-message-queue
3
- Version: 8.2.0
4
- Summary: Python message queuing with Redis and message deduplication
5
- License: MIT
6
- License-File: LICENSE
7
- Keywords: redis,message-queue,deduplication,task-queue
8
- Author: Elijas
9
- Author-email: 4084885+Elijas@users.noreply.github.com
10
- Requires-Python: >=3.12,<4.0
11
- Classifier: Development Status :: 5 - Production/Stable
12
- Classifier: Intended Audience :: Developers
13
- Classifier: License :: OSI Approved :: MIT License
14
- Classifier: Programming Language :: Python :: 3
15
- Classifier: Programming Language :: Python :: 3.12
16
- Classifier: Programming Language :: Python :: 3.13
17
- Classifier: Programming Language :: Python :: 3.14
18
- Classifier: Topic :: Software Development :: Libraries
19
- Classifier: Topic :: System :: Distributed Computing
20
- Requires-Dist: redis (>=5.0.0,<8.0.0)
21
- Requires-Dist: tenacity (>=8.1.0)
22
- Project-URL: Homepage, https://github.com/Elijas/redis-message-queue
23
- Project-URL: Issues, https://github.com/Elijas/redis-message-queue/issues
24
- Project-URL: Repository, https://github.com/Elijas/redis-message-queue
25
- Description-Content-Type: text/markdown
26
-
27
1
  # redis-message-queue
28
2
 
29
- [![PyPI Version](https://img.shields.io/badge/v8.0.1-version?color=43cd0f&style=flat&label=pypi)](https://pypi.org/project/redis-message-queue)
3
+ [![PyPI Version](https://img.shields.io/pypi/v/redis-message-queue?color=43cd0f&style=flat&label=pypi)](https://pypi.org/project/redis-message-queue)
30
4
  [![PyPI Downloads](https://img.shields.io/pypi/dm/redis-message-queue?color=43cd0f&style=flat&label=downloads)](https://pypistats.org/packages/redis-message-queue)
31
5
  [![License: MIT](https://img.shields.io/badge/License-MIT-43cd0f.svg?style=flat&label=license)](LICENSE)
32
6
  [![Maintained: yes](https://img.shields.io/badge/yes-43cd0f.svg?style=flat&label=maintained)](https://github.com/Elijas/redis-message-queue/issues)
@@ -37,7 +11,7 @@ Description-Content-Type: text/markdown
37
11
  **Lightweight Python message queuing with Redis and built-in publish-side deduplication.** Deduplicate publishes within a TTL window, with optional crash recovery — across any number of producers and consumers.
38
12
 
39
13
  ```bash
40
- pip install "redis-message-queue>=8.0.0,<9.0.0"
14
+ pip install "redis-message-queue>=8.2.2,<9.0.0"
41
15
  ```
42
16
 
43
17
  Requires Redis server >= 6.2.
@@ -48,6 +22,7 @@ Redis must be running locally first: use `redis-server` or
48
22
  `docker run -p 6379:6379 redis:7`.
49
23
 
50
24
  ```python
25
+ import json
51
26
  from redis import Redis
52
27
  from redis_message_queue import RedisMessageQueue
53
28
 
@@ -61,7 +36,8 @@ queue = RedisMessageQueue(
61
36
  queue.publish({"id": "msg-1", "text": "hello"})
62
37
  with queue.process_message() as message:
63
38
  if message is not None:
64
- print(f"got {message['text']}")
39
+ payload = json.loads(message)
40
+ print(f"got {payload['text']}")
65
41
  # Expected output: got hello
66
42
  ```
67
43
 
@@ -72,13 +48,15 @@ with queue.process_message() as message:
72
48
  > synchronous. If your handler is `async def`, returns a coroutine, or returns
73
49
  > any other awaitable, use `redis_message_queue.asyncio.RedisMessageQueue`.
74
50
  > The sync context manager does not inspect the handler's return value; an
75
- > unawaited coroutine can be dropped while the message is acked. An ergonomic
76
- > callback API that detects this is planned for v8.1.
51
+ > unawaited coroutine can be dropped while the message is acked. For sync
52
+ > callback-style handlers, use `process_message_callback(handler)`: it checks
53
+ > for awaitable returns before acking and raises `TypeError` if one is returned.
77
54
 
78
55
  ### Async quickstart
79
56
 
80
57
  ```python
81
58
  import asyncio
59
+ import json
82
60
  from redis.asyncio import Redis
83
61
  from redis_message_queue.asyncio import RedisMessageQueue
84
62
 
@@ -92,7 +70,8 @@ async def main():
92
70
  )
93
71
  await queue.publish({"id": "msg-1", "text": "hello"})
94
72
  async with queue.process_message() as message:
95
- print(f"got {message['text']}")
73
+ payload = json.loads(message)
74
+ print(f"got {payload['text']}")
96
75
  await client.aclose()
97
76
 
98
77
  asyncio.run(main()) # Expected output: got hello
@@ -228,7 +207,9 @@ so concurrent publishers cannot race above the configured cap. Overload policies
228
207
  - `drop_oldest` removes the oldest pending message (`RPOP`) before enqueueing the
229
208
  new message. This is silent data loss by design; deduplication markers for
230
209
  dropped messages are not removed, so a dropped duplicate may still be
231
- suppressed until its dedup TTL expires.
210
+ suppressed until its dedup TTL expires. The current event contract emits
211
+ `publish/success` for the new message, but no separate `on_event` signal for
212
+ the dropped message.
232
213
  - `block` retries the atomic check until space opens or
233
214
  `pending_overload_block_timeout_seconds` elapses (default: 1.0), then raises
234
215
  `QueueBackpressureError`.
@@ -500,6 +481,14 @@ gateway = RedisGateway(
500
481
  queue = RedisMessageQueue("q", gateway=gateway)
501
482
  ```
502
483
 
484
+ When `gateway=` is supplied, queue-level constructor defaults are not copied
485
+ into the gateway. For example, `RedisMessageQueue(..., gateway=gateway)`
486
+ leaves visibility timeout and dead-letter routing disabled unless
487
+ `message_visibility_timeout_seconds` and `max_delivery_count` are configured on
488
+ the gateway itself. Passing the queue-level default values
489
+ `visibility_timeout_seconds=300` or `max_delivery_count=10` with `gateway=`
490
+ does not transfer those settings to the gateway.
491
+
503
492
  The retry knobs configure an internal `tenacity` strategy: exponential
504
493
  backoff with jitter, retry on transient Redis errors only, capped at
505
494
  `retry_budget_seconds`. The budget is monotonic elapsed time from the first attempt (including attempt duration), not inter-attempt delay; it is unaffected by Python-host NTP jumps. A single attempt that takes longer than the budget results in zero retries. Setting `retry_budget_seconds=0` disables retry
@@ -732,8 +721,8 @@ queue = RedisMessageQueue("jobs", client=client, on_event=on_event)
732
721
  ```
733
722
 
734
723
  Events cover publish, dedup hits, claim/empty polls, reclaim, ack/nack,
735
- completed/failed cleanup, DLQ moves, heartbeat renewal, stale leases, cleanup
736
- and trim failures, and retry attempts. Callback exceptions are logged and
724
+ completed/failed cleanup, DLQ moves, heartbeat renewal, stale leases, drain,
725
+ cleanup and trim failures, and retry attempts. Callback exceptions are logged and
737
726
  reported with `RuntimeWarning`, but never propagate into queue operations.
738
727
  `on_event` is telemetry only: use it for metrics, tracing, and logging, not for
739
728
  sagas, follow-up writes, billing callbacks, or other correctness-critical
@@ -826,6 +815,23 @@ Pre-commit and mid-flight exceptions:
826
815
  despite the exception. Treat them as "operation did not succeed from the
827
816
  caller's perspective", not "Redis did not commit".
828
817
 
818
+ #### Drain events
819
+
820
+ `drain()` and `close()` on the sync queue, and `drain()` and `aclose()` on the
821
+ async queue, emit `drain` events:
822
+
823
+ - `drain/start` when the queue-local drain flag is set.
824
+ - `drain/success` when pending claim IDs were recovered or no gateway drain
825
+ hook is present.
826
+ - `drain/skipped` when the queue was already drained and the cached successful
827
+ result is returned.
828
+ - `drain/failure` when pending claim recovery times out or otherwise leaves
829
+ unresolved claim IDs.
830
+
831
+ Drain events use `timeout_seconds` for the caller-supplied timeout,
832
+ `pending_claim_ids` for the number of unresolved local claim IDs when known,
833
+ and `exception_type` / `error` on failure.
834
+
829
835
  #### Intentionally silent paths
830
836
 
831
837
  The following operations have no `on_event` surface by design:
@@ -840,9 +846,11 @@ The following operations have no `on_event` surface by design:
840
846
  it back to pending, and returns `false`. Python translates that into
841
847
  `claim_empty/skipped`, the same shape as an empty poll. This is intentional
842
848
  fail-safe behavior; the message is not lost.
843
- - **`drain()` / `close()` / `aclose()` lifecycle:** explicit shutdown
844
- operations do not emit lifecycle events. Pending-claim-drain recovery work
845
- counts as `claim_reclaim` events when reached.
849
+ - **`drop_oldest` evictions:** when publish backpressure uses
850
+ `pending_overload_policy="drop_oldest"`, the oldest pending message is
851
+ discarded before the new message is enqueued. The successful enqueue emits
852
+ `publish/success`, but there is no separate drop event for the discarded
853
+ message in the current feature set.
846
854
  - **Non-claim-loop retry attempts:** tenacity retries in deduplicated publish,
847
855
  ack/remove, move-to-completed/failed, and lease renewal collapse into the
848
856
  terminal operation's failure event. There is no per-attempt event for those
@@ -1041,7 +1049,7 @@ v6.0.0 is a non-breaking-defaults release that adds new public APIs. v5 code con
1041
1049
 
1042
1050
  - `max_pending_length=N` caps pending-list depth; with `pending_overload_policy="raise"` (default) producers see `QueueBackpressureError` when the cap is hit; `"block"` waits up to `pending_overload_block_timeout_seconds`; `"drop_oldest"` evicts silently, so use it only when data loss is acceptable.
1043
1051
  - `queue.drain(timeout=...)` (sync) and `await queue.aclose(timeout=...)` (async) are explicit graceful-shutdown hooks. They refuse new claims and recover pending claim IDs but do not cancel in-flight handlers; join or await your worker separately.
1044
- - `on_event=callback` receives a `QueueEvent` dataclass for every publish/claim/ack/reclaim/dedup/cleanup lifecycle event. Use it for metrics, tracing, and structured logging. See [`examples/production/observability.py`](examples/production/observability.py) for the adapter pattern.
1052
+ - `on_event=callback` receives a `QueueEvent` dataclass for publish/claim/ack/reclaim/dedup/cleanup/drain lifecycle events. Use it for metrics, tracing, and structured logging. See [`examples/production/observability.py`](examples/production/observability.py) for the adapter pattern.
1045
1053
  - See [`examples/production/backpressure.py`](examples/production/backpressure.py) and [`examples/production/graceful_shutdown.py`](examples/production/graceful_shutdown.py) for sync production patterns, with async siblings under [`examples/production/asyncio/`](examples/production/asyncio/).
1046
1054
 
1047
1055
  > When using a pre-fork app server (gunicorn `--preload`, uvicorn workers that import the app at master startup), call `make_queue()` from your worker startup hook - NOT at module import. See [Fork safety](#fork-safety-and-pre-fork-servers) for why.
@@ -1090,14 +1098,13 @@ Try the [examples](https://github.com/Elijas/redis-message-queue/tree/main/examp
1090
1098
 
1091
1099
  ```bash
1092
1100
  # Two publishers
1093
- poetry run python -m examples.send_messages
1094
- poetry run python -m examples.send_messages
1101
+ uv run python -m examples.send_messages
1102
+ uv run python -m examples.send_messages
1095
1103
 
1096
1104
  # Three consumers
1097
- poetry run python -m examples.receive_messages
1098
- poetry run python -m examples.receive_messages
1099
- poetry run python -m examples.receive_messages
1105
+ uv run python -m examples.receive_messages
1106
+ uv run python -m examples.receive_messages
1107
+ uv run python -m examples.receive_messages
1100
1108
  ```
1101
1109
 
1102
1110
  ![GitHub Repo stars](https://img.shields.io/github/stars/elijas/redis-message-queue?style=flat&color=fcfcfc&labelColor=white&logo=github&logoColor=black&label=stars)
1103
-
@@ -1,11 +1,11 @@
1
- [tool.poetry]
1
+ [project]
2
2
  name = "redis-message-queue"
3
- version = "8.2.0"
3
+ version = "8.2.2"
4
4
  description = "Python message queuing with Redis and message deduplication"
5
- authors = ["Elijas <4084885+Elijas@users.noreply.github.com>"]
5
+ authors = [{ name = "Elijas", email = "4084885+Elijas@users.noreply.github.com" }]
6
6
  readme = "README.md"
7
- license = "MIT"
8
- include = ["redis_message_queue/py.typed"]
7
+ license = { text = "MIT" }
8
+ license-files = ["LICENSE"]
9
9
  keywords = ["redis", "message-queue", "deduplication", "task-queue"]
10
10
  classifiers = [
11
11
  "Development Status :: 5 - Production/Stable",
@@ -16,26 +16,65 @@ classifiers = [
16
16
  "Topic :: Software Development :: Libraries",
17
17
  "Topic :: System :: Distributed Computing",
18
18
  ]
19
+ requires-python = ">=3.12,<4.0"
20
+ dependencies = [
21
+ "redis>=5.0.1,<8.0.0",
22
+ "tenacity>=8.1.0",
23
+ ]
19
24
 
20
- [tool.poetry.urls]
25
+ [project.urls]
21
26
  Homepage = "https://github.com/Elijas/redis-message-queue"
22
27
  Repository = "https://github.com/Elijas/redis-message-queue"
23
28
  Issues = "https://github.com/Elijas/redis-message-queue/issues"
24
29
 
25
- [tool.poetry.dependencies]
26
- python = "^3.12"
27
- redis = ">=5.0.0,<8.0.0"
28
- tenacity = ">=8.1.0"
30
+ [dependency-groups]
31
+ dev = [
32
+ "bump-my-version>=1.1.2",
33
+ "mypy",
34
+ "ruff",
35
+ ]
36
+ test = [
37
+ "fakeredis[lua]",
38
+ "pytest",
39
+ "pytest-asyncio",
40
+ "pytest-cov",
41
+ ]
42
+
43
+ [tool.uv]
44
+ default-groups = ["dev", "test"]
29
45
 
30
- [tool.poetry.group.test.dependencies]
31
- pytest = "*"
32
- pytest-asyncio = "*"
33
- pytest-cov = "*"
34
- fakeredis = {version = "*", extras = ["lua"]}
46
+ ##############################
47
+ ### BUMP VERSION
48
+ ##############################
49
+
50
+ [tool.bumpversion]
51
+ current_version = "8.2.2"
52
+ parse = "(?P<major>\\d+)\\.(?P<minor>\\d+)\\.(?P<patch>\\d+)"
53
+ serialize = ["{major}.{minor}.{patch}"]
54
+ search = "{current_version}"
55
+ replace = "{new_version}"
56
+ regex = false
57
+ ignore_missing_version = false
58
+ ignore_missing_files = false
59
+ tag = false
60
+ sign_tags = false
61
+ tag_name = "v{new_version}"
62
+ tag_message = "Bump version: {current_version} -> {new_version}"
63
+ allow_dirty = true
64
+ commit = true
65
+ message = "chore(release): bump version from {current_version} to {new_version}"
66
+ moveable_tags = []
67
+ commit_args = ""
68
+ setup_hooks = []
69
+ pre_commit_hooks = []
70
+ post_commit_hooks = []
35
71
 
36
72
  [build-system]
37
- requires = ["poetry-core"]
38
- build-backend = "poetry.core.masonry.api"
73
+ requires = ["hatchling"]
74
+ build-backend = "hatchling.build"
75
+
76
+ [tool.hatch.build]
77
+ packages = ["redis_message_queue"]
39
78
 
40
79
  [tool.ruff]
41
80
  target-version = "py312"
@@ -45,6 +84,12 @@ line-length = 120
45
84
  select = ["E", "F", "I", "W"]
46
85
  ignore = ["E731"]
47
86
 
87
+ [tool.mypy]
88
+ python_version = "3.12"
89
+ files = ["redis_message_queue/"]
90
+ explicit_package_bases = true
91
+ show_error_codes = true
92
+
48
93
  [tool.pytest.ini_options]
49
94
  asyncio_default_fixture_loop_scope = "function"
50
95
  filterwarnings = [
@@ -301,8 +301,8 @@ class RedisGateway(AbstractRedisGateway):
301
301
 
302
302
  def _emit_event(
303
303
  self,
304
- operation: EventOperation,
305
- outcome: EventOutcome,
304
+ operation: EventOperation | str,
305
+ outcome: EventOutcome | str,
306
306
  *,
307
307
  message_id: str | None = None,
308
308
  claim_id: str | None = None,
@@ -328,7 +328,7 @@ class RedisGateway(AbstractRedisGateway):
328
328
 
329
329
  def _emit_repeated_event(
330
330
  self,
331
- operation: EventOperation,
331
+ operation: EventOperation | str,
332
332
  attempts: list[_MessageAttemptEvent],
333
333
  *,
334
334
  destination_queue: str | None = None,
@@ -275,8 +275,8 @@ class RedisGateway(AbstractRedisGateway):
275
275
 
276
276
  async def _emit_event(
277
277
  self,
278
- operation: EventOperation,
279
- outcome: EventOutcome,
278
+ operation: EventOperation | str,
279
+ outcome: EventOutcome | str,
280
280
  *,
281
281
  message_id: str | None = None,
282
282
  claim_id: str | None = None,
@@ -302,7 +302,7 @@ class RedisGateway(AbstractRedisGateway):
302
302
 
303
303
  async def _emit_repeated_event(
304
304
  self,
305
- operation: EventOperation,
305
+ operation: EventOperation | str,
306
306
  attempts: list[_MessageAttemptEvent],
307
307
  *,
308
308
  destination_queue: str | None = None,
@@ -11,7 +11,6 @@ from typing import AsyncIterator, Awaitable, Callable, Literal, Optional, TypeVa
11
11
  import redis.asyncio
12
12
  import redis.exceptions
13
13
 
14
- from redis_message_queue._callable_utils import is_async_callable
15
14
  from redis_message_queue._config import (
16
15
  DEFAULT_PENDING_OVERLOAD_BLOCK_TIMEOUT_SECONDS,
17
16
  validate_dedup_configuration,
@@ -332,8 +331,12 @@ def _derive_dead_letter_queue(name: str, key_separator: str) -> str:
332
331
  return f"{name}{key_separator}{_AUTO_DEAD_LETTER_QUEUE_SUFFIX}"
333
332
 
334
333
 
335
- def _bind_dead_letter_gateway_to_queue(gateway: AbstractRedisGateway, queue_pending_key: str) -> None:
336
- """Reject reuse of a dead-letter-enabled gateway across different queues.
334
+ def _bind_dead_letter_gateway_to_queue(
335
+ gateway: AbstractRedisGateway,
336
+ queue_pending_key: str,
337
+ queue_processing_key: str,
338
+ ) -> None:
339
+ """Validate and bind a dead-letter-enabled gateway to this queue.
337
340
 
338
341
  The check is not thread-safe: constructing ``RedisMessageQueue`` instances
339
342
  concurrently on multiple threads with the same DLQ-enabled gateway can
@@ -346,6 +349,12 @@ def _bind_dead_letter_gateway_to_queue(gateway: AbstractRedisGateway, queue_pend
346
349
  if max_delivery_count is None:
347
350
  return
348
351
 
352
+ if gateway.dead_letter_queue in (queue_pending_key, queue_processing_key):
353
+ raise ConfigurationError(
354
+ "'dead_letter_queue' must be distinct from the queue's pending and processing Redis keys. "
355
+ "Use a separate Redis list key for poison messages."
356
+ )
357
+
349
358
  bound_pending_key = getattr(gateway, _GATEWAY_BOUND_PENDING_QUEUE_ATTR, None)
350
359
  if bound_pending_key is None:
351
360
  setattr(gateway, _GATEWAY_BOUND_PENDING_QUEUE_ATTR, queue_pending_key)
@@ -451,7 +460,7 @@ class _LeaseHeartbeat:
451
460
  stacklevel=2,
452
461
  )
453
462
 
454
- async def _emit(self, operation: EventOperation, outcome: EventOutcome, **kwargs: object) -> None:
463
+ async def _emit(self, operation: EventOperation | str, outcome: EventOutcome | str, **kwargs: object) -> None:
455
464
  if self._emit_event is not None:
456
465
  await self._emit_event(operation, outcome, **kwargs)
457
466
 
@@ -597,6 +606,11 @@ class RedisMessageQueue:
597
606
  auto-derived dead-letter queue. Set it to ``None`` for unlimited
598
607
  redelivery.
599
608
 
609
+ When ``gateway=`` is supplied, queue-level defaults are not transferred
610
+ to the gateway. Configure lease, dead-letter, and backpressure settings
611
+ such as ``message_visibility_timeout_seconds``, ``max_delivery_count``,
612
+ and ``max_pending_length`` on the gateway itself.
613
+
600
614
  ``deduplication=True`` requires ``get_deduplication_key`` to be a
601
615
  callable that returns a non-empty string. Use a stable logical ID for
602
616
  the deduplication keyspace.
@@ -636,11 +650,11 @@ class RedisMessageQueue:
636
650
  ``GracefulInterruptHandler()`` for prompt Ctrl-C / termination handling
637
651
  in polling waits. ``on_heartbeat_failure`` is a zero-argument callable
638
652
  or coroutine callable invoked when lease renewal fails. ``on_event`` is
639
- telemetry only: an async callback receiving best-effort QueueEvent
640
- lifecycle notifications. Callback failures are logged and converted to
641
- RuntimeWarning without influencing ack/nack or any other message
642
- outcome. Do not use it for correctness-critical callbacks or follow-up
643
- writes.
653
+ telemetry only: a callable returning an awaitable and receiving
654
+ best-effort QueueEvent lifecycle notifications. Callback failures are
655
+ logged and converted to RuntimeWarning without influencing ack/nack or
656
+ any other message outcome. Do not use it for correctness-critical
657
+ callbacks or follow-up writes.
644
658
  """
645
659
  self.key = QueueKeyManager(name, key_separator=key_separator)
646
660
  if not isinstance(deduplication, bool):
@@ -742,8 +756,6 @@ class RedisMessageQueue:
742
756
  )
743
757
  if on_event is not None and not callable(on_event):
744
758
  raise TypeError(f"'on_event' must be callable, got {type(on_event).__name__}.")
745
- if on_event is not None and not is_async_callable(on_event):
746
- raise TypeError("'on_event' must be an async callable.")
747
759
  self._queue_name = name
748
760
  self._on_event = on_event
749
761
  # Queue-local soft-drain flag. See sync queue ``_draining`` docstring
@@ -825,7 +837,7 @@ class RedisMessageQueue:
825
837
  "'max_pending_length' cannot be provided alongside 'gateway'."
826
838
  " Configure publish backpressure on the gateway directly instead."
827
839
  )
828
- _bind_dead_letter_gateway_to_queue(gateway, self.key.pending)
840
+ _bind_dead_letter_gateway_to_queue(gateway, self.key.pending, self.key.processing)
829
841
  self._max_delivery_count = None
830
842
  self._redis = gateway
831
843
  elif client is None:
@@ -867,8 +879,8 @@ class RedisMessageQueue:
867
879
 
868
880
  async def _emit_event(
869
881
  self,
870
- operation: EventOperation,
871
- outcome: EventOutcome,
882
+ operation: EventOperation | str,
883
+ outcome: EventOutcome | str,
872
884
  *,
873
885
  message_id: str | None = None,
874
886
  claim_id: str | None = None,
@@ -273,8 +273,12 @@ def _derive_dead_letter_queue(name: str, key_separator: str) -> str:
273
273
  return f"{name}{key_separator}{_AUTO_DEAD_LETTER_QUEUE_SUFFIX}"
274
274
 
275
275
 
276
- def _bind_dead_letter_gateway_to_queue(gateway: AbstractRedisGateway, queue_pending_key: str) -> None:
277
- """Reject reuse of a dead-letter-enabled gateway across different queues.
276
+ def _bind_dead_letter_gateway_to_queue(
277
+ gateway: AbstractRedisGateway,
278
+ queue_pending_key: str,
279
+ queue_processing_key: str,
280
+ ) -> None:
281
+ """Validate and bind a dead-letter-enabled gateway to this queue.
278
282
 
279
283
  The check is not thread-safe: constructing ``RedisMessageQueue`` instances
280
284
  concurrently on multiple threads with the same DLQ-enabled gateway can
@@ -287,6 +291,12 @@ def _bind_dead_letter_gateway_to_queue(gateway: AbstractRedisGateway, queue_pend
287
291
  if max_delivery_count is None:
288
292
  return
289
293
 
294
+ if gateway.dead_letter_queue in (queue_pending_key, queue_processing_key):
295
+ raise ConfigurationError(
296
+ "'dead_letter_queue' must be distinct from the queue's pending and processing Redis keys. "
297
+ "Use a separate Redis list key for poison messages."
298
+ )
299
+
290
300
  bound_pending_key = getattr(gateway, _GATEWAY_BOUND_PENDING_QUEUE_ATTR, None)
291
301
  if bound_pending_key is None:
292
302
  setattr(gateway, _GATEWAY_BOUND_PENDING_QUEUE_ATTR, queue_pending_key)
@@ -407,7 +417,7 @@ class _LeaseHeartbeat:
407
417
  stacklevel=2,
408
418
  )
409
419
 
410
- def _emit(self, operation: EventOperation, outcome: EventOutcome, **kwargs: object) -> None:
420
+ def _emit(self, operation: EventOperation | str, outcome: EventOutcome | str, **kwargs: object) -> None:
411
421
  if self._emit_event is not None:
412
422
  self._emit_event(operation, outcome, **kwargs)
413
423
 
@@ -549,6 +559,11 @@ class RedisMessageQueue:
549
559
  auto-derived dead-letter queue. Set it to ``None`` for unlimited
550
560
  redelivery.
551
561
 
562
+ When ``gateway=`` is supplied, queue-level defaults are not transferred
563
+ to the gateway. Configure lease, dead-letter, and backpressure settings
564
+ such as ``message_visibility_timeout_seconds``, ``max_delivery_count``,
565
+ and ``max_pending_length`` on the gateway itself.
566
+
552
567
  ``deduplication=True`` requires ``get_deduplication_key`` to be a
553
568
  callable that returns a non-empty string. Use a stable logical ID for
554
569
  the deduplication keyspace.
@@ -786,7 +801,7 @@ class RedisMessageQueue:
786
801
  "'max_pending_length' cannot be provided alongside 'gateway'."
787
802
  " Configure publish backpressure on the gateway directly instead."
788
803
  )
789
- _bind_dead_letter_gateway_to_queue(gateway, self.key.pending)
804
+ _bind_dead_letter_gateway_to_queue(gateway, self.key.pending, self.key.processing)
790
805
  self._max_delivery_count = None
791
806
  self._redis = gateway
792
807
  elif client is None:
@@ -827,8 +842,8 @@ class RedisMessageQueue:
827
842
 
828
843
  def _emit_event(
829
844
  self,
830
- operation: EventOperation,
831
- outcome: EventOutcome,
845
+ operation: EventOperation | str,
846
+ outcome: EventOutcome | str,
832
847
  *,
833
848
  message_id: str | None = None,
834
849
  claim_id: str | None = None,
@@ -1031,8 +1046,9 @@ class RedisMessageQueue:
1031
1046
  does not inspect handler return values; if your handler returns a
1032
1047
  coroutine or other awaitable, the awaitable can be dropped while the
1033
1048
  message is acked. Use ``redis_message_queue.asyncio.RedisMessageQueue``
1034
- for async handlers. An ergonomic callback API that detects this is
1035
- planned for v8.1.
1049
+ for async handlers. For sync callback-style handlers, use
1050
+ ``process_message_callback(handler)`` so awaitable returns are detected
1051
+ before acking.
1036
1052
 
1037
1053
  If the process is killed mid-handler, the claimed message and lease
1038
1054
  metadata remain in Redis until a later consumer claim triggers