redis-message-queue 8.2.5__tar.gz → 8.2.6__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {redis_message_queue-8.2.5 → redis_message_queue-8.2.6}/PKG-INFO +12 -7
- {redis_message_queue-8.2.5 → redis_message_queue-8.2.6}/README.md +11 -6
- {redis_message_queue-8.2.5 → redis_message_queue-8.2.6}/pyproject.toml +2 -2
- {redis_message_queue-8.2.5 → redis_message_queue-8.2.6}/redis_message_queue/_redis_gateway.py +65 -11
- {redis_message_queue-8.2.5 → redis_message_queue-8.2.6}/redis_message_queue/asyncio/_redis_gateway.py +65 -11
- {redis_message_queue-8.2.5 → redis_message_queue-8.2.6}/redis_message_queue/asyncio/redis_message_queue.py +46 -29
- {redis_message_queue-8.2.5 → redis_message_queue-8.2.6}/redis_message_queue/redis_message_queue.py +46 -29
- {redis_message_queue-8.2.5 → redis_message_queue-8.2.6}/.gitignore +0 -0
- {redis_message_queue-8.2.5 → redis_message_queue-8.2.6}/LICENSE +0 -0
- {redis_message_queue-8.2.5 → redis_message_queue-8.2.6}/redis_message_queue/__init__.py +0 -0
- {redis_message_queue-8.2.5 → redis_message_queue-8.2.6}/redis_message_queue/_abstract_redis_gateway.py +0 -0
- {redis_message_queue-8.2.5 → redis_message_queue-8.2.6}/redis_message_queue/_callable_utils.py +0 -0
- {redis_message_queue-8.2.5 → redis_message_queue-8.2.6}/redis_message_queue/_config.py +0 -0
- {redis_message_queue-8.2.5 → redis_message_queue-8.2.6}/redis_message_queue/_event.py +0 -0
- {redis_message_queue-8.2.5 → redis_message_queue-8.2.6}/redis_message_queue/_exceptions.py +0 -0
- {redis_message_queue-8.2.5 → redis_message_queue-8.2.6}/redis_message_queue/_payload_limits.py +0 -0
- {redis_message_queue-8.2.5 → redis_message_queue-8.2.6}/redis_message_queue/_queue_key_manager.py +0 -0
- {redis_message_queue-8.2.5 → redis_message_queue-8.2.6}/redis_message_queue/_redis_cluster.py +0 -0
- {redis_message_queue-8.2.5 → redis_message_queue-8.2.6}/redis_message_queue/_stored_message.py +0 -0
- {redis_message_queue-8.2.5 → redis_message_queue-8.2.6}/redis_message_queue/asyncio/__init__.py +0 -0
- {redis_message_queue-8.2.5 → redis_message_queue-8.2.6}/redis_message_queue/asyncio/_abstract_redis_gateway.py +0 -0
- {redis_message_queue-8.2.5 → redis_message_queue-8.2.6}/redis_message_queue/interrupt_handler/__init__.py +0 -0
- {redis_message_queue-8.2.5 → redis_message_queue-8.2.6}/redis_message_queue/interrupt_handler/_event_driven.py +0 -0
- {redis_message_queue-8.2.5 → redis_message_queue-8.2.6}/redis_message_queue/interrupt_handler/_implementation.py +0 -0
- {redis_message_queue-8.2.5 → redis_message_queue-8.2.6}/redis_message_queue/interrupt_handler/_interface.py +0 -0
- {redis_message_queue-8.2.5 → redis_message_queue-8.2.6}/redis_message_queue/py.typed +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: redis-message-queue
|
|
3
|
-
Version: 8.2.
|
|
3
|
+
Version: 8.2.6
|
|
4
4
|
Summary: Python message queuing with Redis and message deduplication
|
|
5
5
|
Project-URL: Homepage, https://github.com/Elijas/redis-message-queue
|
|
6
6
|
Project-URL: Repository, https://github.com/Elijas/redis-message-queue
|
|
@@ -34,7 +34,7 @@ Description-Content-Type: text/markdown
|
|
|
34
34
|
**Lightweight Python message queuing with Redis and built-in publish-side deduplication.** Deduplicate publishes within a TTL window, with optional crash recovery — across any number of producers and consumers.
|
|
35
35
|
|
|
36
36
|
```bash
|
|
37
|
-
pip install "redis-message-queue>=8.2.
|
|
37
|
+
pip install "redis-message-queue>=8.2.6,<9.0.0"
|
|
38
38
|
```
|
|
39
39
|
|
|
40
40
|
Requires Redis server >= 6.2.
|
|
@@ -46,6 +46,7 @@ Redis must be running locally first: use `redis-server` or
|
|
|
46
46
|
|
|
47
47
|
```python
|
|
48
48
|
import json
|
|
49
|
+
from uuid import uuid4
|
|
49
50
|
from redis import Redis
|
|
50
51
|
from redis_message_queue import RedisMessageQueue
|
|
51
52
|
|
|
@@ -56,7 +57,8 @@ queue = RedisMessageQueue(
|
|
|
56
57
|
deduplication=True,
|
|
57
58
|
get_deduplication_key=lambda msg: msg["id"],
|
|
58
59
|
)
|
|
59
|
-
|
|
60
|
+
message = {"id": f"msg-{uuid4().hex}", "text": "hello"}
|
|
61
|
+
queue.publish(message)
|
|
60
62
|
with queue.process_message() as message:
|
|
61
63
|
if message is not None:
|
|
62
64
|
payload = json.loads(message)
|
|
@@ -80,6 +82,7 @@ with queue.process_message() as message:
|
|
|
80
82
|
```python
|
|
81
83
|
import asyncio
|
|
82
84
|
import json
|
|
85
|
+
from uuid import uuid4
|
|
83
86
|
from redis.asyncio import Redis
|
|
84
87
|
from redis_message_queue.asyncio import RedisMessageQueue
|
|
85
88
|
|
|
@@ -91,10 +94,12 @@ async def main():
|
|
|
91
94
|
deduplication=True,
|
|
92
95
|
get_deduplication_key=lambda msg: msg["id"],
|
|
93
96
|
)
|
|
94
|
-
|
|
97
|
+
message = {"id": f"msg-{uuid4().hex}", "text": "hello"}
|
|
98
|
+
await queue.publish(message)
|
|
95
99
|
async with queue.process_message() as message:
|
|
96
|
-
|
|
97
|
-
|
|
100
|
+
if message is not None:
|
|
101
|
+
payload = json.loads(message)
|
|
102
|
+
print(f"got {payload['text']}")
|
|
98
103
|
await client.aclose()
|
|
99
104
|
|
|
100
105
|
asyncio.run(main()) # Expected output: got hello
|
|
@@ -1044,7 +1049,7 @@ created:
|
|
|
1044
1049
|
dead-letter handling is required.`
|
|
1045
1050
|
|
|
1046
1051
|
**AC-16: redis-py is capped below 8.0.0.** The package dependency is
|
|
1047
|
-
`redis>=5.0.
|
|
1052
|
+
`redis>=5.0.1,<8.0.0` until redis-py 8 RESP3-default behavior is verified.
|
|
1048
1053
|
Users on redis-py 7.x and earlier are unaffected. If you installed a redis-py
|
|
1049
1054
|
8.0.0 beta explicitly, downgrade with `pip install "redis<8.0.0"`.
|
|
1050
1055
|
|
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
**Lightweight Python message queuing with Redis and built-in publish-side deduplication.** Deduplicate publishes within a TTL window, with optional crash recovery — across any number of producers and consumers.
|
|
12
12
|
|
|
13
13
|
```bash
|
|
14
|
-
pip install "redis-message-queue>=8.2.
|
|
14
|
+
pip install "redis-message-queue>=8.2.6,<9.0.0"
|
|
15
15
|
```
|
|
16
16
|
|
|
17
17
|
Requires Redis server >= 6.2.
|
|
@@ -23,6 +23,7 @@ Redis must be running locally first: use `redis-server` or
|
|
|
23
23
|
|
|
24
24
|
```python
|
|
25
25
|
import json
|
|
26
|
+
from uuid import uuid4
|
|
26
27
|
from redis import Redis
|
|
27
28
|
from redis_message_queue import RedisMessageQueue
|
|
28
29
|
|
|
@@ -33,7 +34,8 @@ queue = RedisMessageQueue(
|
|
|
33
34
|
deduplication=True,
|
|
34
35
|
get_deduplication_key=lambda msg: msg["id"],
|
|
35
36
|
)
|
|
36
|
-
|
|
37
|
+
message = {"id": f"msg-{uuid4().hex}", "text": "hello"}
|
|
38
|
+
queue.publish(message)
|
|
37
39
|
with queue.process_message() as message:
|
|
38
40
|
if message is not None:
|
|
39
41
|
payload = json.loads(message)
|
|
@@ -57,6 +59,7 @@ with queue.process_message() as message:
|
|
|
57
59
|
```python
|
|
58
60
|
import asyncio
|
|
59
61
|
import json
|
|
62
|
+
from uuid import uuid4
|
|
60
63
|
from redis.asyncio import Redis
|
|
61
64
|
from redis_message_queue.asyncio import RedisMessageQueue
|
|
62
65
|
|
|
@@ -68,10 +71,12 @@ async def main():
|
|
|
68
71
|
deduplication=True,
|
|
69
72
|
get_deduplication_key=lambda msg: msg["id"],
|
|
70
73
|
)
|
|
71
|
-
|
|
74
|
+
message = {"id": f"msg-{uuid4().hex}", "text": "hello"}
|
|
75
|
+
await queue.publish(message)
|
|
72
76
|
async with queue.process_message() as message:
|
|
73
|
-
|
|
74
|
-
|
|
77
|
+
if message is not None:
|
|
78
|
+
payload = json.loads(message)
|
|
79
|
+
print(f"got {payload['text']}")
|
|
75
80
|
await client.aclose()
|
|
76
81
|
|
|
77
82
|
asyncio.run(main()) # Expected output: got hello
|
|
@@ -1021,7 +1026,7 @@ created:
|
|
|
1021
1026
|
dead-letter handling is required.`
|
|
1022
1027
|
|
|
1023
1028
|
**AC-16: redis-py is capped below 8.0.0.** The package dependency is
|
|
1024
|
-
`redis>=5.0.
|
|
1029
|
+
`redis>=5.0.1,<8.0.0` until redis-py 8 RESP3-default behavior is verified.
|
|
1025
1030
|
Users on redis-py 7.x and earlier are unaffected. If you installed a redis-py
|
|
1026
1031
|
8.0.0 beta explicitly, downgrade with `pip install "redis<8.0.0"`.
|
|
1027
1032
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "redis-message-queue"
|
|
3
|
-
version = "8.2.
|
|
3
|
+
version = "8.2.6"
|
|
4
4
|
description = "Python message queuing with Redis and message deduplication"
|
|
5
5
|
authors = [{ name = "Elijas", email = "4084885+Elijas@users.noreply.github.com" }]
|
|
6
6
|
readme = "README.md"
|
|
@@ -48,7 +48,7 @@ default-groups = ["dev", "test"]
|
|
|
48
48
|
##############################
|
|
49
49
|
|
|
50
50
|
[tool.bumpversion]
|
|
51
|
-
current_version = "8.2.
|
|
51
|
+
current_version = "8.2.6"
|
|
52
52
|
parse = "(?P<major>\\d+)\\.(?P<minor>\\d+)\\.(?P<patch>\\d+)"
|
|
53
53
|
serialize = ["{major}.{minor}.{patch}"]
|
|
54
54
|
search = "{current_version}"
|
{redis_message_queue-8.2.5 → redis_message_queue-8.2.6}/redis_message_queue/_redis_gateway.py
RENAMED
|
@@ -291,6 +291,7 @@ class RedisGateway(AbstractRedisGateway):
|
|
|
291
291
|
self._pending_overload_policy = pending_overload_policy
|
|
292
292
|
self._pending_overload_block_timeout_seconds = pending_overload_block_timeout_seconds
|
|
293
293
|
self._pending_claim_ids: dict[str, list[str]] = {}
|
|
294
|
+
self._in_flight_claim_ids: dict[str, set[str]] = {}
|
|
294
295
|
self._recovering_claim_ids: dict[str, set[str]] = {}
|
|
295
296
|
self._pending_claim_ids_lock = threading.Lock()
|
|
296
297
|
self._drain_pending_claim_ids_lock = threading.Lock()
|
|
@@ -715,10 +716,31 @@ class RedisGateway(AbstractRedisGateway):
|
|
|
715
716
|
return None
|
|
716
717
|
|
|
717
718
|
pending_claim_id_to_share: str | None = None
|
|
719
|
+
active_claim_id: str | None = None
|
|
720
|
+
|
|
721
|
+
def begin_active_claim(claim_id: str) -> None:
|
|
722
|
+
nonlocal active_claim_id
|
|
723
|
+
if active_claim_id == claim_id:
|
|
724
|
+
return
|
|
725
|
+
if active_claim_id is not None:
|
|
726
|
+
self._finish_in_flight_claim_id(to_queue, active_claim_id)
|
|
727
|
+
self._begin_in_flight_claim_id(to_queue, claim_id)
|
|
728
|
+
active_claim_id = claim_id
|
|
729
|
+
|
|
730
|
+
def finish_active_claim() -> None:
|
|
731
|
+
nonlocal active_claim_id
|
|
732
|
+
if active_claim_id is None:
|
|
733
|
+
return
|
|
734
|
+
self._finish_in_flight_claim_id(to_queue, active_claim_id)
|
|
735
|
+
active_claim_id = None
|
|
736
|
+
|
|
718
737
|
try:
|
|
719
738
|
if self._message_wait_interval_seconds == 0:
|
|
720
739
|
claim_id = uuid.uuid4().hex
|
|
721
740
|
claim_may_need_recovery = False
|
|
741
|
+
begin_active_claim(claim_id)
|
|
742
|
+
if self._is_interrupted(is_interrupted):
|
|
743
|
+
return None
|
|
722
744
|
try:
|
|
723
745
|
claimed_message = claim_message(from_queue, to_queue, claim_id)
|
|
724
746
|
except Exception as exc:
|
|
@@ -768,6 +790,8 @@ class RedisGateway(AbstractRedisGateway):
|
|
|
768
790
|
claim_may_need_recovery = False
|
|
769
791
|
last_retryable_exception: Exception | None = None
|
|
770
792
|
while True:
|
|
793
|
+
if active_claim_id is None:
|
|
794
|
+
begin_active_claim(claim_id)
|
|
771
795
|
if self._is_interrupted(is_interrupted):
|
|
772
796
|
if claim_may_need_recovery:
|
|
773
797
|
pending_claim_id_to_share = claim_id
|
|
@@ -796,6 +820,7 @@ class RedisGateway(AbstractRedisGateway):
|
|
|
796
820
|
return claimed_message
|
|
797
821
|
claim_may_need_recovery = False
|
|
798
822
|
last_retryable_exception = None
|
|
823
|
+
finish_active_claim()
|
|
799
824
|
claim_id = uuid.uuid4().hex
|
|
800
825
|
|
|
801
826
|
remaining = deadline - time.monotonic()
|
|
@@ -836,6 +861,7 @@ class RedisGateway(AbstractRedisGateway):
|
|
|
836
861
|
finally:
|
|
837
862
|
if pending_claim_id_to_share is not None:
|
|
838
863
|
self._set_pending_claim_id(to_queue, pending_claim_id_to_share)
|
|
864
|
+
finish_active_claim()
|
|
839
865
|
|
|
840
866
|
def wait_for_message_and_move(self, from_queue: str, to_queue: str) -> ClaimedMessage | MessageData | None:
|
|
841
867
|
if self._is_interrupted():
|
|
@@ -1127,6 +1153,19 @@ class RedisGateway(AbstractRedisGateway):
|
|
|
1127
1153
|
if claim_id not in pending_claim_ids:
|
|
1128
1154
|
pending_claim_ids.append(claim_id)
|
|
1129
1155
|
|
|
1156
|
+
def _begin_in_flight_claim_id(self, processing_queue: str, claim_id: str) -> None:
|
|
1157
|
+
with self._pending_claim_ids_lock:
|
|
1158
|
+
self._in_flight_claim_ids.setdefault(processing_queue, set()).add(claim_id)
|
|
1159
|
+
|
|
1160
|
+
def _finish_in_flight_claim_id(self, processing_queue: str, claim_id: str) -> None:
|
|
1161
|
+
with self._pending_claim_ids_lock:
|
|
1162
|
+
in_flight_claim_ids = self._in_flight_claim_ids.get(processing_queue)
|
|
1163
|
+
if in_flight_claim_ids is None:
|
|
1164
|
+
return
|
|
1165
|
+
in_flight_claim_ids.discard(claim_id)
|
|
1166
|
+
if not in_flight_claim_ids:
|
|
1167
|
+
self._in_flight_claim_ids.pop(processing_queue, None)
|
|
1168
|
+
|
|
1130
1169
|
def _finish_pending_claim_recovery(
|
|
1131
1170
|
self,
|
|
1132
1171
|
processing_queue: str,
|
|
@@ -1328,15 +1367,29 @@ class RedisGateway(AbstractRedisGateway):
|
|
|
1328
1367
|
with self._pending_claim_ids_lock:
|
|
1329
1368
|
pending = self._pending_claim_ids.get(processing_queue)
|
|
1330
1369
|
if not pending:
|
|
1331
|
-
|
|
1332
|
-
|
|
1333
|
-
|
|
1334
|
-
|
|
1335
|
-
|
|
1336
|
-
|
|
1337
|
-
|
|
1338
|
-
|
|
1339
|
-
|
|
1370
|
+
in_flight = self._in_flight_claim_ids.get(processing_queue)
|
|
1371
|
+
if not in_flight:
|
|
1372
|
+
return True
|
|
1373
|
+
claim_id = None
|
|
1374
|
+
else:
|
|
1375
|
+
recovering = self._recovering_claim_ids.setdefault(processing_queue, set())
|
|
1376
|
+
claim_id = next(
|
|
1377
|
+
(cid for cid in pending if cid not in recovering and cid not in skipped_unresolved),
|
|
1378
|
+
None,
|
|
1379
|
+
)
|
|
1380
|
+
if claim_id is None:
|
|
1381
|
+
break
|
|
1382
|
+
recovering.add(claim_id)
|
|
1383
|
+
if claim_id is None:
|
|
1384
|
+
if deadline_monotonic is None:
|
|
1385
|
+
time.sleep(_VISIBILITY_TIMEOUT_POLL_INTERVAL_SECONDS)
|
|
1386
|
+
else:
|
|
1387
|
+
remaining = deadline_monotonic - time.monotonic()
|
|
1388
|
+
if remaining <= 0:
|
|
1389
|
+
last_error = TimeoutError("drain pending-claim recovery deadline expired")
|
|
1390
|
+
break
|
|
1391
|
+
time.sleep(min(_VISIBILITY_TIMEOUT_POLL_INTERVAL_SECONDS, remaining))
|
|
1392
|
+
continue
|
|
1340
1393
|
clear = False
|
|
1341
1394
|
try:
|
|
1342
1395
|
try:
|
|
@@ -1385,6 +1438,7 @@ class RedisGateway(AbstractRedisGateway):
|
|
|
1385
1438
|
self._finish_pending_claim_recovery(processing_queue, claim_id, clear=clear)
|
|
1386
1439
|
with self._pending_claim_ids_lock:
|
|
1387
1440
|
pending = self._pending_claim_ids.get(processing_queue)
|
|
1388
|
-
|
|
1441
|
+
in_flight = self._in_flight_claim_ids.get(processing_queue)
|
|
1442
|
+
if pending or in_flight:
|
|
1389
1443
|
self._last_drain_error = last_error
|
|
1390
|
-
return not pending
|
|
1444
|
+
return not pending and not in_flight
|
|
@@ -265,6 +265,7 @@ class RedisGateway(AbstractRedisGateway):
|
|
|
265
265
|
self._pending_overload_policy = pending_overload_policy
|
|
266
266
|
self._pending_overload_block_timeout_seconds = pending_overload_block_timeout_seconds
|
|
267
267
|
self._pending_claim_ids: dict[str, list[str]] = {}
|
|
268
|
+
self._in_flight_claim_ids: dict[str, set[str]] = {}
|
|
268
269
|
self._recovering_claim_ids: dict[str, set[str]] = {}
|
|
269
270
|
self._pending_claim_ids_lock = threading.Lock()
|
|
270
271
|
self._drain_pending_claim_ids_lock = asyncio.Lock()
|
|
@@ -694,10 +695,31 @@ class RedisGateway(AbstractRedisGateway):
|
|
|
694
695
|
return None
|
|
695
696
|
|
|
696
697
|
pending_claim_id_to_share: str | None = None
|
|
698
|
+
active_claim_id: str | None = None
|
|
699
|
+
|
|
700
|
+
def begin_active_claim(claim_id: str) -> None:
|
|
701
|
+
nonlocal active_claim_id
|
|
702
|
+
if active_claim_id == claim_id:
|
|
703
|
+
return
|
|
704
|
+
if active_claim_id is not None:
|
|
705
|
+
self._finish_in_flight_claim_id(to_queue, active_claim_id)
|
|
706
|
+
self._begin_in_flight_claim_id(to_queue, claim_id)
|
|
707
|
+
active_claim_id = claim_id
|
|
708
|
+
|
|
709
|
+
def finish_active_claim() -> None:
|
|
710
|
+
nonlocal active_claim_id
|
|
711
|
+
if active_claim_id is None:
|
|
712
|
+
return
|
|
713
|
+
self._finish_in_flight_claim_id(to_queue, active_claim_id)
|
|
714
|
+
active_claim_id = None
|
|
715
|
+
|
|
697
716
|
try:
|
|
698
717
|
if self._message_wait_interval_seconds == 0:
|
|
699
718
|
claim_id = uuid.uuid4().hex
|
|
700
719
|
claim_may_need_recovery = False
|
|
720
|
+
begin_active_claim(claim_id)
|
|
721
|
+
if self._is_interrupted(is_interrupted):
|
|
722
|
+
return None
|
|
701
723
|
try:
|
|
702
724
|
claimed_message = await claim_message(from_queue, to_queue, claim_id)
|
|
703
725
|
except Exception as exc:
|
|
@@ -748,6 +770,8 @@ class RedisGateway(AbstractRedisGateway):
|
|
|
748
770
|
claim_may_need_recovery = False
|
|
749
771
|
last_retryable_exception: Exception | None = None
|
|
750
772
|
while True:
|
|
773
|
+
if active_claim_id is None:
|
|
774
|
+
begin_active_claim(claim_id)
|
|
751
775
|
if self._is_interrupted(is_interrupted):
|
|
752
776
|
if claim_may_need_recovery:
|
|
753
777
|
pending_claim_id_to_share = claim_id
|
|
@@ -776,6 +800,7 @@ class RedisGateway(AbstractRedisGateway):
|
|
|
776
800
|
return claimed_message
|
|
777
801
|
claim_may_need_recovery = False
|
|
778
802
|
last_retryable_exception = None
|
|
803
|
+
finish_active_claim()
|
|
779
804
|
claim_id = uuid.uuid4().hex
|
|
780
805
|
|
|
781
806
|
remaining = deadline - loop.time()
|
|
@@ -816,6 +841,7 @@ class RedisGateway(AbstractRedisGateway):
|
|
|
816
841
|
finally:
|
|
817
842
|
if pending_claim_id_to_share is not None:
|
|
818
843
|
self._set_pending_claim_id(to_queue, pending_claim_id_to_share)
|
|
844
|
+
finish_active_claim()
|
|
819
845
|
|
|
820
846
|
async def wait_for_message_and_move(self, from_queue: str, to_queue: str) -> ClaimedMessage | MessageData | None:
|
|
821
847
|
if self._is_interrupted():
|
|
@@ -1107,6 +1133,19 @@ class RedisGateway(AbstractRedisGateway):
|
|
|
1107
1133
|
if claim_id not in pending_claim_ids:
|
|
1108
1134
|
pending_claim_ids.append(claim_id)
|
|
1109
1135
|
|
|
1136
|
+
def _begin_in_flight_claim_id(self, processing_queue: str, claim_id: str) -> None:
|
|
1137
|
+
with self._pending_claim_ids_lock:
|
|
1138
|
+
self._in_flight_claim_ids.setdefault(processing_queue, set()).add(claim_id)
|
|
1139
|
+
|
|
1140
|
+
def _finish_in_flight_claim_id(self, processing_queue: str, claim_id: str) -> None:
|
|
1141
|
+
with self._pending_claim_ids_lock:
|
|
1142
|
+
in_flight_claim_ids = self._in_flight_claim_ids.get(processing_queue)
|
|
1143
|
+
if in_flight_claim_ids is None:
|
|
1144
|
+
return
|
|
1145
|
+
in_flight_claim_ids.discard(claim_id)
|
|
1146
|
+
if not in_flight_claim_ids:
|
|
1147
|
+
self._in_flight_claim_ids.pop(processing_queue, None)
|
|
1148
|
+
|
|
1110
1149
|
def _finish_pending_claim_recovery(
|
|
1111
1150
|
self,
|
|
1112
1151
|
processing_queue: str,
|
|
@@ -1308,15 +1347,29 @@ class RedisGateway(AbstractRedisGateway):
|
|
|
1308
1347
|
with self._pending_claim_ids_lock:
|
|
1309
1348
|
pending = self._pending_claim_ids.get(processing_queue)
|
|
1310
1349
|
if not pending:
|
|
1311
|
-
|
|
1312
|
-
|
|
1313
|
-
|
|
1314
|
-
|
|
1315
|
-
|
|
1316
|
-
|
|
1317
|
-
|
|
1318
|
-
|
|
1319
|
-
|
|
1350
|
+
in_flight = self._in_flight_claim_ids.get(processing_queue)
|
|
1351
|
+
if not in_flight:
|
|
1352
|
+
return True
|
|
1353
|
+
claim_id = None
|
|
1354
|
+
else:
|
|
1355
|
+
recovering = self._recovering_claim_ids.setdefault(processing_queue, set())
|
|
1356
|
+
claim_id = next(
|
|
1357
|
+
(cid for cid in pending if cid not in recovering and cid not in skipped_unresolved),
|
|
1358
|
+
None,
|
|
1359
|
+
)
|
|
1360
|
+
if claim_id is None:
|
|
1361
|
+
break
|
|
1362
|
+
recovering.add(claim_id)
|
|
1363
|
+
if claim_id is None:
|
|
1364
|
+
if deadline_monotonic is None:
|
|
1365
|
+
await asyncio.sleep(_VISIBILITY_TIMEOUT_POLL_INTERVAL_SECONDS)
|
|
1366
|
+
else:
|
|
1367
|
+
remaining = deadline_monotonic - loop.time()
|
|
1368
|
+
if remaining <= 0:
|
|
1369
|
+
last_error = TimeoutError("drain pending-claim recovery deadline expired")
|
|
1370
|
+
break
|
|
1371
|
+
await asyncio.sleep(min(_VISIBILITY_TIMEOUT_POLL_INTERVAL_SECONDS, remaining))
|
|
1372
|
+
continue
|
|
1320
1373
|
clear = False
|
|
1321
1374
|
try:
|
|
1322
1375
|
try:
|
|
@@ -1365,6 +1418,7 @@ class RedisGateway(AbstractRedisGateway):
|
|
|
1365
1418
|
self._finish_pending_claim_recovery(processing_queue, claim_id, clear=clear)
|
|
1366
1419
|
with self._pending_claim_ids_lock:
|
|
1367
1420
|
pending = self._pending_claim_ids.get(processing_queue)
|
|
1368
|
-
|
|
1421
|
+
in_flight = self._in_flight_claim_ids.get(processing_queue)
|
|
1422
|
+
if pending or in_flight:
|
|
1369
1423
|
self._last_drain_error = last_error
|
|
1370
|
-
return not pending
|
|
1424
|
+
return not pending and not in_flight
|
|
@@ -488,18 +488,22 @@ class _LeaseHeartbeat:
|
|
|
488
488
|
if current_task is not None and current_task.cancelling() > 0:
|
|
489
489
|
raise
|
|
490
490
|
logger.exception("on_heartbeat_failure callback raised an exception")
|
|
491
|
-
warnings.
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
491
|
+
with warnings.catch_warnings():
|
|
492
|
+
warnings.simplefilter("always", RuntimeWarning)
|
|
493
|
+
warnings.warn(
|
|
494
|
+
f"on_heartbeat_failure callback raised {type(exc).__name__}",
|
|
495
|
+
RuntimeWarning,
|
|
496
|
+
stacklevel=1,
|
|
497
|
+
)
|
|
496
498
|
except Exception as exc:
|
|
497
499
|
logger.exception("on_heartbeat_failure callback raised an exception")
|
|
498
|
-
warnings.
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
500
|
+
with warnings.catch_warnings():
|
|
501
|
+
warnings.simplefilter("always", RuntimeWarning)
|
|
502
|
+
warnings.warn(
|
|
503
|
+
f"on_heartbeat_failure callback raised {type(exc).__name__}",
|
|
504
|
+
RuntimeWarning,
|
|
505
|
+
stacklevel=1,
|
|
506
|
+
)
|
|
503
507
|
|
|
504
508
|
async def _run(self) -> None:
|
|
505
509
|
try:
|
|
@@ -535,13 +539,15 @@ class _LeaseHeartbeat:
|
|
|
535
539
|
exception_type=type(exc).__name__,
|
|
536
540
|
error=exc,
|
|
537
541
|
)
|
|
538
|
-
warnings.
|
|
539
|
-
"
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
542
|
+
with warnings.catch_warnings():
|
|
543
|
+
warnings.simplefilter("always", RuntimeWarning)
|
|
544
|
+
warnings.warn(
|
|
545
|
+
"Failed to renew message lease "
|
|
546
|
+
f"({_warning_exception_name(exc)}); message will be reclaimed by another consumer "
|
|
547
|
+
"when the visibility timeout expires",
|
|
548
|
+
RuntimeWarning,
|
|
549
|
+
stacklevel=1,
|
|
550
|
+
)
|
|
545
551
|
await self._invoke_failure_callback()
|
|
546
552
|
return
|
|
547
553
|
if not renewed:
|
|
@@ -956,15 +962,22 @@ class RedisMessageQueue:
|
|
|
956
962
|
pending_claim_ids = getattr(self._redis, "_pending_claim_ids", None)
|
|
957
963
|
if not isinstance(pending_claim_ids, dict):
|
|
958
964
|
return None
|
|
965
|
+
in_flight_claim_ids = getattr(self._redis, "_in_flight_claim_ids", None)
|
|
959
966
|
|
|
960
967
|
lock = getattr(self._redis, "_pending_claim_ids_lock", None)
|
|
961
968
|
if lock is None:
|
|
962
969
|
pending = pending_claim_ids.get(self.key.processing)
|
|
963
|
-
|
|
970
|
+
in_flight = in_flight_claim_ids.get(self.key.processing) if isinstance(in_flight_claim_ids, dict) else None
|
|
971
|
+
pending_count = len(pending) if pending is not None else 0
|
|
972
|
+
in_flight_count = len(in_flight) if in_flight is not None else 0
|
|
973
|
+
return pending_count + in_flight_count
|
|
964
974
|
|
|
965
975
|
with lock:
|
|
966
976
|
pending = pending_claim_ids.get(self.key.processing)
|
|
967
|
-
|
|
977
|
+
in_flight = in_flight_claim_ids.get(self.key.processing) if isinstance(in_flight_claim_ids, dict) else None
|
|
978
|
+
pending_count = len(pending) if pending is not None else 0
|
|
979
|
+
in_flight_count = len(in_flight) if in_flight is not None else 0
|
|
980
|
+
return pending_count + in_flight_count
|
|
968
981
|
|
|
969
982
|
def _drain_failure_error(self, timeout_seconds: float | None, pending_claim_ids: int | None) -> BaseException:
|
|
970
983
|
last_drain_error = getattr(self._redis, "_last_drain_error", None)
|
|
@@ -1466,16 +1479,20 @@ class RedisMessageQueue:
|
|
|
1466
1479
|
async with self._aclose_lock:
|
|
1467
1480
|
cleanup_lease_counter = getattr(self._redis, "_cleanup_drained_lease_token_counter", None)
|
|
1468
1481
|
if self._aclose_result is not None:
|
|
1469
|
-
|
|
1470
|
-
|
|
1471
|
-
|
|
1472
|
-
|
|
1473
|
-
|
|
1474
|
-
|
|
1475
|
-
|
|
1476
|
-
|
|
1477
|
-
|
|
1478
|
-
|
|
1482
|
+
pending_claim_ids = self._pending_claim_ids_count()
|
|
1483
|
+
if pending_claim_ids:
|
|
1484
|
+
self._aclose_result = None
|
|
1485
|
+
else:
|
|
1486
|
+
if cleanup_lease_counter is not None:
|
|
1487
|
+
await _await_preserving_cancellation(cleanup_lease_counter(self.key.processing))
|
|
1488
|
+
await self._emit_event(
|
|
1489
|
+
"drain",
|
|
1490
|
+
"skipped",
|
|
1491
|
+
duration_ms=_duration_ms(started_at),
|
|
1492
|
+
timeout_seconds=timeout_seconds,
|
|
1493
|
+
pending_claim_ids=pending_claim_ids,
|
|
1494
|
+
)
|
|
1495
|
+
return self._aclose_result
|
|
1479
1496
|
|
|
1480
1497
|
async with self._publish_lock:
|
|
1481
1498
|
self._draining = True
|
{redis_message_queue-8.2.5 → redis_message_queue-8.2.6}/redis_message_queue/redis_message_queue.py
RENAMED
|
@@ -447,18 +447,22 @@ class _LeaseHeartbeat:
|
|
|
447
447
|
)
|
|
448
448
|
except asyncio.CancelledError as exc:
|
|
449
449
|
logger.exception("on_heartbeat_failure callback raised an exception")
|
|
450
|
-
warnings.
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
450
|
+
with warnings.catch_warnings():
|
|
451
|
+
warnings.simplefilter("always", RuntimeWarning)
|
|
452
|
+
warnings.warn(
|
|
453
|
+
f"on_heartbeat_failure callback raised {type(exc).__name__}",
|
|
454
|
+
RuntimeWarning,
|
|
455
|
+
stacklevel=1,
|
|
456
|
+
)
|
|
455
457
|
except Exception as exc:
|
|
456
458
|
logger.exception("on_heartbeat_failure callback raised an exception")
|
|
457
|
-
warnings.
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
459
|
+
with warnings.catch_warnings():
|
|
460
|
+
warnings.simplefilter("always", RuntimeWarning)
|
|
461
|
+
warnings.warn(
|
|
462
|
+
f"on_heartbeat_failure callback raised {type(exc).__name__}",
|
|
463
|
+
RuntimeWarning,
|
|
464
|
+
stacklevel=1,
|
|
465
|
+
)
|
|
462
466
|
|
|
463
467
|
def _run(self) -> None:
|
|
464
468
|
# No explicit _is_interrupted() check here. Heartbeat lifetime is owned
|
|
@@ -488,13 +492,15 @@ class _LeaseHeartbeat:
|
|
|
488
492
|
exception_type=type(exc).__name__,
|
|
489
493
|
error=exc,
|
|
490
494
|
)
|
|
491
|
-
warnings.
|
|
492
|
-
"
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
495
|
+
with warnings.catch_warnings():
|
|
496
|
+
warnings.simplefilter("always", RuntimeWarning)
|
|
497
|
+
warnings.warn(
|
|
498
|
+
"Failed to renew message lease "
|
|
499
|
+
f"({_warning_exception_name(exc)}); message will be reclaimed by another consumer "
|
|
500
|
+
"when the visibility timeout expires",
|
|
501
|
+
RuntimeWarning,
|
|
502
|
+
stacklevel=1,
|
|
503
|
+
)
|
|
498
504
|
self._invoke_failure_callback()
|
|
499
505
|
return
|
|
500
506
|
if not renewed:
|
|
@@ -914,15 +920,22 @@ class RedisMessageQueue:
|
|
|
914
920
|
pending_claim_ids = getattr(self._redis, "_pending_claim_ids", None)
|
|
915
921
|
if not isinstance(pending_claim_ids, dict):
|
|
916
922
|
return None
|
|
923
|
+
in_flight_claim_ids = getattr(self._redis, "_in_flight_claim_ids", None)
|
|
917
924
|
|
|
918
925
|
lock = getattr(self._redis, "_pending_claim_ids_lock", None)
|
|
919
926
|
if lock is None:
|
|
920
927
|
pending = pending_claim_ids.get(self.key.processing)
|
|
921
|
-
|
|
928
|
+
in_flight = in_flight_claim_ids.get(self.key.processing) if isinstance(in_flight_claim_ids, dict) else None
|
|
929
|
+
pending_count = len(pending) if pending is not None else 0
|
|
930
|
+
in_flight_count = len(in_flight) if in_flight is not None else 0
|
|
931
|
+
return pending_count + in_flight_count
|
|
922
932
|
|
|
923
933
|
with lock:
|
|
924
934
|
pending = pending_claim_ids.get(self.key.processing)
|
|
925
|
-
|
|
935
|
+
in_flight = in_flight_claim_ids.get(self.key.processing) if isinstance(in_flight_claim_ids, dict) else None
|
|
936
|
+
pending_count = len(pending) if pending is not None else 0
|
|
937
|
+
in_flight_count = len(in_flight) if in_flight is not None else 0
|
|
938
|
+
return pending_count + in_flight_count
|
|
926
939
|
|
|
927
940
|
def _drain_failure_error(self, timeout_seconds: float | None, pending_claim_ids: int | None) -> BaseException:
|
|
928
941
|
last_drain_error = getattr(self._redis, "_last_drain_error", None)
|
|
@@ -1426,16 +1439,20 @@ class RedisMessageQueue:
|
|
|
1426
1439
|
with self._drain_lock:
|
|
1427
1440
|
cleanup_lease_counter = getattr(self._redis, "_cleanup_drained_lease_token_counter", None)
|
|
1428
1441
|
if self._drain_result is True:
|
|
1429
|
-
|
|
1430
|
-
|
|
1431
|
-
|
|
1432
|
-
|
|
1433
|
-
|
|
1434
|
-
|
|
1435
|
-
|
|
1436
|
-
|
|
1437
|
-
|
|
1438
|
-
|
|
1442
|
+
pending_claim_ids = self._pending_claim_ids_count()
|
|
1443
|
+
if pending_claim_ids:
|
|
1444
|
+
self._drain_result = None
|
|
1445
|
+
else:
|
|
1446
|
+
if cleanup_lease_counter is not None:
|
|
1447
|
+
cleanup_lease_counter(self.key.processing)
|
|
1448
|
+
self._emit_event(
|
|
1449
|
+
"drain",
|
|
1450
|
+
"skipped",
|
|
1451
|
+
duration_ms=_duration_ms(started_at),
|
|
1452
|
+
timeout_seconds=timeout_seconds,
|
|
1453
|
+
pending_claim_ids=pending_claim_ids,
|
|
1454
|
+
)
|
|
1455
|
+
return True
|
|
1439
1456
|
|
|
1440
1457
|
with self._publish_lock:
|
|
1441
1458
|
self._draining = True
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{redis_message_queue-8.2.5 → redis_message_queue-8.2.6}/redis_message_queue/_callable_utils.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{redis_message_queue-8.2.5 → redis_message_queue-8.2.6}/redis_message_queue/_payload_limits.py
RENAMED
|
File without changes
|
{redis_message_queue-8.2.5 → redis_message_queue-8.2.6}/redis_message_queue/_queue_key_manager.py
RENAMED
|
File without changes
|
{redis_message_queue-8.2.5 → redis_message_queue-8.2.6}/redis_message_queue/_redis_cluster.py
RENAMED
|
File without changes
|
{redis_message_queue-8.2.5 → redis_message_queue-8.2.6}/redis_message_queue/_stored_message.py
RENAMED
|
File without changes
|
{redis_message_queue-8.2.5 → redis_message_queue-8.2.6}/redis_message_queue/asyncio/__init__.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|