redis-message-queue 8.2.2__tar.gz → 8.2.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {redis_message_queue-8.2.2 → redis_message_queue-8.2.4}/PKG-INFO +2 -2
- {redis_message_queue-8.2.2 → redis_message_queue-8.2.4}/README.md +1 -1
- {redis_message_queue-8.2.2 → redis_message_queue-8.2.4}/pyproject.toml +2 -2
- {redis_message_queue-8.2.2 → redis_message_queue-8.2.4}/redis_message_queue/_config.py +50 -2
- {redis_message_queue-8.2.2 → redis_message_queue-8.2.4}/redis_message_queue/_redis_gateway.py +91 -12
- {redis_message_queue-8.2.2 → redis_message_queue-8.2.4}/redis_message_queue/asyncio/_redis_gateway.py +91 -12
- {redis_message_queue-8.2.2 → redis_message_queue-8.2.4}/redis_message_queue/asyncio/redis_message_queue.py +25 -1
- {redis_message_queue-8.2.2 → redis_message_queue-8.2.4}/redis_message_queue/redis_message_queue.py +19 -1
- {redis_message_queue-8.2.2 → redis_message_queue-8.2.4}/.gitignore +0 -0
- {redis_message_queue-8.2.2 → redis_message_queue-8.2.4}/LICENSE +0 -0
- {redis_message_queue-8.2.2 → redis_message_queue-8.2.4}/redis_message_queue/__init__.py +0 -0
- {redis_message_queue-8.2.2 → redis_message_queue-8.2.4}/redis_message_queue/_abstract_redis_gateway.py +0 -0
- {redis_message_queue-8.2.2 → redis_message_queue-8.2.4}/redis_message_queue/_callable_utils.py +0 -0
- {redis_message_queue-8.2.2 → redis_message_queue-8.2.4}/redis_message_queue/_event.py +0 -0
- {redis_message_queue-8.2.2 → redis_message_queue-8.2.4}/redis_message_queue/_exceptions.py +0 -0
- {redis_message_queue-8.2.2 → redis_message_queue-8.2.4}/redis_message_queue/_payload_limits.py +0 -0
- {redis_message_queue-8.2.2 → redis_message_queue-8.2.4}/redis_message_queue/_queue_key_manager.py +0 -0
- {redis_message_queue-8.2.2 → redis_message_queue-8.2.4}/redis_message_queue/_redis_cluster.py +0 -0
- {redis_message_queue-8.2.2 → redis_message_queue-8.2.4}/redis_message_queue/_stored_message.py +0 -0
- {redis_message_queue-8.2.2 → redis_message_queue-8.2.4}/redis_message_queue/asyncio/__init__.py +0 -0
- {redis_message_queue-8.2.2 → redis_message_queue-8.2.4}/redis_message_queue/asyncio/_abstract_redis_gateway.py +0 -0
- {redis_message_queue-8.2.2 → redis_message_queue-8.2.4}/redis_message_queue/interrupt_handler/__init__.py +0 -0
- {redis_message_queue-8.2.2 → redis_message_queue-8.2.4}/redis_message_queue/interrupt_handler/_event_driven.py +0 -0
- {redis_message_queue-8.2.2 → redis_message_queue-8.2.4}/redis_message_queue/interrupt_handler/_implementation.py +0 -0
- {redis_message_queue-8.2.2 → redis_message_queue-8.2.4}/redis_message_queue/interrupt_handler/_interface.py +0 -0
- {redis_message_queue-8.2.2 → redis_message_queue-8.2.4}/redis_message_queue/py.typed +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: redis-message-queue
|
|
3
|
-
Version: 8.2.
|
|
3
|
+
Version: 8.2.4
|
|
4
4
|
Summary: Python message queuing with Redis and message deduplication
|
|
5
5
|
Project-URL: Homepage, https://github.com/Elijas/redis-message-queue
|
|
6
6
|
Project-URL: Repository, https://github.com/Elijas/redis-message-queue
|
|
@@ -34,7 +34,7 @@ Description-Content-Type: text/markdown
|
|
|
34
34
|
**Lightweight Python message queuing with Redis and built-in publish-side deduplication.** Deduplicate publishes within a TTL window, with optional crash recovery — across any number of producers and consumers.
|
|
35
35
|
|
|
36
36
|
```bash
|
|
37
|
-
pip install "redis-message-queue>=8.2.
|
|
37
|
+
pip install "redis-message-queue>=8.2.4,<9.0.0"
|
|
38
38
|
```
|
|
39
39
|
|
|
40
40
|
Requires Redis server >= 6.2.
|
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
**Lightweight Python message queuing with Redis and built-in publish-side deduplication.** Deduplicate publishes within a TTL window, with optional crash recovery — across any number of producers and consumers.
|
|
12
12
|
|
|
13
13
|
```bash
|
|
14
|
-
pip install "redis-message-queue>=8.2.
|
|
14
|
+
pip install "redis-message-queue>=8.2.4,<9.0.0"
|
|
15
15
|
```
|
|
16
16
|
|
|
17
17
|
Requires Redis server >= 6.2.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "redis-message-queue"
|
|
3
|
-
version = "8.2.
|
|
3
|
+
version = "8.2.4"
|
|
4
4
|
description = "Python message queuing with Redis and message deduplication"
|
|
5
5
|
authors = [{ name = "Elijas", email = "4084885+Elijas@users.noreply.github.com" }]
|
|
6
6
|
readme = "README.md"
|
|
@@ -48,7 +48,7 @@ default-groups = ["dev", "test"]
|
|
|
48
48
|
##############################
|
|
49
49
|
|
|
50
50
|
[tool.bumpversion]
|
|
51
|
-
current_version = "8.2.
|
|
51
|
+
current_version = "8.2.4"
|
|
52
52
|
parse = "(?P<major>\\d+)\\.(?P<minor>\\d+)\\.(?P<patch>\\d+)"
|
|
53
53
|
serialize = ["{major}.{minor}.{patch}"]
|
|
54
54
|
search = "{current_version}"
|
|
@@ -585,6 +585,54 @@ return removed
|
|
|
585
585
|
"""
|
|
586
586
|
)
|
|
587
587
|
|
|
588
|
+
RETURN_MESSAGE_TO_PENDING_LUA_SCRIPT = (
|
|
589
|
+
_LUA_KEY_TYPE_GUARD
|
|
590
|
+
+ """
|
|
591
|
+
local err = redis_message_queue_require_type(KEYS[1], 'list')
|
|
592
|
+
if err then
|
|
593
|
+
return err
|
|
594
|
+
end
|
|
595
|
+
|
|
596
|
+
local err = redis_message_queue_require_type(KEYS[2], 'list')
|
|
597
|
+
if err then
|
|
598
|
+
return err
|
|
599
|
+
end
|
|
600
|
+
|
|
601
|
+
local err = redis_message_queue_require_type(KEYS[3], 'hash')
|
|
602
|
+
if err then
|
|
603
|
+
return err
|
|
604
|
+
end
|
|
605
|
+
|
|
606
|
+
local err = redis_message_queue_require_type(KEYS[4], 'hash')
|
|
607
|
+
if err then
|
|
608
|
+
return err
|
|
609
|
+
end
|
|
610
|
+
|
|
611
|
+
local err = redis_message_queue_require_type(KEYS[5], 'string')
|
|
612
|
+
if err then
|
|
613
|
+
return err
|
|
614
|
+
end
|
|
615
|
+
|
|
616
|
+
local cached_result = redis.call('GET', KEYS[5])
|
|
617
|
+
if cached_result then
|
|
618
|
+
return tonumber(cached_result)
|
|
619
|
+
end
|
|
620
|
+
|
|
621
|
+
redis.call('RPUSH', KEYS[2], ARGV[1])
|
|
622
|
+
local removed = redis.call('LREM', KEYS[1], 1, ARGV[1])
|
|
623
|
+
if removed == 1 then
|
|
624
|
+
redis.call('HDEL', KEYS[3], ARGV[2])
|
|
625
|
+
redis.call('HDEL', KEYS[4], ARGV[1])
|
|
626
|
+
redis.call('DEL', KEYS[6])
|
|
627
|
+
else
|
|
628
|
+
redis.call('LREM', KEYS[2], 1, ARGV[1])
|
|
629
|
+
end
|
|
630
|
+
|
|
631
|
+
redis.call('SET', KEYS[5], tostring(removed), 'PX', tonumber(ARGV[3]))
|
|
632
|
+
return removed
|
|
633
|
+
"""
|
|
634
|
+
)
|
|
635
|
+
|
|
588
636
|
REMOVE_MESSAGE_LUA_SCRIPT = (
|
|
589
637
|
_LUA_KEY_TYPE_GUARD
|
|
590
638
|
+ """
|
|
@@ -876,8 +924,6 @@ while claim_attempts < 100 do
|
|
|
876
924
|
|
|
877
925
|
local count = redis.call('HINCRBY', KEYS[6], stored, 1)
|
|
878
926
|
if max_delivery_count > 0 and count > max_delivery_count then
|
|
879
|
-
redis.call('LREM', KEYS[2], 1, stored)
|
|
880
|
-
redis.call('HDEL', KEYS[6], stored)
|
|
881
927
|
-- Strip envelope to store raw payload in DLQ, consistent with completed/failed queues.
|
|
882
928
|
-- The per-delivery UUID in the envelope is lost; see README dead-letter notes.
|
|
883
929
|
local dead_letter_value = stored
|
|
@@ -886,6 +932,8 @@ while claim_attempts < 100 do
|
|
|
886
932
|
dead_letter_value = envelope['payload']
|
|
887
933
|
end
|
|
888
934
|
redis.call('LPUSH', KEYS[7], dead_letter_value)
|
|
935
|
+
redis.call('LREM', KEYS[2], 1, stored)
|
|
936
|
+
redis.call('HDEL', KEYS[6], stored)
|
|
889
937
|
table.insert(dead_lettered_events, {redis_message_queue_message_id(stored), tostring(count)})
|
|
890
938
|
else
|
|
891
939
|
return store_claim_and_return(stored)
|
{redis_message_queue-8.2.2 → redis_message_queue-8.2.4}/redis_message_queue/_redis_gateway.py
RENAMED
|
@@ -31,6 +31,7 @@ from redis_message_queue._config import (
|
|
|
31
31
|
REMOVE_MESSAGE_LUA_SCRIPT,
|
|
32
32
|
REMOVE_MESSAGE_WITH_LEASE_TOKEN_LUA_SCRIPT,
|
|
33
33
|
RENEW_MESSAGE_LEASE_LUA_SCRIPT,
|
|
34
|
+
RETURN_MESSAGE_TO_PENDING_LUA_SCRIPT,
|
|
34
35
|
_ChainedInterrupt,
|
|
35
36
|
build_retry_strategy,
|
|
36
37
|
is_redis_retryable_exception,
|
|
@@ -79,6 +80,8 @@ _VISIBILITY_TIMEOUT_POLL_INTERVAL_SECONDS = 0.25
|
|
|
79
80
|
_PENDING_OVERLOAD_INITIAL_BACKOFF_SECONDS = 0.010
|
|
80
81
|
_PENDING_OVERLOAD_MAX_BACKOFF_SECONDS = 0.500
|
|
81
82
|
_CLAIM_STORE_FAILED_LUA_SENTINEL_BYTES = CLAIM_STORE_FAILED_LUA_SENTINEL.encode("utf-8")
|
|
83
|
+
_PENDING_QUEUE_SUFFIX = "pending"
|
|
84
|
+
_PROCESSING_QUEUE_SUFFIX = "processing"
|
|
82
85
|
|
|
83
86
|
|
|
84
87
|
class _DrainDeadlineExceeded(Exception):
|
|
@@ -703,6 +706,8 @@ class RedisGateway(AbstractRedisGateway):
|
|
|
703
706
|
clear=clear_pending_claim_id,
|
|
704
707
|
)
|
|
705
708
|
if recovered_claim is not None:
|
|
709
|
+
if self._message_visibility_timeout_seconds is None:
|
|
710
|
+
self._delete_claim_result_key(self._claim_result_key(to_queue, pending_claim_id))
|
|
706
711
|
self._emit_event("claim_reclaim", "success", claim_id=pending_claim_id)
|
|
707
712
|
return recovered_claim
|
|
708
713
|
|
|
@@ -810,6 +815,8 @@ class RedisGateway(AbstractRedisGateway):
|
|
|
810
815
|
pending_claim_id_to_share = claim_id
|
|
811
816
|
raise
|
|
812
817
|
if recovered_claim is not None:
|
|
818
|
+
if self._message_visibility_timeout_seconds is None:
|
|
819
|
+
self._delete_claim_result_key(self._claim_result_key(to_queue, claim_id))
|
|
813
820
|
self._emit_event("claim_reclaim", "success", claim_id=claim_id)
|
|
814
821
|
return recovered_claim
|
|
815
822
|
self._emit_event(
|
|
@@ -1168,7 +1175,6 @@ class RedisGateway(AbstractRedisGateway):
|
|
|
1168
1175
|
_raise_if_drain_deadline_expired(deadline_monotonic)
|
|
1169
1176
|
if cached_claim is None:
|
|
1170
1177
|
return None
|
|
1171
|
-
self._delete_claim_result_key(claim_result_key, deadline_monotonic=deadline_monotonic)
|
|
1172
1178
|
_raise_if_drain_deadline_expired(deadline_monotonic)
|
|
1173
1179
|
return cached_claim
|
|
1174
1180
|
|
|
@@ -1225,6 +1231,54 @@ class RedisGateway(AbstractRedisGateway):
|
|
|
1225
1231
|
_raise_if_drain_deadline_expired(deadline_monotonic)
|
|
1226
1232
|
return ClaimedMessage(stored_message=stored_message, lease_token=lease_token)
|
|
1227
1233
|
|
|
1234
|
+
def _pending_queue_from_processing_queue(self, processing_queue: str) -> str:
|
|
1235
|
+
if not processing_queue.endswith(_PROCESSING_QUEUE_SUFFIX):
|
|
1236
|
+
raise RuntimeError(f"cannot derive pending queue key from processing queue {processing_queue!r}")
|
|
1237
|
+
return f"{processing_queue.removesuffix(_PROCESSING_QUEUE_SUFFIX)}{_PENDING_QUEUE_SUFFIX}"
|
|
1238
|
+
|
|
1239
|
+
def _return_recovered_non_visibility_timeout_claim_to_pending(
|
|
1240
|
+
self,
|
|
1241
|
+
processing_queue: str,
|
|
1242
|
+
stored_message: MessageData,
|
|
1243
|
+
claim_id: str,
|
|
1244
|
+
*,
|
|
1245
|
+
deadline_monotonic: float | None,
|
|
1246
|
+
) -> bool:
|
|
1247
|
+
pending_queue = self._pending_queue_from_processing_queue(processing_queue)
|
|
1248
|
+
operation_id = uuid.uuid4().hex
|
|
1249
|
+
operation_result_key = self._operation_result_key(processing_queue, operation_id)
|
|
1250
|
+
|
|
1251
|
+
try:
|
|
1252
|
+
_raise_if_drain_deadline_expired(deadline_monotonic)
|
|
1253
|
+
result = _call_with_drain_deadline(
|
|
1254
|
+
lambda: self._eval(
|
|
1255
|
+
RETURN_MESSAGE_TO_PENDING_LUA_SCRIPT,
|
|
1256
|
+
6,
|
|
1257
|
+
processing_queue,
|
|
1258
|
+
pending_queue,
|
|
1259
|
+
self._claim_result_ids_key(processing_queue),
|
|
1260
|
+
self._claim_result_backrefs_key(processing_queue),
|
|
1261
|
+
operation_result_key,
|
|
1262
|
+
self._claim_result_key(processing_queue, claim_id),
|
|
1263
|
+
stored_message,
|
|
1264
|
+
claim_id,
|
|
1265
|
+
self._operation_result_ttl_ms(),
|
|
1266
|
+
),
|
|
1267
|
+
deadline_monotonic=deadline_monotonic,
|
|
1268
|
+
)
|
|
1269
|
+
_raise_if_drain_deadline_expired(deadline_monotonic)
|
|
1270
|
+
return bool(_coerce_lua_count(result))
|
|
1271
|
+
except RedisMessageQueueError as exc:
|
|
1272
|
+
_set_exception_context(
|
|
1273
|
+
exc,
|
|
1274
|
+
queue=processing_queue,
|
|
1275
|
+
message_id=extract_stored_message_id(stored_message),
|
|
1276
|
+
operation="drain",
|
|
1277
|
+
)
|
|
1278
|
+
raise
|
|
1279
|
+
finally:
|
|
1280
|
+
self._delete_operation_result_key(operation_result_key)
|
|
1281
|
+
|
|
1228
1282
|
def _is_interrupted(self, is_interrupted: BaseGracefulInterruptHandler | None = None) -> bool:
|
|
1229
1283
|
return (self._interrupt is not None and self._interrupt.is_interrupted()) or (
|
|
1230
1284
|
is_interrupted is not None and is_interrupted.is_interrupted()
|
|
@@ -1241,8 +1295,10 @@ class RedisGateway(AbstractRedisGateway):
|
|
|
1241
1295
|
Walks the same recovery path as ``_wait_for_claim`` but without
|
|
1242
1296
|
gating on the interrupt flag, so a soft shutdown can flush
|
|
1243
1297
|
ambiguous-claim state that would otherwise be dropped on process
|
|
1244
|
-
exit (AA-05-F2).
|
|
1245
|
-
the
|
|
1298
|
+
exit (AA-05-F2). No-visibility-timeout recoveries are returned to
|
|
1299
|
+
the pending queue before their claim ids are cleared. Returns True
|
|
1300
|
+
if no pending ids remain; False if the deadline fired or Redis
|
|
1301
|
+
errors prevented full drain.
|
|
1246
1302
|
"""
|
|
1247
1303
|
with self._drain_pending_claim_ids_lock:
|
|
1248
1304
|
self._last_drain_error = None
|
|
@@ -1258,11 +1314,8 @@ class RedisGateway(AbstractRedisGateway):
|
|
|
1258
1314
|
deadline_monotonic: float | None,
|
|
1259
1315
|
) -> bool:
|
|
1260
1316
|
"""Recover every in-memory pending claim id for ``processing_queue``."""
|
|
1261
|
-
|
|
1262
|
-
|
|
1263
|
-
else:
|
|
1264
|
-
recover = self._recover_pending_non_visibility_timeout_claim
|
|
1265
|
-
skipped_transient: set[str] = set()
|
|
1317
|
+
has_visibility_timeout = self._message_visibility_timeout_seconds is not None
|
|
1318
|
+
skipped_unresolved: set[str] = set()
|
|
1266
1319
|
last_error: BaseException | None = None
|
|
1267
1320
|
while True:
|
|
1268
1321
|
# ``>=`` (not ``>``) makes ``timeout=0`` deterministically take
|
|
@@ -1278,7 +1331,7 @@ class RedisGateway(AbstractRedisGateway):
|
|
|
1278
1331
|
return True
|
|
1279
1332
|
recovering = self._recovering_claim_ids.setdefault(processing_queue, set())
|
|
1280
1333
|
claim_id = next(
|
|
1281
|
-
(cid for cid in pending if cid not in recovering and cid not in
|
|
1334
|
+
(cid for cid in pending if cid not in recovering and cid not in skipped_unresolved),
|
|
1282
1335
|
None,
|
|
1283
1336
|
)
|
|
1284
1337
|
if claim_id is None:
|
|
@@ -1287,8 +1340,34 @@ class RedisGateway(AbstractRedisGateway):
|
|
|
1287
1340
|
clear = False
|
|
1288
1341
|
try:
|
|
1289
1342
|
try:
|
|
1290
|
-
|
|
1291
|
-
|
|
1343
|
+
if has_visibility_timeout:
|
|
1344
|
+
self._recover_pending_visibility_timeout_claim(
|
|
1345
|
+
processing_queue,
|
|
1346
|
+
claim_id,
|
|
1347
|
+
deadline_monotonic=deadline_monotonic,
|
|
1348
|
+
)
|
|
1349
|
+
clear = True
|
|
1350
|
+
continue
|
|
1351
|
+
|
|
1352
|
+
recovered_claim = self._recover_pending_non_visibility_timeout_claim(
|
|
1353
|
+
processing_queue,
|
|
1354
|
+
claim_id,
|
|
1355
|
+
deadline_monotonic=deadline_monotonic,
|
|
1356
|
+
)
|
|
1357
|
+
if recovered_claim is None:
|
|
1358
|
+
clear = True
|
|
1359
|
+
elif self._return_recovered_non_visibility_timeout_claim_to_pending(
|
|
1360
|
+
processing_queue,
|
|
1361
|
+
recovered_claim,
|
|
1362
|
+
claim_id,
|
|
1363
|
+
deadline_monotonic=deadline_monotonic,
|
|
1364
|
+
):
|
|
1365
|
+
clear = True
|
|
1366
|
+
else:
|
|
1367
|
+
last_error = RuntimeError(
|
|
1368
|
+
f"drain recovered claim {claim_id!r} but message was not present in processing queue"
|
|
1369
|
+
)
|
|
1370
|
+
skipped_unresolved.add(claim_id)
|
|
1292
1371
|
except _DrainDeadlineExceeded:
|
|
1293
1372
|
last_error = TimeoutError("drain pending-claim recovery deadline expired")
|
|
1294
1373
|
break
|
|
@@ -1301,7 +1380,7 @@ class RedisGateway(AbstractRedisGateway):
|
|
|
1301
1380
|
claim_id,
|
|
1302
1381
|
type(exc).__name__,
|
|
1303
1382
|
)
|
|
1304
|
-
|
|
1383
|
+
skipped_unresolved.add(claim_id)
|
|
1305
1384
|
finally:
|
|
1306
1385
|
self._finish_pending_claim_recovery(processing_queue, claim_id, clear=clear)
|
|
1307
1386
|
with self._pending_claim_ids_lock:
|
|
@@ -29,6 +29,7 @@ from redis_message_queue._config import (
|
|
|
29
29
|
REMOVE_MESSAGE_LUA_SCRIPT,
|
|
30
30
|
REMOVE_MESSAGE_WITH_LEASE_TOKEN_LUA_SCRIPT,
|
|
31
31
|
RENEW_MESSAGE_LEASE_LUA_SCRIPT,
|
|
32
|
+
RETURN_MESSAGE_TO_PENDING_LUA_SCRIPT,
|
|
32
33
|
_ChainedInterrupt,
|
|
33
34
|
build_retry_strategy,
|
|
34
35
|
is_redis_retryable_exception,
|
|
@@ -78,6 +79,8 @@ _VISIBILITY_TIMEOUT_POLL_INTERVAL_SECONDS = 0.25
|
|
|
78
79
|
_PENDING_OVERLOAD_INITIAL_BACKOFF_SECONDS = 0.010
|
|
79
80
|
_PENDING_OVERLOAD_MAX_BACKOFF_SECONDS = 0.500
|
|
80
81
|
_CLAIM_STORE_FAILED_LUA_SENTINEL_BYTES = CLAIM_STORE_FAILED_LUA_SENTINEL.encode("utf-8")
|
|
82
|
+
_PENDING_QUEUE_SUFFIX = "pending"
|
|
83
|
+
_PROCESSING_QUEUE_SUFFIX = "processing"
|
|
81
84
|
|
|
82
85
|
|
|
83
86
|
class _DrainDeadlineExceeded(Exception):
|
|
@@ -682,6 +685,8 @@ class RedisGateway(AbstractRedisGateway):
|
|
|
682
685
|
clear=clear_pending_claim_id,
|
|
683
686
|
)
|
|
684
687
|
if recovered_claim is not None:
|
|
688
|
+
if self._message_visibility_timeout_seconds is None:
|
|
689
|
+
await self._delete_claim_result_key(self._claim_result_key(to_queue, pending_claim_id))
|
|
685
690
|
await self._emit_event("claim_reclaim", "success", claim_id=pending_claim_id)
|
|
686
691
|
return recovered_claim
|
|
687
692
|
|
|
@@ -790,6 +795,8 @@ class RedisGateway(AbstractRedisGateway):
|
|
|
790
795
|
pending_claim_id_to_share = claim_id
|
|
791
796
|
raise
|
|
792
797
|
if recovered_claim is not None:
|
|
798
|
+
if self._message_visibility_timeout_seconds is None:
|
|
799
|
+
await self._delete_claim_result_key(self._claim_result_key(to_queue, claim_id))
|
|
793
800
|
await self._emit_event("claim_reclaim", "success", claim_id=claim_id)
|
|
794
801
|
return recovered_claim
|
|
795
802
|
await self._emit_event(
|
|
@@ -1148,7 +1155,6 @@ class RedisGateway(AbstractRedisGateway):
|
|
|
1148
1155
|
_raise_if_drain_deadline_expired(deadline_monotonic)
|
|
1149
1156
|
if cached_claim is None:
|
|
1150
1157
|
return None
|
|
1151
|
-
await self._delete_claim_result_key(claim_result_key, deadline_monotonic=deadline_monotonic)
|
|
1152
1158
|
_raise_if_drain_deadline_expired(deadline_monotonic)
|
|
1153
1159
|
return cached_claim
|
|
1154
1160
|
|
|
@@ -1206,6 +1212,54 @@ class RedisGateway(AbstractRedisGateway):
|
|
|
1206
1212
|
_raise_if_drain_deadline_expired(deadline_monotonic)
|
|
1207
1213
|
return ClaimedMessage(stored_message=stored_message, lease_token=lease_token)
|
|
1208
1214
|
|
|
1215
|
+
def _pending_queue_from_processing_queue(self, processing_queue: str) -> str:
|
|
1216
|
+
if not processing_queue.endswith(_PROCESSING_QUEUE_SUFFIX):
|
|
1217
|
+
raise RuntimeError(f"cannot derive pending queue key from processing queue {processing_queue!r}")
|
|
1218
|
+
return f"{processing_queue.removesuffix(_PROCESSING_QUEUE_SUFFIX)}{_PENDING_QUEUE_SUFFIX}"
|
|
1219
|
+
|
|
1220
|
+
async def _return_recovered_non_visibility_timeout_claim_to_pending(
|
|
1221
|
+
self,
|
|
1222
|
+
processing_queue: str,
|
|
1223
|
+
stored_message: MessageData,
|
|
1224
|
+
claim_id: str,
|
|
1225
|
+
*,
|
|
1226
|
+
deadline_monotonic: float | None,
|
|
1227
|
+
) -> bool:
|
|
1228
|
+
pending_queue = self._pending_queue_from_processing_queue(processing_queue)
|
|
1229
|
+
operation_id = uuid.uuid4().hex
|
|
1230
|
+
operation_result_key = self._operation_result_key(processing_queue, operation_id)
|
|
1231
|
+
|
|
1232
|
+
try:
|
|
1233
|
+
_raise_if_drain_deadline_expired(deadline_monotonic)
|
|
1234
|
+
result = await _call_with_drain_deadline(
|
|
1235
|
+
lambda: self._eval(
|
|
1236
|
+
RETURN_MESSAGE_TO_PENDING_LUA_SCRIPT,
|
|
1237
|
+
6,
|
|
1238
|
+
processing_queue,
|
|
1239
|
+
pending_queue,
|
|
1240
|
+
self._claim_result_ids_key(processing_queue),
|
|
1241
|
+
self._claim_result_backrefs_key(processing_queue),
|
|
1242
|
+
operation_result_key,
|
|
1243
|
+
self._claim_result_key(processing_queue, claim_id),
|
|
1244
|
+
stored_message,
|
|
1245
|
+
claim_id,
|
|
1246
|
+
self._operation_result_ttl_ms(),
|
|
1247
|
+
),
|
|
1248
|
+
deadline_monotonic=deadline_monotonic,
|
|
1249
|
+
)
|
|
1250
|
+
_raise_if_drain_deadline_expired(deadline_monotonic)
|
|
1251
|
+
return bool(_coerce_lua_count(result))
|
|
1252
|
+
except RedisMessageQueueError as exc:
|
|
1253
|
+
_set_exception_context(
|
|
1254
|
+
exc,
|
|
1255
|
+
queue=processing_queue,
|
|
1256
|
+
message_id=extract_stored_message_id(stored_message),
|
|
1257
|
+
operation="drain",
|
|
1258
|
+
)
|
|
1259
|
+
raise
|
|
1260
|
+
finally:
|
|
1261
|
+
await self._delete_operation_result_key(operation_result_key)
|
|
1262
|
+
|
|
1209
1263
|
def _is_interrupted(self, is_interrupted: BaseGracefulInterruptHandler | None = None) -> bool:
|
|
1210
1264
|
return (self._interrupt is not None and self._interrupt.is_interrupted()) or (
|
|
1211
1265
|
is_interrupted is not None and is_interrupted.is_interrupted()
|
|
@@ -1222,8 +1276,10 @@ class RedisGateway(AbstractRedisGateway):
|
|
|
1222
1276
|
Kept separate from the sync implementation per AF11 (sync/async
|
|
1223
1277
|
gateways stay duplicated). Walks the same recovery path as
|
|
1224
1278
|
``_wait_for_claim`` but without gating on the interrupt flag so a
|
|
1225
|
-
soft shutdown can flush ambiguous-claim state.
|
|
1226
|
-
|
|
1279
|
+
soft shutdown can flush ambiguous-claim state. No-visibility-timeout
|
|
1280
|
+
recoveries are returned to the pending queue before their claim ids
|
|
1281
|
+
are cleared. Returns True if no pending ids remain; False on deadline
|
|
1282
|
+
expiry or Redis errors.
|
|
1227
1283
|
"""
|
|
1228
1284
|
async with self._drain_pending_claim_ids_lock:
|
|
1229
1285
|
self._last_drain_error = None
|
|
@@ -1239,11 +1295,8 @@ class RedisGateway(AbstractRedisGateway):
|
|
|
1239
1295
|
deadline_monotonic: float | None,
|
|
1240
1296
|
) -> bool:
|
|
1241
1297
|
"""Recover every in-memory pending claim id for ``processing_queue``."""
|
|
1242
|
-
|
|
1243
|
-
|
|
1244
|
-
else:
|
|
1245
|
-
recover = self._recover_pending_non_visibility_timeout_claim
|
|
1246
|
-
skipped_transient: set[str] = set()
|
|
1298
|
+
has_visibility_timeout = self._message_visibility_timeout_seconds is not None
|
|
1299
|
+
skipped_unresolved: set[str] = set()
|
|
1247
1300
|
loop = asyncio.get_running_loop()
|
|
1248
1301
|
last_error: BaseException | None = None
|
|
1249
1302
|
while True:
|
|
@@ -1258,7 +1311,7 @@ class RedisGateway(AbstractRedisGateway):
|
|
|
1258
1311
|
return True
|
|
1259
1312
|
recovering = self._recovering_claim_ids.setdefault(processing_queue, set())
|
|
1260
1313
|
claim_id = next(
|
|
1261
|
-
(cid for cid in pending if cid not in recovering and cid not in
|
|
1314
|
+
(cid for cid in pending if cid not in recovering and cid not in skipped_unresolved),
|
|
1262
1315
|
None,
|
|
1263
1316
|
)
|
|
1264
1317
|
if claim_id is None:
|
|
@@ -1267,8 +1320,34 @@ class RedisGateway(AbstractRedisGateway):
|
|
|
1267
1320
|
clear = False
|
|
1268
1321
|
try:
|
|
1269
1322
|
try:
|
|
1270
|
-
|
|
1271
|
-
|
|
1323
|
+
if has_visibility_timeout:
|
|
1324
|
+
await self._recover_pending_visibility_timeout_claim(
|
|
1325
|
+
processing_queue,
|
|
1326
|
+
claim_id,
|
|
1327
|
+
deadline_monotonic=deadline_monotonic,
|
|
1328
|
+
)
|
|
1329
|
+
clear = True
|
|
1330
|
+
continue
|
|
1331
|
+
|
|
1332
|
+
recovered_claim = await self._recover_pending_non_visibility_timeout_claim(
|
|
1333
|
+
processing_queue,
|
|
1334
|
+
claim_id,
|
|
1335
|
+
deadline_monotonic=deadline_monotonic,
|
|
1336
|
+
)
|
|
1337
|
+
if recovered_claim is None:
|
|
1338
|
+
clear = True
|
|
1339
|
+
elif await self._return_recovered_non_visibility_timeout_claim_to_pending(
|
|
1340
|
+
processing_queue,
|
|
1341
|
+
recovered_claim,
|
|
1342
|
+
claim_id,
|
|
1343
|
+
deadline_monotonic=deadline_monotonic,
|
|
1344
|
+
):
|
|
1345
|
+
clear = True
|
|
1346
|
+
else:
|
|
1347
|
+
last_error = RuntimeError(
|
|
1348
|
+
f"drain recovered claim {claim_id!r} but message was not present in processing queue"
|
|
1349
|
+
)
|
|
1350
|
+
skipped_unresolved.add(claim_id)
|
|
1272
1351
|
except _DrainDeadlineExceeded:
|
|
1273
1352
|
last_error = TimeoutError("drain pending-claim recovery deadline expired")
|
|
1274
1353
|
break
|
|
@@ -1281,7 +1360,7 @@ class RedisGateway(AbstractRedisGateway):
|
|
|
1281
1360
|
claim_id,
|
|
1282
1361
|
type(exc).__name__,
|
|
1283
1362
|
)
|
|
1284
|
-
|
|
1363
|
+
skipped_unresolved.add(claim_id)
|
|
1285
1364
|
finally:
|
|
1286
1365
|
self._finish_pending_claim_recovery(processing_queue, claim_id, clear=clear)
|
|
1287
1366
|
with self._pending_claim_ids_lock:
|
|
@@ -483,6 +483,16 @@ class _LeaseHeartbeat:
|
|
|
483
483
|
result = self._on_heartbeat_failure()
|
|
484
484
|
if inspect.isawaitable(result):
|
|
485
485
|
await result
|
|
486
|
+
except asyncio.CancelledError as exc:
|
|
487
|
+
current_task = asyncio.current_task()
|
|
488
|
+
if current_task is not None and current_task.cancelling() > 0:
|
|
489
|
+
raise
|
|
490
|
+
logger.exception("on_heartbeat_failure callback raised an exception")
|
|
491
|
+
warnings.warn(
|
|
492
|
+
f"on_heartbeat_failure callback raised {type(exc).__name__}",
|
|
493
|
+
RuntimeWarning,
|
|
494
|
+
stacklevel=1,
|
|
495
|
+
)
|
|
486
496
|
except Exception as exc:
|
|
487
497
|
logger.exception("on_heartbeat_failure callback raised an exception")
|
|
488
498
|
warnings.warn(
|
|
@@ -920,6 +930,18 @@ class RedisMessageQueue:
|
|
|
920
930
|
"Use an async callable or return an awaitable."
|
|
921
931
|
)
|
|
922
932
|
await result
|
|
933
|
+
except asyncio.CancelledError as exc:
|
|
934
|
+
current_task = asyncio.current_task()
|
|
935
|
+
if current_task is not None and current_task.cancelling() > 0:
|
|
936
|
+
raise
|
|
937
|
+
logger.exception("on_event callback raised an exception")
|
|
938
|
+
with warnings.catch_warnings():
|
|
939
|
+
warnings.simplefilter("always", RuntimeWarning)
|
|
940
|
+
warnings.warn(
|
|
941
|
+
f"on_event callback raised {type(exc).__name__}",
|
|
942
|
+
RuntimeWarning,
|
|
943
|
+
stacklevel=2,
|
|
944
|
+
)
|
|
923
945
|
except Exception as exc:
|
|
924
946
|
logger.exception("on_event callback raised an exception")
|
|
925
947
|
with warnings.catch_warnings():
|
|
@@ -1414,7 +1436,9 @@ class RedisMessageQueue:
|
|
|
1414
1436
|
calls raise ``QueueDrainedError``. It then awaits the gateway's
|
|
1415
1437
|
pending-claim-id recovery loop. Returns ``True`` if all pending claim
|
|
1416
1438
|
ids were recovered, ``False`` if the deadline fired or a transient
|
|
1417
|
-
Redis error left ids pending.
|
|
1439
|
+
Redis error left ids pending. In no-visibility-timeout queues,
|
|
1440
|
+
recovered messages are returned to pending before the claim id is
|
|
1441
|
+
cleared.
|
|
1418
1442
|
|
|
1419
1443
|
Unlike ``asyncio.CancelledError`` (hard-abort, leaves messages
|
|
1420
1444
|
claimed for VT-reclaim), ``aclose()`` is the explicit-drain
|
{redis_message_queue-8.2.2 → redis_message_queue-8.2.4}/redis_message_queue/redis_message_queue.py
RENAMED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import asyncio
|
|
1
2
|
import hashlib
|
|
2
3
|
import inspect
|
|
3
4
|
import logging
|
|
@@ -444,6 +445,13 @@ class _LeaseHeartbeat:
|
|
|
444
445
|
"'on_heartbeat_failure' returned an awaitable; "
|
|
445
446
|
"use the async RedisMessageQueue from redis_message_queue.asyncio instead"
|
|
446
447
|
)
|
|
448
|
+
except asyncio.CancelledError as exc:
|
|
449
|
+
logger.exception("on_heartbeat_failure callback raised an exception")
|
|
450
|
+
warnings.warn(
|
|
451
|
+
f"on_heartbeat_failure callback raised {type(exc).__name__}",
|
|
452
|
+
RuntimeWarning,
|
|
453
|
+
stacklevel=1,
|
|
454
|
+
)
|
|
447
455
|
except Exception as exc:
|
|
448
456
|
logger.exception("on_heartbeat_failure callback raised an exception")
|
|
449
457
|
warnings.warn(
|
|
@@ -883,6 +891,15 @@ class RedisMessageQueue:
|
|
|
883
891
|
"'on_event' returned an awaitable; use the async RedisMessageQueue "
|
|
884
892
|
"from redis_message_queue.asyncio instead"
|
|
885
893
|
)
|
|
894
|
+
except asyncio.CancelledError as exc:
|
|
895
|
+
logger.exception("on_event callback raised an exception")
|
|
896
|
+
with warnings.catch_warnings():
|
|
897
|
+
warnings.simplefilter("always", RuntimeWarning)
|
|
898
|
+
warnings.warn(
|
|
899
|
+
f"on_event callback raised {type(exc).__name__}",
|
|
900
|
+
RuntimeWarning,
|
|
901
|
+
stacklevel=2,
|
|
902
|
+
)
|
|
886
903
|
except Exception as exc:
|
|
887
904
|
logger.exception("on_event callback raised an exception")
|
|
888
905
|
with warnings.catch_warnings():
|
|
@@ -1374,7 +1391,8 @@ class RedisMessageQueue:
|
|
|
1374
1391
|
calls raise ``QueueDrainedError``. It then walks the gateway's
|
|
1375
1392
|
in-memory ``_pending_claim_ids`` to recover any ambiguous claims that
|
|
1376
1393
|
an interrupt-aware shutdown would otherwise drop on process exit
|
|
1377
|
-
(AA-05-F2).
|
|
1394
|
+
(AA-05-F2). In no-visibility-timeout queues, recovered messages are
|
|
1395
|
+
returned to pending before the claim id is cleared.
|
|
1378
1396
|
|
|
1379
1397
|
``timeout`` bounds the pending-claim-id recovery loop in seconds;
|
|
1380
1398
|
``None`` waits indefinitely, ``0`` skips the loop entirely. The
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{redis_message_queue-8.2.2 → redis_message_queue-8.2.4}/redis_message_queue/_callable_utils.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{redis_message_queue-8.2.2 → redis_message_queue-8.2.4}/redis_message_queue/_payload_limits.py
RENAMED
|
File without changes
|
{redis_message_queue-8.2.2 → redis_message_queue-8.2.4}/redis_message_queue/_queue_key_manager.py
RENAMED
|
File without changes
|
{redis_message_queue-8.2.2 → redis_message_queue-8.2.4}/redis_message_queue/_redis_cluster.py
RENAMED
|
File without changes
|
{redis_message_queue-8.2.2 → redis_message_queue-8.2.4}/redis_message_queue/_stored_message.py
RENAMED
|
File without changes
|
{redis_message_queue-8.2.2 → redis_message_queue-8.2.4}/redis_message_queue/asyncio/__init__.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|