redis-message-queue 8.2.2__tar.gz → 8.2.4__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (26) hide show
  1. {redis_message_queue-8.2.2 → redis_message_queue-8.2.4}/PKG-INFO +2 -2
  2. {redis_message_queue-8.2.2 → redis_message_queue-8.2.4}/README.md +1 -1
  3. {redis_message_queue-8.2.2 → redis_message_queue-8.2.4}/pyproject.toml +2 -2
  4. {redis_message_queue-8.2.2 → redis_message_queue-8.2.4}/redis_message_queue/_config.py +50 -2
  5. {redis_message_queue-8.2.2 → redis_message_queue-8.2.4}/redis_message_queue/_redis_gateway.py +91 -12
  6. {redis_message_queue-8.2.2 → redis_message_queue-8.2.4}/redis_message_queue/asyncio/_redis_gateway.py +91 -12
  7. {redis_message_queue-8.2.2 → redis_message_queue-8.2.4}/redis_message_queue/asyncio/redis_message_queue.py +25 -1
  8. {redis_message_queue-8.2.2 → redis_message_queue-8.2.4}/redis_message_queue/redis_message_queue.py +19 -1
  9. {redis_message_queue-8.2.2 → redis_message_queue-8.2.4}/.gitignore +0 -0
  10. {redis_message_queue-8.2.2 → redis_message_queue-8.2.4}/LICENSE +0 -0
  11. {redis_message_queue-8.2.2 → redis_message_queue-8.2.4}/redis_message_queue/__init__.py +0 -0
  12. {redis_message_queue-8.2.2 → redis_message_queue-8.2.4}/redis_message_queue/_abstract_redis_gateway.py +0 -0
  13. {redis_message_queue-8.2.2 → redis_message_queue-8.2.4}/redis_message_queue/_callable_utils.py +0 -0
  14. {redis_message_queue-8.2.2 → redis_message_queue-8.2.4}/redis_message_queue/_event.py +0 -0
  15. {redis_message_queue-8.2.2 → redis_message_queue-8.2.4}/redis_message_queue/_exceptions.py +0 -0
  16. {redis_message_queue-8.2.2 → redis_message_queue-8.2.4}/redis_message_queue/_payload_limits.py +0 -0
  17. {redis_message_queue-8.2.2 → redis_message_queue-8.2.4}/redis_message_queue/_queue_key_manager.py +0 -0
  18. {redis_message_queue-8.2.2 → redis_message_queue-8.2.4}/redis_message_queue/_redis_cluster.py +0 -0
  19. {redis_message_queue-8.2.2 → redis_message_queue-8.2.4}/redis_message_queue/_stored_message.py +0 -0
  20. {redis_message_queue-8.2.2 → redis_message_queue-8.2.4}/redis_message_queue/asyncio/__init__.py +0 -0
  21. {redis_message_queue-8.2.2 → redis_message_queue-8.2.4}/redis_message_queue/asyncio/_abstract_redis_gateway.py +0 -0
  22. {redis_message_queue-8.2.2 → redis_message_queue-8.2.4}/redis_message_queue/interrupt_handler/__init__.py +0 -0
  23. {redis_message_queue-8.2.2 → redis_message_queue-8.2.4}/redis_message_queue/interrupt_handler/_event_driven.py +0 -0
  24. {redis_message_queue-8.2.2 → redis_message_queue-8.2.4}/redis_message_queue/interrupt_handler/_implementation.py +0 -0
  25. {redis_message_queue-8.2.2 → redis_message_queue-8.2.4}/redis_message_queue/interrupt_handler/_interface.py +0 -0
  26. {redis_message_queue-8.2.2 → redis_message_queue-8.2.4}/redis_message_queue/py.typed +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: redis-message-queue
3
- Version: 8.2.2
3
+ Version: 8.2.4
4
4
  Summary: Python message queuing with Redis and message deduplication
5
5
  Project-URL: Homepage, https://github.com/Elijas/redis-message-queue
6
6
  Project-URL: Repository, https://github.com/Elijas/redis-message-queue
@@ -34,7 +34,7 @@ Description-Content-Type: text/markdown
34
34
  **Lightweight Python message queuing with Redis and built-in publish-side deduplication.** Deduplicate publishes within a TTL window, with optional crash recovery — across any number of producers and consumers.
35
35
 
36
36
  ```bash
37
- pip install "redis-message-queue>=8.2.2,<9.0.0"
37
+ pip install "redis-message-queue>=8.2.4,<9.0.0"
38
38
  ```
39
39
 
40
40
  Requires Redis server >= 6.2.
@@ -11,7 +11,7 @@
11
11
  **Lightweight Python message queuing with Redis and built-in publish-side deduplication.** Deduplicate publishes within a TTL window, with optional crash recovery — across any number of producers and consumers.
12
12
 
13
13
  ```bash
14
- pip install "redis-message-queue>=8.2.2,<9.0.0"
14
+ pip install "redis-message-queue>=8.2.4,<9.0.0"
15
15
  ```
16
16
 
17
17
  Requires Redis server >= 6.2.
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "redis-message-queue"
3
- version = "8.2.2"
3
+ version = "8.2.4"
4
4
  description = "Python message queuing with Redis and message deduplication"
5
5
  authors = [{ name = "Elijas", email = "4084885+Elijas@users.noreply.github.com" }]
6
6
  readme = "README.md"
@@ -48,7 +48,7 @@ default-groups = ["dev", "test"]
48
48
  ##############################
49
49
 
50
50
  [tool.bumpversion]
51
- current_version = "8.2.2"
51
+ current_version = "8.2.4"
52
52
  parse = "(?P<major>\\d+)\\.(?P<minor>\\d+)\\.(?P<patch>\\d+)"
53
53
  serialize = ["{major}.{minor}.{patch}"]
54
54
  search = "{current_version}"
@@ -585,6 +585,54 @@ return removed
585
585
  """
586
586
  )
587
587
 
588
+ RETURN_MESSAGE_TO_PENDING_LUA_SCRIPT = (
589
+ _LUA_KEY_TYPE_GUARD
590
+ + """
591
+ local err = redis_message_queue_require_type(KEYS[1], 'list')
592
+ if err then
593
+ return err
594
+ end
595
+
596
+ local err = redis_message_queue_require_type(KEYS[2], 'list')
597
+ if err then
598
+ return err
599
+ end
600
+
601
+ local err = redis_message_queue_require_type(KEYS[3], 'hash')
602
+ if err then
603
+ return err
604
+ end
605
+
606
+ local err = redis_message_queue_require_type(KEYS[4], 'hash')
607
+ if err then
608
+ return err
609
+ end
610
+
611
+ local err = redis_message_queue_require_type(KEYS[5], 'string')
612
+ if err then
613
+ return err
614
+ end
615
+
616
+ local cached_result = redis.call('GET', KEYS[5])
617
+ if cached_result then
618
+ return tonumber(cached_result)
619
+ end
620
+
621
+ redis.call('RPUSH', KEYS[2], ARGV[1])
622
+ local removed = redis.call('LREM', KEYS[1], 1, ARGV[1])
623
+ if removed == 1 then
624
+ redis.call('HDEL', KEYS[3], ARGV[2])
625
+ redis.call('HDEL', KEYS[4], ARGV[1])
626
+ redis.call('DEL', KEYS[6])
627
+ else
628
+ redis.call('LREM', KEYS[2], 1, ARGV[1])
629
+ end
630
+
631
+ redis.call('SET', KEYS[5], tostring(removed), 'PX', tonumber(ARGV[3]))
632
+ return removed
633
+ """
634
+ )
635
+
588
636
  REMOVE_MESSAGE_LUA_SCRIPT = (
589
637
  _LUA_KEY_TYPE_GUARD
590
638
  + """
@@ -876,8 +924,6 @@ while claim_attempts < 100 do
876
924
 
877
925
  local count = redis.call('HINCRBY', KEYS[6], stored, 1)
878
926
  if max_delivery_count > 0 and count > max_delivery_count then
879
- redis.call('LREM', KEYS[2], 1, stored)
880
- redis.call('HDEL', KEYS[6], stored)
881
927
  -- Strip envelope to store raw payload in DLQ, consistent with completed/failed queues.
882
928
  -- The per-delivery UUID in the envelope is lost; see README dead-letter notes.
883
929
  local dead_letter_value = stored
@@ -886,6 +932,8 @@ while claim_attempts < 100 do
886
932
  dead_letter_value = envelope['payload']
887
933
  end
888
934
  redis.call('LPUSH', KEYS[7], dead_letter_value)
935
+ redis.call('LREM', KEYS[2], 1, stored)
936
+ redis.call('HDEL', KEYS[6], stored)
889
937
  table.insert(dead_lettered_events, {redis_message_queue_message_id(stored), tostring(count)})
890
938
  else
891
939
  return store_claim_and_return(stored)
@@ -31,6 +31,7 @@ from redis_message_queue._config import (
31
31
  REMOVE_MESSAGE_LUA_SCRIPT,
32
32
  REMOVE_MESSAGE_WITH_LEASE_TOKEN_LUA_SCRIPT,
33
33
  RENEW_MESSAGE_LEASE_LUA_SCRIPT,
34
+ RETURN_MESSAGE_TO_PENDING_LUA_SCRIPT,
34
35
  _ChainedInterrupt,
35
36
  build_retry_strategy,
36
37
  is_redis_retryable_exception,
@@ -79,6 +80,8 @@ _VISIBILITY_TIMEOUT_POLL_INTERVAL_SECONDS = 0.25
79
80
  _PENDING_OVERLOAD_INITIAL_BACKOFF_SECONDS = 0.010
80
81
  _PENDING_OVERLOAD_MAX_BACKOFF_SECONDS = 0.500
81
82
  _CLAIM_STORE_FAILED_LUA_SENTINEL_BYTES = CLAIM_STORE_FAILED_LUA_SENTINEL.encode("utf-8")
83
+ _PENDING_QUEUE_SUFFIX = "pending"
84
+ _PROCESSING_QUEUE_SUFFIX = "processing"
82
85
 
83
86
 
84
87
  class _DrainDeadlineExceeded(Exception):
@@ -703,6 +706,8 @@ class RedisGateway(AbstractRedisGateway):
703
706
  clear=clear_pending_claim_id,
704
707
  )
705
708
  if recovered_claim is not None:
709
+ if self._message_visibility_timeout_seconds is None:
710
+ self._delete_claim_result_key(self._claim_result_key(to_queue, pending_claim_id))
706
711
  self._emit_event("claim_reclaim", "success", claim_id=pending_claim_id)
707
712
  return recovered_claim
708
713
 
@@ -810,6 +815,8 @@ class RedisGateway(AbstractRedisGateway):
810
815
  pending_claim_id_to_share = claim_id
811
816
  raise
812
817
  if recovered_claim is not None:
818
+ if self._message_visibility_timeout_seconds is None:
819
+ self._delete_claim_result_key(self._claim_result_key(to_queue, claim_id))
813
820
  self._emit_event("claim_reclaim", "success", claim_id=claim_id)
814
821
  return recovered_claim
815
822
  self._emit_event(
@@ -1168,7 +1175,6 @@ class RedisGateway(AbstractRedisGateway):
1168
1175
  _raise_if_drain_deadline_expired(deadline_monotonic)
1169
1176
  if cached_claim is None:
1170
1177
  return None
1171
- self._delete_claim_result_key(claim_result_key, deadline_monotonic=deadline_monotonic)
1172
1178
  _raise_if_drain_deadline_expired(deadline_monotonic)
1173
1179
  return cached_claim
1174
1180
 
@@ -1225,6 +1231,54 @@ class RedisGateway(AbstractRedisGateway):
1225
1231
  _raise_if_drain_deadline_expired(deadline_monotonic)
1226
1232
  return ClaimedMessage(stored_message=stored_message, lease_token=lease_token)
1227
1233
 
1234
+ def _pending_queue_from_processing_queue(self, processing_queue: str) -> str:
1235
+ if not processing_queue.endswith(_PROCESSING_QUEUE_SUFFIX):
1236
+ raise RuntimeError(f"cannot derive pending queue key from processing queue {processing_queue!r}")
1237
+ return f"{processing_queue.removesuffix(_PROCESSING_QUEUE_SUFFIX)}{_PENDING_QUEUE_SUFFIX}"
1238
+
1239
+ def _return_recovered_non_visibility_timeout_claim_to_pending(
1240
+ self,
1241
+ processing_queue: str,
1242
+ stored_message: MessageData,
1243
+ claim_id: str,
1244
+ *,
1245
+ deadline_monotonic: float | None,
1246
+ ) -> bool:
1247
+ pending_queue = self._pending_queue_from_processing_queue(processing_queue)
1248
+ operation_id = uuid.uuid4().hex
1249
+ operation_result_key = self._operation_result_key(processing_queue, operation_id)
1250
+
1251
+ try:
1252
+ _raise_if_drain_deadline_expired(deadline_monotonic)
1253
+ result = _call_with_drain_deadline(
1254
+ lambda: self._eval(
1255
+ RETURN_MESSAGE_TO_PENDING_LUA_SCRIPT,
1256
+ 6,
1257
+ processing_queue,
1258
+ pending_queue,
1259
+ self._claim_result_ids_key(processing_queue),
1260
+ self._claim_result_backrefs_key(processing_queue),
1261
+ operation_result_key,
1262
+ self._claim_result_key(processing_queue, claim_id),
1263
+ stored_message,
1264
+ claim_id,
1265
+ self._operation_result_ttl_ms(),
1266
+ ),
1267
+ deadline_monotonic=deadline_monotonic,
1268
+ )
1269
+ _raise_if_drain_deadline_expired(deadline_monotonic)
1270
+ return bool(_coerce_lua_count(result))
1271
+ except RedisMessageQueueError as exc:
1272
+ _set_exception_context(
1273
+ exc,
1274
+ queue=processing_queue,
1275
+ message_id=extract_stored_message_id(stored_message),
1276
+ operation="drain",
1277
+ )
1278
+ raise
1279
+ finally:
1280
+ self._delete_operation_result_key(operation_result_key)
1281
+
1228
1282
  def _is_interrupted(self, is_interrupted: BaseGracefulInterruptHandler | None = None) -> bool:
1229
1283
  return (self._interrupt is not None and self._interrupt.is_interrupted()) or (
1230
1284
  is_interrupted is not None and is_interrupted.is_interrupted()
@@ -1241,8 +1295,10 @@ class RedisGateway(AbstractRedisGateway):
1241
1295
  Walks the same recovery path as ``_wait_for_claim`` but without
1242
1296
  gating on the interrupt flag, so a soft shutdown can flush
1243
1297
  ambiguous-claim state that would otherwise be dropped on process
1244
- exit (AA-05-F2). Returns True if no pending ids remain; False if
1245
- the deadline fired or transient Redis errors prevented full drain.
1298
+ exit (AA-05-F2). No-visibility-timeout recoveries are returned to
1299
+ the pending queue before their claim ids are cleared. Returns True
1300
+ if no pending ids remain; False if the deadline fired or Redis
1301
+ errors prevented full drain.
1246
1302
  """
1247
1303
  with self._drain_pending_claim_ids_lock:
1248
1304
  self._last_drain_error = None
@@ -1258,11 +1314,8 @@ class RedisGateway(AbstractRedisGateway):
1258
1314
  deadline_monotonic: float | None,
1259
1315
  ) -> bool:
1260
1316
  """Recover every in-memory pending claim id for ``processing_queue``."""
1261
- if self._message_visibility_timeout_seconds is not None:
1262
- recover = self._recover_pending_visibility_timeout_claim
1263
- else:
1264
- recover = self._recover_pending_non_visibility_timeout_claim
1265
- skipped_transient: set[str] = set()
1317
+ has_visibility_timeout = self._message_visibility_timeout_seconds is not None
1318
+ skipped_unresolved: set[str] = set()
1266
1319
  last_error: BaseException | None = None
1267
1320
  while True:
1268
1321
  # ``>=`` (not ``>``) makes ``timeout=0`` deterministically take
@@ -1278,7 +1331,7 @@ class RedisGateway(AbstractRedisGateway):
1278
1331
  return True
1279
1332
  recovering = self._recovering_claim_ids.setdefault(processing_queue, set())
1280
1333
  claim_id = next(
1281
- (cid for cid in pending if cid not in recovering and cid not in skipped_transient),
1334
+ (cid for cid in pending if cid not in recovering and cid not in skipped_unresolved),
1282
1335
  None,
1283
1336
  )
1284
1337
  if claim_id is None:
@@ -1287,8 +1340,34 @@ class RedisGateway(AbstractRedisGateway):
1287
1340
  clear = False
1288
1341
  try:
1289
1342
  try:
1290
- recover(processing_queue, claim_id, deadline_monotonic=deadline_monotonic)
1291
- clear = True
1343
+ if has_visibility_timeout:
1344
+ self._recover_pending_visibility_timeout_claim(
1345
+ processing_queue,
1346
+ claim_id,
1347
+ deadline_monotonic=deadline_monotonic,
1348
+ )
1349
+ clear = True
1350
+ continue
1351
+
1352
+ recovered_claim = self._recover_pending_non_visibility_timeout_claim(
1353
+ processing_queue,
1354
+ claim_id,
1355
+ deadline_monotonic=deadline_monotonic,
1356
+ )
1357
+ if recovered_claim is None:
1358
+ clear = True
1359
+ elif self._return_recovered_non_visibility_timeout_claim_to_pending(
1360
+ processing_queue,
1361
+ recovered_claim,
1362
+ claim_id,
1363
+ deadline_monotonic=deadline_monotonic,
1364
+ ):
1365
+ clear = True
1366
+ else:
1367
+ last_error = RuntimeError(
1368
+ f"drain recovered claim {claim_id!r} but message was not present in processing queue"
1369
+ )
1370
+ skipped_unresolved.add(claim_id)
1292
1371
  except _DrainDeadlineExceeded:
1293
1372
  last_error = TimeoutError("drain pending-claim recovery deadline expired")
1294
1373
  break
@@ -1301,7 +1380,7 @@ class RedisGateway(AbstractRedisGateway):
1301
1380
  claim_id,
1302
1381
  type(exc).__name__,
1303
1382
  )
1304
- skipped_transient.add(claim_id)
1383
+ skipped_unresolved.add(claim_id)
1305
1384
  finally:
1306
1385
  self._finish_pending_claim_recovery(processing_queue, claim_id, clear=clear)
1307
1386
  with self._pending_claim_ids_lock:
@@ -29,6 +29,7 @@ from redis_message_queue._config import (
29
29
  REMOVE_MESSAGE_LUA_SCRIPT,
30
30
  REMOVE_MESSAGE_WITH_LEASE_TOKEN_LUA_SCRIPT,
31
31
  RENEW_MESSAGE_LEASE_LUA_SCRIPT,
32
+ RETURN_MESSAGE_TO_PENDING_LUA_SCRIPT,
32
33
  _ChainedInterrupt,
33
34
  build_retry_strategy,
34
35
  is_redis_retryable_exception,
@@ -78,6 +79,8 @@ _VISIBILITY_TIMEOUT_POLL_INTERVAL_SECONDS = 0.25
78
79
  _PENDING_OVERLOAD_INITIAL_BACKOFF_SECONDS = 0.010
79
80
  _PENDING_OVERLOAD_MAX_BACKOFF_SECONDS = 0.500
80
81
  _CLAIM_STORE_FAILED_LUA_SENTINEL_BYTES = CLAIM_STORE_FAILED_LUA_SENTINEL.encode("utf-8")
82
+ _PENDING_QUEUE_SUFFIX = "pending"
83
+ _PROCESSING_QUEUE_SUFFIX = "processing"
81
84
 
82
85
 
83
86
  class _DrainDeadlineExceeded(Exception):
@@ -682,6 +685,8 @@ class RedisGateway(AbstractRedisGateway):
682
685
  clear=clear_pending_claim_id,
683
686
  )
684
687
  if recovered_claim is not None:
688
+ if self._message_visibility_timeout_seconds is None:
689
+ await self._delete_claim_result_key(self._claim_result_key(to_queue, pending_claim_id))
685
690
  await self._emit_event("claim_reclaim", "success", claim_id=pending_claim_id)
686
691
  return recovered_claim
687
692
 
@@ -790,6 +795,8 @@ class RedisGateway(AbstractRedisGateway):
790
795
  pending_claim_id_to_share = claim_id
791
796
  raise
792
797
  if recovered_claim is not None:
798
+ if self._message_visibility_timeout_seconds is None:
799
+ await self._delete_claim_result_key(self._claim_result_key(to_queue, claim_id))
793
800
  await self._emit_event("claim_reclaim", "success", claim_id=claim_id)
794
801
  return recovered_claim
795
802
  await self._emit_event(
@@ -1148,7 +1155,6 @@ class RedisGateway(AbstractRedisGateway):
1148
1155
  _raise_if_drain_deadline_expired(deadline_monotonic)
1149
1156
  if cached_claim is None:
1150
1157
  return None
1151
- await self._delete_claim_result_key(claim_result_key, deadline_monotonic=deadline_monotonic)
1152
1158
  _raise_if_drain_deadline_expired(deadline_monotonic)
1153
1159
  return cached_claim
1154
1160
 
@@ -1206,6 +1212,54 @@ class RedisGateway(AbstractRedisGateway):
1206
1212
  _raise_if_drain_deadline_expired(deadline_monotonic)
1207
1213
  return ClaimedMessage(stored_message=stored_message, lease_token=lease_token)
1208
1214
 
1215
+ def _pending_queue_from_processing_queue(self, processing_queue: str) -> str:
1216
+ if not processing_queue.endswith(_PROCESSING_QUEUE_SUFFIX):
1217
+ raise RuntimeError(f"cannot derive pending queue key from processing queue {processing_queue!r}")
1218
+ return f"{processing_queue.removesuffix(_PROCESSING_QUEUE_SUFFIX)}{_PENDING_QUEUE_SUFFIX}"
1219
+
1220
+ async def _return_recovered_non_visibility_timeout_claim_to_pending(
1221
+ self,
1222
+ processing_queue: str,
1223
+ stored_message: MessageData,
1224
+ claim_id: str,
1225
+ *,
1226
+ deadline_monotonic: float | None,
1227
+ ) -> bool:
1228
+ pending_queue = self._pending_queue_from_processing_queue(processing_queue)
1229
+ operation_id = uuid.uuid4().hex
1230
+ operation_result_key = self._operation_result_key(processing_queue, operation_id)
1231
+
1232
+ try:
1233
+ _raise_if_drain_deadline_expired(deadline_monotonic)
1234
+ result = await _call_with_drain_deadline(
1235
+ lambda: self._eval(
1236
+ RETURN_MESSAGE_TO_PENDING_LUA_SCRIPT,
1237
+ 6,
1238
+ processing_queue,
1239
+ pending_queue,
1240
+ self._claim_result_ids_key(processing_queue),
1241
+ self._claim_result_backrefs_key(processing_queue),
1242
+ operation_result_key,
1243
+ self._claim_result_key(processing_queue, claim_id),
1244
+ stored_message,
1245
+ claim_id,
1246
+ self._operation_result_ttl_ms(),
1247
+ ),
1248
+ deadline_monotonic=deadline_monotonic,
1249
+ )
1250
+ _raise_if_drain_deadline_expired(deadline_monotonic)
1251
+ return bool(_coerce_lua_count(result))
1252
+ except RedisMessageQueueError as exc:
1253
+ _set_exception_context(
1254
+ exc,
1255
+ queue=processing_queue,
1256
+ message_id=extract_stored_message_id(stored_message),
1257
+ operation="drain",
1258
+ )
1259
+ raise
1260
+ finally:
1261
+ await self._delete_operation_result_key(operation_result_key)
1262
+
1209
1263
  def _is_interrupted(self, is_interrupted: BaseGracefulInterruptHandler | None = None) -> bool:
1210
1264
  return (self._interrupt is not None and self._interrupt.is_interrupted()) or (
1211
1265
  is_interrupted is not None and is_interrupted.is_interrupted()
@@ -1222,8 +1276,10 @@ class RedisGateway(AbstractRedisGateway):
1222
1276
  Kept separate from the sync implementation per AF11 (sync/async
1223
1277
  gateways stay duplicated). Walks the same recovery path as
1224
1278
  ``_wait_for_claim`` but without gating on the interrupt flag so a
1225
- soft shutdown can flush ambiguous-claim state. Returns True if no
1226
- pending ids remain; False on deadline expiry or transient errors.
1279
+ soft shutdown can flush ambiguous-claim state. No-visibility-timeout
1280
+ recoveries are returned to the pending queue before their claim ids
1281
+ are cleared. Returns True if no pending ids remain; False on deadline
1282
+ expiry or Redis errors.
1227
1283
  """
1228
1284
  async with self._drain_pending_claim_ids_lock:
1229
1285
  self._last_drain_error = None
@@ -1239,11 +1295,8 @@ class RedisGateway(AbstractRedisGateway):
1239
1295
  deadline_monotonic: float | None,
1240
1296
  ) -> bool:
1241
1297
  """Recover every in-memory pending claim id for ``processing_queue``."""
1242
- if self._message_visibility_timeout_seconds is not None:
1243
- recover = self._recover_pending_visibility_timeout_claim
1244
- else:
1245
- recover = self._recover_pending_non_visibility_timeout_claim
1246
- skipped_transient: set[str] = set()
1298
+ has_visibility_timeout = self._message_visibility_timeout_seconds is not None
1299
+ skipped_unresolved: set[str] = set()
1247
1300
  loop = asyncio.get_running_loop()
1248
1301
  last_error: BaseException | None = None
1249
1302
  while True:
@@ -1258,7 +1311,7 @@ class RedisGateway(AbstractRedisGateway):
1258
1311
  return True
1259
1312
  recovering = self._recovering_claim_ids.setdefault(processing_queue, set())
1260
1313
  claim_id = next(
1261
- (cid for cid in pending if cid not in recovering and cid not in skipped_transient),
1314
+ (cid for cid in pending if cid not in recovering and cid not in skipped_unresolved),
1262
1315
  None,
1263
1316
  )
1264
1317
  if claim_id is None:
@@ -1267,8 +1320,34 @@ class RedisGateway(AbstractRedisGateway):
1267
1320
  clear = False
1268
1321
  try:
1269
1322
  try:
1270
- await recover(processing_queue, claim_id, deadline_monotonic=deadline_monotonic)
1271
- clear = True
1323
+ if has_visibility_timeout:
1324
+ await self._recover_pending_visibility_timeout_claim(
1325
+ processing_queue,
1326
+ claim_id,
1327
+ deadline_monotonic=deadline_monotonic,
1328
+ )
1329
+ clear = True
1330
+ continue
1331
+
1332
+ recovered_claim = await self._recover_pending_non_visibility_timeout_claim(
1333
+ processing_queue,
1334
+ claim_id,
1335
+ deadline_monotonic=deadline_monotonic,
1336
+ )
1337
+ if recovered_claim is None:
1338
+ clear = True
1339
+ elif await self._return_recovered_non_visibility_timeout_claim_to_pending(
1340
+ processing_queue,
1341
+ recovered_claim,
1342
+ claim_id,
1343
+ deadline_monotonic=deadline_monotonic,
1344
+ ):
1345
+ clear = True
1346
+ else:
1347
+ last_error = RuntimeError(
1348
+ f"drain recovered claim {claim_id!r} but message was not present in processing queue"
1349
+ )
1350
+ skipped_unresolved.add(claim_id)
1272
1351
  except _DrainDeadlineExceeded:
1273
1352
  last_error = TimeoutError("drain pending-claim recovery deadline expired")
1274
1353
  break
@@ -1281,7 +1360,7 @@ class RedisGateway(AbstractRedisGateway):
1281
1360
  claim_id,
1282
1361
  type(exc).__name__,
1283
1362
  )
1284
- skipped_transient.add(claim_id)
1363
+ skipped_unresolved.add(claim_id)
1285
1364
  finally:
1286
1365
  self._finish_pending_claim_recovery(processing_queue, claim_id, clear=clear)
1287
1366
  with self._pending_claim_ids_lock:
@@ -483,6 +483,16 @@ class _LeaseHeartbeat:
483
483
  result = self._on_heartbeat_failure()
484
484
  if inspect.isawaitable(result):
485
485
  await result
486
+ except asyncio.CancelledError as exc:
487
+ current_task = asyncio.current_task()
488
+ if current_task is not None and current_task.cancelling() > 0:
489
+ raise
490
+ logger.exception("on_heartbeat_failure callback raised an exception")
491
+ warnings.warn(
492
+ f"on_heartbeat_failure callback raised {type(exc).__name__}",
493
+ RuntimeWarning,
494
+ stacklevel=1,
495
+ )
486
496
  except Exception as exc:
487
497
  logger.exception("on_heartbeat_failure callback raised an exception")
488
498
  warnings.warn(
@@ -920,6 +930,18 @@ class RedisMessageQueue:
920
930
  "Use an async callable or return an awaitable."
921
931
  )
922
932
  await result
933
+ except asyncio.CancelledError as exc:
934
+ current_task = asyncio.current_task()
935
+ if current_task is not None and current_task.cancelling() > 0:
936
+ raise
937
+ logger.exception("on_event callback raised an exception")
938
+ with warnings.catch_warnings():
939
+ warnings.simplefilter("always", RuntimeWarning)
940
+ warnings.warn(
941
+ f"on_event callback raised {type(exc).__name__}",
942
+ RuntimeWarning,
943
+ stacklevel=2,
944
+ )
923
945
  except Exception as exc:
924
946
  logger.exception("on_event callback raised an exception")
925
947
  with warnings.catch_warnings():
@@ -1414,7 +1436,9 @@ class RedisMessageQueue:
1414
1436
  calls raise ``QueueDrainedError``. It then awaits the gateway's
1415
1437
  pending-claim-id recovery loop. Returns ``True`` if all pending claim
1416
1438
  ids were recovered, ``False`` if the deadline fired or a transient
1417
- Redis error left ids pending.
1439
+ Redis error left ids pending. In no-visibility-timeout queues,
1440
+ recovered messages are returned to pending before the claim id is
1441
+ cleared.
1418
1442
 
1419
1443
  Unlike ``asyncio.CancelledError`` (hard-abort, leaves messages
1420
1444
  claimed for VT-reclaim), ``aclose()`` is the explicit-drain
@@ -1,3 +1,4 @@
1
+ import asyncio
1
2
  import hashlib
2
3
  import inspect
3
4
  import logging
@@ -444,6 +445,13 @@ class _LeaseHeartbeat:
444
445
  "'on_heartbeat_failure' returned an awaitable; "
445
446
  "use the async RedisMessageQueue from redis_message_queue.asyncio instead"
446
447
  )
448
+ except asyncio.CancelledError as exc:
449
+ logger.exception("on_heartbeat_failure callback raised an exception")
450
+ warnings.warn(
451
+ f"on_heartbeat_failure callback raised {type(exc).__name__}",
452
+ RuntimeWarning,
453
+ stacklevel=1,
454
+ )
447
455
  except Exception as exc:
448
456
  logger.exception("on_heartbeat_failure callback raised an exception")
449
457
  warnings.warn(
@@ -883,6 +891,15 @@ class RedisMessageQueue:
883
891
  "'on_event' returned an awaitable; use the async RedisMessageQueue "
884
892
  "from redis_message_queue.asyncio instead"
885
893
  )
894
+ except asyncio.CancelledError as exc:
895
+ logger.exception("on_event callback raised an exception")
896
+ with warnings.catch_warnings():
897
+ warnings.simplefilter("always", RuntimeWarning)
898
+ warnings.warn(
899
+ f"on_event callback raised {type(exc).__name__}",
900
+ RuntimeWarning,
901
+ stacklevel=2,
902
+ )
886
903
  except Exception as exc:
887
904
  logger.exception("on_event callback raised an exception")
888
905
  with warnings.catch_warnings():
@@ -1374,7 +1391,8 @@ class RedisMessageQueue:
1374
1391
  calls raise ``QueueDrainedError``. It then walks the gateway's
1375
1392
  in-memory ``_pending_claim_ids`` to recover any ambiguous claims that
1376
1393
  an interrupt-aware shutdown would otherwise drop on process exit
1377
- (AA-05-F2).
1394
+ (AA-05-F2). In no-visibility-timeout queues, recovered messages are
1395
+ returned to pending before the claim id is cleared.
1378
1396
 
1379
1397
  ``timeout`` bounds the pending-claim-id recovery loop in seconds;
1380
1398
  ``None`` waits indefinitely, ``0`` skips the loop entirely. The