vs-queue 0.1.3__tar.gz → 0.1.5__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {vs_queue-0.1.3/vs_queue.egg-info → vs_queue-0.1.5}/PKG-INFO +10 -2
- {vs_queue-0.1.3 → vs_queue-0.1.5}/README.md +8 -0
- {vs_queue-0.1.3 → vs_queue-0.1.5}/pyproject.toml +2 -2
- {vs_queue-0.1.3 → vs_queue-0.1.5}/vs_queue/base/vs_base_queue.py +1 -1
- vs_queue-0.1.5/vs_queue/error/vs_message_rejected.py +2 -0
- {vs_queue-0.1.3 → vs_queue-0.1.5}/vs_queue/manager/vs_queue_manager.py +17 -6
- {vs_queue-0.1.3 → vs_queue-0.1.5}/vs_queue/provider/vs_rabbitmq_queue.py +13 -6
- {vs_queue-0.1.3 → vs_queue-0.1.5}/vs_queue/provider/vs_redis_queue.py +72 -17
- vs_queue-0.1.5/vs_queue/schema/__init__.py +0 -0
- {vs_queue-0.1.3 → vs_queue-0.1.5/vs_queue.egg-info}/PKG-INFO +10 -2
- {vs_queue-0.1.3 → vs_queue-0.1.5}/vs_queue.egg-info/SOURCES.txt +2 -0
- {vs_queue-0.1.3 → vs_queue-0.1.5}/vs_queue.egg-info/requires.txt +1 -1
- {vs_queue-0.1.3 → vs_queue-0.1.5}/LICENSE.txt +0 -0
- {vs_queue-0.1.3 → vs_queue-0.1.5}/setup.cfg +0 -0
- {vs_queue-0.1.3 → vs_queue-0.1.5}/vs_queue/__init__.py +0 -0
- {vs_queue-0.1.3 → vs_queue-0.1.5}/vs_queue/base/__init__.py +0 -0
- {vs_queue-0.1.3 → vs_queue-0.1.5}/vs_queue/base/vs_base_consumer.py +0 -0
- {vs_queue-0.1.3 → vs_queue-0.1.5}/vs_queue/decorator/__init__.py +0 -0
- {vs_queue-0.1.3 → vs_queue-0.1.5}/vs_queue/decorator/vs_queue_listener.py +0 -0
- {vs_queue-0.1.3/vs_queue/factory → vs_queue-0.1.5/vs_queue/error}/__init__.py +0 -0
- {vs_queue-0.1.3/vs_queue/manager → vs_queue-0.1.5/vs_queue/factory}/__init__.py +0 -0
- {vs_queue-0.1.3 → vs_queue-0.1.5}/vs_queue/factory/vs_queue_factory.py +0 -0
- {vs_queue-0.1.3/vs_queue/provider → vs_queue-0.1.5/vs_queue/manager}/__init__.py +0 -0
- {vs_queue-0.1.3/vs_queue/registry → vs_queue-0.1.5/vs_queue/provider}/__init__.py +0 -0
- {vs_queue-0.1.3 → vs_queue-0.1.5}/vs_queue/py.typed +0 -0
- {vs_queue-0.1.3/vs_queue/retry → vs_queue-0.1.5/vs_queue/registry}/__init__.py +0 -0
- {vs_queue-0.1.3 → vs_queue-0.1.5}/vs_queue/registry/vs_queue_registry.py +0 -0
- {vs_queue-0.1.3/vs_queue/schema → vs_queue-0.1.5/vs_queue/retry}/__init__.py +0 -0
- {vs_queue-0.1.3 → vs_queue-0.1.5}/vs_queue/retry/vs_retry_policy.py +0 -0
- {vs_queue-0.1.3 → vs_queue-0.1.5}/vs_queue/schema/vs_message.py +0 -0
- {vs_queue-0.1.3 → vs_queue-0.1.5}/vs_queue/schema/vs_retention.py +0 -0
- {vs_queue-0.1.3 → vs_queue-0.1.5}/vs_queue.egg-info/dependency_links.txt +0 -0
- {vs_queue-0.1.3 → vs_queue-0.1.5}/vs_queue.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: vs-queue
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.5
|
|
4
4
|
Summary: Pluggable async message queue library for Viveka Sutra — Redis Streams, RabbitMQ, retry, dead letter, and annotation-based consumers
|
|
5
5
|
License: Apache License
|
|
6
6
|
Version 2.0, January 2004
|
|
@@ -202,7 +202,7 @@ Description-Content-Type: text/markdown
|
|
|
202
202
|
License-File: LICENSE.txt
|
|
203
203
|
Requires-Dist: pydantic>=2.0
|
|
204
204
|
Provides-Extra: redis
|
|
205
|
-
Requires-Dist: redis>=
|
|
205
|
+
Requires-Dist: redis>=8.1; extra == "redis"
|
|
206
206
|
Requires-Dist: hiredis>=2.0; extra == "redis"
|
|
207
207
|
Provides-Extra: rabbitmq
|
|
208
208
|
Requires-Dist: aio-pika>=9.0; extra == "rabbitmq"
|
|
@@ -1246,3 +1246,11 @@ Decorator. Registers a `VsBaseConsumer` subclass for auto-discovery by `VsQueueM
|
|
|
1246
1246
|
```bash
|
|
1247
1247
|
./run_tests.sh
|
|
1248
1248
|
```
|
|
1249
|
+
|
|
1250
|
+
## Agent request/reply integration
|
|
1251
|
+
|
|
1252
|
+
`vs-agent[queue]` provides `VsQueueActionConsumer`, using `VsBaseConsumer`, `VsQueueManager`, `VsMessage`, and the selected provider. Callers place `reply_to`, `correlation_id`, and authentication credentials in message headers. The agent validates/authenticates, executes guards and the action, then publishes a new response before returning for acknowledgement. Response headers never inherit credentials.
|
|
1253
|
+
|
|
1254
|
+
`VsQueueManager.register(queue_name, consumer, retry_policy=...)` forwards retry policies to providers; `register_listeners()` applies the retry settings declared by `@queue_listener`. `VsMessageRejected` marks permanent failures, which bypass retries and are dead-lettered when configured. Retry/DLQ publication failures must not acknowledge the original delivery. Providers overwrite `source_queue` with the actual subscribed queue before retrying; callers cannot redirect retries through that header.
|
|
1255
|
+
|
|
1256
|
+
Redis replays the current consumer's pending entries after a subscription restart and periodically uses XAUTOCLAIM to recover stale deliveries from dead consumers (Redis servers supporting XAUTOCLAIM). Its `credentials["claim_idle_ms"]` option defaults to 300000; set it above the longest expected processing time, or zero to disable reclaim. An overdue message may execute more than once. Durable idempotency is the application's responsibility. Older servers log when stale recovery is unavailable.
|
|
@@ -1031,3 +1031,11 @@ Decorator. Registers a `VsBaseConsumer` subclass for auto-discovery by `VsQueueM
|
|
|
1031
1031
|
```bash
|
|
1032
1032
|
./run_tests.sh
|
|
1033
1033
|
```
|
|
1034
|
+
|
|
1035
|
+
## Agent request/reply integration
|
|
1036
|
+
|
|
1037
|
+
`vs-agent[queue]` provides `VsQueueActionConsumer`, using `VsBaseConsumer`, `VsQueueManager`, `VsMessage`, and the selected provider. Callers place `reply_to`, `correlation_id`, and authentication credentials in message headers. The agent validates/authenticates, executes guards and the action, then publishes a new response before returning for acknowledgement. Response headers never inherit credentials.
|
|
1038
|
+
|
|
1039
|
+
`VsQueueManager.register(queue_name, consumer, retry_policy=...)` forwards retry policies to providers; `register_listeners()` applies the retry settings declared by `@queue_listener`. `VsMessageRejected` marks permanent failures, which bypass retries and are dead-lettered when configured. Retry/DLQ publication failures must not acknowledge the original delivery. Providers overwrite `source_queue` with the actual subscribed queue before retrying; callers cannot redirect retries through that header.
|
|
1040
|
+
|
|
1041
|
+
Redis replays the current consumer's pending entries after a subscription restart and periodically uses XAUTOCLAIM to recover stale deliveries from dead consumers (Redis servers supporting XAUTOCLAIM). Its `credentials["claim_idle_ms"]` option defaults to 300000; set it above the longest expected processing time, or zero to disable reclaim. An overdue message may execute more than once. Durable idempotency is the application's responsibility. Older servers log when stale recovery is unavailable.
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "vs-queue"
|
|
7
|
-
version = "0.1.
|
|
7
|
+
version = "0.1.5"
|
|
8
8
|
description = "Pluggable async message queue library for Viveka Sutra — Redis Streams, RabbitMQ, retry, dead letter, and annotation-based consumers"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.11"
|
|
@@ -29,7 +29,7 @@ dependencies = [
|
|
|
29
29
|
|
|
30
30
|
[project.optional-dependencies]
|
|
31
31
|
redis = [
|
|
32
|
-
"redis>=
|
|
32
|
+
"redis>=8.1",
|
|
33
33
|
"hiredis>=2.0",
|
|
34
34
|
]
|
|
35
35
|
rabbitmq = [
|
|
@@ -38,7 +38,7 @@ class VsBaseQueue(ABC):
|
|
|
38
38
|
pass
|
|
39
39
|
|
|
40
40
|
@abstractmethod
|
|
41
|
-
async def subscribe(self, queue: str, consumer: "VsBaseConsumer") -> None:
|
|
41
|
+
async def subscribe(self, queue: str, consumer: "VsBaseConsumer", retry_policy=None) -> None:
|
|
42
42
|
pass
|
|
43
43
|
|
|
44
44
|
async def blpop(self, key: str, timeout: int = 30) -> Optional[str]:
|
|
@@ -5,6 +5,7 @@ from uuid import uuid4
|
|
|
5
5
|
|
|
6
6
|
from vs_queue.base.vs_base_consumer import VsBaseConsumer
|
|
7
7
|
from vs_queue.base.vs_base_queue import VsBaseQueue
|
|
8
|
+
from vs_queue.retry.vs_retry_policy import VsRetryPolicy
|
|
8
9
|
|
|
9
10
|
_logger = logging.getLogger(__name__)
|
|
10
11
|
|
|
@@ -19,7 +20,7 @@ class VsQueueManager:
|
|
|
19
20
|
self._connected = False
|
|
20
21
|
self._listeners_registered = False
|
|
21
22
|
|
|
22
|
-
async def register(self, queue_name: str, consumer: VsBaseConsumer) -> str:
|
|
23
|
+
async def register(self, queue_name: str, consumer: VsBaseConsumer, retry_policy: Optional[VsRetryPolicy] = None) -> str:
|
|
23
24
|
if not self._connected:
|
|
24
25
|
await self._queue.connect()
|
|
25
26
|
self._connected = True
|
|
@@ -29,7 +30,7 @@ class VsQueueManager:
|
|
|
29
30
|
self._consumers[uid] = (queue_name, consumer)
|
|
30
31
|
|
|
31
32
|
task = asyncio.create_task(
|
|
32
|
-
self._run(queue_name, consumer),
|
|
33
|
+
self._run(queue_name, consumer, retry_policy),
|
|
33
34
|
name=f"consumer:{queue_name}:{uid}",
|
|
34
35
|
)
|
|
35
36
|
self._tasks[uid] = task
|
|
@@ -44,7 +45,11 @@ class VsQueueManager:
|
|
|
44
45
|
from vs_queue.decorator.vs_queue_listener import get_registered_listeners
|
|
45
46
|
for consumer_class, meta in get_registered_listeners():
|
|
46
47
|
for _ in range(meta.concurrency):
|
|
47
|
-
await self.register(meta.queue, consumer_class()
|
|
48
|
+
await self.register(meta.queue, consumer_class(), VsRetryPolicy(
|
|
49
|
+
max_retries=meta.max_retries,
|
|
50
|
+
backoff_seconds=meta.retry_backoff_seconds,
|
|
51
|
+
dead_letter_queue=meta.dead_letter_queue,
|
|
52
|
+
))
|
|
48
53
|
_logger.info(f"Listeners registered | queue={meta.queue} concurrency={meta.concurrency}")
|
|
49
54
|
|
|
50
55
|
def get_listeners(self) -> Dict[str, List[VsBaseConsumer]]:
|
|
@@ -53,6 +58,10 @@ class VsQueueManager:
|
|
|
53
58
|
result.setdefault(queue_name, []).append(consumer)
|
|
54
59
|
return result
|
|
55
60
|
|
|
61
|
+
def has_stopped_listeners(self) -> bool:
|
|
62
|
+
"""Whether a registered subscription task has terminated."""
|
|
63
|
+
return any(task.done() for task in self._tasks.values())
|
|
64
|
+
|
|
56
65
|
async def stop_listener(self, queue_name: str, uuid: Optional[str] = None) -> None:
|
|
57
66
|
if uuid and uuid in self._consumers:
|
|
58
67
|
actual_queue, _ = self._consumers[uuid]
|
|
@@ -92,14 +101,16 @@ class VsQueueManager:
|
|
|
92
101
|
queue_name, _ = self._consumers.pop(uid, (None, None))
|
|
93
102
|
_logger.info(f"Listener stopped | queue={queue_name} uuid={uid}")
|
|
94
103
|
|
|
95
|
-
async def _run(self, queue_name: str, consumer: VsBaseConsumer) -> None:
|
|
104
|
+
async def _run(self, queue_name: str, consumer: VsBaseConsumer, retry_policy: Optional[VsRetryPolicy] = None) -> None:
|
|
96
105
|
attempts = 0
|
|
97
106
|
while True:
|
|
98
107
|
try:
|
|
99
|
-
|
|
108
|
+
if retry_policy is None:
|
|
109
|
+
await self._queue.subscribe(queue_name, consumer)
|
|
110
|
+
else:
|
|
111
|
+
await self._queue.subscribe(queue_name, consumer, retry_policy=retry_policy)
|
|
100
112
|
break
|
|
101
113
|
except asyncio.CancelledError:
|
|
102
|
-
await consumer.stop()
|
|
103
114
|
break
|
|
104
115
|
except Exception as e:
|
|
105
116
|
attempts += 1
|
|
@@ -6,6 +6,7 @@ from vs_queue.base.vs_base_consumer import VsBaseConsumer
|
|
|
6
6
|
from vs_queue.base.vs_base_queue import VsBaseQueue
|
|
7
7
|
from vs_queue.retry.vs_retry_policy import VsRetryPolicy
|
|
8
8
|
from vs_queue.schema.vs_message import VsMessage
|
|
9
|
+
from vs_queue.error.vs_message_rejected import VsMessageRejected
|
|
9
10
|
|
|
10
11
|
_logger = logging.getLogger(__name__)
|
|
11
12
|
|
|
@@ -87,6 +88,7 @@ class VsRabbitMQQueue(VsBaseQueue):
|
|
|
87
88
|
headers=data["headers"],
|
|
88
89
|
payload=data["payload"],
|
|
89
90
|
)
|
|
91
|
+
message.headers["source_queue"] = queue
|
|
90
92
|
_logger.info(f"Message received | queue={queue} id={message.id} retry_count={message.retry_count}")
|
|
91
93
|
try:
|
|
92
94
|
await consumer.handle(message)
|
|
@@ -94,15 +96,20 @@ class VsRabbitMQQueue(VsBaseQueue):
|
|
|
94
96
|
_logger.debug(f"Message processed | queue={queue} id={message.id}")
|
|
95
97
|
except Exception as e:
|
|
96
98
|
await consumer.on_error(message, e)
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
99
|
+
try:
|
|
100
|
+
if isinstance(e, VsMessageRejected):
|
|
101
|
+
if retry_policy and retry_policy.dead_letter_queue:
|
|
102
|
+
await self.publish(retry_policy.dead_letter_queue, message)
|
|
103
|
+
await amqp_message.ack()
|
|
104
|
+
elif retry_policy:
|
|
105
|
+
await retry_policy.handle_failure(message, e, self)
|
|
100
106
|
await amqp_message.ack()
|
|
101
107
|
else:
|
|
102
108
|
await amqp_message.nack(requeue=False)
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
109
|
+
_logger.error(f"Message failed, no retry policy | id={message.id} error={e}")
|
|
110
|
+
except Exception:
|
|
111
|
+
await amqp_message.nack(requeue=True)
|
|
112
|
+
raise
|
|
106
113
|
finally:
|
|
107
114
|
await consumer.stop()
|
|
108
115
|
_logger.info(f"Unsubscribed | queue={queue} consumer={type(consumer).__name__}")
|
|
@@ -4,30 +4,38 @@ import logging
|
|
|
4
4
|
import time
|
|
5
5
|
from typing import Any, Dict, Optional
|
|
6
6
|
|
|
7
|
-
import redis.asyncio as aioredis
|
|
8
|
-
|
|
9
7
|
from vs_queue.base.vs_base_consumer import VsBaseConsumer
|
|
10
8
|
from vs_queue.base.vs_base_queue import VsBaseQueue
|
|
11
9
|
from vs_queue.retry.vs_retry_policy import VsRetryPolicy
|
|
12
10
|
from vs_queue.schema.vs_message import VsMessage
|
|
11
|
+
from vs_queue.error.vs_message_rejected import VsMessageRejected
|
|
13
12
|
|
|
14
13
|
_logger = logging.getLogger(__name__)
|
|
15
14
|
|
|
15
|
+
_RECONNECT_MIN_BACKOFF = 1
|
|
16
|
+
_RECONNECT_MAX_BACKOFF = 30
|
|
17
|
+
|
|
16
18
|
|
|
17
19
|
class VsRedisQueue(VsBaseQueue):
|
|
18
20
|
|
|
19
21
|
def __init__(self, host: str, port: int, credentials: Optional[Dict[str, Any]] = None):
|
|
20
22
|
super().__init__(host, port, credentials)
|
|
21
|
-
self._client
|
|
23
|
+
self._client = None
|
|
22
24
|
self._last_trim: Dict[str, float] = {}
|
|
23
25
|
|
|
24
26
|
async def connect(self) -> None:
|
|
27
|
+
import redis.asyncio as aioredis
|
|
25
28
|
password = self._credentials.get("password")
|
|
26
29
|
self._client = aioredis.Redis(
|
|
27
30
|
host=self._host,
|
|
28
31
|
port=self._port,
|
|
29
32
|
password=password,
|
|
30
33
|
decode_responses=True,
|
|
34
|
+
# redis-py 8.1 defaults this to 5s, which kills any blocking read
|
|
35
|
+
# that blocks for longer. Dead connections are caught below instead.
|
|
36
|
+
socket_timeout=None,
|
|
37
|
+
socket_keepalive=True,
|
|
38
|
+
health_check_interval=30,
|
|
31
39
|
)
|
|
32
40
|
await self._client.ping()
|
|
33
41
|
|
|
@@ -81,26 +89,68 @@ class VsRedisQueue(VsBaseQueue):
|
|
|
81
89
|
return value
|
|
82
90
|
|
|
83
91
|
async def subscribe(self, queue: str, consumer: VsBaseConsumer, retry_policy: Optional[VsRetryPolicy] = None) -> None:
|
|
92
|
+
from redis.exceptions import ConnectionError as RedisConnectionError
|
|
93
|
+
from redis.exceptions import ResponseError as RedisResponseError
|
|
94
|
+
from redis.exceptions import TimeoutError as RedisTimeoutError
|
|
84
95
|
group = f"{queue}:group"
|
|
85
|
-
consumer_name = f"{queue}:consumer"
|
|
96
|
+
consumer_name = f"{queue}:consumer:{consumer.uuid or 'default'}"
|
|
86
97
|
|
|
87
98
|
try:
|
|
88
99
|
await self._client.xgroup_create(queue, group, id="0", mkstream=True)
|
|
89
|
-
except
|
|
90
|
-
|
|
100
|
+
except RedisResponseError as e:
|
|
101
|
+
if "BUSYGROUP" not in str(e):
|
|
102
|
+
raise # anything other than "already exists" is worth shouting about
|
|
103
|
+
_logger.debug(f"Consumer group already exists | queue={queue} group={group}")
|
|
91
104
|
|
|
92
105
|
await consumer.start()
|
|
93
106
|
_logger.info(f"Subscribed | queue={queue} consumer={type(consumer).__name__}")
|
|
107
|
+
backoff = _RECONNECT_MIN_BACKOFF
|
|
108
|
+
# Recover this consumer's unacknowledged messages before accepting new ones.
|
|
109
|
+
pending = True
|
|
110
|
+
claim_idle_ms = int(self._credentials.get("claim_idle_ms", 300000))
|
|
111
|
+
claim_cursor = "0-0"
|
|
112
|
+
next_claim = 0.0
|
|
94
113
|
try:
|
|
95
114
|
while True:
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
115
|
+
try:
|
|
116
|
+
results = []
|
|
117
|
+
if not pending and claim_idle_ms > 0 and time.monotonic() >= next_claim:
|
|
118
|
+
try:
|
|
119
|
+
claimed = await self._client.xautoclaim(
|
|
120
|
+
queue, group, consumer_name, min_idle_time=claim_idle_ms,
|
|
121
|
+
start_id=claim_cursor, count=1,
|
|
122
|
+
)
|
|
123
|
+
claim_cursor = claimed[0]
|
|
124
|
+
if claimed[1]:
|
|
125
|
+
results = [(queue, claimed[1])]
|
|
126
|
+
except RedisResponseError as e:
|
|
127
|
+
if "unknown command" not in str(e).lower():
|
|
128
|
+
raise
|
|
129
|
+
_logger.warning("Redis does not support XAUTOCLAIM; stale pending recovery is disabled")
|
|
130
|
+
claim_idle_ms = 0
|
|
131
|
+
next_claim = time.monotonic() + 30
|
|
132
|
+
if not results:
|
|
133
|
+
results = await self._client.xreadgroup(
|
|
134
|
+
groupname=group,
|
|
135
|
+
consumername=consumer_name,
|
|
136
|
+
streams={queue: "0" if pending else ">"},
|
|
137
|
+
count=1,
|
|
138
|
+
block=5000,
|
|
139
|
+
)
|
|
140
|
+
except asyncio.CancelledError:
|
|
141
|
+
raise
|
|
142
|
+
except (RedisConnectionError, RedisTimeoutError, OSError) as e:
|
|
143
|
+
# Redis wandered off. Wait a bit and ask again — the pool reconnects for us.
|
|
144
|
+
_logger.warning(
|
|
145
|
+
f"Queue read failed, retrying in {backoff}s | queue={queue} error={e}"
|
|
146
|
+
)
|
|
147
|
+
await asyncio.sleep(backoff)
|
|
148
|
+
backoff = min(backoff * 2, _RECONNECT_MAX_BACKOFF)
|
|
149
|
+
continue
|
|
150
|
+
|
|
151
|
+
backoff = _RECONNECT_MIN_BACKOFF
|
|
152
|
+
if not results or not any(messages for _, messages in results):
|
|
153
|
+
pending = False
|
|
104
154
|
continue
|
|
105
155
|
|
|
106
156
|
for _, messages in results:
|
|
@@ -112,17 +162,22 @@ class VsRedisQueue(VsBaseQueue):
|
|
|
112
162
|
headers=json.loads(data["headers"]),
|
|
113
163
|
payload=json.loads(data["payload"]),
|
|
114
164
|
)
|
|
165
|
+
message.headers["source_queue"] = queue
|
|
115
166
|
_logger.info(f"Message received | queue={queue} id={message.id} retry_count={message.retry_count}")
|
|
116
167
|
try:
|
|
117
168
|
await consumer.handle(message)
|
|
118
169
|
await self._client.xack(queue, group, msg_id)
|
|
119
170
|
_logger.debug(f"Message processed | queue={queue} id={message.id}")
|
|
171
|
+
except VsMessageRejected as e:
|
|
172
|
+
await consumer.on_error(message, e)
|
|
173
|
+
if retry_policy and retry_policy.dead_letter_queue:
|
|
174
|
+
await self.publish(retry_policy.dead_letter_queue, message)
|
|
175
|
+
await self._client.xack(queue, group, msg_id)
|
|
120
176
|
except Exception as e:
|
|
121
177
|
await consumer.on_error(message, e)
|
|
122
178
|
if retry_policy:
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
await self._client.xack(queue, group, msg_id)
|
|
179
|
+
await retry_policy.handle_failure(message, e, self)
|
|
180
|
+
await self._client.xack(queue, group, msg_id)
|
|
126
181
|
else:
|
|
127
182
|
_logger.error(f"Message failed, no retry policy | id={message.id} error={e}")
|
|
128
183
|
finally:
|
|
File without changes
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: vs-queue
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.5
|
|
4
4
|
Summary: Pluggable async message queue library for Viveka Sutra — Redis Streams, RabbitMQ, retry, dead letter, and annotation-based consumers
|
|
5
5
|
License: Apache License
|
|
6
6
|
Version 2.0, January 2004
|
|
@@ -202,7 +202,7 @@ Description-Content-Type: text/markdown
|
|
|
202
202
|
License-File: LICENSE.txt
|
|
203
203
|
Requires-Dist: pydantic>=2.0
|
|
204
204
|
Provides-Extra: redis
|
|
205
|
-
Requires-Dist: redis>=
|
|
205
|
+
Requires-Dist: redis>=8.1; extra == "redis"
|
|
206
206
|
Requires-Dist: hiredis>=2.0; extra == "redis"
|
|
207
207
|
Provides-Extra: rabbitmq
|
|
208
208
|
Requires-Dist: aio-pika>=9.0; extra == "rabbitmq"
|
|
@@ -1246,3 +1246,11 @@ Decorator. Registers a `VsBaseConsumer` subclass for auto-discovery by `VsQueueM
|
|
|
1246
1246
|
```bash
|
|
1247
1247
|
./run_tests.sh
|
|
1248
1248
|
```
|
|
1249
|
+
|
|
1250
|
+
## Agent request/reply integration
|
|
1251
|
+
|
|
1252
|
+
`vs-agent[queue]` provides `VsQueueActionConsumer`, using `VsBaseConsumer`, `VsQueueManager`, `VsMessage`, and the selected provider. Callers place `reply_to`, `correlation_id`, and authentication credentials in message headers. The agent validates/authenticates, executes guards and the action, then publishes a new response before returning for acknowledgement. Response headers never inherit credentials.
|
|
1253
|
+
|
|
1254
|
+
`VsQueueManager.register(queue_name, consumer, retry_policy=...)` forwards retry policies to providers; `register_listeners()` applies the retry settings declared by `@queue_listener`. `VsMessageRejected` marks permanent failures, which bypass retries and are dead-lettered when configured. Retry/DLQ publication failures must not acknowledge the original delivery. Providers overwrite `source_queue` with the actual subscribed queue before retrying; callers cannot redirect retries through that header.
|
|
1255
|
+
|
|
1256
|
+
Redis replays the current consumer's pending entries after a subscription restart and periodically uses XAUTOCLAIM to recover stale deliveries from dead consumers (Redis servers supporting XAUTOCLAIM). Its `credentials["claim_idle_ms"]` option defaults to 300000; set it above the longest expected processing time, or zero to disable reclaim. An overdue message may execute more than once. Durable idempotency is the application's responsibility. Older servers log when stale recovery is unavailable.
|
|
@@ -13,6 +13,8 @@ vs_queue/base/vs_base_consumer.py
|
|
|
13
13
|
vs_queue/base/vs_base_queue.py
|
|
14
14
|
vs_queue/decorator/__init__.py
|
|
15
15
|
vs_queue/decorator/vs_queue_listener.py
|
|
16
|
+
vs_queue/error/__init__.py
|
|
17
|
+
vs_queue/error/vs_message_rejected.py
|
|
16
18
|
vs_queue/factory/__init__.py
|
|
17
19
|
vs_queue/factory/vs_queue_factory.py
|
|
18
20
|
vs_queue/manager/__init__.py
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|