vs-queue 0.1.4__tar.gz → 0.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. {vs_queue-0.1.4/vs_queue.egg-info → vs_queue-0.2.0}/PKG-INFO +101 -8
  2. {vs_queue-0.1.4 → vs_queue-0.2.0}/README.md +100 -7
  3. {vs_queue-0.1.4 → vs_queue-0.2.0}/pyproject.toml +1 -1
  4. vs_queue-0.2.0/vs_queue/base/vs_base_queue.py +111 -0
  5. vs_queue-0.2.0/vs_queue/error/vs_message_rejected.py +2 -0
  6. {vs_queue-0.1.4 → vs_queue-0.2.0}/vs_queue/manager/vs_queue_manager.py +24 -7
  7. vs_queue-0.2.0/vs_queue/provider/vs_rabbitmq_queue.py +157 -0
  8. vs_queue-0.2.0/vs_queue/provider/vs_redis_queue.py +273 -0
  9. vs_queue-0.2.0/vs_queue/schema/__init__.py +0 -0
  10. {vs_queue-0.1.4 → vs_queue-0.2.0}/vs_queue/schema/vs_message.py +1 -1
  11. {vs_queue-0.1.4 → vs_queue-0.2.0}/vs_queue/schema/vs_retention.py +4 -0
  12. vs_queue-0.2.0/vs_queue/security/__init__.py +0 -0
  13. vs_queue-0.2.0/vs_queue/security/vs_connection_security.py +66 -0
  14. vs_queue-0.2.0/vs_queue/security/vs_message_signer.py +60 -0
  15. {vs_queue-0.1.4 → vs_queue-0.2.0/vs_queue.egg-info}/PKG-INFO +101 -8
  16. {vs_queue-0.1.4 → vs_queue-0.2.0}/vs_queue.egg-info/SOURCES.txt +6 -1
  17. vs_queue-0.1.4/vs_queue/base/vs_base_queue.py +0 -48
  18. vs_queue-0.1.4/vs_queue/provider/vs_rabbitmq_queue.py +0 -108
  19. vs_queue-0.1.4/vs_queue/provider/vs_redis_queue.py +0 -157
  20. {vs_queue-0.1.4 → vs_queue-0.2.0}/LICENSE.txt +0 -0
  21. {vs_queue-0.1.4 → vs_queue-0.2.0}/setup.cfg +0 -0
  22. {vs_queue-0.1.4 → vs_queue-0.2.0}/vs_queue/__init__.py +0 -0
  23. {vs_queue-0.1.4 → vs_queue-0.2.0}/vs_queue/base/__init__.py +0 -0
  24. {vs_queue-0.1.4 → vs_queue-0.2.0}/vs_queue/base/vs_base_consumer.py +0 -0
  25. {vs_queue-0.1.4 → vs_queue-0.2.0}/vs_queue/decorator/__init__.py +0 -0
  26. {vs_queue-0.1.4 → vs_queue-0.2.0}/vs_queue/decorator/vs_queue_listener.py +0 -0
  27. {vs_queue-0.1.4/vs_queue/factory → vs_queue-0.2.0/vs_queue/error}/__init__.py +0 -0
  28. {vs_queue-0.1.4/vs_queue/manager → vs_queue-0.2.0/vs_queue/factory}/__init__.py +0 -0
  29. {vs_queue-0.1.4 → vs_queue-0.2.0}/vs_queue/factory/vs_queue_factory.py +0 -0
  30. {vs_queue-0.1.4/vs_queue/provider → vs_queue-0.2.0/vs_queue/manager}/__init__.py +0 -0
  31. {vs_queue-0.1.4/vs_queue/registry → vs_queue-0.2.0/vs_queue/provider}/__init__.py +0 -0
  32. {vs_queue-0.1.4 → vs_queue-0.2.0}/vs_queue/py.typed +0 -0
  33. {vs_queue-0.1.4/vs_queue/retry → vs_queue-0.2.0/vs_queue/registry}/__init__.py +0 -0
  34. {vs_queue-0.1.4 → vs_queue-0.2.0}/vs_queue/registry/vs_queue_registry.py +0 -0
  35. {vs_queue-0.1.4/vs_queue/schema → vs_queue-0.2.0/vs_queue/retry}/__init__.py +0 -0
  36. {vs_queue-0.1.4 → vs_queue-0.2.0}/vs_queue/retry/vs_retry_policy.py +0 -0
  37. {vs_queue-0.1.4 → vs_queue-0.2.0}/vs_queue.egg-info/dependency_links.txt +0 -0
  38. {vs_queue-0.1.4 → vs_queue-0.2.0}/vs_queue.egg-info/requires.txt +0 -0
  39. {vs_queue-0.1.4 → vs_queue-0.2.0}/vs_queue.egg-info/top_level.txt +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: vs-queue
3
- Version: 0.1.4
3
+ Version: 0.2.0
4
4
  Summary: Pluggable async message queue library for Viveka Sutra — Redis Streams, RabbitMQ, retry, dead letter, and annotation-based consumers
5
5
  License: Apache License
6
6
  Version 2.0, January 2004
@@ -414,6 +414,56 @@ password = guest
414
414
  vhost = /
415
415
  ```
416
416
 
417
+ ### Connection security
418
+
419
+ TLS and authentication are set through the `credentials` dict passed to the provider or to `VsQueueFactory.create`. TLS is off by default, so a local broker works with no extra settings.
420
+
421
+ | Key | Default | Description |
422
+ |---|---|---|
423
+ | `ssl` | `False` | Encrypt the connection with TLS. The server certificate is verified. |
424
+ | `ssl_ca_certs` | system CAs | Path to a CA bundle for verifying the broker certificate |
425
+ | `ssl_certfile` | — | Client certificate, for mutual TLS |
426
+ | `ssl_keyfile` | — | Private key for the client certificate |
427
+ | `ssl_check_hostname` | `True` | Check that the certificate matches the broker host |
428
+ | `username` | — | Redis ACL user or RabbitMQ user |
429
+ | `password` | — | Password for that user |
430
+
431
+ Boolean values may be real booleans or the strings `true` / `false`, so they can come straight from a config file.
432
+
433
+ ```python
434
+ queue = VsQueueFactory.create(
435
+ provider="redis",
436
+ host="redis.internal",
437
+ port=6380,
438
+ credentials={
439
+ "username": "queue_worker",
440
+ "password": "secret",
441
+ "ssl": True,
442
+ "ssl_ca_certs": "/etc/ssl/certs/internal-ca.pem",
443
+ },
444
+ )
445
+ ```
446
+
447
+ When the host is not `localhost`, `127.0.0.1` or `::1`, `connect()` logs a warning if TLS is off and another if no credentials were given. It still connects — the warnings are there so an insecure remote setup does not go unnoticed. RabbitMQ falls back to `guest` for the username and for the password, each independently, when it is not supplied.
448
+
449
+ ### Message signing
450
+
451
+ Anything with write access to the broker can otherwise publish a message that consumers will trust. To prevent that, give every publisher and consumer of a queue the same `signing_key`:
452
+
453
+ | Key | Default | Description |
454
+ |---|---|---|
455
+ | `signing_key` | — | Shared secret, at least 32 characters. Turns signing and verification on. |
456
+
457
+ With a key set:
458
+
459
+ - `publish()` adds an HMAC-SHA256 signature in the `x-vs-signature` header, covering `id`, `timestamp`, `retry_count`, `headers` and `payload`.
460
+ - Consumers verify it before `handle()` runs. An unsigned or tampered message is never handled or retried; it is dead-lettered with `reject_reason=bad_signature` (when a DLQ is configured) and acknowledged.
461
+ - The signature header is removed before the message reaches `handle()`, and re-signed whenever the library re-publishes (retries, DLQ).
462
+
463
+ Signing is opt-in. Without a key nothing is signed or checked, and a remote broker logs a warning at `connect()`. Because publishers and consumers must agree, roll the key out to every service before relying on it — a consumer with a key rejects messages from a publisher without one.
464
+
465
+ Signing does not stop replay: a captured, validly signed message can be published again. Make handlers idempotent on `message.id` where that matters.
466
+
417
467
  ---
418
468
 
419
469
  ## VsMessage
@@ -435,7 +485,7 @@ message = VsMessage(
435
485
  |---|---|---|---|
436
486
  | `id` | `str` | `uuid4().hex` | Unique message ID, auto-generated |
437
487
  | `timestamp` | `datetime` | `now(UTC)` | Message creation time, auto-set |
438
- | `retry_count` | `int` | `0` | Number of processing attempts. Managed by the library — do not set manually. |
488
+ | `retry_count` | `int` | `0` | Number of processing attempts. Managed by the library — do not set manually. Must be `>= 0`; a negative value on the wire is treated as malformed. |
439
489
  | `headers` | `Dict[str, str]` | `{}` | Caller-defined metadata: type, trace_id, routing info, etc. |
440
490
  | `payload` | `Dict[str, Any]` | `None` | Message body. Structure is defined by the caller. |
441
491
 
@@ -732,6 +782,22 @@ class DlqConsumer(VsBaseConsumer):
732
782
  )
733
783
  ```
734
784
 
785
+ **Malformed messages:**
786
+
787
+ A delivery that cannot be parsed into a `VsMessage` (missing fields, invalid JSON, wrong types) is never passed to `handle` and never retried. The provider logs it, publishes a new message to the DLQ when one is configured, and acknowledges the original so it cannot be redelivered. With no DLQ it is logged and dropped. The DLQ message looks like this:
788
+
789
+ | Field | Value |
790
+ |---|---|
791
+ | `headers["reject_reason"]` | `malformed`, or `bad_signature` when [signing](#message-signing) is on and verification failed |
792
+ | `headers["source_queue"]` | Queue the bad delivery arrived on |
793
+ | `headers["source_id"]` | Redis stream ID of the bad entry (empty for RabbitMQ) |
794
+ | `headers["error"]` | Exception class raised while parsing |
795
+ | `payload["raw"]` | The delivery exactly as received |
796
+
797
+ Redis entries that come back empty because retention trimmed them while still pending are acknowledged and skipped without dead-lettering.
798
+
799
+ **DLQ retention (Redis):** nothing consumes a DLQ by default, so on `subscribe()` a Redis DLQ with no retention policy of its own gets `DEFAULT_DLQ_RETENTION` — 7 days and at most 10,000 messages. Register your own with `set_retention()` to change it. RabbitMQ DLQs are left alone, because changing a live queue's arguments makes RabbitMQ refuse the declare.
800
+
735
801
  **Example — replaying a DLQ message:**
736
802
 
737
803
  ```python
@@ -768,7 +834,7 @@ queue.set_retention("vs_trace:logs", VsRetention(ttl_seconds=604800))
768
834
  | `ttl_seconds` | Drop messages older than this |
769
835
  | `max_len` | Keep at most this many messages |
770
836
 
771
- Applied the first time the queue is published to or subscribed. Queues with no policy are unbounded, same as before.
837
+ Applied the first time the queue is published to or subscribed. Queues with no policy have no age or length cap. On Redis, processed messages are still removed (see [delete on ack](#redis-streams-backend)), so only unprocessed backlog builds up. Redis DLQs get a default policy — see [Dead Letter Queue](#dead-letter-queue).
772
838
 
773
839
  **Redis Streams** — no native per-message TTL, so:
774
840
  - `max_len` → `XADD ... MAXLEN ~ n` on every publish
@@ -825,14 +891,20 @@ logging.getLogger("vs_queue").setLevel(logging.DEBUG)
825
891
  **How it works:**
826
892
  - `publish` → `XADD queue * field value ...`
827
893
  - `subscribe` → `XREADGROUP GROUP group consumer STREAMS queue >`
828
- - On success → `XACK` removes the message from the pending entries list
894
+ - On success → `XACK` removes the message from the pending entries list, then `XDEL` deletes the entry from the stream
895
+ - Every acknowledged entry (success, retried, dead-lettered, malformed) is deleted the same way, so finished work and its payload do not stay in Redis. A failed `XDEL` is logged and does not trigger a retry. Set `credentials["delete_on_ack"] = False` if something else replays the stream.
829
896
  - On failure → retry policy runs; if max retries exceeded → DLQ
830
897
 
831
898
  **Credentials:**
832
899
 
833
900
  | Key | Description |
834
901
  |---|---|
902
+ | `username` | Redis ACL user. Omit to authenticate as the `default` user. |
835
903
  | `password` | Redis AUTH password |
904
+ | `ssl`, `ssl_*` | TLS settings — see [Connection security](#connection-security) |
905
+ | `claim_idle_ms` | Idle time before a stale delivery is reclaimed (default `300000`, `0` disables) |
906
+ | `delete_on_ack` | Delete stream entries once acknowledged (default `True`) |
907
+ | `signing_key` | Shared HMAC key — see [Message signing](#message-signing) |
836
908
 
837
909
  **Example:**
838
910
 
@@ -868,6 +940,10 @@ These are created automatically on first `subscribe()`. If the stream does not e
868
940
  | `username` | `guest` | RabbitMQ username |
869
941
  | `password` | `guest` | RabbitMQ password |
870
942
  | `vhost` | `/` | RabbitMQ virtual host |
943
+ | `ssl`, `ssl_*` | off | TLS settings — see [Connection security](#connection-security) |
944
+ | `signing_key` | — | Shared HMAC key — see [Message signing](#message-signing) |
945
+
946
+ Each of `username` and `password` falls back to `guest` when it is missing, and a remote broker without both logs a warning.
871
947
 
872
948
  **Example:**
873
949
 
@@ -885,7 +961,8 @@ await queue.connect()
885
961
  **Acknowledgement behaviour:**
886
962
  - On success → `ack()` removes the message
887
963
  - On failure with retry → `ack()` the original, re-publish with incremented `retry_count`
888
- - On failure beyond max retries → `nack(requeue=False)` + publish to DLQ
964
+ - On failure beyond max retries → publish to DLQ, then `ack()` the original
965
+ - On a malformed body → publish the raw body to the DLQ (when configured) and `ack()`
889
966
 
890
967
  ---
891
968
 
@@ -1125,7 +1202,7 @@ Standard message envelope. Every message published and consumed by `vs-queue` is
1125
1202
  |---|---|---|---|
1126
1203
  | `id` | `str` | `uuid4().hex` | Auto-generated unique ID |
1127
1204
  | `timestamp` | `datetime` | `now(UTC)` | Auto-set creation time |
1128
- | `retry_count` | `int` | `0` | Managed by retry policy |
1205
+ | `retry_count` | `int` | `0` | Managed by retry policy. Must be `>= 0`. |
1129
1206
  | `headers` | `Dict[str, str]` | `{}` | Caller metadata |
1130
1207
  | `payload` | `Dict[str, Any]` | `None` | Message body |
1131
1208
 
@@ -1174,7 +1251,7 @@ Abstract base. Extend and implement `handle` to process messages.
1174
1251
  | `handle(message)` | Yes | Process a message. Raise to trigger retry. |
1175
1252
  | `start()` | No — no-op by default | Setup before consuming begins |
1176
1253
  | `stop()` | No — no-op by default | Cleanup on shutdown — called on graceful cancel |
1177
- | `on_error(message, error)` | No — no-op by default | Called on `handle` failure before retry |
1254
+ | `on_error(message, error)` | No — no-op by default | Called on `handle` failure before retry. An exception raised here is logged and does not stop the consumer. |
1178
1255
 
1179
1256
  ---
1180
1257
 
@@ -1182,6 +1259,12 @@ Abstract base. Extend and implement `handle` to process messages.
1182
1259
 
1183
1260
  Lifecycle manager for all consumers.
1184
1261
 
1262
+ | Constructor parameter | Default | Description |
1263
+ |---|---|---|
1264
+ | `queue` | — | The provider to consume from |
1265
+ | `max_consumer_restarts` | `3` | Restarts allowed before a crashing consumer is given up on |
1266
+ | `healthy_after_seconds` | `60.0` | A consumer that ran this long before crashing gets its restart count reset |
1267
+
1185
1268
  | Method | Description |
1186
1269
  |---|---|
1187
1270
  | `register(queue_name, consumer)` | Register a consumer, auto-connect, start immediately. Returns UUID. |
@@ -1243,6 +1326,16 @@ Decorator. Registers a `VsBaseConsumer` subclass for auto-discovery by `VsQueueM
1243
1326
 
1244
1327
  ## Running Tests
1245
1328
 
1329
+ The provider, security and manager tests live in `vs-agent/tests/test_queue_providers.py` and use broker fakes, so no Redis or RabbitMQ is needed:
1330
+
1246
1331
  ```bash
1247
- ./run_tests.sh
1332
+ cd ../vs-agent && PYTHONPATH=../vs-queue python -m unittest tests.test_queue_providers
1248
1333
  ```
1334
+
1335
+ ## Agent request/reply integration
1336
+
1337
+ `vs-agent[queue]` provides `VsQueueActionConsumer`, using `VsBaseConsumer`, `VsQueueManager`, `VsMessage`, and the selected provider. Callers place `reply_to`, `correlation_id`, and authentication credentials in message headers. The agent validates/authenticates, executes guards and the action, then publishes a new response before returning for acknowledgement. Response headers never inherit credentials.
1338
+
1339
+ `VsQueueManager.register(queue_name, consumer, retry_policy=...)` forwards retry policies to providers; `register_listeners()` applies the retry settings declared by `@queue_listener`. `VsMessageRejected` marks permanent failures, which bypass retries and are dead-lettered when configured. Retry/DLQ publication failures must not acknowledge the original delivery. Providers overwrite `source_queue` with the actual subscribed queue before retrying; callers cannot redirect retries through that header.
1340
+
1341
+ Redis replays the current consumer's pending entries after a subscription restart and periodically uses XAUTOCLAIM to recover stale deliveries from dead consumers (Redis servers supporting XAUTOCLAIM). Its `credentials["claim_idle_ms"]` option defaults to 300000; set it above the longest expected processing time, or zero to disable reclaim. An overdue message may execute more than once. Durable idempotency is the application's responsibility. Older servers log when stale recovery is unavailable.
@@ -199,6 +199,56 @@ password = guest
199
199
  vhost = /
200
200
  ```
201
201
 
202
+ ### Connection security
203
+
204
+ TLS and authentication are set through the `credentials` dict passed to the provider or to `VsQueueFactory.create`. TLS is off by default, so a local broker works with no extra settings.
205
+
206
+ | Key | Default | Description |
207
+ |---|---|---|
208
+ | `ssl` | `False` | Encrypt the connection with TLS. The server certificate is verified. |
209
+ | `ssl_ca_certs` | system CAs | Path to a CA bundle for verifying the broker certificate |
210
+ | `ssl_certfile` | — | Client certificate, for mutual TLS |
211
+ | `ssl_keyfile` | — | Private key for the client certificate |
212
+ | `ssl_check_hostname` | `True` | Check that the certificate matches the broker host |
213
+ | `username` | — | Redis ACL user or RabbitMQ user |
214
+ | `password` | — | Password for that user |
215
+
216
+ Boolean values may be real booleans or the strings `true` / `false`, so they can come straight from a config file.
217
+
218
+ ```python
219
+ queue = VsQueueFactory.create(
220
+ provider="redis",
221
+ host="redis.internal",
222
+ port=6380,
223
+ credentials={
224
+ "username": "queue_worker",
225
+ "password": "secret",
226
+ "ssl": True,
227
+ "ssl_ca_certs": "/etc/ssl/certs/internal-ca.pem",
228
+ },
229
+ )
230
+ ```
231
+
232
+ When the host is not `localhost`, `127.0.0.1` or `::1`, `connect()` logs a warning if TLS is off and another if no credentials were given. It still connects — the warnings are there so an insecure remote setup does not go unnoticed. RabbitMQ falls back to `guest` for the username and for the password, each independently, when it is not supplied.
233
+
234
+ ### Message signing
235
+
236
+ Anything with write access to the broker can otherwise publish a message that consumers will trust. To prevent that, give every publisher and consumer of a queue the same `signing_key`:
237
+
238
+ | Key | Default | Description |
239
+ |---|---|---|
240
+ | `signing_key` | — | Shared secret, at least 32 characters. Turns signing and verification on. |
241
+
242
+ With a key set:
243
+
244
+ - `publish()` adds an HMAC-SHA256 signature in the `x-vs-signature` header, covering `id`, `timestamp`, `retry_count`, `headers` and `payload`.
245
+ - Consumers verify it before `handle()` runs. An unsigned or tampered message is never handled or retried; it is dead-lettered with `reject_reason=bad_signature` (when a DLQ is configured) and acknowledged.
246
+ - The signature header is removed before the message reaches `handle()`, and re-signed whenever the library re-publishes (retries, DLQ).
247
+
248
+ Signing is opt-in. Without a key nothing is signed or checked, and a remote broker logs a warning at `connect()`. Because publishers and consumers must agree, roll the key out to every service before relying on it — a consumer with a key rejects messages from a publisher without one.
249
+
250
+ Signing does not stop replay: a captured, validly signed message can be published again. Make handlers idempotent on `message.id` where that matters.
251
+
202
252
  ---
203
253
 
204
254
  ## VsMessage
@@ -220,7 +270,7 @@ message = VsMessage(
220
270
  |---|---|---|---|
221
271
  | `id` | `str` | `uuid4().hex` | Unique message ID, auto-generated |
222
272
  | `timestamp` | `datetime` | `now(UTC)` | Message creation time, auto-set |
223
- | `retry_count` | `int` | `0` | Number of processing attempts. Managed by the library — do not set manually. |
273
+ | `retry_count` | `int` | `0` | Number of processing attempts. Managed by the library — do not set manually. Must be `>= 0`; a negative value on the wire is treated as malformed. |
224
274
  | `headers` | `Dict[str, str]` | `{}` | Caller-defined metadata: type, trace_id, routing info, etc. |
225
275
  | `payload` | `Dict[str, Any]` | `None` | Message body. Structure is defined by the caller. |
226
276
 
@@ -517,6 +567,22 @@ class DlqConsumer(VsBaseConsumer):
517
567
  )
518
568
  ```
519
569
 
570
+ **Malformed messages:**
571
+
572
+ A delivery that cannot be parsed into a `VsMessage` (missing fields, invalid JSON, wrong types) is never passed to `handle` and never retried. The provider logs it, publishes a new message to the DLQ when one is configured, and acknowledges the original so it cannot be redelivered. With no DLQ it is logged and dropped. The DLQ message looks like this:
573
+
574
+ | Field | Value |
575
+ |---|---|
576
+ | `headers["reject_reason"]` | `malformed`, or `bad_signature` when [signing](#message-signing) is on and verification failed |
577
+ | `headers["source_queue"]` | Queue the bad delivery arrived on |
578
+ | `headers["source_id"]` | Redis stream ID of the bad entry (empty for RabbitMQ) |
579
+ | `headers["error"]` | Exception class raised while parsing |
580
+ | `payload["raw"]` | The delivery exactly as received |
581
+
582
+ Redis entries that come back empty because retention trimmed them while still pending are acknowledged and skipped without dead-lettering.
583
+
584
+ **DLQ retention (Redis):** nothing consumes a DLQ by default, so on `subscribe()` a Redis DLQ with no retention policy of its own gets `DEFAULT_DLQ_RETENTION` — 7 days and at most 10,000 messages. Register your own with `set_retention()` to change it. RabbitMQ DLQs are left alone, because changing a live queue's arguments makes RabbitMQ refuse the declare.
585
+
520
586
  **Example — replaying a DLQ message:**
521
587
 
522
588
  ```python
@@ -553,7 +619,7 @@ queue.set_retention("vs_trace:logs", VsRetention(ttl_seconds=604800))
553
619
  | `ttl_seconds` | Drop messages older than this |
554
620
  | `max_len` | Keep at most this many messages |
555
621
 
556
- Applied the first time the queue is published to or subscribed. Queues with no policy are unbounded, same as before.
622
+ Applied the first time the queue is published to or subscribed. Queues with no policy have no age or length cap. On Redis, processed messages are still removed (see [delete on ack](#redis-streams-backend)), so only unprocessed backlog builds up. Redis DLQs get a default policy — see [Dead Letter Queue](#dead-letter-queue).
557
623
 
558
624
  **Redis Streams** — no native per-message TTL, so:
559
625
  - `max_len` → `XADD ... MAXLEN ~ n` on every publish
@@ -610,14 +676,20 @@ logging.getLogger("vs_queue").setLevel(logging.DEBUG)
610
676
  **How it works:**
611
677
  - `publish` → `XADD queue * field value ...`
612
678
  - `subscribe` → `XREADGROUP GROUP group consumer STREAMS queue >`
613
- - On success → `XACK` removes the message from the pending entries list
679
+ - On success → `XACK` removes the message from the pending entries list, then `XDEL` deletes the entry from the stream
680
+ - Every acknowledged entry (success, retried, dead-lettered, malformed) is deleted the same way, so finished work and its payload do not stay in Redis. A failed `XDEL` is logged and does not trigger a retry. Set `credentials["delete_on_ack"] = False` if something else replays the stream.
614
681
  - On failure → retry policy runs; if max retries exceeded → DLQ
615
682
 
616
683
  **Credentials:**
617
684
 
618
685
  | Key | Description |
619
686
  |---|---|
687
+ | `username` | Redis ACL user. Omit to authenticate as the `default` user. |
620
688
  | `password` | Redis AUTH password |
689
+ | `ssl`, `ssl_*` | TLS settings — see [Connection security](#connection-security) |
690
+ | `claim_idle_ms` | Idle time before a stale delivery is reclaimed (default `300000`, `0` disables) |
691
+ | `delete_on_ack` | Delete stream entries once acknowledged (default `True`) |
692
+ | `signing_key` | Shared HMAC key — see [Message signing](#message-signing) |
621
693
 
622
694
  **Example:**
623
695
 
@@ -653,6 +725,10 @@ These are created automatically on first `subscribe()`. If the stream does not e
653
725
  | `username` | `guest` | RabbitMQ username |
654
726
  | `password` | `guest` | RabbitMQ password |
655
727
  | `vhost` | `/` | RabbitMQ virtual host |
728
+ | `ssl`, `ssl_*` | off | TLS settings — see [Connection security](#connection-security) |
729
+ | `signing_key` | — | Shared HMAC key — see [Message signing](#message-signing) |
730
+
731
+ Each of `username` and `password` falls back to `guest` when it is missing, and a remote broker without both logs a warning.
656
732
 
657
733
  **Example:**
658
734
 
@@ -670,7 +746,8 @@ await queue.connect()
670
746
  **Acknowledgement behaviour:**
671
747
  - On success → `ack()` removes the message
672
748
  - On failure with retry → `ack()` the original, re-publish with incremented `retry_count`
673
- - On failure beyond max retries → `nack(requeue=False)` + publish to DLQ
749
+ - On failure beyond max retries → publish to DLQ, then `ack()` the original
750
+ - On a malformed body → publish the raw body to the DLQ (when configured) and `ack()`
674
751
 
675
752
  ---
676
753
 
@@ -910,7 +987,7 @@ Standard message envelope. Every message published and consumed by `vs-queue` is
910
987
  |---|---|---|---|
911
988
  | `id` | `str` | `uuid4().hex` | Auto-generated unique ID |
912
989
  | `timestamp` | `datetime` | `now(UTC)` | Auto-set creation time |
913
- | `retry_count` | `int` | `0` | Managed by retry policy |
990
+ | `retry_count` | `int` | `0` | Managed by retry policy. Must be `>= 0`. |
914
991
  | `headers` | `Dict[str, str]` | `{}` | Caller metadata |
915
992
  | `payload` | `Dict[str, Any]` | `None` | Message body |
916
993
 
@@ -959,7 +1036,7 @@ Abstract base. Extend and implement `handle` to process messages.
959
1036
  | `handle(message)` | Yes | Process a message. Raise to trigger retry. |
960
1037
  | `start()` | No — no-op by default | Setup before consuming begins |
961
1038
  | `stop()` | No — no-op by default | Cleanup on shutdown — called on graceful cancel |
962
- | `on_error(message, error)` | No — no-op by default | Called on `handle` failure before retry |
1039
+ | `on_error(message, error)` | No — no-op by default | Called on `handle` failure before retry. An exception raised here is logged and does not stop the consumer. |
963
1040
 
964
1041
  ---
965
1042
 
@@ -967,6 +1044,12 @@ Abstract base. Extend and implement `handle` to process messages.
967
1044
 
968
1045
  Lifecycle manager for all consumers.
969
1046
 
1047
+ | Constructor parameter | Default | Description |
1048
+ |---|---|---|
1049
+ | `queue` | — | The provider to consume from |
1050
+ | `max_consumer_restarts` | `3` | Restarts allowed before a crashing consumer is given up on |
1051
+ | `healthy_after_seconds` | `60.0` | A consumer that ran this long before crashing gets its restart count reset |
1052
+
970
1053
  | Method | Description |
971
1054
  |---|---|
972
1055
  | `register(queue_name, consumer)` | Register a consumer, auto-connect, start immediately. Returns UUID. |
@@ -1028,6 +1111,16 @@ Decorator. Registers a `VsBaseConsumer` subclass for auto-discovery by `VsQueueM
1028
1111
 
1029
1112
  ## Running Tests
1030
1113
 
1114
+ The provider, security and manager tests live in `vs-agent/tests/test_queue_providers.py` and use broker fakes, so no Redis or RabbitMQ is needed:
1115
+
1031
1116
  ```bash
1032
- ./run_tests.sh
1117
+ cd ../vs-agent && PYTHONPATH=../vs-queue python -m unittest tests.test_queue_providers
1033
1118
  ```
1119
+
1120
+ ## Agent request/reply integration
1121
+
1122
+ `vs-agent[queue]` provides `VsQueueActionConsumer`, using `VsBaseConsumer`, `VsQueueManager`, `VsMessage`, and the selected provider. Callers place `reply_to`, `correlation_id`, and authentication credentials in message headers. The agent validates/authenticates, executes guards and the action, then publishes a new response before returning for acknowledgement. Response headers never inherit credentials.
1123
+
1124
+ `VsQueueManager.register(queue_name, consumer, retry_policy=...)` forwards retry policies to providers; `register_listeners()` applies the retry settings declared by `@queue_listener`. `VsMessageRejected` marks permanent failures, which bypass retries and are dead-lettered when configured. Retry/DLQ publication failures must not acknowledge the original delivery. Providers overwrite `source_queue` with the actual subscribed queue before retrying; callers cannot redirect retries through that header.
1125
+
1126
+ Redis replays the current consumer's pending entries after a subscription restart and periodically uses XAUTOCLAIM to recover stale deliveries from dead consumers (Redis servers supporting XAUTOCLAIM). Its `credentials["claim_idle_ms"]` option defaults to 300000; set it above the longest expected processing time, or zero to disable reclaim. An overdue message may execute more than once. Durable idempotency is the application's responsibility. Older servers log when stale recovery is unavailable.
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "vs-queue"
7
- version = "0.1.4"
7
+ version = "0.2.0"
8
8
  description = "Pluggable async message queue library for Viveka Sutra — Redis Streams, RabbitMQ, retry, dead letter, and annotation-based consumers"
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.11"
@@ -0,0 +1,111 @@
1
+ import logging
2
+ from abc import ABC, abstractmethod
3
+ from typing import Any, Dict, Optional
4
+
5
+ from vs_queue.schema.vs_message import VsMessage
6
+ from vs_queue.schema.vs_retention import VsRetention
7
+ from vs_queue.security.vs_connection_security import VsConnectionSecurity
8
+ from vs_queue.security.vs_message_signer import SIGNATURE_HEADER, VsMessageSigner, VsSignatureError
9
+
10
+ _logger = logging.getLogger(__name__)
11
+
12
+
13
+ class VsBaseQueue(ABC):
14
+
15
+ def __init__(self, host: str, port: int, credentials: Optional[Dict[str, Any]] = None):
16
+ self._host = host
17
+ self._port = port
18
+ self._credentials = credentials or {}
19
+ self._security = VsConnectionSecurity(host, self._credentials)
20
+ self._signer = VsMessageSigner.from_credentials(self._credentials)
21
+ self._retention: Dict[str, VsRetention] = {}
22
+
23
+ def set_retention(self, queue: str, retention: VsRetention) -> None:
24
+ """Register a retention policy for a queue. Applied the first time the
25
+ queue is published to or subscribed. Call before connect()."""
26
+ self._retention[queue] = retention
27
+
28
+ def _retention_for(self, queue: str) -> Optional[VsRetention]:
29
+ return self._retention.get(queue)
30
+
31
+ def _wire_headers(self, message: VsMessage, timestamp: str) -> Dict[str, str]:
32
+ """Headers to put on the wire, freshly signed when signing is on.
33
+ Any signature the message arrived with is dropped — we sign what we send."""
34
+ headers = {key: value for key, value in message.headers.items() if key != SIGNATURE_HEADER}
35
+ if self._signer:
36
+ headers[SIGNATURE_HEADER] = self._signer.sign(
37
+ message.id, timestamp, message.retry_count, headers, message.payload or {},
38
+ )
39
+ return headers
40
+
41
+ def _verify(self, message: VsMessage, timestamp: Any) -> VsMessage:
42
+ """Check the signature against the wire timestamp, then strip it off.
43
+ Raises VsSignatureError when signing is on and the message doesn't add up."""
44
+ # Step 1: take the signature off so handlers never see or forward it
45
+ signature = message.headers.pop(SIGNATURE_HEADER, None)
46
+
47
+ # Step 2: no signer means no checking — that's the opt-in deal
48
+ if self._signer:
49
+ self._signer.verify(
50
+ signature, message.id, timestamp, message.retry_count,
51
+ message.headers, message.payload or {},
52
+ )
53
+ return message
54
+
55
+ def _warn_if_unsigned(self, provider: str) -> None:
56
+ """Nag once at connect when a remote broker's messages go unsigned."""
57
+ if not self._signer and not self._security.is_local():
58
+ _logger.warning(
59
+ f"{provider} messages are not signed | host={self._host} "
60
+ f"— set credentials['signing_key'] on every publisher and consumer"
61
+ )
62
+
63
+ @staticmethod
64
+ def _malformed_envelope(queue: str, source_id: Any, raw: Any, error: Exception) -> VsMessage:
65
+ """Wrap an unparseable or unverified delivery in a fresh message for the DLQ.
66
+ The original never became a trusted VsMessage, so it rides along as raw payload."""
67
+ reason = "bad_signature" if isinstance(error, VsSignatureError) else "malformed"
68
+ return VsMessage(
69
+ headers={
70
+ "reject_reason": reason,
71
+ "source_queue": queue,
72
+ "source_id": str(source_id),
73
+ "error": type(error).__name__,
74
+ },
75
+ payload={"raw": raw},
76
+ )
77
+
78
+ @staticmethod
79
+ async def _notify_error(consumer: "VsBaseConsumer", message: VsMessage, error: Exception) -> None:
80
+ """Call consumer.on_error without letting a broken hook sink the consumer."""
81
+ try:
82
+ await consumer.on_error(message, error)
83
+ except Exception:
84
+ # The error handler had an error. Log it and move on with our lives.
85
+ _logger.exception(f"on_error hook failed | id={message.id!r}")
86
+
87
+ @abstractmethod
88
+ async def connect(self) -> None:
89
+ pass
90
+
91
+ @abstractmethod
92
+ async def disconnect(self) -> None:
93
+ pass
94
+
95
+ @abstractmethod
96
+ async def health_check(self) -> bool:
97
+ pass
98
+
99
+ @abstractmethod
100
+ async def publish(self, queue: str, message: VsMessage) -> None:
101
+ pass
102
+
103
+ @abstractmethod
104
+ async def subscribe(self, queue: str, consumer: "VsBaseConsumer", retry_policy=None) -> None:
105
+ pass
106
+
107
+ async def blpop(self, key: str, timeout: int = 30) -> Optional[str]:
108
+ raise NotImplementedError(f"{type(self).__name__} does not support blpop")
109
+
110
+
111
+ from vs_queue.base.vs_base_consumer import VsBaseConsumer
@@ -0,0 +1,2 @@
1
+ class VsMessageRejected(Exception):
2
+ """Permanent failure: dead-letter (when configured) rather than retrying."""
@@ -1,25 +1,28 @@
1
1
  import asyncio
2
2
  import logging
3
+ import time
3
4
  from typing import Dict, List, Optional, Tuple
4
5
  from uuid import uuid4
5
6
 
6
7
  from vs_queue.base.vs_base_consumer import VsBaseConsumer
7
8
  from vs_queue.base.vs_base_queue import VsBaseQueue
9
+ from vs_queue.retry.vs_retry_policy import VsRetryPolicy
8
10
 
9
11
  _logger = logging.getLogger(__name__)
10
12
 
11
13
 
12
14
  class VsQueueManager:
13
15
 
14
- def __init__(self, queue: VsBaseQueue, max_consumer_restarts: int = 3):
16
+ def __init__(self, queue: VsBaseQueue, max_consumer_restarts: int = 3, healthy_after_seconds: float = 60.0):
15
17
  self._queue = queue
16
18
  self._max_consumer_restarts = max_consumer_restarts
19
+ self._healthy_after_seconds = healthy_after_seconds
17
20
  self._consumers: Dict[str, Tuple[str, VsBaseConsumer]] = {} # uuid -> (queue_name, consumer)
18
21
  self._tasks: Dict[str, asyncio.Task] = {} # uuid -> task
19
22
  self._connected = False
20
23
  self._listeners_registered = False
21
24
 
22
- async def register(self, queue_name: str, consumer: VsBaseConsumer) -> str:
25
+ async def register(self, queue_name: str, consumer: VsBaseConsumer, retry_policy: Optional[VsRetryPolicy] = None) -> str:
23
26
  if not self._connected:
24
27
  await self._queue.connect()
25
28
  self._connected = True
@@ -29,7 +32,7 @@ class VsQueueManager:
29
32
  self._consumers[uid] = (queue_name, consumer)
30
33
 
31
34
  task = asyncio.create_task(
32
- self._run(queue_name, consumer),
35
+ self._run(queue_name, consumer, retry_policy),
33
36
  name=f"consumer:{queue_name}:{uid}",
34
37
  )
35
38
  self._tasks[uid] = task
@@ -44,7 +47,11 @@ class VsQueueManager:
44
47
  from vs_queue.decorator.vs_queue_listener import get_registered_listeners
45
48
  for consumer_class, meta in get_registered_listeners():
46
49
  for _ in range(meta.concurrency):
47
- await self.register(meta.queue, consumer_class())
50
+ await self.register(meta.queue, consumer_class(), VsRetryPolicy(
51
+ max_retries=meta.max_retries,
52
+ backoff_seconds=meta.retry_backoff_seconds,
53
+ dead_letter_queue=meta.dead_letter_queue,
54
+ ))
48
55
  _logger.info(f"Listeners registered | queue={meta.queue} concurrency={meta.concurrency}")
49
56
 
50
57
  def get_listeners(self) -> Dict[str, List[VsBaseConsumer]]:
@@ -53,6 +60,10 @@ class VsQueueManager:
53
60
  result.setdefault(queue_name, []).append(consumer)
54
61
  return result
55
62
 
63
+ def has_stopped_listeners(self) -> bool:
64
+ """Whether a registered subscription task has terminated."""
65
+ return any(task.done() for task in self._tasks.values())
66
+
56
67
  async def stop_listener(self, queue_name: str, uuid: Optional[str] = None) -> None:
57
68
  if uuid and uuid in self._consumers:
58
69
  actual_queue, _ = self._consumers[uuid]
@@ -92,16 +103,22 @@ class VsQueueManager:
92
103
  queue_name, _ = self._consumers.pop(uid, (None, None))
93
104
  _logger.info(f"Listener stopped | queue={queue_name} uuid={uid}")
94
105
 
95
- async def _run(self, queue_name: str, consumer: VsBaseConsumer) -> None:
106
+ async def _run(self, queue_name: str, consumer: VsBaseConsumer, retry_policy: Optional[VsRetryPolicy] = None) -> None:
96
107
  attempts = 0
97
108
  while True:
109
+ started = time.monotonic()
98
110
  try:
99
- await self._queue.subscribe(queue_name, consumer)
111
+ if retry_policy is None:
112
+ await self._queue.subscribe(queue_name, consumer)
113
+ else:
114
+ await self._queue.subscribe(queue_name, consumer, retry_policy=retry_policy)
100
115
  break
101
116
  except asyncio.CancelledError:
102
- await consumer.stop()
103
117
  break
104
118
  except Exception as e:
119
+ # A long healthy run earns a clean slate — old crashes are forgiven
120
+ if time.monotonic() - started >= self._healthy_after_seconds:
121
+ attempts = 0
105
122
  attempts += 1
106
123
  if attempts > self._max_consumer_restarts:
107
124
  _logger.critical(