taskqueue-toolkit 0.1.1__tar.gz → 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {taskqueue_toolkit-0.1.1 → taskqueue_toolkit-0.2.0}/PKG-INFO +1 -1
- {taskqueue_toolkit-0.1.1 → taskqueue_toolkit-0.2.0}/pyproject.toml +1 -1
- {taskqueue_toolkit-0.1.1 → taskqueue_toolkit-0.2.0}/pyproject.toml.orig +1 -1
- {taskqueue_toolkit-0.1.1 → taskqueue_toolkit-0.2.0}/src/taskqueue_toolkit/queue/pubsub.py +8 -0
- {taskqueue_toolkit-0.1.1 → taskqueue_toolkit-0.2.0}/src/taskqueue_toolkit/queue/rabbitmq.py +38 -2
- {taskqueue_toolkit-0.1.1 → taskqueue_toolkit-0.2.0}/src/taskqueue_toolkit/queue/redis_streams.py +22 -0
- {taskqueue_toolkit-0.1.1 → taskqueue_toolkit-0.2.0}/src/taskqueue_toolkit/queue/sns.py +10 -0
- {taskqueue_toolkit-0.1.1 → taskqueue_toolkit-0.2.0}/src/taskqueue_toolkit/queue/sqs.py +8 -0
- {taskqueue_toolkit-0.1.1 → taskqueue_toolkit-0.2.0}/src/taskqueue_toolkit/queue/task_queue.py +14 -0
- {taskqueue_toolkit-0.1.1 → taskqueue_toolkit-0.2.0}/src/taskqueue_toolkit/testing/pubsub.py +26 -9
- {taskqueue_toolkit-0.1.1 → taskqueue_toolkit-0.2.0}/src/taskqueue_toolkit/testing/rabbitmq.py +34 -10
- {taskqueue_toolkit-0.1.1 → taskqueue_toolkit-0.2.0}/src/taskqueue_toolkit/testing/redis_streams.py +28 -0
- {taskqueue_toolkit-0.1.1 → taskqueue_toolkit-0.2.0}/src/taskqueue_toolkit/testing/sns.py +32 -11
- {taskqueue_toolkit-0.1.1 → taskqueue_toolkit-0.2.0}/src/taskqueue_toolkit/testing/sqs.py +33 -11
- {taskqueue_toolkit-0.1.1 → taskqueue_toolkit-0.2.0}/LICENSE +0 -0
- {taskqueue_toolkit-0.1.1 → taskqueue_toolkit-0.2.0}/README.md +0 -0
- {taskqueue_toolkit-0.1.1 → taskqueue_toolkit-0.2.0}/src/taskqueue_toolkit/__init__.py +0 -0
- {taskqueue_toolkit-0.1.1 → taskqueue_toolkit-0.2.0}/src/taskqueue_toolkit/outbox/__init__.py +0 -0
- {taskqueue_toolkit-0.1.1 → taskqueue_toolkit-0.2.0}/src/taskqueue_toolkit/outbox/orm.py +0 -0
- {taskqueue_toolkit-0.1.1 → taskqueue_toolkit-0.2.0}/src/taskqueue_toolkit/outbox/relay.py +0 -0
- {taskqueue_toolkit-0.1.1 → taskqueue_toolkit-0.2.0}/src/taskqueue_toolkit/outbox/repository.py +0 -0
- {taskqueue_toolkit-0.1.1 → taskqueue_toolkit-0.2.0}/src/taskqueue_toolkit/py.typed +0 -0
- {taskqueue_toolkit-0.1.1 → taskqueue_toolkit-0.2.0}/src/taskqueue_toolkit/queue/__init__.py +0 -0
- {taskqueue_toolkit-0.1.1 → taskqueue_toolkit-0.2.0}/src/taskqueue_toolkit/queue/aws_session.py +0 -0
- {taskqueue_toolkit-0.1.1 → taskqueue_toolkit-0.2.0}/src/taskqueue_toolkit/queue/dsn.py +0 -0
- {taskqueue_toolkit-0.1.1 → taskqueue_toolkit-0.2.0}/src/taskqueue_toolkit/queue/factory.py +0 -0
- {taskqueue_toolkit-0.1.1 → taskqueue_toolkit-0.2.0}/src/taskqueue_toolkit/queue/registry.py +0 -0
- {taskqueue_toolkit-0.1.1 → taskqueue_toolkit-0.2.0}/src/taskqueue_toolkit/testing/__init__.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: taskqueue-toolkit
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.2.0
|
|
4
4
|
Summary: Broker-agnostic async task queue (RabbitMQ, SQS, SNS, Redis Streams, Pub/Sub) with a Postgres outbox pattern
|
|
5
5
|
Author: Walid Boughdiri
|
|
6
6
|
Author-email: Walid Boughdiri <walid.boughdiri@gmail.com>
|
|
@@ -26,6 +26,7 @@ class PubsubQueuedTask[T]:
|
|
|
26
26
|
_subscriber: pubsub_v1.SubscriberClient
|
|
27
27
|
_subscription_path: str
|
|
28
28
|
_ack_id: str
|
|
29
|
+
delivery_count: int
|
|
29
30
|
task: T
|
|
30
31
|
|
|
31
32
|
async def ack(self) -> None:
|
|
@@ -154,9 +155,16 @@ class PubsubTaskQueue[T]:
|
|
|
154
155
|
timeout=_PULL_TIMEOUT_SECONDS,
|
|
155
156
|
)
|
|
156
157
|
for received in response.received_messages:
|
|
158
|
+
# delivery_attempt is only populated when the subscription
|
|
159
|
+
# has a dead-letter policy configured — absent (None)
|
|
160
|
+
# otherwise, in which case this reports 1 rather than
|
|
161
|
+
# silently misreporting every message as never-redelivered
|
|
162
|
+
# forever.
|
|
163
|
+
delivery_count = received.delivery_attempt or 1
|
|
157
164
|
yield PubsubQueuedTask(
|
|
158
165
|
_subscriber=self._subscriber,
|
|
159
166
|
_subscription_path=subscription_path,
|
|
160
167
|
_ack_id=received.ack_id,
|
|
168
|
+
delivery_count=delivery_count,
|
|
161
169
|
task=self.decode(received.message.data),
|
|
162
170
|
)
|
|
@@ -6,6 +6,7 @@ from contextlib import suppress
|
|
|
6
6
|
from dataclasses import dataclass, field
|
|
7
7
|
|
|
8
8
|
import aio_pika
|
|
9
|
+
from pamqp.common import FieldTable
|
|
9
10
|
|
|
10
11
|
from taskqueue_toolkit.queue.dsn import RabbitMqDsn
|
|
11
12
|
from taskqueue_toolkit.queue.task_queue import Decoder, Encoder
|
|
@@ -15,11 +16,33 @@ logger = logging.getLogger(__name__)
|
|
|
15
16
|
Connect = Callable[[str], Awaitable["aio_pika.abc.AbstractRobustConnection"]]
|
|
16
17
|
|
|
17
18
|
|
|
19
|
+
def _delivery_count(message: aio_pika.abc.AbstractIncomingMessage) -> int:
|
|
20
|
+
"""1 on first delivery, incremented on every redelivery (a
|
|
21
|
+
nack(requeue=True), a crashed/disconnected consumer, ...). Sourced from
|
|
22
|
+
x-delivery-count, which RabbitMQ maintains natively for quorum queues —
|
|
23
|
+
see the queue declaration in RabbitMqTaskQueue for why the queue type
|
|
24
|
+
matters here (classic queues don't track this at all; only a
|
|
25
|
+
dead-letter hop shows up there, via x-death, which a plain requeue never
|
|
26
|
+
triggers)."""
|
|
27
|
+
headers = message.headers or {}
|
|
28
|
+
count = headers.get("x-delivery-count")
|
|
29
|
+
if count is None:
|
|
30
|
+
return 1
|
|
31
|
+
# RabbitMQ always sends this header as a long-int field value — the
|
|
32
|
+
# broad FieldValue union is what the type carries at rest, not what
|
|
33
|
+
# actually arrives here.
|
|
34
|
+
return 1 + int(count) # type: ignore[arg-type]
|
|
35
|
+
|
|
36
|
+
|
|
18
37
|
@dataclass(slots=True)
|
|
19
38
|
class RabbitMqQueuedTask[T]:
|
|
20
39
|
_message: aio_pika.abc.AbstractIncomingMessage
|
|
21
40
|
task: T
|
|
22
41
|
|
|
42
|
+
@property
|
|
43
|
+
def delivery_count(self) -> int:
|
|
44
|
+
return _delivery_count(self._message)
|
|
45
|
+
|
|
23
46
|
async def ack(self) -> None:
|
|
24
47
|
await self._message.ack()
|
|
25
48
|
|
|
@@ -36,11 +59,22 @@ class RabbitMqTaskQueue[T]:
|
|
|
36
59
|
# connection strategy the caller already has; defaults to the real SDK.
|
|
37
60
|
connect: Connect = field(default=aio_pika.connect_robust)
|
|
38
61
|
|
|
62
|
+
def _queue_arguments(self) -> FieldTable:
|
|
63
|
+
# Quorum queues (not RabbitMQ's classic queue type) are what expose
|
|
64
|
+
# x-delivery-count on redelivery — a classic queue tracks no such
|
|
65
|
+
# counter at all, so delivery_count would always read 1. This is
|
|
66
|
+
# also RabbitMQ's own recommended queue type for task-queue-style
|
|
67
|
+
# workloads (durable, replicated, no classic-queue mirroring), so
|
|
68
|
+
# this isn't a special case adopted only for the counter.
|
|
69
|
+
return {"x-queue-type": "quorum"}
|
|
70
|
+
|
|
39
71
|
async def publish(self, task: T) -> None:
|
|
40
72
|
connection = await self.connect(self.dsn.url)
|
|
41
73
|
try:
|
|
42
74
|
channel = await connection.channel()
|
|
43
|
-
queue = await channel.declare_queue(
|
|
75
|
+
queue = await channel.declare_queue(
|
|
76
|
+
self.dsn.queue_name, durable=True, arguments=self._queue_arguments()
|
|
77
|
+
)
|
|
44
78
|
await channel.default_exchange.publish(
|
|
45
79
|
aio_pika.Message(
|
|
46
80
|
body=self.encode(task),
|
|
@@ -57,7 +91,9 @@ class RabbitMqTaskQueue[T]:
|
|
|
57
91
|
try:
|
|
58
92
|
channel = await connection.channel()
|
|
59
93
|
await channel.set_qos(prefetch_count=1)
|
|
60
|
-
queue = await channel.declare_queue(
|
|
94
|
+
queue = await channel.declare_queue(
|
|
95
|
+
self.dsn.queue_name, durable=True, arguments=self._queue_arguments()
|
|
96
|
+
)
|
|
61
97
|
|
|
62
98
|
async for message in queue.iterator():
|
|
63
99
|
yield RabbitMqQueuedTask(_message=message, task=self.decode(message.body))
|
{taskqueue_toolkit-0.1.1 → taskqueue_toolkit-0.2.0}/src/taskqueue_toolkit/queue/redis_streams.py
RENAMED
|
@@ -29,6 +29,7 @@ class RedisStreamsQueuedTask[T]:
|
|
|
29
29
|
_stream: str
|
|
30
30
|
_group: str
|
|
31
31
|
_message_id: str
|
|
32
|
+
delivery_count: int
|
|
32
33
|
task: T
|
|
33
34
|
|
|
34
35
|
async def ack(self) -> None:
|
|
@@ -61,6 +62,20 @@ class RedisStreamsTaskQueue[T]:
|
|
|
61
62
|
def _client(self) -> redis.Redis:
|
|
62
63
|
return self.client_factory(self.dsn)
|
|
63
64
|
|
|
65
|
+
async def _delivery_count(
|
|
66
|
+
self, client: redis.Redis, stream: str, group: str, message_id: bytes
|
|
67
|
+
) -> int:
|
|
68
|
+
entries = await client.xpending_range(
|
|
69
|
+
name=stream, groupname=group, min=message_id, max=message_id, count=1
|
|
70
|
+
)
|
|
71
|
+
if not entries:
|
|
72
|
+
# Already ack'd/claimed away between xreadgroup and this lookup
|
|
73
|
+
# — vanishingly unlikely for our single-consumer-per-group
|
|
74
|
+
# usage, but "never delivered before" (1) is the only sane
|
|
75
|
+
# fallback rather than raising.
|
|
76
|
+
return 1
|
|
77
|
+
return int(entries[0]["times_delivered"])
|
|
78
|
+
|
|
64
79
|
async def _ensure_group(self, client: redis.Redis) -> None:
|
|
65
80
|
try:
|
|
66
81
|
await client.xgroup_create(self.dsn.stream_name, self.dsn.group, id="0", mkstream=True)
|
|
@@ -106,11 +121,18 @@ class RedisStreamsTaskQueue[T]:
|
|
|
106
121
|
)
|
|
107
122
|
for _stream_name, messages in response:
|
|
108
123
|
for message_id, fields in messages:
|
|
124
|
+
# xreadgroup itself never reports a delivery
|
|
125
|
+
# count — it's only visible via XPENDING, so
|
|
126
|
+
# this is a second round trip per message.
|
|
127
|
+
delivery_count = await self._delivery_count(
|
|
128
|
+
client, stream, group, message_id
|
|
129
|
+
)
|
|
109
130
|
yield RedisStreamsQueuedTask(
|
|
110
131
|
_redis=client,
|
|
111
132
|
_stream=stream,
|
|
112
133
|
_group=group,
|
|
113
134
|
_message_id=message_id.decode(),
|
|
135
|
+
delivery_count=delivery_count,
|
|
114
136
|
task=self.decode(fields[_FIELD.encode()]),
|
|
115
137
|
)
|
|
116
138
|
finally:
|
|
@@ -65,6 +65,7 @@ class SnsQueuedTask[T]:
|
|
|
65
65
|
_client: SQSClient
|
|
66
66
|
_queue_url: str
|
|
67
67
|
_receipt_handle: str
|
|
68
|
+
delivery_count: int
|
|
68
69
|
task: T
|
|
69
70
|
|
|
70
71
|
async def ack(self) -> None:
|
|
@@ -154,12 +155,21 @@ class SnsTaskQueue[T]:
|
|
|
154
155
|
QueueUrl=queue_url,
|
|
155
156
|
MaxNumberOfMessages=1,
|
|
156
157
|
WaitTimeSeconds=_LONG_POLL_WAIT_SECONDS,
|
|
158
|
+
# Not returned unless asked for explicitly — this is the
|
|
159
|
+
# source for delivery_count below. SNS fan-out delivers
|
|
160
|
+
# into a plain SQS subscriber queue, so the same
|
|
161
|
+
# attribute SqsTaskQueue reads applies here too.
|
|
162
|
+
AttributeNames=["ApproximateReceiveCount"],
|
|
157
163
|
)
|
|
158
164
|
for message in response.get("Messages", []):
|
|
165
|
+
receive_count = int(
|
|
166
|
+
message.get("Attributes", {}).get("ApproximateReceiveCount", "1")
|
|
167
|
+
)
|
|
159
168
|
envelope = _unwrap_sns_envelope(message["Body"])
|
|
160
169
|
yield SnsQueuedTask(
|
|
161
170
|
_client=sqs,
|
|
162
171
|
_queue_url=queue_url,
|
|
163
172
|
_receipt_handle=message["ReceiptHandle"],
|
|
173
|
+
delivery_count=receive_count,
|
|
164
174
|
task=self.decode(base64.b64decode(envelope)),
|
|
165
175
|
)
|
|
@@ -35,6 +35,7 @@ class SqsQueuedTask[T]:
|
|
|
35
35
|
_client: SQSClient
|
|
36
36
|
_queue_url: str
|
|
37
37
|
_receipt_handle: str
|
|
38
|
+
delivery_count: int
|
|
38
39
|
task: T
|
|
39
40
|
|
|
40
41
|
async def ack(self) -> None:
|
|
@@ -95,11 +96,18 @@ class SqsTaskQueue[T]:
|
|
|
95
96
|
QueueUrl=queue_url,
|
|
96
97
|
MaxNumberOfMessages=1,
|
|
97
98
|
WaitTimeSeconds=_LONG_POLL_WAIT_SECONDS,
|
|
99
|
+
# Not returned unless asked for explicitly — this is the
|
|
100
|
+
# source for delivery_count below.
|
|
101
|
+
AttributeNames=["ApproximateReceiveCount"],
|
|
98
102
|
)
|
|
99
103
|
for message in response.get("Messages", []):
|
|
104
|
+
receive_count = int(
|
|
105
|
+
message.get("Attributes", {}).get("ApproximateReceiveCount", "1")
|
|
106
|
+
)
|
|
100
107
|
yield SqsQueuedTask(
|
|
101
108
|
_client=client,
|
|
102
109
|
_queue_url=queue_url,
|
|
103
110
|
_receipt_handle=message["ReceiptHandle"],
|
|
111
|
+
delivery_count=receive_count,
|
|
104
112
|
task=self.decode(base64.b64decode(message["Body"])),
|
|
105
113
|
)
|
{taskqueue_toolkit-0.1.1 → taskqueue_toolkit-0.2.0}/src/taskqueue_toolkit/queue/task_queue.py
RENAMED
|
@@ -28,6 +28,20 @@ class QueuedTask(Protocol[T_co]):
|
|
|
28
28
|
@property
|
|
29
29
|
def task(self) -> T_co: ...
|
|
30
30
|
|
|
31
|
+
@property
|
|
32
|
+
def delivery_count(self) -> int:
|
|
33
|
+
"""How many times this message has been delivered, including the
|
|
34
|
+
current delivery — 1 the first time a consumer sees it, 2 after one
|
|
35
|
+
requeue, and so on. Sourced from each broker's own native counter
|
|
36
|
+
(RabbitMQ's x-death header behind a dead-letter-to-self queue, SQS's
|
|
37
|
+
ApproximateReceiveCount, Pub/Sub's delivery_attempt, Redis Streams'
|
|
38
|
+
pending-entry delivery count) rather than tracked by this package —
|
|
39
|
+
a consumer that wants to give up after N attempts reads this and
|
|
40
|
+
calls nack(requeue=False) itself; this package has no opinion on
|
|
41
|
+
what N should be, that's a decision about the task's own business
|
|
42
|
+
cost of retrying, not a queueing concern."""
|
|
43
|
+
...
|
|
44
|
+
|
|
31
45
|
async def ack(self) -> None:
|
|
32
46
|
"""Mark the task as successfully processed."""
|
|
33
47
|
...
|
|
@@ -40,6 +40,7 @@ class _Message(NamedTuple):
|
|
|
40
40
|
class _ReceivedMessage(NamedTuple):
|
|
41
41
|
ack_id: str
|
|
42
42
|
message: _Message
|
|
43
|
+
delivery_attempt: int
|
|
43
44
|
|
|
44
45
|
|
|
45
46
|
class _PullResponse(NamedTuple):
|
|
@@ -54,11 +55,20 @@ class _PublishFuture:
|
|
|
54
55
|
return self._message_id
|
|
55
56
|
|
|
56
57
|
|
|
58
|
+
@dataclass(slots=True)
|
|
59
|
+
class _Envelope:
|
|
60
|
+
"""A message plus its own delivery attempt count — mirrors Pub/Sub's
|
|
61
|
+
delivery_attempt (1 on first delivery, incremented on each redelivery)."""
|
|
62
|
+
|
|
63
|
+
data: bytes
|
|
64
|
+
delivery_attempt: int = 0
|
|
65
|
+
|
|
66
|
+
|
|
57
67
|
@dataclass(slots=True)
|
|
58
68
|
class _Subscription:
|
|
59
69
|
topic_path: str
|
|
60
|
-
pending: deque[
|
|
61
|
-
in_flight: dict[str,
|
|
70
|
+
pending: deque[_Envelope] = field(default_factory=deque)
|
|
71
|
+
in_flight: dict[str, _Envelope] = field(default_factory=dict)
|
|
62
72
|
|
|
63
73
|
|
|
64
74
|
@dataclass(slots=True)
|
|
@@ -79,7 +89,7 @@ class _FakePublisherClient:
|
|
|
79
89
|
def publish(self, topic_path: str, data: bytes) -> _PublishFuture:
|
|
80
90
|
for sub in self._broker._subscriptions.values():
|
|
81
91
|
if sub.topic_path == topic_path:
|
|
82
|
-
sub.pending.append(data)
|
|
92
|
+
sub.pending.append(_Envelope(data=data))
|
|
83
93
|
return _PublishFuture(str(uuid.uuid4()))
|
|
84
94
|
|
|
85
95
|
|
|
@@ -108,10 +118,17 @@ class _FakeSubscriberClient:
|
|
|
108
118
|
)
|
|
109
119
|
received = []
|
|
110
120
|
for _ in range(min(max_messages, len(sub.pending))):
|
|
111
|
-
|
|
121
|
+
envelope = sub.pending.popleft()
|
|
122
|
+
envelope.delivery_attempt += 1
|
|
112
123
|
ack_id = str(uuid.uuid4())
|
|
113
|
-
sub.in_flight[ack_id] =
|
|
114
|
-
received.append(
|
|
124
|
+
sub.in_flight[ack_id] = envelope
|
|
125
|
+
received.append(
|
|
126
|
+
_ReceivedMessage(
|
|
127
|
+
ack_id=ack_id,
|
|
128
|
+
message=_Message(data=envelope.data),
|
|
129
|
+
delivery_attempt=envelope.delivery_attempt,
|
|
130
|
+
)
|
|
131
|
+
)
|
|
115
132
|
return _PullResponse(received_messages=received)
|
|
116
133
|
|
|
117
134
|
def acknowledge(self, *, subscription: str, ack_ids: list[str]) -> None:
|
|
@@ -128,9 +145,9 @@ class _FakeSubscriberClient:
|
|
|
128
145
|
if sub is None:
|
|
129
146
|
return
|
|
130
147
|
for ack_id in ack_ids:
|
|
131
|
-
|
|
132
|
-
if
|
|
133
|
-
sub.pending.appendleft(
|
|
148
|
+
envelope = sub.in_flight.pop(ack_id, None)
|
|
149
|
+
if envelope is not None and ack_deadline_seconds == 0:
|
|
150
|
+
sub.pending.appendleft(envelope)
|
|
134
151
|
|
|
135
152
|
|
|
136
153
|
@dataclass(slots=True)
|
{taskqueue_toolkit-0.1.1 → taskqueue_toolkit-0.2.0}/src/taskqueue_toolkit/testing/rabbitmq.py
RENAMED
|
@@ -19,22 +19,34 @@ from dataclasses import dataclass, field
|
|
|
19
19
|
__all__ = ["FakeRabbitMqBroker"]
|
|
20
20
|
|
|
21
21
|
|
|
22
|
+
@dataclass(slots=True)
|
|
23
|
+
class _Envelope:
|
|
24
|
+
"""A message plus its own redelivery count — tracked per in-flight copy
|
|
25
|
+
of the message, not by content, so two messages with identical bodies
|
|
26
|
+
(or the same message redelivered many times) each carry their own
|
|
27
|
+
count."""
|
|
28
|
+
|
|
29
|
+
body: bytes
|
|
30
|
+
delivery_count: int = 0
|
|
31
|
+
|
|
32
|
+
|
|
22
33
|
@dataclass(slots=True)
|
|
23
34
|
class _Queue:
|
|
24
35
|
"""One named queue's backlog plus messages currently delivered but not
|
|
25
36
|
yet ack()'d/nack()'d — mirrors a real broker's per-queue state."""
|
|
26
37
|
|
|
27
|
-
pending: deque[
|
|
28
|
-
unacked: dict[int,
|
|
38
|
+
pending: deque[_Envelope] = field(default_factory=deque)
|
|
39
|
+
unacked: dict[int, _Envelope] = field(default_factory=dict)
|
|
29
40
|
_next_id: int = 0
|
|
30
41
|
|
|
31
42
|
def push(self, body: bytes) -> None:
|
|
32
|
-
self.pending.append(body)
|
|
43
|
+
self.pending.append(_Envelope(body=body))
|
|
33
44
|
|
|
34
45
|
def requeue(self, delivery_id: int) -> None:
|
|
35
|
-
|
|
36
|
-
if
|
|
37
|
-
|
|
46
|
+
envelope = self.unacked.pop(delivery_id, None)
|
|
47
|
+
if envelope is not None:
|
|
48
|
+
envelope.delivery_count += 1
|
|
49
|
+
self.pending.appendleft(envelope)
|
|
38
50
|
|
|
39
51
|
def drop(self, delivery_id: int) -> None:
|
|
40
52
|
self.unacked.pop(delivery_id, None)
|
|
@@ -47,6 +59,7 @@ class _Queue:
|
|
|
47
59
|
@dataclass(slots=True)
|
|
48
60
|
class _FakeIncomingMessage:
|
|
49
61
|
body: bytes
|
|
62
|
+
headers: dict[str, int]
|
|
50
63
|
_delivery_id: int
|
|
51
64
|
_queue: _Queue
|
|
52
65
|
|
|
@@ -74,10 +87,19 @@ class _FakeAioPikaQueue:
|
|
|
74
87
|
async def __anext__(self) -> _FakeIncomingMessage:
|
|
75
88
|
while not self._queue.pending:
|
|
76
89
|
await asyncio.sleep(0)
|
|
77
|
-
|
|
90
|
+
envelope = self._queue.pending.popleft()
|
|
78
91
|
delivery_id = self._queue.next_delivery_id()
|
|
79
|
-
self._queue.unacked[delivery_id] =
|
|
80
|
-
|
|
92
|
+
self._queue.unacked[delivery_id] = envelope
|
|
93
|
+
# Mirrors a real quorum queue's x-delivery-count: absent (no header)
|
|
94
|
+
# on first delivery, present from the first redelivery onward — see
|
|
95
|
+
# RabbitMqTaskQueue's _delivery_count(), which reads this as
|
|
96
|
+
# "1 + count" so a never-redelivered message still reports 1.
|
|
97
|
+
headers = (
|
|
98
|
+
{} if envelope.delivery_count == 0 else {"x-delivery-count": envelope.delivery_count}
|
|
99
|
+
)
|
|
100
|
+
return _FakeIncomingMessage(
|
|
101
|
+
body=envelope.body, headers=headers, _delivery_id=delivery_id, _queue=self._queue
|
|
102
|
+
)
|
|
81
103
|
|
|
82
104
|
|
|
83
105
|
@dataclass(slots=True)
|
|
@@ -100,7 +122,9 @@ class _FakeChannel:
|
|
|
100
122
|
async def set_qos(self, *, prefetch_count: int) -> None:
|
|
101
123
|
return None
|
|
102
124
|
|
|
103
|
-
async def declare_queue(
|
|
125
|
+
async def declare_queue(
|
|
126
|
+
self, name: str, *, durable: bool = True, arguments: dict[str, str] | None = None
|
|
127
|
+
) -> _FakeAioPikaQueue:
|
|
104
128
|
queue = self._broker._queue_for(name)
|
|
105
129
|
self.default_exchange = _FakeExchange(_queue=queue)
|
|
106
130
|
return _FakeAioPikaQueue(name=name, _queue=queue)
|
{taskqueue_toolkit-0.1.1 → taskqueue_toolkit-0.2.0}/src/taskqueue_toolkit/testing/redis_streams.py
RENAMED
|
@@ -32,6 +32,10 @@ class _Group:
|
|
|
32
32
|
# (a real broker's Pending Entries List) — nack(requeue=True) leaves an
|
|
33
33
|
# entry here; xreadgroup's "0" branch redelivers from here first.
|
|
34
34
|
pending_entries: list[tuple[bytes, dict[bytes, bytes]]] = field(default_factory=list)
|
|
35
|
+
# Mirrors a real group's per-entry times_delivered (visible via
|
|
36
|
+
# XPENDING) — 0 before the entry's first delivery, matching a real
|
|
37
|
+
# broker where a never-yet-read entry isn't in the PEL at all.
|
|
38
|
+
times_delivered: dict[bytes, int] = field(default_factory=dict)
|
|
35
39
|
|
|
36
40
|
|
|
37
41
|
@dataclass(slots=True)
|
|
@@ -89,6 +93,8 @@ class _FakeRedisClient:
|
|
|
89
93
|
# per-group, matching this package's single-consumer-per-group
|
|
90
94
|
# usage pattern.
|
|
91
95
|
delivered = grp.pending_entries[: count or len(grp.pending_entries)]
|
|
96
|
+
for entry_id, _fields in delivered:
|
|
97
|
+
grp.times_delivered[entry_id] = grp.times_delivered.get(entry_id, 0) + 1
|
|
92
98
|
return [(name.encode(), delivered)]
|
|
93
99
|
|
|
94
100
|
# ">" branch: deliver new entries the group hasn't seen yet.
|
|
@@ -97,6 +103,8 @@ class _FakeRedisClient:
|
|
|
97
103
|
if new_entries:
|
|
98
104
|
stream.cursor[group_key] = cursor + len(new_entries)
|
|
99
105
|
grp.pending_entries.extend(new_entries)
|
|
106
|
+
for entry_id, _fields in new_entries:
|
|
107
|
+
grp.times_delivered[entry_id] = grp.times_delivered.get(entry_id, 0) + 1
|
|
100
108
|
return [(name.encode(), new_entries)]
|
|
101
109
|
|
|
102
110
|
# Nothing new — a real broker blocks up to `block` ms; the fake
|
|
@@ -113,8 +121,28 @@ class _FakeRedisClient:
|
|
|
113
121
|
ids = set(entry_ids)
|
|
114
122
|
before = len(grp.pending_entries)
|
|
115
123
|
grp.pending_entries = [e for e in grp.pending_entries if e[0] not in ids]
|
|
124
|
+
for entry_id in ids:
|
|
125
|
+
grp.times_delivered.pop(entry_id, None)
|
|
116
126
|
return before - len(grp.pending_entries)
|
|
117
127
|
|
|
128
|
+
async def xpending_range(
|
|
129
|
+
self,
|
|
130
|
+
name: str,
|
|
131
|
+
groupname: str,
|
|
132
|
+
min: bytes,
|
|
133
|
+
max: bytes,
|
|
134
|
+
count: int,
|
|
135
|
+
consumername: str | None = None,
|
|
136
|
+
) -> list[dict[str, object]]:
|
|
137
|
+
stream = self._broker._stream_for(name.encode())
|
|
138
|
+
grp = stream.groups.get(groupname.encode())
|
|
139
|
+
if grp is None:
|
|
140
|
+
return []
|
|
141
|
+
times_delivered = grp.times_delivered.get(min)
|
|
142
|
+
if times_delivered is None:
|
|
143
|
+
return []
|
|
144
|
+
return [{"message_id": min, "times_delivered": times_delivered}]
|
|
145
|
+
|
|
118
146
|
async def delete(self, *names: str) -> int:
|
|
119
147
|
count = 0
|
|
120
148
|
for name in names:
|
|
@@ -37,10 +37,19 @@ class _Exceptions:
|
|
|
37
37
|
QueueDoesNotExist: type[Exception] = QueueDoesNotExist
|
|
38
38
|
|
|
39
39
|
|
|
40
|
+
@dataclass(slots=True)
|
|
41
|
+
class _Envelope:
|
|
42
|
+
"""A message plus its own receive count — mirrors SQS's
|
|
43
|
+
ApproximateReceiveCount on the subscriber queue behind the topic."""
|
|
44
|
+
|
|
45
|
+
body: str
|
|
46
|
+
receive_count: int = 0
|
|
47
|
+
|
|
48
|
+
|
|
40
49
|
@dataclass(slots=True)
|
|
41
50
|
class _Queue:
|
|
42
|
-
pending: deque[
|
|
43
|
-
in_flight: dict[str,
|
|
51
|
+
pending: deque[_Envelope] = field(default_factory=deque)
|
|
52
|
+
in_flight: dict[str, _Envelope] = field(default_factory=dict)
|
|
44
53
|
|
|
45
54
|
|
|
46
55
|
def _topic_arn(name: str) -> str:
|
|
@@ -74,7 +83,7 @@ class _FakeSnsClient:
|
|
|
74
83
|
)
|
|
75
84
|
for queue_url in self._broker._topics.get(TopicArn, set()):
|
|
76
85
|
queue = self._broker._queues.setdefault(queue_url, _Queue())
|
|
77
|
-
queue.pending.append(
|
|
86
|
+
queue.pending.append(_Envelope(body=envelope))
|
|
78
87
|
return {"MessageId": str(uuid.uuid4())}
|
|
79
88
|
|
|
80
89
|
|
|
@@ -105,15 +114,27 @@ class _FakeSqsClient:
|
|
|
105
114
|
return None
|
|
106
115
|
|
|
107
116
|
async def receive_message(
|
|
108
|
-
self,
|
|
109
|
-
|
|
117
|
+
self,
|
|
118
|
+
*,
|
|
119
|
+
QueueUrl: str,
|
|
120
|
+
MaxNumberOfMessages: int = 1,
|
|
121
|
+
WaitTimeSeconds: int = 0,
|
|
122
|
+
AttributeNames: list[str] | None = None,
|
|
123
|
+
) -> dict[str, list[dict[str, object]]]:
|
|
110
124
|
queue = self._broker._queues.setdefault(QueueUrl, _Queue())
|
|
111
125
|
messages = []
|
|
112
126
|
for _ in range(min(MaxNumberOfMessages, len(queue.pending))):
|
|
113
|
-
|
|
127
|
+
envelope = queue.pending.popleft()
|
|
128
|
+
envelope.receive_count += 1
|
|
114
129
|
receipt_handle = str(uuid.uuid4())
|
|
115
|
-
queue.in_flight[receipt_handle] =
|
|
116
|
-
|
|
130
|
+
queue.in_flight[receipt_handle] = envelope
|
|
131
|
+
message: dict[str, object] = {
|
|
132
|
+
"Body": envelope.body,
|
|
133
|
+
"ReceiptHandle": receipt_handle,
|
|
134
|
+
}
|
|
135
|
+
if AttributeNames and "ApproximateReceiveCount" in AttributeNames:
|
|
136
|
+
message["Attributes"] = {"ApproximateReceiveCount": str(envelope.receive_count)}
|
|
137
|
+
messages.append(message)
|
|
117
138
|
return {"Messages": messages}
|
|
118
139
|
|
|
119
140
|
async def delete_message(self, *, QueueUrl: str, ReceiptHandle: str) -> None:
|
|
@@ -124,9 +145,9 @@ class _FakeSqsClient:
|
|
|
124
145
|
self, *, QueueUrl: str, ReceiptHandle: str, VisibilityTimeout: int
|
|
125
146
|
) -> None:
|
|
126
147
|
queue = self._broker._queues.setdefault(QueueUrl, _Queue())
|
|
127
|
-
|
|
128
|
-
if
|
|
129
|
-
queue.pending.appendleft(
|
|
148
|
+
envelope = queue.in_flight.pop(ReceiptHandle, None)
|
|
149
|
+
if envelope is not None and VisibilityTimeout == 0:
|
|
150
|
+
queue.pending.appendleft(envelope)
|
|
130
151
|
|
|
131
152
|
|
|
132
153
|
class _FakeSnsContext(AbstractAsyncContextManager[tuple["_FakeSnsClient", "_FakeSqsClient"]]):
|
|
@@ -33,10 +33,20 @@ class _Exceptions:
|
|
|
33
33
|
QueueDoesNotExist: type[Exception] = QueueDoesNotExist
|
|
34
34
|
|
|
35
35
|
|
|
36
|
+
@dataclass(slots=True)
|
|
37
|
+
class _Envelope:
|
|
38
|
+
"""A message plus its own receive count — tracked per in-flight copy of
|
|
39
|
+
the message, mirroring SQS's ApproximateReceiveCount (1 on first
|
|
40
|
+
receive, incremented on each redelivery)."""
|
|
41
|
+
|
|
42
|
+
body: str
|
|
43
|
+
receive_count: int = 0
|
|
44
|
+
|
|
45
|
+
|
|
36
46
|
@dataclass(slots=True)
|
|
37
47
|
class _Queue:
|
|
38
|
-
pending: deque[
|
|
39
|
-
in_flight: dict[str,
|
|
48
|
+
pending: deque[_Envelope] = field(default_factory=deque)
|
|
49
|
+
in_flight: dict[str, _Envelope] = field(default_factory=dict)
|
|
40
50
|
|
|
41
51
|
|
|
42
52
|
@dataclass(slots=True)
|
|
@@ -58,19 +68,31 @@ class _FakeSqsClient:
|
|
|
58
68
|
|
|
59
69
|
async def send_message(self, *, QueueUrl: str, MessageBody: str) -> dict[str, str]:
|
|
60
70
|
queue = self._broker._queues.setdefault(QueueUrl, _Queue())
|
|
61
|
-
queue.pending.append(
|
|
71
|
+
queue.pending.append(_Envelope(body=MessageBody))
|
|
62
72
|
return {"MessageId": str(uuid.uuid4())}
|
|
63
73
|
|
|
64
74
|
async def receive_message(
|
|
65
|
-
self,
|
|
66
|
-
|
|
75
|
+
self,
|
|
76
|
+
*,
|
|
77
|
+
QueueUrl: str,
|
|
78
|
+
MaxNumberOfMessages: int = 1,
|
|
79
|
+
WaitTimeSeconds: int = 0,
|
|
80
|
+
AttributeNames: list[str] | None = None,
|
|
81
|
+
) -> dict[str, list[dict[str, object]]]:
|
|
67
82
|
queue = self._broker._queues.setdefault(QueueUrl, _Queue())
|
|
68
83
|
messages = []
|
|
69
84
|
for _ in range(min(MaxNumberOfMessages, len(queue.pending))):
|
|
70
|
-
|
|
85
|
+
envelope = queue.pending.popleft()
|
|
86
|
+
envelope.receive_count += 1
|
|
71
87
|
receipt_handle = str(uuid.uuid4())
|
|
72
|
-
queue.in_flight[receipt_handle] =
|
|
73
|
-
|
|
88
|
+
queue.in_flight[receipt_handle] = envelope
|
|
89
|
+
message: dict[str, object] = {
|
|
90
|
+
"Body": envelope.body,
|
|
91
|
+
"ReceiptHandle": receipt_handle,
|
|
92
|
+
}
|
|
93
|
+
if AttributeNames and "ApproximateReceiveCount" in AttributeNames:
|
|
94
|
+
message["Attributes"] = {"ApproximateReceiveCount": str(envelope.receive_count)}
|
|
95
|
+
messages.append(message)
|
|
74
96
|
return {"Messages": messages}
|
|
75
97
|
|
|
76
98
|
async def delete_message(self, *, QueueUrl: str, ReceiptHandle: str) -> None:
|
|
@@ -81,9 +103,9 @@ class _FakeSqsClient:
|
|
|
81
103
|
self, *, QueueUrl: str, ReceiptHandle: str, VisibilityTimeout: int
|
|
82
104
|
) -> None:
|
|
83
105
|
queue = self._broker._queues.setdefault(QueueUrl, _Queue())
|
|
84
|
-
|
|
85
|
-
if
|
|
86
|
-
queue.pending.appendleft(
|
|
106
|
+
envelope = queue.in_flight.pop(ReceiptHandle, None)
|
|
107
|
+
if envelope is not None and VisibilityTimeout == 0:
|
|
108
|
+
queue.pending.appendleft(envelope)
|
|
87
109
|
|
|
88
110
|
|
|
89
111
|
class _FakeClientContext(AbstractAsyncContextManager["_FakeSqsClient"]):
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{taskqueue_toolkit-0.1.1 → taskqueue_toolkit-0.2.0}/src/taskqueue_toolkit/outbox/__init__.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{taskqueue_toolkit-0.1.1 → taskqueue_toolkit-0.2.0}/src/taskqueue_toolkit/outbox/repository.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{taskqueue_toolkit-0.1.1 → taskqueue_toolkit-0.2.0}/src/taskqueue_toolkit/queue/aws_session.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{taskqueue_toolkit-0.1.1 → taskqueue_toolkit-0.2.0}/src/taskqueue_toolkit/testing/__init__.py
RENAMED
|
File without changes
|