taskqueue-toolkit 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- taskqueue_toolkit/__init__.py +0 -0
- taskqueue_toolkit/outbox/__init__.py +0 -0
- taskqueue_toolkit/outbox/orm.py +47 -0
- taskqueue_toolkit/outbox/relay.py +59 -0
- taskqueue_toolkit/outbox/repository.py +107 -0
- taskqueue_toolkit/py.typed +0 -0
- taskqueue_toolkit/queue/__init__.py +0 -0
- taskqueue_toolkit/queue/aws_session.py +19 -0
- taskqueue_toolkit/queue/dsn.py +147 -0
- taskqueue_toolkit/queue/factory.py +94 -0
- taskqueue_toolkit/queue/pubsub.py +162 -0
- taskqueue_toolkit/queue/rabbitmq.py +74 -0
- taskqueue_toolkit/queue/redis_streams.py +117 -0
- taskqueue_toolkit/queue/registry.py +53 -0
- taskqueue_toolkit/queue/sns.py +162 -0
- taskqueue_toolkit/queue/sqs.py +103 -0
- taskqueue_toolkit/queue/task_queue.py +62 -0
- taskqueue_toolkit/testing/__init__.py +58 -0
- taskqueue_toolkit/testing/pubsub.py +149 -0
- taskqueue_toolkit/testing/rabbitmq.py +135 -0
- taskqueue_toolkit/testing/redis_streams.py +141 -0
- taskqueue_toolkit/testing/sns.py +155 -0
- taskqueue_toolkit/testing/sqs.py +109 -0
- taskqueue_toolkit-0.1.0.dist-info/METADATA +163 -0
- taskqueue_toolkit-0.1.0.dist-info/RECORD +27 -0
- taskqueue_toolkit-0.1.0.dist-info/WHEEL +4 -0
- taskqueue_toolkit-0.1.0.dist-info/licenses/LICENSE +202 -0
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
"""An in-memory double of the slice of google-cloud-pubsub that
|
|
2
|
+
PubsubTaskQueue calls, for testing code that uses this package without a
|
|
3
|
+
real Pub/Sub project or emulator.
|
|
4
|
+
|
|
5
|
+
Usage:
|
|
6
|
+
broker = FakePubsubBroker()
|
|
7
|
+
queue = PubsubTaskQueue(
|
|
8
|
+
dsn=dsn,
|
|
9
|
+
encode=encode,
|
|
10
|
+
decode=decode,
|
|
11
|
+
publisher_factory=broker.publisher,
|
|
12
|
+
subscriber_factory=broker.subscriber,
|
|
13
|
+
)
|
|
14
|
+
|
|
15
|
+
All methods on the fake clients are synchronous, matching the real (sync
|
|
16
|
+
gRPC) SDK — PubsubTaskQueue always calls through asyncio.to_thread, never
|
|
17
|
+
awaits these directly.
|
|
18
|
+
|
|
19
|
+
Each FakePubsubBroker instance owns its own isolated state — create a new
|
|
20
|
+
one per test rather than sharing one, and there is nothing to reset between
|
|
21
|
+
tests.
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
from __future__ import annotations
|
|
25
|
+
|
|
26
|
+
import uuid
|
|
27
|
+
from collections import deque
|
|
28
|
+
from dataclasses import dataclass, field
|
|
29
|
+
from typing import NamedTuple
|
|
30
|
+
|
|
31
|
+
from google.api_core.exceptions import AlreadyExists
|
|
32
|
+
|
|
33
|
+
__all__ = ["FakePubsubBroker"]
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class _Message(NamedTuple):
|
|
37
|
+
data: bytes
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
class _ReceivedMessage(NamedTuple):
|
|
41
|
+
ack_id: str
|
|
42
|
+
message: _Message
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
class _PullResponse(NamedTuple):
|
|
46
|
+
received_messages: list[_ReceivedMessage]
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
class _PublishFuture:
|
|
50
|
+
def __init__(self, message_id: str) -> None:
|
|
51
|
+
self._message_id = message_id
|
|
52
|
+
|
|
53
|
+
def result(self, timeout: float | None = None) -> str:
|
|
54
|
+
return self._message_id
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
@dataclass(slots=True)
|
|
58
|
+
class _Subscription:
|
|
59
|
+
topic_path: str
|
|
60
|
+
pending: deque[bytes] = field(default_factory=deque)
|
|
61
|
+
in_flight: dict[str, bytes] = field(default_factory=dict)
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
@dataclass(slots=True)
|
|
65
|
+
class _FakePublisherClient:
|
|
66
|
+
_broker: FakePubsubBroker
|
|
67
|
+
|
|
68
|
+
def topic_path(self, project: str, topic: str) -> str:
|
|
69
|
+
return f"projects/{project}/topics/{topic}"
|
|
70
|
+
|
|
71
|
+
def create_topic(self, *, name: str) -> None:
|
|
72
|
+
if name in self._broker._topics:
|
|
73
|
+
raise AlreadyExists(f"Topic already exists: {name}") # type: ignore[no-untyped-call]
|
|
74
|
+
self._broker._topics.add(name)
|
|
75
|
+
|
|
76
|
+
def delete_topic(self, *, topic: str) -> None:
|
|
77
|
+
self._broker._topics.discard(topic)
|
|
78
|
+
|
|
79
|
+
def publish(self, topic_path: str, data: bytes) -> _PublishFuture:
|
|
80
|
+
for sub in self._broker._subscriptions.values():
|
|
81
|
+
if sub.topic_path == topic_path:
|
|
82
|
+
sub.pending.append(data)
|
|
83
|
+
return _PublishFuture(str(uuid.uuid4()))
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
@dataclass(slots=True)
|
|
87
|
+
class _FakeSubscriberClient:
|
|
88
|
+
_broker: FakePubsubBroker
|
|
89
|
+
|
|
90
|
+
def subscription_path(self, project: str, subscription: str) -> str:
|
|
91
|
+
return f"projects/{project}/subscriptions/{subscription}"
|
|
92
|
+
|
|
93
|
+
def create_subscription(self, *, name: str, topic: str) -> None:
|
|
94
|
+
if name in self._broker._subscriptions:
|
|
95
|
+
raise AlreadyExists( # type: ignore[no-untyped-call]
|
|
96
|
+
f"Subscription already exists: {name}"
|
|
97
|
+
)
|
|
98
|
+
self._broker._subscriptions[name] = _Subscription(topic_path=topic)
|
|
99
|
+
|
|
100
|
+
def delete_subscription(self, *, subscription: str) -> None:
|
|
101
|
+
self._broker._subscriptions.pop(subscription, None)
|
|
102
|
+
|
|
103
|
+
def pull(
|
|
104
|
+
self, *, subscription: str, max_messages: int = 1, timeout: float | None = None
|
|
105
|
+
) -> _PullResponse:
|
|
106
|
+
sub = self._broker._subscriptions.setdefault(
|
|
107
|
+
subscription, _Subscription(topic_path=subscription)
|
|
108
|
+
)
|
|
109
|
+
received = []
|
|
110
|
+
for _ in range(min(max_messages, len(sub.pending))):
|
|
111
|
+
data = sub.pending.popleft()
|
|
112
|
+
ack_id = str(uuid.uuid4())
|
|
113
|
+
sub.in_flight[ack_id] = data
|
|
114
|
+
received.append(_ReceivedMessage(ack_id=ack_id, message=_Message(data=data)))
|
|
115
|
+
return _PullResponse(received_messages=received)
|
|
116
|
+
|
|
117
|
+
def acknowledge(self, *, subscription: str, ack_ids: list[str]) -> None:
|
|
118
|
+
sub = self._broker._subscriptions.get(subscription)
|
|
119
|
+
if sub is None:
|
|
120
|
+
return
|
|
121
|
+
for ack_id in ack_ids:
|
|
122
|
+
sub.in_flight.pop(ack_id, None)
|
|
123
|
+
|
|
124
|
+
def modify_ack_deadline(
|
|
125
|
+
self, *, subscription: str, ack_ids: list[str], ack_deadline_seconds: int
|
|
126
|
+
) -> None:
|
|
127
|
+
sub = self._broker._subscriptions.get(subscription)
|
|
128
|
+
if sub is None:
|
|
129
|
+
return
|
|
130
|
+
for ack_id in ack_ids:
|
|
131
|
+
data = sub.in_flight.pop(ack_id, None)
|
|
132
|
+
if data is not None and ack_deadline_seconds == 0:
|
|
133
|
+
sub.pending.appendleft(data)
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
@dataclass(slots=True)
|
|
137
|
+
class FakePubsubBroker:
|
|
138
|
+
"""An isolated in-memory Pub/Sub double. Pass `broker.publisher` and
|
|
139
|
+
`broker.subscriber` as the `publisher_factory=`/`subscriber_factory=`
|
|
140
|
+
arguments to PubsubTaskQueue in place of the real SDK client classes."""
|
|
141
|
+
|
|
142
|
+
_topics: set[str] = field(default_factory=set, init=False)
|
|
143
|
+
_subscriptions: dict[str, _Subscription] = field(default_factory=dict, init=False)
|
|
144
|
+
|
|
145
|
+
def publisher(self) -> _FakePublisherClient:
|
|
146
|
+
return _FakePublisherClient(_broker=self)
|
|
147
|
+
|
|
148
|
+
def subscriber(self) -> _FakeSubscriberClient:
|
|
149
|
+
return _FakeSubscriberClient(_broker=self)
|
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
"""An in-memory double of the slice of aio_pika that RabbitMqTaskQueue
|
|
2
|
+
calls, for testing code that uses this package without a real broker.
|
|
3
|
+
|
|
4
|
+
Usage:
|
|
5
|
+
broker = FakeRabbitMqBroker()
|
|
6
|
+
queue = RabbitMqTaskQueue(dsn=dsn, encode=encode, decode=decode, connect=broker.connect)
|
|
7
|
+
|
|
8
|
+
Each FakeRabbitMqBroker instance owns its own isolated state — create a new
|
|
9
|
+
one per test (or per broker you want to simulate) rather than sharing one,
|
|
10
|
+
and there is nothing to reset between tests.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import asyncio
|
|
16
|
+
from collections import deque
|
|
17
|
+
from dataclasses import dataclass, field
|
|
18
|
+
|
|
19
|
+
__all__ = ["FakeRabbitMqBroker"]
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
@dataclass(slots=True)
|
|
23
|
+
class _Queue:
|
|
24
|
+
"""One named queue's backlog plus messages currently delivered but not
|
|
25
|
+
yet ack()'d/nack()'d — mirrors a real broker's per-queue state."""
|
|
26
|
+
|
|
27
|
+
pending: deque[bytes] = field(default_factory=deque)
|
|
28
|
+
unacked: dict[int, bytes] = field(default_factory=dict)
|
|
29
|
+
_next_id: int = 0
|
|
30
|
+
|
|
31
|
+
def push(self, body: bytes) -> None:
|
|
32
|
+
self.pending.append(body)
|
|
33
|
+
|
|
34
|
+
def requeue(self, delivery_id: int) -> None:
|
|
35
|
+
body = self.unacked.pop(delivery_id, None)
|
|
36
|
+
if body is not None:
|
|
37
|
+
self.pending.appendleft(body)
|
|
38
|
+
|
|
39
|
+
def drop(self, delivery_id: int) -> None:
|
|
40
|
+
self.unacked.pop(delivery_id, None)
|
|
41
|
+
|
|
42
|
+
def next_delivery_id(self) -> int:
|
|
43
|
+
self._next_id += 1
|
|
44
|
+
return self._next_id
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
@dataclass(slots=True)
|
|
48
|
+
class _FakeIncomingMessage:
|
|
49
|
+
body: bytes
|
|
50
|
+
_delivery_id: int
|
|
51
|
+
_queue: _Queue
|
|
52
|
+
|
|
53
|
+
async def ack(self) -> None:
|
|
54
|
+
self._queue.drop(self._delivery_id)
|
|
55
|
+
|
|
56
|
+
async def nack(self, *, requeue: bool = True) -> None:
|
|
57
|
+
if requeue:
|
|
58
|
+
self._queue.requeue(self._delivery_id)
|
|
59
|
+
else:
|
|
60
|
+
self._queue.drop(self._delivery_id)
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
@dataclass(slots=True)
|
|
64
|
+
class _FakeAioPikaQueue:
|
|
65
|
+
name: str
|
|
66
|
+
_queue: _Queue
|
|
67
|
+
|
|
68
|
+
def iterator(self) -> _FakeAioPikaQueue:
|
|
69
|
+
return self
|
|
70
|
+
|
|
71
|
+
def __aiter__(self) -> _FakeAioPikaQueue:
|
|
72
|
+
return self
|
|
73
|
+
|
|
74
|
+
async def __anext__(self) -> _FakeIncomingMessage:
|
|
75
|
+
while not self._queue.pending:
|
|
76
|
+
await asyncio.sleep(0)
|
|
77
|
+
body = self._queue.pending.popleft()
|
|
78
|
+
delivery_id = self._queue.next_delivery_id()
|
|
79
|
+
self._queue.unacked[delivery_id] = body
|
|
80
|
+
return _FakeIncomingMessage(body=body, _delivery_id=delivery_id, _queue=self._queue)
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
@dataclass(slots=True)
|
|
84
|
+
class _FakeExchange:
|
|
85
|
+
_queue: _Queue
|
|
86
|
+
|
|
87
|
+
async def publish(self, message: object, *, routing_key: str) -> None:
|
|
88
|
+
body = getattr(message, "body", b"")
|
|
89
|
+
self._queue.push(body)
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
@dataclass(slots=True)
|
|
93
|
+
class _FakeChannel:
|
|
94
|
+
_broker: FakeRabbitMqBroker
|
|
95
|
+
# Bound by declare_queue() — this fake routes directly to a named queue
|
|
96
|
+
# rather than modeling real exchange/routing-key bindings, so there's
|
|
97
|
+
# nothing meaningful to set until then.
|
|
98
|
+
default_exchange: _FakeExchange | None = field(default=None, init=False)
|
|
99
|
+
|
|
100
|
+
async def set_qos(self, *, prefetch_count: int) -> None:
|
|
101
|
+
return None
|
|
102
|
+
|
|
103
|
+
async def declare_queue(self, name: str, *, durable: bool = True) -> _FakeAioPikaQueue:
|
|
104
|
+
queue = self._broker._queue_for(name)
|
|
105
|
+
self.default_exchange = _FakeExchange(_queue=queue)
|
|
106
|
+
return _FakeAioPikaQueue(name=name, _queue=queue)
|
|
107
|
+
|
|
108
|
+
async def queue_delete(self, name: str) -> None:
|
|
109
|
+
self._broker._queues.pop(name, None)
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
@dataclass(slots=True)
|
|
113
|
+
class _FakeConnection:
|
|
114
|
+
_broker: FakeRabbitMqBroker
|
|
115
|
+
closed: bool = False
|
|
116
|
+
|
|
117
|
+
async def channel(self) -> _FakeChannel:
|
|
118
|
+
return _FakeChannel(_broker=self._broker)
|
|
119
|
+
|
|
120
|
+
async def close(self) -> None:
|
|
121
|
+
self.closed = True
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
@dataclass(slots=True)
|
|
125
|
+
class FakeRabbitMqBroker:
|
|
126
|
+
"""An isolated in-memory RabbitMQ double. Pass `broker.connect` as the
|
|
127
|
+
`connect=` argument to RabbitMqTaskQueue in place of aio_pika.connect_robust."""
|
|
128
|
+
|
|
129
|
+
_queues: dict[str, _Queue] = field(default_factory=dict, init=False)
|
|
130
|
+
|
|
131
|
+
def _queue_for(self, name: str) -> _Queue:
|
|
132
|
+
return self._queues.setdefault(name, _Queue())
|
|
133
|
+
|
|
134
|
+
async def connect(self, url: str) -> _FakeConnection:
|
|
135
|
+
return _FakeConnection(_broker=self)
|
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
"""An in-memory double of the slice of redis.asyncio that
|
|
2
|
+
RedisStreamsTaskQueue calls, for testing code that uses this package
|
|
3
|
+
without a real Redis — with real consumer-group semantics (a pending
|
|
4
|
+
entries list per group, "0" re-reads a consumer's own unacked entries
|
|
5
|
+
before ">" reads new ones), since that ordering is what nack(requeue=True)
|
|
6
|
+
relies on.
|
|
7
|
+
|
|
8
|
+
Usage:
|
|
9
|
+
broker = FakeRedisStreamsBroker()
|
|
10
|
+
queue = RedisStreamsTaskQueue(
|
|
11
|
+
dsn=dsn, encode=encode, decode=decode, client_factory=broker.client
|
|
12
|
+
)
|
|
13
|
+
|
|
14
|
+
Each FakeRedisStreamsBroker instance owns its own isolated state — create a
|
|
15
|
+
new one per test rather than sharing one, and there is nothing to reset
|
|
16
|
+
between tests.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
import asyncio
|
|
22
|
+
from dataclasses import dataclass, field
|
|
23
|
+
|
|
24
|
+
from redis.exceptions import ResponseError
|
|
25
|
+
|
|
26
|
+
__all__ = ["FakeRedisStreamsBroker"]
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
@dataclass(slots=True)
|
|
30
|
+
class _Group:
|
|
31
|
+
# Entries delivered to a consumer but not yet XACKed, in delivery order
|
|
32
|
+
# (a real broker's Pending Entries List) — nack(requeue=True) leaves an
|
|
33
|
+
# entry here; xreadgroup's "0" branch redelivers from here first.
|
|
34
|
+
pending_entries: list[tuple[bytes, dict[bytes, bytes]]] = field(default_factory=list)
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
@dataclass(slots=True)
|
|
38
|
+
class _Stream:
|
|
39
|
+
entries: list[tuple[bytes, dict[bytes, bytes]]] = field(default_factory=list)
|
|
40
|
+
next_id: int = 1
|
|
41
|
+
groups: dict[bytes, _Group] = field(default_factory=dict)
|
|
42
|
+
# Position in `entries` each group has already delivered up to, for the
|
|
43
|
+
# ">" (new entries) branch of xreadgroup.
|
|
44
|
+
cursor: dict[bytes, int] = field(default_factory=dict)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
@dataclass(slots=True)
|
|
48
|
+
class _FakeRedisClient:
|
|
49
|
+
_broker: FakeRedisStreamsBroker
|
|
50
|
+
|
|
51
|
+
async def ping(self) -> bool:
|
|
52
|
+
return True
|
|
53
|
+
|
|
54
|
+
async def xgroup_create(
|
|
55
|
+
self, name: str, group: str, *, id: str = "0", mkstream: bool = False
|
|
56
|
+
) -> None:
|
|
57
|
+
stream = self._broker._stream_for(name.encode())
|
|
58
|
+
group_key = group.encode()
|
|
59
|
+
if group_key in stream.groups:
|
|
60
|
+
raise ResponseError("BUSYGROUP Consumer Group name already exists")
|
|
61
|
+
stream.groups[group_key] = _Group()
|
|
62
|
+
stream.cursor[group_key] = 0
|
|
63
|
+
|
|
64
|
+
async def xadd(self, name: str, fields: dict[str, bytes]) -> bytes:
|
|
65
|
+
stream = self._broker._stream_for(name.encode())
|
|
66
|
+
entry_id = f"{stream.next_id}-0".encode()
|
|
67
|
+
stream.next_id += 1
|
|
68
|
+
encoded_fields = {key.encode(): value for key, value in fields.items()}
|
|
69
|
+
stream.entries.append((entry_id, encoded_fields))
|
|
70
|
+
return entry_id
|
|
71
|
+
|
|
72
|
+
async def xreadgroup(
|
|
73
|
+
self,
|
|
74
|
+
group: str,
|
|
75
|
+
consumer: str,
|
|
76
|
+
streams: dict[str, str],
|
|
77
|
+
*,
|
|
78
|
+
count: int | None = None,
|
|
79
|
+
block: int | None = None,
|
|
80
|
+
) -> list[tuple[bytes, list[tuple[bytes, dict[bytes, bytes]]]]]:
|
|
81
|
+
((name, read_id),) = streams.items()
|
|
82
|
+
stream = self._broker._stream_for(name.encode())
|
|
83
|
+
group_key = group.encode()
|
|
84
|
+
grp = stream.groups[group_key]
|
|
85
|
+
|
|
86
|
+
if read_id == "0":
|
|
87
|
+
# Re-deliver this consumer's own still-pending entries — this
|
|
88
|
+
# fake doesn't track per-consumer PELs separately, only
|
|
89
|
+
# per-group, matching this package's single-consumer-per-group
|
|
90
|
+
# usage pattern.
|
|
91
|
+
delivered = grp.pending_entries[: count or len(grp.pending_entries)]
|
|
92
|
+
return [(name.encode(), delivered)]
|
|
93
|
+
|
|
94
|
+
# ">" branch: deliver new entries the group hasn't seen yet.
|
|
95
|
+
cursor = stream.cursor[group_key]
|
|
96
|
+
new_entries = stream.entries[cursor : cursor + (count or 1)]
|
|
97
|
+
if new_entries:
|
|
98
|
+
stream.cursor[group_key] = cursor + len(new_entries)
|
|
99
|
+
grp.pending_entries.extend(new_entries)
|
|
100
|
+
return [(name.encode(), new_entries)]
|
|
101
|
+
|
|
102
|
+
# Nothing new — a real broker blocks up to `block` ms; the fake
|
|
103
|
+
# yields control once so callers polling in a loop don't spin the
|
|
104
|
+
# event loop hot, then returns empty like a timed-out BLOCK would.
|
|
105
|
+
await asyncio.sleep(0)
|
|
106
|
+
return [(name.encode(), [])]
|
|
107
|
+
|
|
108
|
+
async def xack(self, name: str, group: str, *entry_ids: bytes) -> int:
|
|
109
|
+
stream = self._broker._stream_for(name.encode())
|
|
110
|
+
grp = stream.groups.get(group.encode())
|
|
111
|
+
if grp is None:
|
|
112
|
+
return 0
|
|
113
|
+
ids = set(entry_ids)
|
|
114
|
+
before = len(grp.pending_entries)
|
|
115
|
+
grp.pending_entries = [e for e in grp.pending_entries if e[0] not in ids]
|
|
116
|
+
return before - len(grp.pending_entries)
|
|
117
|
+
|
|
118
|
+
async def delete(self, *names: str) -> int:
|
|
119
|
+
count = 0
|
|
120
|
+
for name in names:
|
|
121
|
+
if self._broker._streams.pop(name.encode(), None) is not None:
|
|
122
|
+
count += 1
|
|
123
|
+
return count
|
|
124
|
+
|
|
125
|
+
async def aclose(self) -> None:
|
|
126
|
+
return None
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
@dataclass(slots=True)
|
|
130
|
+
class FakeRedisStreamsBroker:
|
|
131
|
+
"""An isolated in-memory Redis Streams double. Pass `broker.client` as
|
|
132
|
+
the `client_factory=` argument to RedisStreamsTaskQueue in place of the
|
|
133
|
+
default redis.Redis.from_url-based factory."""
|
|
134
|
+
|
|
135
|
+
_streams: dict[bytes, _Stream] = field(default_factory=dict, init=False)
|
|
136
|
+
|
|
137
|
+
def _stream_for(self, name: bytes) -> _Stream:
|
|
138
|
+
return self._streams.setdefault(name, _Stream())
|
|
139
|
+
|
|
140
|
+
def client(self, dsn: object) -> _FakeRedisClient:
|
|
141
|
+
return _FakeRedisClient(_broker=self)
|
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
"""An in-memory double of the slice of aioboto3's SNS+SQS clients that
|
|
2
|
+
SnsTaskQueue calls, for testing code that uses this package without a real
|
|
3
|
+
topic/queue.
|
|
4
|
+
|
|
5
|
+
Usage:
|
|
6
|
+
broker = FakeSnsBroker()
|
|
7
|
+
queue = SnsTaskQueue(dsn=dsn, encode=encode, decode=decode, client_factory=broker.client)
|
|
8
|
+
|
|
9
|
+
publish() fans out directly into whichever SQS queues are subscribed to the
|
|
10
|
+
topic, wrapping the body in the same {"Type": ..., "Message": ...} envelope
|
|
11
|
+
real SNS-to-SQS delivery uses, since SnsTaskQueue.consume() unwraps that
|
|
12
|
+
envelope.
|
|
13
|
+
|
|
14
|
+
Each FakeSnsBroker instance owns its own isolated state — create a new one
|
|
15
|
+
per test rather than sharing one, and there is nothing to reset between
|
|
16
|
+
tests.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
import json
|
|
22
|
+
import uuid
|
|
23
|
+
from collections import deque
|
|
24
|
+
from contextlib import AbstractAsyncContextManager
|
|
25
|
+
from dataclasses import dataclass, field
|
|
26
|
+
|
|
27
|
+
__all__ = ["FakeSnsBroker"]
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class QueueDoesNotExist(Exception):
|
|
31
|
+
"""Raised by the fake SQS client the same way the real SDK's
|
|
32
|
+
client.exceptions.QueueDoesNotExist would be."""
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
@dataclass(slots=True)
|
|
36
|
+
class _Exceptions:
|
|
37
|
+
QueueDoesNotExist: type[Exception] = QueueDoesNotExist
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
@dataclass(slots=True)
|
|
41
|
+
class _Queue:
|
|
42
|
+
pending: deque[dict[str, str]] = field(default_factory=deque)
|
|
43
|
+
in_flight: dict[str, dict[str, str]] = field(default_factory=dict)
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _topic_arn(name: str) -> str:
|
|
47
|
+
return f"arn:aws:sns:fake:000000000000:{name}"
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
@dataclass(slots=True)
|
|
51
|
+
class _FakeSnsClient:
|
|
52
|
+
_broker: FakeSnsBroker
|
|
53
|
+
|
|
54
|
+
async def create_topic(self, *, Name: str) -> dict[str, str]:
|
|
55
|
+
arn = _topic_arn(Name)
|
|
56
|
+
self._broker._topics.setdefault(arn, set())
|
|
57
|
+
return {"TopicArn": arn}
|
|
58
|
+
|
|
59
|
+
async def delete_topic(self, *, TopicArn: str) -> None:
|
|
60
|
+
self._broker._topics.pop(TopicArn, None)
|
|
61
|
+
|
|
62
|
+
async def subscribe(self, *, TopicArn: str, Protocol: str, Endpoint: str) -> dict[str, str]:
|
|
63
|
+
self._broker._topics.setdefault(TopicArn, set()).add(Endpoint)
|
|
64
|
+
return {"SubscriptionArn": f"{TopicArn}:{uuid.uuid4()}"}
|
|
65
|
+
|
|
66
|
+
async def publish(self, *, TopicArn: str, Message: str) -> dict[str, str]:
|
|
67
|
+
envelope = json.dumps(
|
|
68
|
+
{
|
|
69
|
+
"Type": "Notification",
|
|
70
|
+
"MessageId": str(uuid.uuid4()),
|
|
71
|
+
"TopicArn": TopicArn,
|
|
72
|
+
"Message": Message,
|
|
73
|
+
}
|
|
74
|
+
)
|
|
75
|
+
for queue_url in self._broker._topics.get(TopicArn, set()):
|
|
76
|
+
queue = self._broker._queues.setdefault(queue_url, _Queue())
|
|
77
|
+
queue.pending.append({"Body": envelope})
|
|
78
|
+
return {"MessageId": str(uuid.uuid4())}
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
@dataclass(slots=True)
|
|
82
|
+
class _FakeSqsClient:
|
|
83
|
+
_broker: FakeSnsBroker
|
|
84
|
+
exceptions: _Exceptions = field(default_factory=_Exceptions)
|
|
85
|
+
|
|
86
|
+
async def get_queue_url(self, *, QueueName: str) -> dict[str, str]:
|
|
87
|
+
if QueueName not in self._broker._queues:
|
|
88
|
+
raise self.exceptions.QueueDoesNotExist(QueueName)
|
|
89
|
+
return {"QueueUrl": QueueName}
|
|
90
|
+
|
|
91
|
+
async def create_queue(self, *, QueueName: str) -> dict[str, str]:
|
|
92
|
+
self._broker._queues.setdefault(QueueName, _Queue())
|
|
93
|
+
return {"QueueUrl": QueueName}
|
|
94
|
+
|
|
95
|
+
async def get_queue_attributes(
|
|
96
|
+
self, *, QueueUrl: str, AttributeNames: list[str]
|
|
97
|
+
) -> dict[str, dict[str, str]]:
|
|
98
|
+
# This fake uses the queue URL as its own ARN too — good enough
|
|
99
|
+
# since nothing here parses ARN structure, only threads the value
|
|
100
|
+
# through to subscribe()'s Endpoint, which this fake also keys by
|
|
101
|
+
# queue URL.
|
|
102
|
+
return {"Attributes": {"QueueArn": QueueUrl}}
|
|
103
|
+
|
|
104
|
+
async def set_queue_attributes(self, *, QueueUrl: str, Attributes: dict[str, str]) -> None:
|
|
105
|
+
return None
|
|
106
|
+
|
|
107
|
+
async def receive_message(
|
|
108
|
+
self, *, QueueUrl: str, MaxNumberOfMessages: int = 1, WaitTimeSeconds: int = 0
|
|
109
|
+
) -> dict[str, list[dict[str, str]]]:
|
|
110
|
+
queue = self._broker._queues.setdefault(QueueUrl, _Queue())
|
|
111
|
+
messages = []
|
|
112
|
+
for _ in range(min(MaxNumberOfMessages, len(queue.pending))):
|
|
113
|
+
body = queue.pending.popleft()
|
|
114
|
+
receipt_handle = str(uuid.uuid4())
|
|
115
|
+
queue.in_flight[receipt_handle] = body
|
|
116
|
+
messages.append({"Body": body["Body"], "ReceiptHandle": receipt_handle})
|
|
117
|
+
return {"Messages": messages}
|
|
118
|
+
|
|
119
|
+
async def delete_message(self, *, QueueUrl: str, ReceiptHandle: str) -> None:
|
|
120
|
+
queue = self._broker._queues.setdefault(QueueUrl, _Queue())
|
|
121
|
+
queue.in_flight.pop(ReceiptHandle, None)
|
|
122
|
+
|
|
123
|
+
async def change_message_visibility(
|
|
124
|
+
self, *, QueueUrl: str, ReceiptHandle: str, VisibilityTimeout: int
|
|
125
|
+
) -> None:
|
|
126
|
+
queue = self._broker._queues.setdefault(QueueUrl, _Queue())
|
|
127
|
+
body = queue.in_flight.pop(ReceiptHandle, None)
|
|
128
|
+
if body is not None and VisibilityTimeout == 0:
|
|
129
|
+
queue.pending.appendleft(body)
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
class _FakeSnsContext(AbstractAsyncContextManager[tuple["_FakeSnsClient", "_FakeSqsClient"]]):
|
|
133
|
+
def __init__(self, broker: FakeSnsBroker) -> None:
|
|
134
|
+
self._broker = broker
|
|
135
|
+
|
|
136
|
+
async def __aenter__(self) -> tuple[_FakeSnsClient, _FakeSqsClient]:
|
|
137
|
+
return _FakeSnsClient(_broker=self._broker), _FakeSqsClient(_broker=self._broker)
|
|
138
|
+
|
|
139
|
+
async def __aexit__(self, *exc_info: object) -> None:
|
|
140
|
+
return None
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
@dataclass(slots=True)
|
|
144
|
+
class FakeSnsBroker:
|
|
145
|
+
"""An isolated in-memory SNS+SQS double. Pass `broker.client` as the
|
|
146
|
+
`client_factory=` argument to SnsTaskQueue in place of the default
|
|
147
|
+
aioboto3-session-based factory."""
|
|
148
|
+
|
|
149
|
+
_topics: dict[str, set[str]] = field(default_factory=dict, init=False)
|
|
150
|
+
_queues: dict[str, _Queue] = field(default_factory=dict, init=False)
|
|
151
|
+
|
|
152
|
+
def client(
|
|
153
|
+
self, dsn: object
|
|
154
|
+
) -> AbstractAsyncContextManager[tuple[_FakeSnsClient, _FakeSqsClient]]:
|
|
155
|
+
return _FakeSnsContext(self)
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
"""An in-memory double of the slice of aioboto3's SQS client that
|
|
2
|
+
SqsTaskQueue calls, for testing code that uses this package without a real
|
|
3
|
+
queue.
|
|
4
|
+
|
|
5
|
+
Usage:
|
|
6
|
+
broker = FakeSqsBroker()
|
|
7
|
+
queue = SqsTaskQueue(dsn=dsn, encode=encode, decode=decode, client_factory=broker.client)
|
|
8
|
+
|
|
9
|
+
Each FakeSqsBroker instance owns its own isolated state — create a new one
|
|
10
|
+
per test rather than sharing one, and there is nothing to reset between
|
|
11
|
+
tests.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import uuid
|
|
17
|
+
from collections import deque
|
|
18
|
+
from contextlib import AbstractAsyncContextManager
|
|
19
|
+
from dataclasses import dataclass, field
|
|
20
|
+
|
|
21
|
+
__all__ = ["FakeSqsBroker"]
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class QueueDoesNotExist(Exception):
|
|
25
|
+
"""Raised by the fake client the same way the real SDK's
|
|
26
|
+
client.exceptions.QueueDoesNotExist would be — SqsTaskQueue catches
|
|
27
|
+
this via `except client.exceptions.QueueDoesNotExist`, which this fake
|
|
28
|
+
client exposes as an instance attribute for exactly that reason."""
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
@dataclass(slots=True)
|
|
32
|
+
class _Exceptions:
|
|
33
|
+
QueueDoesNotExist: type[Exception] = QueueDoesNotExist
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
@dataclass(slots=True)
|
|
37
|
+
class _Queue:
|
|
38
|
+
pending: deque[dict[str, str]] = field(default_factory=deque)
|
|
39
|
+
in_flight: dict[str, dict[str, str]] = field(default_factory=dict)
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
@dataclass(slots=True)
|
|
43
|
+
class _FakeSqsClient:
|
|
44
|
+
_broker: FakeSqsBroker
|
|
45
|
+
exceptions: _Exceptions = field(default_factory=_Exceptions)
|
|
46
|
+
|
|
47
|
+
async def get_queue_url(self, *, QueueName: str) -> dict[str, str]:
|
|
48
|
+
if QueueName not in self._broker._queues:
|
|
49
|
+
raise self.exceptions.QueueDoesNotExist(QueueName)
|
|
50
|
+
return {"QueueUrl": QueueName}
|
|
51
|
+
|
|
52
|
+
async def create_queue(self, *, QueueName: str) -> dict[str, str]:
|
|
53
|
+
self._broker._queues.setdefault(QueueName, _Queue())
|
|
54
|
+
return {"QueueUrl": QueueName}
|
|
55
|
+
|
|
56
|
+
async def delete_queue(self, *, QueueUrl: str) -> None:
|
|
57
|
+
self._broker._queues.pop(QueueUrl, None)
|
|
58
|
+
|
|
59
|
+
async def send_message(self, *, QueueUrl: str, MessageBody: str) -> dict[str, str]:
|
|
60
|
+
queue = self._broker._queues.setdefault(QueueUrl, _Queue())
|
|
61
|
+
queue.pending.append({"Body": MessageBody})
|
|
62
|
+
return {"MessageId": str(uuid.uuid4())}
|
|
63
|
+
|
|
64
|
+
async def receive_message(
|
|
65
|
+
self, *, QueueUrl: str, MaxNumberOfMessages: int = 1, WaitTimeSeconds: int = 0
|
|
66
|
+
) -> dict[str, list[dict[str, str]]]:
|
|
67
|
+
queue = self._broker._queues.setdefault(QueueUrl, _Queue())
|
|
68
|
+
messages = []
|
|
69
|
+
for _ in range(min(MaxNumberOfMessages, len(queue.pending))):
|
|
70
|
+
body = queue.pending.popleft()
|
|
71
|
+
receipt_handle = str(uuid.uuid4())
|
|
72
|
+
queue.in_flight[receipt_handle] = body
|
|
73
|
+
messages.append({"Body": body["Body"], "ReceiptHandle": receipt_handle})
|
|
74
|
+
return {"Messages": messages}
|
|
75
|
+
|
|
76
|
+
async def delete_message(self, *, QueueUrl: str, ReceiptHandle: str) -> None:
|
|
77
|
+
queue = self._broker._queues.setdefault(QueueUrl, _Queue())
|
|
78
|
+
queue.in_flight.pop(ReceiptHandle, None)
|
|
79
|
+
|
|
80
|
+
async def change_message_visibility(
|
|
81
|
+
self, *, QueueUrl: str, ReceiptHandle: str, VisibilityTimeout: int
|
|
82
|
+
) -> None:
|
|
83
|
+
queue = self._broker._queues.setdefault(QueueUrl, _Queue())
|
|
84
|
+
body = queue.in_flight.pop(ReceiptHandle, None)
|
|
85
|
+
if body is not None and VisibilityTimeout == 0:
|
|
86
|
+
queue.pending.appendleft(body)
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
class _FakeClientContext(AbstractAsyncContextManager["_FakeSqsClient"]):
|
|
90
|
+
def __init__(self, broker: FakeSqsBroker) -> None:
|
|
91
|
+
self._broker = broker
|
|
92
|
+
|
|
93
|
+
async def __aenter__(self) -> _FakeSqsClient:
|
|
94
|
+
return _FakeSqsClient(_broker=self._broker)
|
|
95
|
+
|
|
96
|
+
async def __aexit__(self, *exc_info: object) -> None:
|
|
97
|
+
return None
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
@dataclass(slots=True)
|
|
101
|
+
class FakeSqsBroker:
|
|
102
|
+
"""An isolated in-memory SQS double. Pass `broker.client` as the
|
|
103
|
+
`client_factory=` argument to SqsTaskQueue in place of the default
|
|
104
|
+
aioboto3-session-based factory."""
|
|
105
|
+
|
|
106
|
+
_queues: dict[str, _Queue] = field(default_factory=dict, init=False)
|
|
107
|
+
|
|
108
|
+
def client(self, dsn: object) -> AbstractAsyncContextManager[_FakeSqsClient]:
|
|
109
|
+
return _FakeClientContext(self)
|