taskqueue-toolkit 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- taskqueue_toolkit/__init__.py +0 -0
- taskqueue_toolkit/outbox/__init__.py +0 -0
- taskqueue_toolkit/outbox/orm.py +47 -0
- taskqueue_toolkit/outbox/relay.py +59 -0
- taskqueue_toolkit/outbox/repository.py +107 -0
- taskqueue_toolkit/py.typed +0 -0
- taskqueue_toolkit/queue/__init__.py +0 -0
- taskqueue_toolkit/queue/aws_session.py +19 -0
- taskqueue_toolkit/queue/dsn.py +147 -0
- taskqueue_toolkit/queue/factory.py +94 -0
- taskqueue_toolkit/queue/pubsub.py +162 -0
- taskqueue_toolkit/queue/rabbitmq.py +74 -0
- taskqueue_toolkit/queue/redis_streams.py +117 -0
- taskqueue_toolkit/queue/registry.py +53 -0
- taskqueue_toolkit/queue/sns.py +162 -0
- taskqueue_toolkit/queue/sqs.py +103 -0
- taskqueue_toolkit/queue/task_queue.py +62 -0
- taskqueue_toolkit/testing/__init__.py +58 -0
- taskqueue_toolkit/testing/pubsub.py +149 -0
- taskqueue_toolkit/testing/rabbitmq.py +135 -0
- taskqueue_toolkit/testing/redis_streams.py +141 -0
- taskqueue_toolkit/testing/sns.py +155 -0
- taskqueue_toolkit/testing/sqs.py +109 -0
- taskqueue_toolkit-0.1.0.dist-info/METADATA +163 -0
- taskqueue_toolkit-0.1.0.dist-info/RECORD +27 -0
- taskqueue_toolkit-0.1.0.dist-info/WHEEL +4 -0
- taskqueue_toolkit-0.1.0.dist-info/licenses/LICENSE +202 -0
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import logging
|
|
4
|
+
from collections.abc import AsyncIterator, Awaitable, Callable
|
|
5
|
+
from contextlib import suppress
|
|
6
|
+
from dataclasses import dataclass, field
|
|
7
|
+
|
|
8
|
+
import aio_pika
|
|
9
|
+
|
|
10
|
+
from taskqueue_toolkit.queue.dsn import RabbitMqDsn
|
|
11
|
+
from taskqueue_toolkit.queue.task_queue import Decoder, Encoder
|
|
12
|
+
|
|
13
|
+
logger = logging.getLogger(__name__)
|
|
14
|
+
|
|
15
|
+
Connect = Callable[[str], Awaitable["aio_pika.abc.AbstractRobustConnection"]]
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
@dataclass(slots=True)
|
|
19
|
+
class RabbitMqQueuedTask[T]:
|
|
20
|
+
_message: aio_pika.abc.AbstractIncomingMessage
|
|
21
|
+
task: T
|
|
22
|
+
|
|
23
|
+
async def ack(self) -> None:
|
|
24
|
+
await self._message.ack()
|
|
25
|
+
|
|
26
|
+
async def nack(self, *, requeue: bool) -> None:
|
|
27
|
+
await self._message.nack(requeue=requeue)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
@dataclass(slots=True)
|
|
31
|
+
class RabbitMqTaskQueue[T]:
|
|
32
|
+
dsn: RabbitMqDsn
|
|
33
|
+
encode: Encoder[T]
|
|
34
|
+
decode: Decoder[T]
|
|
35
|
+
# Overridable for tests (inject a fake connection) or to share/pool a
|
|
36
|
+
# connection strategy the caller already has; defaults to the real SDK.
|
|
37
|
+
connect: Connect = field(default=aio_pika.connect_robust)
|
|
38
|
+
|
|
39
|
+
async def publish(self, task: T) -> None:
|
|
40
|
+
connection = await self.connect(self.dsn.url)
|
|
41
|
+
try:
|
|
42
|
+
channel = await connection.channel()
|
|
43
|
+
queue = await channel.declare_queue(self.dsn.queue_name, durable=True)
|
|
44
|
+
await channel.default_exchange.publish(
|
|
45
|
+
aio_pika.Message(
|
|
46
|
+
body=self.encode(task),
|
|
47
|
+
delivery_mode=aio_pika.DeliveryMode.PERSISTENT,
|
|
48
|
+
),
|
|
49
|
+
routing_key=queue.name,
|
|
50
|
+
)
|
|
51
|
+
logger.info("task published", extra={"queue": self.dsn.queue_name})
|
|
52
|
+
finally:
|
|
53
|
+
await connection.close()
|
|
54
|
+
|
|
55
|
+
async def consume(self) -> AsyncIterator[RabbitMqQueuedTask[T]]:
|
|
56
|
+
connection = await self.connect(self.dsn.url)
|
|
57
|
+
try:
|
|
58
|
+
channel = await connection.channel()
|
|
59
|
+
await channel.set_qos(prefetch_count=1)
|
|
60
|
+
queue = await channel.declare_queue(self.dsn.queue_name, durable=True)
|
|
61
|
+
|
|
62
|
+
async for message in queue.iterator():
|
|
63
|
+
yield RabbitMqQueuedTask(_message=message, task=self.decode(message.body))
|
|
64
|
+
finally:
|
|
65
|
+
# The consumer (a worker's `async for`) can stop iterating at any
|
|
66
|
+
# point — graceful shutdown, an unhandled error, etc. — which
|
|
67
|
+
# raises GeneratorExit right here, inside the yield above. At
|
|
68
|
+
# that point there's no time left to round-trip a clean
|
|
69
|
+
# basic.cancel RPC with the broker (the event loop is usually
|
|
70
|
+
# already shutting down too), so don't try — just drop the
|
|
71
|
+
# connection. RabbitMQ notices the disconnect and requeues
|
|
72
|
+
# whatever message was left unacknowledged.
|
|
73
|
+
with suppress(Exception):
|
|
74
|
+
await connection.close()
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import logging
|
|
4
|
+
from collections.abc import AsyncIterator, Callable
|
|
5
|
+
from dataclasses import dataclass, field
|
|
6
|
+
from typing import cast
|
|
7
|
+
|
|
8
|
+
import redis.asyncio as redis
|
|
9
|
+
from redis.exceptions import ResponseError
|
|
10
|
+
|
|
11
|
+
from taskqueue_toolkit.queue.dsn import RedisStreamsDsn
|
|
12
|
+
from taskqueue_toolkit.queue.task_queue import Decoder, Encoder
|
|
13
|
+
|
|
14
|
+
logger = logging.getLogger(__name__)
|
|
15
|
+
|
|
16
|
+
_LONG_POLL_BLOCK_MS = 10_000
|
|
17
|
+
_FIELD = "payload"
|
|
18
|
+
|
|
19
|
+
ClientFactory = Callable[[RedisStreamsDsn], "redis.Redis"]
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _default_client_factory(dsn: RedisStreamsDsn) -> redis.Redis:
|
|
23
|
+
return redis.Redis.from_url(dsn.url)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
@dataclass(slots=True)
|
|
27
|
+
class RedisStreamsQueuedTask[T]:
|
|
28
|
+
_redis: redis.Redis
|
|
29
|
+
_stream: str
|
|
30
|
+
_group: str
|
|
31
|
+
_message_id: str
|
|
32
|
+
task: T
|
|
33
|
+
|
|
34
|
+
async def ack(self) -> None:
|
|
35
|
+
await self._redis.xack(self._stream, self._group, self._message_id)
|
|
36
|
+
|
|
37
|
+
async def nack(self, *, requeue: bool) -> None:
|
|
38
|
+
if requeue:
|
|
39
|
+
# Leaving the message unacknowledged (in the group's Pending
|
|
40
|
+
# Entries List) is enough — consume()'s next pass re-reads
|
|
41
|
+
# pending entries (id="0") before new ones, so it's redelivered
|
|
42
|
+
# without needing an explicit "give it back" call the way SQS's
|
|
43
|
+
# change_message_visibility does.
|
|
44
|
+
return
|
|
45
|
+
# No per-message dead-letter action here either — same reasoning as
|
|
46
|
+
# SQS: permanently dropping a task means removing it from the PEL,
|
|
47
|
+
# same as a successful ack.
|
|
48
|
+
await self._redis.xack(self._stream, self._group, self._message_id)
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
@dataclass(slots=True)
|
|
52
|
+
class RedisStreamsTaskQueue[T]:
|
|
53
|
+
dsn: RedisStreamsDsn
|
|
54
|
+
encode: Encoder[T]
|
|
55
|
+
decode: Decoder[T]
|
|
56
|
+
# Overridable for tests (inject a fake client) or a shared/pooled
|
|
57
|
+
# connection the caller already manages; defaults to a fresh connection
|
|
58
|
+
# from the DSN's URL per call, matching the real SDK's usual usage.
|
|
59
|
+
client_factory: ClientFactory = field(default=_default_client_factory)
|
|
60
|
+
|
|
61
|
+
def _client(self) -> redis.Redis:
|
|
62
|
+
return self.client_factory(self.dsn)
|
|
63
|
+
|
|
64
|
+
async def _ensure_group(self, client: redis.Redis) -> None:
|
|
65
|
+
try:
|
|
66
|
+
await client.xgroup_create(self.dsn.stream_name, self.dsn.group, id="0", mkstream=True)
|
|
67
|
+
except ResponseError as exc:
|
|
68
|
+
if "BUSYGROUP" not in str(exc):
|
|
69
|
+
raise
|
|
70
|
+
|
|
71
|
+
async def publish(self, task: T) -> None:
|
|
72
|
+
client = self._client()
|
|
73
|
+
try:
|
|
74
|
+
await client.xadd(self.dsn.stream_name, {_FIELD: self.encode(task)})
|
|
75
|
+
logger.info("task published", extra={"stream": self.dsn.stream_name})
|
|
76
|
+
finally:
|
|
77
|
+
await client.aclose()
|
|
78
|
+
|
|
79
|
+
async def consume(self) -> AsyncIterator[RedisStreamsQueuedTask[T]]:
|
|
80
|
+
client = self._client()
|
|
81
|
+
try:
|
|
82
|
+
await self._ensure_group(client)
|
|
83
|
+
stream = self.dsn.stream_name
|
|
84
|
+
group = self.dsn.group
|
|
85
|
+
consumer = self.dsn.consumer
|
|
86
|
+
|
|
87
|
+
while True:
|
|
88
|
+
# Re-deliver our own still-pending entries first (id="0") —
|
|
89
|
+
# covers a crash between XREADGROUP and XACK on a previous
|
|
90
|
+
# run — then fall through to new entries (id=">").
|
|
91
|
+
for read_id in ("0", ">"):
|
|
92
|
+
# redis-py's XReadGroupResponse type is a broad union
|
|
93
|
+
# (it also covers the decode_responses=True str shape);
|
|
94
|
+
# without decode_responses the actual shape is always
|
|
95
|
+
# this nested list of (stream_name, entries) in bytes.
|
|
96
|
+
raw_response = await client.xreadgroup(
|
|
97
|
+
group,
|
|
98
|
+
consumer,
|
|
99
|
+
{stream: read_id},
|
|
100
|
+
count=1,
|
|
101
|
+
block=_LONG_POLL_BLOCK_MS if read_id == ">" else None,
|
|
102
|
+
)
|
|
103
|
+
response = cast(
|
|
104
|
+
"list[tuple[bytes, list[tuple[bytes, dict[bytes, bytes]]]]]",
|
|
105
|
+
raw_response,
|
|
106
|
+
)
|
|
107
|
+
for _stream_name, messages in response:
|
|
108
|
+
for message_id, fields in messages:
|
|
109
|
+
yield RedisStreamsQueuedTask(
|
|
110
|
+
_redis=client,
|
|
111
|
+
_stream=stream,
|
|
112
|
+
_group=group,
|
|
113
|
+
_message_id=message_id.decode(),
|
|
114
|
+
task=self.decode(fields[_FIELD.encode()]),
|
|
115
|
+
)
|
|
116
|
+
finally:
|
|
117
|
+
await client.aclose()
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from collections.abc import Callable
|
|
4
|
+
from typing import Any
|
|
5
|
+
from urllib.parse import urlsplit
|
|
6
|
+
|
|
7
|
+
from taskqueue_toolkit.queue.task_queue import Decoder, Encoder, TaskQueue
|
|
8
|
+
|
|
9
|
+
# A handler takes the full DSN string plus this call's encode/decode pair
|
|
10
|
+
# and returns a ready-to-use TaskQueue — parsing and construction happen
|
|
11
|
+
# together so registering a broker is one call, not a parser plus a
|
|
12
|
+
# separate constructor to keep in sync.
|
|
13
|
+
TaskQueueHandler = Callable[[str, Encoder[Any], Decoder[Any]], TaskQueue[Any]]
|
|
14
|
+
|
|
15
|
+
_registry: dict[str, TaskQueueHandler] = {}
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class SchemeAlreadyRegisteredError(Exception):
|
|
19
|
+
def __init__(self, scheme: str) -> None:
|
|
20
|
+
super().__init__(
|
|
21
|
+
f"A handler is already registered for scheme {scheme!r}. "
|
|
22
|
+
"Each scheme can only map to one broker handler."
|
|
23
|
+
)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def register_scheme(scheme: str, handler: TaskQueueHandler) -> None:
|
|
27
|
+
"""Make create_task_queue() recognize a DSN scheme this package doesn't
|
|
28
|
+
ship a built-in adapter for (e.g. "kafka://...").
|
|
29
|
+
|
|
30
|
+
This is the extension point for brokers outside the five this package
|
|
31
|
+
integrates directly (amqp/redis/sqs/sns/pubsub, handled internally and
|
|
32
|
+
checked for exhaustiveness by mypy) — a consumer that needs a broker
|
|
33
|
+
this package doesn't know about registers it once, at import/startup
|
|
34
|
+
time, instead of forking the package.
|
|
35
|
+
"""
|
|
36
|
+
if scheme in _registry:
|
|
37
|
+
raise SchemeAlreadyRegisteredError(scheme)
|
|
38
|
+
_registry[scheme] = handler
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def unregister_scheme(scheme: str) -> None:
|
|
42
|
+
"""Mainly useful for tests that register a fake handler and want to
|
|
43
|
+
clean up afterwards."""
|
|
44
|
+
_registry.pop(scheme, None)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def resolve_registered_scheme(dsn: str) -> TaskQueueHandler | None:
|
|
48
|
+
"""Look up a handler registered via register_scheme() for this DSN's
|
|
49
|
+
scheme, or None if nothing is registered for it (including every scheme
|
|
50
|
+
this package already handles internally — those never reach the
|
|
51
|
+
registry)."""
|
|
52
|
+
scheme = urlsplit(dsn).scheme
|
|
53
|
+
return _registry.get(scheme)
|
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import base64
|
|
4
|
+
import json
|
|
5
|
+
import logging
|
|
6
|
+
from collections.abc import AsyncIterator, Callable
|
|
7
|
+
from contextlib import AbstractAsyncContextManager, asynccontextmanager
|
|
8
|
+
from dataclasses import dataclass, field
|
|
9
|
+
|
|
10
|
+
import aioboto3
|
|
11
|
+
from types_aiobotocore_sns.client import SNSClient
|
|
12
|
+
from types_aiobotocore_sqs.client import SQSClient
|
|
13
|
+
|
|
14
|
+
from taskqueue_toolkit.queue.aws_session import aws_session_kwargs
|
|
15
|
+
from taskqueue_toolkit.queue.dsn import SnsDsn
|
|
16
|
+
from taskqueue_toolkit.queue.task_queue import Decoder, Encoder
|
|
17
|
+
|
|
18
|
+
logger = logging.getLogger(__name__)
|
|
19
|
+
|
|
20
|
+
_LONG_POLL_WAIT_SECONDS = 10
|
|
21
|
+
_SUBSCRIBER_QUEUE_SUFFIX = "-subscriber"
|
|
22
|
+
|
|
23
|
+
ClientFactory = Callable[[SnsDsn], AbstractAsyncContextManager[tuple[SNSClient, SQSClient]]]
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
@asynccontextmanager
|
|
27
|
+
async def _default_client_factory(dsn: SnsDsn) -> AsyncIterator[tuple[SNSClient, SQSClient]]:
|
|
28
|
+
session = aioboto3.Session()
|
|
29
|
+
async with (
|
|
30
|
+
session.client("sns", **aws_session_kwargs(dsn)) as sns,
|
|
31
|
+
session.client("sqs", **aws_session_kwargs(dsn)) as sqs,
|
|
32
|
+
):
|
|
33
|
+
yield sns, sqs
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _sqs_policy_allowing_sns_publish(queue_arn: str, topic_arn: str) -> str:
|
|
37
|
+
return json.dumps(
|
|
38
|
+
{
|
|
39
|
+
"Version": "2012-10-17",
|
|
40
|
+
"Statement": [
|
|
41
|
+
{
|
|
42
|
+
"Effect": "Allow",
|
|
43
|
+
"Principal": {"Service": "sns.amazonaws.com"},
|
|
44
|
+
"Action": "sqs:SendMessage",
|
|
45
|
+
"Resource": queue_arn,
|
|
46
|
+
"Condition": {"ArnEquals": {"aws:SourceArn": topic_arn}},
|
|
47
|
+
}
|
|
48
|
+
],
|
|
49
|
+
}
|
|
50
|
+
)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def _unwrap_sns_envelope(body: str) -> str:
|
|
54
|
+
"""SNS-to-SQS delivery wraps the published message in a JSON envelope
|
|
55
|
+
(Type, MessageId, TopicArn, Message, Timestamp, Signature, ...) — the
|
|
56
|
+
payload we actually published is in the "Message" field."""
|
|
57
|
+
return str(json.loads(body)["Message"])
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
@dataclass(slots=True)
|
|
61
|
+
class SnsQueuedTask[T]:
|
|
62
|
+
_client: SQSClient
|
|
63
|
+
_queue_url: str
|
|
64
|
+
_receipt_handle: str
|
|
65
|
+
task: T
|
|
66
|
+
|
|
67
|
+
async def ack(self) -> None:
|
|
68
|
+
await self._client.delete_message(
|
|
69
|
+
QueueUrl=self._queue_url, ReceiptHandle=self._receipt_handle
|
|
70
|
+
)
|
|
71
|
+
|
|
72
|
+
async def nack(self, *, requeue: bool) -> None:
|
|
73
|
+
if requeue:
|
|
74
|
+
await self._client.change_message_visibility(
|
|
75
|
+
QueueUrl=self._queue_url,
|
|
76
|
+
ReceiptHandle=self._receipt_handle,
|
|
77
|
+
VisibilityTimeout=0,
|
|
78
|
+
)
|
|
79
|
+
else:
|
|
80
|
+
await self._client.delete_message(
|
|
81
|
+
QueueUrl=self._queue_url, ReceiptHandle=self._receipt_handle
|
|
82
|
+
)
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
@dataclass(slots=True)
|
|
86
|
+
class SnsTaskQueue[T]:
|
|
87
|
+
"""SNS is fan-out only — there's no receive/consume on a topic itself.
|
|
88
|
+
|
|
89
|
+
consume() needs a queue to actually poll, so this adapter provisions one
|
|
90
|
+
dedicated SQS queue subscribed to the topic (both create_topic and
|
|
91
|
+
subscribe are idempotent in the AWS API, so calling this on every
|
|
92
|
+
publish()/consume() is safe — no separate "already set up" state to
|
|
93
|
+
track). publish() only ever talks to SNS; consume() only ever talks to
|
|
94
|
+
the subscriber queue.
|
|
95
|
+
"""
|
|
96
|
+
|
|
97
|
+
dsn: SnsDsn
|
|
98
|
+
encode: Encoder[T]
|
|
99
|
+
decode: Decoder[T]
|
|
100
|
+
# Overridable for tests (inject fake clients) or a shared/pooled session
|
|
101
|
+
# the caller already manages; defaults to a fresh aioboto3 session per
|
|
102
|
+
# call, matching the real SDK's usual usage.
|
|
103
|
+
client_factory: ClientFactory = field(default=_default_client_factory)
|
|
104
|
+
|
|
105
|
+
async def _get_or_create_topic_arn(self, sns: SNSClient) -> str:
|
|
106
|
+
response = await sns.create_topic(Name=self.dsn.topic_name)
|
|
107
|
+
return response["TopicArn"]
|
|
108
|
+
|
|
109
|
+
async def _get_or_create_subscriber_queue(
|
|
110
|
+
self, sqs: SQSClient, sns: SNSClient, topic_arn: str
|
|
111
|
+
) -> str:
|
|
112
|
+
queue_name = self.dsn.topic_name + _SUBSCRIBER_QUEUE_SUFFIX
|
|
113
|
+
try:
|
|
114
|
+
queue_url = (await sqs.get_queue_url(QueueName=queue_name))["QueueUrl"]
|
|
115
|
+
except sqs.exceptions.QueueDoesNotExist:
|
|
116
|
+
queue_url = (await sqs.create_queue(QueueName=queue_name))["QueueUrl"]
|
|
117
|
+
|
|
118
|
+
queue_arn = (
|
|
119
|
+
await sqs.get_queue_attributes(QueueUrl=queue_url, AttributeNames=["QueueArn"])
|
|
120
|
+
)["Attributes"]["QueueArn"]
|
|
121
|
+
|
|
122
|
+
await sqs.set_queue_attributes(
|
|
123
|
+
QueueUrl=queue_url,
|
|
124
|
+
Attributes={"Policy": _sqs_policy_allowing_sns_publish(queue_arn, topic_arn)},
|
|
125
|
+
)
|
|
126
|
+
await sns.subscribe(TopicArn=topic_arn, Protocol="sqs", Endpoint=queue_arn)
|
|
127
|
+
return queue_url
|
|
128
|
+
|
|
129
|
+
async def publish(self, task: T) -> None:
|
|
130
|
+
async with self.client_factory(self.dsn) as (sns, sqs):
|
|
131
|
+
topic_arn = await self._get_or_create_topic_arn(sns)
|
|
132
|
+
# SNS fan-out only delivers to subscribers that already exist at
|
|
133
|
+
# publish time — a message published before the subscriber queue
|
|
134
|
+
# is provisioned is silently dropped, unlike RabbitMQ/SQS/Redis
|
|
135
|
+
# Streams where publishing ahead of any consumer still keeps the
|
|
136
|
+
# message. Ensuring the subscription here, not just in consume(),
|
|
137
|
+
# is what makes the first publish after a cold deploy not lose
|
|
138
|
+
# its message.
|
|
139
|
+
await self._get_or_create_subscriber_queue(sqs, sns, topic_arn)
|
|
140
|
+
message = base64.b64encode(self.encode(task)).decode()
|
|
141
|
+
await sns.publish(TopicArn=topic_arn, Message=message)
|
|
142
|
+
logger.info("task published", extra={"topic": self.dsn.topic_name})
|
|
143
|
+
|
|
144
|
+
async def consume(self) -> AsyncIterator[SnsQueuedTask[T]]:
|
|
145
|
+
async with self.client_factory(self.dsn) as (sns, sqs):
|
|
146
|
+
topic_arn = await self._get_or_create_topic_arn(sns)
|
|
147
|
+
queue_url = await self._get_or_create_subscriber_queue(sqs, sns, topic_arn)
|
|
148
|
+
|
|
149
|
+
while True:
|
|
150
|
+
response = await sqs.receive_message(
|
|
151
|
+
QueueUrl=queue_url,
|
|
152
|
+
MaxNumberOfMessages=1,
|
|
153
|
+
WaitTimeSeconds=_LONG_POLL_WAIT_SECONDS,
|
|
154
|
+
)
|
|
155
|
+
for message in response.get("Messages", []):
|
|
156
|
+
envelope = _unwrap_sns_envelope(message["Body"])
|
|
157
|
+
yield SnsQueuedTask(
|
|
158
|
+
_client=sqs,
|
|
159
|
+
_queue_url=queue_url,
|
|
160
|
+
_receipt_handle=message["ReceiptHandle"],
|
|
161
|
+
task=self.decode(base64.b64decode(envelope)),
|
|
162
|
+
)
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import base64
|
|
4
|
+
import logging
|
|
5
|
+
from collections.abc import AsyncIterator, Callable
|
|
6
|
+
from contextlib import AbstractAsyncContextManager
|
|
7
|
+
from dataclasses import dataclass, field
|
|
8
|
+
from typing import cast
|
|
9
|
+
|
|
10
|
+
import aioboto3
|
|
11
|
+
from types_aiobotocore_sqs.client import SQSClient
|
|
12
|
+
|
|
13
|
+
from taskqueue_toolkit.queue.aws_session import aws_session_kwargs
|
|
14
|
+
from taskqueue_toolkit.queue.dsn import SqsDsn
|
|
15
|
+
from taskqueue_toolkit.queue.task_queue import Decoder, Encoder
|
|
16
|
+
|
|
17
|
+
logger = logging.getLogger(__name__)
|
|
18
|
+
|
|
19
|
+
_LONG_POLL_WAIT_SECONDS = 10
|
|
20
|
+
|
|
21
|
+
ClientFactory = Callable[[SqsDsn], AbstractAsyncContextManager[SQSClient]]
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _default_client_factory(dsn: SqsDsn) -> AbstractAsyncContextManager[SQSClient]:
|
|
25
|
+
return cast(
|
|
26
|
+
"AbstractAsyncContextManager[SQSClient]",
|
|
27
|
+
aioboto3.Session().client("sqs", **aws_session_kwargs(dsn)),
|
|
28
|
+
)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
@dataclass(slots=True)
|
|
32
|
+
class SqsQueuedTask[T]:
|
|
33
|
+
_client: SQSClient
|
|
34
|
+
_queue_url: str
|
|
35
|
+
_receipt_handle: str
|
|
36
|
+
task: T
|
|
37
|
+
|
|
38
|
+
async def ack(self) -> None:
|
|
39
|
+
await self._client.delete_message(
|
|
40
|
+
QueueUrl=self._queue_url, ReceiptHandle=self._receipt_handle
|
|
41
|
+
)
|
|
42
|
+
|
|
43
|
+
async def nack(self, *, requeue: bool) -> None:
|
|
44
|
+
if requeue:
|
|
45
|
+
# Make the message immediately visible again instead of waiting
|
|
46
|
+
# out its VisibilityTimeout — SQS has no direct "nack" RPC, this
|
|
47
|
+
# is the idiomatic way to force redelivery on demand.
|
|
48
|
+
await self._client.change_message_visibility(
|
|
49
|
+
QueueUrl=self._queue_url,
|
|
50
|
+
ReceiptHandle=self._receipt_handle,
|
|
51
|
+
VisibilityTimeout=0,
|
|
52
|
+
)
|
|
53
|
+
else:
|
|
54
|
+
# SQS has no per-call dead-letter action; permanently dropping a
|
|
55
|
+
# task means removing it the same way a successful ack does. A
|
|
56
|
+
# queue-level DLQ (RedrivePolicy) is a deployment concern, not
|
|
57
|
+
# something this adapter configures per message.
|
|
58
|
+
await self._client.delete_message(
|
|
59
|
+
QueueUrl=self._queue_url, ReceiptHandle=self._receipt_handle
|
|
60
|
+
)
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
@dataclass(slots=True)
|
|
64
|
+
class SqsTaskQueue[T]:
|
|
65
|
+
dsn: SqsDsn
|
|
66
|
+
encode: Encoder[T]
|
|
67
|
+
decode: Decoder[T]
|
|
68
|
+
# Overridable for tests (inject a fake client) or a shared/pooled
|
|
69
|
+
# session the caller already manages; defaults to a fresh aioboto3
|
|
70
|
+
# session per call, matching the real SDK's usual usage.
|
|
71
|
+
client_factory: ClientFactory = field(default=_default_client_factory)
|
|
72
|
+
|
|
73
|
+
async def _get_queue_url(self, client: SQSClient) -> str:
|
|
74
|
+
try:
|
|
75
|
+
response = await client.get_queue_url(QueueName=self.dsn.queue_name)
|
|
76
|
+
except client.exceptions.QueueDoesNotExist:
|
|
77
|
+
response = await client.create_queue(QueueName=self.dsn.queue_name)
|
|
78
|
+
return response["QueueUrl"]
|
|
79
|
+
|
|
80
|
+
async def publish(self, task: T) -> None:
|
|
81
|
+
async with self.client_factory(self.dsn) as client:
|
|
82
|
+
queue_url = await self._get_queue_url(client)
|
|
83
|
+
body = base64.b64encode(self.encode(task)).decode()
|
|
84
|
+
await client.send_message(QueueUrl=queue_url, MessageBody=body)
|
|
85
|
+
logger.info("task published", extra={"queue": self.dsn.queue_name})
|
|
86
|
+
|
|
87
|
+
async def consume(self) -> AsyncIterator[SqsQueuedTask[T]]:
|
|
88
|
+
async with self.client_factory(self.dsn) as client:
|
|
89
|
+
queue_url = await self._get_queue_url(client)
|
|
90
|
+
|
|
91
|
+
while True:
|
|
92
|
+
response = await client.receive_message(
|
|
93
|
+
QueueUrl=queue_url,
|
|
94
|
+
MaxNumberOfMessages=1,
|
|
95
|
+
WaitTimeSeconds=_LONG_POLL_WAIT_SECONDS,
|
|
96
|
+
)
|
|
97
|
+
for message in response.get("Messages", []):
|
|
98
|
+
yield SqsQueuedTask(
|
|
99
|
+
_client=client,
|
|
100
|
+
_queue_url=queue_url,
|
|
101
|
+
_receipt_handle=message["ReceiptHandle"],
|
|
102
|
+
task=self.decode(base64.b64decode(message["Body"])),
|
|
103
|
+
)
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from collections.abc import AsyncIterator, Callable
|
|
4
|
+
from typing import Protocol, TypeVar
|
|
5
|
+
|
|
6
|
+
T = TypeVar("T")
|
|
7
|
+
T_co = TypeVar("T_co", covariant=True)
|
|
8
|
+
|
|
9
|
+
Encoder = Callable[[T], bytes]
|
|
10
|
+
Decoder = Callable[[bytes], T]
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class QueuedTask(Protocol[T_co]):
|
|
14
|
+
"""A task handed to a consumer, together with how to close it out.
|
|
15
|
+
|
|
16
|
+
ack() and nack() are the only broker-specific detail a consumer needs:
|
|
17
|
+
the concrete queue implementation decides what "acknowledged" or
|
|
18
|
+
"requeue this" actually means (delete from RabbitMQ, delete from SQS,
|
|
19
|
+
reset a Redis Streams pending entry, ...).
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
@property
|
|
23
|
+
def task(self) -> T_co: ...
|
|
24
|
+
|
|
25
|
+
async def ack(self) -> None:
|
|
26
|
+
"""Mark the task as successfully processed."""
|
|
27
|
+
...
|
|
28
|
+
|
|
29
|
+
async def nack(self, *, requeue: bool) -> None:
|
|
30
|
+
"""Mark the task as failed. requeue=True retries it; False drops it
|
|
31
|
+
for good (or routes it to a dead-letter destination, if the broker
|
|
32
|
+
and its configuration support one — this call itself never
|
|
33
|
+
configures that, only triggers the broker's existing behavior)."""
|
|
34
|
+
...
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class TaskQueue(Protocol[T]):
|
|
38
|
+
"""A broker-agnostic queue of tasks of type T.
|
|
39
|
+
|
|
40
|
+
Deliberately minimal — publish(), consume() (an AsyncIterator of
|
|
41
|
+
QueuedTask[T]), and ack()/nack(requeue=...) on each received task — so it
|
|
42
|
+
stays implementable by brokers with very different delivery models
|
|
43
|
+
(AMQP queues, SQS, SNS fan-out, Redis Streams consumer groups, Pub/Sub)
|
|
44
|
+
without the contract assuming anything specific to one of them: no
|
|
45
|
+
routing keys, no topics/partitions, no consumer groups, no FIFO groups.
|
|
46
|
+
|
|
47
|
+
T is never assumed to be any particular shape. Each concrete
|
|
48
|
+
implementation is handed an Encoder[T]/Decoder[T] pair at construction
|
|
49
|
+
time — this package has no opinion on how your task type serializes,
|
|
50
|
+
only on how it moves through a queue once serialized.
|
|
51
|
+
"""
|
|
52
|
+
|
|
53
|
+
async def publish(self, task: T) -> None:
|
|
54
|
+
"""Enqueue a task for a consumer to pick up."""
|
|
55
|
+
...
|
|
56
|
+
|
|
57
|
+
def consume(self) -> AsyncIterator[QueuedTask[T]]:
|
|
58
|
+
"""Yield tasks as they become available, one at a time. Each yielded
|
|
59
|
+
task must be ack()'d or nack()'d by the consumer — the queue
|
|
60
|
+
implementation decides what happens to a task that's never
|
|
61
|
+
acknowledged (redelivery, visibility timeout expiry, etc.)."""
|
|
62
|
+
...
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
"""In-memory doubles of each broker adapter, for testing code that uses
|
|
2
|
+
this package without a real RabbitMQ/SQS/SNS/Redis/Pub-Sub.
|
|
3
|
+
|
|
4
|
+
Each Fake*Broker is a real, working implementation of the relevant slice of
|
|
5
|
+
its SDK — not a call-recording mock — so it exercises the same code paths
|
|
6
|
+
(FIFO ordering, ack/nack, consumer groups, fan-out, ...) a real broker
|
|
7
|
+
would. Create one instance per test; there is no global state to reset.
|
|
8
|
+
|
|
9
|
+
Importing a given Fake*Broker requires the same optional extra as the
|
|
10
|
+
adapter it doubles (e.g. FakeRabbitMqBroker needs no extra since it has no
|
|
11
|
+
SDK dependency of its own, but FakeRedisStreamsBroker needs
|
|
12
|
+
`pip install taskqueue-toolkit[redis-streams]`, FakeSqsBroker/FakeSnsBroker
|
|
13
|
+
need `[aws]`, FakePubsubBroker needs `[pubsub]`) — this module only imports
|
|
14
|
+
the specific fake actually accessed, via module-level __getattr__, so
|
|
15
|
+
`import taskqueue_toolkit.testing` itself never requires every extra to be
|
|
16
|
+
installed at once.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
from typing import TYPE_CHECKING
|
|
22
|
+
|
|
23
|
+
if TYPE_CHECKING:
|
|
24
|
+
from taskqueue_toolkit.testing.pubsub import FakePubsubBroker
|
|
25
|
+
from taskqueue_toolkit.testing.rabbitmq import FakeRabbitMqBroker
|
|
26
|
+
from taskqueue_toolkit.testing.redis_streams import FakeRedisStreamsBroker
|
|
27
|
+
from taskqueue_toolkit.testing.sns import FakeSnsBroker
|
|
28
|
+
from taskqueue_toolkit.testing.sqs import FakeSqsBroker
|
|
29
|
+
|
|
30
|
+
__all__ = [
|
|
31
|
+
"FakePubsubBroker",
|
|
32
|
+
"FakeRabbitMqBroker",
|
|
33
|
+
"FakeRedisStreamsBroker",
|
|
34
|
+
"FakeSnsBroker",
|
|
35
|
+
"FakeSqsBroker",
|
|
36
|
+
]
|
|
37
|
+
|
|
38
|
+
_EXPORTS = {
|
|
39
|
+
"FakePubsubBroker": ("taskqueue_toolkit.testing.pubsub", "FakePubsubBroker"),
|
|
40
|
+
"FakeRabbitMqBroker": ("taskqueue_toolkit.testing.rabbitmq", "FakeRabbitMqBroker"),
|
|
41
|
+
"FakeRedisStreamsBroker": (
|
|
42
|
+
"taskqueue_toolkit.testing.redis_streams",
|
|
43
|
+
"FakeRedisStreamsBroker",
|
|
44
|
+
),
|
|
45
|
+
"FakeSnsBroker": ("taskqueue_toolkit.testing.sns", "FakeSnsBroker"),
|
|
46
|
+
"FakeSqsBroker": ("taskqueue_toolkit.testing.sqs", "FakeSqsBroker"),
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def __getattr__(name: str) -> object:
|
|
51
|
+
try:
|
|
52
|
+
module_name, attr_name = _EXPORTS[name]
|
|
53
|
+
except KeyError:
|
|
54
|
+
raise AttributeError(f"module {__name__!r} has no attribute {name!r}") from None
|
|
55
|
+
import importlib
|
|
56
|
+
|
|
57
|
+
module = importlib.import_module(module_name)
|
|
58
|
+
return getattr(module, attr_name)
|