taskqueue-toolkit 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,74 @@
1
+ from __future__ import annotations
2
+
3
+ import logging
4
+ from collections.abc import AsyncIterator, Awaitable, Callable
5
+ from contextlib import suppress
6
+ from dataclasses import dataclass, field
7
+
8
+ import aio_pika
9
+
10
+ from taskqueue_toolkit.queue.dsn import RabbitMqDsn
11
+ from taskqueue_toolkit.queue.task_queue import Decoder, Encoder
12
+
13
+ logger = logging.getLogger(__name__)
14
+
15
+ Connect = Callable[[str], Awaitable["aio_pika.abc.AbstractRobustConnection"]]
16
+
17
+
18
+ @dataclass(slots=True)
19
+ class RabbitMqQueuedTask[T]:
20
+ _message: aio_pika.abc.AbstractIncomingMessage
21
+ task: T
22
+
23
+ async def ack(self) -> None:
24
+ await self._message.ack()
25
+
26
+ async def nack(self, *, requeue: bool) -> None:
27
+ await self._message.nack(requeue=requeue)
28
+
29
+
30
+ @dataclass(slots=True)
31
+ class RabbitMqTaskQueue[T]:
32
+ dsn: RabbitMqDsn
33
+ encode: Encoder[T]
34
+ decode: Decoder[T]
35
+ # Overridable for tests (inject a fake connection) or to share/pool a
36
+ # connection strategy the caller already has; defaults to the real SDK.
37
+ connect: Connect = field(default=aio_pika.connect_robust)
38
+
39
+ async def publish(self, task: T) -> None:
40
+ connection = await self.connect(self.dsn.url)
41
+ try:
42
+ channel = await connection.channel()
43
+ queue = await channel.declare_queue(self.dsn.queue_name, durable=True)
44
+ await channel.default_exchange.publish(
45
+ aio_pika.Message(
46
+ body=self.encode(task),
47
+ delivery_mode=aio_pika.DeliveryMode.PERSISTENT,
48
+ ),
49
+ routing_key=queue.name,
50
+ )
51
+ logger.info("task published", extra={"queue": self.dsn.queue_name})
52
+ finally:
53
+ await connection.close()
54
+
55
+ async def consume(self) -> AsyncIterator[RabbitMqQueuedTask[T]]:
56
+ connection = await self.connect(self.dsn.url)
57
+ try:
58
+ channel = await connection.channel()
59
+ await channel.set_qos(prefetch_count=1)
60
+ queue = await channel.declare_queue(self.dsn.queue_name, durable=True)
61
+
62
+ async for message in queue.iterator():
63
+ yield RabbitMqQueuedTask(_message=message, task=self.decode(message.body))
64
+ finally:
65
+ # The consumer (a worker's `async for`) can stop iterating at any
66
+ # point — graceful shutdown, an unhandled error, etc. — which
67
+ # raises GeneratorExit right here, inside the yield above. At
68
+ # that point there's no time left to round-trip a clean
69
+ # basic.cancel RPC with the broker (the event loop is usually
70
+ # already shutting down too), so don't try — just drop the
71
+ # connection. RabbitMQ notices the disconnect and requeues
72
+ # whatever message was left unacknowledged.
73
+ with suppress(Exception):
74
+ await connection.close()
@@ -0,0 +1,117 @@
1
+ from __future__ import annotations
2
+
3
+ import logging
4
+ from collections.abc import AsyncIterator, Callable
5
+ from dataclasses import dataclass, field
6
+ from typing import cast
7
+
8
+ import redis.asyncio as redis
9
+ from redis.exceptions import ResponseError
10
+
11
+ from taskqueue_toolkit.queue.dsn import RedisStreamsDsn
12
+ from taskqueue_toolkit.queue.task_queue import Decoder, Encoder
13
+
14
+ logger = logging.getLogger(__name__)
15
+
16
+ _LONG_POLL_BLOCK_MS = 10_000
17
+ _FIELD = "payload"
18
+
19
+ ClientFactory = Callable[[RedisStreamsDsn], "redis.Redis"]
20
+
21
+
22
+ def _default_client_factory(dsn: RedisStreamsDsn) -> redis.Redis:
23
+ return redis.Redis.from_url(dsn.url)
24
+
25
+
26
+ @dataclass(slots=True)
27
+ class RedisStreamsQueuedTask[T]:
28
+ _redis: redis.Redis
29
+ _stream: str
30
+ _group: str
31
+ _message_id: str
32
+ task: T
33
+
34
+ async def ack(self) -> None:
35
+ await self._redis.xack(self._stream, self._group, self._message_id)
36
+
37
+ async def nack(self, *, requeue: bool) -> None:
38
+ if requeue:
39
+ # Leaving the message unacknowledged (in the group's Pending
40
+ # Entries List) is enough — consume()'s next pass re-reads
41
+ # pending entries (id="0") before new ones, so it's redelivered
42
+ # without needing an explicit "give it back" call the way SQS's
43
+ # change_message_visibility does.
44
+ return
45
+ # No per-message dead-letter action here either — same reasoning as
46
+ # SQS: permanently dropping a task means removing it from the PEL,
47
+ # same as a successful ack.
48
+ await self._redis.xack(self._stream, self._group, self._message_id)
49
+
50
+
51
+ @dataclass(slots=True)
52
+ class RedisStreamsTaskQueue[T]:
53
+ dsn: RedisStreamsDsn
54
+ encode: Encoder[T]
55
+ decode: Decoder[T]
56
+ # Overridable for tests (inject a fake client) or a shared/pooled
57
+ # connection the caller already manages; defaults to a fresh connection
58
+ # from the DSN's URL per call, matching the real SDK's usual usage.
59
+ client_factory: ClientFactory = field(default=_default_client_factory)
60
+
61
+ def _client(self) -> redis.Redis:
62
+ return self.client_factory(self.dsn)
63
+
64
+ async def _ensure_group(self, client: redis.Redis) -> None:
65
+ try:
66
+ await client.xgroup_create(self.dsn.stream_name, self.dsn.group, id="0", mkstream=True)
67
+ except ResponseError as exc:
68
+ if "BUSYGROUP" not in str(exc):
69
+ raise
70
+
71
+ async def publish(self, task: T) -> None:
72
+ client = self._client()
73
+ try:
74
+ await client.xadd(self.dsn.stream_name, {_FIELD: self.encode(task)})
75
+ logger.info("task published", extra={"stream": self.dsn.stream_name})
76
+ finally:
77
+ await client.aclose()
78
+
79
+ async def consume(self) -> AsyncIterator[RedisStreamsQueuedTask[T]]:
80
+ client = self._client()
81
+ try:
82
+ await self._ensure_group(client)
83
+ stream = self.dsn.stream_name
84
+ group = self.dsn.group
85
+ consumer = self.dsn.consumer
86
+
87
+ while True:
88
+ # Re-deliver our own still-pending entries first (id="0") —
89
+ # covers a crash between XREADGROUP and XACK on a previous
90
+ # run — then fall through to new entries (id=">").
91
+ for read_id in ("0", ">"):
92
+ # redis-py's XReadGroupResponse type is a broad union
93
+ # (it also covers the decode_responses=True str shape);
94
+ # without decode_responses the actual shape is always
95
+ # this nested list of (stream_name, entries) in bytes.
96
+ raw_response = await client.xreadgroup(
97
+ group,
98
+ consumer,
99
+ {stream: read_id},
100
+ count=1,
101
+ block=_LONG_POLL_BLOCK_MS if read_id == ">" else None,
102
+ )
103
+ response = cast(
104
+ "list[tuple[bytes, list[tuple[bytes, dict[bytes, bytes]]]]]",
105
+ raw_response,
106
+ )
107
+ for _stream_name, messages in response:
108
+ for message_id, fields in messages:
109
+ yield RedisStreamsQueuedTask(
110
+ _redis=client,
111
+ _stream=stream,
112
+ _group=group,
113
+ _message_id=message_id.decode(),
114
+ task=self.decode(fields[_FIELD.encode()]),
115
+ )
116
+ finally:
117
+ await client.aclose()
@@ -0,0 +1,53 @@
1
+ from __future__ import annotations
2
+
3
+ from collections.abc import Callable
4
+ from typing import Any
5
+ from urllib.parse import urlsplit
6
+
7
+ from taskqueue_toolkit.queue.task_queue import Decoder, Encoder, TaskQueue
8
+
9
+ # A handler takes the full DSN string plus this call's encode/decode pair
10
+ # and returns a ready-to-use TaskQueue — parsing and construction happen
11
+ # together so registering a broker is one call, not a parser plus a
12
+ # separate constructor to keep in sync.
13
+ TaskQueueHandler = Callable[[str, Encoder[Any], Decoder[Any]], TaskQueue[Any]]
14
+
15
+ _registry: dict[str, TaskQueueHandler] = {}
16
+
17
+
18
+ class SchemeAlreadyRegisteredError(Exception):
19
+ def __init__(self, scheme: str) -> None:
20
+ super().__init__(
21
+ f"A handler is already registered for scheme {scheme!r}. "
22
+ "Each scheme can only map to one broker handler."
23
+ )
24
+
25
+
26
+ def register_scheme(scheme: str, handler: TaskQueueHandler) -> None:
27
+ """Make create_task_queue() recognize a DSN scheme this package doesn't
28
+ ship a built-in adapter for (e.g. "kafka://...").
29
+
30
+ This is the extension point for brokers outside the five this package
31
+ integrates directly (amqp/redis/sqs/sns/pubsub, handled internally and
32
+ checked for exhaustiveness by mypy) — a consumer that needs a broker
33
+ this package doesn't know about registers it once, at import/startup
34
+ time, instead of forking the package.
35
+ """
36
+ if scheme in _registry:
37
+ raise SchemeAlreadyRegisteredError(scheme)
38
+ _registry[scheme] = handler
39
+
40
+
41
+ def unregister_scheme(scheme: str) -> None:
42
+ """Mainly useful for tests that register a fake handler and want to
43
+ clean up afterwards."""
44
+ _registry.pop(scheme, None)
45
+
46
+
47
+ def resolve_registered_scheme(dsn: str) -> TaskQueueHandler | None:
48
+ """Look up a handler registered via register_scheme() for this DSN's
49
+ scheme, or None if nothing is registered for it (including every scheme
50
+ this package already handles internally — those never reach the
51
+ registry)."""
52
+ scheme = urlsplit(dsn).scheme
53
+ return _registry.get(scheme)
@@ -0,0 +1,162 @@
1
+ from __future__ import annotations
2
+
3
+ import base64
4
+ import json
5
+ import logging
6
+ from collections.abc import AsyncIterator, Callable
7
+ from contextlib import AbstractAsyncContextManager, asynccontextmanager
8
+ from dataclasses import dataclass, field
9
+
10
+ import aioboto3
11
+ from types_aiobotocore_sns.client import SNSClient
12
+ from types_aiobotocore_sqs.client import SQSClient
13
+
14
+ from taskqueue_toolkit.queue.aws_session import aws_session_kwargs
15
+ from taskqueue_toolkit.queue.dsn import SnsDsn
16
+ from taskqueue_toolkit.queue.task_queue import Decoder, Encoder
17
+
18
+ logger = logging.getLogger(__name__)
19
+
20
+ _LONG_POLL_WAIT_SECONDS = 10
21
+ _SUBSCRIBER_QUEUE_SUFFIX = "-subscriber"
22
+
23
+ ClientFactory = Callable[[SnsDsn], AbstractAsyncContextManager[tuple[SNSClient, SQSClient]]]
24
+
25
+
26
+ @asynccontextmanager
27
+ async def _default_client_factory(dsn: SnsDsn) -> AsyncIterator[tuple[SNSClient, SQSClient]]:
28
+ session = aioboto3.Session()
29
+ async with (
30
+ session.client("sns", **aws_session_kwargs(dsn)) as sns,
31
+ session.client("sqs", **aws_session_kwargs(dsn)) as sqs,
32
+ ):
33
+ yield sns, sqs
34
+
35
+
36
+ def _sqs_policy_allowing_sns_publish(queue_arn: str, topic_arn: str) -> str:
37
+ return json.dumps(
38
+ {
39
+ "Version": "2012-10-17",
40
+ "Statement": [
41
+ {
42
+ "Effect": "Allow",
43
+ "Principal": {"Service": "sns.amazonaws.com"},
44
+ "Action": "sqs:SendMessage",
45
+ "Resource": queue_arn,
46
+ "Condition": {"ArnEquals": {"aws:SourceArn": topic_arn}},
47
+ }
48
+ ],
49
+ }
50
+ )
51
+
52
+
53
+ def _unwrap_sns_envelope(body: str) -> str:
54
+ """SNS-to-SQS delivery wraps the published message in a JSON envelope
55
+ (Type, MessageId, TopicArn, Message, Timestamp, Signature, ...) — the
56
+ payload we actually published is in the "Message" field."""
57
+ return str(json.loads(body)["Message"])
58
+
59
+
60
+ @dataclass(slots=True)
61
+ class SnsQueuedTask[T]:
62
+ _client: SQSClient
63
+ _queue_url: str
64
+ _receipt_handle: str
65
+ task: T
66
+
67
+ async def ack(self) -> None:
68
+ await self._client.delete_message(
69
+ QueueUrl=self._queue_url, ReceiptHandle=self._receipt_handle
70
+ )
71
+
72
+ async def nack(self, *, requeue: bool) -> None:
73
+ if requeue:
74
+ await self._client.change_message_visibility(
75
+ QueueUrl=self._queue_url,
76
+ ReceiptHandle=self._receipt_handle,
77
+ VisibilityTimeout=0,
78
+ )
79
+ else:
80
+ await self._client.delete_message(
81
+ QueueUrl=self._queue_url, ReceiptHandle=self._receipt_handle
82
+ )
83
+
84
+
85
+ @dataclass(slots=True)
86
+ class SnsTaskQueue[T]:
87
+ """SNS is fan-out only — there's no receive/consume on a topic itself.
88
+
89
+ consume() needs a queue to actually poll, so this adapter provisions one
90
+ dedicated SQS queue subscribed to the topic (both create_topic and
91
+ subscribe are idempotent in the AWS API, so calling this on every
92
+ publish()/consume() is safe — no separate "already set up" state to
93
+ track). publish() only ever talks to SNS; consume() only ever talks to
94
+ the subscriber queue.
95
+ """
96
+
97
+ dsn: SnsDsn
98
+ encode: Encoder[T]
99
+ decode: Decoder[T]
100
+ # Overridable for tests (inject fake clients) or a shared/pooled session
101
+ # the caller already manages; defaults to a fresh aioboto3 session per
102
+ # call, matching the real SDK's usual usage.
103
+ client_factory: ClientFactory = field(default=_default_client_factory)
104
+
105
+ async def _get_or_create_topic_arn(self, sns: SNSClient) -> str:
106
+ response = await sns.create_topic(Name=self.dsn.topic_name)
107
+ return response["TopicArn"]
108
+
109
+ async def _get_or_create_subscriber_queue(
110
+ self, sqs: SQSClient, sns: SNSClient, topic_arn: str
111
+ ) -> str:
112
+ queue_name = self.dsn.topic_name + _SUBSCRIBER_QUEUE_SUFFIX
113
+ try:
114
+ queue_url = (await sqs.get_queue_url(QueueName=queue_name))["QueueUrl"]
115
+ except sqs.exceptions.QueueDoesNotExist:
116
+ queue_url = (await sqs.create_queue(QueueName=queue_name))["QueueUrl"]
117
+
118
+ queue_arn = (
119
+ await sqs.get_queue_attributes(QueueUrl=queue_url, AttributeNames=["QueueArn"])
120
+ )["Attributes"]["QueueArn"]
121
+
122
+ await sqs.set_queue_attributes(
123
+ QueueUrl=queue_url,
124
+ Attributes={"Policy": _sqs_policy_allowing_sns_publish(queue_arn, topic_arn)},
125
+ )
126
+ await sns.subscribe(TopicArn=topic_arn, Protocol="sqs", Endpoint=queue_arn)
127
+ return queue_url
128
+
129
+ async def publish(self, task: T) -> None:
130
+ async with self.client_factory(self.dsn) as (sns, sqs):
131
+ topic_arn = await self._get_or_create_topic_arn(sns)
132
+ # SNS fan-out only delivers to subscribers that already exist at
133
+ # publish time — a message published before the subscriber queue
134
+ # is provisioned is silently dropped, unlike RabbitMQ/SQS/Redis
135
+ # Streams where publishing ahead of any consumer still keeps the
136
+ # message. Ensuring the subscription here, not just in consume(),
137
+ # is what makes the first publish after a cold deploy not lose
138
+ # its message.
139
+ await self._get_or_create_subscriber_queue(sqs, sns, topic_arn)
140
+ message = base64.b64encode(self.encode(task)).decode()
141
+ await sns.publish(TopicArn=topic_arn, Message=message)
142
+ logger.info("task published", extra={"topic": self.dsn.topic_name})
143
+
144
+ async def consume(self) -> AsyncIterator[SnsQueuedTask[T]]:
145
+ async with self.client_factory(self.dsn) as (sns, sqs):
146
+ topic_arn = await self._get_or_create_topic_arn(sns)
147
+ queue_url = await self._get_or_create_subscriber_queue(sqs, sns, topic_arn)
148
+
149
+ while True:
150
+ response = await sqs.receive_message(
151
+ QueueUrl=queue_url,
152
+ MaxNumberOfMessages=1,
153
+ WaitTimeSeconds=_LONG_POLL_WAIT_SECONDS,
154
+ )
155
+ for message in response.get("Messages", []):
156
+ envelope = _unwrap_sns_envelope(message["Body"])
157
+ yield SnsQueuedTask(
158
+ _client=sqs,
159
+ _queue_url=queue_url,
160
+ _receipt_handle=message["ReceiptHandle"],
161
+ task=self.decode(base64.b64decode(envelope)),
162
+ )
@@ -0,0 +1,103 @@
1
+ from __future__ import annotations
2
+
3
+ import base64
4
+ import logging
5
+ from collections.abc import AsyncIterator, Callable
6
+ from contextlib import AbstractAsyncContextManager
7
+ from dataclasses import dataclass, field
8
+ from typing import cast
9
+
10
+ import aioboto3
11
+ from types_aiobotocore_sqs.client import SQSClient
12
+
13
+ from taskqueue_toolkit.queue.aws_session import aws_session_kwargs
14
+ from taskqueue_toolkit.queue.dsn import SqsDsn
15
+ from taskqueue_toolkit.queue.task_queue import Decoder, Encoder
16
+
17
+ logger = logging.getLogger(__name__)
18
+
19
+ _LONG_POLL_WAIT_SECONDS = 10
20
+
21
+ ClientFactory = Callable[[SqsDsn], AbstractAsyncContextManager[SQSClient]]
22
+
23
+
24
+ def _default_client_factory(dsn: SqsDsn) -> AbstractAsyncContextManager[SQSClient]:
25
+ return cast(
26
+ "AbstractAsyncContextManager[SQSClient]",
27
+ aioboto3.Session().client("sqs", **aws_session_kwargs(dsn)),
28
+ )
29
+
30
+
31
+ @dataclass(slots=True)
32
+ class SqsQueuedTask[T]:
33
+ _client: SQSClient
34
+ _queue_url: str
35
+ _receipt_handle: str
36
+ task: T
37
+
38
+ async def ack(self) -> None:
39
+ await self._client.delete_message(
40
+ QueueUrl=self._queue_url, ReceiptHandle=self._receipt_handle
41
+ )
42
+
43
+ async def nack(self, *, requeue: bool) -> None:
44
+ if requeue:
45
+ # Make the message immediately visible again instead of waiting
46
+ # out its VisibilityTimeout — SQS has no direct "nack" RPC, this
47
+ # is the idiomatic way to force redelivery on demand.
48
+ await self._client.change_message_visibility(
49
+ QueueUrl=self._queue_url,
50
+ ReceiptHandle=self._receipt_handle,
51
+ VisibilityTimeout=0,
52
+ )
53
+ else:
54
+ # SQS has no per-call dead-letter action; permanently dropping a
55
+ # task means removing it the same way a successful ack does. A
56
+ # queue-level DLQ (RedrivePolicy) is a deployment concern, not
57
+ # something this adapter configures per message.
58
+ await self._client.delete_message(
59
+ QueueUrl=self._queue_url, ReceiptHandle=self._receipt_handle
60
+ )
61
+
62
+
63
+ @dataclass(slots=True)
64
+ class SqsTaskQueue[T]:
65
+ dsn: SqsDsn
66
+ encode: Encoder[T]
67
+ decode: Decoder[T]
68
+ # Overridable for tests (inject a fake client) or a shared/pooled
69
+ # session the caller already manages; defaults to a fresh aioboto3
70
+ # session per call, matching the real SDK's usual usage.
71
+ client_factory: ClientFactory = field(default=_default_client_factory)
72
+
73
+ async def _get_queue_url(self, client: SQSClient) -> str:
74
+ try:
75
+ response = await client.get_queue_url(QueueName=self.dsn.queue_name)
76
+ except client.exceptions.QueueDoesNotExist:
77
+ response = await client.create_queue(QueueName=self.dsn.queue_name)
78
+ return response["QueueUrl"]
79
+
80
+ async def publish(self, task: T) -> None:
81
+ async with self.client_factory(self.dsn) as client:
82
+ queue_url = await self._get_queue_url(client)
83
+ body = base64.b64encode(self.encode(task)).decode()
84
+ await client.send_message(QueueUrl=queue_url, MessageBody=body)
85
+ logger.info("task published", extra={"queue": self.dsn.queue_name})
86
+
87
+ async def consume(self) -> AsyncIterator[SqsQueuedTask[T]]:
88
+ async with self.client_factory(self.dsn) as client:
89
+ queue_url = await self._get_queue_url(client)
90
+
91
+ while True:
92
+ response = await client.receive_message(
93
+ QueueUrl=queue_url,
94
+ MaxNumberOfMessages=1,
95
+ WaitTimeSeconds=_LONG_POLL_WAIT_SECONDS,
96
+ )
97
+ for message in response.get("Messages", []):
98
+ yield SqsQueuedTask(
99
+ _client=client,
100
+ _queue_url=queue_url,
101
+ _receipt_handle=message["ReceiptHandle"],
102
+ task=self.decode(base64.b64decode(message["Body"])),
103
+ )
@@ -0,0 +1,62 @@
1
+ from __future__ import annotations
2
+
3
+ from collections.abc import AsyncIterator, Callable
4
+ from typing import Protocol, TypeVar
5
+
6
+ T = TypeVar("T")
7
+ T_co = TypeVar("T_co", covariant=True)
8
+
9
+ Encoder = Callable[[T], bytes]
10
+ Decoder = Callable[[bytes], T]
11
+
12
+
13
+ class QueuedTask(Protocol[T_co]):
14
+ """A task handed to a consumer, together with how to close it out.
15
+
16
+ ack() and nack() are the only broker-specific detail a consumer needs:
17
+ the concrete queue implementation decides what "acknowledged" or
18
+ "requeue this" actually means (delete from RabbitMQ, delete from SQS,
19
+ reset a Redis Streams pending entry, ...).
20
+ """
21
+
22
+ @property
23
+ def task(self) -> T_co: ...
24
+
25
+ async def ack(self) -> None:
26
+ """Mark the task as successfully processed."""
27
+ ...
28
+
29
+ async def nack(self, *, requeue: bool) -> None:
30
+ """Mark the task as failed. requeue=True retries it; False drops it
31
+ for good (or routes it to a dead-letter destination, if the broker
32
+ and its configuration support one — this call itself never
33
+ configures that, only triggers the broker's existing behavior)."""
34
+ ...
35
+
36
+
37
+ class TaskQueue(Protocol[T]):
38
+ """A broker-agnostic queue of tasks of type T.
39
+
40
+ Deliberately minimal — publish(), consume() (an AsyncIterator of
41
+ QueuedTask[T]), and ack()/nack(requeue=...) on each received task — so it
42
+ stays implementable by brokers with very different delivery models
43
+ (AMQP queues, SQS, SNS fan-out, Redis Streams consumer groups, Pub/Sub)
44
+ without the contract assuming anything specific to one of them: no
45
+ routing keys, no topics/partitions, no consumer groups, no FIFO groups.
46
+
47
+ T is never assumed to be any particular shape. Each concrete
48
+ implementation is handed an Encoder[T]/Decoder[T] pair at construction
49
+ time — this package has no opinion on how your task type serializes,
50
+ only on how it moves through a queue once serialized.
51
+ """
52
+
53
+ async def publish(self, task: T) -> None:
54
+ """Enqueue a task for a consumer to pick up."""
55
+ ...
56
+
57
+ def consume(self) -> AsyncIterator[QueuedTask[T]]:
58
+ """Yield tasks as they become available, one at a time. Each yielded
59
+ task must be ack()'d or nack()'d by the consumer — the queue
60
+ implementation decides what happens to a task that's never
61
+ acknowledged (redelivery, visibility timeout expiry, etc.)."""
62
+ ...
@@ -0,0 +1,58 @@
1
+ """In-memory doubles of each broker adapter, for testing code that uses
2
+ this package without a real RabbitMQ/SQS/SNS/Redis/Pub-Sub.
3
+
4
+ Each Fake*Broker is a real, working implementation of the relevant slice of
5
+ its SDK — not a call-recording mock — so it exercises the same code paths
6
+ (FIFO ordering, ack/nack, consumer groups, fan-out, ...) a real broker
7
+ would. Create one instance per test; there is no global state to reset.
8
+
9
+ Importing a given Fake*Broker requires the same optional extra as the
10
+ adapter it doubles (e.g. FakeRabbitMqBroker needs no extra since it has no
11
+ SDK dependency of its own, but FakeRedisStreamsBroker needs
12
+ `pip install taskqueue-toolkit[redis-streams]`, FakeSqsBroker/FakeSnsBroker
13
+ need `[aws]`, FakePubsubBroker needs `[pubsub]`) — this module only imports
14
+ the specific fake actually accessed, via module-level __getattr__, so
15
+ `import taskqueue_toolkit.testing` itself never requires every extra to be
16
+ installed at once.
17
+ """
18
+
19
+ from __future__ import annotations
20
+
21
+ from typing import TYPE_CHECKING
22
+
23
+ if TYPE_CHECKING:
24
+ from taskqueue_toolkit.testing.pubsub import FakePubsubBroker
25
+ from taskqueue_toolkit.testing.rabbitmq import FakeRabbitMqBroker
26
+ from taskqueue_toolkit.testing.redis_streams import FakeRedisStreamsBroker
27
+ from taskqueue_toolkit.testing.sns import FakeSnsBroker
28
+ from taskqueue_toolkit.testing.sqs import FakeSqsBroker
29
+
30
+ __all__ = [
31
+ "FakePubsubBroker",
32
+ "FakeRabbitMqBroker",
33
+ "FakeRedisStreamsBroker",
34
+ "FakeSnsBroker",
35
+ "FakeSqsBroker",
36
+ ]
37
+
38
+ _EXPORTS = {
39
+ "FakePubsubBroker": ("taskqueue_toolkit.testing.pubsub", "FakePubsubBroker"),
40
+ "FakeRabbitMqBroker": ("taskqueue_toolkit.testing.rabbitmq", "FakeRabbitMqBroker"),
41
+ "FakeRedisStreamsBroker": (
42
+ "taskqueue_toolkit.testing.redis_streams",
43
+ "FakeRedisStreamsBroker",
44
+ ),
45
+ "FakeSnsBroker": ("taskqueue_toolkit.testing.sns", "FakeSnsBroker"),
46
+ "FakeSqsBroker": ("taskqueue_toolkit.testing.sqs", "FakeSqsBroker"),
47
+ }
48
+
49
+
50
+ def __getattr__(name: str) -> object:
51
+ try:
52
+ module_name, attr_name = _EXPORTS[name]
53
+ except KeyError:
54
+ raise AttributeError(f"module {__name__!r} has no attribute {name!r}") from None
55
+ import importlib
56
+
57
+ module = importlib.import_module(module_name)
58
+ return getattr(module, attr_name)