taskqueue-toolkit 0.1.0__py3-none-any.whl

taskqueue_toolkit/outbox/orm.py

@@ -0,0 +1,47 @@
+from __future__ import annotations
+
+import uuid
+from datetime import datetime
+
+from sqlalchemy import DateTime, LargeBinary, String, Text
+from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
+
+
+class Base(DeclarativeBase):
+    pass
+
+
+# Values for OutboxRow.status. Not a DB enum on purpose — SQLite (handy for
+# tests) and Postgres would need separate enum handling, and a plain indexed
+# string is enough for the states polled by a relay.
+#
+# pending -> claimed -> published            (happy path)
+# pending -> claimed -> pending               (publish failed, retry)
+# pending -> claimed -> failed                (publish failed too many times)
+#
+# "claimed" exists so a relay can release its SELECT ... FOR UPDATE SKIP
+# LOCKED lock immediately after marking rows claimed, rather than holding a
+# transaction open across a network call to the broker — the lock's only
+# job is to make the claim itself atomic across concurrent relays.
+STATUS_PENDING = "pending"
+STATUS_CLAIMED = "claimed"
+STATUS_PUBLISHED = "published"
+STATUS_FAILED = "failed"
+
+
+class OutboxRow(Base):
+    __tablename__ = "outbox"
+
+    id: Mapped[uuid.UUID] = mapped_column(primary_key=True)
+    # Free-form, caller-assigned identifier (e.g. an aggregate/entity id) —
+    # not interpreted by this package, only indexed so a caller can look up
+    # "what's pending for this project/order/tenant" without decoding every
+    # payload. Optional: a caller with no natural correlation key just
+    # leaves it unset.
+    correlation_id: Mapped[str | None] = mapped_column(String(255), index=True, nullable=True)
+    payload: Mapped[bytes] = mapped_column(LargeBinary)
+    status: Mapped[str] = mapped_column(String(20), default=STATUS_PENDING, index=True)
+    attempts: Mapped[int] = mapped_column(default=0)
+    created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
+    published_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
+    last_error: Mapped[str | None] = mapped_column(Text, nullable=True)

taskqueue_toolkit/outbox/relay.py

@@ -0,0 +1,59 @@
+from __future__ import annotations
+
+import asyncio
+import logging
+from dataclasses import dataclass
+
+from taskqueue_toolkit.outbox.repository import OutboxRepository
+from taskqueue_toolkit.queue.task_queue import TaskQueue
+
+logger = logging.getLogger(__name__)
+
+_DEFAULT_BATCH_SIZE = 20
+_DEFAULT_POLL_INTERVAL_SECONDS = 2.0
+
+
+@dataclass(slots=True)
+class OutboxRelay[T]:
+    """Delivers outbox entries to the broker.
+
+    This is the only bridge between the outbox's storage (the durability
+    guarantee — an entry committed in the same transaction as the state
+    change it follows from is never lost even if the broker is briefly
+    unreachable) and TaskQueue[T] (the actual delivery mechanism, whichever
+    broker is configured). Neither side ever talks to the other directly.
+    """
+
+    outbox: OutboxRepository[T]
+    task_queue: TaskQueue[T]
+    batch_size: int = _DEFAULT_BATCH_SIZE
+
+    async def relay_once(self) -> int:
+        """Claim and publish one batch of pending entries. Returns how many
+        were claimed, so a caller can decide whether to poll again
+        immediately (batch was full) or wait (batch was empty/partial)."""
+        entries = await self.outbox.claim_pending(self.batch_size)
+        for entry in entries:
+            try:
+                await self.task_queue.publish(entry.task)
+            except Exception as exc:  # noqa: BLE001
+                logger.warning(
+                    "outbox entry failed to publish",
+                    extra={"outbox_id": str(entry.id)},
+                    exc_info=exc,
+                )
+                await self.outbox.mark_failed(entry.id, str(exc))
+            else:
+                await self.outbox.mark_published(entry.id)
+                logger.info("outbox entry published", extra={"outbox_id": str(entry.id)})
+        return len(entries)
+
+    async def run_forever(
+        self, poll_interval_seconds: float = _DEFAULT_POLL_INTERVAL_SECONDS
+    ) -> None:
+        """Poll indefinitely. Intended to run as a background task/process,
+        not inside a request handler."""
+        while True:
+            claimed = await self.relay_once()
+            if claimed < self.batch_size:
+                await asyncio.sleep(poll_interval_seconds)

taskqueue_toolkit/outbox/repository.py

@@ -0,0 +1,107 @@
+from __future__ import annotations
+
+import uuid
+from collections.abc import Callable
+from dataclasses import dataclass
+from datetime import UTC, datetime
+
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from taskqueue_toolkit.outbox.orm import (
+    STATUS_CLAIMED,
+    STATUS_FAILED,
+    STATUS_PENDING,
+    STATUS_PUBLISHED,
+    OutboxRow,
+)
+from taskqueue_toolkit.queue.task_queue import Decoder, Encoder
+
+_MAX_ATTEMPTS = 5
+
+
+@dataclass(frozen=True, slots=True)
+class OutboxEntry[T]:
+    id: uuid.UUID
+    task: T
+    attempts: int
+
+
+@dataclass(slots=True)
+class OutboxRepository[T]:
+    session: AsyncSession
+    encode: Encoder[T]
+    decode: Decoder[T]
+    # Optional: derive a caller-meaningful correlation_id (e.g. an aggregate
+    # id) from the task, purely for indexed lookups — this package never
+    # interprets it itself. Leave unset if the task type has no natural key.
+    correlation_id: Callable[[T], str] | None = None
+
+    async def enqueue(self, task: T) -> None:
+        """Add a task to the outbox within the caller's transaction.
+
+        Deliberately does not commit — the caller writes this in the same
+        transaction as whatever state change the task follows from, so
+        either both land or neither does. The relay is what actually
+        delivers the task to the broker later.
+        """
+        self.session.add(
+            OutboxRow(
+                id=uuid.uuid4(),
+                correlation_id=self.correlation_id(task) if self.correlation_id else None,
+                payload=self.encode(task),
+                status=STATUS_PENDING,
+                attempts=0,
+                created_at=datetime.now(UTC),
+            )
+        )
+
+    async def claim_pending(self, limit: int) -> list[OutboxEntry[T]]:
+        """Atomically mark up to `limit` pending entries as claimed and
+        return them, skipping rows already locked by another relay
+        (SKIP LOCKED) so multiple relay instances can run concurrently
+        without claiming the same task twice. The row lock is only held for
+        this one UPDATE — not across the network call to the broker that
+        follows — callers report the outcome via mark_published()/
+        mark_failed() afterwards.
+
+        Only verified against Postgres (this package's own test suite races
+        multiple relays against a real Postgres to confirm no double-claim).
+        MySQL 8.0+/MariaDB 10.6+ support the same SKIP LOCKED syntax via
+        SQLAlchemy but aren't tested here — treat that combination as
+        unverified. Older MySQL/MariaDB and SQLite don't give this method
+        its concurrency guarantee at all.
+        """
+        result = await self.session.execute(
+            select(OutboxRow)
+            .where(OutboxRow.status == STATUS_PENDING)
+            .order_by(OutboxRow.created_at)
+            .limit(limit)
+            .with_for_update(skip_locked=True)
+        )
+        rows = result.scalars().all()
+        entries = [
+            OutboxEntry(id=row.id, task=self.decode(row.payload), attempts=row.attempts)
+            for row in rows
+        ]
+        for row in rows:
+            row.status = STATUS_CLAIMED
+        await self.session.commit()
+        return entries
+
+    async def mark_published(self, entry_id: uuid.UUID) -> None:
+        row = await self.session.get(OutboxRow, entry_id)
+        if row is None:
+            return
+        row.status = STATUS_PUBLISHED
+        row.published_at = datetime.now(UTC)
+        await self.session.commit()
+
+    async def mark_failed(self, entry_id: uuid.UUID, error: str) -> None:
+        row = await self.session.get(OutboxRow, entry_id)
+        if row is None:
+            return
+        row.attempts += 1
+        row.last_error = error
+        row.status = STATUS_FAILED if row.attempts >= _MAX_ATTEMPTS else STATUS_PENDING
+        await self.session.commit()

taskqueue_toolkit/queue/aws_session.py

@@ -0,0 +1,19 @@
+from __future__ import annotations
+
+from taskqueue_toolkit.queue.dsn import SnsDsn, SqsDsn
+
+
+def aws_session_kwargs(dsn: SqsDsn | SnsDsn) -> dict[str, str]:
+    """boto3-style client() kwargs shared by every AWS-backed TaskQueue
+    implementation (SQS, SNS). Only includes keys that were actually set, so
+    real AWS auth (IAM role, env vars picked up by botocore itself) still
+    works when the DSN leaves them out — only LocalStack testing needs them.
+    """
+    kwargs = {"region_name": dsn.region}
+    if dsn.endpoint_url:
+        kwargs["endpoint_url"] = dsn.endpoint_url
+    if dsn.access_key_id:
+        kwargs["aws_access_key_id"] = dsn.access_key_id
+    if dsn.secret_access_key:
+        kwargs["aws_secret_access_key"] = dsn.secret_access_key
+    return kwargs

taskqueue_toolkit/queue/dsn.py

@@ -0,0 +1,147 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from urllib.parse import parse_qs, urlsplit
+
+
+class UnsupportedDsnSchemeError(Exception):
+    def __init__(self, scheme: str) -> None:
+        super().__init__(
+            f"Unsupported task queue DSN scheme: {scheme!r} "
+            "(expected one of amqp, redis, sqs, sns, pubsub)"
+        )
+
+
+def _query(raw_query: str) -> dict[str, str]:
+    return {key: values[0] for key, values in parse_qs(raw_query).items()}
+
+
+@dataclass(frozen=True, slots=True)
+class RabbitMqDsn:
+    url: str
+    queue_name: str
+
+
+@dataclass(frozen=True, slots=True)
+class RedisStreamsDsn:
+    url: str
+    stream_name: str
+    group: str
+    consumer: str
+
+
+@dataclass(frozen=True, slots=True)
+class SqsDsn:
+    region: str
+    queue_name: str
+    endpoint_url: str
+    access_key_id: str
+    secret_access_key: str
+    use_iam_role: bool
+
+
+@dataclass(frozen=True, slots=True)
+class SnsDsn:
+    region: str
+    topic_name: str
+    endpoint_url: str
+    access_key_id: str
+    secret_access_key: str
+    use_iam_role: bool
+
+
+@dataclass(frozen=True, slots=True)
+class PubsubDsn:
+    project_id: str
+    topic_name: str
+    subscription_name: str
+    emulator_host: str
+
+
+TaskQueueDsn = RabbitMqDsn | RedisStreamsDsn | SqsDsn | SnsDsn | PubsubDsn
+
+_DEFAULT_QUEUE_NAME = "tasks"
+_DEFAULT_STREAM_GROUP = "workers"
+_DEFAULT_STREAM_CONSUMER = "worker-1"
+
+
+def parse_task_queue_dsn(dsn: str) -> TaskQueueDsn:
+    """Parse a single connection string into a broker-specific, typed config.
+
+    The scheme picks the broker (and therefore which TaskQueue
+    implementation to build) — everything else the connection needs travels
+    in the same string: host/credentials in the URL itself, broker-specific
+    extras (queue/topic/group names, region, emulator endpoint, ...) as
+    query params. One string per deployment instead of a broker-name setting
+    plus a parallel block of per-broker settings.
+
+    Examples:
+      amqp://guest:guest@host:5672/?queue=my.tasks
+      redis://:password@host:6379/0?stream=my.tasks&group=workers&consumer=worker-1
+      sqs://eu-west-3/my.tasks?endpoint_url=...&access_key_id=...&secret_access_key=...
+      sns://eu-west-3/my-tasks?endpoint_url=...&access_key_id=...&secret_access_key=...
+      pubsub://project-id/my-tasks?subscription=my-tasks-subscriber&emulator_host=localhost:8085
+    """
+    parts = urlsplit(dsn)
+    query = _query(parts.query)
+
+    if parts.scheme in ("amqp", "amqps"):
+        return RabbitMqDsn(
+            url=dsn.split("?", 1)[0],
+            queue_name=query.get("queue", _DEFAULT_QUEUE_NAME),
+        )
+
+    if parts.scheme in ("redis", "rediss"):
+        return RedisStreamsDsn(
+            url=dsn.split("?", 1)[0],
+            stream_name=query.get("stream", _DEFAULT_QUEUE_NAME),
+            group=query.get("group", _DEFAULT_STREAM_GROUP),
+            consumer=query.get("consumer", _DEFAULT_STREAM_CONSUMER),
+        )
+
+    if parts.scheme == "sqs":
+        return SqsDsn(
+            region=parts.hostname or "",
+            queue_name=parts.path.lstrip("/"),
+            endpoint_url=query.get("endpoint_url", ""),
+            access_key_id=query.get("access_key_id", ""),
+            secret_access_key=query.get("secret_access_key", ""),
+            use_iam_role=query.get("iam_role", "").lower() == "true",
+        )
+
+    if parts.scheme == "sns":
+        return SnsDsn(
+            region=parts.hostname or "",
+            topic_name=parts.path.lstrip("/"),
+            endpoint_url=query.get("endpoint_url", ""),
+            access_key_id=query.get("access_key_id", ""),
+            secret_access_key=query.get("secret_access_key", ""),
+            use_iam_role=query.get("iam_role", "").lower() == "true",
+        )
+
+    if parts.scheme == "pubsub":
+        topic_name = parts.path.lstrip("/")
+        return PubsubDsn(
+            project_id=parts.hostname or "",
+            topic_name=topic_name,
+            subscription_name=query.get("subscription", f"{topic_name}-subscriber"),
+            emulator_host=query.get("emulator_host", ""),
+        )
+
+    raise UnsupportedDsnSchemeError(parts.scheme)
+
+
+def missing_production_aws_auth(dsn: TaskQueueDsn) -> bool:
+    """True when an sqs/sns DSN has no way to authenticate against real AWS:
+    no LocalStack endpoint, no explicit keys, and no iam_role=true opt-in.
+    Not applicable (returns False) to every other DSN type. Split out from
+    parsing so a caller's own production fail-fast check can call it without
+    needing to know anything about AWS-specific DSN fields.
+    """
+    if not isinstance(dsn, SqsDsn | SnsDsn):
+        return False
+    return (
+        not dsn.endpoint_url
+        and not dsn.use_iam_role
+        and (not dsn.access_key_id or not dsn.secret_access_key)
+    )

taskqueue_toolkit/queue/factory.py

@@ -0,0 +1,94 @@
+from __future__ import annotations
+
+from typing import assert_never
+from urllib.parse import urlsplit
+
+from taskqueue_toolkit.queue.dsn import (
+    PubsubDsn,
+    RabbitMqDsn,
+    RedisStreamsDsn,
+    SnsDsn,
+    SqsDsn,
+    UnsupportedDsnSchemeError,
+    missing_production_aws_auth,
+    parse_task_queue_dsn,
+)
+from taskqueue_toolkit.queue.pubsub import PubsubTaskQueue
+from taskqueue_toolkit.queue.rabbitmq import RabbitMqTaskQueue
+from taskqueue_toolkit.queue.redis_streams import RedisStreamsTaskQueue
+from taskqueue_toolkit.queue.registry import resolve_registered_scheme
+from taskqueue_toolkit.queue.sns import SnsTaskQueue
+from taskqueue_toolkit.queue.sqs import SqsTaskQueue
+from taskqueue_toolkit.queue.task_queue import Decoder, Encoder, TaskQueue
+
+_BUILT_IN_SCHEMES = frozenset({"amqp", "amqps", "redis", "rediss", "sqs", "sns", "pubsub"})
+
+
+class MissingAwsAuthError(Exception):
+    def __init__(self, environment: str) -> None:
+        super().__init__(
+            f"environment={environment!r} but the task queue DSN has no way "
+            "to authenticate against AWS: no endpoint_url (LocalStack), no "
+            "access_key_id/secret_access_key, and no iam_role=true. Add "
+            "credentials, or add '&iam_role=true' if auth is handled by an "
+            "IAM role (ECS task role, EKS service account, EC2 instance "
+            "profile) — refusing to build the queue with no AWS auth "
+            "configured outside development."
+        )
+
+
+def create_task_queue[T](
+    dsn: str,
+    *,
+    encode: Encoder[T],
+    decode: Decoder[T],
+    environment: str = "development",
+) -> TaskQueue[T]:
+    """Build the TaskQueue implementation described by a single DSN. The
+    scheme picks the adapter — amqp/amqps -> RabbitMQ, redis/rediss -> Redis
+    Streams, sqs/sns/pubsub -> themselves — and everything else the
+    connection needs (queue/topic names, region, credentials, emulator
+    target, ...) travels in the same string. This is the one place in this
+    package allowed to know every concrete adapter exists; callers depend on
+    TaskQueue[T] only, so switching broker is a DSN change, not a code
+    change.
+
+    encode/decode are required because this package has no opinion on how
+    your task type T serializes — every adapter needs a bytes-in/bytes-out
+    pair to actually move a T through its broker.
+
+    `environment` gates one fail-fast check: outside development, an
+    sqs/sns DSN with no LocalStack endpoint, no explicit keys, and no
+    iam_role=true opt-in is almost certainly a forgotten credential, not a
+    real deployment — refuse to build the adapter rather than let it fail
+    confusingly on the first network call.
+
+    A scheme this package doesn't ship a built-in adapter for is looked up
+    in the registry (see registry.register_scheme()) before giving up with
+    UnsupportedDsnSchemeError — that's how a broker outside this package's
+    five (amqp/redis/sqs/sns/pubsub) gets plugged in.
+    """
+    scheme = urlsplit(dsn).scheme
+    if scheme not in _BUILT_IN_SCHEMES:
+        handler = resolve_registered_scheme(dsn)
+        if handler is not None:
+            return handler(dsn, encode, decode)
+        raise UnsupportedDsnSchemeError(scheme)
+
+    parsed = parse_task_queue_dsn(dsn)
+
+    if environment != "development" and missing_production_aws_auth(parsed):
+        raise MissingAwsAuthError(environment)
+
+    if isinstance(parsed, RabbitMqDsn):
+        return RabbitMqTaskQueue(dsn=parsed, encode=encode, decode=decode)
+    if isinstance(parsed, RedisStreamsDsn):
+        return RedisStreamsTaskQueue(dsn=parsed, encode=encode, decode=decode)
+    if isinstance(parsed, SqsDsn):
+        return SqsTaskQueue(dsn=parsed, encode=encode, decode=decode)
+    if isinstance(parsed, SnsDsn):
+        return SnsTaskQueue(dsn=parsed, encode=encode, decode=decode)
+    if isinstance(parsed, PubsubDsn):
+        return PubsubTaskQueue(dsn=parsed, encode=encode, decode=decode)
+
+    assert_never(parsed)

taskqueue_toolkit/queue/pubsub.py

@@ -0,0 +1,162 @@
+from __future__ import annotations
+
+import asyncio
+import logging
+import os
+from collections.abc import AsyncIterator, Callable
+from contextlib import suppress
+from dataclasses import dataclass, field
+
+from google.api_core.exceptions import AlreadyExists
+from google.cloud import pubsub_v1
+
+from taskqueue_toolkit.queue.dsn import PubsubDsn
+from taskqueue_toolkit.queue.task_queue import Decoder, Encoder
+
+logger = logging.getLogger(__name__)
+
+_PULL_TIMEOUT_SECONDS = 10.0
+
+PublisherFactory = Callable[[], "pubsub_v1.PublisherClient"]
+SubscriberFactory = Callable[[], "pubsub_v1.SubscriberClient"]
+
+
+@dataclass(slots=True)
+class PubsubQueuedTask[T]:
+    _subscriber: pubsub_v1.SubscriberClient
+    _subscription_path: str
+    _ack_id: str
+    task: T
+
+    async def ack(self) -> None:
+        await asyncio.to_thread(
+            self._subscriber.acknowledge,
+            subscription=self._subscription_path,
+            ack_ids=[self._ack_id],
+        )
+
+    async def nack(self, *, requeue: bool) -> None:
+        if requeue:
+            # No dedicated "nack" RPC — resetting the ack deadline to 0
+            # makes the message immediately re-deliverable, same idea as
+            # SQS's change_message_visibility(VisibilityTimeout=0).
+            await asyncio.to_thread(
+                self._subscriber.modify_ack_deadline,
+                subscription=self._subscription_path,
+                ack_ids=[self._ack_id],
+                ack_deadline_seconds=0,
+            )
+        else:
+            await asyncio.to_thread(
+                self._subscriber.acknowledge,
+                subscription=self._subscription_path,
+                ack_ids=[self._ack_id],
+            )
+
+
+@dataclass(slots=True)
+class PubsubTaskQueue[T]:
+    """Pub/Sub is fan-out only, like SNS — no receive on a topic directly.
+
+    consume() needs a subscription to pull from, so this adapter provisions
+    one dedicated subscription per topic (create_topic and create_subscription
+    both raise AlreadyExists — not silently no-op like AWS's idempotent
+    create_topic/subscribe — so that's caught explicitly rather than relied
+    on to be a no-op).
+
+    The official client (google-cloud-pubsub) is synchronous gRPC, not
+    asyncio-native — every call here runs through asyncio.to_thread rather
+    than reimplementing an async gRPC stack for a single adapter.
+    """
+
+    dsn: PubsubDsn
+    encode: Encoder[T]
+    decode: Decoder[T]
+    # Overridable for tests (inject fake clients) or a shared client the
+    # caller already manages; each defaults to the real SDK client.
+    publisher_factory: PublisherFactory = field(default=pubsub_v1.PublisherClient)
+    subscriber_factory: SubscriberFactory = field(default=pubsub_v1.SubscriberClient)
+    _publisher_client: pubsub_v1.PublisherClient | None = field(default=None, init=False)
+    _subscriber_client: pubsub_v1.SubscriberClient | None = field(default=None, init=False)
+
+    def _apply_emulator_host(self) -> None:
+        # The official client only knows to target an emulator via this env
+        # var (it swaps in an insecure channel + anonymous credentials as a
+        # unit) — there's no supported constructor kwarg that does the same
+        # thing as reliably. Setting it here, from the DSN, right before the
+        # first client is built keeps the emulator target fully described by
+        # the DSN instead of requiring a second, separate env var.
+        if self.dsn.emulator_host:
+            os.environ["PUBSUB_EMULATOR_HOST"] = self.dsn.emulator_host
+
+    @property
+    def _publisher(self) -> pubsub_v1.PublisherClient:
+        # Built lazily, not at __init__ time (e.g. via default_factory) —
+        # constructing the client eagerly authenticates against GCP
+        # immediately, which fails outside an environment with real or
+        # emulated credentials even if this instance is never actually used.
+        if self._publisher_client is None:
+            self._apply_emulator_host()
+            self._publisher_client = self.publisher_factory()
+        return self._publisher_client
+
+    @property
+    def _subscriber(self) -> pubsub_v1.SubscriberClient:
+        if self._subscriber_client is None:
+            self._apply_emulator_host()
+            self._subscriber_client = self.subscriber_factory()
+        return self._subscriber_client
+
+    def _topic_path(self) -> str:
+        return str(self._publisher.topic_path(self.dsn.project_id, self.dsn.topic_name))
+
+    def _subscription_path(self) -> str:
+        return str(
+            self._subscriber.subscription_path(self.dsn.project_id, self.dsn.subscription_name)
+        )
+
+    async def _ensure_topic(self) -> str:
+        topic_path = self._topic_path()
+        with suppress(AlreadyExists):
+            await asyncio.to_thread(self._publisher.create_topic, name=topic_path)
+        return topic_path
+
+    async def _ensure_subscription(self, topic_path: str) -> str:
+        subscription_path = self._subscription_path()
+        with suppress(AlreadyExists):
+            await asyncio.to_thread(
+                self._subscriber.create_subscription, name=subscription_path, topic=topic_path
+            )
+        return subscription_path
+
+    async def publish(self, task: T) -> None:
+        topic_path = await self._ensure_topic()
+        # Pub/Sub fan-out only delivers to subscriptions that already exist
+        # at publish time — a message published before the subscription is
+        # provisioned is silently dropped, same trap as SNS. Ensuring it
+        # here, not just in consume(), keeps the first publish after a cold
+        # deploy from being lost.
+        await self._ensure_subscription(topic_path)
+
+        future = self._publisher.publish(topic_path, self.encode(task))
+        await asyncio.to_thread(future.result, timeout=10)
+        logger.info("task published", extra={"topic": self.dsn.topic_name})
+
+    async def consume(self) -> AsyncIterator[PubsubQueuedTask[T]]:
+        topic_path = await self._ensure_topic()
+        subscription_path = await self._ensure_subscription(topic_path)
+
+        while True:
+            response = await asyncio.to_thread(
+                self._subscriber.pull,
+                subscription=subscription_path,
+                max_messages=1,
+                timeout=_PULL_TIMEOUT_SECONDS,
+            )
+            for received in response.received_messages:
+                yield PubsubQueuedTask(
+                    _subscriber=self._subscriber,
+                    _subscription_path=subscription_path,
+                    _ack_id=received.ack_id,
+                    task=self.decode(received.message.data),
+                )