ringo-task-queue 0.1.0.dev8__py3-none-macosx_13_0_arm64.whl

ringo_task_queue/client.py

@@ -0,0 +1,1261 @@
+"""Connection management and the user-facing client/queue APIs."""
+
+from __future__ import annotations
+
+import asyncio
+import os
+import secrets
+import sys
+from datetime import timedelta
+from itertools import islice
+from typing import TYPE_CHECKING, Any, Awaitable, Callable, Coroutine, Iterable, Literal
+
+import grpc
+from grpc_health.v1 import health_pb2, health_pb2_grpc
+
+from . import _pb
+from ._proto.ringo.v1 import queue_pb2, queue_pb2_grpc
+from ._supervisor_policy import TaskTimeouts
+from ._version import PROTOCOL_MAJOR, PROTOCOL_MINOR, SDK_VERSION
+from .concurrency import AdaptiveConcurrency, ResourceConcurrency
+from .errors import (
+    IncompatibleVersionError,
+    InvalidArgumentError,
+    RingoError,
+    UnavailableError,
+    map_rpc_error,
+)
+from .models import (
+    AttemptPage,
+    BatchResult,
+    EnqueueResult,
+    PostgresStorage,
+    QueueStats,
+    SQLiteStorage,
+    Task,
+    TaskSpec,
+)
+
+if TYPE_CHECKING:  # pragma: no cover
+    from .daemon import LocalDaemon
+    from .shared import SharedLocalDaemon
+    from .worker import Worker
+
+SDK_NAME = "ringo-python"
+
+DEFAULT_RPC_TIMEOUT = 30.0
+_DEFAULT_MAX_MESSAGE_BYTES = 16 * 1024 * 1024
+_MAX_ENQUEUE_BATCH = 1000
+_RECONNECT_ATTEMPTS = 3
+_BACKOFF_SECONDS = (0.2, 0.6, 1.5)
+#: GetQueueStats is additive protocol 1.4; older servers reject the RPC.
+_STATS_MIN_PROTOCOL_MINOR = 4
+#: Maximum number of task-type filters accepted by ``TaskQueue.stats``.
+_STATS_MAX_TASK_TYPES = 100
+_ATTEMPT_HISTORY_MIN_PROTOCOL_MINOR = 5
+_ATTEMPT_PAGE_DEFAULT_LIMIT = 32
+_ATTEMPT_PAGE_MAX_LIMIT = 100
+_MAX_ATTEMPT_CURSOR_BYTES = 1024
+
+#: Supported values for the remote ``compression`` option: ``"none"``
+#: (the default) or ``"gzip"``. ``None`` means "not specified": the channel
+#: default applies and a legacy raw ``channel_options`` compression entry
+#: keeps its meaning. The runtime validation in
+#: :func:`_normalize_compression` still rejects any other value for dynamic
+#: callers.
+CompressionOption = Literal["none", "gzip"]
+_COMPRESSION_CHOICES: tuple[str, ...] = ("none", "gzip")
+#: gRPC channel option that controls the same thing as ``compression``. An
+#: explicit ``compression`` (even ``"none"``) combined with this raw option is
+#: an ambiguity error, mirroring the Node SDK.
+_COMPRESSION_CHANNEL_OPTION = "grpc.default_compression_algorithm"
+
+
+def _normalize_compression(value: CompressionOption | None) -> grpc.Compression | None:
+    """Map the public ``compression`` option onto ``grpc.Compression``.
+
+    ``None`` (unspecified) maps to ``None`` so the channel default applies.
+    Anything else outside ``_COMPRESSION_CHOICES`` is a clear
+    ``InvalidArgumentError`` raised before any network activity.
+    """
+    if value is None:
+        return None
+    if not isinstance(value, str) or value not in _COMPRESSION_CHOICES:
+        raise InvalidArgumentError(
+            f"unsupported compression {value!r}; expected one of"
+            f" {', '.join(_COMPRESSION_CHOICES)}"
+        )
+    if value == "gzip":
+        return grpc.Compression.Gzip
+    return grpc.Compression.NoCompression
+
+
+def _check_compression_channel_option_conflict(
+    channel_options: Iterable[tuple[str, Any]] | None,
+    compression: grpc.Compression | None,
+) -> None:
+    """Reject an explicit ``compression`` alongside the raw gRPC option.
+
+    When ``compression`` is unspecified (``None``), raw ``channel_options``
+    keep their legacy meaning for advanced users.
+    """
+    if compression is None:
+        return
+    for key, _ in channel_options or ():
+        if key == _COMPRESSION_CHANNEL_OPTION:
+            raise InvalidArgumentError(
+                f"compression and channel_options[{_COMPRESSION_CHANNEL_OPTION!r}]"
+                " must not both be set"
+            )
+
+
+def _consume_task_exception(task: asyncio.Task[Any]) -> None:
+    if not task.cancelled():
+        task.exception()
+
+
+def _parse_endpoint(endpoint: str) -> tuple[str, bool]:
+    """Return ``(authority, use_tls)`` for an endpoint string."""
+    if not endpoint:
+        raise InvalidArgumentError("endpoint is required")
+    use_tls = False
+    rest = endpoint
+    for scheme, tls in (
+        ("grpc://", False),
+        ("http://", False),
+        ("grpcs://", True),
+        ("https://", True),
+    ):
+        if rest.startswith(scheme):
+            rest = rest[len(scheme) :]
+            use_tls = tls
+            break
+    rest = rest.rstrip("/")
+    if ":" not in rest:
+        rest = f"{rest}:7233"
+    return rest, use_tls
+
+
+#: Bounded automatic restarts of a crashed local daemon (design §31).
+_MAX_DAEMON_RESTARTS = 3
+
+
+class _Connection:
+    """Owns the grpc.aio channel, negotiation, and retried unary calls."""
+
+    def __init__(
+        self,
+        endpoint: str,
+        token: str | None = None,
+        *,
+        local_token: str | None = None,
+        local_generation: str | None = None,
+        channel_options: Iterable[tuple[str, Any]] | None = None,
+        compression: grpc.Compression | None = None,
+    ) -> None:
+        self._endpoint, self._tls = _parse_endpoint(endpoint)
+        self._token = token
+        self._local_token = local_token
+        self._local_generation = local_generation
+        self._attach_call: Any = None
+        self._attach_reader_task: asyncio.Task[Any] | None = None
+        self._shared_disconnect: Callable[[], Awaitable[None]] | None = None
+        self._channel_options = list(channel_options or ())
+        configured = {key for key, _ in self._channel_options}
+        for key in ("grpc.max_receive_message_length", "grpc.max_send_message_length"):
+            if key not in configured:
+                self._channel_options.append((key, _DEFAULT_MAX_MESSAGE_BYTES))
+        if local_token is not None:
+            configured = {key for key, _ in self._channel_options}
+            for key, value in (
+                ("grpc.keepalive_time_ms", 15000),
+                ("grpc.keepalive_timeout_ms", 10000),
+                ("grpc.keepalive_permit_without_calls", 1),
+            ):
+                if key not in configured:
+                    self._channel_options.append((key, value))
+        _check_compression_channel_option_conflict(self._channel_options, compression)
+        # Channel-level default compression; survives every reconnect because
+        # _open_channel() re-applies it to each new aio channel. ``None`` means
+        # unspecified, so legacy raw channel options decide the encoding.
+        self._compression = compression
+        self._channel: grpc.aio.Channel | None = None
+        self._stub: queue_pb2_grpc.QueueServiceStub | None = None
+        self._session_minor = PROTOCOL_MINOR
+        self._session_token = ""
+        self._lock = asyncio.Lock()
+        self._closed = False
+        #: Optional client-provided reconnect hook (daemon restart + channel
+        #: rebuild). Invoked under ``_lock`` instead of plain ``negotiate``.
+        self.on_reconnect: Callable[[], Awaitable[None]] | None = None
+
+    # -- lifecycle ---------------------------------------------------------
+
+    def _open_channel(self) -> None:
+        if self._closed:
+            raise RingoError("connection is closed")
+        if self._tls:
+            credentials = grpc.ssl_channel_credentials()
+            self._channel = grpc.aio.secure_channel(
+                self._endpoint,
+                credentials,
+                options=self._channel_options,
+                compression=self._compression,
+            )
+        else:
+            self._channel = grpc.aio.insecure_channel(
+                self._endpoint,
+                options=self._channel_options,
+                compression=self._compression,
+            )
+        self._stub = queue_pb2_grpc.QueueServiceStub(self._channel)
+
+    async def connect(self, *, check_health: bool = False) -> None:
+        self._open_channel()
+        if check_health:
+            await self._await_serving()
+        if self._local_token is not None:
+            await self.attach_local()
+        await self.negotiate()
+
+    async def attach_local(self) -> None:
+        """Register this SDK process as a shared-daemon member."""
+        if self._local_token is None or self._local_generation is None:
+            return
+        stub = self._require_stub()
+        try:
+            request = queue_pb2.AttachLocalRequest(generation=self._local_generation)
+            call = stub.AttachLocal(request, metadata=self._metadata())
+            response = await call.read()
+            if response is grpc.aio.EOF or response is None:
+                raise UnavailableError(
+                    "shared daemon closed AttachLocal during handshake"
+                )
+            if self._closed:
+                call.cancel()
+                raise RingoError("connection is closed")
+            if response.generation != self._local_generation:
+                call.cancel()
+                raise UnavailableError(
+                    "shared daemon generation changed during AttachLocal"
+                )
+        except grpc.aio.AioRpcError as exc:
+            raise map_rpc_error(exc) from exc
+        self._attach_call = call
+        self._attach_reader_task = asyncio.create_task(
+            self._consume_attach(call), name="ringo-shared-attach"
+        )
+
+    async def _consume_attach(self, call: Any) -> None:
+        try:
+            while True:
+                response = await call.read()
+                if response is grpc.aio.EOF:
+                    break
+                if response is None:
+                    break
+                if response.generation != self._local_generation:
+                    break
+        except asyncio.CancelledError:
+            return
+        except grpc.aio.AioRpcError:
+            pass
+        finally:
+            if call is self._attach_call:
+                self._attach_call = None
+                if not self._closed and self._shared_disconnect is not None:
+                    asyncio.ensure_future(self._shared_disconnect())
+
+    def set_shared_session(
+        self, token: str, generation: str, on_disconnect: Callable[[], Awaitable[None]]
+    ) -> None:
+        self._local_token = token
+        self._local_generation = generation
+        self._shared_disconnect = on_disconnect
+
+    async def _close_attach(self) -> None:
+        task, self._attach_reader_task = self._attach_reader_task, None
+        call, self._attach_call = self._attach_call, None
+        if call is not None:
+            try:
+                call.cancel()
+            except Exception:
+                pass
+        if task is not None and task is not asyncio.current_task():
+            task.cancel()
+            await asyncio.gather(task, return_exceptions=True)
+
+    async def _await_serving(self, timeout: float = 5.0) -> None:
+        """Wait until the health service reports SERVING (local daemon
+        readiness is announced on stdout before the gRPC health state may be
+        observable, so poll briefly instead of assuming)."""
+        assert self._channel is not None
+        health_stub = health_pb2_grpc.HealthStub(self._channel)
+        deadline = asyncio.get_running_loop().time() + timeout
+        last: Exception | None = None
+        while True:
+            if self._closed:
+                raise RingoError("connection is closed")
+            try:
+                response = await health_stub.Check(
+                    health_pb2.HealthCheckRequest(),
+                    timeout=2.0,
+                    metadata=self._metadata(),
+                )
+                if response.status == health_pb2.HealthCheckResponse.SERVING:
+                    return
+                last = RingoError("daemon health status is not SERVING")
+            except grpc.aio.AioRpcError as exc:
+                last = exc
+            if asyncio.get_running_loop().time() >= deadline:
+                raise RingoError(f"daemon did not report SERVING health: {last}")
+            await asyncio.sleep(0.05)
+
+    async def reset(self, endpoint: str) -> None:
+        """Point the connection at a new endpoint (local daemon restart)."""
+        if self._closed:
+            raise RingoError("connection is closed")
+        await self._close_attach()
+        self._endpoint, self._tls = _parse_endpoint(endpoint)
+        channel, self._channel = self._channel, None
+        self._stub = None
+        if channel is not None:
+            await channel.close()
+        self._open_channel()
+
+    async def negotiate(self) -> None:
+        """Perform the major=1/minor=0 version handshake (ADR 0002)."""
+        stub = self._require_stub()
+        request = queue_pb2.NegotiateRequest(
+            sdk_name=SDK_NAME,
+            sdk_version=SDK_VERSION,
+            protocol_major=PROTOCOL_MAJOR,
+            protocol_minor=PROTOCOL_MINOR,
+        )
+        try:
+            response = await stub.Negotiate(
+                request, timeout=DEFAULT_RPC_TIMEOUT, metadata=self._metadata()
+            )
+        except grpc.aio.AioRpcError as exc:
+            raise map_rpc_error(exc) from exc
+        if self._closed:
+            raise RingoError("connection is closed")
+        if response.protocol_major != PROTOCOL_MAJOR:
+            raise IncompatibleVersionError(
+                f"server speaks protocol major {response.protocol_major},"
+                f" SDK requires major {PROTOCOL_MAJOR}; upgrade one side"
+            )
+        self._session_minor = min(response.protocol_minor, PROTOCOL_MINOR)
+        self._session_token = response.session_token
+
+    async def close(self) -> None:
+        self._closed = True
+        await self._close_attach()
+        channel, self._channel = self._channel, None
+        self._stub = None
+        if channel is not None:
+            await channel.close()
+
+    # -- helpers -----------------------------------------------------------
+
+    def _require_stub(self) -> queue_pb2_grpc.QueueServiceStub:
+        if self._closed or self._stub is None:
+            raise RingoError("connection is closed")
+        return self._stub
+
+    def _metadata(self) -> list[tuple[str, str]]:
+        metadata: list[tuple[str, str]] = []
+        if self._token:
+            metadata.append(("authorization", f"Bearer {self._token}"))
+        if self._local_token:
+            metadata.append(("x-ringo-local-token", self._local_token))
+        return metadata
+
+    def request_context(self) -> queue_pb2.RequestContext:
+        return queue_pb2.RequestContext(
+            protocol_major=PROTOCOL_MAJOR,
+            protocol_minor=self._session_minor,
+            session_token=self._session_token,
+        )
+
+    @property
+    def endpoint(self) -> str:
+        return self._endpoint
+
+    @property
+    def protocol_minor(self) -> int:
+        """Protocol minor negotiated for the current session."""
+        return self._session_minor
+
+    async def call(
+        self,
+        name: str,
+        request: Any,
+        *,
+        timeout: float = DEFAULT_RPC_TIMEOUT,
+        retry_safe: bool = False,
+    ) -> Any:
+        """Issue a single unary RPC.
+
+        Submission is never silently replayed: an ``UNAVAILABLE`` may mean the
+        server committed the operation (a lost ACK response, a claimed lease)
+        so the original error is raised after best-effort recovery (daemon
+        restart + fresh negotiation) prepares the connection for *future*
+        calls. Only genuinely read-only RPCs (task, history, and stats reads)
+        pass ``retry_safe=True`` and may be re-sent, each time with a freshly
+        negotiated ``RequestContext``.
+        """
+        rpc, sent_token = await self._begin_unary(name, request, timeout=timeout)
+        cause: BaseException | None = None
+        err: RingoError | None = None
+        try:
+            return await rpc
+        except grpc.aio.AioRpcError as exc:
+            cause = exc
+            err = map_rpc_error(exc)
+        except asyncio.CancelledError as exc:
+            # grpc.aio raises CancelledError when a reconnect closes its old
+            # channel. Preserve genuine caller cancellation, but surface this
+            # transport race as UNAVAILABLE.
+            task = asyncio.current_task()
+            if task is not None and task.cancelling():
+                raise
+            cause = exc
+            err = UnavailableError("RPC interrupted by connection recovery")
+
+        assert err is not None and cause is not None
+        if self._closed:
+            raise err from cause
+        if isinstance(err, IncompatibleVersionError):
+            # Session rejection happens in authorization before the operation
+            # can execute. It is therefore safe to negotiate a fresh short-lived
+            # token and retry once, including ordinary token expiry with no
+            # concurrent daemon restart.
+            await self._recover_best_effort()
+            async with self._lock:
+                rotated = self._session_token != sent_token
+            if rotated:
+                rpc, _ = await self._begin_unary(name, request, timeout=timeout)
+                try:
+                    return await rpc
+                except grpc.aio.AioRpcError as retry_exc:
+                    raise map_rpc_error(retry_exc) from retry_exc
+            raise err from cause
+        if not isinstance(err, UnavailableError):
+            raise err from cause
+
+        # Recovery prepares the connection for future calls. Mutations are not
+        # replayed because an unavailable response may hide a committed write.
+        await self._recover_best_effort()
+        if not retry_safe:
+            raise err from cause
+        rpc, _ = await self._begin_unary(name, request, timeout=timeout)
+        try:
+            return await rpc
+        except grpc.aio.AioRpcError as retry_exc:
+            raise map_rpc_error(retry_exc) from retry_exc
+
+    async def _begin_unary(
+        self,
+        name: str,
+        request: Any,
+        *,
+        timeout: float,
+    ) -> tuple[Any, str]:
+        """Snapshot a stub and fresh session context outside reconnect races."""
+        async with self._lock:
+            # Check every send, including a read retry after renegotiation.
+            if (
+                name == "GetTaskAttempts"
+                and self._session_minor < _ATTEMPT_HISTORY_MIN_PROTOCOL_MINOR
+            ):
+                raise IncompatibleVersionError(
+                    "attempt history requires protocol minor >= 5 (GetTaskAttempts);"
+                    f" negotiated minor is {self._session_minor}"
+                )
+            if name == "ProbeQueue" and self._session_minor < 7:
+                raise IncompatibleVersionError(
+                    "queue eligibility probe requires protocol minor >= 7 (ProbeQueue);"
+                    f" negotiated minor is {self._session_minor}"
+                )
+            request.context.CopyFrom(self.request_context())
+            stub = self._require_stub()
+            token = self._session_token
+            rpc = getattr(stub, name)(
+                request, timeout=timeout, metadata=self._metadata()
+            )
+            return rpc, token
+
+    async def _recover_best_effort(self) -> None:
+        """Bounded reconnect so later calls can succeed; failures here are
+        swallowed because the original call error is what the caller sees."""
+        for attempt in range(_RECONNECT_ATTEMPTS):
+            try:
+                await asyncio.sleep(
+                    _BACKOFF_SECONDS[min(attempt, len(_BACKOFF_SECONDS) - 1)]
+                )
+                async with self._lock:
+                    if self._closed:
+                        return
+                    if self.on_reconnect is not None:
+                        await self.on_reconnect()
+                    else:
+                        await self.negotiate()
+                return
+            except Exception:
+                continue
+
+
+def new_worker_id() -> str:
+    return f"py-{os.getpid()}-{secrets.token_hex(4)}"
+
+
+# ---------------------------------------------------------------------------
+# Public client
+
+
+class RingoClient:
+    """A connected (or connectable) Ringo client.
+
+    Use as an async context manager, or call :meth:`start` / :meth:`close`
+    explicitly::
+
+        async with Ringo.connect("127.0.0.1:7233") as client:
+            queue = client.queue("crawl")
+    """
+
+    def __init__(
+        self,
+        *,
+        endpoint: str | None = None,
+        token: str | None = None,
+        daemon: "LocalDaemon | None" = None,
+        shared_daemon: "SharedLocalDaemon | None" = None,
+        channel_options: Iterable[tuple[str, Any]] | None = None,
+        compression: CompressionOption | None = None,
+    ) -> None:
+        self._endpoint = endpoint
+        self._token = token
+        self._channel_options = channel_options
+        # The value is always validated, but ``compression`` only applies to
+        # remote connections; local mode stays on the (uncompressed) default.
+        normalized = _normalize_compression(compression)
+        if daemon is None and shared_daemon is None:
+            _check_compression_channel_option_conflict(
+                self._channel_options, normalized
+            )
+            self._compression = normalized
+            self._compression_label: CompressionOption = (
+                compression if compression is not None else "none"
+            )
+        else:
+            # Local mode never enables compression; an explicit value is
+            # validated above and then ignored (source compatibility).
+            self._compression = None
+            self._compression_label = "none"
+        self._daemon = daemon
+        self._shared_daemon = shared_daemon
+        if daemon is not None and shared_daemon is not None:
+            raise InvalidArgumentError(
+                "daemon and shared_daemon are mutually exclusive"
+            )
+        # ``_conn`` is built in start(): a local client learns the daemon's
+        # random loopback port only after the daemon announces readiness.
+        self._conn: _Connection | None = None
+        self._queues: dict[str, TaskQueue] = {}
+        self._workers: set[Worker] = set()
+        self._started = False
+        self._closed = False
+        # Start, reconnect and final resource cleanup share one lifecycle lane.
+        # Reconnect enters from the connection's session lock; cleanup must not
+        # acquire that lock while holding this one.
+        self._start_lock = asyncio.Lock()
+        self._close_lock = asyncio.Lock()
+        self._close_task: asyncio.Task[None] | None = None
+        self._daemon_restart_count = 0
+
+    # -- lifecycle ---------------------------------------------------------
+
+    def _check_open(self) -> None:
+        if self._closed:
+            raise RingoError("client is closed")
+
+    async def start(self) -> "RingoClient":
+        async with self._start_lock:
+            self._check_open()
+            # Register before allocating resources: a runtime STOP fence rejects
+            # late starts, and an in-flight start is visible to unified drain.
+            module = sys.modules.get("ringo_task_queue._supervisor_runtime")
+            runtime = None if module is None else module.current_runtime()
+            if runtime is not None:
+                runtime.register_client(self)
+            if not self._started:
+                try:
+                    for attempt in range(2 if self._shared_daemon is not None else 1):
+                        if self._daemon is not None:
+                            self._endpoint = await self._daemon.start()
+                        elif self._shared_daemon is not None:
+                            self._endpoint = await self._shared_daemon.start()
+                        self._check_open()
+                        conn = _Connection(
+                            self._endpoint or "",
+                            self._token,
+                            local_token=(
+                                self._shared_daemon.token
+                                if self._shared_daemon
+                                else None
+                            ),
+                            local_generation=(
+                                self._shared_daemon.generation
+                                if self._shared_daemon
+                                else None
+                            ),
+                            channel_options=self._channel_options,
+                            compression=self._compression,
+                        )
+                        self._conn = conn
+                        conn.on_reconnect = self._reconnect
+                        if self._shared_daemon is not None:
+                            conn.set_shared_session(
+                                self._shared_daemon.token or "",
+                                self._shared_daemon.generation or "",
+                                self._shared_stream_lost,
+                            )
+                        try:
+                            await conn.connect(
+                                check_health=self._daemon is not None
+                                or self._shared_daemon is not None
+                            )
+                            self._check_open()
+                            break
+                        except Exception:
+                            await conn.close()
+                            self._conn = None
+                            if (
+                                self._closed
+                                or self._shared_daemon is None
+                                or attempt > 0
+                            ):
+                                raise
+                            # Stale PID/port, wrong token or generation is never
+                            # accepted as identity. Re-elect once, boundedly.
+                            self._shared_daemon.invalidate_current()
+                except BaseException:
+                    # Fast cleanup: never leak a half-started channel or local daemon.
+                    failed_conn = self._conn
+                    self._conn = None
+                    if failed_conn is not None:
+                        await failed_conn.close()
+                    await self._stop_daemon()
+                    raise
+                self._started = True
+        return self
+
+    def _require_conn(self) -> _Connection:
+        conn = self._conn
+        if conn is None:
+            raise RingoError(
+                "client is not started; use 'async with' or await client.start()"
+            )
+        return conn
+
+    async def _reconnect(self) -> None:
+        """Rotate resources in the lifecycle lane; close wins at every boundary."""
+        async with self._start_lock:
+            self._check_open()
+            conn = self._require_conn()
+            daemon = self._daemon
+            shared = self._shared_daemon
+            if daemon is not None and daemon.is_dead:
+                if self._daemon_restart_count >= _MAX_DAEMON_RESTARTS:
+                    raise UnavailableError(
+                        f"local daemon restart budget ({_MAX_DAEMON_RESTARTS}) exhausted"
+                    )
+                self._daemon_restart_count += 1
+                delay = 0.3 * (2 ** (self._daemon_restart_count - 1))
+                await asyncio.sleep(delay)
+                self._check_open()
+                endpoint = await daemon.start()
+                self._check_open()
+                await conn.reset(endpoint)
+                self._check_open()
+            elif shared is not None:
+                try:
+                    endpoint = await shared.refresh()
+                    self._check_open()
+                    await conn.reset(endpoint)
+                    self._check_open()
+                    conn.set_shared_session(
+                        shared.token or "",
+                        shared.generation or "",
+                        self._shared_stream_lost,
+                    )
+                    await conn.attach_local()
+                    self._check_open()
+                    await conn.negotiate()
+                    self._check_open()
+                except Exception:
+                    if not self._closed:
+                        shared.invalidate_current()
+                    raise
+                return
+            await conn.negotiate()
+            self._check_open()
+
+    async def _shared_stream_lost(self) -> None:
+        """Proactively reconnect an idle shared client after AttachLocal EOF."""
+        if self._closed or not self._started or self._shared_daemon is None:
+            return
+        conn = self._conn
+        if conn is None or conn._closed:
+            return
+        # A stream can report EOF more than once while a replacement is being
+        # opened. Serialize recovery through the connection's normal lock.
+        for attempt in range(_MAX_DAEMON_RESTARTS + 1):
+            if self._closed:
+                return
+            try:
+                await asyncio.sleep(
+                    _BACKOFF_SECONDS[min(attempt, len(_BACKOFF_SECONDS) - 1)]
+                )
+                async with conn._lock:
+                    await self._reconnect()
+                return
+            except Exception:
+                continue
+
+    async def __aenter__(self) -> "RingoClient":
+        return await self.start()
+
+    async def close(self) -> None:
+        """Drain and stop once; caller cancellation cannot cancel cleanup."""
+        async with self._close_lock:
+            if self._close_task is None:
+                self._closed = True
+                self._close_task = asyncio.create_task(
+                    self._close_impl(), name="ringo-client-close"
+                )
+                self._close_task.add_done_callback(_consume_task_exception)
+            close_task = self._close_task
+        await asyncio.shield(close_task)
+
+    async def _close_impl(self) -> None:
+        """Attempt every cleanup step and aggregate any failures."""
+        errors: list[Exception] = []
+        # 1. Drain workers (stop taking tasks, wait grace, cancel remainder).
+        for worker in list(self._workers):
+            try:
+                await worker.close()
+            except Exception as exc:  # keep cleaning up the rest
+                errors.append(exc)
+        self._workers.clear()
+        # 2. Fence the current channel first: this interrupts a pending handshake
+        # without acquiring the session lock (reconnect holds it). Work remains
+        # usable through the worker grace period above.
+        conn = self._conn
+        if conn is not None:
+            try:
+                await conn.close()
+            except Exception as exc:
+                errors.append(exc)
+        # 3. An in-progress daemon.start cannot be abandoned. Wait for its
+        # bounded completion, then reconcile resources under the lifecycle lane.
+        # Closed checks prevent it from publishing a new channel in the meantime.
+        async with self._start_lock:
+            conn, self._conn = self._conn, None
+            if conn is not None:
+                try:
+                    await conn.close()
+                except Exception as exc:
+                    errors.append(exc)
+            try:
+                await self._stop_daemon()
+            except Exception as exc:
+                errors.append(exc)
+            self._started = False
+        if errors:
+            raise ExceptionGroup("errors while closing Ringo client", errors)
+
+    async def _stop_daemon(self) -> None:
+        daemon, self._daemon = self._daemon, None
+        if daemon is not None:
+            await daemon.stop()
+
+    async def __aexit__(self, exc_type, exc, tb) -> None:
+        try:
+            await self.close()
+        except ExceptionGroup as cleanup_errors:
+            # A business exception takes precedence; cleanup failures must
+            # not mask it, but should not vanish either.
+            if exc is None:
+                raise
+            for error in cleanup_errors.exceptions:
+                exc.__cause__ = exc.__cause__ or error
+
+    @property
+    def is_local(self) -> bool:
+        return self._daemon is not None or self._shared_daemon is not None
+
+    @property
+    def compression(self) -> CompressionOption:
+        """The effective channel compression: ``"none"`` (default) or ``"gzip"``."""
+        return self._compression_label
+
+    # -- queues ------------------------------------------------------------
+
+    def queue(self, name: str) -> "TaskQueue":
+        if not name:
+            raise InvalidArgumentError("queue name is required")
+        if self._closed:
+            raise RingoError("client is closed")
+        existing = self._queues.get(name)
+        if existing is None:
+            existing = TaskQueue(self, name)
+            self._queues[name] = existing
+        return existing
+
+    def _register_worker(self, worker: "Worker") -> None:
+        self._workers.add(worker)
+
+    def _unregister_worker(self, worker: "Worker") -> None:
+        self._workers.discard(worker)
+
+
+class Ringo:
+    """Factory for local (embedded daemon) and remote clients.
+
+    Both factories are synchronous and return a client whose
+    ``async with``/``await start()`` performs the actual connect and
+    negotiation, so ``async with Ringo.local(...)`` works directly.
+    """
+
+    @staticmethod
+    def local(
+        data_dir: str | os.PathLike[str],
+        storage: str | SQLiteStorage | PostgresStorage = "sqlite",
+        daemon_path: str | os.PathLike[str] | None = None,
+        *,
+        shared: bool = False,
+        startup_timeout: float = 15.0,
+        stop_timeout: float = 10.0,
+    ) -> RingoClient:
+        """Start a private daemon, or opt into a shared SQLite daemon."""
+        if not isinstance(shared, bool):
+            raise InvalidArgumentError("shared must be a boolean")
+        if storage == "sqlite" or isinstance(storage, SQLiteStorage):
+            storage_name = "sqlite"
+            postgres_dsn = None
+        elif isinstance(storage, PostgresStorage):
+            storage_name = "postgres"
+            postgres_dsn = storage.dsn
+        else:
+            raise InvalidArgumentError(f"unsupported local storage {storage!r}")
+        if shared and storage_name != "sqlite":
+            raise InvalidArgumentError("shared local mode supports SQLite only")
+        # Imported here so remote-only users never pay for subprocess imports.
+        if shared:
+            from .shared import SharedLocalDaemon
+
+            shared_daemon = SharedLocalDaemon(
+                data_dir, daemon_path=daemon_path, startup_timeout=startup_timeout
+            )
+            return RingoClient(shared_daemon=shared_daemon)
+        from .daemon import LocalDaemon
+
+        daemon = LocalDaemon(
+            data_dir,
+            storage=storage_name,
+            postgres_dsn=postgres_dsn,
+            daemon_path=daemon_path,
+            startup_timeout=startup_timeout,
+            stop_timeout=stop_timeout,
+        )
+        return RingoClient(daemon=daemon)
+
+    @staticmethod
+    def connect(
+        endpoint: str,
+        token: str | None = None,
+        *,
+        channel_options: Iterable[tuple[str, Any]] | None = None,
+        compression: CompressionOption | None = None,
+    ) -> RingoClient:
+        """Create a client for a remote (or already-running) endpoint.
+
+        ``compression`` is an optional whole-channel setting: ``"none"``
+        (the default) or ``"gzip"``. ``None`` means "not specified", leaving
+        the gRPC channel default (and any raw ``channel_options`` compression
+        entry) in charge. An explicit value combined with the raw channel
+        option ``grpc.default_compression_algorithm`` raises
+        :class:`InvalidArgumentError`; any other value does so as well,
+        immediately and before any network activity.
+        """
+        return RingoClient(
+            endpoint=endpoint,
+            token=token,
+            channel_options=channel_options,
+            compression=compression,
+        )
+
+
+# ---------------------------------------------------------------------------
+# TaskQueue
+
+
+Handler = Callable[[Task, Any], Coroutine[Any, Any, None]]
+
+
+class TaskQueue:
+    """A client bound to one logical queue (``queue + uniqueKey`` dedupe)."""
+
+    def __init__(self, client: RingoClient, name: str) -> None:
+        self._client = client
+        self.name = name
+
+    # -- enqueue ------------------------------------------------------------
+
+    async def enqueue(
+        self,
+        spec: TaskSpec | None = None,
+        /,
+        **kwargs: Any,
+    ) -> EnqueueResult:
+        """Enqueue one task; duplicates return ``created=False`` (no raise)."""
+        if spec is None:
+            spec = TaskSpec(**kwargs)
+        request = queue_pb2.EnqueueRequest(queue=self.name, task=spec.to_proto())
+        response = await self._client._require_conn().call("Enqueue", request)
+        return EnqueueResult.from_proto(response.result)
+
+    async def enqueue_many(self, specs: Iterable[TaskSpec]) -> BatchResult:
+        """Enqueue any number of tasks, auto-batching under the server limit.
+
+        The server enforces a hard limit of ``_MAX_ENQUEUE_BATCH`` (1000)
+        tasks per gRPC call.  This method transparently splits larger
+        iterables into sequential sub-batches, preserving creation order,
+        and aggregates the per-item results into a single :class:`BatchResult`.
+
+        Fail-fast: if any sub-batch fails, the exception propagates and no
+        partial result is returned.
+        """
+        specs = list(specs)
+        all_results: list[EnqueueResult] = []
+        total_created = 0
+        total_duplicates = 0
+
+        for start in range(0, len(specs), _MAX_ENQUEUE_BATCH):
+            chunk = specs[start : start + _MAX_ENQUEUE_BATCH]
+            request = queue_pb2.EnqueueManyRequest(
+                queue=self.name, tasks=[s.to_proto() for s in chunk]
+            )
+            response = await self._client._require_conn().call("EnqueueMany", request)
+            all_results.extend(EnqueueResult.from_proto(r) for r in response.results)
+            total_created += response.created_count
+            total_duplicates += response.duplicate_count
+
+        return BatchResult(
+            results=all_results,
+            created_count=total_created,
+            duplicate_count=total_duplicates,
+        )
+
+    # -- queries ------------------------------------------------------------
+
+    async def get_by_id(self, task_id: str) -> Task:
+        if not task_id:
+            raise InvalidArgumentError("task_id is required")
+        request = queue_pb2.GetTaskRequest(queue=self.name, task_id=task_id)
+        response = await self._client._require_conn().call(
+            "GetTask", request, retry_safe=True
+        )
+        return Task.from_proto(response.task)
+
+    async def get_by_unique_key(self, unique_key: str) -> Task:
+        if not unique_key:
+            raise InvalidArgumentError("unique_key is required")
+        request = queue_pb2.GetTaskRequest(queue=self.name, unique_key=unique_key)
+        response = await self._client._require_conn().call(
+            "GetTask", request, retry_safe=True
+        )
+        return Task.from_proto(response.task)
+
+    async def get_attempts(
+        self,
+        unique_key: str,
+        *,
+        limit: int = _ATTEMPT_PAGE_DEFAULT_LIMIT,
+        cursor: str | None = None,
+    ) -> AttemptPage:
+        """Read oldest-first history by business key, not the bounded Task preview.
+
+        Cursors retain the first page's history high-water mark. Pages can be
+        shorter than ``limit`` due to the byte budget; follow ``next_cursor``
+        until None. Restart without a cursor to see later appended attempts.
+        """
+        return await self._get_attempts("unique_key", unique_key, limit, cursor)
+
+    async def get_attempts_by_id(
+        self,
+        task_id: str,
+        *,
+        limit: int = _ATTEMPT_PAGE_DEFAULT_LIMIT,
+        cursor: str | None = None,
+    ) -> AttemptPage:
+        """Read a bounded durable history page by explicit server task ID."""
+        return await self._get_attempts("task_id", task_id, limit, cursor)
+
+    async def _get_attempts(
+        self, selector: str, key: str, limit: int, cursor: str | None
+    ) -> AttemptPage:
+        if not isinstance(key, str) or not key:
+            raise InvalidArgumentError(f"{selector} is required")
+        if (
+            isinstance(limit, bool)
+            or not isinstance(limit, int)
+            or not 1 <= limit <= _ATTEMPT_PAGE_MAX_LIMIT
+        ):
+            raise InvalidArgumentError("limit must be an integer between 1 and 100")
+        if cursor is not None:
+            if not isinstance(cursor, str) or not cursor:
+                raise InvalidArgumentError("cursor must be a nonempty string or None")
+            try:
+                cursor_bytes = len(cursor.encode("utf-8"))
+            except UnicodeEncodeError as exc:
+                raise InvalidArgumentError("cursor must be valid UTF-8") from exc
+            if cursor_bytes > _MAX_ATTEMPT_CURSOR_BYTES:
+                raise InvalidArgumentError("cursor must be at most 1024 UTF-8 bytes")
+        conn = self._client._require_conn()
+        if conn.protocol_minor < _ATTEMPT_HISTORY_MIN_PROTOCOL_MINOR:
+            raise IncompatibleVersionError(
+                "attempt history requires protocol minor >= 5 (GetTaskAttempts);"
+                f" negotiated minor is {conn.protocol_minor}"
+            )
+        request = queue_pb2.GetTaskAttemptsRequest(
+            queue=self.name, limit=limit, cursor=cursor or ""
+        )
+        setattr(request, selector, key)
+        response = await conn.call("GetTaskAttempts", request, retry_safe=True)
+        return AttemptPage.from_proto(response)
+
+    async def stats(self, task_types: Iterable[str] | None = None) -> QueueStats:
+        """Read a point-in-time per-status count snapshot for this queue.
+
+        ``task_types`` of ``None`` or empty counts every task type; otherwise
+        only tasks matching any of the given types are included. All six
+        canonical statuses are always present (zero included); ``active`` on
+        the result is ``SCHEDULED + READY + LEASED``. This is a per-queue,
+        per-instance observation: the SDK never fabricates a cross-instance
+        aggregate or a time-window summary.
+
+        The read-only ``GetQueueStats`` RPC is additive protocol 1.4: when the
+        negotiated session minor is lower, this raises
+        :class:`IncompatibleVersionError` before any network activity.
+        ``NOT_FOUND`` (unknown queue) and ``UNAVAILABLE`` propagate unchanged.
+        More than 100 ``task_types`` is an :class:`InvalidArgumentError` raised
+        before any RPC; at most 101 elements are consumed from the iterable.
+        """
+        selected: list[str] | None = None
+        if task_types is not None:
+            # islice caps consumption so an unbounded iterable stops at 101.
+            selected = list(islice(task_types, _STATS_MAX_TASK_TYPES + 1))
+            if len(selected) > _STATS_MAX_TASK_TYPES:
+                raise InvalidArgumentError(
+                    f"task_types must contain at most {_STATS_MAX_TASK_TYPES}"
+                    f" values; got more than {_STATS_MAX_TASK_TYPES}"
+                )
+        conn = self._client._require_conn()
+        if conn.protocol_minor < _STATS_MIN_PROTOCOL_MINOR:
+            raise IncompatibleVersionError(
+                f"queue stats requires protocol minor >= {_STATS_MIN_PROTOCOL_MINOR}"
+                f" (GetQueueStats); negotiated minor is {conn.protocol_minor}"
+            )
+        request = queue_pb2.GetQueueStatsRequest(
+            queue=self.name, task_types=selected or []
+        )
+        response = await conn.call("GetQueueStats", request, retry_safe=True)
+        return QueueStats.from_proto(response)
+
+    async def retry(self, unique_key: str) -> Task:
+        """Reactivate a SUCCEEDED/DEAD task by its business unique key.
+
+        This always resolves through the proto ``unique_key`` field; use
+        :meth:`retry_by_id` for the task-ID form. No string is ever guessed.
+        """
+        if not unique_key:
+            raise InvalidArgumentError("unique_key is required")
+        request = queue_pb2.RetryTaskRequest(queue=self.name)
+        request.unique_key = unique_key
+        response = await self._client._require_conn().call("RetryTask", request)
+        return Task.from_proto(response.task)
+
+    async def retry_by_id(self, task_id: str) -> Task:
+        """Reactivate a SUCCEEDED/DEAD task by its server task ID."""
+        if not task_id:
+            raise InvalidArgumentError("task_id is required")
+        request = queue_pb2.RetryTaskRequest(queue=self.name)
+        request.task_id = task_id
+        response = await self._client._require_conn().call("RetryTask", request)
+        return Task.from_proto(response.task)
+
+    # -- manual claim ---------------------------------------------------------
+
+    async def claim(
+        self,
+        limit: int = 1,
+        *,
+        task_types: Iterable[str] | None = None,
+        wait_timeout: timedelta | None = None,
+        lease_duration: timedelta | None = None,
+        worker_id: str | None = None,
+    ) -> list["LeasedTask"]:
+        """Manually claim tasks (advanced API; not the default worker path)."""
+        if limit < 1 or limit > _MAX_ENQUEUE_BATCH:
+            raise InvalidArgumentError(
+                f"limit must be between 1 and {_MAX_ENQUEUE_BATCH}"
+            )
+        if wait_timeout is not None and wait_timeout < timedelta(0):
+            raise InvalidArgumentError("wait_timeout must not be negative")
+        if lease_duration is not None and lease_duration < timedelta(0):
+            raise InvalidArgumentError("lease_duration must not be negative")
+        request = queue_pb2.ClaimRequest(
+            queue=self.name,
+            worker_id=worker_id or new_worker_id(),
+            task_types=list(task_types or []),
+            limit=limit,
+        )
+        if wait_timeout is not None:
+            request.wait_timeout.CopyFrom(_pb.td_to_dur(wait_timeout))
+        if lease_duration is not None:
+            request.lease_duration.CopyFrom(_pb.td_to_dur(lease_duration))
+        response = await self._client._require_conn().call("Claim", request)
+        return [LeasedTask(self, t) for t in response.tasks]
+
+    # -- worker ---------------------------------------------------------------
+
+    def worker(
+        self,
+        handler: Handler,
+        *,
+        task_types: Iterable[str] | None = None,
+        concurrency: int | ResourceConcurrency | AdaptiveConcurrency = 1,
+        weight: int = 1,
+        lease_duration: timedelta = timedelta(seconds=60),
+        grace_period: timedelta = timedelta(seconds=30),
+        on_idle: Callable[[], Any] | None = None,
+        task_timeouts: TaskTimeouts | None = None,
+    ) -> "Worker":
+        """Create a credit-controlled worker over the frozen Work stream.
+
+        ``on_idle`` is an optional zero-argument callable (sync or awaitable)
+        for bounded batch runs: once the worker has registered and sent its
+        initial credit, the SDK polls queue activity every 250 ms via the
+        read-only ``GetQueueActivity`` RPC scoped to this worker's
+        ``task_types``; after the activity count has been continuously zero
+        for at least 1 s the callback fires exactly once, inline on the
+        ``run()`` coroutine. A callback that returns normally leaves the
+        worker running (and never fires again); any exception it raises —
+        including ``SystemExit`` — propagates verbatim out of
+        ``await worker.run()`` without becoming a task NACK. Workers without
+        ``on_idle`` never send activity RPCs and stay compatible with
+        protocol minor 0 servers; ``on_idle`` requires minor >= 1 and raises
+        :class:`IncompatibleVersionError` from ``run()`` otherwise. Never
+        call ``worker.close()`` from inside the callback (it would wait on
+        the very coroutine executing it).
+        """
+        from .worker import Worker
+
+        if not callable(handler):
+            raise InvalidArgumentError("handler must be callable")
+        if on_idle is not None and not callable(on_idle):
+            raise InvalidArgumentError("on_idle must be callable")
+        if isinstance(concurrency, bool) or not isinstance(
+            concurrency, (int, ResourceConcurrency, AdaptiveConcurrency)
+        ):
+            raise InvalidArgumentError(
+                "concurrency must be an integer, ResourceConcurrency, or AdaptiveConcurrency"
+            )
+        if isinstance(concurrency, int) and (
+            concurrency < 1 or concurrency > 2**31 - 1
+        ):
+            raise InvalidArgumentError(
+                "initial fixed concurrency must be between 1 and 2**31-1"
+            )
+        if isinstance(weight, bool) or not isinstance(weight, int) or weight < 1:
+            raise InvalidArgumentError("weight must be a positive integer")
+        if lease_duration < timedelta(0):
+            raise InvalidArgumentError("lease_duration must not be negative")
+        if grace_period < timedelta(0):
+            raise InvalidArgumentError("grace_period must not be negative")
+        if task_timeouts is not None and not isinstance(task_timeouts, TaskTimeouts):
+            raise InvalidArgumentError("task_timeouts must be TaskTimeouts or None")
+        worker = Worker(
+            self,
+            handler,
+            task_types=list(task_types or []),
+            concurrency=concurrency,
+            lease_duration=lease_duration,
+            grace_period=grace_period,
+            on_idle=on_idle,
+            weight=weight,
+            task_timeouts=task_timeouts,
+        )
+        self._client._register_worker(worker)
+        return worker
+
+
+# ---------------------------------------------------------------------------
+# Manual lease
+
+
+class LeasedTask:
+    """A task claimed via :meth:`TaskQueue.claim` with its current lease."""
+
+    def __init__(self, queue: TaskQueue, msg: queue_pb2.LeasedTask) -> None:
+        self._queue = queue
+        self.task = Task.from_proto(msg.task)
+        self.attempt: int = msg.lease.attempt
+        self.worker_id: str = msg.lease.worker_id
+        self._token: str = msg.lease.lease_token
+        self._lease_until = _pb.ts_to_dt(msg.lease.until)
+
+    @property
+    def lease_until(self):
+        return self._lease_until
+
+    def _ref(self) -> queue_pb2.LeaseRef:
+        return queue_pb2.LeaseRef(
+            task_id=self.task.id,
+            attempt=self.attempt,
+            lease_token=self._token,
+        )
+
+    async def _terminal(self, name: str, request: Any) -> None:
+        request.lease.CopyFrom(self._ref())
+        conn = self._queue._client._require_conn()
+        await conn.call(name, request)
+
+    async def ack(self) -> None:
+        """Acknowledge success for the current attempt/lease token."""
+        await self._terminal("Ack", queue_pb2.AckRequest())
+
+    async def nack(self, error: str = "", *, delay: timedelta | None = None) -> None:
+        """Negative-acknowledge; the task is retried per policy."""
+        request = queue_pb2.NackRequest(error=error)
+        if delay is not None:
+            request.retry_delay_override.CopyFrom(_pb.td_to_dur(delay))
+        await self._terminal("Nack", request)
+
+    async def reject(self, error: str = "") -> None:
+        """Permanently fail the task (DEAD) for the current attempt."""
+        await self._terminal("Reject", queue_pb2.RejectRequest(error=error))
+
+    async def extend_lease(self, extension: timedelta) -> None:
+        """Extend the lease to ``server_now + extension`` (absolute reset)."""
+        if extension <= timedelta(0):
+            raise InvalidArgumentError("extension must be positive")
+        request = queue_pb2.ExtendLeaseRequest()
+        request.extension.CopyFrom(_pb.td_to_dur(extension))
+        request.lease.CopyFrom(self._ref())
+        conn = self._queue._client._require_conn()
+        response = await conn.call("ExtendLease", request)
+        self._lease_until = _pb.ts_to_dt(response.lease.until)