ringo-task-queue 0.1.0.dev7__py3-none-win_amd64.whl → 0.1.0.dev9__py3-none-win_amd64.whl

ringo_task_queue/worker.py

@@ -9,15 +9,17 @@ from __future__ import annotations
 import asyncio
 import inspect
 import math
+import sys
 from collections import deque
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from datetime import datetime, timedelta, timezone
-from typing import Any, Awaitable, Callable, Iterable
+from typing import TYPE_CHECKING, Any, Awaitable, Callable, Iterable
 
 import grpc
 
 from . import _pb
 from ._proto.ringo.v1 import queue_pb2
+from ._supervisor_policy import TaskTimeouts
 from .concurrency import AdaptiveConcurrency, ResourceConcurrency, _Member
 from .errors import (
     IncompatibleVersionError,
@@ -32,6 +34,16 @@ from .errors import (
 )
 from .models import Task
 
+if TYPE_CHECKING:
+    from ._supervisor_watchdog import _WorkerWatchdog
+
+
+def _current_runtime() -> Any:
+    # A normal SDK install does not import the process-supervision runtime.
+    module = sys.modules.get("ringo_task_queue._supervisor_runtime")
+    return None if module is None else module.current_runtime()
+
+
 _MAX_ERROR_BYTES = 256 * 1024
 
 #: Fixed on_idle monitor cadence (design §10.6). These are deliberately not
@@ -41,9 +53,15 @@ _IDLE_POLL_SECONDS = 0.25
 _IDLE_QUIET_SECONDS = 1.0
 #: GetQueueActivity was introduced in protocol minor 1.
 _IDLE_MIN_PROTOCOL_MINOR = 1
+_ORDERED_REQUEST_IDS_MIN_PROTOCOL_MINOR = 6
+_MAX_REQUEST_ID = 2**64 - 1
+
+
+class _WatchdogFenced(Exception):
+    """A stopped deadline invocation abandoned during graceful drain."""
 
 
-def _consume_task_exception(task: asyncio.Task[Any]) -> None:
+def _consume_task_exception(task: asyncio.Future[Any]) -> None:
     if not task.cancelled():
         task.exception()
 
@@ -125,12 +143,19 @@ class TaskContext:
     async def wait_cancelled(self) -> None:
         await self._cancel_event.wait()
 
+    def mark_progress(self) -> None:
+        """Mark local business progress without sending any RPC."""
+        watchdog = self._worker._watchdog
+        if watchdog is not None:
+            watchdog.mark_progress(self._assignment)
+
     async def report_progress(
         self, current: float, total: float, message: str = ""
     ) -> None:
-        """Report progress for this attempt and await its correlated result."""
+        """Mark local progress, report it and await its correlated result."""
         if self.cancelled:
             raise LeaseLostError("task lease is no longer valid")
+        self.mark_progress()
         await self._worker._report_progress(self._assignment, current, total, message)
 
     def _cancel(self) -> None:
@@ -156,6 +181,8 @@ class _Assignment:
     started: bool = False
     session_epoch: int = 0
     session: asyncio.Event | None = None
+    renew_deadline: float | None = None
+    watchdog_expired: bool = False
 
     @property
     def identity(self) -> tuple[str, int, str]:
@@ -170,6 +197,39 @@ class _Assignment:
         )
 
 
+@dataclass
+class _WorkSource:
+    """One Work call owns its correlation and transport lifetime.
+
+    Reader/writer tasks keep this object even after a replacement becomes the
+    current source. No receipt, EOF or timeout from an old call can touch the
+    replacement's pending Futures, registration or transport.
+    """
+
+    epoch: int
+    call: Any
+    minor: int = 0
+    token: str = ""
+    ordered_request_ids: bool = False
+    done: asyncio.Event = field(default_factory=asyncio.Event)
+    registered: asyncio.Event = field(default_factory=asyncio.Event)
+    outbox: asyncio.Queue[queue_pb2.WorkRequest | None] = field(
+        default_factory=asyncio.Queue
+    )
+    pending: dict[int, asyncio.Future[queue_pb2.WorkResult]] = field(
+        default_factory=dict
+    )
+    next_request_id: int = 1
+    error: BaseException | None = None
+    reader: asyncio.Task[Any] | None = None
+    writer: asyncio.Task[Any] | None = None
+    publisher: asyncio.Task[None] | None = None
+    cleanup: asyncio.Task[None] | None = None
+    # Diagnostics only: accepted credit never participates in admission itself.
+    accepted_credit: int = 0
+    credit_generation: int = -1
+
+
 class Worker:
     """A credit-controlled asynchronous task worker.
 
@@ -190,6 +250,7 @@ class Worker:
         grace_period: timedelta,
         on_idle: Callable[[], Any] | None = None,
         weight: int = 1,
+        task_timeouts: TaskTimeouts | None = None,
     ) -> None:
         if isinstance(concurrency, bool) or not isinstance(
             concurrency, (int, ResourceConcurrency, AdaptiveConcurrency)
@@ -214,7 +275,6 @@ class Worker:
             if isinstance(concurrency, int)
             else max(1, concurrency.capacity)
         )
-        self._configured_concurrency = self._concurrency
         self._weight = weight
         self._lease_duration = lease_duration
         self._grace_period = max(0.0, grace_period.total_seconds())
@@ -246,15 +306,10 @@ class Worker:
         self._command_timeout = 30.0
         self._close_lock = asyncio.Lock()
         self._close_task: asyncio.Task[None] | None = None
-        self._session_done: asyncio.Event | None = None
-        self._call: Any = None
-        self._outbox: asyncio.Queue[Any] | None = None
-        self._reader_task: asyncio.Task[Any] | None = None
-        self._writer_task: asyncio.Task[Any] | None = None
-        self._registered: asyncio.Event | None = None
-        self._stream_error: BaseException | None = None
-        self._next_request_id = 1
-        self._pending: dict[int, asyncio.Future[queue_pb2.WorkResult]] = {}
+        self._source: _WorkSource | None = None
+        # Recovery history is distinct from a live source's error. A clean EOF
+        # remains clean, while exhausted recovery can report its earlier cause.
+        self._last_stream_error: BaseException | None = None
         # A successor lease can arrive before the predecessor's NACK receipt
         # is consumed. Only an identical complete LeaseRef is a duplicate.
         self._inflight: dict[tuple[str, int, str], _Assignment] = {}
@@ -271,13 +326,32 @@ class Worker:
         self._publisher_lock = asyncio.Lock()
         self._session_epoch = 0
         self._publisher_closed = False
-        self._probe: tuple[_Member, int] | None = None
-        self._detached = False
-        self._controller_registered = False
+        self._fixed_capacity_generation = 0
+        self._fixed_publication_task: asyncio.Task[None] | None = None
+        self._fixed_publication_source: tuple[int, asyncio.Event] | None = None
+        self._fixed_publication_waiters: list[tuple[int, asyncio.Future[None]]] = []
         self._fixed_dynamic_target = False
+        if task_timeouts is not None and not isinstance(task_timeouts, TaskTimeouts):
+            raise ValueError("task_timeouts must be TaskTimeouts or None")
+        self._runtime = _current_runtime()
+        self._task_timeouts = (
+            self._runtime.task_timeouts
+            if task_timeouts is None and self._runtime is not None
+            else task_timeouts
+        )
+        self._watchdog: _WorkerWatchdog | None = None
+        self._watchdog_error: RuntimeError | None = None
+        if (
+            self._runtime is not None
+            and self._runtime.policy.no_start_timeout is not None
+        ) or (self._task_timeouts is not None and self._task_timeouts.enabled):
+            from ._supervisor_watchdog import _WorkerWatchdog
+
+            self._watchdog = _WorkerWatchdog(
+                self, self._task_timeouts or TaskTimeouts(), self._runtime
+            )
         if self._controller is not None:
             self._member = self._controller._attach(self, weight)
-            self._controller_registered = True
 
     @staticmethod
     def _new_worker_id() -> str:
@@ -286,6 +360,42 @@ class Worker:
 
         return new_worker_id()
 
+    # Read-only private views keep diagnostics and existing benchmark probes
+    # useful without maintaining a second set of mutable transport state.
+    @property
+    def _session_done(self) -> asyncio.Event | None:
+        return self._source.done if self._source is not None else None
+
+    @property
+    def _call(self) -> Any:
+        return self._source.call if self._source is not None else None
+
+    @property
+    def _outbox(self) -> asyncio.Queue[queue_pb2.WorkRequest | None] | None:
+        return self._source.outbox if self._source is not None else None
+
+    @property
+    def _pending(self) -> dict[int, asyncio.Future[queue_pb2.WorkResult]]:
+        return self._source.pending if self._source is not None else {}
+
+    @property
+    def _registered(self) -> asyncio.Event | None:
+        return self._source.registered if self._source is not None else None
+
+    @property
+    def _reader_task(self) -> asyncio.Task[Any] | None:
+        return self._source.reader if self._source is not None else None
+
+    @property
+    def _writer_task(self) -> asyncio.Task[Any] | None:
+        return self._source.writer if self._source is not None else None
+
+    @property
+    def _stream_error(self) -> BaseException | None:
+        return (
+            self._source.error if self._source is not None else self._last_stream_error
+        )
+
     @property
     def worker_id(self) -> str:
         return self._worker_id
@@ -331,6 +441,10 @@ class Worker:
         self._run_started = True
         failures = 0
         try:
+            if self._runtime is not None:
+                self._runtime.register_worker(self)
+            if self._watchdog is not None:
+                self._watchdog.start()
             conn = self._client._require_conn()
             if (
                 self._controller is not None or self._fixed_dynamic_target
@@ -367,7 +481,7 @@ class Worker:
                         # downgrade that makes on_idle unservable; surface
                         # it instead of burning the reconnect budget.
                         raise
-                    self._stream_error = exc
+                    self._last_stream_error = exc
                     failures += 1
                     if self._close_started.is_set():
                         break
@@ -389,11 +503,19 @@ class Worker:
                     )
                 if not self._close_started.is_set():
                     await asyncio.sleep(0)
+            if self._watchdog_error is not None:
+                raise self._watchdog_error
         finally:
             try:
                 await self._cancel_assignments()
             finally:
-                self._run_finished.set()
+                try:
+                    if self._watchdog is not None:
+                        await self._watchdog.stop()
+                finally:
+                    if self._runtime is not None:
+                        self._runtime.unregister_worker(self)
+                    self._run_finished.set()
 
     async def close(self) -> None:
         """Drain and stop once; caller cancellation cannot cancel cleanup."""
@@ -417,7 +539,6 @@ class Worker:
         try:
             if self._controller is not None and self._member is not None:
                 await self._controller._detach(self._member)
-                self._detached = True
             self._publisher_closed = True
             # Drain keeps this Work session alive for handler ACK/renewal.
             # Close is not a session replacement: changing its epoch here
@@ -430,10 +551,10 @@ class Worker:
                     await asyncio.wait_for(
                         self._send_command(
                             queue_pb2.WorkRequest(
-                                request_id=self._allocate_request_id(),
                                 drain=queue_pb2.WorkerDrain(begin=True),
                             ),
-                            wait=True,
+                            expected_epoch=self._session_epoch,
+                            expected_session=session,
                         ),
                         timeout=max(
                             0.1,
@@ -690,9 +811,36 @@ class Worker:
         idle candidate) under the fresh token. It is neither a callback
         failure nor a reconnect-worthy stream error.
         """
+        source = self._source
+        if source is None or source.done is not session:
+            return
         self._idle_candidate = False
         self._idle_zero_since = None
-        session.set()
+        self._end_source(source)
+
+    def _end_source(
+        self, source: _WorkSource, error: BaseException | None = None
+    ) -> None:
+        """Synchronously end exactly one source before any cleanup wait.
+
+        Failing pending Futures here replaces per-command session-wait tasks.
+        A graceful Worker.close merely drains; it calls this only when the
+        transport actually ends or the grace period has expired.
+        """
+        if error is not None and source.error is None:
+            source.error = error
+            if self._source is source:
+                self._last_stream_error = error
+        source.done.set()
+        failure = source.error or UnavailableError("Work stream closed")
+        for future in source.pending.values():
+            if not future.done():
+                future.set_exception(failure)
+        source.pending.clear()
+        try:
+            source.call.cancel()
+        except Exception:
+            pass
 
     async def _run_session(self) -> None:
         conn = self._client._require_conn()
@@ -721,27 +869,48 @@ class Worker:
         async with conn._lock:
             stub = conn._require_stub()
             reg.context.CopyFrom(conn.request_context())
+            # Mode belongs to this registration snapshot, not to a later unary
+            # negotiation that may rotate the connection's token and minor.
+            minor = reg.context.protocol_minor
+            if (
+                self._controller is not None or self._fixed_dynamic_target
+            ) and minor < 2:
+                raise IncompatibleVersionError(
+                    "dynamic concurrency requires protocol minor >= 2; "
+                    f"negotiated minor is {minor}"
+                )
+            if self._on_idle is not None and minor < _IDLE_MIN_PROTOCOL_MINOR:
+                raise IncompatibleVersionError(
+                    "on_idle requires a server speaking protocol minor"
+                    f" >= {_IDLE_MIN_PROTOCOL_MINOR}; negotiated minor is {minor}"
+                )
+            ordered = minor >= _ORDERED_REQUEST_IDS_MIN_PROTOCOL_MINOR
+            if ordered:
+                reg.ordered_request_ids = True
             call = stub.Work(metadata=conn._metadata())
-        self._call = call
-        self._session_done = asyncio.Event()
-        self._registered = asyncio.Event()
-        self._stream_error = None
-        self._outbox = asyncio.Queue()
-        self._pending.clear()
-        self._writer_task = asyncio.create_task(
-            self._write_loop(call), name="ringo-worker-writer"
+        source = _WorkSource(
+            self._session_epoch,
+            call,
+            minor=minor,
+            token=reg.context.session_token,
+            ordered_request_ids=ordered,
         )
-        self._reader_task = asyncio.create_task(
-            self._read_loop(call), name="ringo-worker-reader"
+        self._source = source
+        self._last_stream_error = None
+        source.writer = asyncio.create_task(
+            self._write_loop(source), name="ringo-worker-writer"
+        )
+        source.reader = asyncio.create_task(
+            self._read_loop(source), name="ringo-worker-reader"
         )
         try:
-            await self._enqueue_frame(queue_pb2.WorkRequest(register=reg))
-            await self._wait_event_or_session(self._registered)
+            self._publish_frame(source, queue_pb2.WorkRequest(register=reg))
+            await self._wait_event_or_session(source.registered, source)
             if self._force_cancel.is_set():
                 return
             # Initial credit is sent only after WorkerRegistered was observed.
             if self._controller is None:
-                await self._send_credit(self._concurrency)
+                await self._publish_fixed_capacity()
             else:
                 self._controller._check_loop()
                 await self._controller._start_member(self._member)
@@ -751,108 +920,111 @@ class Worker:
                 # is bound to the token this stream was registered with: a
                 # later unary renegotiation that rotates the token disarms
                 # the monitor until the next session.
-                self._idle_session_token = reg.context.session_token
+                self._idle_session_token = source.token
                 self._idle_candidate = True
                 self._idle_zero_since = None
-            await self._wait_event_or_session(self._session_done)
-            if self._stream_error is not None and not self._close_started.is_set():
-                raise self._stream_error
+            await source.done.wait()
+            if source.error is not None and not self._close_started.is_set():
+                raise source.error
         finally:
-            await self._stop_session()
-            self._call = None
-            self._session_done = None
-            self._registered = None
+            await self._stop_session(source)
+            if self._source is source:
+                self._source = None
 
-    async def _wait_event_or_session(self, event: asyncio.Event) -> None:
-        if self._session_done is None:
-            return
+    async def _wait_event_or_session(
+        self, event: asyncio.Event, source: _WorkSource
+    ) -> None:
+        # Registration is a cold-path Event race. Commands await only their
+        # private Future; every source-end path fails those Futures directly.
         event_task = asyncio.create_task(event.wait())
-        done_task = asyncio.create_task(self._session_done.wait())
-        done, pending = await asyncio.wait(
-            (event_task, done_task), return_when=asyncio.FIRST_COMPLETED
-        )
-        for task in pending:
-            task.cancel()
-        await asyncio.gather(*pending, return_exceptions=True)
-        if done_task in done and not event.is_set():
-            if self._stream_error is not None:
-                raise self._stream_error
-            raise UnavailableError("Work stream closed")
+        done_task = asyncio.create_task(source.done.wait())
+        try:
+            done, _ = await asyncio.wait(
+                (event_task, done_task), return_when=asyncio.FIRST_COMPLETED
+            )
+            if done_task in done and not event.is_set():
+                raise source.error or UnavailableError("Work stream closed")
+        finally:
+            for task in (event_task, done_task):
+                if not task.done():
+                    task.cancel()
+            await asyncio.gather(event_task, done_task, return_exceptions=True)
+
+    async def _stop_session(self, source: _WorkSource | None = None) -> None:
+        source = self._source if source is None else source
+        if source is None:
+            return
+        # Fail command Futures before any lock/join can block teardown. One
+        # shielded owner survives repeated caller cancellation and concurrent
+        # close/run cleanup, including cancellation before its first step.
+        self._end_source(source)
+        if source.cleanup is None:
+            task = asyncio.create_task(
+                self._stop_source(source),
+                name=f"ringo-stop-{self._worker_id}-{source.epoch}",
+            )
+            source.cleanup = task
+            self._capacity_tasks.add(task)
+            task.add_done_callback(self._capacity_tasks.discard)
+            task.add_done_callback(_consume_task_exception)
+        await asyncio.shield(source.cleanup)
 
-    async def _stop_session(self) -> None:
-        session = self._session_done
-        if session is not None:
-            session.set()
-        call, self._call = self._call, None
-        writer, self._writer_task = self._writer_task, None
-        reader, self._reader_task = self._reader_task, None
-        outbox, self._outbox = self._outbox, None
-        if outbox is not None and writer is not None and not writer.done():
-            await outbox.put(None)
-        if call is not None:
-            try:
-                call.cancel()
-            except Exception:
-                pass
-        # Unblock publishers before waiting for a reader that may itself be
-        # awaiting a controller publication boundary. Cancel Python tasks as
-        # well as the transport; call.cancel alone cannot interrupt that await.
-        for future in self._pending.values():
-            if not future.done():
-                future.set_exception(UnavailableError("Work stream closed"))
-        self._pending.clear()
+    async def _stop_source(self, source: _WorkSource) -> None:
+        source.outbox.put_nowait(None)
         tasks = [
-            t
-            for t in (reader, writer)
-            if t is not None and t is not asyncio.current_task()
+            task
+            for task in (source.reader, source.writer, source.publisher)
+            if task is not None and task is not asyncio.current_task()
         ]
         for task in tasks:
             task.cancel()
         controller, member = self._controller, self._member
-        if controller is not None and member is not None:
-            # Schedule controller fencing before a reader join can be cancelled.
-            # It owns only stream/probe liveness, never a still-running handler's
-            # permit, and must survive a second cancellation of run() cleanup.
-            lost = asyncio.create_task(controller._stream_lost(member))
-            self._capacity_tasks.add(lost)
-            lost.add_done_callback(self._capacity_tasks.discard)
-            lost.add_done_callback(_consume_task_exception)
-            await asyncio.shield(lost)
+        if controller is not None and member is not None and self._source is source:
+            # The run loop joins this cleanup before admitting its replacement.
+            # Only unused old-stream credit is reclaimed; running handlers keep
+            # their permits until their real coroutine settles.
+            await controller._stream_lost(member)
         if tasks:
             await asyncio.gather(*tasks, return_exceptions=True)
 
-    async def _enqueue_frame(self, frame: queue_pb2.WorkRequest) -> None:
-        outbox = self._outbox
-        if outbox is None:
-            raise UnavailableError("Work stream is not connected")
-        await outbox.put(frame)
+    def _publish_frame(self, source: _WorkSource, frame: queue_pb2.WorkRequest) -> None:
+        """Synchronous FIFO insertion; callers own validation and correlation."""
+        source.outbox.put_nowait(frame)
 
-    async def _write_loop(self, call: Any) -> None:
+    async def _write_loop(self, source: _WorkSource) -> None:
         try:
-            while True:
-                frame = await self._outbox.get() if self._outbox is not None else None
-                if frame is None:
+            while not source.done.is_set():
+                frame = await source.outbox.get()
+                if frame is None or source.done.is_set():
                     try:
-                        done_writing = getattr(call, "done_writing", None)
+                        done_writing = getattr(source.call, "done_writing", None)
                         if done_writing is not None:
                             await done_writing()
                     except Exception:
                         pass
                     return
-                await call.write(frame)
+                await source.call.write(frame)
         except asyncio.CancelledError:
             raise
         except Exception as exc:
-            self._stream_error = self._map_transport(exc)
-            if self._session_done is not None:
-                self._session_done.set()
+            self._end_source(source, self._map_transport(exc))
+        finally:
+            self._end_source(source)
 
-    async def _read_loop(self, call: Any) -> None:
+    async def _read_loop(self, source: _WorkSource) -> None:
         try:
-            async for response in call:
+            async for response in source.call:
+                if source.done.is_set():
+                    return
                 body = response.WhichOneof("body")
                 if body == "registered":
+                    if self._source is not source:
+                        continue
                     registered = response.registered
+                    if registered.ordered_request_ids != source.ordered_request_ids:
+                        raise IncompatibleVersionError(
+                            "Work registration did not confirm the requested ID mode"
+                        )
                     if registered.HasField("lease_duration"):
                         self._lease_duration = (
                             _pb.dur_to_td(registered.lease_duration)
@@ -864,23 +1036,22 @@ class Worker:
                         and registered.max_concurrency
                     ):
                         self._concurrency = registered.max_concurrency
-                    if self._registered is not None:
-                        self._registered.set()
+                    source.registered.set()
                 elif body == "task":
-                    await self._on_assignment(response.task)
+                    await self._on_assignment(response.task, source=source)
                 elif body == "result":
                     result = response.result
-                    future = self._pending.pop(result.request_id, None)
+                    future = source.pending.pop(result.request_id, None)
                     if future is not None and not future.done():
                         future.set_result(result)
-                # Heartbeats are unsolicited liveness frames and need no action.
+                # Unknown/late results and heartbeats keep their existing
+                # non-fatal behavior; they never correlate into another source.
         except asyncio.CancelledError:
             raise
         except Exception as exc:
-            self._stream_error = self._map_transport(exc)
+            self._end_source(source, self._map_transport(exc))
         finally:
-            if self._session_done is not None:
-                self._session_done.set()
+            self._end_source(source)
 
     @staticmethod
     def _map_transport(exc: BaseException) -> BaseException:
@@ -890,9 +1061,15 @@ class Worker:
             return map_rpc_error(exc)
         return UnavailableError(f"Work stream failed: {_safe_error(exc)}")
 
-    async def _on_assignment(self, leased: queue_pb2.LeasedTask) -> None:
+    async def _on_assignment(
+        self, leased: queue_pb2.LeasedTask, *, source: _WorkSource | None = None
+    ) -> None:
+        source = self._source if source is None else source
         if (
-            self._force_cancel.is_set()
+            source is None
+            or self._source is not source
+            or source.done.is_set()
+            or self._force_cancel.is_set()
             or not leased.HasField("task")
             or not leased.HasField("lease")
         ):
@@ -904,6 +1081,8 @@ class Worker:
             attempt=leased.lease.attempt,
             token=leased.lease.lease_token,
             until=until,
+            renew_deadline=asyncio.get_running_loop().time()
+            + (until - datetime.now(timezone.utc)).total_seconds(),
         )
         context = TaskContext(
             self,
@@ -919,8 +1098,8 @@ class Worker:
         # a duplicate, and never overwrite either lease's cleanup ownership.
         if assignment.identity in self._inflight:
             return
-        assignment.session_epoch = self._session_epoch
-        assignment.session = self._session_done
+        assignment.session_epoch = source.epoch
+        assignment.session = source.done
         if self._controller is not None and self._member is not None:
             # Consuming the reserved probe is the linearization point at which
             # an assignment becomes a running controller permit.
@@ -929,6 +1108,15 @@ class Worker:
                 # remains server-owned and will naturally expire if this was a
                 # transport race.
                 return
+            if self._source is not source or source.done.is_set():
+                assignment.alive = False
+                context._cancel()
+                await self._release_controller_permit(assignment)
+                return
+        # A complete short task can arrive and ACK between two activity polls.
+        # Acceptance itself restarts quiet time; old/duplicate/rejected frames
+        # returned above must never reset the current source's observation.
+        self._idle_zero_since = None
         self._inflight[assignment.identity] = assignment
         assignment.renew_task = asyncio.create_task(
             self._renew_loop(assignment),
@@ -938,6 +1126,7 @@ class Worker:
             # Absolute credit updates can race assignments already in transit.
             # Retain and renew such leases, but gate actual handler entry locally.
             self._waiting_assignments.append(assignment)
+            self._fixed_capacity_generation += 1
             self._start_waiting_assignments()
         else:
             self._start_handler(assignment)
@@ -955,7 +1144,12 @@ class Worker:
             assignment = self._waiting_assignments.popleft()
             if not assignment.alive or assignment.session_epoch != self._session_epoch:
                 continue
-            if assignment.until <= datetime.now(timezone.utc):
+            expired = (
+                assignment.renew_deadline <= asyncio.get_running_loop().time()
+                if assignment.renew_deadline is not None
+                else assignment.until <= datetime.now(timezone.utc)
+            )
+            if expired:
                 self._fail_session(
                     UnavailableError("queued lease reached its local deadline"),
                     epoch=assignment.session_epoch,
@@ -981,6 +1175,21 @@ class Worker:
     def _handler_done(self, assignment: _Assignment, task: asyncio.Task[Any]) -> None:
         _consume_task_exception(task)
         self._handler_tasks.discard(task)
+        if task.cancelled() and assignment.alive:
+            # Task.cancel before its first step never enters the coroutine's
+            # cancellation handler. Stop that handlerless lease synchronously;
+            # the existing release owner still accounts for actual Task.done.
+            self._fail_session(
+                UnavailableError("task handler cancelled before settlement"),
+                epoch=assignment.session_epoch,
+                session=assignment.session,
+            )
+            self._invalidate_assignment(assignment)
+            if self._inflight.get(assignment.identity) is assignment:
+                self._inflight.pop(assignment.identity)
+            renewal = assignment.renew_task
+            if renewal is not None and not renewal.done():
+                renewal.cancel()
         self._schedule_capacity_release(assignment)
 
     def _schedule_capacity_release(self, assignment: _Assignment) -> None:
@@ -992,6 +1201,11 @@ class Worker:
     async def _release_finished_capacity(self, assignment: _Assignment) -> None:
         epoch, session = self._session_epoch, self._session_done
         try:
+            renewal = assignment.renew_task
+            if not assignment.alive and renewal is not None and not renewal.done():
+                # A done callback can cancel renewal before the handler ever
+                # ran. Join that SDK task without releasing another assignment.
+                await asyncio.gather(renewal, return_exceptions=True)
             await self._release_capacity_if_settled(assignment)
         except asyncio.CancelledError:
             raise
@@ -1007,27 +1221,43 @@ class Worker:
         epoch: int | None = None,
         session: asyncio.Event | None = None,
     ) -> None:
-        if epoch is not None and (
-            epoch != self._session_epoch or session is not self._session_done
-        ):
+        source = self._source
+        if source is None:
+            return
+        if epoch is not None and (epoch != source.epoch or session is not source.done):
             return
-        self._stream_error = error
-        if self._session_done is not None:
-            self._session_done.set()
-        if self._call is not None:
-            self._call.cancel()
+        self._end_source(source, error)
 
     async def _handle_assignment(self, assignment: _Assignment) -> None:
         try:
             context = assignment.context
             assert context is not None
-            if not assignment.alive or self._force_cancel.is_set():
+            if not assignment.alive:
                 return
-            result = self._handler(assignment.task, context)
-            if inspect.isawaitable(result):
-                await result
+            if self._force_cancel.is_set() or (
+                assignment.session is not None and assignment.session.is_set()
+            ):
+                # Reader bursts can schedule a handler and observe EOF before
+                # its first step. The immutable origin Event, not a replacement
+                # source or close_started, gates entry into application code.
+                await self._release_assignment(assignment, cancel_context=True)
+                return
+            # One disabled-path check; no monitor task, clock read or RPC for
+            # ordinary workers. The enabled helper enters immediately before
+            # the real handler call and exits before any terminal receipt.
+            watchdog = self._watchdog
+            if watchdog is None:
+                result = self._handler(assignment.task, context)
+                if inspect.isawaitable(result):
+                    await result
+            else:
+                await watchdog.invoke(assignment, context)
             if assignment.alive and not self._force_cancel.is_set():
                 await self._finish(assignment, "ack")
+        except _WatchdogFenced:
+            # A deadline racing graceful close abandons only its own stopped
+            # invocation; other handlers keep their live drain/ACK/renewal lane.
+            await self._release_assignment(assignment, cancel_context=True)
         except asyncio.CancelledError:
             # Cancellation is not a NACK, but an application-cancelled handler
             # must not retain a live, perpetually renewed lease and capacity.
@@ -1051,6 +1281,38 @@ class Worker:
         # Capacity belongs to the complete handler task, including receipt
         # consumption; its done callback releases the slot exactly once.
 
+    def _watchdog_unresponsive(self, assignment: _Assignment, reason: str) -> None:
+        """Fence before notifying: never NACK while old business still runs."""
+        self._invalidate_assignment(assignment)
+        renewal = assignment.renew_task
+        if renewal is not None:
+            renewal.cancel()
+        # Do not recycle local capacity; the real handler's done callback owns
+        # it even if run() returns while the application swallows cancellation.
+        self._close_started.set()
+        self._publisher_closed = True
+        source = self._source
+        if source is not None:
+            # This is an intentional Worker-wide hard fence, not a stale RPC
+            # result: even a replacement source must stop admitting new work.
+            self._end_source(
+                source, UnavailableError("unresponsive timed-out handler; Worker fenced")
+            )
+        detail = {
+            "worker_id": self._worker_id,
+            "queue": self._queue.name,
+            "task_id": assignment.task.id,
+            "attempt": assignment.attempt,
+            "reason": reason,
+        }
+        if self._runtime is not None:
+            self._runtime.request_restart("task cancellation grace exceeded", detail)
+        else:
+            self._watchdog_error = RuntimeError(
+                "task cancellation grace exceeded: Worker stopped accepting tasks; "
+                "a handler is still running; use process supervision for hard isolation"
+            )
+
     async def _finish(
         self,
         assignment: _Assignment,
@@ -1092,9 +1354,22 @@ class Worker:
                 reject=queue_pb2.RejectRequest(lease=assignment.ref, error=text)
             )
         try:
+            # Task.cancel() can be pending after a handler returns/throws without
+            # suspending. Queue.put() on the unbounded outbox is a fast path, so
+            # deliver cancellation before any terminal frame reaches the writer.
+            # Keep this inside our cancellation cleanup, including NACK/reject
+            # callers in handler except blocks. A consumed cancellation remains
+            # a valid normal result; its Task.cancelling() count is not a gate.
+            await asyncio.sleep(0)
+            if assignment.watchdog_expired and (
+                not assignment.alive
+                or self._close_started.is_set()
+                or self._force_cancel.is_set()
+            ):
+                await self._release_assignment(assignment, cancel_context=True)
+                return
             result = await self._send_command(
                 request,
-                wait=True,
                 expected_epoch=assignment.session_epoch,
                 expected_session=assignment.session,
             )
@@ -1117,14 +1392,6 @@ class Worker:
             )
             await self._release_assignment(assignment, cancel_context=True)
             return
-        if result is None:
-            self._fail_session(
-                UnavailableError("terminal command returned no receipt"),
-                epoch=assignment.session_epoch,
-                session=assignment.session,
-            )
-            await self._release_assignment(assignment, cancel_context=True)
-            return
         detail = result.error if result.HasField("error") else None
         if detail is not None and detail.code == queue_pb2.ERROR_CODE_LEASE_LOST:
             await self._lease_lost(assignment)
@@ -1191,8 +1458,14 @@ class Worker:
         await self._release_capacity_if_settled(assignment)
 
     async def _renew_loop(self, assignment: _Assignment) -> None:
+        loop = asyncio.get_running_loop()
+        if assignment.renew_deadline is None:
+            assignment.renew_deadline = (
+                loop.time()
+                + (assignment.until - datetime.now(timezone.utc)).total_seconds()
+            )
         while assignment.alive and not self._force_cancel.is_set():
-            remaining = (assignment.until - datetime.now(timezone.utc)).total_seconds()
+            remaining = assignment.renew_deadline - loop.time()
             delay = max(0.01, remaining - self._lease_duration.total_seconds() / 3)
             try:
                 await asyncio.wait_for(self._force_cancel.wait(), timeout=delay)
@@ -1207,10 +1480,11 @@ class Worker:
                     extension=_pb.td_to_dur(self._lease_duration),
                 )
             )
+            sent_at = loop.time()
+            sent_wall = datetime.now(timezone.utc)
             try:
                 result = await self._send_command(
                     request,
-                    wait=True,
                     expected_epoch=assignment.session_epoch,
                     expected_session=assignment.session,
                 )
@@ -1222,14 +1496,6 @@ class Worker:
                 )
                 await self._lease_lost(assignment)
                 return
-            if result is None:
-                self._fail_session(
-                    UnavailableError("lease renewal returned no receipt"),
-                    epoch=assignment.session_epoch,
-                    session=assignment.session,
-                )
-                await self._lease_lost(assignment)
-                return
             detail = result.error if result.HasField("error") else None
             if detail is not None and detail.code == queue_pb2.ERROR_CODE_LEASE_LOST:
                 await self._lease_lost(assignment)
@@ -1242,10 +1508,22 @@ class Worker:
                 )
                 await self._lease_lost(assignment)
                 return
-            # ExtendLeaseResponse is not carried in WorkResult. The protocol
-            # defines the new deadline as server_now + extension; this local
-            # estimate is the only safe deadline available to the SDK.
-            assignment.until = datetime.now(timezone.utc) + self._lease_duration
+            if (
+                not assignment.alive
+                or self._force_cancel.is_set()
+                or assignment.session_epoch != self._session_epoch
+                or assignment.session is not self._session_done
+                or assignment.session is None
+                or assignment.session.is_set()
+            ):
+                return
+            # WorkResult has no renewed deadline. Server processing cannot
+            # precede publication, so send-time + extension is a conservative
+            # lower bound, including queued writes/backpressure. Receipt time
+            # must not grant an extra lease. Monotonic scheduling survives wall
+            # clock changes; keep the private wall-clock field conservative too.
+            assignment.renew_deadline = sent_at + self._lease_duration.total_seconds()
+            assignment.until = sent_wall + self._lease_duration
 
     async def _report_progress(
         self, assignment: _Assignment, current: float, total: float, message: str
@@ -1262,12 +1540,9 @@ class Worker:
         )
         result = await self._send_command(
             request,
-            wait=True,
             expected_epoch=assignment.session_epoch,
             expected_session=assignment.session,
         )
-        if result is None:
-            return
         detail = result.error if result.HasField("error") else None
         if detail is not None and detail.code == queue_pb2.ERROR_CODE_LEASE_LOST:
             await self._lease_lost(assignment)
@@ -1275,53 +1550,142 @@ class Worker:
         if detail is not None and detail.code != queue_pb2.ERROR_CODE_UNSPECIFIED:
             raise error_from_detail(detail)
 
-    async def _send_credit(self, available: int) -> None:
-        # ``available`` is intentionally not trusted across retries.  A task
-        # may arrive between a rejected snapshot and its retry, so every
-        # credit-capacity retry samples local in-flight state again.
-        await self._publish_fixed_capacity()
-
     async def _publish_fixed_capacity(self) -> None:
         epoch, session = self._session_epoch, self._session_done
-        async with self._publisher_lock:
-            for attempt in range(20):
-                if (
-                    self._publisher_closed
-                    or self._close_started.is_set()
-                    or epoch != self._session_epoch
-                    or self._session_done is None
-                    or self._session_done.is_set()
-                ):
-                    return
-                available = max(
-                    0,
-                    self._concurrency
-                    - self._fixed_running
-                    - len(self._waiting_assignments),
-                )
-                update = queue_pb2.CreditUpdate(available=available)
-                if self._fixed_dynamic_target:
-                    update.target_concurrency = self._concurrency
-                result = await self._send_command(
-                    queue_pb2.WorkRequest(credit=update),
-                    wait=True,
-                    expected_epoch=epoch,
-                    expected_session=session,
-                )
-                if (
-                    result is None
-                    or not result.HasField("error")
-                    or result.error.code == queue_pb2.ERROR_CODE_UNSPECIFIED
-                ):
-                    return
-                if (
-                    result.error.metadata.get("reason") != "credit_capacity"
-                    or attempt == 19
-                ):
-                    error = error_from_detail(result.error)
-                    self._fail_session(error, epoch=epoch, session=session)
-                    raise error
-                await asyncio.sleep(0.01)
+        if (
+            self._publisher_closed
+            or self._close_started.is_set()
+            or session is None
+            or session.is_set()
+        ):
+            return
+        self._fixed_capacity_generation += 1
+        generation = self._fixed_capacity_generation
+        task = self._fixed_publication_task
+        new_publisher = (
+            task is None
+            or task.done()
+            or self._fixed_publication_source != (epoch, session)
+        )
+        if new_publisher:
+            self._fixed_publication_waiters = []
+            self._fixed_publication_source = (epoch, session)
+        waiters = self._fixed_publication_waiters
+        accepted: asyncio.Future[None] = asyncio.get_running_loop().create_future()
+        accepted.add_done_callback(_consume_task_exception)
+        waiters.append((generation, accepted))
+        if new_publisher:
+            task = asyncio.create_task(
+                self._drain_fixed_capacity(epoch, session, waiters),
+                name=f"ringo-credit-{self._worker_id}-{epoch}",
+            )
+            self._fixed_publication_task = task
+            assert self._source is not None
+            self._source.publisher = task
+            self._capacity_tasks.add(task)
+            task.add_done_callback(
+                lambda completed: self._fixed_publication_done(completed, waiters)
+            )
+        # A resize waits for its generation's accepted snapshot, not for the
+        # publisher to become idle: a continuously busy stream may never idle.
+        # Caller/grace cancellation cannot abandon the shared publication.
+        await asyncio.shield(accepted)
+
+    def _fixed_publication_done(
+        self,
+        task: asyncio.Task[None],
+        waiters: list[tuple[int, asyncio.Future[None]]],
+    ) -> None:
+        self._capacity_tasks.discard(task)
+        error = asyncio.CancelledError() if task.cancelled() else task.exception()
+        # Task.cancel() before the coroutine's first step never enters its
+        # finally. Complete this source's waiters even at that boundary.
+        for _, future in waiters:
+            if not future.done():
+                if error is None:
+                    future.set_result(None)
+                else:
+                    future.set_exception(error)
+        waiters.clear()
+
+    async def _drain_fixed_capacity(
+        self,
+        epoch: int,
+        session: asyncio.Event,
+        waiters: list[tuple[int, asyncio.Future[None]]],
+    ) -> None:
+        error: BaseException | None = None
+        try:
+            async with self._publisher_lock:
+                while True:
+                    for attempt in range(20):
+                        if (
+                            self._publisher_closed
+                            or self._close_started.is_set()
+                            or epoch != self._session_epoch
+                            or session is not self._session_done
+                            or session.is_set()
+                        ):
+                            return
+                        # Absolute credit is consumed by Assign frames. Coalesce
+                        # requests, never equal numeric values; an in-transit
+                        # assignment dirties the generation even if a completion
+                        # leaves occupancy unchanged. Retry re-samples all state.
+                        generation = self._fixed_capacity_generation
+                        available = max(
+                            0,
+                            self._concurrency
+                            - self._fixed_running
+                            - len(self._waiting_assignments),
+                        )
+                        update = queue_pb2.CreditUpdate(available=available)
+                        if self._fixed_dynamic_target:
+                            update.target_concurrency = self._concurrency
+                        result = await self._send_command(
+                            queue_pb2.WorkRequest(credit=update),
+                            expected_epoch=epoch,
+                            expected_session=session,
+                        )
+                        if (
+                            not result.HasField("error")
+                            or result.error.code == queue_pb2.ERROR_CODE_UNSPECIFIED
+                        ):
+                            source = self._source
+                            if source is not None and source.done is session:
+                                source.accepted_credit = available
+                                source.credit_generation = generation
+                            break
+                        if (
+                            result.error.metadata.get("reason") != "credit_capacity"
+                            or attempt == 19
+                        ):
+                            raise error_from_detail(result.error)
+                        await asyncio.sleep(0.01)
+                    remaining = []
+                    for requested, future in waiters:
+                        if requested <= generation:
+                            if not future.done():
+                                future.set_result(None)
+                        else:
+                            remaining.append((requested, future))
+                    waiters[:] = remaining
+                    if generation == self._fixed_capacity_generation:
+                        return
+        except BaseException as exc:
+            error = exc
+            if not isinstance(exc, asyncio.CancelledError):
+                self._fail_session(exc, epoch=epoch, session=session)
+            raise
+        finally:
+            # Session replacement/close releases only this publisher's waiters.
+            # A late old failure never completes or cancels a replacement batch.
+            for _, future in waiters:
+                if not future.done():
+                    if error is None:
+                        future.set_result(None)
+                    else:
+                        future.set_exception(error)
+            waiters.clear()
 
     async def _publish_controller_initial(self, capacity: int) -> None:
         async with self._publisher_lock:
@@ -1397,6 +1761,8 @@ class Worker:
         probe_member: _Member | None = None,
         probe_generation: int = 0,
     ) -> bool:
+        controller = self._controller
+        assert controller is not None
         epoch, session = self._session_epoch, self._session_done
         for attempt in range(20):
             if (
@@ -1417,38 +1783,24 @@ class Worker:
             # target and available are one atomic protocol snapshot.  A
             # credit_capacity rejection must be retried with a fresh target /
             # in-flight sample, never by replaying the old snapshot.
-            current_target = (
-                self._controller.capacity
-                if self._controller is not None and not explicit_target
-                else target
-            )
-            current_available = available
-            if self._controller is None:
-                # An assignment may have been sent by the server before the
-                # rejected credit result reached this reader. Re-sample the
-                # now-materialized local assignments instead of replaying the
-                # stale fixed-capacity snapshot.
-                current_available = max(
-                    0,
-                    current_target
-                    - self._fixed_running
-                    - len(self._waiting_assignments),
-                )
+            current_target = controller.capacity if not explicit_target else target
             update = queue_pb2.CreditUpdate(
-                available=max(0, current_available),
+                available=max(0, available),
                 target_concurrency=current_target,
             )
             result = await self._send_command(
                 queue_pb2.WorkRequest(credit=update),
-                wait=True,
                 expected_epoch=epoch,
                 expected_session=session,
             )
             if (
-                result is None
-                or not result.HasField("error")
+                not result.HasField("error")
                 or result.error.code == queue_pb2.ERROR_CODE_UNSPECIFIED
             ):
+                source = self._source
+                if source is not None and source.done is session:
+                    source.accepted_credit = available
+                    source.credit_generation = probe_generation
                 return True
             if (
                 result.error.metadata.get("reason") != "credit_capacity"
@@ -1460,72 +1812,57 @@ class Worker:
             await asyncio.sleep(0.01)
         return False
 
-    def _allocate_request_id(self) -> int:
-        rid = self._next_request_id
-        self._next_request_id = 1 if rid == 2**64 - 1 else rid + 1
-        return rid
-
     async def _send_command(
         self,
         request: queue_pb2.WorkRequest,
         *,
-        wait: bool,
         expected_epoch: int | None = None,
         expected_session: asyncio.Event | None = None,
-    ) -> queue_pb2.WorkResult | None:
+    ) -> queue_pb2.WorkResult:
+        source = self._source
+        if source is None or source.done.is_set():
+            raise UnavailableError("Work stream is not connected")
         if expected_epoch is not None and (
-            expected_epoch != self._session_epoch
-            or expected_session is not self._session_done
+            expected_epoch != source.epoch or expected_session is not source.done
         ):
             raise UnavailableError("command belongs to a replaced Work session")
-        if self._session_done is None or self._session_done.is_set():
-            raise UnavailableError("Work stream is not connected")
-        epoch, session = self._session_epoch, self._session_done
-        rid = request.request_id or self._allocate_request_id()
+        if source.next_request_id > _MAX_REQUEST_ID:
+            error = UnavailableError(
+                "Work request ID space exhausted; reconnect required"
+            )
+            self._end_source(source, error)
+            raise error
+        # This is the actual FIFO publication point, shared by every producer
+        # including drain. No await separates source validation, numbering,
+        # pending registration and insertion. Even an immediate receipt can
+        # find its Future, and cancellation never rolls a consumed ID back.
+        rid = source.next_request_id
+        source.next_request_id += 1
         request.request_id = rid
-        loop = asyncio.get_running_loop()
-        future: asyncio.Future[queue_pb2.WorkResult] = loop.create_future()
-        self._pending[rid] = future
+        future: asyncio.Future[queue_pb2.WorkResult] = (
+            asyncio.get_running_loop().create_future()
+        )
+        source.pending[rid] = future
         try:
-            await self._enqueue_frame(request)
-            if not wait:
-                return None
-            return await asyncio.wait_for(
-                self._wait_command_or_session(future), timeout=self._command_timeout
-            )
+            self._publish_frame(source, request)
+            async with asyncio.timeout(self._command_timeout):
+                return await future
         except asyncio.CancelledError:
-            if self._pending.get(rid) is future:
-                self._pending.pop(rid)
             if future.done() and not future.cancelled():
                 future.exception()
             raise
         except Exception as exc:
-            if self._pending.get(rid) is future:
-                self._pending.pop(rid)
+            # Source ownership, rather than the current Work call, fences a
+            # timeout or publication failure. Unknown mutations are not replayed.
+            self._end_source(source, exc)
             if future.done() and not future.cancelled():
+                # Publication can fail before we ever await this Future. End
+                # the source first, then consume its newly installed exception.
                 future.exception()
-            self._fail_session(exc, epoch=epoch, session=session)
             raise
-
-    async def _wait_command_or_session(
-        self, future: asyncio.Future[queue_pb2.WorkResult]
-    ) -> queue_pb2.WorkResult:
-        session = self._session_done
-        if session is None:
-            raise UnavailableError("Work stream is not connected")
-        session_wait = asyncio.create_task(session.wait())
-        try:
-            done, _ = await asyncio.wait(
-                (future, session_wait), return_when=asyncio.FIRST_COMPLETED
-            )
-            if future in done:
-                return future.result()
-            if future.done():
-                return future.result()
-            raise self._stream_error or UnavailableError("Work stream closed")
         finally:
-            session_wait.cancel()
-            await asyncio.gather(session_wait, return_exceptions=True)
+            if source.pending.get(rid) is future:
+                source.pending.pop(rid)
 
 
 __all__ = ["TaskContext", "Worker"]