ringo-task-queue 0.1.0.dev7__py3-none-win_amd64.whl → 0.1.0.dev8__py3-none-win_amd64.whl

ringo_task_queue/_supervisor_watchdog.py

@@ -0,0 +1,429 @@
+"""Opt-in, event-loop-local Worker observation; never owns business capacity."""
+
+from __future__ import annotations
+
+import asyncio
+import inspect
+import logging
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING, Any
+
+from ._proto.ringo.v1 import queue_pb2
+from ._supervisor_policy import TaskTimeouts
+from .worker import _WatchdogFenced
+
+if TYPE_CHECKING:
+    from .worker import TaskContext, Worker, _Assignment, _WorkSource
+
+_LOG = logging.getLogger(__name__)
+_WATCHDOG_SAMPLE_SECONDS = 0.1
+_STALL_PROBE_SECONDS = 5.0
+_STALL_MIN_PROTOCOL_MINOR = 7
+
+
+@dataclass
+class _Execution:
+    assignment: _Assignment
+    started: float
+    deadline: float | None
+    baseline_cancels: int
+    progress: float | None = None
+    expired: bool = False
+    reason: str = ""
+    cancelled_at: float | None = None
+    cancel_count: int = 0
+    cancel_marker: object = field(default_factory=object)
+    escalated: bool = False
+
+
+class _WorkerWatchdog:
+    def __init__(self, worker: Worker, timeouts: TaskTimeouts, runtime: Any) -> None:
+        self.worker = worker
+        self.timeouts = timeouts
+        self.runtime = runtime
+        self.executions: dict[tuple[str, int, str], _Execution] = {}
+        self.last_started: float | None = None
+        self.start_generation = 0
+        self._task: asyncio.Task[None] | None = None
+        self._probe_task: asyncio.Task[None] | None = None
+        self._stall_source: _WorkSource | None = None
+        self._stall_key: tuple[int, ...] | None = None
+        self._eligible_since: float | None = None
+        self._true_since: float | None = None
+        self._reported = False
+        self._next_probe = 0.0
+        self._old_protocol_source: _WorkSource | None = None
+
+    def start(self) -> None:
+        if self._task is None:
+            self._task = asyncio.create_task(
+                self._run(), name=f"ringo-watchdog-{self.worker.worker_id}"
+            )
+
+    async def stop(self) -> None:
+        tasks = [task for task in (self._task, self._probe_task) if task is not None]
+        for task in tasks:
+            task.cancel()
+        if tasks:
+            await asyncio.gather(*tasks, return_exceptions=True)
+
+    async def invoke(self, assignment: _Assignment, context: TaskContext) -> None:
+        """The budget wraps *only* the actual business invocation/await."""
+        task = asyncio.current_task()
+        assert task is not None
+        now = asyncio.get_running_loop().time()
+        budget = self.timeouts.for_type(assignment.task.task_type)
+        execution = _Execution(
+            assignment,
+            now,
+            None if budget is None else now + budget.total_seconds(),
+            task.cancelling(),
+        )
+        self.executions[assignment.identity] = execution
+        self.last_started = now
+        self.start_generation += 1
+        self._reset_stall()
+        try:
+            try:
+                result = self.worker._handler(assignment.task, context)
+                if inspect.isawaitable(result):
+                    await result
+            except BaseException as exc:
+                self._check_return_deadline(execution)
+                if execution.expired:
+                    if self._owns_timeout(execution, exc):
+                        raise TimeoutError(execution.reason) from None
+                    if self.worker._close_started.is_set():
+                        raise _WatchdogFenced() from None
+                    if not isinstance(exc, asyncio.CancelledError):
+                        raise asyncio.CancelledError(
+                            "expired task execution fenced"
+                        ) from None
+                raise
+            else:
+                self._check_return_deadline(execution)
+                if execution.expired:
+                    if self._owns_timeout(execution, None):
+                        raise TimeoutError(execution.reason)
+                    if self.worker._close_started.is_set():
+                        raise _WatchdogFenced()
+                    # Swallowing SDK cancellation never permits success after
+                    # expiry, including a simultaneous external cancellation.
+                    raise asyncio.CancelledError("expired task execution fenced")
+        finally:
+            # No terminal receipt or capacity-release wait is budgeted.
+            self.executions.pop(assignment.identity, None)
+
+    def _check_return_deadline(self, execution: _Execution) -> None:
+        # A synchronous handler can keep this event loop from sampling. Once it
+        # really returns, classify its elapsed budget without cancelling an
+        # already stopped invocation (which would suppress the timeout NACK).
+        if execution.expired:
+            return
+        now = asyncio.get_running_loop().time()
+        if execution.deadline is not None and now >= execution.deadline:
+            execution.expired = True
+            execution.reason = "task execution deadline exceeded"
+        elif (
+            self.timeouts.progress_timeout is not None
+            and execution.progress is not None
+            and now - execution.progress
+            >= self.timeouts.progress_timeout.total_seconds()
+        ):
+            execution.expired = True
+            execution.reason = "task business progress deadline exceeded"
+        if execution.expired:
+            execution.assignment.watchdog_expired = True
+            execution.cancel_count = execution.baseline_cancels
+
+    def external_snapshot(self) -> dict[str, Any] | None:
+        """Read-only health-thread view, including a blocked event loop.
+
+        No event-loop API, locks, RPC, lease token or payload participates. A
+        failed/racing sample is unknown, never proof of an idle healthy process.
+        """
+        try:
+            earliest: tuple[float, _Execution] | None = None
+            progress_budget = self.timeouts.progress_timeout
+            for execution in self.executions.copy().values():
+                deadline = execution.deadline
+                progress = execution.progress
+                if progress_budget is not None and progress is not None:
+                    progress_deadline = progress + progress_budget.total_seconds()
+                    deadline = (
+                        progress_deadline
+                        if deadline is None
+                        else min(deadline, progress_deadline)
+                    )
+                if deadline is not None and (
+                    earliest is None or deadline < earliest[0]
+                ):
+                    earliest = (deadline, execution)
+            if earliest is None:
+                return None
+            deadline, execution = earliest
+            assignment = execution.assignment
+            return {
+                "execution_id": (
+                    f"{id(self.worker)}:{assignment.session_epoch}:"
+                    f"{assignment.task.id}:{assignment.attempt}:{execution.started:.9f}"
+                ),
+                "deadline": deadline,
+                "cancel_grace": self.timeouts.cancel_grace.total_seconds(),
+            }
+        except (AttributeError, RuntimeError, TypeError, ValueError):
+            return {"unknown": True}
+
+    def mark_progress(self, assignment: _Assignment) -> None:
+        execution = self.executions.get(assignment.identity)
+        if execution is not None and not execution.expired and assignment.alive:
+            execution.progress = asyncio.get_running_loop().time()
+
+    def _owns_timeout(self, execution: _Execution, exc: BaseException | None) -> bool:
+        assignment = execution.assignment
+        task = assignment.handler_task
+        worker = self.worker
+        source = worker._source
+        now = asyncio.get_running_loop().time()
+        return bool(
+            not execution.escalated
+            and assignment.alive
+            and not worker._close_started.is_set()
+            and not worker._force_cancel.is_set()
+            and source is not None
+            and not source.done.is_set()
+            and assignment.session is source.done
+            and assignment.session_epoch == source.epoch
+            and (assignment.renew_deadline is None or now < assignment.renew_deadline)
+            and task is not None
+            # An additional application/close/lease cancellation is not ours.
+            and task.cancelling() <= execution.cancel_count
+            and execution.cancel_count
+            == execution.baseline_cancels + int(execution.cancelled_at is not None)
+            and (
+                not isinstance(exc, asyncio.CancelledError)
+                or exc.args == (execution.cancel_marker,)
+            )
+        )
+
+    async def _run(self) -> None:
+        while True:
+            self.sample()
+            await asyncio.sleep(_WATCHDOG_SAMPLE_SECONDS)
+
+    def sample(self) -> None:
+        now = asyncio.get_running_loop().time()
+        progress_budget = self.timeouts.progress_timeout
+        grace = self.timeouts.cancel_grace.total_seconds()
+        for execution in tuple(self.executions.values()):
+            assignment = execution.assignment
+            task = assignment.handler_task
+            if task is None or task.done():
+                continue
+            if not execution.expired:
+                # A dead lease/source does not prove the old business coroutine
+                # stopped. Its finite execution budget still requires escalation.
+                reason = ""
+                if execution.deadline is not None and now >= execution.deadline:
+                    reason = "task execution deadline exceeded"
+                elif (
+                    progress_budget is not None
+                    and execution.progress is not None
+                    and now - execution.progress >= progress_budget.total_seconds()
+                ):
+                    reason = "task business progress deadline exceeded"
+                if not reason:
+                    continue
+                # Expiry is independent of Task cancellation: a handler may
+                # consume CancelledError, uncancel itself or return normally.
+                execution.expired = True
+                assignment.watchdog_expired = True
+                execution.reason = reason
+                execution.cancelled_at = now
+                execution.cancel_count = task.cancelling() + 1
+                task.cancel(execution.cancel_marker)
+            elif (
+                not execution.escalated
+                and execution.cancelled_at is not None
+                and now - execution.cancelled_at >= grace
+            ):
+                execution.escalated = True
+                self.worker._watchdog_unresponsive(assignment, execution.reason)
+        runtime = self.runtime
+        if runtime is not None:
+            runtime.update_worker(
+                str(id(self.worker)),
+                {
+                    "queue": self.worker._queue.name[:32],
+                    "source_epoch": self.worker._session_epoch,
+                    "running": len(self.executions),
+                    "inflight": len(self.worker._inflight),
+                    "last_handler_started": self.last_started,
+                    "expired": sum(item.expired for item in self.executions.values()),
+                    "stall_suspected": self._reported,
+                },
+            )
+        self._sample_stall(now)
+
+    def _reset_stall(self) -> None:
+        self._eligible_since = None
+        self._true_since = None
+        self._reported = False
+
+    def _eligible_source(self) -> _WorkSource | None:
+        worker = self.worker
+        source = worker._source
+        if (
+            source is None
+            or source.done.is_set()
+            or not source.registered.is_set()
+            or worker._close_started.is_set()
+            or worker._force_cancel.is_set()
+            or worker._publisher_closed
+            or worker._client._require_conn()._session_token != source.token
+            or source.accepted_credit <= 0
+        ):
+            return None
+        if worker._controller is None:
+            if (
+                worker._concurrency
+                - worker._fixed_running
+                - len(worker._waiting_assignments)
+                <= 0
+                or source.credit_generation != worker._fixed_capacity_generation
+            ):
+                return None
+        else:
+            member = worker._member
+            if (
+                member is None
+                or not member.attached
+                or not member.session_ready
+                or not member.probe
+                or member.revoking
+                or worker._controller._grant_paused
+                or worker._controller._closed
+                or source.credit_generation != member.probe_generation
+            ):
+                return None
+        return source
+
+    def _qualification_key(self, source: _WorkSource) -> tuple[int, ...]:
+        # Successful credit/probe generations change even if pause/revoke and
+        # resume/regrant both happen between sampler ticks. Old evidence cannot
+        # survive that gap, nor can an RPC begun under the old qualification.
+        member = self.worker._member
+        return (
+            id(source),
+            source.epoch,
+            source.credit_generation,
+            source.accepted_credit,
+            self.worker._fixed_capacity_generation
+            if member is None
+            else member.probe_generation,
+        )
+
+    def _sample_stall(self, now: float) -> None:
+        runtime = self.runtime
+        timeout = None if runtime is None else runtime.policy.no_start_timeout
+        if timeout is None:
+            return
+        source = self._eligible_source()
+        if source is None:
+            self._stall_source = None
+            self._stall_key = None
+            self._reset_stall()
+            return
+        if source.minor < _STALL_MIN_PROTOCOL_MINOR:
+            if self._old_protocol_source is not source:
+                self._old_protocol_source = source
+                _LOG.warning(
+                    "Worker %s no-start observation skipped: negotiated protocol "
+                    "minor %s is older than %s",
+                    self.worker.worker_id,
+                    source.minor,
+                    _STALL_MIN_PROTOCOL_MINOR,
+                )
+            self._reset_stall()
+            return
+        key = self._qualification_key(source)
+        if source is not self._stall_source or key != self._stall_key:
+            self._stall_source = source
+            self._stall_key = key
+            self._reset_stall()
+        if self._eligible_since is None:
+            self._eligible_since = now
+        if (
+            now - self._eligible_since >= timeout.total_seconds()
+            and now >= self._next_probe
+            and (self._probe_task is None or self._probe_task.done())
+            and not self._reported
+        ):
+            self._next_probe = now + _STALL_PROBE_SECONDS
+            self._probe_task = asyncio.create_task(
+                self._probe_once(source, self.start_generation, key),
+                name=f"ringo-stall-probe-{self.worker.worker_id}",
+            )
+
+    async def _probe_once(
+        self, source: _WorkSource, generation: int, key: tuple[int, ...]
+    ) -> None:
+        if (
+            self._eligible_source() is not source
+            or self._qualification_key(source) != key
+            or self._stall_key != key
+            or self.start_generation != generation
+        ):
+            if self._stall_key == key:
+                self._reset_stall()
+            return
+        try:
+            response = await self.worker._client._require_conn().call(
+                "ProbeQueue",
+                queue_pb2.ProbeQueueRequest(
+                    queue=self.worker._queue.name, task_types=self.worker._task_types
+                ),
+                retry_safe=True,
+            )
+        except asyncio.CancelledError:
+            raise
+        except Exception:  # noqa: BLE001 - any failed observation is inconclusive
+            _LOG.debug(
+                "Worker queue probe inconclusive; resetting stall evidence",
+                exc_info=True,
+            )
+            if self._stall_key == key:
+                self._reset_stall()
+            return
+        if (
+            self._eligible_source() is not source
+            or self._stall_source is not source
+            or self._qualification_key(source) != key
+            or self._stall_key != key
+            or self.start_generation != generation
+            or not response.has_eligible_tasks
+        ):
+            if self._stall_key == key:
+                self._reset_stall()
+            return
+        now = asyncio.get_running_loop().time()
+        if self._true_since is None:
+            self._true_since = now
+            return
+        timeout = self.runtime.policy.no_start_timeout.total_seconds()
+        if now - self._true_since < timeout:
+            return
+        self._reported = True
+        detail = {
+            "worker_id": self.worker.worker_id,
+            "queue": self.worker._queue.name,
+            "source_epoch": source.epoch,
+            "confirmed_seconds": now - self._true_since,
+        }
+        _LOG.warning(
+            "Worker %s suspected consumption stall: eligible tasks repeatedly "
+            "observed without a handler start (competing consumers may race)",
+            self.worker.worker_id,
+        )
+        if self.runtime.policy.restart_on_stall:
+            self.runtime.request_restart("worker consumption stalled", detail)

ringo_task_queue/_version.py

@@ -1,8 +1,8 @@
 """Release-line constants shared by runtime packaging checks."""
 
-SDK_VERSION = "0.1.0.dev7"
+SDK_VERSION = "0.1.0.dev8"
 PROTOCOL_MAJOR = 1
-PROTOCOL_MINOR = 5
+PROTOCOL_MINOR = 7
 SCHEMA_VERSION = 3
 
 

ringo_task_queue/bin/manifest.json

@@ -4,17 +4,17 @@
       "arch": "amd64",
       "filename": "ringo-task-queue-windows-amd64.exe",
       "os": "windows",
-      "sha256": "ccd740bf5420172d14007e8f173707756a11fa19abaa0dab3e3f7ee7b2b5068f",
-      "size": 23388160
+      "sha256": "f3383f5d2d23c234aae02c7227b388b044eeab047161c29e7439e8d865451c59",
+      "size": 23455744
     }
   ],
-  "buildCommit": "9f4025f570de21d62ad67605af8a35e4c5072a8c",
-  "buildDate": "2026-10-13T09:37:17Z",
+  "buildCommit": "51e4b33d658033555b2d3e80dbd3fa8798703236+dirty.952661cfb73eb399664f4cf815c76e7eea3a66513442cea31c2c2acdfc4d8838",
+  "buildDate": "2026-10-15T06:19:37Z",
   "manifestVersion": 1,
   "name": "ringo-task-queue",
   "protocolMajor": 1,
-  "protocolMinor": 5,
-  "releaseManifestSha256": "f394eb5487562bb2cc13e0fc1ac833580307442f5cc33fc4e557df43801e039e",
+  "protocolMinor": 7,
+  "releaseManifestSha256": "27ffcd2b578c852a220c8353624902e1643c0cd0d40d577dca9d041ddca3a954",
   "schemaVersion": 3,
-  "version": "0.1.0-dev.7"
+  "version": "0.1.0-dev.8"
 }