ringo-task-queue 0.1.0.dev6__py3-none-win_amd64.whl → 0.1.0.dev8__py3-none-win_amd64.whl

ringo_task_queue/supervisor.py

@@ -0,0 +1,575 @@
+"""Opt-in, lightweight supervision of an importable Python application.
+
+    from datetime import timedelta
+    from ringo_task_queue.supervisor import supervise, SupervisorPolicy, TaskTimeouts
+
+    if __name__ == "__main__":
+        raise SystemExit(supervise(
+            "myapp:main",
+            policy=SupervisorPolicy(recycle_after=timedelta(hours=12)),
+            task_timeouts=TaskTimeouts(default=timedelta(minutes=45)),
+        ))
+
+The entrypoint is a zero-argument sync or async callable. This blocking launcher
+belongs outside the application's event loop. Normal completion is not a crash.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import math
+import os
+import random
+import re
+import secrets
+import signal
+import socket
+import subprocess
+import sys
+import threading
+import time
+from collections import deque
+from dataclasses import dataclass
+from typing import Any
+
+from ._supervisor_ipc import HEALTH_INTERVAL, Channel
+from ._supervisor_policy import SupervisorPolicy, TaskTimeouts
+
+__all__ = ["SupervisorPolicy", "TaskTimeouts", "supervise"]
+_LOG = logging.getLogger(__name__)
+_ENTRYPOINT = re.compile(
+    r"^[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)*:[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)*$"
+)
+_MONITOR_INTERVAL = 0.1
+_MEMORY_INTERVAL = 1.0
+_CLEANUP_TIMEOUT = 5.0
+_WINDOWS_CREATE_SUSPENDED = 0x00000004
+
+
+class _UserStop(BaseException):
+    pass
+
+
+class _CleanupFailed(RuntimeError):
+    pass
+
+
+class _Reader:
+    def __init__(self, channel: Channel) -> None:
+        self.channel = channel
+        self.ready = False
+        self.ready_at = time.monotonic()
+        self.worker_budgets: dict[str, float | None] = {}
+        self.worker_loop_ids: dict[str, str] = {}
+        self.budget_changed_at = time.monotonic()
+        self.health_serial = 0
+        self.entrypoint_done = False
+        self.entrypoint_failed = False
+        self.entrypoint_exit = 0
+        self.restart: dict[str, Any] | None = None
+        self.fatal = False
+        self.closed = False
+        self.last_health = time.monotonic()
+        self.health: dict[str, Any] = {}
+        self.thread = threading.Thread(
+            target=self._run, name="ringo-supervisor-reader", daemon=True
+        )
+        self.thread.start()
+
+    def _run(self) -> None:
+        try:
+            while True:
+                message = self.channel.receive()
+                if message is None:
+                    return
+                kind = message.get("kind")
+                if kind == "ready":
+                    self.ready_at = time.monotonic()
+                    self.ready = True
+                elif kind == "health":
+                    self.health_serial += 1
+                    message["_serial"] = self.health_serial
+                    self.last_health = time.monotonic()
+                    self.health = message
+                elif kind == "worker_budget":
+                    worker_id = str(message["worker_id"])
+                    self.worker_budgets[worker_id] = message.get("fallback_timeout")
+                    self.worker_loop_ids[worker_id] = str(message.get("loop_id", ""))
+                    self.budget_changed_at = time.monotonic()
+                elif kind == "worker_stopped":
+                    worker_id = str(message["worker_id"])
+                    self.worker_budgets.pop(worker_id, None)
+                    self.worker_loop_ids.pop(worker_id, None)
+                    self.budget_changed_at = time.monotonic()
+                elif kind == "restart" and self.restart is None:
+                    self.restart = message
+                elif kind == "entrypoint_done":
+                    self.entrypoint_exit = message.get("exit_code", 0)
+                    self.entrypoint_done = True
+                elif kind == "entrypoint_failed":
+                    self.entrypoint_failed = True
+                elif kind == "fatal":
+                    self.fatal = True
+        except (OSError, ValueError):
+            pass
+        finally:
+            self.closed = True
+
+
+@dataclass
+class _Generation:
+    process: subprocess.Popen
+    tree: Any
+    channel: Channel
+    reader: _Reader
+    started: float
+    max_rss_bytes: int | None = None
+    cleanup_fault: str | None = None
+
+
+def _spawn(
+    entrypoint: str,
+    policy: SupervisorPolicy,
+    task_timeouts: TaskTimeouts,
+    stop: threading.Event | None = None,
+) -> _Generation:
+    from ._supervisor_process import ProcessTree
+
+    token = secrets.token_hex(32)
+    generation = secrets.token_hex(16)
+    environment = dict(os.environ)
+    # Nested/ancestor markers must not establish this generation's ownership.
+    environment.pop("RINGO_SUPERVISOR_JOB", None)
+    listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    listener.bind(("127.0.0.1", 0))
+    listener.listen(4)
+    listener.settimeout(0.1)
+    environment.update(
+        {
+            "RINGO_SUPERVISOR_PORT": str(listener.getsockname()[1]),
+            "RINGO_SUPERVISOR_TOKEN": token,
+            "RINGO_SUPERVISOR_GENERATION": generation,
+        }
+    )
+    options: dict[str, Any] = {
+        "env": environment,
+        "stdin": subprocess.DEVNULL,
+        "close_fds": True,
+    }
+    if os.name == "nt":
+        options["creationflags"] = (
+            subprocess.CREATE_NEW_PROCESS_GROUP | _WINDOWS_CREATE_SUSPENDED
+        )
+    else:
+        options["start_new_session"] = True
+    process = None
+    tree = None
+    channel = None
+    started = time.monotonic()
+    try:
+        process = subprocess.Popen(
+            [sys.executable, "-m", "ringo_task_queue._supervisor_child", entrypoint],
+            **options,
+        )
+        # Windows must contain even a venv redirector *before it can spawn the
+        # real interpreter*. The later authenticated GO also gates app imports.
+        tree = ProcessTree(process.pid)
+        if stop is not None and stop.is_set():
+            raise _UserStop()
+        if os.name == "nt":
+            tree.resume_root()
+        deadline = started + policy.startup_timeout.total_seconds()
+        while time.monotonic() < deadline:
+            if stop is not None and stop.is_set():
+                raise _UserStop()
+            if process.poll() is not None:
+                raise RuntimeError(
+                    "supervised child exited before control-channel startup"
+                )
+            try:
+                connection, _ = listener.accept()
+            except TimeoutError:
+                continue
+            connection.settimeout(min(1.0, max(0.01, deadline - time.monotonic())))
+            candidate = Channel(connection)
+            try:
+                hello = candidate.receive(
+                    timeout=min(1.0, max(0.01, deadline - time.monotonic()))
+                )
+                valid = (
+                    hello is not None
+                    and hello.get("kind") == "hello"
+                    and isinstance(hello.get("token"), str)
+                    and secrets.compare_digest(hello["token"], token)
+                    and hello.get("generation") == generation
+                )
+            except (OSError, ValueError):
+                valid = False
+            if not valid:
+                candidate.close()
+                continue
+            connection.settimeout(None)
+            channel = candidate
+            channel.send(
+                {
+                    "kind": "configure",
+                    "generation": generation,
+                    "windows_job": os.name == "nt",
+                    "policy": policy._to_wire(),
+                    "task_timeouts": task_timeouts._to_wire(),
+                }
+            )
+            return _Generation(
+                process,
+                tree,
+                channel,
+                _Reader(channel),
+                started,
+                max_rss_bytes=policy.max_rss_bytes,
+            )
+        raise TimeoutError("supervised child control-channel startup timed out")
+    except BaseException:
+        if tree is not None:
+            try:
+                tree.kill()
+                if not tree.wait_empty(_CLEANUP_TIMEOUT):
+                    raise _CleanupFailed(
+                        "supervisor startup cleanup did not empty the owned process tree"
+                    )
+            except Exception as cleanup_error:
+                raise _CleanupFailed(
+                    "supervisor startup resource cleanup failed; refusing replacement"
+                ) from cleanup_error
+            finally:
+                tree.close()
+        elif process is not None and process.poll() is None:
+            process.kill()
+        if process is not None:
+            process.wait(timeout=_CLEANUP_TIMEOUT)
+        if channel is not None:
+            channel.close()
+        raise
+    finally:
+        listener.close()
+
+
+def _external_overdue(
+    health: dict[str, Any], now: float
+) -> set[tuple[str, float, float]]:
+    overdue = set()
+    for execution in health.get("executions", []):
+        if execution.get("unknown"):
+            continue
+        identity = execution.get("execution_id")
+        values = [
+            execution.get(name) for name in ("sampled_at", "deadline", "cancel_grace")
+        ]
+        if not isinstance(identity, str) or not all(
+            isinstance(value, (int, float))
+            and not isinstance(value, bool)
+            and math.isfinite(value)
+            for value in values
+        ):
+            continue
+        sampled, deadline, grace = values
+        if (
+            grace >= 0
+            and 0 <= now - sampled <= 2 * HEALTH_INTERVAL
+            and sampled >= deadline + grace
+        ):
+            overdue.add((identity, float(deadline), float(grace)))
+    return overdue
+
+
+def _monitor(
+    generation: _Generation,
+    policy: SupervisorPolicy,
+    stop: threading.Event | None = None,
+) -> str:
+    reader = generation.reader
+    soft_since: float | None = None
+    last_sample = float("-inf")
+    last_health_serial = 0
+    previous_health: dict[str, Any] = {}
+    while True:
+        if stop is not None and stop.is_set():
+            return "interrupted"
+        now = time.monotonic()
+        if reader.fatal:
+            return "invalid_entrypoint"
+        if reader.restart is not None:
+            reason = str(reader.restart.get("reason", "requested_restart"))[:128]
+            _LOG.warning("Supervised child requested recovery: %r", reason)
+            return "requested_restart"
+        code = generation.process.poll()
+        if code is not None:
+            # Drain already-buffered readiness/fatal frames before classifying
+            # a very short application; process exit can beat the reader thread.
+            reader.thread.join(timeout=0.2)
+            if reader.fatal:
+                return "invalid_entrypoint"
+            return "normal_exit" if code == 0 and reader.ready else "crash"
+        if (
+            not reader.ready
+            and now - generation.started >= policy.startup_timeout.total_seconds()
+        ):
+            return "startup_timeout"
+        if reader.entrypoint_failed:
+            return "crash"
+        if reader.entrypoint_done:
+            return "normal_exit" if reader.entrypoint_exit == 0 else "crash"
+        if reader.closed:
+            return "channel_lost"
+        health = reader.health
+        serial = health.get("_serial", 0)
+        if serial != last_health_serial:
+            # Missing a frame might hide progress recovery or an unknown read.
+            # Revalidate the preceding sample's freshness now, and match the
+            # effective deadline/grace as well as the execution identity.
+            if serial == last_health_serial + 1 and (
+                _external_overdue(health, now) & _external_overdue(previous_health, now)
+            ):
+                return "business_deadline_unresponsive"
+            previous_health = health
+            last_health_serial = serial
+        # If native code monopolizes the GIL, even the reporting thread cannot
+        # snapshot actual entries. Only universally bounded active Workers
+        # permit this conservative maximum-budget liveness fallback.
+        budgets = tuple(reader.worker_budgets.copy().values())
+        if reader.ready and budgets and all(value is not None for value in budgets):
+            fallback = max(value for value in budgets if value is not None)
+            baseline = max(
+                reader.last_health, reader.ready_at, reader.budget_changed_at
+            )
+            if now - baseline >= fallback:
+                return "business_observation_timeout"
+            active_loop_ids = {
+                reader.worker_loop_ids.get(worker_id)
+                for worker_id in reader.worker_budgets.copy()
+            }
+            for loop in health.get("loops", []):
+                if (
+                    loop.get("id") in active_loop_ids
+                    and now - max(loop["last_tick"], reader.budget_changed_at)
+                    >= fallback
+                ):
+                    return "business_observation_timeout"
+        if now - last_sample >= _MEMORY_INTERVAL:
+            # Membership sampling also retains identities for cleanup on POSIX.
+            rss = generation.tree.memory_bytes()
+            last_sample = now
+            if policy.max_rss_bytes is not None and rss >= policy.max_rss_bytes:
+                _LOG.warning(
+                    "Owned process RSS reached hard recovery threshold: %d bytes", rss
+                )
+                return "hard_memory"
+            if policy.soft_rss_bytes is not None and rss >= policy.soft_rss_bytes:
+                if soft_since is None:
+                    soft_since = now
+                if now - soft_since >= policy.rss_grace.total_seconds():
+                    _LOG.warning(
+                        "Owned process RSS persistently exceeded soft threshold: %d bytes",
+                        rss,
+                    )
+                    return "soft_memory"
+            else:
+                soft_since = None
+        if reader.ready and policy.heartbeat_timeout is not None:
+            timeout = policy.heartbeat_timeout.total_seconds()
+            if now - reader.last_health >= timeout:
+                return "health_channel_timeout"
+            for loop in reader.health.get("loops", []):
+                last_tick = loop.get("last_tick")
+                if isinstance(last_tick, (int, float)) and now - last_tick >= timeout:
+                    return "event_loop_timeout"
+        if (
+            reader.ready
+            and policy.recycle_after is not None
+            and now - generation.started >= policy.recycle_after.total_seconds()
+        ):
+            return "planned_recycle"
+        time.sleep(_MONITOR_INTERVAL)
+
+
+def _cleanup(generation: _Generation, grace: float) -> bool:
+    cleaned = False
+    try:
+        try:
+            generation.channel.send({"kind": "stop"})
+        except OSError:
+            pass
+        deadline = time.monotonic() + grace
+        last_memory = float("-inf")
+        max_rss = getattr(generation, "max_rss_bytes", None)
+        while time.monotonic() < deadline:
+            if generation.tree.wait_empty(0):
+                break
+            now = time.monotonic()
+            if max_rss is not None and now - last_memory >= _MEMORY_INTERVAL:
+                # A long healthy-task drain must not disable the hard resource
+                # guard after a soft threshold or planned recycle started it.
+                try:
+                    rss = generation.tree.memory_bytes()
+                except Exception:  # noqa: BLE001 - Unknown RSS is not a zero reading.
+                    generation.cleanup_fault = "resource_observation_error"
+                    _LOG.exception(
+                        "RSS observation failed during drain; forcing owned cleanup"
+                    )
+                    break
+                last_memory = now
+                if rss >= max_rss:
+                    generation.cleanup_fault = "hard_memory"
+                    _LOG.warning(
+                        "Hard RSS threshold reached during drain: %d bytes", rss
+                    )
+                    break
+            time.sleep(min(0.05, max(0.0, deadline - time.monotonic())))
+        if not generation.tree.wait_empty(0):
+            generation.tree.kill()
+        if not generation.tree.wait_empty(_CLEANUP_TIMEOUT):
+            _LOG.error(
+                "Owned resources did not exit; refusing to launch another generation"
+            )
+        else:
+            generation.process.wait(timeout=_CLEANUP_TIMEOUT)
+            cleaned = True
+    except Exception:  # noqa: BLE001 - Unknown native failures must block replacement.
+        _LOG.exception("Resource cleanup failed; refusing to launch another generation")
+    finally:
+        for close in (
+            generation.channel.close,
+            lambda: generation.reader.thread.join(timeout=1),
+            generation.tree.close,
+        ):
+            try:
+                close()
+            except Exception:  # noqa: BLE001 - Finish every independent owned cleanup.
+                cleaned = False
+                _LOG.exception("Resource owner close failed; refusing replacement")
+    return cleaned
+
+
+def supervise(
+    entrypoint: str,
+    *,
+    policy: SupervisorPolicy | None = None,
+    task_timeouts: TaskTimeouts | None = None,
+) -> int:
+    """Run a zero-argument ``module:function`` in supervised fresh processes.
+
+    Normal return/exit(0) ends supervision. Explicit recycling is independent
+    of exit status. Faults have a bounded, cross-generation restart window;
+    user interruption stops rather than restarting. Returns a process exit
+    status (0 success, 130 interruption, 74 cleanup/ownership failure,
+    75 exhausted recovery budget, 78 invalid entrypoint/configuration).
+
+    Import this module explicitly. On POSIX, install
+    ``ringo-task-queue[supervisor]`` for identity-aware process sampling.
+    """
+    if (
+        not isinstance(entrypoint, str)
+        or not _ENTRYPOINT.fullmatch(entrypoint)
+        or entrypoint.split(":", 1)[0] == "__main__"
+    ):
+        raise ValueError(
+            "entrypoint must be an importable 'module:function', not __main__ or a closure"
+        )
+    if threading.current_thread() is not threading.main_thread():
+        raise RuntimeError("supervise must run in the launcher's main thread")
+    try:
+        asyncio.get_running_loop()
+    except RuntimeError:
+        pass
+    else:
+        raise RuntimeError(
+            "supervise is blocking; call it outside the application event loop"
+        )
+    runtime_module = sys.modules.get("ringo_task_queue._supervisor_runtime")
+    if runtime_module is not None and runtime_module.current_runtime() is not None:
+        raise RuntimeError("nested process supervision is not supported")
+    policy = policy if policy is not None else SupervisorPolicy()
+    task_timeouts = task_timeouts if task_timeouts is not None else TaskTimeouts()
+    if not isinstance(policy, SupervisorPolicy) or not isinstance(
+        task_timeouts, TaskTimeouts
+    ):
+        raise ValueError(  # noqa: TRY004 - Consistent declarative policy validation.
+            "policy/task_timeouts must use their respective configuration types"
+        )
+    faults: deque[float] = deque()
+    old_term = signal.getsignal(signal.SIGTERM)
+    old_int = signal.getsignal(signal.SIGINT)
+    stop = threading.Event()
+
+    def interrupted(_signal, _frame):
+        # Repeated signals must not unwind kill/wait or leak a resource owner.
+        stop.set()
+
+    signal.signal(signal.SIGTERM, interrupted)
+    signal.signal(signal.SIGINT, interrupted)
+    generation: _Generation | None = None
+    try:
+        while True:
+            try:
+                if stop.is_set():
+                    return 130
+                generation = _spawn(entrypoint, policy, task_timeouts, stop)
+                reason = _monitor(generation, policy, stop)
+            except _CleanupFailed:
+                _LOG.exception(
+                    "Startup cleanup failed; no replacement will be launched"
+                )
+                return 74
+            except (KeyboardInterrupt, _UserStop):
+                if generation is not None:
+                    _cleanup(generation, policy.shutdown_grace.total_seconds())
+                    generation = None
+                return 130
+            except Exception:  # noqa: BLE001 - Isolate arbitrary generation failures.
+                _LOG.exception("Supervised generation failed")
+                reason = "generation_error"
+            if generation is not None:
+                grace = policy.shutdown_grace.total_seconds()
+                if reason in ("hard_memory", "crash", "invalid_entrypoint"):
+                    grace = min(grace, 1.0)
+                cleaned = _cleanup(generation, grace)
+                cleanup_fault = getattr(generation, "cleanup_fault", None)
+                if reason == "planned_recycle" and cleanup_fault is not None:
+                    # A hard resource fault discovered while draining is not
+                    # an unlimited series of free planned replacements.
+                    reason = cleanup_fault
+                generation = None
+                if not cleaned:
+                    return 74
+            if reason == "interrupted" or stop.is_set():
+                return 130
+            if reason == "normal_exit":
+                return 0
+            if reason == "invalid_entrypoint":
+                return 78
+            if reason != "planned_recycle":
+                now = time.monotonic()
+                while (
+                    faults and now - faults[0] >= policy.restart_window.total_seconds()
+                ):
+                    faults.popleft()
+                if len(faults) >= policy.max_restarts:
+                    _LOG.error(
+                        "Supervisor recovery budget exhausted; last reason=%s", reason
+                    )
+                    return 75
+                faults.append(now)
+                delay = min(30.0, 0.3 * 2 ** min(len(faults) - 1, 7))
+                try:
+                    if stop.wait(delay * random.uniform(0.8, 1.2)):
+                        return 130
+                except (KeyboardInterrupt, _UserStop):
+                    return 130
+            _LOG.info("Starting replacement generation; reason=%s", reason)
+    finally:
+        try:
+            if generation is not None:
+                _cleanup(generation, 0)
+        finally:
+            signal.signal(signal.SIGTERM, old_term)
+            signal.signal(signal.SIGINT, old_int)