pz-cli 5.0.0__py3-none-any.whl

pz_foundation/managed_workers.py

@@ -0,0 +1,1117 @@
+from __future__ import annotations
+
+import fcntl
+import json
+import os
+import signal
+import subprocess
+import time
+from dataclasses import asdict, dataclass, field
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any
+
+from pz_core.core.errors import PzError
+from pz_core.core.extensions import WorkerRoleSpec
+from pz_core.project_runtime_cli import build_project_pz_child_env, resolve_project_pz_executable as _resolve_project_pz_executable
+from pz_core.runtime.plan import RuntimePlan
+
+from .observability import get_logger
+from .worker_replicas import WorkerReplicaCountResolver
+
+
+_MANAGED_WORKER_WORKSPACE_ENV = "PZ_WORKER_WORKSPACE_DIR"
+_FOUNDATION_APP_ID = "pz_foundation"
+_MANAGED_WORKER_LOGGER_NAME = "pz_foundation.managed_workers"
+
+
+def _utc_now() -> datetime:
+    return datetime.now(UTC)
+
+
+def _project_pz_env(plan: RuntimePlan) -> str:
+    try:
+        payload = getattr(getattr(plan, "settings", None), "data", {}) or {}
+        project = payload.get("project") if isinstance(payload, dict) else {}
+        value = project.get("pz_env") if isinstance(project, dict) else None
+        return str(value or "develop").strip() or "develop"
+    except Exception:
+        return "develop"
+
+
+@dataclass(frozen=True, slots=True)
+class ManagedWorkerSpec:
+    extension_id: str
+    owner_app: str
+    name: str
+    app_id: str
+    tool_name: str
+    auto_start: bool = False
+    auto_start_setting: str = ""
+    restart: str = "always"
+    worker_kind: str = "infrastructure_worker"
+    replica_count_setting: str = ""
+    description: str = ""
+
+
+@dataclass(frozen=True, slots=True)
+class ManagedWorkerPaths:
+    worker_root: Path
+    state_path: Path
+    heartbeat_path: Path
+    pid_path: Path
+    lock_path: Path
+
+
+@dataclass(frozen=True, slots=True)
+class ManagedWorkerStatus:
+    name: str
+    app_id: str
+    tool_name: str
+    extension_id: str
+    owner_app: str
+    auto_start: bool
+    restart: str
+    worker_kind: str
+    running: bool
+    healthy: bool
+    phase: str
+    pid: int | None
+    worker_id: str | None
+    replica_slot: int
+    target_count: int
+    last_heartbeat_at: str | None
+    state_path: str
+    heartbeat_path: str
+    lock_path: str
+    command: tuple[str, ...]
+    state: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass(frozen=True, slots=True)
+class _CrashLoopPolicy:
+    max_failures: int = 5
+    failure_window_seconds: float = 60.0
+    base_backoff_seconds: float = 5.0
+    max_backoff_seconds: float = 300.0
+
+
+def _replica_name(spec: ManagedWorkerSpec, replica_slot: int) -> str:
+    return spec.name if int(replica_slot) <= 1 else f"{spec.name}#{int(replica_slot)}"
+
+
+def _read_mapping_member(container: Any, key: str) -> Any:
+    if container is None:
+        return None
+    if isinstance(container, dict):
+        return container.get(key)
+    return getattr(container, key, None)
+
+
+def _foundation_app_settings(plan: RuntimePlan) -> dict[str, Any]:
+    try:
+        app_settings = getattr(getattr(plan, "settings", None), "app_settings", None)
+        payload = app_settings(_FOUNDATION_APP_ID) if callable(app_settings) else {}
+    except Exception:
+        return {}
+    return payload if isinstance(payload, dict) else {}
+
+
+def _coerce_int(value: Any, *, default: int, minimum: int) -> int:
+    try:
+        parsed = int(value)
+    except (TypeError, ValueError):
+        parsed = int(default)
+    return max(int(minimum), parsed)
+
+
+def _coerce_float(value: Any, *, default: float, minimum: float) -> float:
+    try:
+        parsed = float(value)
+    except (TypeError, ValueError):
+        parsed = float(default)
+    return max(float(minimum), parsed)
+
+
+def _managed_worker_crash_loop_policy(plan: RuntimePlan) -> _CrashLoopPolicy:
+    managed_workers = _read_mapping_member(_foundation_app_settings(plan), "managed_workers")
+    crash_loop = _read_mapping_member(managed_workers, "crash_loop")
+    return _CrashLoopPolicy(
+        max_failures=_coerce_int(
+            _read_mapping_member(crash_loop, "max_failures"),
+            default=5,
+            minimum=1,
+        ),
+        failure_window_seconds=_coerce_float(
+            _read_mapping_member(crash_loop, "failure_window_seconds"),
+            default=60.0,
+            minimum=1.0,
+        ),
+        base_backoff_seconds=_coerce_float(
+            _read_mapping_member(crash_loop, "base_backoff_seconds"),
+            default=5.0,
+            minimum=0.0,
+        ),
+        max_backoff_seconds=_coerce_float(
+            _read_mapping_member(crash_loop, "max_backoff_seconds"),
+            default=300.0,
+            minimum=0.0,
+        ),
+    )
+
+
+def discover_managed_workers(plan: RuntimePlan) -> tuple[ManagedWorkerSpec, ...]:
+    discovered: list[ManagedWorkerSpec] = []
+    names: set[str] = set()
+    for extension in plan.extension_registry.by_kind("worker.role"):
+        if not isinstance(extension, WorkerRoleSpec):
+            raise PzError(
+                code="PZ_FOUNDATION_MANAGED_WORKER_DECL_INVALID",
+                message="worker.role declarations must use WorkerRoleSpec.",
+                stage="worker",
+                owner=str(getattr(extension, "owner_app", "") or "pz_foundation"),
+                action="discover_managed_workers",
+            )
+        app_id = str(extension.owner_app).strip()
+        tool_name = str(extension.tool_name).strip()
+        name = str(extension.name or f"{app_id}.{tool_name}").strip() or f"{app_id}.{tool_name}"
+        if name in names:
+            raise PzError(
+                code="PZ_FOUNDATION_MANAGED_WORKER_NAME_CONFLICT",
+                message=f"Managed worker name `{name}` is declared more than once.",
+                stage="worker",
+                owner=str(getattr(extension, "owner_app", "") or "pz_foundation"),
+                action="discover_managed_workers",
+            )
+        names.add(name)
+        discovered.append(
+            ManagedWorkerSpec(
+                extension_id=str(getattr(extension, "id", "") or ""),
+                owner_app=str(getattr(extension, "owner_app", "") or ""),
+                name=name,
+                app_id=app_id,
+                tool_name=tool_name,
+                auto_start=bool(extension.auto_start),
+                auto_start_setting=str(getattr(extension, "auto_start_setting", "") or "").strip(),
+                restart=str(extension.restart or "always").strip() or "always",
+                worker_kind=(
+                    str(getattr(extension, "worker_kind", "") or "infrastructure_worker").strip()
+                    or "infrastructure_worker"
+                ),
+                replica_count_setting=str(getattr(extension, "replica_count_setting", "") or "").strip(),
+                description=str(extension.description or extension.title or "").strip(),
+            )
+        )
+    return tuple(discovered)
+
+
+def _target_count_for_spec(plan: RuntimePlan, spec: ManagedWorkerSpec) -> int:
+    return WorkerReplicaCountResolver(plan).target_count(spec)
+
+
+def _coerce_bool(value: object, default: bool) -> bool:
+    if isinstance(value, bool):
+        return value
+    if value is None:
+        return bool(default)
+    if isinstance(value, str):
+        text = value.strip().lower()
+        if not text:
+            return bool(default)
+        if text in {"1", "true", "yes", "on"}:
+            return True
+        if text in {"0", "false", "no", "off"}:
+            return False
+    return bool(value)
+
+
+def _auto_start_for_spec(plan: RuntimePlan, spec: ManagedWorkerSpec) -> bool:
+    setting_name = str(spec.auto_start_setting or "").strip()
+    if not setting_name:
+        return bool(spec.auto_start)
+    try:
+        app_settings = getattr(getattr(plan, "settings", None), "app_settings", None)
+        if callable(app_settings):
+            payload = app_settings(spec.app_id) or {}
+            if isinstance(payload, dict) and setting_name in payload:
+                return _coerce_bool(payload.get(setting_name), bool(spec.auto_start))
+    except Exception:
+        pass
+    return bool(spec.auto_start)
+
+
+def _worker_targets(plan: RuntimePlan, specs: tuple[ManagedWorkerSpec, ...]) -> list[dict[str, Any]]:
+    targets: list[dict[str, Any]] = []
+    for spec in specs:
+        target_count = _target_count_for_spec(plan, spec)
+        for replica_slot in range(1, target_count + 1):
+            targets.append(
+                {
+                    "app_id": spec.app_id,
+                    "base_name": spec.name,
+                    "name": _replica_name(spec, replica_slot),
+                    "replica_slot": replica_slot,
+                    "target_count": target_count,
+                    "tool_name": spec.tool_name,
+                    "worker_kind": spec.worker_kind,
+                }
+            )
+    return targets
+
+
+def _managed_worker_root(plan: RuntimePlan) -> Path:
+    return plan.project.runtime_dir / "managed_workers"
+
+
+def _managed_worker_logs_dir(plan: RuntimePlan) -> Path:
+    return _managed_worker_root(plan) / "logs"
+
+
+def _managed_worker_workspace_root(
+    plan: RuntimePlan,
+    spec: ManagedWorkerSpec,
+    *,
+    replica_slot: int = 1,
+) -> Path:
+    worker_root = _managed_worker_root(plan) / "workers" / spec.app_id / spec.tool_name
+    if int(replica_slot) > 1:
+        worker_root = worker_root / "replicas" / str(int(replica_slot))
+    return worker_root
+
+
+def managed_worker_paths(
+    plan: RuntimePlan,
+    spec: ManagedWorkerSpec,
+    *,
+    replica_slot: int = 1,
+) -> ManagedWorkerPaths:
+    worker_root = _managed_worker_workspace_root(plan, spec, replica_slot=replica_slot)
+    return ManagedWorkerPaths(
+        worker_root=worker_root,
+        state_path=worker_root / "state.json",
+        heartbeat_path=worker_root / "heartbeat.json",
+        pid_path=worker_root / "worker.pid",
+        lock_path=worker_root / "worker.lock",
+    )
+
+
+def _load_json(path: Path) -> dict[str, Any]:
+    if not path.is_file():
+        return {}
+    try:
+        payload = json.loads(path.read_text(encoding="utf-8"))
+    except Exception:
+        return {}
+    return payload if isinstance(payload, dict) else {}
+
+
+def _read_pid(path: Path) -> int | None:
+    if not path.is_file():
+        return None
+    raw = path.read_text(encoding="utf-8").strip()
+    if not raw:
+        return None
+    try:
+        pid = int(raw)
+    except ValueError:
+        return None
+    return pid if pid > 0 else None
+
+
+def _is_process_alive(pid: int | None) -> bool:
+    if pid is None:
+        return False
+    try:
+        os.kill(pid, 0)
+    except ProcessLookupError:
+        return False
+    except PermissionError:
+        return True
+    return True
+
+
+def _reconcile_worker_runtime_files(
+    plan: RuntimePlan,
+    spec: ManagedWorkerSpec,
+    *,
+    replica_slot: int = 1,
+    pid: int | None,
+    reason: str,
+) -> None:
+    paths = managed_worker_paths(plan, spec, replica_slot=replica_slot)
+    state = _load_json(paths.state_path)
+    heartbeat = _load_json(paths.heartbeat_path)
+    worker_id = str(state.get("worker_id") or heartbeat.get("worker_id") or "").strip()
+    token = str(state.get("token") or heartbeat.get("token") or "").strip()
+    owner = str(state.get("owner") or heartbeat.get("owner") or spec.app_id).strip() or spec.app_id
+    stopped_state = {
+        "owner": owner,
+        "worker_id": worker_id,
+        "token": token,
+        "replica_slot": int(state.get("replica_slot") or heartbeat.get("replica_slot") or replica_slot),
+        "queue": state.get("queue") or spec.tool_name,
+        "kind": state.get("kind") or "default",
+        "ttl_seconds": state.get("ttl_seconds"),
+        "stale_seconds": state.get("stale_seconds"),
+        "max_wait_seconds": state.get("max_wait_seconds"),
+        "once": state.get("once"),
+        "concurrency": state.get("concurrency"),
+        "poll_interval_seconds": state.get("poll_interval_seconds"),
+        "shutdown_timeout_seconds": state.get("shutdown_timeout_seconds"),
+        "phase": "stopped",
+        "reason": reason,
+    }
+    paths.worker_root.mkdir(parents=True, exist_ok=True)
+    paths.state_path.write_text(
+        json.dumps(
+            {key: value for key, value in stopped_state.items() if value is not None},
+            ensure_ascii=True,
+            sort_keys=True,
+        )
+        + "\n",
+        encoding="utf-8",
+    )
+    stopped_heartbeat = {
+        "owner": owner,
+        "worker_id": worker_id,
+        "token": token,
+        "replica_slot": stopped_state["replica_slot"],
+        "phase": "stopped",
+        "pid": pid,
+        "timestamp": _utc_now().isoformat(),
+    }
+    paths.heartbeat_path.write_text(
+        json.dumps(
+            {key: value for key, value in stopped_heartbeat.items() if value not in (None, "")},
+            ensure_ascii=True,
+            sort_keys=True,
+        )
+        + "\n",
+        encoding="utf-8",
+    )
+    if paths.lock_path.exists():
+        paths.lock_path.unlink()
+    if paths.pid_path.exists():
+        paths.pid_path.unlink()
+
+
+def _write_worker_crash_loop_runtime_files(
+    plan: RuntimePlan,
+    spec: ManagedWorkerSpec,
+    *,
+    replica_slot: int = 1,
+    pid: int | None,
+    return_code: int | None,
+    restart_count: int,
+    failure_count: int,
+    failure_window_seconds: float,
+    log_path: Path,
+) -> None:
+    paths = managed_worker_paths(plan, spec, replica_slot=replica_slot)
+    state = _load_json(paths.state_path)
+    heartbeat = _load_json(paths.heartbeat_path)
+    worker_id = str(state.get("worker_id") or heartbeat.get("worker_id") or "").strip()
+    token = str(state.get("token") or heartbeat.get("token") or "").strip()
+    owner = str(state.get("owner") or heartbeat.get("owner") or spec.app_id).strip() or spec.app_id
+    updated_at = _utc_now().isoformat()
+    crash_state = {
+        "owner": owner,
+        "worker_id": worker_id,
+        "token": token,
+        "replica_slot": int(state.get("replica_slot") or heartbeat.get("replica_slot") or replica_slot),
+        "queue": state.get("queue") or spec.tool_name,
+        "kind": state.get("kind") or "default",
+        "ttl_seconds": state.get("ttl_seconds"),
+        "stale_seconds": state.get("stale_seconds"),
+        "max_wait_seconds": state.get("max_wait_seconds"),
+        "once": state.get("once"),
+        "concurrency": state.get("concurrency"),
+        "poll_interval_seconds": state.get("poll_interval_seconds"),
+        "shutdown_timeout_seconds": state.get("shutdown_timeout_seconds"),
+        "phase": "crash_loop_backoff",
+        "reason": f"crash_loop:exit:{return_code}",
+        "pid": pid,
+        "return_code": return_code,
+        "restart_count": int(restart_count),
+        "crash_loop_failures": int(failure_count),
+        "crash_loop_window_seconds": float(failure_window_seconds),
+        "log_path": str(log_path),
+        "updated_at": updated_at,
+    }
+    paths.worker_root.mkdir(parents=True, exist_ok=True)
+    paths.state_path.write_text(
+        json.dumps(
+            {key: value for key, value in crash_state.items() if value is not None},
+            ensure_ascii=True,
+            sort_keys=True,
+        )
+        + "\n",
+        encoding="utf-8",
+    )
+    crash_heartbeat = {
+        "owner": owner,
+        "worker_id": worker_id,
+        "token": token,
+        "replica_slot": crash_state["replica_slot"],
+        "phase": "crash_loop_backoff",
+        "pid": pid,
+        "timestamp": updated_at,
+    }
+    paths.heartbeat_path.write_text(
+        json.dumps(
+            {key: value for key, value in crash_heartbeat.items() if value not in (None, "")},
+            ensure_ascii=True,
+            sort_keys=True,
+        )
+        + "\n",
+        encoding="utf-8",
+    )
+    if paths.lock_path.exists():
+        paths.lock_path.unlink()
+    if paths.pid_path.exists():
+        paths.pid_path.unlink()
+
+
+def resolve_project_pz_executable(project_dir: Path) -> str:
+    try:
+        return _resolve_project_pz_executable(project_dir)
+    except PzError as exc:
+        raise PzError(
+            code="PZ_FOUNDATION_MANAGED_WORKER_PZ_MISSING",
+            message=str(exc.message),
+            detail=exc.detail,
+            stage="worker",
+            owner="pz_foundation",
+            action="run_managed_workers",
+        ) from exc
+
+
+def managed_worker_subprocess_env(
+    plan: RuntimePlan,
+    *,
+    workspace_dir: Path | None = None,
+) -> dict[str, str]:
+    env = build_project_pz_child_env(plan.project.project_dir, pz_env=_project_pz_env(plan))
+    if workspace_dir is not None:
+        env[_MANAGED_WORKER_WORKSPACE_ENV] = str(workspace_dir)
+    return env
+
+
+def managed_worker_command(
+    plan: RuntimePlan,
+    spec: ManagedWorkerSpec,
+    *,
+    replica_slot: int = 1,
+) -> tuple[str, ...]:
+    project_dir = str(plan.project.project_dir.resolve())
+    pz_executable = resolve_project_pz_executable(plan.project.project_dir)
+    return (
+        pz_executable,
+        spec.app_id,
+        spec.tool_name,
+        "--project-dir",
+        project_dir,
+        "--replica-slot",
+        str(int(replica_slot)),
+    )
+
+
+def build_managed_worker_status(
+    plan: RuntimePlan,
+    spec: ManagedWorkerSpec,
+    *,
+    replica_slot: int = 1,
+    target_count: int = 1,
+) -> ManagedWorkerStatus:
+    paths = managed_worker_paths(plan, spec, replica_slot=replica_slot)
+    state = _load_json(paths.state_path)
+    heartbeat = _load_json(paths.heartbeat_path)
+    phase = str(state.get("phase") or "missing").strip() or "missing"
+    pid_raw = heartbeat.get("pid")
+    pid = pid_raw if isinstance(pid_raw, int) else _read_pid(paths.pid_path)
+    running = paths.lock_path.exists() and _is_process_alive(pid)
+    if not running and phase in {"starting", "started", "stopping", "bound"}:
+        phase = "stopped"
+    raw_worker_id = state.get("worker_id") or heartbeat.get("worker_id")
+    worker_id = str(raw_worker_id).strip() if isinstance(raw_worker_id, str) and raw_worker_id.strip() else None
+    raw_heartbeat_at = heartbeat.get("timestamp")
+    last_heartbeat_at = str(raw_heartbeat_at).strip() if isinstance(raw_heartbeat_at, str) and raw_heartbeat_at.strip() else None
+    healthy = bool(running and phase not in {"missing", "stopped"})
+    return ManagedWorkerStatus(
+        name=_replica_name(spec, replica_slot),
+        app_id=spec.app_id,
+        tool_name=spec.tool_name,
+        extension_id=spec.extension_id,
+        owner_app=spec.owner_app,
+        auto_start=_auto_start_for_spec(plan, spec),
+        restart=spec.restart,
+        worker_kind=spec.worker_kind,
+        running=running,
+        healthy=healthy,
+        phase=phase,
+        pid=pid,
+        worker_id=worker_id,
+        replica_slot=int(replica_slot),
+        target_count=max(1, int(target_count)),
+        last_heartbeat_at=last_heartbeat_at,
+        state_path=str(paths.state_path),
+        heartbeat_path=str(paths.heartbeat_path),
+        lock_path=str(paths.lock_path),
+        command=managed_worker_command(plan, spec, replica_slot=replica_slot),
+        state=state,
+    )
+
+
+def _select_managed_workers(
+    plan: RuntimePlan,
+    *,
+    names: tuple[str, ...] = (),
+    auto_start_only: bool = False,
+) -> tuple[ManagedWorkerSpec, ...]:
+    specs = discover_managed_workers(plan)
+    if auto_start_only:
+        specs = tuple(spec for spec in specs if _auto_start_for_spec(plan, spec))
+    if not names:
+        return specs
+    wanted = {name.strip() for name in names if name.strip()}
+    selected = tuple(
+        spec
+        for spec in specs
+        if spec.name in wanted or spec.extension_id in wanted or f"{spec.app_id}.{spec.tool_name}" in wanted
+    )
+    missing = sorted(
+        wanted
+        - {spec.name for spec in selected}
+        - {spec.extension_id for spec in selected}
+        - {f"{spec.app_id}.{spec.tool_name}" for spec in selected}
+    )
+    if missing:
+        raise PzError(
+            code="PZ_FOUNDATION_MANAGED_WORKER_NOT_FOUND",
+            message=f"Unknown managed worker selection: {', '.join(missing)}",
+            stage="worker",
+            owner="pz_foundation",
+            action="run_managed_workers",
+        )
+    return selected
+
+
+def select_managed_worker_specs(
+    plan: RuntimePlan,
+    *,
+    names: tuple[str, ...] = (),
+    auto_start_only: bool = False,
+) -> tuple[ManagedWorkerSpec, ...]:
+    return _select_managed_workers(plan, names=names, auto_start_only=auto_start_only)
+
+
+def list_managed_worker_statuses(
+    plan: RuntimePlan,
+    *,
+    names: tuple[str, ...] = (),
+    auto_start_only: bool = False,
+) -> tuple[ManagedWorkerStatus, ...]:
+    statuses: list[ManagedWorkerStatus] = []
+    for spec in _select_managed_workers(plan, names=names, auto_start_only=auto_start_only):
+        target_count = _target_count_for_spec(plan, spec)
+        for replica_slot in range(1, target_count + 1):
+            statuses.append(
+                build_managed_worker_status(
+                    plan,
+                    spec,
+                    replica_slot=replica_slot,
+                    target_count=target_count,
+                )
+            )
+    return tuple(statuses)
+
+
+@dataclass
+class _RunningProcess:
+    spec: ManagedWorkerSpec
+    replica_slot: int
+    process: subprocess.Popen[Any]
+    log_path: Path
+    log_handle: Any
+    restart_count: int = 0
+
+    @property
+    def pid(self) -> int:
+        return int(self.process.pid)
+
+
+class ManagedWorkerSupervisor:
+    def __init__(
+        self,
+        plan: RuntimePlan,
+        *,
+        names: tuple[str, ...] = (),
+        auto_start_only: bool = True,
+        poll_interval_seconds: float = 1.0,
+        owner_pid: int | None = None,
+        popen_factory=subprocess.Popen,
+    ) -> None:
+        self.plan = plan
+        self.specs = _select_managed_workers(plan, names=names, auto_start_only=auto_start_only)
+        self.poll_interval_seconds = max(0.2, float(poll_interval_seconds))
+        self.owner_pid = int(owner_pid) if owner_pid is not None and int(owner_pid) > 0 else None
+        self._popen_factory = popen_factory
+        self._shutdown_requested = False
+        self._running: dict[str, _RunningProcess] = {}
+        self._restart_counts: dict[str, int] = {}
+        self._failure_history: dict[str, list[float]] = {}
+        self._restart_after: dict[str, float] = {}
+        self._crash_looped: dict[str, dict[str, Any]] = {}
+        self._crash_loop_policy = _managed_worker_crash_loop_policy(plan)
+        self._logger = get_logger(_MANAGED_WORKER_LOGGER_NAME)
+        self._manager_root = _managed_worker_root(self.plan)
+        self._lock_path = self._manager_root / "supervisor.lock"
+        self._state_path = self._manager_root / "supervisor.state.json"
+        self._lock_fd: Any = None
+        self._previous_handlers: dict[int, Any] = {}
+
+    def run(self) -> int:
+        if not self.specs:
+            self._write_state(phase="idle")
+            return 0
+        self._acquire_lock()
+        self._install_signal_handlers()
+        try:
+            for spec in self.specs:
+                self._ensure_replica(spec)
+            self._write_state(phase="running")
+            while not self._shutdown_requested:
+                if self.owner_pid is not None and not _is_process_alive(self.owner_pid):
+                    self._shutdown_requested = True
+                    break
+                self._poll_once()
+                time.sleep(self.poll_interval_seconds)
+            return 0
+        finally:
+            self._shutdown_all()
+            self._write_state(phase="stopped")
+            self._restore_signal_handlers()
+            self._release_lock()
+
+    def _install_signal_handlers(self) -> None:
+        def _handle_signal(_signum: int, _frame) -> None:
+            self._shutdown_requested = True
+
+        for signum in (signal.SIGINT, signal.SIGTERM):
+            try:
+                self._previous_handlers[signum] = signal.getsignal(signum)
+                signal.signal(signum, _handle_signal)
+            except (OSError, ValueError):
+                continue
+
+    def _restore_signal_handlers(self) -> None:
+        for signum, handler in self._previous_handlers.items():
+            try:
+                signal.signal(signum, handler)
+            except (OSError, ValueError):
+                continue
+        self._previous_handlers.clear()
+
+    def _acquire_lock(self) -> None:
+        self._manager_root.mkdir(parents=True, exist_ok=True)
+        self._lock_fd = self._lock_path.open("a+", encoding="utf-8")
+        try:
+            fcntl.flock(self._lock_fd.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
+        except BlockingIOError as exc:
+            raise PzError(
+                code="PZ_FOUNDATION_MANAGED_WORKER_SUPERVISOR_LOCKED",
+                message=f"Managed worker supervisor already holds lock at `{self._lock_path}`.",
+                stage="worker",
+                owner="pz_foundation",
+                action="run_managed_workers",
+            ) from exc
+        self._lock_fd.seek(0)
+        self._lock_fd.truncate()
+        self._lock_fd.write(f"{os.getpid()}\n")
+        self._lock_fd.flush()
+
+    def _release_lock(self) -> None:
+        if self._lock_fd is None:
+            return
+        try:
+            fcntl.flock(self._lock_fd.fileno(), fcntl.LOCK_UN)
+        except Exception:
+            pass
+        self._lock_fd.close()
+        self._lock_fd = None
+        if self._lock_path.exists():
+            self._lock_path.unlink()
+
+    def _write_state(self, *, phase: str) -> None:
+        payload = {
+            "phase": phase,
+            "pid": os.getpid(),
+            "owner_pid": self.owner_pid,
+            "pz_env": _project_pz_env(self.plan),
+            "updated_at": _utc_now().isoformat(),
+            "worker_targets": _worker_targets(self.plan, self.specs),
+            "workers": [
+                {
+                    "name": _replica_name(item.spec, item.replica_slot),
+                    "app_id": item.spec.app_id,
+                    "tool_name": item.spec.tool_name,
+                    "replica_slot": item.replica_slot,
+                    "pid": item.process.pid,
+                    "restart_count": item.restart_count,
+                    "log_path": str(item.log_path),
+                }
+                for item in self._running.values()
+            ],
+            "restart_after": {
+                key: datetime.fromtimestamp(value, UTC).isoformat()
+                for key, value in sorted(self._restart_after.items())
+            },
+            "crash_looped_workers": [
+                {**payload, "name": key}
+                for key, payload in sorted(self._crash_looped.items())
+            ],
+        }
+        self._manager_root.mkdir(parents=True, exist_ok=True)
+        self._state_path.write_text(json.dumps(payload, ensure_ascii=True, sort_keys=True) + "\n", encoding="utf-8")
+
+    def _ensure_replica(self, spec: ManagedWorkerSpec) -> None:
+        target_count = _target_count_for_spec(self.plan, spec)
+        for replica_slot in range(1, target_count + 1):
+            key = _replica_name(spec, replica_slot)
+            current = self._running.get(key)
+            if current is not None and current.process.poll() is None:
+                continue
+            if key in self._crash_looped:
+                continue
+            restart_after = self._restart_after.get(key)
+            if restart_after is not None and time.time() < restart_after:
+                continue
+            status = build_managed_worker_status(
+                self.plan,
+                spec,
+                replica_slot=replica_slot,
+                target_count=target_count,
+            )
+            if status.healthy:
+                continue
+            restart_count = self._restart_counts.get(key, current.restart_count if current is not None else 0)
+            self._restart_after.pop(key, None)
+            self._spawn(spec, replica_slot=replica_slot, restart_count=restart_count)
+
+    def _spawn(self, spec: ManagedWorkerSpec, *, replica_slot: int = 1, restart_count: int = 0) -> None:
+        logs_dir = _managed_worker_logs_dir(self.plan)
+        logs_dir.mkdir(parents=True, exist_ok=True)
+        log_path = logs_dir / f"{_replica_name(spec, replica_slot)}.log"
+        command = managed_worker_command(self.plan, spec, replica_slot=replica_slot)
+        log_handle = log_path.open("a", encoding="utf-8")
+        env = managed_worker_subprocess_env(
+            self.plan,
+            workspace_dir=_managed_worker_workspace_root(self.plan, spec, replica_slot=replica_slot),
+        )
+        process = self._popen_factory(
+            command,
+            cwd=str(self.plan.project.project_dir),
+            stdout=log_handle,
+            stderr=subprocess.STDOUT,
+            text=True,
+            env=env,
+        )
+        self._running[_replica_name(spec, replica_slot)] = _RunningProcess(
+            spec=spec,
+            replica_slot=replica_slot,
+            process=process,
+            log_path=log_path,
+            log_handle=log_handle,
+            restart_count=restart_count,
+        )
+        self._write_state(phase="running")
+
+    def _record_worker_exit(self, item: _RunningProcess, *, return_code: int) -> bool:
+        key = _replica_name(item.spec, item.replica_slot)
+        restart_count = item.restart_count + 1
+        self._restart_counts[key] = restart_count
+        now = time.time()
+        policy = self._crash_loop_policy
+        window_start = now - policy.failure_window_seconds
+        history = [value for value in self._failure_history.get(key, []) if value >= window_start]
+        history.append(now)
+        self._failure_history[key] = history
+        if len(history) >= policy.max_failures:
+            payload = {
+                "app_id": item.spec.app_id,
+                "tool_name": item.spec.tool_name,
+                "replica_slot": item.replica_slot,
+                "pid": item.process.pid,
+                "return_code": return_code,
+                "restart_count": restart_count,
+                "failure_count": len(history),
+                "failure_window_seconds": policy.failure_window_seconds,
+                "log_path": str(item.log_path),
+            }
+            self._crash_looped[key] = payload
+            self._restart_after.pop(key, None)
+            _write_worker_crash_loop_runtime_files(
+                self.plan,
+                item.spec,
+                replica_slot=item.replica_slot,
+                pid=item.process.pid,
+                return_code=return_code,
+                restart_count=restart_count,
+                failure_count=len(history),
+                failure_window_seconds=policy.failure_window_seconds,
+                log_path=item.log_path,
+            )
+            self._logger.error(
+                "managed worker stopped after crash loop name=%s app=%s tool=%s replica_slot=%s "
+                "pid=%s return_code=%s restart_count=%s failure_count=%s failure_window_seconds=%s log_path=%s",
+                key,
+                item.spec.app_id,
+                item.spec.tool_name,
+                item.replica_slot,
+                item.process.pid,
+                return_code,
+                restart_count,
+                len(history),
+                policy.failure_window_seconds,
+                item.log_path,
+            )
+            return False
+        backoff = min(
+            policy.max_backoff_seconds,
+            policy.base_backoff_seconds * (2 ** max(0, len(history) - 1)),
+        )
+        self._restart_after[key] = now + backoff
+        self._logger.warning(
+            "managed worker exited; restart scheduled name=%s app=%s tool=%s replica_slot=%s "
+            "pid=%s return_code=%s restart_count=%s failure_count=%s backoff_seconds=%s log_path=%s",
+            key,
+            item.spec.app_id,
+            item.spec.tool_name,
+            item.replica_slot,
+            item.process.pid,
+            return_code,
+            restart_count,
+            len(history),
+            backoff,
+            item.log_path,
+        )
+        return True
+
+    def _poll_once(self) -> None:
+        for spec_name, item in tuple(self._running.items()):
+            return_code = item.process.poll()
+            if return_code is None:
+                continue
+            try:
+                item.log_handle.close()
+            except Exception:
+                pass
+            del self._running[spec_name]
+            _reconcile_worker_runtime_files(
+                self.plan,
+                item.spec,
+                replica_slot=item.replica_slot,
+                pid=item.process.pid,
+                reason=f"exit:{return_code}",
+            )
+            if not self._shutdown_requested and item.spec.restart != "never":
+                self._record_worker_exit(item, return_code=int(return_code))
+        if not self._shutdown_requested:
+            for spec in self.specs:
+                self._ensure_replica(spec)
+        self._write_state(phase="running")
+
+    def _shutdown_all(self) -> None:
+        for item in self._running.values():
+            if item.process.poll() is None:
+                item.process.terminate()
+        deadline = time.time() + 10.0
+        for item in self._running.values():
+            remaining = max(0.1, deadline - time.time())
+            try:
+                item.process.wait(timeout=remaining)
+            except subprocess.TimeoutExpired:
+                item.process.kill()
+                item.process.wait(timeout=1.0)
+            _reconcile_worker_runtime_files(
+                self.plan,
+                item.spec,
+                replica_slot=item.replica_slot,
+                pid=item.process.pid,
+                reason="supervisor_shutdown",
+            )
+            try:
+                item.log_handle.close()
+            except Exception:
+                pass
+        self._running.clear()
+        self._restart_after.clear()
+
+
+def managed_worker_status_payload(
+    plan: RuntimePlan,
+    *,
+    names: tuple[str, ...] = (),
+    auto_start_only: bool = False,
+) -> dict[str, Any]:
+    statuses = list_managed_worker_statuses(plan, names=names, auto_start_only=auto_start_only)
+    supervisor_pid = _read_supervisor_pid(plan)
+    supervisor_state = _load_json(_supervisor_state_path(plan))
+    return {
+        "status": "ok",
+        "code": "PZ_FOUNDATION_MANAGED_WORKERS_STATUS_OK",
+        "stage": "worker",
+        "owner": "pz_foundation",
+        "action": "managed_workers_status",
+        "project_dir": str(plan.project.project_dir),
+        "supervisor": {
+            "running": bool(supervisor_pid and _is_process_alive(supervisor_pid)),
+            "pid": supervisor_pid,
+            "phase": str(supervisor_state.get("phase") or "missing"),
+            "owner_pid": supervisor_state.get("owner_pid"),
+            "pz_env": str(supervisor_state.get("pz_env") or ""),
+            "worker_targets": supervisor_state.get("worker_targets") if isinstance(supervisor_state.get("worker_targets"), list) else [],
+        },
+        "workers": [asdict(item) for item in statuses],
+    }
+
+
+def managed_worker_supervisor_signature(
+    plan: RuntimePlan,
+    *,
+    names: tuple[str, ...] = (),
+    auto_start_only: bool = False,
+) -> dict[str, Any]:
+    specs = _select_managed_workers(plan, names=names, auto_start_only=auto_start_only)
+    return {
+        "pz_env": _project_pz_env(plan),
+        "worker_targets": _worker_targets(plan, specs),
+    }
+
+
+def _supervisor_runtime_root(plan: RuntimePlan) -> Path:
+    return _managed_worker_root(plan)
+
+
+def _supervisor_lock_path(plan: RuntimePlan) -> Path:
+    return _supervisor_runtime_root(plan) / "supervisor.lock"
+
+
+def _supervisor_state_path(plan: RuntimePlan) -> Path:
+    return _supervisor_runtime_root(plan) / "supervisor.state.json"
+
+
+def _read_supervisor_pid(plan: RuntimePlan) -> int | None:
+    state = _load_json(_supervisor_state_path(plan))
+    pid = state.get("pid")
+    if isinstance(pid, int) and pid > 0:
+        return pid
+    lock_path = _supervisor_lock_path(plan)
+    if not lock_path.is_file():
+        return None
+    raw = lock_path.read_text(encoding="utf-8").strip()
+    try:
+        pid = int(raw)
+    except ValueError:
+        return None
+    return pid if pid > 0 else None
+
+
+def _terminate_pid(pid: int, *, force: bool = False) -> None:
+    os.kill(pid, signal.SIGKILL if force else signal.SIGTERM)
+
+
+def stop_managed_workers(
+    plan: RuntimePlan,
+    *,
+    timeout_seconds: float = 15.0,
+    names: tuple[str, ...] = (),
+    auto_start_only: bool = False,
+    stop_supervisor: bool = True,
+) -> dict[str, Any]:
+    specs = select_managed_worker_specs(plan, names=names, auto_start_only=auto_start_only)
+    state_path = _supervisor_state_path(plan)
+    lock_path = _supervisor_lock_path(plan)
+    worker_statuses = list_managed_worker_statuses(plan, names=names, auto_start_only=auto_start_only)
+    worker_pids = [
+        status.pid
+        for status in worker_statuses
+        if isinstance(status.pid, int) and status.pid > 0 and status.running
+    ]
+    supervisor_pid = _read_supervisor_pid(plan) if stop_supervisor else None
+    signaled: list[int] = []
+    if supervisor_pid is not None and _is_process_alive(supervisor_pid):
+        _terminate_pid(supervisor_pid)
+        signaled.append(supervisor_pid)
+    for pid in worker_pids:
+        if _is_process_alive(pid):
+            _terminate_pid(pid)
+            signaled.append(pid)
+    deadline = time.time() + max(0.5, timeout_seconds)
+    tracked = [pid for pid in {*(worker_pids), *([supervisor_pid] if supervisor_pid else []), *signaled} if pid]
+    while time.time() < deadline and any(_is_process_alive(pid) for pid in tracked):
+        time.sleep(0.1)
+    forced: list[int] = []
+    for pid in tracked:
+        if _is_process_alive(pid):
+            _terminate_pid(pid, force=True)
+            forced.append(pid)
+    for spec in specs:
+        target_count = _target_count_for_spec(plan, spec)
+        for replica_slot in range(1, target_count + 1):
+            _reconcile_worker_runtime_files(
+                plan,
+                spec,
+                replica_slot=replica_slot,
+                pid=None,
+                reason="managed_stop",
+            )
+    if stop_supervisor:
+        _supervisor_runtime_root(plan).mkdir(parents=True, exist_ok=True)
+        state_path.write_text(
+            json.dumps(
+                {
+                    "phase": "stopped",
+                    "pid": supervisor_pid,
+                    "updated_at": _utc_now().isoformat(),
+                    "pz_env": _project_pz_env(plan),
+                    "worker_targets": _worker_targets(plan, specs),
+                    "workers": [],
+                },
+                ensure_ascii=True,
+                sort_keys=True,
+            )
+            + "\n",
+            encoding="utf-8",
+        )
+        if lock_path.exists():
+            lock_path.unlink()
+    return {
+        "status": "ok",
+        "code": "PZ_FOUNDATION_MANAGED_WORKERS_STOPPED",
+        "stage": "worker",
+        "owner": "pz_foundation",
+        "action": "managed_workers_stop",
+        "project_dir": str(plan.project.project_dir),
+        "supervisor_pid": supervisor_pid,
+        "supervisor_stopped": bool(stop_supervisor),
+        "signaled_pids": sorted(signaled),
+        "forced_pids": sorted(forced),
+        "workers": [spec.name for spec in specs],
+    }
+
+
+__all__ = [
+    "ManagedWorkerSpec",
+    "ManagedWorkerStatus",
+    "ManagedWorkerSupervisor",
+    "build_managed_worker_status",
+    "discover_managed_workers",
+    "list_managed_worker_statuses",
+    "managed_worker_command",
+    "managed_worker_paths",
+    "managed_worker_supervisor_signature",
+    "managed_worker_status_payload",
+    "select_managed_worker_specs",
+    "stop_managed_workers",
+]