pz-cli 5.0.0__py3-none-any.whl

pz_foundation/cli/workers.py

@@ -0,0 +1,565 @@
+from __future__ import annotations
+
+import argparse
+import json
+import subprocess
+import sys
+import time
+from pathlib import Path
+
+from pz_core.core.extensions import CliToolSpec
+from pz_core.runtime import compile_runtime_plan
+from pz_core.runtime.errors import as_pz_error
+from pz_foundation.managed_workers import (
+    ManagedWorkerSupervisor,
+    _is_process_alive,
+    managed_worker_supervisor_signature,
+    managed_worker_subprocess_env,
+    managed_worker_status_payload,
+    resolve_project_pz_executable,
+    stop_managed_workers,
+)
+
+
+def _coerce_bool(value: object, default: bool) -> bool:
+    if isinstance(value, bool):
+        return value
+    if value is None:
+        return bool(default)
+    if isinstance(value, str):
+        text = value.strip().lower()
+        if not text:
+            return bool(default)
+        if text in {"1", "true", "yes", "on"}:
+            return True
+        if text in {"0", "false", "no", "off"}:
+            return False
+    return bool(value)
+
+
+def _coerce_positive_int(value: object) -> int | None:
+    if value is None:
+        return None
+    try:
+        parsed = int(value)
+    except (TypeError, ValueError):
+        return None
+    return parsed if parsed > 0 else None
+
+
+def _force_restart_on_host_start(resolved_plan) -> bool:  # noqa: ANN001
+    try:
+        settings = getattr(getattr(resolved_plan, "settings", None), "app_settings", None)
+        payload = settings("pz_foundation") if callable(settings) else {}
+    except Exception:
+        return True
+    managed_workers = payload.get("managed_workers") if isinstance(payload, dict) else {}
+    if not isinstance(managed_workers, dict):
+        return True
+    return _coerce_bool(managed_workers.get("force_restart_on_host_start"), True)
+
+
+def _build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="pz workers",
+        description=(
+            "Manage detached managed workers declared by the current RuntimePlan.\n\n"
+            "Without names and without --all, commands target auto-start workers only. "
+            "Names match worker name, extension id, or <app_id>.<tool_name>."
+        ),
+        epilog=(
+            "Examples:\n"
+            "  pz workers list\n"
+            "  pz workers status --json\n"
+            "  pz workers restart mission_carrier.task_center_worker\n"
+            "  pz workers restart --all --start-timeout 90\n"
+            "  pz workers run --detach --owner-pid 12345"
+        ),
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+    )
+    parser.add_argument(
+        "--project-dir",
+        dest="project_dir",
+        default=None,
+        help="Project root containing pz.jsonc. Defaults to the discovered current project.",
+    )
+    parser.add_argument(
+        "--json",
+        action="store_true",
+        dest="json_output",
+        default=argparse.SUPPRESS,
+        help="Emit machine-readable JSON for commands that produce status payloads.",
+    )
+    subparsers = parser.add_subparsers(dest="command", required=False)
+
+    list_parser = subparsers.add_parser(
+        "list",
+        help="List managed worker targets and runtime state.",
+        description="List managed worker targets and runtime state for the current project.",
+    )
+    _add_worker_selection_arguments(list_parser)
+    _add_json_argument(list_parser)
+
+    status_parser = subparsers.add_parser(
+        "status",
+        help="Show the same managed worker status projection as list.",
+        description="Show the managed worker status projection for scripts and operators.",
+    )
+    _add_worker_selection_arguments(status_parser)
+    _add_json_argument(status_parser)
+
+    run_parser = subparsers.add_parser(
+        "run",
+        help="Run the managed worker supervisor loop.",
+        description=(
+            "Run the managed worker supervisor. This is the low-level entry used by "
+            "pz run and restart; day-to-day operator use should prefer restart/status/stop."
+        ),
+    )
+    _add_worker_selection_arguments(run_parser)
+    run_parser.add_argument(
+        "--poll-interval",
+        type=float,
+        default=1.0,
+        help="Seconds between supervisor reconciliation loops. Default: 1.0.",
+    )
+    run_parser.add_argument(
+        "--owner-pid",
+        type=int,
+        default=None,
+        help="Optional parent host PID. The detached supervisor stops when this owner exits.",
+    )
+    run_parser.add_argument(
+        "--detach",
+        action="store_true",
+        help="Start the supervisor as a detached background process and return immediately.",
+    )
+    run_parser.add_argument(
+        "--force-restart",
+        action="store_true",
+        default=argparse.SUPPRESS,
+        help="Stop any matching existing supervisor/workers before starting the detached supervisor.",
+    )
+    run_parser.add_argument(
+        "--no-force-restart",
+        action="store_false",
+        dest="force_restart",
+        default=argparse.SUPPRESS,
+        help="Keep a healthy matching supervisor when possible for this invocation.",
+    )
+
+    stop_parser = subparsers.add_parser(
+        "stop",
+        help="Stop the managed worker supervisor and matching workers.",
+        description="Stop the managed worker supervisor and matching worker replicas.",
+    )
+    _add_worker_selection_arguments(stop_parser)
+    stop_parser.add_argument(
+        "--timeout",
+        type=float,
+        default=15.0,
+        help="Seconds to wait for graceful shutdown before forceful termination. Default: 15.0.",
+    )
+    _add_json_argument(stop_parser)
+
+    restart_parser = subparsers.add_parser(
+        "restart",
+        help="Stop matching workers and launch a detached supervisor.",
+        description="Stop matching workers, launch a detached supervisor, then read back status.",
+    )
+    _add_worker_selection_arguments(restart_parser)
+    restart_parser.add_argument(
+        "--poll-interval",
+        type=float,
+        default=1.0,
+        help="Seconds between supervisor reconciliation loops after restart. Default: 1.0.",
+    )
+    restart_parser.add_argument(
+        "--owner-pid",
+        type=int,
+        default=None,
+        help="Optional parent host PID passed to the detached supervisor.",
+    )
+    restart_parser.add_argument(
+        "--timeout",
+        type=float,
+        default=15.0,
+        help="Seconds to wait while stopping existing workers. Default: 15.0.",
+    )
+    restart_parser.add_argument(
+        "--start-timeout",
+        type=float,
+        default=90.0,
+        help="Seconds to wait for restarted worker status before returning. Default: 90.0.",
+    )
+    _add_json_argument(restart_parser)
+    return parser
+
+
+def _add_worker_selection_arguments(parser: argparse.ArgumentParser) -> None:
+    parser.add_argument(
+        "names",
+        nargs="*",
+        help="Optional worker selectors: worker name, extension id, or <app_id>.<tool_name>.",
+    )
+    parser.add_argument(
+        "--project-dir",
+        dest="project_dir",
+        default=None,
+        help="Project root containing pz.jsonc. Defaults to the discovered current project.",
+    )
+    parser.add_argument(
+        "--all",
+        action="store_true",
+        dest="include_non_autostart",
+        help="Include non-auto-start worker roles when no explicit names are provided.",
+    )
+
+
+def _add_json_argument(parser: argparse.ArgumentParser) -> None:
+    parser.add_argument(
+        "--json",
+        action="store_true",
+        dest="json_output",
+        default=argparse.SUPPRESS,
+        help="Emit machine-readable JSON.",
+    )
+
+
+def _emit(payload: dict[str, object], *, json_output: bool) -> int:
+    if json_output:
+        print(json.dumps(payload, ensure_ascii=True, sort_keys=True))
+        return 0
+    action = payload.get("action")
+    status = payload.get("status")
+    if action or status:
+        print(f"{action or 'managed_workers'} status={status or 'ok'}")
+    supervisor = payload.get("supervisor")
+    if isinstance(supervisor, dict):
+        print(
+            f"supervisor running={supervisor.get('running')} "
+            f"pid={supervisor.get('pid')} phase={supervisor.get('phase')}"
+        )
+    workers = payload.get("workers")
+    if not isinstance(workers, list):
+        return 0
+    print(f"managed_workers={len(workers)}")
+    for item in workers:
+        if isinstance(item, str):
+            print(item)
+            continue
+        if not isinstance(item, dict):
+            continue
+        print(
+            f"{item.get('name')} app={item.get('app_id')} tool={item.get('tool_name')} "
+            f"kind={item.get('worker_kind') or 'infrastructure_worker'} "
+            f"phase={item.get('phase')} running={item.get('running')} auto_start={item.get('auto_start')}"
+        )
+    return 0
+
+
+def _detached_supervisor_command(
+    resolved_plan,
+    *,
+    names: tuple[str, ...],
+    include_non_autostart: bool,
+    poll_interval: float,
+    owner_pid: int | None,
+) -> tuple[str, ...]:
+    project_dir = str(Path(resolved_plan.project.project_dir).resolve())
+    pz_executable = resolve_project_pz_executable(Path(project_dir))
+    command: list[str] = [
+        pz_executable,
+        "pz_foundation",
+        "workers",
+        "run",
+        "--project-dir",
+        project_dir,
+        "--poll-interval",
+        str(float(poll_interval)),
+    ]
+    if include_non_autostart:
+        command.append("--all")
+    if owner_pid is not None and int(owner_pid) > 0:
+        command.extend(["--owner-pid", str(int(owner_pid))])
+    command.extend(names)
+    return tuple(command)
+
+
+def _launch_detached_supervisor(
+    resolved_plan,
+    *,
+    names: tuple[str, ...],
+    include_non_autostart: bool,
+    poll_interval: float,
+    owner_pid: int | None,
+    force_restart: bool,
+) -> int:
+    auto_start_only = not include_non_autostart and not names
+    requested_owner_pid = _coerce_positive_int(owner_pid)
+    command = _detached_supervisor_command(
+        resolved_plan,
+        names=names,
+        include_non_autostart=include_non_autostart,
+        poll_interval=poll_interval,
+        owner_pid=requested_owner_pid,
+    )
+    logs_dir = Path(resolved_plan.project.runtime_dir) / "managed_workers" / "logs"
+    logs_dir.mkdir(parents=True, exist_ok=True)
+    log_path = logs_dir / "supervisor.log"
+    desired_signature = managed_worker_supervisor_signature(
+        resolved_plan,
+        names=names,
+        auto_start_only=not include_non_autostart and not names,
+    )
+    worker_targets = list(desired_signature.get("worker_targets") or [])
+    existing = managed_worker_status_payload(
+        resolved_plan,
+        names=names,
+        auto_start_only=auto_start_only,
+    )
+    supervisor = existing.get("supervisor") if isinstance(existing.get("supervisor"), dict) else {}
+    workers = tuple(item for item in existing.get("workers", []) if isinstance(item, dict))
+    existing_owner_pid = _coerce_positive_int(supervisor.get("owner_pid"))
+    owner_stale = False
+    if requested_owner_pid is not None:
+        owner_stale = existing_owner_pid != requested_owner_pid
+        if not owner_stale and existing_owner_pid is not None and not _is_process_alive(existing_owner_pid):
+            owner_stale = True
+    if not worker_targets:
+        if bool(supervisor.get("running")):
+            stop_managed_workers(
+                resolved_plan,
+                timeout_seconds=10.0,
+                names=names,
+                auto_start_only=auto_start_only,
+            )
+        return 0
+    if force_restart:
+        if bool(supervisor.get("running")) or workers:
+            stop_managed_workers(
+                resolved_plan,
+                timeout_seconds=10.0,
+                names=names,
+                auto_start_only=auto_start_only,
+            )
+        supervisor = {}
+        workers = ()
+    signature_matches = (
+        str(supervisor.get("pz_env") or "") == str(desired_signature.get("pz_env") or "")
+        and list(supervisor.get("worker_targets") or []) == worker_targets
+    )
+    all_workers_healthy = bool(workers) and all(bool(item.get("healthy")) for item in workers)
+    if bool(supervisor.get("running")) and owner_stale:
+        stop_managed_workers(
+            resolved_plan,
+            timeout_seconds=10.0,
+            names=names,
+            auto_start_only=auto_start_only,
+        )
+        supervisor = {}
+        workers = ()
+        signature_matches = False
+        all_workers_healthy = False
+    if bool(supervisor.get("running")) and signature_matches:
+        return 0
+    if all_workers_healthy:
+        return 0
+    if bool(supervisor.get("running")) and not signature_matches:
+        stop_managed_workers(
+            resolved_plan,
+            timeout_seconds=10.0,
+            names=names,
+            auto_start_only=auto_start_only,
+        )
+    log_handle = log_path.open("a", encoding="utf-8")
+    try:
+        subprocess.Popen(
+            command,
+            cwd=str(resolved_plan.project.project_dir),
+            stdout=log_handle,
+            stderr=subprocess.STDOUT,
+            text=True,
+            env=managed_worker_subprocess_env(resolved_plan),
+            start_new_session=True,
+            close_fds=True,
+        )
+    finally:
+        log_handle.close()
+    return 0
+
+
+def _restart_detached_supervisor(
+    resolved_plan,
+    *,
+    names: tuple[str, ...],
+    include_non_autostart: bool,
+    poll_interval: float,
+    owner_pid: int | None,
+    timeout_seconds: float,
+    start_timeout_seconds: float,
+) -> dict[str, object]:
+    auto_start_only = not include_non_autostart and not names
+    stop_payload = stop_managed_workers(
+        resolved_plan,
+        timeout_seconds=float(timeout_seconds),
+        names=names,
+        auto_start_only=auto_start_only,
+    )
+    launch_code = _launch_detached_supervisor(
+        resolved_plan,
+        names=names,
+        include_non_autostart=include_non_autostart,
+        poll_interval=float(poll_interval),
+        owner_pid=owner_pid,
+        force_restart=False,
+    )
+    status_payload = _wait_for_restart_status(
+        resolved_plan,
+        names=names,
+        auto_start_only=auto_start_only,
+        timeout_seconds=float(start_timeout_seconds),
+    )
+    return {
+        "status": "ok" if launch_code == 0 else "error",
+        "code": "PZ_FOUNDATION_MANAGED_WORKERS_RESTARTED",
+        "stage": "worker",
+        "owner": "pz_foundation",
+        "action": "managed_workers_restart",
+        "project_dir": str(Path(resolved_plan.project.project_dir).resolve()),
+        "launch_requested": launch_code == 0,
+        "stop": stop_payload,
+        "supervisor": status_payload.get("supervisor"),
+        "workers": status_payload.get("workers", []),
+    }
+
+
+def _wait_for_restart_status(
+    resolved_plan,
+    *,
+    names: tuple[str, ...],
+    auto_start_only: bool,
+    timeout_seconds: float = 5.0,
+) -> dict[str, object]:
+    deadline = time.time() + max(0.0, float(timeout_seconds))
+    payload = managed_worker_status_payload(
+        resolved_plan,
+        names=names,
+        auto_start_only=auto_start_only,
+    )
+    while time.time() < deadline:
+        supervisor = payload.get("supervisor") if isinstance(payload.get("supervisor"), dict) else {}
+        workers = payload.get("workers") if isinstance(payload.get("workers"), list) else []
+        if bool(supervisor.get("running")) or any(isinstance(item, dict) and item.get("running") for item in workers):
+            return payload
+        time.sleep(0.1)
+        payload = managed_worker_status_payload(
+            resolved_plan,
+            names=names,
+            auto_start_only=auto_start_only,
+        )
+    return payload
+
+
+def main(argv: list[str] | None = None, *, project_dir: str | None = None, plan=None) -> int:
+    parser = _build_parser()
+    ns = parser.parse_args(list(argv or []))
+    command = ns.command or "list"
+    json_output = bool(getattr(ns, "json_output", False))
+    resolved_plan = plan or compile_runtime_plan(project_dir=project_dir or ns.project_dir)
+    include_non_autostart = bool(getattr(ns, "include_non_autostart", False))
+    names = tuple(str(item).strip() for item in getattr(ns, "names", ()) if str(item).strip())
+    auto_start_only = not include_non_autostart and not names
+    force_restart = (
+        bool(ns.force_restart)
+        if hasattr(ns, "force_restart")
+        else _force_restart_on_host_start(resolved_plan)
+    )
+    try:
+        if command in {"list", "status"}:
+            payload = managed_worker_status_payload(
+                resolved_plan,
+                names=names,
+                auto_start_only=auto_start_only,
+            )
+            return _emit(payload, json_output=json_output)
+        if command == "run":
+            owner_pid = getattr(ns, "owner_pid", None)
+            if bool(getattr(ns, "detach", False)):
+                return _launch_detached_supervisor(
+                    resolved_plan,
+                    names=names,
+                    include_non_autostart=include_non_autostart,
+                    poll_interval=float(ns.poll_interval),
+                    owner_pid=owner_pid,
+                    force_restart=force_restart,
+                )
+            supervisor = ManagedWorkerSupervisor(
+                resolved_plan,
+                names=names,
+                auto_start_only=auto_start_only,
+                poll_interval_seconds=float(ns.poll_interval),
+                owner_pid=owner_pid,
+            )
+            return supervisor.run()
+        if command == "stop":
+            payload = stop_managed_workers(
+                resolved_plan,
+                timeout_seconds=float(ns.timeout),
+                names=names,
+                auto_start_only=auto_start_only,
+            )
+            return _emit(payload, json_output=json_output)
+        if command == "restart":
+            payload = _restart_detached_supervisor(
+                resolved_plan,
+                names=names,
+                include_non_autostart=include_non_autostart,
+                poll_interval=float(ns.poll_interval),
+                owner_pid=getattr(ns, "owner_pid", None),
+                timeout_seconds=float(ns.timeout),
+                start_timeout_seconds=float(ns.start_timeout),
+            )
+            return _emit(payload, json_output=json_output)
+        parser.print_help()
+        return 0
+    except Exception as exc:
+        error = as_pz_error(
+            exc,
+            code="PZ_FOUNDATION_MANAGED_WORKERS_FAILED",
+            stage="worker",
+            owner="pz_foundation",
+            action="managed_workers",
+        )
+        if json_output:
+            payload = error.to_dict()
+            payload["status"] = "error"
+            print(json.dumps(payload, ensure_ascii=True, sort_keys=True))
+        else:
+            sys.stderr.write(f"{error}\n")
+        return 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main(sys.argv[1:]))
+
+
+class ManagedWorkersCliTool:
+    def configure(self, parser) -> None:  # noqa: ANN001
+        return None
+
+    def run(self, ctx) -> int:  # noqa: ANN001
+        return main(
+            list(getattr(ctx, "argv", ()) or ()),
+            project_dir=str(ctx.project.project_dir),
+            plan=ctx.plan,
+        )
+
+
+tool = CliToolSpec(
+    id="pz_foundation.workers",
+    owner_app="pz_foundation",
+    command=("workers",),
+    summary="Manage project-attached worker runtimes.",
+    tool=ManagedWorkersCliTool(),
+)

pz_foundation/configs/default.env

@@ -0,0 +1,15 @@
+# Secret placeholders for pz_foundation.
+pz_foundation__database__password=
+pz_foundation__database__ssl_password=
+pz_foundation__cache__password=
+pz_foundation__mail__password=
+pz_foundation__observability__logger__feishu__webhook=
+pz_foundation__observability__logger__feishu__secret=
+pz_foundation__observability__logger__feishu__webhook_error_event=
+pz_foundation__observability__logger__feishu__secret_error_event=
+pz_foundation__observability__logger__feishu__webhook_warning_event=
+pz_foundation__observability__logger__feishu__secret_warning_event=
+pz_foundation__observability__logger__feishu__webhook_custom_event=
+pz_foundation__observability__logger__feishu__secret_custom_event=
+pz_foundation__storage__access_key_id=
+pz_foundation__storage__secret_access_key=